diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cf716839e..962247df2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -221,8 +221,27 @@ jobs: version: "~> v2" args: release --clean env: + # The v-prefixed Go module tag lands on this commit after this step; + # pin the current tag so GoReleaser never picks that one instead. + GORELEASER_CURRENT_TAG: ${{ steps.version.outputs.VERSION }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Go modules only resolve v-prefixed SemVer tags; Docker, npm and GitHub + # releases keep the bare version. Never forced: the Go checksum database + # pins the first commit a module version points to. + - name: Push Go module tag + env: + VERSION: ${{ steps.version.outputs.VERSION }} + run: | + set -euo pipefail + if git ls-remote --exit-code --tags origin "refs/tags/v${VERSION}" >/dev/null; then + echo "v${VERSION} already exists" + else + git tag "v${VERSION}" + git push origin "v${VERSION}" + fi + curl -fsS "https://proxy.golang.org/github.com/erpc/erpc/@v/v${VERSION}.info" + - name: Attest build provenance uses: actions/attest-build-provenance@96278af6caaf10aea03fd8d33a09a777ca52d62f # v3.2.0 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 749149b13..ae568a75d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,9 @@ permissions: jobs: units: runs-on: "${{ github.repository_owner == 'erpc' && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-24.04' }}" + permissions: + contents: read + id-token: write # Codecov OIDC upload timeout-minutes: 30 steps: - name: Harden the runner (Audit all outbound calls) @@ -35,7 +38,25 @@ jobs: - name: Build run: make build - name: Test with the Go CLI - run: make test-fast + run: make test-fast COVER_DIR=${{ runner.temp }}/coverage + - name: Merge coverage profiles + if: ${{ !cancelled() }} + run: | + set -euo pipefail + DIR="${{ runner.temp }}/coverage" + # Every shard profiles all packages, so blocks repeat across files: + # keep one row per block, summing its hit counts. + awk 'FNR==1 {next} {split($0, f, " "); k=f[1] " " f[2]; n[k]+=f[3]} END {print "mode: atomic"; for (k in n) print k, n[k]}' "$DIR"/*.out > "$DIR/coverage.txt" + TOTAL=$(go tool cover -func="$DIR/coverage.txt" | awk '/^total:/ {print $NF}') + echo "Total coverage: $TOTAL" | tee -a "$GITHUB_STEP_SUMMARY" + - name: Upload coverage to Codecov + if: ${{ !cancelled() && github.event_name == 'push' }} + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + with: + use_oidc: true + files: ${{ runner.temp }}/coverage/coverage.txt + disable_search: true + fail_ci_if_error: false # a Codecov outage must not turn main red security: runs-on: ubuntu-24.04 diff --git a/Makefile b/Makefile index 4edce4b8c..aa3ae25a2 100644 --- a/Makefile +++ b/Makefile @@ -58,12 +58,21 @@ test: @go test ./cmd/... -count 1 -parallel 1 @go test $$(ls -d */ | grep -v "cmd/" | grep -v "test/" | awk '{print "./" $$1 "..."}') -covermode=atomic -v -race -count 1 -parallel 1 -timeout 30m -failfast=false +# COVER_DIR= makes test-fast write one coverage profile per test process +# into ; -coverpkg counts lines any test reaches, not only its own package. +ifdef COVER_DIR +COVER_FLAGS := -cover -covermode=atomic -coverpkg=./... +cover_out = -coverprofile=$(COVER_DIR)/$(1).out +cover_bin = -test.coverprofile=$(COVER_DIR)/$(1).out +endif + .PHONY: test-fast test-fast: @go clean -testcache - @go test ./cmd/... -count 1 -parallel 1 -v + @$(if $(COVER_DIR),mkdir -p $(COVER_DIR)) + @go test ./cmd/... -count 1 -parallel 1 -v $(COVER_FLAGS) $(call cover_out,cmd) @# Compile erpc test binary once - @go test -c -o /tmp/erpc.test ./erpc + @go test -c -o /tmp/erpc.test $(COVER_FLAGS) ./erpc @# Run erpc shards in parallel (each process has isolated gock state) @# DSL shard (requires OpenAI API key, non-blocking) @/tmp/erpc.test -test.v -test.timeout 5m -test.run "^TestConsensusPolicy_DSL" || true & @@ -90,18 +99,18 @@ test-fast: SHARD_CONTAINER_CACHE="^TestEvmJsonRpcCache_(DynamoDB|Redis)$$"; \ SKIP_ALL="$$SHARD_CONSENSUS|$$SHARD_NETWORK|$$SHARD_HTTPSERVER|$$SHARD_INTEGRITY|$$SHARD_CONTAINER_CACHE|^TestConsensusPolicy_DSL|^TestHttp_"; \ pids=(); \ - /tmp/erpc.test -test.v -test.timeout 10m -test.run "$$SHARD_CONSENSUS" -test.parallel 4 & pids+=($$!); \ - /tmp/erpc.test -test.v -test.timeout 10m -test.run "$$SHARD_NETWORK" & pids+=($$!); \ - /tmp/erpc.test -test.v -test.timeout 10m -test.run "$$SHARD_HTTPSERVER" & pids+=($$!); \ - /tmp/erpc.test -test.v -test.timeout 10m -test.run "$$SHARD_INTEGRITY" & pids+=($$!); \ - /tmp/erpc.test -test.v -test.timeout 15m -test.run "$$SHARD_CONTAINER_CACHE" & pids+=($$!); \ - /tmp/erpc.test -test.v -test.timeout 10m -test.skip "$$SKIP_ALL" & pids+=($$!); \ + /tmp/erpc.test -test.v -test.timeout 10m -test.run "$$SHARD_CONSENSUS" -test.parallel 4 $(call cover_bin,erpc-consensus) & pids+=($$!); \ + /tmp/erpc.test -test.v -test.timeout 10m -test.run "$$SHARD_NETWORK" $(call cover_bin,erpc-network) & pids+=($$!); \ + /tmp/erpc.test -test.v -test.timeout 10m -test.run "$$SHARD_HTTPSERVER" $(call cover_bin,erpc-httpserver) & pids+=($$!); \ + /tmp/erpc.test -test.v -test.timeout 10m -test.run "$$SHARD_INTEGRITY" $(call cover_bin,erpc-integrity) & pids+=($$!); \ + /tmp/erpc.test -test.v -test.timeout 15m -test.run "$$SHARD_CONTAINER_CACHE" $(call cover_bin,erpc-container-cache) & pids+=($$!); \ + /tmp/erpc.test -test.v -test.timeout 10m -test.skip "$$SKIP_ALL" $(call cover_bin,erpc-rest) & pids+=($$!); \ fail=0; for pid in "$${pids[@]}"; do wait "$$pid" || fail=1; done; \ exit $$fail' @# TestHttp_ tests run after parallel shards (sensitive to system load, only ~5s) - @/tmp/erpc.test -test.v -test.timeout 5m -test.run "^TestHttp_" + @/tmp/erpc.test -test.v -test.timeout 5m -test.run "^TestHttp_" $(call cover_bin,erpc-http) @# Run all other (non-erpc) packages normally - @go test $$(ls -d */ | grep -v "cmd/" | grep -v "test/" | grep -v "erpc/" | awk '{print "./" $$1 "..."}') -count 1 -parallel 4 -v -timeout 30m -failfast=false + @go test $$(ls -d */ | grep -v "cmd/" | grep -v "test/" | grep -v "erpc/" | awk '{print "./" $$1 "..."}') -count 1 -parallel 4 -v -timeout 30m -failfast=false $(COVER_FLAGS) $(call cover_out,packages) .PHONY: test-race test-race: diff --git a/README.md b/README.md index e431c5cbf..7f3e42140 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,11 @@ eRPC Hero

+[![GitHub Stars](https://img.shields.io/github/stars/erpc/erpc?style=flat&logo=github&label=star&colorA=000000&colorB=000000)](https://github.com/erpc/erpc/stargazers) [![Build Status](https://img.shields.io/github/actions/workflow/status/erpc/erpc/release.yml?branch=main&style=flat&colorA=000000&colorB=000000)](https://github.com/erpc/erpc/actions/workflows/release.yml) [![Docs](https://img.shields.io/badge/docs-get%20started-brightgreen?style=flat&colorA=000000&colorB=000000)](https://docs.erpc.cloud/) +[![Go Reference](https://img.shields.io/badge/go-reference-blue?style=flat&logo=go&colorA=000000&colorB=000000)](https://pkg.go.dev/github.com/erpc/erpc) +[![Coverage](https://img.shields.io/codecov/c/github/erpc/erpc?style=flat&logo=codecov&colorA=000000&colorB=000000)](https://app.codecov.io/gh/erpc/erpc) [![License](https://img.shields.io/github/license/erpc/erpc?style=flat&colorA=000000&colorB=000000)](https://github.com/erpc/erpc/blob/main/LICENSE) [![Contributors](https://img.shields.io/github/contributors/erpc/erpc?style=flat&colorA=000000&colorB=000000)](https://github.com/erpc/erpc/graphs/contributors) [![Telegram](https://img.shields.io/endpoint?logo=telegram&url=https%3A%2F%2Ftg.sumanjay.workers.dev%2Ferpc_cloud&style=flat&colorA=000000&colorB=000000&label=telegram)](https://t.me/erpc_cloud) diff --git a/architecture/evm/cov_blocks_test.go b/architecture/evm/cov_blocks_test.go new file mode 100644 index 000000000..16d5693ec --- /dev/null +++ b/architecture/evm/cov_blocks_test.go @@ -0,0 +1,574 @@ +package evm + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func covBlocksResp(t *testing.T, req *common.NormalizedRequest, result string) *common.NormalizedResponse { + t.Helper() + resp := common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(result), nil), + ) + if req != nil { + resp.WithRequest(req) + } + return resp +} + +func covBlocksReq(method string, params ...interface{}) *common.NormalizedRequest { + if params == nil { + params = []interface{}{} + } + return common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest(method, params)) +} + +// covBlocksChanIDReq carries an id that cannot be marshalled, so any attempt +// to synthesize a response for it fails. +func covBlocksChanIDReq(method string, params ...interface{}) *common.NormalizedRequest { + if params == nil { + params = []interface{}{} + } + jrq := common.NewJsonRpcRequest(method, params) + jrq.ID = make(chan int) + return common.NewNormalizedRequestFromJsonRpcRequest(jrq) +} + +func covBlocksBadBodyReq() *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":`)) +} + +// covBlocksPanicNetwork panics when asked for the latest tip, like a network +// whose state poller has not been wired yet. +type covBlocksPanicNetwork struct { + queryTestNetwork +} + +func (n *covBlocksPanicNetwork) EvmHighestLatestBlockNumber(ctx context.Context) int64 { + panic("state poller not initialized") +} + +// covBlocksForwardUpstream is a config upstream with a pluggable Forward. +type covBlocksForwardUpstream struct { + queryTestConfigUpstream + forwardFn func(ctx context.Context, nq *common.NormalizedRequest) (*common.NormalizedResponse, error) +} + +func (u *covBlocksForwardUpstream) Forward(ctx context.Context, nq *common.NormalizedRequest, byPassMethodExclusion bool, isHedgeAttempt bool) (*common.NormalizedResponse, error) { + return u.forwardFn(ctx, nq) +} + +func TestCovBlocks_BuildGetBlockByNumberRequest(t *testing.T) { + cases := []struct { + name string + in interface{} + want string + wantErr bool + }{ + {name: "latest_tag", in: "latest", want: "latest"}, + {name: "earliest_tag", in: "earliest", want: "earliest"}, + {name: "hex_string_kept_verbatim", in: "0x10", want: "0x10"}, + {name: "unknown_tag", in: "newest", wantErr: true}, + {name: "int", in: 16, want: "0x10"}, + {name: "int64", in: int64(255), want: "0xff"}, + {name: "float64_not_normalizable", in: float64(16), wantErr: true}, + {name: "unsupported_type", in: []int{1}, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + jrq, err := BuildGetBlockByNumberRequest(tc.in, true) + if tc.wantErr { + require.Error(t, err) + assert.Nil(t, jrq) + assert.Contains(t, err.Error(), "invalid block number or tag for eth_getBlockByNumber") + return + } + require.NoError(t, err) + assert.Equal(t, "eth_getBlockByNumber", jrq.Method) + assert.Equal(t, []interface{}{tc.want, true}, jrq.Params) + }) + } +} + +func TestCovBlocks_EnforceHighestBlock_Edges(t *testing.T) { + ctx := context.Background() + network := &testNetwork{highestLatest: 0x100} + enforce := &common.RequestDirectives{EnforceHighestBlock: true} + + t.Run("unparsable_request_body_fails", func(t *testing.T) { + req := covBlocksBadBodyReq() + req.SetDirectives(enforce) + out, err := enforceHighestBlock(ctx, network, req, covBlocksResp(t, nil, `{"number":"0x1"}`), nil) + assert.Nil(t, out) + assert.Error(t, err) + }) + + passThrough := []struct { + name string + params []interface{} + }{ + {name: "no_params", params: []interface{}{}}, + {name: "numeric_block_param", params: []interface{}{float64(5), false}}, + {name: "pending_tag", params: []interface{}{"pending"}}, + } + for _, tc := range passThrough { + t.Run(tc.name, func(t *testing.T) { + req := covBlocksReq("eth_getBlockByNumber", tc.params...) + req.SetDirectives(enforce) + resp := covBlocksResp(t, req, `{"number":"0x1"}`) + out, err := enforceHighestBlock(ctx, network, req, resp, nil) + require.NoError(t, err) + assert.Same(t, resp, out) + }) + } + + for _, tag := range []string{"latest", "finalized"} { + t.Run("malformed_block_number_in_"+tag+"_response_fails", func(t *testing.T) { + req := covBlocksReq("eth_getBlockByNumber", tag, false) + req.SetDirectives(enforce) + resp := covBlocksResp(t, req, `{"number":"0xzz"}`) + out, err := enforceHighestBlock(ctx, network, req, resp, nil) + assert.Nil(t, out) + require.Error(t, err) + }) + } + + t.Run("upstream_error_short_circuits", func(t *testing.T) { + req := covBlocksReq("eth_getBlockByNumber", "latest", false) + req.SetDirectives(enforce) + boom := errors.New("upstream down") + out, err := networkPostForward_eth_getBlockByNumber(ctx, network, req, nil, boom) + assert.Nil(t, out) + assert.Same(t, boom, err) + }) +} + +func TestCovBlocks_PickHighestBlock(t *testing.T) { + ctx := context.Background() + boom := errors.New("re-fetch failed") + type pick int + const ( + pickNil pick = iota + pickX + pickY + ) + cases := []struct { + name string + x, y string // "" means nil response + err error + want pick + wantErr error + }{ + {name: "both_empty_with_error", x: `null`, y: "", err: boom, want: pickNil, wantErr: boom}, + {name: "both_nil_without_error", want: pickNil}, + {name: "y_missing_keeps_x", x: `{"number":"0x5"}`, want: pickX}, + {name: "x_without_number_keeps_y", x: `{"hash":"0x1"}`, y: `{"number":"0x5"}`, want: pickY}, + {name: "y_without_number_keeps_x", x: `{"number":"0x5"}`, y: `{"hash":"0x1"}`, want: pickX}, + {name: "x_unparsable_number_keeps_y", x: `{"number":"0xzz"}`, y: `{"number":"0x5"}`, want: pickY}, + {name: "y_unparsable_number_keeps_x", x: `{"number":"0x5"}`, y: `{"number":"0xzz"}`, want: pickX}, + {name: "x_higher_wins", x: `{"number":"0x6"}`, y: `{"number":"0x5"}`, want: pickX}, + {name: "y_higher_wins", x: `{"number":"0x5"}`, y: `{"number":"0x6"}`, want: pickY}, + {name: "tie_prefers_y", x: `{"number":"0x5"}`, y: `{"number":"0x5"}`, want: pickY}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var x, y *common.NormalizedResponse + if tc.x != "" { + x = covBlocksResp(t, nil, tc.x) + } + if tc.y != "" { + y = covBlocksResp(t, nil, tc.y) + } + out, err := pickHighestBlock(ctx, x, y, tc.err) + if tc.wantErr != nil { + assert.Same(t, tc.wantErr, err) + } else { + assert.NoError(t, err) + } + switch tc.want { + case pickNil: + assert.Nil(t, out) + case pickX: + assert.Same(t, x, out) + case pickY: + assert.Same(t, y, out) + } + }) + } +} + +func TestCovBlocks_ExtractBlockReference_Errors(t *testing.T) { + ctx := context.Background() + + t.Run("nil_request_and_response", func(t *testing.T) { + ref, bn, err := ExtractBlockReferenceFromRequest(ctx, nil) + assert.Equal(t, "", ref) + assert.Zero(t, bn) + assert.NoError(t, err) + + ref, bn, err = ExtractBlockReferenceFromResponse(ctx, nil) + assert.Equal(t, "", ref) + assert.Zero(t, bn) + assert.NoError(t, err) + + ts, err := ExtractBlockTimestampFromResponse(ctx, nil) + assert.Zero(t, ts) + assert.NoError(t, err) + }) + + t.Run("unparsable_request_body", func(t *testing.T) { + _, _, err := ExtractBlockReferenceFromRequest(ctx, covBlocksBadBodyReq()) + assert.Error(t, err) + + _, _, err = ResolveCacheBlockRef(ctx, covBlocksBadBodyReq(), nil) + assert.Error(t, err) + + resp := covBlocksResp(t, covBlocksBadBodyReq(), `{"number":"0x1"}`) + _, _, err = ExtractBlockReferenceFromResponse(ctx, resp) + assert.Error(t, err) + }) + + t.Run("malformed_last_valid_response_surfaces_error_keeping_request_ref", func(t *testing.T) { + req := covBlocksReq("eth_getTransactionReceipt", "0xabc") + lvr := covBlocksResp(t, req, `{"blockNumber":"0xzz"}`) + require.True(t, req.SetLastValidResponse(ctx, lvr)) + + ref, bn, err := ExtractBlockReferenceFromRequest(ctx, req) + require.Error(t, err) + assert.Equal(t, "*", ref) + assert.Zero(t, bn) + }) + + t.Run("nil_json_rpc_inputs", func(t *testing.T) { + jrr := common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`{"number":"0x1"}`), nil) + _, _, err := extractRefFromJsonRpcResponse(ctx, nil, nil, jrr) + assert.EqualError(t, err, "cannot extract block reference when json-rpc request is nil") + + jrq := common.NewJsonRpcRequest("eth_getBlockByNumber", []interface{}{"0x1", false}) + _, _, err = extractRefFromJsonRpcResponse(ctx, nil, jrq, nil) + assert.EqualError(t, err, "cannot extract block reference when json-rpc response is nil") + }) +} + +func TestCovBlocks_ExtractBlockTimestamp(t *testing.T) { + ctx := context.Background() + cases := []struct { + name string + result string + want int64 + wantErr bool + }{ + {name: "decimal_string", result: `{"timestamp":"1700000000"}`, want: 1700000000}, + {name: "hex_string", result: `{"timestamp":"0x6553f100"}`, want: 0x6553f100}, + {name: "null_result", result: `null`, want: 0}, + {name: "invalid_hex", result: `{"timestamp":"0xzz"}`, wantErr: true}, + {name: "invalid_decimal", result: `{"timestamp":"soon"}`, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ts, err := ExtractBlockTimestampFromResponse(ctx, covBlocksResp(t, nil, tc.result)) + if tc.wantErr { + assert.Error(t, err) + assert.Zero(t, ts) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, ts) + }) + } +} + +func TestCovBlocks_ResolveCacheBlockRef_TipFallbacks(t *testing.T) { + ctx := context.Background() + + t.Run("finalized_tag_resolves_to_network_finalized_tip", func(t *testing.T) { + req := covBlocksReq("eth_getBlockByNumber", "finalized", false) + req.SetNetwork(&queryTestNetwork{finalized: 0x30}) + ref, bn, err := ResolveCacheBlockRef(ctx, req, nil) + require.NoError(t, err) + assert.Equal(t, "0x30", ref) + assert.Equal(t, int64(0x30), bn) + }) + + t.Run("panicking_tip_lookup_falls_back_to_tag_literal", func(t *testing.T) { + req := covBlocksReq("eth_getBlockByNumber", "latest", false) + req.SetNetwork(&covBlocksPanicNetwork{}) + var ref string + var err error + require.NotPanics(t, func() { ref, _, err = ResolveCacheBlockRef(ctx, req, nil) }) + require.NoError(t, err) + assert.Equal(t, "latest", ref) + }) +} + +func TestCovBlocks_BlockNumberHooks(t *testing.T) { + ctx := context.Background() + enforce := &common.RequestDirectives{EnforceHighestBlock: true} + staleCounter := func(upstreamID string) float64 { + return testutil.ToFloat64(telemetry.MetricUpstreamStaleLatestBlock.WithLabelValues( + "test-project", "", "test", upstreamID, "eth_blockNumber", + )) + } + + run := func(t *testing.T, tip int64, respHex string, ups common.Upstream) (*common.NormalizedResponse, error) { + t.Helper() + req := covBlocksReq("eth_blockNumber") + req.SetDirectives(enforce) + jrr, err := common.NewJsonRpcResponse(1, respHex, nil) + require.NoError(t, err) + resp := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) + if ups != nil { + resp.SetUpstream(ups) + } + return networkPostForward_eth_blockNumber(ctx, &testNetwork{highestLatest: tip}, req, resp, nil) + } + + t.Run("stale_upstream_answer_is_raised_and_counted", func(t *testing.T) { + ups := common.NewFakeUpstream("covblocks-stale") + before := staleCounter("covblocks-stale") + out, err := run(t, 0x1000, "0x800", ups) + require.NoError(t, err) + _, bn, err := ExtractBlockReferenceFromResponse(ctx, out) + require.NoError(t, err) + assert.Equal(t, int64(0x1000), bn) + assert.Same(t, ups, out.Upstream(), "corrected response keeps upstream attribution") + assert.False(t, out.FromCache()) + assert.Equal(t, before+1, staleCounter("covblocks-stale")) + }) + + t.Run("ahead_upstream_answer_is_pinned_without_stale_metric", func(t *testing.T) { + ups := common.NewFakeUpstream("covblocks-ahead") + before := staleCounter("covblocks-ahead") + out, err := run(t, 0x1000, "0x1002", ups) + require.NoError(t, err) + _, bn, err := ExtractBlockReferenceFromResponse(ctx, out) + require.NoError(t, err) + assert.Equal(t, int64(0x1000), bn) + assert.Same(t, ups, out.Upstream()) + assert.Equal(t, before, staleCounter("covblocks-ahead")) + }) + + t.Run("malformed_block_number_fails", func(t *testing.T) { + out, err := run(t, 0x1000, "0xzz", nil) + assert.Nil(t, out) + assert.Error(t, err) + }) + + t.Run("upstream_error_passes_through", func(t *testing.T) { + boom := errors.New("timeout") + req := covBlocksReq("eth_blockNumber") + req.SetDirectives(enforce) + out, err := networkPostForward_eth_blockNumber(ctx, &testNetwork{highestLatest: 1}, req, nil, boom) + assert.Nil(t, out) + assert.Same(t, boom, err) + }) + + t.Run("unmarshalable_request_id_fails_synthesis", func(t *testing.T) { + req := covBlocksChanIDReq("eth_blockNumber") + req.SetDirectives(enforce) + jrr, err := common.NewJsonRpcResponse(1, "0x1", nil) + require.NoError(t, err) + resp := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) + out, err := networkPostForward_eth_blockNumber(ctx, &testNetwork{highestLatest: 0x10}, req, resp, nil) + assert.Nil(t, out) + assert.Error(t, err) + + handled, pre, err := networkPreForward_eth_blockNumber(ctx, &testNetwork{highestLatest: 0x10}, req) + assert.True(t, handled) + assert.Nil(t, pre) + assert.Error(t, err) + }) +} + +func TestCovBlocks_SendRawTransaction_UpstreamHookPassThrough(t *testing.T) { + ctx := context.Background() + net := &queryTestNetwork{cfg: &common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}} + ups := &covBlocksForwardUpstream{forwardFn: func(ctx context.Context, nq *common.NormalizedRequest) (*common.NormalizedResponse, error) { + t.Fatal("no verification probe expected") + return nil, nil + }} + alreadyKnown := common.NewErrEndpointNonceException(errors.New("already known"), common.NonceExceptionReasonAlreadyKnown) + + cases := []struct { + name string + req *common.NormalizedRequest + re error + }{ + {name: "nonce_code_without_nonce_exception_type", req: makeSendRawTxRequest(t), re: &common.BaseError{Code: common.ErrCodeEndpointNonceException, Message: "nonce"}}, + {name: "unknown_nonce_reason", req: makeSendRawTxRequest(t), re: common.NewErrEndpointNonceException(errors.New("replacement"), common.NonceExceptionReason("replacement_underpriced"))}, + {name: "unparsable_request_body", req: covBlocksBadBodyReq(), re: alreadyKnown}, + {name: "no_params", req: covBlocksReq("eth_sendRawTransaction"), re: alreadyKnown}, + {name: "non_string_param", req: covBlocksReq("eth_sendRawTransaction", float64(1)), re: alreadyKnown}, + {name: "non_hex_param", req: covBlocksReq("eth_sendRawTransaction", "0xnothex"), re: alreadyKnown}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rs := common.NewNormalizedResponse() + out, err := upstreamPostForward_eth_sendRawTransaction(ctx, net, ups, tc.req, rs, tc.re) + assert.Same(t, rs, out) + assert.Same(t, tc.re, err) + }) + } + + t.Run("already_known_with_unmarshalable_id_fails_synthesis", func(t *testing.T) { + req := covBlocksChanIDReq("eth_sendRawTransaction", sendRawTxFixture) + out, err := upstreamPostForward_eth_sendRawTransaction(ctx, net, ups, req, nil, alreadyKnown) + assert.Nil(t, out) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to create JSON-RPC response") + }) +} + +func TestCovBlocks_SendRawTransaction_NonceTooLowVerification(t *testing.T) { + ctx := context.Background() + net := &queryTestNetwork{cfg: &common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}} + vendorMsg := "nonce too low: next nonce 7, tx nonce 3" + nonceTooLow := common.NewErrEndpointNonceException( + common.NewErrJsonRpcExceptionInternal(-32000, common.JsonRpcErrorServerSideException, vendorMsg, nil, nil), + common.NonceExceptionReasonNonceTooLow, + ) + probed := 0 + ups := &covBlocksForwardUpstream{forwardFn: func(ctx context.Context, nq *common.NormalizedRequest) (*common.NormalizedResponse, error) { + probed++ + m, _ := nq.Method() + assert.Equal(t, "eth_getTransactionByHash", m) + return common.NewNormalizedResponse().WithJsonRpcResponse(common.MustNewJsonRpcResponseFromBytes( + []byte(`1`), + []byte(`{"hash":"`+sendRawTxFixtureHash+`"}`), + []byte(`{"code":-32000,"message":"internal error"}`), + )), nil + }} + + out, err := upstreamPostForward_eth_sendRawTransaction(ctx, net, ups, makeSendRawTxRequest(t), nil, nonceTooLow) + assert.Nil(t, out) + require.Error(t, err) + assert.Equal(t, 1, probed) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointClientSideException)) + var jre *common.ErrJsonRpcExceptionInternal + require.True(t, errors.As(err, &jre)) + assert.Equal(t, common.JsonRpcErrorTransactionRejected, jre.NormalizedCode()) + assert.Equal(t, vendorMsg, jre.Message, "the upstream wording is preserved") +} + +func TestCovBlocks_SendRawTransaction_NetworkHookPassThrough(t *testing.T) { + ctx := context.Background() + cfg := &common.NetworkConfig{Evm: &common.EvmNetworkConfig{}} + + t.Run("unparsable_tx_keeps_exhausted_error", func(t *testing.T) { + net := &queryTestNetwork{cfg: cfg, forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + t.Fatal("no probe expected") + return nil, nil + }} + exhausted := makeExhaustedError() + rs := common.NewNormalizedResponse() + out, err := networkPostForward_eth_sendRawTransaction(ctx, net, covBlocksReq("eth_sendRawTransaction", "0x02"), rs, exhausted) + assert.Same(t, rs, out) + assert.Same(t, exhausted, err) + }) + + t.Run("probe_response_with_error_payload_keeps_exhausted_error", func(t *testing.T) { + net := &queryTestNetwork{cfg: cfg, forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return common.NewNormalizedResponse().WithJsonRpcResponse(common.MustNewJsonRpcResponseFromBytes( + []byte(`1`), + []byte(`{"hash":"`+sendRawTxFixtureHash+`"}`), + []byte(`{"code":-32000,"message":"internal error"}`), + )), nil + }} + exhausted := makeExhaustedError() + out, err := networkPostForward_eth_sendRawTransaction(ctx, net, makeSendRawTxRequest(t), nil, exhausted) + assert.Nil(t, out) + assert.Same(t, exhausted, err) + }) +} + +func TestCovBlocks_CreateNormalizedNonceTooLowError(t *testing.T) { + cases := []struct { + name string + in error + want string + }{ + {name: "plain_cause_message", in: common.NewErrEndpointNonceException(errors.New("nonce too low: address 0xabc, tx: 3 state: 7"), common.NonceExceptionReasonNonceTooLow), want: "nonce too low: address 0xabc, tx: 3 state: 7"}, + {name: "no_cause_uses_exception_message", in: common.NewErrEndpointNonceException(nil, common.NonceExceptionReasonNonceTooLow), want: "transaction with this nonce already exists"}, + {name: "foreign_error_uses_generic_message", in: errors.New("connection reset"), want: "nonce too low"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := createNormalizedNonceTooLowError(tc.in) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointClientSideException)) + var jre *common.ErrJsonRpcExceptionInternal + require.True(t, errors.As(err, &jre)) + assert.Equal(t, common.JsonRpcErrorTransactionRejected, jre.NormalizedCode()) + assert.Equal(t, tc.want, jre.Message) + }) + } +} + +// TestCovBlocks_DetailedTracing toggles the process-wide detailed-tracing flag, +// so it must stay sequential (no t.Parallel), like integrity_tracing_test.go. +func TestCovBlocks_DetailedTracing(t *testing.T) { + prev := common.IsTracingDetailed + common.IsTracingDetailed = true + t.Cleanup(func() { common.IsTracingDetailed = prev }) + ctx := context.Background() + + t.Run("latest_block_records_timestamp_distance", func(t *testing.T) { + network := &testNetwork{highestLatest: 0x20} + req := covBlocksReq("eth_getBlockByNumber", "latest", false) + ts := time.Now().Unix() - 100 + resp := covBlocksResp(t, req, fmt.Sprintf(`{"number":"0x10","hash":"0x01","timestamp":"0x%x"}`, ts)) + + out, err := networkPostForward_eth_getBlockByNumber(ctx, network, req, resp, nil) + require.NoError(t, err) + assert.Same(t, resp, out) + gauge := testutil.ToFloat64(telemetry.MetricNetworkLatestBlockTimestampDistance.WithLabelValues("test-project", "test", "network_response")) + assert.GreaterOrEqual(t, gauge, float64(100)) + assert.Less(t, gauge, float64(160)) + }) + + t.Run("block_refs_resolve_the_same_under_detailed_tracing", func(t *testing.T) { + cached := covBlocksReq("eth_getBlockByNumber", "0x10", false) + cached.SetEvmBlockRef("cached-ref") + cached.SetEvmBlockNumber(int64(99)) + ref, bn, err := ExtractBlockReferenceFromRequest(ctx, cached) + require.NoError(t, err) + assert.Equal(t, "cached-ref", ref) + assert.Equal(t, int64(99), bn) + + req := covBlocksReq("eth_getBlockByNumber", "0x10", false) + ref, bn, err = ExtractBlockReferenceFromRequest(ctx, req) + require.NoError(t, err) + assert.Equal(t, "16", ref) + assert.Equal(t, int64(16), bn) + + ref, bn, err = ExtractBlockReferenceFromResponse(ctx, covBlocksResp(t, req, `{"number":"0x11"}`)) + require.NoError(t, err) + assert.Equal(t, "17", ref) + assert.Equal(t, int64(17), bn) + + tsv, err := ExtractBlockTimestampFromResponse(ctx, covBlocksResp(t, nil, `{"timestamp":"0x20"}`)) + require.NoError(t, err) + assert.Equal(t, int64(32), tsv) + }) + + t.Run("block_number_ahead_of_tip_is_pinned", func(t *testing.T) { + req := covBlocksReq("eth_blockNumber") + req.SetDirectives(&common.RequestDirectives{EnforceHighestBlock: true}) + jrr, err := common.NewJsonRpcResponse(1, "0x30", nil) + require.NoError(t, err) + resp := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) + out, err := networkPostForward_eth_blockNumber(ctx, &testNetwork{highestLatest: 0x20}, req, resp, nil) + require.NoError(t, err) + _, bn, err := ExtractBlockReferenceFromResponse(ctx, out) + require.NoError(t, err) + assert.Equal(t, int64(0x20), bn) + }) +} diff --git a/architecture/evm/cov_cache_test.go b/architecture/evm/cov_cache_test.go new file mode 100644 index 000000000..2bb70594e --- /dev/null +++ b/architecture/evm/cov_cache_test.go @@ -0,0 +1,1000 @@ +package evm + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "math/rand/v2" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" + "github.com/klauspost/compress/zstd" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// covCacheConn is a race-safe in-memory connector stub. Unlike the testify +// data.MockConnector it never reflects over the request metadata, so it can be +// used by several fan-out goroutines at once under -race. Values are keyed by +// range key only (the partition key carries the block ref, which differs +// between the reverse and main index lookups). +type covCacheConn struct { + id string + mu sync.Mutex + store map[string][]byte + getErr error + getBytes []byte // when non-nil, returned for every Get instead of store + setErr error + gets atomic.Int32 + sets atomic.Int32 + lastTTL *time.Duration + lastPK string + lastIdx string + setValue []byte +} + +var _ data.Connector = (*covCacheConn)(nil) + +func covCacheNewConn(id string) *covCacheConn { + return &covCacheConn{id: id, store: map[string][]byte{}} +} + +func (c *covCacheConn) Id() string { return c.id } + +func (c *covCacheConn) Get(_ context.Context, index, partitionKey, rangeKey string, _ interface{}) ([]byte, error) { + c.gets.Add(1) + c.mu.Lock() + defer c.mu.Unlock() + c.lastIdx = index + if c.getErr != nil { + return nil, c.getErr + } + if c.getBytes != nil { + return c.getBytes, nil + } + v, ok := c.store[rangeKey] + if !ok { + return nil, common.NewErrRecordNotFound(partitionKey, rangeKey, c.id) + } + return v, nil +} + +func (c *covCacheConn) Set(_ context.Context, partitionKey, rangeKey string, value []byte, ttl *time.Duration) error { + c.sets.Add(1) + c.mu.Lock() + defer c.mu.Unlock() + if c.setErr != nil { + return c.setErr + } + cp := append([]byte(nil), value...) + c.store[rangeKey] = cp + c.setValue = cp + c.lastPK = partitionKey + c.lastTTL = ttl + return nil +} + +func (c *covCacheConn) Delete(context.Context, string, string) error { return nil } +func (c *covCacheConn) List(context.Context, string, int, string) ([]data.KeyValuePair, string, error) { + return nil, "", nil +} +func (c *covCacheConn) Lock(context.Context, string, time.Duration) (data.DistributedLock, error) { + return nil, errors.New("not supported") +} +func (c *covCacheConn) WatchCounterInt64(context.Context, string) (<-chan data.CounterInt64State, func(), error) { + return nil, func() {}, nil +} +func (c *covCacheConn) PublishCounterInt64(context.Context, string, data.CounterInt64State) error { + return nil +} + +func (c *covCacheConn) snapshot() (pk string, idx string, val []byte, ttl *time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + return c.lastPK, c.lastIdx, c.setValue, c.lastTTL +} + +type covCachePolicyOpt func(*common.CachePolicyConfig) + +func covCacheWithEmpty(e common.CacheEmptyBehavior) covCachePolicyOpt { + return func(c *common.CachePolicyConfig) { c.Empty = e } +} + +func covCacheWithTTL(d time.Duration) covCachePolicyOpt { + return func(c *common.CachePolicyConfig) { c.TTL = common.FixedDuration(d) } +} + +func covCachePolicy(t *testing.T, conn data.Connector, method string, fin common.DataFinalityState, opts ...covCachePolicyOpt) *data.CachePolicy { + t.Helper() + cfg := &common.CachePolicyConfig{ + Connector: conn.Id(), + Network: "*", + Method: method, + Finality: fin, + } + for _, o := range opts { + o(cfg) + } + p, err := data.NewCachePolicy(cfg, conn) + require.NoError(t, err) + return p +} + +func covCacheNew(lg zerolog.Logger, project string, policies ...*data.CachePolicy) *EvmJsonRpcCache { + return &EvmJsonRpcCache{projectId: project, logger: &lg, policies: policies} +} + +func covCacheDiscard(level zerolog.Level) zerolog.Logger { + return zerolog.New(io.Discard).Level(level) +} + +func covCacheReq(method, params string, ntw common.Network) *common.NormalizedRequest { + req := common.NewNormalizedRequest([]byte( + fmt.Sprintf(`{"jsonrpc":"2.0","method":%q,"params":%s,"id":7}`, method, params), + )) + if ntw != nil { + req.SetNetwork(ntw) + } + return req +} + +func covCacheResp(req *common.NormalizedRequest, result interface{}) *common.NormalizedResponse { + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(common.MustNewJsonRpcResponse(7, result, nil)) +} + +func covCacheBlock(num string) map[string]interface{} { + return map[string]interface{}{"number": num, "hash": "0x" + strings.Repeat("ab", 32)} +} + +func TestCovCache_NewEvmJsonRpcCache(t *testing.T) { + ctx := context.Background() + lg := covCacheDiscard(zerolog.Disabled) + memConn := func(id string, tags ...string) *common.ConnectorConfig { + return &common.ConnectorConfig{ + Id: id, + Driver: common.DriverMemory, + Tags: tags, + Memory: &common.MemoryConnectorConfig{MaxItems: 100, MaxTotalSize: "1MB"}, + } + } + + t.Run("InvalidConnectorDriver", func(t *testing.T) { + _, err := NewEvmJsonRpcCache(ctx, &lg, &common.CacheConfig{ + Connectors: []*common.ConnectorConfig{{Id: "bad", Driver: "nope"}}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to create connector bad") + var drvErr *common.ErrInvalidConnectorDriver + assert.ErrorAs(t, err, &drvErr) + }) + + t.Run("PolicyReferencesUnknownConnector", func(t *testing.T) { + _, err := NewEvmJsonRpcCache(ctx, &lg, &common.CacheConfig{ + Connectors: []*common.ConnectorConfig{memConn("m1")}, + Policies: []*common.CachePolicyConfig{{Connector: "missing", Network: "*", Method: "*"}}, + }) + require.Error(t, err) + assert.Equal(t, "connector missing not found for policy", err.Error()) + }) + + t.Run("InvalidPolicySize", func(t *testing.T) { + bad := "lots" + _, err := NewEvmJsonRpcCache(ctx, &lg, &common.CacheConfig{ + Connectors: []*common.ConnectorConfig{memConn("m1")}, + Policies: []*common.CachePolicyConfig{{Connector: "m1", Network: "*", Method: "*", MaxItemSize: &bad}}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to create policy") + assert.Contains(t, err.Error(), "invalid maxItemSize") + }) + + t.Run("ConnectorTagsGatePolicies", func(t *testing.T) { + c, err := NewEvmJsonRpcCache(ctx, &lg, &common.CacheConfig{ + Connectors: []*common.ConnectorConfig{memConn("tagged", "systx"), memConn("plain")}, + Policies: []*common.CachePolicyConfig{ + {Connector: "tagged", Network: "*", Method: "*"}, + {Connector: "plain", Network: "*", Method: "*"}, + }, + }) + require.NoError(t, err) + require.Len(t, c.policies, 2) + ok, err := c.policies[0].MatchesUpstreamSelector("alchemy") + require.NoError(t, err) + assert.False(t, ok, "tagged connector must be excluded by a non-matching selector") + ok, _ = c.policies[0].MatchesUpstreamSelector("systx") + assert.True(t, ok) + ok, _ = c.policies[1].MatchesUpstreamSelector("alchemy") + assert.True(t, ok, "untagged connector is always eligible") + assert.False(t, c.compressionEnabled) + }) + + t.Run("CompressionLevels", func(t *testing.T) { + cases := []struct { + level string + threshold int + wantLevel zstd.EncoderLevel + wantThr int + }{ + {"", 0, zstd.SpeedFastest, 512}, + {"fastest", 10, zstd.SpeedFastest, 10}, + {"DEFAULT", 0, zstd.SpeedDefault, 512}, + {"better", 0, zstd.SpeedBetterCompression, 512}, + {"best", 2048, zstd.SpeedBestCompression, 2048}, + {"turbo", 0, zstd.SpeedFastest, 512}, + } + for _, tc := range cases { + t.Run("level="+tc.level, func(t *testing.T) { + c, err := NewEvmJsonRpcCache(ctx, &lg, &common.CacheConfig{ + Compression: &common.CompressionConfig{Enabled: util.BoolPtr(true), ZstdLevel: tc.level, Threshold: tc.threshold}, + }) + require.NoError(t, err) + assert.True(t, c.compressionEnabled) + assert.Equal(t, tc.wantLevel, c.compressionLevel) + assert.Equal(t, tc.wantThr, c.compressionThreshold) + + clone := c.WithProjectId("p1") + assert.Equal(t, "p1", clone.projectId) + assert.Equal(t, tc.wantLevel, clone.compressionLevel) + assert.Equal(t, tc.wantThr, clone.compressionThreshold) + }) + } + }) + + t.Run("CompressionDisabledExplicitly", func(t *testing.T) { + c, err := NewEvmJsonRpcCache(ctx, &lg, &common.CacheConfig{ + Compression: &common.CompressionConfig{Enabled: util.BoolPtr(false)}, + }) + require.NoError(t, err) + assert.False(t, c.compressionEnabled) + assert.Nil(t, c.encoderPool) + }) +} + +func TestCovCache_IsObjectNull(t *testing.T) { + var nilCache *EvmJsonRpcCache + assert.True(t, nilCache.IsObjectNull()) + assert.True(t, (&EvmJsonRpcCache{}).IsObjectNull()) + lg := covCacheDiscard(zerolog.Disabled) + assert.False(t, covCacheNew(lg, "p").IsObjectNull()) +} + +func TestCovCache_SetThenGetRoundTrip(t *testing.T) { + ctx := context.Background() + ntw := &testNetwork{finalityState: common.DataFinalityStateFinalized} + conn := covCacheNewConn("rt") + c := covCacheNew(covCacheDiscard(zerolog.TraceLevel), "covcache-rt", + covCachePolicy(t, conn, "eth_getBlockByNumber", common.DataFinalityStateFinalized, covCacheWithTTL(time.Hour))) + + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw) + require.NoError(t, c.Set(ctx, req, covCacheResp(req, covCacheBlock("0x10")))) + pk, _, stored, ttl := conn.snapshot() + assert.Equal(t, "test-network:16", pk, "numeric block keys the partition by decimal block number") + require.NotNil(t, ttl) + assert.Equal(t, time.Hour, *ttl) + assert.Contains(t, string(stored), `"number":"0x10"`) + + getReq := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw) + resp, err := c.Get(ctx, getReq) + require.NoError(t, err) + require.NotNil(t, resp) + assert.True(t, resp.FromCache()) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + assert.Contains(t, jrr.GetResultString(), `"number":"0x10"`) + assert.Equal(t, int64(7), jrr.ID()) + _, idx, _, _ := conn.snapshot() + assert.Equal(t, data.ConnectorMainIndex, idx) + + // A different block number is a different key: miss. + miss, err := c.Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x11",false]`, ntw)) + require.NoError(t, err) + assert.Nil(t, miss) +} + +func TestCovCache_GetFinalityPolicySelection(t *testing.T) { + ctx := context.Background() + cases := []struct { + name string + finality common.DataFinalityState + want map[string]int32 // connector id -> expected Get calls + }{ + {"Finalized", common.DataFinalityStateFinalized, map[string]int32{"fin": 1, "unfin": 1, "rt": 0}}, + {"Unfinalized", common.DataFinalityStateUnfinalized, map[string]int32{"fin": 0, "unfin": 1, "rt": 0}}, + {"Realtime", common.DataFinalityStateRealtime, map[string]int32{"fin": 0, "unfin": 0, "rt": 1}}, + {"Unknown", common.DataFinalityStateUnknown, map[string]int32{"fin": 1, "unfin": 1, "rt": 1}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + conns := map[string]*covCacheConn{ + "fin": covCacheNewConn("fin"), + "unfin": covCacheNewConn("unfin"), + "rt": covCacheNewConn("rt"), + } + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-fin", + covCachePolicy(t, conns["fin"], "*", common.DataFinalityStateFinalized), + covCachePolicy(t, conns["unfin"], "*", common.DataFinalityStateUnfinalized), + covCachePolicy(t, conns["rt"], "*", common.DataFinalityStateRealtime), + ) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, &testNetwork{finalityState: tc.finality}) + resp, err := c.Get(ctx, req) + require.NoError(t, err) + assert.Nil(t, resp, "all connectors are empty") + for id, want := range tc.want { + assert.Equal(t, want, conns[id].gets.Load(), "connector %s", id) + } + }) + } +} + +func TestCovCache_GetPolicyMatching(t *testing.T) { + ctx := context.Background() + ntw := &testNetwork{finalityState: common.DataFinalityStateFinalized} + + t.Run("SameConnectorDedupedAcrossPolicies", func(t *testing.T) { + conn := covCacheNewConn("shared") + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-dedup", + covCachePolicy(t, conn, "eth_*", common.DataFinalityStateFinalized), + covCachePolicy(t, conn, "*", common.DataFinalityStateFinalized), + ) + resp, err := c.Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw)) + require.NoError(t, err) + assert.Nil(t, resp) + assert.Equal(t, int32(1), conn.gets.Load()) + }) + + t.Run("MethodAndParamsMismatchSkipsCache", func(t *testing.T) { + conn := covCacheNewConn("pm") + p, err := data.NewCachePolicy(&common.CachePolicyConfig{ + Connector: "pm", Network: "*", Method: "eth_getBlockByNumber", + Params: []interface{}{"0x20"}, Finality: common.DataFinalityStateFinalized, + }, conn) + require.NoError(t, err) + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-pm", p) + resp, err := c.Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw)) + require.NoError(t, err) + assert.Nil(t, resp) + resp, err = c.Get(ctx, covCacheReq("eth_getBalance", `["0xabc","0x10"]`, ntw)) + require.NoError(t, err) + assert.Nil(t, resp) + assert.Equal(t, int32(0), conn.gets.Load(), "no policy matched, connector never consulted") + + _, err = c.Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x20",false]`, ntw)) + require.NoError(t, err) + assert.Equal(t, int32(1), conn.gets.Load(), "matching params consult the connector") + }) + + t.Run("InvalidMethodPatternErrorsOnGetAndSet", func(t *testing.T) { + conn := covCacheNewConn("bad") + c := covCacheNew(covCacheDiscard(zerolog.TraceLevel), "covcache-badpat", + covCachePolicy(t, conn, "(", common.DataFinalityStateFinalized)) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw) + _, err := c.Get(ctx, req) + require.Error(t, err) + assert.Contains(t, err.Error(), "expected closing parenthesis") + + err = c.Set(ctx, req, covCacheResp(req, covCacheBlock("0x10"))) + require.Error(t, err) + assert.Contains(t, err.Error(), "expected closing parenthesis") + assert.Equal(t, int32(0), conn.sets.Load()) + }) + + t.Run("InvalidRequestBody", func(t *testing.T) { + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-badbody", + covCachePolicy(t, covCacheNewConn("x"), "*", common.DataFinalityStateUnknown)) + req := common.NewNormalizedRequest([]byte(`{not json`)) + _, err := c.Get(ctx, req) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal), "got %v", err) + + req2 := common.NewNormalizedRequest([]byte(`{not json`)) + err = c.Set(ctx, req2, common.NewNormalizedResponse().WithJsonRpcResponse(common.MustNewJsonRpcResponse(1, "0x1", nil))) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal), "got %v", err) + }) +} + +func TestCovCache_GetConnectorOutcomes(t *testing.T) { + ctx := context.Background() + ntw := &testNetwork{finalityState: common.DataFinalityStateFinalized} + newCache := func(t *testing.T, conn *covCacheConn, opts ...covCachePolicyOpt) *EvmJsonRpcCache { + return covCacheNew(covCacheDiscard(zerolog.DebugLevel), "covcache-outcomes", + covCachePolicy(t, conn, "*", common.DataFinalityStateFinalized, opts...)) + } + + semanticMisses := []error{ + common.NewErrRecordNotFound("pk", "rk", "x"), + common.NewErrRecordExpired("pk", "rk", "x", 2, 1), + common.NewErrEndpointMissingData(errors.New("range outside available"), nil), + } + for _, e := range semanticMisses { + t.Run("SemanticMiss/"+e.(interface{ CodeChain() string }).CodeChain(), func(t *testing.T) { + conn := covCacheNewConn("sem") + conn.getErr = e + resp, err := newCache(t, conn).Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw)) + require.NoError(t, err) + assert.Nil(t, resp) + assert.Equal(t, int32(1), conn.gets.Load()) + }) + } + + t.Run("ConnectorErrorFallsThroughAsMiss", func(t *testing.T) { + conn := covCacheNewConn("err") + conn.getErr = errors.New("redis: connection refused") + resp, err := newCache(t, conn).Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw)) + require.NoError(t, err, "connector errors are not surfaced to the caller") + assert.Nil(t, resp) + }) + + t.Run("EmptyBytesIsMiss", func(t *testing.T) { + conn := covCacheNewConn("eb") + conn.getBytes = []byte{} + resp, err := newCache(t, conn).Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw)) + require.NoError(t, err) + assert.Nil(t, resp) + assert.Equal(t, int32(1), conn.gets.Load()) + }) + + t.Run("UnknownMethodHasNoBlockRef", func(t *testing.T) { + conn := covCacheNewConn("um") + conn.getBytes = []byte(`"0x1"`) + resp, err := newCache(t, conn).Get(ctx, covCacheReq("custom_unknownMethod", `[]`, ntw)) + require.NoError(t, err) + assert.Nil(t, resp) + assert.Equal(t, int32(0), conn.gets.Load(), "no block ref: the connector must not be read") + }) + + t.Run("MalformedBlockNumber", func(t *testing.T) { + conn := covCacheNewConn("mb") + conn.getBytes = []byte(`{"number":"0x1"}`) + c := newCache(t, conn) + req := covCacheReq("eth_getBlockByNumber", `["0xzz",false]`, ntw) + resp, err := c.Get(ctx, req) + require.NoError(t, err) + assert.Nil(t, resp) + assert.Equal(t, int32(0), conn.gets.Load()) + + setReq := covCacheReq("eth_getBlockByNumber", `["0xzz",false]`, ntw) + err = c.Set(ctx, setReq, covCacheResp(setReq, covCacheBlock("0x1"))) + require.Error(t, err) + assert.Equal(t, int32(0), conn.sets.Load()) + }) + + t.Run("UnhashableParamFailsKeyGeneration", func(t *testing.T) { + conn := covCacheNewConn("uh") + conn.getBytes = []byte(`{"blockNumber":"0x1"}`) + c := newCache(t, conn) + mk := func() *common.NormalizedRequest { + r := common.NewNormalizedRequestFromJsonRpcRequest( + common.NewJsonRpcRequest("eth_getTransactionReceipt", []interface{}{int64(5)}), + ) + r.SetNetwork(ntw) + return r + } + resp, err := c.Get(ctx, mk()) + require.NoError(t, err) + assert.Nil(t, resp) + assert.Equal(t, int32(0), conn.gets.Load()) + + req := mk() + err = c.Set(ctx, req, covCacheResp(req, map[string]interface{}{"blockNumber": "0x1"})) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported type for value during hash") + }) +} + +func TestCovCache_GetEmptyishBehavior(t *testing.T) { + ctx := context.Background() + ntw := &testNetwork{finalityState: common.DataFinalityStateFinalized} + cases := []struct { + name string + empty common.CacheEmptyBehavior + wantHit bool + }{ + {"Ignore", common.CacheEmptyBehaviorIgnore, false}, + {"Allow", common.CacheEmptyBehaviorAllow, true}, + {"Only", common.CacheEmptyBehaviorOnly, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + conn := covCacheNewConn("em") + conn.getBytes = []byte(`null`) + c := covCacheNew(covCacheDiscard(zerolog.DebugLevel), "covcache-empty", + covCachePolicy(t, conn, "*", common.DataFinalityStateFinalized, covCacheWithEmpty(tc.empty))) + resp, err := c.Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw)) + require.NoError(t, err) + if !tc.wantHit { + assert.Nil(t, resp) + return + } + require.NotNil(t, resp) + assert.True(t, resp.FromCache()) + assert.True(t, resp.IsResultEmptyish()) + }) + } + + t.Run("IgnoredEmptyLetsPeerServe", func(t *testing.T) { + empty := covCacheNewConn("empty") + empty.getBytes = []byte(`null`) + full := covCacheNewConn("full") + full.getBytes = []byte(`{"number":"0x10"}`) + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-empty-peer", + covCachePolicy(t, empty, "*", common.DataFinalityStateFinalized), + covCachePolicy(t, full, "*", common.DataFinalityStateFinalized), + ) + resp, err := c.Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw)) + require.NoError(t, err) + require.NotNil(t, resp) + jrr, _ := resp.JsonRpcResponse() + assert.Equal(t, `{"number":"0x10"}`, jrr.GetResultString()) + }) +} + +func TestCovCache_GetHitLogging(t *testing.T) { + ctx := context.Background() + ntw := &testNetwork{finalityState: common.DataFinalityStateFinalized} + cases := []struct { + name string + level zerolog.Level + stored string + }{ + {"TraceJsonResult", zerolog.TraceLevel, `{"number":"0x10"}`}, + {"TraceNonJsonResult", zerolog.TraceLevel, `0x10`}, + {"InfoLevel", zerolog.InfoLevel, `{"number":"0x10"}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + conn := covCacheNewConn("lg") + conn.getBytes = []byte(tc.stored) + c := covCacheNew(covCacheDiscard(tc.level), "covcache-log", + covCachePolicy(t, conn, "*", common.DataFinalityStateFinalized)) + resp, err := c.Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw)) + require.NoError(t, err) + require.NotNil(t, resp) + jrr, _ := resp.JsonRpcResponse() + assert.Equal(t, tc.stored, jrr.GetResultString()) + }) + } +} + +func TestCovCache_GetDirectives(t *testing.T) { + ctx := context.Background() + ntw := &testNetwork{finalityState: common.DataFinalityStateFinalized} + const project = "covcache-directives" + + setup := func(t *testing.T) (*EvmJsonRpcCache, *covCacheConn, *data.CachePolicy) { + conn := covCacheNewConn("systx-cache") + conn.getBytes = []byte(`{"number":"0x10"}`) + p := covCachePolicy(t, conn, "*", common.DataFinalityStateFinalized) + p.SetConnectorTags([]string{"systx"}) + return covCacheNew(covCacheDiscard(zerolog.Disabled), project, p), conn, p + } + + t.Run("UseUpstreamExcludesTaggedConnector", func(t *testing.T) { + c, conn, p := setup(t) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw) + req.SetDirectives(&common.RequestDirectives{UseUpstream: "alchemy"}) + miss := telemetry.MetricCacheGetSuccessMissTotal.WithLabelValues(project, "test", "eth_getBlockByNumber", conn.Id(), p.String(), p.GetTTL().String(), "empty_result") + before := testutil.ToFloat64(miss) + resp, err := c.Get(ctx, req) + require.NoError(t, err) + assert.Nil(t, resp) + assert.Equal(t, int32(0), conn.gets.Load()) + assert.Equal(t, before+1, testutil.ToFloat64(miss), "a fully-skipped fan-out is attributed to the first policy") + }) + + t.Run("UseUpstreamAdmitsTaggedConnector", func(t *testing.T) { + c, conn, _ := setup(t) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw) + req.SetDirectives(&common.RequestDirectives{UseUpstream: "systx"}) + resp, err := c.Get(ctx, req) + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, int32(1), conn.gets.Load()) + }) + + t.Run("SkipCacheReadPattern", func(t *testing.T) { + c, conn, _ := setup(t) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw) + req.SetDirectives(&common.RequestDirectives{SkipCacheRead: "systx*"}) + resp, err := c.Get(ctx, req) + require.NoError(t, err) + assert.Nil(t, resp) + assert.Equal(t, int32(0), conn.gets.Load()) + + req2 := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw) + req2.SetDirectives(&common.RequestDirectives{SkipCacheRead: "redis*"}) + resp, err = c.Get(ctx, req2) + require.NoError(t, err) + require.NotNil(t, resp, "non-matching skip pattern still reads") + }) +} + +func TestCovCache_GetLogsRangeObservation(t *testing.T) { + ctx := context.Background() + ntw := &testNetwork{finalityState: common.DataFinalityStateFinalized} + cases := []struct { + name string + params string + }{ + {"TagRange", `[{"fromBlock":"0x1","toBlock":"latest"}]`}, + {"ConcreteRange", `[{"fromBlock":"0x1","toBlock":"0x5"}]`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + conn := covCacheNewConn("logs") + conn.getBytes = []byte(`[{"address":"0xabc","blockNumber":"0x2"}]`) + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-logs", + covCachePolicy(t, conn, "eth_getLogs", common.DataFinalityStateFinalized)) + resp, err := c.Get(ctx, covCacheReq("eth_getLogs", tc.params, ntw)) + require.NoError(t, err) + require.NotNil(t, resp) + assert.True(t, resp.FromCache()) + }) + } +} + +func TestCovCache_DetailedTracing(t *testing.T) { + common.IsTracingDetailed = true + t.Cleanup(func() { common.IsTracingDetailed = false }) + ctx := context.Background() + ntw := &testNetwork{finalityState: common.DataFinalityStateFinalized} + conn := covCacheNewConn("tr") + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-trace", + covCachePolicy(t, conn, "*", common.DataFinalityStateFinalized)) + + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw) + require.NoError(t, c.Set(ctx, req, covCacheResp(req, covCacheBlock("0x10")))) + assert.Equal(t, int32(1), conn.sets.Load()) + + resp, err := c.Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw)) + require.NoError(t, err) + require.NotNil(t, resp) +} + +func TestCovCache_SetDecisions(t *testing.T) { + ctx := context.Background() + + t.Run("UnknownMethodIsNeverCached", func(t *testing.T) { + for _, lvl := range []zerolog.Level{zerolog.TraceLevel, zerolog.DebugLevel} { + conn := covCacheNewConn("um") + c := covCacheNew(covCacheDiscard(lvl), "covcache-set", + covCachePolicy(t, conn, "*", common.DataFinalityStateUnknown)) + req := covCacheReq("custom_unknownMethod", `[]`, &testNetwork{finalityState: common.DataFinalityStateUnknown}) + require.NoError(t, c.Set(ctx, req, covCacheResp(req, "0x1"))) + assert.Equal(t, int32(0), conn.sets.Load(), "level %s", lvl) + } + }) + + t.Run("NoMatchingPolicyIsNoop", func(t *testing.T) { + conn := covCacheNewConn("nm") + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-set", + covCachePolicy(t, conn, "eth_getBalance", common.DataFinalityStateFinalized)) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, &testNetwork{finalityState: common.DataFinalityStateFinalized}) + require.NoError(t, c.Set(ctx, req, covCacheResp(req, covCacheBlock("0x10")))) + assert.Equal(t, int32(0), conn.sets.Load()) + }) + + t.Run("ErrorResponseIsNotCached", func(t *testing.T) { + conn := covCacheNewConn("er") + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-set", + covCachePolicy(t, conn, "*", common.DataFinalityStateFinalized, covCacheWithEmpty(common.CacheEmptyBehaviorAllow))) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, &testNetwork{finalityState: common.DataFinalityStateFinalized}) + resp := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse( + common.MustNewJsonRpcResponse(7, nil, common.NewErrJsonRpcExceptionExternal(-32000, "execution reverted", "")), + ) + require.NoError(t, c.Set(ctx, req, resp)) + assert.Equal(t, int32(0), conn.sets.Load()) + }) + + t.Run("EmptyBehaviorOnSet", func(t *testing.T) { + cases := []struct { + name string + empty common.CacheEmptyBehavior + result interface{} + wantSets int32 + }{ + {"IgnoreSkipsEmpty", common.CacheEmptyBehaviorIgnore, nil, 0}, + {"IgnoreStoresNonEmpty", common.CacheEmptyBehaviorIgnore, covCacheBlock("0x10"), 1}, + {"AllowStoresEmpty", common.CacheEmptyBehaviorAllow, nil, 1}, + {"OnlyStoresEmpty", common.CacheEmptyBehaviorOnly, nil, 1}, + {"OnlySkipsNonEmpty", common.CacheEmptyBehaviorOnly, covCacheBlock("0x10"), 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + conn := covCacheNewConn("eb") + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-set", + covCachePolicy(t, conn, "*", common.DataFinalityStateFinalized, covCacheWithEmpty(tc.empty))) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, &testNetwork{finalityState: common.DataFinalityStateFinalized}) + require.NoError(t, c.Set(ctx, req, covCacheResp(req, tc.result))) + assert.Equal(t, tc.wantSets, conn.sets.Load()) + }) + } + }) + + t.Run("SizeLimits", func(t *testing.T) { + small, large := "10B", "1KB" + cases := []struct { + name string + min, max *string + wantSets int32 + }{ + {"TooLarge", nil, &small, 0}, + {"TooSmall", &large, nil, 0}, + {"WithinLimits", &small, &large, 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + conn := covCacheNewConn("sz") + p, err := data.NewCachePolicy(&common.CachePolicyConfig{ + Connector: "sz", Network: "*", Method: "*", Finality: common.DataFinalityStateFinalized, + MinItemSize: tc.min, MaxItemSize: tc.max, + }, conn) + require.NoError(t, err) + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-set", p) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, &testNetwork{finalityState: common.DataFinalityStateFinalized}) + require.NoError(t, c.Set(ctx, req, covCacheResp(req, covCacheBlock("0x10")))) + assert.Equal(t, tc.wantSets, conn.sets.Load()) + }) + } + }) + + t.Run("EmptyResultBeyondConfidenceHead", func(t *testing.T) { + cases := []struct { + name string + head int64 + wantSets int32 + }{ + {"FutureBlockNotCached", 50, 0}, + {"PastBlockCached", 200, 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + conn := covCacheNewConn("fb") + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-set", + covCachePolicy(t, conn, "*", common.DataFinalityStateUnfinalized, covCacheWithEmpty(common.CacheEmptyBehaviorAllow))) + ntw := &testNetwork{ + finalityState: common.DataFinalityStateUnfinalized, + highestLatest: tc.head, + cfg: &common.NetworkConfig{Architecture: common.ArchitectureEvm, Evm: &common.EvmNetworkConfig{}}, + } + req := covCacheReq("eth_getBlockByNumber", `["0x64",false]`, ntw) + require.NoError(t, c.Set(ctx, req, covCacheResp(req, nil))) + assert.Equal(t, tc.wantSets, conn.sets.Load()) + }) + } + }) + + t.Run("EnforceHighestBlockSkipsLaggingRealtime", func(t *testing.T) { + const project = "covcache-enforce" + cases := []struct { + name string + tip int64 + enforce bool + wantSets int32 + }{ + {"BehindTipSkipped", 100, true, 0}, + {"AtTipCached", 16, true, 1}, + {"NotEnforcedCached", 100, false, 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + conn := covCacheNewConn("bn") + p := covCachePolicy(t, conn, "eth_blockNumber", common.DataFinalityStateRealtime, covCacheWithTTL(2*time.Second)) + c := covCacheNew(covCacheDiscard(zerolog.Disabled), project, p) + req := covCacheReq("eth_blockNumber", `[]`, &testNetwork{finalityState: common.DataFinalityStateRealtime, highestLatest: tc.tip}) + req.SetDirectives(&common.RequestDirectives{EnforceHighestBlock: tc.enforce}) + skipped := telemetry.MetricCacheSetSkippedTotal.WithLabelValues(project, "test", "eth_blockNumber", "bn", p.String(), p.GetTTL().String()) + before := testutil.ToFloat64(skipped) + require.NoError(t, c.Set(ctx, req, covCacheResp(req, "0x10"))) + assert.Equal(t, tc.wantSets, conn.sets.Load()) + assert.Equal(t, before+float64(1-tc.wantSets), testutil.ToFloat64(skipped)) + }) + } + }) + + t.Run("NoNetworkUsesFixedTTL", func(t *testing.T) { + conn := covCacheNewConn("nn") + c := covCacheNew(covCacheDiscard(zerolog.DebugLevel), "covcache-set", + covCachePolicy(t, conn, "*", common.DataFinalityStateUnknown, covCacheWithTTL(5*time.Second))) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, nil) + require.NoError(t, c.Set(ctx, req, covCacheResp(req, covCacheBlock("0x10")))) + pk, _, _, ttl := conn.snapshot() + assert.Equal(t, "n/a:16", pk) + require.NotNil(t, ttl) + assert.Equal(t, 5*time.Second, *ttl) + }) + + t.Run("AllMatchingPoliciesWritten", func(t *testing.T) { + a, b := covCacheNewConn("a"), covCacheNewConn("b") + other := covCacheNewConn("other") + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-set", + covCachePolicy(t, a, "eth_getBlockByNumber", common.DataFinalityStateFinalized), + covCachePolicy(t, b, "*", common.DataFinalityStateFinalized), + covCachePolicy(t, other, "*", common.DataFinalityStateUnfinalized), + ) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, &testNetwork{finalityState: common.DataFinalityStateFinalized}) + require.NoError(t, c.Set(ctx, req, covCacheResp(req, covCacheBlock("0x10")))) + assert.Equal(t, int32(1), a.sets.Load()) + assert.Equal(t, int32(1), b.sets.Load()) + assert.Equal(t, int32(0), other.sets.Load(), "finality mismatch on set") + }) +} + +func TestCovCache_SetConnectorErrors(t *testing.T) { + ctx := context.Background() + ntw := &testNetwork{finalityState: common.DataFinalityStateFinalized} + + t.Run("SingleFailureReturnedAsIs", func(t *testing.T) { + boom := errors.New("dynamodb: ProvisionedThroughputExceededException") + bad, good := covCacheNewConn("bad"), covCacheNewConn("good") + bad.setErr = boom + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-seterr", + covCachePolicy(t, bad, "*", common.DataFinalityStateFinalized), + covCachePolicy(t, good, "*", common.DataFinalityStateFinalized), + ) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw) + err := c.Set(ctx, req, covCacheResp(req, covCacheBlock("0x10"))) + require.ErrorIs(t, err, boom) + assert.Equal(t, int32(1), good.sets.Load(), "a failing peer does not block other writes") + }) + + t.Run("MultipleFailuresAggregated", func(t *testing.T) { + a, b := covCacheNewConn("a"), covCacheNewConn("b") + a.setErr = errors.New("redis: i/o timeout") + b.setErr = errors.New("postgres: connection reset") + c := covCacheNew(covCacheDiscard(zerolog.Disabled), "covcache-seterr", + covCachePolicy(t, a, "*", common.DataFinalityStateFinalized), + covCachePolicy(t, b, "*", common.DataFinalityStateFinalized), + ) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw) + err := c.Set(ctx, req, covCacheResp(req, covCacheBlock("0x10"))) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to set cache for 2 policies") + assert.Contains(t, err.Error(), "redis: i/o timeout") + assert.Contains(t, err.Error(), "postgres: connection reset") + }) +} + +func TestCovCache_CompressionRoundTrip(t *testing.T) { + ctx := context.Background() + lg := covCacheDiscard(zerolog.Disabled) + c, err := NewEvmJsonRpcCache(ctx, &lg, &common.CacheConfig{ + Compression: &common.CompressionConfig{Enabled: util.BoolPtr(true), Threshold: 64}, + }) + require.NoError(t, err) + conn := covCacheNewConn("z") + c.SetPolicies([]*data.CachePolicy{covCachePolicy(t, conn, "*", common.DataFinalityStateFinalized)}) + ntw := &testNetwork{finalityState: common.DataFinalityStateFinalized} + + block := covCacheBlock("0x10") + block["extraData"] = "0x" + strings.Repeat("00", 1024) + req := covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw) + require.NoError(t, c.Set(ctx, req, covCacheResp(req, block))) + _, _, stored, _ := conn.snapshot() + require.True(t, c.isCompressed(stored), "large value must be stored zstd-compressed") + assert.Less(t, len(stored), 1024) + + resp, err := c.Get(ctx, covCacheReq("eth_getBlockByNumber", `["0x10",false]`, ntw)) + require.NoError(t, err) + require.NotNil(t, resp) + jrr, _ := resp.JsonRpcResponse() + assert.Contains(t, jrr.GetResultString(), strings.Repeat("00", 1024)) + + // Small values stay below threshold and are stored verbatim. + req2 := covCacheReq("eth_getBlockByNumber", `["0x11",false]`, ntw) + require.NoError(t, c.Set(ctx, req2, covCacheResp(req2, "0x1"))) + _, _, stored, _ = conn.snapshot() + assert.Equal(t, `"0x1"`, string(stored)) +} + +func TestCovCache_CompressionHelpers(t *testing.T) { + lg := covCacheDiscard(zerolog.Disabled) + c, err := NewEvmJsonRpcCache(context.Background(), &lg, &common.CacheConfig{ + Compression: &common.CompressionConfig{Enabled: util.BoolPtr(true), Threshold: 16}, + }) + require.NoError(t, err) + + t.Run("BelowThresholdUntouched", func(t *testing.T) { + in := []byte("short") + out, ok := c.compressValueBytes(in) + assert.False(t, ok) + assert.Equal(t, in, out) + }) + + t.Run("IncompressibleStoredRaw", func(t *testing.T) { + in := make([]byte, 64) + _, _ = rand.NewChaCha8([32]byte{42}).Read(in) + out, ok := c.compressValueBytes(in) + assert.False(t, ok, "zstd output larger than input must be discarded") + assert.Equal(t, in, out) + }) + + t.Run("CompressibleRoundTrip", func(t *testing.T) { + in := bytes.Repeat([]byte("abcd"), 100) + out, ok := c.compressValueBytes(in) + require.True(t, ok) + require.True(t, c.isCompressed(out)) + back, err := c.decompressValueBytes(out) + require.NoError(t, err) + assert.Equal(t, in, back) + }) + + t.Run("DecompressPassThroughUncompressed", func(t *testing.T) { + in := []byte(`{"a":1}`) + out, err := c.decompressValueBytes(in) + require.NoError(t, err) + assert.Equal(t, in, out) + }) + + t.Run("DecompressCorruptFrame", func(t *testing.T) { + corrupt := append([]byte{0x28, 0xB5, 0x2F, 0xFD}, bytes.Repeat([]byte{0xFF}, 16)...) + _, err := c.decompressValueBytes(corrupt) + require.Error(t, err) + }) + + t.Run("EmptyPools", func(t *testing.T) { + nilPool := &sync.Pool{New: func() interface{} { return nil }} + broken := &EvmJsonRpcCache{ + logger: &lg, + compressionEnabled: true, + compressionThreshold: 1, + encoderPool: nilPool, + decoderPool: nilPool, + } + in := bytes.Repeat([]byte("x"), 32) + out, ok := broken.compressValueBytes(in) + assert.False(t, ok) + assert.Equal(t, in, out) + + compressed, ok := c.compressValueBytes(bytes.Repeat([]byte("abcd"), 100)) + require.True(t, ok) + _, err := broken.decompressValueBytes(compressed) + require.Error(t, err) + assert.Equal(t, "failed to get decoder from pool", err.Error()) + }) + + t.Run("InvalidEncoderLevelStoresUncompressed", func(t *testing.T) { + bad, err := NewEvmJsonRpcCache(context.Background(), &lg, &common.CacheConfig{ + Compression: &common.CompressionConfig{Enabled: util.BoolPtr(true), Threshold: 1}, + }) + require.NoError(t, err) + // The pool's New reads the level lazily; an invalid level makes zstd + // refuse to build an encoder, so the value must be stored as-is. + bad.compressionLevel = zstd.EncoderLevel(99) + in := bytes.Repeat([]byte("abcd"), 100) + out, ok := bad.compressValueBytes(in) + assert.False(t, ok) + assert.Equal(t, in, out) + }) + + t.Run("ClosedDecoderFailsReset", func(t *testing.T) { + closed := &EvmJsonRpcCache{ + logger: &lg, + compressionEnabled: true, + decoderPool: &sync.Pool{New: func() interface{} { + d, err := zstd.NewReader(nil) + if err != nil { + return nil + } + d.Close() + return d + }}, + } + compressed, ok := c.compressValueBytes(bytes.Repeat([]byte("abcd"), 100)) + require.True(t, ok) + _, err := closed.decompressValueBytes(compressed) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to reset zstd decoder") + }) +} diff --git a/architecture/evm/cov_errnorm_test.go b/architecture/evm/cov_errnorm_test.go new file mode 100644 index 000000000..7c2d46cfc --- /dev/null +++ b/architecture/evm/cov_errnorm_test.go @@ -0,0 +1,592 @@ +package evm + +import ( + "encoding/json" + "errors" + "net/http" + "testing" + + bdscommon "github.com/blockchain-data-standards/manifesto/common" + "github.com/erpc/erpc/common" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// covErrNormInternal returns the ErrJsonRpcExceptionInternal carried in err's chain. +func covErrNormInternal(t *testing.T, err error) *common.ErrJsonRpcExceptionInternal { + t.Helper() + jre := &common.ErrJsonRpcExceptionInternal{} + if !errors.As(err, &jre) { + t.Fatalf("expected ErrJsonRpcExceptionInternal in chain, got %T: %v", err, err) + } + return jre +} + +func covErrNormResponse(method string) *common.NormalizedResponse { + if method == "" { + return nil + } + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","method":"` + method + `","params":[],"id":1}`)) + return common.NewNormalizedResponse().WithRequest(req) +} + +// HTTP failures without a JSON-RPC error body are classified by status code alone. +func TestCovErrNorm_HttpStatusWithoutJsonRpcError(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + status int + method string + wantCode common.ErrorCode + wantNormalized common.JsonRpcErrorNumber + wantMessage string + }{ + {name: "500 is server-side", status: 500, wantCode: common.ErrCodeEndpointServerSideException, + wantNormalized: common.JsonRpcErrorServerSideException, wantMessage: "unexpected http failure with status code 500"}, + {name: "503 is server-side", status: 503, wantCode: common.ErrCodeEndpointServerSideException, + wantNormalized: common.JsonRpcErrorServerSideException, wantMessage: "unexpected http failure with status code 503"}, + {name: "429 is capacity exceeded", status: 429, wantCode: common.ErrCodeEndpointCapacityExceeded, + wantNormalized: common.JsonRpcErrorCapacityExceeded, wantMessage: "unexpected http failure with status code 429"}, + {name: "402 is billing", status: 402, wantCode: common.ErrCodeEndpointBillingIssue, + wantNormalized: common.JsonRpcErrorCapacityExceeded, wantMessage: "unexpected http failure with status code 402"}, + {name: "401 is unauthorized", status: 401, wantCode: common.ErrCodeEndpointUnauthorized, + wantNormalized: common.JsonRpcErrorUnauthorized, wantMessage: "unexpected http failure with status code 401"}, + {name: "403 is unauthorized", status: 403, wantCode: common.ErrCodeEndpointUnauthorized, + wantNormalized: common.JsonRpcErrorUnauthorized, wantMessage: "unexpected http failure with status code 403"}, + {name: "405 is unsupported", status: 405, method: "eth_getProof", wantCode: common.ErrCodeEndpointUnsupported, + wantNormalized: common.JsonRpcErrorUnsupportedException, wantMessage: "The method eth_getProof does not exist/is not available"}, + {name: "415 is unsupported without request", status: 415, wantCode: common.ErrCodeEndpointUnsupported, + wantNormalized: common.JsonRpcErrorUnsupportedException, wantMessage: "The method does not exist/is not available"}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + r := &http.Response{StatusCode: tc.status, Header: http.Header{}} + err := ExtractJsonRpcError(r, covErrNormResponse(tc.method), nil, nil) + if !common.HasErrorCode(err, tc.wantCode) { + t.Fatalf("expected %s, got %T: %v", tc.wantCode, err, err) + } + jre := covErrNormInternal(t, err) + if got := jre.NormalizedCode(); got != tc.wantNormalized { + t.Fatalf("normalized code: got %d, want %d", got, tc.wantNormalized) + } + if got := jre.OriginalCode(); got != int(common.JsonRpcErrorServerSideException) { + t.Fatalf("original code: got %d, want %d", got, common.JsonRpcErrorServerSideException) + } + if jre.Message != tc.wantMessage { + t.Fatalf("message: got %q, want %q", jre.Message, tc.wantMessage) + } + if got := jre.Details["statusCode"]; got != tc.status { + t.Fatalf("details.statusCode: got %v, want %d", got, tc.status) + } + }) + } +} + +// The upstream status code is preserved on server-side exceptions. +func TestCovErrNorm_ServerSidePreservesStatusCode(t *testing.T) { + t.Parallel() + + r := &http.Response{StatusCode: 502, Header: http.Header{}} + err := ExtractJsonRpcError(r, nil, nil, nil) + sc, ok := err.(interface{ ErrorStatusCode() int }) + if !ok { + t.Fatalf("expected ErrorStatusCode on %T", err) + } + if sc.ErrorStatusCode() != 502 { + t.Fatalf("ErrorStatusCode: got %d, want 502", sc.ErrorStatusCode()) + } +} + +// The "data" field is folded into details: Alchemy's "Reverted " prefix is +// stripped, structured data passes through as-is, and both feed text checks. +func TestCovErrNorm_DataFieldShapes(t *testing.T) { + t.Parallel() + + const revertData = "0x08c379a00000000000000000000000000000000000000000000000000000000000000020" + + t.Run("alchemy Reverted prefix is stripped", func(t *testing.T) { + t.Parallel() + r := &http.Response{StatusCode: 200, Header: http.Header{}} + jr := common.MustNewJsonRpcResponse(1, nil, + common.NewErrJsonRpcExceptionExternal(3, "execution reverted", "Reverted "+revertData)) + err := ExtractJsonRpcError(r, covErrNormResponse("eth_call"), jr, nil) + if !common.HasErrorCode(err, common.ErrCodeEndpointExecutionException) { + t.Fatalf("expected ErrEndpointExecutionException, got %T: %v", err, err) + } + jre := covErrNormInternal(t, err) + if jre.NormalizedCode() != common.JsonRpcErrorEvmReverted { + t.Fatalf("normalized code: got %d", jre.NormalizedCode()) + } + if got := jre.Details["data"]; got != revertData { + t.Fatalf("details.data: got %v, want %s", got, revertData) + } + if common.IsRetryableTowardNetwork(err) { + t.Fatalf("eth_call revert must not be retried toward the network") + } + }) + + t.Run("plain string data is kept verbatim", func(t *testing.T) { + t.Parallel() + r := &http.Response{StatusCode: 200, Header: http.Header{}} + jr := common.MustNewJsonRpcResponse(1, nil, + common.NewErrJsonRpcExceptionExternal(3, "execution reverted", revertData)) + err := ExtractJsonRpcError(r, nil, jr, nil) + jre := covErrNormInternal(t, err) + if got := jre.Details["data"]; got != revertData { + t.Fatalf("details.data: got %v, want %s", got, revertData) + } + }) + + t.Run("structured data passes through and is text-matched", func(t *testing.T) { + t.Parallel() + r := &http.Response{StatusCode: 200, Header: http.Header{}} + jrErr := common.NewErrJsonRpcExceptionExternal(-32000, "call failed", "") + dataMap := map[string]interface{}{"reason": "header not found"} + jrErr.Data = dataMap + jr := common.MustNewJsonRpcResponse(1, nil, jrErr) + err := ExtractJsonRpcError(r, nil, jr, nil) + // The message alone matches nothing; the "header not found" lives only in data. + if !common.HasErrorCode(err, common.ErrCodeEndpointMissingData) { + t.Fatalf("expected ErrEndpointMissingData from data text, got %T: %v", err, err) + } + jre := covErrNormInternal(t, err) + got, ok := jre.Details["data"].(map[string]interface{}) + if !ok || got["reason"] != "header not found" { + t.Fatalf("details.data must pass the map through, got %#v", jre.Details["data"]) + } + if jre.Message != "call failed" { + t.Fatalf("message must stay the vendor message, got %q", jre.Message) + } + }) + + t.Run("data parsed from raw error body", func(t *testing.T) { + t.Parallel() + r := &http.Response{StatusCode: 200, Header: http.Header{}} + jr := common.MustNewJsonRpcResponseFromBytes([]byte("1"), nil, + []byte(`{"code":-32600,"message":"Invalid Request","data":{"message":"Cancelled due to validation errors in batch request"}}`)) + err := ExtractJsonRpcError(r, nil, jr, nil) + if !common.HasErrorCode(err, common.ErrCodeEndpointClientSideException) { + t.Fatalf("expected ErrEndpointClientSideException, got %T: %v", err, err) + } + jre := covErrNormInternal(t, err) + if jre.NormalizedCode() != common.JsonRpcErrorCallException { + t.Fatalf("normalized code: got %d, want %d", jre.NormalizedCode(), common.JsonRpcErrorCallException) + } + if !common.IsRetryableTowardNetwork(err) { + t.Fatalf("batch validation cancellation must stay retryable so the batch can be split") + } + }) +} + +// Table of JSON-RPC error code/message → erpc classification for branches not +// exercised by error_normalizer_test.go. +func TestCovErrNorm_MessageClassification(t *testing.T) { + t.Parallel() + + type tcase struct { + name string + status int + code int + message string + data interface{} + method string + wantCode common.ErrorCode + wantNormalized common.JsonRpcErrorNumber + wantMessage string // empty: vendor message verbatim + wantRetryNw bool + } + cases := []tcase{ + // Range-too-large. + {name: "alchemy block range hint", code: -32602, message: "Log response size exceeded. You can make eth_getLogs requests with up to a 2K block range and no limit on the response size, or you can request any block range with a cap of 10K logs in the response. Based on your parameters, this block range should work: [0x1, 0x2]", + wantCode: common.ErrCodeEndpointRequestTooLarge, wantNormalized: common.JsonRpcErrorEvmLargeRange, wantMessage: "request exceeded max allowed range: Log response size exceeded. You can make eth_getLogs requests with up to a 2K block range and no limit on the response size, or you can request any block range with a cap of 10K logs in the response. Based on your parameters, this block range should work: [0x1, 0x2]", wantRetryNw: true}, + {name: "maximum blocks distance", code: -32000, message: "maximum 10000 blocks distance between fromBlock and toBlock", + wantCode: common.ErrCodeEndpointRequestTooLarge, wantNormalized: common.JsonRpcErrorEvmLargeRange, wantMessage: "request exceeded max allowed range: maximum 10000 blocks distance between fromBlock and toBlock", wantRetryNw: true}, + + // Billing vs capacity. + {name: "alchemy monthly capacity is billing", code: 429, message: "Monthly capacity limit exceeded. Visit https://dashboard.alchemyapi.io/settings/billing to upgrade your scaling policy for continued service.", + wantCode: common.ErrCodeEndpointBillingIssue, wantNormalized: common.JsonRpcErrorCapacityExceeded, wantRetryNw: true}, + {name: "free tier is billing", code: -32000, message: "You have reached the free tier daily limit", + wantCode: common.ErrCodeEndpointBillingIssue, wantNormalized: common.JsonRpcErrorCapacityExceeded, wantRetryNw: true}, + {name: "rate limit message", code: -32005, message: "rate limit exceeded", + wantCode: common.ErrCodeEndpointCapacityExceeded, wantNormalized: common.JsonRpcErrorCapacityExceeded, wantRetryNw: true}, + {name: "429 status with generic message", status: 429, code: -32000, message: "slow down", + wantCode: common.ErrCodeEndpointCapacityExceeded, wantNormalized: common.JsonRpcErrorCapacityExceeded, wantRetryNw: true}, + + // Block tags. + {name: "pending block not available", code: -32000, message: "pending block is not available", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorClientSideException, wantRetryNw: true}, + {name: "finalized commitment unsupported", code: -32602, message: "finalized is not a supported commitment", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorClientSideException, wantRetryNw: true}, + + // Known missing data. + {name: "missing trie node", code: -32000, message: "missing trie node 0000000000000000000000000000000000000000000000000000000000000000 (path ) ", + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryNw: true}, + {name: "required historical state", code: -32000, message: "required historical state is not available", + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryNw: true}, + + // Node timeout. + {name: "execution timeout", code: -32000, message: "execution timeout", + wantCode: common.ErrCodeEndpointServerSideException, wantNormalized: common.JsonRpcErrorNodeTimeout, wantRetryNw: true}, + + // Reverts. + {name: "VM Exception", code: -32000, message: "VM Exception while processing transaction: revert", + wantCode: common.ErrCodeEndpointExecutionException, wantNormalized: common.JsonRpcErrorEvmReverted}, + {name: "intrinsic gas too high (case-insensitive)", code: -32000, message: "Intrinsic Gas Too High", + wantCode: common.ErrCodeEndpointExecutionException, wantNormalized: common.JsonRpcErrorEvmReverted}, + {name: "revert on sendRawTransaction is retried", code: 3, message: "execution reverted", method: "eth_sendRawTransaction", + wantCode: common.ErrCodeEndpointExecutionException, wantNormalized: common.JsonRpcErrorEvmReverted, wantRetryNw: true}, + + // EVM halts ("EVM error: "). + {name: "berachain InvalidJump", code: -32000, message: "EVM error: InvalidJump", method: "eth_call", + wantCode: common.ErrCodeEndpointExecutionException, wantNormalized: common.JsonRpcErrorEvmReverted, wantMessage: "revert: invalid jump destination"}, + {name: "revm OutOfGas halt", code: -32603, message: "EVM error: OutOfGas(Basic)", method: "eth_estimateGas", + wantCode: common.ErrCodeEndpointExecutionException, wantNormalized: common.JsonRpcErrorEvmReverted, wantMessage: "revert: OutOfGas(Basic)"}, + {name: "halt reason trimmed", code: -32603, message: "EVM error: StackOverflow ", + wantCode: common.ErrCodeEndpointExecutionException, wantNormalized: common.JsonRpcErrorEvmReverted, wantMessage: "revert: StackOverflow"}, + {name: "halt on sendRawTransaction is retried", code: -32603, message: "EVM error: InvalidFEOpcode", method: "eth_sendRawTransaction", + wantCode: common.ErrCodeEndpointExecutionException, wantNormalized: common.JsonRpcErrorEvmReverted, wantMessage: "revert: InvalidFEOpcode", wantRetryNw: true}, + + // Nonce / duplicate. + {name: "geth already known", code: -32000, message: "already known", method: "eth_sendRawTransaction", + wantCode: common.ErrCodeEndpointNonceException, wantNormalized: common.JsonRpcErrorTransactionRejected}, + {name: "nonce too low", code: -32000, message: "nonce too low: next nonce 5, tx nonce 4", method: "eth_sendRawTransaction", + wantCode: common.ErrCodeEndpointNonceException, wantNormalized: common.JsonRpcErrorTransactionRejected}, + + // Transaction rejected / out of gas. + {name: "-32003 code", code: -32003, message: "transaction underpriced", method: "eth_sendRawTransaction", + wantCode: common.ErrCodeEndpointExecutionException, wantNormalized: common.JsonRpcErrorTransactionRejected, wantRetryNw: true}, + {name: "gas too low on eth_call", code: -32000, message: "intrinsic gas too low", method: "eth_call", + wantCode: common.ErrCodeEndpointExecutionException, wantNormalized: common.JsonRpcErrorTransactionRejected}, + + // Not found / disabled. + {name: "method not found", code: -32601, message: "Method not found", + wantCode: common.ErrCodeEndpointUnsupported, wantNormalized: common.JsonRpcErrorUnsupportedException, wantRetryNw: true}, + {name: "module disabled", code: -32000, message: "debug module is disabled", + wantCode: common.ErrCodeEndpointUnsupported, wantNormalized: common.JsonRpcErrorUnsupportedException, wantRetryNw: true}, + {name: "block does not exist", code: -32000, message: "this block does not exist", + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryNw: true}, + {name: "Block status not available", code: -32014, message: "Block status not available", + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryNw: true}, + {name: "Transaction history not available", code: -32011, message: "Transaction history is not available from this node", + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryNw: true}, + {name: "state not available", code: -32000, message: "state root not found", + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryNw: true}, + {name: "tenderly bare not found", code: -32000, message: "not found", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorClientSideException, wantRetryNw: true}, + {name: "Project not found", code: -32000, message: "Project not found", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorClientSideException, wantRetryNw: true}, + {name: "data not available yet", code: -32000, message: "data not available yet", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorClientSideException, wantRetryNw: true}, + + // Unsupported. + {name: "Unsupported method", code: -32000, message: "Unsupported method: trace_block", method: "trace_block", + wantCode: common.ErrCodeEndpointUnsupported, wantNormalized: common.JsonRpcErrorUnsupportedException, wantMessage: "The method trace_block does not exist/is not available", wantRetryNw: true}, + {name: "-32004 code", code: -32004, message: "nope", method: "eth_getProof", + wantCode: common.ErrCodeEndpointUnsupported, wantNormalized: common.JsonRpcErrorUnsupportedException, wantMessage: "The method eth_getProof does not exist/is not available", wantRetryNw: true}, + {name: "plan restriction", code: -32000, message: "trace_filter is not included in your current plan", method: "trace_filter", + wantCode: common.ErrCodeEndpointUnsupported, wantNormalized: common.JsonRpcErrorUnsupportedException, wantMessage: "The method trace_filter does not exist/is not available", wantRetryNw: true}, + {name: "transaction type not supported", code: -32000, message: "transaction type not supported", method: "eth_sendRawTransaction", + wantCode: common.ErrCodeEndpointUnsupported, wantNormalized: common.JsonRpcErrorUnsupportedException, wantMessage: "The method eth_sendRawTransaction does not exist/is not available", wantRetryNw: true}, + + // Malformed transactions. + {name: "typed transaction too short", code: -32000, message: "typed transaction too short", method: "eth_sendRawTransaction", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorInvalidArgument}, + {name: "rlp expected input list", code: -32000, message: "rlp: expected input list for types.LegacyTx", method: "eth_sendRawTransaction", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorInvalidArgument}, + {name: "rlp value size exceeds", code: -32000, message: "rlp: value size exceeds available input length", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorInvalidArgument}, + {name: "invalid transaction", code: -32000, message: "Invalid transaction signature", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorInvalidArgument}, + + // Retryable argument-shape errors. + {name: "tx of type", code: -32000, message: "tx of type 4 is not allowed", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorUnsupportedException, wantRetryNw: true}, + {name: "envio block id map", code: -32000, message: "invalid type: map, expected BlockNumber, 'latest', or 'earliest'", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorUnsupportedException, wantRetryNw: true}, + {name: "generic -32602", code: -32602, message: "bad argument shape", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorInvalidArgument, wantRetryNw: true}, + + // -32600 that is not a batch-validation cancellation falls through to the server-side fallback. + {name: "-32600 with unrelated data map", code: -32600, message: "oops", data: map[string]interface{}{"message": "something else"}, + wantCode: common.ErrCodeEndpointServerSideException, wantNormalized: common.JsonRpcErrorClientSideException, wantRetryNw: true}, + {name: "-32600 with string data", code: -32600, message: "oops", data: "plain", + wantCode: common.ErrCodeEndpointServerSideException, wantNormalized: common.JsonRpcErrorClientSideException, wantRetryNw: true}, + {name: "-32600 without inner message", code: -32600, message: "oops", data: map[string]interface{}{"other": "x"}, + wantCode: common.ErrCodeEndpointServerSideException, wantNormalized: common.JsonRpcErrorClientSideException, wantRetryNw: true}, + + // Non-retryable user mistakes. + {name: "param is required", code: -32000, message: "fromBlock param is required", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorInvalidArgument}, + {name: "invalid argument", code: -32000, message: "invalid argument 0: hex string has length 3, want 40 for common.Address", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorInvalidArgument}, + {name: "Bad request input parameters", code: -32000, message: "Bad request input parameters", + wantCode: common.ErrCodeEndpointClientSideException, wantNormalized: common.JsonRpcErrorInvalidArgument}, + + // Unauthorized. + {name: "invalid api key", code: -32000, message: "invalid api key provided", + wantCode: common.ErrCodeEndpointUnauthorized, wantNormalized: common.JsonRpcErrorUnauthorized, wantRetryNw: true}, + {name: "key is inactive", code: -32000, message: "API key is inactive", + wantCode: common.ErrCodeEndpointUnauthorized, wantNormalized: common.JsonRpcErrorUnauthorized, wantRetryNw: true}, + {name: "403 status with jsonrpc error", status: 403, code: -32000, message: "forbidden", + wantCode: common.ErrCodeEndpointUnauthorized, wantNormalized: common.JsonRpcErrorUnauthorized, wantRetryNw: true}, + + // Fallback keeps the vendor code as the normalized code. + {name: "unclassified error falls back to server-side", code: -32099, message: "something exploded", + wantCode: common.ErrCodeEndpointServerSideException, wantNormalized: common.JsonRpcErrorNumber(-32099), wantRetryNw: true}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + status := tc.status + if status == 0 { + status = 200 + } + r := &http.Response{StatusCode: status, Header: http.Header{}} + jrErr := common.NewErrJsonRpcExceptionExternal(tc.code, tc.message, "") + if tc.data != nil { + jrErr.Data = tc.data + } + jr := common.MustNewJsonRpcResponse(1, nil, jrErr) + + err := ExtractJsonRpcError(r, covErrNormResponse(tc.method), jr, nil) + if !common.HasErrorCode(err, tc.wantCode) { + t.Fatalf("expected %s, got %T: %v", tc.wantCode, err, err) + } + jre := covErrNormInternal(t, err) + if got := jre.NormalizedCode(); got != tc.wantNormalized { + t.Fatalf("normalized code: got %d, want %d", got, tc.wantNormalized) + } + if got := jre.OriginalCode(); got != tc.code { + t.Fatalf("original code: got %d, want %d", got, tc.code) + } + wantMsg := tc.wantMessage + if wantMsg == "" { + wantMsg = tc.message + } + if jre.Message != wantMsg { + t.Fatalf("message: got %q, want %q", jre.Message, wantMsg) + } + if got := common.IsRetryableTowardNetwork(err); got != tc.wantRetryNw { + t.Fatalf("IsRetryableTowardNetwork: got %v, want %v", got, tc.wantRetryNw) + } + }) + } +} + +// Unsupported errors keep the raw vendor text for diagnostics. +func TestCovErrNorm_UnsupportedKeepsVendorMessage(t *testing.T) { + t.Parallel() + + r := &http.Response{StatusCode: 200, Header: http.Header{}} + jr := common.MustNewJsonRpcResponse(1, nil, + common.NewErrJsonRpcExceptionExternal(-32000, "the method is not whitelisted", "")) + err := ExtractJsonRpcError(r, covErrNormResponse("debug_traceBlockByNumber"), jr, nil) + jre := covErrNormInternal(t, err) + if got := jre.Details["vendorMessage"]; got != "the method is not whitelisted" { + t.Fatalf("details.vendorMessage: got %v", got) + } +} + +// 200 OK responses whose result is an Error(string) ABI payload are reverts. +func TestCovErrNorm_RevertPayloadInResult(t *testing.T) { + t.Parallel() + + const payload = "0x08c379a0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000046e6f706500000000000000000000000000000000000000000000000000000000" + + cases := []struct { + name string + method string + result string + wantErr bool + }{ + {name: "Error(string) selector", method: "eth_call", result: `"` + payload + `"`, wantErr: true}, + {name: "ordinary hex result", method: "eth_call", result: `"0x0000000000000000000000000000000000000000000000000000000000000001"`, wantErr: false}, + {name: "trace result without timeout", method: "trace_block", result: `[{"action":{}}]`, wantErr: false}, + {name: "short result", method: "eth_blockNumber", result: `"0x1"`, wantErr: false}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + r := &http.Response{StatusCode: 200, Header: http.Header{}} + jr := common.MustNewJsonRpcResponseFromBytes([]byte("1"), []byte(tc.result), nil) + err := ExtractJsonRpcError(r, covErrNormResponse(tc.method), jr, nil) + if !tc.wantErr { + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + return + } + if !common.HasErrorCode(err, common.ErrCodeEndpointExecutionException) { + t.Fatalf("expected ErrEndpointExecutionException, got %T: %v", err, err) + } + jre := covErrNormInternal(t, err) + if jre.NormalizedCode() != common.JsonRpcErrorEvmReverted { + t.Fatalf("normalized code: got %d", jre.NormalizedCode()) + } + if jre.Message != "transaction reverted" { + t.Fatalf("message: got %q", jre.Message) + } + raw, ok := jre.Details["data"].(json.RawMessage) + if !ok || string(raw) != tc.result { + t.Fatalf("details.data: got %#v, want raw %s", jre.Details["data"], tc.result) + } + }) + } +} + +func TestCovErrNorm_GrpcNilAndOk(t *testing.T) { + t.Parallel() + + if err := ExtractGrpcError(nil, nil); err != nil { + t.Fatalf("nil status must yield nil, got %v", err) + } + if err := ExtractGrpcError(status.New(codes.OK, "fine"), nil); err != nil { + t.Fatalf("OK status must yield nil, got %v", err) + } +} + +// Plain gRPC status codes (no BDS details) map to erpc error kinds. +func TestCovErrNorm_GrpcStatusCodes(t *testing.T) { + t.Parallel() + + cases := []struct { + code codes.Code + wantCode common.ErrorCode + wantNormalized common.JsonRpcErrorNumber + wantRetryNw bool + }{ + {codes.Unimplemented, common.ErrCodeEndpointUnsupported, common.JsonRpcErrorUnsupportedException, true}, + {codes.InvalidArgument, common.ErrCodeEndpointClientSideException, common.JsonRpcErrorInvalidArgument, false}, + {codes.ResourceExhausted, common.ErrCodeEndpointCapacityExceeded, common.JsonRpcErrorCapacityExceeded, true}, + {codes.DeadlineExceeded, common.ErrCodeEndpointServerSideException, common.JsonRpcErrorNodeTimeout, true}, + {codes.Unauthenticated, common.ErrCodeEndpointUnauthorized, common.JsonRpcErrorUnauthorized, true}, + {codes.PermissionDenied, common.ErrCodeEndpointUnauthorized, common.JsonRpcErrorUnauthorized, true}, + {codes.NotFound, common.ErrCodeEndpointMissingData, common.JsonRpcErrorMissingData, true}, + {codes.OutOfRange, common.ErrCodeEndpointMissingData, common.JsonRpcErrorMissingData, true}, + {codes.Internal, common.ErrCodeEndpointServerSideException, common.JsonRpcErrorServerSideException, true}, + {codes.Unknown, common.ErrCodeEndpointServerSideException, common.JsonRpcErrorServerSideException, true}, + {codes.Unavailable, common.ErrCodeEndpointServerSideException, common.JsonRpcErrorServerSideException, true}, + {codes.Aborted, common.ErrCodeEndpointServerSideException, common.JsonRpcErrorServerSideException, true}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.code.String(), func(t *testing.T) { + t.Parallel() + ups := common.NewFakeUpstream("rpc-grpc-1") + msg := "bds says " + tc.code.String() + err := ExtractGrpcError(status.New(tc.code, msg), ups) + if !common.HasErrorCode(err, tc.wantCode) { + t.Fatalf("expected %s, got %T: %v", tc.wantCode, err, err) + } + jre := covErrNormInternal(t, err) + if got := jre.NormalizedCode(); got != tc.wantNormalized { + t.Fatalf("normalized code: got %d, want %d", got, tc.wantNormalized) + } + if jre.Message != msg { + t.Fatalf("message: got %q, want %q", jre.Message, msg) + } + if jre.Details["grpcCode"] != tc.code.String() || jre.Details["grpcMessage"] != msg { + t.Fatalf("grpc details: got %v / %v", jre.Details["grpcCode"], jre.Details["grpcMessage"]) + } + if jre.Details["upstreamId"] != "rpc-grpc-1" { + t.Fatalf("upstreamId: got %v", jre.Details["upstreamId"]) + } + if _, has := jre.Details["bdsErrorCode"]; has { + t.Fatalf("plain gRPC status must not carry bdsErrorCode") + } + if got := common.IsRetryableTowardNetwork(err); got != tc.wantRetryNw { + t.Fatalf("IsRetryableTowardNetwork: got %v, want %v", got, tc.wantRetryNw) + } + }) + } +} + +// BDS error details take precedence over the gRPC status code. +func TestCovErrNorm_GrpcBdsErrorCodes(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + bdsCode bdscommon.ErrorCode + wantCode common.ErrorCode + wantNormalized common.JsonRpcErrorNumber + wantRetryNw bool + }{ + {"unsupported block tag", bdscommon.ErrorCode_UNSUPPORTED_BLOCK_TAG, common.ErrCodeEndpointUnsupported, common.JsonRpcErrorUnsupportedException, true}, + {"unsupported method", bdscommon.ErrorCode_UNSUPPORTED_METHOD, common.ErrCodeEndpointUnsupported, common.JsonRpcErrorUnsupportedException, true}, + {"range outside available", bdscommon.ErrorCode_RANGE_OUTSIDE_AVAILABLE, common.ErrCodeEndpointMissingData, common.JsonRpcErrorMissingData, true}, + {"invalid parameter", bdscommon.ErrorCode_INVALID_PARAMETER, common.ErrCodeEndpointClientSideException, common.JsonRpcErrorInvalidArgument, false}, + {"invalid request", bdscommon.ErrorCode_INVALID_REQUEST, common.ErrCodeEndpointClientSideException, common.JsonRpcErrorInvalidArgument, false}, + {"rate limited", bdscommon.ErrorCode_RATE_LIMITED, common.ErrCodeEndpointCapacityExceeded, common.JsonRpcErrorCapacityExceeded, true}, + {"timeout", bdscommon.ErrorCode_TIMEOUT_ERROR, common.ErrCodeEndpointServerSideException, common.JsonRpcErrorNodeTimeout, true}, + // RANGE_TOO_LARGE maps to gRPC ResourceExhausted; BDS detail must win over capacity-exceeded. + {"range too large", bdscommon.ErrorCode_RANGE_TOO_LARGE, common.ErrCodeEndpointRequestTooLarge, common.JsonRpcErrorEvmLargeRange, true}, + {"internal", bdscommon.ErrorCode_INTERNAL_ERROR, common.ErrCodeEndpointServerSideException, common.JsonRpcErrorServerSideException, true}, + // DATA_NOT_FOUND has no BDS mapping and falls to gRPC Unknown. + {"unmapped bds code falls back to grpc code", bdscommon.ErrorCode_DATA_NOT_FOUND, common.ErrCodeEndpointServerSideException, common.JsonRpcErrorServerSideException, true}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + st := bdscommon.NewError(tc.bdsCode, "bds failure"). + WithCause(errors.New("disk read failed")). + WithDetail("fromBlock", "100"). + ToGRPCStatus() + + err := ExtractGrpcError(st, nil) + if !common.HasErrorCode(err, tc.wantCode) { + t.Fatalf("expected %s, got %T: %v", tc.wantCode, err, err) + } + jre := covErrNormInternal(t, err) + if got := jre.NormalizedCode(); got != tc.wantNormalized { + t.Fatalf("normalized code: got %d, want %d", got, tc.wantNormalized) + } + if jre.Message != "bds failure" { + t.Fatalf("message: got %q", jre.Message) + } + if jre.Details["upstreamId"] != "n/a" { + t.Fatalf("upstreamId without upstream: got %v", jre.Details["upstreamId"]) + } + if jre.Details["bdsErrorCode"] != tc.bdsCode { + t.Fatalf("bdsErrorCode: got %v, want %v", jre.Details["bdsErrorCode"], tc.bdsCode) + } + if jre.Details["cause"] != "disk read failed" { + t.Fatalf("cause: got %v", jre.Details["cause"]) + } + // "100" is JSON-decoded by the BDS layer into a number. + if jre.Details["fromBlock"] != float64(100) { + t.Fatalf("fromBlock detail: got %#v", jre.Details["fromBlock"]) + } + if got := common.IsRetryableTowardNetwork(err); got != tc.wantRetryNw { + t.Fatalf("IsRetryableTowardNetwork: got %v, want %v", got, tc.wantRetryNw) + } + }) + } +} + +// A BDS error without cause or details still classifies and omits "cause". +func TestCovErrNorm_GrpcBdsWithoutCause(t *testing.T) { + t.Parallel() + + st := (&bdscommon.BaseError{Code: bdscommon.ErrorCode_RATE_LIMITED, Message: "slow down"}).ToGRPCStatus() + err := ExtractGrpcError(st, nil) + if !common.HasErrorCode(err, common.ErrCodeEndpointCapacityExceeded) { + t.Fatalf("expected ErrEndpointCapacityExceeded, got %T: %v", err, err) + } + jre := covErrNormInternal(t, err) + if _, has := jre.Details["cause"]; has { + t.Fatalf("cause must be absent, got %v", jre.Details["cause"]) + } +} diff --git a/architecture/evm/cov_integ_test.go b/architecture/evm/cov_integ_test.go new file mode 100644 index 000000000..246c2a651 --- /dev/null +++ b/architecture/evm/cov_integ_test.go @@ -0,0 +1,972 @@ +package evm + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/architecture/evm/integrity" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// covIntegNet is a scripted network: it records the directives of every +// forwarded request and answers through forward. +type covIntegNet struct { + common.Network + id string + cfg *common.NetworkConfig + mu sync.Mutex + dirs []*common.RequestDirectives + calls int + forward func(req *common.NormalizedRequest) (*common.NormalizedResponse, error) +} + +func newCovIntegNet(id string, cfg *common.NetworkConfig, fwd func(req *common.NormalizedRequest) (*common.NormalizedResponse, error)) *covIntegNet { + return &covIntegNet{id: id, cfg: cfg, forward: fwd} +} + +func (n *covIntegNet) Id() string { return n.id } +func (n *covIntegNet) Label() string { return "covnet" } +func (n *covIntegNet) ProjectId() string { return "test" } +func (n *covIntegNet) Config() *common.NetworkConfig { return n.cfg } +func (n *covIntegNet) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + n.mu.Lock() + n.dirs = append(n.dirs, req.Directives()) + n.calls++ + n.mu.Unlock() + return n.forward(req) +} + +func (n *covIntegNet) callCount() int { + n.mu.Lock() + defer n.mu.Unlock() + return n.calls +} + +func (n *covIntegNet) lastDirs() *common.RequestDirectives { + n.mu.Lock() + defer n.mu.Unlock() + if len(n.dirs) == 0 { + return nil + } + return n.dirs[len(n.dirs)-1] +} + +// covIntegFleetNet adds upstream enumeration (what the state prober needs). +type covIntegFleetNet struct { + *covIntegNet + ups []common.Upstream +} + +func (n *covIntegFleetNet) EvmAllUpstreams(ctx context.Context) []common.Upstream { return n.ups } + +// covIntegGroupNet carves the selector "grp" out as its own node group. +type covIntegGroupNet struct{ *covIntegNet } + +func (n *covIntegGroupNet) EvmUpstreamGroupForSelector(ctx context.Context, sel string) (string, string) { + if sel == "grp" { + return "g:grp", "lane-grp" + } + return "", "" +} + +// covIntegHeadNet reports network-wide latest/finalized heights. +type covIntegHeadNet struct { + *covIntegNet + latest, finalized int64 +} + +func (n *covIntegHeadNet) EvmHighestLatestBlockNumber(ctx context.Context) int64 { return n.latest } +func (n *covIntegHeadNet) EvmHighestFinalizedBlockNumber(ctx context.Context) int64 { return n.finalized } + +// covIntegPlainUp hides every EVM capability of the wrapped upstream. +type covIntegPlainUp struct{ common.Upstream } + +// covIntegWriterUp can record a proven head but is not an EvmUpstream. +type covIntegWriterUp struct { + common.Upstream + proven atomic.Int64 +} + +func (u *covIntegWriterUp) EvmSetStateProvenBlock(n int64) { u.proven.Store(n) } + +func covIntegResp(result string) *common.NormalizedResponse { + return common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`"1"`), []byte(result), nil)) +} + +func covIntegHeaderJSON(num int64, hash, parent string) string { + return fmt.Sprintf(`{"number":"0x%x","hash":"%s","parentHash":"%s"}`, num, hash, parent) +} + +// covIntegChainForward serves a fakeChain's blocks by number or by hash. +func covIntegChainForward(chain *fakeChain) func(req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return func(req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + jrq, err := req.JsonRpcRequest() + if err != nil { + return nil, err + } + ref, _ := jrq.Params[0].(string) + chain.mu.Lock() + chain.fetches++ + var b fakeBlock + var found bool + if jrq.Method == "eth_getBlockByHash" { + b, found = chain.byHash[ref] + } else { + var num int64 + _, _ = fmt.Sscanf(ref, "0x%x", &num) + b, found = chain.byNumber[num] + } + chain.mu.Unlock() + if !found { + return covIntegResp("null"), nil + } + return covIntegResp(covIntegHeaderJSON(b.num, b.hash, b.parent)), nil + } +} + +func covIntegFetches(chain *fakeChain) int { + chain.mu.Lock() + defer chain.mu.Unlock() + return chain.fetches +} + +func covIntegStreak(p *stateProber, id string) int64 { + v, ok := p.disprovedStreak.Load(id) + if !ok { + return 0 + } + return v.(*atomic.Int64).Load() +} + +func TestCovInteg_DoOnceJoinsInflightFetch(t *testing.T) { + var mu sync.Mutex + inflight := map[string]*flight[int]{} + f := &flight[int]{val: 7, ok: true} + f.wg.Add(1) + inflight["k"] = f + + type res struct { + v int + ok bool + } + done := make(chan res, 1) + called := atomic.Bool{} + go func() { + v, ok := doOnce(&mu, inflight, "k", func() (int, bool) { + called.Store(true) + return 99, false + }) + done <- res{v, ok} + }() + f.wg.Done() + got := <-done + assert.Equal(t, res{7, true}, got, "a joiner must receive the in-flight result") + assert.False(t, called.Load(), "a joiner must never issue its own fetch") +} + +func TestCovInteg_ChainViewStateTransitions(t *testing.T) { + t.Run("adoptFollowed ignores headers without a hash", func(t *testing.T) { + c := newChainView(nil, 8, "", "", nil) + c.adoptFollowed(5, nil) + c.adoptFollowed(5, &integrity.Header{Number: "0x5"}) + _, _, ok := c.FollowedRange() + assert.False(t, ok) + _, ok = c.HashAt(5) + assert.False(t, ok) + }) + + t.Run("observeHeader and observeReceipts ignore empty keys and values", func(t *testing.T) { + c := newChainView(nil, 8, "", "", nil) + c.observeHeader("", &integrity.Header{Hash: "0xa"}) + c.observeHeader("0xa", nil) + c.observeReceipts("", []integrity.Receipt{{BlockHash: "0xb"}}) + c.observeReceipts("0xb", nil) + assert.Empty(t, c.headers) + _, ok := c.CachedReceipts("0xb") + assert.False(t, ok) + }) + + t.Run("CachedReceipts serves only warm blocks", func(t *testing.T) { + c := newChainView(nil, 8, "", "", nil) + c.observeReceipts("0xbb", []integrity.Receipt{{TransactionHash: "0x1", BlockHash: "0xbb"}}) + got, ok := c.CachedReceipts("0xbb") + require.True(t, ok) + assert.Equal(t, "0x1", got[0].TransactionHash) + _, ok = c.CachedReceipts("0xcold") + assert.False(t, ok) + }) + + t.Run("eviction drops reconfirmation stamps below the window", func(t *testing.T) { + c := newChainView(nil, 2, "", "", nil) + c.observe(1, "0x1", nil) + c.mu.Lock() + c.reconfirmedAt[1] = time.Now() + c.mu.Unlock() + c.observe(10, "0xa", nil) + c.mu.RLock() + _, kept := c.reconfirmedAt[1] + c.mu.RUnlock() + assert.False(t, kept) + }) + + t.Run("a followed segment fully below the window is forgotten", func(t *testing.T) { + c := newChainView(nil, 2, "", "", nil) + c.adoptFollowed(5, &integrity.Header{Number: "0x5", Hash: "0x5"}) + _, _, ok := c.FollowedRange() + require.True(t, ok) + c.observe(10, "0xa", nil) // lo=8 > followHead=5 + _, _, ok = c.FollowedRange() + assert.False(t, ok, "the range must not claim heights that were evicted") + }) + + t.Run("finality unknown when the getter reports no finalized height", func(t *testing.T) { + c := newChainView(nil, 8, "", "", func() int64 { return 0 }) + assert.Equal(t, "unknown", c.finalityLabel(5)) + }) +} + +func TestCovInteg_Reconcile(t *testing.T) { + ctx := context.Background() + + t.Run("malformed branch heads are refused", func(t *testing.T) { + c := newChainView(nil, 8, "", "", nil) + for _, h := range []*integrity.Header{ + nil, + {Number: "0x5"}, + {Hash: "0xa"}, + {Hash: "0xa", Number: "0xzz"}, + {Hash: "0xa", Number: "0x0", ParentHash: "0xp"}, // parent below genesis + } { + depth, ok := c.reconcile(ctx, h) + assert.False(t, ok) + assert.Zero(t, depth) + } + }) + + t.Run("sparse view with no held parent anchors the branch", func(t *testing.T) { + c := newChainView(nil, 8, "", "", nil) + depth, ok := c.reconcile(ctx, &integrity.Header{Number: "0xa", Hash: "0xnew10", ParentHash: "0xnew9"}) + require.True(t, ok) + assert.Equal(t, 1, depth) + assert.Equal(t, "0xnew10", mustHashAt(t, c, 10)) + from, to, ok := c.FollowedRange() + require.True(t, ok) + assert.Equal(t, [2]int64{10, 10}, [2]int64{from, to}) + }) + + t.Run("diverging parent that cannot be fetched fails", func(t *testing.T) { + c := newChainView(nil, 8, "", "", nil) + c.observe(9, "0xheld9", nil) + _, ok := c.reconcile(ctx, &integrity.Header{Number: "0xa", Hash: "0xnew10", ParentHash: "0xnew9"}) + assert.False(t, ok) + assert.Equal(t, "0xheld9", mustHashAt(t, c, 9)) + }) + + t.Run("parent claiming the wrong height fails", func(t *testing.T) { + net := newCovIntegNet("cov-integ-rec", nil, func(req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return covIntegResp(covIntegHeaderJSON(5, "0xnew9", "0xnew8")), nil + }) + c := newChainView(net, 8, "", "", nil) + c.observe(9, "0xheld9", nil) + _, ok := c.reconcile(ctx, &integrity.Header{Number: "0xa", Hash: "0xnew10", ParentHash: "0xnew9"}) + assert.False(t, ok) + assert.Equal(t, "0xheld9", mustHashAt(t, c, 9)) + }) + + t.Run("no common ancestor within the window fails", func(t *testing.T) { + net := newCovIntegNet("cov-integ-rec2", nil, func(req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return covIntegResp(covIntegHeaderJSON(9, "0xnew9", "0xnew8")), nil + }) + c := newChainView(net, 1, "", "", nil) + c.observe(9, "0xheld9", nil) + _, ok := c.reconcile(ctx, &integrity.Header{Number: "0xa", Hash: "0xnew10", ParentHash: "0xnew9"}) + assert.False(t, ok) + assert.Equal(t, "0xheld9", mustHashAt(t, c, 9)) + assert.Equal(t, 1, net.callCount()) + assert.Equal(t, "true", net.lastDirs().SkipCacheRead, "the ancestry walk must bypass the cache") + }) +} + +func TestCovInteg_ResolveAnchors(t *testing.T) { + ctx := context.Background() + + t.Run("a by-number answer for another height is discarded", func(t *testing.T) { + net := newCovIntegNet("cov-integ-anchor", nil, func(req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return covIntegResp(covIntegHeaderJSON(0x11, "0xwrong", "0xp")), nil + }) + c := newChainView(net, 8, "", "", nil) + h, ok := c.resolveHeader(ctx, "eth_getBlockByNumber", "0x10", false) + assert.False(t, ok) + assert.Nil(t, h) + _, pinned := c.HashAt(0x11) + assert.False(t, pinned) + _, pinned = c.HashAt(0x10) + assert.False(t, pinned) + }) + + receiptsCases := []struct { + name string + body string + }{ + {"another block's receipts", `[{"blockHash":"0xother","blockNumber":"0x10","transactionHash":"0x1"}]`}, + {"an empty receipt set", `[]`}, + } + for _, tc := range receiptsCases { + t.Run(tc.name+" is canonical-unavailable and not cached", func(t *testing.T) { + net := newCovIntegNet("cov-integ-rcpt", nil, func(req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return covIntegResp(tc.body), nil + }) + c := newChainView(net, 8, "", "", nil) + got, ok := c.receiptsByHash(ctx, "0xwant") + assert.False(t, ok) + assert.Nil(t, got) + _, cached := c.CachedReceipts("0xwant") + assert.False(t, cached) + }) + } +} + +func TestCovInteg_HasVerifiedEmptyTraceBlock(t *testing.T) { + net := &covIntegGroupNet{newCovIntegNet("cov-integ-trace", nil, nil)} + mkReq := func(body string, withNet bool, dirs *common.RequestDirectives) *common.NormalizedRequest { + r := common.NewNormalizedRequest([]byte(body)) + if withNet { + r.SetNetwork(net) + } + if dirs != nil { + r.SetDirectives(dirs) + } + return r + } + traceBody := func(ref string) string { + return fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":"trace_block","params":["%s"]}`, ref) + } + store := func(group string, h *integrity.Header, n int64) { + v := newChainView(net, 32, "", "", nil) + v.adoptFollowed(n, h) + key := chainViewKey{network: net, group: group} + chainViewStore.Store(key, v) + t.Cleanup(func() { chainViewStore.Delete(key) }) + } + empty := func(num, hash, gas string) *integrity.Header { + return &integrity.Header{Number: num, Hash: hash, GasUsed: gas, RawTransactions: []any{}} + } + store("", empty("0x10", "0xh10", "0x0"), 0x10) + + assert.False(t, HasVerifiedEmptyTraceBlock(nil)) + assert.False(t, HasVerifiedEmptyTraceBlock(mkReq(traceBody("0x10"), false, nil)), "no network") + assert.False(t, HasVerifiedEmptyTraceBlock(mkReq(`not-json`, true, nil)), "unparseable request") + assert.False(t, HasVerifiedEmptyTraceBlock(mkReq(traceBody("16"), true, nil)), "decimal ref is not a hex block") + assert.False(t, HasVerifiedEmptyTraceBlock(mkReq(traceBody("0xzz"), true, nil)), "malformed hex") + assert.False(t, HasVerifiedEmptyTraceBlock(mkReq(traceBody("0x11"), true, nil)), "outside the followed range") + assert.True(t, HasVerifiedEmptyTraceBlock(mkReq(traceBody("0x10"), true, nil))) + + t.Run("group-scoped request reads only its group's view", func(t *testing.T) { + dirs := &common.RequestDirectives{UseUpstream: "grp"} + assert.False(t, HasVerifiedEmptyTraceBlock(mkReq(traceBody("0x10"), true, dirs)), "group view absent") + store("g:grp", empty("0x20", "0xh20", "0x0"), 0x20) + assert.True(t, HasVerifiedEmptyTraceBlock(mkReq(traceBody("0x20"), true, dirs))) + }) + + t.Run("header disagreeing with the pinned height or carrying gas is not proof", func(t *testing.T) { + net2 := &covIntegGroupNet{newCovIntegNet("cov-integ-trace2", nil, nil)} + for _, tc := range []struct { + name string + h *integrity.Header + }{ + {"wrong number", empty("0x31", "0xh30", "0x0")}, + {"non-zero gas", empty("0x30", "0xh30", "0x5208")}, + } { + v := newChainView(net2, 32, "", "", nil) + v.adoptFollowed(0x30, tc.h) + key := chainViewKey{network: net2} + chainViewStore.Store(key, v) + r := common.NewNormalizedRequest([]byte(traceBody("0x30"))) + r.SetNetwork(net2) + assert.False(t, HasVerifiedEmptyTraceBlock(r), tc.name) + chainViewStore.Delete(key) + } + }) +} + +func TestCovInteg_GroupChainView(t *testing.T) { + ctx := context.Background() + assert.Nil(t, groupChainView(ctx, nil, "")) + + t.Run("selector carving a group gets its own pinned view", func(t *testing.T) { + net := &covIntegGroupNet{newCovIntegNet("cov-integ-gcv-grp", nil, nil)} + t.Cleanup(func() { + chainViewStore.Delete(chainViewKey{network: net}) + chainViewStore.Delete(chainViewKey{network: net, group: "g:grp"}) + }) + g := groupChainView(ctx, net, "grp") + wide := groupChainView(ctx, net, "") + require.NotSame(t, g, wide) + assert.Equal(t, "grp", g.selector) + assert.Equal(t, "lane-grp", g.group) + assert.Empty(t, wide.selector) + assert.Same(t, wide, groupChainView(ctx, net, "not-a-group"), "a non-group selector falls back to the network view") + assert.Same(t, g, groupChainView(ctx, net, "grp"), "views are created once") + assert.Equal(t, defaultReorgWindow, g.window) + assert.Nil(t, g.finalized) + }) + + t.Run("integrity config sizes the window and starts follower and prober", func(t *testing.T) { + cfg := &common.NetworkConfig{ + Evm: &common.EvmNetworkConfig{ChainId: 1}, + Integrity: &common.IntegrityConfig{IntegritySettings: common.IntegritySettings{ + ReorgWindow: 7, + Follow: &common.IntegrityFollowConfig{Enabled: util.BoolPtr(true), Interval: common.Duration(time.Hour)}, + StateProbe: &common.IntegrityStateProbeConfig{Enabled: util.BoolPtr(true), Interval: common.Duration(5 * time.Second)}, + }}, + } + net := &covIntegHeadNet{covIntegNet: newCovIntegNet("cov-integ-gcv-follow", cfg, nil), finalized: 42} + v := groupChainView(ctx, net, "") + p := stateProberFor(net.Id()) + t.Cleanup(func() { + if v.follower != nil { + close(v.follower.stop) + } + chainViewStore.Delete(chainViewKey{network: net}) + if p != nil { + stateProbers.Delete(net.Id()) + close(p.work) + } + }) + assert.Equal(t, 7, v.window) + require.NotNil(t, v.follower) + assert.Equal(t, time.Hour, v.follower.interval) + require.NotNil(t, p) + assert.Equal(t, 5*time.Second, p.interval) + assert.Same(t, v, p.view) + assert.Equal(t, "finalized", v.finalityLabel(42), "finality comes from the network") + assert.Equal(t, "unfinalized", v.finalityLabel(43)) + }) + + t.Run("network without a head getter yields no latest func", func(t *testing.T) { + assert.Nil(t, networkLatest(newCovIntegNet("x", nil, nil))) + hn := &covIntegHeadNet{covIntegNet: newCovIntegNet("y", nil, nil), latest: 9} + fn := networkLatest(hn) + require.NotNil(t, fn) + assert.Equal(t, int64(9), fn(ctx)) + }) +} + +func TestCovInteg_ObserveViews(t *testing.T) { + ctx := context.Background() + + t.Run("observeBlockView ignores nil inputs and non-header results", func(t *testing.T) { + c := newChainView(nil, 8, "", "", nil) + observeBlockView(ctx, nil, covIntegResp(covIntegHeaderJSON(1, "0x1", "0x0")), "eth_getblockbynumber") + observeBlockView(ctx, c, nil, "eth_getblockbynumber") + observeBlockView(ctx, c, covIntegResp(`"0x1"`), "eth_getblockbynumber") + observeBlockView(ctx, c, covIntegResp(`{"number":"0x1"}`), "eth_getblockbynumber") + assert.Empty(t, c.canonical) + assert.Empty(t, c.headers) + }) + + t.Run("observeNarrowView pins only from EVM upstreams with known finality", func(t *testing.T) { + anchor := covIntegResp(`{"blockNumber":"0x10","blockHash":"0xbb"}`) + evmUp := common.NewFakeUpstream("u", common.WithEvmStatePoller(common.NewFakeEvmStatePoller(0x30, 0x20))) + + c := newChainView(nil, 32, "", "", nil) + observeNarrowView(ctx, nil, evmUp, anchor) + observeNarrowView(ctx, c, evmUp, nil) + observeNarrowView(ctx, c, covIntegPlainUp{evmUp}, anchor) + _, ok := c.HashAt(0x10) + assert.False(t, ok, "a non-EVM upstream has no finality to gate the pin") + + observeNarrowView(ctx, c, evmUp, covIntegResp(`{"blockNumber":"0x11","blockHash":""}`)) + _, ok = c.HashAt(0x11) + assert.False(t, ok, "an anchor without a hash pins nothing") + + observeNarrowView(ctx, c, evmUp, anchor) + assert.Equal(t, "0xbb", mustHashAt(t, c, 0x10)) + }) +} + +func TestCovInteg_ChainFollower(t *testing.T) { + ctx := context.Background() + newFollower := func(t *testing.T, chain *fakeChain, head *int64, cfg *common.IntegrityFollowConfig) (*chainView, *chainFollower, *covIntegNet) { + t.Helper() + net := newCovIntegNet("cov-integ-follow-"+t.Name(), nil, covIntegChainForward(chain)) + v := newChainView(net, 32, "", "", nil) + f := newChainFollower(v, func(context.Context) int64 { return *head }, cfg) + return v, f, net + } + + t.Run("config overrides interval and per-tick bound", func(t *testing.T) { + chain := newFakeChain() + chain.build(100, 110, "a", "0xgenesis") + head := int64(100) + v, f, _ := newFollower(t, chain, &head, &common.IntegrityFollowConfig{MaxBlocksPerTick: 2, Interval: common.Duration(time.Minute)}) + assert.Equal(t, time.Minute, f.interval) + f.advance(ctx) + head = 110 + f.advance(ctx) + assert.Equal(t, int64(102), mustFollowHead(t, v), "catch-up is bounded per tick") + }) + + t.Run("no head means no work", func(t *testing.T) { + chain := newFakeChain() + head := int64(0) + v, f, _ := newFollower(t, chain, &head, nil) + f.advance(ctx) + assert.Zero(t, covIntegFetches(chain)) + _, _, ok := v.FollowedRange() + assert.False(t, ok) + }) + + t.Run("bootstrap that cannot fetch the head does not start following", func(t *testing.T) { + chain := newFakeChain() + head := int64(100) + v, f, _ := newFollower(t, chain, &head, nil) + f.advance(ctx) + assert.Equal(t, 1, covIntegFetches(chain)) + _, _, ok := v.FollowedRange() + assert.False(t, ok) + }) + + t.Run("a missing block stops the walk where it is", func(t *testing.T) { + chain := newFakeChain() + chain.build(100, 102, "a", "0xgenesis") + head := int64(100) + v, f, _ := newFollower(t, chain, &head, nil) + f.advance(ctx) + head = 105 + f.advance(ctx) + assert.Equal(t, int64(102), mustFollowHead(t, v)) + }) + + t.Run("a cancelled tick advances nothing", func(t *testing.T) { + chain := newFakeChain() + chain.build(100, 105, "a", "0xgenesis") + head := int64(100) + v, f, _ := newFollower(t, chain, &head, nil) + f.advance(ctx) + head = 105 + cctx, cancel := context.WithCancel(ctx) + cancel() + f.advance(cctx) + assert.Equal(t, int64(100), mustFollowHead(t, v)) + }) + + t.Run("an evicted parent re-anchors on the next block", func(t *testing.T) { + chain := newFakeChain() + chain.build(100, 101, "a", "0xgenesis") + head := int64(100) + v, f, _ := newFollower(t, chain, &head, nil) + f.advance(ctx) + v.mu.Lock() + delete(v.canonical, 100) + v.mu.Unlock() + head = 101 + f.advance(ctx) + assert.Equal(t, int64(101), mustFollowHead(t, v)) + assert.Equal(t, chain.hashAt(101), mustHashAt(t, v, 101)) + }) + + t.Run("a new followed head is handed to the network's prober", func(t *testing.T) { + chain := newFakeChain() + chain.build(100, 103, "a", "0xgenesis") + head := int64(100) + v, f, net := newFollower(t, chain, &head, nil) + p := &stateProber{work: make(chan int64, 1)} + stateProbers.Store(net.Id(), p) + t.Cleanup(func() { stateProbers.Delete(net.Id()) }) + f.advance(ctx) + head = 103 + f.advance(ctx) + require.Equal(t, int64(103), mustFollowHead(t, v)) + select { + case got := <-p.work: + assert.Equal(t, int64(103), got) + default: + t.Fatal("the prober was not handed the new head") + } + }) + + t.Run("settled heights are read from cache, the tip fresh", func(t *testing.T) { + chain := newFakeChain() + chain.build(100, 110, "a", "0xgenesis") + net := newCovIntegNet("cov-integ-follow-fresh", nil, covIntegChainForward(chain)) + v := newChainView(net, 32, "", "", func() int64 { return 105 }) + _, ok := v.fetchFollowHeader(ctx, 104) + require.True(t, ok) + assert.Empty(t, net.lastDirs().SkipCacheRead) + _, ok = v.fetchFollowHeader(ctx, 106) + require.True(t, ok) + assert.Equal(t, "true", net.lastDirs().SkipCacheRead) + }) + + t.Run("run returns once stopped without ticking", func(t *testing.T) { + calls := atomic.Int32{} + v := newChainView(nil, 8, "", "", nil) + f := newChainFollower(v, func(context.Context) int64 { calls.Add(1); return 1 }, nil) + close(f.stop) + f.run() + assert.Zero(t, calls.Load()) + }) +} + +func TestCovInteg_StateProber(t *testing.T) { + ctx := context.Background() + const head = int64(1000) + hdr := &integrity.Header{Number: "0x3e8", Hash: "0xhead", ParentHash: "0xpar"} + + // callAnswer scripts eth_call; proofAnswer scripts eth_getProof. + type script struct { + callRes string + callErr error + callNil bool + proofRes string + proofErr error + } + mkNet := func(id string, sc *script, ups ...common.Upstream) *covIntegFleetNet { + return &covIntegFleetNet{ + covIntegNet: newCovIntegNet(id, nil, func(req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + jrq, _ := req.JsonRpcRequest() + switch jrq.Method { + case "eth_call": + if sc.callNil { + return nil, nil + } + if sc.callErr != nil { + return nil, sc.callErr + } + return covIntegResp(sc.callRes), nil + case "eth_getProof": + if sc.proofErr != nil { + return nil, sc.proofErr + } + return covIntegResp(sc.proofRes), nil + } + return nil, fmt.Errorf("unexpected %s", jrq.Method) + }), + ups: ups, + } + } + mkProber := func(net common.Network, interval time.Duration) *stateProber { + v := newChainView(net, 32, "", "", nil) + v.adoptFollowed(head, hdr) + return &stateProber{ + network: net, view: v, interval: interval, + ctxProbe: integrity.ChainStateContextProbe(1), + lastProbe: map[string]time.Time{}, work: make(chan int64, 1), + } + } + atHead := fmt.Sprintf(`"0x%064x"`, head) + + t.Run("context probe outcomes", func(t *testing.T) { + cases := []struct { + name string + sc script + ignore bool + want probeOutcome + forward bool + }{ + {name: "match", sc: script{callRes: atHead}, want: probeMatch, forward: true}, + {name: "eth_call excluded by config", ignore: true, want: probeUnknown}, + {name: "transport error", sc: script{callErr: errors.New("dial tcp: i/o timeout")}, want: probeUnknown, forward: true}, + {name: "nil response", sc: script{callNil: true}, want: probeUnknown, forward: true}, + {name: "no returndata", sc: script{callRes: `"0x"`}, want: probeUnknown, forward: true}, + {name: "unparseable returndata", sc: script{callRes: `"0xzz"`}, want: probeUnknown, forward: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + u := common.NewFakeUpstream("u1") + if tc.ignore { + u.Config().IgnoreMethods = []string{"eth_call"} + } + net := mkNet("cov-integ-sp-ctx", &tc.sc, u) + p := mkProber(net, 0) + assert.Equal(t, tc.want, p.probeContext(ctx, u, head)) + assert.Equal(t, tc.forward, net.callCount() > 0) + }) + } + }) + + t.Run("proof probe outcomes", func(t *testing.T) { + root := "0x" + "1111111111111111111111111111111111111111111111111111111111111111"[:62] // 31 bytes + cases := []struct { + name string + sc script + root string + latched bool + want probeOutcome + latch bool + forward bool + }{ + {name: "no verified stateRoot", root: "", want: probeUnknown}, + {name: "already latched unsupported", root: root, latched: true, want: probeUnknown, latch: true}, + {name: "transient error is not latched", root: root, sc: script{proofErr: errors.New("connection reset by peer")}, want: probeUnknown, forward: true}, + {name: "method-ignored error latches", root: root, sc: script{proofErr: common.NewErrUpstreamMethodIgnored("eth_getProof", "u1")}, want: probeUnknown, latch: true, forward: true}, + {name: "endpoint-unsupported error latches", root: root, sc: script{proofErr: common.NewErrEndpointUnsupported(errors.New("nope"))}, want: probeUnknown, latch: true, forward: true}, + {name: "empty account proof", root: root, sc: script{proofRes: `{"accountProof":[]}`}, want: probeUnknown, forward: true}, + {name: "stateRoot not 32 bytes", root: root, sc: script{proofRes: `{"accountProof":["0xabcd"]}`}, want: probeUnknown, forward: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + u := common.NewFakeUpstream("u1") + net := mkNet("cov-integ-sp-proof", &tc.sc, u) + p := mkProber(net, 0) + if tc.latched { + p.proofUnsupported.Store("u1", true) + } + h := &integrity.Header{StateRoot: tc.root} + assert.Equal(t, tc.want, p.probeProof(ctx, u, head, h)) + _, latched := p.proofUnsupported.Load("u1") + assert.Equal(t, tc.latch, latched) + assert.Equal(t, tc.forward, net.callCount() > 0) + }) + } + }) + + t.Run("no evidence at all leaves the streak untouched", func(t *testing.T) { + u := common.NewFakeUpstream("u1").(*common.FakeUpstream) + u.Config().IgnoreMethods = []string{"eth_call", "eth_getProof"} + net := mkNet("cov-integ-sp-none", &script{}, u) + p := mkProber(net, 0) + p.noteDisproved(u) + p.probeUpstream(ctx, u, head, &integrity.Header{StateRoot: "0x" + fmt.Sprintf("%064x", 1)}) + assert.Equal(t, int64(1), covIntegStreak(p, "u1")) + assert.Zero(t, u.EvmStateProvenBlock()) + assert.Zero(t, net.callCount()) + }) + + t.Run("a match from a non-writer clears the streak but records no head", func(t *testing.T) { + u := covIntegPlainUp{common.NewFakeUpstream("u1")} + net := mkNet("cov-integ-sp-plain", &script{callRes: atHead}, u) + p := mkProber(net, 0) + p.noteDisproved(u) + p.probeUpstream(ctx, u, head, hdr) + assert.Zero(t, covIntegStreak(p, "u1")) + }) + + t.Run("a writer that is not an EVM upstream still gets its proven head", func(t *testing.T) { + u := &covIntegWriterUp{Upstream: common.NewFakeUpstream("u1")} + net := mkNet("cov-integ-sp-writer", &script{callRes: atHead}, u) + p := mkProber(net, 0) + p.probeUpstream(ctx, u, head, hdr) + assert.Equal(t, head, u.proven.Load()) + }) + + t.Run("latest probe without returndata is no evidence", func(t *testing.T) { + u := common.NewFakeUpstream("u1", common.WithEvmStatePoller(common.NewFakeEvmStatePoller(head, head-10))) + net := mkNet("cov-integ-sp-latest", &script{callRes: `"0x"`}, u) + net.cfg = &common.NetworkConfig{Methods: &common.MethodsConfig{Definitions: map[string]*common.CacheMethodConfig{ + "eth_call": {TranslateLatestTag: util.BoolPtr(false)}, + }}} + p := mkProber(net, 0) + assert.Equal(t, probeUnknown, p.probeLatest(ctx, u)) + assert.Equal(t, 1, net.callCount()) + assert.Zero(t, fakeTrackerOf(t, u).MisbehaviorCount) + }) + + t.Run("probeAll needs an enumerable network", func(t *testing.T) { + net := newCovIntegNet("cov-integ-sp-noenum", nil, func(req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return covIntegResp(atHead), nil + }) + p := mkProber(net, 0) + p.probeAll(head) + assert.Zero(t, net.callCount()) + }) + + t.Run("an upstream probed within the interval is skipped", func(t *testing.T) { + u := common.NewFakeUpstream("u1").(*common.FakeUpstream) + u.Config().IgnoreMethods = []string{"eth_getProof"} + net := mkNet("cov-integ-sp-due", &script{callRes: atHead}, u) + p := mkProber(net, time.Hour) + p.probeAll(head) + first := net.callCount() + require.Positive(t, first) + p.probeAll(head) + assert.Equal(t, first, net.callCount()) + assert.Equal(t, head, u.EvmStateProvenBlock()) + }) + + t.Run("run drains handed heads; the newest replaces a queued one", func(t *testing.T) { + u := common.NewFakeUpstream("u1").(*common.FakeUpstream) + u.Config().IgnoreMethods = []string{"eth_getProof"} + net := mkNet("cov-integ-sp-run", &script{callRes: atHead}, u) + p := mkProber(net, 0) + p.onNewHead(5) + p.onNewHead(head) + close(p.work) + p.run() + assert.Equal(t, head, u.EvmStateProvenBlock(), "only the newest head is probed") + assert.Equal(t, 1, net.callCount()) + }) + + t.Run("nil network has empty project label", func(t *testing.T) { + assert.Empty(t, (&stateProber{}).projectId()) + }) +} + +func TestCovInteg_StartStateProber(t *testing.T) { + assert.Nil(t, stateProberFor("cov-integ-no-such-network")) + + cfg := &common.NetworkConfig{Evm: &common.EvmNetworkConfig{ChainId: 42161}} + net := newCovIntegNet("cov-integ-start", cfg, nil) + v1 := newChainView(net, 8, "", "", nil) + v2 := newChainView(net, 8, "", "", nil) + startStateProber(net, v1, &common.IntegrityStateProbeConfig{Interval: common.Duration(3 * time.Second)}) + p := stateProberFor(net.Id()) + require.NotNil(t, p) + t.Cleanup(func() { + stateProbers.Delete(net.Id()) + close(p.work) + }) + assert.Equal(t, 3*time.Second, p.interval) + assert.Equal(t, integrity.ChainStateContextProbe(42161), p.ctxProbe, "the probe target follows the chain architecture") + + startStateProber(net, v2, nil) + assert.Same(t, p, stateProberFor(net.Id()), "one prober per network") + assert.Same(t, v1, p.view) + + net2 := newCovIntegNet("cov-integ-start-default", nil, nil) + startStateProber(net2, v1, nil) + p2 := stateProberFor(net2.Id()) + require.NotNil(t, p2) + t.Cleanup(func() { + stateProbers.Delete(net2.Id()) + close(p2.work) + }) + assert.Equal(t, stateProbeMinInterval, p2.interval) +} + +func TestCovInteg_IsMethodUnsupportedErr(t *testing.T) { + cases := []struct { + err error + want bool + }{ + {nil, false}, + {common.NewErrEndpointUnsupported(errors.New("x")), true}, + {common.NewErrUpstreamMethodIgnored("eth_getProof", "u1"), true}, + {errors.New("the method eth_getProof does not exist/is not available"), true}, + {errors.New("Method not found"), true}, + {errors.New("eth_getProof not supported"), true}, + {errors.New("execution reverted"), false}, + } + for _, tc := range cases { + assert.Equal(t, tc.want, isMethodUnsupportedErr(tc.err), "%v", tc.err) + } +} + +func TestCovInteg_Resolver(t *testing.T) { + ctx := context.Background() + assert.Nil(t, newIntegrityResolver(ctx, nil, nil, "")) + + t.Run("finality needs an EVM upstream", func(t *testing.T) { + r := &integrityResolver{upstream: covIntegPlainUp{common.NewFakeUpstream("u")}} + fin, known := r.IsFinalized(ctx, 5) + assert.False(t, fin) + assert.False(t, known) + }) + + t.Run("no view means canonical unavailable", func(t *testing.T) { + r := &integrityResolver{} + _, ok := r.CanonicalReceipts(ctx, "0xbb") + assert.False(t, ok) + _, ok = r.CanonicalHeader(ctx, "0x10") + assert.False(t, ok) + }) + + t.Run("refs route by hash, number, or tag", func(t *testing.T) { + hash := "0x" + fmt.Sprintf("%064x", 0xabc) + net := newCovIntegNet("cov-integ-resolver", nil, func(req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + jrq, _ := req.JsonRpcRequest() + ref := jrq.Params[0].(string) + switch { + case jrq.Method == "eth_getBlockByHash" && ref == hash: + return covIntegResp(covIntegHeaderJSON(0x20, hash, "0xp")), nil + case jrq.Method == "eth_getBlockByNumber" && ref == "0x10": + return covIntegResp(covIntegHeaderJSON(0x10, "0xn10", "0xp")), nil + case jrq.Method == "eth_getBlockByNumber" && ref == "latest": + return covIntegResp(covIntegHeaderJSON(0x30, "0xlatest", "0xp")), nil + } + return covIntegResp("null"), nil + }) + r := &integrityResolver{network: net, view: newChainView(net, 64, "", "", nil)} + + h, ok := r.CanonicalHeader(ctx, hash) + require.True(t, ok) + assert.Equal(t, hash, h.Hash) + _, pinned := r.view.HashAt(0x20) + assert.False(t, pinned, "a by-hash lookup never pins") + + h, ok = r.CanonicalHeader(ctx, "0x10") + require.True(t, ok) + assert.Equal(t, "0xn10", h.Hash) + assert.Equal(t, "0xn10", mustHashAt(t, r.view, 0x10)) + + h, ok = r.CanonicalHeader(ctx, "latest") + require.True(t, ok) + assert.Equal(t, "0xlatest", h.Hash) + assert.Equal(t, "0xlatest", mustHashAt(t, r.view, 0x30)) + }) +} + +func TestCovInteg_ConfigOverridesAndOverlay(t *testing.T) { + t.Run("override without an enable flag only touches checks the level enabled", func(t *testing.T) { + cs, _ := compileIntegritySettings(&common.IntegritySettings{ + Level: "intrinsic", + Checks: map[string]*common.IntegrityCheckConfig{ + "receiptVsBlock": {Params: map[string]string{"k": "v"}}, + "bloomMatch": {Params: map[string]string{"mode": "superset"}}, + "indexMagnitude": nil, + }, + }, 0) + assert.False(t, cs.For("receiptVsBlock").Enabled, "params alone must not enable a check above the level") + require.True(t, cs.For("bloomMatch").Enabled) + assert.Equal(t, "superset", cs.For("bloomMatch").Params["mode"]) + assert.True(t, cs.For("indexMagnitude").Enabled, "a nil override is ignored") + }) + + t.Run("a selected profile overlays every set field as an independent copy", func(t *testing.T) { + fls := false + profile := &common.IntegritySettings{ + Budget: &common.IntegrityBudgetConfig{MaxPerSecond: 5}, + InvalidBehavior: &common.IntegrityInvalidBehaviorConfig{Finalized: "hardReject"}, + AutoCorrectWhenPossible: &fls, + Checks: map[string]*common.IntegrityCheckConfig{"bloomMatch": {OnFailure: "recordOnly"}}, + } + cfg := &common.IntegrityConfig{ + IntegritySettings: common.IntegritySettings{Level: "intrinsic"}, + HeaderMode: "profiles", + Profiles: map[string]*common.IntegritySettings{"p": profile}, + } + got := resolveRequestSettings(cfg, "p") + require.NotNil(t, got) + assert.Equal(t, "intrinsic", got.Level, "an unset level keeps the base") + require.NotNil(t, got.Budget) + assert.Equal(t, 5, got.Budget.MaxPerSecond) + require.NotNil(t, got.InvalidBehavior) + assert.Equal(t, "hardReject", got.InvalidBehavior.Finalized) + require.NotNil(t, got.AutoCorrectWhenPossible) + assert.False(t, *got.AutoCorrectWhenPossible) + require.Contains(t, got.Checks, "bloomMatch") + assert.Equal(t, "recordOnly", got.Checks["bloomMatch"].OnFailure) + + got.Budget.MaxPerSecond = 99 + got.Checks["bloomMatch"].OnFailure = "hardReject" + assert.Equal(t, 5, profile.Budget.MaxPerSecond, "the profile must not be mutated through the result") + assert.Equal(t, "recordOnly", profile.Checks["bloomMatch"].OnFailure) + + _, policy, _, autoCorrect := func() (integrity.CheckSet, integrity.ReorgPolicy, bool, bool) { + cs, p := compileIntegritySettings(resolveRequestSettings(cfg, "p"), 0) + return cs, p, false, *resolveRequestSettings(cfg, "p").AutoCorrectWhenPossible + }() + assert.Equal(t, integrity.BehaviorError, policy.Finalized) + assert.False(t, autoCorrect) + }) +} diff --git a/architecture/evm/cov_logs_test.go b/architecture/evm/cov_logs_test.go new file mode 100644 index 000000000..f97f96b85 --- /dev/null +++ b/architecture/evm/cov_logs_test.go @@ -0,0 +1,484 @@ +package evm + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "sync" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// covLogsWriteFailer fails the failAt-th Write call (1-based) and records +// every byte written before it. +type covLogsWriteFailer struct { + failAt int + calls int + buf bytes.Buffer +} + +var covLogsErrWrite = errors.New("client connection reset") + +func (w *covLogsWriteFailer) Write(p []byte) (int, error) { + w.calls++ + if w.calls == w.failAt { + return 0, covLogsErrWrite + } + return w.buf.Write(p) +} + +// covLogsBrokenResult is a result writer whose WriteTo and Size both fail, +// standing in for a sub-response whose body can no longer be read. +type covLogsBrokenResult struct{ err error } + +func (b *covLogsBrokenResult) WriteTo(io.Writer, bool) (int64, error) { return 0, b.err } +func (b *covLogsBrokenResult) IsResultEmptyish() bool { return false } +func (b *covLogsBrokenResult) Size(...context.Context) (int, error) { return 0, b.err } + +func covLogsBrokenResponse(err error) *common.JsonRpcResponse { + jrr := &common.JsonRpcResponse{} + jrr.SetResultWriter(&covLogsBrokenResult{err: err}) + return jrr +} + +func covLogsResult(body string) *common.NormalizedResponse { + return common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(body), nil), + ) +} + +func covLogsAddr(c string) string { return "0x" + strings.Repeat(c, 40) } +func covLogsTopic(c string) string { return "0x" + strings.Repeat(c, 64) } + +func TestCovLogs_ResolveBlockTagForGetLogs(t *testing.T) { + ctx := context.Background() + n := &queryTestNetwork{latest: 100, finalized: 90} + cases := []struct { + name string + net common.Network + in string + wantHex string + wantNum int64 + }{ + {"empty", n, "", "", 0}, + {"hex", n, "0x10", "0x10", 16}, + {"malformed hex", n, "0xzz", "", 0}, + {"latest", n, "latest", "0x64", 100}, + {"finalized", n, "finalized", "0x5a", 90}, + {"safe is never resolved", n, "safe", "", 0}, + {"latest without state", &queryTestNetwork{}, "latest", "", 0}, + {"no network", nil, "latest", "", 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + hx, bn := resolveBlockTagForGetLogs(ctx, tc.net, tc.in) + assert.Equal(t, tc.wantHex, hx) + assert.Equal(t, tc.wantNum, bn) + }) + } +} + +func TestCovLogs_GetLogsConcreteRangeSize_Rejects(t *testing.T) { + ctx := context.Background() + size, ok := getLogsConcreteRangeSize(ctx, nil) + assert.False(t, ok) + assert.Zero(t, size) + + cases := []struct { + name string + params []interface{} + }{ + {"no params", []interface{}{}}, + {"filter is not an object", []interface{}{"0x1"}}, + {"malformed fromBlock", []interface{}{map[string]interface{}{"fromBlock": "0xzz", "toBlock": "0x2"}}}, + {"malformed toBlock", []interface{}{map[string]interface{}{"fromBlock": "0x1", "toBlock": "0xzz"}}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + size, ok := getLogsConcreteRangeSize(ctx, common.NewJsonRpcRequest("eth_getLogs", tc.params)) + assert.False(t, ok) + assert.Zero(t, size) + }) + } + size, ok = getLogsConcreteRangeSize(ctx, common.NewJsonRpcRequest("eth_getLogs", []interface{}{ + map[string]interface{}{"fromBlock": "0x1", "toBlock": "0xa"}, + })) + assert.True(t, ok) + assert.Equal(t, float64(10), size) +} + +func TestCovLogs_CheckGetLogsFilterValues(t *testing.T) { + cases := []struct { + name string + filter map[string]interface{} + wantErr string + }{ + {"uppercase 0X prefix is accepted", map[string]interface{}{"address": "0X" + strings.Repeat("A", 40)}, ""}, + {"null topic positions are wildcards", map[string]interface{}{"topics": []interface{}{nil, covLogsTopic("b")}}, ""}, + {"topics given as a string", map[string]interface{}{"topics": covLogsTopic("b")}, "topics: expected an array"}, + {"address array holding a number", map[string]interface{}{"address": []interface{}{covLogsAddr("a"), 7.0}}, "address[1]: expected a hex string"}, + {"address given as a number", map[string]interface{}{"address": 7.0}, "address: expected a hex string or an array"}, + {"address without prefix", map[string]interface{}{"address": strings.Repeat("a", 40)}, "address: expected 20 bytes"}, + {"address with non-hex digit", map[string]interface{}{"address": "0x" + strings.Repeat("g", 40)}, "address: expected 20 bytes"}, + {"topic OR-list with short entry", map[string]interface{}{"topics": []interface{}{[]interface{}{covLogsTopic("c"), "0x01"}}}, "topics[0][1]: expected 32 bytes"}, + {"five topic positions", map[string]interface{}{"topics": []interface{}{nil, nil, nil, nil, nil}}, "at most 4 positions, got 5"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := checkGetLogsFilterValues(tc.filter) + if tc.wantErr == "" { + assert.NoError(t, err) + return + } + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeInvalidRequest)) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +func TestCovLogs_CheckGetLogsHardLimits_NoEvmConfig(t *testing.T) { + ctx := context.Background() + for name, cfg := range map[string]*common.NetworkConfig{"nil config": nil, "nil evm": {}} { + t.Run(name, func(t *testing.T) { + n := &queryTestNetwork{cfg: cfg} + from, to, resolved, err := checkGetLogsHardLimits(ctx, n, "0x1", "0x186a0", 1000, 1000) + require.NoError(t, err, "without evm config no hard limit applies") + assert.True(t, resolved) + assert.Equal(t, int64(1), from) + assert.Equal(t, int64(100000), to) + }) + } +} + +func TestCovLogs_ProjectPreForward_PassThrough(t *testing.T) { + ctx := context.Background() + n := &queryTestNetwork{cfg: &common.NetworkConfig{Evm: &common.EvmNetworkConfig{GetLogsMaxAllowedRange: 1}}} + + handled, resp, err := projectPreForward_eth_getLogs(ctx, nil, createTestRequest(map[string]interface{}{})) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + handled, _, err = projectPreForward_eth_getLogs(ctx, n, nil) + assert.False(t, handled) + assert.NoError(t, err) + + cases := map[string]*common.NormalizedRequest{ + "unparseable body": common.NewNormalizedRequest([]byte(`{not json`)), + "no params": common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest("eth_getLogs", []interface{}{})), + "positional param": common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest("eth_getLogs", []interface{}{"0x1"})), + } + for name, rq := range cases { + t.Run(name, func(t *testing.T) { + handled, resp, err := projectPreForward_eth_getLogs(ctx, n, rq) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + }) + } +} + +func TestCovLogs_NetworkPreForward_Guards(t *testing.T) { + ctx := context.Background() + + handled, _, err := networkPreForward_eth_getLogs(ctx, nil, nil, createTestRequest(map[string]interface{}{})) + assert.False(t, handled) + assert.NoError(t, err) + + t.Run("unparseable body is refused", func(t *testing.T) { + handled, resp, err := networkPreForward_eth_getLogs(ctx, &queryTestNetwork{}, nil, common.NewNormalizedRequest([]byte(`{not json`))) + assert.True(t, handled) + assert.Nil(t, resp) + require.Error(t, err) + }) + + t.Run("no evm config passes a resolved range through", func(t *testing.T) { + n := &queryTestNetwork{forwardFn: func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error) { + t.Fatal("must not forward") + return nil, nil + }} + handled, resp, err := networkPreForward_eth_getLogs(ctx, n, nil, createTestRequest(map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x1000"})) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + }) + + t.Run("split threshold above max range is capped and does not split an in-cap range", func(t *testing.T) { + n := &queryTestNetwork{ + cfg: &common.NetworkConfig{Evm: &common.EvmNetworkConfig{GetLogsMaxAllowedRange: 10}}, + forwardFn: func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error) { + t.Fatal("must not forward") + return nil, nil + }, + } + ups := []common.Upstream{ + nil, + &queryTestConfigUpstream{cfg: &common.UpstreamConfig{}}, + &queryTestConfigUpstream{cfg: &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{GetLogsAutoSplittingRangeThreshold: 100}}}, + } + handled, resp, err := networkPreForward_eth_getLogs(ctx, n, ups, createTestRequest(map[string]interface{}{"fromBlock": "0x1", "toBlock": "0xa"})) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + }) +} + +func TestCovLogs_NetworkPreForward_ProactiveSplitSkipsUnconfiguredUpstreams(t *testing.T) { + ctx := context.Background() + var mu sync.Mutex + var seen []string + n := &queryTestNetwork{ + cfg: &common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}, + forwardFn: func(_ context.Context, r *common.NormalizedRequest) (*common.NormalizedResponse, error) { + jrq, _ := r.JsonRpcRequest() + f := jrq.Params[0].(map[string]interface{}) + mu.Lock() + seen = append(seen, f["fromBlock"].(string)+"-"+f["toBlock"].(string)+"/"+r.Directives().SkipCacheRead) + mu.Unlock() + return covLogsResult(`["` + f["fromBlock"].(string) + `"]`), nil + }, + } + ups := []common.Upstream{ + nil, + &queryTestConfigUpstream{cfg: &common.UpstreamConfig{}}, + &queryTestConfigUpstream{cfg: &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{GetLogsAutoSplittingRangeThreshold: 2}}}, + } + rq := createTestRequest(map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x4"}) + rq.SetDirectives(&common.RequestDirectives{SkipCacheRead: "true"}) + + handled, resp, err := networkPreForward_eth_getLogs(ctx, n, ups, rq) + require.NoError(t, err) + require.True(t, handled) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + var buf bytes.Buffer + _, err = jrr.WriteResultTo(&buf, false) + require.NoError(t, err) + assert.JSONEq(t, `["0x1","0x3"]`, buf.String()) + assert.ElementsMatch(t, []string{"0x1-0x2/true", "0x3-0x4/true"}, seen) + assert.Equal(t, common.CompositeTypeLogsSplitProactive, rq.CompositeType()) +} + +func TestCovLogs_UpstreamPreForward_Guards(t *testing.T) { + ctx := context.Background() + enforce := &common.NetworkConfig{Evm: &common.EvmNetworkConfig{Integrity: &common.EvmIntegrityConfig{EnforceGetLogsBlockRange: util.BoolPtr(true)}}} + n := &queryTestNetwork{cfg: enforce, latest: 100} + u := common.NewFakeUpstream("rpc1") + + t.Run("non-evm upstream is passed through", func(t *testing.T) { + handled, resp, err := upstreamPreForward_eth_getLogs(ctx, n, &queryTestConfigUpstream{}, createTestRequest(map[string]interface{}{"fromBlock": "0x5", "toBlock": "0x1"})) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + }) + + t.Run("unparseable body is refused", func(t *testing.T) { + handled, _, err := upstreamPreForward_eth_getLogs(ctx, n, u, common.NewNormalizedRequest([]byte(`{not json`))) + assert.True(t, handled) + require.Error(t, err) + }) + + cases := map[string]*common.NormalizedRequest{ + "no params": common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest("eth_getLogs", []interface{}{})), + "positional param": common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest("eth_getLogs", []interface{}{"0x1"})), + "unresolvable safe": createTestRequest(map[string]interface{}{"fromBlock": "safe", "toBlock": "0x1"}), + "unresolvable bounds": createTestRequest(map[string]interface{}{"fromBlock": "0x1", "toBlock": "pending"}), + } + for name, rq := range cases { + t.Run(name, func(t *testing.T) { + handled, resp, err := upstreamPreForward_eth_getLogs(ctx, n, u, rq) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + }) + } + + t.Run("inverted range after resolving latest", func(t *testing.T) { + handled, _, err := upstreamPreForward_eth_getLogs(ctx, n, u, createTestRequest(map[string]interface{}{"fromBlock": "0xc8", "toBlock": "latest"})) + assert.True(t, handled) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeInvalidRequest)) + assert.Contains(t, err.Error(), "fromBlock (200) must be less than or equal to toBlock (100)") + }) +} + +func TestCovLogs_NetworkPostForward_SplitOnErrorFallbacks(t *testing.T) { + ctx := context.Background() + tooLarge := common.NewErrEndpointRequestTooLarge(errors.New("query returned more than 10000 results"), common.EvmBlockRangeTooLarge) + cfg := &common.NetworkConfig{Evm: &common.EvmNetworkConfig{GetLogsSplitOnError: util.BoolPtr(true)}} + + t.Run("unsplittable single-block request keeps the upstream error", func(t *testing.T) { + n := &queryTestNetwork{cfg: cfg, forwardFn: func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error) { + t.Fatal("must not forward") + return nil, nil + }} + orig := common.NewNormalizedResponse() + rs, err := networkPostForward_eth_getLogs(ctx, n, createTestRequest(map[string]interface{}{"fromBlock": "0x7", "toBlock": "0x7", "address": covLogsAddr("a")}), orig, tooLarge) + assert.Same(t, orig, rs) + assert.Same(t, tooLarge, err) + }) + + t.Run("failed sub-request keeps the original error and response", func(t *testing.T) { + var mu sync.Mutex + var skips []string + n := &queryTestNetwork{cfg: cfg, forwardFn: func(_ context.Context, r *common.NormalizedRequest) (*common.NormalizedResponse, error) { + mu.Lock() + skips = append(skips, r.Directives().SkipCacheRead) + mu.Unlock() + return nil, errors.New("upstream timeout") + }} + rq := createTestRequest(map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x2"}) + rq.SetDirectives(&common.RequestDirectives{SkipCacheRead: "conn-a"}) + orig := common.NewNormalizedResponse() + rs, err := networkPostForward_eth_getLogs(ctx, n, rq, orig, tooLarge) + assert.Same(t, orig, rs) + assert.Same(t, tooLarge, err) + assert.Equal(t, []string{"conn-a", "conn-a"}, skips, "sub-requests inherit the parent's skip-cache-read directive") + }) + + t.Run("derived sub-requests never split again", func(t *testing.T) { + n := &queryTestNetwork{cfg: cfg} + rq := createTestRequest(map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x2"}) + rq.SetParentRequestId("parent") + rs, err := networkPostForward_eth_getLogs(ctx, n, rq, nil, tooLarge) + assert.Nil(t, rs) + assert.Same(t, tooLarge, err) + }) +} + +func TestCovLogs_SplitEthGetLogsRequest_Invalid(t *testing.T) { + cases := []struct { + name string + rq *common.NormalizedRequest + wantErr string + }{ + {"unparseable body", common.NewNormalizedRequest([]byte(`{not json`)), ""}, + {"no params", common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest("eth_getLogs", []interface{}{})), "invalid params length"}, + {"positional param", common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest("eth_getLogs", []interface{}{"0x1"})), "invalid filter format"}, + {"malformed fromBlock", createTestRequest(map[string]interface{}{"fromBlock": "0xzz", "toBlock": "0x2"}), "invalid syntax"}, + {"tag toBlock", createTestRequest(map[string]interface{}{"fromBlock": "0x1", "toBlock": "latest"}), "invalid toBlock"}, + {"malformed toBlock", createTestRequest(map[string]interface{}{"fromBlock": "0x1", "toBlock": "0xzz"}), "invalid syntax"}, + {"single topic0 cannot split", createTestRequest(map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x1", "topics": []interface{}{[]interface{}{covLogsTopic("a")}}}), "cannot be split further"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + subs, err := splitEthGetLogsRequest(tc.rq) + require.Error(t, err) + assert.Nil(t, subs) + if tc.wantErr != "" { + assert.Contains(t, err.Error(), tc.wantErr) + } + }) + } +} + +func TestCovLogs_ExecuteSubRequests_FailureModes(t *testing.T) { + ctx := context.Background() + subs := []ethGetLogsSubRequest{{fromBlock: 1, toBlock: 1}, {fromBlock: 2, toBlock: 2}} + newReq := func() *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getLogs","params":[{"fromBlock":"0x1","toBlock":"0x2"}],"id":1}`)) + } + brokenErr := errors.New("sub-response body already released") + cases := []struct { + name string + forward func(r *common.NormalizedRequest) (*common.NormalizedResponse, error) + wantErr string + }{ + { + name: "nil response without error", + forward: func(*common.NormalizedRequest) (*common.NormalizedResponse, error) { return nil, nil }, + wantErr: "unexpected empty json-rpc response", + }, + { + name: "json-rpc error in sub-response", + forward: func(*common.NormalizedRequest) (*common.NormalizedResponse, error) { + return common.NewNormalizedResponse().WithJsonRpcResponse(common.MustNewJsonRpcResponseFromBytes( + []byte(`1`), nil, []byte(`{"code":-32005,"message":"query returned more than 10000 results"}`), + )), nil + }, + wantErr: "query returned more than 10000 results", + }, + { + name: "unreadable sub-response body", + forward: func(*common.NormalizedRequest) (*common.NormalizedResponse, error) { + return common.NewNormalizedResponse().WithJsonRpcResponse(covLogsBrokenResponse(brokenErr)), nil + }, + wantErr: brokenErr.Error(), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + n := &queryTestNetwork{ + cfg: &common.NetworkConfig{Evm: &common.EvmNetworkConfig{GetLogsSplitConcurrency: 1}}, + forwardFn: func(_ context.Context, r *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return tc.forward(r) + }, + } + merged, fromCache, err := executeGetLogsSubRequests(ctx, n, newReq(), subs, "") + require.Error(t, err) + assert.Nil(t, merged) + assert.False(t, fromCache) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } + + t.Run("unmarshalable parent id fails the merge", func(t *testing.T) { + n := &queryTestNetwork{forwardFn: func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return covLogsResult(`["log"]`), nil + }} + jrq := common.NewJsonRpcRequest("eth_getLogs", []interface{}{map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x2"}}) + jrq.ID = make(chan int) + merged, _, err := executeGetLogsSubRequests(ctx, n, common.NewNormalizedRequestFromJsonRpcRequest(jrq), subs, "") + require.Error(t, err) + assert.Nil(t, merged) + }) +} + +func TestCovLogs_MultiResponseWriter_WriteErrors(t *testing.T) { + newWriter := func() *GetLogsMultiResponseWriter { + return NewGetLogsMultiResponseWriter([]*common.JsonRpcResponse{ + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`["a"]`), nil), + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`["b"]`), nil), + }) + } + // Write order: "[", `"a"`, ",", `"b"`, "]". + for failAt, wantPrefix := range map[int]string{1: ``, 2: `[`, 3: `["a"`, 4: `["a",`, 5: `["a","b"`} { + w := &covLogsWriteFailer{failAt: failAt} + n, err := newWriter().WriteTo(w, false) + assert.ErrorIs(t, err, covLogsErrWrite, "write %d", failAt) + assert.Equal(t, wantPrefix, w.buf.String(), "write %d", failAt) + assert.Equal(t, int64(len(wantPrefix)), n, "write %d", failAt) + } + + var ok bytes.Buffer + n, err := newWriter().WriteTo(&ok, false) + require.NoError(t, err) + assert.Equal(t, `["a","b"]`, ok.String()) + assert.Equal(t, int64(9), n) +} + +func TestCovLogs_MultiResponseWriter_EmptyishAndSize(t *testing.T) { + assert.True(t, NewGetLogsMultiResponseWriter(nil).IsResultEmptyish()) + assert.True(t, NewGetLogsMultiResponseWriter([]*common.JsonRpcResponse{ + nil, + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`[]`), nil), + }).IsResultEmptyish()) + assert.False(t, NewGetLogsMultiResponseWriter([]*common.JsonRpcResponse{ + nil, + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`["x"]`), nil), + }).IsResultEmptyish()) + + sizeErr := errors.New("size unavailable") + size, err := NewGetLogsMultiResponseWriter([]*common.JsonRpcResponse{ + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`["x"]`), nil), + covLogsBrokenResponse(sizeErr), + }).Size() + assert.ErrorIs(t, err, sizeErr) + assert.Zero(t, size) +} diff --git a/architecture/evm/cov_misc_test.go b/architecture/evm/cov_misc_test.go new file mode 100644 index 000000000..6143593e8 --- /dev/null +++ b/architecture/evm/cov_misc_test.go @@ -0,0 +1,933 @@ +package evm + +import ( + "context" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +// --------------------------------------------------------------------------- +// multicall3 +// --------------------------------------------------------------------------- + +// covMiscMcUpstream is an upstream carrying a Multicall3 batcher and a +// pluggable Forward that receives the aggregate3 eth_call. +type covMiscMcUpstream struct { + common.Upstream + b *Multicall3Batcher + forward func(ctx context.Context, nq *common.NormalizedRequest) (*common.NormalizedResponse, error) +} + +func (u *covMiscMcUpstream) EvmMulticall3Batcher() *Multicall3Batcher { return u.b } +func (u *covMiscMcUpstream) Forward(ctx context.Context, nq *common.NormalizedRequest, _ bool, _ bool) (*common.NormalizedResponse, error) { + return u.forward(ctx, nq) +} + +func covMiscMcNew(t *testing.T, maxCalls int, maxWait time.Duration, forward func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error)) *covMiscMcUpstream { + id := "mc-" + strings.NewReplacer("/", "-", " ", "_").Replace(t.Name()) + return &covMiscMcUpstream{ + Upstream: common.NewFakeUpstream(id), + b: NewMulticall3Batcher(&common.EvmMulticall3Config{MaxCalls: maxCalls, MaxWait: common.Duration(maxWait)}), + forward: forward, + } +} + +func covMiscMcCount(u *covMiscMcUpstream, outcome string) float64 { + return testutil.ToFloat64(telemetry.MetricUpstreamMulticall3CallsTotal.WithLabelValues("test-project", "n/a", u.Id(), u.VendorName(), outcome)) +} + +func covMiscMcCall(id int, to, data, block string) *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(fmt.Sprintf( + `{"jsonrpc":"2.0","id":%d,"method":"eth_call","params":[{"to":"%s","data":"%s"},"%s"]}`, id, to, data, block))) +} + +func covMiscWord(v uint64) []byte { + var w [32]byte + binary.BigEndian.PutUint64(w[24:], v) + return w[:] +} + +// covMiscAgg3 ABI-encodes an aggregate3 return value: (bool, bytes)[]. +func covMiscAgg3(results []aggregate3Result) []byte { + out := append(covMiscWord(32), covMiscWord(uint64(len(results)))...) + offset := uint64(32 * len(results)) + for _, r := range results { + out = append(out, covMiscWord(offset)...) + offset += uint64(96 + padded32(len(r.data))) + } + for _, r := range results { + s := uint64(0) + if r.success { + s = 1 + } + out = append(out, covMiscWord(s)...) + out = append(out, covMiscWord(64)...) + out = append(out, covMiscWord(uint64(len(r.data)))...) + out = append(out, r.data...) + out = append(out, make([]byte, padded32(len(r.data))-len(r.data))...) + } + return out +} + +// covMiscAgg3Targets reads the inner-call targets back out of an aggregate3 +// eth_call the batcher sent. +func covMiscAgg3Targets(t *testing.T, nq *common.NormalizedRequest) (to string, targets [][]byte) { + jrq, err := nq.JsonRpcRequest() + require.NoError(t, err) + call := jrq.Params[0].(map[string]interface{}) + raw, err := hex.DecodeString(strings.TrimPrefix(call["data"].(string), "0x")) + require.NoError(t, err) + require.Equal(t, aggregate3Selector, raw[:4]) + body := raw[4:] + count := int(binary.BigEndian.Uint64(body[56:64])) + head := 64 + for i := range count { + off := int(binary.BigEndian.Uint64(body[head+32*i+24 : head+32*i+32])) + tuple := head + off + targets = append(targets, body[tuple+12:tuple+32]) + } + return call["to"].(string), targets +} + +func covMiscMcResult(result string) *common.NormalizedResponse { + return common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(result), nil)) +} + +type covMiscMcOut struct { + handled bool + resp *common.NormalizedResponse + err error +} + +// covMiscMcRun submits every request concurrently and returns the hook's +// answers in request order. +func covMiscMcRun(ctxs []context.Context, n common.Network, u *covMiscMcUpstream, reqs []*common.NormalizedRequest) []covMiscMcOut { + outs := make([]covMiscMcOut, len(reqs)) + var wg sync.WaitGroup + for i := range reqs { + wg.Add(1) + go func(i int) { + defer wg.Done() + h, r, e := upstreamPreForward_eth_call(ctxs[i], n, u, reqs[i]) + outs[i] = covMiscMcOut{h, r, e} + }(i) + } + wg.Wait() + return outs +} + +func covMiscBg(k int) []context.Context { + out := make([]context.Context, k) + for i := range out { + out[i] = context.Background() + } + return out +} + +func covMiscResultString(t *testing.T, r *common.NormalizedResponse) string { + jrr, err := r.JsonRpcResponse() + require.NoError(t, err) + var s string + require.NoError(t, json.Unmarshal(jrr.GetResultBytes(), &s)) + return s +} + +func TestCovMisc_Multicall3_PacksFullBatchAndAnswersEachCaller(t *testing.T) { + n := &queryTestNetwork{} + addrA, addrB := covLogsAddr("a"), covLogsAddr("b") + var gotDeadline bool + u := covMiscMcNew(t, 2, time.Minute, nil) + u.forward = func(ctx context.Context, nq *common.NormalizedRequest) (*common.NormalizedResponse, error) { + _, gotDeadline = ctx.Deadline() + assert.Equal(t, common.CompositeTypeMulticall3, nq.CompositeType()) + to, targets := covMiscAgg3Targets(t, nq) + assert.Equal(t, multicall3Address, to) + res := make([]aggregate3Result, len(targets)) + for i, tg := range targets { + res[i] = aggregate3Result{success: true, data: tg} + } + return covMiscMcResult(`"0x` + hex.EncodeToString(covMiscAgg3(res)) + `"`), nil + } + before := covMiscMcCount(u, multicall3Packed) + + reqs := []*common.NormalizedRequest{covMiscMcCall(1, addrA, "0x01", "0x10"), covMiscMcCall(2, addrB, "0x02", "0x10")} + outs := covMiscMcRun(covMiscBg(2), n, u, reqs) + + for i, want := range []string{addrA, addrB} { + require.NoError(t, outs[i].err) + require.True(t, outs[i].handled, "call %d is answered from the pack", i) + assert.Equal(t, want, covMiscResultString(t, outs[i].resp), "each caller gets its own return data") + assert.Same(t, u, outs[i].resp.Upstream()) + assert.Same(t, u, reqs[i].LastUpstream()) + } + assert.False(t, gotDeadline, "callers without a deadline give the pack none") + assert.Equal(t, before+2, covMiscMcCount(u, multicall3Packed)) +} + +func TestCovMisc_Multicall3_PackDeadlineIsLastCallers(t *testing.T) { + n := &queryTestNetwork{} + now := time.Now() + ctx1, c1 := context.WithDeadline(context.Background(), now.Add(time.Hour)) + defer c1() + ctx2, c2 := context.WithDeadline(context.Background(), now.Add(2*time.Hour)) + defer c2() + var deadline time.Time + u := covMiscMcNew(t, 2, time.Minute, func(ctx context.Context, nq *common.NormalizedRequest) (*common.NormalizedResponse, error) { + deadline, _ = ctx.Deadline() + return covMiscMcResult(`"0x` + hex.EncodeToString(covMiscAgg3([]aggregate3Result{{true, []byte{1}}, {true, []byte{2}}})) + `"`), nil + }) + outs := covMiscMcRun([]context.Context{ctx2, ctx1}, n, u, []*common.NormalizedRequest{ + covMiscMcCall(1, covLogsAddr("a"), "0x01", "latest"), + covMiscMcCall(2, covLogsAddr("b"), "0x02", "latest"), + }) + assert.True(t, outs[0].handled) + assert.True(t, outs[1].handled) + assert.WithinDuration(t, now.Add(2*time.Hour), deadline, time.Second) +} + +func TestCovMisc_Multicall3_FailedPackSendsEveryCallAlone(t *testing.T) { + cases := []struct { + name string + forward func() (*common.NormalizedResponse, error) + }{ + {"forward error", func() (*common.NormalizedResponse, error) { return nil, errors.New("upstream timeout") }}, + {"nil response", func() (*common.NormalizedResponse, error) { return nil, nil }}, + {"json-rpc error", func() (*common.NormalizedResponse, error) { + return common.NewNormalizedResponse().WithJsonRpcResponse(common.MustNewJsonRpcResponseFromBytes( + []byte(`1`), nil, []byte(`{"code":-32000,"message":"execution reverted"}`))), nil + }}, + {"result is not a string", func() (*common.NormalizedResponse, error) { return covMiscMcResult(`123`), nil }}, + {"result is not hex", func() (*common.NormalizedResponse, error) { return covMiscMcResult(`"0xzz"`), nil }}, + {"malformed aggregate3", func() (*common.NormalizedResponse, error) { return covMiscMcResult(`"0x01"`), nil }}, + {"result count mismatch", func() (*common.NormalizedResponse, error) { + return covMiscMcResult(`"0x` + hex.EncodeToString(covMiscAgg3([]aggregate3Result{{true, []byte{1}}})) + `"`), nil + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + u := covMiscMcNew(t, 2, time.Minute, func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return tc.forward() + }) + before := covMiscMcCount(u, multicall3Failed) + outs := covMiscMcRun(covMiscBg(2), &queryTestNetwork{}, u, []*common.NormalizedRequest{ + covMiscMcCall(1, covLogsAddr("a"), "0x01", "latest"), + covMiscMcCall(2, covLogsAddr("b"), "0x02", "latest"), + }) + for _, o := range outs { + assert.False(t, o.handled) + assert.Nil(t, o.resp) + assert.NoError(t, o.err) + } + assert.Equal(t, before+2, covMiscMcCount(u, multicall3Failed)) + }) + } +} + +func TestCovMisc_Multicall3_NoCodeMarksOlderBlocksIneligible(t *testing.T) { + var forwards int + var mu sync.Mutex + u := covMiscMcNew(t, 2, time.Minute, func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error) { + mu.Lock() + forwards++ + mu.Unlock() + return covMiscMcResult(`"0x"`), nil + }) + n := &queryTestNetwork{} + outs := covMiscMcRun(covMiscBg(2), n, u, []*common.NormalizedRequest{ + covMiscMcCall(1, covLogsAddr("a"), "0x01", "0x10"), + covMiscMcCall(2, covLogsAddr("b"), "0x02", "0x10"), + }) + assert.False(t, outs[0].handled) + assert.False(t, outs[1].handled) + assert.Equal(t, int64(16), u.b.absentAtOrBelow.Load()) + + before := covMiscMcCount(u, multicall3Ineligible) + handled, _, err := upstreamPreForward_eth_call(context.Background(), n, u, covMiscMcCall(3, covLogsAddr("c"), "0x03", "0x5")) + assert.False(t, handled) + assert.NoError(t, err) + assert.Equal(t, before+1, covMiscMcCount(u, multicall3Ineligible), "a block before deployment is never packed") + assert.Equal(t, 1, forwards) +} + +func TestCovMisc_Multicall3_RevertedInPackLearnsBypassOnlyOnCleanAloneSuccess(t *testing.T) { + addrA, addrB := covLogsAddr("a"), covLogsAddr("b") + u := covMiscMcNew(t, 2, time.Minute, func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return covMiscMcResult(`"0x` + hex.EncodeToString(covMiscAgg3([]aggregate3Result{{false, nil}, {false, nil}})) + `"`), nil + }) + n := &queryTestNetwork{} + reqs := []*common.NormalizedRequest{covMiscMcCall(1, addrA, "0x01", "latest"), covMiscMcCall(2, addrB, "0x02", "latest")} + before := covMiscMcCount(u, multicall3Reverted) + outs := covMiscMcRun(covMiscBg(2), n, u, reqs) + assert.False(t, outs[0].handled) + assert.False(t, outs[1].handled) + assert.Equal(t, before+2, covMiscMcCount(u, multicall3Reverted)) + + rawA, _ := hex.DecodeString(strings.TrimPrefix(addrA, "0x")) + rawB, _ := hex.DecodeString(strings.TrimPrefix(addrB, "0x")) + + // A failed alone too (json-rpc error): the contract is not learned. + errResp := common.NewNormalizedResponse().WithJsonRpcResponse(common.MustNewJsonRpcResponseFromBytes( + []byte(`1`), nil, []byte(`{"code":3,"message":"execution reverted"}`))) + upstreamPostForward_eth_call(u, reqs[0], errResp, nil) + assert.False(t, u.b.bypassed(rawA)) + // A post for A again finds nothing awaiting. + upstreamPostForward_eth_call(u, reqs[0], covMiscMcResult(`"0x01"`), nil) + assert.False(t, u.b.bypassed(rawA)) + + // B succeeded alone: B's calls bypass the pack from now on. + upstreamPostForward_eth_call(u, reqs[1], covMiscMcResult(`"0x01"`), nil) + assert.True(t, u.b.bypassed(rawB)) + + beforeBypass := covMiscMcCount(u, multicall3Bypassed) + handled, _, err := upstreamPreForward_eth_call(context.Background(), n, u, covMiscMcCall(3, addrB, "0x02", "latest")) + assert.False(t, handled) + assert.NoError(t, err) + assert.Equal(t, beforeBypass+1, covMiscMcCount(u, multicall3Bypassed)) +} + +func TestCovMisc_Multicall3_PostForwardTransportErrorLearnsNothing(t *testing.T) { + addrA := covLogsAddr("d") + rawA, _ := hex.DecodeString(strings.TrimPrefix(addrA, "0x")) + u := covMiscMcNew(t, 2, time.Minute, nil) + u.forward = func(_ context.Context, nq *common.NormalizedRequest) (*common.NormalizedResponse, error) { + // Pack order follows arrival: answer by target, A reverts. + _, targets := covMiscAgg3Targets(t, nq) + res := make([]aggregate3Result, len(targets)) + for i, tg := range targets { + if string(tg) == string(rawA) { + res[i] = aggregate3Result{success: false} + } else { + res[i] = aggregate3Result{success: true, data: []byte{9}} + } + } + return covMiscMcResult(`"0x` + hex.EncodeToString(covMiscAgg3(res)) + `"`), nil + } + reqs := []*common.NormalizedRequest{covMiscMcCall(1, addrA, "0x01", "latest"), covMiscMcCall(2, covLogsAddr("e"), "0x02", "latest")} + outs := covMiscMcRun(covMiscBg(2), &queryTestNetwork{}, u, reqs) + assert.False(t, outs[0].handled) + assert.True(t, outs[1].handled) + assert.Equal(t, "0x09", covMiscResultString(t, outs[1].resp)) + + upstreamPostForward_eth_call(u, reqs[0], nil, errors.New("connection reset")) + assert.False(t, u.b.bypassed(rawA)) +} + +func TestCovMisc_Multicall3_UnencodableCallerIdFallsBackAlone(t *testing.T) { + u := covMiscMcNew(t, 2, time.Minute, func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return covMiscMcResult(`"0x` + hex.EncodeToString(covMiscAgg3([]aggregate3Result{{true, []byte{1}}, {true, []byte{2}}})) + `"`), nil + }) + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{map[string]interface{}{"to": covLogsAddr("a"), "data": "0x01"}, "latest"}) + jrq.ID = make(chan int) + bad := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + before := covMiscMcCount(u, multicall3Failed) + outs := covMiscMcRun(covMiscBg(2), &queryTestNetwork{}, u, []*common.NormalizedRequest{ + covMiscMcCall(1, covLogsAddr("b"), "0x02", "latest"), bad, + }) + assert.True(t, outs[0].handled) + assert.False(t, outs[1].handled, "an answer that cannot carry the caller's id is not served") + assert.Equal(t, before+1, covMiscMcCount(u, multicall3Failed)) +} + +func TestCovMisc_Multicall3_AloneCanceledAndIneligible(t *testing.T) { + n := &queryTestNetwork{} + noForward := func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error) { + t.Error("a lone or ineligible call must not be packed") + return nil, nil + } + + t.Run("a lone call is flushed by the timer and sent alone", func(t *testing.T) { + u := covMiscMcNew(t, 5, time.Millisecond, noForward) + before := covMiscMcCount(u, multicall3Alone) + handled, resp, err := upstreamPreForward_eth_call(context.Background(), n, u, covMiscMcCall(1, covLogsAddr("a"), "0x01", "latest")) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + assert.Equal(t, before+1, covMiscMcCount(u, multicall3Alone)) + }) + + t.Run("a canceled caller stops waiting", func(t *testing.T) { + u := covMiscMcNew(t, 5, 10*time.Millisecond, noForward) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + before := covMiscMcCount(u, multicall3Canceled) + handled, _, err := upstreamPreForward_eth_call(ctx, n, u, covMiscMcCall(1, covLogsAddr("a"), "0x01", "latest")) + assert.False(t, handled) + assert.NoError(t, err) + assert.Equal(t, before+1, covMiscMcCount(u, multicall3Canceled)) + }) + + t.Run("an unencodable block selector is ineligible", func(t *testing.T) { + u := covMiscMcNew(t, 5, time.Minute, noForward) + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{map[string]interface{}{"to": covLogsAddr("a")}, make(chan int)}) + before := covMiscMcCount(u, multicall3Ineligible) + handled, _, err := upstreamPreForward_eth_call(context.Background(), n, u, common.NewNormalizedRequestFromJsonRpcRequest(jrq)) + assert.False(t, handled) + assert.NoError(t, err) + assert.Equal(t, before+1, covMiscMcCount(u, multicall3Ineligible)) + }) + + t.Run("upstreams without a batcher and pack requests pass through", func(t *testing.T) { + rq := covMiscMcCall(1, covLogsAddr("a"), "0x01", "latest") + handled, _, _ := upstreamPreForward_eth_call(context.Background(), n, common.NewFakeUpstream("plain"), rq) + assert.False(t, handled) + handled, _, _ = upstreamPreForward_eth_call(context.Background(), n, &covMiscMcUpstream{Upstream: common.NewFakeUpstream("nil-batcher")}, rq) + assert.False(t, handled) + u := covMiscMcNew(t, 5, time.Minute, noForward) + pack := covMiscMcCall(2, covLogsAddr("a"), "0x01", "latest") + pack.SetCompositeType(common.CompositeTypeMulticall3) + handled, _, _ = upstreamPreForward_eth_call(context.Background(), n, u, pack) + assert.False(t, handled) + // Post-forward on such upstreams is a no-op. + upstreamPostForward_eth_call(common.NewFakeUpstream("plain"), rq, nil, nil) + upstreamPostForward_eth_call(&covMiscMcUpstream{Upstream: common.NewFakeUpstream("nil-batcher")}, rq, nil, nil) + }) +} + +func TestCovMisc_Multicall3CallOf(t *testing.T) { + to := covLogsAddr("a") + cases := []struct { + name string + body string + ok bool + wantData string + wantGas string + wantFrom string + }{ + {"unparseable", `{not json`, false, "", "", ""}, + {"block missing", `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"` + to + `"}]}`, false, "", "", ""}, + {"block null", `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"` + to + `"},null]}`, false, "", "", ""}, + {"call not an object", `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":["` + to + `","latest"]}`, false, "", "", ""}, + {"non-string field", `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"` + to + `","gas":21000},"latest"]}`, false, "", "", ""}, + {"value cannot be carried", `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"` + to + `","value":"0x1"},"latest"]}`, false, "", "", ""}, + {"non-zero sender", `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"` + to + `","from":"` + covLogsAddr("1") + `"},"latest"]}`, false, "", "", ""}, + {"data and input disagree", `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"` + to + `","data":"0x01","input":"0x02"},"latest"]}`, false, "", "", ""}, + {"bad target", `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0x1234"},"latest"]}`, false, "", "", ""}, + {"bad calldata", `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"` + to + `","data":"0xzz"},"latest"]}`, false, "", "", ""}, + {"null fields ignored, zero sender and gas kept", `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"` + to + `","value":null,"gas":"0x5208","from":"0x0000000000000000000000000000000000000000","data":"0xAB","input":"0xab"},"latest"]}`, true, "ab", "0x5208", "0x0000000000000000000000000000000000000000"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + target, data, block, gas, from, ok := multicall3CallOf(context.Background(), common.NewNormalizedRequest([]byte(tc.body))) + require.Equal(t, tc.ok, ok) + if !ok { + assert.Nil(t, target) + return + } + assert.Equal(t, strings.TrimPrefix(to, "0x"), hex.EncodeToString(target)) + assert.Equal(t, tc.wantData, hex.EncodeToString(data)) + assert.Equal(t, "latest", block) + assert.Equal(t, tc.wantGas, gas) + assert.Equal(t, tc.wantFrom, from) + }) + } +} + +func TestCovMisc_Multicall3BlockNumber(t *testing.T) { + cases := []struct { + in interface{} + want int64 + }{ + {"0x10", 16}, + {"latest", 0}, + {map[string]interface{}{"blockNumber": "0x20"}, 32}, + {"0x" + strings.Repeat("f", 20), 0}, + {true, 0}, + } + for _, tc := range cases { + assert.Equal(t, tc.want, multicall3BlockNumber(tc.in), "%v", tc.in) + } +} + +func TestCovMisc_DecodeAggregate3(t *testing.T) { + valid := covMiscAgg3([]aggregate3Result{{true, []byte{0xaa, 0xbb}}, {false, nil}}) + got, ok := decodeAggregate3(valid) + require.True(t, ok) + require.Len(t, got, 2) + assert.Equal(t, aggregate3Result{success: true, data: []byte{0xaa, 0xbb}}, got[0]) + assert.False(t, got[1].success) + assert.Empty(t, got[1].data) + + // Layout of valid: [0]=32 array offset, [32]=count 2, [64],[96]=element + // offsets (64, 192), element 0 at 128: success, dataRel, length, data. + setWord := func(raw []byte, at int, v uint64) []byte { + out := append([]byte(nil), raw...) + copy(out[at:at+32], covMiscWord(v)) + return out + } + highByte := func(raw []byte, at int) []byte { + out := append([]byte(nil), raw...) + out[at] = 1 + return out + } + L := uint64(len(valid)) + cases := map[string][]byte{ + "too short": valid[:31], + "array offset high bytes": highByte(valid, 0), + "array offset beyond input": setWord(valid, 0, L+1), + "count word beyond input": setWord(valid, 0, L), + "count exceeds head space": setWord(valid, 32, (L-64)/32+1), + "element offset high bytes": highByte(valid, 64), + "success is not a bool": setWord(valid, 128, 2), + "data offset high bytes": highByte(valid, 160), + "length word beyond input": setWord(valid, 160, L-128), + "data runs past end of input": setWord(valid, 192, L), + } + for name, raw := range cases { + t.Run(name, func(t *testing.T) { + res, ok := decodeAggregate3(raw) + assert.False(t, ok) + assert.Nil(t, res) + }) + } +} + +// --------------------------------------------------------------------------- +// hooks dispatch +// --------------------------------------------------------------------------- + +func TestCovMisc_Hooks_UnparseableRequestFailsEveryHook(t *testing.T) { + ctx := context.Background() + n := &queryTestNetwork{} + u := common.NewFakeUpstream("rpc1") + bad := func() *common.NormalizedRequest { return common.NewNormalizedRequest([]byte(`{not json`)) } + rs := common.NewNormalizedResponse() + upErr := errors.New("upstream failed") + + _, _, err := HandleProjectPreForward(ctx, n, bad()) + assert.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal)) + _, _, err = HandleNetworkPreForward(ctx, n, nil, bad()) + assert.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal)) + out, err := HandleNetworkPostForward(ctx, n, bad(), rs, upErr) + assert.Same(t, rs, out) + assert.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal)) + _, _, err = HandleUpstreamPreForward(ctx, n, u, bad(), false) + assert.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal)) + out, err = HandleUpstreamPostForward(ctx, n, u, bad(), rs, upErr, false) + assert.Same(t, rs, out) + assert.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal)) +} + +func TestCovMisc_Hooks_TraceFilterDispatch(t *testing.T) { + ctx := context.Background() + inverted := func(method string) *common.NormalizedRequest { + return common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest(method, []interface{}{ + map[string]interface{}{"fromBlock": "0x10", "toBlock": "0x5"}, + })) + } + n := &queryTestNetwork{ + cfg: &common.NetworkConfig{Evm: &common.EvmNetworkConfig{Integrity: &common.EvmIntegrityConfig{EnforceGetLogsBlockRange: util.BoolPtr(true)}}}, + latest: 100, + forwardFn: func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error) { + t.Fatal("must not forward") + return nil, nil + }, + } + for _, method := range []string{"trace_filter", "arbtrace_filter"} { + t.Run(method, func(t *testing.T) { + handled, resp, err := HandleProjectPreForward(ctx, n, common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest(method, []interface{}{ + map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x5"}, + }))) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + + handled, _, err = HandleNetworkPreForward(ctx, n, nil, inverted(method)) + assert.True(t, handled) + assert.True(t, common.HasErrorCode(err, common.ErrCodeInvalidRequest)) + assert.Contains(t, err.Error(), "fromBlock (16) must be less than or equal to toBlock (5)") + + handled, _, err = HandleUpstreamPreForward(ctx, n, common.NewFakeUpstream("rpc1"), inverted(method), false) + assert.True(t, handled) + assert.True(t, common.HasErrorCode(err, common.ErrCodeInvalidRequest)) + + upErr := errors.New("upstream timeout") + rs := common.NewNormalizedResponse() + out, err := HandleNetworkPostForward(ctx, n, inverted(method), rs, upErr) + assert.Same(t, rs, out) + assert.Same(t, upErr, err, "without splitOnError the upstream error is kept") + }) + } +} + +// --------------------------------------------------------------------------- +// integrity record path (hooks.go) + export +// --------------------------------------------------------------------------- + +// covMiscIntegrityNetwork carries an integrity config under a network id of +// its own (unique per call, so -count=N reruns also get a fresh slot in the +// process-wide exporter cache). +type covMiscIntegrityNetwork struct { + common.Network + id string + cfg *common.NetworkConfig +} + +func (n *covMiscIntegrityNetwork) Config() *common.NetworkConfig { return n.cfg } +func (n *covMiscIntegrityNetwork) Id() string { return n.id } +func (n *covMiscIntegrityNetwork) ProjectId() string { return "test-project" } +func (n *covMiscIntegrityNetwork) Label() string { return n.id } + +var covMiscNetSeq atomic.Int64 + +func covMiscIndexNetwork(id, dir string, settings func(*common.IntegritySettings)) *covMiscIntegrityNetwork { + id = fmt.Sprintf("%s-%d", id, covMiscNetSeq.Add(1)) + s := common.IntegritySettings{ + Checks: map[string]*common.IntegrityCheckConfig{"indexMagnitude": {Enabled: util.BoolPtr(true)}}, + } + if settings != nil { + settings(&s) + } + ic := &common.IntegrityConfig{IntegritySettings: s} + if dir != "" { + ic.MisbehaviorsDestination = &common.MisbehaviorsDestinationConfig{Type: common.MisbehaviorsDestinationTypeFile, Path: dir} + } + return &covMiscIntegrityNetwork{id: id, cfg: &common.NetworkConfig{Integrity: ic}} +} + +func covMiscReceipt(t *testing.T, logs int) (*common.NormalizedRequest, *common.NormalizedResponse) { + idx := make([]string, logs) + for i := range idx { + idx[i] = "0xfffffff7" + } + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getTransactionReceipt","params":["0xabf61f02a6c77b28a9465a2256e26d2fe25714b60bb8edabb7d0ce794fba932e"]}`)) + jrr := common.MustNewJsonRpcResponseFromBytes([]byte("1"), receiptResultBytes(idx), nil) + return req, common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) +} + +func covMiscReadCatches(t *testing.T, dir string) []integrityCatchRecord { + files, err := filepath.Glob(filepath.Join(dir, "*")) + require.NoError(t, err) + var out []integrityCatchRecord + for _, f := range files { + content, err := os.ReadFile(f) + require.NoError(t, err) + for _, line := range strings.Split(strings.TrimSpace(string(content)), "\n") { + if line == "" { + continue + } + var rec integrityCatchRecord + require.NoError(t, json.Unmarshal([]byte(line), &rec)) + out = append(out, rec) + } + } + return out +} + +func TestCovMisc_Integrity_RecordedViolationsAreServedCountedAndArchived(t *testing.T) { + cases := []struct { + name string + settings func(*common.IntegritySettings) + verdict string + }{ + {"recordOnly without autoCorrect", func(s *common.IntegritySettings) { + s.Checks["indexMagnitude"].OnFailure = "recordOnly" + s.AutoCorrectWhenPossible = util.BoolPtr(false) + }, "record_only"}, + {"observeOnly suppresses a hard reject", func(s *common.IntegritySettings) { + s.ObserveOnly = util.BoolPtr(true) + }, "would_reject"}, + } + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + n := covMiscIndexNetwork(fmt.Sprintf("evm:covmisc-rec-%d", i), dir, tc.settings) + u := common.NewFakeUpstream("covmisc-rec-up") + labels := []string{"test-project", "", n.id, u.Id(), "eth_gettransactionreceipt", "indexMagnitude", tc.verdict, "unknown"} + before := testutil.ToFloat64(telemetry.MetricIntegrityViolation.WithLabelValues(labels...)) + + req, rs := covMiscReceipt(t, 2) + out, err := HandleUpstreamPostForward(context.Background(), n, u, req, rs, nil, false) + require.NoError(t, err, "a recorded violation is served") + assert.Same(t, rs, out) + assert.False(t, rs.IsIntegrityRejected()) + assert.False(t, req.IntegrityCaught()) + assert.Equal(t, before+1, testutil.ToFloat64(telemetry.MetricIntegrityViolation.WithLabelValues(labels...))) + + recs := covMiscReadCatches(t, dir) + require.Len(t, recs, 1) + assert.Equal(t, tc.verdict, recs[0].Verdict) + assert.Equal(t, "indexMagnitude", recs[0].Check) + assert.Equal(t, "deterministic", recs[0].Class) + assert.Equal(t, n.id, recs[0].Network) + assert.Equal(t, u.Id(), recs[0].Upstream) + assert.Contains(t, recs[0].Reason, "0xfffffff7") + assert.Contains(t, recs[0].Response, `"logIndex":"0xfffffff7"`) + }) + } +} + +func TestCovMisc_Integrity_DetailedTracingCapsRecordedBodies(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp), sdktrace.WithSampler(sdktrace.AlwaysSample())) + common.SetTracerProviderForTest(tp) + t.Cleanup(func() { common.IsTracingEnabled = false; common.IsTracingDetailed = false }) + common.IsTracingDetailed = true + + dir := t.TempDir() + n := covMiscIndexNetwork("evm:covmisc-trace", dir, func(s *common.IntegritySettings) { + s.Checks["indexMagnitude"].OnFailure = "recordOnly" + s.AutoCorrectWhenPossible = util.BoolPtr(false) + }) + req, rs := covMiscReceipt(t, 700) + jrr, err := rs.JsonRpcResponse() + require.NoError(t, err) + require.Greater(t, len(jrr.GetResultBytes()), maxTraceResponseBytes) + + _, err = HandleUpstreamPostForward(context.Background(), n, common.NewFakeUpstream("covmisc-trace-up"), req, rs, nil, false) + require.NoError(t, err) + + var s *tracetest.SpanStub + spans := exp.GetSpans() + for i := range spans { + if spans[i].Name == "Integrity.Validate" { + s = &spans[i] + } + } + require.NotNil(t, s) + attrs := map[string]interface{}{} + for _, a := range s.Attributes { + attrs[string(a.Key)] = a.Value.AsInterface() + } + assert.Equal(t, "record_only", attrs["integrity.outcome"]) + assert.Equal(t, true, attrs["integrity.response_truncated"]) + assert.Len(t, attrs["integrity.response"], maxTraceResponseBytes) + assert.Equal(t, "record_only", attrs["integrity.check.indexMagnitude"]) + require.Len(t, s.Events, 1) + assert.Equal(t, "integrity.record_only", s.Events[0].Name) + + recs := covMiscReadCatches(t, dir) + require.Len(t, recs, 1) + assert.Len(t, recs[0].Response, maxExportBody, "the archived body is capped") +} + +func TestCovMisc_ExportIntegrityCatch(t *testing.T) { + ctx := context.Background() + u := common.NewFakeUpstream("covmisc-exp-up") + + t.Run("no destination archives nothing", func(t *testing.T) { + n := covMiscIndexNetwork("evm:covmisc-exp-none", "", nil) + assert.Nil(t, networkIntegrityExporter(n)) + exportIntegrityCatch(ctx, n, u, nil, "eth_getlogs", "reject", "c", "deterministic", "unknown", "r") + }) + + t.Run("records without a response and survives a lost directory", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "catches") + n := covMiscIndexNetwork("evm:covmisc-exp-file", dir, nil) + first := networkIntegrityExporter(n) + require.NotNil(t, first) + assert.Same(t, first, networkIntegrityExporter(n), "one exporter per network") + + exportIntegrityCatch(ctx, n, u, nil, "eth_getLogs", "reject", "getLogsFilterSanity", "deterministic", "finalized", "log outside requested range") + recs := covMiscReadCatches(t, dir) + require.Len(t, recs, 1) + assert.Equal(t, "getLogsFilterSanity", recs[0].Check) + assert.Equal(t, "finalized", recs[0].Finality) + assert.Empty(t, recs[0].Response) + + // Close flushes every exporter; a later catch reopens its file. + CloseIntegrityExporters() + require.NoError(t, os.RemoveAll(dir)) + exportIntegrityCatch(ctx, n, u, nil, "eth_getLogs", "reject", "c2", "deterministic", "unknown", "r2") + _, statErr := os.Stat(dir) + assert.True(t, os.IsNotExist(statErr), "a failed append is logged, not retried into a new directory") + + require.NoError(t, os.MkdirAll(dir, 0o750)) + exportIntegrityCatch(ctx, n, u, nil, "eth_getLogs", "reject", "c3", "deterministic", "unknown", "r3") + recs = covMiscReadCatches(t, dir) + require.Len(t, recs, 1) + assert.Equal(t, "c3", recs[0].Check) + CloseIntegrityExporters() + }) +} + +// --------------------------------------------------------------------------- +// pin_latest +// --------------------------------------------------------------------------- + +func TestCovMisc_LatestTagParamIndexAndPinParams(t *testing.T) { + for method, want := range map[string]int{"eth_call": 1, "eth_getStorageAt": 2} { + idx, ok := LatestTagParamIndex(method) + assert.True(t, ok) + assert.Equal(t, want, idx) + } + _, ok := LatestTagParamIndex("eth_getBlockByNumber") + assert.False(t, ok, "block-by-number is never pinned") + + cases := []struct { + name string + params []interface{} + idx int + want []interface{} + pinned bool + }{ + {"absent selector means latest", []interface{}{"0xabc"}, 1, []interface{}{"0xabc", "0x64"}, true}, + {"explicit latest", []interface{}{"0xabc", "latest"}, 1, []interface{}{"0xabc", "0x64"}, true}, + {"params too short", []interface{}{"0xabc"}, 2, []interface{}{"0xabc"}, false}, + {"concrete block", []interface{}{"0xabc", "0x10"}, 1, []interface{}{"0xabc", "0x10"}, false}, + {"object selector", []interface{}{"0xabc", map[string]interface{}{"blockNumber": "0x1"}}, 1, nil, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, pinned := PinLatestParams(tc.params, tc.idx, 100) + assert.Equal(t, tc.pinned, pinned) + if tc.want != nil { + assert.Equal(t, tc.want, got) + } + if pinned { + assert.NotSame(t, &tc.params[0], &got[0], "the input slice is never mutated") + assert.NotEqual(t, "0x64", tc.params[min(tc.idx, len(tc.params)-1)]) + } + }) + } +} + +func TestCovMisc_PinBlockTagInBody(t *testing.T) { + body := []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xabc","latest"]}`) + cases := []struct { + name string + body []byte + idx int + height int64 + pinned bool + }{ + {"empty body", nil, 1, 100, false}, + {"unknown height", body, 1, 0, false}, + {"not json", []byte(`{not json`), 1, 100, false}, + {"selector index beyond params", body, 2, 100, false}, + {"selector is not latest", []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xabc","0x5"]}`), 1, 100, false}, + {"latest is pinned", body, 1, 100, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out, pinned := PinBlockTagInBody(tc.body, tc.idx, tc.height) + assert.Equal(t, tc.pinned, pinned) + if !pinned { + assert.Equal(t, tc.body, out) + return + } + var req map[string]interface{} + require.NoError(t, json.Unmarshal(out, &req)) + assert.Equal(t, []interface{}{"0xabc", "0x64"}, req["params"]) + assert.Equal(t, "eth_getBalance", req["method"]) + }) + } +} + +func TestCovMisc_PinLatestTag(t *testing.T) { + ctx := context.Background() + pinUp := &queryTestConfigUpstream{cfg: &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{PinLatestTag: util.BoolPtr(true)}}} + balance := func(block string) *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xabc","` + block + `"]}`)) + } + cases := []struct { + name string + n common.Network + u common.Upstream + r *common.NormalizedRequest + wantParams []interface{} + wantHeight int64 + wantPinned bool + }{ + {"upstream does not pin", &queryTestNetwork{latest: 100}, &queryTestConfigUpstream{cfg: &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{PinLatestTag: util.BoolPtr(false)}}}, balance("latest"), nil, 0, false}, + {"unparseable request", &queryTestNetwork{latest: 100}, pinUp, common.NewNormalizedRequest([]byte(`{not json`)), nil, 0, false}, + {"concrete block is left alone", &queryTestNetwork{latest: 100}, pinUp, balance("0x5"), nil, 0, false}, + {"head unknown: must not send latest", &queryTestNetwork{}, pinUp, balance("latest"), nil, 0, true}, + {"latest pinned to head", &queryTestNetwork{latest: 100}, pinUp, balance("latest"), []interface{}{"0xabc", "0x64"}, 100, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + params, height, pinned := PinLatestTag(ctx, tc.n, tc.u, tc.r) + assert.Equal(t, tc.wantPinned, pinned) + assert.Equal(t, tc.wantHeight, height) + assert.Equal(t, tc.wantParams, params) + }) + } +} + +// --------------------------------------------------------------------------- +// json_rpc params helpers +// --------------------------------------------------------------------------- + +type covMiscOpaque struct{ v int } + +func TestCovMisc_DeepCopyParams(t *testing.T) { + assert.Nil(t, deepCopyParams(nil)) + + op := &covMiscOpaque{v: 1} + orig := []interface{}{ + map[string]interface{}{"address": []interface{}{"0xa", nil}, "n": 1.5}, + "latest", + op, + } + cp := deepCopyParams(orig) + require.Equal(t, orig, cp) + + cp[0].(map[string]interface{})["address"].([]interface{})[0] = "0xchanged" + cp[0].(map[string]interface{})["n"] = 2.0 + assert.Equal(t, "0xa", orig[0].(map[string]interface{})["address"].([]interface{})[0], "nested slices are copied") + assert.Equal(t, 1.5, orig[0].(map[string]interface{})["n"], "nested maps are copied") + assert.Same(t, op, cp[2], "unknown types are shared as-is") +} + +func TestCovMisc_ReplaceParamAtPath(t *testing.T) { + params := []interface{}{ + map[string]interface{}{"fromBlock": "latest", "nested": map[string]interface{}{"tag": "latest"}}, + "latest", + } + cases := []struct { + name string + path []interface{} + changed bool + check func(t *testing.T, out []interface{}) + }{ + {"empty path", nil, false, nil}, + {"index out of range", []interface{}{5}, false, nil}, + {"negative index", []interface{}{-1}, false, nil}, + {"string key on non-object", []interface{}{1, "fromBlock"}, false, nil}, + {"missing key is not created", []interface{}{0, "toBlock"}, false, nil}, + {"missing nested key", []interface{}{0, "nested", "absent"}, false, nil}, + {"unsupported path element", []interface{}{0, 1.5}, false, nil}, + {"string key at root", []interface{}{"fromBlock"}, false, nil}, + {"top-level index", []interface{}{1}, true, func(t *testing.T, out []interface{}) { + assert.Equal(t, "0x64", out[1]) + }}, + {"nested object key", []interface{}{0, "nested", "tag"}, true, func(t *testing.T, out []interface{}) { + assert.Equal(t, "0x64", out[0].(map[string]interface{})["nested"].(map[string]interface{})["tag"]) + assert.Equal(t, "latest", out[0].(map[string]interface{})["fromBlock"]) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out, changed := replaceParamAtPath(params, tc.path, "0x64") + assert.Equal(t, tc.changed, changed) + if !changed { + assert.Equal(t, params, out) + return + } + tc.check(t, out) + // The input is never mutated. + assert.Equal(t, "latest", params[1]) + assert.Equal(t, "latest", params[0].(map[string]interface{})["nested"].(map[string]interface{})["tag"]) + }) + } +} diff --git a/architecture/evm/cov_poller_test.go b/architecture/evm/cov_poller_test.go new file mode 100644 index 000000000..802a32aac --- /dev/null +++ b/architecture/evm/cov_poller_test.go @@ -0,0 +1,1257 @@ +package evm + +import ( + "context" + "errors" + "fmt" + "math" + "strings" + "sync" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/health" + "github.com/erpc/erpc/telemetry" + promUtil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// covPollerHandler scripts one upstream answer per JSON-RPC call. +type covPollerHandler func(method string, params []interface{}) (*common.NormalizedResponse, error) + +// covPollerUpstream is a scriptable common.EvmUpstream: Forward delegates to a +// swappable handler and counts calls per method; EvmGetChainId and Cordon are +// observable so the chain-identity gate can be asserted on the poll paths. +type covPollerUpstream struct { + cfg *common.UpstreamConfig + logger zerolog.Logger + + mu sync.Mutex + handler covPollerHandler + calls map[string]int + chainId string + chainErr error + chainIdCalls int + cordoned bool + cordonReason string +} + +func (u *covPollerUpstream) setHandler(h covPollerHandler) { + u.mu.Lock() + defer u.mu.Unlock() + u.handler = h +} + +func (u *covPollerUpstream) callCount(method string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.calls[method] +} + +func (u *covPollerUpstream) isCordoned() bool { + u.mu.Lock() + defer u.mu.Unlock() + return u.cordoned +} + +func (u *covPollerUpstream) getChainIdCalls() int { + u.mu.Lock() + defer u.mu.Unlock() + return u.chainIdCalls +} + +func (u *covPollerUpstream) Id() string { return u.cfg.Id } +func (u *covPollerUpstream) VendorName() string { return "covpoller" } +func (u *covPollerUpstream) NetworkId() string { return "evm:123" } +func (u *covPollerUpstream) NetworkLabel() string { return "evm:123" } +func (u *covPollerUpstream) Config() *common.UpstreamConfig { return u.cfg } +func (u *covPollerUpstream) Logger() *zerolog.Logger { return &u.logger } +func (u *covPollerUpstream) Vendor() common.Vendor { return nil } +func (u *covPollerUpstream) Tracker() common.HealthTracker { return nil } +func (u *covPollerUpstream) IgnoreMethod(string) {} +func (u *covPollerUpstream) Uncordon(_, _ string) {} +func (u *covPollerUpstream) ShouldHandleMethod(string) (bool, error) { + return true, nil +} +func (u *covPollerUpstream) Forward(_ context.Context, nq *common.NormalizedRequest, _, _ bool) (*common.NormalizedResponse, error) { + jrq, err := nq.JsonRpcRequest() + if err != nil { + return nil, err + } + u.mu.Lock() + u.calls[jrq.Method]++ + h := u.handler + u.mu.Unlock() + return h(jrq.Method, jrq.Params) +} +func (u *covPollerUpstream) Cordon(_, reason string) { + u.mu.Lock() + defer u.mu.Unlock() + u.cordoned = true + u.cordonReason = reason +} +func (u *covPollerUpstream) EvmGetChainId(context.Context) (string, error) { + u.mu.Lock() + defer u.mu.Unlock() + u.chainIdCalls++ + return u.chainId, u.chainErr +} +func (u *covPollerUpstream) EvmIsBlockFinalized(context.Context, int64, bool) (bool, error) { + return false, nil +} +func (u *covPollerUpstream) EvmAssertBlockAvailability(context.Context, string, common.AvailbilityConfidence, bool, int64) (bool, error) { + return true, nil +} +func (u *covPollerUpstream) EvmSyncingState() common.EvmSyncingState { + return common.EvmSyncingStateUnknown +} +func (u *covPollerUpstream) EvmStatePoller() common.EvmStatePoller { return nil } +func (u *covPollerUpstream) EvmEffectiveLatestBlock() int64 { return 0 } +func (u *covPollerUpstream) EvmEffectiveFinalizedBlock() int64 { return 0 } +func (u *covPollerUpstream) EvmBlockAvailabilityBounds() (int64, int64) { + return math.MinInt64, math.MaxInt64 +} + +var _ common.EvmUpstream = (*covPollerUpstream)(nil) + +// covPollerNew builds a poller over a covPollerUpstream (chainId 123, eth_chainId +// answering "123") backed by an in-memory shared state registry. UpdateMaxWait +// is generous so the foreground TryUpdateIfStale always applies the scripted +// answer synchronously, even under -race. +func covPollerNew(t *testing.T, id string, h covPollerHandler) (*EvmStatePoller, *covPollerUpstream, context.Context) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + logger := zerolog.Nop() + up := &covPollerUpstream{ + cfg: &common.UpstreamConfig{ + Id: id, + Type: common.UpstreamTypeEvm, + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + logger: logger, + handler: h, + calls: map[string]int{}, + chainId: "123", + } + ssr, err := data.NewSharedStateRegistry(ctx, &logger, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: common.DriverMemory, + Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}, + }, + UpdateMaxWait: common.Duration(10 * time.Second), + }) + require.NoError(t, err) + tracker := health.NewTracker(&logger, "test", 2*time.Second) + return NewEvmStatePoller("test", ctx, &logger, up, tracker, ssr), up, ctx +} + +// covPollerFastDebounce makes every poll past ~1ms eligible to fetch again. +func covPollerFastDebounce(p *EvmStatePoller) { + p.stateMu.Lock() + p.debounceInterval = time.Millisecond + p.stateMu.Unlock() +} + +func covPollerResult(result string) *common.NormalizedResponse { + return common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(result), nil)) +} + +func covPollerRpcError(code int, msg string) *common.NormalizedResponse { + return common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), nil, + []byte(fmt.Sprintf(`{"code":%d,"message":%q}`, code, msg)))) +} + +func covPollerBlock(num int64) *common.NormalizedResponse { + return covPollerResult(fmt.Sprintf(`{"number":"0x%x","hash":"0x%064x","timestamp":"0x%x"}`, num, num+1, 1_700_000_000+num)) +} + +// covPollerBlockTag returns the first param of eth_getBlockByNumber. +func covPollerBlockTag(params []interface{}) string { + if len(params) == 0 { + return "" + } + s, _ := params[0].(string) + return s +} + +// covPollerBlocks answers latest/finalized with fixed heights and eth_syncing false. +func covPollerBlocks(latest, finalized int64) covPollerHandler { + return func(method string, params []interface{}) (*common.NormalizedResponse, error) { + switch method { + case "eth_getBlockByNumber": + switch covPollerBlockTag(params) { + case "latest": + return covPollerBlock(latest), nil + case "finalized": + return covPollerBlock(finalized), nil + } + case "eth_syncing": + return covPollerResult(`false`), nil + } + return nil, common.NewErrEndpointUnsupported(errors.New("unexpected " + method)) + } +} + +// covPollerHistory answers latest with `latest` and a numbered block with full +// data only from `earliest` upward (null below), like a pruned node. +func covPollerHistory(latest, earliest int64) covPollerHandler { + return func(method string, params []interface{}) (*common.NormalizedResponse, error) { + if method != "eth_getBlockByNumber" { + return nil, common.NewErrEndpointUnsupported(errors.New("unexpected " + method)) + } + tag := covPollerBlockTag(params) + if tag == "latest" { + return covPollerBlock(latest), nil + } + n, err := common.HexToInt64(tag) + if err != nil { + return nil, err + } + if n < earliest { + return covPollerResult(`null`), nil + } + return covPollerBlock(n), nil + } +} + +func covPollerUnsupportedErrors() map[string]error { + cause := errors.New("the method eth_getBlockByNumber does not exist/is not available") + return map[string]error{ + "EndpointUnsupported": common.NewErrEndpointUnsupported(cause), + "UpstreamSkipped": common.NewErrUpstreamRequestSkipped(cause, "u"), + "MethodIgnored": common.NewErrUpstreamMethodIgnored("eth_getBlockByNumber", "u"), + "EndpointMissingData": common.NewErrEndpointMissingData(cause, nil), + "ClientSideException": common.NewErrEndpointClientSideException(cause), + "EmptyResult(no err)": nil, + } +} + +// --------------------------------------------------------------------------- +// Latest block polling +// --------------------------------------------------------------------------- + +func TestCovPoller_PollLatest_GivesUpAfterTenUnsupportedAnswers(t *testing.T) { + for name, uerr := range covPollerUnsupportedErrors() { + t.Run(name, func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-latest-unsup", func(method string, params []interface{}) (*common.NormalizedResponse, error) { + if uerr == nil { + return covPollerResult(`null`), nil + } + return nil, uerr + }) + covPollerFastDebounce(p) + + v, err := p.PollLatestBlockNumber(ctx) + require.NoError(t, err, "unsupported answers are not surfaced as poll errors") + assert.Zero(t, v) + d := p.GetDiagnostics() + assert.Equal(t, 1, d.LatestBlockFailureCount) + assert.False(t, d.SkipLatestBlockCheck, "one failure must not give up yet") + + require.Eventually(t, func() bool { + _, _ = p.PollLatestBlockNumber(ctx) + return p.GetDiagnostics().SkipLatestBlockCheck + }, 5*time.Second, 2*time.Millisecond) + + d = p.GetDiagnostics() + assert.Equal(t, 10, d.LatestBlockFailureCount) + assert.False(t, d.LatestBlockSuccessfulOnce) + assert.Contains(t, d.LatestBlockDetectionIssue, "latest block check disabled") + + calls := up.callCount("eth_getBlockByNumber") + v, err = p.PollLatestBlockNumber(ctx) + require.NoError(t, err) + assert.Zero(t, v) + assert.Equal(t, calls, up.callCount("eth_getBlockByNumber"), "after giving up the upstream is not asked again") + }) + } +} + +func TestCovPoller_PollLatest_UnsupportedAfterSuccessNeverGivesUp(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-latest-after-success", covPollerBlocks(100, 90)) + covPollerFastDebounce(p) + + v, err := p.PollLatestBlockNumber(ctx) + require.NoError(t, err) + require.Equal(t, int64(100), v) + + up.setHandler(func(string, []interface{}) (*common.NormalizedResponse, error) { + return nil, common.NewErrEndpointUnsupported(errors.New("gone")) + }) + require.Eventually(t, func() bool { + _, _ = p.PollLatestBlockNumber(ctx) + return up.callCount("eth_getBlockByNumber") >= 4 + }, 5*time.Second, 2*time.Millisecond) + + d := p.GetDiagnostics() + assert.True(t, d.LatestBlockSuccessfulOnce) + assert.Zero(t, d.LatestBlockFailureCount, "failures after a success do not count toward giving up") + assert.False(t, d.SkipLatestBlockCheck) + assert.Empty(t, d.LatestBlockDetectionIssue) + assert.Equal(t, int64(100), p.LatestBlock(), "a failed poll keeps the last known head") +} + +func TestCovPoller_PollLatest_TransientErrorsAreReturnedNotCounted(t *testing.T) { + serverErr := common.NewErrEndpointServerSideException(errors.New("upstream 502"), nil, 502) + p, _, ctx := covPollerNew(t, "cov-latest-transient", func(string, []interface{}) (*common.NormalizedResponse, error) { + return nil, serverErr + }) + + _, err := p.PollLatestBlockNumber(ctx) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointServerSideException)) + d := p.GetDiagnostics() + assert.Zero(t, d.LatestBlockFailureCount) + assert.False(t, d.SkipLatestBlockCheck) + // A server-side failure is already recorded by tryForward; the poller adds nothing. + m := p.tracker.GetUpstreamMethodMetrics(p.upstream, "eth_getBlockByNumber", common.DataFinalityStateAll) + assert.Zero(t, m.ErrorsTotal.Load()) +} + +func TestCovPoller_PollLatest_OpenCircuitBreakerIsRecordedOnTracker(t *testing.T) { + cbErr := common.NewErrFailsafeCircuitBreakerOpen(common.ScopeUpstream, nil, nil) + p, _, ctx := covPollerNew(t, "cov-latest-cb", func(string, []interface{}) (*common.NormalizedResponse, error) { + return nil, cbErr + }) + + _, err := p.PollLatestBlockNumber(ctx) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeFailsafeCircuitBreakerOpen)) + + m := p.tracker.GetUpstreamMethodMetrics(p.upstream, "eth_getBlockByNumber", common.DataFinalityStateAll) + assert.Equal(t, int64(1), m.RequestsTotal.Load()) + assert.Equal(t, int64(1), m.ErrorsTotal.Load(), "short-circuited polls must still move the error rate") +} + +func TestCovPoller_PollLatest_MalformedBlockAnswersAreErrors(t *testing.T) { + cases := []struct { + name string + resp *common.NormalizedResponse + expect string + }{ + {"json-rpc error", covPollerRpcError(-32000, "header not found"), "header not found"}, + {"block without number", covPollerResult(`{"hash":"0xabc"}`), "cannot get block number from block data"}, + {"unparseable number", covPollerResult(`{"number":"0xzz"}`), ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p, _, ctx := covPollerNew(t, "cov-latest-malformed", func(string, []interface{}) (*common.NormalizedResponse, error) { + return tc.resp, nil + }) + v, err := p.PollLatestBlockNumber(ctx) + require.Error(t, err) + if tc.expect != "" { + assert.Contains(t, err.Error(), tc.expect) + } + assert.Zero(t, v) + assert.Zero(t, p.LatestBlock()) + assert.Zero(t, p.GetDiagnostics().LatestBlockFailureCount, "malformed answers are transient, not 'unsupported'") + }) + } +} + +func TestCovPoller_PollLatest_IsMonotonicAndIgnoresSmallRollbacks(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-latest-monotonic", covPollerBlocks(100, 90)) + covPollerFastDebounce(p) + + v, err := p.PollLatestBlockNumber(ctx) + require.NoError(t, err) + require.Equal(t, int64(100), v) + + up.setHandler(covPollerBlocks(95, 90)) // lagging backend, within rollback tolerance + calls := up.callCount("eth_getBlockByNumber") + require.Eventually(t, func() bool { + _, _ = p.PollLatestBlockNumber(ctx) + return up.callCount("eth_getBlockByNumber") > calls + }, 5*time.Second, 2*time.Millisecond) + assert.Equal(t, int64(100), p.LatestBlock(), "small rollback must be ignored") + + p.SuggestLatestBlock(50) + assert.Equal(t, int64(100), p.LatestBlock(), "lower suggestion must be ignored") + + up.setHandler(covPollerBlocks(120, 90)) + require.Eventually(t, func() bool { + _, _ = p.PollLatestBlockNumber(ctx) + return p.LatestBlock() == 120 + }, 5*time.Second, 2*time.Millisecond) + assert.True(t, p.GetDiagnostics().LatestBlockSuccessfulOnce) +} + +func TestCovPoller_PollLatest_MajorMoveChainIdGate(t *testing.T) { + t.Run("forward jump from another chain is rejected and cordons", func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-latest-gate-fwd", covPollerBlocks(5_000_000, 90)) + up.chainId = "999" + p.latestBlockShared.TryUpdate(ctx, 1000) + + _, err := p.PollLatestBlockNumber(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1000), p.LatestBlock()) + assert.True(t, up.isCordoned()) + assert.Contains(t, up.cordonReason, "chain identity mismatch on major latest head move") + }) + + t.Run("large rollback from another chain is rejected and cordons", func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-latest-gate-back", covPollerBlocks(1000, 90)) + up.chainId = "999" + p.latestBlockShared.TryUpdate(ctx, 5_000_000) + + _, err := p.PollLatestBlockNumber(ctx) + require.NoError(t, err) + assert.Equal(t, int64(5_000_000), p.LatestBlock()) + assert.True(t, up.isCordoned(), "a major backward move is gated just like a forward one") + }) + + t.Run("verified major jump is accepted", func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-latest-gate-ok", covPollerBlocks(5_000_000, 90)) + p.latestBlockShared.TryUpdate(ctx, 1000) + + v, err := p.PollLatestBlockNumber(ctx) + require.NoError(t, err) + assert.Equal(t, int64(5_000_000), v) + assert.Equal(t, 1, up.getChainIdCalls()) + assert.False(t, up.isCordoned()) + }) + + t.Run("upstream without EvmGetChainId cannot be verified and passes", func(t *testing.T) { + lg := zerolog.Nop() + cu := &canaryUpstream{ + logger: lg, + cfg: &common.UpstreamConfig{ + Id: "cov-latest-gate-plain", Type: common.UpstreamTypeEvm, + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + handler: func(string, string) (*common.NormalizedResponse, error) { return nil, nil }, + } + base, _, _ := covPollerNew(t, "cov-latest-gate-plain-base", covPollerBlocks(1, 1)) + p := NewEvmStatePoller("test", context.Background(), &lg, cu, base.tracker, base.sharedStateRegistry) + assert.True(t, p.verifyChainIdOnMajorHeadMove(context.Background(), "latest", 1000, 5_000_000)) + }) +} + +// --------------------------------------------------------------------------- +// Finalized block polling +// --------------------------------------------------------------------------- + +func TestCovPoller_PollFinalized_GivesUpAfterTenUnsupportedAnswers(t *testing.T) { + for _, name := range []string{"EmptyResult(no err)", "ClientSideException", "EndpointUnsupported"} { + uerr := covPollerUnsupportedErrors()[name] + t.Run(name, func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-fin-unsup", func(method string, params []interface{}) (*common.NormalizedResponse, error) { + if uerr == nil { + return covPollerResult(`null`), nil + } + return nil, uerr + }) + covPollerFastDebounce(p) + + v, err := p.PollFinalizedBlockNumber(ctx) + require.NoError(t, err) + assert.Zero(t, v) + assert.Equal(t, 1, p.GetDiagnostics().FinalizedBlockFailureCount) + + require.Eventually(t, func() bool { + _, _ = p.PollFinalizedBlockNumber(ctx) + return p.GetDiagnostics().SkipFinalizedCheck + }, 5*time.Second, 2*time.Millisecond) + + d := p.GetDiagnostics() + assert.Equal(t, 10, d.FinalizedBlockFailureCount) + assert.Contains(t, d.FinalizedBlockDetectionIssue, "finalized block check disabled") + + calls := up.callCount("eth_getBlockByNumber") + v, err = p.PollFinalizedBlockNumber(ctx) + require.NoError(t, err) + assert.Zero(t, v) + assert.Equal(t, calls, up.callCount("eth_getBlockByNumber")) + }) + } +} + +func TestCovPoller_PollFinalized_UnsupportedAfterSuccessNeverGivesUp(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-fin-after-success", covPollerBlocks(100, 90)) + covPollerFastDebounce(p) + + v, err := p.PollFinalizedBlockNumber(ctx) + require.NoError(t, err) + require.Equal(t, int64(90), v) + + up.setHandler(func(string, []interface{}) (*common.NormalizedResponse, error) { + return covPollerResult(`null`), nil + }) + require.Eventually(t, func() bool { + _, _ = p.PollFinalizedBlockNumber(ctx) + return up.callCount("eth_getBlockByNumber") >= 4 + }, 5*time.Second, 2*time.Millisecond) + + d := p.GetDiagnostics() + assert.True(t, d.FinalizedBlockSuccessfulOnce) + assert.Zero(t, d.FinalizedBlockFailureCount) + assert.False(t, d.SkipFinalizedCheck) + assert.Equal(t, int64(90), p.FinalizedBlock()) +} + +func TestCovPoller_PollFinalized_OpenCircuitBreakerIsRecordedOnTracker(t *testing.T) { + cbErr := common.NewErrFailsafeCircuitBreakerOpen(common.ScopeUpstream, nil, nil) + p, _, ctx := covPollerNew(t, "cov-fin-cb", func(string, []interface{}) (*common.NormalizedResponse, error) { + return nil, cbErr + }) + + _, err := p.PollFinalizedBlockNumber(ctx) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeFailsafeCircuitBreakerOpen)) + m := p.tracker.GetUpstreamMethodMetrics(p.upstream, "eth_getBlockByNumber", common.DataFinalityStateAll) + assert.Equal(t, int64(1), m.RequestsTotal.Load()) + assert.Equal(t, int64(1), m.ErrorsTotal.Load()) + assert.Zero(t, p.GetDiagnostics().FinalizedBlockFailureCount) +} + +func TestCovPoller_PollFinalized_MajorJumpFromAnotherChainIsRejected(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-fin-gate", covPollerBlocks(5_000_100, 5_000_000)) + up.chainId = "999" + covPollerFastDebounce(p) + p.finalizedBlockShared.TryUpdate(ctx, 1000) // marks the shared value fresh + + require.Eventually(t, func() bool { + v, err := p.PollFinalizedBlockNumber(ctx) + require.NoError(t, err) + require.Equal(t, int64(1000), v) + return up.isCordoned() + }, 5*time.Second, 2*time.Millisecond) + assert.Equal(t, int64(1000), p.FinalizedBlock()) + assert.Contains(t, up.cordonReason, "major finalized head move") +} + +// --------------------------------------------------------------------------- +// Debounce resolution +// --------------------------------------------------------------------------- + +func TestCovPoller_ResolveDebounce_DynamicMultiplierScalesBlockTime(t *testing.T) { + p, up, _ := covPollerNew(t, "cov-debounce-mult", covPollerBlocks(1, 1)) + feedBlockTime(t, p, up, 1, 12) // 12s blocks + + half := 0.5 + bt := p.tracker.GetNetworkBlockTime(up.NetworkId()) + assert.Equal(t, time.Duration(float64(bt)*half), p.resolveDebounce(&common.EvmNetworkConfig{DynamicBlockTimeDebounceMultiplier: &half})) + + zero := 0.0 + assert.Equal(t, time.Duration(float64(bt)*common.DefaultDynamicBlockTimeDebounceMultiplier), + p.resolveDebounce(&common.EvmNetworkConfig{DynamicBlockTimeDebounceMultiplier: &zero}), + "a non-positive multiplier falls back to the default") +} + +func TestCovPoller_ResolveFinalizedDebounce(t *testing.T) { + const base = 2 * time.Second + half := 0.5 + zero := 0.0 + + newPoller := func(t *testing.T) (*EvmStatePoller, *covPollerUpstream, context.Context) { + p, up, ctx := covPollerNew(t, "cov-fin-debounce", covPollerBlocks(1, 1)) + feedBlockTime(t, p, up, 1, 12) + p.finalizedBlockShared.TryUpdate(ctx, 1_000_000) + p.latestBlockShared.TryUpdate(ctx, 1_000_100) + return p, up, ctx + } + + t.Run("widened to a fraction of the finality window", func(t *testing.T) { + p, _, _ := newPoller(t) + got := p.resolveFinalizedDebounce(&common.EvmNetworkConfig{FinalizedDebounceDepthMultiplier: &half}, base) + bt := p.tracker.GetNetworkBlockTime(p.upstream.NetworkId()) + require.Greater(t, time.Duration(float64(100)*float64(bt)*half), base) + assert.Equal(t, time.Duration(float64(100)*float64(bt)*half), got) + }) + + t.Run("never narrower than the regular debounce", func(t *testing.T) { + p, _, _ := newPoller(t) + got := p.resolveFinalizedDebounce(&common.EvmNetworkConfig{FinalizedDebounceDepthMultiplier: &half}, time.Hour) + assert.Equal(t, time.Hour, got) + }) + + t.Run("non-positive multiplier keeps the regular debounce", func(t *testing.T) { + p, _, _ := newPoller(t) + assert.Equal(t, base, p.resolveFinalizedDebounce(&common.EvmNetworkConfig{FinalizedDebounceDepthMultiplier: &zero}, base)) + }) + + t.Run("unset multiplier keeps the regular debounce", func(t *testing.T) { + p, _, _ := newPoller(t) + assert.Equal(t, base, p.resolveFinalizedDebounce(&common.EvmNetworkConfig{}, base)) + assert.Equal(t, base, p.resolveFinalizedDebounce(nil, base)) + }) + + t.Run("upstream behind the network finalized keeps the regular debounce", func(t *testing.T) { + p, up, _ := newPoller(t) + // The tracker learns of a higher finalized block than this poller holds. + p.tracker.SetFinalizedBlockNumber(up, 1_000_050) + require.Equal(t, int64(1_000_050), p.tracker.GetNetworkFinalizedBlock(up.NetworkId())) + assert.Equal(t, base, p.resolveFinalizedDebounce(&common.EvmNetworkConfig{FinalizedDebounceDepthMultiplier: &half}, base)) + }) +} + +// --------------------------------------------------------------------------- +// Suggest / refresh / callbacks +// --------------------------------------------------------------------------- + +func TestCovPoller_SuggestLatestBlock_MajorJumpWhileVerificationInFlightIsDropped(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-suggest-inflight", covPollerBlocks(1, 1)) + p.latestBlockShared.TryUpdate(ctx, 1000) + + p.latestMajorVerifyInProgress.Lock() + p.SuggestLatestBlock(5_000_000) + // The busy path returns synchronously without spawning any verification. + assert.Equal(t, int64(1000), p.LatestBlock()) + assert.Zero(t, up.getChainIdCalls()) + p.latestMajorVerifyInProgress.Unlock() + + p.SuggestLatestBlock(5_000_000) + require.Eventually(t, func() bool { return p.LatestBlock() == 5_000_000 }, 5*time.Second, 5*time.Millisecond) + assert.Equal(t, 1, up.getChainIdCalls()) +} + +func TestCovPoller_VerifyThenSuggest_StaleSuggestionIsDiscarded(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-suggest-stale", covPollerBlocks(1, 1)) + p.latestBlockShared.TryUpdate(ctx, 1000) + + // Queued major suggestion that another path already overtook. + p.verifyThenSuggestLatestBlock(500) + require.Eventually(t, func() bool { + if p.latestMajorVerifyInProgress.TryLock() { + p.latestMajorVerifyInProgress.Unlock() + return true + } + return false + }, 5*time.Second, 5*time.Millisecond) + assert.Equal(t, int64(1000), p.LatestBlock()) + assert.Zero(t, up.getChainIdCalls(), "an overtaken suggestion is not verified") +} + +func TestCovPoller_SuggestFinalizedBlock_SkippedWhileUpdateInProgress(t *testing.T) { + p, _, _ := covPollerNew(t, "cov-suggest-fin-busy", covPollerBlocks(1, 1)) + + p.finalizedUpdateInProgress.Lock() + p.SuggestFinalizedBlock(500) + assert.Zero(t, p.FinalizedBlock(), "a concurrent suggestion is dropped synchronously") + p.finalizedUpdateInProgress.Unlock() + + p.SuggestFinalizedBlock(500) + require.Eventually(t, func() bool { return p.FinalizedBlock() == 500 }, 5*time.Second, 5*time.Millisecond) + + p.SuggestFinalizedBlock(400) + require.Eventually(t, func() bool { + if p.finalizedUpdateInProgress.TryLock() { + p.finalizedUpdateInProgress.Unlock() + return true + } + return false + }, 5*time.Second, 5*time.Millisecond) + assert.Equal(t, int64(500), p.FinalizedBlock(), "lower finalized suggestions never move the head back") +} + +func TestCovPoller_OnLatestBlock_FiresForEveryAdvance(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-onlatest", covPollerBlocks(300, 1)) + got := make(chan int64, 8) + p.OnLatestBlock(func(v int64) { got <- v }) + + p.SuggestLatestBlock(200) + assert.Equal(t, int64(200), <-got) + + _, err := p.PollLatestBlockNumber(ctx) + require.NoError(t, err) + assert.Equal(t, int64(300), <-got, "polled advances fire the same callback") + assert.Equal(t, 1, up.callCount("eth_getBlockByNumber")) + + p.SuggestLatestBlock(250) + select { + case v := <-got: + t.Fatalf("callback fired for a non-advance: %d", v) + default: + } +} + +func TestCovPoller_RefreshLatestBlock(t *testing.T) { + t.Run("nil-safe on unusable pollers", func(t *testing.T) { + var nilPoller *EvmStatePoller + assert.True(t, nilPoller.IsObjectNull()) + nilPoller.RefreshLatestBlock() + assert.Nil(t, nilPoller.GetDiagnostics()) + + up := &covPollerUpstream{cfg: &common.UpstreamConfig{Id: "x"}, calls: map[string]int{}} + bare := &EvmStatePoller{upstream: up} + assert.False(t, bare.IsObjectNull()) + bare.RefreshLatestBlock() + bare.OnLatestBlock(func(int64) {}) + assert.Zero(t, up.callCount("eth_getBlockByNumber"), "no shared counter, nothing to refresh") + }) + + t.Run("skips while another refresh is in flight", func(t *testing.T) { + p, up, _ := covPollerNew(t, "cov-refresh-inflight", covPollerBlocks(10, 1)) + p.latestRefreshInFlight.Store(true) + p.RefreshLatestBlock() + assert.False(t, p.latestRefreshInFlight.CompareAndSwap(false, true), "flag untouched by the skipped call") + assert.Zero(t, up.callCount("eth_getBlockByNumber")) + }) + + t.Run("failed refresh releases the in-flight flag", func(t *testing.T) { + p, up, _ := covPollerNew(t, "cov-refresh-fail", func(string, []interface{}) (*common.NormalizedResponse, error) { + return nil, common.NewErrEndpointServerSideException(errors.New("boom"), nil, 500) + }) + p.RefreshLatestBlock() + require.Eventually(t, func() bool { + return up.callCount("eth_getBlockByNumber") == 1 && !p.latestRefreshInFlight.Load() + }, 5*time.Second, 5*time.Millisecond) + assert.Zero(t, p.LatestBlock()) + }) + + t.Run("successful refresh advances the head", func(t *testing.T) { + p, _, _ := covPollerNew(t, "cov-refresh-ok", covPollerBlocks(77, 1)) + p.RefreshLatestBlock() + require.Eventually(t, func() bool { return p.LatestBlock() == 77 }, 5*time.Second, 5*time.Millisecond) + }) +} + +// --------------------------------------------------------------------------- +// Shared state integration, config, diagnostics +// --------------------------------------------------------------------------- + +func TestCovPoller_LargeRollbackIsReportedToTracker(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-rollback-gauge", covPollerBlocks(1, 1)) + + p.latestBlockShared.TryUpdate(ctx, 10_000) + p.latestBlockShared.TryUpdate(ctx, 3_000) // beyond tolerance: a real reorg/correction + p.finalizedBlockShared.TryUpdate(ctx, 9_000) + p.finalizedBlockShared.TryUpdate(ctx, 2_000) + + assert.Equal(t, int64(3_000), p.LatestBlock()) + assert.Equal(t, int64(2_000), p.FinalizedBlock()) + latestGauge := telemetry.MetricUpstreamBlockHeadLargeRollback.WithLabelValues("test", up.VendorName(), up.NetworkLabel(), up.Id(), "latest") + finalizedGauge := telemetry.MetricUpstreamBlockHeadLargeRollback.WithLabelValues("test", up.VendorName(), up.NetworkLabel(), up.Id(), "finalized") + assert.Equal(t, float64(7_000), promUtil.ToFloat64(latestGauge)) + assert.Equal(t, float64(7_000), promUtil.ToFloat64(finalizedGauge)) +} + +func TestCovPoller_SetNetworkConfig(t *testing.T) { + p, _, _ := covPollerNew(t, "cov-netcfg", covPollerBlocks(1, 1)) + + p.SetNetworkConfig(nil) + p.SetNetworkConfig(&common.NetworkConfig{Architecture: common.ArchitectureEvm}) + assert.Nil(t, p.cfg, "nil configs are ignored") + assert.Equal(t, "n/a", p.networkLabel) + + p.SetNetworkConfig(&common.NetworkConfig{Architecture: common.ArchitectureEvm, Evm: &common.EvmNetworkConfig{ChainId: 123}}) + assert.Equal(t, "evm:123", p.networkLabel) + + p.SetNetworkConfig(&common.NetworkConfig{Architecture: common.ArchitectureEvm, Alias: "mainnet", Evm: &common.EvmNetworkConfig{ChainId: 123}}) + assert.Equal(t, "mainnet", p.networkLabel, "alias wins over the network id") +} + +func TestCovPoller_GetDiagnostics_OperatorSkipAndEarliest(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-diag", covPollerHistory(100, 42)) + skip := true + up.cfg.Evm.SkipSyncingCheck = &skip + + _, err := p.PollEarliestBlockNumber(ctx, common.EvmProbeBlockHeader, time.Hour) + require.NoError(t, err) + + d := p.GetDiagnostics() + assert.True(t, d.SkipSyncingCheck, "operator-configured skip is reflected") + assert.NotEmpty(t, d.SyncingCheckError) + require.Contains(t, d.EarliestByProbe, common.EvmProbeBlockHeader) + info := d.EarliestByProbe[common.EvmProbeBlockHeader] + assert.Equal(t, int64(42), info.EarliestBlock) + assert.False(t, info.SchedulerRunning) + assert.Equal(t, int64(100), d.LatestBlock) +} + +// --------------------------------------------------------------------------- +// Syncing detection +// --------------------------------------------------------------------------- + +func TestCovPoller_FetchSyncingState(t *testing.T) { + cases := []struct { + name string + resp *common.NormalizedResponse + err error + syncing bool + wantErr string + skip bool + }{ + {name: "bool true", resp: covPollerResult(`true`), syncing: true}, + {name: "bool false", resp: covPollerResult(`false`)}, + {name: "geth progress object", resp: covPollerResult(`{"currentBlock":"0x10","highestBlock":"0x20"}`), syncing: true}, + {name: "arbitrum msgCount", resp: covPollerResult(`{"msgCount":12}`), syncing: true}, + {name: "Ok true", resp: covPollerResult(`{"Ok":true}`)}, + {name: "Ok false", resp: covPollerResult(`{"Ok":false}`), syncing: true}, + {name: "ok true", resp: covPollerResult(`{"ok":true}`)}, + {name: "ok false", resp: covPollerResult(`{"ok":false}`), syncing: true}, + {name: "unknown object", resp: covPollerResult(`{"foo":1}`), wantErr: "cannot parse syncing state"}, + {name: "string result", resp: covPollerResult(`"yes"`), wantErr: "cannot parse syncing state"}, + {name: "not json", resp: covPollerResult(`nope`), wantErr: "cannot parse syncing state"}, + {name: "json-rpc error", resp: covPollerRpcError(-32601, "the method eth_syncing does not exist/is not available"), wantErr: "does not exist/is not available"}, + {name: "unsupported marks skip", err: common.NewErrEndpointUnsupported(errors.New("no")), wantErr: "does not support", skip: true}, + {name: "client error marks skip", err: common.NewErrEndpointClientSideException(errors.New("bad")), wantErr: "client-side", skip: true}, + {name: "server error keeps checking", err: common.NewErrEndpointServerSideException(errors.New("down"), nil, 503), wantErr: "internal error on remote endpoint"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p, _, ctx := covPollerNew(t, "cov-syncing", func(string, []interface{}) (*common.NormalizedResponse, error) { + return tc.resp, tc.err + }) + syncing, err := p.fetchSyncingState(ctx) + if tc.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + } else { + require.NoError(t, err) + assert.Equal(t, tc.syncing, syncing) + } + assert.Equal(t, tc.skip, p.GetDiagnostics().SkipSyncingCheck) + }) + } +} + +func TestCovPoller_Poll_SyncingStateTransitions(t *testing.T) { + var mu sync.Mutex + syncing := true + h := func(method string, params []interface{}) (*common.NormalizedResponse, error) { + if method == "eth_syncing" { + mu.Lock() + defer mu.Unlock() + return covPollerResult(fmt.Sprintf("%t", syncing)), nil + } + return covPollerBlocks(100, 90)(method, params) + } + setSyncing := func(v bool) { + mu.Lock() + syncing = v + mu.Unlock() + } + p, up, ctx := covPollerNew(t, "cov-syncing-poll", h) + + require.NoError(t, p.Poll(ctx)) + assert.Equal(t, common.EvmSyncingStateSyncing, p.SyncingState()) + assert.Equal(t, int64(100), p.LatestBlock()) + assert.Equal(t, int64(90), p.FinalizedBlock()) + + setSyncing(false) + require.NoError(t, p.Poll(ctx)) + assert.Equal(t, common.EvmSyncingStateUnknown, p.SyncingState(), "one 'synced' answer is not yet trusted") + + setSyncing(true) + require.NoError(t, p.Poll(ctx)) + assert.Equal(t, common.EvmSyncingStateSyncing, p.SyncingState(), "a syncing answer resets confirmations") + + setSyncing(false) + for range FullySyncedThreshold - 2 { + require.NoError(t, p.Poll(ctx)) + assert.Equal(t, common.EvmSyncingStateUnknown, p.SyncingState()) + } + require.NoError(t, p.Poll(ctx)) + assert.Equal(t, common.EvmSyncingStateNotSyncing, p.SyncingState()) + + calls := up.callCount("eth_syncing") + require.NoError(t, p.Poll(ctx)) + assert.Equal(t, calls, up.callCount("eth_syncing"), "fully synced nodes are not asked again") + + p.SetSyncingState(common.EvmSyncingStateSyncing) + assert.Equal(t, common.EvmSyncingStateSyncing, p.SyncingState()) +} + +func TestCovPoller_Poll_JoinsErrorsFromEachTracker(t *testing.T) { + p, _, ctx := covPollerNew(t, "cov-poll-errs", func(method string, params []interface{}) (*common.NormalizedResponse, error) { + switch method { + case "eth_getBlockByNumber": + if covPollerBlockTag(params) == "latest" { + return nil, common.NewErrEndpointServerSideException(errors.New("latest down"), nil, 502) + } + return covPollerBlock(90), nil + case "eth_syncing": + return nil, common.NewErrEndpointServerSideException(errors.New("syncing down"), nil, 502) + } + return nil, errors.New("unexpected") + }) + + err := p.Poll(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "latest down") + assert.Contains(t, err.Error(), "syncing down") + assert.Equal(t, int64(90), p.FinalizedBlock(), "a healthy tracker still updates when others fail") + assert.Equal(t, common.EvmSyncingStateUnknown, p.SyncingState()) +} + +// --------------------------------------------------------------------------- +// Earliest block detection +// --------------------------------------------------------------------------- + +func TestCovPoller_PollEarliestBlockNumber(t *testing.T) { + t.Run("bootstraps latest then binary-searches the earliest block", func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-earliest-ok", covPollerHistory(100, 42)) + assert.Zero(t, p.EarliestBlock(common.EvmProbeBlockHeader), "unknown probe reads as 0") + + v, err := p.PollEarliestBlockNumber(ctx, common.EvmProbeBlockHeader, time.Hour) + require.NoError(t, err) + assert.Equal(t, int64(42), v) + assert.Equal(t, int64(42), p.EarliestBlock(common.EvmProbeBlockHeader)) + assert.Equal(t, int64(100), p.LatestBlock()) + + calls := up.callCount("eth_getBlockByNumber") + v, err = p.PollEarliestBlockNumber(ctx, common.EvmProbeBlockHeader, time.Hour) + require.NoError(t, err) + assert.Equal(t, int64(42), v) + assert.Equal(t, calls, up.callCount("eth_getBlockByNumber"), "fresh earliest is served without probing") + }) + + t.Run("latest unavailable fails the detection", func(t *testing.T) { + p, _, ctx := covPollerNew(t, "cov-earliest-nolatest", func(string, []interface{}) (*common.NormalizedResponse, error) { + return nil, common.NewErrEndpointServerSideException(errors.New("latest down"), nil, 502) + }) + _, err := p.PollEarliestBlockNumber(ctx, common.EvmProbeBlockHeader, time.Hour) + require.Error(t, err) + assert.Contains(t, err.Error(), "latest down") + assert.Zero(t, p.EarliestBlock(common.EvmProbeBlockHeader)) + }) + + t.Run("latest empty yields no earliest bound", func(t *testing.T) { + p, _, ctx := covPollerNew(t, "cov-earliest-emptylatest", func(string, []interface{}) (*common.NormalizedResponse, error) { + return covPollerResult(`null`), nil + }) + v, err := p.PollEarliestBlockNumber(ctx, common.EvmProbeBlockHeader, time.Hour) + require.NoError(t, err) + assert.Zero(t, v) + }) +} + +func TestCovPoller_InitializeEarliestDetection(t *testing.T) { + plus := int64(0) + availability := func() *common.EvmBlockAvailabilityConfig { + return &common.EvmBlockAvailabilityConfig{ + Lower: &common.EvmAvailabilityBoundConfig{EarliestBlockPlus: &plus, UpdateRate: common.Duration(time.Hour)}, + Upper: &common.EvmAvailabilityBoundConfig{EarliestBlockPlus: &plus, UpdateRate: common.Duration(30 * time.Minute)}, + } + } + + t.Run("latest poll error skips this cycle", func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-init-err", func(string, []interface{}) (*common.NormalizedResponse, error) { + return nil, common.NewErrEndpointServerSideException(errors.New("down"), nil, 502) + }) + up.cfg.Evm.BlockAvailability = availability() + p.initializeEarliestBlockDetectionAndStartScheduler(ctx) + assert.Nil(t, p.GetDiagnostics().EarliestByProbe) + assert.Equal(t, 1, up.callCount("eth_getBlockByNumber"), "only the latest poll was attempted") + }) + + t.Run("latest still empty skips this cycle", func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-init-empty", func(string, []interface{}) (*common.NormalizedResponse, error) { + return covPollerResult(`null`), nil + }) + up.cfg.Evm.BlockAvailability = availability() + p.initializeEarliestBlockDetectionAndStartScheduler(ctx) + assert.Nil(t, p.GetDiagnostics().EarliestByProbe) + }) + + t.Run("detects once and starts one scheduler per probe", func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-init-ok", covPollerHistory(100, 42)) + up.cfg.Evm.BlockAvailability = availability() + + p.initializeEarliestBlockDetectionAndStartScheduler(ctx) + d := p.GetDiagnostics() + require.Contains(t, d.EarliestByProbe, common.EvmProbeBlockHeader, "an empty probe defaults to blockHeader") + assert.Equal(t, int64(42), d.EarliestByProbe[common.EvmProbeBlockHeader].EarliestBlock) + assert.True(t, d.EarliestByProbe[common.EvmProbeBlockHeader].SchedulerRunning) + + calls := up.callCount("eth_getBlockByNumber") + p.initializeEarliestBlockDetectionAndStartScheduler(ctx) + assert.Equal(t, calls, up.callCount("eth_getBlockByNumber"), "initial detection runs once per instance") + }) +} + +// --------------------------------------------------------------------------- +// Availability probes +// --------------------------------------------------------------------------- + +func TestCovPoller_BinarySearchEarliest_Bounds(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-bsearch", covPollerHistory(1000, 321)) + + v, err := p.binarySearchEarliest(ctx, "", -10, 1000) + require.NoError(t, err) + assert.Equal(t, int64(321), v, "empty probe defaults to blockHeader and negative low is clamped") + + calls := up.callCount("eth_getBlockByNumber") + v, err = p.binarySearchEarliest(ctx, common.EvmProbeBlockHeader, 0, -1) + require.NoError(t, err) + assert.Zero(t, v) + assert.Equal(t, calls, up.callCount("eth_getBlockByNumber"), "an empty range probes nothing") + + p2, _, _ := covPollerNew(t, "cov-bsearch-genesis", covPollerHistory(1000, 1)) + v, err = p2.binarySearchEarliest(ctx, common.EvmProbeBlockHeader, 0, 1000) + require.NoError(t, err) + assert.Equal(t, int64(1), v, "block-1 fast path") +} + +func TestCovPoller_CheckProbe_DispatchAndUnknown(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-probe-dispatch", covPollerHistory(1000, 0)) + + ok, unsupported, err := p.checkProbe(ctx, "", 5) + require.NoError(t, err) + assert.True(t, ok) + assert.False(t, unsupported) + + before := up.callCount("eth_getBlockByNumber") + ok, unsupported, err = p.checkProbe(ctx, "bogusProbe", 5) + require.NoError(t, err) + assert.False(t, ok) + assert.True(t, unsupported, "unknown probes report unsupported") + assert.Equal(t, before, up.callCount("eth_getBlockByNumber")) +} + +func TestCovPoller_CheckBlockHeaderProbe(t *testing.T) { + cases := []struct { + name string + block int64 + resp *common.NormalizedResponse + err error + want bool + }{ + {name: "negative block", block: -1, want: false}, + {name: "rpc error", block: 5, resp: covPollerRpcError(-32000, "missing trie node")}, + {name: "null", block: 5, resp: covPollerResult(`null`)}, + {name: "transport error", block: 5, err: errors.New("dial tcp: connection refused")}, + {name: "available", block: 5, resp: covPollerBlock(5), want: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-probe-header", func(string, []interface{}) (*common.NormalizedResponse, error) { + return tc.resp, tc.err + }) + ok, unsupported, err := p.checkProbe(ctx, common.EvmProbeBlockHeader, tc.block) + require.NoError(t, err) + assert.False(t, unsupported) + assert.Equal(t, tc.want, ok) + if tc.block < 0 { + assert.Zero(t, up.callCount("eth_getBlockByNumber")) + } + }) + } +} + +func TestCovPoller_CheckEventLogsProbe(t *testing.T) { + type logsResp struct { + resp *common.NormalizedResponse + err error + } + cases := []struct { + name string + block int64 + header *common.NormalizedResponse + logs *logsResp + ok bool + unsupported bool + }{ + {name: "negative block", block: -1}, + {name: "block missing", block: 5, header: covPollerResult(`null`)}, + {name: "block header rpc error", block: 5, header: covPollerRpcError(-32000, "missing trie node")}, + {name: "block without hash", block: 5, header: covPollerResult(`{"number":"0x5"}`)}, + {name: "one log", block: 5, header: covPollerBlock(5), logs: &logsResp{resp: covPollerResult(`[{"logIndex":"0x0"}]`)}, ok: true}, + {name: "no logs", block: 5, header: covPollerBlock(5), logs: &logsResp{resp: covPollerResult(`[]`)}}, + {name: "non-array result", block: 5, header: covPollerBlock(5), logs: &logsResp{resp: covPollerResult(`{"a":1}`)}}, + {name: "method not found", block: 5, header: covPollerBlock(5), logs: &logsResp{resp: covPollerRpcError(-32601, "the method eth_getLogs does not exist/is not available")}, unsupported: true}, + {name: "other rpc error", block: 5, header: covPollerBlock(5), logs: &logsResp{resp: covPollerRpcError(-32005, "query returned more than 10000 results")}}, + {name: "unsupported error", block: 5, header: covPollerBlock(5), logs: &logsResp{err: common.NewErrEndpointUnsupported(errors.New("no logs"))}, unsupported: true}, + {name: "ignored method", block: 5, header: covPollerBlock(5), logs: &logsResp{err: common.NewErrUpstreamMethodIgnored("eth_getLogs", "u")}, unsupported: true}, + {name: "transport error", block: 5, header: covPollerBlock(5), logs: &logsResp{err: errors.New("i/o timeout")}}, + {name: "empty response", block: 5, header: covPollerBlock(5), logs: &logsResp{}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-probe-logs", func(method string, params []interface{}) (*common.NormalizedResponse, error) { + switch method { + case "eth_getBlockByNumber": + return tc.header, nil + case "eth_getLogs": + require.NotNil(t, tc.logs, "eth_getLogs must not be called") + if f, ok := params[0].(map[string]interface{}); ok { + assert.True(t, strings.HasPrefix(fmt.Sprint(f["blockHash"]), "0x")) + } + return tc.logs.resp, tc.logs.err + } + return nil, errors.New("unexpected") + }) + ok, unsupported, err := p.checkProbe(ctx, common.EvmProbeEventLogs, tc.block) + require.NoError(t, err) + assert.Equal(t, tc.ok, ok) + assert.Equal(t, tc.unsupported, unsupported) + if tc.logs == nil { + assert.Zero(t, up.callCount("eth_getLogs")) + } + }) + } +} + +func TestCovPoller_CheckCallStateProbe_FallbackPaths(t *testing.T) { + cases := []struct { + name string + callErr error + balResp *common.NormalizedResponse + balErr error + ok bool + unsupported bool + balanceUsed bool + }{ + {name: "eth_call unsupported", callErr: common.NewErrEndpointUnsupported(errors.New("no")), unsupported: true}, + {name: "balance unsupported", callErr: errors.New("execution reverted"), balErr: common.NewErrUpstreamRequestSkipped(errors.New("x"), "u"), unsupported: true, balanceUsed: true}, + {name: "balance transport error", callErr: errors.New("execution reverted"), balErr: errors.New("i/o timeout"), balanceUsed: true}, + {name: "balance rpc error", callErr: errors.New("execution reverted"), balResp: covPollerRpcError(-32000, "missing trie node"), balanceUsed: true}, + {name: "balance null", callErr: errors.New("execution reverted"), balResp: covPollerResult(`null`), balanceUsed: true}, + {name: "balance empty response", callErr: errors.New("execution reverted"), balanceUsed: true}, + {name: "balance answered", callErr: errors.New("execution reverted"), balResp: covPollerResult(`"0x0"`), ok: true, balanceUsed: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-probe-callstate", func(method string, params []interface{}) (*common.NormalizedResponse, error) { + switch method { + case "eth_call": + return nil, tc.callErr + case "eth_getBalance": + return tc.balResp, tc.balErr + } + return nil, errors.New("unexpected") + }) + ok, unsupported, err := p.checkProbe(ctx, common.EvmProbeCallState, 0x99) + require.NoError(t, err) + assert.Equal(t, tc.ok, ok) + assert.Equal(t, tc.unsupported, unsupported) + assert.Equal(t, tc.balanceUsed, up.callCount("eth_getBalance") == 1) + }) + } + + t.Run("negative block", func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-probe-callstate-neg", covPollerBlocks(1, 1)) + ok, unsupported, err := p.checkProbe(ctx, common.EvmProbeCallState, -3) + require.NoError(t, err) + assert.False(t, ok) + assert.False(t, unsupported) + assert.Zero(t, up.callCount("eth_call")) + }) +} + +func TestCovPoller_CheckTraceDataProbe(t *testing.T) { + notFound := func(m string) *common.NormalizedResponse { + return covPollerRpcError(-32601, "the method "+m+" does not exist/is not available") + } + type ans struct { + resp *common.NormalizedResponse + err error + } + cases := []struct { + name string + header *common.NormalizedResponse + answers map[string]ans + ok bool + unsupported bool + calls []string + }{ + { + name: "trace_block answers", + header: covPollerBlock(9), + answers: map[string]ans{ + "trace_block": {resp: covPollerResult(`[{"type":"call"}]`)}, + }, + ok: true, + calls: []string{"trace_block"}, + }, + { + name: "debug_traceBlockByHash answers after trace_block is missing", + header: covPollerBlock(9), + answers: map[string]ans{ + "trace_block": {resp: notFound("trace_block")}, + "debug_traceBlockByHash": {resp: covPollerResult(`[{"result":{}}]`)}, + }, + ok: true, + calls: []string{"trace_block", "debug_traceBlockByHash"}, + }, + { + name: "parity replay answers last", + header: covPollerBlock(9), + answers: map[string]ans{ + "trace_block": {err: common.NewErrEndpointUnsupported(errors.New("no"))}, + "debug_traceBlockByHash": {resp: covPollerResult(`[]`)}, + "trace_replayBlockTransactions": {resp: covPollerResult(`[{"trace":[]}]`)}, + }, + ok: true, + calls: []string{"trace_block", "debug_traceBlockByHash", "trace_replayBlockTransactions"}, + }, + { + name: "all methods unsupported", + header: covPollerBlock(9), + answers: map[string]ans{ + "trace_block": {resp: notFound("trace_block")}, + "debug_traceBlockByHash": {err: common.NewErrUpstreamMethodIgnored("debug_traceBlockByHash", "u")}, + "trace_replayBlockTransactions": {resp: notFound("trace_replayBlockTransactions")}, + }, + unsupported: true, + calls: []string{"trace_block", "debug_traceBlockByHash", "trace_replayBlockTransactions"}, + }, + { + name: "mixed failures are not-available, not unsupported", + header: covPollerBlock(9), + answers: map[string]ans{ + "trace_block": {resp: notFound("trace_block")}, + "debug_traceBlockByHash": {err: errors.New("i/o timeout")}, + "trace_replayBlockTransactions": {resp: covPollerRpcError(-32000, "missing trie node")}, + }, + calls: []string{"trace_block", "debug_traceBlockByHash", "trace_replayBlockTransactions"}, + }, + { + name: "empty responses", + header: covPollerBlock(9), + answers: map[string]ans{ + "trace_block": {}, + "debug_traceBlockByHash": {resp: covPollerResult(`null`)}, + "trace_replayBlockTransactions": {resp: covPollerResult(`[]`)}, + }, + calls: []string{"trace_block", "debug_traceBlockByHash", "trace_replayBlockTransactions"}, + }, + { + name: "block not found", + header: covPollerResult(`null`), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-probe-trace", func(method string, _ []interface{}) (*common.NormalizedResponse, error) { + if method == "eth_getBlockByNumber" { + return tc.header, nil + } + a, ok := tc.answers[method] + require.True(t, ok, "unexpected call to %s", method) + return a.resp, a.err + }) + ok, unsupported, err := p.checkProbe(ctx, common.EvmProbeTraceData, 9) + require.NoError(t, err) + assert.Equal(t, tc.ok, ok) + assert.Equal(t, tc.unsupported, unsupported) + for _, m := range []string{"trace_block", "debug_traceBlockByHash", "trace_replayBlockTransactions"} { + want := 0 + for _, c := range tc.calls { + if c == m { + want = 1 + } + } + assert.Equal(t, want, up.callCount(m), m) + } + }) + } + + t.Run("negative block", func(t *testing.T) { + p, up, ctx := covPollerNew(t, "cov-probe-trace-neg", covPollerBlocks(1, 1)) + ok, unsupported, err := p.checkProbe(ctx, common.EvmProbeTraceData, -1) + require.NoError(t, err) + assert.False(t, ok) + assert.False(t, unsupported) + assert.Zero(t, up.callCount("eth_getBlockByNumber")) + }) +} diff --git a/architecture/evm/cov_trace_test.go b/architecture/evm/cov_trace_test.go new file mode 100644 index 000000000..511660a6f --- /dev/null +++ b/architecture/evm/cov_trace_test.go @@ -0,0 +1,657 @@ +package evm + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "sync" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// covTraceNet returns a queryTestNetwork whose Forward answers every +// trace_filter sub-request with a one-element array naming the sub-range, so +// merged output reveals both the split boundaries and the merge order. +func covTraceNet(cfg *common.NetworkConfig, latest int64, rec *covTraceRecorder) *queryTestNetwork { + return &queryTestNetwork{ + cfg: cfg, + latest: latest, + forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + jrq, err := req.JsonRpcRequest(ctx) + if err != nil { + return nil, err + } + jrq.RLock() + method := jrq.Method + filter := jrq.Params[0].(map[string]interface{}) + fb, _ := filter["fromBlock"].(string) + tb, _ := filter["toBlock"].(string) + jrq.RUnlock() + if rec != nil { + rec.add(method, fb, tb, req) + } + return common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`[{"from":"`+fb+`","to":"`+tb+`"}]`), nil), + ), nil + }, + } +} + +type covTraceSub struct { + method, from, to, skipCache string + parent interface{} + fromAddress interface{} +} + +type covTraceRecorder struct { + mu sync.Mutex + subs []covTraceSub +} + +func (r *covTraceRecorder) add(method, fb, tb string, req *common.NormalizedRequest) { + jrq, _ := req.JsonRpcRequest() + jrq.RLock() + fa := jrq.Params[0].(map[string]interface{})["fromAddress"] + jrq.RUnlock() + r.mu.Lock() + defer r.mu.Unlock() + r.subs = append(r.subs, covTraceSub{ + method: method, + from: fb, + to: tb, + skipCache: req.Directives().SkipCacheRead, + parent: req.ParentRequestId(), + fromAddress: fa, + }) +} + +func (r *covTraceRecorder) ranges() map[string]string { + r.mu.Lock() + defer r.mu.Unlock() + out := map[string]string{} + for _, s := range r.subs { + out[s.from] = s.to + } + return out +} + +// covTraceResult decodes the "result" member of a merged JSON-RPC response. +func covTraceResult(t *testing.T, jrr *common.JsonRpcResponse) ([]map[string]string, interface{}) { + t.Helper() + var buf bytes.Buffer + _, err := jrr.WriteTo(&buf) + require.NoError(t, err) + var out struct { + ID interface{} `json:"id"` + Result []map[string]string `json:"result"` + } + require.NoError(t, json.Unmarshal(buf.Bytes(), &out), buf.String()) + return out.Result, out.ID +} + +func covTraceReq(t *testing.T, method string, id interface{}, filter map[string]interface{}) *common.NormalizedRequest { + t.Helper() + jrq := common.NewJsonRpcRequest(method, []interface{}{filter}) + if id != nil { + require.NoError(t, jrq.SetID(id)) + } + return common.NewNormalizedRequestFromJsonRpcRequest(jrq) +} + +func covTraceHistogram(t *testing.T, method string) (uint64, float64) { + t.Helper() + obs := telemetry.MetricNetworkEvmTraceFilterRangeRequested.WithLabelValues( + "test-project", "evm:1", method, "n/a", common.DataFinalityStateUnknown.String(), + ) + m := &dto.Metric{} + require.NoError(t, obs.(prometheus.Metric).Write(m)) + return m.GetHistogram().GetSampleCount(), m.GetHistogram().GetSampleSum() +} + +func covTraceForcedSplits(method, dimension string) float64 { + return testutil.ToFloat64(telemetry.MetricNetworkEvmTraceFilterForcedSplits.WithLabelValues( + "test-project", "evm:1", method, dimension, "n/a", "unknown", + )) +} + +func TestCovTrace_ProjectPreForward_RangeHistogram(t *testing.T) { + ctx := context.Background() + net := &queryTestNetwork{cfg: &common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}, latest: 0x10, finalized: 0x8} + + cases := []struct { + name string + method string + filter interface{} + wantCount uint64 + wantSum float64 + }{ + {name: "hex_range_observed", method: "trace_filter", filter: map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x4"}, wantCount: 1, wantSum: 4}, + {name: "latest_tag_resolved_to_tip", method: "arbtrace_filter", filter: map[string]interface{}{"fromBlock": "0x1", "toBlock": "latest"}, wantCount: 1, wantSum: 16}, + {name: "finalized_tag_resolved", method: "trace_filter", filter: map[string]interface{}{"fromBlock": "finalized", "toBlock": "0x8"}, wantCount: 1, wantSum: 1}, + {name: "genesis_from_block_not_observed", method: "trace_filter", filter: map[string]interface{}{"fromBlock": "0x0", "toBlock": "0x4"}}, + {name: "inverted_range_not_observed", method: "trace_filter", filter: map[string]interface{}{"fromBlock": "0x5", "toBlock": "0x4"}}, + {name: "unresolvable_safe_tag_not_observed", method: "trace_filter", filter: map[string]interface{}{"fromBlock": "safe", "toBlock": "0x4"}}, + {name: "missing_bounds_not_observed", method: "trace_filter", filter: map[string]interface{}{}}, + {name: "invalid_hex_not_observed", method: "trace_filter", filter: map[string]interface{}{"fromBlock": "0xzz", "toBlock": "0x4"}}, + {name: "non_object_filter_ignored", method: "trace_filter", filter: "0x1"}, + {name: "other_method_ignored", method: "eth_getLogs", filter: map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x4"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + beforeCount, beforeSum := covTraceHistogram(t, "trace_filter") + beforeArbCount, beforeArbSum := covTraceHistogram(t, "arbtrace_filter") + req := common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest(tc.method, []interface{}{tc.filter})) + + handled, resp, err := projectPreForward_trace_filter(ctx, net, req) + require.NoError(t, err) + assert.False(t, handled) + assert.Nil(t, resp) + + afterCount, afterSum := covTraceHistogram(t, "trace_filter") + afterArbCount, afterArbSum := covTraceHistogram(t, "arbtrace_filter") + gotCount := (afterCount - beforeCount) + (afterArbCount - beforeArbCount) + gotSum := (afterSum - beforeSum) + (afterArbSum - beforeArbSum) + assert.Equal(t, tc.wantCount, gotCount) + assert.Equal(t, tc.wantSum, gotSum) + }) + } + + t.Run("empty_params_and_nil_inputs_are_noops", func(t *testing.T) { + req := common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest("trace_filter", []interface{}{})) + handled, resp, err := projectPreForward_trace_filter(ctx, net, req) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + + handled, _, err = projectPreForward_trace_filter(ctx, nil, req) + assert.False(t, handled) + assert.NoError(t, err) + handled, _, err = projectPreForward_trace_filter(ctx, net, nil) + assert.False(t, handled) + assert.NoError(t, err) + }) +} + +func TestCovTrace_NetworkPreForward_PassThrough(t *testing.T) { + ctx := context.Background() + evmCfg := &common.NetworkConfig{Evm: &common.EvmNetworkConfig{}} + ups := []common.Upstream{&queryTestConfigUpstream{cfg: &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{TraceFilterAutoSplittingRangeThreshold: 1}}}} + + cases := []struct { + name string + cfg *common.NetworkConfig + method string + params []interface{} + }{ + {name: "other_method", cfg: evmCfg, method: "eth_getLogs", params: []interface{}{map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x10"}}}, + {name: "empty_params", cfg: evmCfg, method: "trace_filter", params: []interface{}{}}, + {name: "non_object_filter", cfg: evmCfg, method: "trace_filter", params: []interface{}{"0x1"}}, + {name: "missing_toBlock", cfg: evmCfg, method: "trace_filter", params: []interface{}{map[string]interface{}{"fromBlock": "0x1"}}}, + {name: "unresolvable_safe_tag", cfg: evmCfg, method: "trace_filter", params: []interface{}{map[string]interface{}{"fromBlock": "safe", "toBlock": "0x10"}}}, + {name: "unparsable_hex", cfg: evmCfg, method: "trace_filter", params: []interface{}{map[string]interface{}{"fromBlock": "0x1", "toBlock": "0xnothex"}}}, + {name: "network_without_evm_config", cfg: &common.NetworkConfig{}, method: "trace_filter", params: []interface{}{map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x10"}}}, + {name: "nil_network_config", cfg: nil, method: "trace_filter", params: []interface{}{map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x10"}}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + net := &queryTestNetwork{cfg: tc.cfg, latest: 0x100, forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + t.Fatalf("no sub-request expected for %s", tc.name) + return nil, nil + }} + req := common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest(tc.method, tc.params)) + handled, resp, err := networkPreForward_trace_filter(ctx, net, ups, req) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + assert.False(t, req.IsCompositeRequest()) + }) + } + + t.Run("nil_inputs", func(t *testing.T) { + handled, _, err := networkPreForward_trace_filter(ctx, nil, ups, covTraceReq(t, "trace_filter", nil, map[string]interface{}{})) + assert.False(t, handled) + assert.NoError(t, err) + handled, _, err = networkPreForward_trace_filter(ctx, &queryTestNetwork{cfg: evmCfg}, ups, nil) + assert.False(t, handled) + assert.NoError(t, err) + }) + + t.Run("already_composite_request_is_not_resplit", func(t *testing.T) { + req := covTraceReq(t, "trace_filter", nil, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x10"}) + req.SetCompositeType(common.CompositeTypeTraceFilterSplitOnError) + handled, _, err := networkPreForward_trace_filter(ctx, &queryTestNetwork{cfg: evmCfg}, ups, req) + assert.False(t, handled) + assert.NoError(t, err) + }) + + t.Run("upstreams_without_evm_config_disable_splitting", func(t *testing.T) { + net := &queryTestNetwork{cfg: evmCfg, forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + t.Fatal("no sub-request expected") + return nil, nil + }} + noEvm := []common.Upstream{ + nil, + &queryTestConfigUpstream{cfg: nil}, + &queryTestConfigUpstream{cfg: &common.UpstreamConfig{}}, + &queryTestConfigUpstream{cfg: &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{TraceFilterAutoSplittingRangeThreshold: 0}}}, + } + req := covTraceReq(t, "trace_filter", nil, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x10"}) + handled, resp, err := networkPreForward_trace_filter(ctx, net, noEvm, req) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + }) + + t.Run("tag_resolved_inverted_range_is_invalid_request", func(t *testing.T) { + net := &queryTestNetwork{cfg: evmCfg, latest: 0x5} + req := covTraceReq(t, "trace_filter", nil, map[string]interface{}{"fromBlock": "0x10", "toBlock": "latest"}) + handled, resp, err := networkPreForward_trace_filter(ctx, net, ups, req) + assert.True(t, handled) + assert.Nil(t, resp) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeInvalidRequest)) + assert.Contains(t, err.Error(), "fromBlock (16) must be less than or equal to toBlock (5)") + }) +} + +func TestCovTrace_NetworkPreForward_ProactiveSplitMerges(t *testing.T) { + ctx := context.Background() + rec := &covTraceRecorder{} + net := covTraceNet(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{TraceFilterSplitConcurrency: 2}}, 0x7, rec) + ups := []common.Upstream{ + &queryTestConfigUpstream{cfg: &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{TraceFilterAutoSplittingRangeThreshold: 5}}}, + &queryTestConfigUpstream{cfg: &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{TraceFilterAutoSplittingRangeThreshold: 3}}}, + } + req := covTraceReq(t, "ARBTRACE_FILTER", 42, map[string]interface{}{ + "fromBlock": "0x1", + "toBlock": "latest", + "fromAddress": []interface{}{"0xaaa"}, + }) + req.SetDirectives(&common.RequestDirectives{SkipCacheRead: "*"}) + + handled, resp, err := networkPreForward_trace_filter(ctx, net, ups, req) + require.NoError(t, err) + require.True(t, handled) + require.NotNil(t, resp) + assert.Equal(t, common.CompositeTypeTraceFilterSplitProactive, req.CompositeType()) + assert.False(t, resp.FromCache()) + assert.Same(t, resp, req.LastValidResponse()) + + // Range 1..7 (latest) with min threshold 3 → [1,3], [4,6], [7,7]. + assert.Equal(t, map[string]string{"0x1": "0x3", "0x4": "0x6", "0x7": "0x7"}, rec.ranges()) + for _, s := range rec.subs { + assert.Equal(t, "arbtrace_filter", s.method, "sub-requests use the lower-cased method") + assert.Equal(t, "*", s.skipCache, "skip-cache-read directive propagates to sub-requests") + assert.Equal(t, 42, s.parent) + assert.Equal(t, []interface{}{"0xaaa"}, s.fromAddress) + } + + jrr, err := resp.JsonRpcResponse(ctx) + require.NoError(t, err) + result, id := covTraceResult(t, jrr) + assert.EqualValues(t, 42, id) + assert.Equal(t, []map[string]string{ + {"from": "0x1", "to": "0x3"}, + {"from": "0x4", "to": "0x6"}, + {"from": "0x7", "to": "0x7"}, + }, result, "sub-results are concatenated in request order") +} + +func TestCovTrace_NetworkPreForward_SubRequestFailure(t *testing.T) { + ctx := context.Background() + boom := errors.New("upstream exploded on second half") + net := &queryTestNetwork{ + cfg: &common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}, + forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + jrq, _ := req.JsonRpcRequest() + jrq.RLock() + fb := jrq.Params[0].(map[string]interface{})["fromBlock"] + jrq.RUnlock() + if fb == "0x3" { + return nil, boom + } + return common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`[]`), nil), + ), nil + }, + } + ups := []common.Upstream{&queryTestConfigUpstream{cfg: &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{TraceFilterAutoSplittingRangeThreshold: 2}}}} + req := covTraceReq(t, "trace_filter", 1, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x4"}) + + handled, resp, err := networkPreForward_trace_filter(ctx, net, ups, req) + assert.True(t, handled) + assert.Nil(t, resp) + require.Error(t, err) + assert.ErrorIs(t, err, boom) +} + +func TestCovTrace_NetworkPostForward(t *testing.T) { + ctx := context.Background() + splitCfg := &common.NetworkConfig{Evm: &common.EvmNetworkConfig{TraceFilterSplitOnError: util.BoolPtr(true)}} + largeRange := common.NewErrJsonRpcExceptionInternal(-32005, common.JsonRpcErrorEvmLargeRange, "query returned more than 10000 results", nil, nil) + + t.Run("normalized_large_range_code_triggers_split_and_releases_original", func(t *testing.T) { + rec := &covTraceRecorder{} + net := covTraceNet(splitCfg, 0, rec) + req := covTraceReq(t, "trace_filter", 7, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x4"}) + req.SetDirectives(&common.RequestDirectives{SkipCacheRead: "true"}) + orig := common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`7`), []byte(`null`), nil), + ) + + out, err := networkPostForward_trace_filter(ctx, net, req, orig, largeRange) + require.NoError(t, err) + require.NotNil(t, out) + assert.NotSame(t, orig, out) + assert.Equal(t, common.CompositeTypeTraceFilterSplitOnError, req.CompositeType()) + assert.Equal(t, map[string]string{"0x1": "0x2", "0x3": "0x4"}, rec.ranges()) + for _, s := range rec.subs { + assert.Equal(t, "true", s.skipCache) + } + jrr, err := out.JsonRpcResponse(ctx) + require.NoError(t, err) + result, id := covTraceResult(t, jrr) + assert.EqualValues(t, 7, id) + assert.Equal(t, []map[string]string{{"from": "0x1", "to": "0x2"}, {"from": "0x3", "to": "0x4"}}, result) + }) + + t.Run("unsplittable_request_keeps_original_error", func(t *testing.T) { + net := covTraceNet(splitCfg, 0, nil) + req := covTraceReq(t, "trace_filter", 1, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x1"}) + out, err := networkPostForward_trace_filter(ctx, net, req, nil, largeRange) + assert.Nil(t, out) + assert.Same(t, largeRange, err) + assert.False(t, req.IsCompositeRequest()) + }) + + t.Run("sub_request_failure_keeps_original_error", func(t *testing.T) { + net := &queryTestNetwork{cfg: splitCfg, forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return nil, errors.New("still too large") + }} + req := covTraceReq(t, "trace_filter", 1, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x4"}) + orig := common.NewNormalizedResponse() + tooLarge := common.NewErrEndpointRequestTooLarge(errors.New("range too large"), common.EvmBlockRangeTooLarge) + out, err := networkPostForward_trace_filter(ctx, net, req, orig, tooLarge) + assert.Same(t, orig, out) + assert.Same(t, tooLarge, err) + }) + + t.Run("non_trace_method_keeps_original_error", func(t *testing.T) { + net := covTraceNet(splitCfg, 0, nil) + req := covTraceReq(t, "eth_getLogs", 1, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x4"}) + out, err := networkPostForward_trace_filter(ctx, net, req, nil, largeRange) + assert.Nil(t, out) + assert.Same(t, largeRange, err) + }) + + t.Run("other_internal_codes_are_not_split", func(t *testing.T) { + net := covTraceNet(splitCfg, 0, nil) + req := covTraceReq(t, "trace_filter", 1, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x4"}) + other := common.NewErrJsonRpcExceptionInternal(-32000, common.JsonRpcErrorServerSideException, "header not found", nil, nil) + out, err := networkPostForward_trace_filter(ctx, net, req, nil, other) + assert.Nil(t, out) + assert.Same(t, other, err) + }) + + t.Run("split_disabled_when_flag_unset_or_config_missing", func(t *testing.T) { + req := covTraceReq(t, "trace_filter", 1, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x4"}) + for _, cfg := range []*common.NetworkConfig{nil, {}, {Evm: &common.EvmNetworkConfig{}}} { + out, err := networkPostForward_trace_filter(ctx, &queryTestNetwork{cfg: cfg}, req, nil, largeRange) + assert.Nil(t, out) + assert.Same(t, largeRange, err) + } + }) +} + +func TestCovTrace_UpstreamPreForward_ParamValidation(t *testing.T) { + ctx := context.Background() + newNet := func() *mockNetwork { + n := new(mockNetwork) + n.On("Id").Return("net1").Maybe() + n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{ + Integrity: &common.EvmIntegrityConfig{EnforceGetLogsBlockRange: util.BoolPtr(true)}, + }}).Maybe() + return n + } + newUp := func() *mockEvmUpstream { + u := new(mockEvmUpstream) + u.On("Id").Return("u1").Maybe() + return u + } + + t.Run("non_evm_upstream_is_skipped", func(t *testing.T) { + n := newNet() + req := covTraceReq(t, "trace_filter", 1, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x2"}) + handled, resp, err := upstreamPreForward_trace_filter(ctx, n, &queryTestConfigUpstream{}, req) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + n.AssertNotCalled(t, "Config") + }) + + passThrough := []struct { + name string + params []interface{} + }{ + {name: "empty_params", params: []interface{}{}}, + {name: "non_object_filter", params: []interface{}{"0x1"}}, + {name: "tag_fromBlock", params: []interface{}{map[string]interface{}{"fromBlock": "latest", "toBlock": "0x2"}}}, + {name: "missing_fromBlock", params: []interface{}{map[string]interface{}{"toBlock": "0x2"}}}, + {name: "tag_toBlock", params: []interface{}{map[string]interface{}{"fromBlock": "0x1", "toBlock": "finalized"}}}, + {name: "missing_toBlock", params: []interface{}{map[string]interface{}{"fromBlock": "0x1"}}}, + } + for _, tc := range passThrough { + t.Run(tc.name, func(t *testing.T) { + u := newUp() + req := common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest("trace_filter", tc.params)) + handled, resp, err := upstreamPreForward_trace_filter(ctx, newNet(), u, req) + assert.False(t, handled) + assert.Nil(t, resp) + assert.NoError(t, err) + u.AssertNotCalled(t, "EvmAssertBlockAvailability", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + } + + malformed := []struct { + name string + filter map[string]interface{} + }{ + {name: "invalid_hex_fromBlock", filter: map[string]interface{}{"fromBlock": "0xzz", "toBlock": "0x2"}}, + {name: "invalid_hex_toBlock", filter: map[string]interface{}{"fromBlock": "0x1", "toBlock": "0xqq"}}, + } + for _, tc := range malformed { + t.Run(tc.name, func(t *testing.T) { + u := newUp() + req := covTraceReq(t, "arbtrace_filter", 1, tc.filter) + handled, resp, err := upstreamPreForward_trace_filter(ctx, newNet(), u, req) + assert.True(t, handled) + assert.Nil(t, resp) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid syntax") + u.AssertNotCalled(t, "EvmAssertBlockAvailability", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + } +} + +func TestCovTrace_SplitTraceFilterRequest_EdgesAndMetrics(t *testing.T) { + net := &queryTestNetwork{cfg: &common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}} + + t.Run("rejects_unsupported_method_and_bad_params", func(t *testing.T) { + cases := []struct { + name string + method string + params []interface{} + want string + }{ + {name: "method", method: "eth_getLogs", params: []interface{}{map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x4"}}, want: "unsupported method: eth_getLogs"}, + {name: "empty_params", method: "trace_filter", params: []interface{}{}, want: "invalid params length"}, + {name: "non_object", method: "trace_filter", params: []interface{}{"0x1"}, want: "invalid filter format"}, + } + for _, tc := range cases { + req := common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest(tc.method, tc.params)) + subs, err := splitTraceFilterRequest(req) + assert.Nil(t, subs, tc.name) + assert.EqualError(t, err, tc.want, tc.name) + } + }) + + cases := []struct { + name string + filter map[string]interface{} + dimension string + want []traceFilterSubRequest + }{ + { + name: "block_range", + filter: map[string]interface{}{"fromBlock": "0xa", "toBlock": "0xc", "toAddress": []interface{}{"0xbbb"}}, + dimension: "block_range", + want: []traceFilterSubRequest{ + {method: "trace_filter", fromBlock: 10, toBlock: 10, toAddress: []interface{}{"0xbbb"}}, + {method: "trace_filter", fromBlock: 11, toBlock: 12, toAddress: []interface{}{"0xbbb"}}, + }, + }, + { + name: "from_address", + filter: map[string]interface{}{"fromBlock": "0x5", "toBlock": "0x5", "fromAddress": []interface{}{"0x1", "0x2", "0x3"}}, + dimension: "from_address", + want: []traceFilterSubRequest{ + {method: "trace_filter", fromBlock: 5, toBlock: 5, fromAddress: []interface{}{"0x1"}}, + {method: "trace_filter", fromBlock: 5, toBlock: 5, fromAddress: []interface{}{"0x2", "0x3"}}, + }, + }, + { + name: "to_address_after_single_from_address", + filter: map[string]interface{}{"fromBlock": "0x5", "toBlock": "0x5", "fromAddress": []interface{}{"0x1"}, "toAddress": []interface{}{"0x8", "0x9"}}, + dimension: "to_address", + want: []traceFilterSubRequest{ + {method: "trace_filter", fromBlock: 5, toBlock: 5, fromAddress: []interface{}{"0x1"}, toAddress: []interface{}{"0x8"}}, + {method: "trace_filter", fromBlock: 5, toBlock: 5, fromAddress: []interface{}{"0x1"}, toAddress: []interface{}{"0x9"}}, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := covTraceReq(t, "TRACE_FILTER", 1, tc.filter) + req.SetNetwork(net) + before := covTraceForcedSplits("trace_filter", tc.dimension) + + got, err := splitTraceFilterRequest(req) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + assert.Equal(t, before+1, covTraceForcedSplits("trace_filter", tc.dimension), + "exactly one forced split recorded for dimension %s", tc.dimension) + }) + } +} + +// covTraceFailingWriter is a result writer whose materialization fails, used +// to exercise the clone failure path of the sub-request merger. +type covTraceFailingWriter struct{} + +func (covTraceFailingWriter) WriteTo(w io.Writer, trimSides bool) (int64, error) { + return 0, errors.New("result stream truncated") +} +func (covTraceFailingWriter) IsResultEmptyish() bool { return false } +func (covTraceFailingWriter) Size(ctx ...context.Context) (int, error) { return 0, nil } + +func TestCovTrace_ExecuteSubRequests_FailureModes(t *testing.T) { + ctx := context.Background() + cfg := &common.NetworkConfig{Evm: &common.EvmNetworkConfig{}} + subs := []traceFilterSubRequest{{method: "trace_filter", fromBlock: 1, toBlock: 1}} + + failingClone := &common.JsonRpcResponse{} + failingClone.SetResultWriter(covTraceFailingWriter{}) + + cases := []struct { + name string + resp func() *common.NormalizedResponse + wantErr string + }{ + { + name: "nil_response_without_error", + resp: func() *common.NormalizedResponse { return nil }, + wantErr: "unexpected empty json-rpc response", + }, + { + name: "json_rpc_error_payload", + resp: func() *common.NormalizedResponse { + return common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), nil, []byte(`{"code":-32000,"message":"trace_filter is not available"}`)), + ) + }, + wantErr: "trace_filter is not available", + }, + { + name: "result_clone_failure", + resp: func() *common.NormalizedResponse { return common.NewNormalizedResponse().WithJsonRpcResponse(failingClone) }, + wantErr: "result stream truncated", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + net := &queryTestNetwork{cfg: cfg, forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return tc.resp(), nil + }} + req := covTraceReq(t, "trace_filter", 1, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x1"}) + merged, fromCache, err := executeTraceFilterSubRequests(ctx, net, req, subs, "") + assert.Nil(t, merged) + assert.False(t, fromCache) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } + + t.Run("failure_metric_counts_each_failed_sub_request", func(t *testing.T) { + net := &queryTestNetwork{cfg: cfg, forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return nil, errors.New("upstream timeout") + }} + req := covTraceReq(t, "trace_filter", 1, map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x3"}) + failures := telemetry.CounterHandle(telemetry.MetricNetworkEvmTraceFilterSplitFailure, "test-project", "evm:1", "arbtrace_filter", "n/a", "unknown") + before := testutil.ToFloat64(failures) + three := []traceFilterSubRequest{ + {method: "arbtrace_filter", fromBlock: 1, toBlock: 1}, + {method: "arbtrace_filter", fromBlock: 2, toBlock: 2}, + {method: "arbtrace_filter", fromBlock: 3, toBlock: 3}, + } + _, _, err := executeTraceFilterSubRequests(ctx, net, req, three, "") + require.Error(t, err) + assert.Equal(t, before+3, testutil.ToFloat64(failures)) + }) + + t.Run("all_sub_responses_from_cache_marks_merged_as_cached", func(t *testing.T) { + net := &queryTestNetwork{cfg: cfg, forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return common.NewNormalizedResponse().WithFromCache(true).WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`[{"from":"c"}]`), nil), + ), nil + }} + req := covTraceReq(t, "trace_filter", "abc", map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x2"}) + two := []traceFilterSubRequest{{method: "trace_filter", fromBlock: 1, toBlock: 1}, {method: "trace_filter", fromBlock: 2, toBlock: 2}} + merged, fromCache, err := executeTraceFilterSubRequests(ctx, net, req, two, "") + require.NoError(t, err) + assert.True(t, fromCache) + result, id := covTraceResult(t, merged) + assert.Equal(t, "abc", id) + assert.Equal(t, []map[string]string{{"from": "c"}, {"from": "c"}}, result) + }) + + t.Run("unmarshalable_request_id_fails_merge", func(t *testing.T) { + net := covTraceNet(cfg, 0, nil) + jrq := common.NewJsonRpcRequest("trace_filter", []interface{}{map[string]interface{}{"fromBlock": "0x1", "toBlock": "0x1"}}) + jrq.ID = make(chan int) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + merged, _, err := executeTraceFilterSubRequests(ctx, net, req, subs, "") + assert.Nil(t, merged) + assert.Error(t, err) + }) +} diff --git a/architecture/evm/error_normalizer.go b/architecture/evm/error_normalizer.go index d9391ff66..ab7524b23 100644 --- a/architecture/evm/error_normalizer.go +++ b/architecture/evm/error_normalizer.go @@ -161,6 +161,19 @@ func ExtractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co return capErr } + // jsonrpsee per-connection eth_subscribe cap (reth -32006): a full socket, not an unhealthy upstream. + if strings.Contains(msg, "Too many subscriptions") { + return common.NewErrEndpointCapacityExceeded( + common.NewErrJsonRpcExceptionInternal( + int(code), + common.JsonRpcErrorCapacityExceeded, + err.Message, + nil, + details, + ), + ) + } + if r.StatusCode == 402 || strings.Contains(msg, "reached the free tier") || strings.Contains(msg, "Monthly capacity limit") || @@ -248,6 +261,35 @@ func ExtractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co ) } + // HyperEVM read precompiles (0x…0800 onwards) execute against HyperCore + // state for the requested block. Many nodes hold that state only for + // their live tip; for any other block they answer + // -32003 "out of gas: gas exhausted during precompiled contract execution: " + // while a node that holds the state answers the same call. This is a + // data gap on the node, not an execution result, so it must not reach + // the generic "out of gas" branch below, which returns it to the caller + // as a final execution exception. eth_sendRawTransaction keeps its + // existing transaction-rejected handling. + if strings.Contains(msg, "gas exhausted during precompiled contract execution") { + isSendRawTx := false + if nr != nil && nr.Request() != nil { + m, _ := nr.Request().Method() + isSendRawTx = strings.EqualFold(m, "eth_sendRawTransaction") + } + if !isSendRawTx { + return common.NewErrEndpointMissingData( + common.NewErrJsonRpcExceptionInternal( + int(code), + common.JsonRpcErrorMissingData, + err.Message, + nil, + details, + ), + upstream, + ) + } + } + //---------------------------------------------------------------- // "Timeouts / node-level" errors //---------------------------------------------------------------- @@ -299,6 +341,31 @@ func ExtractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co } return execErr } + // Aurora (Borealis engine) answers debug_trace* with -32603 for two + // per-transaction outcomes that no other node can change: + // "no EVM execution in transaction 0x…": a NEAR-native transaction + // surfaced in the EVM block, so there is nothing to trace; + // "EVM tracing failed in transaction 0x… due to ContractError {…}": + // the engine's own verdict for that transaction; + // "StandaloneError(TransactionHashNotFound(0x…))": the tracer has no + // EVM record of a transaction the chain does list (engine system + // calls to 0x4444…192b), for old and new blocks alike. + // As a server-side exception every request was retried until the + // failsafe gave up (12 attempts each) and the caller saw + // ErrUpstreamsExhausted instead of the real reason. + if strings.Contains(msg, "no EVM execution in transaction") || + strings.Contains(msg, "EVM tracing failed in transaction") || + strings.Contains(msg, "StandaloneError(TransactionHashNotFound(") { + return common.NewErrEndpointExecutionException( + common.NewErrJsonRpcExceptionInternal( + int(code), + common.JsonRpcErrorCallException, + err.Message, + nil, + details, + ), + ) + } // EVM HALTS reported as "EVM error: ". // // A halt — out of gas, invalid opcode, insufficient funds, stack @@ -394,6 +461,31 @@ func ExtractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co ) } + // HyperEVM (Hyperliquid) returns opaque HyperCore↔HyperEVM boundary + // errors as JSON-RPC -32100 with message "Unexpected error (code=10055)". + // Per Hyperliquid docs that bucket covers nonce conflicts, duplicate + // hashes, insufficient funds, and underpriced replacements. Scope the + // match to eth_sendRawTransaction only — the message is too broad to + // reinterpret as NonceTooLow on other methods (would stop failover). + // Treat as nonce-too-low (verify via eth_getTransactionByHash) rather + // than already-known (blind success). + // https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/hyperevm + if strings.Contains(ml, "code=10055") && + nr != nil && nr.Request() != nil { + if m, _ := nr.Request().Method(); strings.ToLower(m) == "eth_sendrawtransaction" { + return common.NewErrEndpointNonceException( + common.NewErrJsonRpcExceptionInternal( + int(code), + common.JsonRpcErrorTransactionRejected, + err.Message, + nil, + details, + ), + common.NonceExceptionReasonNonceTooLow, + ) + } + } + //---------------------------------------------------------------- // "Mempool policy rejection" errors // Observed from Monad, which surfaces every EthTxPoolDropReason as @@ -485,11 +577,20 @@ func ExtractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co //---------------------------------------------------------------- // "Transaction rejected" or "out of gas" errors // Note: This comes AFTER nonce/duplicate detection to avoid masking those errors. + // + // "gas required exceeds allowance (N)" is the eth_estimateGas verdict of + // go-ethereum's gas estimator (eth/gasestimator) when the call still runs + // out of gas at its cap N (for a sender with a tiny balance, the gas its + // balance can buy). Seen from Arbitrum nitro nodes, which carry that + // estimator in their go-ethereum fork. As a server-side exception every + // upstream was asked and answered the same, and the caller saw + // ErrUpstreamsExhausted instead of the node's reason. //---------------------------------------------------------------- if code == common.JsonRpcErrorTransactionRejected || strings.Contains(msg, "out of gas") || strings.Contains(msg, "gas too low") || + strings.Contains(msg, "gas required exceeds allowance") || strings.Contains(msg, "IntrinsicGas") { execErr := common.NewErrEndpointExecutionException( diff --git a/architecture/evm/error_normalizer_test.go b/architecture/evm/error_normalizer_test.go index 69e5ce99a..266bbe922 100644 --- a/architecture/evm/error_normalizer_test.go +++ b/architecture/evm/error_normalizer_test.go @@ -3,6 +3,7 @@ package evm import ( "errors" "net/http" + "strings" "testing" "github.com/erpc/erpc/common" @@ -103,6 +104,26 @@ func TestExtractJsonRpcError_InsufficientFunds_TracingMethodsRetryable(t *testin message: "insufficient funds for gas * price + value", wantRetryable: false, }, + { + // Aurora's engine verdict for the mined tx; every Aurora node repeats it. + name: "debug_traceTransaction Aurora ERR_OUT_OF_FUND stays non-retried", + method: "debug_traceTransaction", + message: `Error: EVM tracing failed in transaction 0xe373b20df8aa743e52c77d8aae113810157af9e330c8f4165cce7f2f4f759a80 due to ContractError { message: "ERR_OUT_OF_FUND" }`, + wantRetryable: false, + }, + { + name: "debug_traceTransaction Aurora NEAR-native tx stays non-retried", + method: "debug_traceTransaction", + message: "Error: no EVM execution in transaction 0x37c6ca677fb7d9f6fe0d53a72288d89fb37b9483ff90950684de6e697e013a00", + wantRetryable: false, + }, + { + // Aurora engine system call (to 0x4444…192b) the tracer has no EVM record of. + name: "debug_traceTransaction Aurora TransactionHashNotFound stays non-retried", + method: "debug_traceTransaction", + message: "StandaloneError(TransactionHashNotFound(0x1e5f1daaab9c1e10582a8defeb63f0716e8e1071c8a0327e973ff47732f50141))", + wantRetryable: false, + }, } for _, tc := range cases { @@ -162,6 +183,35 @@ func TestExtractJsonRpcError_MonadReserveBalanceViolation(t *testing.T) { } } +// TestExtractJsonRpcError_GasRequiredExceedsAllowance verifies that the +// eth_estimateGas verdict of go-ethereum's gas estimator for a call that runs +// out of gas at its cap — -32000 "gas required exceeds allowance (N)", as an +// Arbitrum nitro node returns it for a sender whose balance buys only N gas — +// is an execution outcome returned to the caller, not a server-side exception +// that sends the same estimate to every upstream. +func TestExtractJsonRpcError_GasRequiredExceedsAllowance(t *testing.T) { + t.Parallel() + + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","method":"eth_estimateGas","params":[],"id":1}`)) + nr := common.NewNormalizedResponse().WithRequest(req) + + r := &http.Response{StatusCode: 200, Header: http.Header{}} + jrErr := common.NewErrJsonRpcExceptionExternal(-32000, "gas required exceeds allowance (6691)", "") + jr := common.MustNewJsonRpcResponse(1, nil, jrErr) + + err := ExtractJsonRpcError(r, nr, jr, nil) + if !common.HasErrorCode(err, common.ErrCodeEndpointExecutionException) { + t.Fatalf("expected ErrEndpointExecutionException, got %T: %v", err, err) + } + if common.IsRetryableTowardNetwork(err) { + t.Fatalf("eth_estimateGas gas required exceeds allowance must not be retried toward the network") + } + if !strings.Contains(err.Error(), "gas required exceeds allowance (6691)") { + t.Fatalf("expected the node's message to be kept, got: %v", err) + } +} + // TestExtractJsonRpcError_ResponseTooBig_JsonRpseeSizeCap verifies that // jsonrpsee's oversized-response rejection — used by reth and anything else // built on it — normalizes to ErrEndpointRequestTooLarge, so the eth_getLogs / @@ -346,6 +396,91 @@ func TestExtractJsonRpcError_MempoolPolicyRejections(t *testing.T) { } } +// HyperEVM embeds 10055 in the message ("Unexpected error (code=10055)"), not +// the JSON-RPC code field (which is typically -32100). Scoped to +// eth_sendRawTransaction only: normalize to nonce-too-low (on-chain verify) +// rather than already-known (blind success), because the same code also covers +// insufficient funds / underpriced. Other methods must keep failover. +func TestExtractJsonRpcError_HyperEVMCode10055(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + method string + code int + message string + wantNTL bool + }{ + { + name: "sendRawTransaction observed HyperEVM wire message", + method: "eth_sendRawTransaction", + code: -32100, + message: "Unexpected error (code=10055)", + wantNTL: true, + }, + { + name: "sendRawTransaction code=10055 with other wrapper wording", + method: "eth_sendRawTransaction", + code: -32603, + message: "rpc error: code=10055 msg=boundary", + wantNTL: true, + }, + { + name: "eth_call keeps server-side / failover behavior", + method: "eth_call", + code: -32100, + message: "Unexpected error (code=10055)", + wantNTL: false, + }, + { + name: "nil request does not match", + method: "", + code: -32100, + message: "Unexpected error (code=10055)", + wantNTL: false, + }, + { + name: "unrelated 10055 digit sequence does not match", + method: "eth_sendRawTransaction", + code: -32000, + message: "internal error reference 10055", + wantNTL: false, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var nr *common.NormalizedResponse + if tc.method != "" { + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","method":"` + tc.method + `","params":[],"id":1}`)) + nr = common.NewNormalizedResponse().WithRequest(req) + } + + r := &http.Response{StatusCode: 200, Header: http.Header{}} + jrErr := common.NewErrJsonRpcExceptionExternal(tc.code, tc.message, "") + jr := common.MustNewJsonRpcResponse(1, nil, jrErr) + + err := ExtractJsonRpcError(r, nr, jr, nil) + if err == nil { + t.Fatalf("expected error, got nil") + } + var ne *common.ErrEndpointNonceException + isNTL := errors.As(err, &ne) && + ne.Details["nonceExceptionReason"] == string(common.NonceExceptionReasonNonceTooLow) + if isNTL != tc.wantNTL { + t.Fatalf("expected nonceTooLow=%v, got %T: %v", tc.wantNTL, err, err) + } + if tc.wantNTL && common.HasErrorCode(err, common.ErrCodeEndpointServerSideException) { + t.Fatalf("10055 must not fall through to server-side exception: %v", err) + } + }) + } +} + // A TX_REPLAY_ATTACK rejection of an already-accepted transaction normalizes // to an "already known" nonce exception; other "replay attack" wording does not. func TestExtractJsonRpcError_ReplayAttackIdempotency(t *testing.T) { @@ -455,3 +590,114 @@ func TestExtractJsonRpcError_RethRevertError(t *testing.T) { }) } } + +// hl-node's reply when a HyperEVM read precompile (0x…0800 onwards) runs +// against a block for which the node holds no HyperCore state. The trailing +// number is the node's RPC gas cap and differs per provider. +const hyperEvmPrecompileStateGap = "out of gas: gas exhausted during precompiled contract execution: 600000000" + +// TestExtractJsonRpcError_HyperEvmPrecompileStateGap verifies that the +// precompile-state gap is classified as missing data, so the network sweep +// moves on to an upstream that holds the state. A generic "out of gas" stays a +// final execution result, and eth_sendRawTransaction keeps its existing +// transaction-rejected handling. +func TestExtractJsonRpcError_HyperEvmPrecompileStateGap(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + method string // empty = no request attached to the response + message string + wantCode common.ErrorCode + wantNormalized common.JsonRpcErrorNumber + wantRetryableNw bool + }{ + {name: "eth_call", method: "eth_call", message: hyperEvmPrecompileStateGap, + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + {name: "eth_estimateGas", method: "eth_estimateGas", message: hyperEvmPrecompileStateGap, + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + {name: "eth_createAccessList", method: "eth_createAccessList", message: hyperEvmPrecompileStateGap, + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + {name: "eth_simulateV1", method: "eth_simulateV1", message: hyperEvmPrecompileStateGap, + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + {name: "debug_traceCall", method: "debug_traceCall", message: hyperEvmPrecompileStateGap, + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + {name: "unknown method", method: "hl_someFutureCall", message: hyperEvmPrecompileStateGap, + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + {name: "no request attached", method: "", message: hyperEvmPrecompileStateGap, + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + {name: "other gas cap", method: "eth_call", message: "out of gas: gas exhausted during precompiled contract execution: 50000000", + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + {name: "plain out of gas stays a final execution result", method: "eth_call", message: "out of gas", + wantCode: common.ErrCodeEndpointExecutionException, wantNormalized: common.JsonRpcErrorTransactionRejected, wantRetryableNw: false}, + {name: "eth_sendRawTransaction keeps transaction-rejected handling", method: "eth_sendRawTransaction", message: hyperEvmPrecompileStateGap, + wantCode: common.ErrCodeEndpointExecutionException, wantNormalized: common.JsonRpcErrorTransactionRejected, wantRetryableNw: true}, + // Pruned-state answers from Alchemy (base eth_getProof) and Nirvana (base eth_getCode/eth_call): + // another archive upstream can serve them, so fail over instead of retrying as a server fault. + {name: "eth_getProof no state found", method: "eth_getProof", message: "no state found for block number 52938851", + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + {name: "eth_getCode state pruned", method: "eth_getCode", message: "state at block #51672341 is pruned", + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + // A node that dropped old history names its floor; an archive upstream can still serve the block. + {name: "eth_getLogs pruned history", method: "eth_getLogs", message: "pruned history unavailable: requested 1000, earliest available 5000", + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + {name: "eth_getBlockByNumber below earliest available", method: "eth_getBlockByNumber", message: "block 100 is before the earliest available block 200", + wantCode: common.ErrCodeEndpointMissingData, wantNormalized: common.JsonRpcErrorMissingData, wantRetryableNw: true}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var nr *common.NormalizedResponse + if tc.method != "" { + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","method":"` + tc.method + `","params":[],"id":1}`)) + nr = common.NewNormalizedResponse().WithRequest(req) + } + + r := &http.Response{StatusCode: 200, Header: http.Header{}} + jrErr := common.NewErrJsonRpcExceptionExternal(-32003, tc.message, "") + jr := common.MustNewJsonRpcResponse(1, nil, jrErr) + + err := ExtractJsonRpcError(r, nr, jr, nil) + if !common.HasErrorCode(err, tc.wantCode) { + t.Fatalf("expected %s, got %T: %v", tc.wantCode, err, err) + } + if got := common.IsRetryableTowardNetwork(err); got != tc.wantRetryableNw { + t.Fatalf("IsRetryableTowardNetwork: got %v, want %v", got, tc.wantRetryableNw) + } + jre := &common.ErrJsonRpcExceptionInternal{} + if !errors.As(err, &jre) { + t.Fatalf("expected ErrJsonRpcExceptionInternal in chain, got %T", err) + } + if got := jre.NormalizedCode(); got != tc.wantNormalized { + t.Fatalf("normalized code: got %d, want %d", got, tc.wantNormalized) + } + if got := jre.Message; got != tc.message { + t.Fatalf("node message must reach the caller verbatim: got %q, want %q", got, tc.message) + } + }) + } +} + +func TestExtractJsonRpcError_TooManySubscriptions_IsCapacityExceeded(t *testing.T) { + t.Parallel() + + r := &http.Response{StatusCode: 200, Header: http.Header{}} + jrErr := common.NewErrJsonRpcExceptionExternal( + -32006, + "Too many subscriptions on the connection", + "Exceeded max limit of 32", + ) + jr := common.MustNewJsonRpcResponse(1, nil, jrErr) + + err := ExtractJsonRpcError(r, nil, jr, nil) + if err == nil { + t.Fatalf("expected error, got nil") + } + if !common.HasErrorCode(err, common.ErrCodeEndpointCapacityExceeded) { + t.Fatalf("expected ErrEndpointCapacityExceeded, got %T: %v", err, err) + } +} diff --git a/architecture/evm/eth_blockNumber.go b/architecture/evm/eth_blockNumber.go index 68c067efb..3f39b705d 100644 --- a/architecture/evm/eth_blockNumber.go +++ b/architecture/evm/eth_blockNumber.go @@ -10,25 +10,24 @@ import ( "go.opentelemetry.io/otel/trace" ) -// networkPreForward_eth_blockNumber answers eth_blockNumber from the served tip -// without dispatching, in majority served-tip mode only. There the post-forward -// hook below pins every response to the tip EXACTLY, so the upstream's answer is -// always discarded: the call buys nothing but vendor cost and latency. The -// default (floor-only) mode keeps forwarding, because a fresher upstream head -// passes through there. An unknown tip (cold pollers) and enforce-highest-block -// =false also keep forwarding. +// networkPreForward_eth_blockNumber answers eth_blockNumber from the served +// head without dispatching whenever enforce-highest-block is on and the head is +// known: the post-forward hook below pins every response to that head EXACTLY, +// so an upstream's answer is always discarded and the call buys nothing but +// vendor cost and latency. Each answer also asks the ballot's pollers for a +// debounced background refresh, so on a quiet network the head a client polls +// for is not held at the ticker interval. An unknown head (cold pollers) and +// enforce-highest-block=false keep forwarding. func networkPreForward_eth_blockNumber(ctx context.Context, network common.Network, nq *common.NormalizedRequest) (handled bool, resp *common.NormalizedResponse, err error) { - cfg := network.Config() - if cfg == nil || !cfg.Evm.ServedTipEnabledFor("latest") { - return false, nil, nil - } if dirs := nq.Directives(); dirs == nil || !dirs.EnforceHighestBlock { return false, nil, nil } - tip := common.EvmHighestLatestBlockNumber(network, context.WithValue(ctx, common.RequestContextKey, nq)) + ctx = context.WithValue(ctx, common.RequestContextKey, nq) + tip := common.EvmHighestLatestBlockNumber(network, ctx) if tip <= 0 { return false, nil, nil } + common.EvmRefreshLatestBlock(network, ctx) hex, err := common.NormalizeHex(tip) if err != nil { return true, nil, err @@ -40,28 +39,18 @@ func networkPreForward_eth_blockNumber(ctx context.Context, network common.Netwo return true, common.NewNormalizedResponse().WithRequest(nq).WithJsonRpcResponse(jrr), nil } -// networkPostForward_eth_blockNumber enforces the network's served tip on -// every eth_blockNumber response regardless of its source — a live upstream OR -// the cache. The correction is a pure in-memory synthesis (poller state), -// never an extra upstream call. -// -// In the default mode the served tip is the corroborated head across upstreams (second-highest, see common.ServedTipPick.Freshest), so -// the enforcement is floor-only: a response behind the tip is raised to it and -// a response at/above it is fresher truth that passes through. In majority -// served-tip mode (EvmServedTipConfig.EnabledFor "latest") the response is -// pinned to the tip EXACTLY — above-tip responses are capped too — because -// "latest" interpolation (resolveBlockTagToHex) anchors block-tagged methods -// (eth_call, eth_getLogs, ...) to that same majority tip: letting a fresher -// head, or a cache entry written from one, through the floor-only check makes -// clients observe eth_blockNumber AHEAD of the block "latest" actually -// executes at, even when the same upstream serves both calls. +// networkPostForward_eth_blockNumber pins every eth_blockNumber response to the +// network's served head EXACTLY, regardless of its source — a live upstream +// (cold head at pre-forward time, or a per-request override path) OR the +// cache. The correction is a pure in-memory synthesis (poller state), never an +// extra upstream call. // -// Positioning this at the network post-forward layer (it used to be a project -// pre-forward wrapper that explicitly skipped FromCache responses) is what -// guarantees cache hits are corrected too: a stale value planted in the -// (possibly shared) cache must not be served below the tip this instance -// already advertises, otherwise clients observe the block number moving -// backwards by the full upstream lag for an entire TTL window. +// Exact, not floor-only, because "latest" interpolation (resolveBlockTagToHex) +// anchors block-tagged methods (eth_call, eth_getLogs, ...) to that same served +// head: a fresher upstream answer, or a cache entry written from one, would +// show clients an eth_blockNumber AHEAD of the block "latest" actually executes +// at; a staler one (e.g. a value planted in a shared cache) would show the +// block number moving backwards for an entire TTL window. // // Gating on the request directives (instead of the deprecated // evm.integrity config) matches eth_getBlockByNumber enforcement: the @@ -113,23 +102,13 @@ func networkPostForward_eth_blockNumber(ctx context.Context, network common.Netw return nr, re } - pinToTip := false - if cfg := network.Config(); cfg != nil && cfg.Evm != nil { - pinToTip = cfg.Evm.ServedTipEnabledFor("latest") - } - if pinToTip { - if blockNumber == highestBlock { - return nr, re - } - } else if highestBlock <= blockNumber { + if blockNumber == highestBlock { return nr, re } - ups := nr.Upstream() if blockNumber > highestBlock { - // Pin-down (served-tip mode only): the response is AHEAD of the - // majority tip — freshness, not staleness, so the stale-block metric - // stays out of it. + // Pin-down: the response is AHEAD of the served head — freshness, not + // staleness, so the stale-block metric stays out of it. evt := network.Logger().Debug(). Str("method", "eth_blockNumber"). Int64("servedTip", highestBlock). @@ -138,7 +117,7 @@ func networkPostForward_eth_blockNumber(ctx context.Context, network common.Netw if ups != nil { evt = evt.Str("upstreamId", ups.Id()) } - evt.Msg("response is ahead of the served tip, pinning to the majority tip") + evt.Msg("response is ahead of the served tip, pinning to the served tip") } else if ups != nil { telemetry.MetricUpstreamStaleLatestBlock.WithLabelValues( network.ProjectId(), diff --git a/architecture/evm/eth_blockNumber_test.go b/architecture/evm/eth_blockNumber_test.go index 748335cf9..91e1c6fad 100644 --- a/architecture/evm/eth_blockNumber_test.go +++ b/architecture/evm/eth_blockNumber_test.go @@ -9,27 +9,17 @@ import ( "github.com/stretchr/testify/require" ) -// Decision matrix for networkPostForward_eth_blockNumber across the two -// served-tip modes (common.EvmServedTipConfig): -// -// default mode: floor only — below-tip responses are raised, at/above -// pass through (a response above the last-computed head is fresher truth). -// majority mode ("latest" in servedTip.enabledFor): pinned EXACTLY — above-tip -// responses are capped too, so eth_blockNumber can never run ahead of the -// majority tip that "latest" interpolation anchors eth_call & friends to. -// -// Both modes fail open on an unknown tip (0) and honor the per-request -// enforce-highest-block directive. +// networkPostForward_eth_blockNumber pins every response EXACTLY to the served +// head in every config: below-head responses are raised, above-head ones are +// capped, so eth_blockNumber can never run ahead of the head "latest" +// interpolation anchors eth_call & friends to. It fails open on an unknown +// head (0) and honors the per-request enforce-highest-block directive. func TestNetworkPostForward_EthBlockNumber(t *testing.T) { - netWith := func(servedTipLatest bool, tip int64) *testNetwork { - evmCfg := &common.EvmNetworkConfig{ChainId: 123} - if servedTipLatest { - evmCfg.ServedTip = &common.EvmServedTipConfig{EnabledFor: []string{"latest"}} - } + netWith := func(tip int64) *testNetwork { return &testNetwork{ cfg: &common.NetworkConfig{ Architecture: common.ArchitectureEvm, - Evm: evmCfg, + Evm: &common.EvmNetworkConfig{ChainId: 123}, }, highestLatest: tip, } @@ -53,57 +43,41 @@ func TestNetworkPostForward_EthBlockNumber(t *testing.T) { return out, bn } - t.Run("MaxMode_BelowTipIsRaisedToTip", func(t *testing.T) { - _, bn := run(t, netWith(false, 0x1000), true, "0x800", false) - assert.Equal(t, int64(0x1000), bn) - }) - - t.Run("MaxMode_AboveTipPassesThrough", func(t *testing.T) { - _, bn := run(t, netWith(false, 0x1000), true, "0x1200", false) - assert.Equal(t, int64(0x1200), bn, - "in default mode a response above the last-computed head is fresher truth and must never be capped") - }) - - t.Run("MaxMode_UnknownTipFailsOpen", func(t *testing.T) { - _, bn := run(t, netWith(false, 0), true, "0x800", false) - assert.Equal(t, int64(0x800), bn) - }) - - t.Run("ServedTip_BelowTipIsRaisedToTip", func(t *testing.T) { - _, bn := run(t, netWith(true, 0x1000), true, "0x800", false) + t.Run("BelowTipIsRaisedToTip", func(t *testing.T) { + _, bn := run(t, netWith(0x1000), true, "0x800", false) assert.Equal(t, int64(0x1000), bn) }) - t.Run("ServedTip_AtTipPassesThrough", func(t *testing.T) { - _, bn := run(t, netWith(true, 0x1000), true, "0x1000", false) + t.Run("AtTipPassesThrough", func(t *testing.T) { + _, bn := run(t, netWith(0x1000), true, "0x1000", false) assert.Equal(t, int64(0x1000), bn) }) - t.Run("ServedTip_AboveTipIsPinnedToTip", func(t *testing.T) { - // The inversion bug: a fresher head passing the floor-only check lets - // eth_blockNumber run ahead of the majority tip that "latest" + t.Run("AboveTipIsPinnedToTip", func(t *testing.T) { + // The inversion bug: a fresher head passing a floor-only check lets + // eth_blockNumber run ahead of the served head that "latest" // interpolation anchors eth_call to (on-chain block.number reads then // come back BELOW eth_blockNumber, even via the same upstream). - _, bn := run(t, netWith(true, 0x1000), true, "0x1002", false) + _, bn := run(t, netWith(0x1000), true, "0x1002", false) assert.Equal(t, int64(0x1000), bn, - "served-tip mode must cap eth_blockNumber to the majority tip") + "eth_blockNumber must be capped to the served head") }) - t.Run("ServedTip_AboveTipCacheHitIsPinnedAndKeepsCacheAttribution", func(t *testing.T) { - out, bn := run(t, netWith(true, 0x1000), true, "0x1002", true) + t.Run("AboveTipCacheHitIsPinnedAndKeepsCacheAttribution", func(t *testing.T) { + out, bn := run(t, netWith(0x1000), true, "0x1002", true) assert.Equal(t, int64(0x1000), bn) assert.True(t, out.FromCache(), "the pinned response was synthesized from a cache hit and must keep cache attribution") }) - t.Run("ServedTip_UnknownTipFailsOpen", func(t *testing.T) { - _, bn := run(t, netWith(true, 0), true, "0x1002", false) + t.Run("UnknownTipFailsOpen", func(t *testing.T) { + _, bn := run(t, netWith(0), true, "0x1002", false) assert.Equal(t, int64(0x1002), bn, "an unknown tip must never pin the response down to a synthesized 0") }) - t.Run("ServedTip_EnforcementDisabledServesRaw", func(t *testing.T) { - _, bn := run(t, netWith(true, 0x1000), false, "0x1002", false) + t.Run("EnforcementDisabledServesRaw", func(t *testing.T) { + _, bn := run(t, netWith(0x1000), false, "0x1002", false) assert.Equal(t, int64(0x1002), bn) }) } diff --git a/architecture/evm/eth_chainId.go b/architecture/evm/eth_chainId.go deleted file mode 100644 index e3909a2c5..000000000 --- a/architecture/evm/eth_chainId.go +++ /dev/null @@ -1,178 +0,0 @@ -package evm - -import ( - "context" - "fmt" - "strconv" - "strings" - - "github.com/erpc/erpc/common" - "github.com/erpc/erpc/util" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -func BuildEthChainIdRequest() (*common.JsonRpcRequest, error) { - jrq := common.NewJsonRpcRequest("eth_chainId", []interface{}{}) - err := jrq.SetID(util.RandomID()) - if err != nil { - return nil, err - } - - return jrq, nil -} - -func projectPreForward_eth_chainId(ctx context.Context, n common.Network, r *common.NormalizedRequest) (handled bool, resp *common.NormalizedResponse, err error) { - if r.ShouldSkipCacheRead("") { - return false, nil, nil // Proceed to actual upstream call - } - - ctx, span := common.StartDetailSpan(ctx, "Project.PreForwardHook.eth_chainId", trace.WithAttributes( - attribute.String("request.id", fmt.Sprintf("%v", r.ID())), - attribute.String("network.id", n.Id()), - )) - defer span.End() - - netCfg := n.Config() - if netCfg == nil || netCfg.Evm == nil || netCfg.Evm.ChainId == 0 { - span.SetAttributes(attribute.Bool("skipped.no_chain_id_config", true)) - return false, nil, nil - } - - chainId := netCfg.Evm.ChainId - hexChainId, err := common.NormalizeHex(chainId) - if err != nil { - span.SetAttributes(attribute.String("error.normalize_hex", err.Error())) - return true, nil, fmt.Errorf("failed to normalize chainId %d to hex: %w", chainId, err) - } - - jrqID := r.ID() - // If original request ID is nil (e.g. for some internal requests), generate one. - if jrqID == nil { - jrqID = util.RandomID() - } - - jrr, err := common.NewJsonRpcResponse(jrqID, hexChainId, nil) - if err != nil { - span.SetAttributes(attribute.String("error.new_jsonrpc_response", err.Error())) - return true, nil, fmt.Errorf("failed to create JsonRpcResponse: %w", err) - } - - nr := common.NewNormalizedResponse(). - WithRequest(r). - WithJsonRpcResponse(jrr) - - span.SetAttributes(attribute.Bool("handled_by_hook", true), attribute.String("chain_id_from_config", hexChainId)) - return true, nr, nil -} - -// networkPreForward_eth_chainId short-circuits eth_chainId at network level using the configured chainId -// when skip-cache-read is not requested. This applies after upstream selection but before sending. -func networkPreForward_eth_chainId(ctx context.Context, n common.Network, _ []common.Upstream, r *common.NormalizedRequest) (handled bool, resp *common.NormalizedResponse, err error) { - if r == nil || n == nil { - return false, nil, nil - } - if r.ShouldSkipCacheRead("") { - return false, nil, nil - } - - ctx, span := common.StartDetailSpan(ctx, "Network.PreForwardHook.eth_chainId", trace.WithAttributes( - attribute.String("request.id", fmt.Sprintf("%v", r.ID())), - attribute.String("network.id", n.Id()), - )) - defer span.End() - - netCfg := n.Config() - if netCfg == nil || netCfg.Evm == nil || netCfg.Evm.ChainId == 0 { - span.SetAttributes(attribute.Bool("skipped.no_chain_id_config", true)) - return false, nil, nil - } - - hexChainId, err := common.NormalizeHex(netCfg.Evm.ChainId) - if err != nil { - span.SetAttributes(attribute.String("error.normalize_hex", err.Error())) - return true, nil, fmt.Errorf("failed to normalize chainId %d to hex: %w", netCfg.Evm.ChainId, err) - } - - jrqID := r.ID() - if jrqID == nil { - jrqID = util.RandomID() - } - jrr, err := common.NewJsonRpcResponse(jrqID, hexChainId, nil) - if err != nil { - span.SetAttributes(attribute.String("error.new_jsonrpc_response", err.Error())) - return true, nil, fmt.Errorf("failed to create JsonRpcResponse: %w", err) - } - - nr := common.NewNormalizedResponse(). - WithRequest(r). - WithJsonRpcResponse(jrr) - span.SetAttributes(attribute.Bool("handled_by_hook", true), attribute.String("chain_id_from_config", hexChainId)) - return true, nr, nil -} - -// upstreamPreForward_eth_chainId short-circuits eth_chainId at upstream level using upstream or network config. -// Respects skip-cache-read directive. -func upstreamPreForward_eth_chainId(ctx context.Context, n common.Network, u common.Upstream, r *common.NormalizedRequest) (handled bool, resp *common.NormalizedResponse, err error) { - if r == nil || n == nil || u == nil { - return false, nil, nil - } - if r.ShouldSkipCacheRead("") { - return false, nil, nil - } - - ctx, span := common.StartDetailSpan(ctx, "Upstream.PreForwardHook.eth_chainId", trace.WithAttributes( - attribute.String("request.id", fmt.Sprintf("%v", r.ID())), - attribute.String("network.id", n.Id()), - attribute.String("upstream.id", u.Id()), - )) - defer span.End() - - var chainId int64 - // Prefer upstream's resolved config (detectFeatures fills this when possible) - if uc := u.Config(); uc != nil && uc.Evm != nil && uc.Evm.ChainId > 0 { - chainId = uc.Evm.ChainId - } - // If still empty, try parsing from upstream NetworkId, e.g. "evm:123" - if chainId == 0 { - if nid := u.NetworkId(); strings.HasPrefix(nid, "evm:") { - if p := strings.TrimPrefix(nid, "evm:"); p != "" { - if v, perr := strconv.ParseInt(p, 10, 64); perr == nil && v > 0 { - chainId = v - } - } - } - } - // Finally, fallback to network-level config - if chainId == 0 { - if nc := n.Config(); nc != nil && nc.Evm != nil && nc.Evm.ChainId > 0 { - chainId = nc.Evm.ChainId - } - } - if chainId == 0 { - span.SetAttributes(attribute.Bool("skipped.no_chain_id_config", true)) - return false, nil, nil - } - - hexChainId, err := common.NormalizeHex(chainId) - if err != nil { - span.SetAttributes(attribute.String("error.normalize_hex", err.Error())) - return true, nil, fmt.Errorf("failed to normalize chainId %d to hex: %w", chainId, err) - } - - jrqID := r.ID() - if jrqID == nil { - jrqID = util.RandomID() - } - jrr, err := common.NewJsonRpcResponse(jrqID, hexChainId, nil) - if err != nil { - span.SetAttributes(attribute.String("error.new_jsonrpc_response", err.Error())) - return true, nil, fmt.Errorf("failed to create JsonRpcResponse: %w", err) - } - - nr := common.NewNormalizedResponse(). - WithRequest(r). - WithJsonRpcResponse(jrr) - span.SetAttributes(attribute.Bool("handled_by_hook", true), attribute.String("chain_id_from_config", hexChainId)) - return true, nr, nil -} diff --git a/architecture/evm/eth_getBlockByNumber_test.go b/architecture/evm/eth_getBlockByNumber_test.go index 97dfe223c..d82971468 100644 --- a/architecture/evm/eth_getBlockByNumber_test.go +++ b/architecture/evm/eth_getBlockByNumber_test.go @@ -65,6 +65,8 @@ func (t *testNetwork) EvmLeaderUpstream(ctx context.Context) common.Upstream { return nil } +func (t *testNetwork) EvmRefreshLatestBlock(ctx context.Context) {} + func (t *testNetwork) GetMethodMetrics(method string) common.TrackedMetrics { return nil } diff --git a/architecture/evm/eth_getLogs_test.go b/architecture/evm/eth_getLogs_test.go index e686e09e5..e77e9091b 100644 --- a/architecture/evm/eth_getLogs_test.go +++ b/architecture/evm/eth_getLogs_test.go @@ -116,6 +116,8 @@ func (m *mockNetwork) EvmLeaderUpstream(ctx context.Context) common.Upstream { return nil } +func (m *mockNetwork) EvmRefreshLatestBlock(ctx context.Context) {} + var _ common.EvmUpstream = (*mockEvmUpstream)(nil) type mockEvmUpstream struct { diff --git a/architecture/evm/evm_state_poller.go b/architecture/evm/evm_state_poller.go index 976370b7f..1f0450918 100644 --- a/architecture/evm/evm_state_poller.go +++ b/architecture/evm/evm_state_poller.go @@ -37,7 +37,7 @@ const ( // represent before it is treated as major and re-verified against a live // eth_chainId (see majorHeadMoveThreshold). One minute is comfortably more // chain than a healthy poller can miss between samples — the default state - // poller interval is 5s — while staying far below the height difference any + // poller interval is 30s — while staying far below the height difference any // two unrelated chains exhibit. chainIdVerifyChainProgress = 60 * time.Second @@ -121,6 +121,9 @@ type EvmStatePoller struct { // keep-fresh advances never touch this. latestMajorVerifyInProgress sync.Mutex + // Single in-flight flag for RefreshLatestBlock's background poll. + latestRefreshInFlight atomic.Bool + // Earliest per probe tracking earliestByProbe map[common.EvmAvailabilityProbeType]data.CounterInt64SharedVariable earliestSchedulerStarted map[common.EvmAvailabilityProbeType]bool @@ -446,6 +449,30 @@ func (e *EvmStatePoller) resolveDebounce(cfg *common.EvmNetworkConfig) time.Dura return 1 * time.Second } +// resolveFinalizedDebounce widens the debounce for finalized polls to a fraction of the +// chain's finality depth in time. Finalized moves once per finality window, so a poll at +// the latest-block cadence mostly returns the number it already has; this bounds how far +// the finalized block can trail to that fraction of the window instead. +// +// An upstream behind the network's corroborated finalized block keeps the regular +// debounce: peers already saw the next finalized block, and a vendor whose load balancer +// mixes stale and fresh backends needs every sample to catch up. Response finality uses +// the serving upstream's own finalized block, and cache hits the lowest across upstreams. +func (e *EvmStatePoller) resolveFinalizedDebounce(cfg *common.EvmNetworkConfig, debounce time.Duration) time.Duration { + if cfg == nil || cfg.FinalizedDebounceDepthMultiplier == nil { + return debounce + } + mult := *cfg.FinalizedDebounceDepthMultiplier + networkId := e.upstream.NetworkId() + finalized := e.finalizedBlockShared.GetValue() + depth := e.latestBlockShared.GetValue() - finalized + blockTime := e.tracker.GetNetworkBlockTime(networkId) + if mult <= 0 || finalized <= 0 || depth <= 0 || blockTime <= 0 || finalized < e.tracker.GetNetworkFinalizedBlock(networkId) { + return debounce + } + return max(debounce, time.Duration(float64(depth)*float64(blockTime)*mult)) +} + // PollLatestBlockNumber fetches the latest block number in a blocking manner. // Respects the debounce interval if configured (if the last poll happened too recently, it reuses the cached value). func (e *EvmStatePoller) PollLatestBlockNumber(ctx context.Context) (int64, error) { @@ -538,6 +565,33 @@ func (e *EvmStatePoller) PollLatestBlockNumber(ctx context.Context) (int64, erro }) } +// RefreshLatestBlock starts a background latest-block poll when the stored +// head is older than the poll debounce and no refresh is already in flight, +// so a head read on the request path does not wait for the next ticker poll. +// It never blocks and spawns at most one goroutine per poller at a time. +func (e *EvmStatePoller) RefreshLatestBlock() { + if e.IsObjectNull() || e.latestBlockShared == nil { + return + } + e.stateMu.RLock() + cfg, skip := e.cfg, e.skipLatestBlockCheck + e.stateMu.RUnlock() + if skip || !e.latestBlockShared.IsStale(e.resolveDebounce(cfg)) { + return + } + if !e.latestRefreshInFlight.CompareAndSwap(false, true) { + return + } + go func() { + defer e.latestRefreshInFlight.Store(false) + ctx, cancel := context.WithTimeout(e.appCtx, 10*time.Second) + defer cancel() + if _, err := e.PollLatestBlockNumber(ctx); err != nil { + e.logger.Debug().Err(err).Msg("on-demand latest block refresh failed in evm state poller") + } + }() +} + func (e *EvmStatePoller) SuggestLatestBlock(blockNumber int64) { // Best-effort, non-blocking update. // @@ -766,7 +820,7 @@ func (e *EvmStatePoller) PollFinalizedBlockNumber(ctx context.Context) (int64, e networkLabel := e.networkLabel e.stateMu.RUnlock() - dbi := e.resolveDebounce(cfg) + dbi := e.resolveFinalizedDebounce(cfg, e.resolveDebounce(cfg)) return e.finalizedBlockShared.TryUpdateIfStale(ctx, dbi, func(ctx context.Context) (int64, error) { e.logger.Trace().Msg("fetching finalized block number for evm state poller") diff --git a/architecture/evm/evm_state_poller_suggest_gate_test.go b/architecture/evm/evm_state_poller_suggest_gate_test.go index 50457d660..5d3a5830f 100644 --- a/architecture/evm/evm_state_poller_suggest_gate_test.go +++ b/architecture/evm/evm_state_poller_suggest_gate_test.go @@ -222,8 +222,11 @@ func TestSuggestFinalizedBlock_MajorJumpMatchingApplies(t *testing.T) { p.SuggestFinalizedBlock(1000) require.Eventually(t, func() bool { return p.FinalizedBlock() == 1000 }, 2*time.Second, 10*time.Millisecond) - p.SuggestFinalizedBlock(1000 + gateTolerance + 1000) // MAJOR + // The first suggestion's goroutine publishes the value before it releases + // the in-progress lock, so a single follow-up suggestion can be skipped. + // Re-suggest until it is taken. require.Eventually(t, func() bool { + p.SuggestFinalizedBlock(1000 + gateTolerance + 1000) // MAJOR return p.FinalizedBlock() == 1000+gateTolerance+1000 }, 2*time.Second, 10*time.Millisecond, "verified major finalized jump must be applied") require.False(t, up.isCordoned()) @@ -236,8 +239,11 @@ func TestSuggestFinalizedBlock_MajorJumpChainIdMismatchDroppedAndCordoned(t *tes p.SuggestFinalizedBlock(1000) require.Eventually(t, func() bool { return p.FinalizedBlock() == 1000 }, 2*time.Second, 10*time.Millisecond) - p.SuggestFinalizedBlock(5_000_000) // MAJOR wrong-chain height - require.Eventually(t, up.isCordoned, 2*time.Second, 10*time.Millisecond) + // Re-suggest until taken: see TestSuggestFinalizedBlock_MajorJumpMatchingApplies. + require.Eventually(t, func() bool { + p.SuggestFinalizedBlock(5_000_000) // MAJOR wrong-chain height + return up.isCordoned() + }, 2*time.Second, 10*time.Millisecond) assert.Equal(t, int64(1000), p.FinalizedBlock(), "bogus major finalized jump must not enter the shared counter") } diff --git a/architecture/evm/hooks.go b/architecture/evm/hooks.go index ef3630a6e..69b2945be 100644 --- a/architecture/evm/hooks.go +++ b/architecture/evm/hooks.go @@ -30,11 +30,13 @@ func HandleProjectPreForward(ctx context.Context, network common.Network, nq *co return false, nil, err } + if handled, resp, err := projectPreForward_localAnswer(ctx, network, nq, method); handled { + return true, resp, err + } + switch strings.ToLower(method) { case "eth_call", "eth_estimategas": return projectPreForward_defaultLatestBlock(ctx, network, nq) - case "eth_chainid": - return projectPreForward_eth_chainId(ctx, network, nq) case "eth_getlogs": return projectPreForward_eth_getLogs(ctx, network, nq) case "trace_filter", "arbtrace_filter": @@ -58,8 +60,6 @@ func HandleNetworkPreForward(ctx context.Context, network common.Network, upstre switch strings.ToLower(method) { case "eth_getlogs": return networkPreForward_eth_getLogs(ctx, network, upstreams, nq) - case "eth_chainid": - return networkPreForward_eth_chainId(ctx, network, upstreams, nq) case "eth_blocknumber": return networkPreForward_eth_blockNumber(ctx, network, nq) case "trace_filter", "arbtrace_filter": @@ -109,8 +109,8 @@ func HandleUpstreamPreForward(ctx context.Context, n common.Network, u common.Up switch methodLower { case "eth_getlogs": return upstreamPreForward_eth_getLogs(ctx, n, u, r) - case "eth_chainid": - return upstreamPreForward_eth_chainId(ctx, n, u, r) + case "eth_call": + return upstreamPreForward_eth_call(ctx, n, u, r) case "trace_filter", "arbtrace_filter": return upstreamPreForward_trace_filter(ctx, n, u, r) default: @@ -143,6 +143,9 @@ func HandleUpstreamPostForward(ctx context.Context, n common.Network, u common.U if methodLower == "eth_sendrawtransaction" { return upstreamPostForward_eth_sendRawTransaction(ctx, n, u, rq, rs, re) } + if methodLower == "eth_call" { + upstreamPostForward_eth_call(u, rq, rs, re) + } // For all other methods, skip validation if there's already an error if re != nil { @@ -221,7 +224,7 @@ func HandleUpstreamPostForward(ctx context.Context, n common.Network, u common.U } span.End() // Per-check attempts/outcomes (pass/skip/reject/reject_recoverable/record_only/reconfirmed/ - // off) — sum = total attempts. "pass" means an actual verification ran; + // off/panic) — sum = total attempts. "pass" means an actual verification ran; // "skip" means the check couldn't evaluate (cold cache, missing wiring). // Higher volume than the violation counter below. for _, oc := range res.Outcomes { diff --git a/architecture/evm/integrity/checks_authenticity.go b/architecture/evm/integrity/checks_authenticity.go index 5e696fd7c..f681b2504 100644 --- a/architecture/evm/integrity/checks_authenticity.go +++ b/architecture/evm/integrity/checks_authenticity.go @@ -46,7 +46,17 @@ func init() { return nil } - recovered, err := gethtypes.Sender(gethtypes.LatestSignerForChainID(tx.ChainId()), &tx) + // A pre-EIP-155 legacy tx (v=27/28) carries no chain id: geth reports + // ChainId()=0, and its chain-id signers panic on a non-positive id. + signer := gethtypes.Signer(gethtypes.HomesteadSigner{}) + if tx.Protected() { + chainID := tx.ChainId() + if chainID == nil || chainID.Sign() <= 0 { + return Skipped + } + signer = gethtypes.LatestSignerForChainID(chainID) + } + recovered, err := gethtypes.Sender(signer, &tx) if err != nil { return nil } diff --git a/architecture/evm/integrity/checks_authenticity_test.go b/architecture/evm/integrity/checks_authenticity_test.go index a8c0d4348..7a99e1ab9 100644 --- a/architecture/evm/integrity/checks_authenticity_test.go +++ b/architecture/evm/integrity/checks_authenticity_test.go @@ -80,4 +80,19 @@ func TestCheck_SenderRecovery(t *testing.T) { nosig, _ := json.Marshal(m) assert.NoError(t, run(t, MethodGetTransactionByHash, nosig, cs)) }) + + // Real mainnet pre-EIP-155 legacy tx (v=27, no chain id). geth's chain-id + // signers panic on it; this used to crash the process. + legacy := []byte(`{"blockHash":"0xdaa690a16bc4a2bf2ff8955ff1cf3bc2155bc9c53f5731b6f8706fdc2b33b82c","blockNumber":"0xa2d2e3","from":"0x3fab184622dc19b6109349b94811493bf2a45362","gas":"0x186a0","gasPrice":"0x174876e800","hash":"0xeddf9e61fb9d8f5111840daef55e5fde0041f5702856532cdbb5a02998033d26","input":"0x604580600e600039806000f350fe7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf3","nonce":"0x0","to":null,"transactionIndex":"0x2a","value":"0x0","type":"0x0","v":"0x1b","r":"0x2222222222222222222222222222222222222222222222222222222222222222","s":"0x2222222222222222222222222222222222222222222222222222222222222222"}`) + + t.Run("pre-EIP-155 tx recovers its sender", func(t *testing.T) { + res := runRes(t, MethodGetTransactionByHash, legacy, cs) + require.NoError(t, res.Err) + assert.Equal(t, []CheckOutcome{{"senderRecovery", "pass"}}, res.Outcomes) + }) + + t.Run("pre-EIP-155 tx with wrong from is rejected", func(t *testing.T) { + bad := mutateField(t, legacy, "from", "0x0000000000000000000000000000000000000001") + require.Error(t, run(t, MethodGetTransactionByHash, bad, cs)) + }) } diff --git a/architecture/evm/integrity/cov_checks_test.go b/architecture/evm/integrity/cov_checks_test.go new file mode 100644 index 000000000..1367c63e3 --- /dev/null +++ b/architecture/evm/integrity/cov_checks_test.go @@ -0,0 +1,600 @@ +package integrity + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "strings" + "testing" + + "github.com/erpc/erpc/common" + gethcommon "github.com/ethereum/go-ethereum/common" + gethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// covChecksFind returns the registered check with the given id. +func covChecksFind(t *testing.T, id string) *Check { + t.Helper() + for _, c := range allChecks { + if c.ID == id { + return c + } + } + t.Fatalf("check %q is not registered", id) + return nil +} + +// covChecksRun runs one check directly against a crafted payload, wiring the +// optional history/resolver the way the engine does. +func covChecksRun(t *testing.T, id, method, raw string, params []any, hist History, r Resolver, cfg CheckConfig) *Violation { + t.Helper() + d := newDecoded(method, []byte(raw)) + d.reqParams = params + ctx := withHistory(withResolver(context.Background(), r), hist) + return covChecksFind(t, id).Run(ctx, d, cfg) +} + +func covChecksAssertFail(t *testing.T, v *Violation, contains string) { + t.Helper() + require.NotNil(t, v, "expected a violation") + require.NotSame(t, Skipped, v, "expected a violation, got skip") + assert.Contains(t, v.Reason, contains) +} + +// covChecksPanicResolver crashes when asked for a canonical header, so the +// engine's panic containment can be observed. +type covChecksPanicResolver struct{} + +func (covChecksPanicResolver) IsFinalized(ctx context.Context, bn int64) (bool, bool) { + return true, true +} +func (covChecksPanicResolver) CanonicalReceipts(ctx context.Context, ref string) ([]Receipt, bool) { + return nil, false +} +func (covChecksPanicResolver) CanonicalHeader(ctx context.Context, ref string) (*Header, bool) { + panic("resolver exploded") +} + +func TestCovChecks_IdentitySkipsAndRejects(t *testing.T) { + h := reqHash + cases := []struct { + name, id, method, raw string + params []any + hist History + want string // "skip", "pass" or a reason substring + }{ + {"byHash no params", "blockByHashIdentity", MethodGetBlockByHash, `{"hash":"` + h + `"}`, nil, nil, "skip"}, + {"byHash block without hash", "blockByHashIdentity", MethodGetBlockByHash, `{"number":"0x1"}`, []any{h}, nil, "skip"}, + {"byNumber no params", "blockByNumberIdentity", MethodGetBlockByNumber, `{"number":"0x1"}`, nil, nil, "skip"}, + {"byNumber overflowing param", "blockByNumberIdentity", MethodGetBlockByNumber, `{"number":"0x1"}`, []any{"0xffffffffffffffff"}, nil, "skip"}, + {"byNumber non-object result", "blockByNumberIdentity", MethodGetBlockByNumber, `"oops"`, []any{"0x1"}, nil, "skip"}, + {"byNumber unparseable response number", "blockByNumberIdentity", MethodGetBlockByNumber, `{"number":"0xzz"}`, []any{"0x1"}, nil, "skip"}, + {"byNumber mismatch", "blockByNumberIdentity", MethodGetBlockByNumber, `{"number":"0x2"}`, []any{"0x1"}, nil, "block 2 but block 1 was requested"}, + {"tx without hash", "txByHashIdentity", MethodGetTransactionByHash, `{"from":"0x1"}`, []any{h}, nil, "skip"}, + {"tx non-object", "txByHashIdentity", MethodGetTransactionByHash, `[1]`, []any{h}, nil, "skip"}, + {"receipt with non-hash param", "receiptIdentity", MethodGetTransactionReceipt, `{"transactionHash":"` + h + `"}`, []any{"0x10"}, nil, "skip"}, + {"receipt without tx hash", "receiptIdentity", MethodGetTransactionReceipt, `{"blockHash":"0xbb"}`, []any{h}, nil, "skip"}, + {"receipt wrong tx", "receiptIdentity", MethodGetTransactionReceipt, `{"transactionHash":"` + otherHash + `"}`, []any{h}, nil, "not the requested"}, + {"txPin no history", "txPinConsistency", MethodGetTransactionByHash, `{"hash":"` + h + `","blockHash":"0xbb","blockNumber":"0x10"}`, []any{h}, nil, "skip"}, + {"txPin bad number", "txPinConsistency", MethodGetTransactionByHash, `{"hash":"` + h + `","blockHash":"0xbb","blockNumber":"0xzz"}`, []any{h}, mockHistory{0x10: "0xbb"}, "skip"}, + {"txPin mismatch", "txPinConsistency", MethodGetTransactionByHash, `{"hash":"` + h + `","blockHash":"0xcc","blockNumber":"0x10"}`, []any{h}, mockHistory{0x10: "0xbb"}, "committed block is 0xbb"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + v := covChecksRun(t, tc.id, tc.method, tc.raw, tc.params, tc.hist, nil, CheckConfig{Enabled: true}) + switch tc.want { + case "skip": + assert.Same(t, Skipped, v) + case "pass": + assert.Nil(t, v) + default: + covChecksAssertFail(t, v, tc.want) + } + }) + } + + t.Run("txPin mismatch disputes the committed height", func(t *testing.T) { + v := covChecksRun(t, "txPinConsistency", MethodGetTransactionByHash, + `{"hash":"`+h+`","blockHash":"0xcc","blockNumber":"0x10"}`, []any{h}, mockHistory{0x10: "0xbb"}, nil, CheckConfig{}) + require.NotNil(t, v) + assert.Equal(t, int64(0x10), v.DisputedPin) + }) +} + +func TestCovChecks_RecomputeSkipsUnmodeledData(t *testing.T) { + cfg := CheckConfig{Enabled: true} + + t.Run("blockHashRecompute: no hash → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "blockHashRecompute", MethodGetBlockByNumber, `{"number":"0x1"}`, nil, nil, nil, cfg)) + }) + t.Run("blockHashRecompute: header missing required fields → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "blockHashRecompute", MethodGetBlockByNumber, `{"hash":"0xabc","number":"0x1"}`, nil, nil, nil, cfg)) + }) + + t.Run("transactionsRootRecompute: no root → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "transactionsRootRecompute", MethodGetBlockByNumber, `{"number":"0x1"}`, nil, nil, nil, cfg)) + }) + t.Run("transactionsRootRecompute: empty tx list → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "transactionsRootRecompute", MethodGetBlockByNumber, + `{"number":"0x1","transactionsRoot":"0x`+strings.Repeat("11", 32)+`","transactions":[]}`, nil, nil, nil, cfg)) + }) + t.Run("transactionsRootRecompute: undecodable tx object → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "transactionsRootRecompute", MethodGetBlockByNumber, + `{"number":"0x1","transactionsRoot":"0x`+strings.Repeat("11", 32)+`","transactions":[{"foo":1}]}`, nil, nil, nil, cfg)) + }) + t.Run("transactionsRootRecompute: tx hash does not recompute → skip", func(t *testing.T) { + raw, _, _ := signedTxJSON(t) + raw = mutateField(t, raw, "hash", otherHash) + body := `{"number":"0x1","transactionsRoot":"0x` + strings.Repeat("11", 32) + `","transactions":[` + string(raw) + `]}` + assert.Same(t, Skipped, covChecksRun(t, "transactionsRootRecompute", MethodGetBlockByNumber, body, nil, nil, nil, cfg)) + }) + + root := "0x" + strings.Repeat("ab", 32) + withRoot := mockResolver{header: &Header{ReceiptsRoot: root}} + t.Run("receiptsRootRecompute: result is not an array → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "receiptsRootRecompute", MethodGetBlockReceipts, `{"a":1}`, nil, nil, withRoot, cfg)) + }) + t.Run("receiptsRootRecompute: no block reference → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "receiptsRootRecompute", MethodGetBlockReceipts, `[{"status":"0x1"}]`, nil, nil, withRoot, cfg)) + }) + t.Run("receiptsRootRecompute: receipt missing required fields → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "receiptsRootRecompute", MethodGetBlockReceipts, `[{"blockHash":"0xbb"}]`, nil, nil, withRoot, cfg)) + }) +} + +func covChecksLog(idx, data string, topics ...string) Log { + return Log{Address: "0xaddr", Topics: topics, Data: data, BlockHash: "0xbb", BlockNumber: "0x10", TransactionHash: "0xt1", LogIndex: idx} +} + +func TestCovChecks_GetLogsCompletenessEdges(t *testing.T) { + cfg := CheckConfig{Enabled: true} + canon := map[string][]Receipt{"0xbb": {{BlockHash: "0xbb", Logs: []Log{covChecksLog("0x0", "0xd1", "0xa")}}}} + hist := histWithReceipts{pins: map[int64]string{0x10: "0xbb"}, receipts: canon} + respLog := `{"address":"0xaddr","topics":["0xa"],"data":"0xd1","blockNumber":"0x10","blockHash":"0xbb","transactionHash":"0xt1","logIndex":"0x0"}` + params := func(filter string) []any { + var p []any + require.NoError(t, common.SonicCfg.Unmarshal([]byte(`[`+filter+`]`), &p)) + return p + } + + t.Run("no filter params → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "getLogsCompleteness", MethodGetLogs, `[`+respLog+`]`, nil, hist, nil, cfg)) + }) + t.Run("removed and hashless logs are ignored; nothing else to compare → skip", func(t *testing.T) { + body := `[{"address":"0xaddr","removed":true,"blockHash":"0xbb","logIndex":"0x0"},{"address":"0xaddr","logIndex":"0x1"}]` + assert.Same(t, Skipped, covChecksRun(t, "getLogsCompleteness", MethodGetLogs, body, params(`{"address":"0xaddr"}`), hist, nil, cfg)) + }) + t.Run("non-concrete range: a warm block verifies → pass", func(t *testing.T) { + assert.Nil(t, covChecksRun(t, "getLogsCompleteness", MethodGetLogs, `[`+respLog+`]`, params(`{"address":"0xaddr"}`), hist, nil, cfg)) + }) + t.Run("concrete range but no resolver: present-block result stands", func(t *testing.T) { + assert.Nil(t, covChecksRun(t, "getLogsCompleteness", MethodGetLogs, `[`+respLog+`]`, + params(`{"address":"0xaddr","fromBlock":"0x10","toBlock":"0x11"}`), hist, nil, cfg)) + }) + t.Run("canonical log with malformed index is not comparable → skip", func(t *testing.T) { + bad := histWithReceipts{pins: hist.pins, receipts: map[string][]Receipt{"0xbb": {{Logs: []Log{covChecksLog("0xzz", "0xd1", "0xa")}}}}} + assert.Same(t, Skipped, covChecksRun(t, "getLogsCompleteness", MethodGetLogs, `[`+respLog+`]`, params(`{"address":"0xaddr"}`), bad, nil, cfg)) + }) + t.Run("absent finalized block whose receipts are cold → skip", func(t *testing.T) { + pinsOnly := histWithReceipts{pins: map[int64]string{0x10: "0xbb", 0x11: "0xcold"}, receipts: canon} + v := covChecksRun(t, "getLogsCompleteness", MethodGetLogs, `[`+respLog+`]`, + params(`{"address":"0xaddr","fromBlock":"0x10","toBlock":"0x11"}`), pinsOnly, mockResolver{finalized: true, known: true}, cfg) + assert.Nil(t, v, "the present block verified; the cold absent block is simply not judged") + }) + t.Run("range at the scan cap skips the absent sweep", func(t *testing.T) { + v := covChecksRun(t, "getLogsCompleteness", MethodGetLogs, `[]`, + params(fmt.Sprintf(`{"address":"0xaddr","fromBlock":"0x0","toBlock":"0x%x"}`, absentBlockScanCap)), hist, mockResolver{finalized: true, known: true}, cfg) + assert.Same(t, Skipped, v) + }) +} + +func TestCovChecks_CompareBlockLogs(t *testing.T) { + f := &LogsFilter{FromBlock: -1, ToBlock: -1, addresses: map[string]struct{}{}} + + t.Run("malformed response logIndex → skip", func(t *testing.T) { + got := covChecksLog("0xzz", "0xd1", "0xa") + v := compareBlockLogs(f, "0xbb", []Receipt{{Logs: []Log{covChecksLog("0x0", "0xd1", "0xa")}}}, []*Log{&got}) + assert.Same(t, Skipped, v) + }) + t.Run("differing topic value → reject", func(t *testing.T) { + got := covChecksLog("0x0", "0xd1", "0xa", "0xb") + v := compareBlockLogs(f, "0xbb", []Receipt{{Logs: []Log{covChecksLog("0x0", "0xd1", "0xa", "0xc")}}}, []*Log{&got}) + covChecksAssertFail(t, v, "topic 1 differs from canonical") + }) + t.Run("many missing logs are listed sorted and truncated", func(t *testing.T) { + var logs []Log + for i := 11; i >= 0; i-- { + logs = append(logs, covChecksLog(fmt.Sprintf("0x%x", i), "0xd", "0xa")) + } + got := covChecksLog("0x0", "0xd", "0xa") + v := compareBlockLogs(f, "0xbb", []Receipt{{Logs: logs}}, []*Log{&got}) + covChecksAssertFail(t, v, "missing 11 filter-matching canonical logs (logIndexes 1, 2, 3, 4, 5, 6, 7, 8, … +3 more)") + }) + t.Run("fmtIndexes", func(t *testing.T) { + assert.Equal(t, "", fmtIndexes(nil)) + assert.Equal(t, "7", fmtIndexes([]int64{7})) + assert.Equal(t, "1, 2", fmtIndexes([]int64{1, 2})) + }) +} + +func TestCovChecks_ReceiptStructuralChecks(t *testing.T) { + cfg := CheckConfig{Enabled: true} + addr := "0x27b26e88f007ec9109648c6da522fcaba06c74d7" + cases := []struct { + name, id, raw, want string + }{ + {"sameBlockHash ignores receipts without a hash", "sameBlockHash", `[{"blockHash":""},{"blockHash":"0xaa"},{"blockHash":"0xAA"}]`, ""}, + {"sameBlockHash mixed", "sameBlockHash", `[{"blockHash":"0xaa"},{"blockHash":""},{"blockHash":"0xbb"}]`, "receipts span multiple blocks"}, + {"txIndex absent is tolerated", "transactionIndexConsistency", `[{"transactionIndex":""},{"transactionIndex":"0x1"}]`, ""}, + {"txIndex invalid hex", "transactionIndexConsistency", `[{"transactionIndex":"0xzz"}]`, "invalid transactionIndex hex at receipt 0"}, + {"logMetadata blockNumber mismatch", "logMetadata", `[{"blockHash":"0xaa","blockNumber":"0x10","logs":[{"blockHash":"0xaa","blockNumber":"0x11"}]}]`, "blockNumber mismatch"}, + {"logMetadata unparseable blockNumber", "logMetadata", `[{"blockHash":"0xaa","blockNumber":"0x10","logs":[{"blockHash":"0xaa","blockNumber":"0xzz"}]}]`, "blockNumber mismatch"}, + {"logMetadata transactionIndex mismatch", "logMetadata", `[{"transactionHash":"0x1","transactionIndex":"0x0","logs":[{"transactionHash":"0x1","transactionIndex":"0x1"}]}]`, "transactionIndex mismatch"}, + {"bloomMatch invalid log address", "bloomMatch", `[{"logsBloom":"0x00","logs":[{"address":"0xzz"}]}]`, "invalid log field for bloom recompute"}, + {"bloomMatch invalid topic", "bloomMatch", `[{"logsBloom":"0x00","logs":[{"address":"` + addr + `","topics":["0xqq"]}]}]`, "invalid log field for bloom recompute"}, + {"bloomMatch invalid bloom hex", "bloomMatch", `[{"logsBloom":"0xzz","logs":[{"address":"` + addr + `"}]}]`, "invalid logsBloom hex"}, + {"logIndexContiguity invalid hex", "logIndexContiguity", `[{"logs":[{"logIndex":"zz"}]}]`, "invalid logIndex hex at log 0"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + v := covChecksRun(t, tc.id, MethodGetBlockReceipts, tc.raw, nil, nil, nil, cfg) + if tc.want == "" { + assert.Nil(t, v) + return + } + covChecksAssertFail(t, v, tc.want) + }) + } + + t.Run("indexMagnitude flags an underflowed log transactionIndex", func(t *testing.T) { + v := covChecksRun(t, "indexMagnitude", MethodGetLogs, `[{"logIndex":"0x1","transactionIndex":"0xfffffff7"}]`, nil, nil, nil, cfg) + covChecksAssertFail(t, v, `log transactionIndex "0xfffffff7" exceeds`) + }) +} + +func TestCovChecks_ConsecutiveSkips(t *testing.T) { + cfg := CheckConfig{Enabled: true} + segWith := func(parent *Header) History { + return &segmentHistory{from: 100, to: 101, headers: map[int64]*Header{100: parent}} + } + full := &Header{Number: "0x64", Hash: "0xparent", BaseFeePerGas: "0x3b9aca00", GasLimit: "0x1c9c380", GasUsed: "0xe4e1c0", Timestamp: "0x10"} + + cases := []struct { + name, id, raw string + hist History + }{ + {"baseFee: genesis number", "baseFeeDerivation", `{"number":"0x0","baseFeePerGas":"0x1"}`, segWith(full)}, + {"baseFee: parent without gas fields", "baseFeeDerivation", `{"number":"0x65","baseFeePerGas":"0x1"}`, segWith(&Header{Number: "0x64", BaseFeePerGas: "0x1"})}, + {"baseFee: parent zero gas limit", "baseFeeDerivation", `{"number":"0x65","baseFeePerGas":"0x1"}`, segWith(&Header{Number: "0x64", BaseFeePerGas: "0x1", GasLimit: "0x0", GasUsed: "0x0"})}, + {"baseFee: unparseable child fee", "baseFeeDerivation", `{"number":"0x65","baseFeePerGas":"0xzz"}`, segWith(full)}, + {"timestamp: absent", "timestampMonotonicity", `{"number":"0x65"}`, segWith(full)}, + {"timestamp: unparseable", "timestampMonotonicity", `{"number":"0x65","timestamp":"0xzz"}`, segWith(full)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, tc.id, MethodGetBlockByNumber, tc.raw, nil, tc.hist, nil, cfg)) + }) + } + + t.Run("fee model params override the mainnet defaults", func(t *testing.T) { + m := feeModelFrom(CheckConfig{Params: map[string]string{"elasticity": "6", "denominator": "250", "minBaseFee": "7"}}) + assert.Equal(t, EIP1559Model{Derivable: true, Elasticity: 6, Denominator: 250, MinBaseFee: 7}, m) + bad := feeModelFrom(CheckConfig{Params: map[string]string{"elasticity": "-1", "denominator": "x", "minBaseFee": "0"}}) + assert.Equal(t, EIP1559Model{Derivable: true, Elasticity: 2, Denominator: 8}, bad, "invalid params fall back") + }) + + t.Run("configured constants drive the verdict", func(t *testing.T) { + parent := &Header{Number: "0x64", Hash: "0xparent", BaseFeePerGas: "0x3b9aca00", GasLimit: "0x1c9c380", GasUsed: "0x1c9c380"} + opCfg := CheckConfig{Enabled: true, Params: map[string]string{"elasticity": "6", "denominator": "250"}} + want, ok := NextBaseFee(parent, feeModelFrom(opCfg)) + require.True(t, ok) + good := fmt.Sprintf(`{"number":"0x65","baseFeePerGas":"0x%x"}`, want) + assert.Nil(t, covChecksRun(t, "baseFeeDerivation", MethodGetBlockByNumber, good, nil, segWith(parent), nil, opCfg)) + v := covChecksRun(t, "baseFeeDerivation", MethodGetBlockByNumber, good, nil, segWith(parent), nil, cfg) + covChecksAssertFail(t, v, "elasticity 2 / denominator 8") + }) +} + +func TestCovChecks_NextBaseFeeEdges(t *testing.T) { + m := EIP1559Model{Derivable: true, Elasticity: 2, Denominator: 8} + t.Run("unparseable parent", func(t *testing.T) { + _, ok := NextBaseFee(&Header{BaseFeePerGas: "", GasLimit: "0x10", GasUsed: "0x1"}, m) + assert.False(t, ok) + _, ok = NextBaseFee(&Header{BaseFeePerGas: "0xzz", GasLimit: "0x10", GasUsed: "0x1"}, m) + assert.False(t, ok) + }) + t.Run("invalid model constants", func(t *testing.T) { + _, ok := NextBaseFee(&Header{BaseFeePerGas: "0x10", GasLimit: "0x10", GasUsed: "0x1"}, EIP1559Model{Elasticity: 0, Denominator: 8}) + assert.False(t, ok) + }) + t.Run("gas target rounds to zero", func(t *testing.T) { + _, ok := NextBaseFee(&Header{BaseFeePerGas: "0x10", GasLimit: "0x1", GasUsed: "0x1"}, m) + assert.False(t, ok) + }) + t.Run("over target always raises by at least one wei", func(t *testing.T) { + // target 50, used 51, fee 1: the proportional delta rounds to 0. + got, ok := NextBaseFee(&Header{BaseFeePerGas: "0x1", GasLimit: "0x64", GasUsed: "0x33"}, m) + require.True(t, ok) + assert.Equal(t, "2", got.String()) + }) + t.Run("empty parent with denominator 1 drains the fee to zero", func(t *testing.T) { + got, ok := NextBaseFee(&Header{BaseFeePerGas: "0x64", GasLimit: "0x64", GasUsed: "0x0"}, EIP1559Model{Elasticity: 2, Denominator: 1}) + require.True(t, ok) + assert.Equal(t, "0", got.String()) + }) + t.Run("hexToBig", func(t *testing.T) { + _, ok := hexToBig("0x") + assert.False(t, ok) + _, ok = hexToBig("0xg1") + assert.False(t, ok) + v, ok := hexToBig("0X1f") + require.True(t, ok) + assert.Equal(t, int64(31), v.Int64()) + }) +} + +func TestCovChecks_TraceEdges(t *testing.T) { + const n = int64(1000) + cfg := CheckConfig{Enabled: true} + + t.Run("followed segment without a header at the height → skip", func(t *testing.T) { + d := traceDecoded(traceResponse("0x10"), n) + ctx := withHistory(context.Background(), &followedHeaders{from: n - 1, to: n + 1, headers: map[int64]*Header{}}) + assert.Same(t, Skipped, runTraceBlockGasReconciliation(ctx, d, cfg)) + }) + t.Run("header gasUsed unparseable → skip", func(t *testing.T) { + d := traceDecoded(traceResponse("0x10"), n) + assert.Same(t, Skipped, runTraceBlockGasReconciliation(segmentAt(n, "0xzz", 1), d, cfg)) + }) + t.Run("frame gasUsed unparseable → skip", func(t *testing.T) { + d := traceDecoded(traceResponse("zz"), n) + assert.Same(t, Skipped, runTraceBlockGasReconciliation(segmentAt(n, "0x10", 1), d, cfg)) + }) + t.Run("non-callTracer entries are not decoded as block traces", func(t *testing.T) { + d := traceDecoded([]byte(`[{"txHash":"0x1","result":{"from":"0x1"}}]`), n) + _, ok := d.BlockTraces() + assert.False(t, ok) + assert.Same(t, Skipped, runTraceBlockGasReconciliation(segmentAt(n, "0x10", 1), d, cfg)) + }) + t.Run("an empty trace list has no frames to judge → skip", func(t *testing.T) { + d := traceDecoded([]byte(`[]`), n) + assert.Same(t, Skipped, runTraceFrameShape(context.Background(), d, cfg)) + }) + t.Run("frame without from → reject", func(t *testing.T) { + d := traceDecoded([]byte(`[{"txHash":"0x1","result":{"type":"CALL","from":"0x1","gasUsed":"0x1","calls":[{"type":"STATICCALL","gasUsed":"0x1"}]}}]`), n) + covChecksAssertFail(t, runTraceFrameShape(context.Background(), d, cfg), "STATICCALL frame at depth 1 has no from address") + }) + t.Run("a null frame is a violation", func(t *testing.T) { + frames := 0 + covChecksAssertFail(t, walkTraceFrame(nil, 0, &frames), "null call frame") + }) + t.Run("trees deeper than the bound stop descending without a verdict", func(t *testing.T) { + // A bad leaf below maxTraceDepth is never read. + leaf := CallFrame{Type: "CALL"} + for range maxTraceDepth + 2 { + leaf = CallFrame{Type: "CALL", From: "0x1", GasUsed: "0x0", Calls: []CallFrame{leaf}} + } + frames := 0 + assert.Nil(t, walkTraceFrame(&leaf, 0, &frames)) + assert.Equal(t, maxTraceDepth+1, frames) + }) + t.Run("oversized trees skip", func(t *testing.T) { + calls := make([]CallFrame, maxTraceFrames+1) + for i := range calls { + calls[i] = CallFrame{Type: "CALL", From: "0x1", GasUsed: "0x0"} + } + d := newDecoded(MethodTraceTransaction, nil) + d.callTraceDone = true + d.callTrace = &CallFrame{Type: "CALL", From: "0x1", GasUsed: "0x0", Calls: calls} + d.blockTracesDone = true + assert.Same(t, Skipped, runTraceFrameShape(context.Background(), d, cfg)) + }) +} + +func TestCovChecks_BlockAndContinuityEdges(t *testing.T) { + cfg := CheckConfig{Enabled: true} + for _, id := range []string{"transactionsRootConsistency", "headerFieldShapes", "txBlockInfo"} { + t.Run(id+": non-object result → no verdict", func(t *testing.T) { + assert.Nil(t, covChecksRun(t, id, MethodGetBlockByNumber, `"0xdead"`, nil, nil, nil, cfg)) + }) + } + t.Run("headerConsensusInvariants: non-object result → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "headerConsensusInvariants", MethodGetBlockByNumber, `"0xdead"`, nil, nil, nil, + CheckConfig{Enabled: true, Params: map[string]string{"zeroDifficulty": "true"}})) + }) + t.Run("headerConsensusInvariants: a bare 0x difficulty is not zero", func(t *testing.T) { + v := covChecksRun(t, "headerConsensusInvariants", MethodGetBlockByNumber, `{"number":"0x1","difficulty":"0x"}`, nil, nil, nil, + CheckConfig{Enabled: true, Params: map[string]string{"zeroDifficulty": "true"}}) + covChecksAssertFail(t, v, "difficulty 0x is non-zero") + }) + t.Run("txFieldUniqueness: hashless txs are ignored", func(t *testing.T) { + h := "0x" + strings.Repeat("aa", 32) + assert.Nil(t, covChecksRun(t, "txFieldUniqueness", MethodGetBlockByNumber, + `{"number":"0x1","transactions":[{"from":"0x1"},{"hash":"`+h+`"},{"from":"0x2"}]}`, nil, nil, nil, cfg)) + }) + t.Run("txBlockInfo: block number mismatch", func(t *testing.T) { + covChecksAssertFail(t, covChecksRun(t, "txBlockInfo", MethodGetBlockByNumber, + `{"hash":"0xaa","number":"0x10","transactions":[{"blockHash":"0xaa","blockNumber":"0x11"}]}`, nil, nil, nil, cfg), "tx 0: block number mismatch") + }) + t.Run("txBlockInfo: invalid transactionIndex", func(t *testing.T) { + covChecksAssertFail(t, covChecksRun(t, "txBlockInfo", MethodGetBlockByNumber, + `{"hash":"0xaa","number":"0x10","transactions":[{"blockHash":"0xaa","blockNumber":"0x10","transactionIndex":"0xzz"}]}`, nil, nil, nil, cfg), "invalid transactionIndex hex") + }) + t.Run("schemaConformance ignores methods it does not model", func(t *testing.T) { + assert.Nil(t, covChecksRun(t, "schemaConformance", MethodGetLogs, `{garbage`, nil, nil, nil, cfg)) + }) + + hist := mockHistory{0x10: "0xaaa", 0x11: "0xbbb"} + cases := []struct{ name, id, raw string }{ + {"linkage: no parentHash", "parentHashLinkage", `{"number":"0x11","hash":"0xbbb"}`}, + {"linkage: genesis", "parentHashLinkage", `{"number":"0x0","hash":"0xbbb","parentHash":"0xaaa"}`}, + {"stability: no hash", "hashStability", `{"number":"0x11","parentHash":"0xaaa"}`}, + {"stability: unparseable number", "hashStability", `{"number":"0xzz","hash":"0xbbb"}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, tc.id, MethodGetBlockByNumber, tc.raw, nil, hist, nil, cfg)) + }) + } +} + +func TestCovChecks_ReceiptVsBlockEdges(t *testing.T) { + cfg := CheckConfig{Enabled: true} + t.Run("non-object receipt → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "receiptVsBlock", MethodGetTransactionReceipt, `[1,2]`, nil, nil, mockResolver{}, cfg)) + }) + t.Run("pending receipt without block hash → skip", func(t *testing.T) { + assert.Same(t, Skipped, covChecksRun(t, "receiptVsBlock", MethodGetTransactionReceipt, `{"transactionHash":"0xaa"}`, nil, nil, mockResolver{}, cfg)) + }) + t.Run("log count differs from canonical → reject", func(t *testing.T) { + r := mockResolver{have: true, receipts: []Receipt{{TransactionHash: "0xaa", Logs: []Log{{LogIndex: "0x0"}, {LogIndex: "0x1"}}}}} + v := covChecksRun(t, "receiptVsBlock", MethodGetTransactionReceipt, + `{"blockHash":"0xbb","blockNumber":"0x10","transactionHash":"0xaa","logs":[{"logIndex":"0x0"}]}`, nil, nil, r, cfg) + covChecksAssertFail(t, v, "receipt has 1 logs but canonical block has 2") + }) +} + +func covChecksUnsignedTxJSON(t *testing.T, inner gethtypes.TxData, from string) string { + t.Helper() + b, err := gethtypes.NewTx(inner).MarshalJSON() + require.NoError(t, err) + var m map[string]json.RawMessage + require.NoError(t, json.Unmarshal(b, &m)) + fb, _ := json.Marshal(from) + m["from"] = fb + out, err := json.Marshal(m) + require.NoError(t, err) + return string(out) +} + +func TestCovChecks_SenderRecoveryEdges(t *testing.T) { + cfg := CheckConfig{Enabled: true} + to := gethcommon.HexToAddress("0x00000000000000000000000000000000000000aa") + from := "0x1111111111111111111111111111111111111111" + + t.Run("non-object result is not judged", func(t *testing.T) { + assert.Nil(t, covChecksRun(t, "senderRecovery", MethodGetTransactionByHash, `"0xdead"`, nil, nil, nil, cfg)) + }) + t.Run("protected tx with a zero chain id → skip", func(t *testing.T) { + raw := covChecksUnsignedTxJSON(t, &gethtypes.DynamicFeeTx{ + ChainID: big.NewInt(0), Nonce: 1, GasTipCap: big.NewInt(1), GasFeeCap: big.NewInt(2), Gas: 21000, To: &to, + Value: big.NewInt(1), V: big.NewInt(0), R: big.NewInt(1), S: big.NewInt(1), + }, from) + assert.Same(t, Skipped, covChecksRun(t, "senderRecovery", MethodGetTransactionByHash, raw, nil, nil, nil, cfg)) + }) + t.Run("unrecoverable signature is not judged", func(t *testing.T) { + // s above secp256k1n/2 is rejected by the signer: no sender to compare. + highS, _ := new(big.Int).SetString("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364140", 16) + raw := covChecksUnsignedTxJSON(t, &gethtypes.DynamicFeeTx{ + ChainID: big.NewInt(1), Nonce: 1, GasTipCap: big.NewInt(1), GasFeeCap: big.NewInt(2), Gas: 21000, To: &to, + Value: big.NewInt(1), V: big.NewInt(0), R: big.NewInt(1), S: highS, + }, from) + assert.Nil(t, covChecksRun(t, "senderRecovery", MethodGetTransactionByHash, raw, nil, nil, nil, cfg)) + }) +} + +func TestCovChecks_LogsFilterRejectsUnmodeledShapes(t *testing.T) { + for _, p := range []string{ + `"0xnotanobject"`, + `{"address":12}`, + `{"topics":[["0xa",1]]}`, + `{"topics":[12]}`, + `{"topics":"0xa"}`, + `{"topics":[null,null,null,null,null]}`, + } { + t.Run(p, func(t *testing.T) { + var params []any + require.NoError(t, common.SonicCfg.Unmarshal([]byte(`[`+p+`]`), ¶ms)) + assert.Nil(t, parseLogsFilter(params)) + }) + } + t.Run("toBlock below fromBlock is not a concrete range", func(t *testing.T) { + f := filt(t, `{"fromBlock":"0x12","toBlock":"0x10"}`) + require.NotNil(t, f) + _, _, ok := f.ConcreteRange() + assert.False(t, ok) + }) +} + +func TestCovChecks_FieldHelpers(t *testing.T) { + v := checkByteLen("block hash", "0xzz", hashLen) + covChecksAssertFail(t, v, "block hash: invalid hex") + assert.Nil(t, checkByteLen("block hash", "", hashLen)) + + _, err := bloomFromLogs([]Log{{Address: "0xnothex"}}) + assert.Error(t, err) + _, err = bloomFromLogs([]Log{{Topics: []string{"0xnothex"}}}) + assert.Error(t, err) + _, err = bloomsEqual("0xnothex", make([]byte, bloomLen)) + assert.Error(t, err) + eq, err := bloomsEqual("0x"+strings.Repeat("00", bloomLen), make([]byte, bloomLen)) + require.NoError(t, err) + assert.True(t, eq) + + assert.False(t, isZeroHex("0x")) + assert.True(t, isZeroHex("0x0000")) +} + +func TestCovChecks_DecodedBlockLocators(t *testing.T) { + cases := []struct { + name, method, raw string + ref string + number int64 + }{ + {"header hash", MethodGetBlockByNumber, `{"hash":"0xh","number":"0x10"}`, "0xh", 16}, + {"header with unparseable number", MethodGetBlockByNumber, `{"hash":"0xh","number":"0xzz"}`, "0xh", -1}, + {"receipt block hash", MethodGetBlockReceipts, `[{"blockHash":"0xr","blockNumber":"0x11"}]`, "0xr", 17}, + {"receipt block number only", MethodGetTransactionReceipt, `{"blockNumber":"0x12"}`, "0x12", 18}, + {"log block hash", MethodGetLogs, `[{"blockHash":"0xl","blockNumber":"0x13"}]`, "0xl", 19}, + {"log block number only", MethodGetLogs, `[{"blockNumber":"0x14"}]`, "0x14", 20}, + {"nothing to locate", MethodGetLogs, `[]`, "", -1}, + {"unrelated method", MethodTraceTransaction, `{"type":"CALL"}`, "", -1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := newDecoded(tc.method, []byte(tc.raw)) + assert.Equal(t, tc.ref, d.BlockRef()) + assert.Equal(t, tc.number, d.BlockNumber()) + }) + } + t.Run("block transactions of an undecodable block are empty", func(t *testing.T) { + d := newDecoded(MethodGetBlockByHash, []byte(`"0xdead"`)) + assert.Empty(t, d.Transactions()) + assert.Empty(t, d.Transactions(), "memoized") + }) +} + +func TestCovChecks_EngineContainment(t *testing.T) { + t.Run("a panicking check yields a panic outcome and no verdict", func(t *testing.T) { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockReceipts","params":["0x10"]}`)) + jrr := common.MustNewJsonRpcResponseFromBytes([]byte("1"), []byte(`[{"blockHash":"0xbb","logs":[]}]`), nil) + rs := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) + res := Validate(context.Background(), Input{ + Method: "eth_getBlockReceipts", Upstream: common.NewFakeUpstream("u"), Response: rs, + Checks: only("receiptsRootRecompute", nil), Resolver: covChecksPanicResolver{}, Reorg: DefaultReorgPolicy(), + }) + assert.NoError(t, res.Err) + assert.Equal(t, "panic", outcomeOf(res, "receiptsRootRecompute")) + }) + t.Run("a response without a JSON-RPC body yields no outcomes", func(t *testing.T) { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":[{}]}`)) + rs := common.NewNormalizedResponse().WithRequest(req) + res := Validate(context.Background(), Input{ + Method: "eth_getLogs", Upstream: common.NewFakeUpstream("u"), Response: rs, + Checks: only("getLogsCompleteness", nil), Reorg: DefaultReorgPolicy(), + }) + assert.NoError(t, res.Err) + assert.Empty(t, res.Outcomes) + }) +} + +func TestCovChecks_ArchitectureLookup(t *testing.T) { + a, ok := ArchitectureByName("arbitrum-nitro") + require.True(t, ok) + assert.Equal(t, ProfileFor(42161).Disable, a.Disable) + _, ok = ArchitectureByName("no-such-architecture") + assert.False(t, ok) +} diff --git a/architecture/evm/integrity/engine.go b/architecture/evm/integrity/engine.go index 4b474214d..f46857a2e 100644 --- a/architecture/evm/integrity/engine.go +++ b/architecture/evm/integrity/engine.go @@ -3,9 +3,11 @@ package integrity import ( "context" "fmt" + "runtime/debug" "strings" "github.com/erpc/erpc/common" + "github.com/rs/zerolog/log" ) // Input is everything Validate needs to check one upstream response. @@ -92,7 +94,8 @@ type Result struct { // canonical fetch — a reorg, not corruption; served), "would_reject" // (observe-only mode suppressed a rejection and served the response anyway — // the count is the client-facing cost enforcement would incur), "off" -// (disabled for this finality or check). +// (disabled for this finality or check), "panic" (the check crashed on this +// response; logged, no verdict, response served). type CheckOutcome struct { CheckID string Outcome string @@ -162,7 +165,11 @@ func Validate(ctx context.Context, in Input) Result { continue } - v := c.Run(ctx, d, cfg) + v, panicked := runCheck(ctx, c, d, cfg) + if panicked { + res.Outcomes = append(res.Outcomes, CheckOutcome{c.ID, "panic"}) + continue + } if v == nil { res.Outcomes = append(res.Outcomes, CheckOutcome{c.ID, "pass"}) continue @@ -187,12 +194,16 @@ func Validate(ctx context.Context, in Input) Result { case PinFresh: // Skipped counts as cleared too: the adopted pin resolved the // dispute and the check has nothing left to verify. - if v2 := c.Run(ctx, d, cfg); v2 == nil || v2 == Skipped { + v2, panicked := runCheck(ctx, c, d, cfg) + if panicked { + res.Outcomes = append(res.Outcomes, CheckOutcome{c.ID, "panic"}) + continue + } + if v2 == nil || v2 == Skipped { res.Outcomes = append(res.Outcomes, CheckOutcome{c.ID, "reconfirmed"}) continue - } else { - v = v2 } + v = v2 case PinRateLimited: // The pin could not be re-checked right now, so this violation // rests on CACHED state we have no fresh evidence for. Hard @@ -257,6 +268,20 @@ func Validate(ctx context.Context, in Input) Result { return res } +// runCheck runs one check and turns a panic into "no verdict". Checks parse +// upstream bytes, so a crash in one must cost that check's verdict, never the +// request or the process. +func runCheck(ctx context.Context, c *Check, d *Decoded, cfg CheckConfig) (v *Violation, panicked bool) { + defer func() { + if r := recover(); r != nil { + log.Error().Str("check", c.ID).Str("method", d.method).Interface("panic", r). + Str("stack", string(debug.Stack())).Msg("integrity: check panicked; no verdict for this response") + v, panicked = nil, true + } + }() + return c.Run(ctx, d, cfg), false +} + // finalityOnce caches the response block's finality for one Validate call. // // It MUST be resolved once and reused. The underlying source is the serving diff --git a/architecture/evm/integrity/reconfirm_test.go b/architecture/evm/integrity/reconfirm_test.go index 5be8c7d52..351adcb9d 100644 --- a/architecture/evm/integrity/reconfirm_test.go +++ b/architecture/evm/integrity/reconfirm_test.go @@ -240,3 +240,66 @@ func TestReconfirm_ReceiptVsBlockPinBranch(t *testing.T) { assert.Equal(t, "reconfirmed", outcomeOf(res, "receiptVsBlock")) assert.Equal(t, "0xnew", hist.pins[0x10]) } + +// registerForTest adds a check for the duration of t, then removes it from the +// method registry and allChecks so other tests are unaffected. The id stays in +// the common validation catalog (same as other test-time RegisterIntegrityCheckID +// calls) — that is harmless. +func registerForTest(t *testing.T, c *Check) { + t.Helper() + register(c) + t.Cleanup(func() { + for _, m := range c.Methods { + list := registry[m] + out := list[:0:0] + for _, x := range list { + if x != c { + out = append(out, x) + } + } + if len(out) == 0 { + delete(registry, m) + } else { + registry[m] = out + } + } + kept := allChecks[:0:0] + for _, x := range allChecks { + if x != c { + kept = append(kept, x) + } + } + allChecks = kept + }) +} + +// Regression: a panic on the post-PinFresh re-run must record "panic", not +// "reconfirmed". Discarding the panicked flag made incomplete verification look +// like successful reorg recovery in erpc_integrity_check. +func TestReconfirm_PanicOnRerunReportsPanic(t *testing.T) { + var calls int + c := &Check{ + ID: "testPanicOnRerun", + Family: FamilyContinuity, + Class: ReorgSensitive, + Methods: []string{"eth_getblockbynumber"}, + Run: func(ctx context.Context, d *Decoded, cfg CheckConfig) *Violation { + calls++ + if calls == 1 { + return failf("disputed pin").disputes(0x10) + } + panic("simulated check panic on reconfirm re-run") + }, + } + registerForTest(t, c) + + hist := &reconfirmingHistory{ + pins: map[int64]string{0x10: "0xold"}, + canonical: map[int64]string{0x10: "0xnew"}, + } + res := validateBlockPolicy(t, blockResult("0x10", "0xnew", "0xparent"), only(c.ID, nil), hist, rejectAll) + assert.NoError(t, res.Err, "a panicked check must not reject the response") + assert.Equal(t, "panic", outcomeOf(res, c.ID)) + assert.Equal(t, 1, hist.calls) + assert.Equal(t, 2, calls) +} diff --git a/architecture/evm/integrity_stateprobe.go b/architecture/evm/integrity_stateprobe.go index 2efea37d9..cc5065820 100644 --- a/architecture/evm/integrity_stateprobe.go +++ b/architecture/evm/integrity_stateprobe.go @@ -21,9 +21,9 @@ import ( // still answer eth_call/eth_getBalance from older state — silently, with a // well-formed response. The claimed head says nothing about the state trie. // -// Two probes, both judged against the chain follower's VERIFIED header for a -// height (the trust anchor — without the follower this verification would be -// circular, judging the node against its own claims): +// Three probes. The first two are judged against the chain follower's +// VERIFIED header for a height (the trust anchor — without the follower this +// verification would be circular, judging the node against its own claims): // // - context probe: eth_call to the Multicall3 getBlockNumber() pinned at // block N. The EVM answers from the execution context it actually used, so @@ -40,6 +40,15 @@ import ( // Not all providers expose eth_getProof — support is DISCOVERED per // upstream, never assumed. // +// - latest probe: the context call at the "latest" tag, judged against the +// head the same upstream reports. A node can hold state at its head and +// answer every pinned call right, yet resolve "latest" for state calls to +// an older block: measured on HyperEVM vendors, one froze 31,733 blocks +// back, another ran ~110 back, both with a current head. It runs only +// where clients' "latest" eth_calls reach the upstream as the tag; where +// eRPC sends a number instead (translateLatestTag, pinLatestTag) the +// node's tag resolution is never used. +// // The prober PUBLISHES evidence; it holds no routing authority of its own. // A success advances the upstream's state-proven head (monotonic) — pure // telemetry, never a routing bound: the proven head advances at probe cadence, @@ -47,10 +56,11 @@ import ( // every honest upstream's claimed head, and a bound built on it would refuse // the network's own advertised tip. A single failure just fails to advance — // a mismatch at the tip can be a fork-transient. Only a SUSTAINED streak of -// wrong-height answers is evidence against a node, and that evidence goes -// where every other proven-bad-data signal already goes: upstream misbehavior -// on the health tracker (the ledger consensus disputes, integrity -// deterministic rejects and wrong-empty responses feed — see noteDisproved). +// wrong-height pinned answers is evidence against a node; a stale "latest" +// is evidence each time (see probeUpstream). Both go where every other +// proven-bad-data signal already goes: upstream misbehavior on the health +// tracker (the ledger consensus disputes, integrity deterministic rejects and +// wrong-empty responses feed — see recordMisbehavior). // Whether and when to exclude is the operator's selection policy's call, via // the existing misbehaviorRateAbove predicate. // The context-probe target is per-ARCHITECTURE (integrity.ChainStateContextProbe): @@ -179,9 +189,17 @@ func (p *stateProber) probeAll(headN int64) { } } -// probeUpstream runs both probes against one upstream and advances its proven +// probeUpstream runs the probes against one upstream and advances its proven // head when the evidence supports it. func (p *stateProber) probeUpstream(ctx context.Context, u common.Upstream, n int64, h *integrity.Header) { + // A stale "latest" is judged against the upstream's own head, so a reorg + // cannot fake it: each one is a wrong answer clients would get, recorded + // at once. A vendor fleet split across healthy and stale nodes (measured: + // 9 of 20 answers 9-15 blocks back) never builds the streak below. + if p.probeLatest(ctx, u) == probeMismatch { + p.recordMisbehavior(u) + } + ctxMatch := p.probeContext(ctx, u, n) proofMatch := p.probeProof(ctx, u, n, h) @@ -226,23 +244,8 @@ const ( // probeContext asks the upstream's EVM which block it executes in when pinned // at N. Stale state answers with the stale height. func (p *stateProber) probeContext(ctx context.Context, u common.Upstream, n int64) probeOutcome { - body := fmt.Sprintf( - `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"%s","data":"%s"},"0x%x"]}`, - p.ctxProbe.To, p.ctxProbe.Data, n) - res, err := p.forwardTo(ctx, u, body) - if err != nil { - p.count(u, "context", "error") - return probeUnknown - } - if res == "" || res == "0x" { - // No returndata: Multicall3 is not deployed on this chain (or this - // node prunes code) — no evidence, not a failure. - p.count(u, "context", "unsupported") - return probeUnknown - } - got, ok := parseHexQuantity(res) + got, ok := p.executedBlock(ctx, u, "context", fmt.Sprintf("0x%x", n)) if !ok { - p.count(u, "context", "error") return probeUnknown } if got != n { @@ -266,6 +269,80 @@ func (p *stateProber) probeContext(ctx context.Context, u common.Upstream, n int return probeMatch } +// stateProbeLatestToleranceBlocks is how far "latest" may run behind the +// upstream's head before it counts as stale. The head is the highest block any +// replica saw from any node behind the upstream's URL, so a healthy fleet can +// answer a block or two under it: healthy vendors measured -1 to +3 on +// Ethereum, Base, Arbitrum and HyperEVM; stale ones -7 and further. Like the +// streak, it only rules out jitter, so it is a constant and not config. +const stateProbeLatestToleranceBlocks = 3 + +// probeLatest asks which block the upstream's EVM executes in at "latest", +// against the head read just before the call. +func (p *stateProber) probeLatest(ctx context.Context, u common.Upstream) probeOutcome { + eu, isEvm := u.(common.EvmUpstream) + if !isEvm { + return probeUnknown + } + head := eu.EvmEffectiveLatestBlock() + if head <= 0 || !p.latestReachesUpstream(u) { + return probeUnknown // no head to judge against, or "latest" never reaches u + } + got, ok := p.executedBlock(ctx, u, "latest", "latest") + if !ok { + return probeUnknown + } + if got < head-stateProbeLatestToleranceBlocks { + p.count(u, "latest", "stale") + log.Warn().Str("network", p.network.Label()).Str("upstream", u.Id()). + Int64("head", head).Int64("executedBlock", got). + Msg("state probe: upstream executes \"latest\" calls in state older than its own head") + return probeMismatch + } + p.count(u, "latest", "match") + return probeMatch +} + +// latestReachesUpstream reports whether a client's eth_call at "latest" reaches +// u as the tag rather than as a block number eRPC chose. +func (p *stateProber) latestReachesUpstream(u common.Upstream) bool { + if cfg := u.Config(); cfg != nil && cfg.Evm != nil && cfg.Evm.PinLatestTag != nil && *cfg.Evm.PinLatestTag { + return false + } + m := getMethodConfig("eth_call", p.network) + return m != nil && m.TranslateLatestTag != nil && !*m.TranslateLatestTag +} + +// executedBlock sends the context probe call at blockParam and returns the +// block the upstream's EVM executed it in. ok is false when the answer is no +// evidence; that outcome is already counted under probe. +func (p *stateProber) executedBlock(ctx context.Context, u common.Upstream, probe, blockParam string) (int64, bool) { + if ok, _ := u.ShouldHandleMethod("eth_call"); !ok { + p.count(u, probe, "unsupported") + return 0, false + } + body := fmt.Sprintf( + `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"%s","data":"%s"},"%s"]}`, + p.ctxProbe.To, p.ctxProbe.Data, blockParam) + res, err := p.forwardTo(ctx, u, body) + if err != nil { + p.count(u, probe, "error") + return 0, false + } + if res == "" || res == "0x" { + // No returndata: Multicall3 is not deployed on this chain (or this + // node prunes code) — no evidence, not a failure. + p.count(u, probe, "unsupported") + return 0, false + } + got, ok := parseHexQuantity(res) + if !ok { + p.count(u, probe, "error") + return 0, false + } + return got, true +} + // probeProof verifies the upstream can produce an account proof rooted at the // VERIFIED stateRoot for N — unforgeable without the state trie. func (p *stateProber) probeProof(ctx context.Context, u common.Upstream, n int64, h *integrity.Header) probeOutcome { @@ -370,6 +447,10 @@ func (p *stateProber) noteDisproved(u common.Upstream) { if v.(*atomic.Int64).Add(1) < stateProbeDisprovedStreak { return } + p.recordMisbehavior(u) +} + +func (p *stateProber) recordMisbehavior(u common.Upstream) { // Under integrity observeOnly no check may influence routing: rejections // are suppressed before they reach misbehavior scoring, so probe disproof // stays consistent and records nothing either. @@ -377,10 +458,10 @@ func (p *stateProber) noteDisproved(u common.Upstream) { return } if ht := u.Tracker(); ht != nil { - // Recorded under the context probe's own method: the probe IS a pinned - // eth_call, and eth_call traffic is what a wrong-height execution - // context silently corrupts. The probe pins the followed (not yet - // finalized) head, hence the unfinalized stratum. + // Recorded under the context probes' own method: they ARE eth_calls, + // and eth_call traffic is what a wrong-height execution context + // silently corrupts. They run at the followed (not yet finalized) head, + // hence the unfinalized stratum. ht.RecordUpstreamMisbehavior(u, "eth_call", common.DataFinalityStateUnfinalized) } } diff --git a/architecture/evm/integrity_stateprobe_disproved_test.go b/architecture/evm/integrity_stateprobe_disproved_test.go index 082960dc3..cdb4f6cef 100644 --- a/architecture/evm/integrity_stateprobe_disproved_test.go +++ b/architecture/evm/integrity_stateprobe_disproved_test.go @@ -176,3 +176,93 @@ func TestStateProber_WrongCanaryFailsSymmetrically(t *testing.T) { "%s: nothing proves under a wrong canary", u.Id()) } } + +// A vendor can answer every pinned call right and still resolve "latest" to old +// state (HyperEVM, 2026-10: one vendor froze 31,733 blocks back, another ran +// ~110 back and then split, 9 of 20 answers 9-15 blocks back, all with a +// current head). Where clients' "latest" eth_calls reach upstreams as the tag, +// each stale answer is one misbehavior record, so a split fleet scores in +// proportion, while a healthy sibling that jitters a few blocks under the +// shared head stays clean. Where eRPC sends a number +// instead (the default translateLatestTag, or the upstream's pinLatestTag), +// the tag is never probed. +func TestStateProber_StaleLatestTag(t *testing.T) { + const head = int64(1000) + rounds := stateProbeDisprovedStreak + 2 + run := func(t *testing.T, methods *common.MethodsConfig) (ups map[string]*common.FakeUpstream, latestCalls map[string]int) { + t.Helper() + ups = map[string]*common.FakeUpstream{} + var list []common.Upstream + for _, id := range []string{"stale", "split", "healthy", "pinned-stale"} { + u := common.NewFakeUpstream(id, common.WithEvmStatePoller(common.NewFakeEvmStatePoller(head, head-10))).(*common.FakeUpstream) + ups[id] = u + list = append(list, u) + } + ups["pinned-stale"].Config().Evm = &common.EvmUpstreamConfig{PinLatestTag: util.BoolPtr(true)} + latestCalls = map[string]int{} + net := &fleetNetwork{ups: list, cfg: &common.NetworkConfig{Methods: methods}} + net.forward = func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + jrq, _ := req.JsonRpcRequest() + if jrq.Method != "eth_call" { + return nil, fmt.Errorf("method not found") + } + id := req.Directives().UseUpstream + executed := head // every node runs pinned calls at the pin + if jrq.Params[1] == "latest" { + latestCalls[id]++ + odd := latestCalls[id]%2 == 1 + switch { + case id == "stale" || id == "pinned-stale": + executed = head - 110 + case id == "split" && !odd: + executed = head - stateProbeLatestToleranceBlocks - 1 + case id == "healthy" && !odd: + executed = head - stateProbeLatestToleranceBlocks + default: + executed = head + 1 + } + } + ret := fmt.Sprintf(`"0x%064x"`, executed) + return common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte(`"1"`), []byte(ret), nil)), nil + } + v := newChainView(net, 32, "", "", nil) + v.adoptFollowed(head, &integrity.Header{ + Number: fmt.Sprintf("0x%x", head), Hash: "0xhead", ParentHash: "0xpar", + }) + p := &stateProber{ + network: net, view: v, interval: 0, + ctxProbe: integrity.ChainStateContextProbe(999), + lastProbe: map[string]time.Time{}, work: make(chan int64, 1), + } + for i := 0; i < rounds; i++ { + p.probeAll(head) + } + return ups, latestCalls + } + + t.Run("latest reaches vendors as the tag: each stale answer is recorded", func(t *testing.T) { + ups, latestCalls := run(t, &common.MethodsConfig{Definitions: map[string]*common.CacheMethodConfig{ + "eth_call": {TranslateLatestTag: util.BoolPtr(false)}, + }}) + assert.Equal(t, rounds, fakeTrackerOf(t, ups["stale"]).MisbehaviorCount, + "right pinned answers must not hide a stale \"latest\"") + assert.Equal(t, rounds/2, fakeTrackerOf(t, ups["split"]).MisbehaviorCount, + "a fleet half stale never builds a streak; each stale answer must still count") + assert.Equal(t, 0, fakeTrackerOf(t, ups["healthy"]).MisbehaviorCount) + for _, id := range []string{"stale", "split", "healthy"} { + assert.EqualValues(t, head, ups[id].EvmStateProvenBlock(), "%s holds state at the pin", id) + } + assert.Equal(t, 0, latestCalls["pinned-stale"], "a pinLatestTag upstream never receives \"latest\"") + assert.Equal(t, 0, fakeTrackerOf(t, ups["pinned-stale"]).MisbehaviorCount) + }) + + t.Run("eRPC translates latest to a number: no latest probe at all", func(t *testing.T) { + ups, latestCalls := run(t, nil) + assert.Empty(t, latestCalls) + for id, u := range ups { + assert.Equal(t, 0, fakeTrackerOf(t, u).MisbehaviorCount, id) + assert.EqualValues(t, head, u.EvmStateProvenBlock(), id) + } + }) +} diff --git a/architecture/evm/integrity_stateprobe_test.go b/architecture/evm/integrity_stateprobe_test.go index 9fbd862f4..e2d4ff755 100644 --- a/architecture/evm/integrity_stateprobe_test.go +++ b/architecture/evm/integrity_stateprobe_test.go @@ -156,7 +156,7 @@ func TestStateProber(t *testing.T) { // A fast-chain fleet where structural probe lag exceeds nothing but the clock. // -// eRPC advertises "latest" as the majority CLAIMED head (PickServedTip), and +// eRPC advertises "latest" as a quorum of CLAIMED heads (PickServedTip), and // clients additionally pin state calls at concrete heights they derive // themselves (an indexer calls eth_getBlockByNumber, then eth_call at that // number). The state prober advances each upstream's PROVEN head only at probe @@ -191,12 +191,12 @@ func TestStateBoundary_TipChosenFromClaimedHeadsMustRoute(t *testing.T) { {"upstream-f", 0}, } - // The tip eRPC advertises: the majority order statistic over CLAIMED heads - // (all live and within a block of each other). - tips := make([]common.ServedTipInput, 0, len(fleet)) + // The tip eRPC advertises: the default-quorum order statistic over CLAIMED + // heads (all live and within a block of each other). + tips := make([]int64, 0, len(fleet)) ups := make([]common.Upstream, 0, len(fleet)) for _, f := range fleet { - tips = append(tips, common.ServedTipInput{UpstreamID: f.id, BlockNumber: head}) + tips = append(tips, head) u := &cadenceLaggedUpstream{ FakeUpstream: common.NewFakeUpstream(f.id).(*common.FakeUpstream), claimed: head, @@ -206,8 +206,8 @@ func TestStateBoundary_TipChosenFromClaimedHeadsMustRoute(t *testing.T) { } ups = append(ups, u) } - tip := common.PickServedTip(tips).Tip - require.Equal(t, head, tip, "sanity: a healthy fleet's majority tip is the head") + tip := common.PickServedTip(tips, 0).Tip + require.Equal(t, head, tip, "sanity: a healthy fleet's served tip is the head") // The precondition, asserted so the fixture cannot rot into triviality: the // advertised tip exceeds the proven head of a strict majority of the fleet diff --git a/architecture/evm/json_rpc_cache_age_test.go b/architecture/evm/json_rpc_cache_age_test.go index 619f4345e..f5b41c9eb 100644 --- a/architecture/evm/json_rpc_cache_age_test.go +++ b/architecture/evm/json_rpc_cache_age_test.go @@ -241,11 +241,13 @@ func TestEvmJsonRpcCache_BlockAgeValidation(t *testing.T) { }) t.Run("TriesNextPolicyAfterAgeRejection", func(t *testing.T) { - // Create two mock connectors - mockConnector1 := &data.MockConnector{} + // Two policies fan out concurrently, so the connectors must not hand the + // shared request to testify: its argument diff reflect-reads the request + // while the other goroutine lazily parses it, which -race reports. + mockConnector1 := &reqBlindConnector{MockConnector: &data.MockConnector{}} mockConnector1.On("Id").Return("mock-connector-1") - mockConnector2 := &data.MockConnector{} + mockConnector2 := &reqBlindConnector{MockConnector: &data.MockConnector{}} mockConnector2.On("Id").Return("mock-connector-2") // Create an old cached response for connector 1 (2 minutes ago) @@ -327,3 +329,13 @@ func TestEvmJsonRpcCache_BlockAgeValidation(t *testing.T) { assert.True(t, resp.FromCache()) }) } + +// reqBlindConnector forwards Get to the testify mock without the request +// metadata, so concurrent policy goroutines never reflect over the request. +type reqBlindConnector struct { + *data.MockConnector +} + +func (c *reqBlindConnector) Get(ctx context.Context, index, partitionKey, rangeKey string, _ interface{}) ([]byte, error) { + return c.MockConnector.Get(ctx, index, partitionKey, rangeKey, nil) +} diff --git a/architecture/evm/local_answer.go b/architecture/evm/local_answer.go new file mode 100644 index 000000000..4164df7a6 --- /dev/null +++ b/architecture/evm/local_answer.go @@ -0,0 +1,30 @@ +package evm + +import ( + "context" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" +) + +// projectPreForward_localAnswer answers a method eRPC can answer itself (see +// common.LocalAnswer) before cache and upstream selection. A request that +// skips the cache read asks for an upstream's answer, so it is routed. +func projectPreForward_localAnswer(ctx context.Context, n common.Network, r *common.NormalizedRequest, method string) (handled bool, resp *common.NormalizedResponse, err error) { + if r.ShouldSkipCacheRead("") { + return false, nil, nil + } + result, ok := common.LocalAnswer(ctx, n, method) + if !ok { + return false, nil, nil + } + id := r.ID() + if id == nil { + id = util.RandomID() + } + jrr, err := common.NewJsonRpcResponse(id, result, nil) + if err != nil { + return true, nil, err + } + return true, common.NewNormalizedResponse().WithRequest(r).WithJsonRpcResponse(jrr), nil +} diff --git a/architecture/evm/multicall3.go b/architecture/evm/multicall3.go new file mode 100644 index 000000000..8712b6d9b --- /dev/null +++ b/architecture/evm/multicall3.go @@ -0,0 +1,515 @@ +package evm + +import ( + "context" + "encoding/binary" + "encoding/hex" + "encoding/json" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" + "go.opentelemetry.io/otel/attribute" +) + +// Multicall3 is deployed at this address on every chain that has it. +const multicall3Address = "0xcA11bde05977b3631167028862bE2a173976CA11" + +// aggregate3((address,bool,bytes)[]) +var aggregate3Selector = []byte{0x82, 0xad, 0x56, 0xcb} + +const ( + multicall3Packed = "packed" + multicall3Alone = "alone" + multicall3Reverted = "reverted" + multicall3Empty = "empty" + multicall3Failed = "failed" + multicall3Ineligible = "ineligible" + multicall3Bypassed = "bypassed" + multicall3Canceled = "canceled" +) + +// Multicall3Batcher packs the eth_calls one upstream receives for the same block +// into a single aggregate3 eth_call. It is best-effort: every call it cannot +// answer exactly goes back to the caller, which sends it alone. +type Multicall3Batcher struct { + maxWait time.Duration + maxCalls int + + mu sync.Mutex + open map[string]*multicall3Batch + + // Highest block at which Multicall3 had no code on this upstream; calls at + // or below it are not packed. + absentAtOrBelow atomic.Int64 + + // Contracts whose calls are never packed: configured (bypassContracts), or + // learned when a call to them reverted inside a pack and then succeeded alone + // (typically a contract that rejects a contract caller). Learned entries last + // for the process; they only ever send calls alone, as without packing. + bypass map[[20]byte]struct{} + learnedBypass sync.Map // [20]byte -> struct{} + // Calls that reverted inside a pack and are being sent alone, by request, + // until the upstream post-forward hook sees how the call alone ended. + awaitingAlone sync.Map // *common.NormalizedRequest -> [20]byte +} + +// A pack shares one block, one gas limit and one sender (none, or the zero +// address). With the callers' own gas limit on the multicall, an inner call +// gets less gas than it would alone: one that runs out fails inside the pack +// and is sent alone, so packing never adds gas. +type multicall3Batch struct { + block interface{} + gas string + from string + entries []*multicall3Entry + timer *time.Timer +} + +type multicall3Entry struct { + ctx context.Context + req *common.NormalizedRequest + target []byte + data []byte + result chan multicall3Result +} + +type multicall3Result struct { + returnData []byte + outcome string +} + +type aggregate3Result struct { + success bool + data []byte +} + +func NewMulticall3Batcher(cfg *common.EvmMulticall3Config) *Multicall3Batcher { + b := &Multicall3Batcher{ + maxWait: cfg.MaxWait.Duration(), + maxCalls: cfg.MaxCalls, + open: make(map[string]*multicall3Batch), + bypass: make(map[[20]byte]struct{}, len(cfg.BypassContracts)), + } + for _, a := range cfg.BypassContracts { + // Validated as 20-byte hex addresses at config load. + if raw, err := common.HexToBytes(a); err == nil && len(raw) == 20 { + b.bypass[[20]byte(raw)] = struct{}{} + } + } + return b +} + +func (b *Multicall3Batcher) bypassed(target []byte) bool { + addr := [20]byte(target) + if _, ok := b.bypass[addr]; ok { + return true + } + _, ok := b.learnedBypass.Load(addr) + return ok +} + +type multicall3Upstream interface { + EvmMulticall3Batcher() *Multicall3Batcher +} + +func upstreamPreForward_eth_call(ctx context.Context, n common.Network, u common.Upstream, nrq *common.NormalizedRequest) (bool, *common.NormalizedResponse, error) { + mu, ok := u.(multicall3Upstream) + if !ok || nrq.CompositeType() == common.CompositeTypeMulticall3 { + return false, nil, nil + } + b := mu.EvmMulticall3Batcher() + if b == nil { + return false, nil, nil + } + + ctx, span := common.StartDetailSpan(ctx, "Upstream.PreForwardHook.eth_call") + defer span.End() + + returnData, outcome := b.submit(ctx, n, u, nrq) + var resp *common.NormalizedResponse + if outcome == multicall3Packed { + if jrr, err := common.NewJsonRpcResponse(nrq.ID(), "0x"+hex.EncodeToString(returnData), nil); err == nil { + resp = common.NewNormalizedResponse().WithRequest(nrq).WithJsonRpcResponse(jrr) + resp.SetUpstream(u) + nrq.SetLastUpstream(u) + nrq.SetLastValidResponse(ctx, resp) + } else { + outcome = multicall3Failed + } + } + span.SetAttributes(attribute.String("multicall3.outcome", outcome)) + telemetry.MetricUpstreamMulticall3CallsTotal.WithLabelValues(n.ProjectId(), nrq.NetworkLabel(), u.Id(), u.VendorName(), outcome).Inc() + return resp != nil, resp, nil +} + +// upstreamPostForward_eth_call learns a bypass when a call that reverted inside +// a pack succeeded alone: the contract answers differently to Multicall3. +func upstreamPostForward_eth_call(u common.Upstream, rq *common.NormalizedRequest, rs *common.NormalizedResponse, re error) { + mu, ok := u.(multicall3Upstream) + if !ok { + return + } + b := mu.EvmMulticall3Batcher() + if b == nil { + return + } + v, ok := b.awaitingAlone.LoadAndDelete(rq) + if !ok || re != nil || rs == nil { + return + } + if jrr, err := rs.JsonRpcResponse(); err != nil || jrr == nil || jrr.Error != nil { + return + } + addr := v.([20]byte) + if _, known := b.learnedBypass.LoadOrStore(addr, struct{}{}); !known && u.Logger() != nil { + u.Logger().Info().Str("contract", "0x"+hex.EncodeToString(addr[:])).Msg("multicall3: contract reverts inside a pack but not alone; its calls are no longer packed") + } +} + +// submit waits for the pack nrq joins and returns its return data when the pack +// answered it, else the outcome that sends it alone. +func (b *Multicall3Batcher) submit(ctx context.Context, n common.Network, u common.Upstream, nrq *common.NormalizedRequest) ([]byte, string) { + target, data, block, gas, from, ok := multicall3CallOf(ctx, nrq) + if !ok { + return nil, multicall3Ineligible + } + if b.bypassed(target) { + return nil, multicall3Bypassed + } + if bn := multicall3BlockNumber(block); bn > 0 && bn <= b.absentAtOrBelow.Load() { + return nil, multicall3Ineligible + } + keyBytes, err := json.Marshal([]interface{}{block, gas, from}) + if err != nil { + return nil, multicall3Ineligible + } + key := string(keyBytes) + + e := &multicall3Entry{ctx: ctx, req: nrq, target: target, data: data, result: make(chan multicall3Result, 1)} + b.mu.Lock() + bt := b.open[key] + if bt == nil { + bt = &multicall3Batch{block: block, gas: gas, from: from} + b.open[key] = bt + bt.timer = time.AfterFunc(b.maxWait, func() { b.flush(key, bt, n, u) }) + } + bt.entries = append(bt.entries, e) + full := len(bt.entries) >= b.maxCalls + if full { + delete(b.open, key) + bt.timer.Stop() + } + b.mu.Unlock() + if full { + go b.send(bt, n, u) + } + + select { + case r := <-e.result: + if r.outcome == multicall3Reverted { + b.awaitingAlone.Store(nrq, [20]byte(target)) + } + return r.returnData, r.outcome + case <-ctx.Done(): + return nil, multicall3Canceled + } +} + +func (b *Multicall3Batcher) flush(key string, bt *multicall3Batch, n common.Network, u common.Upstream) { + b.mu.Lock() + if b.open[key] != bt { + // Sent already because it filled up. + b.mu.Unlock() + return + } + delete(b.open, key) + b.mu.Unlock() + b.send(bt, n, u) +} + +func (b *Multicall3Batcher) send(bt *multicall3Batch, n common.Network, u common.Upstream) { + live := make([]*multicall3Entry, 0, len(bt.entries)) + for _, e := range bt.entries { + if e.ctx.Err() == nil { + live = append(live, e) + } + } + if len(live) < 2 { + for _, e := range live { + e.result <- multicall3Result{outcome: multicall3Alone} + } + return + } + + telemetry.MetricUpstreamMulticall3BatchSize.WithLabelValues(n.ProjectId(), live[0].req.NetworkLabel(), u.Id(), u.VendorName()).Observe(float64(len(live))) + results, ok := b.aggregate(live, bt, n, u) + for i, e := range live { + switch { + case !ok: + e.result <- multicall3Result{outcome: multicall3Failed} + case !results[i].success: + e.result <- multicall3Result{outcome: multicall3Reverted} + case len(results[i].data) == 0: + // The node may answer "0x" differently from a contract that returned + // nothing (retries, empty-result handling), so let it answer alone. + e.result <- multicall3Result{outcome: multicall3Empty} + default: + e.result <- multicall3Result{returnData: results[i].data, outcome: multicall3Packed} + } + } +} + +// aggregate sends the pack as one eth_call through the upstream, so its rate +// limits, retries and credit accounting count it once. +func (b *Multicall3Batcher) aggregate(entries []*multicall3Entry, bt *multicall3Batch, n common.Network, u common.Upstream) ([]aggregate3Result, bool) { + call := map[string]interface{}{ + "to": multicall3Address, + "data": "0x" + hex.EncodeToString(encodeAggregate3(entries)), + } + if bt.gas != "" { + call["gas"] = bt.gas + } + if bt.from != "" { + call["from"] = bt.from + } + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{call, bt.block}) + if err := jrq.SetID(util.RandomID()); err != nil { + return nil, false + } + first := entries[0].req + mrq := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + mrq.SetCompositeType(common.CompositeTypeMulticall3) + mrq.SetNetwork(n) + mrq.SetDirectives(first.Directives().Clone()) + mrq.SetParentRequestId(first.ID()) + mrq.CopyHttpContextFrom(first) + + ctx, cancel := multicall3Context(entries) + defer cancel() + ctx = context.WithValue(ctx, common.RequestContextKey, mrq) + + resp, err := u.Forward(ctx, mrq, false, false) + if resp != nil { + defer resp.Release() + } + if err != nil || resp == nil { + return nil, false + } + jrr, err := resp.JsonRpcResponse(ctx) + if err != nil || jrr == nil || jrr.Error != nil { + return nil, false + } + var resultHex string + if err := common.SonicCfg.Unmarshal(jrr.GetResultBytes(), &resultHex); err != nil { + return nil, false + } + raw, err := common.HexToBytes(resultHex) + if err != nil { + return nil, false + } + if len(raw) == 0 { + // No code at the Multicall3 address: the block predates its deployment. + if bn := multicall3BlockNumber(bt.block); bn > 0 { + for cur := b.absentAtOrBelow.Load(); bn > cur && !b.absentAtOrBelow.CompareAndSwap(cur, bn); cur = b.absentAtOrBelow.Load() { + } + } + return nil, false + } + results, ok := decodeAggregate3(raw) + if !ok || len(results) != len(entries) { + return nil, false + } + return results, true +} + +// multicall3Context outlives any one caller, so a caller that gives up costs +// the others nothing; it ends at the last caller's deadline. +func multicall3Context(entries []*multicall3Entry) (context.Context, context.CancelFunc) { + base := context.WithoutCancel(entries[0].ctx) + var last time.Time + for _, e := range entries { + d, ok := e.ctx.Deadline() + if !ok { + return context.WithCancel(base) + } + if d.After(last) { + last = d + } + } + return context.WithDeadline(base, last) +} + +// multicall3CallOf returns the target, calldata, wire block, gas limit and +// sender of an eth_call that Multicall3 can run exactly: {to, data|input, gas?, +// from?} with no value or state override, since Multicall3 cannot carry those. +// The only sender allowed is the zero address, the one a node assumes when +// none is given: any other sender is what msg.sender would be, and inside the +// pack msg.sender is Multicall3. It is returned so the pack is sent from it +// too, keeping tx.origin exact on nodes whose default sender differs. +func multicall3CallOf(ctx context.Context, nrq *common.NormalizedRequest) (target, data []byte, block interface{}, gas, from string, ok bool) { + jrq, err := nrq.JsonRpcRequest(ctx) + if err != nil || jrq == nil { + return nil, nil, nil, "", "", false + } + jrq.RLock() + defer jrq.RUnlock() + + params := common.WireParams(ctx, nrq, jrq.Params) + if len(params) != 2 || params[1] == nil { + return nil, nil, nil, "", "", false + } + call, isMap := params[0].(map[string]interface{}) + if !isMap { + return nil, nil, nil, "", "", false + } + var to, input string + for k, v := range call { + if v == nil { + continue + } + s, isStr := v.(string) + if !isStr { + return nil, nil, nil, "", "", false + } + switch k { + case "to": + to = s + case "gas": + gas = s + case "from": + sender, err := common.HexToBytes(s) + if err != nil || len(sender) != 20 || [20]byte(sender) != [20]byte{} { + return nil, nil, nil, "", "", false + } + from = "0x" + hex.EncodeToString(sender) + case "data", "input": + if input != "" && !strings.EqualFold(input, s) { + return nil, nil, nil, "", "", false + } + input = s + default: + return nil, nil, nil, "", "", false + } + } + target, err = common.HexToBytes(to) + if err != nil || len(target) != 20 { + return nil, nil, nil, "", "", false + } + if input != "" { + if data, err = common.HexToBytes(input); err != nil { + return nil, nil, nil, "", "", false + } + } + return target, data, params[1], gas, from, true +} + +func multicall3BlockNumber(block interface{}) int64 { + s, _, err := util.ParseBlockParameter(block) + if err != nil || !strings.HasPrefix(s, "0x") { + return 0 + } + bn, err := common.HexToInt64(s) + if err != nil { + return 0 + } + return bn +} + +func encodeAggregate3(entries []*multicall3Entry) []byte { + size := 4 + 64 + 32*len(entries) + for _, e := range entries { + size += 128 + padded32(len(e.data)) + } + out := make([]byte, 0, size) + out = append(out, aggregate3Selector...) + out = appendWord(out, 32) + out = appendWord(out, uint64(len(entries))) + offset := uint64(32 * len(entries)) + for _, e := range entries { + out = appendWord(out, offset) + offset += uint64(128 + padded32(len(e.data))) + } + for _, e := range entries { + out = append(out, make([]byte, 12)...) + out = append(out, e.target...) + out = appendWord(out, 1) // allowFailure + out = appendWord(out, 96) // callData offset inside the tuple + out = appendWord(out, uint64(len(e.data))) + out = append(out, e.data...) + out = append(out, make([]byte, padded32(len(e.data))-len(e.data))...) + } + return out +} + +// decodeAggregate3 decodes (bool success, bytes returnData)[]. Every offset and +// length is bounded by len(raw) before use, so malformed input fails, never panics. +func decodeAggregate3(raw []byte) ([]aggregate3Result, bool) { + word := func(at int) (int, bool) { + if at < 0 || at+32 > len(raw) { + return 0, false + } + for _, c := range raw[at : at+24] { + if c != 0 { + return 0, false + } + } + v := binary.BigEndian.Uint64(raw[at+24 : at+32]) + if v > uint64(len(raw)) { + return 0, false + } + return int(v), true + } + array, ok := word(0) + if !ok { + return nil, false + } + count, ok := word(array) + if !ok { + return nil, false + } + head := array + 32 + if count > (len(raw)-head)/32 { + return nil, false + } + results := make([]aggregate3Result, count) + for i := range results { + rel, ok := word(head + 32*i) + if !ok { + return nil, false + } + elem := head + rel + success, ok := word(elem) + if !ok || success > 1 { + return nil, false + } + dataRel, ok := word(elem + 32) + if !ok { + return nil, false + } + length, ok := word(elem + dataRel) + if !ok { + return nil, false + } + start := elem + dataRel + 32 + if start+length > len(raw) { + return nil, false + } + results[i] = aggregate3Result{success: success == 1, data: raw[start : start+length]} + } + return results, true +} + +func padded32(n int) int { + return (n + 31) / 32 * 32 +} + +func appendWord(out []byte, v uint64) []byte { + var w [32]byte + binary.BigEndian.PutUint64(w[24:], v) + return append(out, w[:]...) +} diff --git a/architecture/evm/network_fake_test.go b/architecture/evm/network_fake_test.go index 5dbe80b50..7a4780d6a 100644 --- a/architecture/evm/network_fake_test.go +++ b/architecture/evm/network_fake_test.go @@ -37,6 +37,7 @@ func (n *queryTestNetwork) EvmHighestFinalizedBlockNumber(ctx context.Context) i return n.finalized } func (n *queryTestNetwork) EvmLeaderUpstream(ctx context.Context) common.Upstream { return nil } +func (n *queryTestNetwork) EvmRefreshLatestBlock(ctx context.Context) {} // queryTestConfigUpstream is a minimal common.Upstream with a given config. type queryTestConfigUpstream struct { diff --git a/architecture/evm/pin_latest.go b/architecture/evm/pin_latest.go new file mode 100644 index 000000000..c9c1e1796 --- /dev/null +++ b/architecture/evm/pin_latest.go @@ -0,0 +1,120 @@ +package evm + +import ( + "context" + "fmt" + + "github.com/erpc/erpc/common" +) + +// Block-selector parameter position of the state methods whose "latest" may be +// pinned to a concrete height: the shadow mirror (shadow.pinBlockTag) and +// upstreams with evm.pinLatestTag. Methods absent here are never pinned: for +// eth_getBlockByNumber and friends, pinning the tag would change the QUESTION, +// not just the evaluation height. +var latestTagParamIndex = map[string]int{ + "eth_call": 1, + "eth_getBalance": 1, + "eth_getCode": 1, + "eth_getTransactionCount": 1, + "eth_getStorageAt": 2, +} + +// LatestTagParamIndex returns the block-selector position of a pinnable method. +func LatestTagParamIndex(method string) (int, bool) { + idx, ok := latestTagParamIndex[method] + return idx, ok +} + +// selectsLatest reports whether the block selector at idx means latest: the +// string "latest", or absent because params end right before it. +func selectsLatest(params []interface{}, idx int) bool { + if len(params) == idx { + return true + } + if len(params) < idx { + return false + } + tag, ok := params[idx].(string) + return ok && tag == "latest" +} + +// PinLatestParams returns a copy of params whose block selector at idx is +// hex(height), when that selector means latest (see selectsLatest). Otherwise +// it returns params unchanged and false. Elements are shared, not copied. +func PinLatestParams(params []interface{}, idx int, height int64) ([]interface{}, bool) { + if !selectsLatest(params, idx) { + return params, false + } + out := make([]interface{}, max(len(params), idx+1)) + copy(out, params) + out[idx] = fmt.Sprintf("0x%x", height) + return out, true +} + +// PinBlockTagInBody rewrites params[idx] from "latest" to the given concrete +// height in a JSON-RPC request body. Returns the rewritten body and whether a +// rewrite happened; any parse trouble, an absent selector or a non-"latest" +// selector returns the body unchanged (fail-open, like every other +// shadow-path guard). +func PinBlockTagInBody(body []byte, idx int, height int64) ([]byte, bool) { + if len(body) == 0 || height <= 0 { + return body, false + } + var req map[string]interface{} + if err := common.SonicCfg.Unmarshal(body, &req); err != nil { + return body, false + } + params, ok := req["params"].([]interface{}) + if !ok || len(params) <= idx { + return body, false + } + pinned, ok := PinLatestParams(params, idx, height) + if !ok { + return body, false + } + req["params"] = pinned + out, err := common.SonicCfg.Marshal(req) + if err != nil { + return body, false + } + return out, true +} + +// PinLatestTag resolves what upstream u must receive for r when u sets +// evm.pinLatestTag and r asks a pinnable method for latest: r's params with +// the block selector set to height, the network head that "latest" +// translation would use (resolveBlockTagToHex). pinned is false when u does +// not pin r. pinned with height 0 means the head is unknown, and u must not be +// sent "latest" at all. +// +// ctx must carry the request (common.RequestContextKey) so the head is the one +// of the request's use-upstream group, as in translation. +func PinLatestTag(ctx context.Context, n common.Network, u common.Upstream, r *common.NormalizedRequest) (params []interface{}, height int64, pinned bool) { + cfg := u.Config() + if cfg == nil || cfg.Evm == nil || cfg.Evm.PinLatestTag == nil || !*cfg.Evm.PinLatestTag { + return nil, 0, false + } + jrq, err := r.JsonRpcRequest(ctx) + if err != nil || jrq == nil { + return nil, 0, false + } + jrq.RLock() + idx, ok := latestTagParamIndex[jrq.Method] + ok = ok && selectsLatest(jrq.Params, idx) + jrq.RUnlock() + if !ok { + return nil, 0, false + } + height = common.EvmHighestLatestBlockNumber(n, ctx) + if height <= 0 { + return nil, 0, true + } + jrq.RLock() + params, ok = PinLatestParams(jrq.Params, idx, height) + jrq.RUnlock() + if !ok { + return nil, 0, false + } + return params, height, true +} diff --git a/architecture/evm/served_tip.go b/architecture/evm/served_tip.go deleted file mode 100644 index 2665d40c8..000000000 --- a/architecture/evm/served_tip.go +++ /dev/null @@ -1,755 +0,0 @@ -package evm - -import ( - "math" - "sort" - "sync" - "sync/atomic" - "time" - - "github.com/erpc/erpc/common" -) - -// ─── Long-term trajectory referee ──────────────────────────────────────────── -// -// PickServedTip is a snapshot of one instant, and so is every filter feeding -// it: they all answer "what do the eligible upstreams say RIGHT NOW". That is -// exactly the assumption a majority stall breaks. If N upstreams freeze inside -// the lag-exclusion threshold (or a fleet-wide shared-counter freeze stalls the -// lag metric itself), the majority IS the stale group, and the 2 upstreams that -// genuinely have the chain's head are outvoted — no instantaneous rule can tell -// that apart from a chain that simply stopped. -// -// One thing does distinguish them: what the network's head has been doing for -// the last ten minutes. TipTrajectory records that track and lets the referee -// answer "which of these groups is where the chain should be by now". -// -// STRUCTURAL LIMITS — this is deliberately NOT the stateful tip pipeline that -// commit 7dee07c0 removed (a velocity gate plus a persisted monotonic clamp, -// which reached an absorbing state where every live tip was rejected forever -// and the served tip froze fleet-wide): -// -// - ADVISORY, NEVER GENERATIVE. The referee only ever chooses among head -// values live upstreams are offering in this very evaluation. It cannot -// invent, remember, or extrapolate a servable block; the trajectory is used -// ONLY to rank the groups those upstreams form. -// - UPWARD-ONLY. It may raise the pick to a corroborated fresher group; it can -// never lower or hold one. A referee that cannot hold a value back cannot -// freeze a tip — the 7dee07c0 failure mode is unreachable, not merely -// unlikely. (Erring low is already the majority picker's job, and the -// regression guard's.) -// - SELF-LIMITING EVIDENCE. Samples are the plain MEDIAN of the live heads — -// never the served value, never the referee's own output. During a majority -// stall the tracker therefore keeps recording the stalled median, the fitted -// velocity decays, and the referee stops participating on its own after -// roughly half a window. Its intervention has a built-in expiry that no -// configuration can extend. -// - CORROBORATION IS EARNED OVER TIME, NOT IN AN INSTANT. Matching the -// trajectory in one sample is not evidence: during a chain halt the expected -// head SWEEPS upward through every fixed offset, so any static fork/pair sits -// "on trajectory" for as long as the sweep takes to cross it, and a routine -// 15-second divergence puts a healthy pair "ahead" for one evaluation. Both -// were confirmed exploits of the instant test. A group must therefore hold -// its place — elected AND within tolerance — for a continuous tipDwellDuration -// before it may win, and must have ADVANCED its own head over that stretch at -// a fraction of the fitted velocity (tipDwellMinVelocityFraction). A frozen -// group can never satisfy the second condition, whatever the first says. -// - FAIL OPEN ON ANY UNCERTAINTY, and no shared state: everything below is -// per-process, in-memory, rebuilt from live observations after a restart. -// Per-pod independence is the point — a shared head counter is how the -// 2026-08 celo trigger propagated fleet-wide in the first place. - -const ( - // tipSampleInterval throttles recording to one sample per evaluation-with- - // this-much-elapsed. Fast enough that the default 10-minute window holds - // 120 samples, slow enough that the O(n log n) refit runs at most 0.2 Hz - // per network+axis while reads stay O(1). It lives in common because the - // config validator derives the allowed trajectoryWindow range from it. - tipSampleInterval = common.ServedTipTrajectorySampleInterval - - // tipMinSamples is the smallest buffer the fit is computed over: below it - // a handful of points could define a "trajectory" on noise alone. - tipMinSamples = 30 - - // tipMaxSampleGap is how large a hole the window may contain — both at its - // newest end (the age of the freshest sample as of the START of an - // evaluation) and ANYWHERE INSIDE it (tipFit.maxGapMs). A gap larger than - // this is where a halt or a burst hides, and a fit whose window straddles - // one is extrapolating across the very thing it cannot see: the residuals - // of the samples before the hole then pull the robust intercept up by a - // whole gap of chain progress, which is exactly how a rogue pair sitting at - // "where the head would be if the chain had not stopped" wins an election. - // Checking only the newest sample's age stood the referee down for a single - // evaluation — the next one, a millisecond later, extrapolated across the - // hole regardless. The gap must therefore leave the WINDOW before the fit - // counts as describing the present. - tipMaxSampleGap = time.Minute - - // tipMinGroupSize is the corroboration rule: one witness is not evidence, - // however perfectly it matches the trajectory. A group must contain at - // least two upstreams to be electable. - // - // It stays 2 for a fleet of ANY size, deliberately: the whole point of the - // referee is that two upstreams which genuinely hold the chain's head must - // be able to outvote a larger stalled group, and scaling the requirement - // with the fleet would hand the stalled majority a veto in exactly the - // topology the referee exists for. What makes two trustworthy is not the - // count but the dwell and velocity conditions below: two witnesses that - // have tracked the network's own trajectory for half a minute, advancing - // their own heads while doing so. - tipMinGroupSize = 2 - - // tipDwellDuration is how long the elected group must hold its place — - // continuously within tolerance of the expected head — before it may - // outvote the majority. Six samples at the 5s throttle: long enough that a - // routine divergence (a 15s poller hiccup on half the fleet, the confirmed - // false-override shape) resolves before it can win, short enough that a - // genuine majority stall is corrected inside one block-explorer refresh. - tipDwellDuration = 30 * time.Second - - // tipMajorityStallSeconds is how far behind the trajectory the MAJORITY pick - // must have fallen — expressed as chain progress in SECONDS, converted to - // blocks through the fitted velocity — before the referee may outvote it. - // - // It exists because nothing else in the decision path tests the referee's - // own premise. The tolerance gate asks whether the ELECTED GROUP is on the - // trajectory; the upward-only rule asks whether that group is above the - // majority. Neither asks the question the override is named for: has the - // majority actually stalled? Without this gate the answer is assumed, and - // the assumption is wrong on any healthy fleet with a persistent upper - // group — see the note on TipTrajectoryDecision.Declined. - // - // The gate is a duration, not a block count, because block counts are not - // comparable across chains: ToleranceFloor's 1024-block default is ~3 hours - // of a 12s chain but only ~10 minutes of a 1.7 blocks/s one, so as an - // absolute floor it binds on slow chains and never binds on fast ones. - // Seconds of missing chain progress mean the same thing everywhere. - // - // 30s matches tipDwellDuration: the elected group must hold its place for - // that long anyway, so requiring the majority to be at least that far - // behind adds no latency to correcting a genuine stall — a frozen majority - // crosses this gate and earns the dwell over the same interval. - tipMajorityStallSeconds = 30.0 - - // tipDwellMinVelocityFraction is the share of the fitted velocity the - // elected group must have delivered FROM ITS OWN HEAD over the dwell. It is - // a lower bound only — closeness to expected already bounds the group from - // above — and its job is to disqualify anything that is not moving: a fork, - // a wrong-chain pair, a frozen shared counter parked at a fixed offset. A - // static group's advance is 0 and the threshold is strictly positive - // whenever the referee is confident at all (confident() requires v > 0 and - // the dwell is ≥30s), so a static group can never be elected — not merely - // unlikely to be. Half the fitted velocity leaves room for a group that is - // catching up in bursts. - tipDwellMinVelocityFraction = 0.5 - - // tipToleranceSigmas (k) scales the window's OWN residual spread into the - // distance-from-expected a group may sit at and still count as - // on-trajectory. The spread is a median absolute residual, which for - // normal-ish noise is ≈0.67σ, so k=6 is ≈4σ: a chain that pauses and - // bursts widens its own gate automatically, while a metronomic chain keeps - // the floor (EvmServedTipConfig.MaxRegressionBlocks). - tipToleranceSigmas = 6.0 - - // tipClusterSeconds derives the cluster width from the fitted velocity: - // heads within this much CHAIN PROGRESS of each other are the same view of - // the head. 10s comfortably covers poller-interval skew and propagation - // between healthy providers, and is far below any stall worth correcting. - tipClusterSeconds = 10.0 - - // tipMinClusterWidth keeps sub-second-block chains (and a small fitted - // velocity generally) from splitting a healthy fleet into singletons. - tipMinClusterWidth = 16 - - // tipMaxClusterWidth is an overflow stop, not a tuning knob: an absurd - // fitted velocity must not produce an int64 conversion of a huge float. A - // too-wide cluster merges everything into one group, which can only make - // the referee defer. - tipMaxClusterWidth = int64(1) << 40 - - // tipMinBufferCapacity / tipMaxBufferCapacity bound the ring derived from - // the configured window (tipBufferCapacity). The cap is shared with the - // config validator, which rejects a window this ring could never span. - tipMinBufferCapacity = 64 - tipMaxBufferCapacity = common.ServedTipTrajectoryMaxSamples -) - -// TipTrajectoryParams are the per-network knobs the referee reads on every -// evaluation. They come straight from EvmServedTipConfig, so a config change -// takes effect without rebuilding the tracker. -type TipTrajectoryParams struct { - // Window is the minimum span of recorded history required before the - // referee may participate at all. <= 0 disables the tracker entirely: - // nothing is recorded and nothing is computed. - Window time.Duration - - // ToleranceFloor is the minimum distance-from-expected a group may sit at - // and still be on-trajectory, in blocks (EvmServedTipConfig's - // MaxRegressionBlocks, defaulted by the caller). The observed residual - // spread can only widen the gate, never narrow it below this. - ToleranceFloor int64 -} - -// TipTrajectoryDecision is the referee's advice for one evaluation. Pick is -// always a value the caller may serve as-is: the majority pick it passed in, -// or a corroborated group's minimum. -type TipTrajectoryDecision struct { - // Pick is the block to serve: the majority pick unless Overrode. - Pick int64 - - // Overrode reports that Pick differs from the majority pick — the only - // outcome that is an intervention, and the one worth an alert. - Overrode bool - - // Declined reports the near miss: the group the trajectory elected sat - // ABOVE the majority pick — an override was on the table — and the referee - // refused it, because the group was not close enough to where the head - // should be, because it had not held its place long enough, or because the - // majority it would have outvoted had not actually stalled. - // - // A fleet in one cluster elects that cluster and sets neither flag. A fleet - // permanently split around its own median sets Declined, NOT Overrode. - // - // It was once claimed here that such a fleet could not even elect its upper - // group, reasoning that the fit is made from the median so no - // permanently-offset group can sit nearer to it. That is wrong. Once the - // split exceeds one cluster width the lower upstreams are singletons, which - // tipMinGroupSize rejects, so the upper group is often the ONLY candidate - // and wins by default however far from the median it sits — it is elected - // against the tolerance, never against the median's distance. What keeps - // election from becoming an override on a healthy fleet is the - // majority-stall gate (tipMajorityStallSeconds), not the geometry of the - // fit. Observed on a four-upstream fleet at a persistent ladder, which - // before that gate took a ~24-block override every ~28 minutes. - Declined bool - - // Expected / Tolerance / VelocityPerSec describe the fit that produced the - // decision. Populated only once the confidence gate passes; for logs. - Expected int64 - Tolerance int64 - VelocityPerSec float64 -} - -// tipSample is one recorded observation of where the network's head was. -type tipSample struct { - atMs int64 - head int64 -} - -// tipFit is the robust linear fit over the buffer, recomputed on insertion and -// read lock-free on every evaluation. -type tipFit struct { - refMs int64 // timestamp of the newest sample in the fit - refHead float64 // FITTED head at refMs (not the raw sample: one poisoned sample must not define the anchor) - vPerSec float64 // robust velocity, blocks/second - spread float64 // median absolute residual around the fit, blocks - spanMs int64 // newest − oldest sample timestamp - maxGapMs int64 // largest interval BETWEEN two consecutive samples in the window - count int -} - -// expectedAt extrapolates the fit to an instant on the tracker's timeline. -func (f *tipFit) expectedAt(nowMs int64) float64 { - return f.refHead + f.vPerSec*float64(nowMs-f.refMs)/1000 -} - -// confident is the gate: unless EVERY condition holds the referee is a no-op -// and the caller's majority pick stands byte-identically. prevSampleAgeMs is -// the age of the freshest sample as of the START of this evaluation — the -// evaluation's own sample must not paper over a gap in the history. -func (f *tipFit) confident(nowMs int64, prevSampleAgeMs int64, p TipTrajectoryParams) bool { - if f == nil || f.count < tipMinSamples { - return false - } - if f.spanMs < p.Window.Milliseconds() { - // Also the warm-up condition: an empty buffer after boot has no span. - return false - } - if prevSampleAgeMs > tipMaxSampleGap.Milliseconds() || f.maxGapMs > tipMaxSampleGap.Milliseconds() { - // Both ends of the same rule: the track must be unbroken up to now AND - // unbroken across the whole window it was fitted over (see - // tipMaxSampleGap). A hole inside the window stands the referee down - // until the hole has scrolled out of the ring. - return false - } - if !(f.vPerSec > 0) || math.IsInf(f.vPerSec, 0) { - // A non-advancing (or NaN) fit describes a halted chain, where the - // majority pick is already the right answer. - return false - } - return !math.IsInf(f.refHead, 0) && !math.IsNaN(f.refHead) && nowMs >= f.refMs -} - -// TipTrajectory is one network+axis's long-term head track: a ring of head -// samples plus the robust fit over them. The zero value is ready to use and -// allocates on first use; a tracker whose Window is <= 0 never allocates at -// all. -type TipTrajectory struct { - mu sync.Mutex - samples []tipSample - next int // ring write cursor - count int - - // window is the TipTrajectoryParams.Window the ring was sized for. The - // params are re-read on every evaluation (a config reload takes effect - // without rebuilding the tracker), but the ring is sized once — so a raised - // window would otherwise demand a span the existing ring can never hold, and - // the referee would stand down forever while looking configured. A change - // therefore rebuilds the ring and clears the fit and the dwell: the tracker - // re-warms from live observations, which is the same inert-then-confident - // path every process takes after a deploy. - window time.Duration - - // ordered / slopes / resid are refit scratch, retained so the periodic fit - // allocates nothing after warm-up. - ordered []tipSample - slopes []float64 - resid []float64 - - // base is the instant the first sample was recorded; every timestamp below - // is milliseconds since it, measured with time.Time's MONOTONIC reading. - // A velocity fit is exactly the thing an NTP step would corrupt (a wall - // clock that jumps injects a slope no chain ever had), so the timeline the - // fit lives on never touches wall time. - base atomic.Pointer[time.Time] - lastSampleMs atomic.Int64 - fit atomic.Pointer[tipFit] - - // dwell is the current corroboration stretch: which group has been the - // elected one, since when, and where its head was then. nil = no stretch in - // progress. Immutable once stored, so a dwell that is simply CONTINUING - // costs one atomic load per evaluation and no write at all. - dwell atomic.Pointer[tipDwell] -} - -// tipDwell is one continuous stretch during which the same group of upstreams -// has been elected by the trajectory and has stayed within tolerance of the -// expected head. Its fields are never mutated after the Store; a change of -// group, a miss, or a loss of confidence replaces or clears the whole struct. -type tipDwell struct { - // group is the order-independent identity of the group's MEMBERS - // (groupIdentity), not of their head values: a stretch belongs to a set of - // upstreams, and any change to that set starts a new one. - group uint64 - - // startMs is when the stretch began, and startHead the group's max head at - // that instant — the two ends of the velocity-agreement measurement. - startMs int64 - startHead int64 -} - -// SampleCount returns how many samples the tracker holds. Diagnostics and -// tests only — the decision path never needs it. -func (t *TipTrajectory) SampleCount() int { - t.mu.Lock() - defer t.mu.Unlock() - return t.count -} - -// Observe records this evaluation's live-head median (throttled) and returns -// the trajectory's advice for it. -// -// `sorted` must be ServedTipPick.Sorted for the LIVE head observations of this -// same evaluation (the caller drops availability-bound-capped upstreams first — -// a serving range is not a head), i.e. already filtered and DESCENDING, and -// `median` the plain majority pick over them. Both the sample and the candidate -// groups come from those heads, so the referee's own output never feeds back -// into its evidence. -func (t *TipTrajectory) Observe(now time.Time, sorted []common.ServedTipInput, median int64, p TipTrajectoryParams) TipTrajectoryDecision { - d := TipTrajectoryDecision{Pick: median} - if p.Window <= 0 || median <= 0 { - return d - } - if len(sorted) < tipMinGroupSize { - // Nothing here can ever produce a decision — no group of two can form — - // so nothing is recorded either. A single-upstream network (the majority - // of chains in a large deployment) therefore allocates no ring, runs no - // fit, and costs two comparisons per evaluation. - return d - } - - nowMs, prevSampleAgeMs := t.record(now, median, p) - - fit := t.fit.Load() - if !fit.confident(nowMs, prevSampleAgeMs, p) { - // No confident fit means no elected group, so any stretch in progress is - // over: a group must re-earn its dwell once the referee can see again. - t.breakDwell() - return d - } - - expected := fit.expectedAt(nowMs) - tolerance := float64(p.ToleranceFloor) - if widened := tipToleranceSigmas * fit.spread; widened > tolerance { - tolerance = widened - } - d.Expected = int64(math.Round(expected)) - d.Tolerance = int64(math.Round(tolerance)) - d.VelocityPerSec = fit.vPerSec - - // Single-linkage clustering over the descending heads: a gap wider than one - // cluster width starts a new group. Candidate groups are those with at - // least tipMinGroupSize members; the winner is the one whose MAX (its best - // claim about the head — using the min would penalise a wide group for its - // slowest member) sits closest to where the trajectory says the head is. - width := tipClusterWidth(fit.vPerSec) - bestDistance := math.Inf(1) - var bestMin, bestMax int64 - var bestGroup uint64 - for i := 0; i < len(sorted); { - j := i + 1 - for j < len(sorted) && sorted[j-1].BlockNumber-sorted[j].BlockNumber <= width { - j++ - } - if j-i >= tipMinGroupSize { - groupMax, groupMin := sorted[i].BlockNumber, sorted[j-1].BlockNumber - if distance := math.Abs(float64(groupMax) - expected); distance < bestDistance { - bestDistance, bestMin, bestMax = distance, groupMin, groupMax - bestGroup = groupIdentity(sorted[i:j]) - } - } - i = j - } - - if bestDistance > tolerance { - // No candidate group at all (+Inf), or the elected one is nowhere near - // the trajectory — nothing here is better evidence than the majority, - // and whatever stretch was in progress just missed. - t.breakDwell() - d.Declined = bestMin > median - return d - } - - // The elected group keeps (or starts) its dwell whether or not it is above - // the majority pick: in steady state the whole fleet is that group, and it - // arrives at a stall with its corroboration already earned. - corroborated := t.corroborate(nowMs, bestGroup, bestMax, fit.vPerSec) - - if bestMin <= median { - // UPWARD-ONLY (see the section comment): the referee exists to let a - // corroborated fresh minority outvote a stalled majority. Lowering or - // holding the tip is the majority picker's and the regression guard's - // job, and is the only direction from which a frozen tip is reachable. - return d - } - if expected-float64(median) <= fit.vPerSec*tipMajorityStallSeconds { - // THE MAJORITY HAS NOT STALLED, so there is nothing here to correct. - // Everything above establishes that some group is on the trajectory and - // sits above the majority; none of it establishes that the majority is - // OFF the trajectory, which is the only thing that justifies trading the - // order statistic's servability guarantee for a fresher tip. - // - // That gap was load-bearing. Once a fleet splits wider than one cluster - // width, the upper group is frequently the ONLY electable group — the - // lower upstreams are singletons, and tipMinGroupSize rejects them — so - // the tolerance gate is the sole remaining check, and at ToleranceFloor's - // 1024-block default it admits essentially any split a real fleet - // produces. Election then implies override, and the referee fires on - // ordinary poller skew. - // - // Production, on a ~1.66 blocks/s chain with four upstreams: majority - // pick 30,699,698, elected group 30,699,722, expected 30,699,716, - // tolerance 1024. The majority was 18 blocks — eleven seconds — off the - // trajectory, and the referee raised the tip by 24 blocks every ~28 - // minutes. Nothing had stalled. - // - // Declined, not silent: an override was genuinely on the table, and the - // near miss belongs in the same counter as the other refusals. - d.Declined = true - return d - } - if !corroborated { - // An override was on the table and the group has not held its place long - // enough (or has not moved its own head) to earn it — the same outcome, - // and the same counter, as a group that sits too far from the - // trajectory. - d.Declined = true - return d - } - - // Serve the group's MINIMUM: every member of a winning group already has - // that block, so the advertised tip stays servable by all of them — the - // same invariant the majority order statistic provides. - d.Pick = bestMin - d.Overrode = true - return d -} - -// corroborate reports whether `group` has EARNED the right to outvote the -// majority, and keeps the dwell state that answers it. -// -// Two conditions, both measured over one continuous stretch of being the -// elected group: -// -// - it has been that group for at least tipDwellDuration. A chain halt sweeps -// the expected head upward through every fixed offset, so an instant match -// says only "the sweep is passing over this group right now"; -// - its own head has advanced over the stretch by at least -// tipDwellMinVelocityFraction of what the fitted velocity says the chain -// produced. A fork, a wrong-chain pair, or a frozen counter parked at a -// plausible offset advances by nothing and can never pass. -// -// Concurrency: several serving goroutines may race here; the only outcomes are -// "the stretch restarts" and "the stretch continues", and a restart is the -// conservative direction (it can delay an override, never cause one). -func (t *TipTrajectory) corroborate(nowMs int64, group uint64, groupMax int64, vPerSec float64) bool { - d := t.dwell.Load() - if d == nil || d.group != group || nowMs < d.startMs { - t.dwell.Store(&tipDwell{group: group, startMs: nowMs, startHead: groupMax}) - return false - } - seconds := float64(nowMs-d.startMs) / 1000 - if seconds < tipDwellDuration.Seconds() { - return false - } - return float64(groupMax-d.startHead) >= tipDwellMinVelocityFraction*vPerSec*seconds -} - -// breakDwell ends any stretch in progress. Load-first, so the common case (no -// stretch, or a fleet that has been in one cluster for hours) writes nothing. -func (t *TipTrajectory) breakDwell() { - if t.dwell.Load() != nil { - t.dwell.Store(nil) - } -} - -// tipGroupIDScratch is how many member ids groupIdentity can sort without -// touching the heap. Real fleets are single digits; anything larger falls back -// to an allocation rather than to a wrong answer. -const tipGroupIDScratch = 32 - -// groupIdentity fingerprints a group by its MEMBERS: the ids are sorted, then -// hashed as one separated stream. Sorting is what makes the identity a property -// of the SET while keeping the hash's avalanche intact. -// -// The obvious cheap alternative — a commutative fold, e.g. summing per-id -// hashes — is structurally broken here, not merely weaker. FNV-1a's last step -// is (h ^ c) * prime, so for two ids sharing a prefix the DIFFERENCE between -// their hashes barely depends on that prefix; a fleet named with a shared -// vendor prefix and a numeric suffix therefore collides on suffix swaps: -// {prism-celo-1, nirvana-celo-2} and {prism-celo-2, nirvana-celo-1} folded to -// the same number. A review found four such colliding pairs inside one -// realistic 12-node fleet, and a collision here is not cosmetic — it lets a -// DIFFERENT group inherit the dwell an earlier one earned, which is exactly the -// corroboration the dwell exists to withhold. -// -// The caller's slice is in head order, so the sort works on a copy: a stack -// array for any realistic fleet, and the request path stays lock-free (a shared -// scratch buffer would have to be taken under the tracker's mutex, serialising -// every evaluation to save an allocation that does not happen). -func groupIdentity(group []common.ServedTipInput) uint64 { - const ( - fnvOffset64 = uint64(14695981039346656037) - fnvPrime64 = uint64(1099511628211) - ) - var stack [tipGroupIDScratch]string - var ids []string - if len(group) <= len(stack) { - ids = stack[:0] - } else { - ids = make([]string, 0, len(group)) - } - for _, in := range group { - ids = append(ids, in.UpstreamID) - } - // Insertion sort, not sort.Strings: the slices are tiny, and the interface - // conversion sort.Strings performs would force the scratch array onto the - // heap on every evaluation. - for i := 1; i < len(ids); i++ { - for j := i; j > 0 && ids[j-1] > ids[j]; j-- { - ids[j-1], ids[j] = ids[j], ids[j-1] - } - } - - h := fnvOffset64 - for _, id := range ids { - for i := 0; i < len(id); i++ { - h ^= uint64(id[i]) - h *= fnvPrime64 - } - // A separator, so {"ab","c"} and {"a","bc"} are different sets. - h ^= 0x1f - h *= fnvPrime64 - } - return h -} - -// record appends a sample if at least tipSampleInterval has passed since the -// last one. It returns `now` on the tracker's monotonic timeline, and the age -// there of the previous freshest sample — the age BEFORE this evaluation's own -// sample, so a gap in the history cannot be papered over by the sample that -// notices it (a value past tipMaxSampleGap when there is no previous sample). -// -// The lock is taken only when a sample is actually due, so the per-request cost -// is two atomic loads. -func (t *TipTrajectory) record(now time.Time, head int64, p TipTrajectoryParams) (nowMs int64, prevAgeMs int64) { - staleSentinel := tipMaxSampleGap.Milliseconds() + 1 - - if base := t.base.Load(); base != nil { - nowMs = int64(now.Sub(*base) / time.Millisecond) - if last := t.lastSampleMs.Load(); nowMs-last < tipSampleInterval.Milliseconds() { - return nowMs, nowMs - last - } - } - - t.mu.Lock() - defer t.mu.Unlock() - - base := t.base.Load() - if base == nil { - at := now - base = &at - t.base.Store(base) - } - nowMs = int64(now.Sub(*base) / time.Millisecond) - - prevAgeMs = staleSentinel - if t.count > 0 { - last := t.lastSampleMs.Load() - if nowMs-last < tipSampleInterval.Milliseconds() { - return nowMs, nowMs - last - } - prevAgeMs = nowMs - last - } - - if t.samples == nil || t.window != p.Window { - // First use, or the operator changed trajectoryWindow under a live - // process (see TipTrajectory.window): size the ring for the window in - // force and start the track again, since neither the old samples' span - // nor a fit over them describes the new configuration. - t.samples = make([]tipSample, tipBufferCapacity(p.Window)) - t.window = p.Window - t.next, t.count = 0, 0 - t.fit.Store(nil) - t.dwell.Store(nil) - } - t.samples[t.next] = tipSample{atMs: nowMs, head: head} - t.next = (t.next + 1) % len(t.samples) - if t.count < len(t.samples) { - t.count++ - } - t.lastSampleMs.Store(nowMs) - t.refit() - - return nowMs, prevAgeMs -} - -// refit recomputes the robust fit over the whole buffer. Called under t.mu, at -// most once per tipSampleInterval. -// -// Velocity is the MEDIAN OF HALF-WINDOW-LAGGED SLOPES: slope_i between sample i -// and sample i+n/2, median over the ~n/2 of them. Two properties matter here -// and full Theil-Sen buys neither: every slope spans about half the window, so -// per-sample noise is divided by a long baseline; and any single poisoned -// sample enters at most two of the ~90 slopes, so the median never moves. -// Theil-Sen's n²/2 pairs (≈16k at n=180) would cost a 128 KB sort per fit for -// the same answer. The intercept is the median residual (the standard robust -// intercept), and the spread is the median absolute residual around the -// resulting line — the window's own noise, which is what widens the tolerance -// for a chain that pauses and bursts. -func (t *TipTrajectory) refit() { - n := t.count - if n < tipMinSamples { - t.fit.Store(nil) - return - } - - // Materialise the ring chronologically. - ord := t.ordered[:0] - if t.count < len(t.samples) { - ord = append(ord, t.samples[:t.count]...) - } else { - ord = append(ord, t.samples[t.next:]...) - ord = append(ord, t.samples[:t.next]...) - } - t.ordered = ord - - newest := ord[n-1] - lag := n / 2 - slopes := t.slopes[:0] - for i := 0; i+lag < n; i++ { - dt := float64(ord[i+lag].atMs-ord[i].atMs) / 1000 - if dt <= 0 { - continue - } - slopes = append(slopes, float64(ord[i+lag].head-ord[i].head)/dt) - } - t.slopes = slopes - if len(slopes) == 0 { - t.fit.Store(nil) - return - } - vPerSec := medianInPlace(slopes) - - // Residuals in coordinates relative to the newest sample: seconds and - // blocks, never epoch milliseconds or 8-digit block numbers, so float64 - // keeps full precision on chains of any height. - resid := t.resid[:0] - var maxGapMs int64 - for i, s := range ord { - if i > 0 { - if gap := s.atMs - ord[i-1].atMs; gap > maxGapMs { - maxGapMs = gap - } - } - ts := float64(s.atMs-newest.atMs) / 1000 - resid = append(resid, float64(s.head-newest.head)-vPerSec*ts) - } - t.resid = resid - intercept := medianInPlace(resid) - for i, r := range resid { - resid[i] = math.Abs(r - intercept) - } - - t.fit.Store(&tipFit{ - refMs: newest.atMs, - refHead: float64(newest.head) + intercept, - vPerSec: vPerSec, - spread: medianInPlace(resid), - spanMs: newest.atMs - ord[0].atMs, - maxGapMs: maxGapMs, - count: n, - }) -} - -// tipBufferCapacity sizes the ring from the configured window: enough samples -// to span it with headroom for jitter, so a window an operator raises still -// becomes reachable instead of silently never satisfying the span gate. The -// hard cap means windows beyond ~85 minutes can never be spanned, and would -// stand the referee down permanently — see EvmServedTipConfig.TrajectoryWindow. -func tipBufferCapacity(window time.Duration) int { - n := int(window/tipSampleInterval)*5/4 + 8 - if n < tipMinBufferCapacity { - return tipMinBufferCapacity - } - if n > tipMaxBufferCapacity { - return tipMaxBufferCapacity - } - return n -} - -// tipClusterWidth is how far apart two heads may be and still be one view of -// the chain: tipClusterSeconds of chain progress, floored at -// tipMinClusterWidth. -func tipClusterWidth(vPerSec float64) int64 { - w := vPerSec * tipClusterSeconds - if !(w > tipMinClusterWidth) { // also catches NaN - return tipMinClusterWidth - } - if w > float64(tipMaxClusterWidth) { - return tipMaxClusterWidth - } - return int64(w) -} - -// medianInPlace sorts xs and returns its median. -func medianInPlace(xs []float64) float64 { - if len(xs) == 0 { - return 0 - } - sort.Float64s(xs) - if n := len(xs); n%2 == 1 { - return xs[n/2] - } - return (xs[len(xs)/2-1] + xs[len(xs)/2]) / 2 -} diff --git a/architecture/evm/served_tip_test.go b/architecture/evm/served_tip_test.go deleted file mode 100644 index 230bf5516..000000000 --- a/architecture/evm/served_tip_test.go +++ /dev/null @@ -1,684 +0,0 @@ -package evm - -import ( - "math" - "sync" - "testing" - "time" - - "github.com/erpc/erpc/common" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// The served-tip contract: PickServedTip returns the freshest block a strict -// MAJORITY of inputs have reached. These tests pin the order-statistic -// semantics and the two protections that motivated the design — one rogue -// far-future tip cannot move the pick, one stuck upstream cannot hold it back. - -func tipsFromInts(blocks ...int64) []common.ServedTipInput { - out := make([]common.ServedTipInput, len(blocks)) - for i, b := range blocks { - out[i] = common.ServedTipInput{UpstreamID: "u" + itoa(i), BlockNumber: b} - } - return out -} - -// ballot is what the network hands the referee: the pick's own descending, -// zero-filtered slice (common.ServedTipPick.Sorted). Tests go through PickServedTip so -// they exercise the same single sort the request path performs. -func ballot(blocks ...int64) []common.ServedTipInput { - return common.PickServedTip(tipsFromInts(blocks...)).Sorted -} - -func itoa(i int) string { - if i == 0 { - return "0" - } - s := "" - for i > 0 { - s = string(rune('0'+(i%10))) + s - i /= 10 - } - return s -} - -// ─── Long-term trajectory referee ──────────────────────────────────────────── -// -// These pin the tracker's own mechanics: the fit it derives from a head track, -// the confidence gate that keeps it inert, and the group election. The -// end-to-end behaviour (which block a network actually serves) is pinned in -// package erpc's networks_served_tip_test.go. - -// trajectoryParams is the referee's config as a network with default settings -// supplies it: a 10-minute window and the 1024-block regression tolerance. -func trajectoryParams() TipTrajectoryParams { - return TipTrajectoryParams{Window: 10 * time.Minute, ToleranceFloor: 1024} -} - -// warmTrajectory feeds `samples` observations one tipSampleInterval apart, -// advancing the head by `perSample` blocks each time, and returns the clock and -// head it stopped at. Every sample goes through Observe, i.e. the same entry -// point the request path uses. -func warmTrajectory(tr *TipTrajectory, start time.Time, head int64, perSample int64, samples int) (time.Time, int64) { - return warmTrajectoryWith(tr, start, head, perSample, samples, trajectoryParams()) -} - -func warmTrajectoryWith(tr *TipTrajectory, start time.Time, head int64, perSample int64, samples int, p TipTrajectoryParams) (time.Time, int64) { - now := start - for i := 0; i < samples; i++ { - tr.Observe(now, ballot(head, head), head, p) - now = now.Add(tipSampleInterval) - head += perSample - } - return now, head -} - -func TestTipTrajectory_FitRecoversVelocityAndExpectedHead(t *testing.T) { - var tr TipTrajectory - // 20 blocks/s (100 blocks per 5s sample), 130 samples ≈ 10m50s of history. - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - - fit := tr.fit.Load() - require.NotNil(t, fit) - assert.InDelta(t, 20.0, fit.vPerSec, 0.001, "robust velocity must recover the 20 blocks/s track") - assert.InDelta(t, 0.0, fit.spread, 0.001, "a perfectly linear track has no residual spread") - - // The head the fit expects one sample-interval on is the next sample's. - assert.InDelta(t, float64(head), fit.expectedAt(int64(now.Sub(*tr.base.Load())/time.Millisecond)), 1.0) -} - -func TestTipTrajectory_PoisonedSampleDoesNotMoveTheFit(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 60) - clean := tr.fit.Load() - require.NotNil(t, clean) - - // One wrong-chain / garbage observation lands in the middle of the track. - tr.Observe(now, ballot(head, head), 999_999_999, trajectoryParams()) - now, _ = warmTrajectory(&tr, now.Add(tipSampleInterval), head+100, 100, 70) - - poisoned := tr.fit.Load() - require.NotNil(t, poisoned) - assert.InDelta(t, clean.vPerSec, poisoned.vPerSec, 0.001, - "a single poisoned sample must not move the median-of-lagged-slopes velocity") - assert.Less(t, poisoned.spread, float64(1024), - "nor may it inflate the residual spread past the tolerance floor") -} - -func TestTipTrajectory_ConfidenceGate(t *testing.T) { - params := trajectoryParams() - - t.Run("ColdBufferIsInert", func(t *testing.T) { - var tr TipTrajectory - now := time.Now() - for i := 0; i < tipMinSamples-1; i++ { - d := tr.Observe(now, ballot(1_000_000+int64(i)*100, 999_000), 999_000, params) - require.False(t, d.Overrode, "sample %d: a cold tracker must never override", i) - now = now.Add(tipSampleInterval) - } - assert.Nil(t, tr.fit.Load(), "no fit below tipMinSamples") - }) - - t.Run("ShortWindowIsInert", func(t *testing.T) { - var tr TipTrajectory - // 60 samples = 5 minutes of span: enough points, not enough window. - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 60) - fit := tr.fit.Load() - require.NotNil(t, fit) - require.Less(t, fit.spanMs, params.Window.Milliseconds()) - - // A stalled majority + a fresh pair: the shape that WOULD be overridden. - d := tr.Observe(now, ballot(head, head, head-20_000, head-20_000, head-20_000), head-20_000, params) - assert.False(t, d.Overrode, "span below the configured window must keep the referee inert") - }) - - t.Run("StaleTrackIsInert", func(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - require.True(t, tr.fit.Load().spanMs >= params.Window.Milliseconds()) - - // Nothing evaluated for longer than the maximum gap: the fit no longer - // describes the present, whatever it says. - now = now.Add(tipMaxSampleGap + time.Second) - d := tr.Observe(now, ballot(head+1200, head+1200, head, head, head), head, params) - assert.False(t, d.Overrode, "a gap past tipMaxSampleGap must stand the referee down") - }) - - t.Run("HaltedChainIsInert", func(t *testing.T) { - var tr TipTrajectory - // A completely flat track: velocity 0, so there is no trajectory to be - // off. The majority pick is already the right answer for a halted chain. - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 0, 130) - require.NotNil(t, tr.fit.Load()) - d := tr.Observe(now, ballot(head+5_000, head+5_000, head, head, head), head, params) - assert.False(t, d.Overrode, "v <= 0 must stand the referee down") - }) - - t.Run("DisabledRecordsNothing", func(t *testing.T) { - var tr TipTrajectory - off := TipTrajectoryParams{Window: 0, ToleranceFloor: 1024} - now := time.Now() - for i := 0; i < 200; i++ { - tr.Observe(now, ballot(1_000_000+int64(i)*100, 1_000_000+int64(i)*100), 1_000_000+int64(i)*100, off) - now = now.Add(tipSampleInterval) - } - assert.Zero(t, tr.SampleCount(), "trajectoryWindow: 0 must record nothing at all") - assert.Nil(t, tr.fit.Load()) - }) -} - -func TestTipTrajectory_GroupElection(t *testing.T) { - params := trajectoryParams() - - t.Run("StalledMajorityLosesToCorroboratedFreshPair", func(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - - // The stall: three heads freeze, two keep the track. The tracker keeps - // sampling the (frozen) majority median throughout — its evidence is - // never its own output. - frozen := head - for i := 0; i < 24; i++ { - now = now.Add(tipSampleInterval) - head += 100 - tr.Observe(now, ballot(head, head, frozen, frozen, frozen), frozen, params) - } - - d := tr.Observe(now.Add(time.Second), ballot(head, head, frozen, frozen, frozen), frozen, params) - assert.True(t, d.Overrode, "the on-trajectory pair must outvote the stalled majority") - assert.Equal(t, head, d.Pick, "the pick is the fresh group's MINIMUM — both members have it") - assert.False(t, d.Declined, "an override is not a near miss") - }) - - t.Run("SingletonIsNeverCorroboration", func(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - - frozen := head - for i := 0; i < 24; i++ { - now = now.Add(tipSampleInterval) - head += 100 - tr.Observe(now, ballot(head, frozen, frozen, frozen, frozen), frozen, params) - } - - d := tr.Observe(now.Add(time.Second), ballot(head, frozen, frozen, frozen, frozen), frozen, params) - assert.False(t, d.Overrode, - "one upstream matching the trajectory perfectly is still one witness") - assert.Equal(t, frozen, d.Pick) - }) - - t.Run("NearMissIsDeclinedNotOverridden", func(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - - // Everything is off the trajectory: the freshest pair is the closest - // group to where the head should be, but still further than the - // tolerance. The referee elects it and then refuses it — a near miss, - // which is the one non-override outcome worth counting. - d := tr.Observe(now, ballot(head+3_000, head+3_000, head-20_000, head-20_000, head-20_000), head-20_000, params) - assert.False(t, d.Overrode) - assert.True(t, d.Declined, "a refused raise is the fallback outcome") - assert.Equal(t, head-20_000, d.Pick) - }) - - t.Run("RunawayPairIsOutsideTolerance", func(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - - // Two upstreams jump far past where the chain can possibly be. - d := tr.Observe(now, ballot(head+500_000, head+500_000, head, head, head), head, params) - assert.False(t, d.Overrode, "a group that far off the trajectory must not win") - assert.Equal(t, head, d.Pick) - }) - - t.Run("UpwardOnly", func(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - - // Three upstreams run ahead, two sit exactly on the trajectory: the - // on-trajectory group wins the election, but electing it would LOWER - // the majority pick, so the referee stands down instead. - d := tr.Observe(now, ballot(head+50_000, head+50_000, head+50_000, head, head), head+50_000, params) - assert.False(t, d.Overrode, "the referee may never lower or hold the pick") - assert.Equal(t, head+50_000, d.Pick) - }) - - t.Run("PersistentSplitOnAHealthyFleetIsDeclinedNotOverridden", func(t *testing.T) { - // The production shape this gate exists for. A ~1.6 blocks/s chain with - // four upstreams, every one of them advancing, split wider than one - // cluster width: the top pair is the only electable group (the other two - // are singletons, which tipMinGroupSize rejects), so it wins the election - // unopposed and sits well inside the 1024-block tolerance. - // - // Nothing has stalled — the majority tracks the chain — so the referee - // must decline. Before the majority-stall gate it let the pair earn a 30s - // dwell and then raised the tip by the width of the split, indefinitely. - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 30_000_000, 8, 130) - - for i := 0; i < 240; i++ { // 20 minutes — far past tipDwellDuration - now = now.Add(tipSampleInterval) - head += 8 - pick := common.PickServedTip(tipsFromInts(head, head, head-24, head-64)) - d := tr.Observe(now, pick.Sorted, pick.Tip, params) - require.False(t, d.Overrode, - "sample %d: a fleet whose majority is on the trajectory must never be overridden (pick %d, majority %d, expected %d)", - i, d.Pick, pick.Tip, d.Expected) - require.Equal(t, pick.Tip, d.Pick, "sample %d: the majority pick must survive intact", i) - } - }) - - t.Run("SlowChainStallIsStillCorrected", func(t *testing.T) { - // The same slow chain, but the majority genuinely stops. The gate is - // chain-relative, so a stall crosses it after tipMajorityStallSeconds - // however few blocks that is — 48 here, against the 2400 a 20 blocks/s - // chain would produce in the same time. - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 30_000_000, 8, 130) - - frozen := head - var d TipTrajectoryDecision - for i := 0; i < 24; i++ { // 2 minutes of a frozen majority - now = now.Add(tipSampleInterval) - head += 8 - d = tr.Observe(now, ballot(head, head, frozen, frozen, frozen), frozen, params) - } - assert.True(t, d.Overrode, "a genuine stall must still be corrected on a slow chain") - assert.Equal(t, head, d.Pick, "the pick is the fresh group's minimum") - }) - - t.Run("HealthyFleetIsOneGroupAndNeverIntervenes", func(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - - // Normal propagation skew, well inside one cluster width. - pick := common.PickServedTip(tipsFromInts(head, head-1, head-3, head-7, head-11)) - d := tr.Observe(now, pick.Sorted, pick.Tip, params) - assert.False(t, d.Overrode) - assert.False(t, d.Declined, "a fleet in one cluster is not even a near miss") - assert.Equal(t, pick.Tip, d.Pick) - }) -} - -func TestTipTrajectory_BurstyChainWidensItsOwnTolerance(t *testing.T) { - var steady, bursty TipTrajectory - - _, _ = warmTrajectory(&steady, time.Now(), 1_000_000, 100, 130) - - // Same average velocity, delivered in bursts: 9 samples of nothing, then a - // 1000-block batch (the L2-sequencer shape). - now, head := time.Now(), int64(1_000_000) - for i := 0; i < 130; i++ { - if i%10 == 9 { - head += 1000 - } - bursty.Observe(now, ballot(head, head), head, trajectoryParams()) - now = now.Add(tipSampleInterval) - } - - steadyFit, burstyFit := steady.fit.Load(), bursty.fit.Load() - require.NotNil(t, steadyFit) - require.NotNil(t, burstyFit) - assert.InDelta(t, steadyFit.vPerSec, burstyFit.vPerSec, 2.0, - "both tracks average the same velocity") - assert.Greater(t, burstyFit.spread*tipToleranceSigmas, float64(1024), - "a chain that pauses and bursts must widen its own tolerance past the floor") - assert.Less(t, steadyFit.spread*tipToleranceSigmas, float64(1024), - "a metronomic chain keeps the configured floor") -} - -func TestTipTrajectory_SampleThrottleAndBufferBound(t *testing.T) { - var tr TipTrajectory - params := trajectoryParams() - now := time.Now() - - // A hot request path: 500 evaluations inside one sample interval. - for i := 0; i < 500; i++ { - tr.Observe(now.Add(time.Duration(i)*time.Millisecond), ballot(1_000_000, 1_000_000), 1_000_000, params) - } - assert.Equal(t, 1, tr.SampleCount(), "the throttle admits one sample per interval, whatever the QPS") - - // And the ring never grows past the capacity derived from the window. - _, _ = warmTrajectory(&tr, now.Add(tipSampleInterval), 1_000_100, 100, 4_000) - assert.Equal(t, tipBufferCapacity(params.Window), tr.SampleCount()) - assert.LessOrEqual(t, tr.fit.Load().spanMs, int64(tipBufferCapacity(params.Window))*tipSampleInterval.Milliseconds()) -} - -// The request path calls Observe from every serving goroutine at once, so the -// throttle's double-checked lock, the refit and the lock-free fit read all have -// to hold up under -race. -func TestTipTrajectory_ConcurrentObserve(t *testing.T) { - var tr TipTrajectory - params := trajectoryParams() - start := time.Now() - - var wg sync.WaitGroup - for g := 0; g < 8; g++ { - wg.Add(1) - go func() { - defer wg.Done() - for i := 0; i < 200; i++ { - head := int64(1_000_000 + i*100) - tr.Observe(start.Add(time.Duration(i)*tipSampleInterval), ballot(head, head-1), head, params) - } - }() - } - wg.Wait() - - assert.Positive(t, tr.SampleCount()) - assert.LessOrEqual(t, tr.SampleCount(), tipBufferCapacity(params.Window), - "the ring must stay bounded however many goroutines record into it") -} - -// ─── Corroboration must be EARNED (dwell + velocity agreement) ─────────────── -// -// Matching the trajectory in one instant is not evidence. These pin the two -// exploits of the instant test, both confirmed by simulation against the first -// version of the referee. - -// in / ballotOf build a ballot with EXPLICIT upstream ids, so a test can change -// a group's MEMBERSHIP without changing any head value. -func in(id string, blk int64) common.ServedTipInput { - return common.ServedTipInput{UpstreamID: id, BlockNumber: blk} -} -func ballotOf(ins ...common.ServedTipInput) []common.ServedTipInput { - return common.PickServedTip(ins).Sorted -} - -// A GENUINE chain halt, with no traffic gap at all: the process evaluates every -// 5s, the honest majority is frozen at the last real head, and two upstreams sit -// at a FIXED higher block (a fork, a wrong-chain pair, a shared counter frozen -// mid-flight). Because the fitted trajectory keeps extrapolating, the expected -// head SWEEPS upward through every fixed offset — so the static pair is "on -// trajectory" for as long as the sweep takes to cross it. Measured against the -// instant test, the +2000 pair won 74 of 240 halt evaluations, i.e. minutes of a -// halted chain served from a fork. -func TestTipTrajectory_HaltSweepCannotElectAStaticGroup(t *testing.T) { - for _, ahead := range []int64{2_000, 6_000, 20_000} { - var tr TipTrajectory - p := trajectoryParams() - now, frozen := warmTrajectory(&tr, time.Now(), 74_500_000, 100, 140) - require.NotNil(t, tr.fit.Load(), "precondition: the referee is warm and confident") - - rogue := frozen + ahead - overrides, contended := 0, 0 - for k := 0; k < 240; k++ { // 20 minutes of halt, sampled every 5s - d := tr.Observe(now, ballotOf( - in("r1", rogue), in("r2", rogue), - in("h1", frozen), in("h2", frozen), in("h3", frozen), - ), frozen, p) - if d.Overrode { - overrides++ - } - if d.Declined { - contended++ - } - now = now.Add(tipSampleInterval) - } - assert.Zero(t, overrides, - "a static group cannot have produced the chain progress the fit describes, "+ - "however close the sweeping expected head passes to it (offset +%d)", ahead) - t.Logf("offset +%-6d : 0 overrides, %d evaluations where the sweep made the static pair the ELECTED group", - ahead, contended) - } -} - -// stalledFleet drives a fleet in which `fresh` upstreams sit exactly on the -// fitted trajectory and the rest are frozen `behind` blocks below it, evaluating -// every tipSampleInterval and reporting the first elapsed time at which the -// referee overrode (or -1). The elapsed times these tests assert on are ABSOLUTE -// literals, deliberately: a dwell pinned in terms of tipDwellDuration is pinned -// by nothing, since zeroing the constant would move the expectation with it. -func stalledFleet(tr *TipTrajectory, now time.Time, head int64, behind int64, run time.Duration, fresh []string, frozenIDs []string) (firstOverride time.Duration, overrides int, at map[time.Duration]bool) { - p := trajectoryParams() - start := now - frozenHead := head - behind - firstOverride, at = -1, map[time.Duration]bool{} - for elapsed := time.Duration(0); elapsed <= run; elapsed += tipSampleInterval { - ins := make([]common.ServedTipInput, 0, len(fresh)+len(frozenIDs)) - for _, id := range fresh { - ins = append(ins, in(id, head+int64(elapsed.Seconds())*20)) - } - for _, id := range frozenIDs { - ins = append(ins, in(id, frozenHead)) - } - pick := common.PickServedTip(ins) - d := tr.Observe(start.Add(elapsed), pick.Sorted, pick.Tip, p) - at[elapsed] = d.Overrode - if d.Overrode { - overrides++ - if firstOverride < 0 { - firstOverride = elapsed - } - } - } - return firstOverride, overrides, at -} - -func TestTipTrajectory_DwellMustBeEarned(t *testing.T) { - // A pair that is on the trajectory, corroborating each other, advancing at - // the chain's own velocity, and separated from the stalled majority from the - // VERY FIRST evaluation — i.e. nothing but elapsed time stands between it and - // an override. It must still wait. - t.Run("AnOnTrajectoryGroupStillWaits", func(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - - _, _, at := stalledFleet(&tr, now, head, 3_000, 45*time.Second, - []string{"u1", "u2"}, []string{"u3", "u4", "u5"}) - - for _, elapsed := range []time.Duration{0, 5 * time.Second, 15 * time.Second, 25 * time.Second} { - assert.False(t, at[elapsed], - "at %v the group has corroborated nothing yet — it has merely been right for %v", elapsed, elapsed) - } - assert.True(t, at[45*time.Second], - "but by 45s of holding that place it has earned the override the stall needs") - }) - - // The confirmed false-override shape, with the geometry taken out of the - // argument: the two clusters are further apart than one cluster width from - // the first sample, so nothing but the dwell prevents an override here. - t.Run("TransientDivergenceNeverWins", func(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - - _, overrides, _ := stalledFleet(&tr, now, head, 1_000, 15*time.Second, - []string{"u1", "u2"}, []string{"u3", "u4"}) - assert.Zero(t, overrides, - "15 seconds of a two-of-four split must never hand the tip to half the fleet") - - // And the fleet re-merging leaves nothing behind. - merged := head + 400 - pick := common.PickServedTip([]common.ServedTipInput{in("u1", merged), in("u2", merged), in("u3", merged), in("u4", merged)}) - d := tr.Observe(now.Add(20*time.Second), pick.Sorted, pick.Tip, trajectoryParams()) - assert.False(t, d.Overrode) - }) - - // A group is a SET of upstreams, not a pair of head values: replacing a - // member restarts the corroboration, because the new set has proven nothing. - // This must reach the dwell's RESET branch (an existing stretch, different - // members), not its first-evaluation nil branch. - t.Run("MembershipChangeRestartsTheDwell", func(t *testing.T) { - var tr TipTrajectory - p := trajectoryParams() - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - - frozen := head - 3_000 - stall := func(elapsed time.Duration, freshA, freshB string) TipTrajectoryDecision { - pick := common.PickServedTip([]common.ServedTipInput{ - in(freshA, head+int64(elapsed.Seconds())*20), in(freshB, head+int64(elapsed.Seconds())*20), - in("u3", frozen), in("u4", frozen), in("u5", frozen), - }) - return tr.Observe(now.Add(elapsed), pick.Sorted, pick.Tip, p) - } - var elapsed time.Duration - for ; elapsed <= 60*time.Second; elapsed += tipSampleInterval { - stall(elapsed, "u1", "u2") - } - require.True(t, stall(elapsed, "u1", "u2").Overrode, "precondition: {u1,u2} has earned its override") - - earned := tr.dwell.Load() - require.NotNil(t, earned, "precondition: a stretch is in progress, so the swap hits the RESET branch") - - elapsed += tipSampleInterval - d := stall(elapsed, "u1", "u6") - assert.False(t, d.Overrode, "a different set of upstreams starts a new dwell") - assert.True(t, d.Declined, "and the refusal is the fallback outcome, not silence") - - restarted := tr.dwell.Load() - require.NotNil(t, restarted) - assert.NotEqual(t, earned.group, restarted.group, "the stretch belongs to the new member set") - assert.Greater(t, restarted.startMs, earned.startMs, "and it starts now, not when the old one did") - }) -} - -// A group's identity must survive being renamed by the fleet's own naming -// convention. The commutative fold this replaced collided on suffix swaps — -// nineteen colliding pairs inside one realistic twelve-node fleet — and a -// collision lets a DIFFERENT set of upstreams inherit a dwell it never earned. -func TestTipTrajectory_GroupIdentityDistinguishesSuffixSwaps(t *testing.T) { - a := []common.ServedTipInput{in("prism-celo-1", 10), in("nirvana-celo-2", 10)} - b := []common.ServedTipInput{in("prism-celo-2", 10), in("nirvana-celo-1", 10)} - - assert.NotEqual(t, groupIdentity(a), groupIdentity(b), - "two different pairs of the same fleet must not share an identity") - assert.Equal(t, groupIdentity(a), groupIdentity([]common.ServedTipInput{a[1], a[0]}), - "but member ORDER must not change it — the identity is a property of the set") - - // The consequence that matters: the earned dwell is not inherited. - var tr TipTrajectory - p := trajectoryParams() - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - frozen := head - 3_000 - stall := func(elapsed time.Duration, freshA, freshB string) TipTrajectoryDecision { - pick := common.PickServedTip([]common.ServedTipInput{ - in(freshA, head+int64(elapsed.Seconds())*20), in(freshB, head+int64(elapsed.Seconds())*20), - in("quicknode-celo-1", frozen), in("quicknode-celo-2", frozen), in("quicknode-celo-3", frozen), - }) - return tr.Observe(now.Add(elapsed), pick.Sorted, pick.Tip, p) - } - var elapsed time.Duration - for ; elapsed <= 60*time.Second; elapsed += tipSampleInterval { - stall(elapsed, "prism-celo-1", "nirvana-celo-2") - } - require.True(t, stall(elapsed, "prism-celo-1", "nirvana-celo-2").Overrode, - "precondition: {prism-celo-1, nirvana-celo-2} earned its override") - - elapsed += tipSampleInterval - d := stall(elapsed, "prism-celo-2", "nirvana-celo-1") - assert.False(t, d.Overrode, - "the suffix-swapped pair is a different set and must earn its own dwell") -} - -// The gap rule is about the WINDOW, not about the newest sample: standing the -// referee down for the single evaluation that notices a hole let the next -// evaluation — a millisecond later — extrapolate straight across it. -func TestTipTrajectory_GapMustLeaveTheWindow(t *testing.T) { - p := trajectoryParams() - - t.Run("TheEvaluationAfterTheGapIsInertToo", func(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 74_500_000, 100, 140) - - // A 10-minute traffic hole: the chain kept going, the process saw none - // of it. Then the stall shape — 2 fresh, 3 frozen far behind. - gap := 10 * time.Minute - now = now.Add(gap) - head += int64(gap.Seconds()) * 20 - frozen := head - 30_000 - tips := ballotOf( - in("a", head), in("b", head), - in("c", frozen), in("d", frozen), in("e", frozen), - ) - - d1 := tr.Observe(now, tips, frozen, p) - d2 := tr.Observe(now.Add(time.Millisecond), tips, frozen, p) - assert.False(t, d1.Overrode, "the evaluation that notices the gap is inert") - assert.False(t, d2.Overrode, - "and so is the next one: the fit still spans the hole it cannot see across") - }) - - t.Run("HaltInsideTheGapCannotLiftExpected", func(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 74_500_000, 100, 140) - - // 10 minutes of neither traffic nor chain progress. The robust intercept - // would otherwise lift the fit's anchor by a whole gap of chain, putting - // "expected" thousands of blocks above the true halted head — exactly - // where a rogue pair would be waiting. - now = now.Add(10 * time.Minute) - _ = tr.Observe(now, ballotOf(in("h1", head), in("h2", head), in("h3", head)), head, p) - - fit := tr.fit.Load() - require.NotNil(t, fit) - nowMs := int64(now.Sub(*tr.base.Load()) / time.Millisecond) - inflation := int64(fit.expectedAt(nowMs)) - head - require.Positive(t, inflation, "precondition: the raw fit does extrapolate across the hole") - - d := tr.Observe(now.Add(time.Millisecond), ballotOf( - in("r1", head+inflation), in("r2", head+inflation), - in("h1", head), in("h2", head), in("h3", head), - ), head, p) - assert.False(t, d.Overrode, - "a pair sitting exactly where the gap-inflated fit expects the head must not win") - assert.Zero(t, d.Expected, "the fit is refused outright, so nothing is even compared to it") - }) -} - -// The params are re-read on every evaluation so a config reload takes effect -// without rebuilding the tracker — but the ring was sized once, at first use. An -// operator RAISING trajectoryWindow therefore asked for a span the existing ring -// could never hold, and the referee stood down forever while looking configured. -// A window change now rebuilds the ring and re-warms. -func TestTipTrajectory_WindowChangeRebuildsTheRing(t *testing.T) { - var tr TipTrajectory - now, head := warmTrajectory(&tr, time.Now(), 1_000_000, 100, 130) - require.NotNil(t, tr.fit.Load(), "precondition: warm and confident at the original window") - require.Greater(t, tr.SampleCount(), tipMinSamples) - - // The same tracker, now asked for a window its ring cannot span. - raised := TipTrajectoryParams{Window: 30 * time.Minute, ToleranceFloor: 1024} - require.Greater(t, tipBufferCapacity(raised.Window), tipBufferCapacity(10*time.Minute), - "precondition: the raised window needs a bigger ring") - - tr.Observe(now, ballot(head, head), head, raised) - assert.Equal(t, 1, tr.SampleCount(), "the track restarts on the new window") - assert.Nil(t, tr.fit.Load(), "and no fit from the old one survives it") - assert.Nil(t, tr.dwell.Load(), "nor any corroboration earned under it") - - // Re-warming at the new window works — i.e. the referee comes back rather - // than standing down forever. - now, _ = warmTrajectoryWith(&tr, now.Add(tipSampleInterval), head+100, 100, 420, raised) - fit := tr.fit.Load() - require.NotNil(t, fit) - assert.GreaterOrEqual(t, fit.spanMs, raised.Window.Milliseconds(), - "the rebuilt ring can span the window the operator asked for") - _ = now -} - -// A ballot that cannot form a group can never produce a decision, so it must not -// pay for one: no ring, no fit, no periodic O(n log n) refit. Most chains in a -// large deployment have exactly one upstream. -func TestTipTrajectory_BallotBelowGroupSizeRecordsNothing(t *testing.T) { - var tr TipTrajectory - p := trajectoryParams() - now := time.Now() - for i := 0; i < 200; i++ { - head := 1_000_000 + int64(i)*100 - tr.Observe(now, ballot(head), head, p) - now = now.Add(tipSampleInterval) - } - assert.Zero(t, tr.SampleCount(), "a single-upstream network must not materialize a head track") - assert.Nil(t, tr.fit.Load()) -} - -func TestTipClusterWidth(t *testing.T) { - // Sub-second-block chains and slow chains alike keep the floor; a fast - // chain gets 10 seconds of its own progress. - assert.Equal(t, int64(tipMinClusterWidth), tipClusterWidth(0.083), "12s blocks → floor") - assert.Equal(t, int64(tipMinClusterWidth), tipClusterWidth(1), "1s blocks → floor") - assert.Equal(t, int64(40), tipClusterWidth(4), "250ms blocks → 10s of progress") - assert.Equal(t, int64(tipMinClusterWidth), tipClusterWidth(math.NaN()), "NaN must fall to the floor") - assert.Equal(t, tipMaxClusterWidth, tipClusterWidth(math.MaxFloat64), "no int64 overflow") -} diff --git a/architecture/evm/util.go b/architecture/evm/util.go index 9f5027a53..bb865ce2b 100644 --- a/architecture/evm/util.go +++ b/architecture/evm/util.go @@ -48,6 +48,10 @@ func IsMissingDataError(err error) bool { strings.Contains(txt, "cannot find transaction") || strings.Contains(txt, "after last accepted block") || strings.Contains(txt, "No state available") || + strings.Contains(txt, "no state found for block") || + (strings.Contains(txt, "state at block") && strings.Contains(txt, "is pruned")) || + strings.Contains(txt, "pruned history unavailable") || + strings.Contains(txt, "earliest available") || (strings.Contains(txt, "historical state") && (strings.Contains(txt, "is not available") || strings.Contains(txt, "unavailable"))) || strings.Contains(txt, "trie does not") || strings.Contains(txt, "greater than latest") || diff --git a/architecture/svm/cov_svm_test.go b/architecture/svm/cov_svm_test.go new file mode 100644 index 000000000..de884b39e --- /dev/null +++ b/architecture/svm/cov_svm_test.go @@ -0,0 +1,893 @@ +package svm + +import ( + "context" + "crypto/rand" + "errors" + "math" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/telemetry" + promUtil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// covSvmBadBody is a request body that cannot be parsed into a JSON-RPC +// envelope, so req.Method() and req.JsonRpcRequest() both fail. +const covSvmBadBody = `{not json` + +func covSvmBadReq() *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(covSvmBadBody)) +} + +func covSvmReqFromParams(method string, params []interface{}) *common.NormalizedRequest { + return common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest(method, params)) +} + +func covSvmResultResp(t *testing.T, req *common.NormalizedRequest, raw string) *common.NormalizedResponse { + t.Helper() + jrr, err := common.NewJsonRpcResponseFromBytes([]byte(`1`), []byte(raw), nil) + require.NoError(t, err) + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) +} + +func covSvmErrorResp(t *testing.T, req *common.NormalizedRequest, code int, msg string) *common.NormalizedResponse { + t.Helper() + jrr, err := common.NewJsonRpcResponse(1, nil, common.NewErrJsonRpcExceptionExternal(code, msg, "")) + require.NoError(t, err) + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) +} + +// ---- handler / unparseable requests ------------------------------------------- + +// Every entry point that resolves the method must surface (or tolerate) an +// unparseable body without fabricating a result. +func TestCovSvm_UnparseableRequest_EveryEntryPoint(t *testing.T) { + t.Parallel() + ctx := context.Background() + h := &SvmArchitectureHandler{} + net := &fakeNetwork{cfg: &common.NetworkConfig{ + Architecture: common.ArchitectureSvm, + Svm: &common.SvmNetworkConfig{Cluster: "mainnet-beta", Commitment: "finalized"}, + }} + + handled, resp, err := h.HandleProjectPreForward(ctx, net, covSvmBadReq()) + require.Error(t, err) + require.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal), "got %v", err) + require.False(t, handled) + require.Nil(t, resp) + + handled, resp, err = h.HandleNetworkPreForward(ctx, net, nil, covSvmBadReq()) + require.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal), "got %v", err) + require.False(t, handled) + require.Nil(t, resp) + + // Post-forward hooks never invent an error: the upstream outcome passes through. + goodReq := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSlot","params":[]}`)) + upResp := covSvmResultResp(t, goodReq, `5`) + got, err := h.HandleNetworkPostForward(ctx, net, covSvmBadReq(), upResp, nil) + require.NoError(t, err) + require.Same(t, upResp, got) + + upErr := common.NewErrEndpointServerSideException(errors.New("boom"), nil, 502) + got, err = h.HandleUpstreamPostForward(ctx, net, &svmUpstreamStub{}, covSvmBadReq(), upResp, upErr, false) + require.Same(t, upErr, err) + require.Same(t, upResp, got) + + // Commitment helpers degrade to their "unknown" values. + require.Equal(t, "", effectiveCommitment(ctx, net, covSvmBadReq())) + require.Equal(t, "", effectiveCommitment(ctx, net, nil)) + require.Equal(t, "none", FailsafeCommitment(ctx, net, covSvmBadReq())) + c, action, idx := resolveCommitment(ctx, net, covSvmBadReq()) + require.Equal(t, "", c) + require.Equal(t, commitmentSkip, action) + require.Equal(t, -1, idx) + + handled, _, err = networkPreForward_injectWriteCommitment(ctx, net, covSvmBadReq()) + require.NoError(t, err) + require.False(t, handled) + + // Slot helpers fail open / report "no bound". + end, ok := RequestRangeEndSlot(ctx, covSvmBadReq(), 7, 100) + require.True(t, ok) + require.EqualValues(t, 7, end, "unknown method resolves to start itself") + _, ok = RequestLowerBoundSlot(ctx, covSvmBadReq()) + require.False(t, ok) + require.Zero(t, MinContextSlotOf(ctx, covSvmBadReq())) + require.Zero(t, MinContextSlotOf(ctx, nil)) + + // Finality cannot be resolved without a method. + require.Equal(t, common.DataFinalityStateUnknown, GetFinality(ctx, net, covSvmBadReq(), nil)) + require.Equal(t, common.DataFinalityStateUnknown, GetFinality(ctx, net, nil, nil)) + + // Request identity cannot be derived either. + _, err = RequestKey(ctx, covSvmBadReq()) + require.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal), "got %v", err) + _, err = RequestKey(ctx, nil) + require.ErrorContains(t, err, "nil request") + _, err = svmRequestKey(nil) + require.ErrorContains(t, err, "nil json-rpc request") +} + +// A context-slot response on a bad request must not feed the poller. +func TestCovSvm_TrackContextSlot_Guards(t *testing.T) { + t.Parallel() + ctx := context.Background() + h := &SvmArchitectureHandler{} + newBalanceReq := func() *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getBalance","params":["Abc"]}`)) + } + + cases := []struct { + name string + req func() *common.NormalizedRequest + resp func(req *common.NormalizedRequest) *common.NormalizedResponse + }{ + {"nil response", newBalanceReq, func(*common.NormalizedRequest) *common.NormalizedResponse { return nil }}, + {"unparseable request", covSvmBadReq, func(*common.NormalizedRequest) *common.NormalizedResponse { + return covSvmResultResp(t, newBalanceReq(), `{"context":{"slot":55},"value":1}`) + }}, + {"json-rpc error response", newBalanceReq, func(r *common.NormalizedRequest) *common.NormalizedResponse { + return covSvmErrorResp(t, r, -32005, "Node is behind by 120 slots") + }}, + {"zero context slot", newBalanceReq, func(r *common.NormalizedRequest) *common.NormalizedResponse { + return covSvmResultResp(t, r, `{"context":{"slot":0},"value":1}`) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + poller := &recordingSvmPoller{} + req := tc.req() + resp := tc.resp(req) + got, err := h.HandleUpstreamPostForward(ctx, nil, &svmUpstreamStub{poller: poller}, req, resp, nil, false) + require.NoError(t, err) + require.Same(t, resp, got) + require.Zero(t, poller.lastSuggested) + require.Zero(t, poller.lastFinalizedSuggested) + }) + } + + // An upstream without a poller yet is skipped, not dereferenced. + req := newBalanceReq() + resp := covSvmResultResp(t, req, `{"context":{"slot":55},"value":1}`) + got, err := h.HandleUpstreamPostForward(ctx, nil, &svmUpstreamStub{}, req, resp, nil, false) + require.NoError(t, err) + require.Same(t, resp, got) +} + +// A successful write passes through untouched: the non-retryable wrap applies +// only to failures. +func TestCovSvm_NonRetryableWrite_SuccessPassesThrough(t *testing.T) { + t.Parallel() + h := &SvmArchitectureHandler{} + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"sendTransaction","params":["tx"]}`)) + resp := covSvmResultResp(t, req, `"5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW"`) + got, err := h.HandleUpstreamPostForward(context.Background(), nil, &svmUpstreamStub{}, req, resp, nil, false) + require.NoError(t, err) + require.Same(t, resp, got) +} + +// ---- hooks: genesis / commitment / getBlock guard / getSlot floor ------------ + +func TestCovSvm_GetGenesisHash_ShortCircuitConditions(t *testing.T) { + t.Parallel() + ctx := context.Background() + h := &SvmArchitectureHandler{} + mainnet := &fakeNetwork{cfg: &common.NetworkConfig{ + Architecture: common.ArchitectureSvm, + Svm: &common.SvmNetworkConfig{Cluster: "mainnet-beta"}, + }} + want, ok := common.KnownGenesisHash("", "mainnet-beta") + require.True(t, ok) + + // skip-cache-read asks for the upstream's live answer. + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getGenesisHash","params":[]}`)) + req.SetDirectives(&common.RequestDirectives{SkipCacheRead: "true"}) + handled, resp, err := h.HandleProjectPreForward(ctx, mainnet, req) + require.NoError(t, err) + require.False(t, handled) + require.Nil(t, resp) + + // No svm cluster configured: nothing to answer from. + noSvm := &fakeNetwork{cfg: &common.NetworkConfig{Architecture: common.ArchitectureSvm}} + req = common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getGenesisHash","params":[]}`)) + handled, resp, err = h.HandleProjectPreForward(ctx, noSvm, req) + require.NoError(t, err) + require.False(t, handled) + require.Nil(t, resp) + + // A request with no id still gets a synthesized answer with an id of its own. + req = covSvmReqFromParams("getGenesisHash", []interface{}{}) + handled, resp, err = h.HandleProjectPreForward(ctx, mainnet, req) + require.NoError(t, err) + require.True(t, handled) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.NotNil(t, jrr.ID()) + require.Equal(t, `"`+want+`"`, string(jrr.GetResultBytes())) +} + +func TestCovSvm_CommitmentInjection_LeavesRequestAloneWhenNotSafe(t *testing.T) { + t.Parallel() + ctx := context.Background() + h := &SvmArchitectureHandler{} + netWith := func(commitment string) *fakeNetwork { + return &fakeNetwork{cfg: &common.NetworkConfig{ + Architecture: common.ArchitectureSvm, + Svm: &common.SvmNetworkConfig{Commitment: commitment}, + }} + } + + cases := []struct { + name string + commitment string + body string + wantParams []interface{} + }{ + {"invalid read default", "rooted", `{"jsonrpc":"2.0","id":1,"method":"getBalance","params":["Abc"]}`, []interface{}{"Abc"}}, + {"invalid write default", "rooted", `{"jsonrpc":"2.0","id":1,"method":"sendTransaction","params":["tx"]}`, []interface{}{"tx"}}, + {"non-object write config slot", "finalized", `{"jsonrpc":"2.0","id":1,"method":"sendTransaction","params":["tx","base64"]}`, []interface{}{"tx", "base64"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := common.NewNormalizedRequest([]byte(tc.body)) + handled, resp, err := h.HandleProjectPreForward(ctx, netWith(tc.commitment), req) + require.NoError(t, err) + require.False(t, handled) + require.Nil(t, resp) + jrq, err := req.JsonRpcRequest() + require.NoError(t, err) + require.Equal(t, tc.wantParams, jrq.Params) + }) + } + + // An invalid default resolves to the node's own default, never to the bogus value. + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getBalance","params":["Abc"]}`)) + require.Equal(t, solanaDefaultCommitment, effectiveCommitment(ctx, netWith("rooted"), req)) +} + +func TestCovSvm_GetBlockGuard_ParamShapes(t *testing.T) { + t.Parallel() + ctx := context.Background() + h := &SvmArchitectureHandler{} + net := &fakeNetwork{ + cfg: &common.NetworkConfig{Architecture: common.ArchitectureSvm, Svm: &common.SvmNetworkConfig{}}, + latestSlot: 1000, + finalizedSlot: 900, + indexedSlot: 1000, + } + + forwarded := []struct { + name string + req *common.NormalizedRequest + }{ + {"no params", common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getBlock","params":[]}`))}, + {"non-numeric slot", common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getBlock","params":["abc"]}`))}, + {"slot zero", common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getBlock","params":[0]}`))}, + {"int slot below tip", covSvmReqFromParams("getBlock", []interface{}{int(800)})}, + } + for _, tc := range forwarded { + t.Run(tc.name, func(t *testing.T) { + handled, resp, err := h.HandleNetworkPreForward(ctx, net, nil, tc.req) + require.NoError(t, err) + require.False(t, handled) + require.Nil(t, resp) + }) + } + + // Native int / int64 params (internally-built requests) are guarded too. + for _, slot := range []interface{}{int(5000), int64(5000)} { + handled, resp, err := h.HandleNetworkPreForward(ctx, net, nil, covSvmReqFromParams("getBlock", []interface{}{slot})) + require.True(t, handled, "%T slot above the finalized tip must be rejected", slot) + require.Nil(t, resp) + require.True(t, common.HasErrorCode(err, common.ErrCodeEndpointMissingData), "got %v", err) + } + + // A network that is not an SvmNetwork has no tips to guard with. + handled, _, err := networkPreForward_getBlock(ctx, suffixSvmNet{id: "svm:x"}, covSvmReqFromParams("getBlock", []interface{}{int64(5000)})) + require.NoError(t, err) + require.False(t, handled) +} + +func TestCovSvm_GetSlotFloor_PassThroughCases(t *testing.T) { + t.Parallel() + ctx := context.Background() + h := &SvmArchitectureHandler{} + processedNet := &fakeNetwork{cfg: &common.NetworkConfig{ + Architecture: common.ArchitectureSvm, + Svm: &common.SvmNetworkConfig{Commitment: "processed"}, + }, latestSlot: 5000} + finalizedColdNet := &fakeNetwork{cfg: &common.NetworkConfig{Architecture: common.ArchitectureSvm}} + processedColdNet := &fakeNetwork{cfg: &common.NetworkConfig{ + Architecture: common.ArchitectureSvm, + Svm: &common.SvmNetworkConfig{Commitment: "processed"}, + }} + newSlotReq := func() *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSlot","params":[]}`)) + } + + cases := []struct { + name string + net common.Network + resp func(req *common.NormalizedRequest) *common.NormalizedResponse + }{ + {"error body", processedNet, func(r *common.NormalizedRequest) *common.NormalizedResponse { + return covSvmErrorResp(t, r, -32005, "Node is unhealthy") + }}, + {"non-numeric result", processedNet, func(r *common.NormalizedRequest) *common.NormalizedResponse { + return covSvmResultResp(t, r, `"abc"`) + }}, + {"zero result", processedNet, func(r *common.NormalizedRequest) *common.NormalizedResponse { + return covSvmResultResp(t, r, `0`) + }}, + {"non-svm network", suffixSvmNet{id: "svm:x"}, func(r *common.NormalizedRequest) *common.NormalizedResponse { + return covSvmResultResp(t, r, `100`) + }}, + {"processed with cold poller", processedColdNet, func(r *common.NormalizedRequest) *common.NormalizedResponse { + return covSvmResultResp(t, r, `100`) + }}, + // Finalized with no poller tip falls back to slot-32; a slot that small + // has no positive fallback, so the response is kept as-is. + {"finalized cold with tiny slot", finalizedColdNet, func(r *common.NormalizedRequest) *common.NormalizedResponse { + return covSvmResultResp(t, r, `10`) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := newSlotReq() + resp := tc.resp(req) + got, err := h.HandleNetworkPostForward(ctx, tc.net, req, resp, nil) + require.NoError(t, err) + require.Same(t, resp, got) + }) + } + + // Cold finalized with a real slot: capped by the one-epoch lag fallback. + req := newSlotReq() + got, err := h.HandleNetworkPostForward(ctx, finalizedColdNet, req, covSvmResultResp(t, req, `1000`), nil) + require.NoError(t, err) + require.EqualValues(t, 968, readSlot(t, got)) +} + +// ---- error normalizer ---------------------------------------------------------- + +func TestCovSvm_Extract_EdgeInputs(t *testing.T) { + t.Parallel() + e := NewJsonRpcErrorExtractor() + + // No HTTP response: nothing to classify. + require.NoError(t, e.Extract(nil, nil, common.MustNewJsonRpcResponse(1, nil, common.NewErrJsonRpcExceptionExternal(-32005, "behind", "")), newSvmStub())) + + // A codeless, messageless auth failure still names the status and uses the + // collision-free -32600 fallback, never -32016. + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} { + err := extract(t, 0, "", status) + require.True(t, common.HasErrorCode(err, common.ErrCodeEndpointUnauthorized), "got %v", err) + require.EqualValues(t, svmCodeInvalidRequest, wireCodeOf(t, err)) + require.Contains(t, err.Error(), "HTTP") + } + + // -32000 with an empty message is the generic retryable server-side bucket. + err := extract(t, -32000, "", http.StatusOK) + require.True(t, common.HasErrorCode(err, common.ErrCodeEndpointServerSideException), "got %v", err) + require.EqualValues(t, -32000, wireCodeOf(t, err)) + require.False(t, isRateLimitMessage("")) +} + +// ---- ledger floor / slot bounds / slot lag --------------------------------------- + +func TestCovSvm_RequestRangeEndSlot_GetBlocksWithLimitMalformed(t *testing.T) { + t.Parallel() + ctx := context.Background() + for _, params := range []string{`[50]`, `[50,-1]`, `[50,1.5]`} { + end, ok := RequestRangeEndSlot(ctx, reqOf(t, "getBlocksWithLimit", params), 50, 80) + require.False(t, ok, "params %s must fail open", params) + require.Zero(t, end) + } + // Native int / int64 limits are accepted and resolve to the tip. + for _, limit := range []interface{}{int(10), int64(10)} { + end, ok := RequestRangeEndSlot(ctx, covSvmReqFromParams("getBlocksWithLimit", []interface{}{int64(50), limit}), 50, 80) + require.True(t, ok) + require.EqualValues(t, 80, end) + } + slot, ok := RequestLowerBoundSlot(ctx, covSvmReqFromParams("getBlock", []interface{}{int(42)})) + require.True(t, ok) + require.EqualValues(t, 42, slot) +} + +func TestCovSvm_ResolveDeclaredBound_NoAnchor(t *testing.T) { + t.Parallel() + plus := int64(10) + // firstAvailableSlotPlus without a poller has no floor to anchor to. + require.Equal(t, int64(math.MinInt64), resolveDeclaredBound(&common.SvmAvailabilityBoundConfig{FirstAvailableSlotPlus: &plus}, true, nil)) + require.Equal(t, int64(math.MaxInt64), resolveDeclaredBound(&common.SvmAvailabilityBoundConfig{FirstAvailableSlotPlus: &plus}, false, &pollerAtSlot{null: true})) + // A bound object with no field set is unbounded. + require.Equal(t, int64(math.MinInt64), resolveDeclaredBound(&common.SvmAvailabilityBoundConfig{}, true, &pollerAtSlot{slot: 5})) + require.Equal(t, int64(math.MaxInt64), resolveDeclaredBound(&common.SvmAvailabilityBoundConfig{}, false, &pollerAtSlot{slot: 5})) +} + +func TestCovSvm_SlotLagFilters_NonSvmAndPollerless(t *testing.T) { + t.Parallel() + nonSvm := newSvmStub() + noPoller := &slotLagUpstream{id: "nopoller"} + nullPoller := &slotLagUpstream{id: "null", poller: &pollerAtSlot{slot: 1, null: true}} + stale := upstreamAt("stale", 100) + fresh := upstreamAt("fresh", 1000) + + got := FilterByFinalizedSlotLag([]common.Upstream{nonSvm, noPoller, nullPoller, stale, fresh}, 10, 1000) + require.Equal(t, []common.Upstream{nonSvm, noPoller, nullPoller, fresh}, got, + "upstreams that cannot be judged stay; only the stale one is dropped") + + require.EqualValues(t, 1000, HighestFinalizedSlot([]common.Upstream{nonSvm, noPoller, nullPoller, stale, fresh})) + require.EqualValues(t, 1000, ReferenceFinalizedSlot([]common.Upstream{nonSvm, noPoller, fresh, stale}, 0)) + + got = FilterByMinContextSlot([]common.Upstream{nonSvm, nullPoller, stale}, 500, true) + require.Equal(t, []common.Upstream{nonSvm, nullPoller}, got) +} + +// ---- cache --------------------------------------------------------------------- + +func TestCovSvm_NewCache_ConfigErrors(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + mem := &common.ConnectorConfig{Id: "mem", Driver: common.DriverMemory, Memory: &common.MemoryConnectorConfig{MaxItems: 10, MaxTotalSize: "1MB"}} + bad := "lots" + + cases := []struct { + name string + cfg *common.CacheConfig + wantErr string + }{ + {"unknown driver", &common.CacheConfig{Connectors: []*common.ConnectorConfig{{Id: "x", Driver: "bogus"}}}, "failed to create connector x"}, + {"policy references missing connector", &common.CacheConfig{ + Connectors: []*common.ConnectorConfig{mem}, + Policies: []*common.CachePolicyConfig{{Connector: "nope", Network: "*", Method: "*"}}, + }, "connector nope not found"}, + {"invalid policy size", &common.CacheConfig{ + Connectors: []*common.ConnectorConfig{mem}, + Policies: []*common.CachePolicyConfig{{Connector: "mem", Network: "*", Method: "*", MaxItemSize: &bad}}, + }, "failed to create policy"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, err := NewSvmJsonRpcCache(ctx, &log.Logger, tc.cfg) + require.ErrorContains(t, err, tc.wantErr) + require.Nil(t, c) + }) + } +} + +func TestCovSvm_NewCache_CompressionLevels(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + enabled := true + blob := []byte(strings.Repeat("SolanaBlockPayload", 200)) + for _, level := range []string{"default", "Better", "best", "turbo"} { + t.Run(level, func(t *testing.T) { + c, err := NewSvmJsonRpcCache(ctx, &log.Logger, &common.CacheConfig{ + Compression: &common.CompressionConfig{Enabled: &enabled, ZstdLevel: level}, + }) + require.NoError(t, err) + out := c.compress(blob) + require.Equal(t, []byte{0x28, 0xB5, 0x2F, 0xFD}, out[:4]) + back, err := c.decompress(out) + require.NoError(t, err) + require.Equal(t, blob, back) + }) + } + + // Incompressible payloads over the threshold are stored verbatim. + c, err := NewSvmJsonRpcCache(ctx, &log.Logger, &common.CacheConfig{ + Compression: &common.CompressionConfig{Enabled: &enabled, Threshold: 64}, + }) + require.NoError(t, err) + random := make([]byte, 256) + _, err = rand.Read(random) + require.NoError(t, err) + random[0] = 0x00 // never starts with the zstd magic + require.Equal(t, random, c.compress(random)) + + // A zstd-framed value with no decoder configured cannot be served. + _, err = (&SvmJsonRpcCache{}).decompress([]byte{0x28, 0xB5, 0x2F, 0xFD, 0x01}) + require.ErrorContains(t, err, "no decoder") +} + +func covSvmPolicy(t *testing.T, conn data.Connector, cfg common.CachePolicyConfig) *data.CachePolicy { + t.Helper() + if cfg.Network == "" { + cfg.Network = "*" + } + if cfg.Method == "" { + cfg.Method = "*" + } + p, err := data.NewCachePolicy(&cfg, conn) + require.NoError(t, err) + return p +} + +func covSvmCache(t *testing.T, projectId string, policies ...*data.CachePolicy) *SvmJsonRpcCache { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + base, err := NewSvmJsonRpcCache(ctx, &log.Logger, &common.CacheConfig{}) + require.NoError(t, err) + c := base.WithProjectId(projectId) + c.SetPolicies(policies) + return c +} + +func covSvmBlockReq(params string) *common.NormalizedRequest { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getBlock","params":` + params + `}`)) + req.SetNetwork(finalizedNetwork{}) + return req +} + +func TestCovSvm_CacheGetSet_PolicyMatchErrorsPropagate(t *testing.T) { + t.Parallel() + conn := data.NewMockConnector("covsvm-badpattern") + c := covSvmCache(t, "covsvm-badpattern", covSvmPolicy(t, conn, common.CachePolicyConfig{ + Network: "(svm:*", Finality: common.DataFinalityStateFinalized, Connector: conn.Id(), + })) + ctx := context.Background() + + resp, err := c.Get(ctx, covSvmBlockReq(`[100]`)) + require.Error(t, err) + require.Nil(t, resp) + + req := covSvmBlockReq(`[100]`) + require.Error(t, c.Set(ctx, req, covSvmResultResp(t, req, `{"blockhash":"x"}`))) + conn.AssertNotCalled(t, "Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + conn.AssertNotCalled(t, "Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) +} + +func TestCovSvm_CacheGet_NoMatchingPolicyIsSkipped(t *testing.T) { + t.Parallel() + const project = "covsvm-skip" + conn := data.NewMockConnector("covsvm-skip") + c := covSvmCache(t, project, covSvmPolicy(t, conn, common.CachePolicyConfig{ + Method: "getAccountInfo", Finality: common.DataFinalityStateFinalized, Connector: conn.Id(), + })) + skipped := telemetry.MetricCacheGetSkippedTotal.WithLabelValues(project, "svm:test", "getBlock") + before := promUtil.ToFloat64(skipped) + + resp, err := c.Get(context.Background(), covSvmBlockReq(`[100]`)) + require.NoError(t, err) + require.Nil(t, resp) + require.Equal(t, 1.0, promUtil.ToFloat64(skipped)-before) + conn.AssertNotCalled(t, "Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) +} + +// Params that cannot be encoded yield no key: Get treats it as a connector +// failure (fall through to upstream) and Set reports it. +func TestCovSvm_CacheKeyEncodingFailure(t *testing.T) { + t.Parallel() + conn := data.NewMockConnector("covsvm-inf") + c := covSvmCache(t, "covsvm-inf", covSvmPolicy(t, conn, common.CachePolicyConfig{ + Finality: common.DataFinalityStateFinalized, Connector: conn.Id(), + })) + ctx := context.Background() + newReq := func() *common.NormalizedRequest { + req := covSvmReqFromParams("getBlock", []interface{}{math.Inf(1)}) + req.SetNetwork(finalizedNetwork{}) + return req + } + + resp, err := c.Get(ctx, newReq()) + require.NoError(t, err) + require.Nil(t, resp) + + req := newReq() + jrr, err := common.NewJsonRpcResponse(1, map[string]interface{}{"blockhash": "x"}, nil) + require.NoError(t, err) + err = c.Set(ctx, req, common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr)) + require.ErrorContains(t, err, "failed to encode svm cache key params") + conn.AssertNotCalled(t, "Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + conn.AssertNotCalled(t, "Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) +} + +// A corrupt compressed entry is a connector error, not a hit and not a panic. +func TestCovSvm_CacheGet_CorruptCompressedEntryIsConnectorError(t *testing.T) { + t.Parallel() + const project = "covsvm-corrupt" + conn := data.NewMockConnector("covsvm-corrupt") + conn.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]byte{0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x01, 0x02}, nil) + c, policies := newMissReasonCache(t, project, conn) + + beforeMiss := missCountsByReason(t, project, policies[0]) + beforeErr := cacheErrorTotalForProject(t, project) + resp, err := c.Get(context.Background(), newMissReasonRequest(t)) + require.NoError(t, err) + require.Nil(t, resp) + afterMiss := missCountsByReason(t, project, policies[0]) + require.Equal(t, 1.0, afterMiss["connector_error"]-beforeMiss["connector_error"]) + require.Equal(t, 0.0, afterMiss["connector_miss"]-beforeMiss["connector_miss"]) + require.Equal(t, 1.0, cacheErrorTotalForProject(t, project)-beforeErr) +} + +func TestCovSvm_CacheSet_NilAndEmptyPayloads(t *testing.T) { + t.Parallel() + conn := data.NewMockConnector("covsvm-empty") + c := covSvmCache(t, "covsvm-empty", covSvmPolicy(t, conn, common.CachePolicyConfig{ + Finality: common.DataFinalityStateFinalized, Connector: conn.Id(), Empty: common.CacheEmptyBehaviorAllow, + })) + ctx := context.Background() + + require.NoError(t, c.Set(ctx, covSvmBlockReq(`[1]`), nil)) + + // Unparseable request with a real response: the parse error is reported. + err := c.Set(ctx, covSvmBadReq(), covSvmResultResp(t, covSvmBlockReq(`[1]`), `{"a":1}`)) + require.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal), "got %v", err) + _, err = c.Get(ctx, covSvmBadReq()) + require.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal), "got %v", err) + + // A response with an id but no result bytes has nothing to store, even + // under a policy that accepts empty results. + req := covSvmBlockReq(`[2]`) + jrr, err := common.NewJsonRpcResponseFromBytes([]byte(`7`), nil, nil) + require.NoError(t, err) + require.NoError(t, c.Set(ctx, req, common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr))) + conn.AssertNotCalled(t, "Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) +} + +func TestCovSvm_CacheSet_SizeLimitAndConnectorFailure(t *testing.T) { + t.Parallel() + const project = "covsvm-setfail" + tiny := "4B" + small := data.NewMockConnector("covsvm-small") + failing := data.NewMockConnector("covsvm-failing") + failing.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(errors.New("dial tcp 10.0.0.1:6379: connect: connection refused")) + healthy := data.NewMockConnector("covsvm-healthy") + healthy.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + + sizePolicy := covSvmPolicy(t, small, common.CachePolicyConfig{ + Finality: common.DataFinalityStateFinalized, Connector: small.Id(), MaxItemSize: &tiny, + }) + c := covSvmCache(t, project, + sizePolicy, + covSvmPolicy(t, failing, common.CachePolicyConfig{Finality: common.DataFinalityStateFinalized, Connector: failing.Id()}), + covSvmPolicy(t, healthy, common.CachePolicyConfig{Finality: common.DataFinalityStateFinalized, Connector: healthy.Id()}), + ) + skipped := telemetry.MetricCacheSetSkippedTotal.WithLabelValues( + project, "svm:test", "getBlock", small.Id(), sizePolicy.String(), ttlString(sizePolicy.GetTTL()), + ) + before := promUtil.ToFloat64(skipped) + + req := covSvmBlockReq(`[3]`) + require.NoError(t, c.Set(context.Background(), req, covSvmResultResp(t, req, `{"blockhash":"abcdef"}`)), + "a failing connector must not fail the request path") + + require.Equal(t, 1.0, promUtil.ToFloat64(skipped)-before) + small.AssertNotCalled(t, "Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + failing.AssertNumberOfCalls(t, "Set", 1) + healthy.AssertNumberOfCalls(t, "Set", 1) +} + +func TestCovSvm_TtlString(t *testing.T) { + t.Parallel() + require.Equal(t, "none", ttlString(nil)) + d := 90 * time.Second + require.Equal(t, "1m30s", ttlString(&d)) +} + +func TestCovSvm_SlotRefAndNilParamsKeys(t *testing.T) { + t.Parallel() + c := &SvmJsonRpcCache{} + pk := func(params []interface{}) string { + t.Helper() + req := covSvmReqFromParams("getAccountInfo", params) + req.SetNetwork(finalizedNetwork{}) + rpc, err := req.JsonRpcRequest() + require.NoError(t, err) + g, _, err := c.generateKeys(req, rpc) + require.NoError(t, err) + return g + } + require.Equal(t, "svm:test:5", pk([]interface{}{"Abc", map[string]interface{}{"minContextSlot": int64(5)}})) + require.Equal(t, "svm:test:77", pk([]interface{}{"Abc", map[string]interface{}{"minContextSlot": "77"}})) + require.Equal(t, "svm:test:*", pk([]interface{}{"Abc", map[string]interface{}{"minContextSlot": ""}})) + require.Equal(t, "*", extractSlotRef(nil)) + + nilKey, err := svmRequestKey(common.NewJsonRpcRequest("getSlot", nil)) + require.NoError(t, err) + emptyKey, err := svmRequestKey(common.NewJsonRpcRequest("getSlot", []interface{}{})) + require.NoError(t, err) + require.Equal(t, emptyKey, nilKey, "absent and empty params are the same call") +} + +// ---- state poller ---------------------------------------------------------------- + +// covSvmFuncUpstream lets a poller test decide each response in code: block, +// panic, or return a nil response. +type covSvmFuncUpstream struct { + fakeUpstreamForPoller + id string + cfg *common.UpstreamConfig + fwd func(ctx context.Context, req *common.NormalizedRequest, key string) (*common.NormalizedResponse, error) +} + +func (u *covSvmFuncUpstream) Id() string { return u.id } +func (u *covSvmFuncUpstream) Config() *common.UpstreamConfig { + if u.cfg != nil { + return u.cfg + } + return &common.UpstreamConfig{Id: u.id, Type: common.UpstreamTypeSvm, Endpoint: "http://" + u.id} +} +func (u *covSvmFuncUpstream) Forward(ctx context.Context, req *common.NormalizedRequest, _, _ bool) (*common.NormalizedResponse, error) { + return u.fwd(ctx, req, requestKey(string(req.Body()))) +} + +func covSvmHealthyAnswer(req *common.NormalizedRequest, key string) (*common.NormalizedResponse, error) { + raw := map[string]string{ + "getHealth": `"ok"`, + "getSlot:processed": `1000`, + "getSlot:finalized": `990`, + "getMaxShredInsertSlot": `1001`, + "oldestBlock": `[200]`, + }[key] + jrr, err := common.NewJsonRpcResponseFromBytes(nil, []byte(raw), nil) + if err != nil { + return nil, err + } + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr), nil +} + +func TestCovSvmPoller_MalformedSlotAnswersAreIgnored(t *testing.T) { + t.Parallel() + up := newScriptedUpstream() + up.script("getHealth", []byte(`"ok"`)) + up.script("getSlot:processed", []byte(`null`)) + up.scriptError("getSlot:finalized", -32005, "Node is behind by 42 slots") + up.script("getMaxShredInsertSlot", []byte(`"abc"`)) + p := newPollerWithUpstream(t, up) + + require.NoError(t, p.Poll(context.Background())) + require.Zero(t, p.LatestSlot()) + require.Zero(t, p.FinalizedSlot()) + require.Zero(t, p.ShredInsertSlot()) + require.True(t, p.IsHealthy(), "a bad slot sample is not a health verdict") +} + +func TestCovSvmPoller_NilHealthResponseIsUnhealthy(t *testing.T) { + t.Parallel() + up := &covSvmFuncUpstream{id: "covsvm-nilhealth", fwd: func(_ context.Context, req *common.NormalizedRequest, key string) (*common.NormalizedResponse, error) { + if key == "getHealth" { + return nil, nil + } + return covSvmHealthyAnswer(req, key) + }} + p := newPollerWithUpstream(t, up) + require.NoError(t, p.Poll(context.Background())) + require.False(t, p.IsHealthy()) + require.EqualValues(t, 1000, p.LatestSlot()) +} + +func TestCovSvmPoller_PanickingUpstreamIsRecovered(t *testing.T) { + t.Parallel() + var panicking atomic.Bool + panicking.Store(true) + // The floor probe runs too; its panic is recovered inside the refresh closure. + up := &covSvmFuncUpstream{id: "covsvm-panic", + fwd: func(_ context.Context, req *common.NormalizedRequest, key string) (*common.NormalizedResponse, error) { + if panicking.Load() { + panic("transport exploded") + } + return covSvmHealthyAnswer(req, key) + }} + p := newPollerWithUpstream(t, up) + + require.NoError(t, p.Poll(context.Background())) + require.Zero(t, p.LatestSlot()) + + // The poller keeps working once the upstream recovers. + panicking.Store(false) + require.NoError(t, p.Poll(context.Background())) + require.EqualValues(t, 1000, p.LatestSlot()) + require.EqualValues(t, 990, p.FinalizedSlot()) +} + +func TestCovSvmPoller_FloorProbeFailuresLeaveFloorUnknown(t *testing.T) { + t.Parallel() + cases := []struct { + name string + script func(up *scriptedUpstream) + }{ + {"negative slot", func(up *scriptedUpstream) { up.script("oldestBlock", []byte(`[-5]`)) }}, + {"unparseable result", func(up *scriptedUpstream) { up.script("oldestBlock", []byte(`"abc"`)) }}, + {"transport error", func(*scriptedUpstream) {}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + up := newScriptedUpstream() + scriptAllFour(up) + tc.script(up) + p := newPollerWithUpstream(t, up) + require.NoError(t, p.Poll(context.Background())) + _, known := p.FirstAvailableSlot() + require.False(t, known) + require.Equal(t, 1, up.callCount("oldestBlock")) + require.EqualValues(t, 1000, p.LatestSlot()) + }) + } +} + +// A floor probe slower than UpdateMaxWait returns without a value; the floor +// becomes known once the background fetch lands. +func TestCovSvmPoller_SlowFloorProbeCompletesAsync(t *testing.T) { + t.Parallel() + release := make(chan struct{}) + up := &covSvmFuncUpstream{id: "covsvm-slowfloor", + fwd: func(ctx context.Context, req *common.NormalizedRequest, key string) (*common.NormalizedResponse, error) { + if key == "oldestBlock" { + select { + case <-release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return covSvmHealthyAnswer(req, key) + }} + p := newPollerWithUpstream(t, up) + + require.NoError(t, p.Poll(context.Background())) + _, known := p.FirstAvailableSlot() + require.False(t, known, "the floor must not be known before the probe answers") + + close(release) + require.Eventually(t, func() bool { + floor, known := p.FirstAvailableSlot() + return known && floor == 200 + }, 5*time.Second, 5*time.Millisecond) +} + +func TestCovSvmPoller_LargeRollbackIsRecorded(t *testing.T) { + t.Parallel() + const id = "covsvm-rollback" + up := &covSvmFuncUpstream{id: id, fwd: func(_ context.Context, req *common.NormalizedRequest, key string) (*common.NormalizedResponse, error) { + return covSvmHealthyAnswer(req, key) + }} + p := newPollerWithUpstream(t, up) + + p.SuggestLatestSlot(5000) + p.SuggestLatestSlot(1000) // 4000 back: beyond DefaultToleratedSlotRollback + require.EqualValues(t, 1000, p.LatestSlot(), "a large rollback is applied, not ignored") + require.Equal(t, 4000.0, promUtil.ToFloat64( + telemetry.MetricUpstreamBlockHeadLargeRollback.WithLabelValues("test", "", "", id, "latest"))) + + p.SuggestFinalizedSlot(3000) + p.SuggestFinalizedSlot(100) + require.EqualValues(t, 100, p.FinalizedSlot()) + require.Equal(t, 2900.0, promUtil.ToFloat64( + telemetry.MetricUpstreamBlockHeadLargeRollback.WithLabelValues("test", "", "", id, "finalized"))) +} + +func TestCovSvmPoller_NonPositiveSuggestionsIgnored(t *testing.T) { + t.Parallel() + p := newTestPoller(t) + p.suggestLatestSlot(0) + p.suggestFinalizedSlot(-1) + p.SuggestFinalizedSlot(0) + require.Zero(t, p.LatestSlot()) + require.Zero(t, p.FinalizedSlot()) + require.Zero(t, p.lastExternalFinalizedAt.Load(), "a rejected suggestion is not freshness evidence") +} + +func TestCovSvmPoller_NoUpstream(t *testing.T) { + t.Parallel() + e := &SvmStatePoller{projectId: "covsvm-noupstream"} + e.applyHealthToRouting() // nothing to cordon; must return before touching upstream + e.gauge(telemetry.MetricUpstreamSvmFirstAvailableSlot).Set(7) + require.Equal(t, 7.0, promUtil.ToFloat64( + telemetry.MetricUpstreamSvmFirstAvailableSlot.WithLabelValues("covsvm-noupstream", "", "", ""))) +} diff --git a/architecture/svm/handler.go b/architecture/svm/handler.go index 82dc00afb..dc997cebc 100644 --- a/architecture/svm/handler.go +++ b/architecture/svm/handler.go @@ -107,6 +107,9 @@ func (h *SvmArchitectureHandler) HandleUpstreamPostForward(ctx context.Context, // this never walks a multi-megabyte getBlock payload. Silent on miss. if err == nil { upstreamPostForward_trackContextSlot(ctx, network, upstream, req, resp) + // History sweep: a 200 answer that is provably incomplete for what the + // caller asked becomes MissingData so the sweep tries the next upstream. + return upstreamPostForward_historySweep(ctx, upstream, req, resp) } return resp, err } diff --git a/architecture/svm/history_sweep.go b/architecture/svm/history_sweep.go new file mode 100644 index 000000000..686427a1a --- /dev/null +++ b/architecture/svm/history_sweep.go @@ -0,0 +1,261 @@ +package svm + +import ( + "bytes" + "cmp" + "context" + "encoding/json" + "fmt" + "slices" + + "github.com/erpc/erpc/common" +) + +// solanaDefaultSignaturesLimit is the page size agave applies to +// getSignaturesForAddress when the caller omits `limit` +// (https://solana.com/docs/rpc/http/getsignaturesforaddress: "Default: 1000"). +// It is a wire-contract fact, so the short-page check can compare against it +// instead of treating an omitted limit as unknowable. +const solanaDefaultSignaturesLimit = 1000 + +// upstreamPostForward_historySweep turns two SUCCESSFUL-but-incomplete answers +// into MissingData so the sweep continues to the next upstream — the same +// "this upstream does not have it, another may" verdict a pruned validator's +// -32011 already produces, for the nodes that answer 200 instead of erroring. +// +// A short page or a null status is also the normal end of history (a young +// address, an `until` cursor, a signature that never landed), so the shape alone +// proves nothing. It continues only when another selected upstream has a known +// floor strictly below this one's (see deeperFloorSelected): only then can the +// pool hold history this upstream cannot see. An unknown floor on either side +// serves the answer as is, the behaviour without this hook. +// +// The verdict is whole-request: the original request is re-sent to the next +// upstream and this response is discarded. There is no splice or merge across +// upstreams. The discarded response stays the request's last-valid response, so +// when the deeper upstream fails the client still receives this answer. +// +// Consensus participant slots (common.IsConsensusSlot) skip this hook: parallel +// upstreams vote on the bodies they returned, and turning identical legitimate +// short pages into MissingData would make consensus agree on an error. +// +// Errors pass through untouched: -32011 / -32020 and friends are classified by +// the error normalizer and fail over on their own. +func upstreamPostForward_historySweep(ctx context.Context, u common.Upstream, r *common.NormalizedRequest, rs *common.NormalizedResponse) (*common.NormalizedResponse, error) { + if common.IsConsensusSlot(ctx) { + return rs, nil + } + if rs == nil || r == nil { + return rs, nil + } + method, err := r.Method() + if err != nil { + return rs, nil + } + switch method { + case "getSignatureStatuses": + return rs, signatureStatusesNullToMissingData(ctx, u, r, rs) + case "getSignaturesForAddress": + return rs, signaturesForAddressShortPageToMissingData(ctx, u, r, rs) + } + return rs, nil +} + +// optionsObjectAt returns the params[idx] options object, or nil when absent or +// not an object (legacy/positional encodings are left alone). +func optionsObjectAt(ctx context.Context, r *common.NormalizedRequest, idx int) map[string]interface{} { + rpcReq, err := r.JsonRpcRequest(ctx) + if err != nil || rpcReq == nil { + return nil + } + rpcReq.RLock() + defer rpcReq.RUnlock() + if idx >= len(rpcReq.Params) { + return nil + } + m, _ := rpcReq.Params[idx].(map[string]interface{}) + if m == nil { + return nil + } + // Shallow copy: callers read it after the lock is released. + out := make(map[string]interface{}, len(m)) + for k, v := range m { + out[k] = v + } + return out +} + +// signatureStatusesNullToMissingData: with `searchTransactionHistory: true` the +// caller asked every signature to be looked up in the full ledger, so a null +// entry means "this upstream did not find it", not "not recently seen". Another +// upstream with deeper history may have it. Without the flag a null is the +// valid recent-status-cache answer and must be served as is. +// +// Any null in the page triggers the sweep (whole-request semantics): the +// request is re-sent unchanged and the next upstream answers all signatures. +func signatureStatusesNullToMissingData(ctx context.Context, u common.Upstream, r *common.NormalizedRequest, rs *common.NormalizedResponse) error { + // Only a literal JSON true enables history search; absent/false/non-bool + // leaves the request on the recent-status path. + if search, _ := optionsObjectAt(ctx, r, 1)["searchTransactionHistory"].(bool); !search { + return nil + } + jrr, err := rs.JsonRpcResponse(ctx) + if err != nil || jrr == nil || jrr.Error != nil { + return nil + } + var result struct { + Value []json.RawMessage `json:"value"` + } + if err := common.SonicCfg.Unmarshal(jrr.GetResultBytes(), &result); err != nil { + return nil + } + nulls := 0 + for _, v := range result.Value { + if bytes.Equal(bytes.TrimSpace(v), []byte("null")) { + nulls++ + } + } + if nulls == 0 || !deeperFloorSelected(u, r) { + return nil + } + return newHistorySweepMissingData( + u, + fmt.Sprintf("getSignatureStatuses with searchTransactionHistory returned %d null status(es) of %d", nulls, len(result.Value)), + map[string]interface{}{"method": "getSignatureStatuses", "nulls": nulls, "total": len(result.Value)}, + ) +} + +// signaturesForAddressShortPageToMissingData: a page shorter than the page size +// the caller asked for may mean this upstream's history for the address ends at +// its floor — a pruned validator stops where its ledger does. A full page +// proves nothing is missing. The effective page size is `limit`, else +// agave's default of 1000. +// +// A `limit` that is present but not a positive integer is not ours to +// interpret: fail open and let the upstream's own answer stand. +func signaturesForAddressShortPageToMissingData(ctx context.Context, u common.Upstream, r *common.NormalizedRequest, rs *common.NormalizedResponse) error { + limit := int64(solanaDefaultSignaturesLimit) + if raw, present := optionsObjectAt(ctx, r, 1)["limit"]; present { + l, ok := toInt64Exact(raw) + if !ok || l <= 0 { + return nil + } + limit = l + } + jrr, err := rs.JsonRpcResponse(ctx) + if err != nil || jrr == nil || jrr.Error != nil { + return nil + } + raw := bytes.TrimSpace(jrr.GetResultBytes()) + if len(raw) == 0 || raw[0] != '[' { + return nil + } + var page []json.RawMessage + if err := common.SonicCfg.Unmarshal(raw, &page); err != nil { + return nil + } + if int64(len(page)) >= limit || !deeperFloorSelected(u, r) { + return nil + } + return newHistorySweepMissingData( + u, + fmt.Sprintf("getSignaturesForAddress returned a short page (%d of limit %d)", len(page), limit), + map[string]interface{}{"method": "getSignaturesForAddress", "returned": len(page), "limit": limit}, + ) +} + +// IsHistoryMethod reports whether method reads arbitrarily old ledger history, +// so the upstream with the deepest floor gives the most complete answer. +func IsHistoryMethod(method string) bool { + switch method { + case "getSignatureStatuses", "getSignaturesForAddress": + return true + } + return false +} + +// OrderByHistoryDepth stably reorders ups so that, inside each run of upstreams +// sharing a failover tier (sameTier), known floors come first, deepest first, and +// unknown floors keep their score order after them. A history read then lands on +// the most complete upstream first and the sweep in upstreamPostForward_historySweep +// rarely needs a second call. Ordering only: every upstream stays eligible. +func OrderByHistoryDepth(ups []common.Upstream, sameTier func(a, b common.Upstream) bool) []common.Upstream { + if len(ups) < 2 { + return ups + } + out := slices.Clone(ups) + for start := 0; start < len(out); { + end := start + 1 + for end < len(out) && sameTier(out[start], out[end]) { + end++ + } + slices.SortStableFunc(out[start:end], func(a, b common.Upstream) int { + fa, ka := upstreamFloor(a) + fb, kb := upstreamFloor(b) + switch { + case ka && kb: + return cmp.Compare(fa, fb) + case ka: + return -1 + case kb: + return 1 + } + return 0 + }) + start = end + } + return out +} + +// deeperFloorSelected reports whether another upstream selected for r has a +// known floor strictly below u's known floor: the only evidence that the pool +// can reach history u cannot. Ties and unknowns are not evidence, so an +// upstream at the pool's deepest floor, or one whose floor is unobserved, has +// the final word. +func deeperFloorSelected(u common.Upstream, r *common.NormalizedRequest) bool { + floor, known := upstreamFloor(u) + if !known { + return false + } + for _, other := range r.Upstreams() { + if other == u { + continue + } + if f, ok := upstreamFloor(other); ok && f < floor { + return true + } + } + return false +} + +func upstreamFloor(u common.Upstream) (int64, bool) { + sup, ok := u.(common.SvmUpstream) + if !ok { + return 0, false + } + poller := sup.SvmStatePoller() + if poller == nil || poller.IsObjectNull() { + return 0, false + } + return poller.FirstAvailableSlot() +} + +// newHistorySweepMissingData builds the incomplete-answer verdict: MissingData +// (so the sweep continues to the next upstream) marked PERMANENT. Permanence +// governs only the time-delayed re-sweep after a completed sweep, never +// cross-upstream failover (that is retryableTowardNetwork, unaffected). Waiting +// one block time cannot lengthen an address's history or make an unseen +// signature land, so re-sweeping every upstream after a delay — until +// EmptyResultMaxAttempts — would only multiply load for the same answer. The +// client then receives the request's last-valid response. +func newHistorySweepMissingData(u common.Upstream, msg string, details map[string]interface{}) error { + err := common.NewErrEndpointMissingData( + common.NewErrJsonRpcExceptionInternal(0, common.JsonRpcErrorMissingData, msg, nil, details), + u, + ) + if me, ok := err.(*common.ErrEndpointMissingData); ok { + me.WithPermanentMissingData(true) + me.WithHistorySweepContinuation(true) + } + return err +} diff --git a/architecture/svm/history_sweep_test.go b/architecture/svm/history_sweep_test.go new file mode 100644 index 000000000..5beb4d4bb --- /dev/null +++ b/architecture/svm/history_sweep_test.go @@ -0,0 +1,312 @@ +package svm + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/require" +) + +func resultResp(t *testing.T, req *common.NormalizedRequest, resultJSON string) *common.NormalizedResponse { + t.Helper() + jrr, err := common.NewJsonRpcResponseFromBytes([]byte(`1`), []byte(resultJSON), nil) + require.NoError(t, err) + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) +} + +func sigPage(n int) string { + items := make([]string, n) + for i := range items { + items[i] = fmt.Sprintf(`{"signature":"sig%d","slot":%d,"err":null}`, i, 100+i) + } + return "[" + strings.Join(items, ",") + "]" +} + +// --------------------------------------------------------------------------- +// getSignatureStatuses — continue on null when searching history +// --------------------------------------------------------------------------- + +func TestPostForward_GetSignatureStatuses_NullContinue(t *testing.T) { + t.Parallel() + ctx := context.Background() + h := &SvmArchitectureHandler{} + net := &fakeNetwork{cfg: &common.NetworkConfig{Architecture: common.ArchitectureSvm}} + up := upstreamWithFloor("u", 300000, 200000, true) + deeper := upstreamWithFloor("archive", 300000, 0, true) + + run := func(t *testing.T, params, result string) (*common.NormalizedResponse, error) { + t.Helper() + req := reqOf(t, "getSignatureStatuses", params) + req.SetUpstreams([]common.Upstream{up, deeper}) + return h.HandleUpstreamPostForward(ctx, net, up, req, resultResp(t, req, result), nil, false) + } + + const withNull = `{"context":{"slot":9},"value":[{"slot":1,"confirmations":null,"err":null,"confirmationStatus":"finalized"},null]}` + const allFound = `{"context":{"slot":9},"value":[{"slot":1,"confirmations":null,"err":null,"confirmationStatus":"finalized"}]}` + + t.Run("flag true + null element is MissingData", func(t *testing.T) { + t.Parallel() + _, err := run(t, `[["a","b"],{"searchTransactionHistory":true}]`, withNull) + require.Error(t, err) + require.True(t, common.HasErrorCode(err, common.ErrCodeEndpointMissingData)) + require.True(t, common.IsRetryableTowardNetwork(err), "the sweep must continue to the next upstream") + require.True(t, common.IsPermanentlyMissingData(err), "waiting cannot make a signature land: no delayed re-sweep") + require.True(t, common.IsHistorySweepContinuation(err), "must not count as wrong-empty misbehavior") + }) + + t.Run("flag true + all non-null is served", func(t *testing.T) { + t.Parallel() + rs, err := run(t, `[["a"],{"searchTransactionHistory":true}]`, allFound) + require.NoError(t, err) + require.NotNil(t, rs) + }) + + t.Run("flag true + empty value is served", func(t *testing.T) { + t.Parallel() + _, err := run(t, `[[],{"searchTransactionHistory":true}]`, `{"context":{"slot":9},"value":[]}`) + require.NoError(t, err) + }) + + t.Run("flag false + null is a valid recent-status answer", func(t *testing.T) { + t.Parallel() + rs, err := run(t, `[["a","b"],{"searchTransactionHistory":false}]`, withNull) + require.NoError(t, err) + require.NotNil(t, rs) + }) + + t.Run("flag absent + null is a valid recent-status answer", func(t *testing.T) { + t.Parallel() + _, err := run(t, `[["a","b"]]`, withNull) + require.NoError(t, err) + _, err = run(t, `[["a","b"],{}]`, withNull) + require.NoError(t, err) + }) + + t.Run("non-boolean flag is not treated as true", func(t *testing.T) { + t.Parallel() + _, err := run(t, `[["a","b"],{"searchTransactionHistory":"true"}]`, withNull) + require.NoError(t, err) + }) + + t.Run("upstream error passes through untouched", func(t *testing.T) { + t.Parallel() + req := reqOf(t, "getSignatureStatuses", `[["a"],{"searchTransactionHistory":true}]`) + upErr := common.NewErrEndpointServerSideException(nil, nil, 500) + _, err := h.HandleUpstreamPostForward(ctx, net, up, req, nil, upErr, false) + require.Same(t, upErr, err) + }) + + t.Run("malformed result fails open", func(t *testing.T) { + t.Parallel() + _, err := run(t, `[["a"],{"searchTransactionHistory":true}]`, `"not-an-envelope"`) + require.NoError(t, err) + }) +} + +// --------------------------------------------------------------------------- +// getSignaturesForAddress — short page is MissingData +// --------------------------------------------------------------------------- + +func TestPostForward_GetSignaturesForAddress_ShortPage(t *testing.T) { + t.Parallel() + ctx := context.Background() + h := &SvmArchitectureHandler{} + net := &fakeNetwork{cfg: &common.NetworkConfig{Architecture: common.ArchitectureSvm}} + up := upstreamWithFloor("u", 300000, 200000, true) + deeper := upstreamWithFloor("archive", 300000, 0, true) + + run := func(t *testing.T, params, result string) (*common.NormalizedResponse, error) { + t.Helper() + req := reqOf(t, "getSignaturesForAddress", params) + req.SetUpstreams([]common.Upstream{up, deeper}) + return h.HandleUpstreamPostForward(ctx, net, up, req, resultResp(t, req, result), nil, false) + } + + t.Run("limit N with N-1 results is MissingData", func(t *testing.T) { + t.Parallel() + _, err := run(t, `["addr",{"limit":5}]`, sigPage(4)) + require.Error(t, err) + require.True(t, common.HasErrorCode(err, common.ErrCodeEndpointMissingData)) + require.True(t, common.IsRetryableTowardNetwork(err)) + require.True(t, common.IsPermanentlyMissingData(err), "waiting cannot lengthen history: no delayed re-sweep") + }) + + t.Run("limit N with N results is accepted", func(t *testing.T) { + t.Parallel() + rs, err := run(t, `["addr",{"limit":5}]`, sigPage(5)) + require.NoError(t, err) + require.NotNil(t, rs) + }) + + t.Run("limit with commitment injected is still honoured", func(t *testing.T) { + t.Parallel() + _, err := run(t, `["addr",{"limit":3,"commitment":"finalized"}]`, sigPage(2)) + require.Error(t, err) + _, err = run(t, `["addr",{"limit":3,"commitment":"finalized"}]`, sigPage(3)) + require.NoError(t, err) + }) + + t.Run("omitted limit uses the 1000 default", func(t *testing.T) { + t.Parallel() + for _, params := range []string{`["addr"]`, `["addr",{"commitment":"finalized"}]`, `["addr",{"before":"x"}]`} { + _, err := run(t, params, sigPage(999)) + require.Errorf(t, err, "999 < 1000 must be short for %s", params) + require.True(t, common.HasErrorCode(err, common.ErrCodeEndpointMissingData)) + + rs, err := run(t, params, sigPage(1000)) + require.NoErrorf(t, err, "a full 1000-page must be accepted for %s", params) + require.NotNil(t, rs) + } + }) + + t.Run("empty page with limit > 0 continues", func(t *testing.T) { + t.Parallel() + _, err := run(t, `["addr",{"limit":10}]`, `[]`) + require.Error(t, err) + require.True(t, common.HasErrorCode(err, common.ErrCodeEndpointMissingData)) + }) + + t.Run("non-array result fails open", func(t *testing.T) { + t.Parallel() + _, err := run(t, `["addr",{"limit":10}]`, `null`) + require.NoError(t, err) + _, err = run(t, `["addr",{"limit":10}]`, `{"a":1}`) + require.NoError(t, err) + }) + + t.Run("invalid limit is not interpreted", func(t *testing.T) { + t.Parallel() + for _, limit := range []string{`0`, `-1`, `1.5`, `"5"`, `null`} { + _, err := run(t, fmt.Sprintf(`["addr",{"limit":%s}]`, limit), sigPage(2)) + require.NoErrorf(t, err, "limit %s must fail open", limit) + } + }) + + // The error classes that already fail over keep doing so: the hook only + // inspects successful responses and never rewrites an upstream error. + t.Run("upstream errors pass through untouched", func(t *testing.T) { + t.Parallel() + req := reqOf(t, "getSignaturesForAddress", `["addr",{"limit":5}]`) + for _, code := range []int{-32020, -32011} { + upErr := extract(t, code, "history unavailable", 200) + gotRs, err := h.HandleUpstreamPostForward(ctx, net, up, req, nil, upErr, false) + require.Nil(t, gotRs) + require.Same(t, upErr, err) + require.Truef(t, common.IsRetryableTowardNetwork(err), "%d must still fail over", code) + } + require.True(t, common.HasErrorCode(extract(t, -32011, "x", 200), common.ErrCodeEndpointMissingData)) + }) +} + +// Methods other than the two history reads are untouched. +func TestPostForward_HistorySweep_OtherMethodsUntouched(t *testing.T) { + t.Parallel() + ctx := context.Background() + h := &SvmArchitectureHandler{} + net := &fakeNetwork{cfg: &common.NetworkConfig{Architecture: common.ArchitectureSvm}} + up := upstreamWithFloor("u", 300000, 0, true) + req := reqOf(t, "getTransaction", `["sig"]`) + _, err := h.HandleUpstreamPostForward(ctx, net, up, req, resultResp(t, req, `null`), nil, false) + require.NoError(t, err) +} + +// --------------------------------------------------------------------------- +// continue only toward a known deeper floor +// --------------------------------------------------------------------------- + +func TestPostForward_HistorySweep_ContinuesOnlyTowardDeeperFloor(t *testing.T) { + t.Parallel() + ctx := context.Background() + h := &SvmArchitectureHandler{} + net := &fakeNetwork{cfg: &common.NetworkConfig{Architecture: common.ArchitectureSvm}} + + archive := upstreamWithFloor("archive", 300000, 0, true) + archive2 := upstreamWithFloor("archive2", 300000, 0, true) + pruned := upstreamWithFloor("pruned", 300000, 200000, true) + unknown := upstreamWithFloor("unknown", 300000, 0, false) + + sigs := func(u common.Upstream, pool ...common.Upstream) error { + req := reqOf(t, "getSignaturesForAddress", `["addr",{"limit":5}]`) + req.SetUpstreams(pool) + _, err := h.HandleUpstreamPostForward(ctx, net, u, req, resultResp(t, req, sigPage(2)), nil, false) + return err + } + statuses := func(u common.Upstream, pool ...common.Upstream) error { + req := reqOf(t, "getSignatureStatuses", `[["a"],{"searchTransactionHistory":true}]`) + req.SetUpstreams(pool) + _, err := h.HandleUpstreamPostForward(ctx, net, u, req, resultResp(t, req, `{"context":{"slot":9},"value":[null]}`), nil, false) + return err + } + + continues := []struct { + name string + u common.Upstream + pool []common.Upstream + }{ + {"pruned with a deeper archive selected", pruned, []common.Upstream{pruned, archive}}, + {"pruned with a deeper archive and an unknown", pruned, []common.Upstream{unknown, pruned, archive}}, + } + for _, tc := range continues { + t.Run("continues: "+tc.name, func(t *testing.T) { + t.Parallel() + for _, err := range []error{sigs(tc.u, tc.pool...), statuses(tc.u, tc.pool...)} { + require.True(t, common.HasErrorCode(err, common.ErrCodeEndpointMissingData)) + require.True(t, common.IsRetryableTowardNetwork(err)) + } + }) + } + + serves := []struct { + name string + u common.Upstream + pool []common.Upstream + }{ + {"deepest upstream", archive, []common.Upstream{pruned, archive}}, + {"tie at the deepest floor", archive, []common.Upstream{archive, archive2}}, + {"deeper upstream's floor unknown", pruned, []common.Upstream{pruned, unknown}}, + {"own floor unknown", unknown, []common.Upstream{unknown, archive}}, + {"no other upstream selected", pruned, []common.Upstream{pruned}}, + {"no pool on the request", pruned, nil}, + } + for _, tc := range serves { + t.Run("serves: "+tc.name, func(t *testing.T) { + t.Parallel() + require.NoError(t, sigs(tc.u, tc.pool...)) + require.NoError(t, statuses(tc.u, tc.pool...)) + }) + } +} + +// The history sweep is for serial sweep continuation only. Under consensus, identical +// valid short pages must stay 200s so participants can agree; MissingData would +// collapse to an agreed error with no last-valid-response fallback. +func TestPostForward_HistorySweep_SkippedUnderConsensusSlot(t *testing.T) { + t.Parallel() + ctx := common.WithConsensusSlot(context.Background()) + h := &SvmArchitectureHandler{} + net := &fakeNetwork{cfg: &common.NetworkConfig{Architecture: common.ArchitectureSvm}} + pruned := upstreamWithFloor("pruned", 300000, 200000, true) + pool := []common.Upstream{pruned, upstreamWithFloor("archive", 300000, 0, true)} + + t.Run("getSignaturesForAddress short page is served", func(t *testing.T) { + t.Parallel() + req := reqOf(t, "getSignaturesForAddress", `["addr",{"limit":5}]`) + req.SetUpstreams(pool) + rs, err := h.HandleUpstreamPostForward(ctx, net, pruned, req, resultResp(t, req, sigPage(2)), nil, false) + require.NoError(t, err) + require.NotNil(t, rs) + }) + + t.Run("getSignatureStatuses null with history search is served", func(t *testing.T) { + t.Parallel() + const withNull = `{"context":{"slot":9},"value":[null]}` + req := reqOf(t, "getSignatureStatuses", `[["a"],{"searchTransactionHistory":true}]`) + req.SetUpstreams(pool) + rs, err := h.HandleUpstreamPostForward(ctx, net, pruned, req, resultResp(t, req, withNull), nil, false) + require.NoError(t, err) + require.NotNil(t, rs) + }) +} diff --git a/architecture/svm/json_rpc_cache.go b/architecture/svm/json_rpc_cache.go index cb683bc50..78d7361d9 100644 --- a/architecture/svm/json_rpc_cache.go +++ b/architecture/svm/json_rpc_cache.go @@ -279,7 +279,7 @@ func ttlString(ttl *time.Duration) string { return ttl.String() } -// Set writes the response to every matching policy concurrently. Failures are +// Set writes the response to every matching policy sequentially. Failures are // logged and swallowed per-policy — a single flaky connector must not fail the // upstream request path. func (c *SvmJsonRpcCache) Set(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) error { @@ -358,17 +358,33 @@ func (c *SvmJsonRpcCache) Set(ctx context.Context, req *common.NormalizedRequest ).Add(float64(len(valueToStore))) } setCtx := data.WithReverseIndex(ctx, req.NetworkId(), req.CacheKeySuffix()) + // Per-connector clock: the loop is sequential, so a shared start would + // fold earlier policies' latency into later connectors' samples and + // mis-attribute "Cache Set Latency (Success)" panels. + setStart := time.Now() if err := connector.Set(setCtx, groupKey, requestKey, valueToStore, ttl); err != nil { + errSummary := common.ErrorSummary(err) telemetry.MetricCacheSetErrorTotal.WithLabelValues( c.projectId, req.NetworkLabel(), rpcReq.Method, - connector.Id(), policy.String(), ttlLabel, - common.ErrorSummary(err), + connector.Id(), policy.String(), ttlLabel, errSummary, ).Inc() + telemetry.MetricCacheSetErrorDuration.WithLabelValues( + c.projectId, req.NetworkLabel(), rpcReq.Method, + connector.Id(), policy.String(), ttlLabel, errSummary, + ).Observe(time.Since(setStart).Seconds()) c.logger.Warn().Err(err).Str("connector", connector.Id()). Str("groupKey", groupKey).Str("requestKey", requestKey). Msg("svm cache set failed") continue } + telemetry.MetricCacheSetSuccessTotal.WithLabelValues( + c.projectId, req.NetworkLabel(), rpcReq.Method, + connector.Id(), policy.String(), ttlLabel, + ).Inc() + telemetry.MetricCacheSetSuccessDuration.WithLabelValues( + c.projectId, req.NetworkLabel(), rpcReq.Method, + connector.Id(), policy.String(), ttlLabel, + ).Observe(time.Since(setStart).Seconds()) } return nil } diff --git a/architecture/svm/json_rpc_cache_set_success_test.go b/architecture/svm/json_rpc_cache_set_success_test.go new file mode 100644 index 000000000..5625f8c45 --- /dev/null +++ b/architecture/svm/json_rpc_cache_set_success_test.go @@ -0,0 +1,101 @@ +package svm + +import ( + "context" + "errors" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/telemetry" + "github.com/prometheus/client_golang/prometheus" + promUtil "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// TestSvmCache_Set_EmitsOutcomeMetrics guards EVM parity on both Set outcomes: +// each must bump its counter and record one latency sample. Without these, +// Solana "Cache Set Latency" panels stay dead. +func TestSvmCache_Set_EmitsOutcomeMetrics(t *testing.T) { + t.Parallel() + + setErr := errors.New("connector down") + cases := []struct { + name string + connErr error + total func(labels []string) prometheus.Counter + duration func(labels []string) prometheus.Observer + }{ + { + name: "success", + total: func(l []string) prometheus.Counter { + return telemetry.MetricCacheSetSuccessTotal.WithLabelValues(l...) + }, + duration: func(l []string) prometheus.Observer { + return telemetry.MetricCacheSetSuccessDuration.WithLabelValues(l...) + }, + }, + { + name: "error", + connErr: setErr, + total: func(l []string) prometheus.Counter { + return telemetry.MetricCacheSetErrorTotal.WithLabelValues(append(l, common.ErrorSummary(setErr))...) + }, + duration: func(l []string) prometheus.Observer { + return telemetry.MetricCacheSetErrorDuration.WithLabelValues(append(l, common.ErrorSummary(setErr))...) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + projectId := t.Name() + conn := data.NewMockConnector("set-outcome-mem") + conn.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(tc.connErr).Once() + + c, policies := newMissReasonCache(t, projectId, conn) + require.Len(t, policies, 1) + policy := policies[0] + + req := newMissReasonRequest(t) + jrr, err := common.NewJsonRpcResponse(1, map[string]interface{}{"blockhash": "abc"}, nil) + require.NoError(t, err) + resp := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) + + labels := []string{ + projectId, + missReasonNetworkLabel, + missReasonMethod, + conn.Id(), + policy.String(), + ttlString(policy.GetTTL()), + } + + beforeTotal := promUtil.ToFloat64(tc.total(labels)) + beforeDuration := histogramSampleCount(t, tc.duration(labels)) + + // Set swallows connector errors so the upstream request path never fails. + require.NoError(t, c.Set(context.Background(), req, resp)) + + require.Equal(t, 1.0, promUtil.ToFloat64(tc.total(labels))-beforeTotal) + require.Equal(t, uint64(1), histogramSampleCount(t, tc.duration(labels))-beforeDuration) + conn.AssertExpectations(t) + }) + } +} + +func histogramSampleCount(t *testing.T, obs prometheus.Observer) uint64 { + t.Helper() + m, ok := obs.(prometheus.Metric) + require.True(t, ok, "observer must also be a prometheus.Metric") + var pb dto.Metric + require.NoError(t, m.Write(&pb)) + h := pb.GetHistogram() + require.NotNil(t, h, "metric must be a histogram") + return h.GetSampleCount() +} diff --git a/architecture/svm/state_poller_test.go b/architecture/svm/state_poller_test.go index fdc9b4bec..5037a81a0 100644 --- a/architecture/svm/state_poller_test.go +++ b/architecture/svm/state_poller_test.go @@ -120,24 +120,6 @@ func (s *scriptedUpstream) Config() *common.UpstreamConfig { return s.fakeUpstreamForPoller.Config() } -// newScriptedUpstreamDeclaringFloor is a scriptedUpstream whose operator -// declared a firstAvailableSlotPlus lower bound — the only config under which -// the poller observes the ledger floor. -func newScriptedUpstreamDeclaringFloor() *scriptedUpstream { - up := newScriptedUpstream() - plus := int64(0) - up.cfg = &common.UpstreamConfig{ - Id: "test-poller", Type: common.UpstreamTypeSvm, Endpoint: "http://x", - Svm: &common.SvmUpstreamConfig{ - Cluster: "mainnet-beta", - BlockAvailability: &common.SvmBlockAvailabilityConfig{ - Lower: &common.SvmAvailabilityBoundConfig{FirstAvailableSlotPlus: &plus}, - }, - }, - } - return up -} - type cordonEvent struct { method string reason string @@ -279,7 +261,7 @@ func newPollerOnCtx(t *testing.T, appCtx context.Context, up common.Upstream) *S func TestSvmStatePoller_Poll_FansOutAllFiveCalls(t *testing.T) { t.Parallel() - up := newScriptedUpstreamDeclaringFloor() + up := newScriptedUpstream() up.script("getHealth", []byte(`"ok"`)) up.script("getSlot:processed", []byte(`1000`)) up.script("getSlot:finalized", []byte(`990`)) @@ -319,9 +301,12 @@ func TestSvmStatePoller_Poll_FansOutAllFiveCalls(t *testing.T) { if !p.IsHealthy() { t.Error("IsHealthy should be true: getHealth ok + lag within threshold") } - if floor, known := p.FirstAvailableSlot(); !known || floor != 200 { - t.Errorf("FirstAvailableSlot = (%d, %v), want (200, true)", floor, known) - } + // The floor fetch may finish after Poll's bounded foreground wait under + // -race; it still lands via the async closure. + require.Eventually(t, func() bool { + floor, known := p.FirstAvailableSlot() + return known && floor == 200 + }, 5*time.Second, 10*time.Millisecond, "FirstAvailableSlot must become (200, true)") } // The ledger floor is the one poll whose absence must never cost us an @@ -329,7 +314,7 @@ func TestSvmStatePoller_Poll_FansOutAllFiveCalls(t *testing.T) { // "no constraint". func TestSvmStatePoller_FirstAvailableSlot_UnknownUntilObserved(t *testing.T) { t.Parallel() - up := newScriptedUpstreamDeclaringFloor() + up := newScriptedUpstream() up.script("getHealth", []byte(`"ok"`)) up.script("getSlot:processed", []byte(`1000`)) up.script("getSlot:finalized", []byte(`990`)) @@ -356,10 +341,9 @@ func TestSvmStatePoller_FirstAvailableSlot_UnknownUntilObserved(t *testing.T) { require.Zero(t, floor) } -// The observed floor is opt-in: an upstream whose operator declared no -// firstAvailableSlotPlus costs no floor probe and carries no floor, however -// many ticks run. -func TestSvmStatePoller_FirstAvailableSlot_NotPolledWithoutDeclaredBound(t *testing.T) { +// Every upstream observes its floor, declared bound or not, and the probe is +// debounced by the staleness window rather than issued on every tick. +func TestSvmStatePoller_FirstAvailableSlot_PolledWithoutDeclaredBound(t *testing.T) { t.Parallel() exact := int64(0) cases := map[string]*common.UpstreamConfig{ @@ -385,10 +369,10 @@ func TestSvmStatePoller_FirstAvailableSlot_NotPolledWithoutDeclaredBound(t *test for range 3 { require.NoError(t, p.Poll(context.Background())) } - require.Zero(t, up.callCount("oldestBlock")) + require.Equal(t, 1, up.callCount("oldestBlock"), "one probe per staleness window") floor, known := p.FirstAvailableSlot() - require.False(t, known) - require.Zero(t, floor) + require.True(t, known) + require.EqualValues(t, 509, floor) }) } } @@ -398,7 +382,7 @@ func TestSvmStatePoller_FirstAvailableSlot_NotPolledWithoutDeclaredBound(t *test // FirstAvailableSlot returns a second value instead of just the counter. func TestSvmStatePoller_FirstAvailableSlot_ZeroIsKnown(t *testing.T) { t.Parallel() - up := newScriptedUpstreamDeclaringFloor() + up := newScriptedUpstream() up.script("getHealth", []byte(`"ok"`)) up.script("getSlot:processed", []byte(`1000`)) up.script("getSlot:finalized", []byte(`990`)) @@ -408,8 +392,13 @@ func TestSvmStatePoller_FirstAvailableSlot_ZeroIsKnown(t *testing.T) { p := newPollerWithUpstream(t, up) require.NoError(t, p.Poll(context.Background())) - floor, known := p.FirstAvailableSlot() - require.True(t, known, "an observed floor of 0 must read as KNOWN") + // Under -race the foreground wait can time out while the fetch completes + // asynchronously; the closure then marks the floor known. + require.Eventually(t, func() bool { + _, known := p.FirstAvailableSlot() + return known + }, 5*time.Second, 10*time.Millisecond, "an observed floor of 0 must read as KNOWN") + floor, _ := p.FirstAvailableSlot() require.Zero(t, floor) } @@ -419,7 +408,7 @@ func TestSvmStatePoller_FirstAvailableSlot_ZeroIsKnown(t *testing.T) { // violate the unknown/fail-open contract. func TestSvmStatePoller_FirstAvailableSlot_EmptyLeavesUnknown(t *testing.T) { t.Parallel() - up := newScriptedUpstreamDeclaringFloor() + up := newScriptedUpstream() up.script("getHealth", []byte(`"ok"`)) up.script("getSlot:processed", []byte(`1000`)) up.script("getSlot:finalized", []byte(`990`)) @@ -440,7 +429,7 @@ func TestSvmStatePoller_FirstAvailableSlot_EmptyLeavesUnknown(t *testing.T) { // a replica may legitimately never perform the fetch itself. func TestSvmStatePoller_FirstAvailableSlot_KnownFromCounterPropagation(t *testing.T) { t.Parallel() - up := newScriptedUpstreamDeclaringFloor() + up := newScriptedUpstream() p := newPollerWithUpstream(t, up) _, known := p.FirstAvailableSlot() @@ -462,7 +451,7 @@ func TestSvmStatePoller_FirstAvailableSlot_KnownFromCounterPropagation(t *testin // doing the fetching. func TestSvmStatePoller_FirstAvailableSlot_KnownWithoutLocalFetch(t *testing.T) { t.Parallel() - up := newScriptedUpstreamDeclaringFloor() + up := newScriptedUpstream() up.script("getHealth", []byte(`"ok"`)) up.script("getSlot:processed", []byte(`300000`)) up.script("getSlot:finalized", []byte(`299900`)) @@ -491,7 +480,7 @@ func TestSvmStatePoller_FirstAvailableSlot_KnownWithoutLocalFetch(t *testing.T) // on every instance except the one doing the fetching. func TestSvmStatePoller_FirstAvailableSlot_PropagatedZeroBecomesKnownOnTick(t *testing.T) { t.Parallel() - up := newScriptedUpstreamDeclaringFloor() + up := newScriptedUpstream() up.script("getHealth", []byte(`"ok"`)) up.script("getSlot:processed", []byte(`300000`)) up.script("getSlot:finalized", []byte(`299900`)) @@ -524,7 +513,7 @@ func TestSvmStatePoller_FirstAvailableSlot_PropagatedZeroBecomesKnownOnTick(t *t // in this file. Another parallel test writing a different floor between Poll // and ToFloat64 would flake the equality check. func TestSvmStatePoller_FirstAvailableSlot_GaugeSetWithoutLocalFetch(t *testing.T) { - up := newScriptedUpstreamDeclaringFloor() + up := newScriptedUpstream() up.script("getHealth", []byte(`"ok"`)) up.script("getSlot:processed", []byte(`300000`)) up.script("getSlot:finalized", []byte(`299900`)) @@ -547,7 +536,7 @@ func TestSvmStatePoller_FirstAvailableSlot_GaugeSetWithoutLocalFetch(t *testing. // advertise a floor near MaxInt64 and evict itself from every slot-named read. func TestSvmStatePoller_FirstAvailableSlot_AboveOwnLatestRejected(t *testing.T) { t.Parallel() - up := newScriptedUpstreamDeclaringFloor() + up := newScriptedUpstream() up.script("getHealth", []byte(`"ok"`)) up.script("getSlot:processed", []byte(`1000`)) up.script("getSlot:finalized", []byte(`990`)) @@ -563,29 +552,56 @@ func TestSvmStatePoller_FirstAvailableSlot_AboveOwnLatestRejected(t *testing.T) } // The floor moves O(1) slot per slot while sitting O(100k) slots below the -// head, so it is polled on its own staleness window rather than every tick. +// head, so by default it is probed once and then frozen, as EVM's +// earliestBlockPlus is at updateRate 0. A firstAvailableSlotPlus bound's +// updateRate re-probes it on that cadence; nothing else does. func TestSvmStatePoller_FirstAvailableSlot_PolledLessOftenThanHealth(t *testing.T) { t.Parallel() - up := newScriptedUpstreamDeclaringFloor() - up.script("getHealth", []byte(`"ok"`)) - up.script("getSlot:processed", []byte(`1000`)) - up.script("getSlot:finalized", []byte(`990`)) - up.script("getMaxShredInsertSlot", []byte(`1002`)) - up.script("oldestBlock", []byte(`[200]`)) - - p := newPollerWithUpstream(t, up) - const polls = 2 * healthPollEveryNTicks - for range polls { - require.NoError(t, p.Poll(context.Background())) + plus := int64(0) + exact := int64(0) + withRate := func(b *common.SvmAvailabilityBoundConfig) *common.UpstreamConfig { + return &common.UpstreamConfig{ + Id: "test-poller", Type: common.UpstreamTypeSvm, Endpoint: "http://x", + Svm: &common.SvmUpstreamConfig{BlockAvailability: &common.SvmBlockAvailabilityConfig{Lower: b}}, + } + } + cases := map[string]struct { + cfg *common.UpstreamConfig + wantFloor int + }{ + "default: frozen after first observation": {nil, 1}, + "firstAvailableSlotPlus updateRate 1ns": { + withRate(&common.SvmAvailabilityBoundConfig{FirstAvailableSlotPlus: &plus, UpdateRate: common.Duration(time.Nanosecond)}), + 2 * healthPollEveryNTicks, + }, + "updateRate on a bound without the floor is ignored": { + withRate(&common.SvmAvailabilityBoundConfig{ExactSlot: &exact, UpdateRate: common.Duration(time.Nanosecond)}), + 1, + }, } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + up := newScriptedUpstream() + up.cfg = tc.cfg + up.script("getHealth", []byte(`"ok"`)) + up.script("getSlot:processed", []byte(`1000`)) + up.script("getSlot:finalized", []byte(`990`)) + up.script("getMaxShredInsertSlot", []byte(`1002`)) + up.script("oldestBlock", []byte(`[200]`)) + + p := newPollerWithUpstream(t, up) + const polls = 2 * healthPollEveryNTicks + for range polls { + require.NoError(t, p.Poll(context.Background())) + time.Sleep(time.Millisecond) + } - // getHealth is tick-throttled (healthPollEveryNTicks); the floor is - // independently debounced by DefaultFirstAvailableSlotStaleness, so it - // stays at one fetch across this burst. - require.Equal(t, polls/healthPollEveryNTicks, up.callCount("getHealth"), - "getHealth runs every Nth poll") - require.Equal(t, 1, up.callCount("oldestBlock"), - "the floor is debounced by its own staleness window, not by the poll gate") + require.Equal(t, polls/healthPollEveryNTicks, up.callCount("getHealth"), + "getHealth runs every Nth poll") + require.Equal(t, tc.wantFloor, up.callCount("oldestBlock"), + "the floor is debounced by its own staleness window, not by the poll gate") + }) + } } func TestSvmStatePoller_Poll_HealthFailureFlipsHealthy(t *testing.T) { @@ -770,7 +786,9 @@ func TestSvmStatePoller_Poll_TrafficGate_SkipCapForcesFullPoll(t *testing.T) { scriptAllFour(up) p := newPollerWithUpstream(t, up) - const debounce = 100 * time.Millisecond + // Wide enough that a loaded -race run cannot age the stamps made right + // before each Poll out of the gate window. + const debounce = 400 * time.Millisecond p.SetDebounceInterval(debounce) // Expected cumulative getSlot fetches after the i-th poll: polls 1-4 are diff --git a/architecture/svm/svm_state_poller.go b/architecture/svm/svm_state_poller.go index fdd7109a3..0e4a20c21 100644 --- a/architecture/svm/svm_state_poller.go +++ b/architecture/svm/svm_state_poller.go @@ -66,20 +66,17 @@ const maxConsecutiveSlotPollSkips = 4 // needs a different health-detection latency than ~2s. const healthPollEveryNTicks = 5 -// DefaultFirstAvailableSlotStaleness is the minimum interval between floor -// probes (getBlocksWithLimit(0, 1)) per upstream. +// floorRetryInterval spaces floor probes (getBlocksWithLimit(0, 1)) per +// upstream until the first observation lands. After that the floor is frozen, +// as EVM's earliestBlockPlus is at updateRate 0, unless a firstAvailableSlotPlus +// bound's updateRate asks for re-probes (floorUpdateRate). // -// Deliberately ~75x the slot time. The floor sits O(100k) slots below the head -// and rises at most one slot per slot, so freshness buys nothing measurable. -// It also needs NO staleness margin on the comparison: a stale floor is always -// LOWER than the true floor, so it can only under-exclude — one wasted round -// trip the sweep already absorbs — and never wrongly exclude. That is the -// opposite of indexedTipStalenessMargin, whose bound moves TOWARD the -// requested slot and therefore does need slack. -// -// The only thing this window bounds is recovery after a floor DROP (a node -// re-provisioned with longer history behind a stable endpoint). -const DefaultFirstAvailableSlotStaleness = 30 * time.Second +// Freezing is safe for routing: a pruning node's floor only rises, so a frozen +// floor is LOWER than the true one. It can only under-exclude — one wasted round +// trip the sweep already absorbs — and never wrongly exclude. A floor DROP (a +// node re-provisioned with longer history behind a stable endpoint) is seen +// only on restart or with updateRate set. +const floorRetryInterval = 30 * time.Second // Static request payloads — avoid allocating on every tick. var ( @@ -221,13 +218,8 @@ func NewSvmStatePoller( // (a peer's floor of 0 into a zero-valued counter). Those paths are closed // by refreshFirstAvailableSlot's not-stale fast path — but ONLY when no // local refresh is still pending a real observation (see - // firstAvailableSlotPending). Gated on declaresFirstAvailableSlotPlus so a - // propagated shared counter (same id/endpoint key, different config) cannot - // export the floor gauge or mark this poller known without opt-in. + // firstAvailableSlotPending). firstAvailableShared.OnValue(func(v int64) { - if !e.declaresFirstAvailableSlotPlus() { - return - } e.firstAvailableSlotPending.Store(false) e.firstAvailableSlotKnown.Store(true) e.gauge(telemetry.MetricUpstreamSvmFirstAvailableSlot).Set(float64(v)) @@ -276,14 +268,18 @@ func (e *SvmStatePoller) IsObjectNull() bool { // down the whole erpc process (which serves every other network too). func (e *SvmStatePoller) recoverPanic(where string) { if r := recover(); r != nil { - e.logger.Error(). - Interface("panic", r). - Str("where", where). - Bytes("stack", debug.Stack()). - Msg("recovered from panic in svm state poller") + e.recoverPanicValue(where, r) } } +func (e *SvmStatePoller) recoverPanicValue(where string, r interface{}) { + e.logger.Error(). + Interface("panic", r). + Str("where", where). + Bytes("stack", debug.Stack()). + Msg("recovered from panic in svm state poller") +} + func (e *SvmStatePoller) Bootstrap(ctx context.Context) error { // The debounce gate defaults to one slot when not configured. It may have // already been set via SetDebounceInterval (config can arrive before @@ -489,10 +485,9 @@ func (e *SvmStatePoller) Poll(ctx context.Context) error { e.nextHealthPoll = e.pollCount } - // The floor is probed ONLY for an upstream whose operator declared a - // firstAvailableSlotPlus bound (see declaresFirstAvailableSlotPlus), the - // same opt-in as EVM's earliestBlockPlus. An upstream with no such bound - // costs no extra call and carries no observed floor at all. + // The floor is probed for every upstream: history routing compares floors + // across the pool (history_sweep.go), and a declared firstAvailableSlotPlus + // bound anchors on it. Observing it gates nothing by itself. // // It runs AFTER the fan-out joins, not inside it, because its // edge validation compares the sampled floor against this upstream's own @@ -502,12 +497,10 @@ func (e *SvmStatePoller) Poll(ctx context.Context) error { // path while the value is fresh, and when it does refresh, the foreground // wait is capped by SharedStateConfig.UpdateMaxWait with the fetch // continuing asynchronously. - if e.declaresFirstAvailableSlotPlus() { - func() { - defer e.recoverPanic("fetchFirstAvailableSlot") - e.refreshFirstAvailableSlot(ctx) - }() - } + func() { + defer e.recoverPanic("fetchFirstAvailableSlot") + e.refreshFirstAvailableSlot(ctx) + }() // A health verdict nobody routes on is not a defense; publish it. e.applyHealthToRouting() @@ -595,24 +588,6 @@ func (e *SvmStatePoller) IsHealthy() bool { return true } -// declaresFirstAvailableSlotPlus reports whether this upstream's operator- -// declared svm.blockAvailability uses a firstAvailableSlotPlus bound on either -// side — the only way an observed floor is consulted. Same spirit as EVM's -// earliestBlockPlus: the floor is polled because a declared bound needs it as -// its anchor, never by default. -func (e *SvmStatePoller) declaresFirstAvailableSlotPlus() bool { - if e.upstream == nil { - return false - } - cfg := e.upstream.Config() - if cfg == nil || cfg.Svm == nil || cfg.Svm.BlockAvailability == nil { - return false - } - ba := cfg.Svm.BlockAvailability - return (ba.Lower != nil && ba.Lower.FirstAvailableSlotPlus != nil) || - (ba.Upper != nil && ba.Upper.FirstAvailableSlotPlus != nil) -} - // refreshFirstAvailableSlot re-reads this upstream's ledger floor, debounced by // the counter itself rather than by the poll gate. TryUpdateIfStale returns on // a fast path when the value is fresh, so calling it every tick costs one @@ -630,10 +605,25 @@ func (e *SvmStatePoller) declaresFirstAvailableSlotPlus() bool { // on a cold poller and leave firstAvailableSlotPlus enforcing from genesis if // the async fetch later fails. func (e *SvmStatePoller) refreshFirstAvailableSlot(ctx context.Context) { - wasStale := e.firstAvailableSlotShared.IsStale(DefaultFirstAvailableSlotStaleness) - slot, err := e.firstAvailableSlotShared.TryUpdateIfStale(ctx, DefaultFirstAvailableSlotStaleness, - func(ctx context.Context) (int64, error) { - slot, err := e.fetchFirstAvailableSlot(ctx) + staleness := e.floorUpdateRate() + if staleness == 0 { + if e.firstAvailableSlotKnown.Load() { + return + } + staleness = floorRetryInterval + } + wasStale := e.firstAvailableSlotShared.IsStale(staleness) + slot, err := e.firstAvailableSlotShared.TryUpdateIfStale(ctx, staleness, + func(ctx context.Context) (slot int64, err error) { + // The shared counter may run this on its own goroutine, outside + // Poll's recoverPanic; a panicking transport must not crash the process. + defer func() { + if r := recover(); r != nil { + e.recoverPanicValue("fetchFirstAvailableSlot", r) + slot, err = 0, fmt.Errorf("panic in floor probe: %v", r) + } + }() + slot, err = e.fetchFirstAvailableSlot(ctx) if err != nil { // Includes -32601 from an upstream that does not implement the // method, and null/empty results. Returning the error leaves @@ -692,6 +682,29 @@ func (e *SvmStatePoller) refreshFirstAvailableSlot(ctx context.Context) { e.firstAvailableSlotPending.Store(true) } +// floorUpdateRate is the smallest updateRate set on a firstAvailableSlotPlus +// bound of this upstream, else 0 (freeze after the first observation). Same +// field, default and min-of-bounds rule as EVM's earliestBlockPlus schedule. +func (e *SvmStatePoller) floorUpdateRate() time.Duration { + if e.upstream == nil { + return 0 + } + cfg := e.upstream.Config() + if cfg == nil || cfg.Svm == nil || cfg.Svm.BlockAvailability == nil { + return 0 + } + var rate time.Duration + for _, b := range []*common.SvmAvailabilityBoundConfig{cfg.Svm.BlockAvailability.Lower, cfg.Svm.BlockAvailability.Upper} { + if b == nil || b.FirstAvailableSlotPlus == nil || b.UpdateRate <= 0 { + continue + } + if r := b.UpdateRate.Duration(); rate == 0 || r < rate { + rate = r + } + } + return rate +} + // fetchFirstAvailableSlot reads the oldest block this upstream can actually // serve: the first element of getBlocksWithLimit(0, 1). Agave answers it from // the same storage getBlock reads — long-term storage (BigTable) when the start diff --git a/auth/auth_metrics.go b/auth/auth_metrics.go new file mode 100644 index 000000000..4d3cad0cb --- /dev/null +++ b/auth/auth_metrics.go @@ -0,0 +1,54 @@ +package auth + +import ( + "errors" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" +) + +// RecordAuthFailed increments erpc_auth_failed_total for ErrAuthUnauthorized +// about to be returned to the client; no-ops for other errors. +func RecordAuthFailed(req *common.NormalizedRequest, err error) { + if !common.HasErrorCode(err, common.ErrCodeAuthUnauthorized) { + return + } + strategy, reason := authFailureLabels(err) + project := "n/a" + network := "n/a" + if req != nil { + if n := req.Network(); n != nil { + project = n.ProjectId() + if lbl := n.Label(); lbl != "" { + network = lbl + } else { + network = n.Id() + } + } + } + telemetry.MetricAuthFailedTotal.WithLabelValues( + project, + network, + strategy, + reason, + "unknown", + ).Inc() +} + +func authFailureLabels(err error) (strategy, reason string) { + var ae *common.ErrAuthUnauthorized + if !errors.As(err, &ae) || ae == nil { + return "n/a", "unauthorized" + } + strategy = "n/a" + if ae.Details != nil { + if s, ok := ae.Details["strategy"].(string); ok && s != "" { + strategy = s + } + } + reason = string(ae.Reason()) + if reason == "" { + reason = "unauthorized" + } + return strategy, reason +} diff --git a/auth/auth_metrics_labels_test.go b/auth/auth_metrics_labels_test.go new file mode 100644 index 000000000..87016e8d1 --- /dev/null +++ b/auth/auth_metrics_labels_test.go @@ -0,0 +1,49 @@ +package auth + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" +) + +// labelNetwork satisfies common.Network for the label lookups RecordAuthFailed +// performs; any other method call panics via the nil embedded interface. +type labelNetwork struct { + common.Network + id, label, project string +} + +func (n *labelNetwork) Id() string { return n.id } +func (n *labelNetwork) Label() string { return n.label } +func (n *labelNetwork) ProjectId() string { return n.project } + +func TestRecordAuthFailed_NetworkLabels(t *testing.T) { + count := func(project, network, strategy, reason string) float64 { + return testutil.ToFloat64(telemetry.MetricAuthFailedTotal.WithLabelValues(project, network, strategy, reason, "unknown")) + } + + t.Run("network label preferred over id", func(t *testing.T) { + req := newRequest("eth_call") + req.SetNetwork(&labelNetwork{id: "evm:1", label: "mainnet", project: "proj-labels"}) + before := count("proj-labels", "mainnet", "network", "ip_not_allowed") + RecordAuthFailed(req, common.NewErrAuthUnauthorized("network", "ip_not_allowed", "nope")) + assert.Equal(t, before+1, count("proj-labels", "mainnet", "network", "ip_not_allowed")) + }) + + t.Run("network id used when label empty", func(t *testing.T) { + req := newRequest("eth_call") + req.SetNetwork(&labelNetwork{id: "evm:137", project: "proj-labels"}) + before := count("proj-labels", "evm:137", "siwe", "parse_failed") + RecordAuthFailed(req, common.NewErrAuthUnauthorized("siwe", "parse_failed", "bad")) + assert.Equal(t, before+1, count("proj-labels", "evm:137", "siwe", "parse_failed")) + }) + + t.Run("missing strategy and reason fall back to defaults", func(t *testing.T) { + before := count("n/a", "n/a", "n/a", "unauthorized") + RecordAuthFailed(nil, common.NewErrAuthUnauthorized("", "", "anon")) + assert.Equal(t, before+1, count("n/a", "n/a", "n/a", "unauthorized")) + }) +} diff --git a/auth/authorizer_registry_test.go b/auth/authorizer_registry_test.go new file mode 100644 index 000000000..bf97db3a8 --- /dev/null +++ b/auth/authorizer_registry_test.go @@ -0,0 +1,407 @@ +package auth + +import ( + "context" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/upstream" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestRateLimiters(t *testing.T, budgets ...*common.RateLimitBudgetConfig) *upstream.RateLimitersRegistry { + t.Helper() + logger := zerolog.Nop() + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ + Store: &common.RateLimitStoreConfig{Driver: "memory"}, + Budgets: budgets, + }, &logger) + require.NoError(t, err) + return rlr +} + +func onePerMinuteBudget(id string) *common.RateLimitBudgetConfig { + return &common.RateLimitBudgetConfig{ + Id: id, + Rules: []*common.RateLimitRuleConfig{{Method: "*", MaxCount: 1, Period: common.RateLimitPeriodMinute}}, + } +} + +func newRequest(method string) *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"` + method + `","params":[],"id":1}`)) +} + +func TestNewAuthorizer_ConfigValidation(t *testing.T) { + logger := zerolog.Nop() + tests := []struct { + name string + cfg *common.AuthStrategyConfig + wantErr string + }{ + {name: "nil config", cfg: nil, wantErr: "auth strategy config is nil"}, + {name: "secret without block", cfg: &common.AuthStrategyConfig{Type: common.AuthTypeSecret}, wantErr: "secret strategy config is nil"}, + {name: "jwt without block", cfg: &common.AuthStrategyConfig{Type: common.AuthTypeJwt}, wantErr: "JWT strategy config is nil"}, + {name: "siwe without block", cfg: &common.AuthStrategyConfig{Type: common.AuthTypeSiwe}, wantErr: "SIWE strategy config is nil"}, + {name: "network without block", cfg: &common.AuthStrategyConfig{Type: common.AuthTypeNetwork}, wantErr: "network strategy config is nil"}, + {name: "database without block", cfg: &common.AuthStrategyConfig{Type: common.AuthTypeDatabase}, wantErr: "database strategy config is nil"}, + {name: "unknown type", cfg: &common.AuthStrategyConfig{Type: "magic"}, wantErr: "unknown auth strategy type: magic"}, + { + name: "network with invalid ip", + cfg: &common.AuthStrategyConfig{Type: common.AuthTypeNetwork, Network: &common.NetworkStrategyConfig{AllowedIPs: []string{"x"}}}, + wantErr: "invalid IP address", + }, + { + name: "jwt with unusable key", + cfg: &common.AuthStrategyConfig{Type: common.AuthTypeJwt, Jwt: &common.JwtStrategyConfig{VerificationKeys: map[string]string{"k": "file:///nonexistent/erpc-test-key.pem"}}}, + wantErr: "failed to read key file", + }, + { + name: "database without connector", + cfg: &common.AuthStrategyConfig{Type: common.AuthTypeDatabase, Database: &common.DatabaseStrategyConfig{}}, + wantErr: "connector config is nil", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + az, err := NewAuthorizer(context.Background(), &logger, "p", tc.cfg, nil, 0) + require.ErrorContains(t, err, tc.wantErr) + assert.Nil(t, az) + }) + } + + t.Run("each supported type builds its strategy", func(t *testing.T) { + cfgs := map[common.AuthType]*common.AuthStrategyConfig{ + common.AuthTypeSecret: {Type: common.AuthTypeSecret, Secret: &common.SecretStrategyConfig{Id: "a", Value: "b"}}, + common.AuthTypeJwt: {Type: common.AuthTypeJwt, Jwt: &common.JwtStrategyConfig{VerificationKeys: map[string]string{"k": "hmac"}}}, + common.AuthTypeSiwe: {Type: common.AuthTypeSiwe, Siwe: &common.SiweStrategyConfig{}}, + common.AuthTypeNetwork: {Type: common.AuthTypeNetwork, Network: &common.NetworkStrategyConfig{AllowedCIDRs: []string{"10.0.0.0/8"}}}, + common.AuthTypeDatabase: {Type: common.AuthTypeDatabase, Database: &common.DatabaseStrategyConfig{ + Connector: &common.ConnectorConfig{Id: "mem", Driver: common.DriverMemory, Memory: &common.MemoryConnectorConfig{MaxItems: 10, MaxTotalSize: "1MB"}}, + }}, + } + for typ, cfg := range cfgs { + az, err := NewAuthorizer(context.Background(), &logger, "p", cfg, nil, 3) + require.NoError(t, err, typ) + assert.True(t, az.strategy.Supports(&AuthPayload{Type: typ}), typ) + assert.Equal(t, 3, az.index) + } + }) +} + +func TestAuthorizer_ShouldApplyToMethod(t *testing.T) { + logger := zerolog.Nop() + tests := []struct { + name string + ignore []string + allow []string + method string + want bool + }{ + {name: "no lists applies to all", method: "eth_call", want: true}, + {name: "ignored exactly", ignore: []string{"eth_call"}, method: "eth_call", want: false}, + {name: "ignored by wildcard", ignore: []string{"debug_*"}, method: "debug_traceCall", want: false}, + {name: "not ignored", ignore: []string{"debug_*"}, method: "eth_call", want: true}, + {name: "allow overrides ignore-all", ignore: []string{"*"}, allow: []string{"eth_getLogs"}, method: "eth_getLogs", want: true}, + {name: "ignore-all still blocks other methods", ignore: []string{"*"}, allow: []string{"eth_getLogs"}, method: "eth_call", want: false}, + {name: "allow list alone keeps default apply", allow: []string{"eth_getLogs"}, method: "eth_call", want: true}, + {name: "invalid ignore pattern is skipped", ignore: []string{"(eth_call", "eth_call"}, method: "eth_call", want: false}, + {name: "invalid allow pattern is skipped", ignore: []string{"*"}, allow: []string{"(eth_call", "eth_call"}, method: "eth_call", want: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + az := &Authorizer{logger: &logger, cfg: &common.AuthStrategyConfig{IgnoreMethods: tc.ignore, AllowMethods: tc.allow}} + assert.Equal(t, tc.want, az.shouldApplyToMethod(tc.method)) + }) + } +} + +func TestAuthRegistry_RateLimitBudgets(t *testing.T) { + logger := zerolog.Nop() + + t.Run("strategy budget is enforced per authorizer", func(t *testing.T) { + rlr := newTestRateLimiters(t, onePerMinuteBudget("strategy-budget")) + reg, err := NewAuthRegistry(context.Background(), &logger, "proj", &common.AuthConfig{ + Strategies: []*common.AuthStrategyConfig{{ + Type: common.AuthTypeSecret, + RateLimitBudget: "strategy-budget", + Secret: &common.SecretStrategyConfig{Id: "alice", Value: "s"}, + }}, + }, rlr) + require.NoError(t, err) + + ap := &AuthPayload{Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "s"}} + req := newRequest("eth_call") + req.SetClientIP("1.2.3.4") + user, err := reg.Authenticate(context.Background(), req, "eth_call", ap) + require.NoError(t, err) + assert.Equal(t, "alice", user.Id) + assert.Equal(t, "alice", req.UserId(), "authenticated user is attached to the request") + + req2 := newRequest("eth_call") + req2.SetClientIP("1.2.3.4") + user, err = reg.Authenticate(context.Background(), req2, "eth_call", ap) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeAuthRateLimitRuleExceeded)) + require.NotNil(t, user, "rate-limited caller is still identified") + var rlErr *common.ErrAuthRateLimitRuleExceeded + require.ErrorAs(t, err, &rlErr) + assert.Equal(t, "strategy-budget", rlErr.Details["budget"]) + assert.Equal(t, "alice", rlErr.Details["userId"]) + assert.Equal(t, "1.2.3.4", rlErr.Details["clientIp"]) + assert.Equal(t, "method:eth_call", rlErr.Details["rule"]) + }) + + t.Run("user budget overrides strategy budget", func(t *testing.T) { + rlr := newTestRateLimiters(t, onePerMinuteBudget("tight"), &common.RateLimitBudgetConfig{ + Id: "roomy", + Rules: []*common.RateLimitRuleConfig{{Method: "*", MaxCount: 1000, Period: common.RateLimitPeriodMinute}}, + }) + reg, err := NewAuthRegistry(context.Background(), &logger, "proj", &common.AuthConfig{ + Strategies: []*common.AuthStrategyConfig{{ + Type: common.AuthTypeSecret, + RateLimitBudget: "tight", + Secret: &common.SecretStrategyConfig{Id: "vip", Value: "s", RateLimitBudget: "roomy"}, + }}, + }, rlr) + require.NoError(t, err) + ap := &AuthPayload{Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "s"}} + for i := range 3 { + user, err := reg.Authenticate(context.Background(), newRequest("eth_call"), "eth_call", ap) + require.NoError(t, err, "attempt %d", i) + assert.Equal(t, "roomy", user.RateLimitBudget) + } + }) + + t.Run("unknown budget surfaces budget-not-found", func(t *testing.T) { + rlr := newTestRateLimiters(t) + reg, err := NewAuthRegistry(context.Background(), &logger, "proj", &common.AuthConfig{ + Strategies: []*common.AuthStrategyConfig{{ + Type: common.AuthTypeSecret, + RateLimitBudget: "missing", + Secret: &common.SecretStrategyConfig{Id: "u", Value: "s"}, + }}, + }, rlr) + require.NoError(t, err) + _, err = reg.Authenticate(context.Background(), newRequest("eth_call"), "eth_call", + &AuthPayload{Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "s"}}) + assert.True(t, common.HasErrorCode(err, "ErrRateLimitBudgetNotFound"), "got %v", err) + }) + + t.Run("nil request uses strategy budget and n/a labels", func(t *testing.T) { + rlr := newTestRateLimiters(t, onePerMinuteBudget("nilreq")) + az, err := NewAuthorizer(context.Background(), &logger, "proj", &common.AuthStrategyConfig{ + Type: common.AuthTypeSecret, + RateLimitBudget: "nilreq", + Secret: &common.SecretStrategyConfig{Id: "u", Value: "s"}, + }, rlr, 0) + require.NoError(t, err) + require.NoError(t, az.acquireRateLimitPermit(context.Background(), nil, "eth_call")) + err = az.acquireRateLimitPermit(context.Background(), nil, "eth_call") + var rlErr *common.ErrAuthRateLimitRuleExceeded + require.ErrorAs(t, err, &rlErr) + assert.Equal(t, "n/a", rlErr.Details["userId"]) + assert.Equal(t, "n/a", rlErr.Details["clientIp"]) + }) + + t.Run("no budget never consults the limiter", func(t *testing.T) { + az := &Authorizer{cfg: &common.AuthStrategyConfig{Type: common.AuthTypeSecret}} + assert.NoError(t, az.acquireRateLimitPermit(context.Background(), newRequest("eth_call"), "eth_call")) + }) +} + +func TestAuthRegistry_Authenticate_Routing(t *testing.T) { + logger := zerolog.Nop() + + t.Run("nil config is rejected", func(t *testing.T) { + _, err := NewAuthRegistry(context.Background(), &logger, "p", nil, nil) + require.ErrorContains(t, err, "auth config is nil") + }) + + t.Run("invalid strategy fails registry creation", func(t *testing.T) { + _, err := NewAuthRegistry(context.Background(), &logger, "p", &common.AuthConfig{ + Strategies: []*common.AuthStrategyConfig{{Type: common.AuthTypeSecret}}, + }, nil) + require.ErrorContains(t, err, "failed to create authorizer for project p with strategy secret") + }) + + t.Run("nil payload is unauthorized", func(t *testing.T) { + reg := newSecretAuthRegistry(t, &common.AuthConfig{}) + _, err := reg.Authenticate(context.Background(), nil, "eth_call", nil) + assert.True(t, common.HasErrorCode(err, common.ErrCodeAuthUnauthorized)) + assert.True(t, common.HasErrorCode(reg.AcceptsPayload(context.Background(), nil, nil), common.ErrCodeAuthUnauthorized)) + }) + + t.Run("no strategies allows everything", func(t *testing.T) { + reg := newSecretAuthRegistry(t, &common.AuthConfig{}) + user, err := reg.Authenticate(context.Background(), nil, "eth_call", &AuthPayload{Type: common.AuthTypeNetwork}) + require.NoError(t, err) + assert.Nil(t, user) + assert.NoError(t, reg.AcceptsPayload(context.Background(), nil, &AuthPayload{Type: common.AuthTypeNetwork})) + }) + + reg := newSecretAuthRegistry(t, &common.AuthConfig{ + Strategies: []*common.AuthStrategyConfig{ + { + Type: common.AuthTypeSecret, + IgnoreMethods: []string{"*"}, + AllowMethods: []string{"eth_getLogs"}, + Secret: &common.SecretStrategyConfig{Id: "logs-only", Value: "logs"}, + }, + { + Type: common.AuthTypeNetwork, + Network: &common.NetworkStrategyConfig{AllowedCIDRs: []string{"10.0.0.0/8"}, IPAsUser: true}, + }, + }, + }) + + t.Run("method scoping skips a strategy", func(t *testing.T) { + ap := &AuthPayload{Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "logs"}} + user, err := reg.Authenticate(context.Background(), newRequest("eth_getLogs"), "eth_getLogs", ap) + require.NoError(t, err) + assert.Equal(t, "logs-only", user.Id) + + _, err = reg.Authenticate(context.Background(), newRequest("eth_call"), "eth_call", ap) + require.Error(t, err) + assert.Contains(t, err.Error(), "no auth strategy matched") + }) + + t.Run("AcceptsPayload ignores method scoping", func(t *testing.T) { + ap := &AuthPayload{Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "logs"}} + assert.NoError(t, reg.AcceptsPayload(context.Background(), nil, ap)) + + err := reg.AcceptsPayload(context.Background(), nil, &AuthPayload{Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "nope"}}) + requireAuthReason(t, err, "secret", "invalid_secret") + + err = reg.AcceptsPayload(context.Background(), nil, &AuthPayload{Type: common.AuthTypeJwt, Jwt: &JwtPayload{Token: "x"}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "no auth strategy matched") + }) + + t.Run("network strategy authenticates by client ip", func(t *testing.T) { + req := newRequest("eth_call") + req.SetClientIP("10.9.8.7") + user, err := reg.Authenticate(context.Background(), req, "eth_call", &AuthPayload{Type: common.AuthTypeNetwork}) + require.NoError(t, err) + assert.Equal(t, "10.9.8.7", user.Id) + assert.Equal(t, "10.9.8.7", req.UserId()) + }) +} + +func TestAuthRegistry_FindDatabaseConnector(t *testing.T) { + logger := zerolog.Nop() + reg, err := NewAuthRegistry(context.Background(), &logger, "p", &common.AuthConfig{ + Strategies: []*common.AuthStrategyConfig{ + {Type: common.AuthTypeSecret, Secret: &common.SecretStrategyConfig{Id: "a", Value: "b"}}, + {Type: common.AuthTypeDatabase, Database: &common.DatabaseStrategyConfig{ + Connector: &common.ConnectorConfig{Id: "keys-db", Driver: common.DriverMemory, Memory: &common.MemoryConnectorConfig{MaxItems: 10, MaxTotalSize: "1MB"}}, + }}, + }, + }, nil) + require.NoError(t, err) + + conn, err := reg.FindDatabaseConnector("keys-db") + require.NoError(t, err) + assert.Equal(t, "keys-db", conn.Id()) + + _, err = reg.FindDatabaseConnector("other") + require.EqualError(t, err, "database connector with ID 'other' not found") +} + +func TestDatabaseStrategy_MemoryConnectorEndToEnd(t *testing.T) { + logger := zerolog.Nop() + ttl := time.Minute + maxSize, maxCost, counters := int64(100), int64(1<<20), int64(1000) + s, err := NewDatabaseStrategy(context.Background(), &logger, &common.DatabaseStrategyConfig{ + Connector: &common.ConnectorConfig{Id: "mem", Driver: common.DriverMemory, Memory: &common.MemoryConnectorConfig{MaxItems: 100, MaxTotalSize: "1MB"}}, + Cache: &common.DatabaseStrategyCacheConfig{TTL: &ttl, MaxSize: &maxSize, MaxCost: &maxCost, NumCounters: &counters}, + MaxWait: common.Duration(time.Second), + }) + require.NoError(t, err) + t.Cleanup(s.Close) + + conn := s.GetConnector() + ctx := context.Background() + put := func(key, value string) { + require.NoError(t, conn.Set(ctx, key, "*", []byte(value), nil)) + } + put("good", `{"userId":"u1","rateLimitBudget":"gold"}`) + put("disabled", `{"userId":"u2","enabled":false}`) + put("garbage", `not-json`) + put("noid", `{"enabled":true}`) + // Ristretto writes are async; Get must observe them. + require.Eventually(t, func() bool { + _, err := conn.Get(ctx, data.ConnectorMainIndex, "noid", "*", nil) + return err == nil + }, 5*time.Second, 5*time.Millisecond) + + auth := func(key string) (*common.User, error) { + return s.Authenticate(ctx, nil, &AuthPayload{Type: common.AuthTypeDatabase, Secret: &SecretPayload{Value: key}}) + } + + assert.True(t, s.Supports(&AuthPayload{Type: common.AuthTypeSecret})) + assert.True(t, s.Supports(&AuthPayload{Type: common.AuthTypeDatabase})) + assert.False(t, s.Supports(&AuthPayload{Type: common.AuthTypeJwt})) + + _, err = s.Authenticate(ctx, nil, &AuthPayload{Type: common.AuthTypeDatabase}) + requireAuthReason(t, err, "database", "missing_secret") + _, err = auth("") + requireAuthReason(t, err, "database", "empty_secret") + + user, err := auth("good") + require.NoError(t, err) + assert.Equal(t, "u1", user.Id) + assert.Equal(t, "gold", user.RateLimitBudget) + + _, err = auth("disabled") + requireAuthReason(t, err, "database", "disabled_key") + _, err = auth("garbage") + requireAuthReason(t, err, "database", "db_record_parse_error") + _, err = auth("noid") + requireAuthReason(t, err, "database", "db_record_missing_user_id") + _, err = auth("unknown") + requireAuthReason(t, err, "database", "invalid_api_key") + + // Positive cache: once cached, the user survives deletion from the store + // until invalidated. + require.Eventually(t, func() bool { + _, found := s.cache.Get("good") + return found + }, 5*time.Second, 5*time.Millisecond) + require.NoError(t, conn.Delete(ctx, "good", "*")) + user, err = auth("good") + require.NoError(t, err) + assert.Equal(t, "u1", user.Id, "cached user served without hitting the store") + + s.InvalidateCache("good") + s.cache.Wait() + _, err = auth("good") + requireAuthReason(t, err, "database", "invalid_api_key") + + // Negative cache: a key that was unknown keeps failing from cache even + // after it is created, until the negative entry expires. + require.Eventually(t, func() bool { + _, found := s.negCache.Get("unknown") + return found + }, 5*time.Second, 5*time.Millisecond) + put("unknown", `{"userId":"late"}`) + _, err = auth("unknown") + requireAuthReason(t, err, "database", "cached_unknown_api_key") + + s.ClearCache() +} + +func TestNewDatabaseStrategy_Errors(t *testing.T) { + logger := zerolog.Nop() + _, err := NewDatabaseStrategy(context.Background(), &logger, nil) + require.ErrorContains(t, err, "database strategy config is nil") + _, err = NewDatabaseStrategy(context.Background(), &logger, &common.DatabaseStrategyConfig{}) + require.ErrorContains(t, err, "connector config is nil") + _, err = NewDatabaseStrategy(context.Background(), &logger, &common.DatabaseStrategyConfig{ + Connector: &common.ConnectorConfig{Id: "m", Driver: common.DriverMemory, Memory: &common.MemoryConnectorConfig{MaxItems: 0}}, + }) + require.ErrorContains(t, err, "failed to create database connector") +} diff --git a/auth/http_payload_test.go b/auth/http_payload_test.go new file mode 100644 index 000000000..04864f52b --- /dev/null +++ b/auth/http_payload_test.go @@ -0,0 +1,247 @@ +package auth + +import ( + "encoding/base64" + "net/http" + "net/url" + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/metadata" +) + +func TestNewPayloadFromHttp_SignalPrecedence(t *testing.T) { + siweMsg := "example.com wants you to sign in" + siweMsgB64 := base64.StdEncoding.EncodeToString([]byte(siweMsg)) + basic := func(creds string) string { + return "Basic " + base64.StdEncoding.EncodeToString([]byte(creds)) + } + + tests := []struct { + name string + headers http.Header + args url.Values + wantType common.AuthType + wantSecret string + wantJwt string + wantSig string + wantMsg string + }{ + { + name: "no signals defaults to network", + wantType: common.AuthTypeNetwork, + }, + { + name: "deprecated token query param wins over everything", + args: url.Values{"token": {"tok"}, "secret": {"sec"}, "jwt": {"j"}}, + headers: http.Header{"X-Erpc-Secret-Token": {"hdr"}, "Authorization": {"Bearer b"}}, + wantType: common.AuthTypeSecret, + wantSecret: "tok", + }, + { + name: "secret query param wins over header", + args: url.Values{"secret": {"sec"}}, + headers: http.Header{"X-Erpc-Secret-Token": {"hdr"}}, + wantType: common.AuthTypeSecret, + wantSecret: "sec", + }, + { + name: "secret header wins over authorization", + headers: http.Header{"X-Erpc-Secret-Token": {"hdr"}, "Authorization": {"Bearer b"}}, + wantType: common.AuthTypeSecret, + wantSecret: "hdr", + }, + { + name: "basic auth password is the secret, username ignored", + headers: http.Header{"Authorization": {basic("alice:pa:ss")}}, + wantType: common.AuthTypeSecret, + wantSecret: "pa:ss", + }, + { + name: "auth scheme is case-insensitive and value is trimmed", + headers: http.Header{"Authorization": {" BASIC " + base64.StdEncoding.EncodeToString([]byte("u:p")) + " "}}, + wantType: common.AuthTypeSecret, + wantSecret: "p", + }, + { + name: "bearer token becomes jwt", + headers: http.Header{"Authorization": {"Bearer eyJ.abc.def"}}, + wantType: common.AuthTypeJwt, + wantJwt: "eyJ.abc.def", + }, + { + name: "unknown authorization scheme falls back to network and shadows jwt query", + headers: http.Header{"Authorization": {"Digest abc"}}, + args: url.Values{"jwt": {"ignored"}}, + wantType: common.AuthTypeNetwork, + }, + { + name: "authorization without a value falls back to network", + headers: http.Header{"Authorization": {"Bearer"}}, + wantType: common.AuthTypeNetwork, + }, + { + name: "jwt query param", + args: url.Values{"jwt": {"q.jwt.tok"}}, + wantType: common.AuthTypeJwt, + wantJwt: "q.jwt.tok", + }, + { + name: "siwe query params with base64 message are decoded", + args: url.Values{"signature": {"0xsig"}, "message": {siweMsgB64}}, + wantType: common.AuthTypeSiwe, + wantSig: "0xsig", + wantMsg: siweMsg, + }, + { + name: "siwe query params with plain message are kept verbatim", + args: url.Values{"signature": {"0xsig"}, "message": {siweMsg}}, + wantType: common.AuthTypeSiwe, + wantSig: "0xsig", + wantMsg: siweMsg, + }, + { + name: "siwe signature without message is ignored", + args: url.Values{"signature": {"0xsig"}}, + wantType: common.AuthTypeNetwork, + }, + { + name: "siwe headers", + headers: http.Header{"X-Siwe-Message": {siweMsgB64}, "X-Siwe-Signature": {"0xhdr"}}, + wantType: common.AuthTypeSiwe, + wantSig: "0xhdr", + wantMsg: siweMsg, + }, + { + name: "siwe message header without signature falls back to network", + headers: http.Header{"X-Siwe-Message": {siweMsgB64}}, + wantType: common.AuthTypeNetwork, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + headers := tc.headers + if headers == nil { + headers = http.Header{} + } + args := tc.args + if args == nil { + args = url.Values{} + } + ap, err := NewPayloadFromHttp("eth_call", "10.0.0.1:1234", headers, args) + require.NoError(t, err) + assert.Equal(t, "eth_call", ap.Method) + assert.Equal(t, tc.wantType, ap.Type) + + if tc.wantSecret != "" { + require.NotNil(t, ap.Secret) + assert.Equal(t, tc.wantSecret, ap.Secret.Value) + } else { + assert.Nil(t, ap.Secret) + } + if tc.wantJwt != "" { + require.NotNil(t, ap.Jwt) + assert.Equal(t, tc.wantJwt, ap.Jwt.Token) + } else { + assert.Nil(t, ap.Jwt) + } + if tc.wantSig != "" { + require.NotNil(t, ap.Siwe) + assert.Equal(t, tc.wantSig, ap.Siwe.Signature) + assert.Equal(t, tc.wantMsg, ap.Siwe.Message) + } else { + assert.Nil(t, ap.Siwe) + } + }) + } +} + +func TestNewPayloadFromHttp_MalformedBasicAuth(t *testing.T) { + tests := []struct { + name string + value string + }{ + {name: "not base64", value: "Basic !!!not-base64!!!"}, + {name: "no colon separator", value: "Basic " + base64.StdEncoding.EncodeToString([]byte("justuser"))}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ap, err := NewPayloadFromHttp("eth_call", "", http.Header{"Authorization": {tc.value}}, url.Values{}) + require.Error(t, err) + assert.Nil(t, ap) + }) + } +} + +func TestNewPayloadFromGrpc_Signals(t *testing.T) { + siweMsg := "grpc.example wants you to sign in" + tests := []struct { + name string + md metadata.MD + wantType common.AuthType + wantSecret string + wantSig string + wantMsg string + wantErr bool + }{ + { + name: "secret token metadata wins over authorization", + md: metadata.Pairs("x-erpc-secret-token", "s3", "authorization", "Bearer x"), + wantType: common.AuthTypeSecret, + wantSecret: "s3", + }, + { + name: "siwe metadata decodes base64 message", + md: metadata.Pairs("x-siwe-message", base64.StdEncoding.EncodeToString([]byte(siweMsg)), "x-siwe-signature", "0xabc"), + wantType: common.AuthTypeSiwe, + wantSig: "0xabc", + wantMsg: siweMsg, + }, + { + name: "siwe message without signature defaults to network", + md: metadata.Pairs("x-siwe-message", siweMsg), + wantType: common.AuthTypeNetwork, + }, + { + name: "empty metadata defaults to network", + md: metadata.MD{}, + wantType: common.AuthTypeNetwork, + }, + { + name: "basic auth that is not base64 is rejected", + md: metadata.Pairs("authorization", "Basic %%%"), + wantErr: true, + }, + { + name: "basic auth without colon is rejected", + md: metadata.Pairs("authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("nocolon"))), + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ap, err := NewPayloadFromGrpc("eth_chainId", tc.md) + if tc.wantErr { + require.Error(t, err) + assert.Nil(t, ap) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantType, ap.Type) + if tc.wantSecret != "" { + require.NotNil(t, ap.Secret) + assert.Equal(t, tc.wantSecret, ap.Secret.Value) + } + if tc.wantSig != "" { + require.NotNil(t, ap.Siwe) + assert.Equal(t, tc.wantSig, ap.Siwe.Signature) + assert.Equal(t, tc.wantMsg, ap.Siwe.Message) + } else { + assert.Nil(t, ap.Siwe) + } + }) + } +} diff --git a/auth/registry.go b/auth/registry.go index a42d5aca4..79a4e925f 100644 --- a/auth/registry.go +++ b/auth/registry.go @@ -45,7 +45,7 @@ func NewAuthRegistry(appCtx context.Context, logger *zerolog.Logger, projectId s // Authenticate checks the authentication payload against all registered strategies func (r *AuthRegistry) Authenticate(ctx context.Context, req *common.NormalizedRequest, method string, ap *AuthPayload) (*common.User, error) { if ap == nil { - return nil, common.NewErrAuthUnauthorized("n/a", "auth payload is nil") + return nil, common.NewErrAuthUnauthorized("n/a", "unauthorized", "auth payload is nil") } if len(r.strategies) == 0 { @@ -111,7 +111,7 @@ func (r *AuthRegistry) Authenticate(ctx context.Context, req *common.NormalizedR // every request. func (r *AuthRegistry) AcceptsPayload(ctx context.Context, req *common.NormalizedRequest, ap *AuthPayload) error { if ap == nil { - return common.NewErrAuthUnauthorized("n/a", "auth payload is nil") + return common.NewErrAuthUnauthorized("n/a", "unauthorized", "auth payload is nil") } if len(r.strategies) == 0 { return nil @@ -137,11 +137,11 @@ func unauthorized(errs []error) error { } if len(errs) == 0 { - return common.NewErrAuthUnauthorized("n/a", "no auth strategy matched make sure correct headers or query strings are provided") + return common.NewErrAuthUnauthorized("n/a", "unauthorized", "no auth strategy matched make sure correct headers or query strings are provided") } // If no strategy matched or succeeded, consider the request unauthorized - return common.NewErrAuthUnauthorized("n/a", errors.Join(errs...).Error()) + return common.NewErrAuthUnauthorized("n/a", "unauthorized", errors.Join(errs...).Error()) } // FindDatabaseConnector finds a database connector by ID from the strategies diff --git a/auth/strategy_auth_metrics_test.go b/auth/strategy_auth_metrics_test.go new file mode 100644 index 000000000..e126bee9a --- /dev/null +++ b/auth/strategy_auth_metrics_test.go @@ -0,0 +1,90 @@ +package auth + +import ( + "context" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func init() { util.ConfigureTestLogger() } + +func authFailedCount(strategy, reason string) float64 { + return testutil.ToFloat64(telemetry.MetricAuthFailedTotal.WithLabelValues( + "n/a", "n/a", strategy, reason, "unknown", + )) +} + +func TestRecordAuthFailed_MetersOncePerClientDenial(t *testing.T) { + registry := newSecretAuthRegistry(t, &common.AuthConfig{ + Strategies: []*common.AuthStrategyConfig{ + {Type: common.AuthTypeSecret, Secret: &common.SecretStrategyConfig{Id: "a", Value: "token-a"}}, + {Type: common.AuthTypeSecret, Secret: &common.SecretStrategyConfig{Id: "b", Value: "token-b"}}, + }, + }) + + t.Run("successful second key does not meter", func(t *testing.T) { + before := authFailedCount("secret", "invalid_secret") + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}`)) + user, err := registry.Authenticate(context.Background(), req, "eth_chainId", &AuthPayload{ + Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "token-b"}, + }) + require.NoError(t, err) + require.Equal(t, "b", user.Id) + assert.Equal(t, before, authFailedCount("secret", "invalid_secret")) + }) + + t.Run("client denial meters once", func(t *testing.T) { + beforeSecret := authFailedCount("secret", "invalid_secret") + beforeJoined := authFailedCount("n/a", "unauthorized") + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}`)) + _, err := registry.Authenticate(context.Background(), req, "eth_chainId", &AuthPayload{ + Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "wrong"}, + }) + require.Error(t, err) + assert.Equal(t, beforeSecret, authFailedCount("secret", "invalid_secret")) + assert.Equal(t, beforeJoined, authFailedCount("n/a", "unauthorized")) + RecordAuthFailed(req, err) + // Two strategies failed → joined ErrAuthUnauthorized(strategy=n/a). + assert.Equal(t, beforeJoined+1, authFailedCount("n/a", "unauthorized")) + assert.Equal(t, beforeSecret, authFailedCount("secret", "invalid_secret")) + }) + + t.Run("single strategy denial keeps strategy reason labels", func(t *testing.T) { + reg := newSecretAuthRegistry(t, &common.AuthConfig{ + Strategies: []*common.AuthStrategyConfig{ + {Type: common.AuthTypeSecret, Secret: &common.SecretStrategyConfig{Id: "a", Value: "token-a"}}, + }, + }) + before := authFailedCount("secret", "invalid_secret") + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}`)) + _, err := reg.Authenticate(context.Background(), req, "eth_chainId", &AuthPayload{ + Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "wrong"}, + }) + require.Error(t, err) + RecordAuthFailed(req, err) + assert.Equal(t, before+1, authFailedCount("secret", "invalid_secret")) + }) +} + +func TestNewErrAuthUnauthorized_AttachesReason(t *testing.T) { + s := NewSecretStrategy(&common.SecretStrategyConfig{Id: "ops", Value: "correct"}) + _, err := s.Authenticate(context.Background(), nil, &AuthPayload{ + Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "wrong"}, + }) + require.Error(t, err) + var authErr *common.ErrAuthUnauthorized + require.ErrorAs(t, err, &authErr) + assert.Equal(t, "invalid_secret", string(authErr.Reason())) +} + +func TestRecordAuthFailed_IgnoresNonAuthErrors(t *testing.T) { + before := authFailedCount("n/a", "unauthorized") + RecordAuthFailed(nil, common.NewErrInvalidRequest(context.Canceled)) + assert.Equal(t, before, authFailedCount("n/a", "unauthorized")) +} diff --git a/auth/strategy_database.go b/auth/strategy_database.go index bf62a2ed8..3b34e63fa 100644 --- a/auth/strategy_database.go +++ b/auth/strategy_database.go @@ -12,7 +12,6 @@ import ( "github.com/dgraph-io/ristretto/v2" "github.com/erpc/erpc/common" "github.com/erpc/erpc/data" - "github.com/erpc/erpc/telemetry" "github.com/rs/zerolog" "golang.org/x/sync/singleflight" ) @@ -121,14 +120,12 @@ func (s *DatabaseStrategy) Supports(ap *AuthPayload) bool { func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.NormalizedRequest, ap *AuthPayload) (*common.User, error) { if ap.Secret == nil { - s.recordAuthFailureMetric(req, "missing_secret") - return nil, common.NewErrAuthUnauthorized("database", "no secret provided") + return nil, common.NewErrAuthUnauthorized("database", "missing_secret", "no secret provided") } apiKey := ap.Secret.Value if apiKey == "" { - s.recordAuthFailureMetric(req, "empty_secret") - return nil, common.NewErrAuthUnauthorized("database", "empty API key") + return nil, common.NewErrAuthUnauthorized("database", "empty_secret", "empty API key") } // Check positive cache first if available @@ -144,8 +141,7 @@ func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.Normali if s.negCache != nil { if _, found := s.negCache.Get(apiKey); found { s.logger.Debug().Str("apiKey", apiKey).Msg("API key found in negative cache") - s.recordAuthFailureMetric(req, "cached_unknown_api_key") - return nil, common.NewErrAuthUnauthorized("database", "invalid API key") + return nil, common.NewErrAuthUnauthorized("database", "cached_unknown_api_key", "invalid API key") } } @@ -157,7 +153,6 @@ func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.Normali // One caller per connectorDownProbeInterval still goes through the real // DB path so we eventually notice recovery; everyone else fast-paths. if u := s.tryFastFailOpen(); u != nil { - s.recordAuthFailureMetric(req, "db_fail_open_fast_path") return u, nil } @@ -182,8 +177,7 @@ func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.Normali // RecordNotFound is a business signal (key really doesn't // exist). The DB is healthy — don't taint connectorDown. s.markConnectorUp() - s.recordAuthFailureMetric(req, "invalid_api_key") - return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", "invalid API key"), neg: true}, nil + return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", "invalid_api_key", "invalid API key"), neg: true}, nil } // Real DB error: flip the connector-down latch so subsequent // requests in this probe window fast-path to fail-open without @@ -197,13 +191,13 @@ func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.Normali Str("driver", string(s.cfg.Connector.Driver)). Str("connectorId", s.cfg.Connector.Id). Msg("database query failed during authentication") - s.recordAuthFailureMetric(req, s.classifyDbError(err)) + reason := s.classifyDbError(err) // Fail-open if configured if u := s.buildFailOpenUser(); u != nil { s.logger.Error().Str("userId", u.Id).Msg("auth DB error; fail-open enabled, granting emergency user") return &authFetchResult{user: u, err: nil, neg: false, skipCache: true}, nil } - return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", fmt.Sprintf("database query failed: %v", err)), neg: false}, nil + return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", reason, fmt.Sprintf("database query failed: %v", err)), neg: false}, nil } // Successful query: the DB is healthy. Clear any stale connectorDown @@ -217,13 +211,11 @@ func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.Normali } if err := json.Unmarshal(valueBytes, &userData); err != nil { s.logger.Error().Err(err).Str("apiKey", apiKey).RawJSON("data", valueBytes).Msg("failed to parse user data from database") - s.recordAuthFailureMetric(req, "db_record_parse_error") - return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", "invalid user data format"), neg: false}, nil + return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", "db_record_parse_error", "invalid user data format"), neg: false}, nil } if userData.UserId == "" { s.logger.Error().Str("apiKey", apiKey).RawJSON("data", valueBytes).Msg("missing user ID in database record") - s.recordAuthFailureMetric(req, "db_record_missing_user_id") - return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", "missing user ID in data"), neg: false}, nil + return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", "db_record_missing_user_id", "missing user ID in data"), neg: false}, nil } enabled := true if userData.Enabled != nil { @@ -231,8 +223,7 @@ func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.Normali } if !enabled { s.logger.Warn().Str("apiKey", apiKey).Str("userId", userData.UserId).Msg("authentication attempt with disabled API key") - s.recordAuthFailureMetric(req, "disabled_key") - return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", "API key is disabled"), neg: true}, nil + return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", "disabled_key", "API key is disabled"), neg: true}, nil } user := &common.User{Id: userData.UserId} if userData.RateLimitBudget != "" { @@ -241,7 +232,6 @@ func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.Normali return &authFetchResult{user: user, err: nil, neg: false}, nil }) if sfErr != nil { - s.recordAuthFailureMetric(req, "internal_error") if u := s.buildFailOpenUser(); u != nil { s.logger.Warn().Str("userId", u.Id).Err(sfErr).Msg("singleflight error; fail-open enabled, granting emergency user") return u, nil @@ -449,30 +439,6 @@ func (s *DatabaseStrategy) Close() { } } -// recordAuthFailureMetric increments the auth failure metric with safe labels -func (s *DatabaseStrategy) recordAuthFailureMetric(req *common.NormalizedRequest, reason string) { - project := "n/a" - network := "n/a" - agent := "unknown" - if req != nil { - // Prefer resolved network/project when available - if n := req.Network(); n != nil { - project = n.ProjectId() - network = req.NetworkLabel() - } else { - network = req.NetworkId() - } - agent = req.AgentName() - } - telemetry.MetricAuthFailedTotal.WithLabelValues( - project, - network, - "database", - reason, - agent, - ).Inc() -} - // classifyDbError converts database errors into a bounded set of reason labels. // // During the 2026-05-13 incident the previous implementation collapsed two diff --git a/auth/strategy_jwt.go b/auth/strategy_jwt.go index 8a0a931d7..7e7ffb65c 100644 --- a/auth/strategy_jwt.go +++ b/auth/strategy_jwt.go @@ -86,18 +86,18 @@ func (s *JwtStrategy) Supports(ap *AuthPayload) bool { func (s *JwtStrategy) Authenticate(ctx context.Context, req *common.NormalizedRequest, ap *AuthPayload) (*common.User, error) { token, _, err := s.parser.ParseUnverified(ap.Jwt.Token, jwt.MapClaims{}) if err != nil { - return nil, common.NewErrAuthUnauthorized("jwt", "failed to parse JWT") + return nil, common.NewErrAuthUnauthorized("jwt", "parse_failed", "failed to parse JWT") } if len(s.cfg.AllowedAlgorithms) > 0 { if !contains(s.cfg.AllowedAlgorithms, token.Method.Alg()) { - return nil, common.NewErrAuthUnauthorized("jwt", "unexpected signing method") + return nil, common.NewErrAuthUnauthorized("jwt", "unexpected_algorithm", "unexpected signing method") } } key, err := s.findVerificationKey(ctx, token) if err != nil { - return nil, common.NewErrAuthUnauthorized("jwt", err.Error()) + return nil, common.NewErrAuthUnauthorized("jwt", "no_verification_key", err.Error()) } // Verify the signature (and standard claims such as exp via jwt.Parse). @@ -108,16 +108,16 @@ func (s *JwtStrategy) Authenticate(ctx context.Context, req *common.NormalizedRe claims, ok := token.Claims.(jwt.MapClaims) if !ok { - return nil, common.NewErrAuthUnauthorized("jwt", "invalid JWT claims") + return nil, common.NewErrAuthUnauthorized("jwt", "invalid_claims", "invalid JWT claims") } if err := s.validateClaims(claims); err != nil { - return nil, common.NewErrAuthUnauthorized("jwt", err.Error()) + return nil, common.NewErrAuthUnauthorized("jwt", "claims_invalid", err.Error()) } id, ok := claims["sub"].(string) if !ok { - return nil, common.NewErrAuthUnauthorized("jwt", "missing 'sub' claim to be used as user id") + return nil, common.NewErrAuthUnauthorized("jwt", "missing_sub", "missing 'sub' claim to be used as user id") } user := &common.User{Id: id} @@ -190,7 +190,7 @@ func errAuthUnauthorizedFromJwtParse(err error) error { return common.NewErrAuthUnauthorizedReason("jwt", common.AuthUnauthorizedReasonTokenExpired) } // Other ValidationError flags (nbf, iat, malformed, …) keep jwt's text. - return common.NewErrAuthUnauthorized("jwt", err.Error()) + return common.NewErrAuthUnauthorized("jwt", "invalid_token", err.Error()) } return common.NewErrAuthUnauthorizedReason("jwt", common.AuthUnauthorizedReasonInvalidSignature) } diff --git a/auth/strategy_jwt_paths_test.go b/auth/strategy_jwt_paths_test.go new file mode 100644 index 000000000..7d20439a7 --- /dev/null +++ b/auth/strategy_jwt_paths_test.go @@ -0,0 +1,306 @@ +package auth + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/golang-jwt/jwt/v4" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func pemPublicKey(t *testing.T, blockType string, pub interface{}) string { + t.Helper() + der, err := x509.MarshalPKIXPublicKey(pub) + require.NoError(t, err) + return string(pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: der})) +} + +func authenticateJwt(s *JwtStrategy, token string) (*common.User, error) { + return s.Authenticate(context.Background(), nil, &AuthPayload{Type: common.AuthTypeJwt, Jwt: &JwtPayload{Token: token}}) +} + +func TestJwtStrategy_ClaimValidation(t *testing.T) { + cfg := &common.JwtStrategyConfig{ + VerificationKeys: map[string]string{"default": testJwtHMACSecret}, + AllowedAlgorithms: []string{"HS256"}, + AllowedIssuers: []string{"https://issuer.example"}, + AllowedAudiences: []string{"erpc"}, + RequiredClaims: []string{"tenant"}, + RateLimitBudgetClaimName: "tier", + } + s := newTestJwtStrategy(t, cfg) + assert.True(t, s.Supports(&AuthPayload{Type: common.AuthTypeJwt})) + assert.False(t, s.Supports(&AuthPayload{Type: common.AuthTypeSecret})) + + base := func() jwt.MapClaims { + return jwt.MapClaims{ + "sub": "svc", + "iss": "https://issuer.example", + "aud": "erpc", + "tenant": "t1", + "tier": "gold", + "exp": time.Now().Add(time.Hour).Unix(), + } + } + + t.Run("valid token carries budget from custom claim", func(t *testing.T) { + user, err := authenticateJwt(s, signTestJWT(t, base())) + require.NoError(t, err) + assert.Equal(t, "svc", user.Id) + assert.Equal(t, "gold", user.RateLimitBudget) + }) + + t.Run("empty budget claim is ignored", func(t *testing.T) { + c := base() + c["tier"] = "" + user, err := authenticateJwt(s, signTestJWT(t, c)) + require.NoError(t, err) + assert.Empty(t, user.RateLimitBudget) + }) + + rejects := []struct { + name string + mutate func(jwt.MapClaims) + reason string + message string + }{ + {name: "missing issuer", mutate: func(c jwt.MapClaims) { delete(c, "iss") }, reason: "claims_invalid", message: "'iss' issuer claim is missing"}, + {name: "foreign issuer", mutate: func(c jwt.MapClaims) { c["iss"] = "https://evil" }, reason: "claims_invalid", message: "the 'https://evil' issuer is not allowed"}, + {name: "missing audience", mutate: func(c jwt.MapClaims) { delete(c, "aud") }, reason: "claims_invalid", message: "'aud' audience claim is missing"}, + {name: "foreign audience", mutate: func(c jwt.MapClaims) { c["aud"] = "other" }, reason: "claims_invalid", message: "the 'other' audience is not allowed"}, + {name: "missing required claim", mutate: func(c jwt.MapClaims) { delete(c, "tenant") }, reason: "claims_invalid", message: "missing required claim: tenant"}, + {name: "non-string subject", mutate: func(c jwt.MapClaims) { c["sub"] = 42 }, reason: "missing_sub"}, + {name: "not-before in the future", mutate: func(c jwt.MapClaims) { c["nbf"] = time.Now().Add(time.Hour).Unix() }, reason: "invalid_token"}, + } + for _, tc := range rejects { + t.Run(tc.name, func(t *testing.T) { + c := base() + tc.mutate(c) + _, err := authenticateJwt(s, signTestJWT(t, c)) + requireAuthReason(t, err, "jwt", tc.reason) + if tc.message != "" { + assert.Contains(t, err.Error(), tc.message) + } + }) + } + + t.Run("garbage token fails parsing", func(t *testing.T) { + _, err := authenticateJwt(s, "not.a.jwt") + requireAuthReason(t, err, "jwt", "parse_failed") + }) + + t.Run("algorithm outside allow list", func(t *testing.T) { + tok := jwt.NewWithClaims(jwt.SigningMethodHS512, base()) + signed, err := tok.SignedString([]byte(testJwtHMACSecret)) + require.NoError(t, err) + _, err = authenticateJwt(s, signed) + requireAuthReason(t, err, "jwt", "unexpected_algorithm") + }) + + t.Run("claims.Valid failure inside validateClaims", func(t *testing.T) { + // jwt.Parse already rejects bad standard claims; validateClaims re-checks + // them for direct callers. + err := s.validateClaims(jwt.MapClaims{"exp": time.Now().Add(-time.Hour).Unix()}) + require.ErrorContains(t, err, "invalid standard claims") + }) + + t.Run("default budget claim name is rlm", func(t *testing.T) { + plain := newTestJwtStrategy(t, &common.JwtStrategyConfig{VerificationKeys: map[string]string{"k": testJwtHMACSecret}}) + user, err := authenticateJwt(plain, signTestJWT(t, jwt.MapClaims{"sub": "a", "rlm": "silver"})) + require.NoError(t, err) + assert.Equal(t, "silver", user.RateLimitBudget) + }) +} + +func TestJwtStrategy_PemKeysAndKeyTypes(t *testing.T) { + rsaKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + ecKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + dir := t.TempDir() + rsaPath := filepath.Join(dir, "rsa.pem") + require.NoError(t, os.WriteFile(rsaPath, []byte(pemPublicKey(t, "PUBLIC KEY", &rsaKey.PublicKey)), 0o600)) + + s := newTestJwtStrategy(t, &common.JwtStrategyConfig{ + VerificationKeys: map[string]string{ + "rsa": "file://" + rsaPath, + "ec": pemPublicKey(t, "EC PUBLIC KEY", &ecKey.PublicKey), + }, + }) + claims := func() jwt.MapClaims { return jwt.MapClaims{"sub": "k", "exp": time.Now().Add(time.Hour).Unix()} } + + t.Run("rsa key loaded from file verifies RS256 by kid", func(t *testing.T) { + user, err := authenticateJwt(s, signTestRS256JWT(t, rsaKey, "rsa", claims())) + require.NoError(t, err) + assert.Equal(t, "k", user.Id) + }) + t.Run("ec pem key verifies ES256 without kid via type fallback", func(t *testing.T) { + tok := jwt.NewWithClaims(jwt.SigningMethodES256, claims()) + signed, err := tok.SignedString(ecKey) + require.NoError(t, err) + user, err := authenticateJwt(s, signed) + require.NoError(t, err) + assert.Equal(t, "k", user.Id) + }) + t.Run("HMAC token finds no compatible asymmetric key", func(t *testing.T) { + _, err := authenticateJwt(s, signTestJWT(t, claims())) + requireAuthReason(t, err, "jwt", "no_verification_key") + }) + t.Run("unsupported signing method finds no key", func(t *testing.T) { + tok := jwt.NewWithClaims(jwt.SigningMethodNone, claims()) + signed, err := tok.SignedString(jwt.UnsafeAllowNoneSignatureType) + require.NoError(t, err) + _, err = authenticateJwt(s, signed) + requireAuthReason(t, err, "jwt", "no_verification_key") + }) + + t.Run("isCompatibleKeyType rejects erroring keyfunc", func(t *testing.T) { + failing := func(*jwt.Token) (interface{}, error) { return nil, assert.AnError } + assert.False(t, isCompatibleKeyType(failing, jwt.SigningMethodHS256)) + }) + + t.Run("parseKey rejects non-string and missing file", func(t *testing.T) { + _, err := parseKey(123) + require.ErrorContains(t, err, "unsupported key type: int") + _, err = parseKey("file://" + filepath.Join(dir, "missing.pem")) + require.ErrorContains(t, err, "failed to read key file") + }) +} + +func TestJwtStrategy_StartupFailsWithoutAnyKeys(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + logger := zerolog.Nop() + _, err := NewJwtStrategy(context.Background(), &logger, &common.JwtStrategyConfig{VerificationJwksUrl: srv.URL}) + require.ErrorContains(t, err, "JWKS endpoint returned status 503") +} + +func TestNewJwksHTTPClient_InsecureSkipVerify(t *testing.T) { + secure := newJwksHTTPClient(&common.JwtStrategyConfig{}) + assert.Nil(t, secure.Transport) + + insecure := newJwksHTTPClient(&common.JwtStrategyConfig{VerificationJwksTlsInsecureSkipVerify: true}) + tr, ok := insecure.Transport.(*http.Transport) + require.True(t, ok) + assert.True(t, tr.TLSClientConfig.InsecureSkipVerify) + assert.Equal(t, defaultJwksHTTPTimeout, insecure.Timeout) +} + +func TestParseJWKS_RejectsBadDocuments(t *testing.T) { + b64 := base64.RawURLEncoding.EncodeToString + logger := zerolog.Nop() + tests := []struct { + name string + body string + wantErr string + }{ + {name: "invalid json", body: "{", wantErr: "failed to parse JWKS JSON"}, + {name: "no keys", body: `{"keys":[]}`, wantErr: "contains no keys"}, + { + name: "only unusable keys", + body: mustJSON(t, jwksDocument{Keys: []jwksKey{ + {Kty: "RSA", Kid: ""}, // no kid + {Kty: "oct", Kid: "sym"}, // unsupported type + {Kty: "RSA", Kid: "bad-n", N: "!!", E: "AQAB"}, // bad modulus + {Kty: "RSA", Kid: "bad-e", N: "AQAB", E: "!!"}, // bad exponent + {Kty: "RSA", Kid: "big-e", N: "AQAB", E: b64(make([]byte, 9))}, // exponent too large + {Kty: "RSA", Kid: "zero-e", N: "AQAB", E: b64([]byte{0})}, // zero exponent + {Kty: "EC", Kid: "bad-crv", Crv: "P-999"}, + {Kty: "EC", Kid: "bad-x", Crv: "P-256", X: "!!", Y: "AA"}, + {Kty: "EC", Kid: "bad-y", Crv: "P-384", X: "AA", Y: "!!"}, + {Kty: "EC", Kid: "off-curve", Crv: "P-521", X: b64([]byte{1}), Y: b64([]byte{1})}, + }}), + wantErr: "contains no usable keys", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + keys, err := parseJWKS([]byte(tc.body), &logger) + require.ErrorContains(t, err, tc.wantErr) + assert.Nil(t, keys) + }) + } + + t.Run("key alg pin rejects tokens signed with another alg", func(t *testing.T) { + rsaKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + keys, err := parseJWKS([]byte(mustJSON(t, jwksDocument{Keys: []jwksKey{rsaJWKFromPrivateKey(t, rsaKey, "pinned")}})), nil) + require.NoError(t, err) + _, err = keys["pinned"](&jwt.Token{Method: jwt.SigningMethodRS512}) + require.ErrorContains(t, err, `requires alg "RS256"`) + }) +} + +func mustJSON(t *testing.T, v interface{}) string { + t.Helper() + b, err := json.Marshal(v) + require.NoError(t, err) + return string(b) +} + +func TestFetchVerificationKeysFromJWKS_TransportErrors(t *testing.T) { + _, err := fetchVerificationKeysFromJWKS(context.Background(), http.DefaultClient, "://bad-url", nil) + require.ErrorContains(t, err, "failed to create JWKS request") + + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + url := srv.URL + srv.Close() + _, err = fetchVerificationKeysFromJWKS(context.Background(), http.DefaultClient, url, nil) + require.ErrorContains(t, err, "failed to fetch JWKS") +} + +func TestJwtStrategy_ScheduledRefreshKeepsStaleKeysOnFailure(t *testing.T) { + rsaKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + doc := mustJSON(t, jwksDocument{Keys: []jwksKey{rsaJWKFromPrivateKey(t, rsaKey, "kid-1")}}) + + var hits atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if hits.Add(1) == 1 { + _, _ = w.Write([]byte(doc)) + return + } + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logger := zerolog.Nop() + s, err := NewJwtStrategy(ctx, &logger, &common.JwtStrategyConfig{ + VerificationJwksUrl: srv.URL, + VerificationJwksRefreshInterval: common.Duration(5 * time.Millisecond), + }) + require.NoError(t, err) + + // Lift the post-startup debounce so the ticker performs a real (failing) fetch. + s.refreshMu.Lock() + s.nextRefresh = time.Time{} + s.refreshMu.Unlock() + + require.Eventually(t, func() bool { return hits.Load() >= 2 }, 5*time.Second, 5*time.Millisecond) + + user, err := authenticateJwt(s, signTestRS256JWT(t, rsaKey, "kid-1", jwt.MapClaims{"sub": "still-ok"})) + require.NoError(t, err) + assert.Equal(t, "still-ok", user.Id) +} diff --git a/auth/strategy_network.go b/auth/strategy_network.go index dd55c0c45..1d62d89bd 100644 --- a/auth/strategy_network.go +++ b/auth/strategy_network.go @@ -51,12 +51,12 @@ func (s *NetworkStrategy) Supports(ap *AuthPayload) bool { func (s *NetworkStrategy) Authenticate(ctx context.Context, req *common.NormalizedRequest, ap *AuthPayload) (*common.User, error) { // Use the client IP resolved by the HTTP ingress and stored on the normalized request if req == nil { - return nil, common.NewErrAuthUnauthorized("network", "request context is missing") + return nil, common.NewErrAuthUnauthorized("network", "missing_request", "request context is missing") } ipStr := req.ClientIP() clientIP := net.ParseIP(ipStr) if clientIP == nil { - return nil, common.NewErrAuthUnauthorized("network", fmt.Sprintf("invalid or missing client IP: %s", ipStr)) + return nil, common.NewErrAuthUnauthorized("network", "invalid_client_ip", fmt.Sprintf("invalid or missing client IP: %s", ipStr)) } // Check if localhost is allowed @@ -93,7 +93,7 @@ func (s *NetworkStrategy) Authenticate(ctx context.Context, req *common.Normaliz } } - return nil, common.NewErrAuthUnauthorized("network", fmt.Sprintf("IP %s is not allowed", clientIP.String())) + return nil, common.NewErrAuthUnauthorized("network", "ip_not_allowed", fmt.Sprintf("IP %s is not allowed", clientIP.String())) } func isLocalhost(ip net.IP) bool { diff --git a/auth/strategy_network_siwe_test.go b/auth/strategy_network_siwe_test.go new file mode 100644 index 000000000..5428ceb88 --- /dev/null +++ b/auth/strategy_network_siwe_test.go @@ -0,0 +1,215 @@ +package auth + +import ( + "context" + "crypto/ecdsa" + "strings" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + "github.com/spruceid/siwe-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func requireAuthReason(t *testing.T, err error, strategy, reason string) { + t.Helper() + require.Error(t, err) + var authErr *common.ErrAuthUnauthorized + require.ErrorAs(t, err, &authErr) + assert.Equal(t, reason, string(authErr.Reason())) + assert.Equal(t, strategy, authErr.Details["strategy"]) +} + +func requestFromIP(ip string) *common.NormalizedRequest { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}`)) + if ip != "" { + req.SetClientIP(ip) + } + return req +} + +func TestNewNetworkStrategy_RejectsInvalidEntries(t *testing.T) { + _, err := NewNetworkStrategy(&common.NetworkStrategyConfig{AllowedIPs: []string{"10.0.0.1", "not-an-ip"}}) + require.ErrorContains(t, err, "invalid IP address: not-an-ip") + + _, err = NewNetworkStrategy(&common.NetworkStrategyConfig{AllowedCIDRs: []string{"10.0.0.0/33"}}) + require.ErrorContains(t, err, "invalid CIDR: 10.0.0.0/33") +} + +func TestNetworkStrategy_Authenticate(t *testing.T) { + cfg := &common.NetworkStrategyConfig{ + AllowedIPs: []string{"203.0.113.7", "2001:db8::1"}, + AllowedCIDRs: []string{"10.1.0.0/16", "192.168.0.0/24"}, + RateLimitBudget: "net-budget", + } + + tests := []struct { + name string + cfg *common.NetworkStrategyConfig + req *common.NormalizedRequest + wantUser string + wantBudget string + wantReason string + }{ + {name: "nil request is rejected", cfg: cfg, req: nil, wantReason: "missing_request"}, + {name: "unresolved client ip is rejected", cfg: cfg, req: requestFromIP(""), wantReason: "invalid_client_ip"}, + {name: "garbage client ip is rejected", cfg: cfg, req: requestFromIP("not-an-ip"), wantReason: "invalid_client_ip"}, + {name: "exact ipv4 match uses ip as user", cfg: cfg, req: requestFromIP("203.0.113.7"), wantUser: "203.0.113.7", wantBudget: "net-budget"}, + {name: "exact ipv6 match", cfg: cfg, req: requestFromIP("2001:db8:0:0:0:0:0:1"), wantUser: "2001:db8::1", wantBudget: "net-budget"}, + {name: "cidr match uses cidr as user by default", cfg: cfg, req: requestFromIP("10.1.200.3"), wantUser: "10.1.0.0/16", wantBudget: "net-budget"}, + { + name: "cidr match uses client ip when ipAsUser is set", + cfg: &common.NetworkStrategyConfig{AllowedCIDRs: []string{"192.168.0.0/24"}, IPAsUser: true}, + req: requestFromIP("192.168.0.42"), + wantUser: "192.168.0.42", + }, + {name: "ip outside every rule is rejected", cfg: cfg, req: requestFromIP("8.8.8.8"), wantReason: "ip_not_allowed"}, + {name: "loopback is rejected unless localhost is allowed", cfg: cfg, req: requestFromIP("127.0.0.1"), wantReason: "ip_not_allowed"}, + { + name: "ipv4 loopback allowed when allowLocalhost", + cfg: &common.NetworkStrategyConfig{AllowLocalhost: true, RateLimitBudget: "local"}, + req: requestFromIP("127.0.0.1"), + wantUser: "127.0.0.1", + wantBudget: "local", + }, + { + name: "ipv6 loopback allowed when allowLocalhost", + cfg: &common.NetworkStrategyConfig{AllowLocalhost: true}, + req: requestFromIP("::1"), + wantUser: "::1", + }, + { + name: "allowLocalhost does not admit non-loopback addresses", + cfg: &common.NetworkStrategyConfig{AllowLocalhost: true}, + req: requestFromIP("10.0.0.1"), + wantReason: "ip_not_allowed", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s, err := NewNetworkStrategy(tc.cfg) + require.NoError(t, err) + + ap := &AuthPayload{Type: common.AuthTypeNetwork} + require.True(t, s.Supports(ap)) + assert.False(t, s.Supports(&AuthPayload{Type: common.AuthTypeSecret})) + + user, err := s.Authenticate(context.Background(), tc.req, ap) + if tc.wantReason != "" { + requireAuthReason(t, err, "network", tc.wantReason) + assert.Nil(t, user) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantUser, user.Id) + assert.Equal(t, tc.wantBudget, user.RateLimitBudget) + }) + } +} + +type siweFixture struct { + key *ecdsa.PrivateKey + address string +} + +func newSiweFixture(t *testing.T) siweFixture { + t.Helper() + key, err := crypto.GenerateKey() + require.NoError(t, err) + return siweFixture{key: key, address: crypto.PubkeyToAddress(key.PublicKey).Hex()} +} + +func (f siweFixture) message(t *testing.T, domain string, opts map[string]interface{}) string { + t.Helper() + msg, err := siwe.InitMessage(domain, f.address, "https://"+domain+"/login", siwe.GenerateNonce(), opts) + require.NoError(t, err) + return msg.String() +} + +func signSiwe(t *testing.T, key *ecdsa.PrivateKey, message string) string { + t.Helper() + sig, err := crypto.Sign(accounts.TextHash([]byte(message)), key) + require.NoError(t, err) + sig[64] += 27 + return hexutil.Encode(sig) +} + +func TestSiweStrategy_Authenticate(t *testing.T) { + signer := newSiweFixture(t) + other := newSiweFixture(t) + s := NewSiweStrategy(&common.SiweStrategyConfig{ + AllowedDomains: []string{"app.example.com", "dapp.example.org"}, + RateLimitBudget: "siwe-budget", + }) + + valid := signer.message(t, "app.example.com", map[string]interface{}{ + "expirationTime": time.Now().Add(time.Hour), + }) + wrongDomain := signer.message(t, "evil.example.net", nil) + expired := signer.message(t, "app.example.com", map[string]interface{}{ + "issuedAt": time.Now().Add(-2 * time.Hour), + "expirationTime": time.Now().Add(-time.Hour), + }) + notYetValid := signer.message(t, "dapp.example.org", map[string]interface{}{ + "notBefore": time.Now().Add(time.Hour), + }) + + tests := []struct { + name string + payload *SiwePayload + wantReason string + }{ + {name: "missing payload", payload: nil, wantReason: "missing_payload"}, + {name: "unparseable message", payload: &SiwePayload{Message: "hello world", Signature: "0x00"}, wantReason: "parse_failed"}, + {name: "signature from a different key", payload: &SiwePayload{Message: valid, Signature: signSiwe(t, other.key, valid)}, wantReason: "invalid_signature"}, + {name: "malformed signature", payload: &SiwePayload{Message: valid, Signature: "0xzz"}, wantReason: "invalid_signature"}, + {name: "domain not in allow list", payload: &SiwePayload{Message: wrongDomain, Signature: signSiwe(t, signer.key, wrongDomain)}, wantReason: "domain_not_allowed"}, + {name: "expired message", payload: &SiwePayload{Message: expired, Signature: signSiwe(t, signer.key, expired)}, wantReason: "message_not_valid_now"}, + {name: "not-before in the future", payload: &SiwePayload{Message: notYetValid, Signature: signSiwe(t, signer.key, notYetValid)}, wantReason: "message_not_valid_now"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ap := &AuthPayload{Type: common.AuthTypeSiwe, Siwe: tc.payload} + require.True(t, s.Supports(ap)) + user, err := s.Authenticate(context.Background(), nil, ap) + requireAuthReason(t, err, "siwe", tc.wantReason) + assert.Nil(t, user) + }) + } + + t.Run("valid signature authenticates the lowercased signer address", func(t *testing.T) { + user, err := s.Authenticate(context.Background(), nil, &AuthPayload{ + Type: common.AuthTypeSiwe, + Siwe: &SiwePayload{Message: valid, Signature: signSiwe(t, signer.key, valid)}, + }) + require.NoError(t, err) + assert.Equal(t, strings.ToLower(signer.address), user.Id) + assert.Equal(t, "siwe-budget", user.RateLimitBudget) + }) + + t.Run("no budget configured leaves user budget empty", func(t *testing.T) { + plain := NewSiweStrategy(&common.SiweStrategyConfig{AllowedDomains: []string{"app.example.com"}}) + user, err := plain.Authenticate(context.Background(), nil, &AuthPayload{ + Type: common.AuthTypeSiwe, + Siwe: &SiwePayload{Message: valid, Signature: signSiwe(t, signer.key, valid)}, + }) + require.NoError(t, err) + assert.Empty(t, user.RateLimitBudget) + assert.False(t, plain.Supports(&AuthPayload{Type: common.AuthTypeJwt})) + }) +} + +func TestSecretStrategy_AssignsConfiguredBudget(t *testing.T) { + s := NewSecretStrategy(&common.SecretStrategyConfig{Id: "ops", Value: "pw", RateLimitBudget: "ops-budget"}) + assert.False(t, s.Supports(&AuthPayload{Type: common.AuthTypeJwt})) + user, err := s.Authenticate(context.Background(), nil, &AuthPayload{Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "pw"}}) + require.NoError(t, err) + assert.Equal(t, "ops", user.Id) + assert.Equal(t, "ops-budget", user.RateLimitBudget) +} diff --git a/auth/strategy_secret.go b/auth/strategy_secret.go index 271e18a26..d0e10fc67 100644 --- a/auth/strategy_secret.go +++ b/auth/strategy_secret.go @@ -22,7 +22,7 @@ func (s *SecretStrategy) Supports(ap *AuthPayload) bool { func (s *SecretStrategy) Authenticate(ctx context.Context, req *common.NormalizedRequest, ap *AuthPayload) (*common.User, error) { if ap.Secret.Value != s.cfg.Value { - return nil, common.NewErrAuthUnauthorized("secret", "invalid secret") + return nil, common.NewErrAuthUnauthorized("secret", "invalid_secret", "invalid secret") } user := &common.User{Id: s.cfg.Id} diff --git a/auth/strategy_siwe.go b/auth/strategy_siwe.go index d1d713f41..a49320200 100644 --- a/auth/strategy_siwe.go +++ b/auth/strategy_siwe.go @@ -25,28 +25,28 @@ func (s *SiweStrategy) Supports(ap *AuthPayload) bool { func (s *SiweStrategy) Authenticate(ctx context.Context, req *common.NormalizedRequest, ap *AuthPayload) (*common.User, error) { if ap.Siwe == nil { - return nil, common.NewErrAuthUnauthorized("siwe", "missing SIWE payload") + return nil, common.NewErrAuthUnauthorized("siwe", "missing_payload", "missing SIWE payload") } // Parse the SIWE message message, err := siwe.ParseMessage(ap.Siwe.Message) if err != nil { - return nil, common.NewErrAuthUnauthorized("siwe", fmt.Sprintf("failed to parse SIWE message: %s", err)) + return nil, common.NewErrAuthUnauthorized("siwe", "parse_failed", fmt.Sprintf("failed to parse SIWE message: %s", err)) } // Verify the signature if _, err := message.VerifyEIP191(ap.Siwe.Signature); err != nil { - return nil, common.NewErrAuthUnauthorized("siwe", fmt.Sprintf("failed to verify SIWE signature: %s", err)) + return nil, common.NewErrAuthUnauthorized("siwe", "invalid_signature", fmt.Sprintf("failed to verify SIWE signature: %s", err)) } // Check if the domain is allowed if !s.isDomainAllowed(message.GetDomain()) { - return nil, common.NewErrAuthUnauthorized("siwe", fmt.Sprintf("domain %s is not allowed", message.GetDomain())) + return nil, common.NewErrAuthUnauthorized("siwe", "domain_not_allowed", fmt.Sprintf("domain %s is not allowed", message.GetDomain())) } // Verify the message is not expired if ok, err := message.ValidNow(); !ok { - return nil, common.NewErrAuthUnauthorized("siwe", fmt.Sprintf("SIWE message expired: %s", err)) + return nil, common.NewErrAuthUnauthorized("siwe", "message_not_valid_now", fmt.Sprintf("SIWE message expired: %s", err)) } user := &common.User{Id: strings.ToLower(message.GetAddress().String())} diff --git a/clients/grpc_bds_client.go b/clients/grpc_bds_client.go index e71b45954..37b513aa4 100644 --- a/clients/grpc_bds_client.go +++ b/clients/grpc_bds_client.go @@ -329,6 +329,12 @@ func (c *GenericGrpcBdsClient) SendRequest(ctx context.Context, req *common.Norm resp, err = c.handleChainId(ctx, conn, req, jrReq) case "getBlock": resp, err = c.handleSvmGetBlock(ctx, conn, req, jrReq) + case "getTransaction": + resp, err = c.handleSvmGetTransaction(ctx, conn, req, jrReq) + case "getBlockTime": + resp, err = c.handleSvmGetBlockTime(ctx, conn, req, jrReq) + case "getBlocksWithLimit": + resp, err = c.handleSvmGetBlocksWithLimit(ctx, conn, req, jrReq) default: err := common.NewErrEndpointUnsupported( fmt.Errorf("unsupported method for gRPC BDS client: %s", jrReq.Method), @@ -1252,7 +1258,7 @@ func (c *GenericGrpcBdsClient) handleSvmGetBlock(ctx context.Context, conn *bdsC grpcSpan.SetAttributes(attribute.String("grpc_error", err.Error())) common.SetTraceSpanError(grpcSpan, err) grpcSpan.End() - return nil, svmMapGetBlockError(err) + return nil, svmMapError(err) } grpcSpan.SetAttributes( attribute.String("slot_status", grpcResp.SlotStatus.String()), @@ -1282,20 +1288,16 @@ func (c *GenericGrpcBdsClient) handleSvmGetBlock(ctx context.Context, conn *bdsC } else { result = svm.BlockToJsonRpc(grpcResp.Block, grpcResp.Signatures) } - - jsonRpcResp := &common.JsonRpcResponse{} - if err := jsonRpcResp.SetID(jrReq.ID); err != nil { - return nil, fmt.Errorf("failed to set ID: %w", err) + // Agave's history leaves these null on old slots where the reader holds + // a value; render what Agave renders. + history := svmHistoryOf(req) + if !history.RecordsBlockTime(slot) { + result["blockTime"] = nil } - resultBytes, err := sonic.Marshal(result) - if err != nil { - return nil, fmt.Errorf("failed to marshal result: %w", err) + if !history.RecordsBlockHeight(slot) { + result["blockHeight"] = nil } - jsonRpcResp.SetResult(resultBytes) - - return common.NewNormalizedResponse(). - WithRequest(req). - WithJsonRpcResponse(jsonRpcResp), nil + return svmResult(req, jrReq, result) } // svmParseSlot accepts the slot as the JSON number it is on the wire. Solana @@ -1345,10 +1347,10 @@ func svmCommitment(s string) svm.Commitment { return svm.Commitment_COMMITMENT_FINALIZED } -// svmMapGetBlockError decides, per gRPC status, whether the miss is benign +// svmMapError decides, per gRPC status, whether an SVM reader miss is benign // (fall through to a live upstream) or a genuine transport failure worth // scoring against the connector. -func svmMapGetBlockError(err error) error { +func svmMapError(err error) error { st, ok := status.FromError(err) if !ok { return fmt.Errorf("gRPC call failed: %w", err) @@ -1361,6 +1363,7 @@ func svmMapGetBlockError(err error) error { fmt.Errorf("slot outside BDS coverage: %s", st.Message()), ) case codes.Unimplemented: + // Includes signature lookups while `--serve-signature-lookups` is off. return common.NewErrEndpointUnsupported( fmt.Errorf("method not served by BDS: %s", st.Message()), ) diff --git a/clients/grpc_bds_methods_test.go b/clients/grpc_bds_methods_test.go new file mode 100644 index 000000000..80c64c9ed --- /dev/null +++ b/clients/grpc_bds_methods_test.go @@ -0,0 +1,694 @@ +package clients + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/blockchain-data-standards/manifesto/evm" + "github.com/bytedance/sonic" + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// scriptedBdsServer answers every bds.evm.RPCQueryService method with a +// configured response (or a configured error) and records the last request +// each method received, so tests can assert both directions of the mapping +// between JSON-RPC and the BDS protobuf API. +type scriptedBdsServer struct { + evm.UnimplementedRPCQueryServiceServer + + calls atomic.Int64 + + mu sync.Mutex + err error + chainID uint64 + block *evm.GetBlockResponse + tx *evm.GetTransactionByHashResponse + receipt *evm.GetTransactionReceiptResponse + receipts *evm.GetBlockReceiptsResponse + logs *evm.GetLogsResponse + headerMD metadata.MD + + byNumber *evm.GetBlockByNumberRequest + byHash *evm.GetBlockByHashRequest + txReq *evm.GetTransactionByHashRequest + receiptReq *evm.GetTransactionReceiptRequest + blockRcpts *evm.GetBlockReceiptsRequest + logsReq *evm.GetLogsRequest +} + +func (s *scriptedBdsServer) enter(ctx context.Context, record func()) error { + s.calls.Add(1) + s.mu.Lock() + defer s.mu.Unlock() + record() + if s.headerMD != nil { + _ = grpc.SetHeader(ctx, s.headerMD) + } + return s.err +} + +func (s *scriptedBdsServer) ChainId(ctx context.Context, _ *evm.ChainIdRequest) (*evm.ChainIdResponse, error) { + if err := s.enter(ctx, func() {}); err != nil { + return nil, err + } + return &evm.ChainIdResponse{ChainId: s.chainID}, nil +} + +func (s *scriptedBdsServer) GetBlockByNumber(ctx context.Context, req *evm.GetBlockByNumberRequest) (*evm.GetBlockResponse, error) { + if err := s.enter(ctx, func() { s.byNumber = req }); err != nil { + return nil, err + } + return orEmpty(s.block, &evm.GetBlockResponse{}), nil +} + +func (s *scriptedBdsServer) GetBlockByHash(ctx context.Context, req *evm.GetBlockByHashRequest) (*evm.GetBlockResponse, error) { + if err := s.enter(ctx, func() { s.byHash = req }); err != nil { + return nil, err + } + return orEmpty(s.block, &evm.GetBlockResponse{}), nil +} + +func (s *scriptedBdsServer) GetTransactionByHash(ctx context.Context, req *evm.GetTransactionByHashRequest) (*evm.GetTransactionByHashResponse, error) { + if err := s.enter(ctx, func() { s.txReq = req }); err != nil { + return nil, err + } + return orEmpty(s.tx, &evm.GetTransactionByHashResponse{}), nil +} + +func (s *scriptedBdsServer) GetTransactionReceipt(ctx context.Context, req *evm.GetTransactionReceiptRequest) (*evm.GetTransactionReceiptResponse, error) { + if err := s.enter(ctx, func() { s.receiptReq = req }); err != nil { + return nil, err + } + return orEmpty(s.receipt, &evm.GetTransactionReceiptResponse{}), nil +} + +func (s *scriptedBdsServer) GetBlockReceipts(ctx context.Context, req *evm.GetBlockReceiptsRequest) (*evm.GetBlockReceiptsResponse, error) { + if err := s.enter(ctx, func() { s.blockRcpts = req }); err != nil { + return nil, err + } + return orEmpty(s.receipts, &evm.GetBlockReceiptsResponse{}), nil +} + +func (s *scriptedBdsServer) GetLogs(ctx context.Context, req *evm.GetLogsRequest) (*evm.GetLogsResponse, error) { + if err := s.enter(ctx, func() { s.logsReq = req }); err != nil { + return nil, err + } + return orEmpty(s.logs, &evm.GetLogsResponse{}), nil +} + +func orEmpty[T any](v *T, empty *T) *T { + if v == nil { + return empty + } + return v +} + +func startScriptedBdsServer(t *testing.T, s *scriptedBdsServer) string { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer() + evm.RegisterRPCQueryServiceServer(srv, s) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + return lis.Addr().String() +} + +func bdsCall(t *testing.T, client *GenericGrpcBdsClient, method, params string) ([]byte, error) { + t.Helper() + req := common.NewNormalizedRequest([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":7,"method":%q,"params":%s}`, method, params))) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.SendRequest(ctx, req) + if err != nil { + require.Nil(t, resp) + return nil, err + } + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.Equal(t, int64(7), jrr.ID(), "response must carry the request id") + return jrr.GetResultBytes(), nil +} + +func hash32(b byte) ([]byte, string) { + h := make([]byte, 32) + for i := range h { + h[i] = b + } + return h, fmt.Sprintf("0x%x", h) +} + +// Each supported JSON-RPC method must reach the matching BDS RPC with the +// parameters translated field by field, and the BDS reply must come back as +// the JSON-RPC result a node would send (null for a miss, [] for no rows). +// The client is armed with chainId 10, so every request that has the optional +// chainId field must carry it. +func TestGrpcBdsClient_RequestResponseMapping(t *testing.T) { + const chainID = uint64(10) + hashBytes, hashHex := hash32(0xab) + addrA := "0x00000000000000000000000000000000000000aa" + addrB := "0x00000000000000000000000000000000000000bb" + _, topicHex := hash32(0x11) + + cases := []struct { + name string + method string + params string + setup func(s *scriptedBdsServer) + check func(t *testing.T, s *scriptedBdsServer, result []byte) + }{ + { + name: "getBlockByHash with full transactions", + method: "eth_getBlockByHash", + params: fmt.Sprintf(`[%q,true]`, hashHex), + setup: func(s *scriptedBdsServer) { + s.block = &evm.GetBlockResponse{Block: &evm.BlockHeader{Number: 0x10, Hash: hashBytes}} + }, + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + require.NotNil(t, s.byHash) + assert.Equal(t, hashBytes, s.byHash.BlockHash) + assert.True(t, s.byHash.IncludeTransactions) + assert.Equal(t, chainID, s.byHash.GetChainId()) + var blk map[string]interface{} + require.NoError(t, sonic.Unmarshal(result, &blk)) + assert.Equal(t, "0x10", blk["number"]) + assert.Equal(t, hashHex, blk["hash"]) + }, + }, + { + name: "getBlockByHash for an unknown block is null", + method: "eth_getBlockByHash", + params: fmt.Sprintf(`[%q,false]`, hashHex), + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + require.NotNil(t, s.byHash) + assert.False(t, s.byHash.IncludeTransactions) + assert.Equal(t, "null", string(result)) + }, + }, + { + name: "getBlockByNumber given a block hash is served by GetBlockByHash", + method: "eth_getBlockByNumber", + params: fmt.Sprintf(`[%q,false]`, hashHex), + setup: func(s *scriptedBdsServer) { + s.block = &evm.GetBlockResponse{Block: &evm.BlockHeader{Number: 0x22, Hash: hashBytes}} + }, + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + assert.Nil(t, s.byNumber, "a hash must not be sent as a block number") + require.NotNil(t, s.byHash) + assert.Equal(t, hashBytes, s.byHash.BlockHash) + assert.Equal(t, chainID, s.byHash.GetChainId()) + var blk map[string]interface{} + require.NoError(t, sonic.Unmarshal(result, &blk)) + assert.Equal(t, "0x22", blk["number"]) + }, + }, + { + name: "getBlockByNumber given an EIP-1898 blockHash object misses as null", + method: "eth_getBlockByNumber", + params: fmt.Sprintf(`[{"blockHash":%q},true]`, hashHex), + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + assert.Nil(t, s.byNumber) + require.NotNil(t, s.byHash) + assert.True(t, s.byHash.IncludeTransactions) + assert.Equal(t, "null", string(result)) + }, + }, + { + name: "getBlockByNumber passes a block tag through and misses as null", + method: "eth_getBlockByNumber", + params: `["finalized",false]`, + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + require.NotNil(t, s.byNumber) + assert.Equal(t, "finalized", s.byNumber.BlockNumber) + assert.Equal(t, chainID, s.byNumber.GetChainId()) + assert.Equal(t, "null", string(result)) + }, + }, + { + name: "getTransactionByHash for an unknown tx is null", + method: "eth_getTransactionByHash", + params: fmt.Sprintf(`[%q]`, hashHex), + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + require.NotNil(t, s.txReq) + assert.Equal(t, hashBytes, s.txReq.TransactionHash) + assert.Equal(t, chainID, s.txReq.GetChainId()) + assert.Equal(t, "null", string(result)) + }, + }, + { + name: "getTransactionReceipt for an unknown tx is null", + method: "eth_getTransactionReceipt", + params: fmt.Sprintf(`[%q]`, hashHex), + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + require.NotNil(t, s.receiptReq) + assert.Equal(t, hashBytes, s.receiptReq.TransactionHash) + assert.Equal(t, chainID, s.receiptReq.GetChainId()) + assert.Equal(t, "null", string(result)) + }, + }, + { + name: "getBlockReceipts by number returns every receipt in order", + method: "eth_getBlockReceipts", + params: `["0x10"]`, + setup: func(s *scriptedBdsServer) { + s.receipts = &evm.GetBlockReceiptsResponse{Receipts: []*evm.Receipt{ + {TransactionIndex: 0, BlockNumber: 0x10}, + {TransactionIndex: 1, BlockNumber: 0x10}, + }} + }, + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + require.NotNil(t, s.blockRcpts) + assert.Equal(t, "0x10", s.blockRcpts.GetBlockNumber()) + assert.Nil(t, s.blockRcpts.BlockHash) + assert.Equal(t, chainID, s.blockRcpts.GetChainId()) + var rcpts []map[string]interface{} + require.NoError(t, sonic.Unmarshal(result, &rcpts)) + require.Len(t, rcpts, 2) + assert.Equal(t, "0x0", rcpts[0]["transactionIndex"]) + assert.Equal(t, "0x1", rcpts[1]["transactionIndex"]) + }, + }, + { + name: "getBlockReceipts by hash with no receipts is an empty array", + method: "eth_getBlockReceipts", + params: fmt.Sprintf(`[%q]`, hashHex), + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + require.NotNil(t, s.blockRcpts) + assert.Equal(t, hashBytes, s.blockRcpts.BlockHash) + assert.Nil(t, s.blockRcpts.BlockNumber) + assert.Equal(t, "[]", string(result)) + }, + }, + { + name: "getLogs maps an address list, topics and range; no logs is []", + method: "eth_getLogs", + params: fmt.Sprintf(`[{"fromBlock":"0x5","toBlock":"0x9","address":[%q,%q],"topics":[%q,null,[%q,%q]]}]`, addrA, addrB, topicHex, topicHex, topicHex), + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + require.NotNil(t, s.logsReq) + assert.Equal(t, uint64(5), s.logsReq.GetFromBlock()) + assert.Equal(t, uint64(9), s.logsReq.GetToBlock()) + assert.Nil(t, s.logsReq.BlockHash) + require.Len(t, s.logsReq.Addresses, 2) + assert.Equal(t, byte(0xaa), s.logsReq.Addresses[0][19]) + assert.Equal(t, byte(0xbb), s.logsReq.Addresses[1][19]) + require.Len(t, s.logsReq.Topics, 3) + assert.Len(t, s.logsReq.Topics[0].Values, 1) + assert.Empty(t, s.logsReq.Topics[1].Values, "null topic is a positional wildcard") + assert.Len(t, s.logsReq.Topics[2].Values, 2) + assert.Equal(t, chainID, s.logsReq.GetChainId()) + assert.Equal(t, "[]", string(result)) + }, + }, + { + name: "getLogs maps a single address and renders each log", + method: "eth_getLogs", + params: fmt.Sprintf(`[{"fromBlock":"0x1","toBlock":"0x1","address":%q}]`, addrA), + setup: func(s *scriptedBdsServer) { + s.logs = &evm.GetLogsResponse{Logs: []*evm.Log{{BlockNumber: 1, LogIndex: 3, Address: make([]byte, 20)}}} + }, + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + require.NotNil(t, s.logsReq) + require.Len(t, s.logsReq.Addresses, 1) + assert.Empty(t, s.logsReq.Topics) + var logs []map[string]interface{} + require.NoError(t, sonic.Unmarshal(result, &logs)) + require.Len(t, logs, 1) + assert.Equal(t, "0x1", logs[0]["blockNumber"]) + assert.Equal(t, "0x3", logs[0]["logIndex"]) + }, + }, + { + name: "chainId renders as a hex quantity", + method: "eth_chainId", + params: `[]`, + check: func(t *testing.T, s *scriptedBdsServer, result []byte) { + assert.Equal(t, `"0xa"`, string(result)) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := &scriptedBdsServer{chainID: chainID} + if tc.setup != nil { + tc.setup(s) + } + client := newTestClient(t, startScriptedBdsServer(t, s)) + client.SetExpectedChainId(chainID) + + result, err := bdsCall(t, client, tc.method, tc.params) + require.NoError(t, err) + s.mu.Lock() + defer s.mu.Unlock() + tc.check(t, s, result) + }) + } +} + +// Malformed parameters must be refused locally — the server never sees the +// call — and surface as an error the caller can see, never as a result. +func TestGrpcBdsClient_InvalidParamsNeverReachServer(t *testing.T) { + _, hashHex := hash32(0xcd) + cases := []struct { + name string + method string + params string + code common.ErrorCode + msg string + }{ + {"getBlockByNumber no includeTx", "eth_getBlockByNumber", `["0x1"]`, common.ErrCodeEndpointTransportFailure, "insufficient params"}, + {"getBlockByNumber bool block", "eth_getBlockByNumber", `[true,false]`, common.ErrCodeEndpointTransportFailure, "failed to parse block parameter"}, + {"getBlockByNumber empty object", "eth_getBlockByNumber", `[{"foo":1},false]`, common.ErrCodeEndpointTransportFailure, "failed to parse block parameter"}, + {"getBlockByNumber string includeTx", "eth_getBlockByNumber", `["0x1","yes"]`, common.ErrCodeEndpointTransportFailure, "invalid includeTransactions"}, + {"getBlockByHash one param", "eth_getBlockByHash", fmt.Sprintf(`[%q]`, hashHex), common.ErrCodeEndpointTransportFailure, "insufficient params"}, + {"getBlockByHash numeric hash", "eth_getBlockByHash", `[1,false]`, common.ErrCodeEndpointTransportFailure, "invalid block hash parameter"}, + {"getBlockByHash non-hex hash", "eth_getBlockByHash", `["0xzz",false]`, common.ErrCodeEndpointTransportFailure, "failed to parse block hash"}, + {"getBlockByHash string includeTx", "eth_getBlockByHash", fmt.Sprintf(`[%q,"no"]`, hashHex), common.ErrCodeEndpointTransportFailure, "invalid includeTransactions"}, + {"getLogs no filter", "eth_getLogs", `[]`, common.ErrCodeEndpointTransportFailure, "insufficient params"}, + {"getLogs filter not object", "eth_getLogs", `["x"]`, common.ErrCodeEndpointTransportFailure, "failed to parse params"}, + {"getLogs numeric blockHash", "eth_getLogs", `[{"blockHash":5}]`, common.ErrCodeEndpointTransportFailure, "failed to parse blockHash"}, + {"getLogs blockHash with range", "eth_getLogs", fmt.Sprintf(`[{"blockHash":%q,"fromBlock":"0x1"}]`, hashHex), common.ErrCodeEndpointTransportFailure, "cannot be combined"}, + {"getLogs non-hex blockHash", "eth_getLogs", `[{"blockHash":"0xzz"}]`, common.ErrCodeEndpointTransportFailure, "failed to parse blockHash"}, + {"getLogs bad fromBlock", "eth_getLogs", `[{"fromBlock":"12","toBlock":"0x1"}]`, common.ErrCodeEndpointTransportFailure, "failed to parse fromBlock"}, + {"getLogs bad toBlock", "eth_getLogs", `[{"fromBlock":"0x1","toBlock":"12"}]`, common.ErrCodeEndpointTransportFailure, "failed to parse toBlock"}, + {"getLogs tag range", "eth_getLogs", `[{"fromBlock":"0x1","toBlock":"latest"}]`, common.ErrCodeEndpointTransportFailure, "special block numbers not yet supported"}, + {"getLogs bad address", "eth_getLogs", `[{"fromBlock":"0x1","toBlock":"0x1","address":"0xzz"}]`, common.ErrCodeEndpointTransportFailure, "failed to parse address"}, + {"getLogs bad address in list", "eth_getLogs", `[{"fromBlock":"0x1","toBlock":"0x1","address":["0xzz"]}]`, common.ErrCodeEndpointTransportFailure, "failed to parse address"}, + {"getLogs bad topic", "eth_getLogs", `[{"fromBlock":"0x1","toBlock":"0x1","topics":["0xzz"]}]`, common.ErrCodeEndpointTransportFailure, "failed to parse topic"}, + {"getTransactionByHash no params", "eth_getTransactionByHash", `[]`, common.ErrCodeEndpointTransportFailure, "insufficient params"}, + {"getTransactionByHash numeric", "eth_getTransactionByHash", `[1]`, common.ErrCodeEndpointTransportFailure, "invalid transaction hash"}, + {"getTransactionByHash bad hex", "eth_getTransactionByHash", `["0xzz"]`, common.ErrCodeEndpointTransportFailure, "failed to parse transaction hash"}, + {"getTransactionReceipt no params", "eth_getTransactionReceipt", `[]`, common.ErrCodeEndpointTransportFailure, "insufficient params"}, + {"getTransactionReceipt numeric", "eth_getTransactionReceipt", `[1]`, common.ErrCodeEndpointTransportFailure, "invalid transaction hash"}, + {"getTransactionReceipt bad hex", "eth_getTransactionReceipt", `["0xzz"]`, common.ErrCodeEndpointTransportFailure, "failed to parse transaction hash"}, + {"getBlockReceipts no params", "eth_getBlockReceipts", `[]`, common.ErrCodeEndpointTransportFailure, "insufficient params"}, + {"getBlockReceipts bool block", "eth_getBlockReceipts", `[true]`, common.ErrCodeEndpointTransportFailure, "failed to parse block parameter"}, + {"svm getBlock no slot", "getBlock", `[]`, common.ErrCodeEndpointTransportFailure, "insufficient params"}, + {"svm getBlock hex slot", "getBlock", `["0x10"]`, common.ErrCodeEndpointTransportFailure, "slot"}, + {"svm getBlock config not object", "getBlock", `[1,"json"]`, common.ErrCodeEndpointTransportFailure, "failed to parse getBlock config"}, + {"svm getBlock raw encoding", "getBlock", `[1,{"encoding":"base64"}]`, common.ErrCodeEndpointUnsupported, "base64"}, + } + + s := &scriptedBdsServer{chainID: 1} + client := newTestClient(t, startScriptedBdsServer(t, s)) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + before := s.calls.Load() + _, err := bdsCall(t, client, tc.method, tc.params) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, tc.code), "want %s, got %v", tc.code, err) + assert.Contains(t, err.Error(), tc.msg) + assert.Equal(t, before, s.calls.Load(), "invalid params must not reach the server") + }) + } +} + +// Every gRPC status a BDS server can answer with must land on the eRPC error +// type the upstream policies key on (retry, failover, missing-data handling), +// keeping the server's message. +func TestGrpcBdsClient_GrpcStatusToErpcError(t *testing.T) { + cases := []struct { + code codes.Code + want common.ErrorCode + }{ + {codes.Unavailable, common.ErrCodeEndpointServerSideException}, + {codes.Internal, common.ErrCodeEndpointServerSideException}, + {codes.Unknown, common.ErrCodeEndpointServerSideException}, + {codes.FailedPrecondition, common.ErrCodeEndpointServerSideException}, + {codes.NotFound, common.ErrCodeEndpointMissingData}, + {codes.OutOfRange, common.ErrCodeEndpointMissingData}, + {codes.InvalidArgument, common.ErrCodeEndpointClientSideException}, + {codes.ResourceExhausted, common.ErrCodeEndpointCapacityExceeded}, + {codes.Unauthenticated, common.ErrCodeEndpointUnauthorized}, + {codes.PermissionDenied, common.ErrCodeEndpointUnauthorized}, + {codes.Unimplemented, common.ErrCodeEndpointUnsupported}, + {codes.Canceled, common.ErrCodeEndpointRequestCanceled}, + {codes.DeadlineExceeded, common.ErrCodeEndpointRequestTimeout}, + } + for _, tc := range cases { + t.Run(tc.code.String(), func(t *testing.T) { + msg := "bds says " + strings.ToLower(tc.code.String()) + s := &scriptedBdsServer{err: status.Error(tc.code, msg)} + client := newTestClient(t, startScriptedBdsServer(t, s)) + _, err := bdsCall(t, client, "eth_chainId", `[]`) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, tc.want), "want %s, got %v", tc.want, err) + assert.Contains(t, err.Error(), msg) + }) + } +} + +// A gRPC failure on any method's call path is classified the same way: the +// handler's wrapping must not hide the status from the classifier. +func TestGrpcBdsClient_EveryMethodPropagatesGrpcStatus(t *testing.T) { + _, hashHex := hash32(0x01) + calls := []struct{ method, params string }{ + {"eth_getBlockByNumber", `["0x1",false]`}, + {"eth_getBlockByNumber", fmt.Sprintf(`[%q,false]`, hashHex)}, + {"eth_getBlockByHash", fmt.Sprintf(`[%q,false]`, hashHex)}, + {"eth_getLogs", `[{"fromBlock":"0x1","toBlock":"0x2"}]`}, + {"eth_getTransactionByHash", fmt.Sprintf(`[%q]`, hashHex)}, + {"eth_getTransactionReceipt", fmt.Sprintf(`[%q]`, hashHex)}, + {"eth_getBlockReceipts", `["0x1"]`}, + {"eth_chainId", `[]`}, + } + s := &scriptedBdsServer{err: status.Error(codes.NotFound, "block not indexed")} + client := newTestClient(t, startScriptedBdsServer(t, s)) + for _, c := range calls { + t.Run(c.method+c.params[:2], func(t *testing.T) { + _, err := bdsCall(t, client, c.method, c.params) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointMissingData), "got %v", err) + assert.Contains(t, err.Error(), "block not indexed") + }) + } +} + +// A request body that is not JSON-RPC fails before any connection is used. +func TestGrpcBdsClient_UnparsableRequestIsUpstreamRequestError(t *testing.T) { + s := &scriptedBdsServer{} + client := newTestClient(t, startScriptedBdsServer(t, s)) + _, err := client.SendRequest(context.Background(), common.NewNormalizedRequest([]byte(`not json`))) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamRequest), "got %v", err) + assert.Zero(t, s.calls.Load()) +} + +// A server answering eth_chainId for another chain than the one this client +// is armed for is a cross-wired endpoint: the caller must get an error, never +// the foreign chain id. +func TestGrpcBdsClient_ChainIdMismatchIsError(t *testing.T) { + s := &scriptedBdsServer{chainID: 56} + client := newTestClient(t, startScriptedBdsServer(t, s)) + client.SetExpectedChainId(1) + + result, err := bdsCall(t, client, "eth_chainId", `[]`) + require.Error(t, err) + assert.Nil(t, result) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointTransportFailure), "got %v", err) + assert.Contains(t, err.Error(), common.ErrCodeEndpointChainIdMismatch) +} + +// An upstream configured for one chain must refuse to build a client against +// an endpoint that proves to serve another; the matching chain builds fine and +// stamps its chainId on requests. +func TestNewGrpcBdsClient_VerifiesConfiguredChainId(t *testing.T) { + s := &scriptedBdsServer{chainID: 1} + addr := startScriptedBdsServer(t, s) + parsedURL, err := url.Parse("grpc://" + addr) + require.NoError(t, err) + logger := zerolog.New(io.Discard) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + wrong := common.NewFakeUpstream("bds-wrong") + wrong.Config().Evm = &common.EvmUpstreamConfig{ChainId: 5} + _, err = NewGrpcBdsClient(ctx, &logger, "prj", wrong, parsedURL, 1, "") + require.Error(t, err) + assert.Contains(t, err.Error(), "cross-wired") + + right := common.NewFakeUpstream("bds-right") + right.Config().Evm = &common.EvmUpstreamConfig{ChainId: 1} + c, err := NewGrpcBdsClient(ctx, &logger, "prj", right, parsedURL, 1, "") + require.NoError(t, err) + t.Cleanup(c.(*GenericGrpcBdsClient).Shutdown) + assert.Equal(t, ClientTypeGrpcBds, c.GetType()) + + _, err = bdsCall(t, c.(*GenericGrpcBdsClient), "eth_getBlockByNumber", `["0x1",false]`) + require.NoError(t, err) + s.mu.Lock() + defer s.mu.Unlock() + require.NotNil(t, s.byNumber) + assert.Equal(t, uint64(1), s.byNumber.GetChainId()) +} + +// Caller-side ends of a request are not upstream failures: an expired caller +// deadline is a request timeout, a caller cancel mid-call is a cancellation. +func TestGrpcBdsClient_CallerContextEndings(t *testing.T) { + addr, wedged, stop := startWedgedServer(t) + defer stop() + client := newTestClient(t, addr) + req := func() *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x1",false]}`)) + } + + expired, cancelExpired := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancelExpired() + _, err := client.SendRequest(expired, req()) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointRequestTimeout), "got %v", err) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := client.SendRequest(ctx, req()) + done <- err + }() + require.Eventually(t, func() bool { return wedged.started.Load() >= 1 }, 5*time.Second, 5*time.Millisecond) + cancel() + select { + case err := <-done: + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointRequestCanceled), "got %v", err) + case <-time.After(5 * time.Second): + t.Fatal("SendRequest did not return after the caller canceled") + } +} + +// A client whose pool has no connection left must fail as a transport +// failure, and QueryClient must report no client rather than panic. +func TestGrpcBdsClient_EmptyPool(t *testing.T) { + s := &scriptedBdsServer{chainID: 1} + client := newTestClient(t, startScriptedBdsServer(t, s)) + require.NotNil(t, client.QueryClient(), "a live pool hands out a query client") + + client.pool.poolMu.Lock() + saved := client.pool.conns + client.pool.conns = nil + client.pool.poolMu.Unlock() + t.Cleanup(func() { + client.pool.poolMu.Lock() + client.pool.conns = saved + client.pool.poolMu.Unlock() + }) + + assert.Nil(t, client.QueryClient()) + _, err := bdsCall(t, client, "eth_chainId", `[]`) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointTransportFailure), "got %v", err) + assert.Contains(t, err.Error(), "no available connections") + assert.Zero(t, s.calls.Load()) +} + +// Errors that carry no gRPC status are transport failures; nil stays nil. +func TestGrpcBdsClient_NormalizeNonGrpcError(t *testing.T) { + client := newClientWithPoolSize(t, 1) + assert.NoError(t, client.normalizeGrpcError(nil)) + err := client.normalizeGrpcError(fmt.Errorf("wrapped: %w", errors.New("connection reset"))) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointTransportFailure), "got %v", err) + assert.Contains(t, err.Error(), "connection reset") +} + +// With detailed tracing on, response metadata the server sends back is +// recorded on the client span: single values as strings, repeated keys as +// lists. +func TestGrpcBdsClient_DetailedTracingRecordsResponseMetadata(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp), sdktrace.WithSampler(sdktrace.AlwaysSample())) + common.SetTracerProviderForTest(tp) + common.IsTracingDetailed = true + t.Cleanup(func() { + common.IsTracingDetailed = false + common.IsTracingEnabled = false + }) + + s := &scriptedBdsServer{ + chainID: 1, + headerMD: metadata.Pairs("x-served-by", "reader-3", "x-shard", "a", "x-shard", "b"), + block: &evm.GetBlockResponse{Block: &evm.BlockHeader{Number: 9, Hash: []byte{1}}}, + } + client := newTestClient(t, startScriptedBdsServer(t, s)) + + _, err := bdsCall(t, client, "eth_chainId", `[]`) + require.NoError(t, err) + _, err = bdsCall(t, client, "eth_getBlockByNumber", `["0x9",false]`) + require.NoError(t, err) + _, hashHex := hash32(0x02) + _, err = bdsCall(t, client, "eth_getBlockByNumber", fmt.Sprintf(`[%q,false]`, hashHex)) + require.NoError(t, err) + require.NoError(t, tp.ForceFlush(context.Background())) + + attrs := map[string]map[string]interface{}{} + for _, sp := range exp.GetSpans() { + if attrs[sp.Name] == nil { + attrs[sp.Name] = map[string]interface{}{} + } + for _, a := range sp.Attributes { + attrs[sp.Name][string(a.Key)] = a.Value.AsInterface() + } + } + send := attrs["GrpcBdsClient.SendRequest"] + require.NotNil(t, send, "SendRequest span missing") + assert.Equal(t, "reader-3", send["grpc.response.metadata.x-served-by"]) + assert.Equal(t, []string{"a", "b"}, send["grpc.response.metadata.x-shard"]) + assert.Equal(t, "7", send["request.id"]) + + byNum := attrs["GrpcBdsClient.GetBlockByNumber"] + require.NotNil(t, byNum, "GetBlockByNumber detail span missing") + assert.Equal(t, true, byNum["response_has_block"]) + assert.Equal(t, int64(9), byNum["response_block_number"]) + assert.Equal(t, "reader-3", byNum["grpc.response.metadata.x-served-by"]) + + byHash := attrs["GrpcBdsClient.GetBlockByHash"] + require.NotNil(t, byHash, "GetBlockByHash detail span missing") + assert.Equal(t, hashHex[2:], byHash["block_hash"]) +} + +// The maintainer's recycle swaps a fresh verified conn into the slot; a +// replacement that proves to serve another chain is discarded and the slot +// keeps its conn. +func TestBdsPool_RecycleConn(t *testing.T) { + s := &scriptedBdsServer{chainID: 1} + client := newTestClient(t, startScriptedBdsServer(t, s)) + p := client.pool + slot0 := func() *bdsConn { + p.poolMu.RLock() + defer p.poolMu.RUnlock() + return p.conns[0] + } + + old := slot0() + p.expectedChainId.Store(2) + p.recycleConn(old, "chainid_mismatch") + assert.Same(t, old, slot0(), "a wrong-chain replacement must not be installed") + + p.expectedChainId.Store(1) + p.recycleConn(old, "closed") + replaced := slot0() + assert.NotSame(t, old, replaced, "a verified replacement takes the slot") + assert.Positive(t, old.closedAt.Load()) + + p.recycleConn(old, "age") + assert.Same(t, replaced, slot0(), "a conn no longer pooled is not recycled again") + + _, err := bdsCall(t, client, "eth_chainId", `[]`) + require.NoError(t, err, "pool keeps serving after a recycle") +} diff --git a/clients/grpc_bds_svm.go b/clients/grpc_bds_svm.go new file mode 100644 index 000000000..3fb8af3ef --- /dev/null +++ b/clients/grpc_bds_svm.go @@ -0,0 +1,381 @@ +package clients + +import ( + "context" + "fmt" + "strconv" + + "github.com/blockchain-data-standards/manifesto/svm" + "github.com/blockchain-data-standards/manifesto/svm/parse" + "github.com/bytedance/sonic" + "github.com/erpc/erpc/common" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// svmMaxBlocksLimit is Agave's getBlocksWithLimit cap; above it the node +// answers -32602, which only the live upstream renders. +const svmMaxBlocksLimit = 500_000 + +// svmUnservable reports a request this path will not answer. It is an +// "unsupported" error, never a failure: the cache treats it as a miss and the +// request goes to a live upstream, which renders whatever Agave would. +func svmUnservable(format string, args ...interface{}) error { + return common.NewErrEndpointUnsupported(fmt.Errorf(format, args...)) +} + +// svmParams decodes the positional params array. +func svmParams(jrReq *common.JsonRpcRequest) ([]interface{}, error) { + jrReq.RLock() + paramsBytes, err := sonic.Marshal(jrReq.Params) + jrReq.RUnlock() + if err != nil { + return nil, fmt.Errorf("failed to marshal params: %w", err) + } + var params []interface{} + if err := sonic.Unmarshal(paramsBytes, ¶ms); err != nil { + return nil, fmt.Errorf("failed to parse params: %w", err) + } + return params, nil +} + +// svmConfigObject returns params[i] as a config object; absent or null is an +// empty config. +func svmConfigObject(params []interface{}, i int) (map[string]interface{}, error) { + if len(params) <= i || params[i] == nil { + return nil, nil + } + cfg, ok := params[i].(map[string]interface{}) + if !ok { + return nil, svmUnservable("param %d is not a config object", i) + } + return cfg, nil +} + +// svmStrictCommitment maps a request commitment the reader can honour. +// "processed" and anything unknown fall through: the reader serves nothing +// below confirmed, and the node renders its own answer for a bad value. +func svmStrictCommitment(v interface{}) (svm.Commitment, error) { + switch v { + case nil, "finalized": + return svm.Commitment_COMMITMENT_FINALIZED, nil + case "confirmed": + return svm.Commitment_COMMITMENT_CONFIRMED, nil + default: + return 0, svmUnservable("commitment %v not served by BDS", v) + } +} + +// svmFinalizedSlot is the network's majority finalized root, or ok=false when +// the request carries no SVM network or no upstream has reported one. The +// reader stores blocks at confirmed and does not read commitment for these +// calls, so this is what proves an answer rooted. +func svmFinalizedSlot(ctx context.Context, req *common.NormalizedRequest) (uint64, bool) { + n, ok := req.Network().(common.SvmNetwork) + if !ok { + return 0, false + } + tip := n.SvmHighestFinalizedSlot(ctx) + if tip <= 0 { + return 0, false + } + return uint64(tip), true +} + +// svmHistoryOf is the Agave history of the request's cluster; the zero value +// (every field recorded) when the request carries no SVM network config. +func svmHistoryOf(req *common.NormalizedRequest) common.SvmHistory { + n := req.Network() + if n == nil { + return common.SvmHistory{} + } + cfg := n.Config() + if cfg == nil || cfg.Svm == nil { + return common.SvmHistory{} + } + return common.KnownSvmHistory(cfg.Svm.Chain, cfg.Svm.Cluster) +} + +// svmResult wraps a rendered result in the request's JSON-RPC envelope. +func svmResult(req *common.NormalizedRequest, jrReq *common.JsonRpcRequest, result interface{}) (*common.NormalizedResponse, error) { + resultBytes, err := sonic.Marshal(result) + if err != nil { + return nil, fmt.Errorf("failed to marshal result: %w", err) + } + jsonRpcResp := &common.JsonRpcResponse{} + if err := jsonRpcResp.SetID(jrReq.ID); err != nil { + return nil, fmt.Errorf("failed to set ID: %w", err) + } + jsonRpcResp.SetResult(resultBytes) + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jsonRpcResp), nil +} + +// svmVersionServable is Agave's validate_version: legacy always, a numeric +// version only at or below the request's cap. An absent or unparsable +// version cannot be proven servable. +func svmVersionServable(version *string, max *uint32) bool { + if version == nil { + return false + } + if *version == "legacy" { + return true + } + n, err := strconv.ParseUint(*version, 10, 32) + return err == nil && max != nil && n <= uint64(*max) +} + +// handleSvmGetTransaction serves Solana `getTransaction` from the reader's +// signature index. +// +// An unset transaction means "this reader has no entry", never "the chain has +// no such transaction": the backfilled history carries no index entries. So +// every answer the reader cannot prove — no entry, a different first +// signature, no transactionIndex, a version the caller did not opt into — +// falls through to a live upstream rather than rendering null. +func (c *GenericGrpcBdsClient) handleSvmGetTransaction(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) { + params, err := svmParams(jrReq) + if err != nil { + return nil, err + } + if len(params) < 1 || len(params) > 2 { + return nil, svmUnservable("getTransaction takes 1 or 2 params, got %d", len(params)) + } + sigStr, ok := params[0].(string) + if !ok { + return nil, svmUnservable("getTransaction signature is not a string") + } + signature, err := svm.Base58Decode(sigStr) + if err != nil || len(signature) != 64 { + return nil, svmUnservable("getTransaction signature is not 64 base58 bytes") + } + + grpcReq := &svm.GetTransactionRequest{Signature: signature} + var wantParsed bool + encoding := func(v interface{}) error { + switch v { + case nil, "json": + return nil + case "jsonParsed": + wantParsed = true + return nil + default: + return svmUnservable("getTransaction encoding %v not served by BDS", v) + } + } + if len(params) == 2 { + // Agave's RpcEncodingConfigWrapper: a config object, or the + // deprecated bare encoding string. + if s, isString := params[1].(string); isString { + if err := encoding(s); err != nil { + return nil, err + } + } else { + cfg, err := svmConfigObject(params, 1) + if err != nil { + return nil, err + } + for key, value := range cfg { + if value == nil { + continue + } + switch key { + case "encoding": + if err := encoding(value); err != nil { + return nil, err + } + case "commitment": + if grpcReq.Commitment, err = svmStrictCommitment(value); err != nil { + return nil, err + } + case "maxSupportedTransactionVersion": + n, isNum := value.(float64) + if !isNum || n < 0 || n > 255 || n != float64(uint8(n)) { + return nil, svmUnservable("maxSupportedTransactionVersion %v is not a u8", value) + } + v := uint32(n) + grpcReq.MaxSupportedTransactionVersion = &v + default: + return nil, svmUnservable("getTransaction config key %q not served by BDS", key) + } + } + } + } + + ctx, grpcSpan := common.StartDetailSpan(ctx, "GrpcBdsClient.SvmGetTransaction", + trace.WithAttributes(attribute.String("commitment", grpcReq.Commitment.String())), + ) + grpcResp, err := callBoundedT(ctx, func(ctx context.Context) (*svm.GetTransactionResponse, error) { + return conn.svmClient.GetTransaction(ctx, grpcReq) + }) + if err != nil { + common.SetTraceSpanError(grpcSpan, err) + grpcSpan.End() + return nil, svmMapError(err) + } + ct := grpcResp.GetTransaction() + grpcSpan.SetAttributes(attribute.Bool("found", ct != nil)) + grpcSpan.End() + + // The index maps a hash of the first signature; a transaction whose first + // signature is not the one asked for is not the answer, whatever the + // reader says. + sigs := ct.GetTransaction().GetSignatures() + if len(sigs) == 0 || string(sigs[0]) != string(signature) { + return nil, svmUnservable("transaction not indexed by BDS") + } + if grpcResp.TransactionIndex == nil { + return nil, svmUnservable("BDS returned no transactionIndex") + } + if !svmVersionServable(ct.Version, grpcReq.MaxSupportedTransactionVersion) { + return nil, svmUnservable("transaction version not servable to this request") + } + + var result map[string]interface{} + if wantParsed { + parse.AttachToTransaction(ct) + result = svm.ConfirmedTransactionToJsonRpcParsed(ct) + } else { + result = svm.ConfirmedTransactionToJsonRpc(ct) + } + if grpcReq.MaxSupportedTransactionVersion == nil { + // Agave renders `version` only for a caller that opted in. + delete(result, "version") + } + result["slot"] = grpcResp.Slot + result["transactionIndex"] = *grpcResp.TransactionIndex + if grpcResp.BlockTime != nil && svmHistoryOf(req).RecordsBlockTime(grpcResp.Slot) { + result["blockTime"] = *grpcResp.BlockTime + } else { + result["blockTime"] = nil + } + return svmResult(req, jrReq, result) +} + +// handleSvmGetBlockTime serves Solana `getBlockTime`. The call takes no +// commitment: Agave reads a rooted time at or below its root and a bank's +// clock above it, so only a slot the network has finalized is answered here. +func (c *GenericGrpcBdsClient) handleSvmGetBlockTime(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) { + params, err := svmParams(jrReq) + if err != nil { + return nil, err + } + if len(params) != 1 { + return nil, svmUnservable("getBlockTime takes 1 param, got %d", len(params)) + } + slot, err := svmParseSlot(params[0]) + if err != nil { + return nil, svmUnservable("%v", err) + } + finalized, ok := svmFinalizedSlot(ctx, req) + if !ok || slot > finalized { + return nil, svmUnservable("slot %d is not finalized on this network", slot) + } + history := svmHistoryOf(req) + if history.GenesisTimeOnly(slot) { + // Agave answers slot 0 with the genesis creation time, which no + // archive row carries. + return nil, svmUnservable("slot 0 block time is the genesis creation time") + } + + ctx, grpcSpan := common.StartDetailSpan(ctx, "GrpcBdsClient.SvmGetBlockTime", + trace.WithAttributes(attribute.Int64("slot", int64(slot))), + ) + grpcResp, err := callBoundedT(ctx, func(ctx context.Context) (*svm.GetBlockTimeResponse, error) { + return conn.svmClient.GetBlockTime(ctx, &svm.GetBlockTimeRequest{Slot: slot}) + }) + if err != nil { + common.SetTraceSpanError(grpcSpan, err) + grpcSpan.End() + return nil, svmMapError(err) + } + grpcSpan.SetAttributes(attribute.String("slot_status", grpcResp.SlotStatus.String())) + grpcSpan.End() + + // A skip is -32007 inside the node's ledger and -32009 below it; that + // window differs by vendor, so only the live upstream can answer it. + if grpcResp.SlotStatus == svm.SlotStatus_SLOT_SKIPPED { + return nil, svmUnservable("slot %d is skipped; not served from BDS", slot) + } + if !history.RecordsBlockTime(slot) { + return svmResult(req, jrReq, nil) + } + if grpcResp.BlockTime == nil { + // A present slot Agave records a time for: the reader holding none + // is not Agave's null. + return nil, svmUnservable("BDS holds no block time for slot %d", slot) + } + return svmResult(req, jrReq, *grpcResp.BlockTime) +} + +// handleSvmGetBlocksWithLimit serves Solana `getBlocksWithLimit`. Agave lists +// the first `limit` rooted blocks from `start`; the reader stops early at its +// tip or a skip-dense range, so only a list of exactly `limit` ascending slots +// ending at a finalized slot is that list. +func (c *GenericGrpcBdsClient) handleSvmGetBlocksWithLimit(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) { + params, err := svmParams(jrReq) + if err != nil { + return nil, err + } + if len(params) < 2 || len(params) > 3 { + return nil, svmUnservable("getBlocksWithLimit takes 2 or 3 params, got %d", len(params)) + } + start, err := svmParseSlot(params[0]) + if err != nil { + return nil, svmUnservable("%v", err) + } + limit, err := svmParseSlot(params[1]) + if err != nil || limit > svmMaxBlocksLimit { + return nil, svmUnservable("limit %v not served by BDS", params[1]) + } + grpcReq := &svm.GetBlocksWithLimitRequest{StartSlot: start, Limit: limit} + cfg, err := svmConfigObject(params, 2) + if err != nil { + return nil, err + } + for key, value := range cfg { + if value == nil { + continue + } + if key != "commitment" { + return nil, svmUnservable("getBlocksWithLimit config key %q not served by BDS", key) + } + if grpcReq.Commitment, err = svmStrictCommitment(value); err != nil { + return nil, err + } + } + if limit == 0 { + return svmResult(req, jrReq, []uint64{}) + } + finalized, ok := svmFinalizedSlot(ctx, req) + if !ok || start > finalized { + return nil, svmUnservable("start slot %d is not finalized on this network", start) + } + + ctx, grpcSpan := common.StartDetailSpan(ctx, "GrpcBdsClient.SvmGetBlocksWithLimit", + trace.WithAttributes( + attribute.Int64("start_slot", int64(start)), + attribute.Int64("limit", int64(limit)), + ), + ) + grpcResp, err := callBoundedT(ctx, func(ctx context.Context) (*svm.GetBlocksResponse, error) { + return conn.svmClient.GetBlocksWithLimit(ctx, grpcReq) + }) + if err != nil { + common.SetTraceSpanError(grpcSpan, err) + grpcSpan.End() + return nil, svmMapError(err) + } + slots := grpcResp.GetSlots() + grpcSpan.SetAttributes(attribute.Int("slots", len(slots))) + grpcSpan.End() + + if uint64(len(slots)) != limit || slots[0] < start || slots[len(slots)-1] > finalized { + return nil, svmUnservable("BDS list is not the rooted list Agave returns") + } + for i := 1; i < len(slots); i++ { + if slots[i] <= slots[i-1] { + return nil, svmUnservable("BDS list is not ascending") + } + } + return svmResult(req, jrReq, slots) +} diff --git a/clients/grpc_bds_svm_test.go b/clients/grpc_bds_svm_test.go index c13605161..cdb9d1b03 100644 --- a/clients/grpc_bds_svm_test.go +++ b/clients/grpc_bds_svm_test.go @@ -742,7 +742,7 @@ func TestSvmMapGetBlockError(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - err := svmMapGetBlockError(tc.in) + err := svmMapError(tc.in) require.Error(t, err) assert.Equal(t, tc.wantBenign, common.HasErrorCode(err, common.ErrCodeEndpointUnsupported), "benign-miss vs. real-failure classification decides whether a broken reader gets scored; got: %v", err) diff --git a/clients/http_json_rpc_client.go b/clients/http_json_rpc_client.go index 736668956..5003e5d66 100644 --- a/clients/http_json_rpc_client.go +++ b/clients/http_json_rpc_client.go @@ -390,7 +390,7 @@ func (c *GenericHttpJsonRpcClient) processBatch(alreadyLocked bool) { batchReq = append(batchReq, common.JsonRpcRequest{ JSONRPC: jrReq.JSONRPC, Method: jrReq.Method, - Params: jrReq.Params, + Params: common.WireParams(req.ctx, req.request, jrReq.Params), ID: jrReq.ID, }) } @@ -702,7 +702,7 @@ func (c *GenericHttpJsonRpcClient) sendSingleRequest(ctx context.Context, req *c requestBody, err := common.SonicCfg.Marshal(common.JsonRpcRequest{ JSONRPC: jrReq.JSONRPC, Method: jrReq.Method, - Params: jrReq.Params, + Params: common.WireParams(ctx, req, jrReq.Params), ID: jrReq.ID, }) jrReq.RUnlock() @@ -781,6 +781,7 @@ func (c *GenericHttpJsonRpcClient) sendSingleRequest(ctx context.Context, req *c func (c *GenericHttpJsonRpcClient) prepareRequest(ctx context.Context, body []byte) (*http.Request, error) { var bodyReader io.Reader = bytes.NewReader(body) var pooledRC io.ReadCloser + contentLength := int64(len(body)) // Check if gzip compression is enabled if c.enableGzip { @@ -810,6 +811,7 @@ func (c *GenericHttpJsonRpcClient) prepareRequest(ctx context.Context, body []by prc := util.NewPooledBufferReadCloser(buf) pooledRC = prc bodyReader = prc + contentLength = int64(buf.Len()) } httpReq, err := http.NewRequestWithContext(ctx, "POST", c.Url.String(), bodyReader) @@ -819,6 +821,10 @@ func (c *GenericHttpJsonRpcClient) prepareRequest(ctx context.Context, body []by } return nil, err } + // NewRequest infers the length only for *bytes.Reader and friends; without + // it the pooled gzip body goes out chunked, which costs more on the wire + // than gzip saves on small requests. + httpReq.ContentLength = contentLength httpReq.Header.Set("Accept-Encoding", "gzip") httpReq.Header.Set("Content-Type", "application/json") diff --git a/clients/http_json_rpc_client_behavior_test.go b/clients/http_json_rpc_client_behavior_test.go new file mode 100644 index 000000000..7c52116e8 --- /dev/null +++ b/clients/http_json_rpc_client_behavior_test.go @@ -0,0 +1,734 @@ +package clients + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/bytedance/sonic" + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// capturedHttpRequest is what an httptest upstream saw for one POST. +type capturedHttpRequest struct { + header http.Header + body []byte + uri string +} + +// recordingUpstream is an httptest JSON-RPC upstream that records every +// request (decompressing gzip bodies) and answers through reply. +type recordingUpstream struct { + *httptest.Server + mu sync.Mutex + reqs []capturedHttpRequest +} + +func newRecordingUpstream(t *testing.T, reply func(w http.ResponseWriter, body []byte)) *recordingUpstream { + t.Helper() + u := &recordingUpstream{} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var reader io.Reader = r.Body + if r.Header.Get("Content-Encoding") == "gzip" { + gz, err := gzip.NewReader(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + reader = gz + } + body, err := io.ReadAll(reader) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + u.mu.Lock() + u.reqs = append(u.reqs, capturedHttpRequest{header: r.Header.Clone(), body: body, uri: r.RequestURI}) + u.mu.Unlock() + reply(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *recordingUpstream) requests() []capturedHttpRequest { + u.mu.Lock() + defer u.mu.Unlock() + return append([]capturedHttpRequest(nil), u.reqs...) +} + +func newHttpTestClient(t *testing.T, endpoint string, cfg *common.JsonRpcUpstreamConfig, proxy *ProxyPool, extractor common.JsonRpcErrorExtractor) *GenericHttpJsonRpcClient { + t.Helper() + parsed, err := url.Parse(endpoint) + require.NoError(t, err) + ups := common.NewFakeUpstream("rpc1") + ups.Config().Type = common.UpstreamTypeEvm + ups.Config().Endpoint = endpoint + ups.Config().JsonRpc = cfg + if extractor == nil { + extractor = &noopErrorExtractor{} + } + logger := zerolog.New(io.Discard) + appCtx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + c, err := NewGenericHttpJsonRpcClient(appCtx, &logger, "prj1", ups, parsed, cfg, proxy, extractor) + require.NoError(t, err) + return c.(*GenericHttpJsonRpcClient) +} + +func rpcReq(id int, method string) *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":%q,"params":[]}`, id, method))) +} + +func batchCfg(maxSize int, maxWait time.Duration) *common.JsonRpcUpstreamConfig { + return &common.JsonRpcUpstreamConfig{ + SupportsBatch: &common.TRUE, + BatchMaxSize: maxSize, + BatchMaxWait: common.Duration(maxWait), + } +} + +// echoBatch answers a batch with one result per request, "r", in reverse +// order so matching by position would be visibly wrong. +func echoBatch(w http.ResponseWriter, body []byte) { + var reqs []map[string]interface{} + if err := sonic.Unmarshal(body, &reqs); err != nil { + http.Error(w, "not a batch", http.StatusBadRequest) + return + } + out := make([]string, 0, len(reqs)) + for i := len(reqs) - 1; i >= 0; i-- { + out = append(out, fmt.Sprintf(`{"jsonrpc":"2.0","id":%v,"result":"r%v"}`, reqs[i]["id"], reqs[i]["id"])) + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, "["+strings.Join(out, ",")+"]") +} + +type batchOutcome struct { + result string + err error +} + +func sendConcurrently(t *testing.T, c *GenericHttpJsonRpcClient, ctx context.Context, ids []int) map[int]batchOutcome { + t.Helper() + var mu sync.Mutex + out := make(map[int]batchOutcome, len(ids)) + var wg sync.WaitGroup + for _, id := range ids { + wg.Add(1) + go func(id int) { + defer wg.Done() + resp, err := c.SendRequest(ctx, rpcReq(id, "eth_blockNumber")) + o := batchOutcome{err: err} + if err == nil { + jrr, jerr := resp.JsonRpcResponse() + if jerr != nil { + o.err = jerr + } else { + o.result = jrr.GetResultString() + } + } + mu.Lock() + out[id] = o + mu.Unlock() + }(id) + } + wg.Wait() + return out +} + +func batchSizes(reqs []capturedHttpRequest) []int { + sizes := make([]int, 0, len(reqs)) + for _, r := range reqs { + var arr []interface{} + if err := sonic.Unmarshal(r.body, &arr); err != nil { + sizes = append(sizes, -1) + continue + } + sizes = append(sizes, len(arr)) + } + sort.Ints(sizes) + return sizes +} + +// Each caller gets the result whose id matches its own request, wherever it +// sits in the upstream's array, and batches are cut at BatchMaxSize. +func TestHttpJsonRpcClient_BatchSplitsAtMaxSizeAndMatchesById(t *testing.T) { + up := newRecordingUpstream(t, echoBatch) + c := newHttpTestClient(t, up.URL, batchCfg(2, time.Minute), nil, nil) + + out := sendConcurrently(t, c, context.Background(), []int{1, 2, 3, 4}) + for id := 1; id <= 4; id++ { + require.NoError(t, out[id].err, "id %d", id) + assert.Equal(t, fmt.Sprintf(`"r%d"`, id), out[id].result) + } + assert.Equal(t, []int{2, 2}, batchSizes(up.requests()), "4 requests with max size 2 make two full batches") +} + +// A partial batch is flushed when BatchMaxWait elapses, and the batch POST +// carries the configured headers and JSON content type. +func TestHttpJsonRpcClient_BatchFlushesOnMaxWait(t *testing.T) { + up := newRecordingUpstream(t, echoBatch) + cfg := batchCfg(100, 20*time.Millisecond) + cfg.Headers = map[string]string{"X-Api-Key": "secret"} + c := newHttpTestClient(t, up.URL, cfg, nil, nil) + + out := sendConcurrently(t, c, context.Background(), []int{7}) + require.NoError(t, out[7].err) + assert.Equal(t, `"r7"`, out[7].result) + + reqs := up.requests() + require.Len(t, reqs, 1) + assert.Equal(t, []int{1}, batchSizes(reqs)) + assert.Equal(t, "secret", reqs[0].header.Get("X-Api-Key")) + assert.Equal(t, "application/json", reqs[0].header.Get("Content-Type")) + assert.Contains(t, reqs[0].header.Get("User-Agent"), "Project/prj1") +} + +// Two callers using the same JSON-RPC id cannot share a batch (the response +// could not be routed); the second one goes out in a batch of its own and +// both still get an answer. +func TestHttpJsonRpcClient_BatchDuplicateIdStartsNewBatch(t *testing.T) { + up := newRecordingUpstream(t, echoBatch) + c := newHttpTestClient(t, up.URL, batchCfg(10, 50*time.Millisecond), nil, nil) + + errs := make(chan error, 2) + for range 2 { + go func() { + _, err := c.SendRequest(context.Background(), rpcReq(5, "eth_chainId")) + errs <- err + }() + } + for range 2 { + require.NoError(t, <-errs) + } + assert.Equal(t, []int{1, 1}, batchSizes(up.requests())) +} + +// Batch responses the client cannot route or understand must fail every +// affected caller with the right error instead of hanging or mixing answers. +func TestHttpJsonRpcClient_BatchResponseShapes(t *testing.T) { + cases := []struct { + name string + status int + body string + gzip bool + check func(t *testing.T, out map[int]batchOutcome) + }{ + { + name: "unknown and missing ids", + status: 200, + body: `[{"jsonrpc":"2.0","id":1,"result":"0x1"},{"jsonrpc":"2.0","id":99,"result":"0x63"},{"jsonrpc":"2.0","result":"0x0"}]`, + check: func(t *testing.T, out map[int]batchOutcome) { + require.NoError(t, out[1].err) + assert.Equal(t, `"0x1"`, out[1].result) + require.Error(t, out[2].err) + assert.Contains(t, out[2].err.Error(), "no response received for request ID: 2") + }, + }, + { + name: "per-item error goes only to its caller", + status: 200, + body: `[{"jsonrpc":"2.0","id":1,"result":"0x1"},{"jsonrpc":"2.0","id":2,"error":{"code":-32000,"message":"header not found"}}]`, + check: func(t *testing.T, out map[int]batchOutcome) { + require.NoError(t, out[1].err) + require.Error(t, out[2].err) + assert.True(t, common.HasErrorCode(out[2].err, common.ErrCodeJsonRpcExceptionInternal), "got %v", out[2].err) + assert.Contains(t, out[2].err.Error(), "header not found") + }, + }, + { + name: "item without result or error is a parse error", + status: 200, + body: `[{"jsonrpc":"2.0","id":1,"result":"0x1"},{"jsonrpc":"2.0","id":2}]`, + check: func(t *testing.T, out map[int]batchOutcome) { + require.NoError(t, out[1].err) + require.Error(t, out[2].err) + assert.Contains(t, out[2].err.Error(), "cannot parse json rpc response from upstream") + }, + }, + { + name: "single object answers every caller", + status: 200, + body: `{"jsonrpc":"2.0","id":1,"result":"0xshared"}`, + check: func(t *testing.T, out map[int]batchOutcome) { + for _, id := range []int{1, 2} { + require.NoError(t, out[id].err) + assert.Equal(t, `"0xshared"`, out[id].result) + } + }, + }, + { + name: "single object error fails every caller", + status: 429, + body: `{"jsonrpc":"2.0","id":null,"error":{"code":-32005,"message":"too many requests"}}`, + check: func(t *testing.T, out map[int]batchOutcome) { + for _, id := range []int{1, 2} { + require.Error(t, out[id].err) + assert.Contains(t, out[id].err.Error(), "too many requests") + } + }, + }, + { + name: "scalar body is malformed", + status: 200, + body: `"just a string"`, + check: func(t *testing.T, out map[int]batchOutcome) { + for _, id := range []int{1, 2} { + require.Error(t, out[id].err) + assert.True(t, common.HasErrorCode(out[id].err, "ErrUpstreamMalformedResponse"), "got %v", out[id].err) + } + }, + }, + { + name: "empty body fails every caller", + status: 502, + body: ``, + check: func(t *testing.T, out map[int]batchOutcome) { + for _, id := range []int{1, 2} { + require.Error(t, out[id].err) + assert.Contains(t, out[id].err.Error(), "empty response") + } + }, + }, + { + name: "gzip encoded batch is decoded", + status: 200, + gzip: true, + body: `[{"jsonrpc":"2.0","id":2,"result":"0x2"},{"jsonrpc":"2.0","id":1,"result":"0x1"}]`, + check: func(t *testing.T, out map[int]batchOutcome) { + require.NoError(t, out[1].err) + require.NoError(t, out[2].err) + assert.Equal(t, `"0x1"`, out[1].result) + assert.Equal(t, `"0x2"`, out[2].result) + }, + }, + { + name: "corrupt gzip body fails every caller", + status: 200, + body: "not gzip at all", + check: func(t *testing.T, out map[int]batchOutcome) { + for _, id := range []int{1, 2} { + require.Error(t, out[id].err) + assert.Contains(t, out[id].err.Error(), "gzip") + } + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + up := newRecordingUpstream(t, func(w http.ResponseWriter, _ []byte) { + body := []byte(tc.body) + if tc.gzip || strings.HasPrefix(tc.name, "corrupt gzip") { + w.Header().Set("Content-Encoding", "gzip") + } + if tc.gzip { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, _ = gz.Write(body) + _ = gz.Close() + body = buf.Bytes() + } + w.WriteHeader(tc.status) + _, _ = w.Write(body) + }) + c := newHttpTestClient(t, up.URL, batchCfg(2, time.Minute), nil, nil) + tc.check(t, sendConcurrently(t, c, context.Background(), []int{1, 2})) + }) + } +} + +// A caller whose context is already done is answered from its own context +// and never sent upstream. +func TestHttpJsonRpcClient_BatchSkipsDoneCallers(t *testing.T) { + up := newRecordingUpstream(t, echoBatch) + c := newHttpTestClient(t, up.URL, batchCfg(10, 10*time.Millisecond), nil, nil) + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + _, err := c.SendRequest(canceled, rpcReq(1, "eth_blockNumber")) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointRequestCanceled), "got %v", err) + + expired, cancelExpired := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancelExpired() + _, err = c.SendRequest(expired, rpcReq(2, "eth_blockNumber")) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointRequestTimeout), "got %v", err) + + assert.Empty(t, up.requests()) +} + +// A queued caller that cancels while the batch is still waiting returns at +// once with a cancellation and is dropped from the batch that goes out. +func TestHttpJsonRpcClient_BatchCallerCancelsWhileQueued(t *testing.T) { + up := newRecordingUpstream(t, echoBatch) + c := newHttpTestClient(t, up.URL, batchCfg(2, time.Minute), nil, nil) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := c.SendRequest(ctx, rpcReq(1, "eth_blockNumber")) + done <- err + }() + require.Eventually(t, func() bool { + c.batchMu.Lock() + defer c.batchMu.Unlock() + return len(c.batchRequests) == 1 + }, 5*time.Second, time.Millisecond) + cancel() + err := <-done + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointRequestCanceled), "got %v", err) + + // The next request fills the batch; the canceled one must not be sent. + resp, err := c.SendRequest(context.Background(), rpcReq(2, "eth_blockNumber")) + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + assert.Equal(t, `"r2"`, jrr.GetResultString()) + assert.Equal(t, []int{1}, batchSizes(up.requests())) +} + +// A batch whose upstream never answers in time fails each caller with a +// request timeout bounded by the earliest caller deadline. +func TestHttpJsonRpcClient_BatchUpstreamTimeout(t *testing.T) { + release := make(chan struct{}) + up := newRecordingUpstream(t, func(w http.ResponseWriter, _ []byte) { <-release }) + t.Cleanup(func() { close(release) }) // runs before up.Close (LIFO) + c := newHttpTestClient(t, up.URL, batchCfg(2, time.Minute), nil, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + out := sendConcurrently(t, c, ctx, []int{1, 2}) + for _, id := range []int{1, 2} { + require.Error(t, out[id].err) + assert.True(t, common.HasErrorCode(out[id].err, common.ErrCodeEndpointRequestTimeout), "got %v", out[id].err) + } +} + +// An unreachable endpoint is a transport failure for every batched caller. +func TestHttpJsonRpcClient_BatchTransportFailure(t *testing.T) { + dead := httptest.NewServer(http.NotFoundHandler()) + addr := dead.URL + dead.Close() + c := newHttpTestClient(t, addr, batchCfg(2, time.Minute), nil, nil) + + out := sendConcurrently(t, c, context.Background(), []int{1, 2}) + for _, id := range []int{1, 2} { + require.Error(t, out[id].err) + assert.True(t, common.HasErrorCode(out[id].err, common.ErrCodeEndpointTransportFailure), "got %v", out[id].err) + } +} + +// Shutting the client down (app context canceled) releases queued callers +// with a cancellation instead of leaving them waiting for the batch timer. +func TestHttpJsonRpcClient_BatchAppShutdownReleasesCallers(t *testing.T) { + up := newRecordingUpstream(t, echoBatch) + parsed, err := url.Parse(up.URL) + require.NoError(t, err) + ups := common.NewFakeUpstream("rpc1") + logger := zerolog.New(io.Discard) + appCtx, appCancel := context.WithCancel(context.Background()) + hc, err := NewGenericHttpJsonRpcClient(appCtx, &logger, "prj1", ups, parsed, batchCfg(10, time.Minute), nil, &noopErrorExtractor{}) + require.NoError(t, err) + c := hc.(*GenericHttpJsonRpcClient) + + done := make(chan error, 1) + go func() { + _, err := c.SendRequest(context.Background(), rpcReq(1, "eth_blockNumber")) + done <- err + }() + require.Eventually(t, func() bool { + c.batchMu.Lock() + defer c.batchMu.Unlock() + return len(c.batchRequests) == 1 + }, 5*time.Second, time.Millisecond) + appCancel() + select { + case err := <-done: + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointRequestCanceled), "got %v", err) + case <-time.After(5 * time.Second): + t.Fatal("queued caller not released on shutdown") + } + assert.Empty(t, up.requests()) +} + +// A body that is not a JSON-RPC request fails before anything is sent. +// (The batch path is not covered: SendRequest dereferences the nil parsed +// request on that error path - see http_json_rpc_client.go SendRequest.) +func TestHttpJsonRpcClient_UnparsableRequest(t *testing.T) { + up := newRecordingUpstream(t, echoBatch) + c := newHttpTestClient(t, up.URL, nil, nil, nil) + _, err := c.SendRequest(context.Background(), common.NewNormalizedRequest([]byte(`{oops`))) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamRequest), "got %v", err) + assert.Empty(t, up.requests()) +} + +// What the single-request path returns for each kind of HTTP reply. +func TestHttpJsonRpcClient_SingleResponseClassification(t *testing.T) { + cases := []struct { + name string + status int + ctype string + body string + gzip bool + wantErr common.ErrorCode + contains string + result string + }{ + {name: "200 result", status: 200, body: `{"jsonrpc":"2.0","id":1,"result":"0x10"}`, result: `"0x10"`}, + {name: "gzip result", status: 200, gzip: true, body: `{"jsonrpc":"2.0","id":1,"result":"0x20"}`, result: `"0x20"`}, + {name: "429 json-rpc error", status: 429, body: `{"jsonrpc":"2.0","id":1,"error":{"code":-32005,"message":"rate limited"}}`, wantErr: common.ErrCodeJsonRpcExceptionInternal, contains: "rate limited"}, + {name: "401 json-rpc error", status: 401, body: `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"unauthorized"}}`, wantErr: common.ErrCodeJsonRpcExceptionInternal, contains: "unauthorized"}, + {name: "403 plain text", status: 403, ctype: "text/plain", body: `forbidden`, wantErr: common.ErrCodeJsonRpcExceptionInternal, contains: "forbidden"}, + {name: "503 html", status: 503, ctype: "text/html", body: `bad gateway`, wantErr: common.ErrCodeJsonRpcExceptionInternal, contains: "bad gateway"}, + {name: "413 plain text", status: 413, ctype: "text/plain", body: `request entity too large`, wantErr: common.ErrCodeJsonRpcExceptionInternal, contains: "request entity too large"}, + {name: "context canceled message", status: 200, body: `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"upstream: context canceled"}}`, wantErr: common.ErrCodeEndpointRequestCanceled}, + {name: "deadline exceeded message", status: 200, body: `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"query: context deadline exceeded"}}`, wantErr: common.ErrCodeEndpointRequestTimeout}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + up := newRecordingUpstream(t, func(w http.ResponseWriter, _ []byte) { + body := []byte(tc.body) + if tc.ctype != "" { + w.Header().Set("Content-Type", tc.ctype) + } + if tc.gzip { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, _ = gz.Write(body) + _ = gz.Close() + body = buf.Bytes() + w.Header().Set("Content-Encoding", "gzip") + } + w.WriteHeader(tc.status) + _, _ = w.Write(body) + }) + c := newHttpTestClient(t, up.URL, nil, nil, nil) + resp, err := c.SendRequest(context.Background(), rpcReq(1, "eth_blockNumber")) + if tc.wantErr == "" { + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + assert.Equal(t, tc.result, jrr.GetResultString()) + return + } + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, tc.wantErr), "want %s, got %v", tc.wantErr, err) + assert.Contains(t, err.Error(), tc.contains) + if tc.wantErr == common.ErrCodeJsonRpcExceptionInternal { + assert.Contains(t, err.Error(), fmt.Sprintf(`"statusCode":%d`, tc.status), "status code must be kept for policies") + } + }) + } +} + +// statusExtractor maps HTTP status to an endpoint error the way the EVM +// extractor does, to show the client hands the extractor the real response. +type statusExtractor struct{} + +func (statusExtractor) Extract(resp *http.Response, _ *common.NormalizedResponse, _ *common.JsonRpcResponse, _ common.Upstream) error { + if resp.StatusCode == http.StatusTooManyRequests { + return common.NewErrEndpointCapacityExceeded(fmt.Errorf("status %d", resp.StatusCode)) + } + return nil +} + +func TestHttpJsonRpcClient_ExtractorDecidesBeforeFallback(t *testing.T) { + up := newRecordingUpstream(t, func(w http.ResponseWriter, _ []byte) { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = io.WriteString(w, `{"jsonrpc":"2.0","id":1,"error":{"code":-32005,"message":"slow down"}}`) + }) + c := newHttpTestClient(t, up.URL, nil, nil, statusExtractor{}) + _, err := c.SendRequest(context.Background(), rpcReq(1, "eth_call")) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointCapacityExceeded), "got %v", err) +} + +// Single-request transport outcomes: unreachable host, caller deadline, +// caller cancel, and a corrupt gzip body. +func TestHttpJsonRpcClient_SingleTransportOutcomes(t *testing.T) { + t.Run("unreachable", func(t *testing.T) { + dead := httptest.NewServer(http.NotFoundHandler()) + addr := dead.URL + dead.Close() + c := newHttpTestClient(t, addr, nil, nil, nil) + _, err := c.SendRequest(context.Background(), rpcReq(1, "eth_blockNumber")) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointTransportFailure), "got %v", err) + }) + + release := make(chan struct{}) + slow := newRecordingUpstream(t, func(w http.ResponseWriter, _ []byte) { <-release }) + t.Cleanup(func() { close(release) }) // runs before slow.Close (LIFO) + + t.Run("deadline", func(t *testing.T) { + c := newHttpTestClient(t, slow.URL, nil, nil, nil) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + _, err := c.SendRequest(ctx, rpcReq(1, "eth_blockNumber")) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointRequestTimeout), "got %v", err) + }) + + t.Run("cancel", func(t *testing.T) { + c := newHttpTestClient(t, slow.URL, nil, nil, nil) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := c.SendRequest(ctx, rpcReq(1, "eth_blockNumber")) + done <- err + }() + require.Eventually(t, func() bool { return len(slow.requests()) >= 2 }, 5*time.Second, time.Millisecond) + cancel() + err := <-done + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointRequestCanceled), "got %v", err) + }) + + t.Run("corrupt gzip", func(t *testing.T) { + up := newRecordingUpstream(t, func(w http.ResponseWriter, _ []byte) { + w.Header().Set("Content-Encoding", "gzip") + _, _ = io.WriteString(w, "plain bytes") + }) + c := newHttpTestClient(t, up.URL, nil, nil, nil) + _, err := c.SendRequest(context.Background(), rpcReq(1, "eth_blockNumber")) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointTransportFailure), "got %v", err) + assert.Contains(t, err.Error(), "gzip") + }) +} + +// The outgoing single request carries configured headers, forwarded client +// headers, gzip encoding when enabled, and the exact JSON-RPC payload. +func TestHttpJsonRpcClient_SingleRequestWireFormat(t *testing.T) { + for _, gz := range []bool{false, true} { + t.Run(fmt.Sprintf("gzip=%v", gz), func(t *testing.T) { + up := newRecordingUpstream(t, func(w http.ResponseWriter, _ []byte) { + _, _ = io.WriteString(w, `{"jsonrpc":"2.0","id":3,"result":"0x1"}`) + }) + enable := gz + cfg := &common.JsonRpcUpstreamConfig{ + EnableGzip: &enable, + Headers: map[string]string{"Authorization": "Bearer abc"}, + } + c := newHttpTestClient(t, up.URL, cfg, nil, nil) + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":3,"method":"eth_getBalance","params":["0xabc","latest"]}`)) + req.ForwardHeaders = http.Header{"X-Forwarded-For": []string{"1.2.3.4"}} + _, err := c.SendRequest(context.Background(), req) + require.NoError(t, err) + + reqs := up.requests() + require.Len(t, reqs, 1) + h := reqs[0].header + assert.Equal(t, "Bearer abc", h.Get("Authorization")) + assert.Equal(t, "1.2.3.4", h.Get("X-Forwarded-For")) + assert.Equal(t, "application/json", h.Get("Content-Type")) + if gz { + assert.Equal(t, "gzip", h.Get("Content-Encoding")) + } else { + assert.Empty(t, h.Get("Content-Encoding")) + } + var sent map[string]interface{} + require.NoError(t, sonic.Unmarshal(reqs[0].body, &sent)) + assert.Equal(t, "eth_getBalance", sent["method"]) + assert.Equal(t, []interface{}{"0xabc", "latest"}, sent["params"]) + assert.EqualValues(t, 3, sent["id"]) + }) + } +} + +// Batches are gzip-compressed too when enabled. +func TestHttpJsonRpcClient_BatchGzipRequest(t *testing.T) { + up := newRecordingUpstream(t, echoBatch) + cfg := batchCfg(2, time.Minute) + cfg.EnableGzip = &common.TRUE + c := newHttpTestClient(t, up.URL, cfg, nil, nil) + out := sendConcurrently(t, c, context.Background(), []int{1, 2}) + require.NoError(t, out[1].err) + require.NoError(t, out[2].err) + reqs := up.requests() + require.Len(t, reqs, 1) + assert.Equal(t, "gzip", reqs[0].header.Get("Content-Encoding")) + assert.Equal(t, []int{2}, batchSizes(reqs)) +} + +// With a proxy pool configured, requests go through the proxy (which sees +// the absolute upstream URI) rather than straight to the upstream. +func TestHttpJsonRpcClient_RoutesThroughProxyPool(t *testing.T) { + proxy := newRecordingUpstream(t, func(w http.ResponseWriter, _ []byte) { + _, _ = io.WriteString(w, `{"jsonrpc":"2.0","id":1,"result":"0xproxied"}`) + }) + logger := zerolog.New(io.Discard) + reg, err := NewProxyPoolRegistry([]*common.ProxyPoolConfig{{ID: "p1", Urls: []string{proxy.URL}}}, &logger) + require.NoError(t, err) + pool, err := reg.GetPool("p1") + require.NoError(t, err) + + const upstreamURL = "http://upstream.invalid:8545/rpc" + c := newHttpTestClient(t, upstreamURL, &common.JsonRpcUpstreamConfig{ProxyPool: "p1"}, pool, nil) + resp, err := c.SendRequest(context.Background(), rpcReq(1, "eth_blockNumber")) + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + assert.Equal(t, `"0xproxied"`, jrr.GetResultString()) + + reqs := proxy.requests() + require.Len(t, reqs, 1) + assert.Equal(t, upstreamURL, reqs[0].uri) +} + +// Proxy pool registry lookups and invalid configurations. +func TestProxyPoolRegistry_Configs(t *testing.T) { + logger := zerolog.New(io.Discard) + + reg, err := NewProxyPoolRegistry(nil, &logger) + require.NoError(t, err) + p, err := reg.GetPool("") + require.NoError(t, err) + assert.Nil(t, p, "empty pool id means direct connection") + _, err = reg.GetPool("missing") + require.Error(t, err) + assert.Contains(t, err.Error(), "missing") + + _, err = NewProxyPoolRegistry([]*common.ProxyPoolConfig{{ID: "empty"}}, &logger) + require.Error(t, err) + assert.Contains(t, err.Error(), "no proxy URLs defined for pool 'empty'") + + _, err = NewProxyPoolRegistry([]*common.ProxyPoolConfig{{ID: "bad", Urls: []string{"http://[::1"}}}, &logger) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid proxy URL") + + reg, err = NewProxyPoolRegistry([]*common.ProxyPoolConfig{{ID: "rr", Urls: []string{"http://a:1", "http://b:2"}}}, &logger) + require.NoError(t, err) + pool, err := reg.GetPool("rr") + require.NoError(t, err) + seen := map[string]bool{} + for range 4 { + hc, err := pool.GetClient() + require.NoError(t, err) + u, err := hc.Transport.(*http.Transport).Proxy(&http.Request{URL: &url.URL{Scheme: "http", Host: "x"}}) + require.NoError(t, err) + seen[u.Host] = true + } + assert.Equal(t, map[string]bool{"a:1": true, "b:2": true}, seen, "clients are handed out round-robin") + + _, err = (&ProxyPool{ID: "none"}).GetClient() + require.Error(t, err) +} diff --git a/clients/ws_json_rpc_client.go b/clients/ws_json_rpc_client.go index 24a689e53..af04bbd9d 100644 --- a/clients/ws_json_rpc_client.go +++ b/clients/ws_json_rpc_client.go @@ -214,6 +214,21 @@ func (c *WsJsonRpcClient) Epoch() uint64 { return c.epoch.Load() } +// Done is closed once the client has shut down for good. +func (c *WsJsonRpcClient) Done() <-chan struct{} { return c.appCtx.Done() } + +// Reconnect closes connection epoch, if still current; readLoop then tears +// it down and re-dials, firing the disconnect and reconnect callbacks. +func (c *WsJsonRpcClient) Reconnect(epoch uint64) { + c.connMu.Lock() + conn := c.conn + current := conn != nil && c.epoch.Load() == epoch + c.connMu.Unlock() + if current { + _ = conn.Close() + } +} + func (c *WsJsonRpcClient) SendRequest(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { ctx, span := common.StartDetailSpan(ctx, "WsJsonRpcClient.SendRequest", trace.WithAttributes( @@ -246,7 +261,7 @@ func (c *WsJsonRpcClient) SendRequest(ctx context.Context, req *common.Normalize "jsonrpc": jrReq.JSONRPC, "id": wireID, "method": method, - "params": jrReq.Params, + "params": common.WireParams(ctx, req, jrReq.Params), }) jrReq.RUnlock() if err != nil { @@ -613,7 +628,7 @@ func (c *WsJsonRpcClient) handleMessage(message []byte, epoch uint64) { var subID string if p.sub != nil && common.SonicCfg.Unmarshal(msg.Result, &subID) == nil && subID != "" { if abandoned { - go c.unsubscribeOrphan(subID, epoch) + go c.UnsubscribeOrphan(subID, epoch) return } // The id outlives the frame it was decoded from. @@ -655,7 +670,7 @@ func (c *WsJsonRpcClient) handleNotification(method string, params []byte, epoch // Nothing owns it: cancel it, unless a subscribe in flight may be // about to claim it. Each later notification retries the cancel. if notifParams.Subscription != "" && !c.subscribeInFlight() { - go c.unsubscribeOrphan(strings.Clone(notifParams.Subscription), epoch) + go c.UnsubscribeOrphan(strings.Clone(notifParams.Subscription), epoch) } return } @@ -689,9 +704,9 @@ func (c *WsJsonRpcClient) writeToConn(conn *websocket.Conn, messageType int, dat return conn.WriteMessage(messageType, data) } -// unsubscribeOrphan cancels a subscription the upstream created after its -// caller gave up. Fire-and-forget: the response is dropped as unknown. -func (c *WsJsonRpcClient) unsubscribeOrphan(subID string, epoch uint64) { +// UnsubscribeOrphan cancels a subscription of connection epoch that nobody +// owns any more. Fire-and-forget: the response is dropped as unknown. +func (c *WsJsonRpcClient) UnsubscribeOrphan(subID string, epoch uint64) { body, err := common.SonicCfg.Marshal(map[string]interface{}{ "jsonrpc": "2.0", "id": c.wireIDCounter.Add(1), diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..fb4518f56 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,3 @@ +ignore: + - "cmd/**" + - "internal/simulator/**" diff --git a/common/architecture_evm.go b/common/architecture_evm.go index c745a6923..63146b087 100644 --- a/common/architecture_evm.go +++ b/common/architecture_evm.go @@ -136,6 +136,9 @@ type EvmStatePoller interface { IsBlockFinalized(blockNumber int64) (bool, error) SuggestFinalizedBlock(blockNumber int64) SuggestLatestBlock(blockNumber int64) + // RefreshLatestBlock polls the latest block in the background when the + // stored head is older than the poll debounce; non-blocking, one in flight. + RefreshLatestBlock() SetNetworkConfig(cfg *NetworkConfig) IsObjectNull() bool EarliestBlock(probe EvmAvailabilityProbeType) int64 diff --git a/common/architecture_svm.go b/common/architecture_svm.go index 72ca1e63e..ff35ee51f 100644 --- a/common/architecture_svm.go +++ b/common/architecture_svm.go @@ -127,3 +127,45 @@ func KnownGenesisHash(chain, cluster string) (string, bool) { h, ok := clusters[cluster] return h, ok } + +// SvmHistory is the slots from which a cluster's Agave history records block +// fields. Below them Agave renders the field null whatever an archive holds; +// a nil bound means the cluster records the field from its first slot. +type SvmHistory struct { + BlockTimeFrom *uint64 + BlockHeightFrom *uint64 +} + +// RecordsBlockTime reports whether Agave renders `blockTime` for slot. +func (h SvmHistory) RecordsBlockTime(slot uint64) bool { + return h.BlockTimeFrom == nil || slot >= *h.BlockTimeFrom +} + +// RecordsBlockHeight reports whether Agave renders `blockHeight` for slot. +func (h SvmHistory) RecordsBlockHeight(slot uint64) bool { + return h.BlockHeightFrom == nil || slot >= *h.BlockHeightFrom +} + +// GenesisTimeOnly reports whether Agave answers slot's `blockTime` with the +// genesis creation time, which no archive row carries: slot 0 on a cluster +// that records no times after it. +func (h SvmHistory) GenesisTimeOnly(slot uint64) bool { + return slot == 0 && h.BlockTimeFrom != nil +} + +// knownSvmHistories holds the clusters whose history omits fields, bisected +// against api.mainnet-beta.solana.com: `blockTime` is null through slot +// 39,824,213 and `blockHeight` through 81,032,024. +var knownSvmHistories = map[string]map[string]SvmHistory{ + SvmChainSolana: { + "mainnet-beta": {BlockTimeFrom: ptrU64(39_824_214), BlockHeightFrom: ptrU64(81_032_025)}, + }, +} + +// KnownSvmHistory returns the Agave history of (chain, cluster); the zero +// value (every field recorded) for any other cluster. +func KnownSvmHistory(chain, cluster string) SvmHistory { + return knownSvmHistories[ResolveSvmChain(chain)][cluster] +} + +func ptrU64(v uint64) *uint64 { return &v } diff --git a/common/config.go b/common/config.go index 3343a4139..c635adf45 100644 --- a/common/config.go +++ b/common/config.go @@ -208,6 +208,16 @@ type WebSocketServerConfig struct { // newPendingTransactions drop their oldest entry; logs close the // connection instead (1013), so the client knows it missed data. SubscriptionBufferSize int `yaml:"subscriptionBufferSize,omitempty" json:"subscriptionBufferSize"` + // MaxConnections caps concurrent client connections on this instance + // (0 = unlimited). Unlike the per-IP cap it holds when client IPs are + // unknown or spoofed. + MaxConnections int `yaml:"maxConnections,omitempty" json:"maxConnections"` + // MaxConnectionsPerIP caps concurrent client connections from one + // resolved client IP (0 = unlimited); see trustedIPForwarders. + MaxConnectionsPerIP int `yaml:"maxConnectionsPerIP,omitempty" json:"maxConnectionsPerIP"` + // MaxSubscriptions caps active subscriptions across all connections on + // this instance (0 = unlimited). + MaxSubscriptions int `yaml:"maxSubscriptions,omitempty" json:"maxSubscriptions"` } // ExecutionHeadersMode controls how much per-request execution detail is @@ -1403,6 +1413,12 @@ type EvmUpstreamConfig struct { // head comparison every request pinned at head+1 in that moment is rerouted even though the // upstream would have answered it. 0 (default) keeps the exact head comparison. HeadLagToleranceBlocks int64 `yaml:"headLagToleranceBlocks,omitempty" json:"headLagToleranceBlocks"` + // PinLatestTag sends this upstream the network head as a concrete block number wherever a + // state call (eth_call, eth_getBalance, ...) asks for "latest" or omits the block, and skips the + // upstream when its blockAvailability upper bound is below that head or the head is unknown. + // For an upstream whose own head lags the network: it answers only blocks it holds, and a block + // it lacks fails as missing data so the next upstream answers, instead of an older "latest". + PinLatestTag *bool `yaml:"pinLatestTag,omitempty" json:"pinLatestTag"` // Deprecated: never read at runtime. Configure data integrity via the network // `integrity` block instead. Retained only so existing YAML still parses. DeprecatedIntegrity *UpstreamIntegrityConfig `yaml:"integrity,omitempty" json:"integrity"` @@ -1423,6 +1439,25 @@ type EvmUpstreamConfig struct { DeprecatedGetLogsMaxBlockRange int64 `yaml:"getLogsMaxBlockRange,omitempty" json:"-"` QueryShim *EvmQueryShimConfig `yaml:"queryShim,omitempty" json:"queryShim"` + // Multicall3 packs eth_calls bound for this upstream that share a block into one + // Multicall3 aggregate3 eth_call, so a vendor billed per request is billed once + // per pack. Nil (default) sends every eth_call alone. + Multicall3 *EvmMulticall3Config `yaml:"multicall3,omitempty" json:"multicall3"` +} + +// EvmMulticall3Config bounds one pack. A call joins the open pack for its block, +// and the pack is sent MaxWait after its first call or when it holds MaxCalls. +// A call the pack cannot answer exactly (multicall failed, inner call reverted +// or returned nothing) is sent alone, as it would be without packing. +type EvmMulticall3Config struct { + MaxWait Duration `yaml:"maxWait,omitempty" json:"maxWait" tstype:"Duration"` + MaxCalls int `yaml:"maxCalls,omitempty" json:"maxCalls"` + // BypassContracts are target addresses whose calls are never packed, for + // contracts that answer differently when the caller is Multicall3 instead + // of the zero address (msg.sender or caller-code checks) without reverting. + // Contracts that revert inside a pack but succeed alone are learned at + // runtime and need no entry. + BypassContracts []string `yaml:"bypassContracts,omitempty" json:"bypassContracts"` } // EvmQueryShimConfig lets an upstream serve the sub-requests (eth_getBlockByNumber, @@ -1561,6 +1596,10 @@ func (c *EvmUpstreamConfig) Copy() *EvmUpstreamConfig { v := *c.SkipSyncingCheck copied.SkipSyncingCheck = &v } + if c.PinLatestTag != nil { + v := *c.PinLatestTag + copied.PinLatestTag = &v + } if c.DeprecatedIntegrity != nil { copied.DeprecatedIntegrity = c.DeprecatedIntegrity.Copy() } @@ -1571,6 +1610,11 @@ func (c *EvmUpstreamConfig) Copy() *EvmUpstreamConfig { if c.QueryShim != nil { copied.QueryShim = c.QueryShim.Copy() } + if c.Multicall3 != nil { + v := *c.Multicall3 + v.BypassContracts = slices.Clone(c.Multicall3.BypassContracts) + copied.Multicall3 = &v + } return copied } @@ -2615,9 +2659,9 @@ type SvmUpstreamConfig struct { // BlockAvailability is an operator restriction on the slot window this // upstream may serve. Only what the operator declares is enforced; the - // upstream's oldest stored block (getBlocksWithLimit(0, 1)) is probed and - // used as a lower anchor solely when a bound uses firstAvailableSlotPlus, - // mirroring EVM's earliestBlockPlus. Unknown floor = unbounded. + // upstream's oldest stored block (getBlocksWithLimit(0, 1)) is probed for + // every upstream and anchors a firstAvailableSlotPlus bound, mirroring EVM's + // earliestBlockPlus. Unknown floor = unbounded. BlockAvailability *SvmBlockAvailabilityConfig `yaml:"blockAvailability,omitempty" json:"blockAvailability"` } @@ -2644,18 +2688,21 @@ func (c *SvmBlockAvailabilityConfig) Copy() *SvmBlockAvailabilityConfig { } // SvmAvailabilityBoundConfig is a single declared slot bound. -// Exactly one field must be set. +// Exactly one of ExactSlot, LatestSlotMinus, FirstAvailableSlotPlus must be set. +// UpdateRate applies only to firstAvailableSlotPlus: how often the floor it +// anchors on is re-probed. 0 freezes it after the first observation, as EVM. type SvmAvailabilityBoundConfig struct { - ExactSlot *int64 `yaml:"exactSlot,omitempty" json:"exactSlot,omitempty"` - LatestSlotMinus *int64 `yaml:"latestSlotMinus,omitempty" json:"latestSlotMinus,omitempty"` - FirstAvailableSlotPlus *int64 `yaml:"firstAvailableSlotPlus,omitempty" json:"firstAvailableSlotPlus,omitempty"` + ExactSlot *int64 `yaml:"exactSlot,omitempty" json:"exactSlot,omitempty"` + LatestSlotMinus *int64 `yaml:"latestSlotMinus,omitempty" json:"latestSlotMinus,omitempty"` + FirstAvailableSlotPlus *int64 `yaml:"firstAvailableSlotPlus,omitempty" json:"firstAvailableSlotPlus,omitempty"` + UpdateRate Duration `yaml:"updateRate,omitempty" json:"updateRate,omitempty" tstype:"Duration"` } func (c *SvmAvailabilityBoundConfig) Copy() *SvmAvailabilityBoundConfig { if c == nil { return nil } - out := &SvmAvailabilityBoundConfig{} + out := &SvmAvailabilityBoundConfig{UpdateRate: c.UpdateRate} if c.ExactSlot != nil { v := *c.ExactSlot out.ExactSlot = &v @@ -2690,8 +2737,7 @@ type EvmNetworkConfig struct { // ServedTip configures how the network derives the "latest"/"finalized" // block it advertises to clients (and enforces via block-availability). - // Nil or disabled selects the default mode (the corroborated latest across eligible - // upstreams, see ServedTipPick.Freshest); set Enabled to opt into the majority tip. See + // Nil uses the defaults: quorum 2, regression guard on, no guarantees. See // EvmServedTipConfig. ServedTip *EvmServedTipConfig `yaml:"servedTip,omitempty" json:"servedTip,omitempty"` GetLogsMaxAllowedRange int64 `yaml:"getLogsMaxAllowedRange,omitempty" json:"getLogsMaxAllowedRange"` @@ -2740,6 +2786,11 @@ type EvmNetworkConfig struct { // Default includes common point-lookup methods like eth_getBlockByNumber, eth_getTransactionByHash, etc. MarkEmptyAsErrorMethods []string `yaml:"markEmptyAsErrorMethods,omitempty" json:"markEmptyAsErrorMethods,omitempty"` + // LocalAnswers turns off eRPC's own answer for a method on this network: + // `{net_peerCount: false}` routes net_peerCount to the upstreams. Every + // method in LocalAnswerMethods is answered by eRPC unless set false here. + LocalAnswers map[string]bool `yaml:"localAnswers,omitempty" json:"localAnswers,omitempty"` + // StripSubscribeFromBlockZero removes a zero `fromBlock` from eth_subscribe // logs filters before forwarding upstream. fromBlock has no meaning for a // live subscription, and backends that prune history reject it. @@ -2753,6 +2804,15 @@ type EvmNetworkConfig struct { // Default: 0.7 (30% under the estimated block time). DynamicBlockTimeDebounceMultiplier *float64 `yaml:"dynamicBlockTimeDebounceMultiplier,omitempty" json:"dynamicBlockTimeDebounceMultiplier,omitempty"` + // FinalizedDebounceDepthMultiplier sets how stale an upstream's finalized block may + // get before the state poller fetches it again, as a fraction of the chain's + // finality depth in time: (latest - finalized) x EMA block time x multiplier. The + // result applies only when it is longer than the regular debounce. Finalized + // advances once per finality window, so polling it at the latest-block cadence + // mostly returns the same number. 0 disables it (finalized uses the regular debounce). + // Default: 0.05 (refresh about 20 times per finality window). + FinalizedDebounceDepthMultiplier *float64 `yaml:"finalizedDebounceDepthMultiplier,omitempty" json:"finalizedDebounceDepthMultiplier,omitempty"` + // BlockUnavailableDelayMultiplier scales the EMA-estimated block time to derive the // retry delay when the requested data isn't available yet (ErrUpstreamBlockUnavailable // or an empty/missing-data point-lookup). When the dynamic block time is known, the @@ -2783,7 +2843,7 @@ type EvmNetworkConfig struct { // (regardless of selection policy, tier or syncing state), without // dispatching it; it never fires while any upstream's head is unknown. // Clients that poll the next block by number otherwise walk every routed - // upstream on each poll. Always on when servedTip is enabled for "latest". + // upstream on each poll. // The head is what the state poller and head subscriptions last reported, so // a block an upstream produced since then gets null until eRPC sees it: only // enable it where heads arrive promptly (e.g. over WebSocket). @@ -2804,30 +2864,25 @@ type EvmNetworkConfig struct { // EvmServedTipConfig controls how the network derives the "latest"/"finalized" // block it advertises (and enforces) from its upstreams. // -// In the default mode the served tip is the corroborated latest block across eligible -// non-syncing upstreams (second-highest, or the only one) — which can still advertise a block a slightly-ahead -// pair has, causing "block not found" churn when requests route to a -// slightly-behind upstream. When a tag is listed in EnabledFor, that tag's -// served value is instead the freshest block a strict MAJORITY of the eligible -// upstreams already have, so interpolated requests land on upstreams that can -// serve the advertised block. +// Both axes use one rule: among the eligible (selection-policy), non-syncing +// upstreams, only the best voter rank present votes (live, then benched, then +// static-capped, then benched and static-capped), and the served block is the +// highest one at least Quorum of those voters already have. The health +// tracker benches an upstream whose head stopped moving while another kept +// advancing, until it catches up again. type EvmServedTipConfig struct { - // EnabledFor lists the block tags whose served value uses the cluster-min tip - // instead of the default corroborated head. Valid entries: "latest" and "finalized" (the - // "safe" tag follows "finalized"). Empty selects the default mode for all tags. - EnabledFor []string `yaml:"enabledFor,omitempty" json:"enabledFor,omitempty"` - - // Deprecated: ClusterDelta configured the former cluster-based picker and is - // ignored — the majority order statistic needs no tuning. Kept only so - // existing configs keep parsing. - ClusterDelta int64 `yaml:"clusterDelta,omitempty" json:"clusterDelta,omitempty"` + // Quorum is how many eligible upstreams must already have a block before it + // is served: an integer >= 1, or "majority" (floor(N/2)+1 of N). Unset + // uses DefaultServedTipQuorum (2). A ballot with fewer voters than the + // quorum serves the block all of them have. + Quorum ServedTipQuorum `yaml:"quorum,omitempty" json:"quorum,omitempty" tstype:"number | 'majority'"` // GuaranteedMethods lists method name patterns (glob; e.g. "trace_*", // "debug_traceBlockByNumber") whose supporting-upstream subset must be able to // serve the advertised latest. For a request on a matching method, "latest" // resolves against the majority of only the upstreams that support it // (membership auto-detected via ShouldHandleMethod — no per-upstream config). - // Empty means only the global (all-eligible) majority is computed. + // Empty means only the global ballot is computed. GuaranteedMethods []string `yaml:"guaranteedMethods,omitempty" json:"guaranteedMethods,omitempty"` // GuaranteedFor lists upstream SELECTORS (id or tag glob — the same @@ -2840,67 +2895,32 @@ type EvmServedTipConfig struct { GuaranteedFor []string `yaml:"guaranteedFor,omitempty" json:"guaranteedFor,omitempty"` // MaxRegressionBlocks is how far below the corroborated LIVE upstream head - // (the second-highest live head) the majority pick may fall before it is - // treated as a poisoned ballot rather than as reality. While a pick is below - // that bound the network keeps serving its last corroborated pick - // (in-process, for at most one minute, then it fails open and serves the - // pick as computed). 0 uses DefaultToleratedBlockHeadRollback (1024) — the - // same rollback tolerance the state poller and health tracker apply to a + // (the second-highest live head) the pick may fall before it is treated as + // a poisoned ballot rather than as reality. While a pick is below that + // bound the network keeps serving its last corroborated pick (in-process, + // for at most one minute, then it fails open and serves the pick as + // computed). 0 uses DefaultToleratedBlockHeadRollback (1024) — the same + // rollback tolerance the state poller and health tracker apply to a // retreating head, so all three layers agree on what counts as a genuine - // deep correction. -1 disables the regression guard entirely (the symmetric - // opposite of trajectoryWindow: 0); no other negative value is accepted. + // deep correction. -1 disables the regression guard entirely; no other + // negative value is accepted. MaxRegressionBlocks int64 `yaml:"maxRegressionBlocks,omitempty" json:"maxRegressionBlocks,omitempty"` - - // TrajectoryWindow is how much recorded head history the trajectory referee - // needs before it may participate in the pick. The referee tracks where this - // network's head has been over the window and, when a stalled group holds - // the majority, serves the corroborated group that is actually where the - // chain should be by now (see architecture/evm's TipTrajectory: advisory, - // upward-only, in-process, and a no-op until every confidence condition - // holds). Unset uses DefaultServedTipTrajectoryWindow (10m); an explicit 0 - // disables the referee entirely — nothing is recorded and the pick is the - // plain majority. Any other value must be between - // MinServedTipTrajectoryWindow and MaxServedTipTrajectoryWindow: below the - // minimum the window is self-defeating (it must be long enough that a stall - // cannot look like a trajectory), and above the maximum the sample ring can - // never span it, so the referee would stand down forever while looking - // configured. WRITE IT AS A DURATION STRING — trajectoryWindow: "10m". A - // bare number is parsed as MILLISECONDS. - TrajectoryWindow *Duration `yaml:"trajectoryWindow,omitempty" json:"trajectoryWindow,omitempty"` } // ShortCircuitFutureBlocksEnabled reports whether a numbered eth_getBlockByNumber -// above every upstream's head returns null without dispatching: when -// ShortCircuitFutureBlocks is true, or servedTip is enabled for "latest". -// Nil-receiver safe. +// above every upstream's head returns null without dispatching. Nil-receiver +// safe. func (c *EvmNetworkConfig) ShortCircuitFutureBlocksEnabled() bool { - if c == nil { - return false - } - if c.ShortCircuitFutureBlocks != nil && *c.ShortCircuitFutureBlocks { - return true - } - return c.ServedTipEnabledFor("latest") + return c != nil && c.ShortCircuitFutureBlocks != nil && *c.ShortCircuitFutureBlocks } -// ServedTipEnabledFor reports whether the majority served tip is enabled for -// the given block axis ("latest" or "finalized"). The "safe" tag resolves to the -// finalized axis, so listing "safe" in EnabledFor enables it for "finalized". -// Anything not listed uses the default corroborated head. Nil-receiver safe. -func (c *EvmNetworkConfig) ServedTipEnabledFor(tag string) bool { +// ServedTipQuorum is the network's configured served-tip quorum (zero = the +// default). Nil-receiver safe. +func (c *EvmNetworkConfig) ServedTipQuorum() ServedTipQuorum { if c == nil || c.ServedTip == nil { - return false - } - for _, t := range c.ServedTip.EnabledFor { - if strings.EqualFold(t, tag) { - return true - } - // "safe" resolves to the finalized axis. - if strings.EqualFold(tag, "finalized") && strings.EqualFold(t, "safe") { - return true - } + return 0 } - return false + return c.ServedTip.Quorum } // EvmIntegrityConfig is deprecated. Use DirectiveDefaultsConfig for validation settings. @@ -3362,8 +3382,29 @@ type RateLimitStoreConfig struct { Redis *RedisConnectorConfig `yaml:"redis,omitempty" json:"redis,omitempty"` CacheKeyPrefix string `yaml:"cacheKeyPrefix,omitempty" json:"cacheKeyPrefix"` NearLimitRatio float32 `yaml:"nearLimitRatio,omitempty" json:"nearLimitRatio"` + // OnFailure decides a request the redis store could not: not connected, + // erroring, slower than redis.getTimeout, or at its in-flight cap. + OnFailure RateLimitStoreFailureMode `yaml:"onFailure,omitempty" json:"onFailure,omitempty"` + // LocalLimitFactor scales every maxCount for the in-process limiter that + // onFailure: local decides with. It counts this replica only, so set it + // near 1/replicas. Required for local; must be > 0 and <= 1. + LocalLimitFactor float64 `yaml:"localLimitFactor,omitempty" json:"localLimitFactor,omitempty"` } +// RateLimitStoreFailureMode is what a redis-backed budget does with a request +// its store could not decide. +type RateLimitStoreFailureMode string + +const ( + // RateLimitStoreFailureAllow (default) admits the request. + RateLimitStoreFailureAllow RateLimitStoreFailureMode = "allow" + // RateLimitStoreFailureDeny rejects the request as rate limited. + RateLimitStoreFailureDeny RateLimitStoreFailureMode = "deny" + // RateLimitStoreFailureLocal decides it with an in-process limiter that + // counts every request, at maxCount × localLimitFactor. + RateLimitStoreFailureLocal RateLimitStoreFailureMode = "local" +) + // CachePartitionKey builds the JSON-RPC cache partition key. // Empty suffix keeps {networkId}:{ref}; a set suffix yields {networkId}:{suffix}:{ref}. // ':' and '\' inside suffix and ref are backslash-escaped so those segments diff --git a/common/config_test.go b/common/config_test.go index 89873afc6..3052028f5 100644 --- a/common/config_test.go +++ b/common/config_test.go @@ -1318,9 +1318,8 @@ func TestEvmNetworkConfig_ShortCircuitFutureBlocksEnabled(t *testing.T) { enabled, disabled := true, false assert.False(t, (&EvmNetworkConfig{ShortCircuitFutureBlocks: &disabled}).ShortCircuitFutureBlocksEnabled()) assert.True(t, (&EvmNetworkConfig{ShortCircuitFutureBlocks: &enabled}).ShortCircuitFutureBlocksEnabled()) - // servedTip for "latest" keeps the short-circuit on, as before. - assert.True(t, (&EvmNetworkConfig{ServedTip: &EvmServedTipConfig{EnabledFor: []string{"latest"}}}).ShortCircuitFutureBlocksEnabled()) - assert.False(t, (&EvmNetworkConfig{ServedTip: &EvmServedTipConfig{EnabledFor: []string{"finalized"}}}).ShortCircuitFutureBlocksEnabled()) + // Only the explicit key turns it on; a servedTip block does not. + assert.False(t, (&EvmNetworkConfig{ServedTip: &EvmServedTipConfig{Quorum: ServedTipQuorumMajority}}).ShortCircuitFutureBlocksEnabled()) } func TestUpstreamConfig_ValidateRateLimitCountMode(t *testing.T) { diff --git a/common/cov_defaults_config_test.go b/common/cov_defaults_config_test.go new file mode 100644 index 000000000..dd6313915 --- /dev/null +++ b/common/cov_defaults_config_test.go @@ -0,0 +1,620 @@ +package common + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/erpc/erpc/util" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + pb "github.com/envoyproxy/go-control-plane/envoy/service/ratelimit/v3" + "gopkg.in/yaml.v3" +) + +func covDefaultsWriteCfg(t *testing.T, fs afero.Fs, name, body string) string { + t.Helper() + require.NoError(t, afero.WriteFile(fs, name, []byte(body), 0o644)) + return name +} + +func covDefaultsSetLegacyHooks(t *testing.T, fn func(*Config) ([]string, error), logger func(string)) { + t.Helper() + prevFn, prevLog := LegacyTranslateFn, LegacyTranslateLogger + LegacyTranslateFn, LegacyTranslateLogger = fn, logger + t.Cleanup(func() { LegacyTranslateFn, LegacyTranslateLogger = prevFn, prevLog }) +} + +func TestCovDefaults_LoadConfig_LegacyHookAndEnvExpansion(t *testing.T) { + t.Setenv("COVDEFAULTS_ENDPOINT", "https://env.example") + fs := afero.NewMemMapFs() + path := covDefaultsWriteCfg(t, fs, "erpc.yaml", ` +logLevel: warn +projects: + - id: main + upstreams: + - id: u1 + endpoint: ${COVDEFAULTS_ENDPOINT} + evm: + chainId: 1 +`) + var warnings []string + covDefaultsSetLegacyHooks(t, func(c *Config) ([]string, error) { + c.Projects[0].Upstreams[0].RateLimitBudget = "" + return []string{"w1", "w2"}, nil + }, func(w string) { warnings = append(warnings, w) }) + + cfg, err := LoadConfig(fs, path, &DefaultOptions{}) + require.NoError(t, err) + assert.Equal(t, "https://env.example", cfg.Projects[0].Upstreams[0].Endpoint) + assert.Equal(t, []string{"w1", "w2"}, warnings, "every translator warning reaches the logger") + assert.Same(t, cfg.Projects[0], cfg.GetProjectConfig("main")) + assert.Nil(t, cfg.GetProjectConfig("other")) +} + +func TestCovDefaults_LoadConfig_FailurePaths(t *testing.T) { + fs := afero.NewMemMapFs() + ok := covDefaultsWriteCfg(t, fs, "ok.yaml", "logLevel: warn\nprojects:\n - id: main\n upstreams:\n - id: u\n endpoint: https://a\n evm:\n chainId: 1\n") + + t.Run("translator error", func(t *testing.T) { + covDefaultsSetLegacyHooks(t, func(*Config) ([]string, error) { return nil, errors.New("boom") }, nil) + _, err := LoadConfig(fs, ok, &DefaultOptions{}) + assert.EqualError(t, err, "legacy config migration: boom") + }) + t.Run("set defaults error", func(t *testing.T) { + covDefaultsSetLegacyHooks(t, nil, nil) + bad := covDefaultsWriteCfg(t, fs, "bad-defaults.yaml", "logLevel: warn\nprojects:\n - id: main\n upstreams:\n - id: u\n endpoint: nosuchvendor://k\n") + _, err := LoadConfig(fs, bad, &DefaultOptions{}) + assert.ErrorContains(t, err, "unsupported vendor name in vendor.settings: nosuchvendor") + }) + t.Run("validate error", func(t *testing.T) { + covDefaultsSetLegacyHooks(t, nil, nil) + bad := covDefaultsWriteCfg(t, fs, "bad-validate.yaml", "logLevel: warn\nprojects:\n - id: main\n upstreams:\n - id: u\n endpoint: https://a\n evm:\n chainId: 1\n - id: u\n endpoint: https://b\n evm:\n chainId: 1\n") + _, err := LoadConfig(fs, bad, &DefaultOptions{}) + require.Error(t, err) + }) + t.Run("ts file that does not exist on disk", func(t *testing.T) { + covDefaultsSetLegacyHooks(t, nil, nil) + p := covDefaultsWriteCfg(t, fs, "/covdefaults-nowhere/erpc.ts", "export default {}") + _, err := LoadConfig(fs, p, &DefaultOptions{}) + require.Error(t, err, "the TS compiler reads the real filesystem, not the afero one") + }) +} + +func TestCovDefaults_LoadConfig_TypeScriptErrors(t *testing.T) { + covDefaultsSetLegacyHooks(t, nil, nil) + dir := t.TempDir() + cases := []struct { + name string + src string + want string + }{ + {"no default export", "export const x = 1;\n", "config object must be default exported"}, + {"unknown field", "export default { logLevel: 'warn', notAField: 1 };\n", "ts config decode"}, + {"runtime throw", "throw new Error('kaboom');\nexport default {};\n", "kaboom"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := filepath.Join(dir, strings.ReplaceAll(tc.name, " ", "_")+".ts") + require.NoError(t, os.WriteFile(p, []byte(tc.src), 0o644)) + _, err := LoadConfig(afero.NewOsFs(), p, &DefaultOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.want) + }) + } +} + +func TestCovDefaults_LegacyYAMLShapes(t *testing.T) { + t.Run("network single failsafe object", func(t *testing.T) { + var n NetworkConfig + require.NoError(t, yaml.Unmarshal([]byte(` +architecture: evm +rateLimitBudget: rb +alias: eth +cacheKeySuffix: s +evm: + chainId: 1 +failover: + onDefaultsExhausted: true +failsafe: + retry: + maxAttempts: 4 +`), &n)) + assert.Equal(t, NetworkArchitecture("evm"), n.Architecture) + assert.Equal(t, "rb", n.RateLimitBudget) + assert.Equal(t, "eth", n.Alias) + assert.Equal(t, "s", n.CacheKeySuffix) + assert.True(t, n.Failover.Enabled()) + require.Len(t, n.Failsafe, 1) + assert.Equal(t, "*", n.Failsafe[0].MatchMethod) + assert.Equal(t, 4, n.Failsafe[0].Retry.MaxAttempts) + assert.Equal(t, "evm:1", n.NetworkId()) + }) + t.Run("network single failsafe with explicit method", func(t *testing.T) { + var n NetworkConfig + require.NoError(t, yaml.Unmarshal([]byte("architecture: evm\nfailsafe:\n matchMethod: eth_call\n"), &n)) + assert.Equal(t, "eth_call", n.Failsafe[0].MatchMethod) + }) + t.Run("network unknown field surfaces as-is", func(t *testing.T) { + var n NetworkConfig + dec := yaml.NewDecoder(strings.NewReader("architecture: evm\nbogusField: 1\n")) + dec.KnownFields(true) + err := dec.Decode(&n) + assert.ErrorContains(t, err, "not found in type") + }) + t.Run("network type error in both shapes keeps original", func(t *testing.T) { + var n NetworkConfig + err := yaml.Unmarshal([]byte("architecture: [1]\n"), &n) + require.Error(t, err) + }) + t.Run("networkDefaults single failsafe", func(t *testing.T) { + var nd NetworkDefaults + require.NoError(t, yaml.Unmarshal([]byte(` +rateLimitBudget: rb +cacheKeySuffix: s +evm: + getLogsMaxAllowedRange: 9 +svm: + chain: fogo +failsafe: + hedge: + maxCount: 2 +`), &nd)) + assert.Equal(t, "rb", nd.RateLimitBudget) + assert.Equal(t, "s", nd.CacheKeySuffix) + assert.Equal(t, int64(9), nd.Evm.GetLogsMaxAllowedRange) + assert.Equal(t, "fogo", nd.Svm.Chain) + require.Len(t, nd.Failsafe, 1) + assert.Equal(t, "*", nd.Failsafe[0].MatchMethod) + assert.Equal(t, 2, nd.Failsafe[0].Hedge.MaxCount) + }) + t.Run("networkDefaults unknown field", func(t *testing.T) { + var nd NetworkDefaults + dec := yaml.NewDecoder(strings.NewReader("bogus: 1\n")) + dec.KnownFields(true) + assert.ErrorContains(t, dec.Decode(&nd), "not found in type") + }) + t.Run("networkDefaults bad type", func(t *testing.T) { + var nd NetworkDefaults + assert.Error(t, yaml.Unmarshal([]byte("rateLimitBudget: [1]\n"), &nd)) + }) + t.Run("upstream single failsafe object with legacy group", func(t *testing.T) { + var u UpstreamConfig + require.NoError(t, yaml.Unmarshal([]byte(` +id: u1 +type: evm +endpoint: https://a +group: fallback +cohort: c1 +rateLimitBudget: rb +routing: + scoreMultipliers: + - errorRate: 3 +failsafe: + timeout: + duration: 2s +`), &u)) + assert.Equal(t, "u1", u.Id) + assert.Equal(t, "rb", u.RateLimitBudget) + require.Len(t, u.Failsafe, 1) + assert.Equal(t, "*", u.Failsafe[0].MatchMethod) + assert.Equal(t, covDefaultsDur(2*time.Second), u.Failsafe[0].Timeout.Duration.Base) + assert.True(t, u.HasTag("tier:fallback")) + assert.True(t, u.HasTag("cohort:c1")) + assert.False(t, u.HasTag("tier:default")) + require.NotNil(t, u.Routing) + assert.Equal(t, 3.0, *u.Routing.ScoreMultipliers[0].ErrorRate) + }) + t.Run("upstream legacy group not duplicated", func(t *testing.T) { + var u UpstreamConfig + require.NoError(t, yaml.Unmarshal([]byte("id: u\ntags: [\"tier:fallback\"]\ngroup: fallback\n"), &u)) + assert.Equal(t, []string{"tier:fallback"}, u.Tags) + }) + t.Run("upstream unknown field", func(t *testing.T) { + var u UpstreamConfig + dec := yaml.NewDecoder(strings.NewReader("id: u\nbogus: 1\n")) + dec.KnownFields(true) + assert.ErrorContains(t, dec.Decode(&u), "not found in type") + }) + t.Run("upstream bad type in both shapes", func(t *testing.T) { + var u UpstreamConfig + assert.Error(t, yaml.Unmarshal([]byte("id: [1]\n"), &u)) + }) + t.Run("project legacy routing fields stashed", func(t *testing.T) { + var p ProjectConfig + require.NoError(t, yaml.Unmarshal([]byte("id: p\nroutingStrategy: score-based\nscoreGranularity: method\n"), &p)) + require.NotNil(t, p.LegacyProject) + assert.Equal(t, "score-based", p.LegacyProject.RoutingStrategy) + assert.Equal(t, "method", p.LegacyProject.ScoreGranularity) + var plain ProjectConfig + require.NoError(t, yaml.Unmarshal([]byte("id: p\n"), &plain)) + assert.Nil(t, plain.LegacyProject) + assert.Error(t, yaml.Unmarshal([]byte("id: [1]\n"), &plain)) + }) + t.Run("selection policy legacy fields stashed", func(t *testing.T) { + var s SelectionPolicyConfig + require.NoError(t, yaml.Unmarshal([]byte("evalFunction: \"(u) => u\"\nresampleCount: 3\n"), &s)) + require.NotNil(t, s.LegacySelectionPolicy) + assert.Equal(t, "(u) => u", s.LegacySelectionPolicy.EvalFunction) + assert.Equal(t, 3, s.LegacySelectionPolicy.ResampleCount) + assert.Error(t, yaml.Unmarshal([]byte("evalInterval: [1]\n"), &s)) + }) + t.Run("directive skipCacheRead normalized to string", func(t *testing.T) { + var d DirectiveDefaultsConfig + require.NoError(t, yaml.Unmarshal([]byte("skipCacheRead: true\n"), &d)) + assert.Equal(t, "true", d.SkipCacheRead) + var dj DirectiveDefaultsConfig + require.NoError(t, json.Unmarshal([]byte(`{"skipCacheRead":"cache-a*"}`), &dj)) + assert.Equal(t, "cache-a*", dj.SkipCacheRead) + var dn DirectiveDefaultsConfig + require.NoError(t, json.Unmarshal([]byte(`{"retryEmpty":true}`), &dn)) + assert.Nil(t, dn.SkipCacheRead) + assert.Error(t, yaml.Unmarshal([]byte("retryEmpty: [1]\n"), &dn)) + assert.Error(t, json.Unmarshal([]byte(`{"retryEmpty":[1]}`), &dn)) + }) +} + +func TestCovDefaults_TimeoutAndHedgeLegacySiblings(t *testing.T) { + t.Run("timeout yaml legacy siblings fold in", func(t *testing.T) { + var c TimeoutPolicyConfig + require.NoError(t, yaml.Unmarshal([]byte("quantile: 0.9\nminDuration: 1s\nmaxDuration: 5s\n"), &c)) + assert.Equal(t, &AdaptiveDuration{Quantile: 0.9, Min: covDefaultsDur(time.Second), Max: covDefaultsDur(5 * time.Second)}, c.Duration) + var keep TimeoutPolicyConfig + require.NoError(t, yaml.Unmarshal([]byte("duration: {base: 2s, quantile: 0.5, min: 100ms, max: 3s}\nquantile: 0.9\nminDuration: 1s\nmaxDuration: 5s\n"), &keep)) + assert.Equal(t, 0.5, keep.Duration.Quantile, "unified form wins over legacy siblings") + assert.Equal(t, covDefaultsDur(100*time.Millisecond), keep.Duration.Min) + assert.Error(t, yaml.Unmarshal([]byte("quantile: [1]\n"), &c)) + }) + t.Run("timeout json forms", func(t *testing.T) { + var c TimeoutPolicyConfig + require.NoError(t, json.Unmarshal([]byte(`{"quantile":0.95,"minDuration":"200ms","maxDuration":1500}`), &c)) + assert.Equal(t, &AdaptiveDuration{Quantile: 0.95, Min: covDefaultsDur(200 * time.Millisecond), Max: covDefaultsDur(1500 * time.Millisecond)}, c.Duration) + var n TimeoutPolicyConfig + require.NoError(t, n.UnmarshalJSON([]byte("null"))) + assert.Nil(t, n.Duration) + assert.ErrorContains(t, json.Unmarshal([]byte(`{"minDuration":"bogus"}`), &c), "timeout.minDuration") + assert.ErrorContains(t, json.Unmarshal([]byte(`{"maxDuration":"bogus"}`), &c), "timeout.maxDuration") + assert.Error(t, json.Unmarshal([]byte(`{"quantile":"x"}`), &c)) + }) + t.Run("hedge yaml and json legacy siblings", func(t *testing.T) { + var h HedgePolicyConfig + require.NoError(t, yaml.Unmarshal([]byte("maxCount: 2\nquantile: 0.8\nminDelay: 50ms\nmaxDelay: 2s\n"), &h)) + assert.Equal(t, 2, h.MaxCount) + assert.Equal(t, &AdaptiveDuration{Quantile: 0.8, Min: covDefaultsDur(50 * time.Millisecond), Max: covDefaultsDur(2 * time.Second)}, h.Delay) + assert.Error(t, yaml.Unmarshal([]byte("maxCount: [1]\n"), &h)) + + var hj HedgePolicyConfig + require.NoError(t, json.Unmarshal([]byte(`{"delay":"100ms","maxCount":3,"quantile":0.7,"minDelay":10,"maxDelay":"1s"}`), &hj)) + assert.Equal(t, 3, hj.MaxCount) + assert.Equal(t, &AdaptiveDuration{Base: covDefaultsDur(100 * time.Millisecond), Quantile: 0.7, Min: covDefaultsDur(10 * time.Millisecond), Max: covDefaultsDur(time.Second)}, hj.Delay) + var hn HedgePolicyConfig + require.NoError(t, hn.UnmarshalJSON(nil)) + assert.Nil(t, hn.Delay) + assert.ErrorContains(t, json.Unmarshal([]byte(`{"minDelay":"x"}`), &hj), "hedge.minDelay") + assert.ErrorContains(t, json.Unmarshal([]byte(`{"maxDelay":"x"}`), &hj), "hedge.maxDelay") + assert.Error(t, json.Unmarshal([]byte(`{"maxCount":"x"}`), &hj)) + }) +} + +func TestCovDefaults_RateLimitPeriodCodec(t *testing.T) { + cases := []struct { + in string + want RateLimitPeriod + }{ + {"1", RateLimitPeriodMinute}, {"second", RateLimitPeriodSecond}, {"1s", RateLimitPeriodSecond}, + {"Minute", RateLimitPeriodMinute}, {"60s", RateLimitPeriodMinute}, {"hour", RateLimitPeriodHour}, + {"3600s", RateLimitPeriodHour}, {"1d", RateLimitPeriodDay}, {"86400s", RateLimitPeriodDay}, + {"7d", RateLimitPeriodWeek}, {"604800s", RateLimitPeriodWeek}, {"30d", RateLimitPeriodMonth}, + {"720h", RateLimitPeriodMonth}, {"365d", RateLimitPeriodYear}, {"8760h", RateLimitPeriodYear}, + {"1000ms", RateLimitPeriodSecond}, {"1m0s", RateLimitPeriodMinute}, {"60m", RateLimitPeriodHour}, + {"1440m", RateLimitPeriodDay}, {"10080m", RateLimitPeriodWeek}, {"43200m", RateLimitPeriodMonth}, + {"525600m", RateLimitPeriodYear}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + var p RateLimitPeriod + require.NoError(t, yaml.Unmarshal([]byte(tc.in), &p)) + assert.Equal(t, tc.want, p) + }) + } + for _, bad := range []string{"99", "2s", "fortnight", "[1]"} { + var p RateLimitPeriod + assert.Error(t, yaml.Unmarshal([]byte(bad), &p), bad) + } + var p RateLimitPeriod + assert.EqualError(t, yaml.Unmarshal([]byte("2s"), &p), "rate limiter period must be one of: second, minute, hour, day, week, month, year (got 2s)") + assert.EqualError(t, yaml.Unmarshal([]byte("{a: 1}"), &p), "invalid period type; expected string enum, integer enum, or duration like '1s'") + + names := map[RateLimitPeriod]string{ + RateLimitPeriodSecond: "second", RateLimitPeriodMinute: "minute", RateLimitPeriodHour: "hour", + RateLimitPeriodDay: "day", RateLimitPeriodWeek: "week", RateLimitPeriodMonth: "month", + RateLimitPeriodYear: "year", RateLimitPeriod(42): "unknown", + } + units := map[RateLimitPeriod]pb.RateLimitResponse_RateLimit_Unit{ + RateLimitPeriodSecond: pb.RateLimitResponse_RateLimit_SECOND, RateLimitPeriodMinute: pb.RateLimitResponse_RateLimit_MINUTE, + RateLimitPeriodHour: pb.RateLimitResponse_RateLimit_HOUR, RateLimitPeriodDay: pb.RateLimitResponse_RateLimit_DAY, + RateLimitPeriodWeek: pb.RateLimitResponse_RateLimit_WEEK, RateLimitPeriodMonth: pb.RateLimitResponse_RateLimit_MONTH, + RateLimitPeriodYear: pb.RateLimitResponse_RateLimit_YEAR, RateLimitPeriod(42): pb.RateLimitResponse_RateLimit_UNKNOWN, + } + for period, name := range names { + assert.Equal(t, name, period.String()) + y, err := period.MarshalYAML() + require.NoError(t, err) + assert.Equal(t, name, y) + j, err := period.MarshalJSON() + require.NoError(t, err) + assert.Equal(t, `"`+name+`"`, string(j)) + assert.Equal(t, units[period], period.Unit()) + } +} + +func TestCovDefaults_RateLimitRuleHelpers(t *testing.T) { + cases := []struct { + rule RateLimitRuleConfig + scope string + countMode string + }{ + {RateLimitRuleConfig{}, "", "request"}, + {RateLimitRuleConfig{PerUser: true, PerNetwork: true, PerIP: true, CountMode: "credit"}, "user,network,ip", "credit"}, + {RateLimitRuleConfig{PerIP: true}, "ip", "request"}, + } + for _, tc := range cases { + assert.Equal(t, tc.scope, tc.rule.ScopeString()) + assert.Equal(t, tc.countMode, tc.rule.CountModeString()) + } + +} + +func TestCovDefaults_SecretsAreRedactedWhenMarshalled(t *testing.T) { + redis := &RedisConnectorConfig{Addr: "r:6379", Username: "u", Password: "hunter2", URI: "redis://u:hunter2@r:6379/0", InitTimeout: covDefaultsDur(time.Second)} + j, err := json.Marshal(redis) + require.NoError(t, err) + assert.NotContains(t, string(j), "hunter2") + var rm map[string]interface{} + require.NoError(t, json.Unmarshal(j, &rm)) + assert.Equal(t, "REDACTED", rm["password"]) + assert.Equal(t, "1s", rm["initTimeout"]) + y, err := yaml.Marshal(redis) + require.NoError(t, err) + assert.NotContains(t, string(y), "hunter2") + assert.Contains(t, string(y), "lockRetryInterval:") + + pg := &PostgreSQLConnectorConfig{ConnectionUri: "postgres://u:pw-secret@db:5432/x", Table: "t", MinConns: 1, MaxConns: 2} + j, err = json.Marshal(pg) + require.NoError(t, err) + assert.NotContains(t, string(j), "pw-secret") + assert.Contains(t, string(j), `"minConns":"1"`) + y, err = yaml.Marshal(pg) + require.NoError(t, err) + assert.NotContains(t, string(y), "pw-secret") + assert.Contains(t, string(y), "table: t") + + aws := &AwsAuthConfig{Mode: "secret", AccessKeyID: "AKIA", SecretAccessKey: "very-secret"} + j, err = json.Marshal(aws) + require.NoError(t, err) + assert.JSONEq(t, `{"mode":"secret","credentialsFile":"","profile":"","accessKeyID":"AKIA","secretAccessKey":"REDACTED"}`, string(j)) + y, err = yaml.Marshal(aws) + require.NoError(t, err) + assert.NotContains(t, string(y), "very-secret") + + prov := &ProviderConfig{Id: "p", Vendor: "alchemy", Settings: VendorSettings{"apiKey": "ak-secret"}, IgnoreNetworks: []string{"evm:5"}} + j, err = json.Marshal(prov) + require.NoError(t, err) + assert.NotContains(t, string(j), "ak-secret") + assert.Contains(t, string(j), `"settings":"REDACTED"`) + y, err = yaml.Marshal(prov) + require.NoError(t, err) + assert.NotContains(t, string(y), "ak-secret") + assert.Contains(t, string(y), "evm:5") + + ups := &UpstreamConfig{Id: "u", Endpoint: "https://node.example/v2/topsecretkey"} + j, err = json.Marshal(ups) + require.NoError(t, err) + assert.NotContains(t, string(j), "topsecretkey") + assert.Contains(t, string(j), "https://node.example#redacted=") + y, err = yaml.Marshal(ups) + require.NoError(t, err) + assert.NotContains(t, string(y), "topsecretkey") + assert.Equal(t, "https://node.example/v2/topsecretkey", ups.Endpoint, "marshalling never mutates the source") + + sec := &SecretStrategyConfig{Id: "s", Value: "tok-secret", RateLimitBudget: "rb"} + j, err = json.Marshal(sec) + require.NoError(t, err) + assert.JSONEq(t, `{"value":"REDACTED"}`, string(j)) + y, err = yaml.Marshal(sec) + require.NoError(t, err) + assert.NotContains(t, string(y), "tok-secret") + assert.Contains(t, string(y), "rateLimitBudget: rb") +} + +func TestCovDefaults_UpstreamCopyIsDeep(t *testing.T) { + lbm := int64(5) + slot := int64(9) + src := &UpstreamConfig{ + Id: "u", + Evm: &EvmUpstreamConfig{ChainId: 1, BlockAvailability: &EvmBlockAvailabilityConfig{Lower: &EvmAvailabilityBoundConfig{LatestBlockMinus: &lbm}}, SkipWhenSyncing: util.BoolPtr(true), SkipSyncingCheck: util.BoolPtr(true), PinLatestTag: util.BoolPtr(true), DeprecatedGetLogsSplitOnError: util.BoolPtr(true), QueryShim: &EvmQueryShimConfig{Enabled: util.BoolPtr(true), AllowedMethods: []string{"eth_queryBlocks"}}, DeprecatedIntegrity: &UpstreamIntegrityConfig{EthGetBlockReceipts: &UpstreamIntegrityEthGetBlockReceiptsConfig{Enabled: true, CheckLogIndexStrictIncrements: util.BoolPtr(true), CheckLogsBloom: util.BoolPtr(true)}}}, + Svm: &SvmUpstreamConfig{Chain: "solana", BlockAvailability: &SvmBlockAvailabilityConfig{Lower: &SvmAvailabilityBoundConfig{LatestSlotMinus: &slot, ExactSlot: &slot, FirstAvailableSlotPlus: &slot}, Upper: &SvmAvailabilityBoundConfig{}}}, + Failsafe: []*FailsafeConfig{{MatchFinality: []DataFinalityState{DataFinalityStateFinalized}, MatchCommitment: []string{"confirmed"}, Retry: &RetryPolicyConfig{MaxAttempts: 2}, CircuitBreaker: &CircuitBreakerPolicyConfig{FailureThresholdCount: 3}, Timeout: &TimeoutPolicyConfig{Duration: &AdaptiveDuration{Base: 1}}, Hedge: &HedgePolicyConfig{MaxCount: 2, Delay: &AdaptiveDuration{Base: 2}}, Consensus: &ConsensusPolicyConfig{ + MaxParticipants: 3, + PunishMisbehavior: &PunishMisbehaviorConfig{DisputeThreshold: 4}, + MisbehaviorsDestination: &MisbehaviorsDestinationConfig{Type: MisbehaviorsDestinationTypeS3, S3: &S3FlushConfig{MaxRecords: 1, Credentials: &AwsAuthConfig{Mode: "env"}}}, + IgnoreFields: map[string][]string{"eth_getLogs": {"a"}}, + PreferHighestValueFor: map[string][]string{"eth_blockNumber": {"result"}}, + MaxWaitOnResult: &AdaptiveDuration{Base: 3}, + MaxWaitOnEmpty: &AdaptiveDuration{Base: 4}, + RequiredParticipants: []*ConsensusRequiredParticipant{nil, {Tag: "t", MinParticipants: 1}}, + }}}, + JsonRpc: &JsonRpcUpstreamConfig{SupportsBatch: util.BoolPtr(true), EnableGzip: util.BoolPtr(true), Headers: map[string]string{"h": "v"}}, + Grpc: &GrpcUpstreamConfig{Headers: map[string]string{"g": "v"}}, + RateLimitAutoTune: &RateLimitAutoTuneConfig{Enabled: util.BoolPtr(true)}, + IgnoreMethods: []string{"a"}, + AllowMethods: []string{"b"}, + } + cp := src.Copy() + require.Equal(t, src, cp, "copy is value-equal") + + // Mutating every nested piece of the copy must leave the source untouched. + *cp.Evm.BlockAvailability.Lower.LatestBlockMinus = 99 + *cp.Evm.SkipWhenSyncing = false + *cp.Evm.PinLatestTag = false + *cp.Evm.DeprecatedGetLogsSplitOnError = false + *cp.Evm.QueryShim.Enabled = false + cp.Evm.QueryShim.AllowedMethods[0] = "x" + *cp.Evm.DeprecatedIntegrity.EthGetBlockReceipts.CheckLogsBloom = false + *cp.Svm.BlockAvailability.Lower.LatestSlotMinus = 0 + cp.Failsafe[0].MatchFinality[0] = DataFinalityStateUnfinalized + cp.Failsafe[0].MatchCommitment[0] = "x" + cp.Failsafe[0].Retry.MaxAttempts = 9 + cp.Failsafe[0].CircuitBreaker.FailureThresholdCount = 9 + cp.Failsafe[0].Timeout.Duration.Base = 9 + cp.Failsafe[0].Hedge.Delay.Base = 9 + cs := cp.Failsafe[0].Consensus + cs.PunishMisbehavior.DisputeThreshold = 9 + cs.MisbehaviorsDestination.S3.Credentials.Mode = "x" + cs.IgnoreFields["eth_getLogs"][0] = "x" + cs.PreferHighestValueFor["eth_blockNumber"][0] = "x" + cs.MaxWaitOnResult.Base = 9 + cs.RequiredParticipants[1].Tag = "x" + cp.Grpc.Headers["g"] = "x" + cp.IgnoreMethods[0] = "x" + cp.AllowMethods[0] = "x" + + assert.Equal(t, int64(5), *src.Evm.BlockAvailability.Lower.LatestBlockMinus) + assert.True(t, *src.Evm.SkipWhenSyncing) + assert.True(t, *src.Evm.PinLatestTag) + assert.True(t, *src.Evm.DeprecatedGetLogsSplitOnError) + assert.True(t, *src.Evm.QueryShim.Enabled) + assert.Equal(t, "eth_queryBlocks", src.Evm.QueryShim.AllowedMethods[0]) + assert.True(t, *src.Evm.DeprecatedIntegrity.EthGetBlockReceipts.CheckLogsBloom) + assert.Equal(t, int64(9), *src.Svm.BlockAvailability.Lower.LatestSlotMinus) + sf := src.Failsafe[0] + assert.Equal(t, DataFinalityStateFinalized, sf.MatchFinality[0]) + assert.Equal(t, "confirmed", sf.MatchCommitment[0]) + assert.Equal(t, 2, sf.Retry.MaxAttempts) + assert.Equal(t, uint(3), sf.CircuitBreaker.FailureThresholdCount) + assert.Equal(t, Duration(1), sf.Timeout.Duration.Base) + assert.Equal(t, Duration(2), sf.Hedge.Delay.Base) + assert.Equal(t, uint(4), sf.Consensus.PunishMisbehavior.DisputeThreshold) + assert.Equal(t, "env", sf.Consensus.MisbehaviorsDestination.S3.Credentials.Mode) + assert.Equal(t, "a", sf.Consensus.IgnoreFields["eth_getLogs"][0]) + assert.Equal(t, "result", sf.Consensus.PreferHighestValueFor["eth_blockNumber"][0]) + assert.Equal(t, Duration(3), sf.Consensus.MaxWaitOnResult.Base) + assert.Equal(t, "t", sf.Consensus.RequiredParticipants[1].Tag) + assert.Nil(t, cp.Failsafe[0].Consensus.RequiredParticipants[0]) + assert.Equal(t, "v", src.Grpc.Headers["g"]) + assert.Equal(t, "a", src.IgnoreMethods[0]) + assert.Equal(t, "b", src.AllowMethods[0]) + + // Nil receivers copy to nil. + assert.Nil(t, (*UpstreamConfig)(nil).Copy()) + assert.Nil(t, (*EvmUpstreamConfig)(nil).Copy()) + assert.Nil(t, (*FailsafeConfig)(nil).Copy()) + assert.Nil(t, (*RetryPolicyConfig)(nil).Copy()) + assert.Nil(t, (*CircuitBreakerPolicyConfig)(nil).Copy()) + assert.Nil(t, (*TimeoutPolicyConfig)(nil).Copy()) + assert.Nil(t, (*HedgePolicyConfig)(nil).Copy()) + assert.Nil(t, (*ConsensusPolicyConfig)(nil).Copy()) + assert.Nil(t, (*MisbehaviorsDestinationConfig)(nil).Copy()) + assert.Nil(t, (*PunishMisbehaviorConfig)(nil).Copy()) + assert.Nil(t, (*UpstreamIntegrityConfig)(nil).Copy()) + assert.Nil(t, (*UpstreamIntegrityEthGetBlockReceiptsConfig)(nil).Copy()) + assert.Nil(t, (*EvmQueryShimConfig)(nil).Copy()) + assert.Nil(t, (*EvmBlockAvailabilityConfig)(nil).Copy()) + assert.Nil(t, (*EvmAvailabilityBoundConfig)(nil).Copy()) + assert.Nil(t, (*SvmUpstreamConfig)(nil).Copy()) + assert.Nil(t, (*SvmBlockAvailabilityConfig)(nil).Copy()) + assert.Nil(t, (*SvmAvailabilityBoundConfig)(nil).Copy()) + assert.Nil(t, (*JsonRpcUpstreamConfig)(nil).Copy()) + assert.Nil(t, (*GrpcUpstreamConfig)(nil).Copy()) + assert.Nil(t, (*RateLimitAutoTuneConfig)(nil).Copy()) +} + +func TestCovDefaults_QueryShimAllowsMethod(t *testing.T) { + cases := []struct { + shim *EvmQueryShimConfig + method string + want bool + }{ + {nil, "eth_queryBlocks", false}, + {&EvmQueryShimConfig{}, "eth_queryBlocks", false}, + {&EvmQueryShimConfig{Enabled: util.BoolPtr(false)}, "eth_queryBlocks", false}, + {&EvmQueryShimConfig{Enabled: util.BoolPtr(true)}, "eth_queryBlocks", true}, + {&EvmQueryShimConfig{Enabled: util.BoolPtr(true), AllowedMethods: []string{"eth_queryL*"}}, "eth_queryLogs", true}, + {&EvmQueryShimConfig{Enabled: util.BoolPtr(true), AllowedMethods: []string{"eth_queryL*"}}, "eth_queryBlocks", false}, + } + for _, tc := range cases { + assert.Equal(t, tc.want, tc.shim.AllowsMethod(tc.method), "%+v %s", tc.shim, tc.method) + } +} + +func TestCovDefaults_NetworkIdAndHelpers(t *testing.T) { + cases := []struct { + n NetworkConfig + want string + }{ + {NetworkConfig{}, ""}, + {NetworkConfig{Architecture: ArchitectureEvm}, ""}, + {NetworkConfig{Architecture: ArchitectureEvm, Evm: &EvmNetworkConfig{ChainId: 137}}, "evm:137"}, + {NetworkConfig{Architecture: ArchitectureSvm}, ""}, + {NetworkConfig{Architecture: ArchitectureSvm, Svm: &SvmNetworkConfig{}}, ""}, + {NetworkConfig{Architecture: ArchitectureSvm, Svm: &SvmNetworkConfig{Cluster: "mainnet-beta"}}, "svm:mainnet-beta"}, + {NetworkConfig{Architecture: ArchitectureSvm, Svm: &SvmNetworkConfig{Chain: "fogo", Cluster: "testnet"}}, "svm:fogo:testnet"}, + {NetworkConfig{Architecture: "btc"}, ""}, + } + for _, tc := range cases { + assert.Equal(t, tc.want, tc.n.NetworkId()) + } + + var nilNet *NetworkConfig + assert.True(t, nilNet.MultiplexingEnabled(), "multiplexing defaults on") + assert.False(t, (*FailoverConfig)(nil).Enabled()) + assert.False(t, (&FailoverConfig{}).Enabled()) + var nilEvm *EvmNetworkConfig + assert.False(t, nilEvm.ShortCircuitFutureBlocksEnabled()) + assert.Equal(t, ServedTipQuorum(0), nilEvm.ServedTipQuorum()) + + assert.Equal(t, "evm:1:42", CachePartitionKey("evm:1", "", "42")) + assert.Equal(t, "evm:1:sfx:42", CachePartitionKey("evm:1", "sfx", "42")) + assert.Equal(t, `evm:1:systx\:foo`, CachePartitionKey("evm:1", "", "systx:foo")) + assert.Equal(t, `evm:1:a\\b:*`, CachePartitionKey("evm:1", `a\b`, "*")) + assert.Equal(t, `svm\:mainnet:bar:1`, CachePartitionKey("svm:mainnet", "bar", "1"), "suffix that would form another network id escapes the network id") +} + +func TestCovDefaults_MetricsTelemetryOptions(t *testing.T) { + assert.Nil(t, (*MetricsConfig)(nil).TelemetryOptions()) + + evict := covDefaultsDur(2 * time.Hour) + m := &MetricsConfig{ + HistogramBuckets: "0.1,1", + HistogramDropLabels: []string{"a"}, + HistogramLabelOverrides: map[string][]string{"h": {"b"}}, + CounterDropLabels: []string{"c"}, + CounterLabelOverrides: map[string][]string{"k": {"d"}}, + Customizations: []*MetricsCustomizationConfig{ + nil, + {Subject: "consensus_*", Action: "drop"}, + {Subject: "x", Buckets: []float64{1, 2}, Labels: []*MetricLabelCustomizationConfig{nil, {Subject: "network", Action: "keep"}}}, + }, + CounterIdleEvictionAfter: &evict, + } + o := m.TelemetryOptions() + assert.Equal(t, "0.1,1", o.HistogramBuckets) + assert.Equal(t, []string{"a"}, o.LegacyLabels.HistogramDropLabels) + assert.Equal(t, map[string][]string{"k": {"d"}}, o.LegacyLabels.CounterLabelOverrides) + require.Len(t, o.Customizations, 2, "nil customizations are skipped") + assert.Equal(t, "consensus_*", o.Customizations[0].Subject) + assert.Equal(t, "drop", o.Customizations[0].Action) + assert.Equal(t, []float64{1, 2}, o.Customizations[1].Buckets) + require.Len(t, o.Customizations[1].Labels, 1) + assert.Equal(t, "network", o.Customizations[1].Labels[0].Subject) + assert.Equal(t, "keep", o.Customizations[1].Labels[0].Action) + require.NotNil(t, o.CounterIdleEvictionAfter) + assert.Equal(t, 2*time.Hour, *o.CounterIdleEvictionAfter) + + assert.Nil(t, (&MetricsConfig{}).TelemetryOptions().CounterIdleEvictionAfter) +} diff --git a/common/cov_defaults_infra_test.go b/common/cov_defaults_infra_test.go new file mode 100644 index 000000000..cca104fa7 --- /dev/null +++ b/common/cov_defaults_infra_test.go @@ -0,0 +1,625 @@ +package common + +import ( + "testing" + "time" + + "github.com/erpc/erpc/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func covDefaultsDur(d time.Duration) Duration { return Duration(d) } + +func TestCovDefaults_ConfigSetDefaults_EmptyConfigGetsMainProject(t *testing.T) { + cfg := &Config{} + require.NoError(t, cfg.SetDefaults(&DefaultOptions{})) + + assert.Equal(t, "INFO", cfg.LogLevel) + assert.Equal(t, "erpc-default", cfg.ClusterKey) + require.NotNil(t, cfg.HealthCheck) + assert.Equal(t, HealthCheckModeNetworks, cfg.HealthCheck.Mode) + assert.Equal(t, EvalAnyInitializedUpstreams, cfg.HealthCheck.DefaultEval) + + // A lone synthetic project gets an alias rule so /evm/1 works without /main. + require.NotNil(t, cfg.Server.Aliasing) + require.Len(t, cfg.Server.Aliasing.Rules, 1) + assert.Equal(t, "*", cfg.Server.Aliasing.Rules[0].MatchDomain) + assert.Equal(t, "main", cfg.Server.Aliasing.Rules[0].ServeProject) + + require.Len(t, cfg.Projects, 1) + p := cfg.Projects[0] + assert.Equal(t, "main", p.Id) + require.Len(t, p.Providers, 1) + assert.Equal(t, "public", p.Providers[0].Id) + assert.Equal(t, "repository", p.Providers[0].Vendor) + + // The synthetic project's upstream defaults were defaulted in turn. + require.Len(t, p.UpstreamDefaults.Failsafe, 1) + assert.Equal(t, 1, p.UpstreamDefaults.Failsafe[0].Retry.MaxAttempts) + assert.Equal(t, covDefaultsDur(500*time.Millisecond), p.UpstreamDefaults.Failsafe[0].Retry.Delay) + assert.Equal(t, float32(1.2), p.UpstreamDefaults.Failsafe[0].Retry.BackoffFactor) + require.Len(t, p.NetworkDefaults.Failsafe, 1) + assert.Equal(t, 2, p.NetworkDefaults.Failsafe[0].Hedge.MaxCount) + assert.Equal(t, covDefaultsDur(100*time.Millisecond), p.NetworkDefaults.Failsafe[0].Hedge.Delay.Min) +} + +func TestCovDefaults_ConfigSetDefaults_EndpointsOptionBecomeUpstreams(t *testing.T) { + cfg := &Config{} + require.NoError(t, cfg.SetDefaults(&DefaultOptions{Endpoints: []string{"https://rpc.covdefaults-a.example", "https://rpc.covdefaults-a.example"}})) + p := cfg.Projects[0] + assert.Empty(t, p.Providers) + require.Len(t, p.Upstreams, 2) + for _, u := range p.Upstreams { + assert.Equal(t, "https://rpc.covdefaults-a.example", u.Endpoint) + assert.Equal(t, UpstreamTypeEvm, u.Type) + // The synthetic project's upstreamDefaults failsafe is copied onto each. + require.Len(t, u.Failsafe, 1) + assert.Equal(t, 1, u.Failsafe[0].Retry.MaxAttempts) + } + // Ids are derived from the host and kept unique by a counter. + assert.NotEqual(t, p.Upstreams[0].Id, p.Upstreams[1].Id) + assert.Contains(t, p.Upstreams[0].Id, "rpc.covdefaults-a.example-") +} + +func TestCovDefaults_ConfigSetDefaults_PropagatesComponents(t *testing.T) { + cfg := &Config{ + ClusterKey: "my-cluster", + Tracing: &TracingConfig{Enabled: true}, + Database: &DatabaseConfig{ + SharedState: &SharedStateConfig{}, + }, + Admin: &AdminConfig{}, + RateLimiters: &RateLimiterConfig{ + Budgets: []*RateLimitBudgetConfig{{Id: "b", Rules: []*RateLimitRuleConfig{{MaxCount: 5, Period: RateLimitPeriod(99)}}}}, + }, + Projects: []*ProjectConfig{{Id: "p1", Upstreams: []*UpstreamConfig{{Endpoint: "https://x.example"}}}}, + } + require.NoError(t, cfg.SetDefaults(nil)) + + assert.Equal(t, "my-cluster", cfg.Database.SharedState.ClusterKey, "shared state inherits the top-level cluster key") + assert.Equal(t, "grpc", string(cfg.Tracing.Protocol)) + require.NotNil(t, cfg.Admin.CORS) + assert.Equal(t, []string{"*"}, cfg.Admin.CORS.AllowedOrigins) + assert.Equal(t, "memory", cfg.RateLimiters.Store.Driver) + rule := cfg.RateLimiters.Budgets[0].Rules[0] + assert.Equal(t, RateLimitPeriodSecond, rule.Period, "invalid period falls back to second") + assert.Equal(t, "*", rule.Method) + assert.Nil(t, cfg.Server.Aliasing, "explicit projects get no implicit aliasing rule") + assert.True(t, cfg.HasRateLimiterBudget("b")) + assert.False(t, cfg.HasRateLimiterBudget("missing")) +} + +func TestCovDefaults_ConfigSetDefaults_ErrorsPropagate(t *testing.T) { + cases := []struct { + name string + cfg *Config + want string + }{ + {"database", &Config{Database: &DatabaseConfig{SharedState: &SharedStateConfig{Connector: &ConnectorConfig{Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}}}}}, "not both"}, + {"admin", &Config{Admin: &AdminConfig{Auth: &AuthConfig{Strategies: []*AuthStrategyConfig{{Type: AuthTypeDatabase, Database: &DatabaseStrategyConfig{Connector: &ConnectorConfig{Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}}}}}}}}, "failed to set defaults for auth"}, + {"project", &Config{Projects: []*ProjectConfig{{Id: "p", Networks: []*NetworkConfig{{SelectionPolicy: &SelectionPolicyConfig{EvalScope: "bogus"}}}}}}, "evalScope"}, + {"rateLimiters", &Config{Projects: []*ProjectConfig{{Id: "p", Upstreams: []*UpstreamConfig{{Endpoint: "https://x"}}}}, RateLimiters: &RateLimiterConfig{Store: &RateLimitStoreConfig{Driver: "redis", Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}}}}, "rate limit store"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := tc.cfg.SetDefaults(nil) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.want) + }) + } +} + +func TestCovDefaults_ServerConfig_DeprecatedHttpPortMigrates(t *testing.T) { + s := &ServerConfig{HttpPort: util.IntPtr(8080)} + require.NoError(t, s.SetDefaults()) + assert.Equal(t, 8080, *s.HttpPortV4) + assert.Equal(t, 9080, *s.HttpPortV6, "v6 port derives as http port + 1000") + assert.Equal(t, 8080, *s.GrpcPortV4) + assert.Equal(t, 9080, *s.GrpcPortV6) + + s2 := &ServerConfig{HttpPortV4: util.IntPtr(7000), HttpPortV6: util.IntPtr(7001), HttpHostV4: util.StringPtr("10.0.0.1")} + require.NoError(t, s2.SetDefaults()) + assert.Equal(t, 7000, *s2.HttpPort, "deprecated field mirrors the v4 port") + assert.Equal(t, 7001, *s2.HttpPortV6) + assert.Equal(t, "10.0.0.1", *s2.GrpcHostV4) + assert.Equal(t, "[::]", *s2.GrpcHostV6) +} + +func TestCovDefaults_ServerConfig_OmittedFieldsAndExplicitWins(t *testing.T) { + s := &ServerConfig{} + require.NoError(t, s.SetDefaults()) + assert.Nil(t, s.ListenV4, "tests do not listen unless FORCE_TEST_LISTEN_V4") + assert.Equal(t, covDefaultsDur(150*time.Second), *s.MaxTimeout) + assert.Equal(t, covDefaultsDur(30*time.Second), *s.ReadTimeout) + assert.Equal(t, covDefaultsDur(120*time.Second), *s.WriteTimeout) + assert.Equal(t, covDefaultsDur(10*time.Second), *s.WaitBeforeShutdown) + assert.Equal(t, covDefaultsDur(10*time.Second), *s.WaitAfterShutdown) + assert.Equal(t, ExecutionHeadersAll, *s.ExecutionHeaders) + assert.False(t, *s.CostHeaders) + assert.True(t, *s.EnableGzip) + assert.True(t, *s.IncludeErrorDetails) + assert.False(t, *s.GrpcEnabled) + assert.True(t, *s.GrpcReflection) + assert.Equal(t, 100*1024*1024, *s.GrpcMaxRecvMsgSize) + assert.Equal(t, []string{"127.0.0.1/8", "::1/128"}, s.TrustedIPForwarders) + assert.Equal(t, []string{}, s.TrustedIPHeaders) + ws := s.WebSocket + assert.False(t, *ws.Enabled) + assert.Equal(t, 4096, ws.ReadBufferSize) + assert.Equal(t, int64(1024*1024), ws.MaxMessageSize) + assert.Equal(t, covDefaultsDur(30*time.Second), *ws.PingInterval) + assert.Equal(t, 100, ws.MaxSubscriptionsPerConnection) + assert.Equal(t, 250, ws.MaxConcurrentRequestsPerConnection) + assert.Equal(t, 256, ws.SubscriptionBufferSize) + + t.Setenv("FORCE_TEST_LISTEN_V4", "true") + forced := &ServerConfig{ + TrustedIPForwarders: []string{"10.0.0.0/8"}, + WebSocket: &WebSocketServerConfig{Enabled: util.BoolPtr(true), ReadBufferSize: 1}, + } + require.NoError(t, forced.SetDefaults()) + assert.True(t, *forced.ListenV4) + assert.Equal(t, []string{"10.0.0.0/8"}, forced.TrustedIPForwarders) + assert.True(t, *forced.WebSocket.Enabled) + assert.Equal(t, 1, forced.WebSocket.ReadBufferSize) +} + +func TestCovDefaults_MetricsAndTracingDefaults(t *testing.T) { + m := &MetricsConfig{} + require.NoError(t, m.SetDefaults()) + assert.Nil(t, m.Enabled, "metrics stay disabled under go test") + assert.Equal(t, "0.0.0.0", *m.HostV4) + assert.Equal(t, "[::]", *m.HostV6) + assert.Equal(t, 4001, *m.Port) + assert.Equal(t, ErrorLabelModeCompact, m.ErrorLabelMode) + assert.Equal(t, covDefaultsDur(24*time.Hour), *m.CounterIdleEvictionAfter) + + explicit := &MetricsConfig{Port: util.IntPtr(9100), ErrorLabelMode: ErrorLabelModeVerbose} + require.NoError(t, explicit.SetDefaults()) + assert.Equal(t, 9100, *explicit.Port) + assert.Equal(t, ErrorLabelModeVerbose, explicit.ErrorLabelMode) + + cases := []struct { + in TracingConfig + wantEndpoint string + wantProtocol TracingProtocol + }{ + {TracingConfig{}, "localhost:4317", TracingProtocolGrpc}, + {TracingConfig{Protocol: TracingProtocolHttp}, "http://localhost:4318", TracingProtocolHttp}, + {TracingConfig{Endpoint: "otel:4317"}, "otel:4317", TracingProtocolGrpc}, + } + for _, tc := range cases { + tr := tc.in + require.NoError(t, tr.SetDefaults()) + assert.Equal(t, tc.wantEndpoint, tr.Endpoint) + assert.Equal(t, tc.wantProtocol, tr.Protocol) + assert.Equal(t, 1.0, tr.SampleRate) + assert.Equal(t, "erpc", tr.ServiceName) + } + tr := &TracingConfig{SampleRate: 0.25, ServiceName: "svc"} + require.NoError(t, tr.SetDefaults()) + assert.Equal(t, 0.25, tr.SampleRate) + assert.Equal(t, "svc", tr.ServiceName) +} + +func TestCovDefaults_AdminAndCORS(t *testing.T) { + a := &AdminConfig{Auth: &AuthConfig{Strategies: []*AuthStrategyConfig{{Secret: &SecretStrategyConfig{Value: "s"}}}}} + require.NoError(t, a.SetDefaults()) + assert.Equal(t, AuthTypeSecret, a.Auth.Strategies[0].Type, "a secret block implies the secret type") + assert.Equal(t, []string{"*"}, a.CORS.AllowedOrigins) + assert.Equal(t, []string{"GET", "POST", "OPTIONS"}, a.CORS.AllowedMethods) + assert.Equal(t, []string{"content-type", "authorization", "x-erpc-secret-token"}, a.CORS.AllowedHeaders) + assert.False(t, *a.CORS.AllowCredentials) + assert.Equal(t, 3600, a.CORS.MaxAge) + + c := &CORSConfig{AllowedOrigins: []string{"https://app"}, AllowedMethods: []string{"POST"}, AllowedHeaders: []string{"x"}, AllowCredentials: util.BoolPtr(true), MaxAge: 5} + require.NoError(t, c.SetDefaults()) + assert.Equal(t, CORSConfig{AllowedOrigins: []string{"https://app"}, AllowedMethods: []string{"POST"}, AllowedHeaders: []string{"x"}, AllowCredentials: util.BoolPtr(true), MaxAge: 5}, *c) + + bad := &AdminConfig{Auth: &AuthConfig{Strategies: []*AuthStrategyConfig{{Type: AuthTypeDatabase, Database: &DatabaseStrategyConfig{Connector: &ConnectorConfig{Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}}}}}}} + assert.ErrorContains(t, bad.SetDefaults(), "failed to set defaults for auth") +} + +func TestCovDefaults_SharedState(t *testing.T) { + s := &SharedStateConfig{} + require.NoError(t, s.SetDefaults("ck")) + assert.Equal(t, "ck", s.ClusterKey) + assert.Equal(t, "memory", s.Connector.Id) + assert.Equal(t, DriverMemory, s.Connector.Driver) + assert.Equal(t, 100_000, s.Connector.Memory.MaxItems) + assert.Equal(t, covDefaultsDur(3*time.Second), s.FallbackTimeout) + assert.Equal(t, covDefaultsDur(4*time.Second), s.LockTtl) + assert.Equal(t, covDefaultsDur(100*time.Millisecond), s.LockMaxWait) + assert.Equal(t, covDefaultsDur(50*time.Millisecond), s.UpdateMaxWait) + + r := &SharedStateConfig{ClusterKey: "own", LockTtl: covDefaultsDur(time.Minute), Connector: &ConnectorConfig{Driver: DriverRedis, Redis: &RedisConnectorConfig{Addr: "r:6380"}}} + require.NoError(t, r.SetDefaults("ck")) + assert.Equal(t, "own", r.ClusterKey) + assert.Equal(t, "redis", r.Connector.Id, "an id-less connector is named after its driver") + assert.Equal(t, "redis://r:6380/0", r.Connector.Redis.URI) + assert.Equal(t, covDefaultsDur(time.Minute), r.LockTtl) + + bad := &SharedStateConfig{Connector: &ConnectorConfig{Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}}} + assert.ErrorContains(t, bad.SetDefaults("ck"), "not both") +} + +func TestCovDefaults_DatabaseCaches(t *testing.T) { + d := &DatabaseConfig{ + EvmJsonRpcCache: &CacheConfig{ + Connectors: []*ConnectorConfig{{Driver: DriverMemory}}, + Policies: []*CachePolicyConfig{{Connector: "cache-memory"}}, + }, + SvmJsonRpcCache: &CacheConfig{Compression: &CompressionConfig{Enabled: util.BoolPtr(false), Algorithm: "zstd", ZstdLevel: "best", Threshold: 10}}, + } + require.NoError(t, d.SetDefaults("ck")) + evm := d.EvmJsonRpcCache + assert.Equal(t, "cache-memory", evm.Connectors[0].Id) + assert.Equal(t, &MemoryConnectorConfig{MaxItems: 100000, MaxTotalSize: "1GB"}, evm.Connectors[0].Memory) + assert.Equal(t, "*", evm.Policies[0].Method) + assert.Equal(t, "*", evm.Policies[0].Network) + assert.Equal(t, CachePolicyAppliesToBoth, evm.Policies[0].AppliesTo) + assert.Equal(t, &CompressionConfig{Enabled: util.BoolPtr(true), Algorithm: "zstd", ZstdLevel: "fastest", Threshold: 1024}, evm.Compression) + assert.Equal(t, &CompressionConfig{Enabled: util.BoolPtr(false), Algorithm: "zstd", ZstdLevel: "best", Threshold: 10}, d.SvmJsonRpcCache.Compression) + + pol := &CachePolicyConfig{Method: "eth_call", Network: "evm:1", AppliesTo: CachePolicyAppliesToGet} + require.NoError(t, pol.SetDefaults()) + assert.Equal(t, "eth_call", pol.Method) + assert.Equal(t, CachePolicyAppliesToGet, pol.AppliesTo) + + for name, c := range map[string]*DatabaseConfig{ + "evm": {EvmJsonRpcCache: &CacheConfig{Connectors: []*ConnectorConfig{{Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}}}}}, + "svm": {SvmJsonRpcCache: &CacheConfig{Connectors: []*ConnectorConfig{{Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}}}}}, + "shared": {SharedState: &SharedStateConfig{Connector: &ConnectorConfig{Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}}}}, + } { + t.Run(name, func(t *testing.T) { + assert.ErrorContains(t, c.SetDefaults("ck"), "not both") + }) + } +} + +func TestCovDefaults_ConnectorDriverInference(t *testing.T) { + cases := []struct { + name string + scope connectorScope + in ConnectorConfig + check func(t *testing.T, c *ConnectorConfig) + }{ + {"memory by driver", connectorScopeCache, ConnectorConfig{Driver: DriverMemory}, func(t *testing.T, c *ConnectorConfig) { + assert.Equal(t, "cache-memory", c.Id) + assert.Equal(t, 100000, c.Memory.MaxItems) + }}, + {"redis block implies driver", connectorScopeCache, ConnectorConfig{Redis: &RedisConnectorConfig{Addr: "h"}}, func(t *testing.T, c *ConnectorConfig) { + assert.Equal(t, DriverRedis, c.Driver) + assert.Equal(t, "redis://h:6379/0", c.Redis.URI) + }}, + {"redis driver without block", connectorScopeSharedState, ConnectorConfig{Driver: DriverRedis}, func(t *testing.T, c *ConnectorConfig) { + require.NotNil(t, c.Redis) + assert.Equal(t, 8, c.Redis.ConnPoolSize) + assert.Equal(t, "", c.Redis.URI) + }}, + {"postgres driver without block", connectorScopeCache, ConnectorConfig{Driver: DriverPostgreSQL}, func(t *testing.T, c *ConnectorConfig) { + assert.Equal(t, "erpc_json_rpc_cache", c.PostgreSQL.Table) + }}, + {"postgres block implies driver", connectorScopeAuth, ConnectorConfig{Id: "mine", PostgreSQL: &PostgreSQLConnectorConfig{}}, func(t *testing.T, c *ConnectorConfig) { + assert.Equal(t, DriverPostgreSQL, c.Driver) + assert.Equal(t, "mine", c.Id) + assert.Equal(t, "erpc_auth", c.PostgreSQL.Table) + }}, + {"dynamodb driver without block", connectorScopeSharedState, ConnectorConfig{Driver: DriverDynamoDB}, func(t *testing.T, c *ConnectorConfig) { + assert.Equal(t, "erpc_shared_state", c.DynamoDB.Table) + }}, + {"dynamodb block implies driver", connectorScopeCache, ConnectorConfig{DynamoDB: &DynamoDBConnectorConfig{Table: "t"}}, func(t *testing.T, c *ConnectorConfig) { + assert.Equal(t, DriverDynamoDB, c.Driver) + assert.Equal(t, "t", c.DynamoDB.Table) + }}, + {"grpc driver without block", connectorScopeCache, ConnectorConfig{Driver: DriverGrpc}, func(t *testing.T, c *ConnectorConfig) { + assert.Equal(t, covDefaultsDur(100*time.Millisecond), c.Grpc.GetTimeout) + }}, + {"grpc block keeps explicit timeout", connectorScopeCache, ConnectorConfig{Grpc: &GrpcConnectorConfig{GetTimeout: covDefaultsDur(time.Second)}}, func(t *testing.T, c *ConnectorConfig) { + assert.Equal(t, DriverGrpc, c.Driver) + assert.Equal(t, covDefaultsDur(time.Second), c.Grpc.GetTimeout) + }}, + {"failsafe lists default match method and skip nils", connectorScopeCache, ConnectorConfig{ + Driver: DriverMemory, + FailsafeForGets: []*FailsafeConfig{nil, {Retry: &RetryPolicyConfig{}}}, + FailsafeForSets: []*FailsafeConfig{nil, {MatchMethod: "eth_*"}}, + }, func(t *testing.T, c *ConnectorConfig) { + assert.Nil(t, c.FailsafeForGets[0]) + assert.Equal(t, "*", c.FailsafeForGets[1].MatchMethod) + assert.Equal(t, 3, c.FailsafeForGets[1].Retry.MaxAttempts) + assert.Equal(t, "eth_*", c.FailsafeForSets[1].MatchMethod) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := tc.in + require.NoError(t, c.SetDefaults(tc.scope)) + tc.check(t, &c) + }) + } +} + +func TestCovDefaults_ConnectorErrorsAreWrapped(t *testing.T) { + badHedge := []*FailsafeConfig{{Consensus: &ConsensusPolicyConfig{MisbehaviorsDestination: &MisbehaviorsDestinationConfig{}}}} + _ = badHedge + cases := []struct { + name string + scope connectorScope + in ConnectorConfig + want string + }{ + {"redis", connectorScopeCache, ConnectorConfig{Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}}, "failed to set defaults for redis connector"}, + {"postgres scope", connectorScope("bogus"), ConnectorConfig{Driver: DriverPostgreSQL}, "failed to set defaults for postgres connector: invalid connector scope: bogus"}, + {"dynamo scope", connectorScope("bogus"), ConnectorConfig{Driver: DriverDynamoDB}, "failed to set defaults for dynamo db connector: invalid connector scope: bogus"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := tc.in + assert.ErrorContains(t, c.SetDefaults(tc.scope), tc.want) + }) + } +} + +func TestCovDefaults_RedisConnector(t *testing.T) { + cases := []struct { + name string + in RedisConnectorConfig + wantURI string + }{ + {"addr without port", RedisConnectorConfig{Addr: "cache.local"}, "redis://cache.local:6379/0"}, + {"addr with scheme prefix and creds", RedisConnectorConfig{Addr: "redis://cache.local:6380", Username: "u", Password: "p@ss", DB: 2}, "redis://u:p%40ss@cache.local:6380/2"}, + {"rediss prefix stripped, tls picks scheme", RedisConnectorConfig{Addr: "rediss://c:1", TLS: &TLSConfig{Enabled: true}}, "rediss://c:1/0"}, + {"uri kept verbatim", RedisConnectorConfig{URI: "redis://x:1/3"}, "redis://x:1/3"}, + {"nothing set leaves uri empty", RedisConnectorConfig{}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := tc.in + require.NoError(t, r.SetDefaults()) + assert.Equal(t, tc.wantURI, r.URI) + if tc.in.Addr != "" { + assert.Empty(t, r.Addr, "discrete fields are folded into the uri") + assert.Empty(t, r.Username) + assert.Empty(t, r.Password) + assert.Zero(t, r.DB) + } + assert.Equal(t, 8, r.ConnPoolSize) + assert.Equal(t, covDefaultsDur(5*time.Second), r.InitTimeout) + assert.Equal(t, covDefaultsDur(time.Second), r.GetTimeout) + assert.Equal(t, covDefaultsDur(3*time.Second), r.SetTimeout) + assert.Equal(t, covDefaultsDur(500*time.Millisecond), r.LockRetryInterval) + }) + } + + explicit := &RedisConnectorConfig{URI: "redis://x", ConnPoolSize: 2, InitTimeout: 1, GetTimeout: 1, SetTimeout: 1, LockRetryInterval: 1} + require.NoError(t, explicit.SetDefaults()) + assert.Equal(t, 2, explicit.ConnPoolSize) + assert.Equal(t, Duration(1), explicit.LockRetryInterval) + + err := (&RedisConnectorConfig{URI: "redis://a", Addr: "b"}).SetDefaults() + assert.EqualError(t, err, "redis connector: provide either 'uri' or 'addr/username/password/db', not both") + + iam := &RedisConnectorConfig{Addr: "c:6379", IAMAuth: &RedisIAMAuthConfig{Enabled: true, CacheName: " MyCache "}} + require.NoError(t, iam.SetDefaults()) + assert.Equal(t, "mycache", iam.IAMAuth.CacheName) + assert.True(t, iam.TLS.Enabled) + assert.Equal(t, "rediss://c:6379/0", iam.URI) +} + +func TestCovDefaults_PostgresConnector(t *testing.T) { + cases := []struct { + scope connectorScope + wantTable string + wantMin int32 + wantMax int32 + }{ + {connectorScopeSharedState, "erpc_shared_state", 4, 32}, + {connectorScopeCache, "erpc_json_rpc_cache", 4, 32}, + {connectorScopeAuth, "erpc_auth", 1, 4}, + } + for _, tc := range cases { + t.Run(string(tc.scope), func(t *testing.T) { + p := &PostgreSQLConnectorConfig{} + require.NoError(t, p.SetDefaults(tc.scope)) + assert.Equal(t, tc.wantTable, p.Table) + assert.Equal(t, tc.wantMin, p.MinConns) + assert.Equal(t, tc.wantMax, p.MaxConns) + assert.Equal(t, covDefaultsDur(5*time.Second), p.InitTimeout) + assert.Equal(t, covDefaultsDur(time.Second), p.GetTimeout) + assert.Equal(t, covDefaultsDur(2*time.Second), p.SetTimeout) + }) + } + explicit := &PostgreSQLConnectorConfig{Table: "t", MinConns: 7, MaxConns: 9} + require.NoError(t, explicit.SetDefaults(connectorScope("anything")), "explicit table skips scope validation") + assert.Equal(t, int32(7), explicit.MinConns) + assert.Equal(t, int32(9), explicit.MaxConns) + + assert.EqualError(t, (&PostgreSQLConnectorConfig{}).SetDefaults("weird"), "invalid connector scope: weird") + + withQuery := &PostgreSQLConnectorConfig{ConnectionUri: "postgres://u@db:5432/x?connect_timeout=5", IAMAuth: &PostgreSQLIAMAuthConfig{Enabled: true}} + require.NoError(t, withQuery.SetDefaults(connectorScopeCache)) + assert.Equal(t, "postgres://u@db:5432/x?connect_timeout=5&sslmode=require", withQuery.ConnectionUri) + + alreadySSL := &PostgreSQLConnectorConfig{ConnectionUri: "postgres://u@db:5432/x?sslmode=verify-full", IAMAuth: &PostgreSQLIAMAuthConfig{Enabled: true}} + require.NoError(t, alreadySSL.SetDefaults(connectorScopeCache)) + assert.Equal(t, "postgres://u@db:5432/x?sslmode=verify-full", alreadySSL.ConnectionUri) + + disabledIAM := &PostgreSQLConnectorConfig{ConnectionUri: "postgres://db/x", IAMAuth: &PostgreSQLIAMAuthConfig{Enabled: false}} + require.NoError(t, disabledIAM.SetDefaults(connectorScopeCache)) + assert.Equal(t, "postgres://db/x", disabledIAM.ConnectionUri) + assert.Empty(t, disabledIAM.IAMAuth.Endpoint) +} + +func TestCovDefaults_DynamoDBConnector(t *testing.T) { + for scope, table := range map[connectorScope]string{ + connectorScopeSharedState: "erpc_shared_state", + connectorScopeCache: "erpc_json_rpc_cache", + connectorScopeAuth: "erpc_auth", + } { + d := &DynamoDBConnectorConfig{} + require.NoError(t, d.SetDefaults(scope)) + assert.Equal(t, &DynamoDBConnectorConfig{ + Table: table, + PartitionKeyName: "groupKey", + RangeKeyName: "requestKey", + ReverseIndexName: "idx_requestKey_groupKey", + TTLAttributeName: "ttl", + InitTimeout: covDefaultsDur(5 * time.Second), + GetTimeout: covDefaultsDur(time.Second), + SetTimeout: covDefaultsDur(2 * time.Second), + StatePollInterval: covDefaultsDur(5 * time.Second), + }, d) + } + explicit := &DynamoDBConnectorConfig{Table: "x", PartitionKeyName: "pk", RangeKeyName: "rk", ReverseIndexName: "ri", TTLAttributeName: "exp", InitTimeout: 1, GetTimeout: 2, SetTimeout: 3, StatePollInterval: 4} + want := *explicit + require.NoError(t, explicit.SetDefaults("bogus")) + assert.Equal(t, want, *explicit) + assert.EqualError(t, (&DynamoDBConnectorConfig{}).SetDefaults("bogus"), "invalid connector scope: bogus") +} + +func TestCovDefaults_AuthStrategies(t *testing.T) { + t.Run("type implies empty sub-config", func(t *testing.T) { + for _, typ := range []AuthType{AuthTypeNetwork, AuthTypeSecret, AuthTypeJwt, AuthTypeSiwe, AuthTypeDatabase} { + s := &AuthStrategyConfig{Type: typ} + require.NoError(t, s.SetDefaults()) + assert.Equal(t, typ, s.Type) + switch typ { + case AuthTypeNetwork: + assert.NotNil(t, s.Network) + case AuthTypeSecret: + assert.NotNil(t, s.Secret) + case AuthTypeJwt: + assert.Equal(t, "rlm", s.Jwt.RateLimitBudgetClaimName) + assert.Zero(t, s.Jwt.VerificationJwksRefreshInterval, "no jwks url, no refresh interval") + case AuthTypeSiwe: + assert.NotNil(t, s.Siwe) + case AuthTypeDatabase: + assert.Equal(t, int64(10000), *s.Database.Cache.MaxSize) + } + } + }) + t.Run("sub-config block sets the type", func(t *testing.T) { + cases := []struct { + in AuthStrategyConfig + want AuthType + }{ + {AuthStrategyConfig{Jwt: &JwtStrategyConfig{}}, AuthTypeJwt}, + {AuthStrategyConfig{Siwe: &SiweStrategyConfig{}}, AuthTypeSiwe}, + {AuthStrategyConfig{Database: &DatabaseStrategyConfig{}}, AuthTypeDatabase}, + {AuthStrategyConfig{Secret: &SecretStrategyConfig{}}, AuthTypeSecret}, + } + for _, tc := range cases { + s := tc.in + require.NoError(t, s.SetDefaults()) + assert.Equal(t, tc.want, s.Type) + } + }) + t.Run("jwt keeps explicit values and defaults jwks refresh", func(t *testing.T) { + j := &JwtStrategyConfig{RateLimitBudgetClaimName: "tier", VerificationJwksUrl: "https://idp/jwks"} + require.NoError(t, j.SetDefaults()) + assert.Equal(t, "tier", j.RateLimitBudgetClaimName) + assert.Equal(t, covDefaultsDur(time.Hour), j.VerificationJwksRefreshInterval) + j2 := &JwtStrategyConfig{VerificationJwksUrl: "https://idp/jwks", VerificationJwksRefreshInterval: covDefaultsDur(time.Minute)} + require.NoError(t, j2.SetDefaults()) + assert.Equal(t, covDefaultsDur(time.Minute), j2.VerificationJwksRefreshInterval) + }) + t.Run("database strategy defaults", func(t *testing.T) { + d := &DatabaseStrategyConfig{Connector: &ConnectorConfig{Driver: DriverPostgreSQL, PostgreSQL: &PostgreSQLConnectorConfig{ConnectionUri: "postgres://db"}}} + require.NoError(t, d.SetDefaults()) + assert.Equal(t, time.Hour, *d.Cache.TTL) + assert.Equal(t, int64(10000), *d.Cache.MaxSize) + assert.Equal(t, int64(1<<30), *d.Cache.MaxCost) + assert.Equal(t, int64(100000), *d.Cache.NumCounters) + assert.Equal(t, 3, d.Retry.MaxAttempts) + assert.Equal(t, covDefaultsDur(100*time.Millisecond), d.Retry.BaseBackoff) + assert.False(t, d.FailOpen.Enabled) + assert.Equal(t, "emergency-failopen", d.FailOpen.UserId) + assert.Equal(t, covDefaultsDur(time.Second), d.MaxWait) + assert.Equal(t, "auth-postgresql", d.Connector.Id) + assert.Equal(t, "erpc_auth", d.Connector.PostgreSQL.Table) + assert.Equal(t, int32(1), d.Connector.PostgreSQL.MinConns) + assert.Equal(t, int32(4), d.Connector.PostgreSQL.MaxConns) + }) + t.Run("database strategy explicit values survive", func(t *testing.T) { + ttl := time.Minute + size, cost, counters := int64(1), int64(2), int64(3) + d := &DatabaseStrategyConfig{ + Connector: &ConnectorConfig{Driver: DriverMemory}, + Cache: &DatabaseStrategyCacheConfig{TTL: &ttl, MaxSize: &size, MaxCost: &cost, NumCounters: &counters}, + Retry: &DatabaseRetryConfig{MaxAttempts: -1, BaseBackoff: covDefaultsDur(time.Second)}, + FailOpen: &DatabaseFailOpenConfig{Enabled: true, UserId: "ops"}, + MaxWait: covDefaultsDur(5 * time.Second), + } + require.NoError(t, d.SetDefaults()) + assert.Equal(t, time.Minute, *d.Cache.TTL) + assert.Equal(t, int64(1), *d.Cache.MaxSize) + assert.Equal(t, 3, d.Retry.MaxAttempts, "non-positive attempts fall back to 3") + assert.Equal(t, covDefaultsDur(time.Second), d.Retry.BaseBackoff) + assert.Equal(t, "ops", d.FailOpen.UserId) + assert.Equal(t, covDefaultsDur(5*time.Second), d.MaxWait) + }) + t.Run("database strategy connector error wrapped", func(t *testing.T) { + s := &AuthStrategyConfig{Database: &DatabaseStrategyConfig{Connector: &ConnectorConfig{Driver: DriverPostgreSQL, PostgreSQL: &PostgreSQLConnectorConfig{}, Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}}}} + assert.ErrorContains(t, s.SetDefaults(), "failed to set defaults for database strategy") + a := &AuthConfig{Strategies: []*AuthStrategyConfig{s}} + assert.ErrorContains(t, a.SetDefaults(), "failed to set defaults for auth strategy") + }) +} + +func TestCovDefaults_RateLimiters(t *testing.T) { + r := &RateLimiterConfig{Store: &RateLimitStoreConfig{Driver: "redis"}, Budgets: []*RateLimitBudgetConfig{ + {Id: "b1", Rules: []*RateLimitRuleConfig{{Method: "eth_call", Period: RateLimitPeriodHour}, {Period: RateLimitPeriod(-3)}}}, + {Id: "b2"}, + }} + require.NoError(t, r.SetDefaults()) + assert.Equal(t, 8, r.Store.Redis.ConnPoolSize, "redis store gets connector defaults") + assert.Equal(t, "eth_call", r.Budgets[0].Rules[0].Method) + assert.Equal(t, RateLimitPeriodHour, r.Budgets[0].Rules[0].Period) + assert.Equal(t, "*", r.Budgets[0].Rules[1].Method) + assert.Equal(t, RateLimitPeriodSecond, r.Budgets[0].Rules[1].Period) + assert.Same(t, r.Budgets[1], (&Config{RateLimiters: r}).RateLimiterBudget("b2")) + assert.Nil(t, (&Config{}).RateLimiterBudget("b2")) + + mem := &RateLimiterConfig{} + require.NoError(t, mem.SetDefaults()) + assert.Equal(t, "memory", mem.Store.Driver) + assert.Nil(t, mem.Store.Redis) +} + +func TestCovDefaults_MisbehaviorsDestinationAndAutoTune(t *testing.T) { + f := &MisbehaviorsDestinationConfig{} + require.NoError(t, f.SetDefaults()) + assert.Equal(t, MisbehaviorsDestinationTypeFile, f.Type) + assert.Equal(t, "{timestampMs}-{method}-{networkId}", f.FilePattern) + assert.Nil(t, f.S3, "file destinations get no s3 block") + + s3 := &MisbehaviorsDestinationConfig{Type: MisbehaviorsDestinationTypeS3, FilePattern: "x"} + require.NoError(t, s3.SetDefaults()) + assert.Equal(t, "x", s3.FilePattern) + assert.Equal(t, &S3FlushConfig{MaxRecords: 100, MaxSize: 1024 * 1024, FlushInterval: covDefaultsDur(60 * time.Second), ContentType: "application/jsonl"}, s3.S3) + + s3e := &MisbehaviorsDestinationConfig{Type: MisbehaviorsDestinationTypeS3, S3: &S3FlushConfig{MaxRecords: 1, MaxSize: 2, FlushInterval: 3, ContentType: "text/plain", Region: "eu"}} + require.NoError(t, s3e.SetDefaults()) + assert.Equal(t, &S3FlushConfig{MaxRecords: 1, MaxSize: 2, FlushInterval: 3, ContentType: "text/plain", Region: "eu"}, s3e.S3) + + // Consensus applies the destination defaults to its own destination. + c := &ConsensusPolicyConfig{MisbehaviorsDestination: &MisbehaviorsDestinationConfig{Type: MisbehaviorsDestinationTypeS3}} + require.NoError(t, c.SetDefaults()) + assert.Equal(t, 100, c.MisbehaviorsDestination.S3.MaxRecords) + + at := &RateLimitAutoTuneConfig{} + require.NoError(t, at.SetDefaults()) + assert.True(t, *at.Enabled) + assert.Equal(t, covDefaultsDur(time.Minute), at.AdjustmentPeriod) + assert.Equal(t, 0.1, at.ErrorRateThreshold) + assert.Equal(t, 1.05, at.IncreaseFactor) + assert.Equal(t, 0.95, at.DecreaseFactor) + assert.Equal(t, 100000, at.MaxBudget) + + at2 := &RateLimitAutoTuneConfig{Enabled: util.BoolPtr(false), IncreaseFactor: 2, DecreaseFactor: 0.5, MaxBudget: 10} + require.NoError(t, at2.SetDefaults()) + assert.False(t, *at2.Enabled) + assert.Equal(t, 2.0, at2.IncreaseFactor) + assert.Equal(t, 10, at2.MaxBudget) +} diff --git a/common/cov_defaults_precedence_test.go b/common/cov_defaults_precedence_test.go new file mode 100644 index 000000000..3e82b306e --- /dev/null +++ b/common/cov_defaults_precedence_test.go @@ -0,0 +1,649 @@ +package common + +import ( + "net/url" + "testing" + "time" + + "github.com/erpc/erpc/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCovDefaults_BuildProviderSettings_Vendors(t *testing.T) { + cases := []struct { + endpoint string + want VendorSettings + }{ + {"alchemy://k1", VendorSettings{"apiKey": "k1"}}, + {"blastapi://k1", VendorSettings{"apiKey": "k1"}}, + {"drpc://k1", VendorSettings{"apiKey": "k1"}}, + {"envio://rpc.hypersync.example", VendorSettings{"rootDomain": "rpc.hypersync.example"}}, + {"etherspot://k1", VendorSettings{"apiKey": "k1"}}, + {"infura://k1", VendorSettings{"apiKey": "k1"}}, + {"llama://k1", VendorSettings{"apiKey": "k1"}}, + {"pimlico://k1", VendorSettings{"apiKey": "k1"}}, + {"thirdweb://cid", VendorSettings{"clientId": "cid"}}, + {"dwellir://k1", VendorSettings{"apiKey": "k1"}}, + {"conduit://k1", VendorSettings{"apiKey": "k1"}}, + {"superchain://github.com", VendorSettings{"registryUrl": "github.com"}}, + {"superchain://github.com/ethereum-optimism/registry", VendorSettings{"registryUrl": "github.com/ethereum-optimism/registry"}}, + {"tenderly://k1", VendorSettings{"apiKey": "k1"}}, + {"onfinality://k1", VendorSettings{"apiKey": "k1"}}, + {"blockpi://k1", VendorSettings{"apiKey": "k1"}}, + {"ankr://k1", VendorSettings{"apiKey": "k1"}}, + {"satelink://k1", VendorSettings{"apiKey": "k1"}}, + {"blockdaemon://k1", VendorSettings{"apiKey": "k1"}}, + {"erpc://erpc.example/main/evm", VendorSettings{"endpoint": "https://erpc.example/main/evm"}}, + {"erpc://erpc.example/main?secret=s3", VendorSettings{"endpoint": "https://erpc.example/main", "secret": "s3"}}, + {"erpc://erpc.example/main?other=1", VendorSettings{"endpoint": "https://erpc.example/main"}}, + {"repository://evm-public-endpoints.erpc.cloud/list?x=1", VendorSettings{"repositoryUrl": "https://evm-public-endpoints.erpc.cloud/list?x=1"}}, + {"routemesh://lb.example/rpc/1/secret-key", VendorSettings{"baseURL": "lb.example", "apiKey": "secret-key"}}, + {"quicknode://k1?tagIds=42", VendorSettings{"apiKey": "k1", "tagIds": 42}}, + {"quicknode://k1?tagIds=abc", VendorSettings{"apiKey": "k1"}}, + {"quicknode://k1?tagIds=a,b", VendorSettings{"apiKey": "k1"}}, + {"quicknode://k1?tagLabels=+prod+", VendorSettings{"apiKey": "k1", "tagLabels": "prod"}}, + {"chainstack://k1?region=a®ion=b", VendorSettings{"apiKey": "k1", "region": []string{"a", "b"}}}, + {"spectrum://team/key/?plan=pro&nodeType=archive", VendorSettings{"apiKey": "team/key", "plan": "pro", "nodeType": "archive"}}, + } + for _, tc := range cases { + t.Run(tc.endpoint, func(t *testing.T) { + u, err := url.Parse(tc.endpoint) + require.NoError(t, err) + got, err := buildProviderSettings(u.Scheme, u) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestCovDefaults_BuildProviderSettings_Errors(t *testing.T) { + cases := []struct { + endpoint string + want string + }{ + {"routemesh://lb.example/v1/1/k", "routemesh endpoint path must be in format /rpc//, got: /v1/1/k"}, + {"nosuchvendor://k", "unsupported vendor name in vendor.settings: nosuchvendor"}, + } + for _, tc := range cases { + u, err := url.Parse(tc.endpoint) + require.NoError(t, err) + _, err = buildProviderSettings(u.Scheme, u) + assert.EqualError(t, err, tc.want) + } + + bad := &url.URL{Scheme: "goldsky", Host: "s", RawQuery: "a=%zz"} + _, err := buildProviderSettings("goldsky", bad) + assert.ErrorContains(t, err, "failed to parse goldsky query parameters") + _, err = buildProviderSettings("quicknode", &url.URL{Host: "k", RawQuery: "a=%zz"}) + assert.ErrorContains(t, err, "failed to parse quicknode query parameters") + _, err = buildProviderSettings("chainstack", &url.URL{Host: "k", RawQuery: "a=%zz"}) + assert.ErrorContains(t, err, "failed to parse chainstack query parameters") + _, err = buildProviderSettings("spectrum", &url.URL{Host: "k", RawQuery: "a=%zz"}) + assert.ErrorContains(t, err, "failed to parse spectrum query parameters") + _, err = buildProviderSettings("erpc", &url.URL{Host: "k", RawQuery: "a=%zz"}) + assert.ErrorContains(t, err, "failed to parse erpc query parameters") +} + +func TestCovDefaults_ConvertUpstreamToProvider(t *testing.T) { + for _, ep := range []string{"http://a", "https://a", "ws://a", "wss://a", "grpc://a", "grpc+bds://a", "grpcs://a"} { + p, err := convertUpstreamToProvider(&UpstreamConfig{Endpoint: ep}) + require.NoError(t, err) + assert.Nil(t, p, "%s is a native endpoint, not a vendor shorthand", ep) + } + + u := &UpstreamConfig{Id: "my-alchemy", Endpoint: "evm+alchemy://KEY", RateLimitBudget: "rl", Evm: &EvmUpstreamConfig{ChainId: 1}} + p, err := convertUpstreamToProvider(u) + require.NoError(t, err) + assert.Equal(t, "my-alchemy", p.Id) + assert.Equal(t, "alchemy", p.Vendor, "evm+ prefix is stripped from the vendor") + assert.Equal(t, VendorSettings{"apiKey": "KEY"}, p.Settings) + assert.Equal(t, "-", p.UpstreamIdTemplate) + ov := p.Overrides["*"] + require.NotNil(t, ov) + assert.Empty(t, ov.Endpoint, "endpoint/id are left for the provider to fill") + assert.NotEmpty(t, ov.Id, "the override is defaulted, which assigns a generated id") + assert.NotEqual(t, "my-alchemy", ov.Id) + assert.Equal(t, "rl", ov.RateLimitBudget) + assert.Equal(t, int64(1), ov.Evm.ChainId) + + anon, err := convertUpstreamToProvider(&UpstreamConfig{Endpoint: "infura://K"}) + require.NoError(t, err) + assert.Contains(t, anon.Id, "infura-", "id-less shorthand gets a vendor-prefixed id") + + _, err = convertUpstreamToProvider(&UpstreamConfig{Id: "x", Endpoint: "nosuch://K"}) + assert.EqualError(t, err, "failed to convert upstream (id: x) to provider: unsupported vendor name in vendor.settings: nosuch") + _, err = convertUpstreamToProvider(&UpstreamConfig{Endpoint: "bad scheme://%%"}) + assert.ErrorContains(t, err, "invalid upstream endpoint while converting to provider") +} + +func TestCovDefaults_ProjectSetDefaults_ShorthandUpstreamBecomesProvider(t *testing.T) { + p := &ProjectConfig{ + Id: "p", + UpstreamDefaults: &UpstreamConfig{ + RateLimitBudget: "global", + Tags: []string{"pool:a"}, + }, + NetworkDefaults: &NetworkDefaults{ + RateLimitBudget: "net-budget", + Failsafe: []*FailsafeConfig{{Retry: &RetryPolicyConfig{MaxAttempts: 7}}}, + }, + Upstreams: []*UpstreamConfig{ + {Id: "plain", Endpoint: "https://node.example", RateLimitBudget: "own"}, + {Id: "drpc-up", Endpoint: "drpc://KEY"}, + }, + Networks: []*NetworkConfig{{Architecture: "evm", Evm: &EvmNetworkConfig{ChainId: 1}}}, + Auth: &AuthConfig{Strategies: []*AuthStrategyConfig{{Secret: &SecretStrategyConfig{Value: "x"}}}}, + CORS: &CORSConfig{}, + } + require.NoError(t, p.SetDefaults(nil)) + + require.Len(t, p.Upstreams, 1, "the vendor shorthand is moved out of upstreams") + plain := p.Upstreams[0] + assert.Equal(t, "own", plain.RateLimitBudget, "explicit upstream value wins over upstreamDefaults") + assert.Equal(t, []string{"pool:a"}, plain.Tags) + require.NotNil(t, plain.RateLimitAutoTune, "a budget implies auto-tune defaults") + assert.True(t, *plain.RateLimitAutoTune.Enabled) + + require.Len(t, p.Providers, 1) + assert.Equal(t, "drpc-up", p.Providers[0].Id) + assert.Equal(t, "drpc", p.Providers[0].Vendor) + assert.Equal(t, "global", p.Providers[0].Overrides["*"].RateLimitBudget) + + n := p.Networks[0] + assert.Equal(t, "net-budget", n.RateLimitBudget) + require.Len(t, n.Failsafe, 1) + assert.Equal(t, 7, n.Failsafe[0].Retry.MaxAttempts) + assert.Equal(t, AuthTypeSecret, p.Auth.Strategies[0].Type) + assert.Equal(t, 3600, p.CORS.MaxAge) +} + +func TestCovDefaults_ProjectSetDefaults_Errors(t *testing.T) { + redisBoth := &ConnectorConfig{Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}} + _ = redisBoth + badPolicy := &SelectionPolicyConfig{EvalScope: "nope"} + cases := []struct { + name string + p *ProjectConfig + want string + }{ + {"networkDefaults", &ProjectConfig{NetworkDefaults: &NetworkDefaults{SelectionPolicy: badPolicy}}, "failed to set defaults for network defaults"}, + {"upstream parse", &ProjectConfig{Upstreams: []*UpstreamConfig{{Endpoint: "://bad"}}}, "failed to set defaults for upstream: failed to parse endpoint"}, + {"upstream convert", &ProjectConfig{Upstreams: []*UpstreamConfig{{Id: "u", Endpoint: "nosuch://k"}}}, "failed to convert upstream (id: u) to provider"}, + {"endpoints option", &ProjectConfig{}, ""}, + {"network", &ProjectConfig{Upstreams: []*UpstreamConfig{{Id: "u", Endpoint: "https://a"}}, Networks: []*NetworkConfig{{SelectionPolicy: &SelectionPolicyConfig{EvalScope: "nope"}}}}, "failed to set defaults for network"}, + {"auth", &ProjectConfig{Upstreams: []*UpstreamConfig{{Id: "u", Endpoint: "https://a"}}, Auth: &AuthConfig{Strategies: []*AuthStrategyConfig{{Database: &DatabaseStrategyConfig{Connector: &ConnectorConfig{Redis: &RedisConnectorConfig{URI: "redis://a", Addr: "b"}}}}}}}, "failed to set defaults for auth"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if tc.want == "" { + err := tc.p.SetDefaults(&DefaultOptions{Endpoints: []string{"://bad"}}) + assert.ErrorContains(t, err, "failed to set defaults for upstream") + return + } + assert.ErrorContains(t, tc.p.SetDefaults(nil), tc.want) + }) + } +} + +func TestCovDefaults_UpstreamSetDefaults_IdsAndTypes(t *testing.T) { + u := &UpstreamConfig{Endpoint: "https://Rpc.CovPrec.example:8545/path"} + require.NoError(t, u.SetDefaults(nil)) + assert.Contains(t, u.Id, "Rpc.CovPrec.example-", "native endpoints derive the id from the host, port dropped") + assert.NotContains(t, u.Id, "8545") + assert.Equal(t, UpstreamTypeEvm, u.Type) + require.NotNil(t, u.Evm, "evm type gets an evm block") + assert.Equal(t, covDefaultsDur(30*time.Second), u.Evm.StatePollerInterval) + assert.Equal(t, EvmNodeTypeUnknown, u.Evm.NodeType) + assert.False(t, *u.Evm.SkipWhenSyncing) + assert.False(t, *u.Evm.SkipSyncingCheck) + assert.Nil(t, u.Evm.BlockAvailability) + assert.NotNil(t, u.JsonRpc) + assert.Nil(t, u.RateLimitAutoTune, "no budget, no auto-tune") + assert.Nil(t, u.IgnoreMethods) + + vendor := &UpstreamConfig{VendorName: "acme", Endpoint: "https://x"} + require.NoError(t, vendor.SetDefaults(nil)) + assert.Contains(t, vendor.Id, "acme-") + + scheme := &UpstreamConfig{Endpoint: "custom://x"} + require.NoError(t, scheme.SetDefaults(nil)) + assert.Contains(t, scheme.Id, "custom-", "non-native endpoints derive the id from the scheme") + + svm := &UpstreamConfig{Id: "s", Type: UpstreamTypeSvm, AllowMethods: []string{"getSlot"}} + require.NoError(t, svm.SetDefaults(nil)) + assert.Nil(t, svm.Evm, "non-evm types get no evm block") + assert.Equal(t, []string{"*"}, svm.IgnoreMethods, "allowMethods implies ignoring everything else") + + both := &UpstreamConfig{Id: "b", AllowMethods: []string{"a"}, IgnoreMethods: []string{"b"}} + require.NoError(t, both.SetDefaults(nil)) + assert.Equal(t, []string{"b"}, both.IgnoreMethods) +} + +func TestCovDefaults_UpstreamSetDefaults_FailsafeMatching(t *testing.T) { + defaults := &UpstreamConfig{ + Failsafe: []*FailsafeConfig{ + {MatchMethod: "eth_get*", Timeout: &TimeoutPolicyConfig{Duration: &AdaptiveDuration{Base: covDefaultsDur(2 * time.Second)}}}, + {MatchMethod: "*", Retry: &RetryPolicyConfig{MaxAttempts: 9}}, + }, + } + u := &UpstreamConfig{ + Id: "u", + Failsafe: []*FailsafeConfig{ + {MatchMethod: "eth_getLogs", Timeout: &TimeoutPolicyConfig{}}, + {MatchMethod: "debug_trace", Retry: &RetryPolicyConfig{}}, + }, + } + require.NoError(t, u.SetDefaults(defaults)) + require.NotNil(t, u.Failsafe[0].Timeout.Duration) + assert.Equal(t, covDefaultsDur(2*time.Second), u.Failsafe[0].Timeout.Duration.Base, "matched default fills omitted timeout") + assert.Equal(t, 9, u.Failsafe[1].Retry.MaxAttempts, "wildcard default matches any method") + + // Upstream with no failsafe of its own copies every default. + empty := &UpstreamConfig{Id: "e"} + require.NoError(t, empty.SetDefaults(defaults)) + require.Len(t, empty.Failsafe, 2) + assert.Equal(t, "eth_get*", empty.Failsafe[0].MatchMethod) + assert.NotSame(t, defaults.Failsafe[0], empty.Failsafe[0], "defaults are copied, not shared") + assert.Equal(t, 9, empty.Failsafe[1].Retry.MaxAttempts) +} + +func TestCovDefaults_UpstreamApplyDefaults_ExplicitWins(t *testing.T) { + ba := int64(500) + defaults := &UpstreamConfig{ + Endpoint: "https://default.example", + Type: UpstreamTypeEvm, + VendorName: "v", + Tags: []string{"pool:x"}, + RateLimitBudget: "d", + RateLimitAutoTune: &RateLimitAutoTuneConfig{MaxBudget: 5}, + AllowMethods: []string{"eth_call"}, + IgnoreMethods: []string{"debug_*"}, + AutoIgnoreUnsupportedMethods: util.BoolPtr(true), + JsonRpc: &JsonRpcUpstreamConfig{BatchMaxSize: 50, SupportsBatch: util.BoolPtr(true)}, + Grpc: &GrpcUpstreamConfig{Headers: map[string]string{"h": "v"}}, + Routing: &UpstreamRoutingConfig{ScoreMultipliers: []*ScoreMultiplierConfig{{Method: "*"}}}, + Svm: &SvmUpstreamConfig{Chain: "fogo", Cluster: "mainnet", CheckGenesisHash: true}, + Evm: &EvmUpstreamConfig{ + ChainId: 1, + NodeType: EvmNodeTypeArchive, + StatePollerInterval: covDefaultsDur(5 * time.Second), + StatePollerDebounce: covDefaultsDur(time.Second), + MaxAvailableRecentBlocks: ba, + GetLogsAutoSplittingRangeThreshold: 1000, + TraceFilterAutoSplittingRangeThreshold: 200, + DeprecatedIntegrity: &UpstreamIntegrityConfig{}, + }, + } + + t.Run("empty upstream inherits everything", func(t *testing.T) { + u := &UpstreamConfig{} + require.NoError(t, u.ApplyDefaults(defaults)) + assert.Equal(t, "https://default.example", u.Endpoint) + assert.Equal(t, "v", u.VendorName) + assert.Equal(t, []string{"pool:x"}, u.Tags) + assert.Equal(t, "d", u.RateLimitBudget) + assert.Same(t, defaults.RateLimitAutoTune, u.RateLimitAutoTune) + assert.Equal(t, []string{"eth_call"}, u.AllowMethods) + assert.Equal(t, []string{"debug_*"}, u.IgnoreMethods) + assert.True(t, *u.AutoIgnoreUnsupportedMethods) + assert.Equal(t, 50, u.JsonRpc.BatchMaxSize) + assert.Equal(t, "v", u.Grpc.Headers["h"]) + u.Grpc.Headers["h"] = "mutated" + assert.Equal(t, "v", defaults.Grpc.Headers["h"], "grpc block is deep-copied") + require.NotNil(t, u.Routing) + assert.NotSame(t, defaults.Routing, u.Routing) + assert.Len(t, u.Routing.ScoreMultipliers, 1) + assert.Equal(t, "fogo", u.Svm.Chain) + assert.NotSame(t, defaults.Svm, u.Svm) + assert.Equal(t, int64(1), u.Evm.ChainId) + assert.Equal(t, EvmNodeTypeArchive, u.Evm.NodeType) + assert.Equal(t, int64(500), u.Evm.MaxAvailableRecentBlocks) + require.NotNil(t, u.Evm.BlockAvailability, "legacy window maps onto blockAvailability") + assert.Equal(t, int64(500), *u.Evm.BlockAvailability.Lower.LatestBlockMinus) + assert.NotNil(t, u.Evm.DeprecatedIntegrity) + }) + + t.Run("partial upstream keeps its own values", func(t *testing.T) { + u := &UpstreamConfig{ + Endpoint: "https://mine", + Tags: []string{"own"}, + RateLimitBudget: "mine", + AllowMethods: []string{}, + JsonRpc: &JsonRpcUpstreamConfig{BatchMaxSize: 1}, + Svm: &SvmUpstreamConfig{}, + Evm: &EvmUpstreamConfig{ChainId: 10}, + } + require.NoError(t, u.ApplyDefaults(defaults)) + assert.Equal(t, "https://mine", u.Endpoint) + assert.Equal(t, []string{"own"}, u.Tags) + assert.Equal(t, "mine", u.RateLimitBudget) + assert.Equal(t, []string{}, u.AllowMethods) + assert.Equal(t, 1, u.JsonRpc.BatchMaxSize) + assert.Equal(t, "fogo", u.Svm.Chain, "empty svm fields fill from defaults") + assert.Equal(t, "mainnet", u.Svm.Cluster) + assert.True(t, u.Svm.CheckGenesisHash) + assert.Equal(t, int64(10), u.Evm.ChainId) + assert.Equal(t, covDefaultsDur(5*time.Second), u.Evm.StatePollerInterval) + assert.Equal(t, covDefaultsDur(time.Second), u.Evm.StatePollerDebounce) + assert.Equal(t, int64(500), u.Evm.MaxAvailableRecentBlocks) + assert.Equal(t, int64(1000), u.Evm.GetLogsAutoSplittingRangeThreshold) + assert.Equal(t, int64(200), u.Evm.TraceFilterAutoSplittingRangeThreshold) + assert.NotNil(t, u.Evm.DeprecatedIntegrity) + }) + + t.Run("explicit blockAvailability in defaults suppresses legacy window", func(t *testing.T) { + lbm := int64(7) + d := &UpstreamConfig{Evm: &EvmUpstreamConfig{MaxAvailableRecentBlocks: 99, BlockAvailability: &EvmBlockAvailabilityConfig{Lower: &EvmAvailabilityBoundConfig{LatestBlockMinus: &lbm}}}} + u := &UpstreamConfig{} + require.NoError(t, u.ApplyDefaults(d)) + assert.Zero(t, u.Evm.MaxAvailableRecentBlocks) + assert.Equal(t, int64(7), *u.Evm.BlockAvailability.Lower.LatestBlockMinus) + assert.NotSame(t, d.Evm.BlockAvailability, u.Evm.BlockAvailability) + + partial := &UpstreamConfig{Evm: &EvmUpstreamConfig{}} + require.NoError(t, partial.ApplyDefaults(d)) + assert.Equal(t, int64(7), *partial.Evm.BlockAvailability.Lower.LatestBlockMinus) + assert.Zero(t, partial.Evm.MaxAvailableRecentBlocks) + }) + + t.Run("nil defaults is a no-op", func(t *testing.T) { + u := &UpstreamConfig{Endpoint: "x"} + require.NoError(t, u.ApplyDefaults(nil)) + assert.Equal(t, &UpstreamConfig{Endpoint: "x"}, u) + }) +} + +func TestCovDefaults_EvmUpstreamSetDefaults(t *testing.T) { + full := &EvmUpstreamConfig{NodeType: EvmNodeTypeFull} + require.NoError(t, full.SetDefaults(nil)) + assert.Equal(t, int64(128), full.MaxAvailableRecentBlocks, "full nodes default to a 128-block window") + assert.Equal(t, int64(128), *full.BlockAvailability.Lower.LatestBlockMinus) + + lbm := int64(3) + defaults := &EvmUpstreamConfig{ + StatePollerDebounce: covDefaultsDur(2 * time.Second), + NodeType: EvmNodeTypeFull, + BlockAvailability: &EvmBlockAvailabilityConfig{Lower: &EvmAvailabilityBoundConfig{LatestBlockMinus: &lbm}}, + SkipWhenSyncing: util.BoolPtr(true), + SkipSyncingCheck: util.BoolPtr(true), + PinLatestTag: util.BoolPtr(true), + Multicall3: &EvmMulticall3Config{MaxCalls: 5}, + } + e := &EvmUpstreamConfig{} + require.NoError(t, e.SetDefaults(defaults)) + assert.Equal(t, covDefaultsDur(2*time.Second), e.StatePollerDebounce) + assert.Equal(t, EvmNodeTypeFull, e.NodeType) + assert.Equal(t, int64(3), *e.BlockAvailability.Lower.LatestBlockMinus, "inherited window beats the full-node 128") + assert.Zero(t, e.MaxAvailableRecentBlocks) + assert.True(t, *e.SkipWhenSyncing) + assert.True(t, *e.SkipSyncingCheck) + assert.True(t, *e.PinLatestTag) + assert.Equal(t, 5, e.Multicall3.MaxCalls) + assert.Equal(t, covDefaultsDur(10*time.Millisecond), e.Multicall3.MaxWait) + assert.NotSame(t, defaults.Multicall3, e.Multicall3) + + legacy := &EvmUpstreamConfig{} + require.NoError(t, legacy.SetDefaults(&EvmUpstreamConfig{MaxAvailableRecentBlocks: 64})) + assert.Equal(t, int64(64), legacy.MaxAvailableRecentBlocks) + assert.Equal(t, int64(64), *legacy.BlockAvailability.Lower.LatestBlockMinus) + + own := &EvmUpstreamConfig{Multicall3: &EvmMulticall3Config{MaxWait: 1, MaxCalls: 2}} + require.NoError(t, own.SetDefaults(nil)) + assert.Equal(t, Duration(1), own.Multicall3.MaxWait) + assert.Equal(t, 2, own.Multicall3.MaxCalls) +} + +func TestCovDefaults_NetworkSetDefaults_NetworkDefaultsPrecedence(t *testing.T) { + dbt, fdm, bud := 2.0, 3.0, 4.0 + defaults := &NetworkDefaults{ + RateLimitBudget: "nd", + CacheKeySuffix: "sfx", + SelectionPolicy: &SelectionPolicyConfig{EvalInterval: covDefaultsDur(time.Minute)}, + DirectiveDefaults: &DirectiveDefaultsConfig{EnforceHighestBlock: util.BoolPtr(false)}, + Multiplexing: util.BoolPtr(false), + Failover: &FailoverConfig{OnDefaultsExhausted: util.BoolPtr(true)}, + Evm: &EvmNetworkConfig{ + Integrity: &EvmIntegrityConfig{EnforceGetLogsBlockRange: util.BoolPtr(false)}, + FallbackStatePollerDebounce: covDefaultsDur(3 * time.Second), + DynamicBlockTimeDebounceMultiplier: &dbt, + FinalizedDebounceDepthMultiplier: &fdm, + BlockUnavailableDelayMultiplier: &bud, + FallbackFinalityDepth: 77, + GetLogsMaxAllowedAddresses: 11, + GetLogsMaxAllowedTopics: 12, + GetLogsSplitOnError: util.BoolPtr(false), + GetLogsSplitConcurrency: 3, + GetLogsMaxResponseBytes: 1024, + TraceFilterSplitOnError: util.BoolPtr(true), + TraceFilterSplitConcurrency: 4, + ServedTip: &EvmServedTipConfig{Quorum: 3}, + EmptyResultConfidence: AvailbilityConfidenceFinalized, + ShortCircuitFutureBlocks: util.BoolPtr(true), + StripSubscribeFromBlockZero: util.BoolPtr(true), + LocalAnswers: map[string]bool{"eth_chainId": true, "net_version": false}, + }, + } + + n := &NetworkConfig{Evm: &EvmNetworkConfig{ChainId: 1, LocalAnswers: map[string]bool{"eth_chainId": false}}} + require.NoError(t, n.SetDefaults(nil, defaults)) + assert.Equal(t, NetworkArchitecture("evm"), n.Architecture) + assert.Equal(t, "nd", n.RateLimitBudget) + assert.Equal(t, "sfx", n.CacheKeySuffix) + assert.Equal(t, covDefaultsDur(time.Minute), n.SelectionPolicy.EvalInterval) + assert.NotSame(t, defaults.SelectionPolicy, n.SelectionPolicy) + assert.False(t, *n.DirectiveDefaults.EnforceHighestBlock, "networkDefaults directive wins over the built-in true") + assert.False(t, *n.DirectiveDefaults.EnforceGetLogsBlockRange, "deprecated evm.integrity migrates into directives") + assert.True(t, *n.DirectiveDefaults.EnforceNonNullTaggedBlocks) + assert.False(t, n.MultiplexingEnabled()) + assert.True(t, n.Failover.Enabled()) + *defaults.Failover.OnDefaultsExhausted = false + assert.True(t, n.Failover.Enabled(), "failover is cloned, not aliased") + *defaults.Failover.OnDefaultsExhausted = true + + e := n.Evm + assert.Equal(t, covDefaultsDur(3*time.Second), e.FallbackStatePollerDebounce) + assert.Equal(t, 2.0, *e.DynamicBlockTimeDebounceMultiplier) + assert.Equal(t, 3.0, *e.FinalizedDebounceDepthMultiplier) + assert.Equal(t, 4.0, *e.BlockUnavailableDelayMultiplier) + assert.Equal(t, int64(77), e.FallbackFinalityDepth) + assert.Equal(t, int64(11), e.GetLogsMaxAllowedAddresses) + assert.Equal(t, int64(12), e.GetLogsMaxAllowedTopics) + assert.False(t, *e.GetLogsSplitOnError) + assert.Equal(t, 3, e.GetLogsSplitConcurrency) + assert.Equal(t, int64(1024), e.GetLogsMaxResponseBytes) + assert.True(t, *e.TraceFilterSplitOnError) + assert.Equal(t, 4, e.TraceFilterSplitConcurrency) + assert.Equal(t, ServedTipQuorum(3), e.ServedTipQuorum()) + assert.NotSame(t, defaults.Evm.ServedTip, e.ServedTip) + assert.Equal(t, AvailbilityConfidenceFinalized, e.EmptyResultConfidence) + assert.True(t, e.ShortCircuitFutureBlocksEnabled()) + assert.True(t, *e.StripSubscribeFromBlockZero) + assert.Equal(t, map[string]bool{"eth_chainId": false, "net_version": false}, e.LocalAnswers, "network's own localAnswers entry wins") + assert.Equal(t, int64(30_000), e.GetLogsMaxAllowedRange, "fields absent from both fall to built-ins") + + // Explicit network values are never overwritten. + own := &NetworkConfig{ + RateLimitBudget: "own", + CacheKeySuffix: "mine", + Multiplexing: util.BoolPtr(true), + Evm: &EvmNetworkConfig{ChainId: 2, GetLogsSplitConcurrency: 99, FallbackFinalityDepth: 5}, + } + require.NoError(t, own.SetDefaults(nil, defaults)) + assert.Equal(t, "own", own.RateLimitBudget) + assert.Equal(t, "mine", own.CacheKeySuffix) + assert.True(t, own.MultiplexingEnabled()) + assert.Equal(t, 99, own.Evm.GetLogsSplitConcurrency) + assert.Equal(t, int64(5), own.Evm.FallbackFinalityDepth) + assert.Equal(t, map[string]bool{"eth_chainId": true, "net_version": false}, own.Evm.LocalAnswers) +} + +func TestCovDefaults_NetworkSetDefaults_ArchitectureAndSvm(t *testing.T) { + lag := int64(0) + defaults := &NetworkDefaults{ + Evm: &EvmNetworkConfig{GetLogsMaxAllowedRange: 5}, + Svm: &SvmNetworkConfig{Chain: "fogo", Commitment: "confirmed", StatePollerDebounce: covDefaultsDur(time.Second), MaxFinalizedSlotLag: &lag, EnforceBlockAvailability: util.BoolPtr(false), Cluster: "ignored"}, + } + + evm := &NetworkConfig{} + require.NoError(t, evm.SetDefaults(nil, defaults)) + assert.Equal(t, NetworkArchitecture("evm"), evm.Architecture, "an empty network adopts the evm defaults block") + assert.Equal(t, int64(5), evm.Evm.GetLogsMaxAllowedRange) + + svm := &NetworkConfig{Architecture: ArchitectureSvm} + require.NoError(t, svm.SetDefaults(nil, defaults)) + assert.Nil(t, svm.Evm, "svm networks never get the evm defaults block") + require.NotNil(t, svm.Svm) + assert.Equal(t, covDefaultsDur(400*time.Millisecond), svm.Svm.StatePollerDebounce, "an svm block created after the merge only gets built-ins") + + svm2 := &NetworkConfig{Svm: &SvmNetworkConfig{Cluster: "devnet"}} + require.NoError(t, svm2.SetDefaults(nil, defaults)) + assert.Equal(t, ArchitectureSvm, svm2.Architecture) + assert.Equal(t, "devnet", svm2.Svm.Cluster, "cluster is identity and never inherited") + assert.Equal(t, "fogo", svm2.Svm.Chain) + assert.Equal(t, "confirmed", svm2.Svm.Commitment) + assert.Equal(t, covDefaultsDur(time.Second), svm2.Svm.StatePollerDebounce) + assert.Equal(t, int64(0), *svm2.Svm.MaxFinalizedSlotLag, "explicit zero lag in defaults is kept") + assert.NotSame(t, defaults.Svm.MaxFinalizedSlotLag, svm2.Svm.MaxFinalizedSlotLag) + assert.False(t, *svm2.Svm.EnforceBlockAvailability) + + bare := &NetworkConfig{Architecture: ArchitectureSvm} + require.NoError(t, bare.SetDefaults(nil, nil)) + assert.Equal(t, MaxShredInsertSlotLagThreshold, *bare.Svm.MaxFinalizedSlotLag) + + bareEvm := &NetworkConfig{Architecture: "evm"} + require.NoError(t, bareEvm.SetDefaults(nil, nil)) + require.NotNil(t, bareEvm.Evm) + assert.Equal(t, int64(DefaultEvmFinalityDepth), bareEvm.Evm.FallbackFinalityDepth) + assert.True(t, *bareEvm.Evm.Integrity.EnforceHighestBlock) + assert.NotNil(t, bareEvm.Methods) +} + +func TestCovDefaults_NetworkSetDefaults_FailsafeMergeAndErrors(t *testing.T) { + defaults := &NetworkDefaults{Failsafe: []*FailsafeConfig{ + {MatchMethod: "eth_*", MatchCommitment: []string{"confirmed", "finalized"}, Retry: &RetryPolicyConfig{MaxAttempts: 4}}, + }} + n := &NetworkConfig{Failsafe: []*FailsafeConfig{ + {MatchMethod: "eth_call", MatchCommitment: []string{"finalized"}, Retry: &RetryPolicyConfig{}}, + {MatchMethod: "eth_call", MatchCommitment: []string{"processed"}, Retry: &RetryPolicyConfig{}}, + }} + require.NoError(t, n.SetDefaults(nil, defaults)) + assert.Equal(t, 4, n.Failsafe[0].Retry.MaxAttempts, "default covering the rule's commitments applies") + assert.Equal(t, 3, n.Failsafe[1].Retry.MaxAttempts, "uncovered commitment falls back to built-ins") + + noDefaults := &NetworkConfig{Failsafe: []*FailsafeConfig{{Retry: &RetryPolicyConfig{}}}} + require.NoError(t, noDefaults.SetDefaults(nil, &NetworkDefaults{})) + assert.Equal(t, 3, noDefaults.Failsafe[0].Retry.MaxAttempts) + nd2 := &NetworkDefaults{SelectionPolicy: &SelectionPolicyConfig{EvalScope: "x"}} + assert.ErrorContains(t, nd2.SetDefaults(), "failed to set defaults for selection policy") + badNet := &NetworkConfig{SelectionPolicy: &SelectionPolicyConfig{EvalScope: "x"}} + assert.ErrorContains(t, badNet.SetDefaults(nil, nil), "failed to set defaults for selection policy") +} + +func TestCovDefaults_NetworkDefaultsSetDefaults_FillsNested(t *testing.T) { + nd := &NetworkDefaults{ + Failsafe: []*FailsafeConfig{{Hedge: &HedgePolicyConfig{}}}, + SelectionPolicy: &SelectionPolicyConfig{}, + Evm: &EvmNetworkConfig{}, + Svm: &SvmNetworkConfig{}, + DirectiveDefaults: &DirectiveDefaultsConfig{EnforceHighestBlock: util.BoolPtr(false)}, + } + require.NoError(t, nd.SetDefaults()) + assert.Equal(t, "*", nd.Failsafe[0].MatchMethod) + assert.Equal(t, 1, nd.Failsafe[0].Hedge.MaxCount) + assert.Equal(t, DefaultSelectionPolicySource, nd.SelectionPolicy.EvalFunc) + assert.NotNil(t, nd.SelectionPolicy.CompiledProgram) + assert.Equal(t, int64(30_000), nd.Evm.GetLogsMaxAllowedRange) + assert.Equal(t, covDefaultsDur(400*time.Millisecond), nd.Svm.StatePollerDebounce) + assert.False(t, *nd.DirectiveDefaults.EnforceHighestBlock) + assert.True(t, *nd.DirectiveDefaults.EnforceGetLogsBlockRange) + + var nilDD *DirectiveDefaultsConfig + assert.NoError(t, nilDD.SetDefaults()) + var nilSvm *SvmNetworkConfig + assert.NoError(t, nilSvm.SetDefaults()) +} + +func TestCovDefaults_FailsafeSetDefaults_PolicyPrecedence(t *testing.T) { + defaults := &FailsafeConfig{ + MatchMethod: "eth_*", + Timeout: &TimeoutPolicyConfig{Duration: &AdaptiveDuration{Base: covDefaultsDur(time.Second)}}, + Retry: &RetryPolicyConfig{MaxAttempts: 5, BackoffFactor: 2, BackoffMaxDelay: 7, Delay: 8, Jitter: 9, EmptyResultMaxAttempts: 4, EmptyResultDelay: 6, EmptyResultIgnore: []string{"eth_getLogs"}}, + Hedge: &HedgePolicyConfig{MaxCount: 3, Delay: &AdaptiveDuration{Base: covDefaultsDur(50 * time.Millisecond)}}, + CircuitBreaker: &CircuitBreakerPolicyConfig{FailureThresholdCount: 1, FailureThresholdCapacity: 2, SuccessThresholdCount: 3, SuccessThresholdCapacity: 4, HalfOpenAfter: 5}, + } + + t.Run("absent policies are created from defaults", func(t *testing.T) { + f := &FailsafeConfig{} + require.NoError(t, f.SetDefaults(defaults)) + assert.Equal(t, "eth_*", f.MatchMethod) + assert.Equal(t, covDefaultsDur(time.Second), f.Timeout.Duration.Base) + assert.NotSame(t, defaults.Timeout.Duration, f.Timeout.Duration) + assert.Equal(t, RetryPolicyConfig{MaxAttempts: 5, BackoffFactor: 2, BackoffMaxDelay: 7, Delay: 8, Jitter: 9, EmptyResultMaxAttempts: 4, EmptyResultDelay: 6, EmptyResultAccept: []string{"eth_getLogs"}}, *f.Retry) + assert.Equal(t, 3, f.Hedge.MaxCount) + assert.Equal(t, covDefaultsDur(50*time.Millisecond), f.Hedge.Delay.Base) + assert.Equal(t, covDefaultsDur(100*time.Millisecond), f.Hedge.Delay.Min) + assert.Equal(t, CircuitBreakerPolicyConfig{FailureThresholdCount: 1, FailureThresholdCapacity: 2, SuccessThresholdCount: 3, SuccessThresholdCapacity: 4, HalfOpenAfter: 5}, *f.CircuitBreaker) + }) + + t.Run("present policies keep explicit fields", func(t *testing.T) { + f := &FailsafeConfig{ + MatchMethod: "eth_call", + Timeout: &TimeoutPolicyConfig{Duration: &AdaptiveDuration{Base: covDefaultsDur(3 * time.Second)}}, + Retry: &RetryPolicyConfig{MaxAttempts: 1, EmptyResultAccept: []string{"x"}}, + Hedge: &HedgePolicyConfig{MaxCount: 9}, + CircuitBreaker: &CircuitBreakerPolicyConfig{FailureThresholdCount: 50}, + } + require.NoError(t, f.SetDefaults(defaults)) + assert.Equal(t, "eth_call", f.MatchMethod) + assert.Equal(t, covDefaultsDur(3*time.Second), f.Timeout.Duration.Base) + assert.Equal(t, 1, f.Retry.MaxAttempts) + assert.Equal(t, float32(2), f.Retry.BackoffFactor) + assert.Equal(t, []string{"x"}, f.Retry.EmptyResultAccept) + assert.Equal(t, 9, f.Hedge.MaxCount) + assert.Equal(t, uint(50), f.CircuitBreaker.FailureThresholdCount) + assert.Equal(t, uint(2), f.CircuitBreaker.FailureThresholdCapacity) + }) + + t.Run("built-ins without defaults", func(t *testing.T) { + f := &FailsafeConfig{Timeout: &TimeoutPolicyConfig{}, Hedge: &HedgePolicyConfig{}, CircuitBreaker: &CircuitBreakerPolicyConfig{}, Consensus: &ConsensusPolicyConfig{}} + require.NoError(t, f.SetDefaults(nil)) + assert.Equal(t, "*", f.MatchMethod) + assert.Nil(t, f.Timeout.Duration, "timeout has no built-in duration") + assert.Equal(t, 1, f.Hedge.MaxCount) + assert.Equal(t, covDefaultsDur(999*time.Second), f.Hedge.Delay.Max) + assert.Equal(t, CircuitBreakerPolicyConfig{FailureThresholdCount: 20, FailureThresholdCapacity: 80, SuccessThresholdCount: 8, SuccessThresholdCapacity: 200, HalfOpenAfter: covDefaultsDur(5 * time.Minute)}, *f.CircuitBreaker) + assert.Equal(t, 5, f.Consensus.MaxParticipants) + }) + + t.Run("retry ignore alias and deprecated delay", func(t *testing.T) { + r := &RetryPolicyConfig{EmptyResultIgnore: []string{"m"}} + require.NoError(t, r.SetDefaults(nil)) + assert.Equal(t, []string{"m"}, r.EmptyResultAccept) + r2 := &RetryPolicyConfig{} + require.NoError(t, r2.SetDefaults(&RetryPolicyConfig{EmptyResultAccept: []string{"a"}})) + assert.Equal(t, []string{"a"}, r2.EmptyResultAccept) + }) +} + +func TestCovDefaults_SelectionPolicyAndEvmNetworkDefaults(t *testing.T) { + s := &SelectionPolicyConfig{EvalFunc: "this is ( not js"} + assert.ErrorContains(t, s.SetDefaults(), "failed to compile selectionPolicy.evalFunc") + + ts := &SelectionPolicyConfig{EvalFunc: tsFunctionSentinelPrefix + "7"} + require.NoError(t, ts.SetDefaults()) + assert.Nil(t, ts.CompiledProgram, "TS sentinels resolve at runtime, never compiled") + assert.Equal(t, tsFunctionSentinelPrefix+"7", ts.EvalFuncOriginal) + assert.True(t, IsTSFunctionSentinel(ts.EvalFunc)) + assert.Equal(t, "7", TSFunctionSentinelID(ts.EvalFunc)) + + dist := int64(5) + e := &EvmNetworkConfig{MaxFutureBlockRetryDistance: &dist, MarkEmptyAsErrorMethods: []string{}} + require.NoError(t, e.SetDefaults()) + assert.Nil(t, e.MaxFutureBlockRetryDistance, "deprecated field is cleared") + assert.Equal(t, []string{}, e.MarkEmptyAsErrorMethods, "explicit empty list is kept") + assert.Equal(t, AvailbilityConfidenceBlockHead, e.EmptyResultConfidence) + assert.Equal(t, DefaultEvmStatePollerDebounce, e.FallbackStatePollerDebounce) + assert.Equal(t, 10, e.TraceFilterSplitConcurrency) + assert.Nil(t, e.TraceFilterSplitOnError) +} diff --git a/common/cov_errors_codes_test.go b/common/cov_errors_codes_test.go new file mode 100644 index 000000000..42050ebe2 --- /dev/null +++ b/common/cov_errors_codes_test.go @@ -0,0 +1,432 @@ +package common + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var covErrorsSentinel = errors.New("cov-errors sentinel cause") + +type covErrorsCtorCase struct { + name string + err error + code ErrorCode + // status is the ErrorStatusCode() result; 0 means the type must not expose one. + status int + msg string + details map[string]interface{} + // wrapsSentinel asserts errors.Is(err, covErrorsSentinel) through the Cause chain. + wrapsSentinel bool +} + +func covErrorsCtorCases() []covErrorsCtorCase { + up := NewFakeUpstream("up-1") + return []covErrorsCtorCase{ + {name: "InvalidRequest", err: NewErrInvalidRequest(covErrorsSentinel), code: ErrCodeInvalidRequest, status: http.StatusBadRequest, msg: "invalid request body or headers", details: map[string]interface{}{"retryableTowardNetwork": false}, wrapsSentinel: true}, + {name: "InvalidUrlPath", err: NewErrInvalidUrlPath("missing project", "/x/y"), code: ErrCodeInvalidUrlPath, status: http.StatusBadRequest, msg: "url path is not valid: missing project", details: map[string]interface{}{"providedPath": "/x/y"}}, + {name: "InvalidConfig", err: NewErrInvalidConfig("bad cfg"), code: "ErrInvalidConfig", msg: "bad cfg"}, + {name: "RequestTimeout", err: NewErrRequestTimeout(1500 * time.Millisecond), code: "ErrRequestTimeout", status: http.StatusGatewayTimeout, msg: "request timed out", details: map[string]interface{}{"timeoutSeconds": 1.5}}, + {name: "InternalServerError", err: NewErrInternalServerError(covErrorsSentinel), code: "ErrInternalServerError", msg: "internal server error", wrapsSentinel: true}, + {name: "AuthUnauthorized", err: NewErrAuthUnauthorized("secret", "bad_token", "token is bad"), code: ErrCodeAuthUnauthorized, status: http.StatusUnauthorized, msg: "token is bad", details: map[string]interface{}{"strategy": "secret", "reason": "bad_token"}}, + {name: "AuthUnauthorizedReason", err: NewErrAuthUnauthorizedReason("jwt", AuthUnauthorizedReasonInvalidSignature), code: ErrCodeAuthUnauthorized, status: http.StatusUnauthorized, msg: "invalid signature", details: map[string]interface{}{"strategy": "jwt", "reason": "invalid_signature"}}, + {name: "AuthRateLimitRuleExceeded", err: NewErrAuthRateLimitRuleExceeded("prj", "jwt", "b1", "r1", "user-1", "1.2.3.4"), code: ErrCodeAuthRateLimitRuleExceeded, status: http.StatusTooManyRequests, msg: "auth-level rate limit rule exceeded", details: map[string]interface{}{"projectId": "prj", "strategy": "jwt", "budget": "b1", "rule": "r1", "userId": "user-1", "clientIp": "1.2.3.4"}}, + {name: "ProjectNotFound", err: NewErrProjectNotFound("main"), code: ErrCodeProjectNotFound, status: http.StatusNotFound, msg: "project 'main' not configured"}, + {name: "ProjectAlreadyExists", err: NewErrProjectAlreadyExists("main"), code: "ErrProjectAlreadyExists", msg: "project already exists", details: map[string]interface{}{"projectId": "main"}}, + {name: "NetworkNotFound", err: NewErrNetworkNotFound("evm:1"), code: ErrCodeNetworkNotFound, status: http.StatusNotFound, msg: "network not configured", details: map[string]interface{}{"networkId": "evm:1"}}, + {name: "UnknownNetworkID", err: NewErrUnknownNetworkID(ArchitectureEvm), code: "ErrUnknownNetworkID", status: http.StatusBadRequest, msg: "could not resolve network ID", details: map[string]interface{}{"architecture": ArchitectureEvm}}, + {name: "UnknownNetworkArchitecture", err: NewErrUnknownNetworkArchitecture("cosmos"), code: "ErrUnknownNetworkArchitecture", status: http.StatusBadRequest, msg: "unknown network architecture", details: map[string]interface{}{"architecture": NetworkArchitecture("cosmos")}}, + {name: "NotImplemented", err: NewErrNotImplemented("eth_sign is not supported"), code: ErrCodeNotImplemented, status: http.StatusNotImplemented, msg: "eth_sign is not supported"}, + {name: "InvalidEvmChainId", err: NewErrInvalidEvmChainId("abc"), code: "ErrInvalidEvmChainId", status: http.StatusBadRequest, msg: "invalid EVM chain ID", details: map[string]interface{}{"chainId": "abc"}}, + {name: "FinalizedBlockUnavailable", err: NewErrFinalizedBlockUnavailable(42), code: ErrCodeFinalizedBlockUnavailable, msg: "finalized/latest blocks are not available", details: map[string]interface{}{"blockNumber": int64(42)}}, + {name: "UpstreamClientInitialization", err: NewErrUpstreamClientInitialization(covErrorsSentinel, up), code: "ErrUpstreamClientInitialization", msg: "could not initialize upstream client", details: map[string]interface{}{"upstreamId": "up-1"}, wrapsSentinel: true}, + {name: "UpstreamMalformedResponse", err: NewErrUpstreamMalformedResponse(covErrorsSentinel, up), code: "ErrUpstreamMalformedResponse", status: http.StatusBadRequest, msg: "malformed response from upstream", details: map[string]interface{}{"upstreamId": "up-1"}, wrapsSentinel: true}, + {name: "NoUpstreamsDefined", err: NewErrNoUpstreamsDefined("main"), code: "ErrNoUpstreamsDefined", status: http.StatusNotFound, msg: "no upstreams defined for project", details: map[string]interface{}{"project": "main"}}, + {name: "NoUpstreamsFound", err: NewErrNoUpstreamsFound("main", "evm:1"), code: "ErrNoUpstreamsFound", status: http.StatusNotFound, msg: "no upstreams found for network 'evm:1' and project 'main'", details: map[string]interface{}{"project": "main", "network": "evm:1"}}, + {name: "NetworkInitializing", err: NewErrNetworkInitializing("main", "evm:1"), code: ErrCodeNetworkInitializing, status: http.StatusServiceUnavailable, msg: "network 'evm:1' for project 'main' is initializing", details: map[string]interface{}{"project": "main", "network": "evm:1"}}, + {name: "NetworkNoUpstreamsAvailable", err: NewErrNetworkNoUpstreamsAvailable("main", "evm:1"), code: ErrCodeNetworkNoUpstreamsAvailable, status: http.StatusNotFound, msg: "no RPC providers are available for network 'evm:1' in project 'main'", details: map[string]interface{}{"project": "main", "network": "evm:1"}}, + {name: "NetworkNotSupported", err: NewErrNetworkNotSupported("main", "evm:1"), code: ErrCodeNetworkNotSupported, status: http.StatusNotFound, msg: "network 'evm:1' is not supported for project 'main'", details: map[string]interface{}{"project": "main", "network": "evm:1"}}, + {name: "UpstreamNetworkNotDetected", err: NewErrUpstreamNetworkNotDetected("main", up), code: "ErrUpstreamNetworkNotDetected", msg: "network not detected for upstream", details: map[string]interface{}{"projectId": "main", "upstreamId": "up-1"}}, + {name: "UpstreamInitialization", err: NewErrUpstreamInitialization(covErrorsSentinel, "up-1"), code: "ErrUpstreamInitialization", msg: "failed to initialize upstream", details: map[string]interface{}{"upstreamId": "up-1"}, wrapsSentinel: true}, + {name: "UpstreamRequestSkipped", err: NewErrUpstreamRequestSkipped(covErrorsSentinel, "up-1"), code: ErrCodeUpstreamRequestSkipped, status: http.StatusNotAcceptable, msg: "skipped forwarding request to upstream", details: map[string]interface{}{"upstreamId": "up-1"}, wrapsSentinel: true}, + {name: "UpstreamBlockUnavailable", err: NewErrUpstreamBlockUnavailable("up-1", 100, 90, 80), code: ErrCodeUpstreamBlockUnavailable, status: http.StatusServiceUnavailable, msg: "upstream does not have the requested block yet", details: map[string]interface{}{"upstreamId": "up-1", "blockNumber": int64(100), "latestBlock": int64(90), "finalizedBlock": int64(80)}}, + {name: "UpstreamMethodIgnored", err: NewErrUpstreamMethodIgnored("eth_call", "up-1"), code: ErrCodeUpstreamMethodIgnored, status: http.StatusNotAcceptable, msg: "method ignored by upstream configuration", details: map[string]interface{}{"method": "eth_call", "upstreamId": "up-1"}}, + {name: "UpstreamSyncing", err: NewErrUpstreamSyncing("up-1"), code: ErrCodeUpstreamSyncing, status: http.StatusUnprocessableEntity, msg: "upstream is syncing", details: map[string]interface{}{"upstreamId": "up-1"}}, + {name: "UpstreamShadowing", err: NewErrUpstreamShadowing("up-1"), code: ErrCodeUpstreamShadowing, msg: "upstream is shadowing", details: map[string]interface{}{"upstreamId": "up-1"}}, + {name: "UpstreamNotAllowed", err: NewErrUpstreamNotAllowed("alchemy-*", "up-1"), code: ErrCodeUpstreamNotAllowed, msg: "upstream not allowed based on use-upstream directive", details: map[string]interface{}{"required": "alchemy-*", "upstreamId": "up-1"}}, + {name: "UpstreamHedgeCancelled", err: NewErrUpstreamHedgeCancelled("up-1", covErrorsSentinel), code: ErrCodeUpstreamHedgeCancelled, msg: "hedged request cancelled", details: map[string]interface{}{"upstreamId": "up-1"}, wrapsSentinel: true}, + {name: "ResponseWriteLock", err: NewErrResponseWriteLock("w-1"), code: "ErrResponseWriteLock", msg: "failed to acquire write lock", details: map[string]interface{}{"writerId": "w-1"}}, + {name: "JsonRpcRequestUnresolvableMethod", err: NewErrJsonRpcRequestUnresolvableMethod("raw"), code: "ErrJsonRpcRequestUnresolvableMethod", msg: "could not resolve method", details: map[string]interface{}{"request": "raw", "retryableTowardNetwork": false}}, + {name: "FailsafeConfiguration", err: NewErrFailsafeConfiguration(covErrorsSentinel, map[string]interface{}{"policy": "retry"}), code: "ErrFailsafeConfiguration", msg: "failed to configure failsafe policy", details: map[string]interface{}{"policy": "retry"}, wrapsSentinel: true}, + {name: "RateLimitBudgetNotFound", err: NewErrRateLimitBudgetNotFound("b1"), code: "ErrRateLimitBudgetNotFound", msg: "rate limit budget not found", details: map[string]interface{}{"budgetId": "b1"}}, + {name: "RateLimitRuleNotFound", err: NewErrRateLimitRuleNotFound("b1", "eth_call"), code: "ErrRateLimitRuleNotFound", msg: "rate limit rule not found", details: map[string]interface{}{"budgetId": "b1", "method": "eth_call"}}, + {name: "ProjectRateLimitRuleExceeded", err: NewErrProjectRateLimitRuleExceeded("main", "b1", "r1"), code: ErrCodeProjectRateLimitRuleExceeded, status: http.StatusTooManyRequests, msg: "project-level rate limit rule exceeded", details: map[string]interface{}{"project": "main", "budget": "b1", "rule": "r1"}}, + {name: "NetworkRateLimitRuleExceeded", err: NewErrNetworkRateLimitRuleExceeded("main", "evm:1", "b1", "r1"), code: ErrCodeNetworkRateLimitRuleExceeded, status: http.StatusTooManyRequests, msg: "network-level rate limit rule exceeded", details: map[string]interface{}{"project": "main", "network": "evm:1", "budget": "b1", "rule": "r1"}}, + {name: "NetworkRequestTimeout", err: NewErrNetworkRequestTimeout(250*time.Millisecond, covErrorsSentinel), code: ErrCodeNetworkRequestTimeout, status: http.StatusGatewayTimeout, msg: "timed out after 250ms", wrapsSentinel: true}, + {name: "UpstreamRateLimitRuleExceeded", err: NewErrUpstreamRateLimitRuleExceeded("up-1", "b1", "r1"), code: ErrCodeUpstreamRateLimitRuleExceeded, status: http.StatusTooManyRequests, msg: "upstream-level rate limit rule exceeded", details: map[string]interface{}{"upstreamId": "up-1", "budget": "b1", "rule": "r1"}}, + {name: "UpstreamExcludedByPolicy", err: NewErrUpstreamExcludedByPolicy("up-1"), code: ErrCodeUpstreamExcludedByPolicy, msg: "upstream excluded by selection policy", details: map[string]interface{}{"upstreamId": "up-1"}}, + {name: "EndpointUnauthorized", err: NewErrEndpointUnauthorized(covErrorsSentinel), code: ErrCodeEndpointUnauthorized, status: http.StatusUnauthorized, msg: "remote endpoint responded with Unauthorized", wrapsSentinel: true}, + {name: "EndpointChainIdMismatch", err: NewErrEndpointChainIdMismatch(10, 1), code: ErrCodeEndpointChainIdMismatch, msg: "cross-wired endpoint", details: map[string]interface{}{"detectedChainId": uint64(10), "expectedChainId": uint64(1)}}, + {name: "EndpointUnsupported", err: NewErrEndpointUnsupported(covErrorsSentinel), code: ErrCodeEndpointUnsupported, status: http.StatusNotAcceptable, msg: "does not support requested method", wrapsSentinel: true}, + {name: "EndpointExecutionException", err: NewErrEndpointExecutionException(covErrorsSentinel), code: ErrCodeEndpointExecutionException, status: http.StatusOK, msg: "execution exception on node", details: map[string]interface{}{"retryableTowardNetwork": false}, wrapsSentinel: true}, + {name: "EndpointServerSideException_DefaultStatus", err: NewErrEndpointServerSideException(covErrorsSentinel, map[string]interface{}{"k": "v"}, 0), code: ErrCodeEndpointServerSideException, status: http.StatusInternalServerError, msg: "an internal error on remote endpoint", details: map[string]interface{}{"k": "v"}, wrapsSentinel: true}, + {name: "EndpointServerSideException_OriginalStatus", err: NewErrEndpointServerSideException(nil, nil, http.StatusBadGateway), code: ErrCodeEndpointServerSideException, status: http.StatusBadGateway, msg: "an internal error on remote endpoint"}, + {name: "EndpointRequestTimeout", err: NewErrEndpointRequestTimeout(3*time.Second, covErrorsSentinel), code: ErrCodeEndpointRequestTimeout, status: http.StatusGatewayTimeout, msg: "remote endpoint request timeout", details: map[string]interface{}{"durationMs": int64(3000)}, wrapsSentinel: true}, + {name: "EndpointRequestCanceled", err: NewErrEndpointRequestCanceled(covErrorsSentinel), code: ErrCodeEndpointRequestCanceled, msg: "remote endpoint request canceled", wrapsSentinel: true}, + {name: "EndpointCapacityExceeded", err: NewErrEndpointCapacityExceeded(covErrorsSentinel), code: ErrCodeEndpointCapacityExceeded, status: http.StatusTooManyRequests, msg: "remote endpoint capacity exceeded", wrapsSentinel: true}, + {name: "EndpointBillingIssue", err: NewErrEndpointBillingIssue(covErrorsSentinel), code: ErrCodeEndpointBillingIssue, status: http.StatusPaymentRequired, msg: "remote endpoint billing issue", wrapsSentinel: true}, + {name: "EndpointMissingData_NoUpstream", err: NewErrEndpointMissingData(covErrorsSentinel, nil), code: ErrCodeEndpointMissingData, status: http.StatusOK, msg: "remote endpoint does not have this data", wrapsSentinel: true}, + {name: "UpstreamNodeTypeMismatch", err: NewErrUpstreamNodeTypeMismatch(covErrorsSentinel, EvmNodeTypeArchive, EvmNodeTypeFull), code: ErrCodeUpstreamNodeTypeMismatch, msg: "node type does not match", details: map[string]interface{}{"expected": EvmNodeTypeArchive, "actual": EvmNodeTypeFull}, wrapsSentinel: true}, + {name: "EndpointRequestTooLarge", err: NewErrEndpointRequestTooLarge(covErrorsSentinel, EvmAddressesTooLarge), code: ErrCodeEndpointRequestTooLarge, status: http.StatusRequestEntityTooLarge, msg: "too large request", details: map[string]interface{}{"complaint": EvmAddressesTooLarge}, wrapsSentinel: true}, + {name: "InvalidConnectorDriver", err: NewErrInvalidConnectorDriver(DriverRedis), code: ErrCodeInvalidConnectorDriver, msg: "invalid store driver", details: map[string]interface{}{"driver": DriverRedis}}, + {name: "RecordNotFound", err: NewErrRecordNotFound("pk", "rk", "memory"), code: ErrCodeRecordNotFound, msg: "record not found", details: map[string]interface{}{"partitionKey": "pk", "rangeKey": "rk", "driver": "memory"}}, + {name: "RecordExpired", err: NewErrRecordExpired("pk", "rk", "redis", 200, 100), code: ErrCodeRecordExpired, msg: "record expired", details: map[string]interface{}{"partitionKey": "pk", "rangeKey": "rk", "driver": "redis", "now": int64(200), "expiration": int64(100)}}, + {name: "ConsensusDispute", err: NewErrConsensusDispute("dispute", nil, []error{covErrorsSentinel}), code: ErrCodeConsensusDispute, status: http.StatusConflict, msg: "dispute", wrapsSentinel: true}, + {name: "ConsensusCompositionDispute", err: NewErrConsensusCompositionDispute("composition", nil, []error{covErrorsSentinel}), code: ErrCodeConsensusCompositionDispute, status: http.StatusConflict, msg: "composition", wrapsSentinel: true}, + {name: "ConsensusLowParticipants", err: NewErrConsensusLowParticipants("too few", nil, []error{covErrorsSentinel}), code: ErrCodeConsensusLowParticipants, status: http.StatusPreconditionFailed, msg: "too few", wrapsSentinel: true}, + {name: "GetLogsExceededMaxAllowedResponseSize", err: NewErrGetLogsExceededMaxAllowedResponseSize(2048, 1024), code: ErrCodeGetLogsExceededMaxAllowedResponseSize, status: http.StatusRequestEntityTooLarge, msg: "getLogs response exceeded max allowed size", details: map[string]interface{}{"responseBytes": int64(2048), "maxAllowedBytes": int64(1024)}}, + {name: "GetLogsExceededMaxAllowedRange", err: NewErrGetLogsExceededMaxAllowedRange(20000, 10000), code: ErrCodeGetLogsExceededMaxAllowedRange, status: http.StatusRequestEntityTooLarge, msg: "exceeded max allowed range", details: map[string]interface{}{"requestRange": int64(20000), "maxAllowedRange": int64(10000)}}, + {name: "GetLogsExceededMaxAllowedAddresses", err: NewErrGetLogsExceededMaxAllowedAddresses(30, 10), code: ErrCodeGetLogsExceededMaxAllowedAddresses, status: http.StatusRequestEntityTooLarge, msg: "exceeded max allowed addresses", details: map[string]interface{}{"requestAddresses": int64(30), "maxAllowedAddresses": int64(10)}}, + {name: "GetLogsExceededMaxAllowedTopics", err: NewErrGetLogsExceededMaxAllowedTopics(5, 4), code: ErrCodeGetLogsExceededMaxAllowedTopics, status: http.StatusRequestEntityTooLarge, msg: "exceeded max allowed topics", details: map[string]interface{}{"requestTopics": int64(5), "maxAllowedTopics": int64(4)}}, + {name: "EndpointContentValidation", err: NewErrEndpointContentValidation(covErrorsSentinel, up), code: ErrCodeEndpointContentValidation, status: http.StatusBadGateway, msg: "failed validation content check", details: map[string]interface{}{"upstreamId": "up-1"}, wrapsSentinel: true}, + {name: "EndpointNonceException", err: NewErrEndpointNonceException(covErrorsSentinel, NonceExceptionReasonNonceTooLow), code: ErrCodeEndpointNonceException, status: http.StatusOK, msg: "transaction with this nonce already exists", details: map[string]interface{}{"nonceExceptionReason": "nonce_too_low", "retryableTowardNetwork": false}, wrapsSentinel: true}, + {name: "SubscriptionNotFound", err: NewErrSubscriptionNotFound("0xabc"), code: ErrCodeSubscriptionNotFound, status: http.StatusNotFound, msg: "subscription not found", details: map[string]interface{}{"subscriptionId": "0xabc"}}, + {name: "NoWsUpstreamAvailable", err: NewErrNoWsUpstreamAvailable("evm:1"), code: ErrCodeNoWsUpstreamAvailable, status: http.StatusBadRequest, msg: "none configured for network evm:1", details: map[string]interface{}{"networkId": "evm:1"}}, + {name: "SubscriptionLimitExceeded", err: NewErrSubscriptionLimitExceeded(5), code: ErrCodeSubscriptionLimitExceeded, status: http.StatusTooManyRequests, msg: "per connection exceeded", details: map[string]interface{}{"maxPerConnection": 5}}, + {name: "InstanceSubscriptionLimitExceeded", err: NewErrInstanceSubscriptionLimitExceeded(50), code: ErrCodeSubscriptionLimitExceeded, status: http.StatusTooManyRequests, msg: "per instance exceeded", details: map[string]interface{}{"maxPerInstance": 50}}, + } +} + +func TestCovErrors_Constructors_CodeStatusDetailsAndJSON(t *testing.T) { + for _, tc := range covErrorsCtorCases() { + t.Run(tc.name, func(t *testing.T) { + var se StandardError + require.True(t, errors.As(tc.err, &se), "constructor must return a StandardError") + base := se.Base() + assert.Equal(t, tc.code, base.Code) + assert.True(t, HasErrorCode(tc.err, tc.code)) + assert.True(t, HasErrorCode(tc.err, "ErrSomethingElse", tc.code), "any matching code in the list matches") + assert.False(t, HasErrorCode(tc.err, "ErrSomethingElse")) + assert.Contains(t, base.Message, tc.msg) + assert.True(t, strings.HasPrefix(tc.err.Error(), string(tc.code)+": "), tc.err.Error()) + + sc, hasStatus := tc.err.(interface{ ErrorStatusCode() int }) + if tc.status == 0 { + assert.False(t, hasStatus, "type unexpectedly maps to an HTTP status") + } else { + require.True(t, hasStatus, "type must map to an HTTP status") + assert.Equal(t, tc.status, sc.ErrorStatusCode()) + } + + for k, v := range tc.details { + assert.Equal(t, v, base.Details[k], "detail %s", k) + } + + if tc.wrapsSentinel { + assert.True(t, errors.Is(tc.err, covErrorsSentinel), "cause must be reachable via errors.Is") + assert.Contains(t, tc.err.Error(), "caused by: "+covErrorsSentinel.Error()) + } else { + assert.False(t, errors.Is(tc.err, covErrorsSentinel)) + } + + raw, err := SonicCfg.Marshal(tc.err) + require.NoError(t, err) + var m map[string]interface{} + require.NoError(t, SonicCfg.Unmarshal(raw, &m), string(raw)) + assert.Equal(t, string(tc.code), m["code"]) + assert.Equal(t, base.Message, m["message"]) + if len(tc.details) > 0 { + md, ok := m["details"].(map[string]interface{}) + require.True(t, ok, string(raw)) + for k := range tc.details { + assert.Contains(t, md, k) + } + } + }) + } +} + +func TestCovErrors_ClientSideException_StatusDependsOnCause(t *testing.T) { + cases := []struct { + name string + cause error + status int + }{ + {"NoCause", nil, http.StatusBadRequest}, + {"PlainCause", covErrorsSentinel, http.StatusBadRequest}, + {"EvmReverted", NewErrJsonRpcExceptionInternal(3, JsonRpcErrorEvmReverted, "execution reverted", nil, nil), http.StatusOK}, + {"CallException", NewErrJsonRpcExceptionInternal(-32000, JsonRpcErrorCallException, "call exception", nil, nil), http.StatusOK}, + {"TransactionRejected", NewErrJsonRpcExceptionInternal(-32003, JsonRpcErrorTransactionRejected, "rejected", nil, nil), http.StatusOK}, + {"InvalidArgument", NewErrJsonRpcExceptionInternal(-32602, JsonRpcErrorInvalidArgument, "bad arg", nil, nil), http.StatusBadRequest}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := NewErrEndpointClientSideException(tc.cause) + assert.True(t, HasErrorCode(err, ErrCodeEndpointClientSideException)) + assert.Equal(t, tc.status, err.(*ErrEndpointClientSideException).ErrorStatusCode()) + assert.True(t, IsRetryableTowardNetwork(err), "retryable toward network by default") + assert.False(t, IsRetryableTowardNetwork(err.WithRetryableTowardNetwork(false))) + }) + } +} + +func TestCovErrors_JsonRpcRequestUnmarshal_Branches(t *testing.T) { + t.Run("BaseErrorCauseIsKept", func(t *testing.T) { + cause := &BaseError{Code: "ErrInner", Message: "inner"} + err := NewErrJsonRpcRequestUnmarshal(cause, []byte(`{bad`)) + be := err.(*ErrJsonRpcRequestUnmarshal) + assert.Equal(t, "failed to unmarshal json-rpc request", be.Message) + assert.Same(t, cause, be.Cause) + assert.Equal(t, "{bad", be.Details["body"]) + assert.Equal(t, 400, be.ErrorStatusCode()) + assert.Equal(t, "ErrJsonRpcRequestUnmarshal <- ErrInner", be.CodeChain()) + }) + t.Run("PlainCauseBecomesMessage", func(t *testing.T) { + err := NewErrJsonRpcRequestUnmarshal(errors.New("unexpected EOF"), []byte(`{`)) + be := err.(*ErrJsonRpcRequestUnmarshal) + assert.Equal(t, "unexpected EOF", be.Message) + assert.Nil(t, be.Cause) + assert.False(t, IsRetryableTowardNetwork(err)) + }) + t.Run("NilCause", func(t *testing.T) { + err := NewErrJsonRpcRequestUnmarshal(nil, nil) + be := err.(*ErrJsonRpcRequestUnmarshal) + assert.Equal(t, "failed to unmarshal json-rpc request", be.Message) + assert.Nil(t, be.Cause) + assert.Equal(t, "", be.Details["body"]) + assert.True(t, IsClientError(err)) + }) +} + +func TestCovErrors_JsonRpcRequestPreparation_RetryableDefault(t *testing.T) { + err := NewErrJsonRpcRequestPreparation(covErrorsSentinel, nil) + assert.Equal(t, false, err.(*ErrJsonRpcRequestPreparation).Details["retryableTowardNetwork"]) + assert.False(t, IsRetryableTowardNetwork(err)) + assert.True(t, errors.Is(err, covErrorsSentinel)) + + kept := NewErrJsonRpcRequestPreparation(nil, map[string]interface{}{"retryableTowardNetwork": true, "x": 1}) + assert.Equal(t, true, kept.(*ErrJsonRpcRequestPreparation).Details["retryableTowardNetwork"]) + assert.Equal(t, 1, kept.(*ErrJsonRpcRequestPreparation).Details["x"]) + assert.True(t, IsRetryableTowardNetwork(kept)) +} + +func TestCovErrors_EndpointTransportFailure_RedactsUrl(t *testing.T) { + u, err := url.Parse("https://rpc.example.com/secret-key") + require.NoError(t, err) + + plain := NewErrEndpointTransportFailure(u, errors.New(`Post "https://rpc.example.com/secret-key": dial tcp: connection refused`)) + tf := plain.(*ErrEndpointTransportFailure) + assert.Equal(t, "rpc.example.com", tf.Details["host"]) + assert.NotContains(t, tf.Cause.Error(), "secret-key") + assert.Contains(t, tf.Cause.Error(), "dial tcp: connection refused") + + std := NewErrEndpointRequestTimeout(time.Second, nil) + wrapped := NewErrEndpointTransportFailure(u, std) + assert.Same(t, std, wrapped.(*ErrEndpointTransportFailure).Cause, "standard errors are kept as-is") + assert.True(t, HasErrorCode(wrapped, ErrCodeEndpointTransportFailure, ErrCodeEndpointRequestTimeout)) + + noCause := NewErrEndpointTransportFailure(u, nil) + assert.Nil(t, noCause.(*ErrEndpointTransportFailure).Cause) +} + +func TestCovErrors_EndpointMissingData_UpstreamDetails(t *testing.T) { + up := NewFakeUpstream("evm-up", WithEvmStatePoller(NewFakeEvmStatePoller(100, 90))) + up.Config().Evm = &EvmUpstreamConfig{MaxAvailableRecentBlocks: 128} + + err := NewErrEndpointMissingData(nil, up) + md := err.(*ErrEndpointMissingData) + assert.Equal(t, "evm-up", md.Details["upstreamId"]) + assert.Equal(t, int64(100), md.Details["latestBlock"]) + assert.Equal(t, int64(90), md.Details["finalizedBlock"]) + assert.Equal(t, int64(128), md.Details["maxAvailableRecentBlocks"]) + assert.Same(t, up, md.Upstream()) + + plainUp := NewFakeUpstream("plain-up") + md2 := NewErrEndpointMissingData(nil, plainUp).(*ErrEndpointMissingData) + assert.Equal(t, "plain-up", md2.Details["upstreamId"]) + assert.NotContains(t, md2.Details, "latestBlock") + assert.NotContains(t, md2.Details, "maxAvailableRecentBlocks") +} + +func TestCovErrors_EndpointContentValidation_NilUpstream(t *testing.T) { + err := NewErrEndpointContentValidation(nil, nil).(*ErrEndpointContentValidation) + assert.NotContains(t, err.Details, "upstreamId") + assert.Nil(t, err.Upstream()) +} + +func TestCovErrors_AuthUnauthorized_Reason(t *testing.T) { + assert.Equal(t, "token expired", AuthUnauthorizedReasonTokenExpired.Message()) + assert.Equal(t, "invalid signature", AuthUnauthorizedReasonInvalidSignature.Message()) + assert.Equal(t, "custom_reason", AuthUnauthorizedReason("custom_reason").Message()) + + var nilErr *ErrAuthUnauthorized + assert.Equal(t, AuthUnauthorizedReason(""), nilErr.Reason()) + assert.Equal(t, AuthUnauthorizedReason(""), (&ErrAuthUnauthorized{}).Reason()) + + fromStr := NewErrAuthUnauthorizedReason("jwt", AuthUnauthorizedReasonTokenExpired).(*ErrAuthUnauthorized) + assert.Equal(t, AuthUnauthorizedReasonTokenExpired, fromStr.Reason()) + assert.Equal(t, "token expired", fromStr.Message) + + typed := &ErrAuthUnauthorized{BaseError{Details: map[string]interface{}{"reason": AuthUnauthorizedReasonInvalidSignature}}} + assert.Equal(t, AuthUnauthorizedReasonInvalidSignature, typed.Reason()) + + wrongType := &ErrAuthUnauthorized{BaseError{Details: map[string]interface{}{"reason": 7}}} + assert.Equal(t, AuthUnauthorizedReason(""), wrongType.Reason()) +} + +func TestCovErrors_UpstreamRequest_Accessors(t *testing.T) { + up := NewFakeUpstream("up-a") + err := NewErrUpstreamRequest(covErrorsSentinel, up, "evm:1", "eth_call", 1500*time.Millisecond, 3, 2, 1) + ur := err.(*ErrUpstreamRequest) + assert.Equal(t, "up-a", ur.UpstreamId()) + assert.Equal(t, 3, ur.Attempts()) + assert.Equal(t, 2, ur.Retries()) + assert.Equal(t, 1, ur.Hedges()) + assert.False(t, ur.FromCache()) + assert.False(t, ur.IsObjectNull()) + assert.Same(t, up, ur.Upstream()) + assert.Equal(t, int64(1500), ur.Details["durationMs"]) + assert.Equal(t, "evm:1", ur.Details["networkId"]) + assert.Equal(t, "eth_call", ur.Details["method"]) + assert.True(t, errors.Is(err, covErrorsSentinel)) + + noUp := NewErrUpstreamRequest(nil, nil, "evm:1", "eth_call", 0, 0, 0, 0).(*ErrUpstreamRequest) + assert.NotContains(t, noUp.Details, "upstreamId") + assert.Equal(t, "", noUp.UpstreamId()) + + emptyId := NewErrUpstreamRequest(nil, NewFakeUpstream(""), "evm:1", "eth_call", 0, 0, 0, 0).(*ErrUpstreamRequest) + assert.NotContains(t, emptyId.Details, "upstreamId") + + bare := &ErrUpstreamRequest{} + assert.True(t, bare.IsObjectNull()) + assert.Equal(t, "", bare.UpstreamId()) + assert.Equal(t, 0, bare.Attempts()) + assert.Equal(t, 0, bare.Retries()) + assert.Equal(t, 0, bare.Hedges()) + + var nilUr *ErrUpstreamRequest + assert.True(t, nilUr.IsObjectNull()) + + wrongTypes := &ErrUpstreamRequest{BaseError: BaseError{Code: ErrCodeUpstreamRequest, Details: map[string]interface{}{ + "upstreamId": 5, "attempts": "3", "retries": 2.0, "hedges": int64(1), + }}} + assert.Equal(t, "", wrongTypes.UpstreamId()) + assert.Equal(t, 0, wrongTypes.Attempts()) + assert.Equal(t, 0, wrongTypes.Retries()) + assert.Equal(t, 0, wrongTypes.Hedges()) +} + +func TestCovErrors_FailsafeErrors_MessagesAndDeepest(t *testing.T) { + start := time.Now().Add(-2 * time.Second) + + type ctor func(Scope, error, *time.Time) error + cases := []struct { + name string + ctor ctor + code ErrorCode + withStart string + noStart string + hasDurMs bool + }{ + {"Timeout", NewErrFailsafeTimeoutExceeded, ErrCodeFailsafeTimeoutExceeded, "failsafe timeout policy exceeded on network-level after ", "failsafe timeout policy exceeded on network-level", false}, + {"Retry", NewErrFailsafeRetryExceeded, ErrCodeFailsafeRetryExceeded, "gave up retrying on network-level after ", "gave up retrying on network-level", true}, + {"CircuitBreaker", NewErrFailsafeCircuitBreakerOpen, ErrCodeFailsafeCircuitBreakerOpen, "failsafe circuit breaker open on network-level after ", "failsafe circuit breaker open on network-level", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + timed := tc.ctor(ScopeNetwork, nil, &start) + var se StandardError + require.True(t, errors.As(timed, &se)) + assert.True(t, strings.HasPrefix(se.Base().Message, tc.withStart), se.Base().Message) + if tc.hasDurMs { + assert.GreaterOrEqual(t, se.Base().Details["durationMs"].(int64), int64(2000)) + } + assert.Equal(t, se.Base().Message, se.DeepestMessage(), "no cause: deepest message is own message") + + untimed := tc.ctor(ScopeNetwork, nil, nil) + require.True(t, errors.As(untimed, &se)) + assert.Equal(t, tc.noStart, se.Base().Message) + assert.Nil(t, se.Base().Details) + + stdCause := NewErrEndpointCapacityExceeded(NewErrJsonRpcExceptionInternal(0, 0, "slow down", nil, nil)) + withStd := tc.ctor(ScopeNetwork, stdCause, nil) + require.True(t, errors.As(withStd, &se)) + assert.Equal(t, tc.noStart+": slow down", se.DeepestMessage()) + assert.True(t, HasErrorCode(withStd, tc.code, ErrCodeEndpointCapacityExceeded)) + + withPlain := tc.ctor(ScopeNetwork, errors.New("boom"), nil) + require.True(t, errors.As(withPlain, &se)) + assert.Equal(t, tc.noStart+": boom", se.DeepestMessage()) + }) + } + assert.Equal(t, http.StatusGatewayTimeout, NewErrFailsafeTimeoutExceeded(ScopeUpstream, nil, nil).(*ErrFailsafeTimeoutExceeded).ErrorStatusCode()) +} + +func TestCovErrors_Classification(t *testing.T) { + up := NewFakeUpstream("up-1") + cases := []struct { + name string + err error + client bool + capacity bool + retryableUpstream bool + retryableNetwork bool + severity Severity + }{ + {"Nil", nil, false, false, false, true, SeverityInfo}, + {"ClientSide", NewErrEndpointClientSideException(nil), true, false, true, true, SeverityInfo}, + {"InvalidRequest", NewErrInvalidRequest(nil), true, false, false, false, SeverityInfo}, + {"GetLogsRange", NewErrGetLogsExceededMaxAllowedRange(2, 1), true, false, true, true, SeverityInfo}, + {"GetLogsAddresses", NewErrGetLogsExceededMaxAllowedAddresses(2, 1), true, false, true, true, SeverityInfo}, + {"GetLogsTopics", NewErrGetLogsExceededMaxAllowedTopics(2, 1), true, false, true, true, SeverityInfo}, + {"GetLogsResponseSize", NewErrGetLogsExceededMaxAllowedResponseSize(2, 1), true, false, true, true, SeverityInfo}, + {"SubscriptionNotFound", NewErrSubscriptionNotFound("s"), true, false, true, true, SeverityInfo}, + {"SubscriptionLimit", NewErrSubscriptionLimitExceeded(1), true, false, true, true, SeverityInfo}, + {"ExecutionException", NewErrEndpointExecutionException(nil), false, false, false, false, SeverityInfo}, + {"BlockUnavailable", NewErrUpstreamBlockUnavailable("u", 1, 0, 0), false, false, true, true, SeverityWarning}, + {"MissingData", NewErrEndpointMissingData(nil, up), false, false, true, true, SeverityWarning}, + {"NetworkNotSupported", NewErrNetworkNotSupported("p", "n"), false, false, true, true, SeverityWarning}, + {"CircuitBreakerOpen", NewErrFailsafeCircuitBreakerOpen(ScopeUpstream, nil, nil), false, false, false, true, SeverityWarning}, + {"RequestSkipped", NewErrUpstreamRequestSkipped(nil, "u"), false, false, false, true, SeverityWarning}, + {"MethodIgnored", NewErrUpstreamMethodIgnored("m", "u"), false, false, false, true, SeverityWarning}, + {"Unsupported", NewErrEndpointUnsupported(nil), false, false, false, true, SeverityWarning}, + {"NotImplemented", NewErrNotImplemented("x"), false, false, false, true, SeverityWarning}, + {"Billing", NewErrEndpointBillingIssue(nil), false, false, false, true, SeverityWarning}, + {"Unauthorized", NewErrEndpointUnauthorized(nil), false, false, false, true, SeverityWarning}, + {"TooLarge", NewErrEndpointRequestTooLarge(nil, EvmBlockRangeTooLarge), false, false, false, true, SeverityWarning}, + {"ContentValidation", NewErrEndpointContentValidation(nil, up), false, false, false, true, SeverityWarning}, + {"ProjectRateLimit", NewErrProjectRateLimitRuleExceeded("p", "b", "r"), false, true, false, true, SeverityWarning}, + {"NetworkRateLimit", NewErrNetworkRateLimitRuleExceeded("p", "n", "b", "r"), false, true, false, true, SeverityWarning}, + {"UpstreamRateLimit", NewErrUpstreamRateLimitRuleExceeded("u", "b", "r"), false, true, false, true, SeverityWarning}, + {"AuthRateLimit", NewErrAuthRateLimitRuleExceeded("p", "s", "b", "r", "u", "ip"), false, true, false, true, SeverityWarning}, + {"EndpointCapacity", NewErrEndpointCapacityExceeded(nil), false, true, false, true, SeverityWarning}, + {"RequestCanceled", NewErrEndpointRequestCanceled(nil), false, false, true, true, SeverityWarning}, + {"ContextCanceled", covErrorsWrap(context.Canceled), false, false, true, true, SeverityWarning}, + {"NonceException", NewErrEndpointNonceException(nil, NonceExceptionReasonAlreadyKnown), false, false, true, false, SeverityCritical}, + {"ServerSide", NewErrEndpointServerSideException(nil, nil, 0), false, false, true, true, SeverityCritical}, + {"Timeout", NewErrEndpointRequestTimeout(time.Second, nil), false, false, true, true, SeverityCritical}, + {"Plain", errors.New("plain"), false, false, true, true, SeverityCritical}, + {"UpstreamRequestWrapsCapacity", NewErrUpstreamRequest(NewErrEndpointCapacityExceeded(nil), up, "evm:1", "eth_call", 0, 1, 0, 0), false, true, false, true, SeverityWarning}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.client, IsClientError(tc.err), "IsClientError") + assert.Equal(t, tc.capacity, IsCapacityIssue(tc.err), "IsCapacityIssue") + if tc.err != nil { + assert.Equal(t, tc.retryableUpstream, IsRetryableTowardsUpstream(tc.err), "IsRetryableTowardsUpstream") + } + assert.Equal(t, tc.retryableNetwork, IsRetryableTowardNetwork(tc.err), "IsRetryableTowardNetwork") + assert.Equal(t, tc.severity, ClassifySeverity(tc.err), "ClassifySeverity") + }) + } +} + +func covErrorsWrap(err error) error { return fmt.Errorf("covErrors wrap: %w", err) } diff --git a/common/cov_errors_grpc_test.go b/common/cov_errors_grpc_test.go new file mode 100644 index 000000000..5436bde38 --- /dev/null +++ b/common/cov_errors_grpc_test.go @@ -0,0 +1,149 @@ +package common + +import ( + "errors" + "testing" + + bdscommon "github.com/blockchain-data-standards/manifesto/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// covErrorsInnerJsonRpc returns the json-rpc exception the gRPC mapper nests as cause. +func covErrorsInnerJsonRpc(t *testing.T, err error) *ErrJsonRpcExceptionInternal { + t.Helper() + var inner *ErrJsonRpcExceptionInternal + require.True(t, errors.As(err, &inner), "expected nested ErrJsonRpcExceptionInternal in %v", err) + return inner +} + +func TestCovErrors_Grpc_NilAndOK(t *testing.T) { + assert.Nil(t, ExtractGrpcErrorFromGrpcStatus(nil, nil)) + assert.Nil(t, ExtractGrpcErrorFromGrpcStatus(status.New(codes.OK, "fine"), NewFakeUpstream("u"))) +} + +func TestCovErrors_Grpc_PlainStatusCodes(t *testing.T) { + cases := []struct { + code codes.Code + errCode ErrorCode + normalized JsonRpcErrorNumber + }{ + {codes.Canceled, ErrCodeEndpointRequestCanceled, 0}, + {codes.Unimplemented, ErrCodeEndpointUnsupported, JsonRpcErrorUnsupportedException}, + {codes.InvalidArgument, ErrCodeEndpointClientSideException, JsonRpcErrorInvalidArgument}, + {codes.ResourceExhausted, ErrCodeEndpointCapacityExceeded, JsonRpcErrorCapacityExceeded}, + {codes.DeadlineExceeded, ErrCodeEndpointRequestTimeout, JsonRpcErrorNodeTimeout}, + {codes.Unauthenticated, ErrCodeEndpointUnauthorized, JsonRpcErrorUnauthorized}, + {codes.PermissionDenied, ErrCodeEndpointUnauthorized, JsonRpcErrorUnauthorized}, + {codes.NotFound, ErrCodeEndpointMissingData, JsonRpcErrorMissingData}, + {codes.OutOfRange, ErrCodeEndpointMissingData, JsonRpcErrorMissingData}, + {codes.Internal, ErrCodeEndpointServerSideException, JsonRpcErrorServerSideException}, + {codes.Unknown, ErrCodeEndpointServerSideException, JsonRpcErrorServerSideException}, + {codes.Unavailable, ErrCodeEndpointServerSideException, JsonRpcErrorServerSideException}, + {codes.AlreadyExists, ErrCodeEndpointServerSideException, JsonRpcErrorServerSideException}, + {codes.Aborted, ErrCodeEndpointServerSideException, JsonRpcErrorServerSideException}, + } + for _, tc := range cases { + for _, withUpstream := range []bool{false, true} { + name := tc.code.String() + var up Upstream + wantUpsId := "n/a" + if withUpstream { + name += "/withUpstream" + up = NewFakeUpstream("bds-up") + wantUpsId = "bds-up" + } + t.Run(name, func(t *testing.T) { + err := ExtractGrpcErrorFromGrpcStatus(status.New(tc.code, "boom from server"), up) + require.Error(t, err) + assert.True(t, HasErrorCode(err, tc.errCode), "got %v", err) + + inner := covErrorsInnerJsonRpc(t, err) + assert.Equal(t, tc.normalized, inner.NormalizedCode()) + assert.Equal(t, int(tc.normalized), inner.OriginalCode()) + assert.Equal(t, "boom from server", inner.Message) + assert.Equal(t, tc.code.String(), inner.Details["grpcCode"]) + assert.Equal(t, "boom from server", inner.Details["grpcMessage"]) + assert.Equal(t, wantUpsId, inner.Details["upstreamId"]) + assert.NotContains(t, inner.Details, "bdsErrorCode") + + if tc.code == codes.InvalidArgument { + assert.False(t, IsRetryableTowardNetwork(err), "invalid argument must not fail over") + } + if tc.errCode == ErrCodeEndpointMissingData { + md := err.(*ErrEndpointMissingData) + if withUpstream { + assert.Equal(t, "bds-up", md.Details["upstreamId"]) + assert.Same(t, up, md.Upstream()) + } else { + assert.NotContains(t, md.Details, "upstreamId") + } + } + }) + } + } +} + +func TestCovErrors_Grpc_BdsErrorCodes(t *testing.T) { + cases := []struct { + bds bdscommon.ErrorCode + errCode ErrorCode + normalized JsonRpcErrorNumber + }{ + {bdscommon.ErrorCode_UNSUPPORTED_BLOCK_TAG, ErrCodeEndpointUnsupported, JsonRpcErrorUnsupportedException}, + {bdscommon.ErrorCode_UNSUPPORTED_METHOD, ErrCodeEndpointUnsupported, JsonRpcErrorUnsupportedException}, + {bdscommon.ErrorCode_RANGE_OUTSIDE_AVAILABLE, ErrCodeEndpointMissingData, JsonRpcErrorMissingData}, + {bdscommon.ErrorCode_INVALID_PARAMETER, ErrCodeEndpointClientSideException, JsonRpcErrorInvalidArgument}, + {bdscommon.ErrorCode_INVALID_REQUEST, ErrCodeEndpointClientSideException, JsonRpcErrorInvalidArgument}, + {bdscommon.ErrorCode_RATE_LIMITED, ErrCodeEndpointCapacityExceeded, JsonRpcErrorCapacityExceeded}, + {bdscommon.ErrorCode_TIMEOUT_ERROR, ErrCodeEndpointRequestTimeout, JsonRpcErrorNodeTimeout}, + {bdscommon.ErrorCode_RANGE_TOO_LARGE, ErrCodeEndpointRequestTooLarge, JsonRpcErrorEvmLargeRange}, + {bdscommon.ErrorCode_INTERNAL_ERROR, ErrCodeEndpointServerSideException, JsonRpcErrorServerSideException}, + // Unmapped BDS codes fall through to the gRPC code switch (Unknown -> server-side). + {bdscommon.ErrorCode_DATA_NOT_FOUND, ErrCodeEndpointServerSideException, JsonRpcErrorServerSideException}, + } + for _, tc := range cases { + t.Run(tc.bds.String(), func(t *testing.T) { + up := NewFakeUpstream("bds-up") + st := bdscommon.NewError(tc.bds, "bds says no"). + WithCause(errors.New("root cause")). + WithDetail("fromBlock", 5). + WithDetail("hint", "narrow it"). + ToGRPCStatus() + + err := ExtractGrpcErrorFromGrpcStatus(st, up) + require.Error(t, err) + assert.True(t, HasErrorCode(err, tc.errCode), "got %v", err) + + inner := covErrorsInnerJsonRpc(t, err) + assert.Equal(t, tc.normalized, inner.NormalizedCode()) + assert.Equal(t, "bds says no", inner.Message) + assert.Equal(t, tc.bds, inner.Details["bdsErrorCode"]) + assert.Equal(t, "root cause", inner.Details["cause"]) + assert.Equal(t, float64(5), inner.Details["fromBlock"], "JSON-encoded BDS details are decoded") + assert.Equal(t, "narrow it", inner.Details["hint"]) + assert.Equal(t, "bds-up", inner.Details["upstreamId"]) + assert.Equal(t, st.Code().String(), inner.Details["grpcCode"]) + + if tc.errCode == ErrCodeEndpointRequestTooLarge { + assert.Equal(t, EvmBlockRangeTooLarge, err.(*ErrEndpointRequestTooLarge).Details["complaint"]) + } + if tc.errCode == ErrCodeEndpointClientSideException { + assert.False(t, IsRetryableTowardNetwork(err)) + } + }) + } +} + +func TestCovErrors_Grpc_BdsWithoutCause(t *testing.T) { + st := bdscommon.NewError(bdscommon.ErrorCode_RATE_LIMITED, "slow down").ToGRPCStatus() + err := ExtractGrpcErrorFromGrpcStatus(st, nil) + assert.True(t, HasErrorCode(err, ErrCodeEndpointCapacityExceeded)) + inner := covErrorsInnerJsonRpc(t, err) + assert.NotContains(t, inner.Details, "cause") + assert.Equal(t, bdscommon.ErrorCode_RATE_LIMITED, inner.Details["bdsErrorCode"]) + assert.Equal(t, "n/a", inner.Details["upstreamId"]) + assert.Equal(t, codes.ResourceExhausted.String(), inner.Details["grpcCode"]) +} diff --git a/common/cov_errors_shape_test.go b/common/cov_errors_shape_test.go new file mode 100644 index 000000000..01194aaa7 --- /dev/null +++ b/common/cov_errors_shape_test.go @@ -0,0 +1,601 @@ +package common + +import ( + "bytes" + "context" + "errors" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// covErrorsEmptyMulti is a multi-error wrapper whose Unwrap yields no children; +// errors.Join cannot produce one (it returns nil instead). +type covErrorsEmptyMulti struct{} + +func (covErrorsEmptyMulti) Error() string { return "empty multi" } +func (covErrorsEmptyMulti) Unwrap() []error { return nil } + +func covErrorsSetLabelMode(t *testing.T, mode LabelMode) { + t.Helper() + SetErrorLabelMode(mode) + t.Cleanup(func() { SetErrorLabelMode(ErrorLabelModeVerbose) }) +} + +func covErrorsUnmarshalObj(t *testing.T, v interface{}) map[string]interface{} { + t.Helper() + raw, err := SonicCfg.Marshal(v) + require.NoError(t, err) + var m map[string]interface{} + require.NoError(t, SonicCfg.Unmarshal(raw, &m), string(raw)) + return m +} + +func covErrorsZerolog(t *testing.T, obj zerolog.LogObjectMarshaler) map[string]interface{} { + t.Helper() + // Test init disables zerolog globally; enable it for this capture only. + prev := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.TraceLevel) + t.Cleanup(func() { zerolog.SetGlobalLevel(prev) }) + var buf bytes.Buffer + lg := zerolog.New(&buf) + lg.Info().Object("err", obj).Send() + var m map[string]interface{} + require.NoError(t, SonicCfg.Unmarshal(buf.Bytes(), &m), buf.String()) + inner, _ := m["err"].(map[string]interface{}) + return inner +} + +func TestCovErrors_SetErrorLabelMode_IgnoresUnknown(t *testing.T) { + covErrorsSetLabelMode(t, ErrorLabelModeCompact) + SetErrorLabelMode("bogus") + assert.Equal(t, "ErrUnknown", strings.Split(ErrorSummary(&BaseError{Code: "ErrUnknown"}), "/")[0]) + assert.Equal(t, "GenericError", ErrorSummary(errors.New("x")), "still compact after an invalid mode") +} + +func TestCovErrors_IsNull(t *testing.T) { + assert.True(t, IsNull(nil)) + assert.True(t, IsNull("")) + assert.True(t, IsNull(&BaseError{})) + assert.False(t, IsNull(&BaseError{Code: "ErrX"})) + assert.False(t, IsNull(NewErrUpstreamSyncing("u"))) + assert.False(t, IsNull(errors.New("plain"))) +} + +func TestCovErrors_ErrorSummary_Compact(t *testing.T) { + covErrorsSetLabelMode(t, ErrorLabelModeCompact) + + jre := NewErrJsonRpcExceptionInternal(-32000, JsonRpcErrorCallException, "call exception", nil, nil) + cases := []struct { + name string + in interface{} + want string + }{ + {"Nil", nil, ""}, + {"UpstreamRequestWithJsonRpcCause", NewErrUpstreamRequest(jre, nil, "evm:1", "eth_call", 0, 1, 0, 0), "ErrUpstreamRequest/-32000"}, + {"SkippedWithStdCause", NewErrUpstreamRequestSkipped(NewErrUpstreamSyncing("u"), "u"), "ErrUpstreamRequestSkipped/ErrUpstreamSyncing"}, + {"OtherWithCauseNoCompound", NewErrEndpointUnsupported(NewErrUpstreamSyncing("u")), "ErrEndpointUnsupported"}, + {"JoinSingle", errors.Join(NewErrUpstreamSyncing("u")), "ErrUpstreamSyncing"}, + {"JoinMany", errors.Join(NewErrUpstreamSyncing("u"), errors.New("raw failure")), "ErrUpstreamSyncing, raw failure"}, + {"EmptyMulti", covErrorsEmptyMulti{}, "UnknownMultipleErrors"}, + {"Deadline", covErrorsWrap(context.DeadlineExceeded), "ContextDeadlineExceeded"}, + {"Canceled", covErrorsWrap(context.Canceled), "ContextCanceled"}, + {"Generic", errors.New("x"), "GenericError"}, + {"String", "some text", "StringError"}, + {"Other", 42, "UnknownError"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, ErrorSummary(tc.in)) + }) + } +} + +func TestCovErrors_ErrorSummary_Verbose(t *testing.T) { + covErrorsSetLabelMode(t, ErrorLabelModeVerbose) + + jre := NewErrJsonRpcExceptionInternal(-32000, JsonRpcErrorCallException, "execution reverted: 0xdeadbeefcafe", nil, nil) + cases := []struct { + name string + in interface{} + want string + }{ + {"StandardChain", NewErrUpstreamRequest(jre, nil, "evm:1", "eth_call", 0, 1, 0, 0), "ErrUpstreamRequest <- -32000 <- ErrJsonRpcExceptionInternal: execution reverted"}, + {"PlainCleaned", errors.New("dial 10.0.0.1 failed\n after 1234 ms"), "dial X.X.X.X failed after XX ms"}, + {"StringCleaned", "trie node abcdef0123 missing", "trie node 0xREDACTED missing"}, + {"OtherCleaned", 12345, "XX"}, + {"TxHash", errors.New("transaction abc123 not found"), "transaction 0xREDACTED not found"}, + {"LongAlnum", errors.New("key " + strings.Repeat("a", 40)), "key XX"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, ErrorSummary(tc.in)) + }) + } + + long := ErrorSummary(errors.New(strings.Repeat("ab ", 300))) + assert.Len(t, long, 512, "messages are capped at 512 chars") +} + +func TestCovErrors_ErrorFingerprint(t *testing.T) { + covErrorsSetLabelMode(t, ErrorLabelModeVerbose) + assert.Equal(t, "ErrUpstreamSyncing upstream is syncing and should not serve requests", ErrorFingerprint(NewErrUpstreamSyncing("u"))) + fp := ErrorFingerprint(errors.New(strings.Repeat("ab ", 200))) + assert.Len(t, fp, 256) + assert.Equal(t, "a b c", ErrorFingerprint(errors.New("a!@#b$%^c"))) +} + +func TestCovErrors_IsClientDisconnect(t *testing.T) { + cases := []struct { + err error + want bool + }{ + {nil, false}, + {context.Canceled, true}, + {covErrorsWrap(context.DeadlineExceeded), true}, + {errors.New("use of closed network connection"), true}, + {errors.New("write tcp: use of closed network connection"), true}, + {errors.New("write: broken pipe"), true}, + {errors.New("read: connection reset by peer"), true}, + {errors.New("ECONNRESET"), true}, + {errors.New("EPIPE"), true}, + {errors.New("server returned 500"), false}, + } + for _, tc := range cases { + assert.Equal(t, tc.want, IsClientDisconnect(tc.err), "%v", tc.err) + } +} + +func TestCovErrors_BaseError_ErrorString(t *testing.T) { + assert.Equal(t, "ErrX: m", (&BaseError{Code: "ErrX", Message: "m"}).Error()) + assert.Equal(t, `ErrX: m -> ({"a":1})`, (&BaseError{Code: "ErrX", Message: "m", Details: map[string]interface{}{"a": 1}}).Error()) + assert.Equal(t, "ErrX: m \ncaused by: c", (&BaseError{Code: "ErrX", Message: "m", Cause: errors.New("c")}).Error()) + assert.Equal(t, "ErrX: m -> ({\"a\":1}) \ncaused by: c", (&BaseError{Code: "ErrX", Message: "m", Cause: errors.New("c"), Details: map[string]interface{}{"a": 1}}).Error()) + + unmarshalable := (&BaseError{Code: "ErrX", Message: "m", Details: map[string]interface{}{"fn": func() {}}}).Error() + assert.True(t, strings.HasPrefix(unmarshalable, "ErrX: m -> (map[fn:"), unmarshalable) +} + +func TestCovErrors_BaseError_ChainAndSearch(t *testing.T) { + leaf := &BaseError{Code: "ErrLeaf", Message: "leaf msg", Details: map[string]interface{}{"k": "leaf"}} + mid := &BaseError{Code: "ErrMid", Message: "mid msg", Cause: leaf} + top := &BaseError{Code: "ErrTop", Message: "top msg", Cause: mid, Details: map[string]interface{}{"own": 1}} + + assert.Equal(t, "ErrTop <- ErrMid <- ErrLeaf", top.CodeChain()) + assert.Equal(t, "leaf msg", top.DeepestMessage()) + assert.Equal(t, "plain", (&BaseError{Code: "E", Message: "m", Cause: errors.New("plain")}).DeepestMessage()) + assert.Equal(t, "E", (&BaseError{Code: "E", Cause: errors.New("plain")}).CodeChain()) + + assert.Equal(t, 1, top.DeepSearch("own")) + assert.Equal(t, "leaf", top.DeepSearch("k")) + assert.Nil(t, top.DeepSearch("missing")) + assert.Nil(t, (&BaseError{Code: "E", Cause: errors.New("plain")}).DeepSearch("k")) + + multi := &BaseError{Code: "ErrMulti", Cause: errors.Join(errors.New("plain"), &BaseError{Code: "ErrA"}, leaf)} + assert.Equal(t, "leaf", multi.DeepSearch("k"), "first standard child that has the key wins") + assert.Nil(t, multi.DeepSearch("missing")) + assert.True(t, multi.HasCode("ErrLeaf")) + assert.False(t, multi.HasCode("ErrNope")) + + assert.True(t, errors.Is(top, leaf)) + assert.Same(t, mid, top.Unwrap()) + assert.Same(t, mid, top.GetCause()) + assert.Equal(t, ErrorCode("ErrTop"), top.GetCode()) + assert.Same(t, top, top.Base()) +} + +func TestCovErrors_BaseError_FlagSetters(t *testing.T) { + var nilBase *BaseError + assert.NotPanics(t, func() { nilBase.WithRetryableTowardNetwork(false) }) + assert.Nil(t, nilBase.WithPermanentMissingData(true)) + + b := &BaseError{Code: ErrCodeEndpointMissingData} + b.WithPermanentMissingData(true) + assert.Equal(t, true, b.Details["permanentMissingData"]) + assert.True(t, IsPermanentlyMissingData(b)) + + r := &BaseError{Code: "ErrR"} + r.WithRetryableTowardNetwork(false) + assert.False(t, IsRetryableTowardNetwork(r)) +} + +func TestCovErrors_BaseError_MarshalJSONShapes(t *testing.T) { + t.Run("NoCodeNoCause_IsJustMessage", func(t *testing.T) { + raw, err := SonicCfg.Marshal(&BaseError{Message: "just text"}) + require.NoError(t, err) + assert.Equal(t, `"just text"`, string(raw)) + }) + t.Run("GenericCodeWithCause_CauseIsString", func(t *testing.T) { + m := covErrorsUnmarshalObj(t, &BaseError{Code: "ErrGeneric", Message: "m", Cause: errors.New("inner")}) + assert.Equal(t, "inner", m["cause"]) + assert.Equal(t, "m", m["message"]) + }) + t.Run("JoinedCauses_Array", func(t *testing.T) { + m := covErrorsUnmarshalObj(t, &BaseError{Code: "ErrTop", Message: "m", Cause: errors.Join(NewErrUpstreamSyncing("u1"), errors.New("raw"))}) + causes, ok := m["cause"].([]interface{}) + require.True(t, ok, "%v", m) + require.Len(t, causes, 2) + assert.Equal(t, "ErrUpstreamSyncing", causes[0].(map[string]interface{})["code"]) + assert.Equal(t, "raw", causes[1]) + }) + t.Run("StandardCause_Nested", func(t *testing.T) { + m := covErrorsUnmarshalObj(t, &BaseError{Code: "ErrTop", Message: "m", Cause: NewErrUpstreamSyncing("u1")}) + c := m["cause"].(map[string]interface{}) + assert.Equal(t, "ErrUpstreamSyncing", c["code"]) + assert.Equal(t, "u1", c["details"].(map[string]interface{})["upstreamId"]) + }) + t.Run("PlainCause_WrappedAsGenericMessage", func(t *testing.T) { + // A plain cause is wrapped as ErrGeneric, which itself marshals as its bare message. + m := covErrorsUnmarshalObj(t, &BaseError{Code: "ErrTop", Message: "m", Cause: errors.New("raw")}) + assert.Equal(t, "raw", m["cause"]) + }) + t.Run("NoCause_OmitsCause", func(t *testing.T) { + m := covErrorsUnmarshalObj(t, &BaseError{Code: "ErrTop", Message: "m", Details: map[string]interface{}{"a": "b"}}) + assert.NotContains(t, m, "cause") + assert.Equal(t, map[string]interface{}{"a": "b"}, m["details"]) + }) +} + +func TestCovErrors_BaseError_Zerolog(t *testing.T) { + var nilBase *BaseError + assert.Empty(t, covErrorsZerolog(t, nilBase)) + + m := covErrorsZerolog(t, &BaseError{Code: "ErrTop", Message: "m", Details: map[string]interface{}{"a": "b"}, Cause: NewErrUpstreamSyncing("u")}) + assert.Equal(t, "ErrTop", m["code"]) + assert.Equal(t, "m", m["message"]) + assert.Equal(t, map[string]interface{}{"a": "b"}, m["details"]) + assert.Equal(t, "ErrUpstreamSyncing", m["cause"].(map[string]interface{})["code"]) + + plain := covErrorsZerolog(t, &BaseError{Code: "ErrTop", Cause: errors.New("raw")}) + assert.Equal(t, "raw", plain["cause"]) + assert.NotContains(t, plain, "details") + + multi := covErrorsZerolog(t, &BaseError{Code: "ErrTop", Cause: errors.Join(NewErrUpstreamSyncing("u"), NewErrUpstreamShadowing("v"))}) + causes, ok := multi["cause"].([]interface{}) + require.True(t, ok, "%v", multi) + assert.Len(t, causes, 2) +} + +func TestCovErrors_TaskFatalError(t *testing.T) { + var nilFatal *TaskFatalError + assert.Equal(t, "fatal task error", nilFatal.Error()) + assert.Equal(t, "fatal task error", (&TaskFatalError{}).Error()) + err := NewTaskFatal(covErrorsSentinel) + assert.Equal(t, covErrorsSentinel.Error(), err.Error()) + assert.True(t, errors.Is(err, covErrorsSentinel)) + var tf interface{ IsTaskFatal() bool } + require.True(t, errors.As(covErrorsWrap(err), &tf)) + assert.True(t, tf.IsTaskFatal()) +} + +func covErrorsAllCategoryCauses(t *testing.T) []error { + t.Helper() + u, err := url.Parse("https://rpc.example.com") + require.NoError(t, err) + return []error{ + NewErrEndpointUnsupported(nil), + NewErrEndpointMissingData(nil, nil), + context.DeadlineExceeded, + NewErrEndpointRequestTimeout(time.Second, nil), + NewErrNetworkRequestTimeout(time.Second, nil), + NewErrFailsafeTimeoutExceeded(ScopeUpstream, nil, nil), + NewErrEndpointServerSideException(nil, nil, 0), + NewErrEndpointCapacityExceeded(nil), + NewErrUpstreamRateLimitRuleExceeded("u", "b", "r"), + NewErrFailsafeCircuitBreakerOpen(ScopeUpstream, nil, nil), + NewErrEndpointBillingIssue(nil), + NewErrEndpointTransportFailure(u, errors.New("refused")), + NewErrUpstreamExcludedByPolicy("u"), + NewErrUpstreamMethodIgnored("m", "u"), + NewErrUpstreamRequestSkipped(nil, "u"), + NewErrEndpointRequestTooLarge(nil, EvmBlockRangeTooLarge), + NewErrGetLogsExceededMaxAllowedRange(2, 1), + NewErrGetLogsExceededMaxAllowedAddresses(2, 1), + NewErrGetLogsExceededMaxAllowedTopics(2, 1), + NewErrEndpointUnauthorized(nil), + NewErrUpstreamSyncing("u"), + NewErrUpstreamBlockUnavailable("u", 1, 0, 0), + NewErrEndpointContentValidation(nil, nil), + errors.New("mystery"), + NewErrUpstreamNodeTypeMismatch(nil, EvmNodeTypeArchive, EvmNodeTypeFull), + NewErrUpstreamHedgeCancelled("u", nil), + NewErrEndpointClientSideException(nil), + NewErrJsonRpcRequestUnmarshal(nil, nil), + NewErrInvalidRequest(nil), + NewErrInvalidUrlPath("r", "/p"), + } +} + +func TestCovErrors_UpstreamsExhausted_SummarizesEveryCategory(t *testing.T) { + ers := &sync.Map{} + for i, e := range covErrorsAllCategoryCauses(t) { + ers.Store(string(rune('a'+i)), e) + } + ers.Store("zz-nil", nil) + ers.Store("zz-not-error", "just a string") + + err := NewErrUpstreamsExhausted(nil, ers, "main", "evm:1", "eth_call", 2*time.Second, 5, 4, 1, 30) + ex := err.(*ErrUpstreamsExhausted) + + want := "1 upstream unsupported method, 1 upstream missing data, 4 upstream timeout, 1 upstream server errors, " + + "2 upstream rate limited, 1 upstream circuit breaker open, 1 upstream billing issues, 1 upstream transport errors, " + + "1 upstream excluded by policy, 1 upstream method ignored, 1 upstream skipped, 4 upstream too large complaints, " + + "1 upstream unauthorized, 2 upstream not synced, 1 upstream validation mismatch, 1 upstream unknown errors, " + + "1 node type mismatches, 1 hedges cancelled, 4 user errors" + assert.Equal(t, want, ex.SummarizeCauses()) + assert.Equal(t, "all upstream attempts failed ("+want+")", ex.Message) + assert.Equal(t, want, ex.DeepestMessage()) + assert.Equal(t, "ErrUpstreamsExhausted", ex.CodeChain()) + assert.Len(t, ex.Errors(), 30, "nil and non-error entries are dropped") + assert.Equal(t, 5, ex.Attempts()) + assert.Equal(t, 4, ex.Retries()) + assert.Equal(t, 1, ex.Hedges()) + assert.Equal(t, int64(2000), ex.Details["durationMs"]) + assert.Equal(t, 30, ex.Details["upstreams"]) + assert.False(t, ex.FromCache()) + assert.False(t, ex.IsObjectNull()) + assert.Nil(t, ex.Request()) + assert.True(t, HasErrorCode(err, ErrCodeUpstreamsExhausted, ErrCodeEndpointBillingIssue)) + assert.True(t, errors.Is(err, context.DeadlineExceeded), "joined causes are reachable via errors.Is") + + m := covErrorsUnmarshalObj(t, err) + causes, ok := m["cause"].([]interface{}) + require.True(t, ok) + assert.Len(t, causes, 30) +} + +func TestCovErrors_UpstreamsExhausted_OrderIsDeterministic(t *testing.T) { + upB := NewFakeUpstream("b-up") + upA := NewFakeUpstream("a-up") + upC := NewFakeUpstream("c-up") + + ers := &sync.Map{} + // Upstream keys (sorted by Id), string keys, and other keys falling back to the error. + ers.Store(upB, NewErrUpstreamRequest(NewErrEndpointServerSideException(nil, nil, 0), upB, "evm:1", "eth_call", 0, 1, 0, 0)) + ers.Store(upA, NewErrUpstreamRequest(NewErrEndpointServerSideException(nil, nil, 0), upA, "evm:1", "eth_call", 0, 1, 0, 0)) + // Non-retryable sorts after every retryable cause, despite the lowest key. + ers.Store("0-terminal", NewErrEndpointExecutionException(nil)) + // Unkeyed (int key): id taken from the upstream carried on the error. + ers.Store(1, NewErrUpstreamRequest(NewErrEndpointServerSideException(nil, nil, 0), upC, "evm:1", "eth_call", 0, 1, 0, 0)) + // Unkeyed with no upstream: falls back to error text; ties broken by text. + ers.Store(2, errors.New("zz plain 2")) + ers.Store(3, errors.New("zz plain 1")) + + for range 5 { + err := NewErrUpstreamsExhausted(nil, ers, "main", "evm:1", "eth_call", 0, 1, 0, 0, 6).(*ErrUpstreamsExhausted) + children := err.Errors() + require.Len(t, children, 6) + assert.Equal(t, "a-up", children[0].(*ErrUpstreamRequest).UpstreamId()) + assert.Equal(t, "b-up", children[1].(*ErrUpstreamRequest).UpstreamId()) + assert.Equal(t, "c-up", children[2].(*ErrUpstreamRequest).UpstreamId()) + assert.Equal(t, "zz plain 1", children[3].Error()) + assert.Equal(t, "zz plain 2", children[4].Error()) + assert.True(t, HasErrorCode(children[5], ErrCodeEndpointExecutionException)) + + ups := err.Upstreams() + require.Len(t, ups, 3) + assert.Equal(t, []string{"a-up", "b-up", "c-up"}, []string{ups[0].Id(), ups[1].Id(), ups[2].Id()}) + assert.Equal(t, "a-up", err.UpstreamId(), "first upstreamId found in children") + } + assert.Nil(t, orderCauses(nil)) +} + +func TestCovErrors_UpstreamsExhausted_EdgeShapes(t *testing.T) { + var nilEx *ErrUpstreamsExhausted + assert.Nil(t, nilEx.Upstreams()) + assert.Nil(t, nilEx.Errors()) + assert.True(t, nilEx.IsObjectNull()) + + empty := NewErrUpstreamsExhausted(nil, &sync.Map{}, "p", "n", "m", 0, 0, 0, 0, 0).(*ErrUpstreamsExhausted) + assert.Equal(t, "all upstream attempts failed", empty.Message, "no causes: no summary suffix") + assert.Nil(t, empty.Cause) + assert.Equal(t, empty.Message, empty.DeepestMessage()) + assert.Equal(t, "ErrUpstreamsExhausted", empty.CodeChain()) + assert.Empty(t, empty.Upstreams()) + assert.Equal(t, "", empty.UpstreamId()) + assert.False(t, IsRetryableTowardNetwork(empty)) + assert.False(t, IsRetryableTowardsUpstream(empty)) + + plainCause := NewErrUpstreamsExhaustedWithCause(errors.New("raw")).(*ErrUpstreamsExhausted) + assert.Equal(t, "", plainCause.SummarizeCauses()) + assert.Equal(t, "", plainCause.DeepestMessage()) + assert.Nil(t, plainCause.Errors()) + assert.Equal(t, "ErrUpstreamsExhausted", plainCause.CodeChain()) + assert.Equal(t, 0, plainCause.Attempts()) + assert.Equal(t, 0, plainCause.Retries()) + assert.Equal(t, 0, plainCause.Hedges()) + + emptyMulti := NewErrUpstreamsExhaustedWithCause(covErrorsEmptyMulti{}).(*ErrUpstreamsExhausted) + assert.Equal(t, "", emptyMulti.DeepestMessage()) + assert.Empty(t, emptyMulti.Errors()) + + wrongTypes := &ErrUpstreamsExhausted{BaseError: BaseError{Details: map[string]interface{}{"attempts": "1", "retries": "1", "hedges": "1", "upstreamId": 3}}} + assert.Equal(t, 0, wrongTypes.Attempts()) + assert.Equal(t, 0, wrongTypes.Retries()) + assert.Equal(t, 0, wrongTypes.Hedges()) + assert.Equal(t, "", wrongTypes.UpstreamId()) + assert.True(t, wrongTypes.IsObjectNull()) +} + +func TestCovErrors_UpstreamsExhausted_Retryability(t *testing.T) { + build := func(children ...error) error { + ers := &sync.Map{} + for i, c := range children { + ers.Store(string(rune('a'+i)), c) + } + return NewErrUpstreamsExhausted(nil, ers, "p", "n", "m", 0, 0, 0, 0, len(children)) + } + + onlyVerdicts := build(NewErrEndpointUnsupported(nil), NewErrUpstreamMethodIgnored("m", "u")) + assert.False(t, IsRetryableTowardNetwork(onlyVerdicts), "every upstream said it does not serve the method") + assert.False(t, IsRetryableTowardsUpstream(onlyVerdicts)) + + mixed := build(NewErrEndpointUnsupported(nil), NewErrEndpointServerSideException(nil, nil, 0)) + assert.True(t, IsRetryableTowardNetwork(mixed)) + assert.True(t, IsRetryableTowardsUpstream(mixed)) + + wrappedVerdict := build(NewErrUpstreamRequest(NewErrEndpointUnsupported(nil), nil, "n", "m", 0, 1, 0, 0)) + assert.False(t, IsRetryableTowardNetwork(wrappedVerdict), "verdict is found through single-cause wrappers") + + // A verdict check never descends into a nested multi-error. + nestedMulti := build(&BaseError{Code: "ErrWrap", Cause: errors.Join(NewErrEndpointUnsupported(nil))}) + assert.False(t, IsRetryableTowardNetwork(nestedMulti), "nested child is non-retryable verdict-only bundle") + + nonTerminalNested := build(&BaseError{Code: "ErrWrap", Cause: errors.Join(NewErrEndpointServerSideException(nil, nil, 0))}) + assert.True(t, IsRetryableTowardNetwork(nonTerminalNested)) + + assert.True(t, IsRetryableTowardNetwork(errors.New("plain"))) + assert.False(t, isUpstreamMethodVerdict(errors.New("plain"))) + assert.False(t, isUpstreamMethodVerdict(&BaseError{Code: "ErrWrap", Cause: errors.Join(NewErrEndpointUnsupported(nil))})) + assert.False(t, isUpstreamMethodVerdict(nil)) + + // Flag below a multi-error wrapper is not consulted by the linear walk. + flagged := &BaseError{Code: "ErrWrap", Cause: errors.Join( + NewErrEndpointExecutionException(nil), + NewErrEndpointServerSideException(nil, nil, 0), + )} + assert.True(t, IsRetryableTowardNetwork(flagged)) + assert.False(t, IsPermanentlyMissingData(&BaseError{Code: "ErrWrap", Cause: errors.Join( + (&BaseError{Code: ErrCodeEndpointMissingData}).WithPermanentMissingData(true), + )})) + assert.True(t, IsPermanentlyMissingData(NewErrUpstreamRequest( + (&BaseError{Code: ErrCodeEndpointMissingData}).WithPermanentMissingData(true), nil, "n", "m", 0, 0, 0, 0))) +} + +func TestCovErrors_NoUpstreamsLeftToSelect_States(t *testing.T) { + req := NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":[]}`)) + u1, u2, u3, u4 := NewFakeUpstream("u1"), NewFakeUpstream("u2"), NewFakeUpstream("u3"), NewFakeUpstream("u4") + req.SetUpstreams([]Upstream{u1, u2, u3, u4}) + req.ErrorsByUpstream.Store(u1, NewErrEndpointMissingData(nil, u1)) + req.ErrorsByUpstream.Store(u2, NewErrEndpointServerSideException(nil, nil, 0)) + req.ConsumedUpstreams.Store(u3, true) + + err := NewErrNoUpstreamsLeftToSelect(req, "all consumed") + be := err.(*ErrNoUpstreamsLeftToSelect) + assert.Equal(t, ErrCodeNoUpstreamsLeftToSelect, be.Code) + assert.Equal(t, "no upstreams left to select, all consumed", be.Message) + assert.Equal(t, map[string]interface{}{ + "method": "eth_getBalance", + "u1": "empty_response", + "u2": "errored", + "u3": "consumed", + "u4": "available", + }, be.Details) +} + +func TestCovErrors_JsonRpcExceptionInternal_Codes(t *testing.T) { + e := NewErrJsonRpcExceptionInternal(-32005, JsonRpcErrorCapacityExceeded, "limited", nil, nil) + assert.Equal(t, JsonRpcErrorCapacityExceeded, e.NormalizedCode()) + assert.Equal(t, -32005, e.OriginalCode()) + assert.Equal(t, "-32005 <- ErrJsonRpcExceptionInternal", e.CodeChain()) + + bare := NewErrJsonRpcExceptionInternal(0, 0, "none", nil, nil) + assert.NotContains(t, bare.Details, "originalCode") + assert.NotContains(t, bare.Details, "normalizedCode") + assert.Equal(t, JsonRpcErrorNumber(0), bare.NormalizedCode()) + assert.Equal(t, 0, bare.OriginalCode()) + + fromJSON := NewErrJsonRpcExceptionInternal(0, 0, "f", nil, map[string]interface{}{"originalCode": float64(-32000)}) + assert.Equal(t, -32000, fromJSON.OriginalCode(), "float64 from decoded JSON is accepted") + weird := NewErrJsonRpcExceptionInternal(0, 0, "w", nil, map[string]interface{}{"originalCode": "x"}) + assert.Equal(t, 0, weird.OriginalCode()) +} + +func TestCovErrors_JsonRpcExceptionExternal(t *testing.T) { + e := NewErrJsonRpcExceptionExternal(-32000, "execution reverted", "0x08c379a0") + assert.Equal(t, "-32000: execution reverted", e.Error()) + assert.Equal(t, "-32000", e.CodeChain()) + assert.False(t, e.HasCode("-32000")) + assert.Equal(t, "execution reverted", e.DeepestMessage()) + assert.Nil(t, e.GetCause()) + + m := covErrorsUnmarshalObj(t, e) + assert.Equal(t, map[string]interface{}{"code": float64(-32000), "message": "execution reverted", "data": "0x08c379a0"}, m) + + z := covErrorsZerolog(t, e) + assert.Equal(t, float64(-32000), z["code"]) + assert.Equal(t, "0x08c379a0", z["data"]) + + noData := covErrorsZerolog(t, &ErrJsonRpcExceptionExternal{Code: 1, Message: "m"}) + assert.NotContains(t, noData, "data") + + var nilExt *ErrJsonRpcExceptionExternal + assert.Empty(t, covErrorsZerolog(t, nilExt)) +} + +func TestCovErrors_ConsensusErrors(t *testing.T) { + type multi interface { + Errors() []error + } + causes := []error{NewErrUpstreamSyncing("a"), errors.New("b")} + for name, ctor := range map[string]func(string, []ParticipantInfo, []error) error{ + "Dispute": NewErrConsensusDispute, + "CompositionDispute": NewErrConsensusCompositionDispute, + "LowParticipants": NewErrConsensusLowParticipants, + } { + t.Run(name, func(t *testing.T) { + withCauses := ctor("msg", nil, causes).(multi) + assert.Len(t, withCauses.Errors(), 2) + assert.True(t, HasErrorCode(withCauses.(error), ErrCodeUpstreamSyncing)) + assert.Nil(t, ctor("msg", nil, nil).(multi).Errors(), "no causes: nil") + + var se StandardError + require.True(t, errors.As(withCauses.(error), &se)) + se.Base().Cause = errors.New("single") + assert.Nil(t, withCauses.Errors(), "non-multi cause: nil") + }) + } + + participants := []ParticipantInfo{ + {Upstream: "a", ResultHash: "h1"}, + {Upstream: "b"}, + {}, + {Upstream: "c", ErrSummary: "ErrTimeout"}, + {ErrSummary: "orphan"}, + } + low := NewErrConsensusLowParticipants("not enough", participants, nil).(*ErrConsensusLowParticipants) + assert.Equal(t, "[a = h1, b = NoResult, c = ErrTimeout, orphan]", low.SummarizeParticipants()) + assert.Equal(t, "ErrConsensusLowParticipants: not enough: [a = h1, b = NoResult, c = ErrTimeout, orphan]", low.DeepestMessage()) + + assert.Equal(t, "", (&ErrConsensusLowParticipants{}).SummarizeParticipants()) + wrong := &ErrConsensusLowParticipants{BaseError{Details: map[string]interface{}{"participants": "x"}}} + assert.Equal(t, "", wrong.SummarizeParticipants()) + + dispute := NewErrConsensusDispute("disagree", nil, []error{NewErrEndpointExecutionException(nil)}) + assert.False(t, IsRetryableTowardNetwork(dispute), "all-terminal dispute children are not retryable") +} + +func TestCovErrors_HasErrorCode_Shapes(t *testing.T) { + assert.False(t, HasErrorCode(nil, ErrCodeUpstreamSyncing)) + assert.True(t, HasErrorCode(errors.Join(errors.New("x"), NewErrUpstreamSyncing("u")), ErrCodeUpstreamSyncing)) + assert.False(t, HasErrorCode(errors.Join(errors.New("x")), ErrCodeUpstreamSyncing)) + assert.False(t, HasErrorCode(errors.New("x"), ErrCodeUpstreamSyncing)) + assert.False(t, HasErrorCode(covErrorsWrap(NewErrUpstreamSyncing("u")), ErrCodeUpstreamSyncing), "fmt %w wrappers are not traversed") +} + +func TestCovErrors_OrderCauses_KeyTieFallsBackToText(t *testing.T) { + up := NewFakeUpstream("same-id") + ers := &sync.Map{} + // An Upstream key and a string key resolve to the same sort key; text breaks the tie. + ers.Store(up, errors.New("zz second")) + ers.Store("same-id", errors.New("aa first")) + got := orderCauses(ers) + require.Len(t, got, 2) + assert.Equal(t, "aa first", got[0].Error()) + assert.Equal(t, "zz second", got[1].Error()) +} + +func TestCovErrors_IsRetryableTowardNetwork_EmptyMultiCause(t *testing.T) { + // A multi-error cause with no children is not a verdict bundle; the linear + // walk stops at it, so only the outer flag counts. + outer := &BaseError{Code: "ErrWrap", Cause: covErrorsEmptyMulti{}} + assert.True(t, IsRetryableTowardNetwork(outer)) + outer.WithRetryableTowardNetwork(false) + assert.False(t, IsRetryableTowardNetwork(outer)) +} diff --git a/common/cov_jsonrpc_helpers_test.go b/common/cov_jsonrpc_helpers_test.go new file mode 100644 index 000000000..275c3ed6f --- /dev/null +++ b/common/cov_jsonrpc_helpers_test.go @@ -0,0 +1,116 @@ +package common + +import ( + "bytes" + "context" + "errors" + "io" + + "github.com/rs/zerolog" +) + +// covJsonrpcEnableLogs lifts the package-wide disabled log level (set by the +// test init) for the duration of one test so log output can be asserted. +func covJsonrpcEnableLogs(t interface{ Cleanup(func()) }) { + prev := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.TraceLevel) + t.Cleanup(func() { zerolog.SetGlobalLevel(prev) }) +} + +// covJsonrpcNetwork is a minimal Network for request/response tests: it only +// answers identity, config, logger and finality. +type covJsonrpcNetwork struct { + Network + id string + label string + cfg *NetworkConfig + finality DataFinalityState + finalityCalls int + logger zerolog.Logger +} + +func (n *covJsonrpcNetwork) Id() string { return n.id } +func (n *covJsonrpcNetwork) Label() string { return n.label } +func (n *covJsonrpcNetwork) Config() *NetworkConfig { return n.cfg } +func (n *covJsonrpcNetwork) Logger() *zerolog.Logger { return &n.logger } +func (n *covJsonrpcNetwork) GetFinality(_ context.Context, _ *NormalizedRequest, _ *NormalizedResponse) DataFinalityState { + n.finalityCalls++ + return n.finality +} + +// covJsonrpcUpstream is an Upstream whose Config may be nil, to exercise the +// "upstream without config" branches. +type covJsonrpcUpstream struct { + Upstream + id string + cfg *UpstreamConfig +} + +func (u *covJsonrpcUpstream) Id() string { return u.id } +func (u *covJsonrpcUpstream) Config() *UpstreamConfig { return u.cfg } + +// covJsonrpcByteWriter is a util.ByteWriter (and ReleasableByteWriter) backed by +// a fixed payload, with optional injected errors. +type covJsonrpcByteWriter struct { + data []byte + writeErr error + sizeErr error + released bool + emptyish bool + writeCall int +} + +func (w *covJsonrpcByteWriter) WriteTo(out io.Writer, trimSides bool) (int64, error) { + w.writeCall++ + if w.writeErr != nil { + return 0, w.writeErr + } + b := w.data + if trimSides && len(b) >= 2 { + b = b[1 : len(b)-1] + } + n, err := out.Write(b) + return int64(n), err +} +func (w *covJsonrpcByteWriter) IsResultEmptyish() bool { return w.emptyish } +func (w *covJsonrpcByteWriter) Size(_ ...context.Context) (int, error) { + if w.sizeErr != nil { + return 0, w.sizeErr + } + return len(w.data), nil +} +func (w *covJsonrpcByteWriter) Release() { w.released = true } + +// covJsonrpcFailWriter fails the Nth Write call (1-based) and accepts the rest. +type covJsonrpcFailWriter struct { + failAt int + calls int + buf bytes.Buffer +} + +var errCovJsonrpcWrite = errors.New("cov-jsonrpc write failed") + +func (w *covJsonrpcFailWriter) Write(p []byte) (int, error) { + w.calls++ + if w.calls == w.failAt { + return 0, errCovJsonrpcWrite + } + return w.buf.Write(p) +} + +// covJsonrpcBody is a ReadCloser with a controllable Close error. +type covJsonrpcBody struct { + io.Reader + closeErr error + closed int +} + +func (b *covJsonrpcBody) Close() error { + b.closed++ + return b.closeErr +} + +// covJsonrpcErrReader always fails to read. +type covJsonrpcErrReader struct{} + +func (covJsonrpcErrReader) Read([]byte) (int, error) { return 0, errors.New("cov-jsonrpc read failed") } diff --git a/common/cov_jsonrpc_jsonrpc_test.go b/common/cov_jsonrpc_jsonrpc_test.go new file mode 100644 index 000000000..a19492ee6 --- /dev/null +++ b/common/cov_jsonrpc_jsonrpc_test.go @@ -0,0 +1,866 @@ +package common + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "math" + "strings" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func covJsonrpcWrite(t *testing.T, r *JsonRpcResponse) string { + t.Helper() + var buf bytes.Buffer + n, err := r.WriteTo(&buf) + require.NoError(t, err) + require.Equal(t, int64(buf.Len()), n) + return buf.String() +} + +func TestCovJsonrpc_NewJsonRpcResponse(t *testing.T) { + r, err := NewJsonRpcResponse(1, map[string]string{"a": "b"}, nil) + require.NoError(t, err) + assert.Equal(t, `{"jsonrpc":"2.0","id":1,"result":{"a":"b"}}`, covJsonrpcWrite(t, r)) + + _, err = NewJsonRpcResponse(1, math.Inf(1), nil) + assert.Error(t, err, "unmarshalable result must be rejected") + _, err = NewJsonRpcResponse(make(chan int), "x", nil) + assert.Error(t, err, "unmarshalable id must be rejected") + + assert.Panics(t, func() { MustNewJsonRpcResponse(1, make(chan int), nil) }) + assert.Panics(t, func() { MustNewJsonRpcResponseFromBytes([]byte(`{`), nil, nil) }) +} + +func TestCovJsonrpc_NewFromBytes_IdForms(t *testing.T) { + cases := []struct { + name string + id string + wantID interface{} + errSub string + }{ + {"number", `42`, int64(42), ""}, + {"string", `"abc"`, "abc", ""}, + {"null", `null`, nil, ""}, + {"object", `{"x":1}`, nil, "unsupported ID type"}, + {"invalid", `{`, nil, "?"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r, err := NewJsonRpcResponseFromBytes([]byte(tc.id), []byte(`1`), nil) + if tc.errSub != "" { + require.Error(t, err) + if tc.errSub != "?" { + assert.Contains(t, err.Error(), tc.errSub) + } + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantID, r.ID()) + // Wire output keeps the id bytes verbatim. + assert.Contains(t, covJsonrpcWrite(t, r), `"id":`+tc.id+`,`) + }) + } + + t.Run("large id keeps verbatim bytes on wire", func(t *testing.T) { + r, err := NewJsonRpcResponseFromBytes([]byte(`18446744073709551615`), []byte(`1`), nil) + require.NoError(t, err) + assert.Contains(t, covJsonrpcWrite(t, r), `"id":18446744073709551615,`) + }) + + t.Run("no id at all", func(t *testing.T) { + r, err := NewJsonRpcResponseFromBytes(nil, []byte(`1`), nil) + require.NoError(t, err) + assert.Nil(t, r.ID()) + }) +} + +func TestCovJsonrpc_SetIDAndSetIDBytes(t *testing.T) { + r := &JsonRpcResponse{} + require.NoError(t, r.SetID("x-1")) + assert.Equal(t, "x-1", r.ID()) + assert.Error(t, r.SetID(make(chan int))) + + r2 := &JsonRpcResponse{} + require.NoError(t, r2.SetIDBytes([]byte(`1.0`))) + assert.Equal(t, int64(1), r2.ID()) + assert.Equal(t, `{"jsonrpc":"2.0","id":1.0}`, covJsonrpcWrite(t, r2), "fractional id written verbatim") + assert.Error(t, r2.SetIDBytes([]byte(`[1]`))) + + // ID() lazily decodes idBytes when no typed id is set. + r3 := &JsonRpcResponse{idBytes: []byte(`"lazy"`)} + assert.Equal(t, "lazy", r3.ID()) + r4 := &JsonRpcResponse{idBytes: []byte(`{bad`)} + assert.Nil(t, r4.ID(), "undecodable id bytes yield nil") +} + +func TestCovJsonrpc_ParseFromStream(t *testing.T) { + t.Run("read error", func(t *testing.T) { + r := &JsonRpcResponse{} + err := r.ParseFromStream(nil, covJsonrpcErrReader{}, 0) + assert.ErrorContains(t, err, "cov-jsonrpc read failed") + }) + + t.Run("invalid json keeps raw data as result", func(t *testing.T) { + r := &JsonRpcResponse{} + err := r.ParseFromStream([]context.Context{context.Background()}, strings.NewReader(`oops`), 4) + require.Error(t, err) + assert.Equal(t, `oops`, r.GetResultString()) + }) + + t.Run("unsupported id type", func(t *testing.T) { + r := &JsonRpcResponse{} + err := r.ParseFromStream(nil, strings.NewReader(`{"id":[1],"result":1}`), 0) + assert.ErrorContains(t, err, "unsupported ID type") + }) + + t.Run("error object", func(t *testing.T) { + r := &JsonRpcResponse{} + require.NoError(t, r.ParseFromStream(nil, strings.NewReader(`{"id":3,"error":{"code":-32000,"message":"execution reverted","data":"0x08c379a0"}}`), 0)) + require.NotNil(t, r.Error) + assert.Equal(t, -32000, r.Error.Code) + assert.Equal(t, "execution reverted", r.Error.Message) + assert.Equal(t, "0x08c379a0", r.Error.Data) + assert.Equal(t, `{"jsonrpc":"2.0","id":3,"error":{"code":-32000,"message":"execution reverted","data":"0x08c379a0"}}`, covJsonrpcWrite(t, r)) + }) +} + +func TestCovJsonrpc_ParseError_Shapes(t *testing.T) { + cases := []struct { + name string + raw string + code int + message string + data interface{} + keepRaw bool + }{ + {"standard", `{"code":-32602,"message":"bad params"}`, -32602, "bad params", nil, true}, + {"nested data", `{"code":3,"message":"reverted","data":{"x":[1]}}`, 3, "reverted", map[string]interface{}{"x": []interface{}{float64(1)}}, true}, + {"empty", ``, int(JsonRpcErrorServerSideException), "unexpected empty response from upstream endpoint", "", false}, + // A literal null error body is decoded as an all-zero object, so it + // falls through to the raw-text message rather than the empty-response one. + {"null", `null`, int(JsonRpcErrorServerSideException), "null", "", false}, + {"only error string", `{"error":"rate limited"}`, int(JsonRpcErrorServerSideException), "rate limited", "", false}, + {"plain string", `"boom"`, int(JsonRpcErrorServerSideException), `"boom"`, "", false}, + {"empty object", `{}`, int(JsonRpcErrorServerSideException), `{}`, "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := &JsonRpcResponse{} + require.NoError(t, r.ParseError(tc.raw)) + require.NotNil(t, r.Error) + assert.Equal(t, tc.code, r.Error.Code) + assert.Equal(t, tc.message, r.Error.Message) + assert.Equal(t, tc.data, r.Error.Data) + if tc.keepRaw { + assert.Equal(t, tc.raw, string(r.errBytes)) + } else { + assert.Nil(t, r.errBytes) + } + }) + } + + t.Run("from bytes ctor", func(t *testing.T) { + r, err := NewJsonRpcResponseFromBytes([]byte(`1`), nil, []byte(`{"error":"nope"}`)) + require.NoError(t, err) + assert.Equal(t, "nope", r.Error.Message) + }) +} + +func TestCovJsonrpc_PeekByPath(t *testing.T) { + ctx := context.Background() + r := MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`{"number":"0x10","tx":[{"h":"0xa"}]}`), nil) + + s, err := r.PeekStringByPath(ctx, "number") + require.NoError(t, err) + assert.Equal(t, "0x10", s) + + s, err = r.PeekStringByPath(ctx, "tx", 0, "h") + require.NoError(t, err) + assert.Equal(t, "0xa", s) + + _, err = r.PeekStringByPath(ctx, "missing") + assert.Error(t, err) + + b, err := r.PeekBytesByPath(ctx, "tx", 0) + require.NoError(t, err) + assert.Equal(t, `{"h":"0xa"}`, string(b)) + + _, err = r.PeekBytesByPath(ctx, "nope") + assert.Error(t, err) + + t.Run("materializes from result writer", func(t *testing.T) { + w := &covJsonrpcByteWriter{data: []byte(`{"blockNumber":"0x5"}`)} + r := &JsonRpcResponse{} + r.SetResultWriter(w) + s, err := r.PeekStringByPath(ctx, "blockNumber") + require.NoError(t, err) + assert.Equal(t, "0x5", s) + assert.Equal(t, `{"blockNumber":"0x5"}`, r.GetResultString(), "writer is materialized into result") + }) + + t.Run("writer failure is surfaced", func(t *testing.T) { + r := &JsonRpcResponse{} + r.SetResultWriter(&covJsonrpcByteWriter{writeErr: errors.New("writer dead")}) + _, err := r.PeekStringByPath(ctx, "x") + assert.ErrorContains(t, err, "writer dead") + _, err = r.PeekBytesByPath(ctx, "x") + assert.ErrorContains(t, err, "writer dead") + }) + + t.Run("empty result", func(t *testing.T) { + r := &JsonRpcResponse{} + _, err := r.PeekStringByPath(ctx, "x") + assert.Error(t, err) + _, err = r.PeekBytesByPath(ctx, "x") + assert.Error(t, err) + }) +} + +func TestCovJsonrpc_SizeAndResultLength(t *testing.T) { + var nilR *JsonRpcResponse + sz, err := nilR.Size() + assert.Zero(t, sz) + assert.NoError(t, err) + + r := &JsonRpcResponse{} + sz, err = r.Size() + assert.Zero(t, sz) + assert.NoError(t, err) + assert.Zero(t, r.ResultLength()) + + r.SetResultWriter(&covJsonrpcByteWriter{data: []byte(`[1,2,3]`)}) + sz, err = r.Size() + require.NoError(t, err) + assert.Equal(t, 7, sz) + assert.Equal(t, 7, r.ResultLength()) + + bad := &JsonRpcResponse{} + bad.SetResultWriter(&covJsonrpcByteWriter{data: []byte(`x`), sizeErr: errors.New("size unknown")}) + _, err = bad.Size() + assert.ErrorContains(t, err, "size unknown") + assert.Zero(t, bad.ResultLength(), "size error reports zero length") + + r.SetResult([]byte(`"0x1"`)) + assert.Equal(t, 5, r.ResultLength()) +} + +func TestCovJsonrpc_Free(t *testing.T) { + var nilR *JsonRpcResponse + nilR.Free() + + w := &covJsonrpcByteWriter{data: []byte(`1`)} + r := MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`{"a":"0x1"}`), []byte(`{"code":1,"message":"m"}`)) + _, err := r.CanonicalHash() + require.NoError(t, err) + r.SetResultWriter(w) + r.Free() + assert.True(t, w.released, "releasable writer is released") + assert.Nil(t, r.GetResultBytes()) + assert.Zero(t, r.ResultLength()) + assert.Equal(t, int64(1), r.ID(), "typed id survives Free") + _, ok := r.canonicalHashWithIgnored.Load(defaultCanonicalHashPlaceholder) + assert.False(t, ok, "hash cache cleared") +} + +func TestCovJsonrpc_MarshalZerologObject(t *testing.T) { + covJsonrpcEnableLogs(t) + render := func(r *JsonRpcResponse) string { + var buf bytes.Buffer + lg := zerolog.New(&buf) + lg.Info().Object("r", r).Send() + return buf.String() + } + var nilR *JsonRpcResponse + assert.Contains(t, render(nilR), `"r":{}`) + + r := MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`{"x":1}`), []byte(`{"code":1,"message":"m"}`)) + out := render(r) + assert.Contains(t, out, `"error":{"code":1,"message":"m"}`) + assert.Contains(t, out, `"result":{"x":1}`) + + notJson := &JsonRpcResponse{errBytes: []byte(``), result: []byte(`123`)} + out = render(notJson) + assert.Contains(t, out, `"error":""`) + assert.Contains(t, out, `"result":"123"`) + + internalErr := &JsonRpcResponse{Error: NewErrJsonRpcExceptionExternal(-1, "internal", "")} + assert.Contains(t, render(internalErr), `"message":"internal"`) + + big := bytes.Repeat([]byte("a"), 320*1024) + bigR := &JsonRpcResponse{result: big} + out = render(bigR) + assert.Contains(t, out, `"resultHead"`) + assert.Contains(t, out, `"resultTail"`) + assert.NotContains(t, out, `"result":`) + + wr := &JsonRpcResponse{} + wr.SetResultWriter(&covJsonrpcByteWriter{emptyish: true}) + assert.Contains(t, render(wr), `"resultWriterEmpty":true`) +} + +func TestCovJsonrpc_WriteTo(t *testing.T) { + t.Run("internal error marshalled lazily", func(t *testing.T) { + r := &JsonRpcResponse{idBytes: []byte(`1`), Error: NewErrJsonRpcExceptionExternal(-32000, "boom", "")} + assert.Equal(t, `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"boom","data":""}}`, covJsonrpcWrite(t, r)) + }) + + t.Run("streamed result", func(t *testing.T) { + r := &JsonRpcResponse{idBytes: []byte(`"a"`)} + r.SetResultWriter(&covJsonrpcByteWriter{data: []byte(`[1]`)}) + assert.Equal(t, `{"jsonrpc":"2.0","id":"a","result":[1]}`, covJsonrpcWrite(t, r)) + }) + + t.Run("streamed result failure", func(t *testing.T) { + r := &JsonRpcResponse{idBytes: []byte(`1`)} + r.SetResultWriter(&covJsonrpcByteWriter{writeErr: errors.New("stream fail")}) + _, err := r.WriteTo(&bytes.Buffer{}) + assert.ErrorContains(t, err, "stream fail") + }) + + t.Run("unmarshalable internal error", func(t *testing.T) { + r := &JsonRpcResponse{idBytes: []byte(`1`), Error: &ErrJsonRpcExceptionExternal{Code: 1, Data: make(chan int)}} + _, err := r.WriteTo(&bytes.Buffer{}) + assert.Error(t, err) + }) + + // Each write step must propagate a writer failure. + type shape struct { + name string + mk func() *JsonRpcResponse + writes int + } + shapes := []shape{ + {"errBytes+result", func() *JsonRpcResponse { + return MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`2`), []byte(`{"code":1,"message":"m"}`)) + }, 6}, + {"lazy error", func() *JsonRpcResponse { + return &JsonRpcResponse{idBytes: []byte(`1`), Error: NewErrJsonRpcExceptionExternal(1, "m", "")} + }, 4}, + {"writer result", func() *JsonRpcResponse { + r := &JsonRpcResponse{idBytes: []byte(`1`)} + r.SetResultWriter(&covJsonrpcByteWriter{data: []byte(`3`)}) + return r + }, 4}, + } + for _, s := range shapes { + for i := 1; i <= s.writes; i++ { + w := &covJsonrpcFailWriter{failAt: i} + _, err := s.mk().WriteTo(w) + assert.ErrorIs(t, err, errCovJsonrpcWrite, "%s: failing write #%d must propagate", s.name, i) + } + } +} + +func TestCovJsonrpc_WriteResultTo(t *testing.T) { + var buf bytes.Buffer + r := &JsonRpcResponse{result: []byte(`[1,2]`)} + n, err := r.WriteResultTo(&buf, false) + require.NoError(t, err) + assert.Equal(t, int64(5), n) + assert.Equal(t, `[1,2]`, buf.String()) + + buf.Reset() + _, err = r.WriteResultTo(&buf, true) + require.NoError(t, err) + assert.Equal(t, `1,2`, buf.String()) + + buf.Reset() + n, err = (&JsonRpcResponse{result: []byte(`[]`)}).WriteResultTo(&buf, true) + require.NoError(t, err) + assert.Zero(t, n) + + n, err = (&JsonRpcResponse{}).WriteResultTo(&buf, false) + require.NoError(t, err) + assert.Zero(t, n) + + buf.Reset() + wr := &JsonRpcResponse{} + wr.SetResultWriter(&covJsonrpcByteWriter{data: []byte(`[7,8]`)}) + _, err = wr.WriteResultTo(&buf, true) + require.NoError(t, err) + assert.Equal(t, `7,8`, buf.String()) +} + +func TestCovJsonrpc_Clone(t *testing.T) { + var nilR *JsonRpcResponse + c, err := nilR.Clone() + assert.Nil(t, c) + assert.NoError(t, err) + + orig := MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`{"a":"0x1"}`), []byte(`{"code":5,"message":"m"}`)) + h, err := orig.CanonicalHash() + require.NoError(t, err) + c, err = orig.Clone() + require.NoError(t, err) + assert.Equal(t, covJsonrpcWrite(t, orig), covJsonrpcWrite(t, c)) + ch, ok := c.canonicalHashWithIgnored.Load(defaultCanonicalHashPlaceholder) + assert.True(t, ok) + assert.Equal(t, h, ch) + // Deep copy: mutating clone bytes never touches the original. + c.result[1] = 'Z' + c.errBytes[0] = '[' + assert.Equal(t, `{"a":"0x1"}`, orig.GetResultString()) + assert.Equal(t, byte('{'), orig.errBytes[0]) + + wr := &JsonRpcResponse{idBytes: []byte(`2`)} + w := &covJsonrpcByteWriter{data: []byte(`"0xff"`)} + wr.SetResultWriter(w) + c2, err := wr.Clone() + require.NoError(t, err) + assert.Equal(t, `"0xff"`, c2.GetResultString()) + assert.Nil(t, c2.resultWriter) +} + +func TestCovJsonrpc_IsResultEmptyish(t *testing.T) { + ctx := context.Background() + var nilR *JsonRpcResponse + assert.True(t, nilR.IsResultEmptyish()) + assert.True(t, nilR.IsResultEmptyish(ctx)) + assert.True(t, (&JsonRpcResponse{}).IsResultEmptyish(ctx)) + assert.False(t, (&JsonRpcResponse{result: []byte(`"0x1"`)}).IsResultEmptyish(ctx)) + assert.True(t, (&JsonRpcResponse{result: []byte(`"0x"`)}).IsResultEmptyish(ctx)) + + w := &JsonRpcResponse{} + w.SetResultWriter(&covJsonrpcByteWriter{emptyish: true}) + assert.True(t, w.IsResultEmptyish(ctx)) + w.SetResultWriter(&covJsonrpcByteWriter{emptyish: false}) + assert.False(t, w.IsResultEmptyish()) +} + +func TestCovJsonrpc_CanonicalHashEdges(t *testing.T) { + var nilR *JsonRpcResponse + h, err := nilR.CanonicalHash() + assert.Equal(t, "", h) + assert.NoError(t, err) + h, err = nilR.CanonicalHashWithIgnoredFields([]string{"a"}) + assert.Equal(t, "", h) + assert.NoError(t, err) + + bad := &JsonRpcResponse{result: []byte(`{bad`)} + _, err = bad.CanonicalHash() + assert.Error(t, err) + _, err = bad.CanonicalHashWithIgnoredFields([]string{"a"}) + assert.Error(t, err) + + // No ignore fields delegates to CanonicalHash. + r := &JsonRpcResponse{result: []byte(`{"a":"0x1"}`)} + h1, err := r.CanonicalHash() + require.NoError(t, err) + h2, err := r.CanonicalHashWithIgnoredFields(nil) + require.NoError(t, err) + assert.Equal(t, h1, h2) + + // Cached with-ignored result is reused for the same slice. + ign := []string{"a"} + a1, err := r.CanonicalHashWithIgnoredFields(ign) + require.NoError(t, err) + r.SetResult([]byte(`{"a":"0x2","b":"0x3"}`)) + a2, err := r.CanonicalHashWithIgnoredFields(ign) + require.NoError(t, err) + assert.Equal(t, a1, a2, "cache keyed by ignore slice") +} + +func TestCovJsonrpc_CanonicalizeEquivalences(t *testing.T) { + hash := func(s string) string { + r := &JsonRpcResponse{result: []byte(s)} + h, err := r.CanonicalHash() + require.NoError(t, err) + return h + } + // Leading zeroes in hex strings, empties and key order do not matter. + assert.Equal(t, hash(`{"a":"0x01","b":[1,"",null,{}],"c":"0x0"}`), hash(`{"b":[1],"a":"0x1"}`)) + // Arrays of only emptyish elements collapse like the empty array. + assert.Equal(t, hash(`{"a":[null,"0x00",[]],"b":1}`), hash(`{"b":1}`)) + // Booleans are never empty. + assert.NotEqual(t, hash(`{"a":false}`), hash(`{}`)) + // Numbers: zero vanishes, others kept. + assert.Equal(t, hash(`{"a":0,"b":2}`), hash(`{"b":2}`)) + assert.NotEqual(t, hash(`{"b":2}`), hash(`{"b":3}`)) + // Nested arrays. + assert.NotEqual(t, hash(`[[1,2],[3]]`), hash(`[[1],[2,3]]`)) +} + +func TestCovJsonrpc_CanonicalizeTo_Direct(t *testing.T) { + cases := []struct { + in interface{} + out string + wrote bool + }{ + {nil, "", false}, + {"0x00ab", "ab", true}, + {"", "", false}, + {[]byte("0x0012"), "12", true}, + {[]byte(`"0x00"`), "", false}, + {[]byte{}, "", false}, + {int64(7), "7", true}, + {json.Number("0"), "", false}, + {map[string]interface{}{"k": "v"}, `{"k":v}`, true}, + {map[string]interface{}{"k": []interface{}{}}, "", false}, + {[]interface{}{"x", true}, "[x,true]", true}, + {[]interface{}{map[string]interface{}{"k": ""}, "y"}, "[y]", true}, + } + for _, tc := range cases { + var buf bytes.Buffer + wrote, err := canonicalizeTo(&buf, tc.in) + require.NoError(t, err, "%#v", tc.in) + assert.Equal(t, tc.wrote, wrote, "%#v", tc.in) + if tc.wrote { + assert.Equal(t, tc.out, buf.String(), "%#v", tc.in) + } + } + + _, err := canonicalizeTo(&bytes.Buffer{}, make(chan int)) + assert.Error(t, err) + _, err = canonicalizeTo(&bytes.Buffer{}, map[string]interface{}{"k": make(chan int)}) + assert.Error(t, err) + _, err = canonicalizeTo(&bytes.Buffer{}, []interface{}{make(chan int)}) + assert.Error(t, err) + + // Writer failures propagate from every structural write. + inputs := []interface{}{ + map[string]interface{}{"a": "1", "b": "2"}, + []interface{}{"1", "2"}, + "abc", + []byte("abc"), + int64(5), + } + for _, in := range inputs { + for i := 1; i <= 8; i++ { + w := &covJsonrpcFailWriter{failAt: i} + _, err := canonicalizeTo(w, in) + if w.calls >= i { + assert.ErrorIs(t, err, errCovJsonrpcWrite, "%#v write #%d", in, i) + } + } + } +} + +func TestCovJsonrpc_IsEmptyishValue(t *testing.T) { + cases := []struct { + v interface{} + want bool + }{ + {nil, true}, + {"", true}, + {"0x", true}, + {"0x000", true}, + {"0x010", false}, + {"abc", false}, + {[]interface{}{}, true}, + {[]interface{}{1}, false}, + {map[string]interface{}{}, true}, + {float64(0), true}, + {float64(1), false}, + {false, false}, + {int64(0), true}, + {int64(3), false}, + {make(chan int), false}, + } + for _, tc := range cases { + assert.Equal(t, tc.want, isEmptyishValue(tc.v), "%#v", tc.v) + } +} + +func TestCovJsonrpc_RemoveFieldsByPaths_Edges(t *testing.T) { + assert.Nil(t, removeFieldsByPaths(nil, []string{"a"})) + obj := map[string]interface{}{"a": 1} + assert.Equal(t, obj, removeFieldsByPaths(obj, nil)) + + // Wildcard on map keys applies nested removal to every value. + got := removeFieldsByPaths(map[string]interface{}{ + "x": map[string]interface{}{"ts": 1, "keep": 2}, + "y": map[string]interface{}{"ts": 3}, + }, []string{"*.ts"}) + assert.Equal(t, map[string]interface{}{ + "x": map[string]interface{}{"keep": 2}, + "y": map[string]interface{}{}, + }, got) + + // Array without wildcard rule is left untouched; primitives pass through. + arr := []interface{}{1, 2} + assert.Equal(t, map[string]interface{}{"a": arr}, removeFieldsByPaths(map[string]interface{}{"a": arr}, []string{"a.b"})) + assert.Equal(t, "s", removeFieldsByPaths("s", []string{"a"})) + // Wildcard leaf "*" on an array (not a nested tree) returns the array unchanged. + assert.Equal(t, arr, removeFieldsByPaths(arr, []string{"*"})) +} + +func TestCovJsonrpc_RemoveLeadingZeroes(t *testing.T) { + assert.Equal(t, "ab", string(removeLeadingZeroes([]byte("0x00ab")))) + assert.Equal(t, "ab", string(removeLeadingZeroes([]byte("0Xab")))) + assert.Equal(t, `1"`, string(removeLeadingZeroes([]byte(`"0x01"`)))) + assert.Nil(t, removeLeadingZeroes([]byte(`"0x00"`))) + assert.Equal(t, "12", string(removeLeadingZeroes([]byte("12")))) +} + +func TestCovJsonrpcRequest_Basics(t *testing.T) { + var nilR *JsonRpcRequest + assert.Equal(t, "", nilR.NetworkIdHint()) + ch, err := nilR.CacheHash() + assert.Equal(t, "", ch) + assert.NoError(t, err) + nilR.InvalidateCacheHash() + _, err = nilR.PeekByPath(0) + assert.ErrorContains(t, err, "empty params") + + r := NewJsonRpcRequest("eth_call", []interface{}{"0x1"}) + assert.Equal(t, "2.0", r.JSONRPC) + assert.False(t, r.IsNotification()) + ctx := context.Background() + r.LockWithTrace(ctx) + r.Unlock() + r.RLockWithTrace(ctx) + r.RUnlock() + assert.Nil(t, r.IDRawBytes(), "programmatic request has no wire id") +} + +func TestCovJsonrpcRequest_UnmarshalIdForms(t *testing.T) { + cases := []struct { + name string + body string + wantID interface{} + randomID bool + raw string + notification bool + errSub string + }{ + {"int", `{"method":"m","id":5}`, int64(5), false, "5", false, ""}, + {"string", `{"method":"m","id":"abc"}`, "abc", false, `"abc"`, false, ""}, + {"empty string", `{"method":"m","id":""}`, nil, true, `""`, false, ""}, + {"null", `{"method":"m","id":null}`, nil, true, "null", false, ""}, + {"missing", `{"method":"m"}`, nil, true, "", true, ""}, + {"fractional", `{"method":"m","id":1.5}`, nil, false, "", false, "cannot be represented as a 64-bit integer"}, + {"too large", `{"method":"m","id":99999999999999999999}`, nil, false, "", false, "cannot be represented"}, + {"no method", `{"id":1}`, nil, false, "", false, `no "method" member`}, + {"bad json", `{`, nil, false, "", false, "?"}, + {"bool id ignored", `{"method":"m","id":true}`, nil, true, "true", false, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := &JsonRpcRequest{} + err := SonicCfg.Unmarshal([]byte(tc.body), r) + if tc.errSub != "" { + require.Error(t, err) + if tc.errSub != "?" { + assert.Contains(t, err.Error(), tc.errSub) + } + return + } + require.NoError(t, err) + if tc.randomID { + _, ok := r.ID.(int64) + assert.True(t, ok, "random id assigned, got %T", r.ID) + } else { + assert.Equal(t, tc.wantID, r.ID) + } + if tc.raw == "" { + assert.Nil(t, r.IDRawBytes()) + } else { + assert.Equal(t, tc.raw, string(r.IDRawBytes())) + } + assert.Equal(t, tc.notification, r.IsNotification()) + }) + } +} + +func TestCovJsonrpcRequest_EthQueryNonArrayParams(t *testing.T) { + r := &JsonRpcRequest{} + err := SonicCfg.Unmarshal([]byte(`{"jsonrpc":"2.0","method":"eth_queryLogs","params":{"a":1},"id":3,"networkId":"evm:1"}`), r) + require.NoError(t, err, "eth_query* with object params decodes so the query parser can answer -32602") + assert.Equal(t, "eth_queryLogs", r.Method) + assert.Nil(t, r.Params) + assert.Equal(t, int64(3), r.ID) + assert.Equal(t, "evm:1", r.NetworkIdHint()) + + other := &JsonRpcRequest{} + assert.Error(t, SonicCfg.Unmarshal([]byte(`{"method":"eth_call","params":{"a":1},"id":3}`), other), + "non eth_query methods keep failing on object params") +} + +func TestCovJsonrpcRequest_ResponseID(t *testing.T) { + parse := func(body string) *JsonRpcRequest { + r := &JsonRpcRequest{} + require.NoError(t, SonicCfg.Unmarshal([]byte(body), r)) + return r + } + big := parse(`{"method":"m","id":9007199254740993}`) + assert.Equal(t, json.RawMessage(`9007199254740993`), big.ResponseID(false)) + + null := parse(`{"method":"m","id":null}`) + assert.Equal(t, json.RawMessage(`null`), null.ResponseID(true), "solana echoes null") + assert.IsType(t, int64(0), null.ResponseID(false), "evm gets the random id") + + prog := NewJsonRpcRequest("m", nil) + prog.ID = "p" + assert.Equal(t, "p", prog.ResponseID(false)) + + // SetID drops the captured wire bytes. + require.NoError(t, big.SetID(int64(1))) + assert.Nil(t, big.IDRawBytes()) + assert.Equal(t, int64(1), big.ResponseID(false)) +} + +func TestCovJsonrpcRequest_CloneKeepsIdRaw(t *testing.T) { + r := &JsonRpcRequest{} + require.NoError(t, SonicCfg.Unmarshal([]byte(`{"method":"m","id":12345678901234567,"params":[null,{"a":[1]}]}`), r)) + c := r.Clone() + assert.Equal(t, r.IDRawBytes(), c.IDRawBytes()) + c.idRaw[0] = '9' + assert.Equal(t, byte('1'), r.idRaw[0], "idRaw deep-copied") + assert.Nil(t, c.Params[0]) + + empty := NewJsonRpcRequest("m", nil).Clone() + assert.Nil(t, empty.Params) + assert.Nil(t, empty.IDRawBytes()) +} + +func TestCovJsonrpcRequest_MarshalZerologObject(t *testing.T) { + covJsonrpcEnableLogs(t) + var buf bytes.Buffer + lg := zerolog.New(&buf) + var nilR *JsonRpcRequest + lg.Info().Object("r", nilR).Send() + assert.Contains(t, buf.String(), `"r":{}`) + + buf.Reset() + r := NewJsonRpcRequest("eth_getBalance", []interface{}{"0xabc", "latest"}) + r.ID = int64(9) + lg.Info().Object("r", r).Send() + assert.Contains(t, buf.String(), `"r":{"method":"eth_getBalance","params":["0xabc","latest"],"id":9}`) +} + +func TestCovJsonrpcRequest_CacheHashTypes(t *testing.T) { + r := NewJsonRpcRequest("m", []interface{}{1, true, false, nil, map[string]interface{}{"k": []interface{}{"A"}}}) + h1, err := r.CacheHash(context.Background()) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(h1, "m:")) + + // int and float64 of same value encode identically. + r2 := NewJsonRpcRequest("m", []interface{}{float64(1), true, false, nil, map[string]interface{}{"k": []interface{}{"a"}}}) + h2, err := r2.CacheHash() + require.NoError(t, err) + assert.Equal(t, h1, h2) + + // Memoized until invalidated. + r.Params[1] = false + h3, _ := r.CacheHash() + assert.Equal(t, h1, h3) + r.InvalidateCacheHash() + h4, err := r.CacheHash() + require.NoError(t, err) + assert.NotEqual(t, h1, h4) + + for _, bad := range []interface{}{struct{}{}, []interface{}{struct{}{}}, map[string]interface{}{"k": struct{}{}}} { + _, err := NewJsonRpcRequest("m", []interface{}{bad}).CacheHash() + assert.ErrorContains(t, err, "unsupported type") + } + + // writeUint zero and multi-digit counts. + var buf bytes.Buffer + writeUint(&buf, 0) + writeUint(&buf, 1234) + assert.Equal(t, "01234", buf.String()) +} + +func TestCovJsonrpcRequest_PeekByPath(t *testing.T) { + r := NewJsonRpcRequest("m", []interface{}{map[string]interface{}{"to": "0x1", "arr": []interface{}{"z"}}}) + v, err := r.PeekByPath(0, "to") + require.NoError(t, err) + assert.Equal(t, "0x1", v) + v, err = r.PeekByPath(0, "arr", 0) + require.NoError(t, err) + assert.Equal(t, "z", v) + + _, err = r.PeekByPath(5) + assert.ErrorContains(t, err, "out of bounds") + _, err = r.PeekByPath(0, 0) + assert.ErrorContains(t, err, "expected array") + _, err = r.PeekByPath("x") + assert.ErrorContains(t, err, "expected map") + _, err = r.PeekByPath(0, "missing") + assert.ErrorContains(t, err, "not found") + _, err = r.PeekByPath(1.5) + assert.ErrorContains(t, err, "unsupported path element type") + _, err = NewJsonRpcRequest("m", nil).PeekByPath(0) + assert.ErrorContains(t, err, "empty params") +} + +func TestCovJsonrpc_TranslateToJsonRpcException(t *testing.T) { + cases := []struct { + name string + err error + code JsonRpcErrorNumber + msg string + }{ + {"auth rate limit", NewErrAuthRateLimitRuleExceeded("p", "s", "b", "r", "u", "ip"), JsonRpcErrorCapacityExceeded, "rate-limit exceeded"}, + {"project rate limit", NewErrProjectRateLimitRuleExceeded("p", "b", "r"), JsonRpcErrorCapacityExceeded, "rate-limit exceeded"}, + {"network rate limit", NewErrNetworkRateLimitRuleExceeded("p", "n", "b", "r"), JsonRpcErrorCapacityExceeded, "rate-limit exceeded"}, + {"upstream rate limit", NewErrUpstreamRateLimitRuleExceeded("u", "b", "r"), JsonRpcErrorCapacityExceeded, "rate-limit exceeded"}, + {"unauthorized", NewErrAuthUnauthorized("secret", "bad token", "nope"), JsonRpcErrorUnauthorized, "unauthorized"}, + {"method ignored", NewErrUpstreamMethodIgnored("eth_x", "u1"), JsonRpcErrorUnsupportedException, "method ignored by upstream"}, + {"subscription not found", NewErrSubscriptionNotFound("0x1"), JsonRpcErrorCallException, "subscription not found"}, + {"request unmarshal", NewErrJsonRpcRequestUnmarshal(errors.New("bad"), []byte("{")), JsonRpcErrorParseException, "failed to parse json-rpc request"}, + {"invalid request", NewErrInvalidRequest(errors.New("bad")), JsonRpcErrorInvalidArgument, "invalid request url and/or body"}, + {"invalid url path", NewErrInvalidUrlPath("bad", "/x"), JsonRpcErrorInvalidArgument, "invalid request url and/or body"}, + {"getLogs range", NewErrGetLogsExceededMaxAllowedRange(100, 10), JsonRpcErrorEvmLargeRange, "getLogs request exceeded max allowed range"}, + {"block unavailable", NewErrUpstreamBlockUnavailable("u", 10, 5, 3), JsonRpcErrorMissingData, "block not available"}, + {"plain error", errors.New("kaboom"), JsonRpcErrorServerSideException, "kaboom"}, + {"standard error", NewErrEndpointServerSideException(errors.New("deep cause"), nil, 500), JsonRpcErrorServerSideException, "deep cause"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out := TranslateToJsonRpcException(tc.err) + jre := &ErrJsonRpcExceptionInternal{} + require.True(t, errors.As(out, &jre)) + assert.Equal(t, tc.code, jre.NormalizedCode()) + assert.Contains(t, jre.Message, tc.msg) + assert.True(t, errors.Is(out, tc.err) || out == tc.err || jre.Cause == tc.err) + }) + } + + t.Run("existing json-rpc exception passes through", func(t *testing.T) { + in := NewErrJsonRpcExceptionInternal(0, JsonRpcErrorEvmReverted, "reverted", nil, nil) + assert.Same(t, in, TranslateToJsonRpcException(in)) + }) +} + +func TestCovJsonrpc_TranslateExhaustedDominance(t *testing.T) { + // Two rate-limited upstreams outvote one server error. + ex := newExhausted(t, []string{"a", "b", "c"}, map[string]error{ + "a": NewErrUpstreamRateLimitRuleExceeded("a", "b", "r"), + "b": NewErrUpstreamRateLimitRuleExceeded("b", "b", "r"), + "c": NewErrEndpointServerSideException(errors.New("x"), nil, 500), + }) + assert.EqualValues(t, JsonRpcErrorCapacityExceeded, clientWireCode(t, TranslateToJsonRpcException(ex))) + + // Only skipped/unsupported causes: the first one is the representative. + ex2 := newExhausted(t, []string{"a", "b"}, map[string]error{ + "a": NewErrUpstreamRequestSkipped(errors.New("skip"), "a"), + "b": NewErrEndpointUnsupported(errors.New("unsupported")), + }) + out := TranslateToJsonRpcException(ex2) + assert.EqualValues(t, JsonRpcErrorServerSideException, clientWireCode(t, out)) + + // Block-unavailable is a verdict only when every non-skipped cause agrees. + ex3 := newExhausted(t, []string{"a", "b"}, map[string]error{ + "a": NewErrUpstreamBlockUnavailable("a", 10, 5, 3), + "b": NewErrUpstreamRequestSkipped(errors.New("skip"), "b"), + }) + assert.EqualValues(t, JsonRpcErrorMissingData, clientWireCode(t, TranslateToJsonRpcException(ex3))) + + // Plain (non-standard) causes are ignored by the scan. + ex4 := newExhausted(t, []string{"a"}, map[string]error{"a": errors.New("plain")}) + assert.EqualValues(t, JsonRpcErrorServerSideException, clientWireCode(t, TranslateToJsonRpcException(ex4))) +} + +func TestCovJsonrpc_HasLinearErrorCode(t *testing.T) { + assert.False(t, hasLinearErrorCode(errors.New("plain"), ErrCodeNotImplemented)) + assert.True(t, hasLinearErrorCode(NewErrFailsafeRetryExceeded(ScopeNetwork, NewErrNotImplemented("x"), nil), ErrCodeNotImplemented)) + assert.False(t, hasLinearErrorCode(nil, ErrCodeNotImplemented)) + assert.Nil(t, findUpstreamsExhausted(errors.New("plain"))) +} diff --git a/common/cov_jsonrpc_request_test.go b/common/cov_jsonrpc_request_test.go new file mode 100644 index 000000000..12bf895c2 --- /dev/null +++ b/common/cov_jsonrpc_request_test.go @@ -0,0 +1,627 @@ +package common + +import ( + "bytes" + "context" + "errors" + "net/http" + "net/url" + "testing" + "time" + + "github.com/erpc/erpc/util" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCovJsonrpcRequest_ContextMarkers(t *testing.T) { + ctx := context.Background() + assert.False(t, IsConsensusSlot(ctx)) + assert.True(t, IsConsensusSlot(WithConsensusSlot(ctx))) + assert.False(t, IsSweepIteration(ctx)) + assert.True(t, IsSweepIteration(WithSweepIteration(ctx))) + + a := NewNormalizedRequest(nil) + b := NewNormalizedRequest(nil) + own := []interface{}{"own"} + over := []interface{}{"override"} + wctx := WithWireParams(ctx, a, over) + assert.Equal(t, over, WireParams(wctx, a, own), "override applies to its own request") + assert.Equal(t, own, WireParams(wctx, b, own), "override never leaks to other requests") + assert.Equal(t, own, WireParams(ctx, a, own)) +} + +func TestCovJsonrpcRequest_DirectivesClone(t *testing.T) { + var nilD *RequestDirectives + assert.Equal(t, &RequestDirectives{}, nilD.Clone()) + + d := &RequestDirectives{ + RetryEmpty: true, RetryPending: true, SkipCacheRead: "redis*", UseUpstream: "a*", + ByPassMethodExclusion: true, SkipInterpolation: true, SkipConsensus: true, + EnforceHighestBlock: true, EnforceGetLogsBlockRange: true, EnforceNonNullTaggedBlocks: true, + IntegritySelector: "strict", + } + c := d.Clone() + assert.Equal(t, d, c) + assert.NotSame(t, d, c) +} + +func TestCovJsonrpcRequest_NilReceiver(t *testing.T) { + var r *NormalizedRequest + ctx := context.Background() + assert.Nil(t, r.ExecState()) + assert.Zero(t, r.CreditUnitsTotal()) + assert.Nil(t, r.CreditUnitsByVendor()) + assert.False(t, r.IsInternal()) + r.SetUser(&User{Id: "x"}) + r.SetUserFromTrustedHeader("x") + assert.Nil(t, r.User()) + assert.Nil(t, r.SetLastUpstream(&covJsonrpcUpstream{id: "a"})) + assert.Nil(t, r.LastUpstream()) + assert.False(t, r.SetLastValidResponse(ctx, NewNormalizedResponse())) + assert.Nil(t, r.LastValidResponse()) + r.ClearLastValidResponse() + r.ClearLastValidResponseIf(NewNormalizedResponse()) + r.MarkIntegrityCaught("c", "f") + assert.False(t, r.IntegrityCaught()) + r.SetIntegrityFallbackResponse(NewNormalizedResponse(), "c", "f", "r") + assert.Nil(t, r.TakeIntegrityFallbackResponse()) + assert.Equal(t, "", r.IntegrityRejectedCheck()) + assert.Equal(t, "", r.IntegrityRejectedFinality()) + r.AddIntegrityOverhead(time.Second) + assert.Zero(t, r.IntegrityOverhead()) + assert.Nil(t, r.Network()) + assert.Nil(t, r.ID()) + assert.Equal(t, "n/a", r.NetworkId()) + assert.Equal(t, "", r.CacheKeySuffix()) + assert.Equal(t, "n/a", r.NetworkLabel()) + assert.Nil(t, r.CacheDal()) + r.SetDirectives(&RequestDirectives{}) + r.SetAllowClientDirectiveMatcher(nil) + assert.False(t, r.ShouldSkipCacheRead("x")) + assert.Nil(t, r.Directives()) + jrq, err := r.JsonRpcRequest() + assert.Nil(t, jrq) + assert.NoError(t, err) + m, err := r.Method() + assert.Equal(t, "", m) + assert.NoError(t, err) + assert.Equal(t, "", r.NetworkIdHint()) + r.SetClientIP("1.2.3.4") + assert.Equal(t, "n/a", r.ClientIP()) + r.SetAgentName("x") + assert.Nil(t, r.EvmBlockRef()) + r.SetEvmBlockRef("x") + assert.Nil(t, r.EvmBlockNumber()) + r.SetEvmBlockNumber(int64(1)) + assert.True(t, HasErrorCode(r.Validate(), ErrCodeInvalidRequest)) + assert.False(t, r.IsCompositeRequest()) + assert.Equal(t, "", r.CompositeType()) + r.SetCompositeType("x") + assert.Nil(t, r.ParentRequestId()) + r.SetParentRequestId(1) + assert.Equal(t, DataFinalityStateUnknown, r.Finality(ctx)) + r.CopyHttpContextFrom(NewNormalizedRequest(nil)) + r.SetUpstreams(nil) + assert.Zero(t, r.UpstreamsCount()) + assert.Nil(t, r.Upstreams()) + assert.False(t, r.MarkEscalatedToFallbacks()) + assert.Equal(t, "n/a", r.UserId()) + assert.Equal(t, "unknown", r.AgentName()) + _, err = r.NextUpstream() + assert.ErrorContains(t, err, "uninitialized request") + var buf bytes.Buffer + lg := zerolog.New(&buf) + r.MarshalZerologObject(lg.Info()) +} + +func TestCovJsonrpcRequest_ExecStateAndCredits(t *testing.T) { + r := NewNormalizedRequest([]byte(`{"method":"eth_call"}`)) + st := r.ExecState() + require.NotNil(t, st) + assert.Same(t, st, r.ExecState(), "exec state is created once") + assert.Zero(t, r.CreditUnitsTotal()) + assert.Nil(t, r.CreditUnitsByVendor()) +} + +func TestCovJsonrpcRequest_IsInternal(t *testing.T) { + r := NewNormalizedRequest(nil) + assert.False(t, r.IsInternal(), "no directives") + r.SetDirectives(&RequestDirectives{IsInternal: true}) + assert.True(t, r.IsInternal()) +} + +func TestCovJsonrpcRequest_UserAndUpstreamAccessors(t *testing.T) { + r := NewNormalizedRequest(nil) + r.SetUser(nil) + assert.Nil(t, r.User()) + assert.Equal(t, "n/a", r.UserId()) + r.SetUser(&User{}) + assert.Equal(t, "n/a", r.UserId(), "empty user id reported as n/a") + r.SetUser(&User{Id: "alice"}) + assert.Equal(t, "alice", r.UserId()) + + assert.Nil(t, r.LastUpstream()) + r.SetLastUpstream(nil) + assert.Nil(t, r.LastUpstream()) + up := &covJsonrpcUpstream{id: "u1", cfg: &UpstreamConfig{Id: "u1"}} + assert.Same(t, r, r.SetLastUpstream(up)) + assert.Same(t, up, r.LastUpstream()) + + assert.Equal(t, "n/a", r.ClientIP()) + r.SetClientIP("") + assert.Equal(t, "n/a", r.ClientIP()) + r.SetClientIP("10.0.0.1") + assert.Equal(t, "10.0.0.1", r.ClientIP()) + + r.SetAgentName("") + assert.Equal(t, "unknown", r.AgentName()) + r.SetAgentName("my-bot") + assert.Equal(t, "my-bot", r.AgentName()) + + r.SetEvmBlockRef(nil) + assert.Nil(t, r.EvmBlockRef()) + r.SetEvmBlockRef("latest") + assert.Equal(t, "latest", r.EvmBlockRef()) + r.SetEvmBlockNumber(nil) + assert.Nil(t, r.EvmBlockNumber()) + r.SetEvmBlockNumber(int64(100)) + assert.Equal(t, int64(100), r.EvmBlockNumber()) + + assert.False(t, r.IsCompositeRequest()) + r.SetCompositeType("") + assert.Equal(t, CompositeTypeNone, r.CompositeType()) + r.SetCompositeType(CompositeTypeLogsSplitOnError) + assert.True(t, r.IsCompositeRequest()) + assert.Equal(t, "", (&NormalizedRequest{}).CompositeType(), "zero-value request has no composite type") + + assert.Nil(t, r.ParentRequestId()) + r.SetParentRequestId(nil) + assert.Nil(t, r.ParentRequestId()) + r.SetParentRequestId("parent-1") + assert.Equal(t, "parent-1", r.ParentRequestId()) + + assert.True(t, r.MarkEscalatedToFallbacks()) + assert.False(t, r.MarkEscalatedToFallbacks(), "escalation happens at most once") + + r.SetUpstreams(nil) + assert.Nil(t, r.Upstreams()) + ups := []Upstream{up, &covJsonrpcUpstream{id: "u2"}} + r.SetUpstreams(ups) + assert.Equal(t, 2, r.UpstreamsCount()) + snap := r.Upstreams() + snap[0] = nil + assert.Same(t, up, r.Upstreams()[0], "snapshot is a copy") + + var cd CacheDAL + r.SetCacheDal(cd) + assert.Nil(t, r.CacheDal()) +} + +func TestCovJsonrpcRequest_IntegrityTracking(t *testing.T) { + r := NewNormalizedRequest(nil) + assert.False(t, r.IntegrityCaught()) + assert.Equal(t, "", r.IntegrityRejectedCheck()) + assert.Equal(t, "", r.IntegrityRejectedFinality()) + + r.MarkIntegrityCaught("", "") + assert.True(t, r.IntegrityCaught()) + assert.Equal(t, "", r.IntegrityRejectedCheck(), "empty check id is not recorded") + + r.MarkIntegrityCaught("receipts-root", "finalized") + assert.Equal(t, "receipts-root", r.IntegrityRejectedCheck()) + assert.Equal(t, "finalized", r.IntegrityRejectedFinality()) + + r.AddIntegrityOverhead(0) + r.AddIntegrityOverhead(-time.Second) + assert.Zero(t, r.IntegrityOverhead(), "non-positive durations ignored") + r.AddIntegrityOverhead(2 * time.Millisecond) + r.AddIntegrityOverhead(3 * time.Millisecond) + assert.Equal(t, 5*time.Millisecond, r.IntegrityOverhead()) + + r.SetIntegrityFallbackResponse(nil, "c", "f", "r") + assert.Nil(t, r.TakeIntegrityFallbackResponse()) +} + +func TestCovJsonrpcRequest_LastValidResponseLifecycle(t *testing.T) { + ctx := context.Background() + r := NewNormalizedRequest([]byte(`{"method":"eth_getBlockByNumber","id":1}`)) + assert.False(t, r.SetLastValidResponse(ctx, nil)) + + up := &covJsonrpcUpstream{id: "u", cfg: &UpstreamConfig{Id: "u"}} + empty := NewNormalizedResponse().SetUpstream(up).WithJsonRpcResponse(MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`null`), nil)) + full := NewNormalizedResponse().SetUpstream(up).WithJsonRpcResponse(MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`{"n":"0x1"}`), nil)) + empty2 := NewNormalizedResponse().WithJsonRpcResponse(MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`[]`), nil)) + + assert.True(t, r.SetLastValidResponse(ctx, empty), "empty accepted when slot empty") + assert.Same(t, up, r.LastUpstream()) + assert.True(t, r.SetLastValidResponse(ctx, full), "non-empty replaces empty") + assert.False(t, r.SetLastValidResponse(ctx, empty2), "empty never replaces non-empty") + assert.Same(t, full, r.LastValidResponse()) + + r.ClearLastValidResponseIf(nil) + r.ClearLastValidResponseIf(empty2) + assert.Same(t, full, r.LastValidResponse(), "conditional clear only drops matching response") + r.ClearLastValidResponseIf(full) + assert.Nil(t, r.LastValidResponse()) + + r.SetLastValidResponse(ctx, full) + r.ClearLastValidResponse() + assert.Nil(t, r.LastValidResponse()) +} + +func TestCovJsonrpcRequest_NetworkAccessors(t *testing.T) { + r := NewNormalizedRequest([]byte(`{"method":"eth_chainId","id":1}`)) + assert.Equal(t, "n/a", r.NetworkId()) + assert.Equal(t, "n/a", r.NetworkLabel()) + assert.Equal(t, "", r.CacheKeySuffix()) + + nw := &covJsonrpcNetwork{id: "evm:1"} + r.SetNetwork(nw) + assert.Same(t, nw, r.Network()) + assert.Equal(t, "evm:1", r.NetworkId()) + assert.Equal(t, "evm:1", r.NetworkLabel(), "label falls back to id") + assert.Equal(t, "", r.CacheKeySuffix(), "nil config yields no suffix") + + nw.label = "mainnet" + nw.cfg = &NetworkConfig{CacheKeySuffix: "v2"} + assert.Equal(t, "mainnet", r.NetworkLabel()) + assert.Equal(t, "v2", r.CacheKeySuffix()) +} + +func TestCovJsonrpcRequest_Finality(t *testing.T) { + ctx := context.Background() + r := NewNormalizedRequest([]byte(`{"method":"eth_call"}`)) + assert.Equal(t, DataFinalityStateUnknown, r.Finality(ctx)) + + nw := &covJsonrpcNetwork{id: "evm:1", finality: DataFinalityStateUnknown} + r.SetNetwork(nw) + assert.Equal(t, DataFinalityStateUnknown, r.Finality(ctx)) + assert.Equal(t, DataFinalityStateUnknown, r.Finality(ctx)) + assert.Equal(t, 2, nw.finalityCalls, "unknown is not cached") + + nw.finality = DataFinalityStateUnfinalized + assert.Equal(t, DataFinalityStateUnfinalized, r.Finality(ctx)) + nw.finality = DataFinalityStateFinalized + assert.Equal(t, DataFinalityStateUnfinalized, r.Finality(ctx), "definitive finality is cached") +} + +func TestCovJsonrpcRequest_ApplyDirectiveDefaults(t *testing.T) { + r := NewNormalizedRequest(nil) + r.ApplyDirectiveDefaults(nil) + assert.Nil(t, r.Directives(), "nil defaults are a no-op") + + up := "primary-*" + r.ApplyDirectiveDefaults(&DirectiveDefaultsConfig{ + RetryEmpty: util.BoolPtr(true), + RetryPending: util.BoolPtr(true), + SkipCacheRead: true, + UseUpstream: &up, + SkipInterpolation: util.BoolPtr(true), + SkipConsensus: util.BoolPtr(true), + EnforceHighestBlock: util.BoolPtr(true), + EnforceGetLogsBlockRange: util.BoolPtr(true), + EnforceNonNullTaggedBlocks: util.BoolPtr(true), + }) + d := r.Directives() + assert.Equal(t, &RequestDirectives{ + RetryEmpty: true, RetryPending: true, SkipCacheRead: "true", UseUpstream: "primary-*", + SkipInterpolation: true, SkipConsensus: true, EnforceHighestBlock: true, + EnforceGetLogsBlockRange: true, EnforceNonNullTaggedBlocks: true, + }, d) + + r.ApplyDirectiveDefaults(&DirectiveDefaultsConfig{RetryEmpty: util.BoolPtr(false)}) + assert.True(t, r.Directives().RetryEmpty, "defaults never overwrite populated directives") + + r2 := NewNormalizedRequest(nil) + r2.ApplyDirectiveDefaults(&DirectiveDefaultsConfig{SkipCacheRead: "redis*"}) + assert.Equal(t, "redis*", r2.Directives().SkipCacheRead) +} + +func TestCovJsonrpcRequest_EnrichFromHttp_AllDirectives(t *testing.T) { + headers := http.Header{} + headers.Set("X-ERPC-Retry-Empty", " TRUE ") + headers.Set("X-ERPC-Retry-Pending", "false") + headers.Set("X-ERPC-Skip-Cache-Read", " memory* ") + headers.Set("X-ERPC-Use-Upstream", "alchemy") + headers.Set("X-ERPC-Skip-Interpolation", "true") + headers.Set("X-ERPC-Skip-Consensus", "true") + headers.Set("X-ERPC-Enforce-Highest-Block", "true") + headers.Set("X-ERPC-Enforce-GetLogs-Range", "true") + headers.Set("X-ERPC-Enforce-Non-Null-Tagged-Blocks", "true") + headers.Set("X-ERPC-Integrity", " strict ") + headers.Set("User-Agent", "Mozilla/5.0 Chrome/120") + + r := NewNormalizedRequest(nil) + r.EnrichFromHttp(headers, nil, UserAgentTrackingModeSimplified) + assert.Equal(t, &RequestDirectives{ + RetryEmpty: true, RetryPending: false, SkipCacheRead: "memory*", UseUpstream: "alchemy", + SkipInterpolation: true, SkipConsensus: true, EnforceHighestBlock: true, + EnforceGetLogsBlockRange: true, EnforceNonNullTaggedBlocks: true, IntegritySelector: "strict", + }, r.Directives()) + assert.Equal(t, "chrome", r.AgentName()) + + // Query params override headers. + q := url.Values{} + q.Set("retry-empty", "false") + q.Set("retry-pending", "true") + q.Set("skip-cache-read", "true") + q.Set("use-upstream", " infura ") + q.Set("skip-interpolation", "false") + q.Set("skip-consensus", "false") + q.Set("enforce-highest-block", "false") + q.Set("enforce-getlogs-range", "false") + q.Set("enforce-non-null-tagged-blocks", "false") + q.Set("integrity", "off") + q.Set("user-agent", "my-custom-agent/1.2") + r.EnrichFromHttp(headers, q, UserAgentTrackingModeRaw) + assert.Equal(t, &RequestDirectives{ + RetryEmpty: false, RetryPending: true, SkipCacheRead: "true", UseUpstream: "infura", + IntegritySelector: "off", + }, r.Directives()) + assert.Equal(t, "my-custom-agent/1.2", r.AgentName(), "raw mode keeps the agent verbatim") + + // No directives: only the agent is captured, directives untouched. + r2 := NewNormalizedRequest(nil) + r2.EnrichFromHttp(http.Header{"User-Agent": []string{"curl/8"}}, nil, UserAgentTrackingModeSimplified) + assert.Nil(t, r2.Directives()) + assert.Equal(t, "curl", r2.AgentName()) + + r3 := NewNormalizedRequest(nil) + r3.EnrichFromHttp(nil, nil, UserAgentTrackingModeSimplified) + assert.Equal(t, "unknown", r3.AgentName()) + + // Copy-on-write: shared defaults are not mutated. + shared := &RequestDirectives{RetryPending: true} + r4 := NewNormalizedRequest(nil) + r4.SetDirectives(shared) + r4.EnrichFromHttp(http.Header{"X-Erpc-Retry-Pending": []string{"false"}}, nil, UserAgentTrackingModeSimplified) + assert.True(t, shared.RetryPending) + assert.False(t, r4.Directives().RetryPending) +} + +func TestCovJsonrpcRequest_DirectiveMatcherEmptyKey(t *testing.T) { + r := NewNormalizedRequest(nil) + r.SetAllowClientDirectiveMatcher(func(string) bool { return true }) + assert.False(t, r.isDirectiveAllowed(""), "unknown directive keys are never allowed under a matcher") + assert.True(t, r.isDirectiveAllowed("retry-empty")) + assert.False(t, hasDirectiveInHeaders(nil)) + assert.False(t, hasDirectiveInQueryParams(nil)) +} + +func TestCovJsonrpcRequest_ShouldSkipCacheRead(t *testing.T) { + cases := []struct { + directive string + connector string + want bool + }{ + {"", "redis-1", false}, + {"false", "redis-1", false}, + {"FALSE", "redis-1", false}, + {"True", "", true}, + {"redis*", "", false}, + {"redis*", "redis-1", true}, + {"redis*", "memory-1", false}, + {"memory*|dynamo*", "dynamo-x", true}, + } + for _, tc := range cases { + r := NewNormalizedRequest(nil) + r.SetDirectives(&RequestDirectives{SkipCacheRead: tc.directive}) + assert.Equal(t, tc.want, r.ShouldSkipCacheRead(tc.connector), "%q vs %q", tc.directive, tc.connector) + } + assert.False(t, NewNormalizedRequest(nil).ShouldSkipCacheRead("x"), "no directives") +} + +func TestCovJsonrpcRequest_MethodAndParsing(t *testing.T) { + ctx := context.Background() + + r := NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBalance","id":"abc","networkId":"evm:10"}`)) + assert.Equal(t, "evm:10", r.NetworkIdHint()) + m, err := r.Method() + require.NoError(t, err) + assert.Equal(t, "eth_getBalance", m) + assert.Equal(t, "abc", r.ID()) + assert.Equal(t, "evm:10", r.NetworkIdHint(), "hint read from cached envelope") + + _, err = NewNormalizedRequest(nil).Method() + assert.True(t, HasErrorCode(err, "ErrJsonRpcRequestUnresolvableMethod"), "got %v", err) + + _, err = NewNormalizedRequest([]byte(`{bad`)).Method() + assert.True(t, HasErrorCode(err, ErrCodeJsonRpcRequestUnmarshal), "got %v", err) + + _, err = NewNormalizedRequest([]byte(`{"method":""}`)).JsonRpcRequest(ctx) + assert.True(t, HasErrorCode(err, "ErrJsonRpcRequestUnresolvableMethod"), "got %v", err) + + assert.Equal(t, "", NewNormalizedRequest([]byte(`{bad`)).NetworkIdHint()) + assert.Equal(t, "", NewNormalizedRequest(nil).NetworkIdHint()) + assert.Nil(t, NewNormalizedRequest([]byte(`{bad`)).ID()) + + // Body is retained until JsonRpcRequest consumes it. + r2 := NewNormalizedRequest([]byte(`{"method":"eth_call","id":1}`)) + assert.NotNil(t, r2.Body()) + _, err = r2.JsonRpcRequest(ctx) + require.NoError(t, err) + assert.Nil(t, r2.Body()) +} + +func TestCovJsonrpcRequest_Validate(t *testing.T) { + assert.True(t, HasErrorCode(NewNormalizedRequest(nil).Validate(), ErrCodeInvalidRequest)) + err := NewNormalizedRequest([]byte(`{bad`)).Validate() + assert.True(t, HasErrorCode(err, ErrCodeInvalidRequest)) + assert.True(t, HasErrorCode(err, ErrCodeJsonRpcRequestUnmarshal)) + + empty := NewNormalizedRequestFromJsonRpcRequest(&JsonRpcRequest{Method: ""}) + err = empty.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "method is required") + + assert.NoError(t, NewNormalizedRequest([]byte(`{"method":"eth_call"}`)).Validate()) +} + +func TestCovJsonrpcRequest_MarshalJSON(t *testing.T) { + body := []byte(`{"method":"eth_call","id":1}`) + b, err := NewNormalizedRequest(body).MarshalJSON() + require.NoError(t, err) + assert.Equal(t, body, b) + + fromJrq := NewNormalizedRequestFromJsonRpcRequest(NewJsonRpcRequest("eth_chainId", nil)) + b, err = fromJrq.MarshalJSON() + require.NoError(t, err) + assert.JSONEq(t, `{"method":"eth_chainId"}`, string(b)) + + b, err = (&NormalizedRequest{}).MarshalJSON() + assert.Nil(t, b) + assert.NoError(t, err) +} + +func TestCovJsonrpcRequest_CacheHash(t *testing.T) { + r := NewNormalizedRequest([]byte(`{"method":"eth_getBalance","params":["0xAbC","latest"],"id":1}`)) + h, err := r.CacheHash(context.Background()) + require.NoError(t, err) + r2 := NewNormalizedRequest([]byte(`{"method":"eth_getBalance","params":["0xabc","latest"],"id":2}`)) + h2, err := r2.CacheHash() + require.NoError(t, err) + assert.Equal(t, h, h2, "id and hex case do not affect cache hash") + + _, err = NewNormalizedRequest([]byte(`{bad`)).CacheHash() + assert.True(t, HasErrorCode(err, ErrCodeJsonRpcRequestUnmarshal)) + + var nilR *NormalizedRequest + _, err = nilR.CacheHash() + assert.ErrorContains(t, err, "not valid to generate cache hash") +} + +func TestCovJsonrpcNormalizedRequest_MarshalZerologObject(t *testing.T) { + covJsonrpcEnableLogs(t) + render := func(r *NormalizedRequest) string { + var buf bytes.Buffer + lg := zerolog.New(&buf) + lg.Info().Object("rq", r).Send() + return buf.String() + } + + out := render(NewNormalizedRequest([]byte(`{"method":"m"}`))) + assert.Contains(t, out, `"lastUpstream":"nil"`) + assert.Contains(t, out, `"body":{"method":"m"}`) + + out = render(NewNormalizedRequest([]byte(`garbage`))) + assert.Contains(t, out, `"body":"garbage"`) + + r := NewNormalizedRequestFromJsonRpcRequest(NewJsonRpcRequest("eth_call", nil)) + r.SetNetwork(&covJsonrpcNetwork{id: "evm:137"}) + r.SetLastUpstream(&covJsonrpcUpstream{id: "u", cfg: &UpstreamConfig{Id: "u"}}) + out = render(r) + assert.Contains(t, out, `"lastUpstream":"u"`) + assert.Contains(t, out, `"networkId":"evm:137"`) + assert.Contains(t, out, `"jsonRpc":{"method":"eth_call"`) + + r.SetLastUpstream(&covJsonrpcUpstream{id: "nocfg"}) + assert.Regexp(t, `"lastUpstream":"0x[0-9a-f]+"`, render(r)) +} + +func TestCovJsonrpcRequest_CopyHttpContextFrom(t *testing.T) { + src := NewNormalizedRequest(nil) + src.SetAgentName("viem") + src.SetUser(&User{Id: "bob"}) + dst := NewNormalizedRequest(nil) + dst.CopyHttpContextFrom(nil) + assert.Equal(t, "unknown", dst.AgentName()) + dst.CopyHttpContextFrom(src) + assert.Equal(t, "viem", dst.AgentName()) + assert.Equal(t, "bob", dst.UserId()) + + dst2 := NewNormalizedRequest(nil) + dst2.CopyHttpContextFrom(NewNormalizedRequest(nil)) + assert.Equal(t, "unknown", dst2.AgentName()) + assert.Nil(t, dst2.User()) +} + +func TestCovJsonrpcRequest_SimplifyAgentName(t *testing.T) { + r := NewNormalizedRequest(nil) + cases := map[string]string{ + "curl/8.1": "curl", + "Wget/1.21": "wget", + "PostmanRuntime/7": "postman", + "insomnia/2023": "insomnia", + "HTTPie/3": "httpie", + "python-requests/2.31": "python", + "node-fetch/1.0": "nodejs", + "javascript-client": "nodejs", + "Go-http-client/1.1": "go", + "go/1.22": "go", + "Java/17": "java", + "rust-reqwest": "rust", + "Ruby": "ruby", + "PHP/8": "php", + "Mozilla Chrome/1": "chrome", + "Mozilla Firefox/1": "firefox", + "Mozilla Safari/1": "safari", + "Mozilla Edge/1": "edge", + "viem/2.0": "viem", + "ethers/6": "ethers", + "web3.py": "web3", + "web3j-client": "web3", + "alchemy-sdk": "alchemy-sdk", + "infura-client": "infura-sdk", + "quicknode-x": "nodejs", // "node" substring wins over quicknode + "QuickNode-SDK/1": "nodejs", + "MyBot/1.0 (+https://x)": "mybot", + "averyveryverylongagentname-xyz": "averyveryverylongag"[:19] + "e", + "(weird)": "other", + " ": "other", + } + for in, want := range cases { + assert.Equal(t, want, r.simplifyAgentName(in), "agent %q", in) + } +} + +func TestCovJsonrpcRequest_NextUpstream_UseUpstreamAndGates(t *testing.T) { + r := NewNormalizedRequest([]byte(`{"method":"eth_call","id":1}`)) + _, err := r.NextUpstream() + assert.True(t, HasErrorCode(err, ErrCodeNoUpstreamsLeftToSelect)) + + a := &covJsonrpcUpstream{id: "alchemy", cfg: &UpstreamConfig{Id: "alchemy"}} + b := &covJsonrpcUpstream{id: "infura", cfg: &UpstreamConfig{Id: "infura", Tags: []string{"family:systx"}}} + r.SetUpstreams([]Upstream{a, b}) + r.SetDirectives(&RequestDirectives{UseUpstream: "family:systx"}) + + got, err := r.NextUpstream() + require.NoError(t, err) + assert.Same(t, b, got, "selector matches upstream tags") + _, err = r.NextUpstream() + assert.True(t, HasErrorCode(err, ErrCodeNoUpstreamsLeftToSelect), "non-matching upstreams are never picked") +} + +func TestCovJsonrpcRequest_MarkUpstreamCompleted(t *testing.T) { + ctx := context.Background() + up := &covJsonrpcUpstream{id: "u", cfg: &UpstreamConfig{Id: "u"}} + + t.Run("canceled hedge frees reservation without recording", func(t *testing.T) { + r := NewNormalizedRequest([]byte(`{"method":"eth_call","id":1}`)) + r.SetUpstreams([]Upstream{up}) + _, err := r.NextUpstream() + require.NoError(t, err) + r.MarkUpstreamCompleted(ctx, up, nil, NewErrEndpointRequestCanceled(errors.New("hedge won"))) + _, recorded := r.ErrorsByUpstream.Load(up) + assert.False(t, recorded) + again, err := r.NextUpstream() + require.NoError(t, err) + assert.Same(t, up, again) + }) + + t.Run("emptyish without parsed json-rpc records missing data", func(t *testing.T) { + r := NewNormalizedRequest([]byte(`{"method":"eth_call","id":1}`)) + r.SetUpstreams([]Upstream{up}) + _, err := r.NextUpstream() + require.NoError(t, err) + // A response whose body never parses still exposes a json-rpc error object; + // an emptyish result is recorded as missing data. + resp := NewNormalizedResponse().WithJsonRpcResponse(MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`"0x"`), nil)) + r.MarkUpstreamCompleted(ctx, up, resp, nil) + v, ok := r.ErrorsByUpstream.Load(up) + require.True(t, ok) + assert.True(t, HasErrorCode(v.(error), ErrCodeEndpointMissingData)) + assert.Contains(t, v.(error).Error(), "emptyish") + }) +} diff --git a/common/cov_jsonrpc_response_test.go b/common/cov_jsonrpc_response_test.go new file mode 100644 index 000000000..ef2068a5e --- /dev/null +++ b/common/cov_jsonrpc_response_test.go @@ -0,0 +1,435 @@ +package common + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "strings" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func covJsonrpcRespFromBody(body string) (*NormalizedResponse, *covJsonrpcBody) { + b := &covJsonrpcBody{Reader: strings.NewReader(body)} + return NewNormalizedResponse().WithBody(b).WithExpectedSize(len(body)), b +} + +func TestCovJsonrpcResponse_NilReceiverAccessors(t *testing.T) { + var r *NormalizedResponse + ctx := context.Background() + + assert.False(t, r.FromCache()) + assert.Nil(t, r.SetFromCache(true)) + assert.Nil(t, r.EvmBlockRef()) + r.SetEvmBlockRef("0x1") + assert.Nil(t, r.EvmBlockNumber()) + r.SetEvmBlockNumber(int64(1)) + assert.Equal(t, DataFinalityStateUnknown, r.Finality(ctx)) + assert.Equal(t, time.Duration(0), r.Duration()) + r.SetDuration(time.Second) + assert.Equal(t, 0, r.Attempts()) + assert.Equal(t, 0, r.Retries()) + assert.Equal(t, 0, r.Hedges()) + assert.Nil(t, r.Upstream()) + assert.Equal(t, "", r.UpstreamId()) + assert.Nil(t, r.SetUpstream(&covJsonrpcUpstream{id: "a"})) + assert.Nil(t, r.WithRequest(NewNormalizedRequest(nil))) + assert.Nil(t, r.WithFromCache(true)) + assert.Nil(t, r.WithFinality(DataFinalityStateFinalized)) + jrr, err := r.JsonRpcResponse(ctx) + assert.Nil(t, jrr) + assert.NoError(t, err) + assert.Nil(t, r.WithBody(io.NopCloser(strings.NewReader("")))) + assert.Nil(t, r.WithExpectedSize(10)) + r.AddRef() + r.DoneRef() + assert.Nil(t, r.Request()) + r.MarkIntegrityRejected() + assert.False(t, r.IsIntegrityRejected()) + assert.True(t, r.IsObjectNull(ctx)) + assert.True(t, r.IsResultEmptyish(ctx), "nil response has no result") + n, err := r.WriteTo(&bytes.Buffer{}) + assert.Zero(t, n) + assert.ErrorContains(t, err, "unexpected nil response") + r.Release() + r.closeBodyOnce() + h, err := r.Hash() + assert.Equal(t, "", h) + assert.NoError(t, err) + h, err = r.HashWithIgnoredFields([]string{"a"}) + assert.Equal(t, "", h) + assert.NoError(t, err) + sz, err := r.Size() + assert.Zero(t, sz) + assert.NoError(t, err) +} + +func TestCovJsonrpcResponse_CountersAndFlags(t *testing.T) { + r := NewNormalizedResponse() + // Unset counters read as zero. + assert.Equal(t, 0, r.Attempts()) + assert.Equal(t, 0, r.Retries()) + assert.Equal(t, 0, r.Hedges()) + + assert.Same(t, r, r.SetAttempts(3).SetRetries(2).SetHedges(1)) + assert.Equal(t, 3, r.Attempts()) + assert.Equal(t, 2, r.Retries()) + assert.Equal(t, 1, r.Hedges()) + + r.SetDuration(150 * time.Millisecond) + assert.Equal(t, 150*time.Millisecond, r.Duration()) + + assert.False(t, r.FromCache()) + r.SetFromCache(true) + assert.True(t, r.FromCache()) + r.WithFromCache(false) + assert.False(t, r.FromCache()) + + // nil values never overwrite stored block refs/numbers. + r.SetEvmBlockRef("0xabc") + r.SetEvmBlockRef(nil) + assert.Equal(t, "0xabc", r.EvmBlockRef()) + r.SetEvmBlockNumber(int64(42)) + r.SetEvmBlockNumber(nil) + assert.Equal(t, int64(42), r.EvmBlockNumber()) + + assert.False(t, r.IsIntegrityRejected()) + r.MarkIntegrityRejected() + assert.True(t, r.IsIntegrityRejected()) +} + +func TestCovJsonrpcResponse_UpstreamAccessors(t *testing.T) { + r := NewNormalizedResponse() + assert.Nil(t, r.Upstream()) + assert.Equal(t, "", r.UpstreamId()) + + r.SetUpstream(nil) + assert.Nil(t, r.Upstream(), "nil upstream must not be stored") + + noCfg := &covJsonrpcUpstream{id: "no-cfg"} + r.SetUpstream(noCfg) + assert.Same(t, noCfg, r.Upstream()) + assert.Equal(t, "", r.UpstreamId(), "upstream without config has no reportable id") + + r2 := NewNormalizedResponse().SetUpstream(&covJsonrpcUpstream{id: "alchemy", cfg: &UpstreamConfig{Id: "alchemy"}}) + assert.Equal(t, "alchemy", r2.UpstreamId()) +} + +func TestCovJsonrpcResponse_Finality(t *testing.T) { + ctx := context.Background() + + t.Run("explicit finality wins and is cached", func(t *testing.T) { + r := NewNormalizedResponse().WithFinality(DataFinalityStateUnfinalized) + assert.Equal(t, DataFinalityStateUnfinalized, r.Finality(ctx)) + }) + + t.Run("no request yields unknown", func(t *testing.T) { + assert.Equal(t, DataFinalityStateUnknown, NewNormalizedResponse().Finality(ctx)) + }) + + t.Run("realtime request finality is adopted", func(t *testing.T) { + req := NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_gasPrice"}`)) + req.finality.Store(DataFinalityStateRealtime) + r := NewNormalizedResponse().WithRequest(req) + assert.Equal(t, DataFinalityStateRealtime, r.Finality(ctx)) + }) + + t.Run("network finality is cached only when definitive", func(t *testing.T) { + nw := &covJsonrpcNetwork{id: "evm:1", finality: DataFinalityStateFinalized} + req := NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber"}`)) + req.SetNetwork(nw) + r := NewNormalizedResponse().WithRequest(req) + assert.Equal(t, DataFinalityStateFinalized, r.Finality(ctx)) + calls := nw.finalityCalls + assert.Equal(t, DataFinalityStateFinalized, r.Finality(ctx)) + assert.Equal(t, calls, nw.finalityCalls, "definitive finality must be served from cache") + + nwU := &covJsonrpcNetwork{id: "evm:1", finality: DataFinalityStateUnknown} + reqU := NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"trace_transaction"}`)) + reqU.SetNetwork(nwU) + rU := NewNormalizedResponse().WithRequest(reqU) + assert.Equal(t, DataFinalityStateUnknown, rU.Finality(ctx)) + before := nwU.finalityCalls + assert.Equal(t, DataFinalityStateUnknown, rU.Finality(ctx)) + assert.Greater(t, nwU.finalityCalls, before, "unknown finality must be recomputed") + }) +} + +func TestCovJsonrpcResponse_JsonRpcResponseParsing(t *testing.T) { + ctx := context.Background() + + t.Run("valid body is parsed once and body closed", func(t *testing.T) { + r, body := covJsonrpcRespFromBody(`{"jsonrpc":"2.0","id":7,"result":"0x10"}`) + jrr, err := r.JsonRpcResponse(ctx) + require.NoError(t, err) + assert.Equal(t, `"0x10"`, jrr.GetResultString()) + assert.Equal(t, int64(7), jrr.ID()) + assert.Equal(t, 1, body.closed) + + again, err := r.JsonRpcResponse() + require.NoError(t, err) + assert.Same(t, jrr, again) + r.Release() + assert.Equal(t, 1, body.closed, "body is closed exactly once") + }) + + t.Run("invalid body yields parse exception with raw text as data", func(t *testing.T) { + r, _ := covJsonrpcRespFromBody(`bad gateway`) + jrr, err := r.JsonRpcResponse(ctx) + require.NoError(t, err) + require.NotNil(t, jrr.Error) + assert.Equal(t, int(JsonRpcErrorParseException), jrr.Error.Code) + assert.Contains(t, jrr.Error.Message, "cannot parse json-rpc response") + assert.Equal(t, `bad gateway`, jrr.Error.Data) + }) + + t.Run("missing body yields parse exception", func(t *testing.T) { + jrr, err := NewNormalizedResponse().JsonRpcResponse() + require.NoError(t, err) + require.NotNil(t, jrr.Error) + assert.Equal(t, int(JsonRpcErrorParseException), jrr.Error.Code) + assert.Equal(t, "no body available to parse JsonRpcResponse", jrr.Error.Message) + }) + + t.Run("close error is logged through the network logger", func(t *testing.T) { + covJsonrpcEnableLogs(t) + for _, closeErr := range []error{context.Canceled, errors.New("disk on fire")} { + var logBuf bytes.Buffer + nw := &covJsonrpcNetwork{id: "evm:1", logger: zerolog.New(&logBuf).Level(zerolog.DebugLevel)} + req := NewNormalizedRequest([]byte(`{"method":"eth_chainId"}`)) + req.SetNetwork(nw) + b := &covJsonrpcBody{Reader: strings.NewReader(`{"id":1,"result":"0x1"}`), closeErr: closeErr} + r := NewNormalizedResponse().WithRequest(req).WithBody(b) + _, err := r.JsonRpcResponse() + require.NoError(t, err) + if errors.Is(closeErr, context.Canceled) { + assert.Contains(t, logBuf.String(), "response body close on canceled request") + assert.Contains(t, logBuf.String(), `"level":"debug"`) + } else { + assert.Contains(t, logBuf.String(), "failed to close response body") + assert.Contains(t, logBuf.String(), `"level":"warn"`) + } + } + }) + + t.Run("close error without network is swallowed", func(t *testing.T) { + b := &covJsonrpcBody{Reader: strings.NewReader(`{"id":1,"result":"0x1"}`), closeErr: errors.New("x")} + r := NewNormalizedResponse().WithBody(b) + jrr, err := r.JsonRpcResponse() + require.NoError(t, err) + assert.Equal(t, `"0x1"`, jrr.GetResultString()) + assert.Equal(t, 1, b.closed) + }) +} + +func TestCovJsonrpcResponse_EmptyishAndObjectNull(t *testing.T) { + ctx := context.Background() + cases := []struct { + name string + jrr *JsonRpcResponse + emptyish bool + objectNull bool + }{ + {"real result", MustNewJsonRpcResponseFromBytes([]byte("1"), []byte(`"0x1"`), nil), false, false}, + {"null result with id", MustNewJsonRpcResponseFromBytes([]byte("1"), []byte(`null`), nil), true, false}, + {"empty array", MustNewJsonRpcResponseFromBytes([]byte("1"), []byte(`[]`), nil), true, false}, + // IsObjectNull compares ID() to untyped 0 (an int), so only a typed int + // zero id with no result and no error is reported as a null object. + {"empty with int zero id", &JsonRpcResponse{id: 0}, true, true}, + {"error only", &JsonRpcResponse{Error: NewErrJsonRpcExceptionExternal(-32000, "x", "")}, true, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := NewNormalizedResponse().WithJsonRpcResponse(tc.jrr) + assert.Equal(t, tc.emptyish, r.IsResultEmptyish(ctx)) + assert.Equal(t, tc.objectNull, r.IsObjectNull(ctx)) + assert.Equal(t, tc.objectNull, r.IsObjectNull()) + }) + } +} + +func TestCovJsonrpcResponse_MarshalAndWriteTo(t *testing.T) { + r := NewNormalizedResponse() + b, err := r.MarshalJSON() + assert.Nil(t, b) + assert.NoError(t, err) + _, err = r.WriteTo(&bytes.Buffer{}) + assert.ErrorContains(t, err, "unexpected empty response") + + jrr := MustNewJsonRpcResponseFromBytes([]byte(`5`), []byte(`{"a":1}`), nil) + r.WithJsonRpcResponse(jrr) + var buf bytes.Buffer + n, err := r.WriteTo(&buf) + require.NoError(t, err) + assert.Equal(t, `{"jsonrpc":"2.0","id":5,"result":{"a":1}}`, buf.String()) + assert.Equal(t, int64(buf.Len()), n) + + // JsonRpcResponse forbids MarshalJSON, so the normalized wrapper surfaces it. + _, err = r.MarshalJSON() + assert.ErrorContains(t, err, "MarshalJSON must not be used") +} + +func TestCovJsonrpcResponse_HashAndSize(t *testing.T) { + r := NewNormalizedResponse().WithJsonRpcResponse( + MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`{"b":"0x2","a":"0x1","ts":"0x5"}`), nil)) + h, err := r.Hash(context.Background()) + require.NoError(t, err) + assert.Len(t, h, 64) + + other := NewNormalizedResponse().WithJsonRpcResponse( + MustNewJsonRpcResponseFromBytes([]byte(`2`), []byte(`{"a":"0x1","b":"0x2","ts":"0x9"}`), nil)) + oh, err := other.Hash() + require.NoError(t, err) + assert.NotEqual(t, h, oh) + + ignore := []string{"ts"} + hi, err := r.HashWithIgnoredFields(ignore) + require.NoError(t, err) + ohi, err := other.HashWithIgnoredFields(ignore) + require.NoError(t, err) + assert.Equal(t, hi, ohi, "ignoring the differing field equalizes hashes") + + sz, err := r.Size() + require.NoError(t, err) + assert.Equal(t, len(`{"b":"0x2","a":"0x1","ts":"0x5"}`), sz) +} + +func TestCovJsonrpcResponse_MarshalZerologObject(t *testing.T) { + covJsonrpcEnableLogs(t) + render := func(r *NormalizedResponse) string { + var buf bytes.Buffer + lg := zerolog.New(&buf) + lg.Info().Object("resp", r).Send() + return buf.String() + } + + var nilResp *NormalizedResponse + assert.Contains(t, render(nilResp), `"resp":{}`) + + r := NewNormalizedResponse().SetAttempts(2).SetRetries(1).SetHedges(1) + out := render(r) + assert.Contains(t, out, `"upstream":"nil"`) + assert.Contains(t, out, `"attempts":2`) + assert.NotContains(t, out, `"jsonRpc"`) + + r.SetUpstream(&covJsonrpcUpstream{id: "x"}) + assert.Regexp(t, `"upstream":"0x[0-9a-f]+"`, render(r), "upstream without config is logged by pointer") + + r2 := NewNormalizedResponse(). + SetUpstream(&covJsonrpcUpstream{id: "up-1", cfg: &UpstreamConfig{Id: "up-1"}}). + WithJsonRpcResponse(MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`"0x1"`), nil)) + r2.SetFromCache(true) + out2 := render(r2) + assert.Contains(t, out2, `"upstream":"up-1"`) + assert.Contains(t, out2, `"fromCache":true`) + assert.Contains(t, out2, `"jsonRpc":{"id":1,"resultSize":5,"result":"0x1"}`) +} + +func TestCovJsonrpcResponse_ReleaseWaitsForRefs(t *testing.T) { + jrr := MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`"0x1"`), nil) + req := NewNormalizedRequest([]byte(`{"method":"eth_chainId"}`)) + r := NewNormalizedResponse().WithJsonRpcResponse(jrr).WithRequest(req) + r.AddRef() + + released := make(chan struct{}) + go func() { + r.Release() + close(released) + }() + select { + case <-released: + t.Fatal("Release returned while a ref was still held") + case <-time.After(20 * time.Millisecond): + } + r.DoneRef() + <-released + + assert.Nil(t, r.Request(), "release detaches the request") + assert.Nil(t, jrr.GetResultBytes(), "release frees the json-rpc result") + _, err := r.WriteTo(&bytes.Buffer{}) + assert.ErrorContains(t, err, "unexpected empty response") + r.Release() // idempotent +} + +func TestCovJsonrpcResponse_LookupResponseMetadata(t *testing.T) { + assert.Nil(t, LookupResponseMetadata(nil)) + assert.Nil(t, LookupResponseMetadata(errors.New("plain"))) + + // StandardError without a cause and without metadata. + assert.Nil(t, LookupResponseMetadata(NewErrNotImplemented("x"))) + + // A NormalizedResponse carried as a cause is found through the chain. + resp := NewNormalizedResponse().SetAttempts(3).SetRetries(2) + md := LookupResponseMetadata(&covJsonrpcMetaErr{NormalizedResponse: resp}) + require.NotNil(t, md) + assert.Equal(t, 3, md.Attempts()) + + wrapped := NewErrFailsafeRetryExceeded(ScopeNetwork, &covJsonrpcMetaErr{NormalizedResponse: resp}, nil) + md2 := LookupResponseMetadata(wrapped) + require.NotNil(t, md2, "metadata must be found through the cause chain") + assert.Equal(t, 2, md2.Retries()) +} + +// covJsonrpcMetaErr is an error that also exposes response metadata. +type covJsonrpcMetaErr struct{ *NormalizedResponse } + +func (covJsonrpcMetaErr) Error() string { return "meta" } + +func TestCovJsonrpcResponse_CopyResponseForRequest(t *testing.T) { + ctx := context.Background() + + t.Run("nil response", func(t *testing.T) { + req := NewNormalizedRequest([]byte(`{"method":"eth_chainId","id":1}`)) + out, err := CopyResponseForRequest(ctx, nil, req) + assert.Nil(t, out) + assert.NoError(t, err) + }) + + t.Run("copies metadata and rebinds id from parsed envelope", func(t *testing.T) { + src := NewNormalizedResponse(). + SetAttempts(2).SetRetries(1).SetHedges(1). + SetUpstream(&covJsonrpcUpstream{id: "u", cfg: &UpstreamConfig{Id: "u"}}). + WithJsonRpcResponse(MustNewJsonRpcResponseFromBytes([]byte(`99`), []byte(`"0xabc"`), nil)) + src.SetFromCache(true) + src.SetEvmBlockRef("0x5") + src.SetEvmBlockNumber(int64(5)) + + req := NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBalance","id":"client-1"}`)) + out, err := CopyResponseForRequest(ctx, src, req) + require.NoError(t, err) + assert.Same(t, req, out.Request()) + assert.Equal(t, 2, out.Attempts()) + assert.Equal(t, 1, out.Retries()) + assert.Equal(t, 1, out.Hedges()) + assert.True(t, out.FromCache()) + assert.Equal(t, "u", out.UpstreamId()) + assert.Equal(t, "0x5", out.EvmBlockRef()) + assert.Equal(t, int64(5), out.EvmBlockNumber()) + var buf bytes.Buffer + _, err = out.WriteTo(&buf) + require.NoError(t, err) + assert.Equal(t, `{"jsonrpc":"2.0","id":"client-1","result":"0xabc"}`, buf.String()) + }) + + t.Run("unparseable request is reported", func(t *testing.T) { + src := NewNormalizedResponse().WithJsonRpcResponse(MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`1`), nil)) + req := NewNormalizedRequest([]byte(`not json`)) + _, err := CopyResponseForRequest(ctx, src, req) + assert.True(t, HasErrorCode(err, ErrCodeJsonRpcRequestUnmarshal), "got %v", err) + }) + + t.Run("source with failing result writer", func(t *testing.T) { + jrr := &JsonRpcResponse{} + jrr.SetResultWriter(&covJsonrpcByteWriter{writeErr: fmt.Errorf("stream broken")}) + src := NewNormalizedResponse().WithJsonRpcResponse(jrr) + req := NewNormalizedRequest([]byte(`{"method":"eth_x","id":1}`)) + _, err := CopyResponseForRequest(ctx, src, req) + assert.ErrorContains(t, err, "stream broken") + }) +} diff --git a/common/cov_misc_behavior_test.go b/common/cov_misc_behavior_test.go new file mode 100644 index 000000000..944e5bf83 --- /dev/null +++ b/common/cov_misc_behavior_test.go @@ -0,0 +1,746 @@ +package common + +import ( + "bytes" + "context" + "crypto/x509" + "os" + "path/filepath" + "runtime" + "sync" + "testing" + "time" + + "github.com/grafana/sobek" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "gopkg.in/yaml.v3" +) + +// ---- timeout_func.go ---- + +func TestCovMiscTimeoutFunc_DisabledAndStatic(t *testing.T) { + logger := zerolog.Nop() + assert.Nil(t, NewTimeoutFunc(&logger, nil)) + assert.Nil(t, NewTimeoutFunc(&logger, &TimeoutPolicyConfig{})) + assert.Nil(t, NewTimeoutFunc(&logger, &TimeoutPolicyConfig{Duration: &AdaptiveDuration{}})) + // Only Min set, no quantile: static Resolve returns Base=0 → no timeout. + assert.Nil(t, NewTimeoutFunc(&logger, &TimeoutPolicyConfig{Duration: &AdaptiveDuration{Min: Duration(time.Second)}})) + + fn := NewTimeoutFunc(&logger, &TimeoutPolicyConfig{Duration: NewStaticDuration(3 * time.Second)}) + require.NotNil(t, fn) + got := fn(context.Background(), nil) + require.NotNil(t, got) + assert.Equal(t, 3*time.Second, *got) +} + +func TestCovMiscTimeoutFunc_QuantilePaths(t *testing.T) { + logger := zerolog.Nop() + ntw := &covMiscNetwork{id: "evm:1", label: "mainnet", finality: DataFinalityStateUnfinalized, metrics: map[string]TrackedMetrics{ + "eth_call": &covMiscMetrics{q: &fakeQuantile{val: 400 * time.Millisecond}}, + "eth_getLogs": &covMiscMetrics{q: &fakeQuantile{val: 0}}, + }} + mkReq := func(method string, withNet bool) *NormalizedRequest { + r := NewNormalizedRequestFromJsonRpcRequest(NewJsonRpcRequest(method, nil)) + if withNet { + r.SetNetwork(ntw) + } + return r + } + ctx := context.Background() + + t.Run("base set: min auto-floors to base/2", func(t *testing.T) { + fn := NewTimeoutFunc(&logger, &TimeoutPolicyConfig{Duration: &AdaptiveDuration{Base: Duration(2 * time.Second), Quantile: 0.9, Max: Duration(10 * time.Second)}}) + require.NotNil(t, fn) + assert.Equal(t, 2400*time.Millisecond, *fn(ctx, mkReq("eth_call", true))) + // no latency data: Resolve → base + min(=1s) = 3s + assert.Equal(t, 3*time.Second, *fn(ctx, mkReq("eth_getLogs", true))) + // cold-start fallbacks use Base + assert.Equal(t, 2*time.Second, *fn(ctx, mkReq("eth_call", false))) + assert.Equal(t, 2*time.Second, *fn(ctx, mkReq("", true))) + assert.Equal(t, 2*time.Second, *fn(ctx, mkReq("eth_unknownMethod", true))) + }) + + t.Run("no base: min auto-floors to 500ms; fallback is max", func(t *testing.T) { + fn := NewTimeoutFunc(&logger, &TimeoutPolicyConfig{Duration: &AdaptiveDuration{Quantile: 0.5, Max: Duration(5 * time.Second)}}) + require.NotNil(t, fn) + assert.Equal(t, 500*time.Millisecond, *fn(ctx, mkReq("eth_call", true)), "400ms raised to 500ms floor") + assert.Equal(t, 5*time.Second, *fn(ctx, mkReq("eth_call", false))) + }) + + t.Run("explicit min is kept", func(t *testing.T) { + fn := NewTimeoutFunc(&logger, &TimeoutPolicyConfig{Duration: &AdaptiveDuration{Quantile: 0.5, Min: Duration(10 * time.Millisecond), Max: Duration(5 * time.Second)}}) + assert.Equal(t, 400*time.Millisecond, *fn(ctx, mkReq("eth_call", true))) + }) + + t.Run("fallback nil when neither base nor max", func(t *testing.T) { + assert.Nil(t, coldStartFallback(&AdaptiveDuration{Quantile: 0.5})) + }) + + t.Run("non-positive resolve falls back", func(t *testing.T) { + // A negative base makes Resolve non-positive even with tracker data. + fn := NewTimeoutFunc(&logger, &TimeoutPolicyConfig{Duration: &AdaptiveDuration{Base: Duration(-time.Second), Quantile: 0.5, Min: Duration(time.Millisecond), Max: Duration(time.Hour)}}) + require.NotNil(t, fn) + assert.Equal(t, time.Millisecond, *fn(ctx, mkReq("eth_call", true)), "min clamp keeps it positive") + }) +} + +// ---- match.go ---- + +type covMiscExec struct { + id string + method string + finality []DataFinalityState +} + +func TestCovMiscSelectExecutor_Priority(t *testing.T) { + getM := func(e covMiscExec) string { return e.method } + getF := func(e covMiscExec) []DataFinalityState { return e.finality } + execs := []covMiscExec{ + {"catchall", "*", nil}, + {"catchall2", "", nil}, + {"finality-only", "", []DataFinalityState{DataFinalityStateRealtime}}, + {"finality-only-2", "*", []DataFinalityState{DataFinalityStateRealtime}}, + {"method", "eth_call", nil}, + {"method-2", "eth_*", nil}, + {"method+finality", "eth_call", []DataFinalityState{DataFinalityStateUnfinalized}}, + {"method+finality-2", "eth_call|eth_x", []DataFinalityState{DataFinalityStateUnfinalized}}, + {"other", "trace_*", []DataFinalityState{DataFinalityStateFinalized}}, + } + cases := []struct { + method string + finality DataFinalityState + want string + }{ + {"eth_call", DataFinalityStateUnfinalized, "method+finality"}, + {"eth_call", DataFinalityStateFinalized, "method"}, + {"eth_getBalance", DataFinalityStateFinalized, "method-2"}, + {"debug_x", DataFinalityStateRealtime, "finality-only"}, + {"debug_x", DataFinalityStateUnknown, "catchall"}, + {"trace_block", DataFinalityStateFinalized, "other"}, + } + for _, tc := range cases { + got, ok := SelectExecutor(execs, tc.method, tc.finality, getM, getF) + require.True(t, ok, tc.method) + assert.Equal(t, tc.want, got.id, "%s/%s", tc.method, tc.finality) + } + + none, ok := SelectExecutor(execs[4:6], "trace_x", DataFinalityStateFinalized, getM, getF) + assert.False(t, ok) + assert.Equal(t, covMiscExec{}, none) + _, ok = SelectExecutor([]covMiscExec(nil), "eth_call", DataFinalityStateFinalized, getM, getF) + assert.False(t, ok) + + assert.True(t, matchMethodPattern("", "x")) + assert.True(t, matchMethodPattern("eth_call", "eth_call")) + assert.False(t, matchMethodPattern("eth_call", "eth_callx")) +} + +// ---- matcher.go ---- + +func TestCovMiscMatcher_NumericComparisons(t *testing.T) { + cases := []struct { + pattern, value string + want bool + }{ + {">=0x10", "0x10", true}, + {">=0x10", "0xf", false}, + {"<=0x10", "0x10", true}, + {"<=0x10", "0x11", false}, + {">0x10", "0x11", true}, + {">0x10", "0x10", false}, + {"<0x10", "0xf", true}, + {"<16", "0xf", true}, + {"=0x10", "0x10", true}, + {"=0x10", " 0x10 ", true}, + {"=0x10", "0x11", false}, + {">=zz", "0x10", false}, + {">=0x10", "0xzz", false}, + {">0x10", "17", false}, // non-hex values fall back to glob + {"", "", true}, + {"", "x", false}, + {"(eth_* & !eth_call) | debug_*", "eth_getLogs", true}, + {"(eth_* & !eth_call) | debug_*", "eth_call", false}, + {"(eth_* & !eth_call) | debug_*", "debug_trace", true}, + {"!!eth_call", "eth_call", true}, + } + for _, tc := range cases { + got, err := WildcardMatch(tc.pattern, tc.value) + require.NoError(t, err, tc.pattern) + assert.Equal(t, tc.want, got, "%q vs %q", tc.pattern, tc.value) + } + + assert.False(t, compareNumbers("1", "1", "~"), "unknown operator never matches") + n, err := parseNumber("0X1f") + require.NoError(t, err) + assert.Equal(t, int64(31), n) + n, err = parseNumber("0xzz1") + require.Error(t, err) + assert.Zero(t, n) + + // unbalanced parens parse but evaluate per grammar + m, err := NewWildcardMatcher("(a") + require.Error(t, err) + assert.Nil(t, m) + ok, err := WildcardMatch("(a", "a") + require.ErrorContains(t, err, "expected closing parenthesis") + assert.False(t, ok) + ok, err = WildcardMatch("(a)|", "x") + require.NoError(t, err) + assert.False(t, ok, "dangling operator matches nothing") +} + +func TestCovMiscMatcher_ValidatePatternErrors(t *testing.T) { + cases := []struct { + pattern string + wantErr string + }{ + {" ", "empty pattern"}, + {"a)", "unmatched closing parenthesis at position 1"}, + {"| a", "operator '|' missing operand at position 0"}, + {"a &", "operator '&' missing operand at position 1"}, + {"(| a)", "invalid left operand for '|' at position 1"}, + {"a | )", "invalid right operand for '|' at position 1"}, + {"a & |b", "invalid right operand for '&' at position 1"}, + {"a !", "NOT operator missing operand at position 1"}, + {"!)", "invalid operand for NOT at position 0"}, + {">=0xzz", "invalid hex number in comparison: >=0xzz"}, + {" n * 2;`), 0o600)) + require.NoError(t, os.WriteFile(entry, []byte(`import { double } from "./helper"; export const answer: number = double(21);`), 0o600)) + + js, err := CompileTypeScript(entry) + require.NoError(t, err) + rt, err := NewRuntime() + require.NoError(t, err) + _, err = rt.Evaluate(js) + require.NoError(t, err) + assert.Equal(t, int64(42), rt.Exports().Get("answer").ToInteger(), "bundled TS evaluates with imports resolved") + + _, err = CompileTypeScript(filepath.Join(dir, "missing.ts")) + require.ErrorContains(t, err, "build failed") + bad := filepath.Join(dir, "bad.ts") + require.NoError(t, os.WriteFile(bad, []byte(`export const x = ;`), 0o600)) + _, err = CompileTypeScript(bad) + require.ErrorContains(t, err, "build failed") + + fn, err := CompileFunction(`(function(a, b) { return a + b })`) + require.NoError(t, err) + res, err := fn(nil, rt.ToValue(2), rt.ToValue(3)) + require.NoError(t, err) + assert.Equal(t, int64(5), res.ToInteger()) + _, err = CompileFunction(`42`) + require.EqualError(t, err, "result is not a function") + _, err = CompileFunction(`function (`) + require.Error(t, err) + + prog, err := CompileProgram(`(x) => x + 1`) + require.NoError(t, err) + v, err := rt.VM().RunProgram(prog) + require.NoError(t, err) + f, ok := sobek.AssertFunction(v) + require.True(t, ok) + out, err := f(nil, rt.ToValue(1)) + require.NoError(t, err) + assert.Equal(t, int64(2), out.ToInteger()) + _, err = CompileProgram(`) =>`) + require.Error(t, err) +} + +func TestCovMiscConsole_LogsByLevel(t *testing.T) { + var buf bytes.Buffer + prevLogger, prevLevel := log.Logger, zerolog.GlobalLevel() + log.Logger = zerolog.New(&buf).Level(zerolog.InfoLevel) + zerolog.SetGlobalLevel(zerolog.TraceLevel) + t.Cleanup(func() { + log.Logger = prevLogger + zerolog.SetGlobalLevel(prevLevel) + }) + + rt, err := NewRuntime() + require.NoError(t, err) + _, err = rt.Evaluate(`console.log("hello", 1, true); console.warn("careful"); console.debug("hidden"); console.trace("hidden2"); console.info("fyi")`) + require.NoError(t, err) + out := buf.String() + assert.Contains(t, out, `"message":"hello 1 true"`) + assert.Contains(t, out, `"level":"warn","message":"careful"`) + assert.Contains(t, out, `"message":"fyi"`) + assert.NotContains(t, out, "hidden", "below logger level is dropped") +} + +func TestCovMiscUtils(t *testing.T) { + assert.Equal(t, []string{"a", "b", "c"}, RemoveDuplicates([]string{"a", "b", "a", "c", "b"})) + assert.Equal(t, []string{}, RemoveDuplicates(nil)) + + for _, in := range []string{`{}`, `[]`, `"s"`, `null`, `true`, `false`} { + assert.True(t, IsSemiValidJson([]byte(in)), in) + } + for _, in := range []string{``, ``, `1`} { + assert.False(t, IsSemiValidJson([]byte(in)), in) + } + + u, err := HexToUint64("0x10") + require.NoError(t, err) + assert.Equal(t, uint64(16), u) + i, err := HexToInt64("0xff") + require.NoError(t, err) + assert.Equal(t, int64(255), i) + b, err := HexToBytes("0x0102") + require.NoError(t, err) + assert.Equal(t, []byte{1, 2}, b) + h, err := NormalizeHex("0x00ff") + require.NoError(t, err) + assert.Equal(t, "0xff", h) +} + +func TestCovMiscTLS_CreateConfig(t *testing.T) { + certFile, keyFile := covMiscWriteCert(t) + + cfg, err := CreateTLSConfig(&TLSConfig{CertFile: certFile, KeyFile: keyFile, CAFile: certFile, InsecureSkipVerify: true}) + require.NoError(t, err) + assert.True(t, cfg.InsecureSkipVerify) + require.Len(t, cfg.Certificates, 1) + require.NotNil(t, cfg.RootCAs) + leaf, err := x509.ParseCertificate(cfg.Certificates[0].Certificate[0]) + require.NoError(t, err) + assert.Equal(t, "cov-misc.local", leaf.Subject.CommonName) + // RootCAs is built from CAFile: the self-signed cert verifies against it. + _, err = leaf.Verify(x509.VerifyOptions{Roots: cfg.RootCAs, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}}) + require.NoError(t, err) + + cfg, err = CreateTLSConfig(&TLSConfig{}) + require.NoError(t, err) + assert.Empty(t, cfg.Certificates) + assert.Nil(t, cfg.RootCAs) + + // cert without key: ignored (both required) + cfg, err = CreateTLSConfig(&TLSConfig{CertFile: certFile}) + require.NoError(t, err) + assert.Empty(t, cfg.Certificates) + + _, err = CreateTLSConfig(&TLSConfig{CertFile: certFile, KeyFile: certFile}) + require.ErrorContains(t, err, "failed to load client cert/key pair") + _, err = CreateTLSConfig(&TLSConfig{CAFile: filepath.Join(t.TempDir(), "nope.pem")}) + require.ErrorContains(t, err, "failed to read CA file") +} diff --git a/common/cov_misc_helpers_test.go b/common/cov_misc_helpers_test.go new file mode 100644 index 000000000..e1215a74f --- /dev/null +++ b/common/cov_misc_helpers_test.go @@ -0,0 +1,176 @@ +package common + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "iter" + "math/big" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" +) + +// covMiscNetwork is a minimal Network for exercising helpers that read a +// network's id/config/metrics/finality. +type covMiscNetwork struct { + id string + label string + cfg *NetworkConfig + metrics map[string]TrackedMetrics + finality DataFinalityState +} + +func (n *covMiscNetwork) Id() string { return n.id } +func (n *covMiscNetwork) Label() string { return n.label } +func (n *covMiscNetwork) ProjectId() string { return "cov-misc-project" } +func (n *covMiscNetwork) Architecture() NetworkArchitecture { return ArchitectureEvm } +func (n *covMiscNetwork) Config() *NetworkConfig { return n.cfg } +func (n *covMiscNetwork) Logger() *zerolog.Logger { l := zerolog.Nop(); return &l } +func (n *covMiscNetwork) GetMethodMetrics(method string) TrackedMetrics { + if m, ok := n.metrics[method]; ok { + return m + } + return nil +} +func (n *covMiscNetwork) Forward(context.Context, *NormalizedRequest) (*NormalizedResponse, error) { + return nil, nil +} +func (n *covMiscNetwork) GetFinality(context.Context, *NormalizedRequest, *NormalizedResponse) DataFinalityState { + return n.finality +} + +// covMiscEvmNetwork adds the EVM view and the routable-upstreams surface. +type covMiscEvmNetwork struct { + *covMiscNetwork + latest, finalized int64 + leader Upstream + refreshed atomic.Int32 + routable []Upstream +} + +func (n *covMiscEvmNetwork) EvmHighestLatestBlockNumber(context.Context) int64 { return n.latest } +func (n *covMiscEvmNetwork) EvmHighestFinalizedBlockNumber(context.Context) int64 { return n.finalized } +func (n *covMiscEvmNetwork) EvmLeaderUpstream(context.Context) Upstream { return n.leader } +func (n *covMiscEvmNetwork) EvmRefreshLatestBlock(context.Context) { n.refreshed.Add(1) } +func (n *covMiscEvmNetwork) EvmRoutableUpstreams(context.Context) iter.Seq[Upstream] { + return func(yield func(Upstream) bool) { + for _, u := range n.routable { + if !yield(u) { + return + } + } + } +} + +type covMiscMetrics struct{ q QuantileTracker } + +func (m *covMiscMetrics) ErrorRate() float64 { return 0 } +func (m *covMiscMetrics) GetResponseQuantiles() QuantileTracker { return m.q } + +// covMiscSyncedUpstream overrides the fake's fixed syncing state. +type covMiscSyncedUpstream struct { + *FakeUpstream + state EvmSyncingState +} + +func (u *covMiscSyncedUpstream) EvmSyncingState() EvmSyncingState { return u.state } + +func covMiscNewSyncedUpstream(id string, state EvmSyncingState) *covMiscSyncedUpstream { + return &covMiscSyncedUpstream{FakeUpstream: NewFakeUpstream(id).(*FakeUpstream), state: state} +} + +// covMiscWriteCert writes a self-signed ECDSA cert/key pair to a temp dir and +// returns (certFile, keyFile). +func covMiscWriteCert(t *testing.T) (string, string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "cov-misc.local"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + keyDer, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + + dir := t.TempDir() + certFile := filepath.Join(dir, "cert.pem") + keyFile := filepath.Join(dir, "key.pem") + require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600)) + require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDer}), 0o600)) + return certFile, keyFile +} + +// covMiscRecordSpans swaps the package tracer for one backed by an in-memory +// span recorder and restores every tracing global on cleanup. +func covMiscRecordSpans(t *testing.T, sampleRate float64, detailed bool, matchers []*ForceTraceMatcher) *tracetest.SpanRecorder { + t.Helper() + prevEnabled, prevDetailed := IsTracingEnabled, IsTracingDetailed + prevTracer, prevProvider, prevMatchers := tracer, tracerProvider, forceTraceMatchers + + sr := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(createTracingSampler(&TracingConfig{SampleRate: sampleRate})), + sdktrace.WithSpanProcessor(sr), + ) + tracer = tp.Tracer(instrumentationName) + tracerProvider = tp + IsTracingEnabled = true + IsTracingDetailed = detailed + forceTraceMatchers = matchers + + t.Cleanup(func() { + _ = tp.Shutdown(context.Background()) + IsTracingEnabled, IsTracingDetailed = prevEnabled, prevDetailed + tracer, tracerProvider, forceTraceMatchers = prevTracer, prevProvider, prevMatchers + }) + return sr +} + +// covMiscDisableTracing forces the disabled path and restores on cleanup. +func covMiscDisableTracing(t *testing.T) { + t.Helper() + prevEnabled, prevDetailed, prevProvider := IsTracingEnabled, IsTracingDetailed, tracerProvider + IsTracingEnabled, IsTracingDetailed = false, false + t.Cleanup(func() { + IsTracingEnabled, IsTracingDetailed, tracerProvider = prevEnabled, prevDetailed, prevProvider + }) +} + +func covMiscAttrs(kvs []attribute.KeyValue) map[string]attribute.Value { + out := make(map[string]attribute.Value, len(kvs)) + for _, kv := range kvs { + out[string(kv.Key)] = kv.Value + } + return out +} + +func covMiscOnlySpan(t *testing.T, sr *tracetest.SpanRecorder) sdktrace.ReadOnlySpan { + t.Helper() + ended := sr.Ended() + require.Len(t, ended, 1) + return ended[0] +} + +var _ trace.Span = defaultNoopSpan diff --git a/common/cov_misc_object_test.go b/common/cov_misc_object_test.go new file mode 100644 index 000000000..0688f8807 --- /dev/null +++ b/common/cov_misc_object_test.go @@ -0,0 +1,218 @@ +package common + +import ( + "reflect" + "testing" + "time" + + "github.com/grafana/sobek" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type covMiscJsInner struct { + Name string `json:"name"` +} + +type covMiscJsTarget struct { + Flag bool `json:"flag"` + Count int `json:"count"` + Size uint32 `json:"size"` + Ratio float64 `json:"ratio"` + Label string `json:"label,omitempty"` + Untagged string // falls back to the Go field name + Timeout Duration `json:"timeout"` + Period RateLimitPeriod `json:"period"` + Inner covMiscJsInner `json:"inner"` + Ptr *int `json:"ptr"` + PtrNull *int `json:"ptrNull"` + Tags []string `json:"tags"` + Nested []covMiscJsInner `json:"nested"` + Fn sobek.Callable `json:"fn"` + Labels map[string]string `json:"labels"` + NullMap map[string]int `json:"nullMap"` + Any interface{} `json:"any"` + Skipped string `json:"skipped"` + unexposed string //nolint:unused + Objects map[string]covMiscJsInner `json:"objects"` +} + +func covMiscEval(t *testing.T, src string) sobek.Value { + t.Helper() + rt, err := NewRuntime() + require.NoError(t, err) + v, err := rt.Evaluate(src) + require.NoError(t, err) + return v +} + +func TestCovMiscObject_MapsEveryKind(t *testing.T) { + v := covMiscEval(t, `({ + flag: true, count: 42, size: 7, ratio: 0.25, label: "lbl", Untagged: "u", + timeout: "1500ms", period: "hour", + inner: {name: "in"}, + ptr: 9, ptrNull: null, + tags: ["a", "b"], + nested: [{name: "n0"}, {name: "n1"}], + fn: function(x) { return x * 2 }, + labels: {k1: "v1", k2: "v2"}, + nullMap: null, + any: {deep: [1, "two"]}, + objects: {o: {name: "obj"}}, + })`) + + var got covMiscJsTarget + got.Skipped = "keep" + require.NoError(t, MapJavascriptObjectToGo(v, &got)) + + assert.True(t, got.Flag) + assert.Equal(t, 42, got.Count) + assert.Equal(t, uint32(7), got.Size) + assert.Equal(t, 0.25, got.Ratio) + assert.Equal(t, "lbl", got.Label) + assert.Equal(t, "u", got.Untagged) + assert.Equal(t, Duration(1500*time.Millisecond), got.Timeout) + assert.Equal(t, RateLimitPeriodHour, got.Period) + assert.Equal(t, "in", got.Inner.Name) + require.NotNil(t, got.Ptr) + assert.Equal(t, 9, *got.Ptr) + assert.Nil(t, got.PtrNull) + assert.Equal(t, []string{"a", "b"}, got.Tags) + assert.Equal(t, []covMiscJsInner{{"n0"}, {"n1"}}, got.Nested) + require.NotNil(t, got.Fn) + res, err := got.Fn(sobek.Undefined(), covMiscEval(t, "21")) + require.NoError(t, err) + assert.Equal(t, int64(42), res.ToInteger()) + assert.Equal(t, map[string]string{"k1": "v1", "k2": "v2"}, got.Labels) + assert.Nil(t, got.NullMap) + assert.Equal(t, map[string]interface{}{"deep": []interface{}{int64(1), "two"}}, got.Any) + assert.Equal(t, "keep", got.Skipped, "absent JS keys leave fields untouched") + assert.Equal(t, map[string]covMiscJsInner{"o": {"obj"}}, got.Objects) +} + +func TestCovMiscObject_NumericDurationIsMilliseconds(t *testing.T) { + var got struct { + Timeout Duration `json:"timeout"` + } + require.NoError(t, MapJavascriptObjectToGo(covMiscEval(t, `({timeout: 250})`), &got)) + assert.Equal(t, Duration(250*time.Millisecond), got.Timeout) +} + +func TestCovMiscObject_RateLimitPeriodForms(t *testing.T) { + type target struct { + P RateLimitPeriod `json:"p"` + } + cases := []struct { + js string + want RateLimitPeriod + wantErr string + }{ + {`"second"`, RateLimitPeriodSecond, ""}, + {`" 1M "`, RateLimitPeriodMinute, ""}, + {`"60s"`, RateLimitPeriodMinute, ""}, + {`"3600s"`, RateLimitPeriodHour, ""}, + {`"1d"`, RateLimitPeriodDay, ""}, + {`"7d"`, RateLimitPeriodWeek, ""}, + {`"720h"`, RateLimitPeriodMonth, ""}, + {`"year"`, RateLimitPeriodYear, ""}, + {`"fortnight"`, 0, "invalid rate limit period: fortnight"}, + {`4`, RateLimitPeriodWeek, ""}, + {`99`, 0, "invalid rate limit period: 99"}, + {`true`, 0, "invalid rate limit period type: bool"}, + } + for _, tc := range cases { + t.Run(tc.js, func(t *testing.T) { + var got target + err := MapJavascriptObjectToGo(covMiscEval(t, "({p: "+tc.js+"})"), &got) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + assert.Contains(t, err.Error(), "failed to set field p") + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got.P) + }) + } +} + +func TestCovMiscObject_Errors(t *testing.T) { + obj := covMiscEval(t, `({})`) + + var notStruct int + require.EqualError(t, MapJavascriptObjectToGo(obj, notStruct), "dest must be a non-nil pointer") + var nilPtr *covMiscJsInner + require.EqualError(t, MapJavascriptObjectToGo(obj, nilPtr), "dest must be a non-nil pointer") + require.EqualError(t, MapJavascriptObjectToGo(obj, ¬Struct), "dest must point to a struct") + + var inner covMiscJsInner + require.NoError(t, MapJavascriptObjectToGo(nil, &inner)) + require.NoError(t, MapJavascriptObjectToGo(sobek.Undefined(), &inner)) + require.NoError(t, MapJavascriptObjectToGo(sobek.Null(), &inner)) + assert.Empty(t, inner.Name) + + cases := []struct { + name string + dest interface{} + js string + wantErr string + }{ + {"bad duration", &struct { + D Duration `json:"d"` + }{}, `({d: "soon"})`, "invalid duration format"}, + {"array expected", &struct { + S []string `json:"s"` + }{}, `({s: "nope"})`, "expected array but got nope"}, + {"array undefined element", &struct { + S []string `json:"s"` + }{}, `({s: [undefined, "x"]})`, "expected array element 0"}, + {"array element error", &struct { + S []Duration `json:"s"` + }{}, `({s: ["bad"]})`, "invalid duration format"}, + {"pointer element error", &struct { + P *Duration `json:"p"` + }{}, `({p: "bad"})`, "invalid duration format"}, + {"func from object", &struct { + F sobek.Callable `json:"f"` + }{}, `({f: {}})`, "field is not a function"}, + {"func from primitive", &struct { + F sobek.Callable `json:"f"` + }{}, `({f: 1})`, "field is not a function"}, + {"map from primitive", &struct { + M map[string]string `json:"m"` + }{}, `({m: 5})`, "expected object but got 5"}, + {"map non-string key", &struct { + M map[int]string `json:"m"` + }{}, `({m: {a: "b"}})`, "unsupported map key type: int"}, + {"map value error", &struct { + M map[string]Duration `json:"m"` + }{}, `({m: {a: "bad"}})`, "failed to set map value for key a"}, + {"nested struct error", &struct { + In struct { + D Duration `json:"d"` + } `json:"in"` + }{}, `({in: {d: "bad"}})`, "failed to set field d"}, + {"unsupported kind", &struct { + C chan int `json:"c"` + }{}, `({c: 1})`, "unsupported kind: chan"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.ErrorContains(t, MapJavascriptObjectToGo(covMiscEval(t, tc.js), tc.dest), tc.wantErr) + }) + } +} + +func TestCovMiscObject_MapSkipsUndefinedValues(t *testing.T) { + var got struct { + M map[string]string `json:"m"` + } + require.NoError(t, MapJavascriptObjectToGo(covMiscEval(t, `({m: {a: "x", b: undefined}})`), &got)) + assert.Equal(t, map[string]string{"a": "x"}, got.M) +} + +func TestCovMiscObject_UnsettableFieldIsNoop(t *testing.T) { + var s struct{ hidden string } + fv := reflect.ValueOf(&s).Elem().Field(0) + require.NoError(t, setJsValueToGoField(covMiscEval(t, `"x"`), fv)) + assert.Empty(t, s.hidden) +} diff --git a/common/cov_misc_tracing_test.go b/common/cov_misc_tracing_test.go new file mode 100644 index 000000000..4e16a9d21 --- /dev/null +++ b/common/cov_misc_tracing_test.go @@ -0,0 +1,565 @@ +package common + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +// Tracing state is process-global; none of these tests run in parallel. + +func TestCovMiscTracing_DisabledIsNoop(t *testing.T) { + covMiscDisableTracing(t) + ctx := context.Background() + + gotCtx, span := StartSpan(ctx, "x") + assert.Equal(t, ctx, gotCtx) + assert.False(t, span.IsRecording()) + assert.Equal(t, trace.SpanContext{}, span.SpanContext()) + assert.Nil(t, span.TracerProvider()) + // noop methods must be callable + span.SetName("n") + span.SetAttributes(attribute.String("a", "b")) + span.AddEvent("e") + span.RecordError(errors.New("boom")) + span.SetStatus(codes.Error, "x") + span.End() + + _, dspan := StartDetailSpan(ctx, "x") + assert.False(t, dspan.IsRecording()) + + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.Header.Set("traceparent", "00-0af7651916cd43dd8448eb211c80319c-b7ad567c75a3d1e5-01") + assert.False(t, trace.SpanContextFromContext(ExtractHTTPRequestTraceContext(r)).IsValid()) + + w := httptest.NewRecorder() + InjectHTTPResponseTraceContext(ctx, w) + assert.Empty(t, w.Header().Get("traceparent")) + + sctx, sspan := StartHTTPServerSpan(ctx, r) + assert.Equal(t, ctx, sctx) + assert.False(t, sspan.IsRecording()) + + assert.Equal(t, ctx, StartRequestSpan(ctx, NewNormalizedRequest([]byte(`{"method":"eth_call"}`)))) + EndRequestSpan(ctx, nil, nil) // no-op + assert.Equal(t, ctx, SetForceTraceNetwork(ctx, "evm:1")) + assert.Equal(t, "", GetForceTraceNetwork(ctx)) + require.NoError(t, ForceFlushTraces(ctx)) +} + +func TestCovMiscTracing_ShutdownWithoutProviderIsNil(t *testing.T) { + prev := tracerProvider + tracerProvider = nil + t.Cleanup(func() { tracerProvider = prev }) + require.NoError(t, ShutdownTracing(context.Background())) +} + +func TestCovMiscTracing_StartSpanRecordsAndDetailGate(t *testing.T) { + sr := covMiscRecordSpans(t, 1, false, nil) + ctx := context.Background() + + _, s := StartSpan(ctx, "Cache.Get", trace.WithAttributes(attribute.String("k", "v"))) + assert.True(t, s.IsRecording()) + s.End() + _, d := StartDetailSpan(ctx, "Detail.Op") + assert.False(t, d.IsRecording(), "detail spans require IsTracingDetailed") + d.End() + + sp := covMiscOnlySpan(t, sr) + assert.Equal(t, "Cache.Get", sp.Name()) + assert.Equal(t, "v", covMiscAttrs(sp.Attributes())["k"].AsString()) + + IsTracingDetailed = true + _, d = StartDetailSpan(ctx, "Detail.Op") + d.End() + require.Len(t, sr.Ended(), 2) + assert.Equal(t, "Detail.Op", sr.Ended()[1].Name()) +} + +func TestCovMiscTracing_SampleRateZeroDropsUnlessForced(t *testing.T) { + sr := covMiscRecordSpans(t, 0, false, nil) + + r := httptest.NewRequest(http.MethodPost, "http://erpc.local/main/evm/1", nil) + _, span := StartHTTPServerSpan(context.Background(), r) + assert.False(t, span.IsRecording()) + span.End() + assert.Empty(t, sr.Ended()) + + r = httptest.NewRequest(http.MethodPost, "http://erpc.local/main/evm/1?force-trace=true", nil) + _, span = StartHTTPServerSpan(context.Background(), r) + assert.True(t, span.IsRecording()) + span.End() + sp := covMiscOnlySpan(t, sr) + attrs := covMiscAttrs(sp.Attributes()) + assert.Equal(t, "header_or_query", attrs["erpc.forced_trace_reason"].AsString()) + assert.True(t, attrs[forceTraceAttributeKey].AsBool()) + assert.Equal(t, trace.SpanKindServer, sp.SpanKind()) +} + +func TestCovMiscTracing_ShouldForceTraceHTTP(t *testing.T) { + cases := []struct { + name string + header string + query string + want bool + }{ + {"none", "", "", false}, + {"header true", "true", "", true}, + {"header 1", "1", "", true}, + {"header yes", "yes", "", true}, + {"header false wins over query", "no", "true", false}, + {"query 1", "", "1", true}, + {"query yes", "", "yes", true}, + {"query other", "", "nah", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + u := "http://x/" + if tc.query != "" { + u += "?" + ForceTraceQueryParam + "=" + tc.query + } + r := httptest.NewRequest(http.MethodGet, u, nil) + if tc.header != "" { + r.Header.Set(ForceTraceHeader, tc.header) + } + assert.Equal(t, tc.want, shouldForceTrace(r)) + }) + } +} + +func TestCovMiscTracing_HTTPServerSpanAttributesAndPropagation(t *testing.T) { + sr := covMiscRecordSpans(t, 1, false, nil) + + r := httptest.NewRequest(http.MethodPost, "http://erpc.local/p/evm/1", nil) + r.Header.Set("User-Agent", "cov-agent") + r.Header.Set("traceparent", "00-0af7651916cd43dd8448eb211c80319c-b7ad567c75a3d1e5-01") + + extracted := ExtractHTTPRequestTraceContext(r) + assert.Equal(t, "0af7651916cd43dd8448eb211c80319c", trace.SpanContextFromContext(extracted).TraceID().String()) + + ctx, span := StartHTTPServerSpan(context.Background(), r) + assert.Equal(t, "0af7651916cd43dd8448eb211c80319c", span.SpanContext().TraceID().String(), "remote parent continues trace") + + w := httptest.NewRecorder() + InjectHTTPResponseTraceContext(ctx, w) + assert.Contains(t, w.Header().Get("traceparent"), span.SpanContext().SpanID().String()) + + EnrichHTTPServerSpan(ctx, 200, nil) + span.End() + + sp := covMiscOnlySpan(t, sr) + attrs := covMiscAttrs(sp.Attributes()) + assert.Equal(t, "POST", attrs["http.method"].AsString()) + assert.Equal(t, "cov-agent", attrs["http.user_agent"].AsString()) + assert.Equal(t, int64(200), attrs["http.status_code"].AsInt64()) + assert.Equal(t, codes.Ok, sp.Status().Code) + assert.Equal(t, "b7ad567c75a3d1e5", sp.Parent().SpanID().String()) +} + +func TestCovMiscTracing_EnrichHTTPServerSpanStatus(t *testing.T) { + cases := []struct { + name string + status int + err error + wantCode codes.Code + wantDesc string + wantErrC string + }{ + {"client error status", 429, nil, codes.Error, "Too Many Requests", ""}, + {"standard error", 500, NewErrInvalidConfig("bad"), codes.Error, "ErrInvalidConfig", "ErrInvalidConfig"}, + {"plain error", 500, errors.New("plain"), codes.Error, "plain", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + sr := covMiscRecordSpans(t, 1, false, nil) + ctx, span := StartSpan(context.Background(), "s") + EnrichHTTPServerSpan(ctx, tc.status, tc.err) + span.End() + sp := covMiscOnlySpan(t, sr) + assert.Equal(t, tc.wantCode, sp.Status().Code) + assert.Contains(t, sp.Status().Description, tc.wantDesc) + attrs := covMiscAttrs(sp.Attributes()) + if tc.wantErrC != "" { + assert.Equal(t, tc.wantErrC, attrs["error.code"].AsString()) + } + if tc.err != nil { + require.Len(t, sp.Events(), 1) + assert.Equal(t, "exception", sp.Events()[0].Name) + } + }) + } + + t.Run("non-recording span untouched", func(t *testing.T) { + covMiscRecordSpans(t, 1, false, nil) + EnrichHTTPServerSpan(context.Background(), 500, errors.New("x")) // no span in ctx: must not panic + }) +} + +func TestCovMiscTracing_SetTraceSpanErrorIgnoresNonErrorsAndNilSpans(t *testing.T) { + sr := covMiscRecordSpans(t, 1, false, nil) + SetTraceSpanError(nil, errors.New("x")) + _, span := StartSpan(context.Background(), "s") + SetTraceSpanError(span, "a string is not an error") + span.End() + SetTraceSpanError(span, errors.New("after end: not recording")) + sp := covMiscOnlySpan(t, sr) + assert.Equal(t, codes.Unset, sp.Status().Code) + assert.Empty(t, sp.Events()) +} + +func TestCovMiscTracing_ForceTraceMatchers(t *testing.T) { + matchers := []*ForceTraceMatcher{ + nil, + {}, // empty: never matches + {Network: "evm:137", Method: "eth_call"}, + {Network: "evm:42161"}, + {Method: "debug_*|trace_*"}, + } + covMiscRecordSpans(t, 0, false, matchers) + + cases := []struct { + network, method string + want bool + reason string + }{ + {"evm:137", "eth_call", true, "network:evm:137,method:eth_call"}, + {"evm:137", "eth_getBalance", false, ""}, + {"evm:42161", "anything", true, "network:evm:42161"}, + {"evm:1", "trace_block", true, "method:trace_block"}, + {"evm:1", "eth_call", false, ""}, + } + for _, tc := range cases { + got, reason := ShouldForceTrace(tc.network, tc.method) + assert.Equal(t, tc.want, got, "%s %s", tc.network, tc.method) + assert.Equal(t, tc.reason, reason) + } + + forceTraceMatchers = nil + got, reason := ShouldForceTrace("evm:137", "eth_call") + assert.False(t, got) + assert.Empty(t, reason) +} + +func TestCovMiscTracing_ForceTraceNetworkContext(t *testing.T) { + covMiscRecordSpans(t, 1, false, nil) + ctx := context.Background() + assert.Equal(t, ctx, SetForceTraceNetwork(ctx, ""), "empty network is not stored") + ctx = SetForceTraceNetwork(ctx, "evm:10") + assert.Equal(t, "evm:10", GetForceTraceNetwork(ctx)) +} + +func TestCovMiscTracing_RequestSpanForcedByNetworkMatcher(t *testing.T) { + sr := covMiscRecordSpans(t, 0, true, []*ForceTraceMatcher{{Network: "evm:10"}}) + + jrq := NewJsonRpcRequest("eth_getBalance", []interface{}{"0xabc", "latest"}) + jrq.ID = 7 + req := NewNormalizedRequestFromJsonRpcRequest(jrq) + + // unmatched network: sampled out + ctx := StartRequestSpan(context.Background(), req) + EndRequestSpan(ctx, nil, nil) + assert.Empty(t, sr.Ended()) + + ctx = SetForceTraceNetwork(context.Background(), "evm:10") + ctx = StartRequestSpan(ctx, req) + EndRequestSpan(ctx, nil, NewErrInvalidConfig("nope")) + + sp := covMiscOnlySpan(t, sr) + assert.Equal(t, "Request.Handle", sp.Name()) + attrs := covMiscAttrs(sp.Attributes()) + assert.Equal(t, "eth_getBalance", attrs["request.method"].AsString()) + assert.Equal(t, "network:evm:10", attrs["erpc.forced_trace_reason"].AsString()) + assert.Equal(t, "7", attrs["request.id"].AsString()) + assert.Equal(t, `["0xabc","latest"]`, attrs["request.jsonrpc.params"].AsString()) + assert.Equal(t, codes.Error, sp.Status().Code) + assert.Equal(t, "ErrInvalidConfig", attrs["error.code"].AsString()) +} + +func TestCovMiscTracing_EndRequestSpanSuccessDetailed(t *testing.T) { + sr := covMiscRecordSpans(t, 1, true, nil) + + ntw := &covMiscNetwork{id: "evm:1", finality: DataFinalityStateFinalized} + req := NewNormalizedRequestFromJsonRpcRequest(NewJsonRpcRequest("eth_blockNumber", nil)) + req.SetNetwork(ntw) + req.SetUser(&User{Id: "alice"}) + + jrr, err := NewJsonRpcResponse(1, "0x10", nil) + require.NoError(t, err) + resp := NewNormalizedResponse(). + WithRequest(req). + WithJsonRpcResponse(jrr). + SetFromCache(true). + SetUpstream(NewFakeUpstream("up-a")). + SetAttempts(3). + SetRetries(2). + SetHedges(1) + + ctx := StartRequestSpan(context.Background(), req) + EndRequestSpan(ctx, resp, nil) + + sp := covMiscOnlySpan(t, sr) + attrs := covMiscAttrs(sp.Attributes()) + assert.Equal(t, codes.Ok, sp.Status().Code) + assert.True(t, attrs["cache.hit"].AsBool()) + assert.Equal(t, "up-a", attrs["upstream.id"].AsString()) + assert.Equal(t, int64(len(`"0x10"`)), attrs["response.result_size"].AsInt64()) + assert.Equal(t, "evm:1", attrs["network.id"].AsString()) + assert.Equal(t, "alice", attrs["user.id"].AsString()) + assert.Equal(t, "finalized", attrs["request.finality"].AsString()) + assert.Equal(t, "finalized", attrs["response.finality"].AsString()) + assert.Equal(t, int64(3), attrs["execution.attempts"].AsInt64()) + assert.Equal(t, int64(2), attrs["execution.retries"].AsInt64()) + assert.Equal(t, int64(1), attrs["execution.hedges"].AsInt64()) + _, hasParams := attrs["request.jsonrpc.params"] + assert.False(t, hasParams, "nil params are not recorded") +} + +func TestCovMiscTracing_EndRequestSpanNonRecordingContext(t *testing.T) { + sr := covMiscRecordSpans(t, 1, false, nil) + EndRequestSpan(context.Background(), NewNormalizedResponse(), nil) + assert.Empty(t, sr.Ended()) +} + +func TestCovMiscTracing_ForceFlush(t *testing.T) { + sr := covMiscRecordSpans(t, 1, false, nil) + _, s := StartSpan(context.Background(), "f") + s.End() + require.NoError(t, ForceFlushTraces(context.Background())) + assert.Len(t, sr.Ended(), 1) +} + +func TestCovMiscTracing_SamplerDecisions(t *testing.T) { + traceID := trace.TraceID{1} + forced := []attribute.KeyValue{attribute.Bool(forceTraceAttributeKey, true)} + notForced := []attribute.KeyValue{attribute.Bool(forceTraceAttributeKey, false), attribute.String("x", "y")} + + cases := []struct { + rate float64 + attrs []attribute.KeyValue + want sdktrace.SamplingDecision + descPart string + }{ + {0, nil, sdktrace.Drop, "AlwaysOffSampler"}, + {0, notForced, sdktrace.Drop, "AlwaysOffSampler"}, + {0, forced, sdktrace.RecordAndSample, "AlwaysOffSampler"}, + {1, nil, sdktrace.RecordAndSample, "AlwaysOnSampler"}, + {0.5, nil, sdktrace.RecordAndSample, "ParentBased"}, // TraceID{1,0..} is below the ratio bound + } + for _, tc := range cases { + s := createTracingSampler(&TracingConfig{SampleRate: tc.rate}) + res := s.ShouldSample(sdktrace.SamplingParameters{ParentContext: context.Background(), TraceID: traceID, Attributes: tc.attrs}) + assert.Equal(t, tc.want, res.Decision, "rate=%v attrs=%v", tc.rate, tc.attrs) + assert.True(t, strings.HasPrefix(s.Description(), "ForceTraceSampler{")) + assert.Contains(t, s.Description(), tc.descPart) + } +} + +func TestCovMiscTracing_HTTPEndpointOptions(t *testing.T) { + cases := []struct { + endpoint string + tls bool + wantN int + wantErr string + }{ + {"collector:4318", false, 2, ""}, + {"collector:4318", true, 1, ""}, + {"https://collector:4318", true, 1, ""}, + {"https://collector:4318/custom", false, 2, ""}, + {"http://collector:4318/", false, 2, ""}, + {"http://collector:4318", true, 0, "tls.enabled needs an https://"}, + {"ftp://collector:1", false, 0, "want http(s)://"}, + {"http://%zz", false, 0, "invalid tracing endpoint"}, + } + for _, tc := range cases { + opts, err := tracingHTTPEndpointOptions(tc.endpoint, tc.tls) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr, tc.endpoint) + continue + } + require.NoError(t, err, tc.endpoint) + assert.Len(t, opts, tc.wantN, tc.endpoint) + } +} + +func TestCovMiscTracing_ExportersWithTLSAndHeaders(t *testing.T) { + certFile, keyFile := covMiscWriteCert(t) + ctx := context.Background() + + goodTLS := &TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile, CAFile: certFile} + badTLS := &TLSConfig{Enabled: true, CAFile: certFile + ".missing"} + + g, err := createTracingGRPCExporter(ctx, &TracingConfig{Endpoint: "127.0.0.1:1", TLS: goodTLS, Headers: map[string]string{"x-k": "v"}}) + require.NoError(t, err) + require.NoError(t, g.Shutdown(ctx)) + g, err = createTracingGRPCExporter(ctx, &TracingConfig{Endpoint: "127.0.0.1:1"}) + require.NoError(t, err) + require.NoError(t, g.Shutdown(ctx)) + _, err = createTracingGRPCExporter(ctx, &TracingConfig{Endpoint: "127.0.0.1:1", TLS: badTLS}) + require.ErrorContains(t, err, "failed to read CA file") + + h, err := createTracingHTTPExporter(ctx, &TracingConfig{Endpoint: "https://127.0.0.1:1", TLS: goodTLS, Headers: map[string]string{"x-k": "v"}}) + require.NoError(t, err) + require.NoError(t, h.Shutdown(ctx)) + _, err = createTracingHTTPExporter(ctx, &TracingConfig{Endpoint: "127.0.0.1:1", TLS: badTLS}) + require.ErrorContains(t, err, "failed to read CA file") +} + +// covMiscResetTracingInit gives InitializeTracing a fresh sync.Once and +// restores every global it may touch. +func covMiscResetTracingInit(t *testing.T) { + t.Helper() + prevEnabled, prevDetailed := IsTracingEnabled, IsTracingDetailed + prevTracer, prevProvider, prevMatchers := tracer, tracerProvider, forceTraceMatchers + prevGlobalTP := otel.GetTracerProvider() + prevProp := otel.GetTextMapPropagator() + prevHandler := otel.GetErrorHandler() + initOnce = sync.Once{} + t.Cleanup(func() { + if tracerProvider != nil && tracerProvider != prevProvider { + _ = tracerProvider.Shutdown(context.Background()) + } + IsTracingEnabled, IsTracingDetailed = prevEnabled, prevDetailed + tracer, tracerProvider, forceTraceMatchers = prevTracer, prevProvider, prevMatchers + otel.SetTracerProvider(prevGlobalTP) + otel.SetTextMapPropagator(prevProp) + otel.SetErrorHandler(prevHandler) + initOnce = sync.Once{} + }) +} + +func TestCovMiscTracing_InitializeDisabled(t *testing.T) { + covMiscResetTracingInit(t) + IsTracingEnabled, IsTracingDetailed = true, true + logger := zerolog.Nop() + + require.NoError(t, InitializeTracing(context.Background(), &logger, nil)) + assert.False(t, IsTracingEnabled) + assert.False(t, IsTracingDetailed) + _, span := StartSpan(context.Background(), "after-disabled-init") + assert.False(t, span.IsRecording()) + + // second call is a no-op thanks to sync.Once, even with an enabled config + require.NoError(t, InitializeTracing(context.Background(), &logger, &TracingConfig{Enabled: true, Protocol: "bogus"})) + assert.False(t, IsTracingEnabled) +} + +func TestCovMiscTracing_InitializeUnsupportedProtocol(t *testing.T) { + covMiscResetTracingInit(t) + IsTracingEnabled = false + logger := zerolog.Nop() + err := InitializeTracing(context.Background(), &logger, &TracingConfig{Enabled: true, Protocol: "carrier-pigeon"}) + require.ErrorContains(t, err, "unsupported tracing protocol: carrier-pigeon") + assert.False(t, IsTracingEnabled) +} + +func TestCovMiscTracing_InitializeExporterError(t *testing.T) { + covMiscResetTracingInit(t) + IsTracingEnabled = false + logger := zerolog.Nop() + err := InitializeTracing(context.Background(), &logger, &TracingConfig{Enabled: true, Protocol: TracingProtocolHttp, Endpoint: "ftp://x:1"}) + require.ErrorContains(t, err, "invalid tracing endpoint") + assert.False(t, IsTracingEnabled) +} + +func TestCovMiscTracing_InitializeHTTPExportsSpans(t *testing.T) { + covMiscResetTracingInit(t) + + var mu sync.Mutex + var bodies [][]byte + got := make(chan struct{}, 8) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var b bytes.Buffer + _, _ = b.ReadFrom(r.Body) + mu.Lock() + bodies = append(bodies, b.Bytes()) + mu.Unlock() + w.WriteHeader(http.StatusOK) + got <- struct{}{} + })) + defer srv.Close() + + // Debug level exercises the otel-logger wiring branch (global test level + // is disabled, so nothing is written). + logger := zerolog.New(io.Discard).Level(zerolog.DebugLevel) + cfg := &TracingConfig{ + Enabled: true, + Protocol: TracingProtocolHttp, + Endpoint: srv.URL, + SampleRate: 1, + Detailed: true, + ServiceName: "cov-misc-svc", + ResourceAttributes: map[string]string{"deployment.env": "test", "skipped": ""}, + ForceTraceMatchers: []*ForceTraceMatcher{{Network: "evm:1", Method: "eth_call"}}, + } + require.NoError(t, InitializeTracing(context.Background(), &logger, cfg)) + assert.True(t, IsTracingEnabled) + assert.True(t, IsTracingDetailed) + forced, reason := ShouldForceTrace("evm:1", "eth_call") + assert.True(t, forced) + assert.Equal(t, "network:evm:1,method:eth_call", reason) + + // The global propagator now carries W3C trace context. + ctxWithSpan, parent := StartSpan(context.Background(), "Parent") + carrier := propagation.MapCarrier{} + otel.GetTextMapPropagator().Inject(ctxWithSpan, carrier) + assert.Contains(t, carrier.Get("traceparent"), parent.SpanContext().TraceID().String()) + parent.End() + + _, span := StartSpan(context.Background(), "Export.Me") + span.End() + require.NoError(t, ForceFlushTraces(context.Background())) + select { + case <-got: + case <-time.After(10 * time.Second): + t.Fatal("collector received no export") + } + mu.Lock() + payload := string(bytes.Join(bodies, nil)) + mu.Unlock() + assert.Contains(t, payload, "Export.Me") + assert.Contains(t, payload, "cov-misc-svc") + assert.Contains(t, payload, "deployment.env") + + require.NoError(t, ShutdownTracing(context.Background())) + tracerProvider = nil +} + +func TestCovMiscTracing_InitializeGRPC(t *testing.T) { + covMiscResetTracingInit(t) + logger := zerolog.Nop() + require.NoError(t, InitializeTracing(context.Background(), &logger, &TracingConfig{ + Enabled: true, Protocol: TracingProtocolGrpc, Endpoint: "127.0.0.1:1", SampleRate: 0.25, + })) + assert.True(t, IsTracingEnabled) + assert.False(t, IsTracingDetailed) + // Fractional rates are parent-based: a remote parent's decision wins. + sampledParent := trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{9}, SpanID: trace.SpanID{1}, TraceFlags: trace.FlagsSampled, Remote: true}) + droppedParent := trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{9}, SpanID: trace.SpanID{2}, Remote: true}) + _, s1 := StartSpan(trace.ContextWithRemoteSpanContext(context.Background(), sampledParent), "child") + _, s2 := StartSpan(trace.ContextWithRemoteSpanContext(context.Background(), droppedParent), "child") + assert.True(t, s1.IsRecording()) + assert.False(t, s2.IsRecording()) + s1.End() + s2.End() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = ShutdownTracing(ctx) // export to a closed port may error; shutdown must return + tracerProvider = nil +} diff --git a/common/cov_misc_values_test.go b/common/cov_misc_values_test.go new file mode 100644 index 000000000..5f8ec3060 --- /dev/null +++ b/common/cov_misc_values_test.go @@ -0,0 +1,348 @@ +package common + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestCovMiscData_FinalityStateStringAndMarshal(t *testing.T) { + cases := []struct { + f DataFinalityState + want string + }{ + {DataFinalityStateFinalized, "finalized"}, + {DataFinalityStateUnfinalized, "unfinalized"}, + {DataFinalityStateRealtime, "realtime"}, + {DataFinalityStateUnknown, "unknown"}, + {DataFinalityStateAll, "invalid(-1)"}, + {DataFinalityState(9), "invalid(9)"}, + } + for _, tc := range cases { + assert.Equal(t, tc.want, tc.f.String()) + y, err := yaml.Marshal(tc.f) + require.NoError(t, err) + assert.Equal(t, tc.want+"\n", string(y)) + j, err := tc.f.MarshalJSON() + require.NoError(t, err) + assert.Equal(t, `"`+tc.want+`"`, string(j)) + } +} + +func TestCovMiscData_FinalityStateUnmarshalYAML(t *testing.T) { + cases := []struct { + in string + want DataFinalityState + wantErr string + }{ + {"finalized", DataFinalityStateFinalized, ""}, + {"0", DataFinalityStateFinalized, ""}, + {"Unfinalized", DataFinalityStateUnfinalized, ""}, + {"1", DataFinalityStateUnfinalized, ""}, + {"REALTIME", DataFinalityStateRealtime, ""}, + {"2", DataFinalityStateRealtime, ""}, + {"unknown", DataFinalityStateUnknown, ""}, + {"3", DataFinalityStateUnknown, ""}, + {"pending", 0, "invalid data finality state: pending"}, + {"[a]", 0, "cannot unmarshal"}, + } + for _, tc := range cases { + var got DataFinalityState + err := yaml.Unmarshal([]byte(tc.in), &got) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr, tc.in) + continue + } + require.NoError(t, err, tc.in) + assert.Equal(t, tc.want, got, tc.in) + } +} + +func TestCovMiscData_CacheEmptyBehavior(t *testing.T) { + assert.Equal(t, "ignore", CacheEmptyBehaviorIgnore.String()) + assert.Equal(t, "allow", CacheEmptyBehaviorAllow.String()) + assert.Equal(t, "only", CacheEmptyBehaviorOnly.String()) + assert.Equal(t, "invalid(7)", CacheEmptyBehavior(7).String()) + assert.Equal(t, "invalid(-1)", CacheEmptyBehavior(-1).String()) + + y, err := yaml.Marshal(CacheEmptyBehaviorAllow) + require.NoError(t, err) + assert.Equal(t, "allow\n", string(y)) + + cases := []struct { + in string + want CacheEmptyBehavior + wantErr string + }{ + {"ignore", CacheEmptyBehaviorIgnore, ""}, + {"0", CacheEmptyBehaviorIgnore, ""}, + {"ALLOW", CacheEmptyBehaviorAllow, ""}, + {"1", CacheEmptyBehaviorAllow, ""}, + {"only", CacheEmptyBehaviorOnly, ""}, + {"2", CacheEmptyBehaviorOnly, ""}, + {"sometimes", 0, "invalid cache empty behavior: sometimes"}, + {"{a: 1}", 0, "cannot unmarshal"}, + } + for _, tc := range cases { + var got CacheEmptyBehavior + err := yaml.Unmarshal([]byte(tc.in), &got) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr, tc.in) + continue + } + require.NoError(t, err, tc.in) + assert.Equal(t, tc.want, got, tc.in) + } +} + +func TestCovMiscData_CachePolicyAppliesTo(t *testing.T) { + assert.Equal(t, "both", CachePolicyAppliesTo("").String(), "empty defaults to both") + assert.Equal(t, "get", CachePolicyAppliesToGet.String()) + y, err := yaml.Marshal(CachePolicyAppliesTo("")) + require.NoError(t, err) + assert.Equal(t, "both\n", string(y)) + j, err := CachePolicyAppliesToSet.MarshalJSON() + require.NoError(t, err) + assert.Equal(t, `"set"`, string(j)) + + cases := []struct { + in string + want CachePolicyAppliesTo + wantErr bool + }{ + {`""`, CachePolicyAppliesToBoth, false}, + {`" Both "`, CachePolicyAppliesToBoth, false}, + {`"GET"`, CachePolicyAppliesToGet, false}, + {`"set"`, CachePolicyAppliesToSet, false}, + {`"delete"`, "", true}, + } + for _, tc := range cases { + var gotJ, gotY CachePolicyAppliesTo + errJ := gotJ.UnmarshalJSON([]byte(tc.in)) + errY := yaml.Unmarshal([]byte(tc.in), &gotY) + if tc.wantErr { + require.ErrorContains(t, errJ, "invalid cache policy appliesTo") + require.ErrorContains(t, errY, "invalid cache policy appliesTo") + continue + } + require.NoError(t, errJ, tc.in) + require.NoError(t, errY, tc.in) + assert.Equal(t, tc.want, gotJ, tc.in) + assert.Equal(t, tc.want, gotY, tc.in) + } + var a CachePolicyAppliesTo + require.Error(t, a.UnmarshalJSON([]byte(`123`))) + require.Error(t, yaml.Unmarshal([]byte(`[x]`), &a)) +} + +func TestCovMiscDuration_YAMLAndJSON(t *testing.T) { + var d Duration + require.NoError(t, yaml.Unmarshal([]byte(`"1m30s"`), &d)) + assert.Equal(t, 90*time.Second, d.Duration()) + assert.Equal(t, "1m30s", d.String()) + + err := yaml.Unmarshal([]byte(`"forever"`), &d) + require.ErrorContains(t, err, "invalid interval/duration format") + + err = yaml.Unmarshal([]byte(`[1]`), &d) + require.EqualError(t, err, "cannot unmarshal duration value") + + y, err := yaml.Marshal(Duration(1500 * time.Millisecond)) + require.NoError(t, err) + assert.Equal(t, "1.5s\n", string(y)) + j, err := Duration(2 * time.Second).MarshalJSON() + require.NoError(t, err) + assert.Equal(t, `"2s"`, string(j)) + + p := Duration(3 * time.Second).Ptr() + *p = Duration(time.Second) + assert.Equal(t, time.Second, p.Duration()) + assert.Equal(t, 4*time.Second, *Duration(4 * time.Second).DurationPtr()) +} + +// Numeric scalars decode as strings first under yaml.v3, so the int/float +// millisecond branches are reachable only via a custom unmarshal func. +func TestCovMiscDuration_NumericFallbacks(t *testing.T) { + asInt := func(v interface{}) error { + switch p := v.(type) { + case *int64: + *p = 250 + return nil + } + return assert.AnError + } + var d Duration + require.NoError(t, d.UnmarshalYAML(asInt)) + assert.Equal(t, 250*time.Millisecond, d.Duration()) + + asFloat := func(v interface{}) error { + if p, ok := v.(*float64); ok { + *p = 12.9 + return nil + } + return assert.AnError + } + require.NoError(t, d.UnmarshalYAML(asFloat)) + assert.Equal(t, 12*time.Millisecond, d.Duration()) +} + +func TestCovMiscAdaptiveDuration_ResolveClampsAndQuantile(t *testing.T) { + cases := []struct { + name string + d *AdaptiveDuration + qt QuantileTracker + want time.Duration + }{ + {"nil", nil, nil, 0}, + {"static base ignores min/max", &AdaptiveDuration{Base: Duration(5 * time.Millisecond), Min: Duration(time.Second)}, nil, 5 * time.Millisecond}, + {"quantile cold start uses min", &AdaptiveDuration{Quantile: 0.9, Min: Duration(100 * time.Millisecond), Max: Duration(time.Second)}, nil, 100 * time.Millisecond}, + {"base plus quantile", &AdaptiveDuration{Base: Duration(10 * time.Millisecond), Quantile: 0.5}, &fakeQuantile{val: 20 * time.Millisecond}, 30 * time.Millisecond}, + {"clamped to max", &AdaptiveDuration{Quantile: 0.5, Max: Duration(50 * time.Millisecond)}, &fakeQuantile{val: time.Second}, 50 * time.Millisecond}, + {"raised to min", &AdaptiveDuration{Quantile: 0.5, Min: Duration(40 * time.Millisecond), Max: Duration(time.Second)}, &fakeQuantile{val: time.Millisecond}, 40 * time.Millisecond}, + } + for _, tc := range cases { + assert.Equal(t, tc.want, tc.d.Resolve(tc.qt), tc.name) + } +} + +func TestCovMiscAdaptiveDuration_Validate(t *testing.T) { + ms := func(n int) Duration { return Duration(time.Duration(n) * time.Millisecond) } + cases := []struct { + d *AdaptiveDuration + wantErr string + }{ + {nil, ""}, + {&AdaptiveDuration{Base: ms(1)}, ""}, + {&AdaptiveDuration{Quantile: 1.5, Base: ms(1)}, "x.quantile must be between 0 and 1"}, + {&AdaptiveDuration{Quantile: -0.1, Base: ms(1)}, "x.quantile must be between 0 and 1"}, + {&AdaptiveDuration{Quantile: 0.5}, "x requires base or max when quantile is set"}, + {&AdaptiveDuration{}, "x must specify at least one of base/quantile/min/max"}, + {&AdaptiveDuration{Base: ms(1), Min: ms(10), Max: ms(5)}, "x.min must be <= x.max"}, + {&AdaptiveDuration{Quantile: 0.5, Max: ms(5)}, ""}, + } + for _, tc := range cases { + err := tc.d.validate("x") + if tc.wantErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.wantErr) + } + } +} + +func TestCovMiscAdaptiveDuration_InheritFrom(t *testing.T) { + src := &AdaptiveDuration{Base: Duration(1), Quantile: 0.5, Min: Duration(2), Max: Duration(3)} + d := &AdaptiveDuration{Base: Duration(9)} + d.inheritFrom(src) + assert.Equal(t, &AdaptiveDuration{Base: Duration(9), Quantile: 0.5, Min: Duration(2), Max: Duration(3)}, d, "set fields win, zero fields inherit") + + full := &AdaptiveDuration{Base: Duration(7), Quantile: 0.1, Min: Duration(8), Max: Duration(9)} + before := *full + full.inheritFrom(src) + assert.Equal(t, before, *full) + + full.inheritFrom(nil) + (*AdaptiveDuration)(nil).inheritFrom(src) + assert.Equal(t, before, *full) + assert.Nil(t, (*AdaptiveDuration)(nil).Copy()) +} + +func TestCovMiscAdaptiveDuration_UnmarshalForms(t *testing.T) { + var d AdaptiveDuration + require.NoError(t, yaml.Unmarshal([]byte(`{base: 100ms, quantile: 0.9, min: 50ms, max: 2s}`), &d)) + assert.Equal(t, AdaptiveDuration{Base: Duration(100 * time.Millisecond), Quantile: 0.9, Min: Duration(50 * time.Millisecond), Max: Duration(2 * time.Second)}, d) + + err := yaml.Unmarshal([]byte(`[1, 2]`), &d) + require.ErrorContains(t, err, "AdaptiveDuration must be a duration scalar or {base, quantile, min, max} object") + + // int/float branches via a custom unmarshal func (yaml.v3 decodes ints into strings first) + d = AdaptiveDuration{} + require.NoError(t, d.UnmarshalYAML(func(v interface{}) error { + if p, ok := v.(*int64); ok { + *p = 300 + return nil + } + return assert.AnError + })) + assert.Equal(t, Duration(300*time.Millisecond), d.Base) + require.NoError(t, d.UnmarshalYAML(func(v interface{}) error { + if p, ok := v.(*float64); ok { + *p = 7.5 + return nil + } + return assert.AnError + })) + assert.Equal(t, Duration(7*time.Millisecond), d.Base) + + jsonCases := []struct { + in string + want AdaptiveDuration + wantErr string + }{ + {`null`, AdaptiveDuration{}, ""}, + {`"250ms"`, AdaptiveDuration{Base: Duration(250 * time.Millisecond)}, ""}, + {`"abc"`, AdaptiveDuration{}, `invalid duration scalar "abc"`}, + {`"\x"`, AdaptiveDuration{}, "invalid"}, + {`125`, AdaptiveDuration{Base: Duration(125 * time.Millisecond)}, ""}, + {`true`, AdaptiveDuration{}, "AdaptiveDuration must be a duration scalar"}, + {`{"base": 10, "quantile": 0.5, "min": "1ms", "max": null}`, AdaptiveDuration{Base: Duration(10 * time.Millisecond), Quantile: 0.5, Min: Duration(time.Millisecond)}, ""}, + {`{"base": "bad"}`, AdaptiveDuration{}, "AdaptiveDuration.base: invalid duration"}, + {`{"min": "bad"}`, AdaptiveDuration{}, "AdaptiveDuration.min: invalid duration"}, + {`{"max": true}`, AdaptiveDuration{}, "AdaptiveDuration.max"}, + {`{"base": "\x"}`, AdaptiveDuration{}, "AdaptiveDuration"}, + {`{"quantile": "x"}`, AdaptiveDuration{}, "AdaptiveDuration object"}, + } + for _, tc := range jsonCases { + var got AdaptiveDuration + err := got.UnmarshalJSON([]byte(tc.in)) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr, tc.in) + continue + } + require.NoError(t, err, tc.in) + assert.Equal(t, tc.want, got, tc.in) + } +} + +func TestCovMiscAdaptiveDuration_MarshalJSONAndCtor(t *testing.T) { + var nilD *AdaptiveDuration + b, err := nilD.MarshalJSON() + require.NoError(t, err) + assert.Equal(t, "null", string(b)) + + d := NewStaticDuration(3 * time.Second) + assert.Equal(t, Duration(3*time.Second), d.Base) + b, err = d.MarshalJSON() + require.NoError(t, err) + assert.JSONEq(t, `{"base":"3s"}`, string(b)) + + var back AdaptiveDuration + require.NoError(t, back.UnmarshalJSON(b)) + assert.Equal(t, *d, back, "object form round-trips") +} + +func TestCovMiscAdaptiveDuration_ResolveForRequest(t *testing.T) { + qd := &AdaptiveDuration{Base: Duration(10 * time.Millisecond), Quantile: 0.9, Min: Duration(5 * time.Millisecond), Max: Duration(time.Second)} + ntw := &covMiscNetwork{id: "evm:1", metrics: map[string]TrackedMetrics{ + "eth_call": &covMiscMetrics{q: &fakeQuantile{val: 90 * time.Millisecond}}, + }} + mkReq := func(method string, withNet bool) *NormalizedRequest { + r := NewNormalizedRequestFromJsonRpcRequest(NewJsonRpcRequest(method, nil)) + if withNet { + r.SetNetwork(ntw) + } + return r + } + + var zero *AdaptiveDuration + assert.Equal(t, time.Duration(0), zero.ResolveForRequest(mkReq("eth_call", true))) + assert.Equal(t, time.Duration(0), qd.ResolveForRequest(nil)) + assert.Equal(t, 7*time.Millisecond, NewStaticDuration(7*time.Millisecond).ResolveForRequest(mkReq("eth_call", true))) + assert.Equal(t, 15*time.Millisecond, qd.ResolveForRequest(mkReq("eth_call", false)), "no network: cold start base+min") + assert.Equal(t, 15*time.Millisecond, qd.ResolveForRequest(mkReq("", true)), "empty method: cold start") + assert.Equal(t, 15*time.Millisecond, qd.ResolveForRequest(mkReq("eth_getLogs", true)), "no tracker: cold start") + assert.Equal(t, 100*time.Millisecond, qd.ResolveForRequest(mkReq("eth_call", true))) +} diff --git a/common/cov_validation_database_test.go b/common/cov_validation_database_test.go new file mode 100644 index 000000000..713a060b3 --- /dev/null +++ b/common/cov_validation_database_test.go @@ -0,0 +1,278 @@ +package common + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func cvMemoryConnector(id string) *ConnectorConfig { + return &ConnectorConfig{Id: id, Driver: DriverMemory, Memory: &MemoryConnectorConfig{MaxItems: 10}} +} + +func cvValidSharedState() *SharedStateConfig { + return &SharedStateConfig{ + Connector: cvMemoryConnector("ss"), + FallbackTimeout: Duration(time.Second), + LockTtl: Duration(4 * time.Second), + LockMaxWait: Duration(100 * time.Millisecond), + UpdateMaxWait: Duration(50 * time.Millisecond), + } +} + +func TestCovValidation_Database(t *testing.T) { + baseline := func() *DatabaseConfig { + return &DatabaseConfig{ + EvmJsonRpcCache: &CacheConfig{Connectors: []*ConnectorConfig{cvMemoryConnector("m")}}, + SvmJsonRpcCache: &CacheConfig{Connectors: []*ConnectorConfig{cvMemoryConnector("m")}}, + SharedState: cvValidSharedState(), + } + } + cvRunCases(t, baseline, (*DatabaseConfig).Validate, []cvCase[DatabaseConfig]{ + {"evm cache invalid", func(d *DatabaseConfig) { d.EvmJsonRpcCache.Connectors[0].Id = "" }, "*.connector.id is required"}, + {"svm cache invalid", func(d *DatabaseConfig) { d.SvmJsonRpcCache.Compression = &CompressionConfig{Algorithm: "gzip"} }, "cache.*.compression.algorithm must be 'zstd'"}, + {"shared state invalid", func(d *DatabaseConfig) { d.SharedState.LockTtl = 0 }, "sharedState.lockTtl is required"}, + }) +} + +func TestCovValidation_SharedState(t *testing.T) { + cvRunCases(t, cvValidSharedState, (*SharedStateConfig).Validate, []cvCase[SharedStateConfig]{ + {"connector missing", func(s *SharedStateConfig) { s.Connector = nil }, "sharedState.connector is required"}, + {"connector invalid", func(s *SharedStateConfig) { s.Connector.Driver = "" }, "database.*.connector.driver is required"}, + {"fallbackTimeout missing", func(s *SharedStateConfig) { s.FallbackTimeout = 0 }, "sharedState.fallbackTimeout is required"}, + {"fallbackTimeout too small", func(s *SharedStateConfig) { s.FallbackTimeout = Duration(50 * time.Millisecond) }, "sharedState.fallbackTimeout should be at least 100ms"}, + {"lockTtl missing", func(s *SharedStateConfig) { s.LockTtl = 0 }, "sharedState.lockTtl is required"}, + {"lockTtl too small", func(s *SharedStateConfig) { s.LockTtl = Duration(500 * time.Millisecond) }, "sharedState.lockTtl should be at least 1s"}, + {"lockTtl shorter than fallbackTimeout", func(s *SharedStateConfig) { + s.FallbackTimeout = Duration(3 * time.Second) + s.LockTtl = Duration(2 * time.Second) + }, "sharedState.lockTtl (2s) should be at least as long as fallbackTimeout (3s)"}, + {"lockMaxWait not below fallbackTimeout", func(s *SharedStateConfig) { s.LockMaxWait = Duration(time.Second) }, "sharedState.lockMaxWait (1s) should be less than fallbackTimeout (1s)"}, + {"updateMaxWait not below fallbackTimeout", func(s *SharedStateConfig) { s.UpdateMaxWait = Duration(2 * time.Second) }, "sharedState.updateMaxWait (2s) should be less than fallbackTimeout (1s)"}, + }) +} + +func TestCovValidation_CacheConfig(t *testing.T) { + baseline := func() *CacheConfig { + return &CacheConfig{ + Connectors: []*ConnectorConfig{cvMemoryConnector("a"), cvMemoryConnector("b")}, + Policies: []*CachePolicyConfig{{ + Network: "*", Method: "*", Connector: "b", + MinItemSize: strPtrCv("1KB"), MaxItemSize: strPtrCv("1MB"), + AppliesTo: CachePolicyAppliesToGet, + Finality: DataFinalityStateRealtime, + TTL: &BlockTimeAdaptiveDuration{BlockTimeMultiplier: 2}, + }}, + Compression: &CompressionConfig{Algorithm: "zstd", ZstdLevel: "better", Threshold: 1024}, + } + } + cvRunCases(t, baseline, (*CacheConfig).Validate, []cvCase[CacheConfig]{ + {"duplicate connector id", func(c *CacheConfig) { c.Connectors[1].Id = "a"; c.Policies[0].Connector = "a" }, "cache.*.connectors.*.id must be unique, 'a' is duplicated"}, + {"invalid connector", func(c *CacheConfig) { c.Connectors[0].Memory = nil }, "database.*.connector.memory is required when driver is memory"}, + {"policy without network", func(c *CacheConfig) { c.Policies[0].Network = "" }, "cache.*.policies.*.network is required"}, + {"policy without method", func(c *CacheConfig) { c.Policies[0].Method = "" }, "cache.*.policies.*.method is required"}, + {"policy without connector", func(c *CacheConfig) { c.Policies[0].Connector = "" }, "cache.*.policies.*.connector is required"}, + {"policy unknown connector", func(c *CacheConfig) { c.Policies[0].Connector = "zz" }, "cache.*.policies.*.connector 'zz' does not exist in cache.connectors"}, + {"bad minItemSize", func(c *CacheConfig) { c.Policies[0].MinItemSize = strPtrCv("lots") }, "cache.*.policies.*.minItemSize is invalid"}, + {"bad maxItemSize", func(c *CacheConfig) { c.Policies[0].MaxItemSize = strPtrCv("-5") }, "cache.*.policies.*.maxItemSize is invalid"}, + {"min above max", func(c *CacheConfig) { c.Policies[0].MinItemSize = strPtrCv("2MB") }, "cache.*.policies.*.minItemSize must be less than or equal to maxItemSize"}, + {"unknown appliesTo", func(c *CacheConfig) { c.Policies[0].AppliesTo = "delete" }, "cache.*.policies.*.appliesTo must be one of: get, set, both"}, + {"negative blockTimeMultiplier", func(c *CacheConfig) { c.Policies[0].TTL.BlockTimeMultiplier = -1 }, "cache.*.policies.*.ttl.blockTimeMultiplier must be >= 0"}, + {"blockTimeMultiplier on finalized", func(c *CacheConfig) { c.Policies[0].Finality = DataFinalityStateFinalized }, "ttl.blockTimeMultiplier is only supported when finality is 'realtime'"}, + {"fixed ttl on finalized", func(c *CacheConfig) { + c.Policies[0].Finality = DataFinalityStateFinalized + c.Policies[0].TTL = &BlockTimeAdaptiveDuration{Fallback: Duration(time.Minute)} + }, ""}, + {"unknown compression algorithm", func(c *CacheConfig) { c.Compression.Algorithm = "lz4" }, "cache.*.compression.algorithm must be 'zstd'"}, + {"unknown zstd level", func(c *CacheConfig) { c.Compression.ZstdLevel = "max" }, "cache.*.compression.zstdLevel must be one of: fastest, default, better, best"}, + {"negative threshold", func(c *CacheConfig) { c.Compression.Threshold = -1 }, "cache.*.compression.threshold must be greater than or equal to 0"}, + }) +} + +func strPtrCv(s string) *string { return &s } + +func cvValidConnector(driver ConnectorDriverType) *ConnectorConfig { + c := &ConnectorConfig{Id: "c1", Driver: driver} + switch driver { + case DriverMemory: + c.Memory = &MemoryConnectorConfig{MaxItems: 100} + case DriverRedis: + c.Redis = &RedisConnectorConfig{URI: "redis://localhost:6379", LockRetryInterval: Duration(200 * time.Millisecond)} + case DriverPostgreSQL: + c.PostgreSQL = &PostgreSQLConnectorConfig{ + ConnectionUri: "postgres://user:pw@localhost:5432/db", + Table: "rpc_cache", + MinConns: 1, MaxConns: 4, + InitTimeout: Duration(time.Second), GetTimeout: Duration(time.Second), SetTimeout: Duration(time.Second), + } + case DriverDynamoDB: + c.DynamoDB = &DynamoDBConnectorConfig{ + Table: "t", PartitionKeyName: "pk", RangeKeyName: "rk", ReverseIndexName: "idx", TTLAttributeName: "ttl", + InitTimeout: Duration(time.Second), GetTimeout: Duration(time.Second), SetTimeout: Duration(time.Second), + StatePollInterval: Duration(time.Second), + } + case DriverGrpc: + c.Grpc = &GrpcConnectorConfig{Servers: []string{"grpc://cache:50051"}, PoolSize: 4, HealthCheckService: "erpc"} + } + return c +} + +func TestCovValidation_ConnectorCommon(t *testing.T) { + cvRunCases(t, func() *ConnectorConfig { return cvValidConnector(DriverMemory) }, (*ConnectorConfig).Validate, []cvCase[ConnectorConfig]{ + {"id missing", func(c *ConnectorConfig) { c.Id = "" }, "*.connector.id is required"}, + {"driver missing", func(c *ConnectorConfig) { c.Driver = "" }, "database.*.connector.driver is required"}, + {"unknown driver", func(c *ConnectorConfig) { c.Driver = "mongo" }, "database.*.connector.driver 'mongo' is invalid must be one of"}, + {"memory with redis", func(c *ConnectorConfig) { c.Redis = &RedisConnectorConfig{URI: "redis://x"} }, "database.*.connector.memory is mutually exclusive"}, + {"redis with postgres", func(c *ConnectorConfig) { + c.Driver, c.Memory = DriverRedis, nil + c.Redis = &RedisConnectorConfig{URI: "redis://x"} + c.PostgreSQL = &PostgreSQLConnectorConfig{} + }, "database.*.connector.redis is mutually exclusive"}, + {"postgres with dynamo", func(c *ConnectorConfig) { + c.Driver, c.Memory = DriverPostgreSQL, nil + c.PostgreSQL = &PostgreSQLConnectorConfig{} + c.DynamoDB = &DynamoDBConnectorConfig{} + }, "database.*.connector.postgresql is mutually exclusive"}, + {"dynamo driver with empty dynamo block", func(c *ConnectorConfig) { + c.Driver, c.Memory = DriverDynamoDB, nil + c.DynamoDB = &DynamoDBConnectorConfig{} + }, "database.*.connector.dynamodb.table is required"}, + {"failsafeForGets consensus", func(c *ConnectorConfig) { + c.FailsafeForGets = []*FailsafeConfig{nil, {Consensus: &ConsensusPolicyConfig{}}} + }, "connector 'c1'.failsafeForGets[1]: consensus is not supported for connector-level failsafe"}, + {"failsafeForSets matchCommitment", func(c *ConnectorConfig) { + c.FailsafeForSets = []*FailsafeConfig{{MatchCommitment: []string{"finalized"}}} + }, "connector 'c1'.failsafeForSets[0]: matchCommitment is not supported for connector-level failsafe"}, + {"plain failsafe accepted", func(c *ConnectorConfig) { + c.FailsafeForGets = []*FailsafeConfig{{MatchMethod: "*"}} + }, ""}, + }) + + // Every driver's own block must be present when that driver is chosen. + for _, tc := range []struct { + driver ConnectorDriverType + wantErr string + }{ + {DriverMemory, "database.*.connector.memory is required when driver is memory"}, + {DriverRedis, "database.*.connector.redis is required when driver is redis"}, + {DriverPostgreSQL, "database.*.connector.postgresql is required when driver is postgresql"}, + {DriverDynamoDB, "database.*.connector.dynamodb is required when driver is dynamodb"}, + } { + t.Run(string(tc.driver)+" block required", func(t *testing.T) { + require.ErrorContains(t, (&ConnectorConfig{Id: "x", Driver: tc.driver}).Validate(), tc.wantErr) + }) + } + t.Run("dynamo with memory block reports the memory exclusivity", func(t *testing.T) { + c := cvValidConnector(DriverDynamoDB) + c.Memory = &MemoryConnectorConfig{} + // memory is checked first, so the memory-side message wins. + require.ErrorContains(t, c.Validate(), "database.*.connector.memory is mutually exclusive") + }) +} + +func TestCovValidation_ConnectorDynamoDB(t *testing.T) { + cvRunCases(t, func() *ConnectorConfig { return cvValidConnector(DriverDynamoDB) }, (*ConnectorConfig).Validate, []cvCase[ConnectorConfig]{ + {"table", func(c *ConnectorConfig) { c.DynamoDB.Table = "" }, "database.*.connector.dynamodb.table is required"}, + {"partitionKeyName", func(c *ConnectorConfig) { c.DynamoDB.PartitionKeyName = "" }, "database.*.connector.dynamodb.partitionKeyName is required"}, + {"rangeKeyName", func(c *ConnectorConfig) { c.DynamoDB.RangeKeyName = "" }, "database.*.connector.dynamodb.rangeKeyName is required"}, + {"reverseIndexName", func(c *ConnectorConfig) { c.DynamoDB.ReverseIndexName = "" }, "database.*.connector.dynamodb.reverseIndexName is required"}, + {"ttlAttributeName", func(c *ConnectorConfig) { c.DynamoDB.TTLAttributeName = "" }, "database.*.connector.dynamodb.ttlAttributeName is required"}, + {"initTimeout", func(c *ConnectorConfig) { c.DynamoDB.InitTimeout = 0 }, "database.*.connector.dynamodb.initTimeout is required"}, + {"getTimeout", func(c *ConnectorConfig) { c.DynamoDB.GetTimeout = 0 }, "database.*.connector.dynamodb.getTimeout is required"}, + {"setTimeout", func(c *ConnectorConfig) { c.DynamoDB.SetTimeout = 0 }, "database.*.connector.dynamodb.setTimeout is required"}, + {"statePollInterval", func(c *ConnectorConfig) { c.DynamoDB.StatePollInterval = 0 }, "database.*.connector.dynamodb.statePollInterval is required"}, + {"with redis block", func(c *ConnectorConfig) { c.Redis = &RedisConnectorConfig{} }, "database.*.connector.redis is mutually exclusive"}, + }) +} + +func TestCovValidation_ConnectorPostgreSQL(t *testing.T) { + cvRunCases(t, func() *ConnectorConfig { return cvValidConnector(DriverPostgreSQL) }, (*ConnectorConfig).Validate, []cvCase[ConnectorConfig]{ + {"connectionUri", func(c *ConnectorConfig) { c.PostgreSQL.ConnectionUri = "" }, "database.*.connector.postgresql.connectionUri is required"}, + {"table", func(c *ConnectorConfig) { c.PostgreSQL.Table = "" }, "database.*.connector.postgresql.table is required"}, + {"minConns", func(c *ConnectorConfig) { c.PostgreSQL.MinConns = 0 }, "database.*.connector.postgresql.minConns is required"}, + {"maxConns", func(c *ConnectorConfig) { c.PostgreSQL.MaxConns = 0 }, "database.*.connector.postgresql.maxConns is required"}, + {"initTimeout", func(c *ConnectorConfig) { c.PostgreSQL.InitTimeout = 0 }, "database.*.connector.postgresql.initTimeout is required"}, + {"getTimeout", func(c *ConnectorConfig) { c.PostgreSQL.GetTimeout = 0 }, "database.*.connector.postgresql.getTimeout is required"}, + {"setTimeout", func(c *ConnectorConfig) { c.PostgreSQL.SetTimeout = 0 }, "database.*.connector.postgresql.setTimeout is required"}, + }) +} + +func TestCovValidation_PostgreSQLIAMAuth(t *testing.T) { + baseline := func() *PostgreSQLConnectorConfig { + c := cvValidConnector(DriverPostgreSQL).PostgreSQL + c.ConnectionUri = "postgres://app@db.example:5432/cache?sslmode=verify-full" + c.IAMAuth = &PostgreSQLIAMAuthConfig{ + Enabled: true, Endpoint: "db.example:5432", DBUser: "app", + Auth: &AwsAuthConfig{Mode: "env"}, + } + return c + } + cvRunCases(t, baseline, (*PostgreSQLConnectorConfig).Validate, []cvCase[PostgreSQLConnectorConfig]{ + {"disabled IAM ignores fields", func(p *PostgreSQLConnectorConfig) { p.IAMAuth = &PostgreSQLIAMAuthConfig{} }, ""}, + {"endpoint missing", func(p *PostgreSQLConnectorConfig) { p.IAMAuth.Endpoint = "" }, "postgresql.iamAuth.endpoint could not be derived from connectionUri"}, + {"endpoint without port", func(p *PostgreSQLConnectorConfig) { p.IAMAuth.Endpoint = "db.example" }, "postgresql.iamAuth.endpoint must include a port (host:port)"}, + {"dbUser missing", func(p *PostgreSQLConnectorConfig) { p.IAMAuth.DBUser = "" }, "postgresql.iamAuth.dbUser could not be derived from connectionUri"}, + {"static password", func(p *PostgreSQLConnectorConfig) { + p.ConnectionUri = "postgres://app:secret@db.example:5432/cache?sslmode=require" + }, `cannot combine IAM auth with a static password in connectionUri (got password for user "app")`}, + {"dbUser mismatch", func(p *PostgreSQLConnectorConfig) { p.IAMAuth.DBUser = "other" }, `postgresql.iamAuth.dbUser "other" does not match the user in connectionUri "app"`}, + {"uri without user uses dbUser", func(p *PostgreSQLConnectorConfig) { + p.ConnectionUri = "postgres://db.example:5432/cache?sslmode=require" + }, ""}, + {"sslmode disable", func(p *PostgreSQLConnectorConfig) { + p.ConnectionUri = "postgres://app@db.example:5432/cache?sslmode=disable" + }, `sslmode must be require, verify-ca, or verify-full (got "disable"`}, + {"sslmode missing", func(p *PostgreSQLConnectorConfig) { + p.ConnectionUri = "postgres://app@db.example:5432/cache" + }, `(got ""`}, + {"bad auth mode", func(p *PostgreSQLConnectorConfig) { p.IAMAuth.Auth.Mode = "imds" }, `postgresql.iamAuth.auth.mode "imds" is invalid; must be file, env, or secret`}, + }) +} + +func TestCovValidation_ConnectorRedis(t *testing.T) { + cvRunCases(t, func() *ConnectorConfig { return cvValidConnector(DriverRedis) }, (*ConnectorConfig).Validate, []cvCase[ConnectorConfig]{ + {"uri missing", func(c *ConnectorConfig) { c.Redis.URI = " " }, "database.*.connector.redis.uri is required"}, + {"bad scheme", func(c *ConnectorConfig) { c.Redis.URI = "tcp://localhost:6379" }, "redis connector: invalid URI scheme, must be 'rediss://' or 'redis://'"}, + {"lockRetryInterval too small", func(c *ConnectorConfig) { c.Redis.LockRetryInterval = Duration(10 * time.Millisecond) }, "redis.lockRetryInterval should be at least 100ms"}, + {"lockRetryInterval unset", func(c *ConnectorConfig) { c.Redis.LockRetryInterval = 0 }, ""}, + {"with memory block", func(c *ConnectorConfig) { c.Memory = &MemoryConnectorConfig{} }, "database.*.connector.memory is mutually exclusive"}, + }) +} + +func TestCovValidation_RedisIAMAuth(t *testing.T) { + baseline := func() *RedisConnectorConfig { + return &RedisConnectorConfig{ + URI: "rediss://app@cache.example:6379", + IAMAuth: &RedisIAMAuthConfig{ + Enabled: true, CacheName: "c", UserID: "app", + Auth: &AwsAuthConfig{Mode: "secret"}, + }, + } + } + cvRunCases(t, baseline, (*RedisConnectorConfig).Validate, []cvCase[RedisConnectorConfig]{ + {"cacheName missing", func(r *RedisConnectorConfig) { r.IAMAuth.CacheName = "" }, "redis.iamAuth.cacheName is required when iamAuth.enabled is true"}, + {"userID missing", func(r *RedisConnectorConfig) { r.IAMAuth.UserID = "" }, "redis.iamAuth.userID is required when iamAuth.enabled is true"}, + {"plaintext scheme", func(r *RedisConnectorConfig) { r.URI = "redis://cache.example:6379" }, "redis.iamAuth requires in-transit TLS: URI must use 'rediss://' scheme"}, + {"static password", func(r *RedisConnectorConfig) { r.URI = "rediss://app:pw@cache.example:6379" }, `redis.iamAuth: cannot combine IAM auth with a static password (got password for user "app")`}, + {"bad auth mode", func(r *RedisConnectorConfig) { r.IAMAuth.Auth.Mode = "web" }, `redis.iamAuth.auth.mode "web" is invalid; must be file, env, or secret`}, + {"disabled IAM over plaintext", func(r *RedisConnectorConfig) { + r.URI = "redis://cache.example:6379" + r.IAMAuth.Enabled = false + }, ""}, + }) +} + +func TestCovValidation_ConnectorGrpc(t *testing.T) { + cvRunCases(t, func() *ConnectorConfig { return cvValidConnector(DriverGrpc) }, (*ConnectorConfig).Validate, []cvCase[ConnectorConfig]{ + {"negative pool", func(c *ConnectorConfig) { c.Grpc.PoolSize = -1 }, "database.*.connector.grpc.poolSize must not be negative"}, + {"pool too large", func(c *ConnectorConfig) { c.Grpc.PoolSize = MaxGrpcConnPoolSize + 1 }, "database.*.connector.grpc.poolSize must be <= 256"}, + {"health check on mounted path", func(c *ConnectorConfig) { c.Grpc.Servers = []string{"grpc://cache:50051/tenant"} }, "database.*.connector.grpc.healthCheckService cannot be used with"}, + {"unparsable server", func(c *ConnectorConfig) { c.Grpc.Servers = []string{"grpc://cache:50051/%zz"} }, "database.*.connector.grpc: invalid gRPC endpoint"}, + {"mounted path without health check", func(c *ConnectorConfig) { + c.Grpc.Servers = []string{"grpc://cache:50051/tenant"} + c.Grpc.HealthCheckService = "" + }, ""}, + {"trailing slash is root", func(c *ConnectorConfig) { c.Grpc.Servers = []string{"grpc://cache:50051/"} }, ""}, + }) +} diff --git a/common/cov_validation_edges_test.go b/common/cov_validation_edges_test.go new file mode 100644 index 000000000..1017c2793 --- /dev/null +++ b/common/cov_validation_edges_test.go @@ -0,0 +1,42 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCovValidation_NetworkDefaultsNilIsValid(t *testing.T) { + var n *NetworkDefaults + require.NoError(t, n.Validate()) +} + +func TestCovValidation_ServedTipGuaranteedMethodsPattern(t *testing.T) { + e := cvValidEvmNetwork(1).Evm + e.ServedTip = &EvmServedTipConfig{GuaranteedMethods: []string{"(eth_call"}} + require.ErrorContains(t, e.Validate(), `network.*.evm.servedTip.guaranteedMethods has invalid pattern "(eth_call"`) +} + +func TestCovValidation_SvmBlockAvailabilityUpperBound(t *testing.T) { + c := &SvmBlockAvailabilityConfig{Upper: &SvmAvailabilityBoundConfig{}} + require.ErrorContains(t, c.Validate(), "upstream.*.svm.blockAvailability.upper is invalid: bound must set exactly one of") +} + +func TestCovValidation_IntegritySettingsEdges(t *testing.T) { + cvRunCases(t, func() *IntegrityConfig { + return &IntegrityConfig{ + IntegritySettings: IntegritySettings{ReorgWindow: 64}, + Profiles: map[string]*IntegritySettings{"lenient": nil, "strict": {Level: "corroborated"}}, + } + }, (*IntegrityConfig).Validate, []cvCase[IntegrityConfig]{ + {"empty profile name", func(i *IntegrityConfig) { i.Profiles[" "] = &IntegritySettings{} }, "integrity.profiles contains an empty profile name"}, + {"invalid profile", func(i *IntegrityConfig) { i.Profiles["strict"].Level = "max" }, "integrity.profiles.strict: integrity.level 'max' is invalid"}, + {"negative reorgWindow", func(i *IntegrityConfig) { i.ReorgWindow = -1 }, "integrity.reorgWindow must be >= 0"}, + {"invalid misbehaviors destination", func(i *IntegrityConfig) { + i.MisbehaviorsDestination = &MisbehaviorsDestinationConfig{Type: "file", Path: "rel"} + }, "integrity.misbehaviorsDestination: consensus.misbehaviorsDestination.path must be an absolute path"}, + {"valid misbehaviors destination", func(i *IntegrityConfig) { + i.MisbehaviorsDestination = &MisbehaviorsDestinationConfig{Type: "file", Path: "/var/erpc"} + }, ""}, + }) +} diff --git a/common/cov_validation_failsafe_test.go b/common/cov_validation_failsafe_test.go new file mode 100644 index 000000000..aca46d99d --- /dev/null +++ b/common/cov_validation_failsafe_test.go @@ -0,0 +1,207 @@ +package common + +import ( + "testing" + "time" + + "github.com/grafana/sobek" + "github.com/stretchr/testify/require" +) + +func TestCovValidation_Failsafe(t *testing.T) { + baseline := func() *FailsafeConfig { + return &FailsafeConfig{ + MatchMethod: "eth_*", + MatchRequestKind: "user", + MatchCommitment: []string{"none", "processed"}, + Timeout: &TimeoutPolicyConfig{Duration: &AdaptiveDuration{Base: Duration(time.Second)}}, + Retry: &RetryPolicyConfig{MaxAttempts: 3, BackoffFactor: 1.5, BackoffMaxDelay: Duration(time.Second)}, + Hedge: &HedgePolicyConfig{Delay: &AdaptiveDuration{Quantile: 0.9, Max: Duration(time.Second)}, MaxCount: 1}, + CircuitBreaker: &CircuitBreakerPolicyConfig{ + HalfOpenAfter: Duration(time.Minute), FailureThresholdCount: 5, FailureThresholdCapacity: 10, + SuccessThresholdCount: 2, SuccessThresholdCapacity: 3, + }, + } + } + cvRunCases(t, baseline, (*FailsafeConfig).Validate, []cvCase[FailsafeConfig]{ + {"matchMethod empty", func(f *FailsafeConfig) { f.MatchMethod = "" }, "failsafe.matchMethod cannot be empty, use '*' to match any method"}, + {"matchRequestKind unknown", func(f *FailsafeConfig) { f.MatchRequestKind = "admin" }, "failsafe.matchRequestKind 'admin' is invalid, must be one of: user | internal | *"}, + {"matchRequestKind internal", func(f *FailsafeConfig) { f.MatchRequestKind = "internal" }, ""}, + + {"timeout without duration", func(f *FailsafeConfig) { f.Timeout.Duration = nil }, "upstream.*.failsafe.timeout.duration is required"}, + {"timeout quantile out of range", func(f *FailsafeConfig) { f.Timeout.Duration = &AdaptiveDuration{Quantile: 1.5, Base: 1} }, "upstream.*.failsafe.timeout.duration.quantile must be between 0 and 1"}, + {"timeout quantile without base or max", func(f *FailsafeConfig) { f.Timeout.Duration = &AdaptiveDuration{Quantile: 0.5} }, "upstream.*.failsafe.timeout.duration requires base or max when quantile is set"}, + {"timeout empty spec", func(f *FailsafeConfig) { f.Timeout.Duration = &AdaptiveDuration{} }, "upstream.*.failsafe.timeout.duration must specify at least one of base/quantile/min/max"}, + {"timeout min above max", func(f *FailsafeConfig) { + f.Timeout.Duration = &AdaptiveDuration{Base: 1, Min: Duration(2 * time.Second), Max: Duration(time.Second)} + }, "upstream.*.failsafe.timeout.duration.min must be <= upstream.*.failsafe.timeout.duration.max"}, + + {"retry backoffFactor zero", func(f *FailsafeConfig) { f.Retry.BackoffFactor = 0 }, "upstream.*.failsafe.retry.backoffFactor must be greater than 0"}, + {"retry backoffMaxDelay zero", func(f *FailsafeConfig) { f.Retry.BackoffMaxDelay = 0 }, "upstream.*.failsafe.retry.backoffMaxDelay is required"}, + + {"hedge without delay", func(f *FailsafeConfig) { f.Hedge.Delay = nil }, "failsafe.hedge.delay is required"}, + {"hedge zero delay", func(f *FailsafeConfig) { f.Hedge.Delay = &AdaptiveDuration{} }, "failsafe.hedge.delay is required"}, + {"hedge bad quantile", func(f *FailsafeConfig) { f.Hedge.Delay.Quantile = -0.1 }, "failsafe.hedge.delay.quantile must be between 0 and 1"}, + + {"cb halfOpenAfter", func(f *FailsafeConfig) { f.CircuitBreaker.HalfOpenAfter = 0 }, "failsafe.circuitBreaker.halfOpenAfter is required"}, + {"cb failure capacity", func(f *FailsafeConfig) { f.CircuitBreaker.FailureThresholdCapacity = 0 }, "failsafe.circuitBreaker.failureThresholdCapacity must be greater than 0"}, + {"cb failure count", func(f *FailsafeConfig) { f.CircuitBreaker.FailureThresholdCount = 0 }, "failsafe.circuitBreaker.failureThresholdCount must be greater than 0"}, + {"cb failure count above capacity", func(f *FailsafeConfig) { f.CircuitBreaker.FailureThresholdCount = 11 }, "failsafe.circuitBreaker.failureThresholdCount must be less than or equal to failureThresholdCapacity"}, + {"cb success count", func(f *FailsafeConfig) { f.CircuitBreaker.SuccessThresholdCount = 0 }, "failsafe.circuitBreaker.successThresholdCount must be greater than 0"}, + {"cb success capacity", func(f *FailsafeConfig) { f.CircuitBreaker.SuccessThresholdCapacity = 0 }, "failsafe.circuitBreaker.successThresholdCapacity must be greater than 0"}, + {"cb success count above capacity", func(f *FailsafeConfig) { f.CircuitBreaker.SuccessThresholdCount = 4 }, "failsafe.circuitBreaker.successThresholdCount must be less than or equal to"}, + }) +} + +func TestCovValidation_Consensus(t *testing.T) { + baseline := func() *ConsensusPolicyConfig { + return &ConsensusPolicyConfig{ + MaxParticipants: 4, + AgreementThreshold: 2, + PunishMisbehavior: &PunishMisbehaviorConfig{ + DisputeThreshold: 3, DisputeWindow: Duration(time.Minute), SitOutPenalty: Duration(time.Minute), + }, + MisbehaviorsDestination: &MisbehaviorsDestinationConfig{Type: MisbehaviorsDestinationTypeFile, Path: "/var/log/erpc"}, + RequiredParticipants: []*ConsensusRequiredParticipant{{Tag: "archive", MinParticipants: 2, MinAgreement: 1}}, + } + } + cvRunCases(t, baseline, (*ConsensusPolicyConfig).Validate, []cvCase[ConsensusPolicyConfig]{ + {"maxParticipants", func(c *ConsensusPolicyConfig) { c.MaxParticipants = 0 }, "consensus.maxParticipants must be greater than 0"}, + {"agreementThreshold", func(c *ConsensusPolicyConfig) { c.AgreementThreshold = 0 }, "consensus.agreementThreshold must be greater than 0"}, + {"threshold above participants", func(c *ConsensusPolicyConfig) { c.AgreementThreshold = 5 }, "consensus.maxParticipants must be greater than or equal to agreementThreshold"}, + {"punish disputeThreshold", func(c *ConsensusPolicyConfig) { c.PunishMisbehavior.DisputeThreshold = 0 }, "consensus.punishMisbehavior.disputeThreshold must be greater than 0"}, + {"punish disputeWindow", func(c *ConsensusPolicyConfig) { c.PunishMisbehavior.DisputeWindow = 0 }, "consensus.punishMisbehavior.disputeWindow must be greater than 0"}, + {"punish sitOutPenalty", func(c *ConsensusPolicyConfig) { c.PunishMisbehavior.SitOutPenalty = 0 }, "consensus.punishMisbehavior.sitOutPenalty must be greater than 0"}, + {"destination invalid", func(c *ConsensusPolicyConfig) { c.MisbehaviorsDestination.Path = "relative/dir" }, "consensus.misbehaviorsDestination is invalid: consensus.misbehaviorsDestination.path must be an absolute path"}, + {"required participant nil", func(c *ConsensusPolicyConfig) { c.RequiredParticipants = append(c.RequiredParticipants, nil) }, "consensus.requiredParticipants[1] must not be null"}, + {"required participant tag", func(c *ConsensusPolicyConfig) { c.RequiredParticipants[0].Tag = " " }, "consensus.requiredParticipants[0].tag is required"}, + {"required participant min", func(c *ConsensusPolicyConfig) { c.RequiredParticipants[0].MinParticipants = 0 }, "consensus.requiredParticipants[0].minParticipants must be greater than 0"}, + {"required participant above max", func(c *ConsensusPolicyConfig) { c.RequiredParticipants[0].MinParticipants = 5 }, "consensus.requiredParticipants[0].minParticipants (5) cannot exceed maxParticipants (4)"}, + {"required participant negative agreement", func(c *ConsensusPolicyConfig) { c.RequiredParticipants[0].MinAgreement = -1 }, "consensus.requiredParticipants[0].minAgreement must not be negative"}, + {"required participant agreement above min", func(c *ConsensusPolicyConfig) { c.RequiredParticipants[0].MinAgreement = 3 }, "consensus.requiredParticipants[0].minAgreement (3) cannot exceed minParticipants (2)"}, + }) +} + +func TestCovValidation_MisbehaviorsDestination(t *testing.T) { + baseline := func() *MisbehaviorsDestinationConfig { + return &MisbehaviorsDestinationConfig{ + Type: "S3", Path: "S3://bucket/prefix", + S3: &S3FlushConfig{ + MaxRecords: 100, MaxSize: 1 << 20, FlushInterval: Duration(time.Minute), + Endpoint: "https://s3.example", + Credentials: &AwsAuthConfig{Mode: "file", CredentialsFile: "/creds", Profile: "default"}, + }, + } + } + cvRunCases(t, baseline, (*MisbehaviorsDestinationConfig).Validate, []cvCase[MisbehaviorsDestinationConfig]{ + {"unknown type", func(m *MisbehaviorsDestinationConfig) { m.Type = "kafka" }, "consensus.misbehaviorsDestination.type must be 'file' or 's3'"}, + {"file without path", func(m *MisbehaviorsDestinationConfig) { m.Type, m.Path = "", " " }, "consensus.misbehaviorsDestination.path is required for file destination"}, + {"file relative path", func(m *MisbehaviorsDestinationConfig) { m.Type, m.Path = MisbehaviorsDestinationTypeFile, "logs" }, "consensus.misbehaviorsDestination.path must be an absolute path for file destination"}, + {"empty type defaults to file", func(m *MisbehaviorsDestinationConfig) { m.Type, m.Path = "", "/tmp/x" }, ""}, + {"s3 without path", func(m *MisbehaviorsDestinationConfig) { m.Path = "" }, "consensus.misbehaviorsDestination.path is required for s3 destination"}, + {"s3 non-s3 path", func(m *MisbehaviorsDestinationConfig) { m.Path = "/local" }, "consensus.misbehaviorsDestination.path must start with s3:// for s3 destination"}, + {"s3 without s3 block", func(m *MisbehaviorsDestinationConfig) { m.S3 = nil }, "consensus.misbehaviorsDestination.s3 is required when type is 's3'"}, + {"s3 negative maxRecords", func(m *MisbehaviorsDestinationConfig) { m.S3.MaxRecords = -1 }, "consensus.misbehaviorsDestination.s3 is invalid: s3.maxRecords must be >= 0"}, + {"s3 negative maxSize", func(m *MisbehaviorsDestinationConfig) { m.S3.MaxSize = -1 }, "s3.maxSize must be >= 0"}, + {"s3 negative flushInterval", func(m *MisbehaviorsDestinationConfig) { m.S3.FlushInterval = -1 }, "s3.flushInterval must be >= 0"}, + {"s3 endpoint bad scheme", func(m *MisbehaviorsDestinationConfig) { m.S3.Endpoint = "ftp://x" }, `s3.endpoint must be a valid http(s) URL, got "ftp://x"`}, + {"s3 endpoint without host", func(m *MisbehaviorsDestinationConfig) { m.S3.Endpoint = "https://" }, `s3.endpoint must be a valid http(s) URL, got "https://"`}, + {"s3 file creds without file", func(m *MisbehaviorsDestinationConfig) { m.S3.Credentials.CredentialsFile = "" }, "s3.credentials.credentialsFile is required when mode is 'file'"}, + {"s3 file creds without profile", func(m *MisbehaviorsDestinationConfig) { m.S3.Credentials.Profile = "" }, "s3.credentials.profile is required when mode is 'file'"}, + {"s3 secret creds incomplete", func(m *MisbehaviorsDestinationConfig) { + m.S3.Credentials = &AwsAuthConfig{Mode: "secret", AccessKeyID: "AK"} + }, "s3.credentials.accessKeyID and secretAccessKey are required when mode is 'secret'"}, + {"s3 secret creds complete", func(m *MisbehaviorsDestinationConfig) { + m.S3.Credentials = &AwsAuthConfig{Mode: "Secret", AccessKeyID: "AK", SecretAccessKey: "SK"} + }, ""}, + {"s3 env creds", func(m *MisbehaviorsDestinationConfig) { m.S3.Credentials = &AwsAuthConfig{Mode: "env"} }, ""}, + {"s3 default chain creds", func(m *MisbehaviorsDestinationConfig) { m.S3.Credentials = &AwsAuthConfig{} }, ""}, + {"s3 unknown creds mode", func(m *MisbehaviorsDestinationConfig) { m.S3.Credentials = &AwsAuthConfig{Mode: "imds"} }, "s3.credentials.mode must be one of: env, file, secret"}, + }) +} + +func cvValidSelectionPolicy(t *testing.T) *SelectionPolicyConfig { + t.Helper() + src := "(upstreams) => upstreams" + prog, err := sobek.Compile("", src, false) + require.NoError(t, err) + return &SelectionPolicyConfig{ + EvalInterval: Duration(time.Minute), + EvalTimeout: Duration(time.Second), + EvalFunc: src, + CompiledProgram: prog, + } +} + +func TestCovValidation_SelectionPolicy(t *testing.T) { + baseline := func() *SelectionPolicyConfig { return cvValidSelectionPolicy(t) } + cvRunCases(t, baseline, (*SelectionPolicyConfig).Validate, []cvCase[SelectionPolicyConfig]{ + {"evalInterval zero", func(s *SelectionPolicyConfig) { s.EvalInterval = 0 }, "selectionPolicy.evalInterval must be greater than 0"}, + {"evalTimeout zero", func(s *SelectionPolicyConfig) { s.EvalTimeout = 0 }, "selectionPolicy.evalTimeout must be greater than 0"}, + {"evalTimeout not below interval", func(s *SelectionPolicyConfig) { s.EvalTimeout = s.EvalInterval }, "selectionPolicy.evalTimeout (1m0s) must be less than evalInterval (1m0s)"}, + {"evalFunc missing", func(s *SelectionPolicyConfig) { s.EvalFunc = "" }, "selectionPolicy.evalFunc is required"}, + {"not compiled", func(s *SelectionPolicyConfig) { s.CompiledProgram = nil }, "selectionPolicy.evalFunc failed to compile (CompiledProgram is nil)"}, + {"ts sentinel needs no program", func(s *SelectionPolicyConfig) { + s.EvalFunc, s.CompiledProgram = tsFunctionSentinelPrefix+"7", nil + }, ""}, + }) +} + +func TestCovValidation_Network(t *testing.T) { + cfg := cvConfigWithBudgets() + baseline := func() *NetworkConfig { + n := cvValidEvmNetwork(1) + n.Failsafe = []*FailsafeConfig{{MatchMethod: "*"}} + n.SelectionPolicy = cvValidSelectionPolicy(t) + n.RateLimitBudget = "plain" + n.Alias = "eth-main_1" + n.CacheKeySuffix = "v1" + n.StaticResponses = []*StaticResponseConfig{{Method: "web3_clientVersion", Response: &StaticResponseBodyConfig{Result: "erpc"}}} + n.Integrity = &IntegrityConfig{HeaderMode: "off"} + return n + } + validate := func(n *NetworkConfig) error { return n.Validate(cfg) } + cvRunCases(t, baseline, validate, []cvCase[NetworkConfig]{ + {"architecture missing", func(n *NetworkConfig) { n.Architecture = "" }, "network.*.architecture is required"}, + {"evm block missing", func(n *NetworkConfig) { n.Evm = nil }, "network.*.evm is required for evm networks"}, + {"evm block invalid", func(n *NetworkConfig) { n.Evm.FallbackFinalityDepth = 0 }, "network.*.evm.fallbackFinalityDepth must be greater than 0"}, + {"failsafe invalid", func(n *NetworkConfig) { n.Failsafe[0].Retry = &RetryPolicyConfig{} }, "upstream.*.failsafe.retry.backoffFactor must be greater than 0"}, + {"consensus allowed at network level", func(n *NetworkConfig) { + n.Failsafe[0].Consensus = &ConsensusPolicyConfig{MaxParticipants: 2, AgreementThreshold: 2} + }, ""}, + {"selection policy invalid", func(n *NetworkConfig) { n.SelectionPolicy.EvalFunc = "" }, "selectionPolicy.evalFunc is required"}, + {"unknown budget", func(n *NetworkConfig) { n.RateLimitBudget = "nope" }, "network.*.rateLimitBudget 'nope' does not exist in config.rateLimiters"}, + {"bad alias", func(n *NetworkConfig) { n.Alias = "eth main" }, "network.*.alias 'eth main' must contain only alphanumeric characters, dash, or underscore"}, + {"bad cacheKeySuffix", func(n *NetworkConfig) { n.CacheKeySuffix = "a/b" }, "network.*.cacheKeySuffix 'a/b' must contain only alphanumeric characters, dash, or underscore"}, + {"nil static response", func(n *NetworkConfig) { n.StaticResponses = append(n.StaticResponses, nil) }, "network.*.staticResponses[1]: entry is nil"}, + {"static response without method", func(n *NetworkConfig) { n.StaticResponses[0].Method = "" }, "network.*.staticResponses[0]: method is required"}, + {"static response without response", func(n *NetworkConfig) { n.StaticResponses[0].Response = nil }, "network.*.staticResponses[0]: response is required"}, + {"static response with both", func(n *NetworkConfig) { + n.StaticResponses[0].Response.Error = &StaticResponseErrorConfig{Code: -32000, Message: "x"} + }, "response must set exactly one of result or error, got both"}, + {"static response with neither", func(n *NetworkConfig) { n.StaticResponses[0].Response.Result = nil }, "response must set exactly one of result or error, got neither"}, + {"static error without message", func(n *NetworkConfig) { + n.StaticResponses[0].Response = &StaticResponseBodyConfig{Error: &StaticResponseErrorConfig{Code: -32000}} + }, "network.*.staticResponses[0]: response.error.message is required"}, + {"static error with message", func(n *NetworkConfig) { + n.StaticResponses[0].Response = &StaticResponseBodyConfig{Error: &StaticResponseErrorConfig{Code: -32601, Message: "disabled"}} + }, ""}, + {"integrity invalid", func(n *NetworkConfig) { n.Integrity.HeaderMode = "verbose" }, "network.*: integrity.headerMode 'verbose' is invalid"}, + }) +} + +func TestCovValidation_EvmNetwork(t *testing.T) { + baseline := func() *EvmNetworkConfig { + e := cvValidEvmNetwork(1).Evm + e.LocalAnswers = map[string]bool{LocalAnswerMethods()[0]: false} + e.SafeBlockSource = "tag:archive" + return e + } + cvRunCases(t, baseline, (*EvmNetworkConfig).Validate, []cvCase[EvmNetworkConfig]{ + {"fallbackFinalityDepth", func(e *EvmNetworkConfig) { e.FallbackFinalityDepth = 0 }, "network.*.evm.fallbackFinalityDepth must be greater than 0"}, + {"fallbackStatePollerDebounce", func(e *EvmNetworkConfig) { e.FallbackStatePollerDebounce = 0 }, "network.*.evm.fallbackStatePollerDebounce is required"}, + {"getLogsMaxAllowedRange", func(e *EvmNetworkConfig) { e.GetLogsMaxAllowedRange = 0 }, "network.*.evm.getLogsMaxAllowedRange must be greater than 0"}, + {"unknown local answer", func(e *EvmNetworkConfig) { e.LocalAnswers["eth_getBalance"] = true }, "network.*.evm.localAnswers.eth_getBalance: eRPC has no local answer for this method"}, + {"bad safeBlockSource", func(e *EvmNetworkConfig) { e.SafeBlockSource = "(tag:archive" }, `network.*.evm.safeBlockSource has invalid selector "(tag:archive"`}, + }) +} diff --git a/common/cov_validation_project_test.go b/common/cov_validation_project_test.go new file mode 100644 index 000000000..e2988be75 --- /dev/null +++ b/common/cov_validation_project_test.go @@ -0,0 +1,335 @@ +package common + +import ( + "testing" + "time" + + "github.com/erpc/erpc/util" + "github.com/stretchr/testify/require" +) + +// cvConfigWithBudgets is the root config Project/Upstream/Network validation +// resolves rate-limit budgets and proxy pools against. +func cvConfigWithBudgets() *Config { + return &Config{ + RateLimiters: &RateLimiterConfig{Budgets: []*RateLimitBudgetConfig{ + {Id: "plain", Rules: []*RateLimitRuleConfig{{Method: "*", MaxCount: 1}}}, + {Id: "priced", Rules: []*RateLimitRuleConfig{{Method: "*", MaxCount: 1}}, CreditUnits: map[string]int64{"eth_call": 2}}, + {Id: "credit-rule", Rules: []*RateLimitRuleConfig{{Method: "*", MaxCount: 1, CountMode: RateLimitCountModeCredit}}}, + }}, + ProxyPools: []*ProxyPoolConfig{{ID: "pool-a", Urls: []string{"http://p"}}, {ID: "pool-b", Urls: []string{"http://q"}}}, + } +} + +func cvValidEvmNetwork(chainId int64) *NetworkConfig { + return &NetworkConfig{ + Architecture: ArchitectureEvm, + Evm: &EvmNetworkConfig{ + ChainId: chainId, + FallbackFinalityDepth: 1024, + FallbackStatePollerDebounce: Duration(time.Second), + GetLogsMaxAllowedRange: 10_000, + }, + } +} + +func TestCovValidation_Project(t *testing.T) { + cfg := cvConfigWithBudgets() + baseline := func() *ProjectConfig { + return &ProjectConfig{ + Id: "main", + Providers: []*ProviderConfig{ + {Id: "alchemy", Vendor: "alchemy", UpstreamIdTemplate: "-"}, + }, + Upstreams: []*UpstreamConfig{ + {Id: "u1", Endpoint: "http://a"}, + {Id: "u2", Endpoint: "http://b"}, + }, + UpstreamDefaults: &UpstreamConfig{Failsafe: []*FailsafeConfig{nil, {MatchMethod: "*"}}}, + NetworkDefaults: &NetworkDefaults{CacheKeySuffix: "v2"}, + Networks: []*NetworkConfig{ + func() *NetworkConfig { n := cvValidEvmNetwork(1); n.Alias = "eth"; return n }(), + cvValidEvmNetwork(10), + }, + Auth: &AuthConfig{Strategies: []*AuthStrategyConfig{{Type: AuthTypeNetwork, Network: &NetworkStrategyConfig{}}}}, + CORS: &CORSConfig{AllowedOrigins: []string{"*"}}, + AllowClientDirectives: util.StringPtr("use-upstream | retry-*"), + RateLimitBudget: "plain", + Integrity: &IntegrityConfig{IntegritySettings: IntegritySettings{Level: "intrinsic"}}, + } + } + validate := func(p *ProjectConfig) error { return p.Validate(cfg) } + cvRunCases(t, baseline, validate, []cvCase[ProjectConfig]{ + {"id missing", func(p *ProjectConfig) { p.Id = "" }, "project id is required"}, + {"invalid provider", func(p *ProjectConfig) { p.Providers[0].Vendor = "" }, "project.*.providers.*.vendor is required"}, + {"duplicate provider", func(p *ProjectConfig) { p.Providers = append(p.Providers, p.Providers[0]) }, "project.*.providers.*.id must be unique, 'alchemy' is duplicated"}, + {"invalid upstream", func(p *ProjectConfig) { p.Upstreams[0].Endpoint = "" }, "upstream.*.endpoint is required"}, + {"duplicate upstream", func(p *ProjectConfig) { p.Upstreams[1].Id = "u1" }, "project.*.upstreams.*.id must be unique, 'u1' is duplicated"}, + {"no upstreams nor providers", func(p *ProjectConfig) { p.Upstreams, p.Providers = nil, nil }, "project.*.upstreams or project.*.providers is required"}, + {"providers only", func(p *ProjectConfig) { p.Upstreams = nil }, ""}, + {"upstreamDefaults consensus", func(p *ProjectConfig) { + p.UpstreamDefaults.Failsafe[1].Consensus = &ConsensusPolicyConfig{} + }, "project.*.upstreamDefaults: failsafe.consensus is only supported for network-level failsafe"}, + {"upstreamDefaults matchCommitment", func(p *ProjectConfig) { + p.UpstreamDefaults.Failsafe[1].MatchCommitment = []string{"confirmed"} + }, "project.*.upstreamDefaults: failsafe.matchCommitment is only supported for network-level failsafe"}, + {"networkDefaults bad suffix", func(p *ProjectConfig) { p.NetworkDefaults.CacheKeySuffix = "v2!" }, "networkDefaults.cacheKeySuffix 'v2!' must contain only alphanumeric characters"}, + {"invalid network", func(p *ProjectConfig) { p.Networks[1].Architecture = "" }, "network.*.architecture is required"}, + {"duplicate network id", func(p *ProjectConfig) { p.Networks[1].Evm.ChainId = 1 }, "project.*.networks.*.id must be unique, 'evm:1' is duplicated"}, + {"duplicate alias", func(p *ProjectConfig) { p.Networks[1].Alias = "eth" }, "project.*.networks.*.alias must be unique, 'eth' is duplicated"}, + {"svm upstream chain mismatch", func(p *ProjectConfig) { + p.Upstreams = append(p.Upstreams, &UpstreamConfig{ + Id: "sol", Type: UpstreamTypeSvm, Endpoint: "http://sol", + Svm: &SvmUpstreamConfig{Chain: "eclipse", Cluster: "mainnet-beta"}, + }) + p.Networks = append(p.Networks, &NetworkConfig{Architecture: ArchitectureSvm, Svm: &SvmNetworkConfig{Cluster: "mainnet-beta"}}) + }, "upstream.*.svm.chain 'eclipse' (upstream 'sol') matches no network.*.svm.chain for cluster 'mainnet-beta'"}, + {"invalid auth", func(p *ProjectConfig) { p.Auth.Strategies = nil }, "project.*.auth.strategies is required"}, + {"invalid cors", func(p *ProjectConfig) { p.CORS.AllowedOrigins = nil }, "*.cors.allowedOrigins is required"}, + {"bad allowClientDirectives", func(p *ProjectConfig) { p.AllowClientDirectives = util.StringPtr("(use-upstream") }, "project.*.allowClientDirectives pattern is invalid"}, + {"empty allowClientDirectives", func(p *ProjectConfig) { p.AllowClientDirectives = util.StringPtr("") }, ""}, + {"unknown budget", func(p *ProjectConfig) { p.RateLimitBudget = "nope" }, "project.*.rateLimitBudget 'nope' does not exist in config.rateLimiters"}, + {"invalid integrity", func(p *ProjectConfig) { p.Integrity.Level = "paranoid" }, "project.*: integrity.level 'paranoid' is invalid"}, + }) +} + +func TestCovValidation_Provider(t *testing.T) { + cfg := cvConfigWithBudgets() + baseline := func() *ProviderConfig { + return &ProviderConfig{ + Id: "p", Vendor: "drpc", UpstreamIdTemplate: "-", + OnlyNetworks: []string{"evm:1", "svm:mainnet-beta"}, + Overrides: map[string]*UpstreamConfig{"evm:1": {RateLimitBudget: "plain"}}, + } + } + validate := func(p *ProviderConfig) error { return p.Validate(cfg) } + cvRunCases(t, baseline, validate, []cvCase[ProviderConfig]{ + {"id missing", func(p *ProviderConfig) { p.Id = "" }, "project.*.providers.*.id is required"}, + {"vendor missing", func(p *ProviderConfig) { p.Vendor = "" }, "project.*.providers.*.vendor is required"}, + {"template missing", func(p *ProviderConfig) { p.UpstreamIdTemplate = "" }, "project.*.providers.*.upstreamIdTemplate is required"}, + {"only and ignore together", func(p *ProviderConfig) { p.IgnoreNetworks = []string{} }, "onlyNetworks and project.*.providers.*.ignoreNetworks are mutually exclusive"}, + {"override skips endpoint but checks budget", func(p *ProviderConfig) { + p.Overrides["evm:1"].RateLimitBudget = "nope" + }, "upstream.*.rateLimitBudget 'nope' does not exist in config.rateLimiters"}, + {"bad onlyNetworks", func(p *ProviderConfig) { p.OnlyNetworks = []string{"evm:0"} }, "project.*.providers.*.onlyNetworks.* 'evm:0' is invalid must be like evm:1"}, + {"bad ignoreNetworks", func(p *ProviderConfig) { + p.OnlyNetworks, p.IgnoreNetworks = nil, []string{"ethereum"} + }, "project.*.providers.*.ignoreNetworks.* 'ethereum' is invalid must be like evm:1"}, + {"valid ignoreNetworks", func(p *ProviderConfig) { p.OnlyNetworks, p.IgnoreNetworks = nil, []string{"evm:137"} }, ""}, + }) +} + +func TestCovValidation_AuthConfig(t *testing.T) { + dbStrategy := func(connId string) *AuthStrategyConfig { + return &AuthStrategyConfig{Type: AuthTypeDatabase, Database: &DatabaseStrategyConfig{Connector: cvMemoryConnector(connId)}} + } + baseline := func() *AuthConfig { + return &AuthConfig{Strategies: []*AuthStrategyConfig{dbStrategy("db1"), dbStrategy("db2")}} + } + cvRunCases(t, baseline, (*AuthConfig).Validate, []cvCase[AuthConfig]{ + {"no strategies", func(a *AuthConfig) { a.Strategies = nil }, "project.*.auth.strategies is required, add at least one strategy"}, + {"invalid strategy", func(a *AuthConfig) { a.Strategies[0].Type = "oauth" }, "auth.*.type 'oauth' is invalid must be one of"}, + {"duplicate database connector", func(a *AuthConfig) { a.Strategies[1].Database.Connector.Id = "db1" }, "database auth strategy connector ID 'db1' is not unique"}, + }) +} + +func TestCovValidation_AuthStrategy(t *testing.T) { + baseline := func() *AuthStrategyConfig { + return &AuthStrategyConfig{Type: AuthTypeSecret, Secret: &SecretStrategyConfig{Value: "s3cret"}} + } + cvRunCases(t, baseline, (*AuthStrategyConfig).Validate, []cvCase[AuthStrategyConfig]{ + {"type missing", func(s *AuthStrategyConfig) { s.Type = "" }, "auth.*.type is required"}, + {"network strategy without block", func(s *AuthStrategyConfig) { s.Type = AuthTypeNetwork }, "auth.*.network is required for network strategy"}, + {"network strategy", func(s *AuthStrategyConfig) { s.Type, s.Network = AuthTypeNetwork, &NetworkStrategyConfig{} }, ""}, + {"secret strategy without block", func(s *AuthStrategyConfig) { s.Secret = nil }, "auth.*.secret is required for secret strategy"}, + {"secret value missing", func(s *AuthStrategyConfig) { s.Secret.Value = "" }, "auth.*.secret.value is required"}, + {"jwt strategy without block", func(s *AuthStrategyConfig) { s.Type = AuthTypeJwt }, "auth.*.jwt is required for jwt strategy"}, + {"jwt strategy invalid", func(s *AuthStrategyConfig) { s.Type, s.Jwt = AuthTypeJwt, &JwtStrategyConfig{} }, "auth.*.jwt.verificationKeys or auth.*.jwt.verificationJwksUrl is required"}, + {"jwt strategy", func(s *AuthStrategyConfig) { + s.Type, s.Jwt = AuthTypeJwt, &JwtStrategyConfig{VerificationKeys: map[string]string{"k": "v"}} + }, ""}, + {"siwe strategy without block", func(s *AuthStrategyConfig) { s.Type = AuthTypeSiwe }, "auth.*.siwe is required for siwe strategy"}, + {"siwe strategy", func(s *AuthStrategyConfig) { s.Type, s.Siwe = AuthTypeSiwe, &SiweStrategyConfig{} }, ""}, + {"database strategy without block", func(s *AuthStrategyConfig) { s.Type = AuthTypeDatabase }, "auth.*.database is required for database strategy"}, + {"database strategy invalid", func(s *AuthStrategyConfig) { + s.Type, s.Database = AuthTypeDatabase, &DatabaseStrategyConfig{} + }, "auth.*.database.connector is required"}, + }) +} + +func TestCovValidation_DatabaseStrategy(t *testing.T) { + i64 := func(v int64) *int64 { return &v } + baseline := func() *DatabaseStrategyConfig { + ttl := time.Minute + return &DatabaseStrategyConfig{ + Connector: cvMemoryConnector("auth"), + Cache: &DatabaseStrategyCacheConfig{TTL: &ttl, MaxSize: i64(10), MaxCost: i64(10), NumCounters: i64(100)}, + } + } + cvRunCases(t, baseline, (*DatabaseStrategyConfig).Validate, []cvCase[DatabaseStrategyConfig]{ + {"connector missing", func(s *DatabaseStrategyConfig) { s.Connector = nil }, "auth.*.database.connector is required"}, + {"negative ttl", func(s *DatabaseStrategyConfig) { d := -time.Second; s.Cache.TTL = &d }, "auth.*.database.cache.ttl must be non-negative"}, + {"zero maxSize", func(s *DatabaseStrategyConfig) { s.Cache.MaxSize = i64(0) }, "auth.*.database.cache.maxSize must be positive"}, + {"zero maxCost", func(s *DatabaseStrategyConfig) { s.Cache.MaxCost = i64(0) }, "auth.*.database.cache.maxCost must be positive"}, + {"negative numCounters", func(s *DatabaseStrategyConfig) { s.Cache.NumCounters = i64(-1) }, "auth.*.database.cache.numCounters must be positive"}, + {"invalid connector", func(s *DatabaseStrategyConfig) { s.Connector.Driver = "x" }, "database.*.connector.driver 'x' is invalid"}, + {"no cache block", func(s *DatabaseStrategyConfig) { s.Cache = nil }, ""}, + }) +} + +func TestCovValidation_Upstream(t *testing.T) { + cfg := cvConfigWithBudgets() + baseline := func() *UpstreamConfig { + return &UpstreamConfig{ + Id: "u1", + Endpoint: "https://rpc.example", + Evm: &EvmUpstreamConfig{ChainId: 1, StatePollerInterval: Duration(time.Second)}, + Failsafe: []*FailsafeConfig{{MatchMethod: "*", Retry: &RetryPolicyConfig{BackoffFactor: 2, BackoffMaxDelay: Duration(time.Second)}}}, + JsonRpc: &JsonRpcUpstreamConfig{SupportsBatch: util.BoolPtr(true), BatchMaxSize: 10, BatchMaxWait: Duration(time.Millisecond), ProxyPool: "pool-b"}, + Grpc: &GrpcUpstreamConfig{PoolSize: 2}, + RateLimitAutoTune: &RateLimitAutoTuneConfig{ + Enabled: util.BoolPtr(true), AdjustmentPeriod: Duration(time.Minute), + ErrorRateThreshold: 0.1, IncreaseFactor: 1.05, DecreaseFactor: 0.9, + }, + RateLimitCountMode: RateLimitCountModeCredit, + RateLimitBudget: "plain", + } + } + validate := func(u *UpstreamConfig) error { return u.Validate(cfg, false) } + cvRunCases(t, baseline, validate, []cvCase[UpstreamConfig]{ + {"endpoint missing", func(u *UpstreamConfig) { u.Endpoint = "" }, "upstream.*.endpoint is required"}, + {"evm invalid", func(u *UpstreamConfig) { u.Evm.StatePollerInterval = 0 }, "upstream.*.evm.statePollerInterval is required"}, + {"svm invalid", func(u *UpstreamConfig) { + u.Type, u.Evm, u.Svm = UpstreamTypeSvm, nil, &SvmUpstreamConfig{} + }, "upstream.*.svm.cluster is required for svm upstreams"}, + {"failsafe invalid", func(u *UpstreamConfig) { u.Failsafe[0].MatchMethod = "" }, "failsafe.matchMethod cannot be empty"}, + {"failsafe consensus", func(u *UpstreamConfig) { + u.Failsafe[0].Consensus = &ConsensusPolicyConfig{MaxParticipants: 3, AgreementThreshold: 2} + }, "upstream 'u1': failsafe.consensus is only supported for network-level failsafe"}, + {"jsonRpc batch without wait", func(u *UpstreamConfig) { u.JsonRpc.BatchMaxWait = 0 }, "jsonRpc.batchMaxWait is required and must be greater than 0"}, + {"jsonRpc batch without size", func(u *UpstreamConfig) { u.JsonRpc.BatchMaxSize = 0 }, "jsonRpc.batchMaxSize must be greater than 0"}, + {"jsonRpc unknown proxy pool", func(u *UpstreamConfig) { u.JsonRpc.ProxyPool = "pool-z" }, "jsonRpc.proxyPool 'pool-z' does not exist in configured proxyPools, must be one of: [pool-a pool-b]"}, + {"jsonRpc batch disabled ignores sizes", func(u *UpstreamConfig) { + u.JsonRpc = &JsonRpcUpstreamConfig{SupportsBatch: util.BoolPtr(false)} + }, ""}, + {"grpc pool negative", func(u *UpstreamConfig) { u.Grpc.PoolSize = -2 }, "upstream.*.grpc.poolSize must not be negative"}, + {"grpc health check on mounted path", func(u *UpstreamConfig) { + u.Endpoint = "grpc://node:443/v1" + u.Evm.ChainId = 0 + u.Grpc.HealthCheckService = "svc" + }, "upstream.*.grpc.healthCheckService cannot be used with"}, + {"autoTune invalid", func(u *UpstreamConfig) { u.RateLimitAutoTune.AdjustmentPeriod = 0 }, "upstream.*.rateLimitAutoTune.adjustmentPeriod is required"}, + {"unknown count mode", func(u *UpstreamConfig) { u.RateLimitCountMode = "tokens" }, "upstream.*.rateLimitCountMode 'tokens' is invalid"}, + {"unknown budget", func(u *UpstreamConfig) { u.RateLimitBudget = "nope" }, "upstream.*.rateLimitBudget 'nope' does not exist in config.rateLimiters"}, + {"credit mode with creditUnits budget", func(u *UpstreamConfig) { u.RateLimitBudget = "priced" }, "but its budget 'priced' also defines creditUnits"}, + {"credit mode with credit-rule budget", func(u *UpstreamConfig) { u.RateLimitBudget = "credit-rule" }, "but its budget 'credit-rule' also sets rules.*.countMode"}, + {"request mode with priced budget", func(u *UpstreamConfig) { + u.RateLimitCountMode, u.RateLimitBudget = RateLimitCountModeRequest, "priced" + }, ""}, + }) + + t.Run("skipEndpointCheck allows empty endpoint", func(t *testing.T) { + u := baseline() + u.Endpoint = "" + u.Evm.ChainId = 0 + require.NoError(t, u.Validate(cfg, true)) + }) +} + +func TestCovValidation_RateLimitAutoTune(t *testing.T) { + baseline := func() *RateLimitAutoTuneConfig { + return &RateLimitAutoTuneConfig{ + Enabled: util.BoolPtr(true), AdjustmentPeriod: Duration(time.Minute), + ErrorRateThreshold: 1, IncreaseFactor: 1.5, DecreaseFactor: 0.5, MinBudget: 0, MaxBudget: 100, + } + } + cvRunCases(t, baseline, (*RateLimitAutoTuneConfig).Validate, []cvCase[RateLimitAutoTuneConfig]{ + {"disabled skips checks", func(r *RateLimitAutoTuneConfig) { r.Enabled = util.BoolPtr(false); r.IncreaseFactor = 0 }, ""}, + {"nil enabled skips checks", func(r *RateLimitAutoTuneConfig) { r.Enabled = nil; r.AdjustmentPeriod = 0 }, ""}, + {"adjustmentPeriod", func(r *RateLimitAutoTuneConfig) { r.AdjustmentPeriod = 0 }, "upstream.*.rateLimitAutoTune.adjustmentPeriod is required"}, + {"errorRateThreshold zero", func(r *RateLimitAutoTuneConfig) { r.ErrorRateThreshold = 0 }, "upstream.*.rateLimitAutoTune.errorRateThreshold must be greater than 0 and less than 1"}, + {"errorRateThreshold above one", func(r *RateLimitAutoTuneConfig) { r.ErrorRateThreshold = 1.01 }, "errorRateThreshold must be greater than 0"}, + {"increaseFactor", func(r *RateLimitAutoTuneConfig) { r.IncreaseFactor = 1 }, "upstream.*.rateLimitAutoTune.increaseFactor must be greater than 1"}, + {"decreaseFactor zero", func(r *RateLimitAutoTuneConfig) { r.DecreaseFactor = 0 }, "upstream.*.rateLimitAutoTune.decreaseFactor must be greater than 0 and less than 1"}, + {"decreaseFactor one", func(r *RateLimitAutoTuneConfig) { r.DecreaseFactor = 1 }, "decreaseFactor must be greater than 0 and less than 1"}, + {"minBudget negative", func(r *RateLimitAutoTuneConfig) { r.MinBudget = -1 }, "upstream.*.rateLimitAutoTune.minBudget must be greater than or equal to 0"}, + {"maxBudget negative", func(r *RateLimitAutoTuneConfig) { r.MaxBudget = -1 }, "upstream.*.rateLimitAutoTune.maxBudget must be greater than or equal to 0"}, + }) +} + +func TestCovValidation_EvmUpstream(t *testing.T) { + i64 := func(v int64) *int64 { return &v } + u := &UpstreamConfig{Endpoint: "https://rpc.example"} + baseline := func() *EvmUpstreamConfig { + return &EvmUpstreamConfig{ + ChainId: 1, + StatePollerInterval: Duration(time.Second), + NodeType: EvmNodeTypeArchive, + BlockAvailability: &EvmBlockAvailabilityConfig{ + Lower: &EvmAvailabilityBoundConfig{LatestBlockMinus: i64(1000), Probe: EvmProbeEventLogs}, + Upper: &EvmAvailabilityBoundConfig{LatestBlockMinus: i64(0)}, + }, + Multicall3: &EvmMulticall3Config{ + MaxWait: Duration(time.Millisecond), MaxCalls: 2, + BypassContracts: []string{"0x" + "11223344556677889900aabbccddeeff00112233"}, + }, + } + } + validate := func(e *EvmUpstreamConfig) error { return e.Validate(u) } + cvRunCases(t, baseline, validate, []cvCase[EvmUpstreamConfig]{ + {"statePollerInterval", func(e *EvmUpstreamConfig) { e.StatePollerInterval = 0 }, "upstream.*.evm.statePollerInterval is required"}, + {"unknown nodeType", func(e *EvmUpstreamConfig) { e.NodeType = "light" }, "upstream.*.evm.nodeType 'light' is invalid must be one of"}, + {"invalid lower bound", func(e *EvmUpstreamConfig) { e.BlockAvailability.Lower = &EvmAvailabilityBoundConfig{} }, "upstream.*.evm.blockAvailability.lower is invalid: bound must set exactly one of"}, + {"invalid upper bound", func(e *EvmUpstreamConfig) { e.BlockAvailability.Upper.Probe = "receipts" }, "upstream.*.evm.blockAvailability.upper is invalid: bound.probe 'receipts' is invalid"}, + {"latestBlockMinus inverted", func(e *EvmUpstreamConfig) { e.BlockAvailability.Lower.LatestBlockMinus = i64(0); e.BlockAvailability.Upper.LatestBlockMinus = i64(5) }, "lower.latestBlockMinus must be >= upper.latestBlockMinus"}, + {"exactBlock inverted", func(e *EvmUpstreamConfig) { + e.BlockAvailability.Lower = &EvmAvailabilityBoundConfig{ExactBlock: i64(10)} + e.BlockAvailability.Upper = &EvmAvailabilityBoundConfig{ExactBlock: i64(5)} + }, "upstream.*.evm.blockAvailability.lower.exactBlock must be <= upper.exactBlock"}, + {"earliestBlockPlus inverted", func(e *EvmUpstreamConfig) { + e.BlockAvailability.Lower = &EvmAvailabilityBoundConfig{EarliestBlockPlus: i64(10)} + e.BlockAvailability.Upper = &EvmAvailabilityBoundConfig{EarliestBlockPlus: i64(5)} + }, "lower.earliestBlockPlus must be <= upper.earliestBlockPlus"}, + {"multicall maxWait", func(e *EvmUpstreamConfig) { e.Multicall3.MaxWait = 0 }, "upstream.*.evm.multicall3.maxWait must be greater than 0"}, + {"multicall maxCalls", func(e *EvmUpstreamConfig) { e.Multicall3.MaxCalls = 1 }, "upstream.*.evm.multicall3.maxCalls must be at least 2"}, + {"multicall short bypass address", func(e *EvmUpstreamConfig) { e.Multicall3.BypassContracts = []string{"0x1234"} }, "upstream.*.evm.multicall3.bypassContracts entry '0x1234' is not a 20-byte hex address"}, + {"multicall non-hex bypass address", func(e *EvmUpstreamConfig) { e.Multicall3.BypassContracts = []string{"0xzz"} }, "bypassContracts entry '0xzz' is not a 20-byte hex address"}, + }) + + t.Run("chainId rejected on non-native endpoint", func(t *testing.T) { + e := baseline() + err := e.Validate(&UpstreamConfig{Endpoint: "alchemy://key"}) + require.ErrorContains(t, err, "upstream.*.evm.chainId must be 0 for non-http endpoints, but '1' is provided") + }) + t.Run("zero chainId accepted on non-native endpoint", func(t *testing.T) { + e := baseline() + e.ChainId = 0 + require.NoError(t, e.Validate(&UpstreamConfig{Endpoint: "alchemy://key"})) + }) +} + +func TestCovValidation_EvmAvailabilityBound(t *testing.T) { + i64 := func(v int64) *int64 { return &v } + baseline := func() *EvmAvailabilityBoundConfig { + return &EvmAvailabilityBoundConfig{EarliestBlockPlus: i64(5), Probe: EvmProbeTraceData, UpdateRate: Duration(time.Minute)} + } + cvRunCases(t, baseline, (*EvmAvailabilityBoundConfig).Validate, []cvCase[EvmAvailabilityBoundConfig]{ + {"none set", func(b *EvmAvailabilityBoundConfig) { b.EarliestBlockPlus = nil }, "bound must set exactly one of: exactBlock, latestBlockMinus, earliestBlockPlus"}, + {"two set", func(b *EvmAvailabilityBoundConfig) { b.ExactBlock = i64(1) }, "bound fields exactBlock, latestBlockMinus, earliestBlockPlus are mutually exclusive"}, + {"exact with probe", func(b *EvmAvailabilityBoundConfig) { + b.EarliestBlockPlus, b.ExactBlock, b.UpdateRate = nil, i64(1), 0 + }, "bound.probe must be empty when exactBlock is set"}, + {"exact with updateRate", func(b *EvmAvailabilityBoundConfig) { + b.EarliestBlockPlus, b.ExactBlock, b.Probe = nil, i64(1), "" + }, "bound.updateRate must be 0 when exactBlock is set"}, + {"exact negative", func(b *EvmAvailabilityBoundConfig) { + *b = EvmAvailabilityBoundConfig{ExactBlock: i64(-1)} + }, "bound.exactBlock must be >= 0"}, + {"exact valid", func(b *EvmAvailabilityBoundConfig) { *b = EvmAvailabilityBoundConfig{ExactBlock: i64(0)} }, ""}, + {"latest negative", func(b *EvmAvailabilityBoundConfig) { b.EarliestBlockPlus, b.LatestBlockMinus = nil, i64(-1) }, "bound.latestBlockMinus must be >= 0"}, + {"earliest negative", func(b *EvmAvailabilityBoundConfig) { b.EarliestBlockPlus = i64(-1) }, "bound.earliestBlockPlus must be >= 0"}, + {"negative updateRate", func(b *EvmAvailabilityBoundConfig) { b.UpdateRate = Duration(-time.Second) }, "bound.updateRate must be >= 0"}, + {"latest with updateRate only warns", func(b *EvmAvailabilityBoundConfig) { b.EarliestBlockPlus, b.LatestBlockMinus = nil, i64(3) }, ""}, + {"empty probe defaults", func(b *EvmAvailabilityBoundConfig) { b.Probe = "" }, ""}, + }) +} diff --git a/common/cov_validation_top_test.go b/common/cov_validation_top_test.go new file mode 100644 index 000000000..486969933 --- /dev/null +++ b/common/cov_validation_top_test.go @@ -0,0 +1,240 @@ +package common + +import ( + "testing" + "time" + + "github.com/erpc/erpc/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// cvCase is one row of a "valid baseline, mutate one field" table: mutate +// breaks exactly one rule and wantErr is the message fragment that rule emits. +type cvCase[T any] struct { + name string + mutate func(*T) + wantErr string +} + +// cvRunCases builds a fresh baseline per row, asserts the untouched baseline +// passes, and that each mutation is rejected with its specific message. +func cvRunCases[T any](t *testing.T, baseline func() *T, validate func(*T) error, cases []cvCase[T]) { + t.Helper() + t.Run("valid baseline accepted", func(t *testing.T) { + require.NoError(t, validate(baseline())) + }) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + v := baseline() + tc.mutate(v) + err := validate(v) + if tc.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorContains(t, err, tc.wantErr) + }) + } +} + +func cvDur(d time.Duration) *Duration { v := Duration(d); return &v } + +func cvValidConfig() *Config { + return &Config{ + Server: &ServerConfig{MaxTimeout: cvDur(30 * time.Second)}, + HealthCheck: &HealthCheckConfig{}, + Projects: []*ProjectConfig{{ + Id: "main", + Upstreams: []*UpstreamConfig{{Id: "u1", Endpoint: "http://rpc.example"}}, + }}, + } +} + +func TestCovValidation_Config(t *testing.T) { + cvRunCases(t, cvValidConfig, (*Config).Validate, []cvCase[Config]{ + {"server missing", func(c *Config) { c.Server = nil }, "server config is required"}, + {"server invalid", func(c *Config) { c.Server.MaxTimeout = nil }, "server.maxTimeout is required"}, + {"healthCheck missing", func(c *Config) { c.HealthCheck = nil }, "healthCheck config is required"}, + {"healthCheck auth invalid", func(c *Config) { c.HealthCheck.Auth = &AuthConfig{} }, "project.*.auth.strategies is required"}, + {"metrics invalid", func(c *Config) { c.Metrics = &MetricsConfig{ErrorLabelMode: "loud"} }, "metrics.errorLabelMode must be either"}, + {"metrics valid", func(c *Config) { c.Metrics = &MetricsConfig{} }, ""}, + {"admin invalid", func(c *Config) { c.Admin = &AdminConfig{CORS: &CORSConfig{}} }, "*.cors.allowedOrigins is required"}, + {"admin valid", func(c *Config) { c.Admin = &AdminConfig{CORS: &CORSConfig{AllowedOrigins: []string{"*"}}} }, ""}, + {"database invalid", func(c *Config) { + c.Database = &DatabaseConfig{SharedState: &SharedStateConfig{}} + }, "sharedState.connector is required"}, + {"projects missing", func(c *Config) { c.Projects = nil }, "projects config is required"}, + {"project invalid", func(c *Config) { c.Projects[0].Id = "" }, "project id is required"}, + {"rateLimiters invalid", func(c *Config) { c.RateLimiters = &RateLimiterConfig{} }, "rateLimiters.store is required"}, + {"rateLimiters valid", func(c *Config) { + c.RateLimiters = &RateLimiterConfig{Store: &RateLimitStoreConfig{Driver: "memory"}} + }, ""}, + {"proxyPool invalid", func(c *Config) { c.ProxyPools = []*ProxyPoolConfig{{}} }, "proxyPool.*.id is required"}, + {"proxyPool valid", func(c *Config) { + c.ProxyPools = []*ProxyPoolConfig{{ID: "p", Urls: []string{"http://proxy"}}} + }, ""}, + {"indexer invalid", func(c *Config) { c.Indexer = &IndexerConfig{DedupWindowSize: -1} }, "indexer.dedupWindowSize must be >= 0"}, + {"indexer zero uses default", func(c *Config) { c.Indexer = &IndexerConfig{} }, ""}, + }) +} + +func TestCovValidation_Server(t *testing.T) { + baseline := func() *ServerConfig { + return &ServerConfig{ + ListenV4: util.BoolPtr(true), + HttpHostV4: util.StringPtr("0.0.0.0"), + HttpPortV4: util.IntPtr(4000), + ListenV6: util.BoolPtr(true), + HttpHostV6: util.StringPtr("[::]"), + HttpPortV6: util.IntPtr(4000), + MaxTimeout: cvDur(time.Second), + TrustedIPForwarders: []string{ + "10.0.0.0/8", " 192.168.1.1 ", "::1", + }, + } + } + cvRunCases(t, baseline, (*ServerConfig).Validate, []cvCase[ServerConfig]{ + {"listenV4 without host", func(s *ServerConfig) { s.HttpHostV4 = nil }, "server.listenV4 is true but server.httpHostV4 is not set"}, + {"listenV4 without port", func(s *ServerConfig) { s.HttpPortV4 = nil }, "server.listenV4 is true but server.httpPortV4 is not set"}, + {"listenV6 without host", func(s *ServerConfig) { s.HttpHostV6 = nil }, "server.listenV6 is true but server.httpHostV6 is not set"}, + {"listenV6 without port", func(s *ServerConfig) { s.HttpPortV6 = nil }, "server.listenV6 is true but server.httpPortV6 is not set"}, + {"listen disabled needs no host", func(s *ServerConfig) { + s.ListenV4, s.ListenV6 = util.BoolPtr(false), util.BoolPtr(false) + s.HttpHostV4, s.HttpHostV6 = nil, nil + }, ""}, + {"maxTimeout nil", func(s *ServerConfig) { s.MaxTimeout = nil }, "server.maxTimeout is required"}, + {"maxTimeout zero", func(s *ServerConfig) { s.MaxTimeout = cvDur(0) }, "server.maxTimeout is required"}, + {"empty forwarder", func(s *ServerConfig) { s.TrustedIPForwarders = []string{" "} }, "server.trustedForwarders contains empty entry"}, + {"bad CIDR", func(s *ServerConfig) { s.TrustedIPForwarders = []string{"10.0.0.0/99"} }, "server.trustedForwarders entry '10.0.0.0/99' is not a valid CIDR"}, + {"bad IP", func(s *ServerConfig) { s.TrustedIPForwarders = []string{"not-an-ip"} }, "server.trustedForwarders entry 'not-an-ip' is not a valid IP address"}, + {"webSocket negative limits", func(s *ServerConfig) { + s.WebSocket = &WebSocketServerConfig{ + PingInterval: cvDur(time.Second), + MaxSubscriptionsPerConnection: 1, + MaxConcurrentRequestsPerConnection: 1, + SubscriptionBufferSize: 1, + MaxConnectionsPerIP: -1, + } + }, "maxConnections, maxConnectionsPerIP and maxSubscriptions must not be negative"}, + {"webSocket nil pingInterval", func(s *ServerConfig) { + s.WebSocket = &WebSocketServerConfig{} + }, "server.webSocket.pingInterval must be greater than 0"}, + }) +} + +func TestCovValidation_Metrics(t *testing.T) { + baseline := func() *MetricsConfig { + return &MetricsConfig{ + Enabled: util.BoolPtr(true), + HostV4: util.StringPtr("0.0.0.0"), + Port: util.IntPtr(4001), + ErrorLabelMode: ErrorLabelModeCompact, + HistogramBuckets: "0.05, 0.5,5", + Customizations: []*MetricsCustomizationConfig{ + {Subject: "erpc_upstream_*", Action: MetricActionDrop}, + }, + } + } + cvRunCases(t, baseline, (*MetricsConfig).Validate, []cvCase[MetricsConfig]{ + {"enabled without any host", func(m *MetricsConfig) { m.HostV4 = nil }, "metrics.hostV4 or metrics.hostV6 is required when metrics.enabled is true"}, + {"enabled with only v6 host", func(m *MetricsConfig) { m.HostV4, m.HostV6 = nil, util.StringPtr("::") }, ""}, + {"enabled without port", func(m *MetricsConfig) { m.Port = nil }, "metrics.port is required when metrics.enabled is true"}, + {"disabled needs no host or port", func(m *MetricsConfig) { + m.Enabled, m.HostV4, m.Port = util.BoolPtr(false), nil, nil + }, ""}, + {"verbose label mode", func(m *MetricsConfig) { m.ErrorLabelMode = ErrorLabelModeVerbose }, ""}, + {"unknown label mode", func(m *MetricsConfig) { m.ErrorLabelMode = "terse" }, "metrics.errorLabelMode must be either 'verbose' or 'compact'"}, + {"non-float bucket", func(m *MetricsConfig) { m.HistogramBuckets = "0.1,abc" }, "metrics.histogramBuckets contains invalid float value: abc"}, + {"malformed customization subject", func(m *MetricsConfig) { + m.Customizations = []*MetricsCustomizationConfig{{Subject: "erpc_*_total", Action: MetricActionDrop}} + }, "may only use '*' as its final character"}, + {"customization that does nothing", func(m *MetricsConfig) { + m.Customizations = []*MetricsCustomizationConfig{{Subject: "erpc_x"}} + }, "has no action, labels or buckets"}, + }) +} + +func TestCovValidation_AdminAndHealthCheck(t *testing.T) { + validAuth := func() *AuthConfig { + return &AuthConfig{Strategies: []*AuthStrategyConfig{{Type: AuthTypeSecret, Secret: &SecretStrategyConfig{Value: "s"}}}} + } + cvRunCases(t, func() *AdminConfig { + return &AdminConfig{Auth: validAuth(), CORS: &CORSConfig{AllowedOrigins: []string{"https://app"}}} + }, (*AdminConfig).Validate, []cvCase[AdminConfig]{ + {"auth invalid", func(a *AdminConfig) { a.Auth.Strategies[0].Secret.Value = "" }, "auth.*.secret.value is required"}, + {"cors without origins", func(a *AdminConfig) { a.CORS.AllowedOrigins = nil }, "*.cors.allowedOrigins is required"}, + }) + + cvRunCases(t, func() *HealthCheckConfig { + return &HealthCheckConfig{Auth: validAuth()} + }, (*HealthCheckConfig).Validate, []cvCase[HealthCheckConfig]{ + {"auth strategy without type", func(h *HealthCheckConfig) { h.Auth.Strategies[0].Type = "" }, "auth.*.type is required"}, + }) +} + +func TestCovValidation_RateLimiters(t *testing.T) { + baseline := func() *RateLimiterConfig { + return &RateLimiterConfig{ + Store: &RateLimitStoreConfig{ + Driver: " Redis ", + Redis: &RedisConnectorConfig{URI: "redis://localhost:6379"}, + NearLimitRatio: 0.8, + }, + Budgets: []*RateLimitBudgetConfig{{ + Id: "b1", + Rules: []*RateLimitRuleConfig{{Method: "*", MaxCount: 10, Period: RateLimitPeriodMinute}}, + CreditUnits: map[string]int64{"eth_call": 2}, + }}, + } + } + cvRunCases(t, baseline, (*RateLimiterConfig).Validate, []cvCase[RateLimiterConfig]{ + {"store missing", func(r *RateLimiterConfig) { r.Store = nil }, "rateLimiters.store is required"}, + {"redis store without redis", func(r *RateLimiterConfig) { r.Store.Redis = nil }, "rateLimiters.store.redis is required when store.type is 'redis'"}, + {"nearLimitRatio too high", func(r *RateLimiterConfig) { r.Store.NearLimitRatio = 1 }, "rateLimiters.store.nearLimitRatio must be > 0 and < 1"}, + {"nearLimitRatio negative", func(r *RateLimiterConfig) { r.Store.NearLimitRatio = -0.1 }, "rateLimiters.store.nearLimitRatio must be > 0 and < 1"}, + {"nearLimitRatio zero means default", func(r *RateLimiterConfig) { r.Store.NearLimitRatio = 0 }, ""}, + {"redis connector invalid", func(r *RateLimiterConfig) { r.Store.Redis.URI = "" }, "rateLimiters.store.redis is invalid: database.*.connector.redis.uri is required"}, + {"memory store", func(r *RateLimiterConfig) { r.Store = &RateLimitStoreConfig{Driver: "memory"} }, ""}, + {"empty driver", func(r *RateLimiterConfig) { r.Store.Driver = "" }, "rateLimiters.store.type '' is invalid must be one of: redis, memory"}, + {"unknown driver", func(r *RateLimiterConfig) { r.Store.Driver = "etcd" }, "rateLimiters.store.type 'etcd' is invalid"}, + {"budget without rules", func(r *RateLimiterConfig) { r.Budgets[0].Rules = nil }, "rateLimiter.*.budget.rules is required"}, + {"rule without method", func(r *RateLimiterConfig) { r.Budgets[0].Rules[0].Method = "" }, "rateLimiter.*.budget.rules.*.method is required"}, + {"negative creditUnits", func(r *RateLimiterConfig) { r.Budgets[0].CreditUnits["eth_call"] = -1 }, "rateLimiter.*.budget.creditUnits.eth_call must not be negative"}, + }) +} + +func TestCovValidation_RateLimitRule(t *testing.T) { + baseline := func() *RateLimitRuleConfig { + return &RateLimitRuleConfig{Method: "eth_*", MaxCount: 5, Period: RateLimitPeriodSecond} + } + cvRunCases(t, baseline, (*RateLimitRuleConfig).Validate, []cvCase[RateLimitRuleConfig]{ + {"deprecated waitTime is ignored", func(r *RateLimitRuleConfig) { r.WaitTime = Duration(time.Second) }, ""}, + {"unknown period", func(r *RateLimitRuleConfig) { r.Period = RateLimitPeriod(99) }, "rateLimiter.*.budget.rules.*.period must be one of"}, + {"year period", func(r *RateLimitRuleConfig) { r.Period = RateLimitPeriodYear }, ""}, + {"request count mode", func(r *RateLimitRuleConfig) { r.CountMode = RateLimitCountModeRequest }, ""}, + {"unknown count mode", func(r *RateLimitRuleConfig) { r.CountMode = "bytes" }, "countMode 'bytes' is invalid, must be one of: request, credit"}, + {"credit mode with zero ceiling", func(r *RateLimitRuleConfig) { + r.CountMode, r.MaxCount = RateLimitCountModeCredit, 0 + }, "maxCount must be greater than 0 when countMode is credit"}, + {"credit mode with ceiling", func(r *RateLimitRuleConfig) { r.CountMode = RateLimitCountModeCredit }, ""}, + }) +} + +func TestCovValidation_ProxyPool(t *testing.T) { + baseline := func() *ProxyPoolConfig { + return &ProxyPoolConfig{ID: "pool", Urls: []string{"http://a", "HTTPS://b", "socks5://c"}} + } + cvRunCases(t, baseline, (*ProxyPoolConfig).Validate, []cvCase[ProxyPoolConfig]{ + {"id missing", func(p *ProxyPoolConfig) { p.ID = "" }, "proxyPool.*.id is required under proxyPools"}, + {"urls missing", func(p *ProxyPoolConfig) { p.Urls = nil }, "proxyPool.*.urls is required under proxyPool.*.id 'pool'"}, + {"unsupported scheme", func(p *ProxyPoolConfig) { p.Urls = []string{"ftp://x"} }, "must be valid HTTP, HTTPS, or SOCKS5 URLs, got: ftp://x"}, + }) +} + +func TestCovValidation_RateLimitBudgetHasCreditRule(t *testing.T) { + b := &RateLimitBudgetConfig{Rules: []*RateLimitRuleConfig{nil, {CountMode: RateLimitCountModeRequest}}} + assert.False(t, b.HasCreditRule()) + b.Rules = append(b.Rules, &RateLimitRuleConfig{CountMode: RateLimitCountModeCredit}) + assert.True(t, b.HasCreditRule()) +} diff --git a/common/defaults.go b/common/defaults.go index 60f33f30e..6c6fdaf01 100644 --- a/common/defaults.go +++ b/common/defaults.go @@ -1365,14 +1365,7 @@ func (p *ProjectConfig) SetDefaults(opts *DefaultOptions) error { if err := repositoryCfg.SetDefaults(p.UpstreamDefaults); err != nil { return fmt.Errorf("failed to set defaults for repository provider: %w", err) } - envioCfg := &ProviderConfig{ - Id: "envio", - Vendor: "envio", - } - if err := envioCfg.SetDefaults(p.UpstreamDefaults); err != nil { - return fmt.Errorf("failed to set defaults for envio provider: %w", err) - } - p.Providers = append(p.Providers, repositoryCfg, envioCfg) + p.Providers = append(p.Providers, repositoryCfg) } } if p.Providers == nil { @@ -2142,6 +2135,23 @@ func (e *EvmUpstreamConfig) SetDefaults(defaults *EvmUpstreamConfig) error { } } + if e.PinLatestTag == nil && defaults != nil { + e.PinLatestTag = defaults.PinLatestTag + } + + if e.Multicall3 == nil && defaults != nil && defaults.Multicall3 != nil { + v := *defaults.Multicall3 + e.Multicall3 = &v + } + if e.Multicall3 != nil { + if e.Multicall3.MaxWait == 0 { + e.Multicall3.MaxWait = Duration(10 * time.Millisecond) + } + if e.Multicall3.MaxCalls == 0 { + e.Multicall3.MaxCalls = 20 + } + } + return nil } @@ -2245,6 +2255,9 @@ func (n *NetworkConfig) SetDefaults(upstreams []*UpstreamConfig, defaults *Netwo if n.Evm.DynamicBlockTimeDebounceMultiplier == nil && defaults.Evm.DynamicBlockTimeDebounceMultiplier != nil { n.Evm.DynamicBlockTimeDebounceMultiplier = defaults.Evm.DynamicBlockTimeDebounceMultiplier } + if n.Evm.FinalizedDebounceDepthMultiplier == nil && defaults.Evm.FinalizedDebounceDepthMultiplier != nil { + n.Evm.FinalizedDebounceDepthMultiplier = defaults.Evm.FinalizedDebounceDepthMultiplier + } if n.Evm.BlockUnavailableDelayMultiplier == nil && defaults.Evm.BlockUnavailableDelayMultiplier != nil { n.Evm.BlockUnavailableDelayMultiplier = defaults.Evm.BlockUnavailableDelayMultiplier } @@ -2291,6 +2304,14 @@ func (n *NetworkConfig) SetDefaults(upstreams []*UpstreamConfig, defaults *Netwo if n.Evm.StripSubscribeFromBlockZero == nil && defaults.Evm.StripSubscribeFromBlockZero != nil { n.Evm.StripSubscribeFromBlockZero = defaults.Evm.StripSubscribeFromBlockZero } + for method, on := range defaults.Evm.LocalAnswers { + if _, set := n.Evm.LocalAnswers[method]; !set { + if n.Evm.LocalAnswers == nil { + n.Evm.LocalAnswers = make(map[string]bool, len(defaults.Evm.LocalAnswers)) + } + n.Evm.LocalAnswers[method] = on + } + } } else if n.Evm == nil && defaults.Evm != nil && n.Svm == nil && n.Architecture != ArchitectureSvm { // Copy EVM defaults only onto networks that are (or can become) EVM. // Without the SVM guard, a mixed project with networkDefaults.evm @@ -2440,6 +2461,7 @@ func migrateLegacyIntegrityChecks(n *NetworkConfig) { const DefaultEvmFinalityDepth = 1024 const DefaultEvmStatePollerDebounce = Duration(5 * time.Second) const DefaultDynamicBlockTimeDebounceMultiplier = 0.7 +const DefaultFinalizedDebounceDepthMultiplier = 0.05 const DefaultBlockUnavailableDelayMultiplier = 1.0 // DefaultToleratedBlockHeadRollback is the tolerance (in blocks) applied when a @@ -2450,37 +2472,12 @@ const DefaultBlockUnavailableDelayMultiplier = 1.0 // and the health tracker so both layers converge on the same head. const DefaultToleratedBlockHeadRollback = 1024 -// DefaultServedTipTrajectoryWindow is how much head history the served-tip -// trajectory referee needs before it may participate in a pick -// (EvmServedTipConfig.TrajectoryWindow). Ten minutes is long enough that a -// majority stall cannot masquerade as the trajectory (the referee's evidence is -// the median of live heads, so a stall of half the window already flattens the -// fit and stands the referee down), and short enough to be warm well within a -// pod's lifetime. -const DefaultServedTipTrajectoryWindow = 10 * time.Minute - -// ServedTipTrajectorySampleInterval and ServedTipTrajectoryMaxSamples are the -// referee's recording cadence and its hard ring cap (architecture/evm's -// tipSampleInterval / tipMaxBufferCapacity, which read them from here so the -// config bounds below cannot drift from the mechanism they describe). -const ( - ServedTipTrajectorySampleInterval = 5 * time.Second - ServedTipTrajectoryMaxSamples = 1024 -) - -// MinServedTipTrajectoryWindow / MaxServedTipTrajectoryWindow bound a configured -// EvmServedTipConfig.TrajectoryWindow to the values that can actually work. -// -// Below a minute the window is self-defeating: it must be long enough that a -// stall cannot look like a trajectory, and a fit over seconds of history is -// noise. Above 80% of what the ring can span (cadence × capacity) the required -// span is never reached, so the referee would stand down forever while looking -// configured — the failure mode a range check exists to prevent, since nothing -// in the metrics distinguishes "never warm" from "never needed". -const ( - MinServedTipTrajectoryWindow = time.Minute - MaxServedTipTrajectoryWindow = ServedTipTrajectorySampleInterval * ServedTipTrajectoryMaxSamples * 4 / 5 -) +// DefaultServedTipQuorum is how many eligible upstreams must have a block +// before the network serves it as "latest"/"finalized" (ServedTipQuorum). +// Two keeps a lone far-ahead head out of every multi-upstream ballot, while a +// ballot reduced to one voter (benched peers, a one-upstream network) still +// serves that voter's head. +const DefaultServedTipQuorum = 2 // DefaultEmptyResultMaxAttempts bounds retries when the requested data isn't on the // upstream yet (empty/missing-data point-lookups, pending tx-lookups, and @@ -2644,6 +2641,10 @@ func (e *EvmNetworkConfig) SetDefaults() error { d := DefaultDynamicBlockTimeDebounceMultiplier e.DynamicBlockTimeDebounceMultiplier = &d } + if e.FinalizedDebounceDepthMultiplier == nil { + d := DefaultFinalizedDebounceDepthMultiplier + e.FinalizedDebounceDepthMultiplier = &d + } if e.BlockUnavailableDelayMultiplier == nil { d := DefaultBlockUnavailableDelayMultiplier e.BlockUnavailableDelayMultiplier = &d @@ -3396,6 +3397,9 @@ func (r *RateLimitStoreConfig) SetDefaults() error { if err := r.Redis.SetDefaults(); err != nil { return fmt.Errorf("failed to set defaults for redis connector: %w", err) } + if r.OnFailure == "" { + r.OnFailure = RateLimitStoreFailureAllow + } } return nil } diff --git a/common/errors.go b/common/errors.go index 10fa02fea..7a151bed6 100644 --- a/common/errors.go +++ b/common/errors.go @@ -232,6 +232,19 @@ func (e *BaseError) WithPermanentMissingData(p bool) *BaseError { return e } +// WithHistorySweepContinuation marks MissingData that exists only to continue a +// serial upstream sweep after an incomplete-but-successful JSON-RPC body — not +// proof that the upstream lacked data other peers had. +func (e *BaseError) WithHistorySweepContinuation(p bool) *BaseError { + if e != nil { + if e.Details == nil { + e.Details = map[string]interface{}{} + } + e.Details["historySweepContinuation"] = p + } + return e +} + func (e *BaseError) GetCode() ErrorCode { return e.Code } @@ -569,13 +582,16 @@ func (r AuthUnauthorizedReason) Message() string { } } -var NewErrAuthUnauthorized = func(strategy string, message string) error { +// NewErrAuthUnauthorized builds ErrAuthUnauthorized with a closed reason label +// for metrics (Details.reason) and a separate client-facing message. +var NewErrAuthUnauthorized = func(strategy, reason, message string) error { return &ErrAuthUnauthorized{ BaseError{ Code: ErrCodeAuthUnauthorized, Message: message, Details: map[string]interface{}{ "strategy": strategy, + "reason": reason, }, }, } @@ -583,16 +599,7 @@ var NewErrAuthUnauthorized = func(strategy string, message string) error { // NewErrAuthUnauthorizedReason builds ErrAuthUnauthorized with Message from reason. var NewErrAuthUnauthorizedReason = func(strategy string, reason AuthUnauthorizedReason) error { - return &ErrAuthUnauthorized{ - BaseError{ - Code: ErrCodeAuthUnauthorized, - Message: reason.Message(), - Details: map[string]interface{}{ - "strategy": strategy, - "reason": reason, - }, - }, - } + return NewErrAuthUnauthorized(strategy, string(reason), reason.Message()) } func (e *ErrAuthUnauthorized) ErrorStatusCode() int { @@ -734,10 +741,12 @@ func (e *ErrUnknownNetworkArchitecture) ErrorStatusCode() int { type ErrNotImplemented struct{ BaseError } +const ErrCodeNotImplemented ErrorCode = "ErrNotImplemented" + var NewErrNotImplemented = func(msg string) error { return &ErrNotImplemented{ BaseError{ - Code: "ErrNotImplemented", + Code: ErrCodeNotImplemented, Message: msg, }, } @@ -2704,6 +2713,28 @@ func IsPermanentlyMissingData(err error) bool { return false } +// IsHistorySweepContinuation reports MissingData emitted to continue a serial +// sweep after an incomplete 200 — not a wrong-empty disagreement with the winner. +func IsHistorySweepContinuation(err error) bool { + for cur := err; cur != nil; { + cse, ok := cur.(StandardError) + if !ok { + break + } + if base := cse.Base(); base != nil && base.Details != nil { + if p, ok := base.Details["historySweepContinuation"].(bool); ok && p { + return true + } + } + next := cse.GetCause() + if _, isMulti := next.(interface{ Unwrap() []error }); isMulti { + break + } + cur = next + } + return false +} + func IsRetryableTowardsUpstream(err error) bool { // Check if this is an exhausted upstreams error with retryable underlying errors if HasErrorCode(err, ErrCodeUpstreamsExhausted) { @@ -2732,6 +2763,11 @@ func IsRetryableTowardsUpstream(err error) bool { ErrCodeUpstreamMethodIgnored, ErrCodeEndpointUnsupported, + // Method the network cannot serve in this configuration (stateful method + // under multiple upstreams, eth_accounts/eth_sign) -> No Retry: this is a + // deterministic pre-forward rejection, not a transient upstream fault. + ErrCodeNotImplemented, + // Do not try when 3rd-party providers run out of monthly capacity or billing issues ErrCodeEndpointBillingIssue, @@ -2792,6 +2828,8 @@ func IsClientError(err error) bool { ErrCodeGetLogsExceededMaxAllowedAddresses, ErrCodeGetLogsExceededMaxAllowedTopics, ErrCodeGetLogsExceededMaxAllowedResponseSize, + ErrCodeSubscriptionNotFound, + ErrCodeSubscriptionLimitExceeded, )) } @@ -3200,6 +3238,20 @@ var NewErrSubscriptionLimitExceeded = func(max int) error { } } +// NewErrInstanceSubscriptionLimitExceeded is the instance-wide counterpart +// of NewErrSubscriptionLimitExceeded (server.webSocket.maxSubscriptions). +var NewErrInstanceSubscriptionLimitExceeded = func(max int) error { + return &ErrSubscriptionLimitExceeded{ + BaseError{ + Code: ErrCodeSubscriptionLimitExceeded, + Message: "maximum subscriptions per instance exceeded", + Details: map[string]interface{}{ + "maxPerInstance": max, + }, + }, + } +} + func (e *ErrSubscriptionLimitExceeded) ErrorStatusCode() int { return http.StatusTooManyRequests } diff --git a/common/errors_retry_test.go b/common/errors_retry_test.go index 23fc89d2d..2c3730165 100644 --- a/common/errors_retry_test.go +++ b/common/errors_retry_test.go @@ -363,6 +363,29 @@ func TestClassifySeverity_NetworkNotSupported(t *testing.T) { }) } +func TestClassifySeverity_NotImplemented(t *testing.T) { + // ErrNotImplemented is a deterministic pre-forward rejection — a stateful + // method under multiple upstreams, or eth_accounts/eth_sign. No upstream + // will ever accept it and no retry or operator action can fix it, so it must + // not page: not retryable, warning severity (not the critical fall-through). + err := NewErrNotImplemented("stateful method requires a single targeted upstream") + assert.False(t, IsRetryableTowardsUpstream(err), + "not-implemented is a deterministic rejection — retrying is pointless") + assert.Equal(t, SeverityWarning, ClassifySeverity(err), + "not-implemented must be warning, not the critical fall-through") +} + +func TestClassifySeverity_SubscriptionNotFound(t *testing.T) { + // eth_unsubscribe for an already-gone / unknown subscription is a benign + // client condition (JSON-RPC convention is to return false, not fault). It + // must classify as a client error → info severity, not the critical + // fall-through that would page on routine WS unsubscribe churn. + err := NewErrSubscriptionNotFound("0xdeadbeef") + assert.True(t, IsClientError(err), "subscription-not-found is a client condition") + assert.Equal(t, SeverityInfo, ClassifySeverity(err), + "subscription-not-found must be info, not the critical fall-through") +} + func BenchmarkIsRetryableTowardNetwork(b *testing.B) { missingDataErr := NewErrEndpointMissingData( NewErrJsonRpcExceptionInternal(-32000, -32014, "missing trie node", nil, nil), diff --git a/common/evm_local_answers.go b/common/evm_local_answers.go new file mode 100644 index 000000000..43d62590a --- /dev/null +++ b/common/evm_local_answers.go @@ -0,0 +1,129 @@ +package common + +import ( + "context" + "fmt" + "iter" + "runtime" + "slices" + "strconv" + "strings" +) + +// A local answer is eRPC's own reply to a method whose true answer does not +// depend on which upstream serves it, or is about eRPC itself, the node the +// client talks to. Routing such a method makes one upstream a single point of +// failure for a constant (upstreams often reject net_* methods) or returns one +// node's view where eRPC fronts several. ok=false leaves the request to the +// upstreams. +type localAnswer func(ctx context.Context, n Network) (result any, ok bool) + +var localAnswers = map[string]localAnswer{ + "eth_chainId": func(_ context.Context, n Network) (any, bool) { + id := evmChainId(n) + if id <= 0 { + return nil, false + } + return fmt.Sprintf("0x%x", id), true + }, + "net_version": func(_ context.Context, n Network) (any, bool) { + id := evmChainId(n) + if id <= 0 { + return nil, false + } + return strconv.FormatInt(id, 10), true + }, + // eRPC is listening while it answers. + "net_listening": func(context.Context, Network) (any, bool) { return true, true }, + // eRPC's peers are the upstreams it can route this network to now. + "net_peerCount": func(ctx context.Context, n Network) (any, bool) { + ups, ok := evmRoutableUpstreams(ctx, n) + if !ok { + return nil, false + } + count := 0 + for range ups { + count++ + } + return fmt.Sprintf("0x%x", count), true + }, + "web3_clientVersion": func(context.Context, Network) (any, bool) { + return fmt.Sprintf("erpc/%s/%s-%s/%s", ErpcVersion, runtime.GOOS, runtime.GOARCH, runtime.Version()), true + }, + // eRPC serves from synced upstreams, so it is not syncing while one of its + // routable upstreams is synced. Otherwise the upstreams answer. + "eth_syncing": func(ctx context.Context, n Network) (any, bool) { + ups, ok := evmRoutableUpstreams(ctx, n) + if !ok { + return nil, false + } + for u := range ups { + if eu, ok := u.(EvmUpstream); ok && eu.EvmSyncingState() == EvmSyncingStateNotSyncing { + return false, true + } + } + return nil, false + }, +} + +// LocalAnswerMethods lists the methods eRPC answers itself unless a network's +// evm.localAnswers sets them false. +func LocalAnswerMethods() []string { + names := make([]string, 0, len(localAnswers)) + for name := range localAnswers { + names = append(names, name) + } + slices.Sort(names) + return names +} + +// LocalAnswer returns eRPC's own answer to method on n. ok=false when the +// method has no local answer, the network turned it off, or eRPC cannot tell +// now; the request then goes to the upstreams. +func LocalAnswer(ctx context.Context, n Network, method string) (result any, ok bool) { + name, answer := lookupLocalAnswer(method) + if answer == nil { + return nil, false + } + if cfg := n.Config(); cfg != nil && cfg.Evm != nil { + if on, set := cfg.Evm.LocalAnswers[name]; set && !on { + return nil, false + } + } + return answer(ctx, n) +} + +func lookupLocalAnswer(method string) (string, localAnswer) { + if answer, ok := localAnswers[method]; ok { + return method, answer + } + for name, answer := range localAnswers { + if strings.EqualFold(name, method) { + return name, answer + } + } + return "", nil +} + +func evmChainId(n Network) int64 { + if cfg := n.Config(); cfg != nil && cfg.Evm != nil { + return cfg.Evm.ChainId + } + return 0 +} + +// EvmRoutableUpstreamsProvider is the optional surface a network exposes its +// routable upstreams through: the selection policy's eligible set in routing +// order, narrowed to the request's use-upstream selector. Asserted narrowly, +// like EvmStateProvenWriter, so EvmNetwork implementations need not change. +type EvmRoutableUpstreamsProvider interface { + EvmRoutableUpstreams(ctx context.Context) iter.Seq[Upstream] +} + +func evmRoutableUpstreams(ctx context.Context, n Network) (iter.Seq[Upstream], bool) { + p, ok := n.(EvmRoutableUpstreamsProvider) + if !ok { + return nil, false + } + return p.EvmRoutableUpstreams(ctx), true +} diff --git a/common/json_rpc.go b/common/json_rpc.go index a967d2217..2c42dca6e 100644 --- a/common/json_rpc.go +++ b/common/json_rpc.go @@ -1418,7 +1418,7 @@ func (r *JsonRpcRequest) UnmarshalJSON(data []byte) error { } aux.JSONRPC = "2.0" - if err := SonicCfg.Unmarshal(data, &aux); err != nil { + if err := SonicCfg.Unmarshal(data, aux); err != nil { // MIP-16 answers eth_query* params that are not an array with // -32602, not a parse error: decode the envelope again with the // params left raw and drop them, so the query parser rejects the @@ -1905,6 +1905,20 @@ func TranslateToJsonRpcException(err error) error { nil, ) } + // The client can free a slot by unsubscribing; -32603 would read as a server fault. + if HasErrorCode(err, ErrCodeSubscriptionLimitExceeded) { + reason := "subscription limit exceeded" + if se, ok := err.(StandardError); ok { + reason = se.DeepestMessage() + } + return NewErrJsonRpcExceptionInternal( + 0, + JsonRpcErrorCapacityExceeded, + reason, + err, + nil, + ) + } if HasErrorCode(err, ErrCodeUpstreamMethodIgnored) { var reason string if se, ok := err.(StandardError); ok { @@ -1983,6 +1997,26 @@ func TranslateToJsonRpcException(err error) error { nil, ) } + // A method eRPC declines to serve (filter lifecycle methods under a + // multi-upstream pool, eth_accounts/eth_sign, subscriptions over HTTP) is + // "method not supported", not an internal fault. Without this it falls + // through to -32603, which clients read as a transient server error: ethers + // only downgrades filters to polling when the error classifies as + // UNSUPPORTED_OPERATION (code -32601 / a "not found|supported" message), so a + // -32603 leaves it re-arming the same call on every block forever. + if HasErrorCode(err, ErrCodeNotImplemented) { + reason := "method not supported" + if se, ok := err.(StandardError); ok { + reason = se.DeepestMessage() + } + return NewErrJsonRpcExceptionInternal( + 0, + JsonRpcErrorUnsupportedException, + reason, + err, + nil, + ) + } var msg = "internal server error" if se, ok := err.(StandardError); ok { msg = se.DeepestMessage() diff --git a/common/json_rpc_test.go b/common/json_rpc_test.go index 1a78528c4..a4cbe5d02 100644 --- a/common/json_rpc_test.go +++ b/common/json_rpc_test.go @@ -772,6 +772,22 @@ func clientWireCode(t *testing.T, err error) JsonRpcErrorNumber { return jre.NormalizedCode() } +// A method eRPC declines to serve — stateful filter methods under a +// multi-upstream pool, eth_accounts/eth_sign, or a subscription over HTTP — +// must reach the client as -32601 "method not supported", not the generic +// -32603. ethers only downgrades eth_getFilterChanges to polling when the error +// classifies as UNSUPPORTED_OPERATION (code -32601 / a "not found|supported" +// message); a -32603 leaves it re-arming the same call on every block forever. +func TestTranslateToJsonRpcException_NotImplementedIsMethodNotFound(t *testing.T) { + err := NewErrNotImplemented("method eth_newFilter is not supported by any configured upstream") + translated := TranslateToJsonRpcException(err) + + assert.EqualValues(t, JsonRpcErrorUnsupportedException, clientWireCode(t, translated), + "ErrNotImplemented must map to -32601 so clients detect an unsupported method") + assert.Contains(t, translated.Error(), "not supported", + "the descriptive reason must survive translation for message-matching clients") +} + func newExhausted(t *testing.T, order []string, causes map[string]error) error { t.Helper() m := &sync.Map{} @@ -1147,3 +1163,12 @@ func TestJsonRpcRequest_CacheHash_EvmHexCaseInsensitive(t *testing.T) { require.Equal(t, hashUpper, hashLower, "EVM hex params are case-insensitive and must share a CacheHash") } + +func TestTranslateToJsonRpcException_SubscriptionLimitIsCapacityNotInternal(t *testing.T) { + err := NewErrSubscriptionLimitExceeded(10) + + assert.Equal(t, SeverityInfo, ClassifySeverity(err), + "subscription limit is a client condition and must never grade critical") + assert.EqualValues(t, JsonRpcErrorCapacityExceeded, clientWireCode(t, TranslateToJsonRpcException(err)), + "subscription limit must map to -32005, not -32603") +} diff --git a/common/network.go b/common/network.go index 8b1367f11..a474eae63 100644 --- a/common/network.go +++ b/common/network.go @@ -37,6 +37,10 @@ type EvmNetwork interface { EvmHighestLatestBlockNumber(ctx context.Context) int64 EvmHighestFinalizedBlockNumber(ctx context.Context) int64 EvmLeaderUpstream(ctx context.Context) Upstream + // EvmRefreshLatestBlock asks the pollers of the request's latest-head + // ballot for a fresh head in the background (see + // EvmStatePoller.RefreshLatestBlock); it never blocks. + EvmRefreshLatestBlock(ctx context.Context) } // SvmNetwork is the SVM-specific view of a Network. Production Network @@ -85,6 +89,14 @@ func EvmLeaderUpstream(n Network, ctx context.Context) Upstream { return nil } +// EvmRefreshLatestBlock calls EvmNetwork.EvmRefreshLatestBlock when n is an +// EVM network. +func EvmRefreshLatestBlock(n Network, ctx context.Context) { + if e, ok := n.(EvmNetwork); ok { + e.EvmRefreshLatestBlock(ctx) + } +} + func IsValidArchitecture(architecture string) bool { switch NetworkArchitecture(architecture) { case ArchitectureEvm, ArchitectureSvm: diff --git a/common/request.go b/common/request.go index d4ddeb9ba..53d363337 100644 --- a/common/request.go +++ b/common/request.go @@ -19,6 +19,7 @@ const ( CompositeTypeLogsSplitProactive = "logs-split-proactive" CompositeTypeTraceFilterSplitOnError = "trace-filter-split-on-error" CompositeTypeTraceFilterSplitProactive = "trace-filter-split-proactive" + CompositeTypeMulticall3 = "multicall3" ) const RequestContextKey ContextKey = "rq" @@ -66,6 +67,32 @@ func IsSweepIteration(ctx context.Context) bool { return v } +// wireParamsContextKey carries the JSON-RPC params one upstream attempt sends +// for one request in place of its own. The request keeps its params, so the +// cache key, response binding and every other upstream see the original question. +type wireParamsContextKey struct{} + +// wireParams binds the override to its request: any other request sent with a +// derived ctx on the same attempt (a probe, a state poll) keeps its own params. +type wireParams struct { + req *NormalizedRequest + params []interface{} +} + +// WithWireParams returns ctx whose upstream attempt sends params for req on the wire. +func WithWireParams(ctx context.Context, req *NormalizedRequest, params []interface{}) context.Context { + return context.WithValue(ctx, wireParamsContextKey{}, wireParams{req: req, params: params}) +} + +// WireParams returns the params an upstream client sends for req under ctx: +// the attempt's override for exactly req when set, else req's own params. +func WireParams(ctx context.Context, req *NormalizedRequest, params []interface{}) []interface{} { + if w, ok := ctx.Value(wireParamsContextKey{}).(wireParams); ok && w.req == req { + return w.params + } + return params +} + type directiveKeyNames struct { header string query string @@ -230,6 +257,10 @@ type NormalizedRequest struct { upstreamMutex sync.Mutex UpstreamIdx uint32 ErrorsByUpstream sync.Map + // Upstream -> its latest block when the ErrorsByUpstream entry was stored. + // Wrong-empty accounting needs the head at answer time: by the time the + // request ends, a lagging upstream may have caught up. + headsAtError sync.Map // Fields for centralized upstream management upstreamList []Upstream // Available upstreams for this request @@ -1522,6 +1553,7 @@ func (r *NormalizedRequest) MarkUpstreamCompleted(ctx context.Context, upstream // permanent errors (method ignored, execution exception). if err != nil { r.ErrorsByUpstream.Store(upstream, err) + r.headsAtError.Store(upstream, latestBlockOf(upstream)) } else if resp != nil && resp.IsResultEmptyish(ctx) { jr, jrErr := resp.JsonRpcResponse(ctx) if jr == nil { @@ -1529,6 +1561,7 @@ func (r *NormalizedRequest) MarkUpstreamCompleted(ctx context.Context, upstream } else { r.ErrorsByUpstream.Store(upstream, NewErrEndpointMissingData(fmt.Errorf("upstream responded emptyish: %v", jr.GetResultString()), upstream)) } + r.headsAtError.Store(upstream, latestBlockOf(upstream)) } // Free from ConsumedUpstreams if the upstream can be retried on a @@ -1545,3 +1578,24 @@ func (r *NormalizedRequest) MarkUpstreamCompleted(ctx context.Context, upstream r.ConsumedUpstreams.Delete(upstream) } } + +// HeadAtError returns the upstream's latest block as eRPC knew it when the +// upstream's ErrorsByUpstream entry was stored; 0 when unknown. +func (r *NormalizedRequest) HeadAtError(upstream Upstream) int64 { + if v, ok := r.headsAtError.Load(upstream); ok { + return v.(int64) + } + return 0 +} + +func latestBlockOf(upstream Upstream) int64 { + eu, ok := upstream.(EvmUpstream) + if !ok { + return 0 + } + sp := eu.EvmStatePoller() + if sp == nil || sp.IsObjectNull() { + return 0 + } + return sp.LatestBlock() +} diff --git a/common/served_tip.go b/common/served_tip.go index aaec8d5a7..288ad0448 100644 --- a/common/served_tip.go +++ b/common/served_tip.go @@ -1,34 +1,25 @@ package common -import "sort" - -// ServedTipInput is a single observation: an upstream's last-known tip block. -// Callers are responsible for excluding syncing or cordoned upstreams BEFORE -// passing observations here; the picker treats every input as a candidate. -type ServedTipInput struct { - // UpstreamID is preserved only for telemetry attribution. It is not used - // in the pick. - UpstreamID string - - // BlockNumber is the upstream's reported tip. Zero or negative values are - // treated as "no data yet" and filtered before picking. - BlockNumber int64 -} +import ( + "fmt" + "slices" + "strconv" + "strings" +) // ServedTipPick is the picker's output. type ServedTipPick struct { // Tip is the value to advertise as latest/finalized: the highest block - // number that a strict MAJORITY of the inputs have already reached, or 0 - // when there are no valid inputs. + // number that at least quorum of the inputs have already reached (all of + // them when there are fewer inputs than the quorum), or 0 when there are + // no valid inputs. Tip int64 // Freshest is the freshest CORROBORATED view: the 2nd-highest valid input // (or the only input when N=1) — the reference for the deliberate-lag // gauge (Freshest - Tip). Using the 2nd-highest instead of the raw max - // means a single rogue far-future upstream cannot inflate the lag gauge - // (the problem the old velocity gate solved via MaxEligible: one - // wrong-chain endpoint used to make the gauge read hundreds of thousands - // of blocks). The absolute per-upstream maxima remain observable via + // means a single rogue far-future upstream cannot inflate the lag gauge. + // The absolute per-upstream maxima remain observable via // erpc_upstream_latest_block_number. Freshest int64 @@ -38,72 +29,129 @@ type ServedTipPick struct { // is the one place a lone far-ahead head is still visible to callers. Max int64 - // Inputs is the number of valid (BlockNumber > 0) observations. + // Inputs is the number of valid (> 0) heads. Inputs int +} + +// ServedTipQuorum is how many voters must already have a block before it is +// served: a fixed count (>= 1), or ServedTipQuorumMajority for floor(N/2)+1 of +// the N voters. The zero value is DefaultServedTipQuorum. In YAML/JSON it is a +// positive integer or the string "majority". +type ServedTipQuorum int + +// ServedTipQuorumMajority requires a strict majority of the voters. +const ServedTipQuorumMajority ServedTipQuorum = -1 + +// Of returns the quorum for a ballot of n voters, capped at n so a ballot with +// fewer voters than the quorum serves the block all of them have. +func (q ServedTipQuorum) Of(n int) int { + need := DefaultServedTipQuorum + switch { + case q == ServedTipQuorumMajority: + need = n/2 + 1 + case q > 0: + need = int(q) + } + return max(1, min(need, n)) +} + +func parseServedTipQuorum(s string) (ServedTipQuorum, error) { + s = strings.TrimSpace(s) + if strings.EqualFold(s, "majority") { + return ServedTipQuorumMajority, nil + } + n, err := strconv.Atoi(s) + if err != nil || n < 1 { + return 0, fmt.Errorf("servedTip.quorum must be an integer >= 1 or \"majority\", got %q", s) + } + return ServedTipQuorum(n), nil +} - // Sorted is the valid inputs, DESCENDING by block number — the order - // statistic's own working slice, exposed so the trajectory referee can - // cluster the very same ballot without sorting it a second time. Nil when - // there are no valid inputs; never mutate it. - Sorted []ServedTipInput +func (q ServedTipQuorum) MarshalYAML() (interface{}, error) { + if q == ServedTipQuorumMajority { + return "majority", nil + } + return int(q), nil } -// PickServedTip returns the freshest block number that a strict majority of -// the eligible upstreams have already reached: the floor(N/2)-th highest head -// (0-indexed, descending). This is the entire served-tip algorithm. +func (q *ServedTipQuorum) UnmarshalYAML(unmarshal func(interface{}) error) error { + var n int + if err := unmarshal(&n); err == nil { + return q.set(strconv.Itoa(n)) + } + var s string + if err := unmarshal(&s); err != nil { + return err + } + return q.set(s) +} + +func (q ServedTipQuorum) MarshalJSON() ([]byte, error) { + if q == ServedTipQuorumMajority { + return []byte(`"majority"`), nil + } + return []byte(strconv.Itoa(int(q))), nil +} + +func (q *ServedTipQuorum) UnmarshalJSON(data []byte) error { + if string(data) == "null" { + return nil + } + return q.set(strings.Trim(string(data), `"`)) +} + +func (q *ServedTipQuorum) set(s string) error { + parsed, err := parseServedTipQuorum(s) + if err != nil { + return err + } + *q = parsed + return nil +} + +// PickServedTip returns the freshest block number that at least q of the +// eligible upstreams have already reached: the q-th highest head (all of them +// when fewer than q voted). This is the entire served-tip algorithm. +// +// heads are the voters' heads; callers exclude syncing, cordoned or benched +// upstreams BEFORE passing them, and heads <= 0 ("no data yet") are ignored. +// It sorts heads in place, so a caller's ballot pays no copy. // -// One order statistic over the live heads provides every protection the -// previous cluster + velocity-gate + persistent-counter pipeline engineered -// separately — with zero state and zero configuration: +// One order statistic over the live heads provides every protection a +// stateful pipeline would have to engineer separately — with zero state: // // - GARBAGE-RESISTANT: a far-future tip from a rogue/wrong-chain upstream -// cannot move the pick unless a strict majority agrees with it. -// - STUCK-RESISTANT: a frozen or lagging upstream cannot hold the pick -// back unless it IS the majority (a halted chain — where holding back is -// the correct answer). -// - SERVABLE: by construction at least floor(N/2)+1 upstreams already have -// the advertised block, so interpolated "latest" requests land on -// upstreams that can actually serve it. +// cannot move the pick unless q-1 others agree with it. +// - SERVABLE: at least q upstreams already have the advertised block, so +// interpolated "latest" requests land on upstreams that can serve it. // - MONOTONIC IN PRACTICE: each input is itself a monotonic, rollback- // tolerant poller counter, and an order statistic over monotonic inputs // only regresses when the ELIGIBLE SET changes — bounded by the live -// head spread (a couple of blocks), the same wobble any load-balanced -// provider exhibits. -// - WEDGE-IMMUNE: nothing is persisted and nothing is predicted — no -// inherited counter, no anchor clock, no block-time estimate, no -// absorbing state. The 2026-06 production incident (served tips silently -// frozen hours in the past, fleet-wide) is structurally impossible here; +// head spread. +// - WEDGE-IMMUNE: nothing is persisted and nothing is predicted. A +// frozen-tip regression (served tips silently stuck hours in the past) +// is structurally impossible here; // networks_served_tip_invariants_test.go (package erpc) pins that class // of outcome forever. // -// Examples (heads descending): N=1 → that head; N=2 → the LOWER (never -// advertise a block only one upstream claims); N=3 → 2nd; N=4 → 3rd; N=5 → 3rd. -func PickServedTip(tips []ServedTipInput) ServedTipPick { - // One sorted slice serves the whole evaluation: the order statistic here - // and the referee's clustering downstream (ServedTipPick.Sorted). The - // UpstreamIDs travel with it because group IDENTITY — not just the head - // values — is what the referee's dwell test is keyed on. - sorted := make([]ServedTipInput, 0, len(tips)) - for _, t := range tips { - if t.BlockNumber > 0 { - sorted = append(sorted, t) - } - } - if len(sorted) == 0 { +// A voter that stopped moving holds the pick back only while it is needed for +// the quorum; the health tracker benches such a voter (see +// health.Tracker.IsHeadBenched) so it leaves the ballot instead. +// +// Examples (heads descending), default q=2: N=1 → that head; N=2 → the lower; +// N=3..∞ → 2nd. Majority: N=3 → 2nd; N=4 → 3rd; N=5 → 3rd. +func PickServedTip(heads []int64, q ServedTipQuorum) ServedTipPick { + slices.Sort(heads) + first, _ := slices.BinarySearch(heads, 1) + valid := heads[first:] + n := len(valid) + if n == 0 { return ServedTipPick{} } - sort.Slice(sorted, func(i, j int) bool { return sorted[i].BlockNumber > sorted[j].BlockNumber }) - freshest := sorted[0].BlockNumber - if len(sorted) > 1 { - // Corroborated freshest: a single rogue far-future tip must not be - // able to inflate the lag reference (see ServedTipPick.Freshest). - freshest = sorted[1].BlockNumber - } return ServedTipPick{ - Tip: sorted[len(sorted)/2].BlockNumber, - Freshest: freshest, - Max: sorted[0].BlockNumber, - Inputs: len(sorted), - Sorted: sorted, + Tip: valid[n-q.Of(n)], + Freshest: valid[max(n-2, 0)], + Max: valid[n-1], + Inputs: n, } } diff --git a/common/served_tip_test.go b/common/served_tip_test.go index 34f8de47b..7af5b5fbb 100644 --- a/common/served_tip_test.go +++ b/common/served_tip_test.go @@ -6,71 +6,66 @@ import ( "github.com/stretchr/testify/assert" ) -// The served-tip contract: PickServedTip returns the freshest block a strict -// MAJORITY of inputs have reached. These tests pin the order-statistic -// semantics and the two protections that motivated the design — one rogue -// far-future tip cannot move the pick, one stuck upstream cannot hold it back. - -func tipsFromInts(blocks ...int64) []ServedTipInput { - out := make([]ServedTipInput, len(blocks)) - for i, b := range blocks { - out[i] = ServedTipInput{UpstreamID: "u" + itoa(i), BlockNumber: b} - } - return out -} -func itoa(i int) string { - if i == 0 { - return "0" - } - s := "" - for i > 0 { - s = string(rune('0'+(i%10))) + s - i /= 10 - } - return s -} +// The served-tip contract: PickServedTip returns the freshest block at least +// quorum inputs have reached. These tests pin the order-statistic semantics +// for the default quorum (2) and "majority", and the two protections that +// motivated the design — one rogue far-future tip cannot move the pick, one +// stuck upstream cannot hold it back. + func TestPickServedTip_MajorityIndexAcrossN(t *testing.T) { // N=1: the only head. - assert.Equal(t, int64(100), PickServedTip(tipsFromInts(100)).Tip) + assert.Equal(t, int64(100), PickServedTip([]int64{100}, ServedTipQuorumMajority).Tip) // N=2: the LOWER — never advertise a block only one upstream claims. - assert.Equal(t, int64(100), PickServedTip(tipsFromInts(200, 100)).Tip) + assert.Equal(t, int64(100), PickServedTip([]int64{200, 100}, ServedTipQuorumMajority).Tip) // N=3: 2nd highest (2 of 3 have it). - assert.Equal(t, int64(101), PickServedTip(tipsFromInts(102, 101, 100)).Tip) + assert.Equal(t, int64(101), PickServedTip([]int64{102, 101, 100}, ServedTipQuorumMajority).Tip) // N=4: 3rd highest (3 of 4 have it). - assert.Equal(t, int64(101), PickServedTip(tipsFromInts(103, 102, 101, 100)).Tip) + assert.Equal(t, int64(101), PickServedTip([]int64{103, 102, 101, 100}, ServedTipQuorumMajority).Tip) // N=5: 3rd highest (3 of 5 have it). - assert.Equal(t, int64(102), PickServedTip(tipsFromInts(104, 103, 102, 101, 100)).Tip) + assert.Equal(t, int64(102), PickServedTip([]int64{104, 103, 102, 101, 100}, ServedTipQuorumMajority).Tip) // N=7: 4th highest (4 of 7 have it). - assert.Equal(t, int64(103), PickServedTip(tipsFromInts(106, 105, 104, 103, 102, 101, 100)).Tip) + assert.Equal(t, int64(103), PickServedTip([]int64{106, 105, 104, 103, 102, 101, 100}, ServedTipQuorumMajority).Tip) +} + +func TestPickServedTip_DefaultQuorumIsSecondHighest(t *testing.T) { + var q ServedTipQuorum // zero value = DefaultServedTipQuorum + assert.Equal(t, int64(100), PickServedTip([]int64{100}, q).Tip) + assert.Equal(t, int64(100), PickServedTip([]int64{200, 100}, q).Tip) + assert.Equal(t, int64(102), PickServedTip([]int64{103, 102, 101, 100, 99}, q).Tip, + "N=5: 2nd highest, where majority would serve the 3rd") + assert.Equal(t, int64(101), PickServedTip([]int64{999_999_999, 101, 100}, q).Tip, + "one rogue still cannot move the pick") + assert.Equal(t, int64(100), PickServedTip([]int64{103, 102, 101, 100}, ServedTipQuorum(4)).Tip) + assert.Equal(t, int64(101), PickServedTip([]int64{101, 100}, ServedTipQuorum(1)).Tip) } func TestPickServedTip_GarbageTipCannotMoveThePick(t *testing.T) { // A rogue upstream reporting a fantasy-future block (wrong chain, // misconfigured endpoint) is just one voice — the majority ignores it. - // This is the abstract/zora prod scenario that used to inflate lag gauges - // and (pre-2026-06 fix) could poison the persistent counter. - p := PickServedTip(tipsFromInts(999_999_999, 101, 100)) + // This is the wrong-chain endpoint shape that used to inflate lag gauges + // and, under the earlier persistent-counter design, could poison the counter. + p := PickServedTip([]int64{999_999_999, 101, 100}, ServedTipQuorumMajority) assert.Equal(t, int64(101), p.Tip) assert.Equal(t, int64(101), p.Freshest, "the lag reference (corroborated freshest) must ignore the lone rogue too") // Even two agreeing rogues lose against a 5-upstream majority. - assert.Equal(t, int64(102), PickServedTip(tipsFromInts(999_999_999, 999_999_999, 102, 101, 100)).Tip) + assert.Equal(t, int64(102), PickServedTip([]int64{999_999_999, 999_999_999, 102, 101, 100}, ServedTipQuorumMajority).Tip) // N=2 with one rogue: the SANE (lower) head wins — the old cluster // tie-break picked the garbage here. - assert.Equal(t, int64(100), PickServedTip(tipsFromInts(999_999_999, 100)).Tip) + assert.Equal(t, int64(100), PickServedTip([]int64{999_999_999, 100}, ServedTipQuorumMajority).Tip) } func TestPickServedTip_StuckUpstreamCannotHoldThePickBack(t *testing.T) { // One frozen/lagging upstream cannot pin the advertised tip while the // majority advances — the inverse of the garbage case. - assert.Equal(t, int64(200), PickServedTip(tipsFromInts(5, 201, 200)).Tip) - assert.Equal(t, int64(201), PickServedTip(tipsFromInts(5, 202, 201, 200, 201)).Tip) + assert.Equal(t, int64(200), PickServedTip([]int64{5, 201, 200}, ServedTipQuorumMajority).Tip) + assert.Equal(t, int64(201), PickServedTip([]int64{5, 202, 201, 200, 201}, ServedTipQuorumMajority).Tip) } func TestPickServedTip_AllAgreeingIsIdentity(t *testing.T) { - p := PickServedTip(tipsFromInts(100, 100, 100)) + p := PickServedTip([]int64{100, 100, 100}, ServedTipQuorumMajority) assert.Equal(t, int64(100), p.Tip) assert.Equal(t, int64(100), p.Freshest) assert.Equal(t, 3, p.Inputs) @@ -78,10 +73,10 @@ func TestPickServedTip_AllAgreeingIsIdentity(t *testing.T) { func TestPickServedTip_ZeroAndEmptyInputs(t *testing.T) { // Zero/negative heads are "no data yet" and filtered. - assert.Equal(t, int64(100), PickServedTip(tipsFromInts(0, 100, 0)).Tip) - assert.Equal(t, 1, PickServedTip(tipsFromInts(0, 100, 0)).Inputs) - assert.Equal(t, int64(0), PickServedTip(tipsFromInts(0, 0)).Tip) - assert.Equal(t, int64(0), PickServedTip(nil).Tip) + assert.Equal(t, int64(100), PickServedTip([]int64{0, 100, 0}, ServedTipQuorumMajority).Tip) + assert.Equal(t, 1, PickServedTip([]int64{0, 100, 0}, ServedTipQuorumMajority).Inputs) + assert.Equal(t, int64(0), PickServedTip([]int64{0, 0}, ServedTipQuorumMajority).Tip) + assert.Equal(t, int64(0), PickServedTip(nil, ServedTipQuorumMajority).Tip) } func TestPickServedTip_TipNeverExceedsFreshestAndIsAlwaysAHead(t *testing.T) { @@ -92,30 +87,29 @@ func TestPickServedTip_TipNeverExceedsFreshestAndIsAlwaysAHead(t *testing.T) { {5, 100, 101, 102, 999999}, } for _, heads := range cases { - p := PickServedTip(tipsFromInts(heads...)) + p := PickServedTip(heads, ServedTipQuorumMajority) assert.LessOrEqual(t, p.Tip, p.Freshest, "heads=%v", heads) assert.Contains(t, heads, p.Tip, "tip must be a real observed head; heads=%v", heads) } } func TestPickServedTip_Scenario_VendorPropagationJitter(t *testing.T) { - // Old: ClusterDelta grouped heads within 1-2 blocks so vendor propagation - // jitter never split agreement. New: the majority head IS inside the - // jitter band — no grouping parameter needed. - assert.Equal(t, int64(100), PickServedTip(tipsFromInts(101, 100, 100)).Tip) - assert.Equal(t, int64(101), PickServedTip(tipsFromInts(101, 101, 100)).Tip, + // The quorum head sits inside the vendor propagation jitter band, so no + // grouping parameter is needed. + assert.Equal(t, int64(100), PickServedTip([]int64{101, 100, 100}, ServedTipQuorumMajority).Tip) + assert.Equal(t, int64(101), PickServedTip([]int64{101, 101, 100}, ServedTipQuorumMajority).Tip, "two fresh vs one 1-block lagger: tip moves forward") } func TestPickServedTip_Scenario_SingleLaggerDoesNotHoldBack(t *testing.T) { // Old: the dominant (fresh) cluster outvoted a stuck/lagging upstream. - assert.Equal(t, int64(101), PickServedTip(tipsFromInts(101, 101, 50)).Tip) + assert.Equal(t, int64(101), PickServedTip([]int64{101, 101, 50}, ServedTipQuorumMajority).Tip) } func TestPickServedTip_Scenario_SingleLeaderDoesNotDefineTip(t *testing.T) { // Old: a lone most-ahead node (flashblocks-style) formed a 1-node cluster // and lost to the agreeing pair. Advertising the loner's head is exactly // the "block not found" churn that motivated served-tip in PR #900. - assert.Equal(t, int64(100), PickServedTip(tipsFromInts(120, 100, 99)).Tip, + assert.Equal(t, int64(100), PickServedTip([]int64{120, 100, 99}, ServedTipQuorumMajority).Tip, "the loner's head must never be advertised") } @@ -123,36 +117,36 @@ func TestPickServedTip_Scenario_MajorityLaggersWin(t *testing.T) { // Old: 4 laggers outvoted 3 leaders by cluster size — the network truth // is what most upstreams can serve. New: same outcome with a fresher // representative (the BEST lagger instead of the worst). - assert.Equal(t, int64(50), PickServedTip(tipsFromInts(103, 102, 101, 50, 49, 48, 47)).Tip) + assert.Equal(t, int64(50), PickServedTip([]int64{103, 102, 101, 50, 49, 48, 47}, ServedTipQuorumMajority).Tip) } func TestPickServedTip_Scenario_BurstCatchupServedImmediately(t *testing.T) { // Old: the velocity gate carried slack+buffer tuned to ALLOW legitimate // burst catch-up (L2 sequencer batches, a halted chain resuming), plus - // fail-open for when that tuning was wrong — mis-tuning is what froze - // prod. New: stateless, so a jump is served the moment a majority + // fail-open for when that tuning was wrong — mis-tuning froze the + // tip. New: stateless, so a jump is served the moment a majority // reports it; there is no window to outrun and nothing to mis-arm. - assert.Equal(t, int64(100), PickServedTip(tipsFromInts(100, 100, 99)).Tip) - assert.Equal(t, int64(1100), PickServedTip(tipsFromInts(1100, 1100, 1099)).Tip) + assert.Equal(t, int64(100), PickServedTip([]int64{100, 100, 99}, ServedTipQuorumMajority).Tip) + assert.Equal(t, int64(1100), PickServedTip([]int64{1100, 1100, 1099}, ServedTipQuorumMajority).Tip) } func TestPickServedTip_Scenario_GarbageCannotInflateLagReference(t *testing.T) { // Old: MaxEligible (the velocity-gated max) kept a rogue far-future tip - // out of the lag gauge — without that, dashboards read "1.8 days behind" - // on healthy chains (the abstract/zora incident). New: Freshest is the + // out of the lag gauge — without that, dashboards read days behind + // on healthy chains (a wrong-chain endpoint). New: Freshest is the // 2nd-highest head, so a single rogue cannot touch the gauge either. - p := PickServedTip(tipsFromInts(999_999_999, 102, 101, 100)) + p := PickServedTip([]int64{999_999_999, 102, 101, 100}, ServedTipQuorumMajority) assert.Equal(t, int64(101), p.Tip) assert.Equal(t, int64(102), p.Freshest, "lag reference ignores the lone rogue") assert.Equal(t, int64(1), p.Freshest-p.Tip, "deliberate lag stays in single digits") } func TestPickServedTip_Scenario_HaltedChainHoldsHonestly(t *testing.T) { - // Old: a halted chain froze the counter (and post-incident, tripped the + // Old: a halted chain froze the counter (and, later, tripped the // stuck watchdog). New: picks over frozen heads keep returning the same // honest consensus value — no invented progress; the advance-age // watchdog still fires at the Network layer. - frozen := tipsFromInts(500, 500, 499) - assert.Equal(t, int64(500), PickServedTip(frozen).Tip) - assert.Equal(t, PickServedTip(frozen), PickServedTip(frozen), "pure function: same inputs, same pick") + frozen := []int64{500, 500, 499} + assert.Equal(t, int64(500), PickServedTip(frozen, ServedTipQuorumMajority).Tip) + assert.Equal(t, PickServedTip(frozen, ServedTipQuorumMajority), PickServedTip(frozen, ServedTipQuorumMajority), "pure function: same inputs, same pick") } diff --git a/common/tracing_core.go b/common/tracing_core.go index 3304d5303..93e343925 100644 --- a/common/tracing_core.go +++ b/common/tracing_core.go @@ -3,6 +3,8 @@ package common import ( "context" "fmt" + "net/url" + "strings" "sync" "github.com/go-logr/zerologr" @@ -205,18 +207,17 @@ func createTracingGRPCExporter(ctx context.Context, cfg *TracingConfig) (*otlptr } func createTracingHTTPExporter(ctx context.Context, cfg *TracingConfig) (*otlptrace.Exporter, error) { - secureOption := otlptracehttp.WithInsecure() - if cfg.TLS != nil && cfg.TLS.Enabled { + tlsEnabled := cfg.TLS != nil && cfg.TLS.Enabled + opts, err := tracingHTTPEndpointOptions(cfg.Endpoint, tlsEnabled) + if err != nil { + return nil, err + } + if tlsEnabled { tlsConfig, err := CreateTLSConfig(cfg.TLS) if err != nil { return nil, err } - secureOption = otlptracehttp.WithTLSClientConfig(tlsConfig) - } - - opts := []otlptracehttp.Option{ - otlptracehttp.WithEndpoint(cfg.Endpoint), - secureOption, + opts = append(opts, otlptracehttp.WithTLSClientConfig(tlsConfig)) } if len(cfg.Headers) > 0 { opts = append(opts, otlptracehttp.WithHeaders(cfg.Headers)) @@ -225,6 +226,38 @@ func createTracingHTTPExporter(ctx context.Context, cfg *TracingConfig) (*otlptr return otlptracehttp.New(ctx, opts...) } +// tracingHTTPEndpointOptions accepts "host:port" or a full URL. WithEndpoint +// takes host:port only; a scheme in it makes every export URL unparsable. +// A URL's scheme decides TLS; host:port falls back to tls.enabled. A URL +// without a path keeps the exporter's default /v1/traces. +func tracingHTTPEndpointOptions(endpoint string, tlsEnabled bool) ([]otlptracehttp.Option, error) { + if !strings.Contains(endpoint, "://") { + opts := []otlptracehttp.Option{otlptracehttp.WithEndpoint(endpoint)} + if !tlsEnabled { + opts = append(opts, otlptracehttp.WithInsecure()) + } + return opts, nil + } + u, err := url.Parse(endpoint) + if err != nil { + return nil, fmt.Errorf("invalid tracing endpoint %q: %w", endpoint, err) + } + if u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + return nil, fmt.Errorf("invalid tracing endpoint %q: want http(s)://host:port[/path] or host:port", endpoint) + } + if u.Scheme == "http" && tlsEnabled { + return nil, fmt.Errorf("invalid tracing endpoint %q: tls.enabled needs an https:// endpoint", endpoint) + } + opts := []otlptracehttp.Option{otlptracehttp.WithEndpoint(u.Host)} + if u.Scheme == "http" { + opts = append(opts, otlptracehttp.WithInsecure()) + } + if u.Path != "" && u.Path != "/" { + opts = append(opts, otlptracehttp.WithURLPath(u.Path)) + } + return opts, nil +} + func createTracingSampler(cfg *TracingConfig) sdktrace.Sampler { var baseSampler sdktrace.Sampler diff --git a/common/tracing_http_exporter_test.go b/common/tracing_http_exporter_test.go new file mode 100644 index 000000000..b5bd61615 --- /dev/null +++ b/common/tracing_http_exporter_test.go @@ -0,0 +1,67 @@ +package common + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/require" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// Every endpoint form the config accepts must reach the receiver; the +// documented "http://host:port" form and the HTTP default once exported +// nothing (erpc/erpc#1276). +func TestTracingHTTPExporter_EndpointForms(t *testing.T) { + var mu sync.Mutex + var paths []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + paths = append(paths, r.URL.Path) + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + hostPort := strings.TrimPrefix(srv.URL, "http://") + + cases := []struct { + endpoint string + wantPath string + }{ + {hostPort, "/v1/traces"}, + {srv.URL, "/v1/traces"}, + {srv.URL + "/", "/v1/traces"}, + {srv.URL + "/otlp/v1/traces", "/otlp/v1/traces"}, + } + for _, tc := range cases { + t.Run(tc.endpoint, func(t *testing.T) { + mu.Lock() + paths = nil + mu.Unlock() + + ctx := context.Background() + exp, err := createTracingHTTPExporter(ctx, &TracingConfig{Endpoint: tc.endpoint}) + require.NoError(t, err) + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) + _, span := tp.Tracer("test").Start(ctx, "op") + span.End() + require.NoError(t, tp.Shutdown(ctx)) + + mu.Lock() + defer mu.Unlock() + require.Equal(t, []string{tc.wantPath}, paths) + }) + } + + for _, bad := range []string{"ftp://x:1", "http://", "http://x:1"} { + cfg := &TracingConfig{Endpoint: bad} + if bad == "http://x:1" { + cfg.TLS = &TLSConfig{Enabled: true} + } + _, err := createTracingHTTPExporter(context.Background(), cfg) + require.Error(t, err, bad) + } +} diff --git a/common/upstream_fake.go b/common/upstream_fake.go index b3f6c9c1a..2272e9e2d 100644 --- a/common/upstream_fake.go +++ b/common/upstream_fake.go @@ -301,6 +301,8 @@ func (p *FakeEvmStatePoller) SuggestLatestBlock(blockNumber int64) { p.latestBlockNumber = blockNumber } +func (p *FakeEvmStatePoller) RefreshLatestBlock() {} + func (p *FakeEvmStatePoller) SyncingState() EvmSyncingState { return EvmSyncingStateUnknown } diff --git a/common/validation.go b/common/validation.go index 895e97e01..f300f298e 100644 --- a/common/validation.go +++ b/common/validation.go @@ -122,6 +122,9 @@ func (s *ServerConfig) Validate() error { if ws.SubscriptionBufferSize < 1 { return fmt.Errorf("server.webSocket.subscriptionBufferSize must be at least 1") } + if ws.MaxConnections < 0 || ws.MaxConnectionsPerIP < 0 || ws.MaxSubscriptions < 0 { + return fmt.Errorf("server.webSocket.maxConnections, maxConnectionsPerIP and maxSubscriptions must not be negative") + } } // Validate trusted IP forwarders if provided (IPs or CIDRs). Support legacy + new field @@ -214,12 +217,26 @@ func (r *RateLimiterConfig) Validate() error { if r.Store.NearLimitRatio != 0 && (r.Store.NearLimitRatio <= 0 || r.Store.NearLimitRatio >= 1) { return fmt.Errorf("rateLimiters.store.nearLimitRatio must be > 0 and < 1") } + switch r.Store.OnFailure { + case "", RateLimitStoreFailureAllow, RateLimitStoreFailureDeny: + if r.Store.LocalLimitFactor != 0 { + return fmt.Errorf("rateLimiters.store.localLimitFactor is only used with onFailure: %s", RateLimitStoreFailureLocal) + } + case RateLimitStoreFailureLocal: + if r.Store.LocalLimitFactor <= 0 || r.Store.LocalLimitFactor > 1 { + return fmt.Errorf("rateLimiters.store.localLimitFactor must be > 0 and <= 1 when onFailure is %s", RateLimitStoreFailureLocal) + } + default: + return fmt.Errorf("rateLimiters.store.onFailure '%s' is invalid, must be one of: %s, %s, %s", r.Store.OnFailure, RateLimitStoreFailureAllow, RateLimitStoreFailureDeny, RateLimitStoreFailureLocal) + } // Validate redis connector if err := r.Store.Redis.Validate(); err != nil { return fmt.Errorf("rateLimiters.store.redis is invalid: %w", err) } case "memory": - // No validation for memory store + if r.Store.OnFailure != "" || r.Store.LocalLimitFactor != 0 { + return fmt.Errorf("rateLimiters.store.onFailure and localLimitFactor apply only to driver: redis") + } case "": fallthrough default: @@ -1177,6 +1194,19 @@ func (e *EvmUpstreamConfig) Validate(u *UpstreamConfig) error { return err } } + if m := e.Multicall3; m != nil { + if m.MaxWait <= 0 { + return fmt.Errorf("upstream.*.evm.multicall3.maxWait must be greater than 0") + } + if m.MaxCalls < 2 { + return fmt.Errorf("upstream.*.evm.multicall3.maxCalls must be at least 2") + } + for _, a := range m.BypassContracts { + if raw, err := HexToBytes(a); err != nil || len(raw) != 20 { + return fmt.Errorf("upstream.*.evm.multicall3.bypassContracts entry '%s' is not a 20-byte hex address", a) + } + } + } return nil } @@ -1666,29 +1696,18 @@ func (e *EvmNetworkConfig) Validate() error { if e.GetLogsMaxAllowedRange == 0 { return fmt.Errorf("network.*.evm.getLogsMaxAllowedRange must be greater than 0") } - if e.ServedTip != nil { - for _, tag := range e.ServedTip.EnabledFor { - switch strings.ToLower(strings.TrimSpace(tag)) { - case "latest", "finalized", "safe": - default: - return fmt.Errorf("network.*.evm.servedTip.enabledFor contains unknown tag %q (valid: latest, finalized, safe)", tag) - } + for method := range e.LocalAnswers { + if !slices.Contains(LocalAnswerMethods(), method) { + return fmt.Errorf("network.*.evm.localAnswers.%s: eRPC has no local answer for this method; one of %v", method, LocalAnswerMethods()) } - if e.ServedTip.ClusterDelta < 0 { - return fmt.Errorf("network.*.evm.servedTip.clusterDelta must be >= 0 (0 auto-derives from block time)") + } + if e.ServedTip != nil { + if q := e.ServedTip.Quorum; q < 0 && q != ServedTipQuorumMajority { + return fmt.Errorf("network.*.evm.servedTip.quorum must be an integer >= 1 or \"majority\"") } if e.ServedTip.MaxRegressionBlocks < -1 { return fmt.Errorf("network.*.evm.servedTip.maxRegressionBlocks must be >= 0, or -1 to disable the regression guard (0 uses the default rollback tolerance)") } - if w := e.ServedTip.TrajectoryWindow; w != nil && w.Duration() != 0 { - if d := w.Duration(); d < MinServedTipTrajectoryWindow || d > MaxServedTipTrajectoryWindow { - return fmt.Errorf( - "network.*.evm.servedTip.trajectoryWindow must be 0 (disables the trajectory referee) "+ - "or between %s and %s, got %s — NOTE a bare number in YAML is parsed as MILLISECONDS, "+ - "so write trajectoryWindow: \"10m\", not trajectoryWindow: 10", - MinServedTipTrajectoryWindow, MaxServedTipTrajectoryWindow, d) - } - } for _, m := range e.ServedTip.GuaranteedMethods { if err := ValidatePattern(m); err != nil { return fmt.Errorf("network.*.evm.servedTip.guaranteedMethods has invalid pattern %q: %w", m, err) @@ -1844,6 +1863,12 @@ func (b *SvmAvailabilityBoundConfig) Validate() error { if b.FirstAvailableSlotPlus != nil && *b.FirstAvailableSlotPlus < 0 { return fmt.Errorf("bound.firstAvailableSlotPlus must be >= 0") } + if b.UpdateRate < 0 { + return fmt.Errorf("bound.updateRate must be >= 0") + } + if b.UpdateRate != 0 && b.FirstAvailableSlotPlus == nil { + return fmt.Errorf("bound.updateRate applies only to firstAvailableSlotPlus") + } return nil } diff --git a/common/validation_servedtip_test.go b/common/validation_servedtip_test.go index 45812d691..78b314ad7 100644 --- a/common/validation_servedtip_test.go +++ b/common/validation_servedtip_test.go @@ -1,8 +1,9 @@ package common import ( + "bytes" + "encoding/json" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -23,28 +24,6 @@ func TestEvmNetworkConfig_Validate_ServedTip(t *testing.T) { require.NoError(t, baseValidEvmNetworkConfig().Validate()) }) - t.Run("valid tags pass (case-insensitive incl. safe)", func(t *testing.T) { - e := baseValidEvmNetworkConfig() - e.ServedTip = &EvmServedTipConfig{EnabledFor: []string{"latest", "Finalized", "safe"}} - require.NoError(t, e.Validate()) - }) - - t.Run("unknown tag rejected", func(t *testing.T) { - e := baseValidEvmNetworkConfig() - e.ServedTip = &EvmServedTipConfig{EnabledFor: []string{"lastest"}} // typo - err := e.Validate() - require.Error(t, err) - assert.Contains(t, err.Error(), "enabledFor") - }) - - t.Run("negative clusterDelta rejected", func(t *testing.T) { - e := baseValidEvmNetworkConfig() - e.ServedTip = &EvmServedTipConfig{ClusterDelta: -1} - err := e.Validate() - require.Error(t, err) - assert.Contains(t, err.Error(), "clusterDelta") - }) - t.Run("maxRegressionBlocks below -1 rejected", func(t *testing.T) { e := baseValidEvmNetworkConfig() e.ServedTip = &EvmServedTipConfig{MaxRegressionBlocks: -2} @@ -55,20 +34,13 @@ func TestEvmNetworkConfig_Validate_ServedTip(t *testing.T) { t.Run("maxRegressionBlocks -1 disables the guard", func(t *testing.T) { e := baseValidEvmNetworkConfig() - e.ServedTip = &EvmServedTipConfig{EnabledFor: []string{"latest"}, MaxRegressionBlocks: -1} - require.NoError(t, e.Validate()) - }) - - t.Run("zero maxRegressionBlocks means the default tolerance", func(t *testing.T) { - e := baseValidEvmNetworkConfig() - e.ServedTip = &EvmServedTipConfig{EnabledFor: []string{"latest"}} + e.ServedTip = &EvmServedTipConfig{MaxRegressionBlocks: -1} require.NoError(t, e.Validate()) }) t.Run("guaranteedFor accepts id and tag selectors", func(t *testing.T) { e := baseValidEvmNetworkConfig() e.ServedTip = &EvmServedTipConfig{ - EnabledFor: []string{"latest"}, GuaranteedFor: []string{"type:internal", "int*", "!type:external"}, } require.NoError(t, e.Validate()) @@ -77,7 +49,6 @@ func TestEvmNetworkConfig_Validate_ServedTip(t *testing.T) { t.Run("guaranteedFor rejects an invalid selector", func(t *testing.T) { e := baseValidEvmNetworkConfig() e.ServedTip = &EvmServedTipConfig{ - EnabledFor: []string{"latest"}, GuaranteedFor: []string{"(type:internal"}, } err := e.Validate() @@ -85,69 +56,69 @@ func TestEvmNetworkConfig_Validate_ServedTip(t *testing.T) { assert.Contains(t, err.Error(), "guaranteedFor") }) - t.Run("empty guaranteedFor is the default", func(t *testing.T) { + t.Run("empty servedTip is the default", func(t *testing.T) { e := baseValidEvmNetworkConfig() - e.ServedTip = &EvmServedTipConfig{EnabledFor: []string{"latest"}} + e.ServedTip = &EvmServedTipConfig{} require.NoError(t, e.Validate()) assert.Empty(t, e.ServedTip.GuaranteedFor) }) - t.Run("zero trajectoryWindow disables the referee", func(t *testing.T) { + t.Run("quorum below -1 rejected", func(t *testing.T) { e := baseValidEvmNetworkConfig() - e.ServedTip = &EvmServedTipConfig{EnabledFor: []string{"latest"}, TrajectoryWindow: Duration(0).Ptr()} - require.NoError(t, e.Validate()) + e.ServedTip = &EvmServedTipConfig{Quorum: -2} + err := e.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "quorum") }) - // A window outside the range the mechanism can honour is a configuration - // that LOOKS enabled and can never act: below a minute a fit is noise, and - // beyond what the sample ring can span the referee stands down forever while - // nothing in the metrics says so. - t.Run("trajectoryWindow range", func(t *testing.T) { - for _, c := range []struct { - name string - window Duration - valid bool - }{ - {"the default", Duration(DefaultServedTipTrajectoryWindow), true}, - {"exactly the minimum", Duration(MinServedTipTrajectoryWindow), true}, - {"exactly the maximum", Duration(MaxServedTipTrajectoryWindow), true}, - {"one second below the minimum", Duration(MinServedTipTrajectoryWindow - time.Second), false}, - {"one second above the maximum", Duration(MaxServedTipTrajectoryWindow + time.Second), false}, - {"negative", Duration(-1), false}, - {"5 milliseconds (a bare YAML 5)", Duration(5 * time.Millisecond), false}, - } { - t.Run(c.name, func(t *testing.T) { - e := baseValidEvmNetworkConfig() - e.ServedTip = &EvmServedTipConfig{EnabledFor: []string{"latest"}, TrajectoryWindow: c.window.Ptr()} - err := e.Validate() - if c.valid { - require.NoError(t, err) - return - } - require.Error(t, err) - assert.Contains(t, err.Error(), "trajectoryWindow") - assert.Contains(t, err.Error(), "MILLISECONDS", - "the error must name the trap it exists for: a bare YAML number is milliseconds") - }) + // End to end through the YAML and JSON an operator actually writes, with + // the config loader's strict decoding. + t.Run("quorum YAML/JSON", func(t *testing.T) { + decode := func(src string) (*EvmServedTipConfig, error) { + var st EvmServedTipConfig + dec := yaml.NewDecoder(bytes.NewBufferString(src)) + dec.KnownFields(true) + return &st, dec.Decode(&st) } - }) - // End to end through the YAML an operator actually writes. - t.Run("trajectoryWindow YAML round-trip", func(t *testing.T) { - e := baseValidEvmNetworkConfig() - require.NoError(t, yaml.Unmarshal([]byte("enabledFor: [latest]\ntrajectoryWindow: \"10m\"\n"), &e.ServedTip)) - require.Equal(t, 10*time.Minute, e.ServedTip.TrajectoryWindow.Duration()) - require.NoError(t, e.Validate()) + st, err := decode("quorum: 3\n") + require.NoError(t, err) + assert.Equal(t, ServedTipQuorum(3), st.Quorum) + assert.Equal(t, 3, st.Quorum.Of(5)) + assert.Equal(t, 2, st.Quorum.Of(2), "a ballot smaller than the quorum serves what all of it has") + + st, err = decode("quorum: majority\n") + require.NoError(t, err) + assert.Equal(t, ServedTipQuorumMajority, st.Quorum) + assert.Equal(t, 3, st.Quorum.Of(5)) + + st, err = decode("guaranteedFor: [\"type:internal\"]\n") + require.NoError(t, err) + assert.Equal(t, DefaultServedTipQuorum, st.Quorum.Of(5), "unset is the default quorum") + + for _, bad := range []string{"quorum: 0\n", "quorum: -1\n", "quorum: half\n"} { + _, err = decode(bad) + require.Error(t, err, bad) + assert.Contains(t, err.Error(), "quorum", bad) + } + for _, removed := range []string{"enabledFor: [latest]\n", "clusterDelta: 3\n", "trajectoryWindow: 10m\n"} { + _, err = decode(removed) + require.Error(t, err, "removed key must be rejected: %s", removed) + } - // A bare number is the trap the error message names. The config loader's - // yaml.v3 rejects it outright here; a Duration that reaches the config - // any other way (a JSON tool, yaml.v2, a programmatic literal) carries - // the millisecond reading instead, which is what the range check above - // catches — 5 means 5ms, never 5 minutes, on every path. - bare := baseValidEvmNetworkConfig() - err := yaml.Unmarshal([]byte("enabledFor: [latest]\ntrajectoryWindow: 5\n"), &bare.ServedTip) - require.Error(t, err, "a bare number must never be read as 5 minutes") - assert.Contains(t, err.Error(), "duration") + out, err := yaml.Marshal(&EvmServedTipConfig{Quorum: ServedTipQuorumMajority}) + require.NoError(t, err) + assert.Equal(t, "quorum: majority\n", string(out)) + + var js EvmServedTipConfig + require.NoError(t, json.Unmarshal([]byte(`{"quorum":"majority"}`), &js)) + assert.Equal(t, ServedTipQuorumMajority, js.Quorum) + require.NoError(t, json.Unmarshal([]byte(`{"quorum":4}`), &js)) + assert.Equal(t, ServedTipQuorum(4), js.Quorum) + require.Error(t, json.Unmarshal([]byte(`{"quorum":0}`), &js)) + raw, err := json.Marshal(&EvmServedTipConfig{Quorum: ServedTipQuorumMajority}) + require.NoError(t, err) + assert.JSONEq(t, `{"quorum":"majority"}`, string(raw)) }) } @@ -157,7 +128,7 @@ func TestEvmNetworkConfig_Validate_ServedTip(t *testing.T) { func TestNetworkConfig_SetDefaults_InheritsServedTip(t *testing.T) { defaults := &NetworkDefaults{ Evm: &EvmNetworkConfig{ - ServedTip: &EvmServedTipConfig{EnabledFor: []string{"latest"}}, + ServedTip: &EvmServedTipConfig{Quorum: ServedTipQuorumMajority}, }, } @@ -165,29 +136,13 @@ func TestNetworkConfig_SetDefaults_InheritsServedTip(t *testing.T) { n := &NetworkConfig{Architecture: ArchitectureEvm, Evm: &EvmNetworkConfig{ChainId: 1}} require.NoError(t, n.SetDefaults(nil, defaults)) require.NotNil(t, n.Evm.ServedTip, "must inherit servedTip from networkDefaults") - assert.True(t, n.Evm.ServedTipEnabledFor("latest")) + assert.Equal(t, ServedTipQuorumMajority, n.Evm.ServedTipQuorum()) // A network that sets its own servedTip keeps it (default does not clobber). n2 := &NetworkConfig{Architecture: ArchitectureEvm, Evm: &EvmNetworkConfig{ ChainId: 1, - ServedTip: &EvmServedTipConfig{EnabledFor: []string{"finalized"}}, + ServedTip: &EvmServedTipConfig{Quorum: 3}, }} require.NoError(t, n2.SetDefaults(nil, defaults)) - assert.False(t, n2.Evm.ServedTipEnabledFor("latest")) - assert.True(t, n2.Evm.ServedTipEnabledFor("finalized"), "explicit network servedTip must win") -} - -func TestServedTipEnabledFor_SafeFollowsFinalized(t *testing.T) { - safe := &EvmNetworkConfig{ServedTip: &EvmServedTipConfig{EnabledFor: []string{"safe"}}} - assert.True(t, safe.ServedTipEnabledFor("finalized"), "safe enables the finalized axis") - assert.False(t, safe.ServedTipEnabledFor("latest")) - - latest := &EvmNetworkConfig{ServedTip: &EvmServedTipConfig{EnabledFor: []string{"latest"}}} - assert.True(t, latest.ServedTipEnabledFor("latest")) - assert.False(t, latest.ServedTipEnabledFor("finalized")) - - // Nil-receiver / nil config safe. - var nilCfg *EvmNetworkConfig - assert.False(t, nilCfg.ServedTipEnabledFor("latest")) - assert.False(t, (&EvmNetworkConfig{}).ServedTipEnabledFor("latest")) + assert.Equal(t, ServedTipQuorum(3), n2.Evm.ServedTipQuorum(), "explicit network servedTip must win") } diff --git a/consensus/misbehavior_test.go b/consensus/misbehavior_test.go new file mode 100644 index 000000000..6b409bbd7 --- /dev/null +++ b/consensus/misbehavior_test.go @@ -0,0 +1,560 @@ +package consensus + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// capturingExporter records every exported misbehavior line. +type capturingExporter struct { + mu sync.Mutex + lines [][]byte + meta [][2]string + err error +} + +func (c *capturingExporter) AppendWithMetadata(line []byte, method, networkId string) error { + c.mu.Lock() + defer c.mu.Unlock() + c.lines = append(c.lines, append([]byte(nil), line...)) + c.meta = append(c.meta, [2]string{method, networkId}) + return c.err +} + +func (c *capturingExporter) records(t *testing.T) []misbehaviorRecord { + t.Helper() + c.mu.Lock() + defer c.mu.Unlock() + out := make([]misbehaviorRecord, 0, len(c.lines)) + for _, l := range c.lines { + var rec misbehaviorRecord + require.NoError(t, json.Unmarshal(l, &rec)) + out = append(out, rec) + } + return out +} + +func newPunishingExecutor(t *testing.T, cfg *config, exp misbehaviorExporter, level zerolog.Level) *executor { + t.Helper() + lg := zerolog.New(zerolog.NewTestWriter(t)).Level(level) + return &executor{consensusPolicy: &consensusPolicy{ + config: cfg, + logger: &lg, + misbehavingUpstreamsLimiter: &sync.Map{}, + misbehavingUpstreamsSitoutTimer: &sync.Map{}, + disputeLogLevel: zerolog.WarnLevel, + exporter: exp, + }} +} + +func cordonState(u common.Upstream) (string, bool) { + return u.(*common.FakeUpstream).CordonedReason() +} + +func misbehaviorCount(u common.Upstream) int { + return u.Tracker().(*common.FakeHealthTracker).MisbehaviorCount +} + +func trackRound(t *testing.T, e *executor, labels metricsLabels, responses []*execResult) *slotResult { + t.Helper() + req := newTestRequest() + ctx := context.WithValue(context.Background(), common.RequestContextKey, req) + lg := zerolog.Nop() + analysis := newConsensusAnalysis(&lg, ctx, e.config, responses) + winner := e.determineWinner(&lg, analysis) + e.trackAndPunishMisbehavingUpstreams(&lg, req, labels, winner, analysis) + return winner +} + +func TestMisbehavior_DissenterIsRecordedAndPunishedAfterThreshold(t *testing.T) { + cfg := &config{ + maxParticipants: 3, + agreementThreshold: 2, + punishMisbehavior: &common.PunishMisbehaviorConfig{ + DisputeThreshold: 1, + DisputeWindow: common.Duration(time.Hour), + SitOutPenalty: common.Duration(time.Hour), + }, + } + e := newPunishingExecutor(t, cfg, nil, zerolog.Disabled) + a := taggedUpstream("good-a") + b := taggedUpstream("good-b") + bad := taggedUpstream("bad-c") + labels := metricsLabels{projectId: "p-punish", networkId: "evm:1", method: "eth_getBalance", category: "eth_getBalance", + finalityStr: "unfinalized", finality: common.DataFinalityStateUnfinalized, userId: "u", agentName: "ag"} + punished := telemetry.MetricConsensusUpstreamPunished.WithLabelValues("p-punish", "evm:1", "bad-c", "u", "ag") + before := testutil.ToFloat64(punished) + + round := func() { + w := trackRound(t, e, labels, []*execResult{ + resultFrom(t, a, "0xaa", 0), + resultFrom(t, b, "0xaa", 1), + resultFrom(t, bad, "0xbb", 2), + }) + require.NoError(t, w.Error) + } + + // First dispute consumes the single burst token: recorded, not punished. + round() + assert.Equal(t, 1, misbehaviorCount(bad)) + assert.Equal(t, 0, misbehaviorCount(a), "agreeing upstreams are never recorded") + tr := bad.Tracker().(*common.FakeHealthTracker) + assert.Equal(t, "eth_getBalance", tr.LastMisbehaviorMethod) + assert.Equal(t, common.DataFinalityStateUnfinalized, tr.LastMisbehaviorFinality) + _, cordoned := cordonState(bad) + assert.False(t, cordoned, "within the dispute budget the upstream keeps serving") + + // Second dispute exceeds the budget: cordoned on every method. + round() + reason, cordoned := cordonState(bad) + assert.True(t, cordoned) + assert.Equal(t, "misbehaving in consensus", reason) + assert.Equal(t, before+1, testutil.ToFloat64(punished)) + + // Third dispute while sitting out does not punish again. + round() + assert.Equal(t, before+1, testutil.ToFloat64(punished), "an upstream already sitting out is not re-punished") + assert.Equal(t, 3, misbehaviorCount(bad)) +} + +func TestMisbehavior_SitOutPenaltyExpiresAndUncordons(t *testing.T) { + cfg := &config{ + maxParticipants: 3, + agreementThreshold: 2, + punishMisbehavior: &common.PunishMisbehaviorConfig{ + DisputeThreshold: 1, + DisputeWindow: common.Duration(time.Hour), + SitOutPenalty: common.Duration(20 * time.Millisecond), + }, + } + e := newPunishingExecutor(t, cfg, nil, zerolog.Disabled) + a := taggedUpstream("good-a") + b := taggedUpstream("good-b") + bad := taggedUpstream("bad-c") + for i := 0; i < 2; i++ { + trackRound(t, e, metricsLabels{method: "eth_getBalance"}, []*execResult{ + resultFrom(t, a, "0xaa", 0), + resultFrom(t, b, "0xaa", 1), + resultFrom(t, bad, "0xbb", 2), + }) + } + _, cordoned := cordonState(bad) + require.True(t, cordoned) + + require.Eventually(t, func() bool { + _, c := cordonState(bad) + _, stillSitting := e.misbehavingUpstreamsSitoutTimer.Load("bad-c") + return !c && !stillSitting + }, 5*time.Second, 5*time.Millisecond, "penalty must end with the upstream uncordoned and forgotten") +} + +func TestMisbehavior_PunishmentGates(t *testing.T) { + a := taggedUpstream("ups-a") + b := taggedUpstream("ups-b") + c := taggedUpstream("ups-c") + d := taggedUpstream("ups-d") + + cases := []struct { + name string + punish *common.PunishMisbehaviorConfig + resps func(t *testing.T) []*execResult + }{ + {"punishment not configured", nil, nil}, + {"zero dispute threshold disables punishment", + &common.PunishMisbehaviorConfig{DisputeThreshold: 0, DisputeWindow: common.Duration(time.Hour)}, nil}, + {"non-positive dispute window disables punishment", + &common.PunishMisbehaviorConfig{DisputeThreshold: 1, DisputeWindow: 0}, nil}, + {"winner without a strict majority of valid participants is not enforced", + &common.PunishMisbehaviorConfig{DisputeThreshold: 1, DisputeWindow: common.Duration(time.Hour), SitOutPenalty: common.Duration(time.Hour)}, + func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xaa", 0), + resultFrom(t, b, "0xaa", 1), + resultFrom(t, c, "0xbb", 2), + resultFrom(t, d, "0xcc", 3), + } + }}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + cfg := &config{maxParticipants: 4, agreementThreshold: 2, punishMisbehavior: tt.punish, + disputeBehavior: common.ConsensusDisputeBehaviorAcceptMostCommonValidResult} + e := newPunishingExecutor(t, cfg, nil, zerolog.Disabled) + bad := taggedUpstream("bad") + resps := tt.resps + if resps == nil { + resps = func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xaa", 0), + resultFrom(t, b, "0xaa", 1), + resultFrom(t, bad, "0xbb", 2), + } + } + } + for i := 0; i < 3; i++ { + trackRound(t, e, metricsLabels{method: "eth_getBalance"}, resps(t)) + } + for _, u := range []common.Upstream{bad, c, d} { + _, cordoned := cordonState(u) + assert.False(t, cordoned, "%s must not be cordoned", u.Id()) + } + _, limiterCreated := e.misbehavingUpstreamsLimiter.Load("bad") + assert.False(t, limiterCreated, "no dispute limiter is created while punishment is gated off") + }) + } +} + +func TestMisbehavior_ErrorsAreCountedNotPunished(t *testing.T) { + cfg := &config{ + maxParticipants: 4, + agreementThreshold: 2, + punishMisbehavior: &common.PunishMisbehaviorConfig{ + DisputeThreshold: 1, DisputeWindow: common.Duration(time.Hour), SitOutPenalty: common.Duration(time.Hour), + }, + } + e := newPunishingExecutor(t, cfg, nil, zerolog.Disabled) + a := taggedUpstream("ups-a") + b := taggedUpstream("ups-b") + reverting := taggedUpstream("ups-revert") + flaky := taggedUpstream("ups-flaky") + labels := metricsLabels{projectId: "p-errs", networkId: "evm:1", category: "eth_call", finalityStr: "unknown", method: "eth_call"} + + errCount := func(ups, rtype, code string) float64 { + return testutil.ToFloat64(telemetry.MetricConsensusUpstreamErrors.WithLabelValues( + "p-errs", "evm:1", ups, "eth_call", "unknown", rtype, code, "", "")) + } + revertBefore := errCount("ups-revert", "consensus_error", "ErrEndpointExecutionException") + timeoutBefore := errCount("ups-flaky", "infrastructure_error", "timeout") + + for i := 0; i < 3; i++ { + trackRound(t, e, labels, []*execResult{ + resultFrom(t, a, "0xaa", 0), + resultFrom(t, b, "0xaa", 1), + errorFrom(reverting, revertedErr(), 2), + errorFrom(flaky, errors.New("i/o timeout"), 3), + }) + } + + assert.Equal(t, revertBefore+3, errCount("ups-revert", "consensus_error", "ErrEndpointExecutionException")) + assert.Equal(t, timeoutBefore+3, errCount("ups-flaky", "infrastructure_error", "timeout")) + for _, u := range []common.Upstream{reverting, flaky} { + assert.Equal(t, 0, misbehaviorCount(u), "errors are not data misbehavior") + _, cordoned := cordonState(u) + assert.False(t, cordoned) + } +} + +func TestMisbehavior_UpstreamErrorCodeLabels(t *testing.T) { + ups := taggedUpstream("ups-x") + cases := []struct { + name string + err error + code string + }{ + {"missing data", missingDataFrom(ups, "header not found"), "ErrEndpointMissingData"}, + {"server side", common.NewErrEndpointServerSideException(errors.New("boom"), nil, 500), "ErrEndpointServerSideException"}, + {"other standard error", common.NewErrUpstreamsExhaustedWithCause(errors.New("x")), string(common.ErrCodeUpstreamsExhausted)}, + {"block not found text", errors.New("block not found"), "block_not_found"}, + {"opaque error fingerprint", errors.New("weird failure"), common.ErrorFingerprint(errors.New("weird failure"))}, + } + for i, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + project := "p-codes-" + string(rune('a'+i)) + cfg := &config{maxParticipants: 3, agreementThreshold: 2} + e := newPunishingExecutor(t, cfg, nil, zerolog.Disabled) + labels := metricsLabels{projectId: project, networkId: "n", category: "m", finalityStr: "f", method: "m"} + // response_type label depends on classification; sum both possible + // values. Counters are process-global, so assert the delta. + count := func() float64 { + return testutil.ToFloat64(telemetry.MetricConsensusUpstreamErrors.WithLabelValues(project, "n", "ups-x", "m", "f", + ResponseTypeConsensusError.String(), tt.code, "", "")) + + testutil.ToFloat64(telemetry.MetricConsensusUpstreamErrors.WithLabelValues(project, "n", "ups-x", "m", "f", + ResponseTypeInfrastructureError.String(), tt.code, "", "")) + } + before := count() + trackRound(t, e, labels, []*execResult{ + resultFrom(t, taggedUpstream("ups-a"), "0xaa", 0), + resultFrom(t, taggedUpstream("ups-b"), "0xaa", 1), + errorFrom(ups, tt.err, 2), + }) + assert.Equal(t, before+1, count(), "error must be labelled with code %q", tt.code) + }) + } +} + +func TestMisbehavior_AllInfrastructureErrorsSkipTracking(t *testing.T) { + exp := &capturingExporter{} + cfg := &config{maxParticipants: 2, agreementThreshold: 2} + e := newPunishingExecutor(t, cfg, exp, zerolog.Disabled) + a := taggedUpstream("ups-a") + b := taggedUpstream("ups-b") + trackRound(t, e, metricsLabels{projectId: "p-infra"}, []*execResult{ + errorFrom(a, errors.New("dial timeout"), 0), + errorFrom(b, errors.New("connection refused"), 1), + }) + assert.Empty(t, exp.records(t)) +} + +func TestMisbehavior_ExportsFullRecordForDisagreement(t *testing.T) { + exp := &capturingExporter{} + cfg := &config{maxParticipants: 4, agreementThreshold: 2, disputeBehavior: common.ConsensusDisputeBehaviorReturnError, + ignoreFields: map[string][]string{"eth_getLogs": {"blockTimestamp"}}} + // Debug logger so the dispute log path (warn level) is emitted too. + e := newPunishingExecutor(t, cfg, exp, zerolog.DebugLevel) + + a := common.NewFakeUpstream("ups-a") + a.Config().VendorName = "alchemy" + b := taggedUpstream("ups-b") + empty := taggedUpstream("ups-empty") + failing := taggedUpstream("ups-fail") + + labels := metricsLabels{projectId: "p-export", networkId: "evm:1", method: "eth_getLogs", category: "eth_getLogs", + finalityStr: "finalized", userId: "user-1"} + winner := trackRound(t, e, labels, []*execResult{ + resultFrom(t, a, "0xaa", 0), + resultFrom(t, b, "0xaa", 1), + resultOf(t, empty, []interface{}{}, 2), + errorFrom(failing, errors.New("dial timeout"), 3), + }) + require.NoError(t, winner.Error) + + recs := exp.records(t) + require.Len(t, recs, 1, "one record per round with misbehavior") + assert.Equal(t, [2]string{"eth_getLogs", "evm:1"}, exp.meta[0]) + rec := recs[0] + assert.Equal(t, "p-export", rec.ProjectID) + assert.Equal(t, "user-1", rec.UserId) + assert.Equal(t, "finalized", rec.Finality) + assert.Equal(t, 4, rec.Policy.MaxParticipants) + assert.Equal(t, 2, rec.Policy.AgreementThreshold) + assert.Equal(t, common.ConsensusDisputeBehaviorReturnError, rec.Policy.DisputeBehavior) + assert.Equal(t, []string{"blockTimestamp"}, rec.Policy.IgnoreFields["eth_getLogs"]) + assert.Contains(t, string(rec.Request), `"eth_getLogs"`) + + assert.Equal(t, "non_empty", rec.Winner.ResponseType) + assert.NotEmpty(t, rec.Winner.Hash) + assert.Positive(t, rec.Winner.Size) + assert.Equal(t, 4, rec.Analysis.TotalParticipants) + assert.Equal(t, 3, rec.Analysis.ValidParticipants) + require.NotNil(t, rec.Analysis.BestByCount) + assert.Equal(t, 2, rec.Analysis.BestByCount.Count) + assert.Len(t, rec.Analysis.Groups, 3) + + byID := map[string]participantSnapshot{} + for _, p := range rec.Participants { + byID[p.UpstreamID] = p + } + require.Len(t, byID, 4) + assert.Equal(t, "alchemy", byID["ups-a"].Vendor) + assert.JSONEq(t, `"0xaa"`, string(byID["ups-a"].Response)) + assert.Equal(t, "empty", byID["ups-empty"].ResponseType) + assert.JSONEq(t, `[]`, string(byID["ups-empty"].Response)) + assert.Equal(t, "infrastructure_error", byID["ups-fail"].ResponseType) + assert.Contains(t, string(byID["ups-fail"].Response), "dial timeout", + "errored participants carry the error as their body") +} + +func TestMisbehavior_ExportRecordsErrorWinnerAndSurvivesExporterFailure(t *testing.T) { + exp := &capturingExporter{err: errors.New("disk full")} + cfg := &config{maxParticipants: 3, agreementThreshold: 2} + e := newPunishingExecutor(t, cfg, exp, zerolog.Disabled) + + // Two empties agree; a non-empty dissents. Winner is the empty result. + winner := trackRound(t, e, metricsLabels{method: "eth_getLogs"}, []*execResult{ + resultOf(t, taggedUpstream("ups-a"), []interface{}{}, 0), + resultOf(t, taggedUpstream("ups-b"), []interface{}{}, 1), + resultFrom(t, taggedUpstream("ups-c"), "0xdata", 2), + }) + require.NoError(t, winner.Error) + recs := exp.records(t) + require.Len(t, recs, 1, "the record is attempted even when the exporter fails") + assert.Equal(t, "empty", recs[0].Winner.ResponseType) + + // Synthesized error winner (no group holds it) uses the best data group as + // the consensus reference and records the error as the winner. + lg := zerolog.Nop() + req := newTestRequest() + ctx := context.WithValue(context.Background(), common.RequestContextKey, req) + analysis := newConsensusAnalysis(&lg, ctx, cfg, []*execResult{ + resultFrom(t, taggedUpstream("ups-a"), "0xaa", 0), + resultFrom(t, taggedUpstream("ups-b"), "0xaa", 1), + resultFrom(t, taggedUpstream("ups-c"), "0xbb", 2), + }) + errWinner := &slotResult{Error: common.NewErrConsensusDispute("forced", nil, nil)} + e.trackAndPunishMisbehavingUpstreams(&lg, req, metricsLabels{method: "eth_getLogs"}, errWinner, analysis) + recs = exp.records(t) + require.Len(t, recs, 2) + assert.Equal(t, "consensus_error", recs[1].Winner.ResponseType) + assert.Equal(t, string(common.ErrCodeConsensusDispute), recs[1].Winner.Hash) +} + +func TestFileMisbehaviorExporter_Lifecycle(t *testing.T) { + lg := zerolog.Nop() + + t.Run("rejects empty and relative paths", func(t *testing.T) { + _, err := newFileMisbehaviorExporter(nil, &lg) + require.Error(t, err) + _, err = newFileMisbehaviorExporter(&common.MisbehaviorsDestinationConfig{}, &lg) + require.Error(t, err) + _, err = newFileMisbehaviorExporter(&common.MisbehaviorsDestinationConfig{Path: "relative/dir"}, &lg) + require.ErrorContains(t, err, "must be absolute") + }) + + t.Run("fails when the base path cannot be created", func(t *testing.T) { + blocker := filepath.Join(t.TempDir(), "file") + require.NoError(t, os.WriteFile(blocker, []byte("x"), 0o600)) + _, err := newFileMisbehaviorExporter(&common.MisbehaviorsDestinationConfig{Path: filepath.Join(blocker, "sub")}, &lg) + require.Error(t, err) + }) + + t.Run("appends one line per record per resolved file and closes handles", func(t *testing.T) { + dir := t.TempDir() + exp, err := newFileMisbehaviorExporter(&common.MisbehaviorsDestinationConfig{ + Path: dir, FilePattern: "{method}-{networkId}", + }, &lg) + require.NoError(t, err) + require.NoError(t, exp.AppendWithMetadata([]byte(`{"n":1}`), "eth_call", "evm:1")) + require.NoError(t, exp.AppendWithMetadata([]byte(`{"n":2}`), "eth_call", "evm:1")) + require.NoError(t, exp.AppendWithMetadata([]byte(`{"n":3}`), "eth_getLogs", "evm:1")) + require.NoError(t, exp.Close()) + assert.Empty(t, exp.fileHandles) + + readLines := func(name string) []string { + f, err := os.Open(filepath.Join(dir, name)) + require.NoError(t, err) + defer f.Close() + var out []string + s := bufio.NewScanner(f) + for s.Scan() { + out = append(out, s.Text()) + } + return out + } + assert.Equal(t, []string{`{"n":1}`, `{"n":2}`}, readLines("eth_call-evm_1.jsonl")) + assert.Equal(t, []string{`{"n":3}`}, readLines("eth_getLogs-evm_1.jsonl")) + }) + + t.Run("reports an error when the target file cannot be opened", func(t *testing.T) { + dir := t.TempDir() + exp, err := newFileMisbehaviorExporter(&common.MisbehaviorsDestinationConfig{Path: dir, FilePattern: "{method}"}, &lg) + require.NoError(t, err) + // A directory squatting on the resolved file name makes open fail. + require.NoError(t, os.Mkdir(filepath.Join(dir, "eth_call.jsonl"), 0o750)) + err = exp.AppendWithMetadata([]byte(`{}`), "eth_call", "evm:1") + require.ErrorContains(t, err, "failed to open file") + }) +} + +func TestCreateMisbehaviorExporter_SelectsImplementation(t *testing.T) { + lg := zerolog.Nop() + + assert.Nil(t, createMisbehaviorExporter(nil, &lg)) + assert.Nil(t, createMisbehaviorExporter(&common.MisbehaviorsDestinationConfig{}, &lg)) + assert.Nil(t, NewMisbehaviorExporter(&common.MisbehaviorsDestinationConfig{ + Type: common.MisbehaviorsDestinationTypeFile, Path: "not-absolute", + }, &lg), "a file exporter that fails to initialize is disabled") + assert.Nil(t, NewMisbehaviorExporter(&common.MisbehaviorsDestinationConfig{ + Type: common.MisbehaviorsDestinationTypeS3, Path: "not-an-s3-uri", + }, &lg), "an S3 exporter with an invalid URI is disabled") + + exp := NewMisbehaviorExporter(&common.MisbehaviorsDestinationConfig{Path: t.TempDir()}, &lg) + require.NotNil(t, exp) + _, isFile := exp.(*fileMisbehaviorExporter) + assert.True(t, isFile, "untyped destinations default to the file exporter") +} + +func TestParseS3Path(t *testing.T) { + cases := []struct { + in string + bucket string + prefix string + wantErrStr string + }{ + {in: "s3://bucket", bucket: "bucket"}, + {in: "s3://bucket/", bucket: "bucket"}, + {in: "s3://bucket/a/b", bucket: "bucket", prefix: "a/b/"}, + {in: "s3://bucket/a/b/", bucket: "bucket", prefix: "a/b/"}, + {in: "s3://", wantErrStr: "no bucket"}, + {in: "/local/path", wantErrStr: "must start with s3://"}, + } + for _, tt := range cases { + t.Run(tt.in, func(t *testing.T) { + bucket, prefix, err := parseS3Path(tt.in) + if tt.wantErrStr != "" { + require.ErrorContains(t, err, tt.wantErrStr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.bucket, bucket) + assert.Equal(t, tt.prefix, prefix) + }) + } +} + +func TestNewConsensus_AppliesConfig(t *testing.T) { + lg := zerolog.Nop() + + _, err := NewConsensus(nil, &lg) + require.Error(t, err) + + yes := true + dir := t.TempDir() + c, err := NewConsensus(&common.ConsensusPolicyConfig{ + MaxParticipants: 3, + AgreementThreshold: 2, + DisputeBehavior: common.ConsensusDisputeBehaviorAcceptMostCommonValidResult, + LowParticipantsBehavior: common.ConsensusLowParticipantsBehaviorReturnError, + PunishMisbehavior: &common.PunishMisbehaviorConfig{DisputeThreshold: 3}, + DisputeLogLevel: "info", + IgnoreFields: map[string][]string{"eth_getBlockByNumber": {"timestamp"}}, + PreferNonEmpty: &yes, + PreferLargerResponses: &yes, + PreferHighestValueFor: map[string][]string{"eth_getTransactionCount": {"result"}}, + MisbehaviorsDestination: &common.MisbehaviorsDestinationConfig{Path: dir}, + RequiredParticipants: []*common.ConsensusRequiredParticipant{{Tag: "t", MinParticipants: 1}}, + }, &lg) + require.NoError(t, err) + p := c.policy + assert.Equal(t, 3, p.maxParticipants) + assert.Equal(t, 2, p.agreementThreshold) + assert.Equal(t, common.ConsensusDisputeBehaviorAcceptMostCommonValidResult, p.disputeBehavior) + assert.Equal(t, common.ConsensusLowParticipantsBehaviorReturnError, p.lowParticipantsBehavior) + assert.Equal(t, uint(3), p.punishMisbehavior.DisputeThreshold) + assert.Equal(t, zerolog.InfoLevel, p.disputeLogLevel) + assert.True(t, p.preferNonEmpty) + assert.True(t, p.preferLargerResponses) + assert.Contains(t, p.ignoreFields, "eth_getBlockByNumber") + assert.Contains(t, p.preferHighestValueFor, "eth_getTransactionCount") + assert.Len(t, p.requiredParticipants, 1) + assert.NotNil(t, p.exporter, "a valid file destination enables export") + + t.Run("invalid dispute log level keeps the warn default", func(t *testing.T) { + c, err := NewConsensus(&common.ConsensusPolicyConfig{MaxParticipants: 1, DisputeLogLevel: "loud"}, &lg) + require.NoError(t, err) + assert.Equal(t, zerolog.WarnLevel, c.policy.disputeLogLevel) + assert.Nil(t, c.policy.exporter) + }) + + t.Run("nil consensus passes straight to the inner function", func(t *testing.T) { + var nilC *Consensus + want := validResponseWithValue("0xinner") + got, err := nilC.Run(context.Background(), newTestRequest(), + func(context.Context, *common.NormalizedRequest) (*common.NormalizedResponse, error) { return want, nil }) + require.NoError(t, err) + assert.Same(t, want, got) + }) +} diff --git a/consensus/rules_leader_error_test.go b/consensus/rules_leader_error_test.go index f1a5e256e..6f014671b 100644 --- a/consensus/rules_leader_error_test.go +++ b/consensus/rules_leader_error_test.go @@ -40,6 +40,11 @@ func (n *leaderNetwork) EvmHighestFinalizedBlockNumber(context.Context) int64 { return 0 } func (n *leaderNetwork) EvmLeaderUpstream(context.Context) common.Upstream { return n.leader } +func (n *leaderNetwork) EvmRefreshLatestBlock(context.Context) {} + +// Without this, a new EvmNetwork method makes the fake silently stop being an +// EvmNetwork and the leader lookup returns nil. +var _ common.EvmNetwork = (*leaderNetwork)(nil) func missingDataFrom(ups common.Upstream, message string) error { cause := common.NewErrJsonRpcExceptionInternal(0, common.JsonRpcErrorMissingData, message, nil, nil) diff --git a/consensus/rules_outcomes_test.go b/consensus/rules_outcomes_test.go new file mode 100644 index 000000000..b22d03d16 --- /dev/null +++ b/consensus/rules_outcomes_test.go @@ -0,0 +1,848 @@ +package consensus + +import ( + "context" + "errors" + "testing" + + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// resultOf builds an attributed successful participant with an arbitrary +// JSON result (string, number, object). +func resultOf(t *testing.T, ups common.Upstream, value interface{}, index int) *execResult { + t.Helper() + jrpc, err := common.NewJsonRpcResponse(1, value, nil) + require.NoError(t, err) + return &execResult{ + Result: common.NewNormalizedResponse().WithJsonRpcResponse(jrpc), + Upstream: ups, + Index: index, + } +} + +func revertedErr() error { + return common.NewErrEndpointExecutionException( + common.NewErrJsonRpcExceptionInternal(3, 3, "execution reverted", nil, nil), + ) +} + +func invalidParamsErr() error { + return common.NewErrEndpointClientSideException( + common.NewErrJsonRpcExceptionInternal(-32602, -32602, "invalid params", nil, nil), + ) +} + +// analyzeFor runs analysis the way the executor does: with the request bound +// to the context so the method, ignoreFields and the block-head leader resolve. +func analyzeFor(t *testing.T, cfg *config, method string, leader common.Upstream, responses []*execResult) *consensusAnalysis { + t.Helper() + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"` + method + `","params":[]}`)) + if leader != nil { + req.SetNetwork(&leaderNetwork{leader: leader}) + } + ctx := context.WithValue(context.Background(), common.RequestContextKey, req) + lg := zerolog.Nop() + return newConsensusAnalysis(&lg, ctx, cfg, responses) +} + +func winnerResultString(t *testing.T, w *slotResult) string { + t.Helper() + require.NotNil(t, w) + require.NoError(t, w.Error) + require.NotNil(t, w.Result) + jrr, err := w.Result.JsonRpcResponse() + require.NoError(t, err) + return jrr.GetResultString() +} + +func TestDetermineWinner_Outcomes(t *testing.T) { + a := taggedUpstream("ups-a") + b := taggedUpstream("ups-b") + c := taggedUpstream("ups-c") + d := taggedUpstream("ups-d") + + acceptMost := common.ConsensusDisputeBehaviorAcceptMostCommonValidResult + returnErr := common.ConsensusDisputeBehaviorReturnError + lowAcceptMost := common.ConsensusLowParticipantsBehaviorAcceptMostCommonValidResult + + type tc struct { + name string + cfg *config + method string + leader common.Upstream + results func(t *testing.T) []*execResult + // exactly one of the expectations below is set; none = an empty result + wantValue string + wantCode common.ErrorCode + wantErrAny []common.ErrorCode + } + + cases := []tc{ + { + name: "tx broadcast returns the single accepted hash despite failures", + cfg: &config{maxParticipants: 3, agreementThreshold: 3}, + method: "eth_sendRawTransaction", + results: func(t *testing.T) []*execResult { + return []*execResult{ + errorFrom(a, errors.New("dial timeout"), 0), + resultFrom(t, b, "0xhash", 1), + errorFrom(c, errors.New("connection reset"), 2), + } + }, + wantValue: `"0xhash"`, + }, + { + name: "highest value wins only among values meeting the threshold", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, preferHighestValueFor: map[string][]string{"eth_getTransactionCount": {"result"}}}, + method: "eth_getTransactionCount", + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0x5", 0), + resultFrom(t, b, "0x5", 1), + resultFrom(t, c, "0x7", 2), + } + }, + wantValue: `"0x5"`, + }, + { + name: "highest value with zero threshold treats a single vote as enough", + cfg: &config{maxParticipants: 4, agreementThreshold: 0, preferHighestValueFor: map[string][]string{"eth_getTransactionCount": {"result"}}}, + method: "eth_getTransactionCount", + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0x5", 0), + resultFrom(t, b, "not-a-number", 1), + errorFrom(c, revertedErr(), 2), + resultOf(t, d, 9, 3), + } + }, + wantValue: `9`, + }, + { + name: "highest value disputes when no value reaches the threshold", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, preferHighestValueFor: map[string][]string{"eth_getTransactionCount": {"result"}}}, + method: "eth_getTransactionCount", + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0x5", 0), + resultFrom(t, b, "0x6", 1), + resultFrom(t, c, "0x7", 2), + } + }, + wantCode: common.ErrCodeConsensusDispute, + }, + { + name: "highest value compares nested fields of object results", + cfg: &config{maxParticipants: 3, agreementThreshold: 1, preferHighestValueFor: map[string][]string{"eth_getBlockByNumber": {"number"}}}, + method: "eth_getBlockByNumber", + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultOf(t, a, map[string]interface{}{"number": "0x10", "hash": "0xaa"}, 0), + resultOf(t, b, map[string]interface{}{"number": "0x20", "hash": "0xbb"}, 1), + resultOf(t, c, map[string]interface{}{"hash": "0xcc"}, 2), + } + }, + wantValue: `{"hash":"0xbb","number":"0x20"}`, + }, + { + name: "highest value without any extractable value falls back to hash agreement", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, preferHighestValueFor: map[string][]string{"eth_getTransactionCount": {"result"}}}, + method: "eth_getTransactionCount", + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "pending", 0), + resultFrom(t, b, "pending", 1), + resultFrom(t, c, "other", 2), + } + }, + wantValue: `"pending"`, + }, + { + name: "only-block-head-leader dispute returns the leader's answer", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: common.ConsensusDisputeBehaviorOnlyBlockHeadLeader}, + method: "eth_call", + leader: c, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xa", 0), + resultFrom(t, b, "0xb", 1), + resultFrom(t, c, "0xc", 2), + } + }, + wantValue: `"0xc"`, + }, + { + name: "only-block-head-leader dispute surfaces the leader's infrastructure error", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: common.ConsensusDisputeBehaviorOnlyBlockHeadLeader}, + method: "eth_call", + leader: c, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xa", 0), + resultFrom(t, b, "0xb", 1), + errorFrom(c, common.NewErrEndpointServerSideException(errors.New("leader down"), nil, 503), 2), + } + }, + wantCode: common.ErrCodeEndpointServerSideException, + }, + { + name: "only-block-head-leader dispute without an elected leader disputes", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: common.ConsensusDisputeBehaviorOnlyBlockHeadLeader}, + method: "eth_call", + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xa", 0), + resultFrom(t, b, "0xb", 1), + } + }, + wantCode: common.ErrCodeConsensusDispute, + }, + { + name: "only-block-head-leader dispute behavior does not override met threshold", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: common.ConsensusDisputeBehaviorOnlyBlockHeadLeader}, + method: "eth_call", + leader: c, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xa", 0), + resultFrom(t, b, "0xa", 1), + resultFrom(t, c, "0xc", 2), + } + }, + wantValue: `"0xa"`, + }, + { + name: "only-block-head-leader on low participants returns the leader's result", + cfg: &config{maxParticipants: 3, agreementThreshold: 3, + lowParticipantsBehavior: common.ConsensusLowParticipantsBehaviorOnlyBlockHeadLeader}, + method: "eth_call", + leader: b, + results: func(t *testing.T) []*execResult { + return []*execResult{ + errorFrom(a, errors.New("dial timeout"), 0), + resultFrom(t, b, "0xb", 1), + } + }, + wantValue: `"0xb"`, + }, + { + name: "only-block-head-leader low-participants behavior ignores rounds with enough participants", + cfg: &config{maxParticipants: 2, agreementThreshold: 2, + lowParticipantsBehavior: common.ConsensusLowParticipantsBehaviorOnlyBlockHeadLeader}, + method: "eth_call", + leader: b, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xa", 0), + resultFrom(t, b, "0xa", 1), + } + }, + wantValue: `"0xa"`, + }, + { + name: "prefer-block-head-leader dispute picks the leader's group", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: common.ConsensusDisputeBehaviorPreferBlockHeadLeader}, + method: "eth_call", + leader: a, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xa", 0), + resultFrom(t, b, "0xb", 1), + resultFrom(t, c, "0xc", 2), + } + }, + wantValue: `"0xa"`, + }, + { + name: "prefer-block-head-leader with an errored leader falls back to dispute", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: common.ConsensusDisputeBehaviorPreferBlockHeadLeader}, + method: "eth_call", + leader: a, + results: func(t *testing.T) []*execResult { + return []*execResult{ + errorFrom(a, errors.New("dial timeout"), 0), + resultFrom(t, b, "0xb", 1), + resultFrom(t, c, "0xc", 2), + } + }, + wantCode: common.ErrCodeConsensusDispute, + }, + { + name: "prefer-larger + accept-most-common below threshold picks the largest", + cfg: &config{maxParticipants: 3, agreementThreshold: 3, disputeBehavior: acceptMost, + preferLargerResponses: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xaa", 0), + resultFrom(t, b, "0xbbbbbb", 1), + resultFrom(t, c, "0xcc", 2), + } + }, + wantValue: `"0xbbbbbb"`, + }, + { + name: "prefer-larger + accept-most-common below threshold with no data disputes", + cfg: &config{maxParticipants: 3, agreementThreshold: 3, disputeBehavior: acceptMost, + preferLargerResponses: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0x", 0), + errorFrom(b, revertedErr(), 1), + errorFrom(c, invalidParamsErr(), 2), + } + }, + wantCode: common.ErrCodeConsensusDispute, + }, + { + name: "accept-most-common + prefer-non-empty overrides an agreed empty result", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: acceptMost, + preferNonEmpty: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0x", 0), + resultFrom(t, b, "0x", 1), + resultFrom(t, c, "0xdata", 2), + } + }, + wantValue: `"0xdata"`, + }, + { + name: "accept-most-common + prefer-non-empty overrides an agreed execution error", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: acceptMost, + preferNonEmpty: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + errorFrom(a, revertedErr(), 0), + errorFrom(b, revertedErr(), 1), + resultFrom(t, c, "0xdata", 2), + } + }, + wantValue: `"0xdata"`, + }, + { + name: "accept-most-common + prefer-non-empty below threshold prefers data over empties", + cfg: &config{maxParticipants: 4, agreementThreshold: 3, disputeBehavior: acceptMost, + preferNonEmpty: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0x", 0), + resultFrom(t, b, "0x", 1), + resultFrom(t, c, "0xdata", 2), + } + }, + wantValue: `"0xdata"`, + }, + { + name: "accept-most-common + prefer-non-empty below threshold prefers data over errors", + cfg: &config{maxParticipants: 4, agreementThreshold: 3, disputeBehavior: acceptMost, + preferNonEmpty: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + errorFrom(a, revertedErr(), 0), + errorFrom(b, revertedErr(), 1), + resultFrom(t, c, "0xdata", 2), + } + }, + wantValue: `"0xdata"`, + }, + { + name: "accept-most-common + prefer-non-empty picks data when data and error both meet threshold", + cfg: &config{maxParticipants: 4, agreementThreshold: 2, disputeBehavior: acceptMost, + preferNonEmpty: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + errorFrom(a, revertedErr(), 0), + errorFrom(b, revertedErr(), 1), + resultFrom(t, c, "0xdata", 2), + resultFrom(t, d, "0xdata", 3), + } + }, + wantValue: `"0xdata"`, + }, + { + name: "return-error + prefer-non-empty disputes an agreed empty when data exists", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: returnErr, + preferNonEmpty: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0x", 0), + resultFrom(t, b, "0x", 1), + resultFrom(t, c, "0xdata", 2), + } + }, + wantCode: common.ErrCodeConsensusDispute, + }, + { + name: "return-error without prefer-non-empty accepts an agreed empty", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: returnErr}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0x", 0), + resultFrom(t, b, "0x", 1), + resultFrom(t, c, "0xdata", 2), + } + }, + wantValue: `"0x"`, + }, + { + name: "tie between two data groups at threshold disputes", + cfg: &config{maxParticipants: 5, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xa", 0), + resultFrom(t, b, "0xa", 1), + resultFrom(t, c, "0xb", 2), + resultFrom(t, d, "0xb", 3), + errorFrom(taggedUpstream("ups-e"), revertedErr(), 4), + } + }, + wantCode: common.ErrCodeConsensusDispute, + }, + { + name: "tie between data and an agreed error at threshold disputes", + cfg: &config{maxParticipants: 4, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + errorFrom(a, revertedErr(), 0), + errorFrom(b, revertedErr(), 1), + resultFrom(t, c, "0xa", 2), + resultFrom(t, d, "0xa", 3), + } + }, + wantCode: common.ErrCodeConsensusDispute, + }, + { + name: "prefer-larger picks the largest of several groups above threshold", + cfg: &config{maxParticipants: 4, agreementThreshold: 2, preferLargerResponses: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xaa", 0), + resultFrom(t, b, "0xaa", 1), + resultFrom(t, c, "0xbbbbbb", 2), + resultFrom(t, d, "0xbbbbbb", 3), + } + }, + wantValue: `"0xbbbbbb"`, + }, + { + name: "prefer-larger with only empty and error groups above threshold disputes", + cfg: &config{maxParticipants: 4, agreementThreshold: 2, preferLargerResponses: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0x", 0), + resultFrom(t, b, "0x", 1), + errorFrom(c, revertedErr(), 2), + errorFrom(d, revertedErr(), 3), + } + }, + wantCode: common.ErrCodeConsensusDispute, + }, + { + name: "prefer-larger + accept-most-common replaces a smaller agreed result", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: acceptMost, + preferLargerResponses: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xaa", 0), + resultFrom(t, b, "0xaa", 1), + resultFrom(t, c, "0xbbbbbb", 2), + } + }, + wantValue: `"0xbbbbbb"`, + }, + { + name: "return-error + prefer-larger disputes a smaller agreed result", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, disputeBehavior: returnErr, + preferLargerResponses: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xaa", 0), + resultFrom(t, b, "0xaa", 1), + resultFrom(t, c, "0xbbbbbb", 2), + } + }, + wantCode: common.ErrCodeConsensusDispute, + }, + { + name: "accept-most-common below threshold returns the unique leading result", + cfg: &config{maxParticipants: 4, agreementThreshold: 3, disputeBehavior: acceptMost}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xb", 0), + resultFrom(t, b, "0xa", 1), + resultFrom(t, c, "0xa", 2), + } + }, + wantValue: `"0xa"`, + }, + { + name: "accept-most-common below threshold returns the unique leading agreed error", + cfg: &config{maxParticipants: 4, agreementThreshold: 3, disputeBehavior: acceptMost}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xb", 0), + errorFrom(b, revertedErr(), 1), + errorFrom(c, revertedErr(), 2), + } + }, + wantCode: common.ErrCodeEndpointExecutionException, + }, + { + name: "low participants + accept-most-common disputes a tie between data", + cfg: &config{maxParticipants: 3, agreementThreshold: 3, lowParticipantsBehavior: lowAcceptMost}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xa", 0), + resultFrom(t, b, "0xb", 1), + errorFrom(c, errors.New("dial timeout"), 2), + } + }, + wantCode: common.ErrCodeConsensusDispute, + }, + { + name: "low participants + accept-most-common falls back to an empty result", + cfg: &config{maxParticipants: 3, agreementThreshold: 3, lowParticipantsBehavior: lowAcceptMost}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0x", 0), + resultOf(t, b, []interface{}{}, 1), + errorFrom(c, errors.New("dial timeout"), 2), + } + }, + wantErrAny: nil, + }, + { + name: "low participants + accept-most-common falls back to an agreed error", + cfg: &config{maxParticipants: 3, agreementThreshold: 3, lowParticipantsBehavior: lowAcceptMost}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + errorFrom(a, revertedErr(), 0), + errorFrom(b, invalidParamsErr(), 1), + } + }, + wantErrAny: []common.ErrorCode{common.ErrCodeEndpointExecutionException, common.ErrCodeEndpointClientSideException}, + }, + { + name: "agreed execution error meeting threshold is returned to the caller", + cfg: &config{maxParticipants: 3, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + errorFrom(a, revertedErr(), 0), + errorFrom(b, revertedErr(), 1), + resultFrom(t, c, "0xa", 2), + } + }, + wantCode: common.ErrCodeEndpointExecutionException, + }, + { + name: "split below threshold without dispute preference disputes", + cfg: &config{maxParticipants: 3, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xa", 0), + resultFrom(t, b, "0xb", 1), + resultFrom(t, c, "0xc", 2), + } + }, + wantCode: common.ErrCodeConsensusDispute, + }, + { + name: "identical infrastructure errors meeting threshold surface that error", + cfg: &config{maxParticipants: 2, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + errorFrom(a, common.NewErrEndpointServerSideException(errors.New("503"), nil, 503), 0), + errorFrom(b, common.NewErrEndpointServerSideException(errors.New("503"), nil, 503), 1), + } + }, + wantCode: common.ErrCodeEndpointServerSideException, + }, + { + name: "low participants with return-error behavior returns low participants", + cfg: &config{maxParticipants: 3, agreementThreshold: 3, + lowParticipantsBehavior: common.ConsensusLowParticipantsBehaviorReturnError}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + resultFrom(t, a, "0xa", 0), + errorFrom(b, errors.New("dial timeout"), 1), + } + }, + wantCode: common.ErrCodeConsensusLowParticipants, + }, + { + name: "no responses at all is low participants", + cfg: &config{maxParticipants: 3, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return nil + }, + wantCode: common.ErrCodeConsensusLowParticipants, + }, + { + name: "a lone infrastructure error below threshold resolves to low participants", + cfg: &config{maxParticipants: 2, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return []*execResult{ + errorFrom(a, errors.New("dial timeout"), 0), + } + }, + wantCode: common.ErrCodeConsensusLowParticipants, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + method := tt.method + if method == "" { + method = "eth_getBalance" + } + analysis := analyzeFor(t, tt.cfg, method, tt.leader, tt.results(t)) + winner := winnerOf(tt.cfg, analysis) + require.NotNil(t, winner) + switch { + case tt.wantValue != "": + assert.JSONEq(t, tt.wantValue, winnerResultString(t, winner)) + case tt.wantCode != "": + require.Error(t, winner.Error) + assert.True(t, common.HasErrorCode(winner.Error, tt.wantCode), + "want %s, got %v", tt.wantCode, winner.Error) + case tt.wantErrAny != nil: + require.Error(t, winner.Error) + assert.True(t, common.HasErrorCode(winner.Error, tt.wantErrAny...), + "want one of %v, got %v", tt.wantErrAny, winner.Error) + default: + // Both empties tie at one vote; either is a valid empty answer. + require.NoError(t, winner.Error) + require.NotNil(t, winner.Result) + assert.True(t, winner.Result.IsResultEmptyish(), "an empty result must be returned") + } + }) + } +} + +func TestShouldShortCircuit_Reasons(t *testing.T) { + a := taggedUpstream("ups-a") + b := taggedUpstream("ups-b") + c := taggedUpstream("ups-c") + + cases := []struct { + name string + cfg *config + method string + results func(t *testing.T) []*execResult + wantReason string // empty = no short-circuit + }{ + { + name: "agreed execution error at threshold short-circuits", + cfg: &config{maxParticipants: 3, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return []*execResult{errorFrom(a, revertedErr(), 0), errorFrom(b, revertedErr(), 1)} + }, + wantReason: "consensus_error_threshold", + }, + { + name: "agreed execution error below threshold keeps waiting", + cfg: &config{maxParticipants: 3, agreementThreshold: 3}, + results: func(t *testing.T) []*execResult { + return []*execResult{errorFrom(a, revertedErr(), 0), errorFrom(b, revertedErr(), 1)} + }, + }, + { + name: "agreed error with highest-value method keeps waiting for values", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, preferHighestValueFor: map[string][]string{"eth_getTransactionCount": {"result"}}}, + method: "eth_getTransactionCount", + results: func(t *testing.T) []*execResult { + return []*execResult{errorFrom(a, revertedErr(), 0), errorFrom(b, revertedErr(), 1)} + }, + }, + { + name: "agreed error under accept-most-common with prefer-non-empty keeps waiting", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, + disputeBehavior: common.ConsensusDisputeBehaviorAcceptMostCommonValidResult, preferNonEmpty: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{errorFrom(a, revertedErr(), 0), errorFrom(b, revertedErr(), 1)} + }, + }, + { + name: "unassailable data lead short-circuits", + cfg: &config{maxParticipants: 3, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return []*execResult{resultFrom(t, a, "0xa", 0), resultFrom(t, b, "0xa", 1)} + }, + wantReason: "unassailable_lead", + }, + { + name: "lead over a dissenter still unassailable with one slot left", + cfg: &config{maxParticipants: 4, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return []*execResult{resultFrom(t, a, "0xa", 0), resultFrom(t, b, "0xa", 1), resultFrom(t, c, "0xb", 2)} + }, + wantReason: "unassailable_lead", + }, + { + name: "lead that remaining slots could tie keeps waiting", + cfg: &config{maxParticipants: 5, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return []*execResult{resultFrom(t, a, "0xa", 0), resultFrom(t, b, "0xa", 1)} + }, + }, + { + name: "prefer-larger keeps waiting while slots remain", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, preferLargerResponses: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{resultFrom(t, a, "0xa", 0), resultFrom(t, b, "0xa", 1)} + }, + }, + { + name: "highest-value method keeps waiting while slots remain", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, preferHighestValueFor: map[string][]string{"eth_getTransactionCount": {"result"}}}, + method: "eth_getTransactionCount", + results: func(t *testing.T) []*execResult { + return []*execResult{resultFrom(t, a, "0x1", 0), resultFrom(t, b, "0x1", 1)} + }, + }, + { + name: "prefer-non-empty with an empty leader keeps waiting", + cfg: &config{maxParticipants: 3, agreementThreshold: 2, preferNonEmpty: true}, + results: func(t *testing.T) []*execResult { + return []*execResult{resultFrom(t, a, "0x", 0), resultFrom(t, b, "0x", 1)} + }, + }, + { + name: "no responses never short-circuits", + cfg: &config{maxParticipants: 3, agreementThreshold: 2}, + results: func(t *testing.T) []*execResult { + return nil + }, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + method := tt.method + if method == "" { + method = "eth_getBalance" + } + analysis := analyzeFor(t, tt.cfg, method, nil, tt.results(t)) + lg := zerolog.Nop() + e := &executor{consensusPolicy: &consensusPolicy{logger: &lg, config: tt.cfg}} + winner := e.determineWinner(&lg, analysis) + reason, ok := e.shouldShortCircuit(winner, analysis) + assert.Equal(t, tt.wantReason != "", ok) + assert.Equal(t, tt.wantReason, reason) + }) + } +} + +func TestAnalysis_IgnoreFieldsMergeResponsesDifferingOnlyInIgnoredFields(t *testing.T) { + a := taggedUpstream("ups-a") + b := taggedUpstream("ups-b") + responses := func(t *testing.T) []*execResult { + return []*execResult{ + resultOf(t, a, map[string]interface{}{"hash": "0x1", "timestamp": "0x2"}, 0), + resultOf(t, b, map[string]interface{}{"hash": "0x1", "timestamp": "0x3"}, 1), + } + } + + t.Run("without ignoreFields the responses disagree", func(t *testing.T) { + cfg := &config{maxParticipants: 2, agreementThreshold: 2} + analysis := analyzeFor(t, cfg, "eth_getBlockByNumber", nil, responses(t)) + assert.Len(t, analysis.groups, 2) + }) + + t.Run("ignored field is excluded from the agreement hash", func(t *testing.T) { + cfg := &config{maxParticipants: 2, agreementThreshold: 2, + ignoreFields: map[string][]string{"eth_getBlockByNumber": {"timestamp"}}} + analysis := analyzeFor(t, cfg, "eth_getBlockByNumber", nil, responses(t)) + require.Len(t, analysis.groups, 1) + winner := winnerOf(cfg, analysis) + require.NoError(t, winner.Error) + }) + + t.Run("ignoreFields for another method has no effect", func(t *testing.T) { + cfg := &config{maxParticipants: 2, agreementThreshold: 2, + ignoreFields: map[string][]string{"eth_getLogs": {"timestamp"}}} + analysis := analyzeFor(t, cfg, "eth_getBlockByNumber", nil, responses(t)) + assert.Len(t, analysis.groups, 2) + }) +} + +func TestAnalysis_ClassifiesUnparseableResponsesAsInfrastructure(t *testing.T) { + cfg := &config{maxParticipants: 2, agreementThreshold: 2} + r := &execResult{Result: nil, Upstream: taggedUpstream("ups-a")} + classifyAndHashResponse(r, context.Background(), cfg) + assert.Equal(t, ResponseTypeInfrastructureError, r.CachedResponseType) + assert.Equal(t, "error:generic", r.CachedHash) +} + +func TestErrorToConsensusHash(t *testing.T) { + cases := []struct { + name string + err error + want string + }{ + {"nil error has no hash", nil, ""}, + {"json-rpc error hashes by normalized code", revertedErr(), "jsonrpc:3"}, + {"standard error without json-rpc cause hashes by its code", + common.NewErrEndpointServerSideException(errors.New("boom"), nil, 500), string(common.ErrCodeEndpointServerSideException)}, + {"plain error hashes generically", errors.New("boom"), "error:generic"}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, errorToConsensusHash(tt.err)) + }) + } +} + +func TestResponseType_String(t *testing.T) { + cases := map[ResponseType]string{ + ResponseTypeNonEmpty: "non_empty", + ResponseTypeEmpty: "empty", + ResponseTypeConsensusError: "consensus_error", + ResponseTypeInfrastructureError: "infrastructure_error", + ResponseType(99): "unknown", + } + for rt, want := range cases { + assert.Equal(t, want, rt.String()) + } +} + +func TestExtractFieldValues(t *testing.T) { + resp := func(t *testing.T, v interface{}) *common.NormalizedResponse { + return resultOf(t, nil, v, 0).Result + } + cases := []struct { + name string + resp func(t *testing.T) *common.NormalizedResponse + fields []string + want []int64 // nil = not extractable + }{ + {"nil response", func(t *testing.T) *common.NormalizedResponse { return nil }, []string{"result"}, nil}, + {"no fields", func(t *testing.T) *common.NormalizedResponse { return resp(t, "0x1") }, nil, nil}, + {"hex string result", func(t *testing.T) *common.NormalizedResponse { return resp(t, "0x1f") }, []string{"result"}, []int64{31}}, + {"numeric result", func(t *testing.T) *common.NormalizedResponse { return resp(t, 42) }, []string{"result"}, []int64{42}}, + {"non-numeric result", func(t *testing.T) *common.NormalizedResponse { return resp(t, "abc") }, []string{"result"}, nil}, + {"nested fields in order", func(t *testing.T) *common.NormalizedResponse { + return resp(t, map[string]interface{}{"nonce": "0x2", "balance": "10"}) + }, []string{"nonce", "balance"}, []int64{2, 10}}, + {"missing nested field", func(t *testing.T) *common.NormalizedResponse { + return resp(t, map[string]interface{}{"nonce": "0x2"}) + }, []string{"nonce", "balance"}, nil}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + got := extractFieldValues(tt.resp(t), tt.fields) + if tt.want == nil { + assert.Nil(t, got) + return + } + require.Len(t, got, len(tt.want)) + for i, w := range tt.want { + assert.Equal(t, w, got[i].Int64()) + } + }) + } +} diff --git a/data/dynamodb_connector_test.go b/data/dynamodb_connector_test.go new file mode 100644 index 000000000..c8b3e8e55 --- /dev/null +++ b/data/dynamodb_connector_test.go @@ -0,0 +1,450 @@ +package data + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ddbTableSeq atomic.Int64 + +func ddbConnectorConfig(endpoint string) *common.DynamoDBConnectorConfig { + return &common.DynamoDBConnectorConfig{ + Endpoint: endpoint, + Region: "us-west-2", + Table: fmt.Sprintf("cov_%d_%d", time.Now().UnixNano()%1_000_000, ddbTableSeq.Add(1)), + PartitionKeyName: "pk", + RangeKeyName: "rk", + ReverseIndexName: "idx_reverse", + TTLAttributeName: "ttl", + InitTimeout: common.Duration(10 * time.Second), + GetTimeout: common.Duration(3 * time.Second), + SetTimeout: common.Duration(3 * time.Second), + StatePollInterval: common.Duration(50 * time.Millisecond), + LockRetryInterval: common.Duration(50 * time.Millisecond), + Auth: &common.AwsAuthConfig{ + Mode: "secret", + AccessKeyID: "fakeKey", + SecretAccessKey: "fakeSecret", + }, + } +} + +func newReadyDynamoConnector(t *testing.T, cfg *common.DynamoDBConnectorConfig) (context.Context, *DynamoDBConnector) { + t.Helper() + logger := zerolog.New(io.Discard) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + c, err := NewDynamoDBConnector(ctx, &logger, "ddb-"+cfg.Table, cfg) + require.NoError(t, err) + require.Equal(t, util.StateReady, c.initializer.State(), "errors: %v", c.initializer.Errors()) + return ctx, c +} + +// putRawItem writes an item directly, for shapes Set never produces (legacy +// string values, already-expired TTLs, values with no payload). +func putRawItem(t *testing.T, c *DynamoDBConnector, item map[string]*dynamodb.AttributeValue) { + t.Helper() + _, err := c.writeClient.PutItem(&dynamodb.PutItemInput{TableName: aws.String(c.table), Item: item}) + require.NoError(t, err) +} + +func ddbItem(pk, rk string) map[string]*dynamodb.AttributeValue { + return map[string]*dynamodb.AttributeValue{"pk": {S: aws.String(pk)}, "rk": {S: aws.String(rk)}} +} + +func pastTTL() *dynamodb.AttributeValue { + return &dynamodb.AttributeValue{N: aws.String(strconv.FormatInt(time.Now().Add(-time.Hour).Unix(), 10))} +} + +func TestDynamoDBConnector_GetSetDelete(t *testing.T) { + ctx, c := newReadyDynamoConnector(t, ddbConnectorConfig(sharedDynamoEndpoint(t))) + assert.Equal(t, "ddb-"+c.table, c.Id()) + + ttl := time.Hour + require.NoError(t, c.Set(ctx, "pk", "with-ttl", []byte("v1"), &ttl)) + require.NoError(t, c.Set(ctx, "pk", "no-ttl", make([]byte, 2048), nil)) + + legacy := ddbItem("pk", "legacy") + legacy["value"] = &dynamodb.AttributeValue{S: aws.String("legacy-string")} + putRawItem(t, c, legacy) + + expired := ddbItem("pk", "expired") + expired["value"] = &dynamodb.AttributeValue{B: []byte("old")} + expired["ttl"] = pastTTL() + putRawItem(t, c, expired) + + novalue := ddbItem("pk", "novalue") + novalue["value"] = &dynamodb.AttributeValue{N: aws.String("1")} + putRawItem(t, c, novalue) + + cases := []struct { + name string + rk string + want string + errCode common.ErrorCode + errMsg string + }{ + {name: "binary value with ttl", rk: "with-ttl", want: "v1"}, + {name: "legacy string value", rk: "legacy", want: "legacy-string"}, + {name: "missing key", rk: "absent", errCode: common.ErrCodeRecordNotFound}, + {name: "expired ttl", rk: "expired", errCode: common.ErrCodeRecordExpired}, + {name: "value neither binary nor string", rk: "novalue", errMsg: "neither binary nor string"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := c.Get(ctx, ConnectorMainIndex, "pk", tc.rk, nil) + switch { + case tc.errCode != "": + assert.True(t, common.HasErrorCode(err, tc.errCode), "want %s, got %v", tc.errCode, err) + case tc.errMsg != "": + require.ErrorContains(t, err, tc.errMsg) + default: + require.NoError(t, err) + assert.Equal(t, tc.want, string(got)) + } + }) + } + + require.NoError(t, c.Delete(ctx, "pk", "with-ttl")) + _, err := c.Get(ctx, ConnectorMainIndex, "pk", "with-ttl", nil) + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "got %v", err) +} + +func TestDynamoDBConnector_ReverseIndexGet(t *testing.T) { + ctx, c := newReadyDynamoConnector(t, ddbConnectorConfig(sharedDynamoEndpoint(t))) + + require.NoError(t, c.Set(ctx, "evm:1:100", "rk", []byte("b100"), nil)) + require.NoError(t, c.Set(ctx, "evm:1:200", "rk", []byte("b200"), nil)) + legacy := ddbItem("evm:2:100", "rk") + legacy["value"] = &dynamodb.AttributeValue{S: aws.String("legacy")} + putRawItem(t, c, legacy) + // Expired item sorting above all live ones must be skipped. + expired := ddbItem("evm:1:999", "rk") + expired["value"] = &dynamodb.AttributeValue{B: []byte("expired")} + expired["ttl"] = pastTTL() + putRawItem(t, c, expired) + // Only an expired item under this range key. + onlyExpired := ddbItem("evm:1:1", "dead-rk") + onlyExpired["value"] = &dynamodb.AttributeValue{B: []byte("x")} + onlyExpired["ttl"] = pastTTL() + putRawItem(t, c, onlyExpired) + bad := ddbItem("evm:3:1", "bad-rk") + bad["value"] = &dynamodb.AttributeValue{N: aws.String("1")} + putRawItem(t, c, bad) + + cases := []struct { + name string + pk, rk string + want string + missing bool + errMsg string + }{ + {name: "prefix picks newest live partition", pk: "evm:1:*", rk: "rk", want: "b200"}, + {name: "exact partition", pk: "evm:1:100", rk: "rk", want: "b100"}, + {name: "bare wildcard spans partitions", pk: "*", rk: "rk", want: "legacy"}, + {name: "legacy string value", pk: "evm:2:*", rk: "rk", want: "legacy"}, + {name: "only expired match", pk: "evm:1:*", rk: "dead-rk", missing: true}, + {name: "unknown range key", pk: "", rk: "nope", missing: true}, + {name: "empty range key rejected", pk: "evm:1:*", rk: "", errMsg: "rangeKey must be a non-empty string"}, + {name: "wildcard range key rejected", pk: "evm:1:*", rk: "r*", errMsg: "not contain wildcards"}, + {name: "value neither binary nor string", pk: "evm:3:*", rk: "bad-rk", errMsg: "neither binary nor string"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := c.Get(ctx, ConnectorReverseIndex, tc.pk, tc.rk, nil) + switch { + case tc.missing: + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "got %v", err) + case tc.errMsg != "": + require.ErrorContains(t, err, tc.errMsg) + default: + require.NoError(t, err) + assert.Equal(t, tc.want, string(got)) + } + }) + } +} + +func TestDynamoDBConnector_List(t *testing.T) { + ctx, c := newReadyDynamoConnector(t, ddbConnectorConfig(sharedDynamoEndpoint(t))) + + require.NoError(t, c.Set(ctx, "a", "1", []byte("va"), nil)) + require.NoError(t, c.Set(ctx, "b", "2", []byte("vb"), nil)) + legacy := ddbItem("c", "3") + legacy["value"] = &dynamodb.AttributeValue{S: aws.String("vc")} + putRawItem(t, c, legacy) + expired := ddbItem("d", "4") + expired["value"] = &dynamodb.AttributeValue{B: []byte("old")} + expired["ttl"] = pastTTL() + putRawItem(t, c, expired) + + collect := func(t *testing.T, index string) map[string]string { + t.Helper() + got := map[string]string{} + token := "" + for range 20 { + items, next, err := c.List(ctx, index, 1, token) + require.NoError(t, err) + for _, it := range items { + got[it.PartitionKey+"|"+it.RangeKey] = string(it.Value) + } + if next == "" { + return got + } + token = next + } + t.Fatal("pagination did not terminate") + return nil + } + + want := map[string]string{"a|1": "va", "b|2": "vb", "c|3": "vc"} + assert.Equal(t, want, collect(t, ConnectorMainIndex), "paged scan returns live items only") + assert.Equal(t, want, collect(t, ConnectorReverseIndex), "reverse index scan sees the same items") + + t.Run("invalid tokens are rejected", func(t *testing.T) { + _, _, err := c.List(ctx, ConnectorMainIndex, 1, "%%%") + require.ErrorContains(t, err, "invalid pagination token") + _, _, err = c.List(ctx, ConnectorMainIndex, 1, base64.StdEncoding.EncodeToString([]byte("nope"))) + require.ErrorContains(t, err, "invalid pagination token format") + }) +} + +func TestDynamoDBConnector_WatchCounter(t *testing.T) { + ctx, c := newReadyDynamoConnector(t, ddbConnectorConfig(sharedDynamoEndpoint(t))) + key := "shared/latest" + + initial := CounterInt64State{Value: 1, UpdatedAt: 10, UpdatedBy: "a"} + b, err := common.SonicCfg.Marshal(initial) + require.NoError(t, err) + require.NoError(t, c.Set(ctx, key, "value", b, nil)) + + updates, cleanup, err := c.WatchCounterInt64(ctx, key) + require.NoError(t, err) + select { + case st := <-updates: + assert.Equal(t, initial, st) + case <-time.After(5 * time.Second): + t.Fatal("no initial value") + } + + // Publish is a no-op; the poller picks up stored values instead. + require.NoError(t, c.PublishCounterInt64(ctx, key, CounterInt64State{Value: 99, UpdatedAt: 99})) + + next := CounterInt64State{Value: 2, UpdatedAt: 20, UpdatedBy: "b"} + b, err = common.SonicCfg.Marshal(next) + require.NoError(t, err) + require.NoError(t, c.Set(ctx, key, "value", b, nil)) + select { + case st := <-updates: + assert.Equal(t, next, st, "poller delivers newer stored value") + case <-time.After(10 * time.Second): + t.Fatal("poller did not deliver update") + } + cleanup() + _, open := <-updates + assert.False(t, open) +} + +func TestDynamoDBConnector_GetSimpleValueShapes(t *testing.T) { + ctx, c := newReadyDynamoConnector(t, ddbConnectorConfig(sharedDynamoEndpoint(t))) + + item := func(key string, v *dynamodb.AttributeValue) { + it := ddbItem(key, "value") + if v != nil { + it["value"] = v + } + putRawItem(t, c, it) + } + item("binary", &dynamodb.AttributeValue{B: []byte(`{"v":1,"t":5}`)}) + item("string", &dynamodb.AttributeValue{S: aws.String(`{"v":2,"t":6}`)}) + item("number", &dynamodb.AttributeValue{N: aws.String("7")}) + item("no-value-attr", nil) + item("zero-ts", &dynamodb.AttributeValue{B: []byte(`{"v":3,"t":0}`)}) + item("bool", &dynamodb.AttributeValue{BOOL: aws.Bool(true)}) + + cases := []struct { + key string + want *CounterInt64State + }{ + {key: "binary", want: &CounterInt64State{Value: 1, UpdatedAt: 5}}, + {key: "string", want: &CounterInt64State{Value: 2, UpdatedAt: 6}}, + {key: "number"}, + {key: "no-value-attr"}, + {key: "zero-ts"}, + {key: "bool"}, + {key: "absent"}, + } + for _, tc := range cases { + t.Run(tc.key, func(t *testing.T) { + st, ok, err := c.getSimpleValue(ctx, tc.key) + require.NoError(t, err) + if tc.want == nil { + assert.False(t, ok) + return + } + require.True(t, ok) + assert.Equal(t, *tc.want, st) + }) + } +} + +func TestDynamoDBConnector_LockErrors(t *testing.T) { + ctx, c := newReadyDynamoConnector(t, ddbConnectorConfig(sharedDynamoEndpoint(t))) + + lock, err := c.Lock(ctx, "k", time.Minute) + require.NoError(t, err) + assert.False(t, lock.IsNil()) + + t.Run("already cancelled context fails before trying", func(t *testing.T) { + cctx, cancel := context.WithCancel(ctx) + cancel() + _, err := c.Lock(cctx, "other", time.Minute) + require.ErrorContains(t, err, "cancelled or timed out") + }) + + t.Run("held lock times out while waiting to retry", func(t *testing.T) { + dctx, cancel := context.WithTimeout(ctx, 300*time.Millisecond) + defer cancel() + start := time.Now() + _, err := c.Lock(dctx, "k", time.Minute) + // The deadline lands either between retries or inside a PutItem + // attempt; both surface as a lock failure for this key, promptly. + require.ErrorContains(t, err, "for key 'k'") + assert.Less(t, time.Since(start), 5*time.Second) + }) + + require.NoError(t, lock.Unlock(ctx)) + + var nilLock *dynamoLock + assert.True(t, nilLock.IsNil()) + + t.Run("non-retryable error is returned immediately", func(t *testing.T) { + cfg := ddbConnectorConfig(sharedDynamoEndpoint(t)) + broken := &DynamoDBConnector{ + logger: c.logger, writeClient: c.writeClient, table: "no_such_table_" + cfg.Table, + partitionKeyName: "pk", rangeKeyName: "rk", setTimeout: time.Second, lockRetryInterval: time.Millisecond, + } + _, err := broken.Lock(ctx, "k", time.Second) + require.ErrorContains(t, err, "failed to acquire lock for key 'k' during attempt") + + dl := &dynamoLock{connector: broken, lockKey: "k:lock"} + assert.Error(t, dl.Unlock(ctx), "unlock against a missing table surfaces the error") + }) +} + +func TestDynamoDBConnector_EnsuresReverseIndexOnExistingTable(t *testing.T) { + endpoint := sharedDynamoEndpoint(t) + cfg := ddbConnectorConfig(endpoint) + ctx, c := newReadyDynamoConnector(t, cfg) + + // Create a table without the reverse index, then point a connector with a + // differently-named index at it: the connector must add the index. + plain := cfg.Table + "_plain" + _, err := c.writeClient.CreateTable(&dynamodb.CreateTableInput{ + TableName: aws.String(plain), + KeySchema: []*dynamodb.KeySchemaElement{ + {AttributeName: aws.String("pk"), KeyType: aws.String("HASH")}, + {AttributeName: aws.String("rk"), KeyType: aws.String("RANGE")}, + }, + AttributeDefinitions: []*dynamodb.AttributeDefinition{ + {AttributeName: aws.String("pk"), AttributeType: aws.String("S")}, + {AttributeName: aws.String("rk"), AttributeType: aws.String("S")}, + }, + BillingMode: aws.String("PAY_PER_REQUEST"), + }) + require.NoError(t, err) + + cfg2 := ddbConnectorConfig(endpoint) + cfg2.Table = plain + _, c2 := newReadyDynamoConnector(t, cfg2) + out, err := c2.writeClient.DescribeTableWithContext(ctx, &dynamodb.DescribeTableInput{TableName: aws.String(plain)}) + require.NoError(t, err) + var names []string + for _, g := range out.Table.GlobalSecondaryIndexes { + names = append(names, *g.IndexName) + } + assert.Contains(t, names, "idx_reverse") + + t.Run("no reverse index configured is a no-op", func(t *testing.T) { + cfg3 := *cfg2 + cfg3.ReverseIndexName = "" + require.NoError(t, ensureGlobalSecondaryIndexes(ctx, c.logger, c.writeClient, &cfg3)) + }) +} + +func TestDynamoDBConnector_InitFailures(t *testing.T) { + logger := zerolog.New(io.Discard) + newConn := func(t *testing.T, cfg *common.DynamoDBConnectorConfig) *DynamoDBConnector { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + c, err := NewDynamoDBConnector(ctx, &logger, "ddb-init", cfg) + require.NoError(t, err) + return c + } + + fatal := map[string]func(*common.DynamoDBConnectorConfig){ + "missing region": func(c *common.DynamoDBConnectorConfig) { c.Region = "" }, + "unsupported auth mode": func(c *common.DynamoDBConnectorConfig) { c.Auth.Mode = "bogus" }, + "missing table": func(c *common.DynamoDBConnectorConfig) { c.Table = "" }, + } + for name, mutate := range fatal { + t.Run(name+" is fatal", func(t *testing.T) { + cfg := ddbConnectorConfig("http://127.0.0.1:1") + mutate(cfg) + c := newConn(t, cfg) + assert.Equal(t, util.StateFatal, c.initializer.State()) + }) + } + + t.Run("every auth mode builds a session", func(t *testing.T) { + for _, auth := range []*common.AwsAuthConfig{ + nil, + {Mode: "env"}, + {Mode: "file", CredentialsFile: "/nonexistent", Profile: "default"}, + {Mode: "secret", AccessKeyID: "a", SecretAccessKey: "b"}, + } { + cfg := ddbConnectorConfig("http://127.0.0.1:1") + cfg.Auth = auth + sess, err := createSession(cfg) + require.NoError(t, err) + assert.Equal(t, "us-west-2", *sess.Config.Region) + } + }) + + t.Run("unreachable endpoint returns promptly and entry points report not initialized", func(t *testing.T) { + cfg := ddbConnectorConfig("http://127.0.0.1:1") + cfg.InitTimeout = common.Duration(300 * time.Millisecond) + start := time.Now() + c := newConn(t, cfg) + assert.Less(t, time.Since(start), 10*time.Second) + assert.NotEqual(t, util.StateReady, c.initializer.State()) + + bare := &DynamoDBConnector{logger: c.logger} + ctx := context.Background() + assert.ErrorContains(t, bare.Set(ctx, "pk", "rk", nil, nil), "not initialized") + _, err := bare.Get(ctx, ConnectorMainIndex, "pk", "rk", nil) + assert.ErrorContains(t, err, "not initialized") + _, err = bare.Lock(ctx, "k", time.Second) + assert.ErrorContains(t, err, "not initialized") + assert.ErrorContains(t, bare.Delete(ctx, "pk", "rk"), "not initialized") + _, _, err = bare.List(ctx, ConnectorMainIndex, 1, "") + assert.ErrorContains(t, err, "not initialized") + _, _, err = bare.WatchCounterInt64(ctx, "k") + assert.ErrorContains(t, err, "not initialized") + }) +} diff --git a/data/grpc.go b/data/grpc.go index cf4e5b0bd..868ad76af 100644 --- a/data/grpc.go +++ b/data/grpc.go @@ -68,7 +68,10 @@ var supportedMethods = map[string]struct{}{ "eth_blockNumber": {}, // SVM. Solana method names are unprefixed, so they cannot collide with the // eth_* entries above and one allowlist serves both architectures. - "getBlock": {}, + "getBlock": {}, + "getTransaction": {}, + "getBlockTime": {}, + "getBlocksWithLimit": {}, } func NewGrpcConnector( diff --git a/data/grpc_connector_test.go b/data/grpc_connector_test.go new file mode 100644 index 000000000..2d98369a0 --- /dev/null +++ b/data/grpc_connector_test.go @@ -0,0 +1,309 @@ +package data + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/blockchain-data-standards/manifesto/evm" + "github.com/erpc/erpc/clients" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// bdsTestServer answers ChainId and GetBlockByNumber the way a BDS reader +// does: each tag maps to a fixed block (number, timestamp). +type bdsTestServer struct { + evm.UnimplementedRPCQueryServiceServer + chainID uint64 + chainErr error + blocks map[string][2]uint64 + calls atomic.Int64 +} + +func (s *bdsTestServer) ChainId(context.Context, *evm.ChainIdRequest) (*evm.ChainIdResponse, error) { + s.calls.Add(1) + if s.chainErr != nil { + return nil, s.chainErr + } + return &evm.ChainIdResponse{ChainId: s.chainID}, nil +} + +func (s *bdsTestServer) GetBlockByNumber(_ context.Context, req *evm.GetBlockByNumberRequest) (*evm.GetBlockResponse, error) { + s.calls.Add(1) + b, ok := s.blocks[req.GetBlockNumber()] + if !ok { + return &evm.GetBlockResponse{}, nil + } + return &evm.GetBlockResponse{Block: &evm.BlockHeader{Number: b[0], Timestamp: b[1], Hash: []byte{0xaa}}}, nil +} + +func startBdsTestServer(t *testing.T, s *bdsTestServer) string { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer() + evm.RegisterRPCQueryServiceServer(srv, s) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + return "grpc://" + lis.Addr().String() +} + +func newTestGrpcConnector(t *testing.T, cfg *common.GrpcConnectorConfig) *GrpcConnector { + t.Helper() + lg := zerolog.Nop() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + gc, err := NewGrpcConnector(ctx, &lg, "grpc-test", cfg) + require.NoError(t, err) + t.Cleanup(func() { + gc.mu.RLock() + defer gc.mu.RUnlock() + for _, cli := range gc.clientByNetwork { + if sd, ok := cli.(clients.ShutdownableClient); ok { + sd.Shutdown() + } + } + }) + return gc +} + +func TestGrpcConnector_ProbesChainIdAndPollsHeads(t *testing.T) { + srv := &bdsTestServer{chainID: 1, blocks: map[string][2]uint64{ + "earliest": {100, 1_000}, + "latest": {500, 5_000}, + "finalized": {400, 0}, + }} + addr := startBdsTestServer(t, srv) + gc := newTestGrpcConnector(t, &common.GrpcConnectorConfig{Servers: []string{addr}, Headers: map[string]string{"x-key": "v"}}) + + require.Equal(t, util.StateReady, gc.initializer.State()) + require.NoError(t, gc.checkReady()) + gc.mu.RLock() + _, ok := gc.clientByNetwork["evm:1"] + gc.mu.RUnlock() + require.True(t, ok, "probed chainId 1 registers the server as evm:1") + + _, known := gc.CacheLatestBlockTimestamp("evm:1") + assert.False(t, known, "head unknown before the first poll") + + gc.pollBlockHeadsOnce(context.Background()) + ts, known := gc.CacheLatestBlockTimestamp("evm:1") + assert.True(t, known) + assert.Equal(t, int64(5_000), ts) + gc.mu.RLock() + assert.Equal(t, uint64(100), gc.earliestByNetwork["evm:1"]) + assert.Equal(t, uint64(500), gc.latestByNetwork["evm:1"]) + assert.Equal(t, uint64(400), gc.finalizedByNetwork["evm:1"]) + gc.mu.RUnlock() + + ctx := context.Background() + t.Run("block before earliest is a fast miss without a round trip", func(t *testing.T) { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x5",false]}`)) + req.SetNetwork(fakeBlockNumberNetwork{}) + req.SetEvmBlockNumber(uint64(5)) + before := srv.calls.Load() + _, err := gc.Get(ctx, ConnectorMainIndex, "pk", "rk", req) + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "got %v", err) + assert.Equal(t, before, srv.calls.Load()) + }) + + t.Run("block at or after earliest is read through", func(t *testing.T) { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}`)) + req.SetNetwork(fakeBlockNumberNetwork{}) + req.SetEvmBlockNumber(uint64(500)) + got, err := gc.Get(ctx, ConnectorMainIndex, "pk", "rk", req) + require.NoError(t, err) + assert.Contains(t, string(got), `"number":"0x1f4"`) + }) + + t.Run("eth_blockNumber reflects the server's latest", func(t *testing.T) { + got, err := gc.Get(ctx, ConnectorMainIndex, "pk", "rk", blockNumberRequest(t)) + require.NoError(t, err) + assert.Equal(t, `"0x1f4"`, string(got)) + }) + + t.Run("a second server for the same chain is ignored", func(t *testing.T) { + dup := newTestGrpcConnector(t, &common.GrpcConnectorConfig{Servers: []string{addr, addr}}) + require.Equal(t, util.StateReady, dup.initializer.State()) + dup.mu.RLock() + assert.Len(t, dup.clientByNetwork, 1) + dup.mu.RUnlock() + }) +} + +func TestGrpcConnector_PollKeepsPriorHeadsWhenTagsAreMissing(t *testing.T) { + srv := &bdsTestServer{chainID: 1, blocks: map[string][2]uint64{"latest": {7, 70}}} + gc := newTestGrpcConnector(t, &common.GrpcConnectorConfig{Servers: []string{startBdsTestServer(t, srv)}}) + gc.mu.Lock() + gc.earliestByNetwork["evm:1"] = 3 + gc.finalizedByNetwork["evm:1"] = 6 + gc.mu.Unlock() + + gc.pollBlockHeadsOnce(context.Background()) + + gc.mu.RLock() + defer gc.mu.RUnlock() + assert.Equal(t, uint64(3), gc.earliestByNetwork["evm:1"], "a null earliest block leaves the prior value") + assert.Equal(t, uint64(6), gc.finalizedByNetwork["evm:1"], "a null finalized block leaves the prior value") + assert.Equal(t, uint64(7), gc.latestByNetwork["evm:1"]) +} + +func TestGrpcConnector_ConfiguredNetworkIdSkipsProbe(t *testing.T) { + srv := &bdsTestServer{chainErr: status.Error(codes.Unimplemented, "no chain id")} + gc := newTestGrpcConnector(t, &common.GrpcConnectorConfig{Servers: []string{startBdsTestServer(t, srv)}, NetworkId: "svm:solana-mainnet"}) + require.Equal(t, util.StateReady, gc.initializer.State()) + assert.Zero(t, srv.calls.Load(), "no eth_chainId probe when the network is configured") + gc.mu.RLock() + _, ok := gc.clientByNetwork["svm:solana-mainnet"] + gc.mu.RUnlock() + assert.True(t, ok) +} + +func TestGrpcConnector_InitFailures(t *testing.T) { + t.Run("nil config", func(t *testing.T) { + lg := zerolog.Nop() + _, err := NewGrpcConnector(context.Background(), &lg, "x", nil) + require.ErrorContains(t, err, "requires config") + }) + + t.Run("failed chainId probe leaves the connector not ready", func(t *testing.T) { + srv := &bdsTestServer{chainErr: status.Error(codes.Unavailable, "down")} + gc := newTestGrpcConnector(t, &common.GrpcConnectorConfig{Servers: []string{startBdsTestServer(t, srv)}}) + assert.NotEqual(t, util.StateReady, gc.initializer.State()) + assert.ErrorContains(t, gc.checkReady(), "no gRPC clients initialized") + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}`)) + req.SetNetwork(fakeBlockNumberNetwork{}) + _, err := gc.Get(context.Background(), ConnectorMainIndex, "pk", "rk", req) + assert.ErrorContains(t, err, "no gRPC clients initialized") + }) + + t.Run("unparsable server url is fatal", func(t *testing.T) { + gc := newTestGrpcConnector(t, &common.GrpcConnectorConfig{Servers: []string{"grpc://%zz"}}) + assert.Equal(t, util.StateFatal, gc.initializer.State()) + }) + + t.Run("no servers yields an empty connector", func(t *testing.T) { + gc := newTestGrpcConnector(t, &common.GrpcConnectorConfig{}) + assert.ErrorContains(t, gc.checkReady(), "no gRPC clients initialized") + assert.ErrorContains(t, (&GrpcConnector{}).checkReady(), "initializer not set") + }) + + t.Run("unreachable bootstrap endpoint fails construction", func(t *testing.T) { + lg := zerolog.Nop() + _, err := NewGrpcConnector(context.Background(), &lg, "x", &common.GrpcConnectorConfig{Bootstrap: "http://127.0.0.1:1/servers"}) + require.ErrorContains(t, err, "bootstrap failed") + _, err = NewGrpcConnector(context.Background(), &lg, "x", &common.GrpcConnectorConfig{Bootstrap: "://bad"}) + require.ErrorContains(t, err, "bootstrap failed") + }) +} + +func TestGrpcConnector_BootstrapDiscoversServers(t *testing.T) { + addr := startBdsTestServer(t, &bdsTestServer{chainID: 10}) + cases := map[string]string{ + "json array": fmt.Sprintf(`[%q]`, addr), + "newline separated": fmt.Sprintf("\n %s \n\n", addr), + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + hs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(hs.Close) + gc := newTestGrpcConnector(t, &common.GrpcConnectorConfig{Bootstrap: hs.URL}) + require.Equal(t, util.StateReady, gc.initializer.State()) + gc.mu.RLock() + _, ok := gc.clientByNetwork["evm:10"] + gc.mu.RUnlock() + assert.True(t, ok) + }) + } +} + +// scriptedClient returns a fixed response/error and records the deadline it +// was called with. +type scriptedClient struct { + clients.GrpcBdsClient + resp *common.NormalizedResponse + err error + deadline atomic.Int64 +} + +func (s *scriptedClient) SendRequest(ctx context.Context, _ *common.NormalizedRequest) (*common.NormalizedResponse, error) { + if dl, ok := ctx.Deadline(); ok { + s.deadline.Store(int64(time.Until(dl))) + } + return s.resp, s.err +} + +func TestGrpcConnector_GetOutcomes(t *testing.T) { + getReq := func() *common.NormalizedRequest { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}`)) + req.SetNetwork(fakeBlockNumberNetwork{}) + return req + } + sendErr := errors.New("transport down") + + t.Run("transport error is returned as is", func(t *testing.T) { + gc := newBlockNumberConnector(t, &scriptedClient{err: sendErr}) + _, err := gc.Get(context.Background(), ConnectorMainIndex, "pk", "rk", getReq()) + assert.ErrorIs(t, err, sendErr) + }) + + t.Run("nil response without error is an empty miss", func(t *testing.T) { + gc := newBlockNumberConnector(t, &scriptedClient{}) + got, err := gc.Get(context.Background(), ConnectorMainIndex, "pk", "rk", getReq()) + assert.NoError(t, err) + assert.Nil(t, got) + }) + + t.Run("missing request metadata or method is a miss", func(t *testing.T) { + gc := newBlockNumberConnector(t, &scriptedClient{}) + _, err := gc.Get(context.Background(), ConnectorMainIndex, "pk", "rk", nil) + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound)) + _, err = gc.Get(context.Background(), ConnectorMainIndex, "pk", "rk", common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"params":[]}`))) + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound)) + }) + + t.Run("getTimeout tightens a longer caller deadline", func(t *testing.T) { + cli := &scriptedClient{resp: blockResponse(t, `{"number":"0x1"}`)} + gc := newBlockNumberConnector(t, cli) + gc.getTimeout = 200 * time.Millisecond + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + got, err := gc.Get(ctx, ConnectorMainIndex, "pk", "rk", getReq()) + require.NoError(t, err) + assert.Equal(t, `{"number":"0x1"}`, string(got)) + assert.LessOrEqual(t, time.Duration(cli.deadline.Load()), 200*time.Millisecond) + }) +} + +func TestGrpcConnector_WriteSideIsUnsupported(t *testing.T) { + g := &GrpcConnector{id: "g"} + ctx := context.Background() + assert.True(t, g.ReadOnly()) + assert.Equal(t, "g", g.Id()) + assert.ErrorContains(t, g.Set(ctx, "pk", "rk", nil, nil), "read-only") + assert.ErrorContains(t, g.Delete(ctx, "pk", "rk"), "read-only") + _, err := g.Lock(ctx, "k", time.Second) + assert.ErrorContains(t, err, "read-only") + _, _, err = g.List(ctx, ConnectorMainIndex, 1, "") + assert.ErrorContains(t, err, "List") + _, cleanup, err := g.WatchCounterInt64(ctx, "k") + assert.ErrorContains(t, err, "WatchCounterInt64") + cleanup() // the returned cleanup is safe to call even on error + assert.ErrorContains(t, g.PublishCounterInt64(ctx, "k", CounterInt64State{}), "PublishCounterInt64") +} diff --git a/data/grpc_svm_test.go b/data/grpc_svm_test.go index be288193f..9ad004936 100644 --- a/data/grpc_svm_test.go +++ b/data/grpc_svm_test.go @@ -87,7 +87,7 @@ func TestGrpcConnectorSupportedMethodsGatesReads(t *testing.T) { {name: "eth_getTransactionReceipt still served", networkId: "evm:1", method: "eth_getTransactionReceipt", params: `["0xabc"]`, served: true}, {name: "eth_getBlockReceipts still served", networkId: "evm:1", method: "eth_getBlockReceipts", params: `["0x1"]`, served: true}, {name: "eth_chainId still served", networkId: "evm:1", method: "eth_chainId", params: `[]`, served: true}, - {name: "unlisted svm method is fast-skipped", networkId: "svm:solana-mainnet", method: "getTransaction", params: `["sig"]`, served: false}, + {name: "unlisted svm method is fast-skipped", networkId: "svm:solana-mainnet", method: "getSignaturesForAddress", params: `["addr"]`, served: false}, {name: "unlisted evm method is fast-skipped", networkId: "evm:1", method: "eth_call", params: `[{}]`, served: false}, } diff --git a/data/postgresql_connector_test.go b/data/postgresql_connector_test.go new file mode 100644 index 000000000..3a4d1d064 --- /dev/null +++ b/data/postgresql_connector_test.go @@ -0,0 +1,506 @@ +package data + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var pgTableSeq atomic.Int64 + +// newPgTable returns a table name unique to this test binary run so tests +// sharing one Postgres server never see each other's rows. +func newPgTable(prefix string) string { + return fmt.Sprintf("%s_%d_%d", prefix, time.Now().UnixNano()%1_000_000, pgTableSeq.Add(1)) +} + +func pgConnectorConfig(uri, table string) *common.PostgreSQLConnectorConfig { + return &common.PostgreSQLConnectorConfig{ + Table: table, + ConnectionUri: uri, + InitTimeout: common.Duration(5 * time.Second), + GetTimeout: common.Duration(2 * time.Second), + SetTimeout: common.Duration(2 * time.Second), + MinConns: 1, + MaxConns: 4, + } +} + +// newReadyPgConnector builds a connector against the shared Postgres server +// and waits until it reports ready. +func newReadyPgConnector(t *testing.T, cfg *common.PostgreSQLConnectorConfig) (context.Context, *PostgreSQLConnector) { + t.Helper() + logger := zerolog.New(io.Discard) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + c, err := NewPostgreSQLConnector(ctx, &logger, "pg-"+cfg.Table, cfg) + require.NoError(t, err) + require.Equal(t, util.StateReady, c.initializer.State(), "errors: %v", c.initializer.Errors()) + return ctx, c +} + +// rawPgPool opens a pool directly against the shared server, for arranging +// rows the connector API cannot produce (already-expired rows, legacy schema). +func rawPgPool(t *testing.T, uri string) *pgxpool.Pool { + t.Helper() + pool, err := pgxpool.Connect(context.Background(), uri) + require.NoError(t, err) + t.Cleanup(pool.Close) + return pool +} + +func insertExpiredPgRow(t *testing.T, pool *pgxpool.Pool, table, pk, rk, value string) { + t.Helper() + _, err := pool.Exec(context.Background(), fmt.Sprintf( + `INSERT INTO %s (partition_key, range_key, value, expires_at) VALUES ($1, $2, $3, NOW() - INTERVAL '1 hour')`, table), + pk, rk, []byte(value)) + require.NoError(t, err) +} + +func TestPostgreSQLConnector_SetGetDelete(t *testing.T) { + uri := sharedPostgresURI(t) + table := newPgTable("pg_crud") + ctx, c := newReadyPgConnector(t, pgConnectorConfig(uri, table)) + + t.Run("missing key is record-not-found", func(t *testing.T) { + _, err := c.Get(ctx, ConnectorMainIndex, "nope", "nope", nil) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "got %v", err) + }) + + t.Run("set without ttl then overwrite with ttl", func(t *testing.T) { + require.NoError(t, c.Set(ctx, "pk1", "rk1", []byte("v1"), nil)) + got, err := c.Get(ctx, ConnectorMainIndex, "pk1", "rk1", nil) + require.NoError(t, err) + assert.Equal(t, []byte("v1"), got) + + ttl := time.Hour + require.NoError(t, c.Set(ctx, "pk1", "rk1", []byte("v2"), &ttl)) + got, err = c.Get(ctx, ConnectorMainIndex, "pk1", "rk1", nil) + require.NoError(t, err) + assert.Equal(t, []byte("v2"), got) + }) + + t.Run("binary values round-trip byte-exact", func(t *testing.T) { + val := []byte{0x00, 0xff, 0x10, 0x00, 0x7f} + require.NoError(t, c.Set(ctx, "bin", "rk", val, nil)) + got, err := c.Get(ctx, ConnectorMainIndex, "bin", "rk", nil) + require.NoError(t, err) + assert.Equal(t, val, got) + }) + + t.Run("entry with short ttl expires", func(t *testing.T) { + ttl := 50 * time.Millisecond + require.NoError(t, c.Set(ctx, "short", "rk", []byte("soon-gone"), &ttl)) + require.Eventually(t, func() bool { + _, err := c.Get(ctx, ConnectorMainIndex, "short", "rk", nil) + return common.HasErrorCode(err, common.ErrCodeRecordNotFound) + }, 5*time.Second, 20*time.Millisecond) + }) + + t.Run("zero ttl stores without expiry", func(t *testing.T) { + zero := time.Duration(0) + require.NoError(t, c.Set(ctx, "zero", "rk", []byte("forever"), &zero)) + got, err := c.Get(ctx, ConnectorMainIndex, "zero", "rk", nil) + require.NoError(t, err) + assert.Equal(t, []byte("forever"), got) + }) + + t.Run("delete removes the entry and is idempotent", func(t *testing.T) { + require.NoError(t, c.Set(ctx, "del", "rk", []byte("x"), nil)) + require.NoError(t, c.Delete(ctx, "del", "rk")) + _, err := c.Get(ctx, ConnectorMainIndex, "del", "rk", nil) + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "got %v", err) + require.NoError(t, c.Delete(ctx, "del", "rk")) + }) +} + +func TestPostgreSQLConnector_WildcardGet(t *testing.T) { + uri := sharedPostgresURI(t) + table := newPgTable("pg_wild") + ctx, c := newReadyPgConnector(t, pgConnectorConfig(uri, table)) + pool := rawPgPool(t, uri) + + rows := []struct{ pk, rk, v string }{ + {"evm:1:100", "eth_getBlockByNumber:abc", "b100"}, + {"evm:1:200", "eth_getBlockByNumber:abc", "b200"}, + {"evm:1:300", "eth_call:foo_bar", "underscore"}, + {"evm:1:300", "eth_call:foo-bar", "dash"}, + {"evm:2:999", "eth_getBlockByNumber:abc", "other-chain"}, + } + for _, r := range rows { + require.NoError(t, c.Set(ctx, r.pk, r.rk, []byte(r.v), nil)) + } + // An expired row sorting above every live one must be skipped. + insertExpiredPgRow(t, pool, table, "evm:1:999", "eth_getBlockByNumber:abc", "expired") + + cases := []struct { + name string + index string + pk, rk string + want string + missing bool + }{ + {name: "reverse index picks highest live partition", index: ConnectorReverseIndex, pk: "evm:1:*", rk: "eth_getBlockByNumber:abc", want: "b200"}, + {name: "reverse index is scoped to the chain prefix", index: ConnectorReverseIndex, pk: "evm:2:*", rk: "eth_getBlockByNumber:abc", want: "other-chain"}, + {name: "reverse index with no match is not-found", index: ConnectorReverseIndex, pk: "evm:3:*", rk: "eth_getBlockByNumber:abc", missing: true}, + {name: "main index range prefix", index: ConnectorMainIndex, pk: "evm:1:100", rk: "eth_getBlockByNumber:*", want: "b100"}, + {name: "underscore in range pattern is literal", index: ConnectorMainIndex, pk: "evm:1:300", rk: "eth_call:foo_b*", want: "underscore"}, + {name: "dash pattern does not match underscore row", index: ConnectorMainIndex, pk: "evm:1:300", rk: "eth_call:foo-b*", want: "dash"}, + {name: "main index only expired match is not-found", index: ConnectorMainIndex, pk: "evm:1:999", rk: "eth_getBlockByNumber:*", missing: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := c.Get(ctx, tc.index, tc.pk, tc.rk, nil) + if tc.missing { + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "got %v", err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, string(got)) + }) + } + + t.Run("expired exact key is not-found", func(t *testing.T) { + _, err := c.Get(ctx, ConnectorMainIndex, "evm:1:999", "eth_getBlockByNumber:abc", nil) + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "got %v", err) + }) +} + +func TestPostgreSQLConnector_List(t *testing.T) { + uri := sharedPostgresURI(t) + table := newPgTable("pg_list") + ctx, c := newReadyPgConnector(t, pgConnectorConfig(uri, table)) + pool := rawPgPool(t, uri) + + for i := 1; i <= 3; i++ { + require.NoError(t, c.Set(ctx, fmt.Sprintf("pk%d", i), "rk", []byte(fmt.Sprintf("v%d", i)), nil)) + } + insertExpiredPgRow(t, pool, table, "pk0", "rk", "expired") + + t.Run("returns live rows ordered by key", func(t *testing.T) { + items, next, err := c.List(ctx, ConnectorMainIndex, 10, "") + require.NoError(t, err) + assert.Empty(t, next) + require.Len(t, items, 3) + for i, it := range items { + assert.Equal(t, fmt.Sprintf("pk%d", i+1), it.PartitionKey) + assert.Equal(t, "rk", it.RangeKey) + assert.Equal(t, fmt.Sprintf("v%d", i+1), string(it.Value)) + } + }) + + t.Run("offset token resumes after the given position", func(t *testing.T) { + token := base64.StdEncoding.EncodeToString([]byte(`{"offset":2}`)) + items, next, err := c.List(ctx, ConnectorMainIndex, 10, token) + require.NoError(t, err) + assert.Empty(t, next) + require.Len(t, items, 1) + assert.Equal(t, "pk3", items[0].PartitionKey) + }) + + t.Run("limit caps the page", func(t *testing.T) { + items, _, err := c.List(ctx, ConnectorMainIndex, 2, "") + require.NoError(t, err) + require.Len(t, items, 2) + assert.Equal(t, "pk1", items[0].PartitionKey) + assert.Equal(t, "pk2", items[1].PartitionKey) + }) + + t.Run("token that is not base64 is rejected", func(t *testing.T) { + _, _, err := c.List(ctx, ConnectorMainIndex, 2, "%%%not-base64") + require.ErrorContains(t, err, "invalid pagination token") + }) + + t.Run("token that is not json is rejected", func(t *testing.T) { + token := base64.StdEncoding.EncodeToString([]byte("not-json")) + _, _, err := c.List(ctx, ConnectorMainIndex, 2, token) + require.ErrorContains(t, err, "invalid pagination token format") + }) +} + +func TestPostgreSQLConnector_Unlock(t *testing.T) { + uri := sharedPostgresURI(t) + ctx, c := newReadyPgConnector(t, pgConnectorConfig(uri, newPgTable("pg_unlock"))) + + lock, err := c.Lock(ctx, "k", time.Second) + require.NoError(t, err) + require.NoError(t, lock.Unlock(ctx)) + // A second unlock has no transaction to commit. + require.ErrorContains(t, lock.Unlock(ctx), "no active transaction") + + var nilLock *postgresLock + assert.True(t, nilLock.IsNil()) +} + +func TestPostgreSQLConnector_WatchAndPublishCounter(t *testing.T) { + uri := sharedPostgresURI(t) + ctx, c := newReadyPgConnector(t, pgConnectorConfig(uri, newPgTable("pg_watch"))) + key := "shared/state-poller:evm:1/latest" + + initial := CounterInt64State{Value: 10, UpdatedAt: 1000, UpdatedBy: "a"} + payload, err := common.SonicCfg.Marshal(initial) + require.NoError(t, err) + require.NoError(t, c.Set(ctx, key, "value", payload, nil)) + + updates, cleanup, err := c.WatchCounterInt64(ctx, key) + require.NoError(t, err) + + select { + case st := <-updates: + assert.Equal(t, initial, st, "watch must deliver the stored value first") + case <-time.After(5 * time.Second): + t.Fatal("no initial value delivered") + } + + // A second watcher on the same key reuses the existing listener. + updates2, cleanup2, err := c.WatchCounterInt64(ctx, key) + require.NoError(t, err) + <-updates2 // initial value + + next := CounterInt64State{Value: 11, UpdatedAt: 2000, UpdatedBy: "b"} + require.Eventually(t, func() bool { + require.NoError(t, c.PublishCounterInt64(ctx, key, next)) + select { + case st := <-updates: + return st == next + case <-time.After(200 * time.Millisecond): + return false + } + }, 10*time.Second, 10*time.Millisecond, "published value must reach watchers") + + select { + case st := <-updates2: + assert.Equal(t, next, st) + case <-time.After(5 * time.Second): + t.Fatal("second watcher did not receive publish") + } + + cleanup() + _, open := <-updates + assert.False(t, open, "cleanup must close the updates channel") + cleanup2() +} + +func TestPostgreSQLConnector_WatchCounterSkipsUnparsableStoredValue(t *testing.T) { + uri := sharedPostgresURI(t) + ctx, c := newReadyPgConnector(t, pgConnectorConfig(uri, newPgTable("pg_watchbad"))) + + require.NoError(t, c.Set(ctx, "k", "value", []byte("not-json"), nil)) + st, ok, err := c.getCurrentValue(ctx, "k") + require.NoError(t, err) + assert.False(t, ok) + assert.Equal(t, CounterInt64State{}, st) + + _, ok, err = c.getCurrentValue(ctx, "absent") + require.NoError(t, err) + assert.False(t, ok, "absent key is reported as missing, not as an error") +} + +func TestPostgreSQLConnector_MigratesLegacyTextValueColumn(t *testing.T) { + uri := sharedPostgresURI(t) + table := newPgTable("pg_legacy") + pool := rawPgPool(t, uri) + _, err := pool.Exec(context.Background(), fmt.Sprintf( + `CREATE TABLE %s (partition_key TEXT, range_key TEXT, value TEXT, PRIMARY KEY (partition_key, range_key))`, table)) + require.NoError(t, err) + _, err = pool.Exec(context.Background(), fmt.Sprintf(`INSERT INTO %s VALUES ('pk', 'rk', 'legacy')`, table)) + require.NoError(t, err) + + ctx, c := newReadyPgConnector(t, pgConnectorConfig(uri, table)) + + got, err := c.Get(ctx, ConnectorMainIndex, "pk", "rk", nil) + require.NoError(t, err) + assert.Equal(t, []byte("legacy"), got, "legacy text rows must survive the BYTEA migration") + + var dataType string + require.NoError(t, pool.QueryRow(context.Background(), + `SELECT data_type FROM information_schema.columns WHERE table_name = $1 AND column_name = 'value'`, table).Scan(&dataType)) + assert.Equal(t, "bytea", dataType) + + ttl := time.Hour + require.NoError(t, c.Set(ctx, "pk2", "rk", []byte{0x00, 0x01}, &ttl), "migrated table must accept writes with ttl") +} + +func TestPostgreSQLConnector_ReconnectSwapsPoolWithoutReapplyingSchema(t *testing.T) { + uri := sharedPostgresURI(t) + cfg := pgConnectorConfig(uri, newPgTable("pg_reconnect")) + ctx, c := newReadyPgConnector(t, cfg) + require.NoError(t, c.Set(ctx, "pk", "rk", []byte("before"), nil)) + + c.connMu.RLock() + oldPool := c.conn + c.connMu.RUnlock() + + // Drop the table: if the second connect re-ran DDL it would recreate it. + pool := rawPgPool(t, uri) + _, err := pool.Exec(context.Background(), fmt.Sprintf("DROP TABLE %s", cfg.Table)) + require.NoError(t, err) + + require.NoError(t, c.connectTask(ctx, cfg)) + + c.connMu.RLock() + newPool := c.conn + c.connMu.RUnlock() + assert.NotSame(t, oldPool, newPool, "reconnect must publish a fresh pool") + + _, err = c.Get(ctx, ConnectorMainIndex, "pk", "rk", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not exist", "schema must not be re-applied on reconnect") +} + +func TestPostgreSQLConnector_ConnectionLossTriggersRecovery(t *testing.T) { + uri := sharedPostgresURI(t) + ctx, c := newReadyPgConnector(t, pgConnectorConfig(uri, newPgTable("pg_recover"))) + + // A non-transport error never demotes a ready connector. + c.handleConnectionFailure(errors.New("syntax error at or near")) + require.Equal(t, util.StateReady, c.initializer.State()) + + c.handleConnectionFailure(io.EOF) + require.NotEqual(t, util.StateReady, c.initializer.State(), "transport failure must mark the connector for reconnect") + + // The pool stays usable while the initializer reconnects in the background. + require.NoError(t, c.Set(ctx, "pk", "rk", []byte("v"), nil)) + + require.Eventually(t, func() bool { + return c.initializer.State() == util.StateReady + }, 20*time.Second, 50*time.Millisecond, "auto-retry must bring the connector back to ready") + + got, err := c.Get(ctx, ConnectorMainIndex, "pk", "rk", nil) + require.NoError(t, err) + assert.Equal(t, []byte("v"), got) +} + +func TestPostgreSQLConnector_CleanupExpiredRemovesOnlyExpiredRows(t *testing.T) { + uri := sharedPostgresURI(t) + table := newPgTable("pg_cleanup") + ctx, c := newReadyPgConnector(t, pgConnectorConfig(uri, table)) + pool := rawPgPool(t, uri) + + require.NoError(t, c.Set(ctx, "live", "rk", []byte("v"), nil)) + insertExpiredPgRow(t, pool, table, "dead1", "rk", "x") + insertExpiredPgRow(t, pool, table, "dead2", "rk", "x") + + countRows := func() int { + var n int + require.NoError(t, pool.QueryRow(context.Background(), fmt.Sprintf("SELECT COUNT(*) FROM %s", table)).Scan(&n)) + return n + } + require.Equal(t, 3, countRows()) + + require.NoError(t, c.cleanupExpired(ctx)) + assert.Equal(t, 1, countRows()) + require.NoError(t, c.cleanupExpired(ctx), "nothing left to delete is not an error") + + t.Run("cleanup loop runs on its ticker and stops with ctx", func(t *testing.T) { + insertExpiredPgRow(t, pool, table, "dead3", "rk", "x") + lg := zerolog.Nop() + loopCtx, stop := context.WithCancel(context.Background()) + p := &PostgreSQLConnector{logger: &lg, conn: pool, table: table, cleanupTicker: time.NewTicker(10 * time.Millisecond)} + defer p.cleanupTicker.Stop() + done := make(chan struct{}) + go func() { p.startCleanup(loopCtx); close(done) }() + require.Eventually(t, func() bool { return countRows() == 1 }, 5*time.Second, 10*time.Millisecond) + stop() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("cleanup loop did not stop on ctx cancel") + } + }) + + t.Run("cleanup loop without ticker returns immediately", func(t *testing.T) { + lg := zerolog.Nop() + p := &PostgreSQLConnector{logger: &lg} + done := make(chan struct{}) + go func() { p.startCleanup(context.Background()); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("startCleanup with nil ticker must return") + } + }) +} + +func TestPostgreSQLConnector_SchemaFailureLeavesConnectorNotReady(t *testing.T) { + uri := sharedPostgresURI(t) + logger := zerolog.New(io.Discard) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // A hyphen is not valid in an unquoted identifier, so CREATE TABLE fails. + c, err := NewPostgreSQLConnector(ctx, &logger, "pg-bad-table", pgConnectorConfig(uri, "bad-table")) + require.NoError(t, err) + assert.NotEqual(t, util.StateReady, c.initializer.State()) + err = c.Set(ctx, "pk", "rk", []byte("v"), nil) + assert.ErrorIs(t, err, ErrConnectorNotReady, "the half-open pool must be discarded") +} + +func TestPostgreSQLConnector_SkipSchemaSetupOnMissingTable(t *testing.T) { + uri := sharedPostgresURI(t) + cfg := pgConnectorConfig(uri, newPgTable("pg_noschema")) + cfg.SkipSchemaSetup = true + ctx, c := newReadyPgConnector(t, cfg) + + _, err := c.Get(ctx, ConnectorMainIndex, "pk", "rk", nil) + require.Error(t, err) + assert.False(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "a missing table is a query error, not a miss") + assert.Equal(t, util.StateReady, c.initializer.State(), "an application-level error must not trigger reconnect") +} + +func TestPostgreSQLConnector_InitFailures(t *testing.T) { + logger := zerolog.New(io.Discard) + + t.Run("unparsable uri is fatal", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, err := NewPostgreSQLConnector(ctx, &logger, "pg-bad-uri", pgConnectorConfig("postgres://%zz", "t")) + require.NoError(t, err) + assert.Equal(t, util.StateFatal, c.initializer.State()) + }) + + t.Run("unsupported iam auth mode is fatal", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cfg := pgConnectorConfig("postgres://u@127.0.0.1:1/db?sslmode=disable", "t") + cfg.IAMAuth = &common.PostgreSQLIAMAuthConfig{Enabled: true, Region: "us-east-1", Auth: &common.AwsAuthConfig{Mode: "bogus"}} + c, err := NewPostgreSQLConnector(ctx, &logger, "pg-bad-iam", cfg) + require.NoError(t, err) + assert.Equal(t, util.StateFatal, c.initializer.State()) + }) + + t.Run("unreachable host returns promptly and every entry point reports not-ready", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cfg := pgConnectorConfig("postgres://u:p@127.0.0.1:1/db?sslmode=disable", "t") + cfg.InitTimeout = common.Duration(300 * time.Millisecond) + start := time.Now() + c, err := NewPostgreSQLConnector(ctx, &logger, "pg-unreachable", cfg) + require.NoError(t, err) + assert.Less(t, time.Since(start), 5*time.Second, "constructor must not hang on an unreachable host") + + assert.ErrorIs(t, c.Set(ctx, "pk", "rk", nil, nil), ErrConnectorNotReady) + _, err = c.Get(ctx, ConnectorMainIndex, "pk", "rk", nil) + assert.ErrorIs(t, err, ErrConnectorNotReady) + _, err = c.Lock(ctx, "k", time.Second) + assert.ErrorIs(t, err, ErrConnectorNotReady) + assert.ErrorIs(t, c.Delete(ctx, "pk", "rk"), ErrConnectorNotReady) + _, _, err = c.List(ctx, ConnectorMainIndex, 1, "") + assert.ErrorIs(t, err, ErrConnectorNotReady) + assert.ErrorIs(t, c.PublishCounterInt64(ctx, "k", CounterInt64State{}), ErrConnectorNotReady) + assert.Equal(t, "pg-unreachable", c.Id()) + }) +} diff --git a/data/redis_connector_test.go b/data/redis_connector_test.go new file mode 100644 index 000000000..1e0711b24 --- /dev/null +++ b/data/redis_connector_test.go @@ -0,0 +1,622 @@ +package data + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "io" + "math/big" + "net" + "os" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/redis/go-redis/v9" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newReadyRedisConnector starts a miniredis server and a connector bound to +// it, waiting until the connector reports ready. +func newReadyRedisConnector(t *testing.T, mutate func(*common.RedisConnectorConfig)) (context.Context, *RedisConnector, *miniredis.Miniredis) { + t.Helper() + s := miniredis.RunT(t) + cfg := &common.RedisConnectorConfig{ + Addr: s.Addr(), + InitTimeout: common.Duration(2 * time.Second), + GetTimeout: common.Duration(2 * time.Second), + SetTimeout: common.Duration(2 * time.Second), + LockRetryInterval: common.Duration(50 * time.Millisecond), + } + if mutate != nil { + mutate(cfg) + } + require.NoError(t, cfg.SetDefaults()) + logger := zerolog.New(io.Discard) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + c, err := NewRedisConnector(ctx, &logger, "redis-test", cfg) + require.NoError(t, err) + require.Equal(t, util.StateReady, c.initializer.State(), "errors: %v", c.initializer.Errors()) + return ctx, c, s +} + +func TestRedisConnector_SetGetDeleteAndTTL(t *testing.T) { + ctx, c, s := newReadyRedisConnector(t, nil) + assert.Equal(t, "redis-test", c.Id()) + + _, err := c.Get(ctx, ConnectorMainIndex, "pk", "missing", nil) + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "got %v", err) + + ttl := time.Minute + big := bytes.Repeat([]byte("x"), 2048) + require.NoError(t, c.Set(ctx, "pk", "big", big, &ttl)) + got, err := c.Get(ctx, ConnectorMainIndex, "pk", "big", nil) + require.NoError(t, err) + assert.Equal(t, big, got) + assert.Equal(t, time.Minute, s.TTL("pk:big")) + + s.FastForward(2 * time.Minute) + _, err = c.Get(ctx, ConnectorMainIndex, "pk", "big", nil) + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "expired key must be a miss, got %v", err) + + require.NoError(t, c.Set(ctx, "pk", "keep", []byte("v"), nil)) + assert.Equal(t, time.Duration(0), s.TTL("pk:keep"), "nil ttl stores a persistent key") + require.NoError(t, c.Delete(ctx, "pk", "keep")) + assert.False(t, s.Exists("pk:keep")) +} + +func TestRedisConnector_ReverseIndexResolution(t *testing.T) { + ctx, c, s := newReadyRedisConnector(t, nil) + rctx := WithReverseIndex(ctx, "evm:1", "") + wildcard := common.CachePartitionKey("evm:1", "", "*") + short := 30 * time.Second + + require.NoError(t, c.Set(rctx, "evm:1:100", "rk-ttl", []byte("with-ttl"), &short)) + require.NoError(t, c.Set(rctx, "evm:1:200", "rk-persist", []byte("persistent"), nil)) + + got, err := c.Get(ctx, ConnectorReverseIndex, wildcard, "rk-ttl", nil) + require.NoError(t, err) + assert.Equal(t, "with-ttl", string(got)) + + got, err = c.Get(ctx, ConnectorReverseIndex, wildcard, "rk-persist", nil) + require.NoError(t, err) + assert.Equal(t, "persistent", string(got)) + + t.Run("dangling reverse entry is a miss", func(t *testing.T) { + // The reverse entry outlives the primary key it points to. + s.Del("evm:1:200:rk-persist") + _, err := c.Get(ctx, ConnectorReverseIndex, wildcard, "rk-persist", nil) + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "got %v", err) + }) + + t.Run("no reverse entry falls back to the literal key", func(t *testing.T) { + _, err := c.Get(ctx, ConnectorReverseIndex, wildcard, "never-set", nil) + assert.True(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "got %v", err) + }) + + t.Run("delete removes primary and reverse entries", func(t *testing.T) { + require.NoError(t, c.Delete(rctx, "evm:1:100", "rk-ttl")) + assert.False(t, s.Exists("evm:1:100:rk-ttl")) + assert.False(t, s.Exists(redisReverseIndexPrefix+"#"+wildcard+"#rk-ttl")) + }) +} + +func TestRedisConnector_List(t *testing.T) { + ctx, c, _ := newReadyRedisConnector(t, nil) + rctx := WithReverseIndex(ctx, "evm:1", "") + require.NoError(t, c.Set(ctx, "a", "1", []byte("va"), nil)) + require.NoError(t, c.Set(ctx, "b", "2", []byte("vb"), nil)) + require.NoError(t, c.Set(rctx, "evm:1:100", "rk", []byte("vblock"), nil)) + + t.Run("main index lists primary keys split on the first colon", func(t *testing.T) { + items, next, err := c.List(ctx, ConnectorMainIndex, 100, "") + require.NoError(t, err) + assert.Empty(t, next) + got := map[string]string{} + for _, it := range items { + got[it.PartitionKey+"|"+it.RangeKey] = string(it.Value) + } + assert.Equal(t, "va", got["a|1"]) + assert.Equal(t, "vb", got["b|2"]) + assert.Equal(t, "vblock", got["evm|1:100:rk"]) + }) + + t.Run("reverse index lists only reverse entries", func(t *testing.T) { + items, _, err := c.List(ctx, ConnectorReverseIndex, 100, "") + require.NoError(t, err) + require.Len(t, items, 1) + assert.Equal(t, common.CachePartitionKey("evm:1", "", "*"), items[0].PartitionKey) + assert.Equal(t, "rk", items[0].RangeKey) + assert.Equal(t, "evm:1:100", string(items[0].Value), "reverse entry value is the concrete partition key") + }) + + t.Run("cursor token pages through keys", func(t *testing.T) { + seen := map[string]bool{} + token := "" + for range 10 { + items, next, err := c.List(ctx, ConnectorMainIndex, 1, token) + require.NoError(t, err) + for _, it := range items { + seen[it.PartitionKey+":"+it.RangeKey] = true + } + if next == "" { + break + } + token = next + } + keys := make([]string, 0, len(seen)) + for k := range seen { + keys = append(keys, k) + } + sort.Strings(keys) + assert.Contains(t, keys, "a:1") + assert.Contains(t, keys, "b:2") + }) + + t.Run("non-numeric token is rejected", func(t *testing.T) { + _, _, err := c.List(ctx, ConnectorMainIndex, 1, "abc") + require.ErrorContains(t, err, "invalid pagination token") + }) +} + +func TestRedisConnector_LockSemantics(t *testing.T) { + ctx, c, s := newReadyRedisConnector(t, nil) + + t.Run("held lock fails fast once tries are exhausted", func(t *testing.T) { + lock, err := c.Lock(ctx, "held", 300*time.Millisecond) + require.NoError(t, err) + assert.False(t, lock.IsNil()) + defer func() { _ = lock.Unlock(ctx) }() + + start := time.Now() + _, err = c.Lock(ctx, "held", 300*time.Millisecond) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to acquire lock for key 'held'") + assert.Less(t, time.Since(start), 5*time.Second) + }) + + t.Run("deadline shorter than the operation buffer still bounds the wait", func(t *testing.T) { + lock, err := c.Lock(ctx, "short-deadline", 5*time.Second) + require.NoError(t, err) + defer func() { _ = lock.Unlock(ctx) }() + + dctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) + defer cancel() + _, err = c.Lock(dctx, "short-deadline", 5*time.Second) + require.Error(t, err) + }) + + t.Run("long deadline is capped by the lock ttl", func(t *testing.T) { + dctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + lock, err := c.Lock(dctx, "long-deadline", time.Second) + require.NoError(t, err) + require.NoError(t, lock.Unlock(ctx)) + }) + + t.Run("unlocking an expired lock reports an error", func(t *testing.T) { + lock, err := c.Lock(ctx, "expires", time.Second) + require.NoError(t, err) + s.FastForward(2 * time.Second) + err = lock.Unlock(ctx) + require.Error(t, err) + }) + + t.Run("unlocking a lock taken over by another owner is refused", func(t *testing.T) { + lock, err := c.Lock(ctx, "stolen", 5*time.Second) + require.NoError(t, err) + require.NoError(t, s.Set("lock:stolen", "someone-else")) + require.Error(t, lock.Unlock(ctx)) + }) + + var nilLock *redisLock + assert.True(t, nilLock.IsNil()) +} + +func TestRedisConnector_WatchAndPublishCounter(t *testing.T) { + ctx, c, _ := newReadyRedisConnector(t, nil) + key := "shared/latest" + + stored := CounterInt64State{Value: 5, UpdatedAt: 100, UpdatedBy: "seed"} + payload, err := common.SonicCfg.Marshal(stored) + require.NoError(t, err) + require.NoError(t, c.Set(ctx, key, "value", payload, nil)) + + updates, cleanup, err := c.WatchCounterInt64(ctx, key) + require.NoError(t, err) + defer cleanup() + + select { + case st := <-updates: + assert.Equal(t, stored, st, "subscriber receives the stored value first") + case <-time.After(5 * time.Second): + t.Fatal("no initial value") + } + + pub := CounterInt64State{Value: 6, UpdatedAt: 200, UpdatedBy: "pub"} + require.NoError(t, c.PublishCounterInt64(ctx, key, pub)) + select { + case st := <-updates: + assert.Equal(t, pub, st) + case <-time.After(5 * time.Second): + t.Fatal("published value not delivered") + } + + t.Run("unparsable and zero-timestamp messages are ignored", func(t *testing.T) { + require.NoError(t, c.client.Publish(ctx, "counter:"+key, "garbage").Err()) + require.NoError(t, c.client.Publish(ctx, "counter:"+key, `{"v":9,"t":0}`).Err()) + valid := CounterInt64State{Value: 7, UpdatedAt: 300} + require.NoError(t, c.PublishCounterInt64(ctx, key, valid)) + select { + case st := <-updates: + assert.Equal(t, valid, st, "only the valid message reaches the subscriber") + case <-time.After(5 * time.Second): + t.Fatal("valid publish not delivered") + } + }) + + t.Run("poll pushes the stored value to subscribers", func(t *testing.T) { + polled := CounterInt64State{Value: 8, UpdatedAt: 400} + b, err := common.SonicCfg.Marshal(polled) + require.NoError(t, err) + require.NoError(t, c.Set(ctx, key, "value", b, nil)) + c.pubsubManager.pollAllKeys() + select { + case st := <-updates: + assert.Equal(t, polled, st) + case <-time.After(5 * time.Second): + t.Fatal("poll did not deliver") + } + }) +} + +func TestRedisPubSubManager_GetCurrentValue(t *testing.T) { + ctx, c, _ := newReadyRedisConnector(t, nil) + m := c.pubsubManager + + cases := []struct { + name string + stored []byte + wantOk bool + }{ + {name: "absent", stored: nil}, + {name: "not json", stored: []byte("{")}, + {name: "zero timestamp", stored: []byte(`{"v":1,"t":0}`)}, + {name: "valid", stored: []byte(`{"v":3,"t":9,"b":"x"}`), wantOk: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + key := "gcv-" + tc.name + if tc.stored != nil { + require.NoError(t, c.Set(ctx, key, "value", tc.stored, nil)) + } + st, ok, err := m.getCurrentValue(ctx, key) + require.NoError(t, err) + assert.Equal(t, tc.wantOk, ok) + if tc.wantOk { + assert.Equal(t, CounterInt64State{Value: 3, UpdatedAt: 9, UpdatedBy: "x"}, st) + } + }) + } +} + +func TestRedisPubSubManager_ReconnectsAfterServerRestart(t *testing.T) { + ctx, c, s := newReadyRedisConnector(t, nil) + key := "restart" + updates, cleanup, err := c.WatchCounterInt64(ctx, key) + require.NoError(t, err) + defer cleanup() + + addr := s.Addr() + s.Close() + require.NoError(t, s.StartAddr(addr)) + + // The message loop must notice the dropped connection and resubscribe; + // keep publishing until a value comes through. + n := int64(0) + require.Eventually(t, func() bool { + n++ + _ = c.PublishCounterInt64(ctx, key, CounterInt64State{Value: n, UpdatedAt: n}) + select { + case st := <-updates: + return st.UpdatedAt > 0 + case <-time.After(100 * time.Millisecond): + return false + } + }, 20*time.Second, 50*time.Millisecond, "subscriber must keep receiving after a redis restart") +} + +func TestRedisPubSubManager_StopClosesSubscribers(t *testing.T) { + s := miniredis.RunT(t) + cfg := &common.RedisConnectorConfig{Addr: s.Addr()} + require.NoError(t, cfg.SetDefaults()) + logger := zerolog.New(io.Discard) + ctx, cancel := context.WithCancel(context.Background()) + c, err := NewRedisConnector(ctx, &logger, "redis-stop", cfg) + require.NoError(t, err) + + _, cleanup, err := c.pubsubManager.Subscribe("k") + require.NoError(t, err) + defer cleanup() + v, ok := c.pubsubManager.subscribers.Load("k") + require.True(t, ok) + sc := v.([]*subscriberChannel)[0] + + cancel() + select { + case <-sc.done: + case <-time.After(5 * time.Second): + t.Fatal("subscriber not closed when the app context ends") + } + _, _, err = c.pubsubManager.Subscribe("k") + require.ErrorContains(t, err, "pubsub manager is stopped") +} + +func TestRedisPubSubManager_SubscribeRestartsStoppedManager(t *testing.T) { + _, c, _ := newReadyRedisConnector(t, nil) + m := c.pubsubManager + m.stop() + require.False(t, m.running.Load()) + + ch, cleanup, err := m.Subscribe("k") + require.NoError(t, err) + defer cleanup() + assert.NotNil(t, ch) + assert.True(t, m.running.Load(), "Subscribe must restart a stopped manager") + + t.Run("start fails without a client", func(t *testing.T) { + lg := zerolog.Nop() + bare := &RedisPubSubManager{connector: &RedisConnector{}, logger: &lg, appCtx: context.Background()} + _, _, err := bare.Subscribe("k") + require.ErrorContains(t, err, "redis client not available") + assert.False(t, bare.running.Load()) + }) +} + +func TestRedisConnector_OperationsFailAfterServerLoss(t *testing.T) { + ctx, c, s := newReadyRedisConnector(t, func(cfg *common.RedisConnectorConfig) { + cfg.GetTimeout = common.Duration(300 * time.Millisecond) + cfg.SetTimeout = common.Duration(300 * time.Millisecond) + }) + s.Close() + + require.Error(t, c.Set(ctx, "pk", "rk", []byte("v"), nil)) + _, err := c.Get(ctx, ConnectorMainIndex, "pk", "rk", nil) + assert.Error(t, err) + assert.False(t, common.HasErrorCode(err, common.ErrCodeRecordNotFound), "transport failure is not a miss") + assert.Error(t, c.Delete(ctx, "pk", "rk")) + _, _, err = c.List(ctx, ConnectorMainIndex, 10, "") + assert.Error(t, err) + assert.Error(t, c.PublishCounterInt64(ctx, "k", CounterInt64State{Value: 1, UpdatedAt: 1})) +} + +func TestRedisConnector_MarkConnectionLost(t *testing.T) { + _, c, _ := newReadyRedisConnector(t, nil) + + benign := []error{ + nil, + redis.Nil, + redis.TxFailedErr, + context.Canceled, + context.DeadlineExceeded, + common.NewErrRecordNotFound("pk", "rk", RedisDriverName), + errors.New("WRONGTYPE Operation against a key holding the wrong kind of value"), + } + for _, err := range benign { + c.markConnectionAsLostIfNecessary(err) + require.Equal(t, util.StateReady, c.initializer.State(), "%v must not mark the connection lost", err) + } + + c.markConnectionAsLostIfNecessary(errors.New("dial tcp: connection refused")) + require.NotEqual(t, util.StateReady, c.initializer.State()) + assert.Error(t, c.checkReady()) + + // The retry finds the existing client healthy and keeps it. + client := c.client + require.Eventually(t, func() bool { return c.initializer.State() == util.StateReady }, 20*time.Second, 50*time.Millisecond) + assert.Same(t, client, c.client, "a healthy client must be reused, not replaced") + + (&RedisConnector{}).markConnectionAsLostIfNecessary(errors.New("connection refused")) // nil initializer is a no-op +} + +func TestRedisConnector_NotReadyEntryPoints(t *testing.T) { + ctx := context.Background() + lg := zerolog.Nop() + cases := map[string]*RedisConnector{ + "nil initializer": {logger: &lg}, + "no client": {logger: &lg, initializer: readyInitializer(t)}, + "no redsync": {logger: &lg, initializer: readyInitializer(t), client: redis.NewClient(&redis.Options{Addr: "127.0.0.1:1"})}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + assert.Error(t, c.Set(ctx, "pk", "rk", nil, nil)) + _, err := c.Get(ctx, ConnectorMainIndex, "pk", "rk", nil) + assert.Error(t, err) + _, err = c.Lock(ctx, "k", time.Second) + assert.Error(t, err) + assert.Error(t, c.Delete(ctx, "pk", "rk")) + _, _, err = c.List(ctx, ConnectorMainIndex, 1, "") + assert.Error(t, err) + _, _, err = c.WatchCounterInt64(ctx, "k") + assert.Error(t, err) + assert.Error(t, c.PublishCounterInt64(ctx, "k", CounterInt64State{})) + }) + } + + t.Run("watch without pubsub manager", func(t *testing.T) { + _, c, _ := newReadyRedisConnector(t, nil) + c.pubsubManager = nil + _, _, err := c.WatchCounterInt64(ctx, "k") + require.ErrorContains(t, err, "pubsub manager not initialized") + }) +} + +// readyInitializer returns an initializer whose single task has succeeded. +func readyInitializer(t *testing.T) *util.Initializer { + t.Helper() + lg := zerolog.Nop() + i := util.NewInitializer(context.Background(), &lg, nil) + require.NoError(t, i.ExecuteTasks(context.Background(), util.NewBootstrapTask("ok", func(context.Context) error { return nil }))) + require.Equal(t, util.StateReady, i.State()) + return i +} + +func TestRedisConnector_InitFailures(t *testing.T) { + logger := zerolog.New(io.Discard) + newConn := func(t *testing.T, cfg *common.RedisConnectorConfig) *RedisConnector { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + c, err := NewRedisConnector(ctx, &logger, "redis-init", cfg) + require.NoError(t, err) + return c + } + + t.Run("unparsable uri is fatal", func(t *testing.T) { + c := newConn(t, &common.RedisConnectorConfig{URI: "mysql://nope"}) + assert.Equal(t, util.StateFatal, c.initializer.State()) + }) + + t.Run("unsupported iam auth mode is fatal", func(t *testing.T) { + c := newConn(t, &common.RedisConnectorConfig{ + URI: "rediss://127.0.0.1:1", + IAMAuth: &common.RedisIAMAuthConfig{Enabled: true, CacheName: "c", UserID: "u", Region: "us-east-1", Auth: &common.AwsAuthConfig{Mode: "bogus"}}, + }) + assert.Equal(t, util.StateFatal, c.initializer.State()) + }) + + t.Run("iam auth against an unreachable host is retryable", func(t *testing.T) { + c := newConn(t, &common.RedisConnectorConfig{ + URI: "rediss://127.0.0.1:1", + InitTimeout: common.Duration(200 * time.Millisecond), + IAMAuth: &common.RedisIAMAuthConfig{Enabled: true, CacheName: "c", UserID: "u", Region: "us-east-1", + Auth: &common.AwsAuthConfig{Mode: "secret", AccessKeyID: "AK", SecretAccessKey: "SK"}}, + }) + st := c.initializer.State() + assert.NotEqual(t, util.StateReady, st) + assert.NotEqual(t, util.StateFatal, st) + }) + + t.Run("uri timeouts fill unset connector timeouts", func(t *testing.T) { + s := miniredis.RunT(t) + c := newConn(t, &common.RedisConnectorConfig{URI: "redis://" + s.Addr() + "?dial_timeout=2s&read_timeout=3s&write_timeout=4s"}) + require.Equal(t, util.StateReady, c.initializer.State()) + assert.Equal(t, 2*time.Second, c.initTimeout) + assert.Equal(t, 3*time.Second, c.getTimeout) + assert.Equal(t, 4*time.Second, c.setTimeout) + }) +} + +// writeSelfSignedCert writes a self-signed server cert/key for 127.0.0.1 and +// returns their paths plus a TLS config that serves it. +func writeSelfSignedCert(t *testing.T) (certFile, keyFile string, serverCfg *tls.Config) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "127.0.0.1"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + keyDer, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + dir := t.TempDir() + certFile = filepath.Join(dir, "cert.pem") + keyFile = filepath.Join(dir, "key.pem") + require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600)) + require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDer}), 0o600)) + pair, err := tls.LoadX509KeyPair(certFile, keyFile) + require.NoError(t, err) + return certFile, keyFile, &tls.Config{Certificates: []tls.Certificate{pair}, MinVersion: tls.VersionTLS12} +} + +func TestRedisConnector_TLS(t *testing.T) { + certFile, keyFile, serverCfg := writeSelfSignedCert(t) + s, err := miniredis.RunTLS(serverCfg) + require.NoError(t, err) + t.Cleanup(s.Close) + logger := zerolog.New(io.Discard) + + connect := func(t *testing.T, cfg *common.RedisConnectorConfig) *RedisConnector { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + cfg.InitTimeout = common.Duration(2 * time.Second) + require.NoError(t, cfg.SetDefaults()) + c, err := NewRedisConnector(ctx, &logger, "redis-tls", cfg) + require.NoError(t, err) + return c + } + + t.Run("yaml tls with the server CA connects over redis://", func(t *testing.T) { + c := connect(t, &common.RedisConnectorConfig{URI: "redis://" + s.Addr(), TLS: &common.TLSConfig{Enabled: true, CAFile: certFile}}) + require.Equal(t, util.StateReady, c.initializer.State(), "errors: %v", c.initializer.Errors()) + require.NoError(t, c.Set(context.Background(), "pk", "rk", []byte("tls"), nil)) + }) + + t.Run("rediss:// merges yaml CA and client cert", func(t *testing.T) { + c := connect(t, &common.RedisConnectorConfig{URI: "rediss://" + s.Addr(), TLS: &common.TLSConfig{Enabled: true, CAFile: certFile, CertFile: certFile, KeyFile: keyFile}}) + require.Equal(t, util.StateReady, c.initializer.State(), "errors: %v", c.initializer.Errors()) + }) + + t.Run("untrusted server certificate is fatal with guidance", func(t *testing.T) { + c := connect(t, &common.RedisConnectorConfig{URI: "redis://" + s.Addr(), TLS: &common.TLSConfig{Enabled: true}}) + require.Equal(t, util.StateFatal, c.initializer.State()) + assert.Contains(t, c.initializer.Errors().Error(), "tls.caFile") + }) + + t.Run("certificate signed by a different CA names the caFile", func(t *testing.T) { + otherCA, otherKey, _ := writeSelfSignedCert(t) + c := connect(t, &common.RedisConnectorConfig{URI: "redis://" + s.Addr(), TLS: &common.TLSConfig{Enabled: true, CAFile: otherCA, CertFile: otherCA, KeyFile: otherKey}}) + require.Equal(t, util.StateFatal, c.initializer.State()) + msg := c.initializer.Errors().Error() + assert.Contains(t, msg, "signed by the provided 'tls.caFile'") + assert.Contains(t, msg, "client certificate") + }) + + t.Run("rediss:// without yaml tls uses the uri default", func(t *testing.T) { + c := connect(t, &common.RedisConnectorConfig{URI: "rediss://" + s.Addr()}) + // The URI-implied config verifies against system CAs, which do not + // trust this self-signed server. + assert.NotEqual(t, util.StateReady, c.initializer.State()) + }) +} + +func TestRedisZerologAdapter_LevelsByMessage(t *testing.T) { + var buf bytes.Buffer + prev := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.DebugLevel) + t.Cleanup(func() { zerolog.SetGlobalLevel(prev) }) + lg := zerolog.New(&buf).Level(zerolog.DebugLevel) + a := &zerologAdapter{logger: &lg} + + a.Printf(context.Background(), "redis: discarding bad PubSub connection: %s\n", "EOF") + assert.Contains(t, buf.String(), `"level":"warn"`) + assert.NotContains(t, buf.String(), `\n"`, "trailing newline is trimmed") + + buf.Reset() + a.Printf(context.Background(), "redis: pool stats %d", 3) + assert.Empty(t, buf.String(), "routine messages are logged at trace level") +} diff --git a/data/redis_ratelimiter_iam.go b/data/redis_ratelimiter_iam.go index f4f54e5ba..22c9cddab 100644 --- a/data/redis_ratelimiter_iam.go +++ b/data/redis_ratelimiter_iam.go @@ -6,7 +6,6 @@ import ( "net/url" "strconv" "strings" - "time" envoyratelimitredis "github.com/envoyproxy/ratelimit/src/redis" "github.com/mediocregopher/radix/v3" @@ -59,12 +58,11 @@ func NewIAMRateLimitClient(cfg *common.RedisConnectorConfig) (envoyratelimitredi return radix.Dial(network, addr, opts...) } + // No 5ms PoolPipelineWindow: PipeDo sends each check as one explicit + // radix.Pipeline, which radix's implicit pipeliner never holds back, + // matching NewClientImpl with window 0 (upstream/ratelimiter_registry.go). pool, err := radix.NewPool("tcp", addr, cfg.ConnPoolSize, radix.PoolConnFunc(df), - // Must match NewClientImpl defaults — remoteAdmissionCap() sizes the - // per-budget admission channel as connPoolSize*32, which depends on - // implicit pipelining at exactly this window and limit. - radix.PoolPipelineWindow(5*time.Millisecond, 32), // Proactively recycle connections before ElastiCache's 12h server-side // force-disconnect; each replacement dials with a fresh IAM token. // ponytail: no jitter (radix lacks the knob); connections spread @@ -110,18 +108,11 @@ func (c *iamRateLimitClient) PipeAppend(pipeline envoyratelimitredis.Pipeline, r return append(pipeline, radix.FlatCmd(rcv, cmd, key, args...)) } -// PipeDo runs each command individually and returns on the first error, -// matching clientImpl.PipeDo's implicit-pipelining path (driver_impl.go). -// The envoy ratelimit library constructs one pipeline per DoLimit call -// (a single INCRBY+EXPIRE pair for one key), so early-return on error -// never abandons commands for a different key mid-pipeline. +// PipeDo sends the whole pipeline in one round trip, matching clientImpl.PipeDo +// without implicit pipelining (driver_impl.go). A Redis pipeline is not a +// transaction: an error still fails the whole check, which onFailure decides. func (c *iamRateLimitClient) PipeDo(pipeline envoyratelimitredis.Pipeline) error { - for _, action := range pipeline { - if err := c.pool.Do(action); err != nil { - return err - } - } - return nil + return c.pool.Do(radix.Pipeline(pipeline...)) } func (c *iamRateLimitClient) Close() error { @@ -134,4 +125,4 @@ func (c *iamRateLimitClient) Close() error { // only called in envoyproxy/ratelimit's own benchmark tests. func (c *iamRateLimitClient) NumActiveConns() int { return 0 } -func (c *iamRateLimitClient) ImplicitPipeliningEnabled() bool { return true } +func (c *iamRateLimitClient) ImplicitPipeliningEnabled() bool { return false } diff --git a/data/shared_containers_test.go b/data/shared_containers_test.go new file mode 100644 index 000000000..2006ee629 --- /dev/null +++ b/data/shared_containers_test.go @@ -0,0 +1,115 @@ +package data + +import ( + "context" + "fmt" + "os" + "sync" + "testing" + "time" + + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +// Shared containers are started lazily, once per test binary, by the first +// test that needs them, and terminated by TestMain. Each test isolates its +// data with its own table name, so sharing a server is safe. +type sharedContainer struct { + once sync.Once + c testcontainers.Container + addr string + err error +} + +var ( + sharedPostgres sharedContainer + sharedDynamo sharedContainer +) + +func TestMain(m *testing.M) { + code := m.Run() + for _, sc := range []*sharedContainer{&sharedPostgres, &sharedDynamo} { + if sc.c != nil { + _ = sc.c.Terminate(context.Background()) + } + } + os.Exit(code) +} + +// sharedPostgresURI returns a connection URI for a Postgres server shared by +// the whole test binary. +func sharedPostgresURI(t *testing.T) string { + t.Helper() + sharedPostgres.once.Do(func() { + ctx := context.Background() + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "postgres:15-alpine", + Env: map[string]string{"POSTGRES_PASSWORD": "password"}, + ExposedPorts: []string{"5432/tcp"}, + WaitingFor: wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(2 * time.Minute), + }, + Started: true, + }) + sharedPostgres.c = c + if err != nil { + sharedPostgres.err = err + return + } + host, err := c.Host(ctx) + if err != nil { + sharedPostgres.err = err + return + } + port, err := c.MappedPort(ctx, "5432") + if err != nil { + sharedPostgres.err = err + return + } + sharedPostgres.addr = fmt.Sprintf("postgres://postgres:password@%s:%s/postgres?sslmode=disable", host, port.Port()) + }) + if sharedPostgres.err != nil { + t.Fatalf("failed to start shared postgres container: %v", sharedPostgres.err) + } + return sharedPostgres.addr +} + +// sharedDynamoEndpoint returns the endpoint of a DynamoDB Local server shared +// by the whole test binary. +func sharedDynamoEndpoint(t *testing.T) string { + t.Helper() + sharedDynamo.once.Do(func() { + ctx := context.Background() + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "amazon/dynamodb-local", + ExposedPorts: []string{"8000/tcp"}, + WaitingFor: wait.ForListeningPort("8000/tcp"), + }, + Started: true, + }) + sharedDynamo.c = c + if err != nil { + sharedDynamo.err = err + return + } + host, err := c.Host(ctx) + if err != nil { + sharedDynamo.err = err + return + } + port, err := c.MappedPort(ctx, "8000") + if err != nil { + sharedDynamo.err = err + return + } + sharedDynamo.addr = fmt.Sprintf("http://%s:%s", host, port.Port()) + }) + if sharedDynamo.err != nil { + t.Fatalf("failed to start shared dynamodb container: %v", sharedDynamo.err) + } + return sharedDynamo.addr +} diff --git a/docs/package.json b/docs/package.json index 8cce96474..20f8c14ca 100644 --- a/docs/package.json +++ b/docs/package.json @@ -13,7 +13,7 @@ "@radix-ui/react-slot": "^1.3.3", "@svgr/webpack": "^8.1.0", "clsx": "^2.1.1", - "next": "^15.5.25", + "next": "^15.5.28", "sharp": "0.35.5", "nextra": "^2.13.4", "nextra-theme-docs": "^2.13.4", diff --git a/docs/pages/_meta.js b/docs/pages/_meta.js index 4bf9a07d8..b169e82c6 100644 --- a/docs/pages/_meta.js +++ b/docs/pages/_meta.js @@ -38,6 +38,11 @@ module.exports = { title: "Reference", }, reference: { title: "Reference", display: "children" }, + "-- Compare": { + type: "separator", + title: "Compare", + }, + compare: { title: "Compare", display: "children" }, // Presets / examples are still accessible via direct URL (/presets/*) // but are not surfaced in the sidebar. presets: { display: "hidden" }, diff --git a/docs/pages/compare/_meta.js b/docs/pages/compare/_meta.js new file mode 100644 index 000000000..b8246f560 --- /dev/null +++ b/docs/pages/compare/_meta.js @@ -0,0 +1,5 @@ +module.exports = { + "erpc-vs-dshackle": { title: "eRPC vs dshackle" }, + "erpc-vs-web3-proxy": { title: "eRPC vs web3-proxy" }, + "erpc-vs-drpc-nodecore": { title: "eRPC vs dRPC NodeCore" }, +}; diff --git a/docs/pages/compare/erpc-vs-drpc-nodecore.mdx b/docs/pages/compare/erpc-vs-drpc-nodecore.mdx new file mode 100644 index 000000000..27b8c3c6d --- /dev/null +++ b/docs/pages/compare/erpc-vs-drpc-nodecore.mdx @@ -0,0 +1,92 @@ +--- +title: eRPC vs dRPC NodeCore +description: eRPC vs drpc nodecore feature by feature, with a source link in every cell, and where NodeCore is the better fit. +--- + +import { LLMsTxtLink, AISection } from "../../components"; + + + +# eRPC vs dRPC NodeCore + +[NodeCore](https://github.com/drpcorg/nodecore) is dRPC's open-source RPC stack in Go. +It covers 15+ chain families over JSON-RPC, REST and gRPC, and links to dRPC's hosted network. +eRPC is an EVM and Solana JSON-RPC proxy built around failover, a finality-aware cache, and cost control. +Every cell below links to the page or line that states it. Facts were checked on 2026-10-22. + +## Feature table + +| | eRPC | dRPC NodeCore | +|---|---|---| +| Language, license | Go, Apache-2.0 ([LICENSE](https://github.com/erpc/erpc/blob/32a8a1e450c6ab5a78477859c85221b6a9058c72/LICENSE#L1)) | Go, MIT ([go.mod](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/go.mod#L3), [LICENSE](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/LICENSE#L1)) | +| Latest release | 0.4.0, 2026-10-11 ([release](https://github.com/erpc/erpc/releases/tag/0.4.0)) | 1.17.1, 2026-10-22 ([release](https://github.com/drpcorg/nodecore/releases/tag/1.17.1)) | +| Chains | EVM and Solana ([networks](/config/projects/networks)) | EVM, Solana, Bitcoin, Starknet, TON, Cosmos, Sui, Aptos and more ([README](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/README.md#L16)) | +| Upstream protocols | JSON-RPC over HTTP and WebSocket, gRPC ([upstreams](/config/projects/upstreams)) | JSON-RPC, WebSocket, REST, Tendermint, gRPC ([upstream config](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/05-upstream-config.md#L593-L640)) | +| Config | YAML or TypeScript ([quick start](/)) | YAML, reload on SIGHUP or file change ([config](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/01-config.md#L270-L290)) | +| Upstream selection | Score from errors, latency, 429s, block lag; JS policy ([selection policies](/config/projects/selection-policies)) | Score from latency, error rate, lag; custom TypeScript scoring ([upstream config](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/05-upstream-config.md#L378-L446)) | +| Failover and retry | Yes, per method ([retry](/config/failsafe/retry)) | Yes, backoff and jitter ([upstream config](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/05-upstream-config.md#L244-L295)) | +| Hedged requests | Yes ([hedge](/config/failsafe/hedge)) | Yes ([upstream config](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/05-upstream-config.md#L273-L295)) | +| Circuit breaker | Yes ([circuit breaker](/config/failsafe/circuit-breaker)) | No named breaker; validators and method bans remove upstreams ([upstream config](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/05-upstream-config.md#L778-L848)) | +| Cross-checking answers | Consensus across any N upstreams ([consensus](/config/failsafe/consensus)) | Quorum over signed responses from dRPC providers ([quorum](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/10-quorum.md#L1-L49)) | +| Integrity | Tip and range guards, opt-in schema checks ([integrity](/config/failsafe/integrity)) | Block numbers never go back; lag, chain-id, log-index validators ([upstream config](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/05-upstream-config.md#L225-L242)) | +| Cache | Memory, Redis, PostgreSQL, DynamoDB; finality-aware ([drivers](/config/database/drivers), [cache](/config/database/evm-json-rpc-cache)) | Memory, Redis, PostgreSQL; finalized-only policies ([cache](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/04-cache.md#L72-L158)) | +| Rate limits | Per client, project, network, upstream ([rate limiters](/config/rate-limiters)) | Upstream side, with auto-tune ([rate limiting](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/06-rate-limiting.md#L1-L48)) | +| Client auth | Secret, JWT, SIWE, IP, database ([auth](/config/auth)) | Token, JWT, API keys with IP, method and contract allowlists ([auth](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/03-auth.md#L66-L215)) | +| `eth_getLogs` splitting | Yes ([getLogs splitting](/reference/evm/getlogs-splitting)) | Not documented ([request_processor.go](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/internal/upstreams/flow/request_processor.go#L318-L331)) | +| JSON-RPC batch | Yes ([batch](/operation/batch)) | Yes ([batch e2e test](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/test/e2e/http/batch_e2e_test.go#L15-L40)) | +| Subscriptions | EVM `eth_subscribe` only; shared across clients ([websocket](/operation/websocket)) | EVM and other chain families, Solana included; shared across clients ([subscriptions](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/13-subscriptions.md#L1-L30)) | +| Metrics, tracing | Prometheus, OpenTelemetry ([monitoring](/operation/monitoring), [tracing](/operation/tracing)) | Prometheus, Pyroscope ([metrics](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/08-prometheus-metrics.md), [server config](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/02-server-config.md#L42)) | +| Archive routing | Block availability per upstream ([upstreams](/config/projects/upstreams)) | Detects each upstream's lowest block and routes by height ([upstream config](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/05-upstream-config.md#L742-L760)) | +| Deployment | Binaries, Docker, Kubernetes manifests, `npx start-erpc` ([docker](/deployment/docker), [kubernetes](/deployment/kubernetes)) | Binaries, Docker, Helm chart ([README](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/README.md#L125-L129), [chart](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/chart/nodecore/README.md#L10)) | +| Hosted option | Managed instances ([cloud](/deployment/cloud)) | dRPC NodeCloud; keys can sync from it ([integration](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/09-integration.md#L12-L30)) | + +"Not documented" means the NodeCore docs do not describe it and a search of its source found no code for it. +The [dRPC NodeCore page](https://drpc.org/docs/nodecore-open-source-rpc-stack) also lists a circuit breaker and OpenTelemetry traces. The open-source repo at the commit above does not document them, so the table does not count them. + +## Where NodeCore is stronger + +- **Chain coverage.** 15+ chain families, with REST, Tendermint and gRPC upstreams ([README](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/README.md#L16)). eRPC serves EVM and Solana only. +- **Subscriptions beyond EVM.** Solana and other chain families, and EVM streams built from all upstreams together ([subscriptions](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/13-subscriptions.md#L1-L30)). eRPC serves only EVM `eth_subscribe`. +- **Signed-response quorum** with dRPC providers ([quorum](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/10-quorum.md#L32-L38)). +- **Contract allowlists per API key** for `eth_call` and `eth_getLogs` ([auth](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/docs/nodecore/03-auth.md#L172-L207)). +- **Official Helm chart** ([chart](https://github.com/drpcorg/nodecore/blob/8d8d79af258b5f7897ec77709eb495c29e744973/chart/nodecore/README.md#L10)). eRPC ships raw manifests ([kubernetes](/deployment/kubernetes)). + +## Where eRPC is stronger + +- Consensus across any upstreams, not only signed ones ([consensus](/config/failsafe/consensus)). +- Circuit breaker and OpenTelemetry tracing in the open-source code ([circuit breaker](/config/failsafe/circuit-breaker), [tracing](/operation/tracing)). +- `eth_getLogs` range splitting ([getLogs splitting](/reference/evm/getlogs-splitting)). +- Rate limits on clients, projects and networks, not only upstreams ([rate limiters](/config/rate-limiters)). +- DynamoDB cache driver and finality-aware cache buckets ([cache](/config/database/evm-json-rpc-cache)). +- Config in TypeScript, with typed selection policies ([selection policies](/config/projects/selection-policies)). + + + +### Sources and snapshot + +- eRPC: `erpc/erpc` at `32a8a1e450c6ab5a78477859c85221b6a9058c72`, docs at docs.erpc.cloud. +- NodeCore: `drpcorg/nodecore` at `8d8d79af258b5f7897ec77709eb495c29e744973` (2026-10-22), docs in `docs/nodecore/`. +- Checked 2026-10-22. NodeCore cuts releases often; re-check rows before you rely on them. Absence claims rest on its docs and a source search of `internal/`. + +### Moving from NodeCore to eRPC + +| NodeCore concept | eRPC equivalent | +|---|---| +| `upstream-config.upstreams[]` | `projects[].upstreams[]` ([upstreams](/config/projects/upstreams)) | +| `chain: ethereum` | `networks[].evm.chainId: 1` ([networks](/config/projects/networks)) | +| `failsafe-config.retry` / `hedge` | `failsafe[].retry` / `failsafe[].hedge` ([retry](/config/failsafe/retry), [hedge](/config/failsafe/hedge)) | +| TypeScript scoring function | selection policy ([selection policies](/config/projects/selection-policies)) | +| cache policies | `database.evmJsonRpcCache.policies` ([cache](/config/database/evm-json-rpc-cache)) | + +### Keep NodeCore when + +- You serve non-EVM, non-Solana chains. +- You route to dRPC providers and want their signed-response quorum. + +### Related pages + +- [Why eRPC?](/why) +- [eRPC vs dshackle](/compare/erpc-vs-dshackle) +- [eRPC vs web3-proxy](/compare/erpc-vs-web3-proxy) + + diff --git a/docs/pages/compare/erpc-vs-dshackle.mdx b/docs/pages/compare/erpc-vs-dshackle.mdx new file mode 100644 index 000000000..cfb79536c --- /dev/null +++ b/docs/pages/compare/erpc-vs-dshackle.mdx @@ -0,0 +1,97 @@ +--- +title: eRPC vs dshackle +description: eRPC vs dshackle (emeraldpay) feature by feature, with a source link in every cell, and where dshackle is the better fit. +--- + +import { LLMsTxtLink, AISection } from "../../components"; + + + +# eRPC vs dshackle + +[dshackle](https://github.com/emeraldpay/dshackle) and eRPC both put one endpoint in front of many blockchain nodes. +dshackle comes from wallet infrastructure: a gRPC API with streams, signed responses, and deep Bitcoin support. +eRPC is an EVM and Solana JSON-RPC proxy built around failover, a finality-aware cache, and cost control. +Every cell below links to the page or line that states it. Facts were checked on 2026-10-22. + +## Feature table + +| | eRPC | dshackle | +|---|---|---| +| Language, license | Go, Apache-2.0 ([LICENSE](https://github.com/erpc/erpc/blob/32a8a1e450c6ab5a78477859c85221b6a9058c72/LICENSE#L1)) | Rust on `master`, Kotlin in v0.17.0; Apache-2.0 ([Cargo.toml](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/Cargo.toml#L1-L4), [LICENSE](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/LICENSE)) | +| Latest release | 0.4.0, 2026-10-11 ([release](https://github.com/erpc/erpc/releases/tag/0.4.0)) | v0.17.0, 2026-04-01 ([release](https://github.com/emeraldpay/dshackle/releases/tag/v0.17.0)) | +| Chains | EVM and Solana ([networks](/config/projects/networks)) | EVM and Bitcoin ([upstream config](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/04-upstream-config.adoc#L13-L34)) | +| Client APIs | JSON-RPC over HTTP and WebSocket ([websocket](/operation/websocket)) | JSON-RPC over HTTP and WebSocket, plus a gRPC API ([methods](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/07-methods.adoc#L1-L30)) | +| Config | YAML or TypeScript ([quick start](/)) | YAML with `include` and `${ENV}` ([server config](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/03-server-config.adoc#L1-L10)) | +| Upstream selection | Score from errors, latency, 429s, block lag ([selection policies](/config/projects/selection-policies)) | Role tiers plus round-robin; lag and sync filters ([upstream config](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/04-upstream-config.adoc#L154-L177)) | +| Failover and retry | Yes, per method ([retry](/config/failsafe/retry)) | Primary, secondary, fallback with growing delay ([upstream config](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/04-upstream-config.adoc#L167-L177)) | +| Hedged requests | Yes ([hedge](/config/failsafe/hedge)) | Not documented ([source tree](https://github.com/emeraldpay/dshackle/tree/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/src)) | +| Circuit breaker | Yes ([circuit breaker](/config/failsafe/circuit-breaker)) | Pauses upstreams that return 429 or overload ([pause.rs](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/src/upstream/pause.rs#L15-L59)) | +| Cross-checking answers | Consensus across N upstreams with a threshold ([consensus](/config/failsafe/consensus)) | Per-method quorum: not lagging, non-empty, max nonce, broadcast ([ethereum.rs](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/src/upstream/methods/ethereum.rs#L64-L90)) | +| Signed responses | No | Yes, names the upstream that answered ([methods](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/07-methods.adoc#L94-L117)) | +| Cache | Memory, Redis, PostgreSQL, DynamoDB; finality-aware ([drivers](/config/database/drivers), [cache](/config/database/evm-json-rpc-cache)) | Memory and Redis; blocks, txs, receipts by hash ([caching](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/10-caching.adoc#L1-L16)) | +| Rate limits | Per client, project, network, upstream ([rate limiters](/config/rate-limiters)) | None for clients ([overload.rs](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/src/upstream/overload.rs#L15-L35)) | +| Client auth | Secret, JWT, SIWE, IP, database ([auth](/config/auth)) | TLS client certificates ([authentication](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/08-authentication.adoc#L68)) | +| `eth_getLogs` splitting | Yes ([getLogs splitting](/reference/evm/getlogs-splitting)) | Not documented; passed through ([ethereum.rs](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/src/upstream/methods/ethereum.rs#L122)) | +| JSON-RPC batch | Yes ([batch](/operation/batch)) | Yes ([reference](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/reference-configuration.adoc#L420-L424)) | +| `eth_subscribe` | newHeads, logs, newPendingTransactions ([websocket](/operation/websocket)) | newHeads, logs, newPendingTransactions, syncing ([egress.rs](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/src/upstream/egress.rs#L42-L45)) | +| Metrics, tracing | Prometheus, OpenTelemetry ([monitoring](/operation/monitoring), [tracing](/operation/tracing)) | Prometheus; JSON access logs, no OpenTelemetry ([monitoring](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/06-monitoring.adoc#L1-L14)) | +| Multi-tenant | Projects, per-user budgets ([projects](/config/projects)) | Not documented ([reference](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/reference-configuration.adoc#L434-L454)) | +| Archive routing | Block availability per upstream ([upstreams](/config/projects/upstreams)) | Labels such as `archive: true`, method on/off ([upstream config](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/04-upstream-config.adoc#L264-L284)) | +| Deployment | Binaries, Docker, Kubernetes manifests, `npx start-erpc` ([docker](/deployment/docker), [kubernetes](/deployment/kubernetes)) | Docker, release zip ([Dockerfile](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/Dockerfile)) | + +"Not documented" means the project's docs do not describe it and a search of its source found no code for it. + +## Where dshackle is stronger + +- **gRPC API with streams.** Head, balance (ETH, ERC-20, BTC with xpub) and tx-status subscriptions ([methods](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/07-methods.adoc#L11-L30)). +- **Signed responses.** A client can prove which upstream produced an answer ([methods](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/07-methods.adoc#L94-L117)). +- **Bitcoin.** bitcoind, Esplora and ZeroMQ notifications ([reference](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/reference-configuration.adoc#L944-L992)). eRPC does not serve Bitcoin. +- **Instance mesh.** dshackle instances link over mTLS gRPC across regions ([architecture](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/01-architecture-intro.adoc#L63-L72)). +- **Fork-aware routing.** Upstreams on the losing fork still serve data below the fork point ([upstream config](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/docs/04-upstream-config.adoc#L119-L150)). + +## Where eRPC is stronger + +- Hedging, circuit breaker and consensus as separate failsafe steps ([failsafe](/config/failsafe)). +- Finality-aware cache on four storage drivers ([cache](/config/database/evm-json-rpc-cache)). +- `eth_getLogs` range splitting ([getLogs splitting](/reference/evm/getlogs-splitting)). +- Client auth and rate limits for public or multi-team endpoints ([auth](/config/auth), [rate limiters](/config/rate-limiters)). +- Solana support ([networks](/config/projects/networks)). + +## Note on dshackle versions + +The latest dshackle release, v0.17.0, is Kotlin. Since April 2026 the `master` branch is a Rust rewrite, version `0.18.0-dev` ([Cargo.toml](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/Cargo.toml#L1-L4)). +The dshackle README says `master` "should not be used in production" ([README](https://github.com/emeraldpay/dshackle/blob/896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78/README.adoc#L308)). +dRPC keeps a separate Kotlin fork, [drpcorg/dshackle](https://github.com/drpcorg/dshackle), with more chains and JWT auth. This page compares `emeraldpay/dshackle`. + + + +### Sources and snapshot + +- eRPC: `erpc/erpc` at `32a8a1e450c6ab5a78477859c85221b6a9058c72`, docs at docs.erpc.cloud. +- dshackle: `emeraldpay/dshackle` `master` at `896ab53b7c8806b5ee481a14aa1e1c04e7bc1a78` (2026-10-13), release `v0.17.0` at `f781e8edc4e35887f684209250412baf530417ae`. +- Checked 2026-10-22. Absence claims ("Not documented") rest on the dshackle docs plus a source search of `master` `src/` and `v0.17.0` `src/main`. No maintainer statement confirms them. + +### Moving from dshackle to eRPC + +| dshackle concept | eRPC equivalent | +|---|---| +| `cluster.upstreams[]` | `projects[].upstreams[]` ([upstreams](/config/projects/upstreams)) | +| `blockchain` per upstream | `networks[].evm.chainId` ([networks](/config/projects/networks)) | +| `role: primary/secondary/fallback` | selection policy ([selection policies](/config/projects/selection-policies)) | +| `methods.enabled[].quorum` | `failsafe[].consensus` ([consensus](/config/failsafe/consensus)) | +| `methods.enabled` / `methods.disabled` | `allowMethods` / `ignoreMethods` ([upstreams](/config/projects/upstreams)) | +| Redis cache | `database.evmJsonRpcCache` with a Redis connector ([cache](/config/database/evm-json-rpc-cache)) | + +### Keep dshackle when + +- Clients use its gRPC streams or signed responses. +- You serve Bitcoin. + +### Related pages + +- [Why eRPC?](/why) +- [eRPC vs web3-proxy](/compare/erpc-vs-web3-proxy) +- [eRPC vs dRPC NodeCore](/compare/erpc-vs-drpc-nodecore) + + diff --git a/docs/pages/compare/erpc-vs-web3-proxy.mdx b/docs/pages/compare/erpc-vs-web3-proxy.mdx new file mode 100644 index 000000000..39305655f --- /dev/null +++ b/docs/pages/compare/erpc-vs-web3-proxy.mdx @@ -0,0 +1,88 @@ +--- +title: eRPC vs web3-proxy +description: eRPC vs web3-proxy (llamanodes) feature by feature, with a source link in every cell, and where web3-proxy is the better fit. +--- + +import { LLMsTxtLink, AISection } from "../../components"; + + + +# eRPC vs web3-proxy + +[web3-proxy](https://github.com/llamanodes/web3-proxy) is the Rust proxy that ran LlamaNodes' public RPC. +It ships a full RPC-as-a-service layer: wallet login, keys, tiers and compute-unit billing. +eRPC is an EVM and Solana JSON-RPC proxy built around failover, a finality-aware cache, and cost control. +Every cell below links to the page or line that states it. Facts were checked on 2026-10-22. + +## Feature table + +| | eRPC | web3-proxy | +|---|---|---| +| Language, license | Go, Apache-2.0 ([LICENSE](https://github.com/erpc/erpc/blob/32a8a1e450c6ab5a78477859c85221b6a9058c72/LICENSE#L1)) | Rust, GPL-3.0 ([LICENSE](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/LICENSE#L1-L2)) | +| Latest release | 0.4.0, 2026-10-11 ([release](https://github.com/erpc/erpc/releases/tag/0.4.0)) | No GitHub releases; last tag v1.43.99, 2023-12-03 ([tags](https://github.com/llamanodes/web3-proxy/tags)) | +| Last commit | 2026-10-22 ([commits](https://github.com/erpc/erpc/commits/main)) | 2023-12-05 ([commit](https://github.com/llamanodes/web3-proxy/commit/f4a71ee0b621c461828956436f6d55372228ec18)) | +| Chains | EVM and Solana, many chains per process ([networks](/config/projects/networks)) | EVM, one `chain_id` per process ([example.toml](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/config/example.toml#L1-L2)) | +| Config | YAML or TypeScript ([quick start](/)) | TOML ([example.toml](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/config/example.toml)) | +| Upstream selection | Score from errors, latency, 429s, block lag ([selection policies](/config/projects/selection-policies)) | Synced head, latency tiers, peak-EWMA load ([one.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/rpcs/one.rs#L270-L365)) | +| Failover and retry | Yes, per method ([retry](/config/failsafe/retry)) | Next ranked upstream on error ([consensus.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/rpcs/consensus.rs#L947-L1013)) | +| Hedged requests | Yes ([hedge](/config/failsafe/hedge)) | No; "fastest" mode is `todo!()` ([many.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/rpcs/many.rs#L583-L590)) | +| Circuit breaker | Yes ([circuit breaker](/config/failsafe/circuit-breaker)) | Not found in source ([src](https://github.com/llamanodes/web3-proxy/tree/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src)) | +| Cross-checking answers | Consensus across N upstreams ([consensus](/config/failsafe/consensus)) | Head-block agreement only; response quorum is `todo!()` ([many.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/rpcs/many.rs#L588)) | +| Cache | Memory, Redis, PostgreSQL, DynamoDB; finality-aware ([drivers](/config/database/drivers), [cache](/config/database/evm-json-rpc-cache)) | In-memory; keys include block hash ([app/mod.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/app/mod.rs#L411-L416), [block_number.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/block_number.rs#L353-L369)) | +| Rate limits | Per client, project, network, upstream ([rate limiters](/config/rate-limiters)) | Per upstream soft/hard limits; per IP and per key tier ([config.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/config.rs#L398-L428), [authorization.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/frontend/authorization.rs#L680-L720)) | +| Client auth | Secret, JWT, SIWE, IP, database ([auth](/config/auth)) | RPC keys, SIWE login, IP/origin/referer allowlists ([authentication.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/frontend/users/authentication.rs#L78-L130)) | +| Billing | No; cost headers for an external meter ([directives](/operation/directives)) | Yes: compute units, balances, deposits, Stripe ([compute_units.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/compute_units.rs#L14-L80)) | +| `eth_getLogs` splitting | Yes ([getLogs splitting](/reference/evm/getlogs-splitting)) | No; ranges over 200,000 blocks rejected ([block_number.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/block_number.rs#L438-L452)) | +| JSON-RPC batch | Yes ([batch](/operation/batch)) | Yes ([app/mod.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/app/mod.rs#L1011-L1017)) | +| `eth_subscribe` | newHeads, logs, newPendingTransactions ([websocket](/operation/websocket)) | newHeads, newPendingTransactions ([ws.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/app/ws.rs#L41-L252)) | +| Metrics, tracing | Prometheus, OpenTelemetry ([monitoring](/operation/monitoring), [tracing](/operation/tracing)) | Prometheus, InfluxDB stats, Sentry ([prometheus.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/prometheus.rs#L15-L44), [Cargo.toml](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/Cargo.toml#L76-L80)) | +| Archive routing | Block availability per upstream ([upstreams](/config/projects/upstreams)) | Probes each upstream's history depth automatically ([one.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/rpcs/one.rs#L368-L490)) | +| Deployment | Binaries, Docker, Kubernetes manifests, `npx start-erpc` ([docker](/deployment/docker), [kubernetes](/deployment/kubernetes)) | Build with cargo or the Dockerfile ([Dockerfile](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/Dockerfile#L204-L216)) | + +"Not found in source" means a search of the web3-proxy source found no code for it. + +## Where web3-proxy is stronger + +- **RPC-as-a-service built in.** Users, sub-users, tiers, compute-unit pricing, on-chain deposits and Stripe ([compute_units.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/compute_units.rs#L14-L80)). eRPC has no billing. +- **Private transaction routing.** `eth_sendRawTransaction` goes to `private_rpcs` such as Flashbots ([app/mod.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/app/mod.rs#L1101-L1185)). +- **ERC-4337 bundler routing** to a separate upstream pool ([app/mod.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/app/mod.rs#L1468-L1477)). +- **Automatic archive-depth probing** per upstream, with no config ([one.rs](https://github.com/llamanodes/web3-proxy/blob/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/rpcs/one.rs#L368-L450)). + +## Where eRPC is stronger + +- Active releases; web3-proxy has had no commit since December 2023. +- Hedging, circuit breaker and response consensus ([failsafe](/config/failsafe)). +- Shared, finality-aware cache on Redis, PostgreSQL or DynamoDB ([cache](/config/database/evm-json-rpc-cache)). +- `eth_getLogs` range splitting instead of a range cap ([getLogs splitting](/reference/evm/getlogs-splitting)). +- Many chains, EVM and Solana, in one process ([networks](/config/projects/networks)). + + + +### Sources and snapshot + +- eRPC: `erpc/erpc` at `32a8a1e450c6ab5a78477859c85221b6a9058c72`, docs at docs.erpc.cloud. +- web3-proxy: `llamanodes/web3-proxy` `main` at `f4a71ee0b621c461828956436f6d55372228ec18` (2023-12-05). The repo is not archived. +- Checked 2026-10-22. "Not found in source" and "No" for hedging, circuit breaker and OpenTelemetry rest on a source search and the `todo!()` branches cited. The code was not run. + +### Moving from web3-proxy to eRPC + +| web3-proxy concept | eRPC equivalent | +|---|---| +| one process per `chain_id` | one process, many `networks[]` ([networks](/config/projects/networks)) | +| `balanced_rpcs` | `projects[].upstreams[]` ([upstreams](/config/projects/upstreams)) | +| `backup = true` upstream | fallback tier in a selection policy ([selection policies](/config/projects/selection-policies)) | +| `soft_limit` / `hard_limit` | upstream rate-limit budget ([rate limiters](/config/rate-limiters)) | +| RPC keys in the URL | `secret` or `database` auth ([auth](/config/auth)) | +| in-memory response cache | `database.evmJsonRpcCache` ([cache](/config/database/evm-json-rpc-cache)) | + +### Keep web3-proxy when + +- You need its built-in billing and user accounts and accept an unmaintained codebase. + +### Related pages + +- [Why eRPC?](/why) +- [eRPC vs dshackle](/compare/erpc-vs-dshackle) +- [eRPC vs dRPC NodeCore](/compare/erpc-vs-drpc-nodecore) + + diff --git a/docs/pages/config/auth.mdx b/docs/pages/config/auth.mdx index 11b98aa65..3dffaa326 100644 --- a/docs/pages/config/auth.mdx +++ b/docs/pages/config/auth.mdx @@ -629,13 +629,14 @@ they naturally expire. Always set `allowedAlgorithms` to prevent algorithm-confu 25. **Admin auth registry DOES support rate-limit budgets.** Unlike the healthcheck scope, the admin `AuthRegistry` is created with the same `rateLimitersRegistry` as projects. Rate-limit budgets on admin auth strategies are applied when configured. 26. **`verificationJwksUrl` must be absolute HTTP(S).** Non-empty values are parsed at startup; scheme must be `http` or `https` and a host is required. Scheme-less hosts, `file://`, and other schemes fail validation even when static `verificationKeys` are also set. [] -27. **JWT `jwt.Parse` failures are classified into `AuthUnauthorizedReason`.** Bad signatures → `invalid_signature` / "invalid signature"; expired tokens (`ValidationErrorExpired`) → `token_expired` / "token expired". If both bits are set, signature wins — claims (including `exp`) are untrusted without a valid sig. Other `ValidationError` flags keep the jwt library text without a `reason` detail. Classification lives in auth (not `common`) to avoid importing `golang-jwt` into `common`. [, ] +27. **JWT `jwt.Parse` failures are classified into closed reasons.** Bad signatures → `invalid_signature`; expired → `token_expired`. If both bits are set, signature wins. Other `ValidationError` flags use `invalid_token` and keep jwt library text as the message. [] +28. **`erpc_auth_failed_total` increments when an `ErrAuthUnauthorized` is returned to the client** (`processErrorBody` / gRPC status mapping via `RecordAuthFailed`). Strategies stamp a closed `Details.reason` via `NewErrAuthUnauthorized`; intermediate strategy rejects on a request that later succeeds are not counted. `agent_name` is always `"unknown"`; `project`/`network` are `"n/a"` unless a resolved network is already on the request. [] ### Observability | Metric | Type | Labels | When it fires | |---|---|---|---| -| `erpc_auth_failed_total` | counter | `project`, `network`, `strategy`, `reason`, `agent_name` | Database strategy only. Reason values: `missing_secret`, `empty_secret`, `cached_unknown_api_key`, `db_fail_open_fast_path`, `db_not_ready`, `db_timeout`, `db_connection`, `db_query_error`, `invalid_api_key`, `disabled_key`, `db_record_parse_error`, `db_record_missing_user_id`, `internal_error`. | +| `erpc_auth_failed_total` | counter | `project`, `network`, `strategy`, `reason`, `agent_name` | Client-facing `ErrAuthUnauthorized` counter via `RecordAuthFailed` (). Closed `reason` sets: **database** — `missing_secret`, `empty_secret`, `cached_unknown_api_key`, `db_not_ready`, `db_timeout`, `db_connection`, `db_query_error`, `invalid_api_key`, `disabled_key`, `db_record_parse_error`, `db_record_missing_user_id`; **jwt** — `parse_failed`, `unexpected_algorithm`, `no_verification_key`, `token_expired`, `invalid_signature`, `invalid_token`, `invalid_claims`, `claims_invalid`, `missing_sub`; **secret** — `invalid_secret`; **siwe** — `missing_payload`, `parse_failed`, `invalid_signature`, `domain_not_allowed`, `message_not_valid_now`; **network** — `missing_request`, `invalid_client_ip`, `ip_not_allowed`. Fallback reason `unauthorized`. `agent_name` always `unknown`. No `user` label. | | `erpc_rate_limits_total` | counter | `project`, `network`, `vendor`, `upstream`, `category`, `finality`, `user`, `agent_name`, `budget`, `scope`, `auth`, `origin` | When auth-level rate-limit budget is exhausted. `origin="auth"`, `auth=":"` (e.g. `"secret:0"`, `"database:1"`). | **Log messages (database strategy):** diff --git a/docs/pages/config/database/shared-state.mdx b/docs/pages/config/database/shared-state.mdx index c85db3756..5582e4b0d 100644 --- a/docs/pages/config/database/shared-state.mdx +++ b/docs/pages/config/database/shared-state.mdx @@ -108,7 +108,7 @@ https://docs.erpc.cloud/config/database/shared-state.llms.txt`} **Foreground latency budget.** `TryUpdateIfStale` is used by the EVM state poller to debounce expensive RPC calls. It launches the upstream RPC in a background goroutine and waits at most `updateMaxWait` for the result. If the RPC finishes within that window the fresh value is returned synchronously; otherwise the stale local value is returned immediately and the RPC completes asynchronously. The foreground path never acquires the distributed lock. -**Rollback handling.** All counter updates enforce a rollback policy. `latestBlock`, `finalizedBlock`, and `servedLatestBlock`/`servedFinalizedBlock` use `ignoreRollbackOf = 1024` — decreases up to 1024 blocks are silently discarded as noise from lagging upstreams; larger gaps (genuine reorgs or provider resets) are accepted and trigger the `OnLargeRollback` callback. `earliestBlock` uses `ignoreRollbackOf = 0` — all decreases are accepted because node pruning genuinely shifts the historical anchor. +**Rollback handling.** All counter updates enforce a rollback policy. `latestBlock`, `finalizedBlock`, and the network-level delivered-head counter use `ignoreRollbackOf = 1024` — decreases up to 1024 blocks are silently discarded as noise from lagging upstreams; larger gaps (genuine reorgs or provider resets) are accepted and trigger the `OnLargeRollback` callback. `earliestBlock` uses `ignoreRollbackOf = 0` — all decreases are accepted because node pruning genuinely shifts the historical anchor. **Timestamp vector clock and conflict resolution.** Each counter state is serialised as `{"v":,"t":,"b":""}`. The `t` field doubles as both a vector clock for conflict resolution and a staleness indicator. `allocateUpdatedAtMs` ensures local timestamps are strictly monotonic even under concurrent goroutines (a CAS spin-loop). Remote state is only adopted when `st.UpdatedAt > currentTs`, providing idempotency and protection against out-of-order network delivery. When a remote update has a newer timestamp but a value that would imply a small rollback (within `ignoreRollbackOf`), `advanceTimestampPast` is called to advance the local timestamp past the remote's so the local (higher) value wins the next reconciliation cycle. @@ -144,7 +144,7 @@ https://docs.erpc.cloud/config/database/shared-state.llms.txt`} **Bootstrap task retry.** If `WatchCounterInt64` fails at startup (connector not ready, network error), `initCounterSync` marks the watch task as failed and the `Initializer` retries it in the background. During this window the counter operates in local-only mode. Source: -**Served-tip partitions.** Beyond per-upstream counters, eRPC also maintains network-wide `servedLatestBlock` and `servedFinalizedBlock` counters, plus per-tag-group partitions for tag-aware routing. The partition key is `"grp:" + hex(sha256(sorted_upstream_ids)[:8])` so equivalent selectors map to the same key. A cap of 16 partitions per network prevents cardinality explosion from pathological configs. Once the cap is reached, new tag-group selectors fall back to the stateless subnet-minimum. +**Served tip.** The served tip itself is not shared: each instance computes it from the shared per-upstream counters (see [Block tracking](/reference/evm/block-tracking)). The only network-level counter is `network///latestBlock`: the highest head any instance has delivered to a WebSocket `newHeads` subscriber, used as a bounded floor for `latest`. Per-group served-tip lanes are in-process telemetry, not shared state. **Fallback when absent.** `erpc/erpc.go:L49-L66` synthesises an in-memory registry if `sharedState == nil`; the same `GetCounterInt64` API works identically but no cross-pod propagation occurs. Init failures are non-fatal: `erpc/init.go` logs a warning and continues with a synthesised in-memory registry. A broken Redis config therefore produces degraded per-pod behavior, not a hard startup failure. @@ -340,7 +340,7 @@ The shared-state layer has no effect on JSON-RPC request or response shapes visi - **Block-number reads** (`latestBlock`, `finalizedBlock`) used by the upstream selector and tag-aware routing come from the in-process atomic copy — zero latency on the request path. - **Block-number refreshes** are debounced via `TryUpdateIfStale`; the foreground caller waits at most `updateMaxWait` (default 50 ms). If the refresh completes within that window the fresh value influences the current request; otherwise the stale value is used and the refresh completes asynchronously for the next request. -- **Served-tip counters** (`servedLatestBlock`, `servedFinalizedBlock`) are updated after a response is selected and do not add latency. They gate the "never serve older than seen" invariant that prevents block-number regressions across requests in a scaled fleet. +- **Delivered-head counter** (`network///latestBlock`) is updated before a `newHeads` notification is delivered and does not add request latency. Other instances honor it as a floor for `latest` only within `maxRetryableBlockDistance` of their freshest live head. - A broken or unreachable shared-state connector degrades to per-pod local state — no error is returned to the client; routing quality degrades silently. - **Operator cordons** (`erpc_cordonUpstream`) are one more shared counter per upstream (`operatorCordon`, value = cordon time in unix-ms, `0` = lifted), with the same propagation, bootstrap fetch and local-first semantics as the block counters. See [Cordoning](/operation/cordoning). @@ -367,7 +367,7 @@ The shared-state layer has no effect on JSON-RPC request or response shapes visi 9. **DynamoDB `PublishCounterInt64` is a no-op.** Cross-pod latency is bounded by `statePollInterval` (default 5 s). Using DynamoDB means significantly higher convergence latency than Redis or PostgreSQL. Source: 10. **`WatchCounterInt64` channel closed on shutdown.** `RedisPubSubManager.stop()` closes all subscriber channels on app shutdown; the registry goroutine observing the channel receives `ok=false` and marks the watch task as failed. This is benign since the app context is also cancelled at that point. Source: 11. **`clusterKey` propagation order.** Explicit `database.sharedState.clusterKey` wins over the inherited top-level `clusterKey`. Source: -12. **Served-tip partition cap at 16 per network.** Once reached, new tag-group selectors fall back to stateless subnet-minimum rather than getting a monotonic counter. Source: +12. **The served tip is per instance.** Pods compute it from the same shared per-upstream heads; nothing about the pick itself is stored. Source: 13. **init.go treats shared-state failure as a warning, not fatal.** A broken Redis config produces degraded per-pod behavior, not a hard startup failure. Source: 14. **`database.sharedState` nil means `SetDefaults` is never called on it.** The synthesised config at `erpc/erpc.go:L49-L66` exists only in local scope and does not write back to `cfg.Database.SharedState`. Source: 15. **Connector `id` auto-assignment differs between shared-state and other scopes.** Shared-state connector id defaults to `string(driver)` (e.g. `"redis"`), not `"shared-state-redis"`. Source: @@ -411,7 +411,7 @@ The shared-state layer has no effect on JSON-RPC request or response shapes visi - [`data/shared_state_variable.go:L364-L707`](https://github.com/erpc/erpc/blob/main/data/shared_state_variable.go#L364-L707) — `TryUpdate`, `TryUpdateIfStale`, `scheduleBackgroundPushCurrent`, lock ordering, rollback handling, timestamp CAS helpers - [`data/shared_state_registry.go:L102-L222`](https://github.com/erpc/erpc/blob/main/data/shared_state_registry.go#L102-L222) — `GetCounterInt64`: idempotent counter creation, bootstrap tasks, `WatchCounterInt64` goroutine - [`data/redis_pubsub_manager.go:L155-L460`](https://github.com/erpc/erpc/blob/main/data/redis_pubsub_manager.go#L155-L460) — `RedisPubSubManager`: single `PSubscribe("counter:*")`, reconnect backoff, periodic poll fallback, copy-on-write subscriber slice -- [`erpc/networks.go:L55-L96`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L55-L96) — served-tip partition key derivation and 16-partition cap +- [`erpc/networks.go:L55-L111`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L55-L111) — delivered-head shared counter (`latestBlockShared`, `NoteObservedLatestBlock`) - [`erpc/erpc.go:L49-L66`](https://github.com/erpc/erpc/blob/main/erpc/erpc.go#L49-L66) — in-memory fallback synthesis when `sharedState == nil` - [`common/defaults.go:L789-L828`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L789-L828) — `SharedStateConfig.SetDefaults`; all default values including connector id override - [`data/shared_state_sources.go`](https://github.com/erpc/erpc/blob/main/data/shared_state_sources.go) — named constants for the `source` field in logs and traces (`UpdateSourceRemoteSync`, `UpdateSourceTryUpdate`, etc.) @@ -420,7 +420,7 @@ The shared-state layer has no effect on JSON-RPC request or response shapes visi ### Related pages - [Connectors](/config/database) — the backing store options (Redis, PostgreSQL, DynamoDB, memory) that power shared state. -- [Tag-aware routing](/config/projects/selection-policies) — uses `servedLatestBlock`/`servedFinalizedBlock` partition counters that shared state keeps monotonic across pods. +- [Tag-aware routing](/config/projects/selection-policies) — `use-upstream` selectors scope the served tip to the matched upstreams; no shared counter is involved. - [EVM state poller](/reference/evm) — the consumer that calls `TryUpdateIfStale` per upstream; registers `OnValue`/`OnLargeRollback` callbacks. - [Survive provider outages](/use-cases/survive-provider-outages) — routing quality under failover depends on consistent block-height awareness across pods. - [Deployment](/deployment) — horizontal scaling guidance where shared state becomes necessary. diff --git a/docs/pages/config/database/svm-json-rpc-cache.mdx b/docs/pages/config/database/svm-json-rpc-cache.mdx index c99cddc0a..c73639617 100644 --- a/docs/pages/config/database/svm-json-rpc-cache.mdx +++ b/docs/pages/config/database/svm-json-rpc-cache.mdx @@ -125,7 +125,7 @@ https://docs.erpc.cloud/config/database/svm-json-rpc-cache.llms.txt`} | 4 | `slotPinnedMethods` at any weaker commitment | `unfinalized` | | 5 | everything else | `realtime` (moving-head read) | -Step 5 is the rule that matters, and it is why step 3 needs an explicit table at all. **`commitment: finalized` on Solana means "the state at the latest ROOTED slot", and the rooted slot advances roughly every 400 ms — it is a moving head, not an immutability promise.** `getBalance`, `getAccountInfo`, `getProgramAccounts`, and `getTokenAccountBalance` at `finalized` therefore answer a *different question every slot*, exactly like EVM's `latest` block tag. Classifying them `finalized` would be a permanent-cache bug: `finalized` is the zero value of the Go enum that a policy with no explicit `finality` matches, and an unset TTL means "no expiry" in every connector — so a later transfer would never invalidate the cached balance. +Step 5 is the rule that matters, and it is why step 3 needs an explicit table at all. **`commitment: finalized` on Solana means "the state at the latest ROOTED slot", and the rooted slot advances roughly every 400 ms — it is a moving head, not an immutability promise.** `getAccountInfo` and `getProgramAccounts` at `finalized` therefore answer a *different question every slot*, exactly like EVM's `latest` block tag. Classifying them `finalized` would be a permanent-cache bug: `finalized` is the zero value of the Go enum that a policy with no explicit `finality` matches, and an unset TTL means "no expiry" in every connector — so a later transfer would never invalidate the cached account. Direct balance reads (`getBalance`, `getTokenAccountBalance`) take that one step further: they are in `neverCacheMethods` and hard-skipped before any policy runs, so a realtime TTL cannot cache them either. Only a response pinned to an explicit positional identifier — a slot number or a transaction signature — is genuinely immutable once that slot is rooted. That is the whole content of `slotPinnedMethods`: @@ -144,10 +144,11 @@ alwaysFinalizedMethods = getInflationReward (defined only over finalized epoch neverCacheMethods = sendTransaction, sendRawTransaction, simulateTransaction, requestAirdrop, getLatestBlockhash, getRecentBlockhash, getFeeForMessage, getSignatureStatuses, getVoteAccounts, getLeaderSchedule, getEpochInfo, - getSlotLeaders, getRecentPerformanceSamples, getRecentPrioritizationFees + getSlotLeaders, getRecentPerformanceSamples, getRecentPrioritizationFees, + getBalance, getTokenAccountBalance ``` -Two categories live in `neverCacheMethods`: **mutating or effectful** calls, where caching would break the at-least-once semantics callers expect, and **transient realtime snapshots** that go stale in under one slot. `getEpochSchedule` is deliberately *excluded* — its constants only change at epoch boundaries (~432,000 slots), so it falls through to the moving-head bucket and is cached under the realtime policy's TTL. +Three categories live in `neverCacheMethods`: **mutating or effectful** calls, where caching would break the at-least-once semantics callers expect; **transient realtime snapshots** that go stale in under one slot; and **direct balance reads** (`getBalance`, `getTokenAccountBalance`) that feed financial decisions and must never return a stale value. `getEpochSchedule` is deliberately *excluded* — its constants only change at epoch boundaries (~432,000 slots), so it falls through to the moving-head bucket and is cached under the realtime policy's TTL. **Never-cache is hard-enforced, not policy-driven.** `realtime` is still a *cacheable* finality at the policy layer, so mapping these methods to `realtime` is not sufficient on its own. `Get` and `Set` both check `neverCacheMethods` **before** policy matching and return early. An operator's stray `finality: realtime` catch-all therefore cannot cache `sendTransaction`. @@ -207,8 +208,9 @@ All fields live under `database.svmJsonRpcCache` and use the **same `CacheConfig method: "*" finality: unfinalized ttl: 800ms - # Everything else: getBalance, getAccountInfo, getProgramAccounts, getSlot, + # Everything else: getAccountInfo, getProgramAccounts, getSlot, # getBlockHeight, getSignaturesForAddress, ... all moving-head reads. + # (getBalance / getTokenAccountBalance are neverCacheMethods — hard-skipped.) - connector: mem network: "*" method: "*" @@ -224,8 +226,9 @@ All fields live under `database.svmJsonRpcCache` and use the **same `CacheConfig // getBlock/getTransaction at confirmed/processed — pinned to a slot, but a // fork switch can still drop it. ~2 slots of tolerance. { connector: "mem", network: "*", method: "*", finality: "unfinalized", ttl: "800ms" }, - // Everything else: getBalance, getAccountInfo, getProgramAccounts, getSlot, + // Everything else: getAccountInfo, getProgramAccounts, getSlot, // getBlockHeight, getSignaturesForAddress, ... all moving-head reads. + // (getBalance / getTokenAccountBalance are neverCacheMethods — hard-skipped.) { connector: "mem", network: "*", method: "*", finality: "realtime", ttl: "2s" }, ], }, @@ -377,7 +380,7 @@ All fields live under `database.svmJsonRpcCache` and use the **same `CacheConfig ### Best practices -- **Always define a `realtime` policy.** It is not an edge case on SVM — it is the majority of your traffic. `getBalance`, `getAccountInfo`, `getProgramAccounts`, `getTokenAccountBalance`, `getSlot`, `getBlockHeight`, and `getSignaturesForAddress` are all `realtime` at *every* commitment level, including `finalized`. A config with only a `finality: finalized` policy caches almost nothing. +- **Always define a `realtime` policy.** It is not an edge case on SVM — it is the majority of your traffic. `getAccountInfo`, `getProgramAccounts`, `getSlot`, `getBlockHeight`, and `getSignaturesForAddress` are all `realtime` at *every* commitment level, including `finalized`. (`getBalance` / `getTokenAccountBalance` are hard-skipped via `neverCacheMethods` and never match a policy.) A config with only a `finality: finalized` policy caches almost nothing. - **Never "fix" a low hit rate by forcing state reads to `finalized`.** There is no config knob to do this on SVM, and that is deliberate — it would be a permanent-cache correctness bug, not a tuning win. If you need higher hit rates on account reads, raise the `realtime` TTL knowingly. - **Keep `realtime` TTL in the 1–3 s range.** One Solana slot is ~400 ms. A 2 s TTL means a caller may observe state up to ~5 slots old; anything much larger starts to surface visibly stale balances. - **Set `unfinalized` TTL to roughly two slots (≈800 ms).** That bucket only ever holds `getBlock`/`getTransaction` below `finalized` commitment, where a fork switch is the risk being bounded. @@ -412,10 +415,13 @@ The SVM cache emits the **same `erpc_cache_*` metric family** as the EVM cache, | `erpc_cache_get_success_miss_total` | counter | Every matched policy missed | | `erpc_cache_get_skipped_total` | counter | No policy matched the request's network/method/finality | | `erpc_cache_get_error_total` | counter | A connector `Get` errored; the walk continues to the next policy | +| `erpc_cache_set_success_total` | counter | A connector `Set` succeeded | | `erpc_cache_set_skipped_total` | counter | Payload rejected by a policy's size limits | | `erpc_cache_set_error_total` | counter | A connector `Set` errored (logged at WARN, swallowed) | -| `erpc_cache_set_original_bytes` / `erpc_cache_set_compressed_bytes` | counter | Pre- and post-zstd payload accounting; the compressed counter only advances when compression actually shrank the value | -| `erpc_cache_get_success_hit_duration` / `erpc_cache_get_success_miss_duration` | histogram | Hit and miss latency | +| `erpc_cache_set_original_bytes_total` / `erpc_cache_set_compressed_bytes_total` | counter | Pre- and post-zstd payload accounting; the compressed counter only advances when compression actually shrank the value | +| `erpc_cache_get_success_hit_duration_seconds` / `erpc_cache_get_success_miss_duration_seconds` / `erpc_cache_set_success_duration_seconds` / `erpc_cache_set_error_duration_seconds` | histogram | Hit, miss, successful-set, and failed-set latency | + +Use the `*_total` **counters** (not duration histograms) for hit/miss rates. `neverCacheMethods` short-circuits stay silent — they are not cache skips and do not inflate skipped counters. **Notably absent on SVM:** `erpc_cache_get_age_guard_reject_total` never fires — there is no block-timestamp age gate. diff --git a/docs/pages/config/example.mdx b/docs/pages/config/example.mdx index f3e2c5488..40ab44499 100644 --- a/docs/pages/config/example.mdx +++ b/docs/pages/config/example.mdx @@ -691,7 +691,7 @@ erpc dump --config erpc.yaml --format json `dump` resolves the effective selection policies (including TypeScript `evalFunc` sentinels back to their source) and prints the complete config as YAML or JSON. -**Defaults cascade.** `SetDefaults` fills every nil/zero field after decode — no field is silently undefined at runtime. If `projects` is empty after loading, a synthetic `main` project with `repository` + `envio` providers is injected automatically. +**Defaults cascade.** `SetDefaults` fills every nil/zero field after decode — no field is silently undefined at runtime. If `projects` is empty after loading, a synthetic `main` project with a `repository` provider is injected automatically. ### Per-section documentation map @@ -728,7 +728,7 @@ erpc dump --config erpc.yaml --format json - `SetDefaults` fills every nil/zero field after decode — no field is silently undefined at runtime. - `Validate` runs after `SetDefaults` and returns an error if any invariant is violated; `erpc validate` exposes the full report. - TypeScript configs are bundled by embedded esbuild (no Node.js required) and evaluated in sobek; `process.env` is pre-populated from `os.Environ()`. -- If `projects` is empty after loading, a synthetic `main` project with `repository` + `envio` providers is injected automatically. +- If `projects` is empty after loading, a synthetic `main` project with a `repository` provider is injected automatically. ### Worked examples @@ -881,7 +881,7 @@ pipeline, env-var semantics, validate/dump tooling — applies to whichever form | `metrics.counterLabelOverrides` | map | nil | **Deprecated** — use `metrics.customizations`. Per-metric label keep-list for counters; key is metric name without `erpc_` prefix. | **Default project synthesis.** When `projects` is empty, `SetDefaults` injects a synthetic `main` project () with: -- `repository` and `envio` providers +- a `repository` provider (id `public`) - An aliasing rule so `/evm/` resolves without a project prefix - Network defaults: `evm.getLogsMaxAllowedRange=30_000`, `getLogsSplitOnError=true`, integrity enforcement enabled - Network-level failsafe: retry(maxAttempts=5), timeout(120s), hedge(p70 quantile, maxCount=2) diff --git a/docs/pages/config/failsafe/integrity.mdx b/docs/pages/config/failsafe/integrity.mdx index 6f7373a0e..7828a79e5 100644 --- a/docs/pages/config/failsafe/integrity.mdx +++ b/docs/pages/config/failsafe/integrity.mdx @@ -128,12 +128,13 @@ at network scope (try another upstream), not retryable at the same upstream, fee Configured under `networks[].directiveDefaults`. On by default; toggle per-request via `X-ERPC-Enforce-*` headers / `?enforce-*=` query params. -**Highest-block enforcement — `eth_blockNumber`.** Reads `dirs.EnforceHighestBlock`. If -the response block is below the network-known highest, eRPC replaces the response with a -synthetic JSON-RPC result containing the highest hex block number — no re-request is -made. Applies to EVERY response source, including cache hits, so a stale value planted in -a shared cache can never be served below the tip this instance knows. The tip is resolved -request-aware: a `use-upstream` selector scopes it to the targeted subset. +**Highest-block enforcement — `eth_blockNumber`.** Reads `dirs.EnforceHighestBlock`. While +the network's served head is known, eRPC answers `eth_blockNumber` from it without +dispatching. Otherwise the request is forwarded, and every response — including cache +hits — is pinned exactly to the served head once it is known: a higher value is lowered, +a lower one raised, with no re-request. The head is resolved request-aware: a +`use-upstream` selector scopes it to the targeted subset. See +[Block tracking](/reference/evm/block-tracking). **Highest-block enforcement — `eth_getBlockByNumber[latest/finalized]`.** Reads `dirs.EnforceHighestBlock`. If the returned block is behind the network-known highest, @@ -162,7 +163,7 @@ not-yet-produced blocks. Fail-open on unknown head or block tags. | +| `enforceHighestBlock` | `*bool` | `true` | Highest-block enforcement for `eth_blockNumber` (answered from the served head; every response pinned to it, incl. cache hits) and `eth_getBlockByNumber[latest/finalized]` (re-fetch). Gates the stale-tip cache write guard. Header: `X-ERPC-Enforce-Highest-Block`. | | `enforceGetLogsBlockRange` | `*bool` | `true` | Pre-screens `eth_getLogs`/`trace_filter` ranges against upstream availability. The actual hooks read `evm.integrity.enforceGetLogsBlockRange` directly. Header: `X-ERPC-Enforce-GetLogs-Range`. | | `enforceNonNullTaggedBlocks` | `*bool` | `true` | Converts null tagged `eth_getBlockByNumber` responses into `ErrEndpointMissingData`. Disable for chains that legitimately return null for some tags (e.g. ZKSync Era). Header: `X-ERPC-Enforce-Non-Null-Tagged-Blocks`. | diff --git a/docs/pages/config/matcher.mdx b/docs/pages/config/matcher.mdx index f3bc57ff2..73a63ab0e 100644 --- a/docs/pages/config/matcher.mdx +++ b/docs/pages/config/matcher.mdx @@ -128,7 +128,7 @@ like `"latest"`), numeric operators fall through to glob matching and will not c Use `0x*` to match hex-format values, and combine with named tags using `|` for full coverage. **`MatchesSelector` and upstream tag awareness.** The `use-upstream` directive and served-tip -partition grouping use `MatchesSelector` rather than plain `WildcardMatch`. The function first +[lane](/reference/lanes) grouping use `MatchesSelector` rather than plain `WildcardMatch`. The function first tests the pattern against the upstream's `id`. If that matches, it returns `true` immediately. If the pattern contains **no `!` anywhere** (purely-positive), it also iterates the upstream's `tags []string` and returns `true` on any tag match. Any `!` anywhere in the pattern — even @@ -231,7 +231,7 @@ connector `failsafeForGets[]`/`failsafeForSets[]`) | Field | Type | Default | Behavior / footguns | |---|---|---|---| -| `networks[].evm.servedTip.guaranteedMethods[]` | []string | — | Each element validated by `ValidatePattern` at config load (the only config field with startup validation). At runtime, `WildcardMatch` against the request method determines whether the served-tip mechanism applies. Source: | +| `networks[].evm.servedTip.guaranteedMethods[]` | []string | — | Each element validated by `ValidatePattern` at config load (as are `servedTip.guaranteedFor[]` and `safeBlockSource`). At runtime, each pattern selects the upstreams whose `ShouldHandleMethod` accepts it ([guarantee subsets](/reference/evm/block-tracking#guarantee-subsets)). Source: | **Provider network overrides** — `providers[].overrides` (map keys) @@ -536,8 +536,8 @@ rule, a heavy-method request consumes from both buckets: - Avoid setting `!` in `use-upstream` selector patterns unless you specifically want to suppress tag matching — any `!` anywhere in the pattern disables tag fallback entirely. - **Validate patterns at startup** where possible: `ValidatePattern` is only automatically called - for `evm.servedTip.guaranteedMethods`; other fields accept invalid patterns silently and fail - at runtime. + for `evm.servedTip.guaranteedMethods`, `evm.servedTip.guaranteedFor` and `evm.safeBlockSource`; + other fields accept invalid patterns silently and fail at runtime. ### Edge cases & gotchas @@ -551,7 +551,7 @@ rule, a heavy-method request consumes from both buckets: 8. **Static response `method` is exact equality, not wildcard.** Unlike every other string matcher, `FindStaticResponseMatch` uses byte equality. A glob pattern in `staticResponses[].method` is silently treated as a literal string. Source: 9. **`CachePolicyConfig.finality` zero-value is `finalized`.** Omitting `finality` in a cache policy gives a finalized-only policy without any warning. Source: 10. **Multiple rate-limit rules fire simultaneously.** All rules whose `method` pattern matches are returned and each independently debits a token bucket. A single request may be throttled by a `"*"` rule and a specific-method rule at the same time. Source: -11. **`ValidatePattern` is called only for `evm.servedTip.guaranteedMethods` at startup.** Other pattern fields (cache policy method, failsafe matchMethod, etc.) are not validated at config load — an invalid pattern silently skips matches at runtime. Source: +11. **`ValidatePattern` is called only for `evm.servedTip.guaranteedMethods`, `evm.servedTip.guaranteedFor` and `evm.safeBlockSource` at startup.** Other pattern fields (cache policy method, failsafe matchMethod, etc.) are not validated at config load — an invalid pattern silently skips matches at runtime. Source: 12. **`WildcardMatch` concurrent calls are safe** (no shared mutable state per call) but `RateLimiterBudget.GetRulesByMethod` holds a `sync.RWMutex` during iteration. Re-tokenization cost is paid under the read lock on every matching call. Source: 13. **`paramToString` on float64 from JSON.** JSON numbers decode as `float64`; `strconv.FormatFloat(v, 'f', -1, 64)` produces `"1234"` not `"1234.0"`. A string pattern `"123*"` would match numeric param `1234`. Source: 14. **Cache `MatchesForGet` returns `true` for `unknown` finality across all policies.** A request with indeterminate finality can be served from a `finalized` policy's cache, potentially returning stale data. Source: diff --git a/docs/pages/config/projects.mdx b/docs/pages/config/projects.mdx index 9948b421b..a3411a1d7 100644 --- a/docs/pages/config/projects.mdx +++ b/docs/pages/config/projects.mdx @@ -107,7 +107,7 @@ behavior (URL shape `/{projectId}/{architecture}/{chainId}`) is covered in | `projects[].id` | string | — (required) | Used as the `project` label on nearly every metric, in URL path routing, gRPC `x-erpc-project`, admin RPC params. Uniqueness enforced at registration: duplicate → `ErrProjectAlreadyExists`. | | `projects[].auth` | `*AuthConfig` | `nil` (no auth → all requests allowed; `AuthenticateConsumer` returns `(nil, nil)`) | When set, an auth registry is built at registration. Requires ≥1 strategy. | | `projects[].cors` | `*CORSConfig` | `nil` (no CORS handling at all; OPTIONS not short-circuited) | Checked after project resolution. | -| `projects[].providers` | `[]*ProviderConfig` | If both `providers` and `upstreams` are empty: two default providers are injected: `{id: "public", vendor: "repository"}` and `{id: "envio", vendor: "envio"}`. | Unique ids required. Per-project providers registry. | +| `projects[].providers` | `[]*ProviderConfig` | If both `providers` and `upstreams` are empty: one default provider is injected: `{id: "public", vendor: "repository"}`. | Unique ids required. Per-project providers registry. | | `projects[].upstreamDefaults` | `*UpstreamConfig` | `nil` | Applied to every upstream via `ApplyDefaults` before that upstream's own `SetDefaults`; also passed to the providers registry. | | `projects[].upstreams` | `[]*UpstreamConfig` | `nil` | Shorthand non-http(s)/grpc endpoints (e.g. `alchemy://KEY`) are converted into providers and removed from the list at `SetDefaults` time. Requires ≥1 of `upstreams`/`providers`. Unique ids required. | | `projects[].networkDefaults` | `*NetworkDefaults` | `nil` | Applied to statically defined networks at `SetDefaults` and to lazy-loaded networks at first request. Its own `rateLimitBudget` is network-level, distinct from project-level. | @@ -213,7 +213,7 @@ All metrics use the `erpc_` namespace. | `erpc_rate_limits_total` | `project`, `network`, `vendor`, `upstream`, `category`, `finality`, `user`, `agent_name`, `budget`, `scope`, `auth`, `origin` | Unified deny counter. Project layer: `origin="project"`, `auth=""`. Auth layer: `origin="auth"`, `auth=":"`. | | `erpc_rate_limiter_budget_max_count` | `budget`, `method`, `scope` | Gauge. | | `erpc_rate_limiter_failopen_total` | `project`, `network`, `user`, `agent_name`, `budget`, `category`, `reason` | Reasons: `admission_full` / `limit_timeout`. Monitor this to detect hard-enforcement gaps. | -| `erpc_auth_failed_total` | `project`, `network`, `strategy`, `reason`, `agent_name` | Incremented inside auth strategies. | +| `erpc_auth_failed_total` | `project`, `network`, `strategy`, `reason`, `agent_name` | Incremented when `ErrAuthUnauthorized` is returned to the client. | | `erpc_shadow_response_identical_total` | — | Shadow traffic comparison outcomes. | | `erpc_shadow_response_mismatch_total` | — | Mismatches log at error level with both hashes. | | `erpc_shadow_response_error_total` | — | Shadow request error outcomes. | diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index 555ae393f..70d2224af 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -102,7 +102,7 @@ Leave unset for chains where upstreams handle `fromBlock: "0x0"` in an `eth_subs ## Future-block polling -Clients that follow a chain by number (for example rollup nodes reading L1) poll `eth_getBlockByNumber` for the block after the head and treat `null` as "not produced yet". No upstream has that block, so each one returns `null` in turn and every poll walks every routed upstream, fallbacks included. Enable `shortCircuitFutureBlocks` to return `null` for a numbered `eth_getBlockByNumber` above the latest head of every upstream of the network (static availability caps included), without dispatching it. A block at or below the last head any upstream reported is still dispatched, and the request is dispatched as usual while any upstream's head is unknown. It is always on when `servedTip.enabledFor` includes `"latest"`. +Clients that follow a chain by number (for example rollup nodes reading L1) poll `eth_getBlockByNumber` for the block after the head and treat `null` as "not produced yet". No upstream has that block, so each one returns `null` in turn and every poll walks every routed upstream, fallbacks included. Enable `shortCircuitFutureBlocks` to return `null` for a numbered `eth_getBlockByNumber` above the latest head of every upstream of the network (static availability caps included), without dispatching it. A block at or below the last head any upstream reported is still dispatched, and the request is dispatched as usual while any upstream's head is unknown. Only this flag turns it on. @@ -226,11 +226,11 @@ https://docs.erpc.cloud/config/projects/networks.llms.txt`} @@ -280,7 +280,7 @@ https://docs.erpc.cloud/config/projects/networks.llms.txt`} **Finality classification** (`erpc/networks.go:L1645-1743`). Explicit `finalized`/`realtime` flags in `methods.definitions` win. Otherwise the block ref/number is extracted from the request, then from the response body (for hash-keyed cache hits). Non-numeric tags → `realtime`; numeric blocks are checked via `EvmIsBlockFinalized` on the serving upstream, then the last upstream tried, then the network-wide lowest-finalized heuristic. -**Served tip.** Default ("max mode"): `EvmHighestLatestBlockNumber` / `EvmHighestFinalizedBlockNumber` return the MAX across policy-eligible, non-syncing upstreams. Setting `evm.servedTip.enabledFor` switches that axis to a cluster-min, cross-pod monotonic counter. Rollback tolerance is 1024 blocks (`DefaultToleratedBlockHeadRollback`, [`architecture/evm/evm_state_poller.go:L27`](https://github.com/erpc/erpc/blob/main/architecture/evm/evm_state_poller.go#L27)). A `use-upstream` selector produces a per-group scoped tip keyed by the SHA-256 (first 8 bytes, hex-encoded) of the sorted matched upstream-id set; unmatched selectors compute a stateless min and emit no gauges. Group partitions are capped at 16 (`maxServedTipPartitions`); a "simple" group selector must be ≤128 chars, no whitespace, at most one leading `!`, and the matched set must be ≥2 upstreams and less than all upstreams. Syncing upstreams are excluded from every head computation (max mode, cluster mode, lowest-finalized, and guaranteed-method floors). `EvmLeaderUpstream` returns the upstream with the highest raw `LatestBlock()`; `EvmLowestFinalizedBlockNumber` returns the min positive effective finalized across non-syncing upstreams. +**Served tip.** Both axes advertise the highest block that `servedTip.quorum` of the network's voters already have, per process and with no stored pick; the rule, benching, guard and floors are in [Block tracking & served tip](/reference/evm/block-tracking#served-tip). A `use-upstream` selector scopes the pick to its matched upstreams and may get its own [lane](/reference/lanes). `EvmLeaderUpstream` returns the upstream with the highest raw `LatestBlock()`; `EvmLowestFinalizedBlockNumber` returns the min positive effective finalized across non-syncing upstreams. ### Config schema @@ -322,6 +322,7 @@ https://docs.erpc.cloud/config/projects/networks.llms.txt`} | `fallbackFinalityDepth` | `int64` | `1024` () | Used when an upstream doesn't expose the `finalized` tag. Must be > 0 after defaults. | | `fallbackStatePollerDebounce` | `Duration` | `5s` () | Static debounce for block polling until dynamic block time is learned. Must be > 0. | | `dynamicBlockTimeDebounceMultiplier` | `*float64` | `0.7` () | Polling debounce = EMA block time × multiplier. Lower → more aggressive polling (fresher data, more upstream load). | +| `finalizedDebounceDepthMultiplier` | `*float64` | `0.05` () | Finalized-poll debounce = (latest − finalized) × EMA block time × multiplier, when longer than the regular debounce. `0` disables it. | | `blockUnavailableDelayMultiplier` | `*float64` | `1.0` () | Retry delay for block-unavailable = EMA block time × multiplier. | | `enforceBlockAvailability` | `*bool` | `nil` (defer; fallback = enabled) | Network-level priority 2 in 5-step chain. `nil` defers to upstream bounds and system defaults. `false` disables enforcement even when an upstream has explicit bounds configured. | | `maxRetryableBlockDistance` | `*int64` | `nil` → `128` at use site () | Block-unavailable errors within this distance of upstream head are retryable; beyond → non-retryable skip. | @@ -334,17 +335,21 @@ https://docs.erpc.cloud/config/projects/networks.llms.txt`} | `traceFilterSplitConcurrency` | `int` | `10` () | Parallelism cap for trace-filter splits. | | `idempotentTransactionBroadcast` | `*bool` | `nil` = **enabled** () | "Already known"/"nonce too low"-verified errors become success-with-tx-hash, making retry/hedge safe for `eth_sendRawTransaction`. | | `markEmptyAsErrorMethods` | `[]string` | `eth_blockNumber`, `eth_getBlockByNumber`, `eth_getTransactionByHash`, and 8 others () | Methods where empty response = error (retried, upstream scored down). `eth_getTransactionReceipt` deliberately excluded. | +| `localAnswers` | `map[string]bool` | all on | Methods eRPC answers itself: `eth_chainId`, `net_version`, `net_listening`, `net_peerCount`, `web3_clientVersion`, `eth_syncing`. `false` routes that method to the upstreams with no eRPC answer. Other keys fail validation. [Local answers](/reference/evm/method-handlers#local-answers). | | `emptyResultConfidence` | `blockHead` \| `finalizedBlock` | `blockHead` () | How confirmed a block must be for empty point-lookups to be retried as missing data. | -| `shortCircuitFutureBlocks` | `*bool` | `nil` = **off**; inherited from `networkDefaults.evm` () | Returns `null` for a numbered `eth_getBlockByNumber` above the highest latest head across every upstream of the network (static caps included; selection policy, tier and syncing state ignored), without dispatching. Fails open while any upstream's head is unknown. Uses latest heads regardless of `emptyResultConfidence`. Always on when `servedTip.enabledFor` includes `latest` (). Does not change what `latest` resolves to. | +| `shortCircuitFutureBlocks` | `*bool` | `nil` = **off**; inherited from `networkDefaults.evm` () | Returns `null` for a numbered `eth_getBlockByNumber` above the highest latest head across every upstream of the network (static caps included; selection policy, tier and syncing state ignored), without dispatching. Fails open while any upstream's head is unknown. Uses latest heads regardless of `emptyResultConfidence`. Only an explicit `true` enables it (). Does not change what `latest` resolves to. | | `safeBlockSource` | `string` (upstream selector) | `""` = inherit or provider-defined routing | Routes standard block-tagged JSON-RPC requests carrying `safe` to matching upstreams and skips cache reads. The tag stays verbatim so those upstreams define both height and hash. Operator routing overrides client `use-upstream`; retry/consensus still applies inside the matching pool. An empty network value inherits `networkDefaults.evm.safeBlockSource` when present. Does not affect `eth_query*` or direct gRPC Query APIs. | | `evm.integrity.enforceHighestBlock` | `*bool` | `true` () | **Deprecated** — migrated into `directiveDefaults.enforceHighestBlock` at `SetDefaults` time when the directive is unset (). Prefer `directiveDefaults.enforceHighestBlock`. | | `evm.integrity.enforceGetLogsBlockRange` | `*bool` | `true` () | **Deprecated** — same migration path as `enforceHighestBlock`. | | `evm.integrity.enforceNonNullTaggedBlocks` | `*bool` | `true` () | **Deprecated** — same migration path. | | `maxFutureBlockRetryDistance` | `*int64` | `nil` | **Deprecated** — warned about and set to nil in `SetDefaults` (). Has no effect; remove from configs. | -| `servedTip` | `EvmServedTipConfig` | `nil` = max mode | Copied wholesale from `networkDefaults.evm.servedTip` when nil. | -| `servedTip.enabledFor` | `[]string` | `[]` (max mode) | Valid: `latest`, `finalized`, `safe`. Listing a tag switches that axis from max-across-upstreams to cluster-min + monotonic clamp via shared state. | -| `servedTip.clusterDelta` | `int64` | `0` = auto-derive from EMA block time, clamped `[2, 10]` | Must be ≥ 0. | -| `servedTip.guaranteedMethods` | `[]string` (glob) | `[]` | Global tip clamped to per-method supporting-set floor (). | +| `servedTip` | `EvmServedTipConfig` | `nil` = the defaults below | Copied wholesale from `networkDefaults.evm.servedTip` when nil. Tunes the [served-tip rule](/reference/evm/block-tracking#served-tip), which runs either way (). | +| `servedTip.quorum` | `int` ≥ 1 \| `"majority"` | `2` | Voters that must already have a block before it is served; capped at the voter count. `"majority"` = `floor(N/2)+1`. Other values fail config load (). | +| `servedTip.maxRegressionBlocks` | `int64` | `0` = 1024 | Tolerance of the [regression guard](/reference/evm/block-tracking#regression-guard) below the corroborated live head. `-1` disables; other negatives fail validation. | +| `servedTip.guaranteedMethods` | `[]string` (glob) | `[]` | Methods whose supporting upstreams must be able to serve the tip ([guarantee subsets](/reference/evm/block-tracking#guarantee-subsets)). Patterns validated at load. | +| `servedTip.guaranteedFor` | `[]string` (selector) | `[]` | Upstream groups (id/tag selector) that must be able to serve the tip ([guarantee subsets](/reference/evm/block-tracking#guarantee-subsets)). Selectors validated at load. | +| `servedTip.enabledFor` | — | — | **Removed**; configs that set it fail to load. Use `servedTip.quorum: majority` (with ≤ 3 voters the default quorum already serves the same block). If it listed `latest`, also set `shortCircuitFutureBlocks: true`, which it used to turn on. | +| `servedTip.clusterDelta`, `servedTip.trajectoryWindow` | — | — | **Removed**; configs that set them fail to load. Delete them; benching replaces the trajectory referee. | #### `projects[].networks[].svm` — SvmNetworkConfig @@ -355,7 +360,7 @@ https://docs.erpc.cloud/config/projects/networks.llms.txt`} | `commitment` | `string` | `""` — **no default** | One of `finalized`, `confirmed`, `processed`. When unset nothing is injected and each node applies its own default (Solana's is `finalized`), so a deviating vendor can disagree on identical requests. Serving and routing assume that documented default; cacheability stays stricter — unpinned slot-addressed reads classify `unfinalized`. Setting it pins one level across the pool so the cache and consensus compare like-for-like. Details: [Commitment & finality](/reference/svm/commitment). | | `statePollerDebounce` | Duration | `400ms` (one slot) () | Minimum interval between poll fan-outs of an upstream's slot/health view. A **gate**, not the ticker period — the ticker stays pinned at 400 ms, so values below it have no effect. Raising it widens the `getBlock` availability guard's staleness margin proportionally. | | `maxFinalizedSlotLag` | `*int64` | `100` slots ≈ 40 s () | Bounds how far an upstream's finalized slot may trail the pool reference before it is excluded from **consensus voting on finalized data**. Applied only when a consensus policy is active AND the request's resolved finality is `Finalized`. **Tri-state**: omitted → `100`; explicit `0` → filter **disabled**; `>0` → that value. The field is a pointer precisely so an explicit `0` is distinguishable from "unset" — a plain integer would collapse the two and make the documented disable switch unreachable. | -| `enforceBlockAvailability` | `*bool` | `nil` → `true` (materialized at read time by `SvmEnforceBlockAvailability()`, ) | Gates the `getBlock`/`getConfirmedBlock` guard that short-circuits requests for slots above the pool's highest indexed slot, saving quota and firing the indexing-lag retry immediately, **and** any operator-declared `upstreams[*].svm.blockAvailability` window. It does **not** enable a history-floor skip by itself: the floor (`getBlocksWithLimit(0, 1)`) is probed only for upstreams that declare a `firstAvailableSlotPlus` bound, and there is no always-on floor or pool-level `-32009`. Set `false` when shred-insert tracking is unavailable or unreliable on a deployment. An explicit `false` in `networkDefaults.svm` **is** honored. | +| `enforceBlockAvailability` | `*bool` | `nil` → `true` (materialized at read time by `SvmEnforceBlockAvailability()`, ) | Gates the `getBlock`/`getConfirmedBlock` guard that short-circuits requests for slots above the pool's highest indexed slot, saving quota and firing the indexing-lag retry immediately, **and** any operator-declared `upstreams[*].svm.blockAvailability` window. It does **not** enable a history-floor skip by itself: the floor (`getBlocksWithLimit(0, 1)`) is probed for every upstream but gates only a declared `firstAvailableSlotPlus` bound, and there is no always-on floor or pool-level `-32009`. Set `false` when shred-insert tracking is unavailable or unreliable on a deployment. An explicit `false` in `networkDefaults.svm` **is** honored. | Not present on `SvmNetworkConfig`: there is no `maxSlotsPerSignaturesQuery`. An early design proposed one as an analogue of EVM's `getLogs` range cap, but `getSignaturesForAddress` is bounded by signature cursors (`before`/`until`), not by a slot range, so the key described semantics Solana does not have and was removed. `minContextSlot` is a node-freshness floor — the minimum bank slot at which a request may be *evaluated* — not a lower bound on returned history, so it cannot serve as a range cap either. @@ -386,7 +391,7 @@ Not present on `SvmNetworkConfig`: there is no `maxSlotsPerSignaturesQuery`. An | `failsafe[]` | `[]FailsafeConfig` | `nil` | Network has none → deep-copied wholesale. Network has some → per-entry merge from the FIRST compatible default (wildcard method + finality match); break on first match (). | | `selectionPolicy` | `SelectionPolicyConfig` | `nil` | Shallow-copied when network's is nil (). | | `directiveDefaults` | `DirectiveDefaultsConfig` | `nil` | Shallow-copied when network's is nil — **no per-field merge** (). | -| `evm` | `EvmNetworkConfig` | `nil` | Struct-copied wholesale when network has no `evm` block — **except onto SVM networks, which never receive it**. Otherwise per-field fill for: `integrity`, `fallbackStatePollerDebounce`, `dynamicBlockTimeDebounceMultiplier`, `blockUnavailableDelayMultiplier`, `fallbackFinalityDepth`, `getLogsMaxAllowed*`, `getLogs*`, `traceFilter*`, `servedTip`, `emptyResultConfidence`, `shortCircuitFutureBlocks`. NOT inherited individually: `chainId`, `enforceBlockAvailability`, `maxRetryableBlockDistance`, `markEmptyAsErrorMethods`, `idempotentTransactionBroadcast` (). | +| `evm` | `EvmNetworkConfig` | `nil` | Struct-copied wholesale when network has no `evm` block — **except onto SVM networks, which never receive it**. Otherwise per-field fill for: `integrity`, `fallbackStatePollerDebounce`, `dynamicBlockTimeDebounceMultiplier`, `finalizedDebounceDepthMultiplier`, `blockUnavailableDelayMultiplier`, `fallbackFinalityDepth`, `getLogsMaxAllowed*`, `getLogs*`, `traceFilter*`, `servedTip`, `emptyResultConfidence`, `shortCircuitFutureBlocks`, and `localAnswers` per method the network does not set. NOT inherited individually: `chainId`, `enforceBlockAvailability`, `maxRetryableBlockDistance`, `markEmptyAsErrorMethods`, `idempotentTransactionBroadcast` (). | | `svm` | `SvmNetworkConfig` | `nil` | Per-field fill: `chain`, `commitment`, `statePollerDebounce`, `maxFinalizedSlotLag`, `enforceBlockAvailability` (). **`cluster` is deliberately NOT inherited** — it is network identity, and inheriting it would let two networks alias one another. The two pointer fields are tested for `nil`, not for their zero value, so an explicit `maxFinalizedSlotLag: 0` or `enforceBlockAvailability: false` set here survives the merge. | | `multiplexing` | `*bool` | `nil` | Value-copied when network's `multiplexing` is nil (). No deep merge — a network must set its own `multiplexing` to deviate from the project-wide default. | @@ -424,9 +429,9 @@ non-obvious choices. evm: getLogsMaxAllowedRange: 30000 getLogsSplitOnError: true - # cluster-min tip on "latest" prevents backward tip jumps across pods + # serve "latest" once a majority of upstreams has it servedTip: - enabledFor: [latest] + quorum: majority failsafe: # NOTE: delay MUST NOT be 0 for realtime/unfinalized — a short non-zero # delay gives shared-state propagation time to catch up after a block @@ -471,8 +476,8 @@ non-obvious choices. evm: { getLogsMaxAllowedRange: 30000, getLogsSplitOnError: true, - // cluster-min tip on "latest" prevents backward tip jumps across pods - servedTip: { enabledFor: ["latest"] }, + // serve "latest" once a majority of upstreams has it + servedTip: { quorum: "majority" }, }, failsafe: [ // NOTE: delay MUST NOT be 0 for realtime/unfinalized — non-zero delay @@ -716,7 +721,7 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} - Use `alias` on every statically-declared chain you control. It makes URLs readable, shows up in every Prometheus label, and costs nothing. - Set `preserveDefaultMethods: true` before adding any custom `methods.definitions` entry. The default `false` silently drops all built-in methods (including `eth_call`, `eth_getLogs`) the moment you add one definition. - Never set `finalized: true` on a method that returns block-specific data (`eth_getBalance`, `eth_getCode`, etc.) — every response will be permanently cached at the first value ever returned. -- Enable `servedTip.enabledFor: [latest, finalized]` in production multi-pod deployments to prevent backward tip jumps across pods that could surface stale reads to clients. +- Use `servedTip.quorum: majority` on pools of 4+ upstreams when "block not found" follows `latest` requests; the default 2 serves a block only two upstreams may have. - Keep `getLogsMaxAllowedRange` at its default 30 000 unless your upstreams document a higher limit — range splits happen automatically on oversized requests but `getLogsSplitConcurrency` caps parallelism at 10 to avoid flooding providers. - For `eth_sendRawTransaction` in high-retry configs, leave `idempotentTransactionBroadcast` enabled (default). Disabling it turns idempotent "already known" errors back into real failures, making retries unsafe. @@ -735,7 +740,7 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} 11. **Static response error code `0` is omitted from the wire response** — `Code` is tagged `json:"code,omitempty"` on int, so `0` disappears from the JSON error object. Always use a non-zero code in production. [`common/errors.go:L2226`](https://github.com/erpc/erpc/blob/main/common/errors.go#L2226) 12. **Failsafe merge picks the FIRST matching default** — ordering of `networkDefaults.failsafe` entries matters when multiple wildcard entries could match. 13. **`stateful: false` on a built-in stateful method is silently overridden** — `MethodsConfig.SetDefaults` force-sets `Stateful = true` for all 6 stateful methods in all three code paths, overwriting any user-supplied `false` without any warning. [`common/defaults.go:L512-573`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L512-L573) -14. **`EnforceGetLogsBlockRange` and `EnforceHighestBlock` directives cannot be overridden per-request via HTTP headers for `eth_getLogs`, `trace_filter`, `arbtrace_filter`, and `eth_blockNumber`** — these hooks read from the config baked in at startup, not live per-request directives. Set `enforceGetLogsBlockRange: false` in `directiveDefaults`, not as an HTTP header. [`architecture/evm/eth_getLogs.go:L282-285`](https://github.com/erpc/erpc/blob/main/architecture/evm/eth_getLogs.go#L282-L285) +14. **`EnforceGetLogsBlockRange` directives cannot be overridden per-request via HTTP headers for `eth_getLogs`, `trace_filter`, and `arbtrace_filter`** — these hooks read from the config baked in at startup, not live per-request directives. Set `enforceGetLogsBlockRange: false` in `directiveDefaults`, not as an HTTP header. `eth_blockNumber` reads the live `enforceHighestBlock` directive, so a header does override it. [`architecture/evm/eth_getLogs.go:L282-285`](https://github.com/erpc/erpc/blob/main/architecture/evm/eth_getLogs.go#L282-L285) 15. **`ErrNetworkRequestTimeout` is produced only in the multiplexer follower path** — a request that is itself the leader experiences `ErrFailsafeTimeoutExceeded` or upstream sweep errors, never `ErrNetworkRequestTimeout`. [`erpc/networks.go:L2079-2084`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L2079-L2084) 16. **`evm.chainId: 0` is not rejected** — `chainId: 0` produces `networkId = evm:0`, which passes `util.IsValidNetworkId` but is excluded by provider `onlyNetworks`/`ignoreNetworks` lists that use the stricter `chainId > 0` check. 17. **`translateLatestTag: false` on a non-historical method collapses all callers to one cache key** — all requests for that method share the literal `"latest"` key; stale data is returned until the cache TTL expires. @@ -744,15 +749,15 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} 20. **Failed lazy bootstrap is user-retryable** — non-fatal bootstrap tasks are retried by the background auto-retry loop and by the next request's `ExecuteTasks`. Only `resolveNetworkConfig` format errors are `TaskFatal` and never retried. [`erpc/networks_registry.go:L263-266`](https://github.com/erpc/erpc/blob/main/erpc/networks_registry.go#L263-L266) 21. **Provider `onlyNetworks` bypasses vendor support checks entirely** — a networkId listed in `onlyNetworks` returns `SupportsNetwork = true` without consulting the vendor's dynamic check; `ignoreNetworks` is evaluated first and short-circuits to false. [`thirdparty/provider.go:L34-49`](https://github.com/erpc/erpc/blob/main/thirdparty/provider.go#L34-L49) 22. **Synthesized lazy network configs become permanently visible** — `ExposeNetworkConfig` appends to `project.Config.Networks` (admin API surface) and never overwrites an existing entry, so lazy networks persist in the admin response after first use. [`erpc/projects.go:L58-77`](https://github.com/erpc/erpc/blob/main/erpc/projects.go#L58-L77) -23. **Syncing upstreams are excluded from every head computation** — max mode, cluster mode, lowest-finalized, and guaranteed-method floor calculations all skip upstreams in `EvmSyncingStateSyncing`. The future-block short-circuit is the exception: it counts every upstream of the network, syncing ones included ([`erpc/networks.go:L935-954`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L935-L954)). [`erpc/networks.go:L327-330`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L327-L330) -24. **Future-block short-circuit compares against the MAX head across every upstream of the network, not the cluster-min served tip** — selection policy, tier (a cordoned fallback can still serve through the escape) and syncing state are ignored, and it fails open while any upstream's head is unknown, so it never nulls a block that any upstream reports having. One upstream whose head is never learned (for example a broken endpoint) therefore keeps it from firing on that network. It runs only when `evm.shortCircuitFutureBlocks` is true or `servedTip.enabledFor` includes `latest`, and always compares latest heads: `emptyResultConfidence: finalizedBlock` does not make it null an unfinalized block an upstream already has. The synthetic null response is never cached, and the short-circuit only fires for concrete block numbers (not tags or hashes). [`erpc/networks.go:L901-954`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L901-L954) +23. **Syncing upstreams are excluded from every head computation** — the served-tip ballots, lowest-finalized, and guaranteed-method floor calculations all skip upstreams in `EvmSyncingStateSyncing`. The future-block short-circuit is the exception: it counts every upstream of the network, syncing ones included ([`erpc/networks.go:L849-868`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L849-L868)). [`erpc/networks.go:L451-470`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L451-L470) +24. **Future-block short-circuit compares against the MAX head across every upstream of the network, not the served tip** — selection policy, tier (a cordoned fallback can still serve through the escape) and syncing state are ignored, and it fails open while any upstream's head is unknown, so it never nulls a block that any upstream reports having. One upstream whose head is never learned (for example a broken endpoint) therefore keeps it from firing on that network. It runs only when `evm.shortCircuitFutureBlocks` is true, and always compares latest heads: `emptyResultConfidence: finalizedBlock` does not make it null an unfinalized block an upstream already has. The synthetic null response is never cached, and the short-circuit only fires for concrete block numbers (not tags or hashes). [`erpc/networks.go:L893-964`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L893-L964) 25. **Wrong-empty misbehavior is suppressed for out-of-bounds blocks** — if the block number in a missing-data response falls outside an upstream's configured `blockAvailability` bounds, the empty response is expected and no misbehavior is recorded against that upstream. [`erpc/networks.go:L1566-1587`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L1566-L1587) 26. **`getLogsMaxAllowedRange: 0` is rejected by validation** — the field must be > 0 after defaults (`common/validation.go:L1309-1311`). Unlike `getLogsMaxAllowedAddresses` and `getLogsMaxAllowedTopics` (which default to 0 = unlimited), `getLogsMaxAllowedRange` defaults to 30 000 and cannot be explicitly zeroed out; setting `0` in a network inheriting from defaults is treated as "unset" and inherits the parent value. [`common/defaults.go:L2092-2094`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L2092-L2094) 27. **Zero projects → implicit `main` project + catch-all aliasing rule** — when the config has no projects, eRPC auto-creates `{matchDomain: "*", serveProject: "main"}`, so `/evm/123` works without a project path segment. [`common/defaults.go:L100-110`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L100-L110) -28. **Selector-scoped tips never pollute network gauges** — stateless scoped picks (unmatched or non-simple selectors) use a sentinel lane and emit no Prometheus gauge; equivalent selectors dedup into one partition keyed by matched-set hash; the cap of 16 partitions is enforced globally per network. [`erpc/networks.go:L98-105`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L98-L105) +28. **Selector-scoped tips never pollute network gauges** — only a simple group selector gets a [lane](/reference/lanes) and its own `lane` series; any other selector's pick writes no gauge. [`erpc/networks.go:L628-655`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L628-L655) 29. **Static response params matching has no wildcard** — there is no glob or `*` support; every params slot must match exactly. To catch a method regardless of params, add an entry with an empty `params` array. To match block 0 (`"0x0"`), be exact — `"0x00"` will not match. [`common/static_response.go:L14-99`](https://github.com/erpc/erpc/blob/main/common/static_response.go#L14-L99) 30. **Same-chain networks share Redis cache keys unless `cacheKeySuffix` is set** — partition keys are `{networkId}:{blockRef}` (`evm:998:64321354`). Two projects on chainId 998 with a shared Redis will HIT each other's bodies. `X-ERPC-Skip-Cache-Read` still writes, so the last fetch poisons the other project. Set `cacheKeySuffix: systx` on one network to get `evm:998:systx:{blockRef}`. Colon is rejected. Reverse-index wildcard is `{networkId}:{suffix}:*`, passed from the known network id and suffix rather than inferred from the concrete key. Memory/Redis `Delete` must use the same `WithReverseIndex` ctx as `Set`. A chain-wide main-key flush of `evm:998:*` still matches suffixed data keys. [`common/config.go:L3022-L3028`](https://github.com/erpc/erpc/blob/main/common/config.go#L3022-L3028), [`data/connector.go`](https://github.com/erpc/erpc/blob/main/data/connector.go) -31. **`shortCircuitFutureBlocks` can briefly null a block that exists** — the head it compares against is what the state poller and head subscriptions last reported. A block an upstream produced after that gets `null` until eRPC sees it, even if the client already saw it via the node's own `newHeads` (the case covered by `TestNetworkAvailability_Issue934_*`, which run with the option off). Enable it only where heads arrive promptly (WebSocket head subscriptions) and clients retry a `null`. [`erpc/networks.go:L901-926`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L901-L926) +31. **`shortCircuitFutureBlocks` can briefly null a block that exists** — the head it compares against is what the state poller and head subscriptions last reported. A block an upstream produced after that gets `null` until eRPC sees it, even if the client already saw it via the node's own `newHeads` (the case covered by `TestNetworkAvailability_Issue934_*`, which run with the option off). Enable it only where heads arrive promptly (WebSocket head subscriptions) and clients retry a `null`. [`erpc/networks.go:L893-936`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L893-L936) ### Observability @@ -765,14 +770,13 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} | `erpc_network_multiplexed_request_total` | counter | `project`, `network`, `category`, `finality`, `user`, `agent_name` | Follower registered with the in-flight deduplicator | | `erpc_network_static_response_served_total` | counter | `project`, `network`, `category` | Static response matched and served | | `erpc_network_timeout_fired_total` | counter | `project`, `network`, `category`, `finality`, `scope` | Failsafe timeout fired; `scope=network` at network level | -| `erpc_network_served_tip_block_number` | gauge | `project`, `network`, `lane`, `axis` | Served-tip value updated; absent in max mode | -| `erpc_network_served_tip_lag_blocks` | gauge | `project`, `network`, `lane`, `axis` | `MaxEligible - served`, clamped ≥ 0; absent in max mode | -| `erpc_network_served_tip_upstream_excluded_total` | counter | `project`, `network`, `upstream`, `axis`, `reason` | Upstream excluded from tip computation (`velocity`\|`outlier`) | | `erpc_network_hedged_request_total` | counter | `project`, `network`, `upstream`, `category`, `attempt`, `finality`, `user`, `agent_name` | Hedge attempt dispatched for this upstream | | `erpc_network_hedge_discards_total` | counter | `project`, `network`, `upstream`, `category`, `attempt`, `hedge`, `finality`, `user`, `agent_name` | Hedge response discarded (a faster response already won) | | `erpc_upstream_wrong_empty_response_total` | counter | `project`, `vendor`, `network`, `upstream`, `category`, `finality`, `user`, `agent_name` | Upstream returned missing-data while another upstream served real data; suppressed when the block is outside the upstream's `blockAvailability` bounds | | `erpc_unexpected_panic_total` | counter | (standard labels) | Panic recovered in the async cache-set goroutine | +Served-tip and bench metrics: see the [metrics reference](/reference/metrics#area-3-block-number-and-tip-tracking). + **Notable logs** (zerolog, at `erpc/networks_registry.go` and `erpc/networks.go` unless noted): - `"registered network alias"` / `"skipping duplicate alias registration with different target"` — alias map events. - `"network initialization ended with zero upstreams"` — bootstrap completed but no upstreams were registered. @@ -786,7 +790,7 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} - Provider task: `"registering N upstream(s) from provider"` (`upstream/registry.go`). **Span attributes** (on the `Network.Forward` span): -- `future_block.short_circuit=true` — the future-block short-circuit returned `null` without dispatching (`evm.shortCircuitFutureBlocks` or served-tip for `latest`). No metric counts these; they are not dispatched, so they do not appear in upstream request metrics. +- `future_block.short_circuit=true` — the future-block short-circuit returned `null` without dispatching (`evm.shortCircuitFutureBlocks: true`). No metric counts these; they are not dispatched, so they do not appear in upstream request metrics. ### Source code entry points diff --git a/docs/pages/config/projects/providers.mdx b/docs/pages/config/projects/providers.mdx index 97db27d71..efb48a09c 100644 --- a/docs/pages/config/projects/providers.mdx +++ b/docs/pages/config/projects/providers.mdx @@ -25,7 +25,7 @@ at all gets a sensible zero-cost default automatically. - One line to cover all chains a vendor supports — no manual network list. - Hot-path chain discovery with no I/O: static maps and lock-free remote caches keep routing decisions off the slow path. -- Zero-config fallback: eRPC auto-injects `repository` + `envio` providers when no upstreams or providers are configured. +- Zero-config fallback: eRPC auto-injects a `repository` provider when no upstreams or providers are configured. ## Quick taste @@ -124,7 +124,7 @@ https://docs.erpc.cloud/config/projects/providers.llms.txt`} **URL shorthand scheme.** Any upstream whose endpoint scheme is not `http`, `https`, `grpc`, or `grpc+bds` is converted to a `ProviderConfig` at config-defaults time. The `evm+` prefix is stripped first (`evm+alchemy://` equals `alchemy://`). The original upstream config (failsafe, rate limits, method filters, etc.) is placed into `overrides: {"*": ...}` so it applies to every generated upstream. The provider id defaults to `-` (process-wide counter per vendor name). Unknown scheme is a config-load error. -**Default providers.** A project with zero upstreams, zero providers, and no CLI `--endpoint` flags automatically gets `{id:"public", vendor:"repository"}` + `{id:"envio", vendor:"envio"}` injected with all defaults, and logs a warning. Setting `providers: []` explicitly still satisfies the zero-providers condition — the injection fires unless you have at least one upstream or one CLI endpoint. +**Default providers.** A project with zero upstreams, zero providers, and no CLI `--endpoint` flags automatically gets `{id:"public", vendor:"repository"}` injected with all defaults, and logs a warning. Setting `providers: []` explicitly still satisfies the zero-providers condition — the injection fires unless you have at least one upstream or one CLI endpoint. **Vendors also serve statically-configured upstreams.** At `NewUpstream`, `LookupByUpstream` finds a vendor by explicit `upstream.vendorName` (exact match, wins — if the name is unknown, `OwnsUpstream` is NOT consulted and the result is nil) or the first vendor whose `OwnsUpstream` matches the endpoint (in registration order). If no vendor matches, `vendorName` is guessed as `unknown-` from the endpoint host. The vendor's `GenerateConfigs` is invoked with `settings == nil` to apply vendor defaults (e.g. dRPC forcing `autoIgnoreUnsupportedMethods: false`, envio's method allow-list). Vendor-specific JSON-RPC error mapping fires before generic normalization: `GetVendorSpecificErrorIfAny` is called on the upstream's attached vendor first. diff --git a/docs/pages/config/projects/selection-policies.mdx b/docs/pages/config/projects/selection-policies.mdx index 458442204..e56f227ca 100644 --- a/docs/pages/config/projects/selection-policies.mdx +++ b/docs/pages/config/projects/selection-policies.mdx @@ -85,8 +85,9 @@ the starting point for your own tweaks: // Outer \`samplesAbove(20)\` gates on aggregate counts so the // predicate doesn't even run on cold-start pods. .excludeIf(any(all(samplesAbove(20), latencyAbove(3000), latencyDeviationAbove(3, { mode: 'majority' })), latencyAbove(10_000))) - // Block-head lag: drop if behind tip by ≥16 blocks or ≥30s. - .excludeIf(any(blockNumberLagAbove(16), blockSecondsLagAbove(30))) + // Block-head lag: drop if behind tip by ≥16 blocks or ≥30s. Not + // probed: the state poller refreshes lag, probes cannot. + .excludeIf(any(blockNumberLagAbove(16), blockSecondsLagAbove(30)), { probe: false }) // Outage safety net: if everyone failed the health excludes, fall // back to the raw set rather than failing closed. .whenEmpty(() => upstreams) @@ -98,10 +99,10 @@ the starting point for your own tweaks: // Hold the primary stable across ticks unless a meaningfully better // option exists for ≥30s. .stickyPrimary({ hysteresis: 0.30, minSwitchInterval: '30s' }) - // Shadow-mirror sampled real traffic to currently-excluded - // upstreams in the background so they accumulate fresh tracker - // samples without touching real user traffic. Per-upstream - // opt-out via \`routing.probe: off\`. + // Shadow-mirror sampled real traffic to upstreams excluded for + // errors, throttling or latency, in the background, so they + // accumulate fresh tracker samples without touching real user + // traffic. Per-upstream opt-out via \`routing.probe: off\`. .probeExcluded({ sampleRate: 0.1, minSamples: 10, minSamplesWindow: '60s', maxConcurrent: 4, timeout: '10s' })`} ts={`projects: [{ id: "main", @@ -123,8 +124,9 @@ the starting point for your own tweaks: // samplesAbove(20) gates cold-start pods. (Full rationale: see the // shipped default_policy.js.) .excludeIf(any(all(samplesAbove(20), latencyAbove(3000), latencyDeviationAbove(3, { mode: "majority" })), latencyAbove(10_000))) - // Block-head lag: drop if behind tip by ≥16 blocks or ≥30s. - .excludeIf(any(blockNumberLagAbove(16), blockSecondsLagAbove(30))) + // Block-head lag: drop if behind tip by ≥16 blocks or ≥30s. Not + // probed: the state poller refreshes lag, probes cannot. + .excludeIf(any(blockNumberLagAbove(16), blockSecondsLagAbove(30)), { probe: false }) // Outage safety net: fall back to the raw set rather than failing closed. .whenEmpty(() => upstreams) // Tier split: prefer non-fallback; use tier:fallback when no primary survives. @@ -228,7 +230,7 @@ The default `evalScope: "network"` creates one slot per network. Setting `evalSc all(samplesAbove(20), latencyAbove(3000), latencyDeviationAbove(3, { mode: 'majority' })), latencyAbove(10_000) )) - .excludeIf(any(blockNumberLagAbove(16), blockSecondsLagAbove(30))) + .excludeIf(any(blockNumberLagAbove(16), blockSecondsLagAbove(30)), { probe: false }) .whenEmpty(() => upstreams) .preferTag('!tier:fallback', { minHealthy: 1, fallback: 'tier:fallback' }) .sortByScore(PREFER_FASTEST) @@ -236,9 +238,9 @@ The default `evalScope: "network"` creates one slot per network. Setting `evalSc .probeExcluded({ sampleRate: 0.1, minSamples: 10, minSamplesWindow: '60s', maxConcurrent: 4, timeout: '10s' }) ``` -Each `excludeIf` step records per-leaf attribution into `erpc_selection_exclusion_total{reason}`. The `whenEmpty` safety net prevents fail-closed behavior during a widespread outage. `stickyPrimary` prevents flapping: the challenger must score more than 30% above the incumbent AND the last switch must be at least 30 seconds ago. `probeExcluded` shadow-mirrors 10% of real traffic to excluded upstreams so re-admission happens the moment their metrics recover. +Each `excludeIf` step records per-leaf attribution into `erpc_selection_exclusion_total{reason}`. The `whenEmpty` safety net prevents fail-closed behavior during a widespread outage. `stickyPrimary` prevents flapping: the challenger must score more than 30% above the incumbent AND the last switch must be at least 30 seconds ago. `probeExcluded` shadow-mirrors 10% of real traffic to upstreams excluded for errors, throttling or latency, so re-admission happens the moment their metrics recover. Lag exclusions are not probed: the state poller re-admits them as soon as the upstream catches up. -**Probe eligibility (which excluded upstreams get shadow traffic).** Not every exclusion benefits from probing, so exclude-family steps record a per-upstream probe verdict: `excludeIf(...)` defaults to probe-**eligible** (the predicate reads traffic metrics that freeze without traffic — probes are how the upstream proves recovery), while `excludeTag(...)` and `removeCordoned()` default to probe-**blocking** (static/admin exclusions don't reverse via traffic data, so probing them only burns quota). Each accepts an override: `excludeTag('tier:reserve', { probe: true })`, `excludeIf(blockNumberLagAbove(16), { probe: false })` — the latter is recommended for lag predicates, whose inputs the state poller refreshes regardless of traffic. Verdicts combine order-insensitively across the whole chain: every exclude step judges every upstream it *would* drop (including upstreams already dropped by earlier steps), and an excluded upstream is probed iff at least one probe-eligible step excludes it **and** no probe-blocking step does — so an upstream that is both erroring and tag-excluded is not probed, regardless of step order. Upstreams excluded only by untracked means (raw `filter`, `take`, …) default to probing. `routing.probe: 'off'` remains the per-upstream hard veto on top. The resolved verdict is visible per upstream in the decision output (`ProbeEligible`). +**Probe eligibility (which excluded upstreams get shadow traffic).** Not every exclusion benefits from probing, so exclude-family steps record a per-upstream probe verdict: `excludeIf(...)` defaults to probe-**eligible** (the predicate reads traffic metrics that freeze without traffic — probes are how the upstream proves recovery), while `excludeTag(...)` and `removeCordoned()` default to probe-**blocking** (static/admin exclusions don't reverse via traffic data, so probing them only burns quota). Each accepts an override: `excludeTag('tier:reserve', { probe: true })`, `excludeIf(blockNumberLagAbove(16), { probe: false })` — the default policy uses the latter for its lag rule, because the state poller refreshes lag regardless of traffic and probes cannot. Verdicts combine order-insensitively across the whole chain: every exclude step judges every upstream it *would* drop (including upstreams already dropped by earlier steps), and an excluded upstream is probed iff at least one probe-eligible step excludes it **and** no probe-blocking step does — so an upstream that is both erroring and tag-excluded is not probed, regardless of step order. Upstreams excluded only by untracked means (raw `filter`, `take`, …) default to probing. `routing.probe: 'off'` remains the per-upstream hard veto on top. The resolved verdict is visible per upstream in the decision output (`ProbeEligible`). **Scoring formula and presets.** `sortByScore` ranks surviving upstreams by: @@ -260,6 +262,8 @@ Per-upstream `routing.scoreMultipliers` on the upstream config merge with the ch **`shadowExcludeIf` — dry-run exclusion rules.** `shadowExcludeIf(predicate)` is the non-destructive counterpart of `excludeIf`. The predicate runs every tick; would-have-been-excluded upstreams stay in the ordered list but each trip increments `erpc_selection_shadow_exclusion_total{upstream, reason}` with the same leaf-slug attribution. Use it to measure a new rule's blast radius before promoting it to `excludeIf`. +**`blockMethods` — per-method exclusion.** `blockMethods({ match, when, probe? })` skips an upstream for single methods while it keeps serving all others. For each upstream and each method with samples that matches `match` (glob or array), `when` runs with `u.metrics` set to that method's numbers (`requestsTotal`, `latencyP`), so `samplesAbove` and `latencyAbove` work; rate predicates do not trip. A trip removes the upstream from that method's list at request time. If blocks would leave a method no upstream, all stay. Blocked upstreams receive probe traffic for that method unless `probe: false`. Each block increments `erpc_selection_method_block_total{method, upstream, reason}`. The step works at `evalScope: 'network'`. + **`includeIf` — conditional re-admission (pool-level quantifiers).** `includeIf(target, condition)` is the dual of `excludeIf`: it ADDS upstreams from the full universe back into the chain when an aggregate condition over the *surviving pool* holds — it never removes anyone. `target` comes first ("include *these* if *condition*"): a tag pattern (`'tier:reserve'`, `!` negation supported) or a selector object `{id, tag, vendor, type, position}` whose facets AND together; an object with no concrete facet (`{}`, `{position:'head'}`) is a deliberate no-op so it can never accidentally admit the entire universe. Admitted upstreams land at the tail by default (`position: 'head'` available in object form), are never duplicated by id, and are ranked by the subsequent `sortByScore` like everyone else. `condition` is a boolean or `(pool, ctx) => boolean` receiving the current chain — which is what enables pool-level ("network-level") questions via native array methods over the per-upstream predicate factories: `p.every(blockSecondsLagAbove(30))`, `p.length < 2`. Mind that native `every` is `true` on an empty array — guard with `p.length > 0 &&` when the empty pool should be handled by a separate explicit `p.length < 1` line. Malformed arguments and throwing conditions degrade to a no-op rather than sinking the eval. Source: [`internal/policy/stdlib/stdlib.js:L1023-L1116`](https://github.com/erpc/erpc/blob/main/internal/policy/stdlib/stdlib.js#L1023-L1116); tests: [`internal/policy/stdlib/include_if_test.go`](https://github.com/erpc/erpc/blob/main/internal/policy/stdlib/include_if_test.go). **Additional stdlib chain methods.** Beyond `excludeIf` / `includeIf`, the JS stdlib installs the following chainable `Array.prototype` methods (source: [`internal/policy/stdlib/stdlib.js`](https://github.com/erpc/erpc/blob/main/internal/policy/stdlib/stdlib.js)): @@ -362,7 +366,7 @@ Defaults: | Field | Type | Default | Footguns | |---|---|---|---| | `evalInterval` | `Duration` | `15s` ([`common/defaults.go:L2532`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L2532)) | `0` disables the ticker (frozen cache); tests only. Should be ≥ 3× `scoreMetricsWindowSize / 10` for stable quantile reads. | -| `evalTimeout` | `Duration` | `100ms` ([`common/defaults.go:L2534-2535`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L2534-2535)) | Exceeded → tick error logged, previous cache retained unchanged. Must be `< evalInterval`. | +| `evalTimeout` | `Duration` | `100ms` ([`common/defaults.go:L3125-3126`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L3125-3126)) | Exceeded → warning logged and `timeout` counted; the eval cannot be interrupted, so its result is still applied. Must be `< evalInterval`. | | `evalScope` | `EvalScope` enum | `"network"` ([`common/defaults.go:L2556-2583`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L2556-2583)) | `"network"` \| `"network-method"` \| `"network-finality"` \| `"network-method-finality"`. Narrower scopes multiply goroutines and JS evals. `scoreMultipliers.method`-specific entries **never match** in `"network"` scope because `ctx.method == "*"`. | | `evalPerMethod` | `*bool` | `nil` — **deprecated** | Pointer-bool alias for the method axis. `SetDefaults` nils it unconditionally and derives `evalScope` instead. Excluded from TS surface (`tstype:"-"`). | | `evalPerFinality` | `*bool` | `nil` — **deprecated** | Same mechanics as `evalPerMethod` for the finality axis. Niled by `SetDefaults`. | @@ -657,6 +661,22 @@ non-obvious choices. .probeExcluded({ sampleRate: 0.1, minSamples: 10, timeout: '10s' }) ``` +**5b. Blocking one slow method on one upstream (`blockMethods`).** Keep an upstream for `eth_call` and logs, but skip it for traces while its trace p99 is over 5s: + +```js +(upstreams, ctx) => + upstreams + .removeCordoned() + .excludeIf(all(samplesAbove(10), errorRateAbove(0.7))) + .blockMethods({ + match: ['debug_trace*', 'trace_*'], + when: all(samplesAbove(20), latencyAbove(5000, 99)), + }) + .whenEmpty(() => upstreams) + .sortByScore(PREFER_FASTEST) + .probeExcluded({ sampleRate: 0.05, minSamples: 10, timeout: '10s' }) +``` + **6. Reserve tier — break-glass inclusion (`includeIf`).** Keep a pricier or rate-limited provider (tagged `tags: ["tier:reserve"]`) out of normal rotation, and bring it in only when the serving pool is *collectively* unfit — every survivor lagging or slow, or none @@ -748,6 +768,7 @@ reserve upstreams to also keep shadow-probe traffic off them while benched: | `erpc_selection_rejection_total` | Counter | `project, network, method, upstream, step` | Per tick × excluded upstream; `step` = first stdlib primitive that dropped the upstream. | | `erpc_selection_exclusion_total` | Counter | `project, network, method, upstream, reason` | Per tick × excluded upstream × leaf predicate slug. | | `erpc_selection_shadow_exclusion_total` | Counter | `project, network, method, upstream, reason` | `shadowExcludeIf` would-have-excluded; upstream stays in rotation. | +| `erpc_selection_method_block_total` | Counter | `project, network, method, upstream, reason` | Per tick × `blockMethods` block; `method` is the blocked method. Upstream stays in rotation for other methods. | | `erpc_selection_excluded_seconds` | Gauge | `project, network, method, upstream` | Seconds continuously excluded; reset to `0` on readmit. | | `erpc_selection_readmit_total` | Counter | `project, network, method, upstream` | Excluded → in-rotation transition. | | `erpc_selection_readmit_age_seconds` | Histogram | `project, network, method` | Distribution of exclusion age at readmit. | @@ -767,7 +788,7 @@ reserve upstreams to also keep shadow-probe traffic off them while benched: An upstream can have `position=0` in the wildcard slot and `position=1` in a per-method slot simultaneously (positions are per-slot). **`erpc_selection_eval_errors_total` `kind` values** (classified at [`internal/policy/slot.go:L460-L467`](https://github.com/erpc/erpc/blob/main/internal/policy/slot.go#L460-L467)): -- `"timeout"` — JS eval exceeded `evalTimeout` (default 100 ms); previous cache retained. +- `"timeout"` — JS eval exceeded `evalTimeout` (default 100 ms); its result is still applied. - `"throw"` — uncaught JS exception (runtime error, user-thrown error). - `"invalid_return"` — eval completed but returned a non-array, null/undefined, entries without `id`, or unknown upstream IDs. diff --git a/docs/pages/config/projects/upstreams.mdx b/docs/pages/config/projects/upstreams.mdx index 44ee4e67a..787ca0793 100644 --- a/docs/pages/config/projects/upstreams.mdx +++ b/docs/pages/config/projects/upstreams.mdx @@ -144,7 +144,7 @@ of the tip (configurable via `networks[].evm.maxRetryableBlockDistance`). `latestSlotMinus`, `firstAvailableSlotPlus`). Only what the operator declares is enforced. The upstream's oldest stored block, probed with `getBlocksWithLimit(0, 1)`, anchors a `firstAvailableSlotPlus` bound, like EVM's -`earliestBlockPlus`; it is probed only for upstreams declaring one. The network-level +`earliestBlockPlus`; it is probed for every upstream. The network-level `svm.enforceBlockAvailability` switch gates the declared window. () () @@ -251,6 +251,11 @@ All paths relative to `projects[*]`. "Default" = value after `SetDefaults`/`Appl | `upstreams[*].evm.statePollerDebounce` | Duration | `0` → inferred from chain block time | Minimum spacing between forced polls. | | `upstreams[*].evm.skipWhenSyncing` | `*bool` | `false` () | Skip requests with `ErrUpstreamSyncing` while the poller reports syncing state. | | `upstreams[*].evm.skipSyncingCheck` | `*bool` | `false` | Disable `eth_syncing` polling for this upstream and hard-set syncing state to `NotSyncing`. Use for nodes that always return a syncing object (e.g. Pharos/Antora) even when fully caught up, which would otherwise cause permanent circuit-breaker false positives. | +| `upstreams[*].evm.pinLatestTag` | `*bool` | `false` (inherits `upstreamDefaults.evm.pinLatestTag`) | For `eth_call`, `eth_getBalance`, `eth_getCode`, `eth_getTransactionCount`, `eth_getStorageAt` asking `"latest"` (or omitting the block): this upstream receives the network head as a hex number; other upstreams, the cache and the response keep `"latest"`. Skipped when the head is above its `blockAvailability.upper` bound or unknown. For an upstream whose own head lags the network: a block it lacks fails as missing data and the next upstream answers. | +| `upstreams[*].evm.multicall3` | object | nil = off (inherits `upstreamDefaults.evm.multicall3`) | Packs `eth_call`s sent to this upstream for the same block into one [Multicall3](https://github.com/mds1/multicall3) `aggregate3` call at `0xcA11…CA11`, so a vendor billed per request bills once per pack. Below retries, hedging, consensus and cache: each call keeps its own. Packed: `{to, data\|input, gas?}` calls with no `from`, `value` or state override; a call with `gas` packs only with calls of the same `gas`, which the multicall carries. A call is sent alone, exactly as without packing, when no other call joins its pack, the multicall fails or is malformed, its inner call reverts, runs out of gas or returns no data, or its block predates Multicall3 on this upstream. Not invisible: a packed call runs with `msg.sender` = Multicall3 instead of the zero address and with less gas left, so a view that returns data based on `msg.sender` or `gasleft()` can answer differently; enable it where the callers' calls do not depend on either. Metrics: `erpc_upstream_multicall3_calls_total{outcome}`, `erpc_upstream_multicall3_batch_size`; requests saved = `packed` − batches. | +| `upstreams[*].evm.multicall3.maxWait` | Duration | `10ms` | A pack is sent this long after its first call. | +| `upstreams[*].evm.multicall3.maxCalls` | int | `20` (min `2`) | A pack is sent at once when it holds this many calls. | +| `upstreams[*].evm.multicall3.bypassContracts` | `[]string` | `[]` | Target addresses never packed: contracts that answer differently, without reverting, when the caller is Multicall3 (`msg.sender` or caller-code checks). A contract whose call reverts inside a pack but succeeds alone is learned at runtime (logged once, for the process lifetime) and needs no entry. | | `upstreams[*].evm.blockAvailability.lower` / `.upper` | object | nil = unbounded side | Each bound sets exactly one of `exactBlock`, `latestBlockMinus`, `earliestBlockPlus`. Cross-bound validation: `latestBlockMinus` lower value must be ≥ upper value. | | `upstreams[*].evm.blockAvailability.{lower,upper}.latestBlockMinus` | `*int64` | nil | Bound = latest − N. Recomputed on each check from state poller. | | `upstreams[*].evm.blockAvailability.{lower,upper}.earliestBlockPlus` | `*int64` | nil | Bound = detected earliest block + N via probe. Earliest `0` before detection computes `0+N`. | @@ -269,10 +274,11 @@ All paths relative to `projects[*]`. "Default" = value after `SetDefaults`/`Appl | `upstreams[*].svm.cluster` | string | `""`; inherited from `upstreamDefaults.svm.cluster` | The cluster this upstream serves. Must match the network-level `svm.cluster` for the upstream to be eligible for that network. Unlike the network side, cluster **is** inherited here — on an upstream it only says "which cluster do I serve", and a pool homogeneous across one cluster is the common case. | | `upstreams[*].svm.chain` | string | `""` → treated as `"solana"`; inherited from `upstreamDefaults.svm.chain` | Which SVM chain this upstream serves. Must match the network-level `svm.chain`. | | `upstreams[*].svm.checkGenesisHash` | bool | `false` | Opts an **unknown** `(chain, cluster)` pair into `getGenesisHash` validation at bootstrap. Known Solana clusters (`mainnet-beta`, `devnet`, `testnet`) are always validated regardless of this flag: a mismatch **or a fetch failure** fails the upstream, catching nodes mis-pointed at the wrong cluster and refusing to register one that could not be verified. For unknown clusters the same check (with no table comparison) runs only when this flag is set, so private and local clusters with no published genesis hash still work. | -| `upstreams[*].svm.blockAvailability.lower` / `.upper` | object | nil = unbounded side | Operator restriction on the slot window this upstream may serve. Each bound sets exactly one of `exactSlot`, `latestSlotMinus`, `firstAvailableSlotPlus`. Only declared bounds are enforced; the observed floor is probed only when a bound uses `firstAvailableSlotPlus`. Gated by `networks[*].svm.enforceBlockAvailability`. () | +| `upstreams[*].svm.blockAvailability.lower` / `.upper` | object | nil = unbounded side | Operator restriction on the slot window this upstream may serve. Each bound sets exactly one of `exactSlot`, `latestSlotMinus`, `firstAvailableSlotPlus`. Only declared bounds are enforced; the observed floor is probed for every upstream but skips nothing unless a bound uses `firstAvailableSlotPlus`. Gated by `networks[*].svm.enforceBlockAvailability`. () | | `upstreams[*].svm.blockAvailability.{lower,upper}.exactSlot` | `*int64` | nil | Fixed slot bound. Must be ≥ 0. | | `upstreams[*].svm.blockAvailability.{lower,upper}.latestSlotMinus` | `*int64` | nil | Bound = poller's `LatestSlot()` − N. Recomputed on each check. Cold poller (latest ≤ 0) fail-opens that side. | -| `upstreams[*].svm.blockAvailability.{lower,upper}.firstAvailableSlotPlus` | `*int64` | nil | Bound = oldest stored block (`getBlocksWithLimit(0, 1)`) + N. Setting it is what opts the upstream in to polling the floor. Unobserved floor fail-opens that side — unknown is not slot 0. | +| `upstreams[*].svm.blockAvailability.{lower,upper}.firstAvailableSlotPlus` | `*int64` | nil | Bound = oldest stored block (`getBlocksWithLimit(0, 1)`) + N. Unobserved floor fail-opens that side — unknown is not slot 0. | +| `upstreams[*].svm.blockAvailability.{lower,upper}.updateRate` | Duration | `0` = freeze after first observation | Floor re-probe cadence, as EVM's `earliestBlockPlus`. Until the first observation the floor retries every `30s`. `firstAvailableSlotPlus` bounds only; the smaller of `lower`/`upper` wins. | | `upstreamDefaults` | `*UpstreamConfig` | nil | Project-level template. Gets its own `SetDefaults(nil)` first. Each upstream runs `ApplyDefaults` then `SetDefaults`. | | `proxyPools[*].id` | string | required | Pool name referenced by `upstreams[*].jsonRpc.proxyPool`. | | `proxyPools[*].urls` | `[]string` | required, min 1 | Proxy URLs. Accepted schemes: `http://`, `https://`, `socks5://` (case-insensitive prefix check — `socks4://` rejected at startup). | @@ -934,7 +940,7 @@ SVM upstream gets its own slot/health poller automatically: [SVM slot tracking & health](/reference/svm/slot-tracking). 25. **SVM never gates on a vendor-reported floor by default.** An upstream's - floor is probed only when it declares `firstAvailableSlotPlus`. Otherwise every + floor is probed, but skips nothing unless it declares `firstAvailableSlotPlus`. Otherwise every slot-named read reaches the upstream; `exactSlot: 0` explicitly declares full history. () 26. **SVM declared bounds fail-open when state is unknown.** A cold poller, an diff --git a/docs/pages/config/rate-limiters.mdx b/docs/pages/config/rate-limiters.mdx index 6945f28fa..ecbb79056 100644 --- a/docs/pages/config/rate-limiters.mdx +++ b/docs/pages/config/rate-limiters.mdx @@ -121,15 +121,30 @@ rules match a single method, all are evaluated in parallel goroutines; the first The backing store is either an in-process sharded memory cache (64 FNV-1a shards, lazy-expiry cleanup) or a Redis-backed Envoy ratelimit store. In memory mode, each eRPC process has independent counters — the effective cluster limit is `maxCount × instanceCount`. -In Redis mode, counters are shared across all instances, but the Redis client **fail-opens** -while the connection is being established, and again on timeout or admission pressure. +In Redis mode, counters are shared across all instances. When Redis cannot decide a +request, `store.onFailure` does. -**Redis fail-open paths.** Three distinct conditions cause the Redis rate limiter to allow -a request instead of blocking: +**Over-limit cache.** Once Redis reports a rule's key over its limit, each replica keeps +that key in a 4 MiB in-process cache until the window ends, and denies it from there. +Other rules in the same request still go to Redis; if every matching key is cached, no +Redis call is made, so the denial holds while Redis is down. Source: -1. **Not yet connected** — `getCache()` returns nil; `TryAcquirePermit` returns `(true, nil)` before even reaching the Redis call. This window lasts from startup until Redis first connects. -2. **Admission semaphore full** — each budget has a bounded channel (`max(256, connPoolSize × 32)` slots). When full, the request is allowed immediately without spawning a goroutine, and `erpc_rate_limiter_remote_admission_shedded_total` increments. This design was added after the 2026-05-07 incident where 25k+ goroutines accumulated during a Redis contention spike. -3. **Per-call timeout** (`getTimeout`, default `1s`) — `doLimitWithTimeout` returns after the timer fires, allows the request, and increments `erpc_rate_limiter_failopen_total{reason="limit_timeout"}`. The spawned goroutine **continues running** and completes the Redis write — the counter is updated even though the caller was not blocked. +**Redis store failures.** Four conditions leave a request undecided by Redis: + +1. **`store_unavailable`** — not connected yet (startup, or first connect failed and retries in the background). +2. **`store_error`** — the Redis call failed (envoy panics on Redis errors; eRPC recovers). +3. **`admission_full`** — each budget caps in-flight Redis calls at `max(256, connPoolSize × 32)`. When full, no call or goroutine is started. This cap was added after the 2026-05-07 incident where 25k+ goroutines accumulated during a Redis contention spike. One request takes one slot, however many rules it matches: all matching rules go to Redis in one pipeline. +4. **`limit_timeout`** — no answer within `getTimeout` (default `1s`). The call keeps running and still updates the Redis counter. + +`store.onFailure` then decides: + +| `onFailure` | Undecided request | Cost | +|---|---|---| +| `allow` (default) | Admitted. `erpc_rate_limiter_failopen_total{reason}` increments. | A Redis incident lifts every limit. | +| `deny` | Rejected as rate limited. | A Redis incident rejects all limited traffic. | +| `local` | Decided by an in-process counter at `maxCount × localLimitFactor`. It counts every request, so it is warm when Redis fails. | One extra in-memory count per request. It counts this replica only, so set `localLimitFactor` near `1/replicas`. | + +Every undecided request increments `erpc_rate_limiter_store_failure_total{budget, reason, decision}`. **Auto-tuner.** When an upstream has `rateLimitBudget` set and `rateLimitAutoTune` is absent from config, eRPC automatically populates `rateLimitAutoTune` with defaults and sets @@ -154,7 +169,7 @@ last wins; there is no coordination. Where `nearLimitThreshold = floor(maxCount × nearLimitRatio)`. Default ratio 0.8: for `maxCount=100`, requests 81–100 are allowed (near-limit stats only); request 101 is blocked. eRPC never surfaces `NEAR_LIMIT` to callers. Source: -**`TryAcquirePermit` flow.** (1) If cache is nil → fail-open `(true, nil)`. (2) `GetRulesByMethod` returns empty → allow (no rules match). (3) If any matching rule uses `perIP`/`perUser`/`perNetwork` and `req == nil` → return error. (4) Single matching rule: evaluate directly without spawning a goroutine. (5) Multiple rules: fan-out goroutines, `atomic.Bool` short-circuits on first `OVER_LIMIT`. Source: +**`TryAcquirePermit` flow.** (1) `GetRulesByMethod` returns empty → allow (no rules match). (2) If any matching rule uses `perIP`/`perUser`/`perNetwork` and `req == nil` → return error. (3) Every matching rule becomes one descriptor of a single `DoLimit` call: one Redis pipeline and one admission slot per request. The first `OVER_LIMIT` descriptor blocks. (4) If the store cannot answer, `store.onFailure` decides. Source: **Credit-weighted counting.** By default each acquisition consumes 1 hit (`HitsAddend: 1`). When an **upstream** sets `rateLimitCountMode: credit`, the upstream layer passes the request's pre-flight vendor credit-unit estimate as the hit weight, so a heavy `eth_getLogs` (e.g. 60 CU on Alchemy) drains the budget 60× faster than a flat request count. A `0`-CU method (e.g. `eth_chainId`) is admitted for free and consumes nothing. This weight rides the standard Envoy `HitsAddend`, so it works on both the memory and Redis stores. Source: @@ -168,7 +183,7 @@ Where `nearLimitThreshold = floor(maxCount × nearLimitRatio)`. Default ratio 0. **Memory store eviction.** Each shard prunes lazily: on every `DoLimit` call, if the shard's last-prune timestamp differs from the current unix second, all buckets with `expiry ≤ now` are deleted. This is O(#buckets) per shard — not O(#keys). Source: -**Admission cap derivation.** `remoteAdmissionCap = max(256, connPoolSize × 32)`. The ×32 multiplier accounts for radix's implicit pipelining window (up to 32 commands per connection). Pool of 8 → cap 256 (floor). Pool of 64 → cap 2048. Source: +**Admission cap derivation.** `remoteAdmissionCap = max(256, connPoolSize × 32)`. It caps goroutines, not Redis capacity: each check sends one explicit pipeline and holds one pool connection for one round trip, with radix's implicit pipelining off. Pool of 8 → cap 256 (floor). Pool of 64 → cap 2048. Source: **Auto-tuner minimum sample floor.** The tuner skips adjustment when `totalCount < 10` — insufficient signal. Counters are reset at the start of `maybeAdjust` regardless, so a sustained pattern of exactly 9 requests per period will perpetually reset the clock. Source: @@ -201,7 +216,7 @@ limiter. Remove `waitTime` from any existing config to suppress the warning. | `rateLimiters.store.redis.tls.insecureSkipVerify` | bool | `false` | Skip server cert verification. Never use in production. Source: | | `rateLimiters.store.redis.connPoolSize` | int | `8` | Radix connection pool size. Derives admission cap: `max(256, connPoolSize × 32)`. Source: | | `rateLimiters.store.redis.initTimeout` | Duration | `5s` | Connection establishment timeout. Source: | -| `rateLimiters.store.redis.getTimeout` | Duration | `1s` | Per-call DoLimit timeout. When exceeded, request **fail-opens** and `erpc_rate_limiter_failopen_total{reason="limit_timeout"}` increments. Source: | +| `rateLimiters.store.redis.getTimeout` | Duration | `1s` | Per-call DoLimit timeout. When exceeded, `store.onFailure` decides the request (`reason="limit_timeout"`). Source: | | `rateLimiters.store.redis.setTimeout` | Duration | `3s` | **Unused by the rate limiter** — present for struct sharing with the data-connector Redis config. Source: | | `rateLimiters.store.redis.lockRetryInterval` | Duration | `500ms` | **Unused by the rate limiter** — present for struct sharing with the data-connector Redis config. Source: | | `rateLimiters.store.redis.iamAuth.enabled` | bool | `false` | Enables AWS ElastiCache IAM auth for the rate-limiter store. When true, `SetDefaults` auto-enables TLS (`rediss://`) and eRPC builds a `radix/v3` pool whose dial function mints a fresh SigV4 token per new connection. Mutually exclusive with a static password. Source: | @@ -211,6 +226,8 @@ limiter. Remove `waitTime` from any existing config to suppress the warning. | `rateLimiters.store.redis.iamAuth.auth.*` | `*AwsAuthConfig` | nil (default chain) | Optional AWS credential source (same fields as `dynamodb.auth`). Omit to use the instance-role / IRSA default chain (recommended on EKS/EC2). Source: | | `rateLimiters.store.nearLimitRatio` | float32 | `0.8` | Envoy near-limit threshold fraction (`nearLimitThreshold = floor(maxCount × nearLimitRatio)`). **Has no effect on eRPC blocking decisions** — eRPC only checks `OVER_LIMIT`; `NEAR_LIMIT` is never returned to callers. Only alters internal Envoy stats counters, which are not exported to Prometheus. Source: | | `rateLimiters.store.cacheKeyPrefix` | string | `"erpc_rl_"` | Prefix for all Redis/memory cache keys. **Footgun:** two eRPC deployments sharing the same Redis with the same prefix will merge their counters. Use distinct prefixes per deployment (`"erpc_prod_"`, `"erpc_staging_"`). Source: | +| `rateLimiters.store.onFailure` | string | `"allow"` | Redis only. `allow` \| `deny` \| `local`: what a request gets when Redis cannot decide it (not connected, error, `admission_full`, `limit_timeout`). See *Redis store failures* above. Source: | +| `rateLimiters.store.localLimitFactor` | float | — | Required with `onFailure: local`, `(0, 1]`. The in-process limit is `min(maxCount, max(ceil(maxCount × localLimitFactor), hits))`, so a credit rule always fits one call (600 × 0.1 with a 200-credit call → 200). Set near `1/replicas`. Source: | #### `rateLimiters.budgets[]` @@ -278,15 +295,12 @@ Redis slowdowns: driver: redis redis: uri: "\${REDIS_URL}" - # 200ms gives the radix pipeline window (5ms) plus RTT comfortable - # headroom while still fast-failing during real outages. - # Never set below ~50ms on pipelined clients: the pipeline window - # itself takes ~5ms and a too-tight cap triggers constant fail-opens. + # 200ms covers RTT with headroom while still fast-failing during + # real outages. Each check is one round trip; implicit pipelining + # is off, so no batching window adds to it. getTimeout: 200ms - # Default is 8 — not enough for 12+ machine fleets. - # With radix implicit pipelining (window=5ms, limit=32), - # effective in-flight = poolSize × 32: poolSize=64 supports - # ~2k concurrent checks/machine without queueing. + # Default is 8 — not enough for 12+ machine fleets. Each in-flight + # check holds one connection; past the pool, checks wait for one. connPoolSize: 64 # Use distinct prefixes per environment — two deployments sharing the # same Redis and same prefix will silently merge their counters. @@ -640,9 +654,9 @@ Full error code reference: to any upstream where you want stable, manually-tuned limits. - **Never use `period: "2h"` or other compound durations.** Only the seven canonical period names are accepted; `"2h"` parses successfully but is rejected at unmarshal time. -- **Watch `erpc_rate_limiter_failopen_total`.** Sustained `reason="limit_timeout"` spikes - mean Redis is too slow; consider raising `getTimeout` or switching to memory mode for - low-latency deployments. +- **Watch `erpc_rate_limiter_store_failure_total`.** Sustained `reason="limit_timeout"` or + `admission_full` means Redis is too slow for `getTimeout`. Pick `onFailure` for what a + Redis incident should do: `local` keeps a per-replica limit, `deny` blocks. ### Edge cases & gotchas @@ -659,7 +673,7 @@ Full error code reference: 11. **`store` block is auto-created with `driver="memory"` when absent.** Declaring `rateLimiters:` without `store.driver: redis` silently uses memory even if Redis fields are present. No warning is emitted. Source: 12. **Wildcard rules affect auto-tuner scope broadly.** A rule `method: "eth_*"` matched by `eth_call` will have its `maxCount` adjusted whenever any `eth_*` method hits the error threshold — not only the method that triggered the change. Source: [`upstream/ratelimiter_autotuner.go:L113`](https://github.com/erpc/erpc/blob/main/upstream/ratelimiter_autotuner.go#L113) 13. **Period `"2h"` is rejected.** `time.ParseDuration("2h")` succeeds but `2*time.Hour` does not equal any canonical period. Only exact matches to `second`, `minute`, `hour`, `day`, `week`, `month`, `year` (and their aliases) are accepted. Source: -14. **Redis DoLimit goroutines outlive their callers.** After `getTimeout` fires, the goroutine continues running and completes the Redis write. The counter is updated even though the caller was allowed through. This prevents counter drift but means Redis receives more writes than the caller observes. Source: [`upstream/ratelimiter_budget.go:L349`](https://github.com/erpc/erpc/blob/main/upstream/ratelimiter_budget.go#L349) +14. **Redis DoLimit goroutines outlive their callers.** After `getTimeout` fires, the goroutine continues running and completes the Redis write. The counter is updated even though `onFailure` already decided the caller. This prevents counter drift but means Redis receives more writes than the caller observes. Source: 15. **Auto-tuner `adjustmentPeriod` clock resets on insufficient samples.** `lastAdjustments[method]` is updated at the start of `maybeAdjust` regardless of whether an actual adjustment occurs. A sustained pattern of exactly 9 requests per period (below the 10-sample floor) will perpetually reset the clock and prevent any evaluation. Source: [`upstream/ratelimiter_autotuner.go:L94`](https://github.com/erpc/erpc/blob/main/upstream/ratelimiter_autotuner.go#L94) 16. **Auto-tuner only increases when `errorRate == 0.0` exactly.** Partial improvement in `(0, threshold]` produces no adjustment — budget will not grow back until error rate reaches exactly zero. Source: [`upstream/ratelimiter_autotuner.go:L107-L112`](https://github.com/erpc/erpc/blob/main/upstream/ratelimiter_autotuner.go#L107-L112) 17. **`minBudget: 0` is no floor.** Default `minBudget=0` means no clamping floor — the auto-tuner can drive `maxCount` to 0, making the budget always-blocked. Set to at least 1. Source: @@ -679,10 +693,11 @@ Full error code reference: |---|---|---|---| | `erpc_rate_limits_total` | counter | `project, network, vendor, upstream, category, finality, user, agent_name, budget, scope, auth, origin` | A budget rule denies a request. `origin` ∈ `{auth, project, network, upstream}` | | `erpc_rate_limiter_budget_max_count` | gauge | `budget, method, scope` | Set at budget initialization and on every auto-tuner adjustment | -| `erpc_rate_limiter_failopen_total` | counter | `project, network, user, agent_name, budget, category, reason` | Rate limiter allowed a request due to error; `reason` ∈ `{admission_full, limit_timeout}` | +| `erpc_rate_limiter_failopen_total` | counter | `project, network, user, agent_name, budget, category, reason` | `onFailure: allow` admitted a request Redis could not decide; `reason` ∈ `{store_unavailable, store_error, admission_full, limit_timeout}` | +| `erpc_rate_limiter_store_failure_total` | counter | `budget, reason, decision` | Every request Redis could not decide; `decision` ∈ `{allow, deny, local_allow, local_deny}` | | `erpc_rate_limiter_remote_inflight` | gauge | `budget` | Incremented when a Redis DoLimit goroutine starts; decremented on completion or panic | -| `erpc_rate_limiter_remote_admission_shedded_total` | counter | `budget` | Admission semaphore was full; request fail-opened without spawning a goroutine | -| `erpc_rate_limiter_remote_duration_seconds` | histogram | `budget, result` | Duration of each Redis DoLimit call; `result` ∈ `{ok, over_limit, fail_open}` | +| `erpc_rate_limiter_remote_admission_shedded_total` | counter | `budget` | Admission semaphore was full; no Redis call was started | +| `erpc_rate_limiter_remote_duration_seconds` | histogram | `budget, result` | Duration of each Redis DoLimit call; `result` ∈ `{ok, over_limit, fail_open}` (`fail_open`: error or timeout) | | `erpc_rate_limiter_budget_decision_total` | counter | `project, network, category, finality, user, agent_name, budget, method, scope, decision` | **DEPRECATED** — replaced by `erpc_rate_limits_total`. Zero call sites in production code. Source: | | `erpc_unexpected_panic_total` | counter | `origin, store, fingerprint` | Redis DoLimit goroutine panicked and was recovered; `origin="ratelimiter-redis-dolimit"` | diff --git a/docs/pages/config/server.mdx b/docs/pages/config/server.mdx index 19e53cddd..1054385ef 100644 --- a/docs/pages/config/server.mdx +++ b/docs/pages/config/server.mdx @@ -106,7 +106,7 @@ https://docs.erpc.cloud/config/server.llms.txt`} **Timeout machinery.** eRPC does NOT use `net/http`'s built-in `TimeoutHandler`. Its own implementation buffers the entire response body in a pooled `bytes.Buffer` and stages headers privately; only when the inner handler finishes within the deadline does it flush to the real connection. On timeout: JSON-RPC `-32603` body at HTTP 200 (POST) or 504 (other). On client cancel: "request cancelled by client" at HTTP 200 (POST) or 503 with empty body (other). Source: -**Request and response gzip.** `enableGzip: true` wraps the handler in a `conditionalGzipWriter`. The first write decides once: if the chunk is smaller than 1024 bytes, no compression is applied; otherwise `Content-Length` is deleted, `Content-Encoding: gzip` + `Vary: Accept-Encoding` are set, and subsequent writes flow through a pooled `gzip.Writer`. Inbound gzip bodies (`Content-Encoding: gzip`) are always accepted and decompressed using a pooled reader, regardless of `enableGzip`. Source: +**Request and response gzip.** `enableGzip: true` wraps the handler in a `conditionalGzipWriter`. Body writes (and any explicit status code) are held back until the cumulative body size reaches 1024 bytes; at that point `Content-Length` is deleted, `Content-Encoding: gzip` is set, and all bytes (buffered and subsequent) flow through a pooled `gzip.Writer`. If the handler finishes or flushes below 1024 bytes, the buffered bytes are sent uncompressed. The decision is cumulative rather than first-write-only because `JsonRpcResponse.WriteTo` streams the ~22-byte envelope prefix before the result. `Vary: Accept-Encoding` is set on every response through the handler, compressed or not. Inbound gzip bodies (`Content-Encoding: gzip`) are always accepted and decompressed using a pooled reader, regardless of `enableGzip`. Source: **gRPC port sharing.** When `grpcEnabled: true` and the gRPC v4 host:port equal the HTTP v4 host:port (the default derivation), the IPv4 HTTP handler multiplexes: HTTP/2 + `Content-Type: application/grpc` → in-process gRPC server (`grpc.Server.ServeHTTP`), bypassing `TimeoutHandler` and `gzipHandler` entirely. Without TLS, gRPC clients reach it over the native cleartext HTTP/2 above. Sharing never applies to IPv6. Because shared-port gRPC rides the HTTP server's connections, it drains with them on shutdown (GOAWAY, in-flight RPCs finish); `GracefulStop` only runs for a standalone gRPC server. To run a standalone gRPC server on a different port, set `grpcPortV4` to a different value. Source: @@ -182,7 +182,7 @@ All fields live under `server.` unless noted. `Duration` accepts Go duration str - `MaxHeaderBytes` = 1 MiB — the only hard inbound size limit - No request-body size limit — `util.ReadAll` copies until EOF with no upper bound; effective limits are `readTimeout` and process memory - Graceful shutdown budget = 30s -- Response compression threshold = 1024 bytes on the first write +- Response compression threshold = 1024 bytes of cumulative body - TLS `MinVersion` = TLS 1.2 ### Worked examples @@ -426,7 +426,7 @@ Note: `writeFatalError` (panics, write failures) and the timeout/cancel paths fo 19. **CORS metric label mismatch.** The label named `project` is populated with the URL path, not the project ID. Source: 20. **`serveArchitecture`-only aliasing requires a project in the URL path.** Combining `serveArchitecture` + `serveChain` without `serveProject` is always `ErrInvalidUrlPath`. Source: 21. **OPTIONS to a project without CORS config falls through to normal request handling.** The OPTIONS early-return lives inside the `CORS != nil` branch; a project with no `cors:` block sends an empty body OPTIONS through the full JSON-RPC path, returning a JSON-RPC error (not a 204). Source: -22. **Gzip decision is first-write-only.** A response whose first chunk is smaller than 1024 bytes is never compressed, even if subsequent writes are large. Source: +22. **An explicit `Flush()` below 1024 bytes commits to uncompressed.** The gzip writer buffers sub-threshold body bytes; a handler that flushes before reaching 1024 cumulative bytes forces passthrough for the rest of the response, however large. The JSON-RPC path does not flush mid-body, so this only affects streaming handlers. Source: 23. **Done/canceled race silently drops the response.** If the inner handler finishes but the request context has already errored (e.g. a race between handler return and deadline), nothing is written to the socket. The timeout layer has already responded. Source: 24. **Counter headers are zero-filled on early errors, not omitted.** Even URL-parse or project-lookup failures emit `X-ERPC-Attempts: 0`, `X-ERPC-Upstream-Attempts: 0`, etc., because `writeCounterHeaders` runs against a nil-safe snapshot. Only `executionHeaders: "off"` suppresses them entirely. Source: 25. **`ErrUnknown` fallback body is not JSON-RPC shaped.** When `processErrorBody` receives an error that survives all unwrapping as neither a `*common.BaseError` nor `common.StandardError`, it produces `{"code":"ErrUnknown","message":"unexpected server error","cause":{...}}` — a struct dump, not `{"jsonrpc":"2.0","error":{...}}`. Clients parsing `response.error.code` as an integer will fail; detection must branch on whether the outer object has a `code` string key vs an `error` object key. Source: diff --git a/docs/pages/free.mdx b/docs/pages/free.mdx index a59f0acfd..43d741f73 100644 --- a/docs/pages/free.mdx +++ b/docs/pages/free.mdx @@ -10,8 +10,8 @@ import { AISection, SourceLink, LLMsTxtLink } from "../components"; # Free & Public RPCs No config file, no API keys, no provider signup. Start eRPC and it immediately -routes every EVM chain through two auto-injected providers — a curated public -endpoint catalog and Envio HyperRPC — backed by eRPC's full resilience stack. +routes every EVM chain through an auto-injected provider — a curated public +endpoint catalog — backed by eRPC's full resilience stack. The fastest way to go from zero to any chain in development or prototyping. ## Quick taste @@ -50,7 +50,7 @@ curl 'http://localhost:4000/main/evm/42161' \ ### How it works When a project has zero `providers`, zero `upstreams`, and no CLI `--endpoint` flags, eRPC -logs a warning and auto-injects two providers before any request is processed: +logs a warning and auto-injects one provider before any request is processed: - **`repository` (id `public`)** — fetches `https://evm-public-endpoints.erpc.cloud` on first request for a chain and re-checks every hour. The catalog is a JSON map of @@ -58,37 +58,27 @@ logs a warning and auto-injects two providers before any request is processed: upstream with `autoIgnoreUnsupportedMethods: true`; `wss`/`ws` entries are silently skipped. Upstream ids embed a redacted endpoint (`https#redacted=ab12c`) so no raw URLs appear in logs or metrics. -- **`envio`** — HyperRPC at `rpc.hypersync.xyz`. A 14-method read-only allow-list is - injected (`eth_chainId`, `eth_blockNumber`, `eth_getBlockByNumber`, `eth_getBlockByHash`, - `eth_getTransactionByHash`, `eth_getTransactionByBlockHashAndIndex`, - `eth_getTransactionByBlockNumberAndIndex`, `eth_getTransactionReceipt`, - `eth_getBlockReceipts`, `eth_getLogs`, `eth_getFilterLogs`, `eth_getFilterChanges`, - `eth_uninstallFilter`, `eth_newFilter`). Write methods fall through to `repository` - upstreams only. - -On the first request for a chain, eRPC schedules bootstrap tasks for both providers and + +On the first request for a chain, eRPC schedules a bootstrap task for the provider and waits up to 30 seconds (capped by the request deadline) polling every 200 ms until at least one upstream is ready. Subsequent requests on the same chain reuse the prepared upstreams. Failed bootstrap tasks are auto-retried with exponential backoff (factor 1.5, 3 s → 130 s). -The hot-path data path is lock-free: both providers cache catalog data in a `RemoteDataCache` +The hot-path data path is lock-free: the provider caches catalog data in a `RemoteDataCache` backed by an `atomic.Pointer`, so reads on warm chains involve no I/O and no mutex. ### Config schema -Auto-injected providers are not configurable through YAML. To understand their exact defaults: +The auto-injected provider is not configurable through YAML. Its exact defaults: | Field | Type | Auto-injected value | Behavior / footguns | |---|---|---|---| -| `providers[].id` | string | `"public"` (repository), `"envio"` (envio) | Appears in upstream task names and metric labels. Do not reuse `"public"` if you later add explicit providers — id collisions cause undefined upstream selection behavior. | -| `providers[].vendor` | string | `"repository"` / `"envio"` | Selects the vendor integration. | +| `providers[].id` | string | `"public"` | Appears in upstream task names and metric labels. Do not reuse `"public"` if you later add explicit providers — id collisions cause undefined upstream selection behavior. | +| `providers[].vendor` | string | `"repository"` | Selects the vendor integration. | | `providers[].settings` | map | `{}` (all vendor defaults) | Cannot be customized without opting out of auto-injection by explicitly adding a provider or upstream. | | `repository` › `repositoryUrl` | string | `"https://evm-public-endpoints.erpc.cloud"` | Remote JSON catalog. Non-integer chain id keys are silently skipped. Only `http`/`https` entries produce upstreams. | | `repository` › `recheckInterval` | duration | `1h` | Catalog freshness window. **YAML footgun**: the value is read with a Go `.(time.Duration)` type assertion; YAML decodes scalars to `string`/`int64`, so the assertion always fails silently and the 1 h default is used regardless of what you write. Only Go programmatic config (`common.VendorSettings{"recheckInterval": 1*time.Hour}`) can override it. | | `repository` › `autoIgnoreUnsupportedMethods` | bool | `true` | Public RPCs vary wildly in method support; unsupported-method errors are treated as permanent and that method is skipped on that upstream. | -| `envio` › `rootDomain` | string | `"rpc.hypersync.xyz"` | URL template `https://{chainId}.{rootDomain}`. | -| `envio` › `apiKey` | string | `""` (omitted from URL) | Appended as a path segment when non-empty. | -| `envio` method allow-list | injected | `ignoreMethods: ["*"]` + 14 explicit `allowMethods` | Listed under "How it works". Any method not in the list is blocked and falls through to `repository` upstreams. | ### Worked examples @@ -125,7 +115,7 @@ projects: apiKey: ${ALCHEMY_KEY} ``` -This disables both `repository` and `envio` auto-injection entirely. +This disables `repository` auto-injection entirely. **3. Pointing `repository` at a private mirror (Go programmatic config only).** Because `recheckInterval` and `repositoryUrl` cannot be overridden from YAML, the only path that @@ -142,17 +132,18 @@ providers: []common.ProviderConfig{{ }} ``` -This is an explicit provider (not auto-injected) so `envio` is no longer auto-added. - -**4. Read-only chain fan-out behind Envio.** Because `envio` covers 14 read methods with -sub-millisecond latency for 61 well-known chains, a workload heavy on `eth_getLogs` or -`eth_getBlockReceipts` benefits from the zero-config default even in staging environments -where you already have API keys — add Envio as an explicit upstream alongside your provider: +**4. Read-only chain fan-out behind Envio.** Envio HyperRPC covers 14 read methods for 61 +well-known chains and needs an API token from [app.envio.dev](https://app.envio.dev/api-tokens). +A workload heavy on `eth_getLogs` or `eth_getBlockReceipts` can add it next to the public catalog: ```yaml -upstreams: - - id: envio-hyper - endpoint: envio://rpc.hypersync.xyz +providers: + - id: public + vendor: repository + - id: envio + vendor: envio + settings: + apiKey: ${ENVIO_API_TOKEN} ``` ### Request/response behavior @@ -162,9 +153,6 @@ upstreams: `/main/arbitrum`. See [URL structure](/reference/evm/url-structure) for all accepted shapes. - Only `POST` and `OPTIONS` reach the proxy; any other method (`GET`, etc.) is silently treated as a healthcheck and returns a JSON health envelope, not a JSON-RPC response. -- Write methods (`eth_sendRawTransaction`, etc.) are not blocked by eRPC — but they cannot - route through the `envio` provider (14-method allow-list). If `repository` upstreams for - the chain are available they will carry the write. - Responses carry standard `X-ERPC-*` headers (attempts, upstream, etc.) even on zero-config setups. - Unknown chain IDs return HTTP 404 with a JSON-RPC error body (`ErrNetworkNotSupported`) @@ -202,38 +190,25 @@ upstreams: used. Only Go programmatic config can override it. Source: 3. **`repository` catalog cold start is not instant** — first request for any chain may - wait for the HTTP fetch (no static fallback). `envio` cold start is fast for 61 known - chains but requires a live `eth_chainId` probe for unknown ones (10 s timeout). - Source: ; -4. **`envio` TLS error on unknown subdomain = unsupported** — Envio's k8s load-balancer - returns a bad certificate for unrecognized chain subdomains; eRPC treats - `"failed to verify certificate"` as `(false, nil)` (chain not supported), not an error. - Source: -5. **`wss`/`ws` endpoints in the repository catalog are silently skipped** — only + wait for the HTTP fetch (no static fallback). + Source: +4. **`wss`/`ws` endpoints in the repository catalog are silently skipped** — only `http`/`https` entries produce upstreams. Source: -6. **Write methods are unavailable via envio** — the 14-method allow-list blocks - `eth_sendRawTransaction` and all state-mutation calls. Writes fall through to `repository` - upstreams only; if no repository upstream supports the method it will fail. -7. **Rate limits on public endpoints are uncontrolled** — no rate-limit budget or per-upstream +5. **Rate limits on public endpoints are uncontrolled** — no rate-limit budget or per-upstream circuit-breaker is configured on auto-injected upstreams. A misbehaving upstream will only be flagged once the upstream's circuit-breaker window fills (if a project-wide policy is configured). -8. **Provider id `"public"` is reserved** by the auto-injected repository provider. Adding +6. **Provider id `"public"` is reserved** by the auto-injected repository provider. Adding an explicit provider with the same id causes upstream-id collisions; the upstream registry will deduplicate by id, silently merging them. Source: -9. **`envio` nil Evm dereference in GenerateConfigs** — `GenerateConfigs` dereferences - `upstream.Evm` without a nil check (envio.go:202). In practice the provider bootstrap - always sets `Evm` for `evm:` networks, but a manually-constructed upstream without `Evm` - would panic here. - Source: ### Observability | Metric | Type | Labels | When it fires | |---|---|---|---| -| `erpc_upstream_request_total` | counter | `project, vendor, network, upstream, ...` | Every request forwarded to a repository or envio upstream. `vendor` label is `"repository"` or `"envio"`. | +| `erpc_upstream_request_total` | counter | `project, vendor, network, upstream, ...` | Every request forwarded to a repository upstream. `vendor` label is `"repository"`. | | `erpc_upstream_request_duration_seconds` | histogram | `project, vendor, network, upstream` | Latency of each upstream call; useful for spotting slow public endpoints. | | `erpc_selection_score` | gauge | `project, network, method, upstream` | Per-upstream score produced by the selection policy (`sortByScore`). Lower = better. | @@ -247,8 +222,6 @@ Notable log messages: - [`common/defaults.go:L1113-L1142`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L1113-L1142) — auto-injection conditions and injected provider construction (`len(providers)==0 && len(upstreams)==0 && no CLI endpoints`) - [`thirdparty/repository.go:L41-L92`](https://github.com/erpc/erpc/blob/main/thirdparty/repository.go#L41-L92) — `SupportsNetwork`, `autoIgnoreUnsupportedMethods` default, cold-start guard (`ErrRemoteCacheCold`) - [`thirdparty/repository.go:L121-L166`](https://github.com/erpc/erpc/blob/main/thirdparty/repository.go#L121-L166) — `GenerateConfigs` fan-out: one upstream per `http`/`https` endpoint, redacted ids -- [`thirdparty/envio.go:L22-L167`](https://github.com/erpc/erpc/blob/main/thirdparty/envio.go#L22-L167) — 61-chain fast path + live `eth_chainId` probe for unknown chains; TLS-error-as-unsupported branch -- [`thirdparty/envio.go:L174-L194`](https://github.com/erpc/erpc/blob/main/thirdparty/envio.go#L174-L194) — 14-method allow-list injection - [`thirdparty/remote_cache.go:L13-L78`](https://github.com/erpc/erpc/blob/main/thirdparty/remote_cache.go#L13-L78) — `RemoteDataCache` design rationale: lock-free hot path, single-flight async refresh, copy-on-write - [`upstream/registry.go:L155-L293`](https://github.com/erpc/erpc/blob/main/upstream/registry.go#L155-L293) — lazy provider bootstrap, 30 s readiness wait, `ErrNetworkInitializing` / `ErrNetworkNotSupported` - [`util/initializer.go:L179-L182`](https://github.com/erpc/erpc/blob/main/util/initializer.go#L179-L182) — exponential retry parameters for failed bootstrap tasks (factor 1.5, 3 s → 130 s) diff --git a/docs/pages/index.mdx b/docs/pages/index.mdx index 6de4ab1af..b60009f6d 100644 --- a/docs/pages/index.mdx +++ b/docs/pages/index.mdx @@ -121,7 +121,7 @@ eRPC is a fault-tolerant EVM RPC proxy and permanent caching layer. The bootstra **TypeScript configs** are compiled in-process via esbuild (IIFE bundle) and evaluated with the sobek JS runtime, so closures and module-level helpers in selection-policy `evalFunc` functions are preserved natively — no function serialisation. -**No-config mode**: when `projects` is empty, a synthetic `main` project with public repository + Envio providers is injected, and `/evm/` resolves without a project prefix. +**No-config mode**: when `projects` is empty, a synthetic `main` project with public repository provider is injected, and `/evm/` resolves without a project prefix. ### Config schema diff --git a/docs/pages/operation/cli.mdx b/docs/pages/operation/cli.mdx index 6fc49c751..bdfc1cb9c 100644 --- a/docs/pages/operation/cli.mdx +++ b/docs/pages/operation/cli.mdx @@ -200,7 +200,7 @@ pprof — build with `-tags pprof` to activate. | `healthCheck` | \*HealthCheckConfig | mode=`networks`, defaultEval=`any:initializedUpstreams` | Health check endpoint behaviour. | | `admin` | \*AdminConfig | nil | Admin JSON-RPC API; CORS defaults to `*` origin with no credentials when section is present. | | `database` | \*DatabaseConfig | nil | EVM JSON-RPC cache and/or shared-state connector. | -| `projects` | []\*ProjectConfig | synthetic `main` project with public + envio providers when empty | Ordered list of project configs. When empty, `SetDefaults` injects a `main` project. | +| `projects` | []\*ProjectConfig | synthetic `main` project with the public repository provider when empty | Ordered list of project configs. When empty, `SetDefaults` injects a `main` project. | | `rateLimiters` | \*RateLimiterConfig | nil | Rate limiter budgets. Budget IDs referenced by projects/upstreams/networks must exist here. | | `proxyPools` | []\*ProxyPoolConfig | nil | HTTP proxy pool definitions. | | `tracing` | \*TracingConfig | nil | OTel tracing; defaults when set: protocol=grpc, endpoint=localhost:4317, sampleRate=1.0, serviceName=erpc. | diff --git a/docs/pages/operation/directives.mdx b/docs/pages/operation/directives.mdx index 925834e75..c2ceeaa91 100644 --- a/docs/pages/operation/directives.mdx +++ b/docs/pages/operation/directives.mdx @@ -174,10 +174,10 @@ Config struct: [`common/config.go:2105-2152`](https://github.com/erpc/erpc/blob/ | 1 | `X-ERPC-Retry-Empty` | `retry-empty` | bool | `retryEmpty` | `false` | Retry on null/empty upstream response. Subject to `EmptyResultMaxAttempts` cap. | [`erpc/network_executor.go:427-442`](https://github.com/erpc/erpc/blob/main/erpc/network_executor.go#L427-L442) | | 2 | `X-ERPC-Retry-Pending` | `retry-pending` | bool | `retryPending` | `false` (struct comment "true by default" is **stale**) | Retry `eth_getTransactionReceipt` / `eth_getTransactionByHash` / `eth_getTransactionByBlockHashAndIndex` / `eth_getTransactionByBlockNumberAndIndex` while tx is pending. `EmptyResultMaxAttempts` cap applies. | [`erpc/network_executor.go:445-459`](https://github.com/erpc/erpc/blob/main/erpc/network_executor.go#L445-L459) | | 3 | `X-ERPC-Skip-Cache-Read` | `skip-cache-read` | string (bool in config YAML normalized to string) | `skipCacheRead` | `""` (no skip) | `"true"` = skip all; `"false"`/`""` = use cache; other string = connector-ID WildcardMatch pattern. YAML `true`/`false` normalized to strings via `fmt.Sprintf("%v", v)`. Header NOT trimmed; query IS trimmed. | [`common/request.go:898-914`](https://github.com/erpc/erpc/blob/main/common/request.go#L898-L914) | -| 4 | `X-ERPC-Use-Upstream` | `use-upstream` | string | `useUpstream` | `""` (no filter) | Selector matched against upstream `id`; for purely-positive patterns also against upstream `Tags` (any tag match wins). Negated patterns (`!`) match ID only. Header NOT trimmed; query IS trimmed. No match → `ErrUpstreamsExhausted`. Activates selector-scoped served-tip partitioning. Stateful methods with multiple matching upstreams fail with `ErrNotImplemented`. | [`upstream/upstream.go:1526-1534`](https://github.com/erpc/erpc/blob/main/upstream/upstream.go#L1526-L1534) | +| 4 | `X-ERPC-Use-Upstream` | `use-upstream` | string | `useUpstream` | `""` (no filter) | Selector matched against upstream `id`; for purely-positive patterns also against upstream `Tags` (any tag match wins). Negated patterns (`!`) match ID only. Header NOT trimmed; query IS trimmed. No match → `ErrUpstreamsExhausted`. Scopes the [served tip](/reference/evm/block-tracking#selector-scoped-served-tip) to the matched upstreams; a simple group selector also gets its own [lane](/reference/lanes). Stateful methods with multiple matching upstreams fail with `ErrNotImplemented`. | [`upstream/upstream.go:1526-1534`](https://github.com/erpc/erpc/blob/main/upstream/upstream.go#L1526-L1534) | | 5 | `X-ERPC-Skip-Interpolation` | `skip-interpolation` | bool | `skipInterpolation` | `false` | Suppresses block-tag → hex substitution in forwarded params. Internal block refs still computed/cached. | [`architecture/evm/json_rpc.go:132`](https://github.com/erpc/erpc/blob/main/architecture/evm/json_rpc.go#L132) | | 6 | `X-ERPC-Skip-Consensus` | `skip-consensus` | bool | `skipConsensus` | `false` | Bypasses consensus branch; uses standard `retry(hedge(upstreamSweep))`. Retry/hedge/breaker/timeout still apply. | [`erpc/network_executor.go:179-188`](https://github.com/erpc/erpc/blob/main/erpc/network_executor.go#L179-L188) | -| 7 | `X-ERPC-Enforce-Highest-Block` | `enforce-highest-block` | bool | `enforceHighestBlock` | **`true`** (SetDefaults) | `eth_getBlockByNumber` with `"latest"`/`"finalized"`: re-routes if response is behind known highest block. Skipped for cached responses. | [`architecture/evm/eth_getBlockByNumber.go:111-170`](https://github.com/erpc/erpc/blob/main/architecture/evm/eth_getBlockByNumber.go#L111-L170) | +| 7 | `X-ERPC-Enforce-Highest-Block` | `enforce-highest-block` | bool | `enforceHighestBlock` | **`true`** (SetDefaults) | `eth_getBlockByNumber` with `"latest"`/`"finalized"`: re-routes if response is behind known highest block. Skipped for cached responses. `eth_blockNumber`: answered from and pinned to the served tip ([details](/reference/evm/block-tracking#eth_blocknumber)). | [`architecture/evm/eth_getBlockByNumber.go:111-170`](https://github.com/erpc/erpc/blob/main/architecture/evm/eth_getBlockByNumber.go#L111-L170), [`architecture/evm/eth_blockNumber.go`](https://github.com/erpc/erpc/blob/main/architecture/evm/eth_blockNumber.go) | | 8 | `X-ERPC-Enforce-GetLogs-Range` | `enforce-getlogs-range` | bool | `enforceGetLogsBlockRange` | **`true`** (SetDefaults) | Pre-forward: rejects `eth_getLogs`/`trace_filter` if block range exceeds `getLogsMaxAllowedRange`. Consumed via legacy `EvmIntegrityConfig` (not a true per-request override — see Edge cases). | [`architecture/evm/eth_getLogs.go:280-288`](https://github.com/erpc/erpc/blob/main/architecture/evm/eth_getLogs.go#L280-L288) | | 9 | `X-ERPC-Enforce-Non-Null-Tagged-Blocks` | `enforce-non-null-tagged-blocks` | bool | `enforceNonNullTaggedBlocks` | **`true`** (SetDefaults) | Treats null block for tag-based `eth_getBlockByNumber` as an error. Disable for chains that legitimately return null for certain tags (e.g., zkSync). | [`architecture/evm/eth_getBlockByNumber.go:304`](https://github.com/erpc/erpc/blob/main/architecture/evm/eth_getBlockByNumber.go#L304) | | 10 | `X-ERPC-Integrity` | `integrity` | string | — | `""` | Per-request data-integrity selection — a level (`off`/`intrinsic`/`corroborated`/`authoritative`) or a configured profile name; honored only when the network's integrity `headerMode` permits. See [Integrity checks](/config/failsafe/integrity). | [`architecture/evm/hooks.go`](https://github.com/erpc/erpc/blob/main/architecture/evm/hooks.go) | @@ -369,7 +369,7 @@ Responses also carry `X-ERPC-Network-Id` and `X-ERPC-Network-Alias` (the network 6. **`EnforceGetLogsBlockRange` cannot be overridden per-request via HTTP.** `SetDefaults` copies `DirectiveDefaults` values into `Evm.Integrity`; the architecture layer reads from the network config struct, not the per-request directive. Source: [`common/defaults.go:1957-1964`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L1957-L1964). 7. **Batch responses emit ONE aggregated `X-ERPC-*` header set.** Counters sum across sub-calls, `X-ERPC-Cache` becomes `PARTIAL:` when mixed, `X-ERPC-Duration` is the slowest sub-call, the trace caps at 50 segments (`X-ERPC-Upstreams-Truncated` carries the overflow count), and the single-winner `X-ERPC-Upstream` is not emitted. Per-sub-call attribution still needs the per-response `id` fields in the body. 8. **`executionHeaders: summary` removes only `X-ERPC-Upstreams`.** All counter and metadata headers still emit. Use `"off"` to suppress everything. -9. **Selector-scoped served-tip partitions are capped.** Beyond `maxServedTipPartitions` per network, no partition is created and the stateless fallback is used silently — no error. Source: [`erpc/networks.go:429`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L429). +9. **Selector-scoped served-tip lanes are capped.** Past the per-network [lane cap](/reference/lanes), the tip is still scoped to the matched upstreams but gets no lane — no error. 10. **`nil *bool` in `DirectiveDefaultsConfig` ≠ `false *bool`.** Nil means "not set — skip"; a `*false` pointer means "explicitly disable". Only non-nil pointers are applied in `ApplyDirectiveDefaults`. Source: [`common/request.go:570-580`](https://github.com/erpc/erpc/blob/main/common/request.go#L570-L580). 11. **`allowClientDirectives` filters HTTP-supplied directives only.** Config-set `directiveDefaults` always apply regardless of the filter. The pattern is pre-compiled at project registration via `NewWildcardMatcher` and evaluated against each directive's query-param key (e.g. `skip-cache-read`, `use-upstream`). `nil` = all allowed; `""` = none allowed; `"!skip-cache-read & !use-upstream"` = all except those two. Does not filter `X-ERPC-Force-Trace` (processed before project resolution). Source: `isDirectiveAllowed` method on `NormalizedRequest` in `common/request.go`, `NewWildcardMatcher` in `common/matcher.go`, `AllowClientDirectives` in `common/config.go`. See [projects config](/config/projects). @@ -396,7 +396,7 @@ Directives have no dedicated Prometheus metrics; their effects appear in existin - [`erpc/http_server.go:1081-1272`](https://github.com/erpc/erpc/blob/main/erpc/http_server.go#L1081-L1272) — `executionHeadersMode`, `setResponseHeaders`, `writeCounterHeaders`, `writeResponseMetadataHeaders`, `writeUpstreamTraceHeaders` - [`erpc/network_executor.go:179-188`](https://github.com/erpc/erpc/blob/main/erpc/network_executor.go#L179-L188) — `SkipConsensus` branch - [`erpc/network_executor.go:374-462`](https://github.com/erpc/erpc/blob/main/erpc/network_executor.go#L374-L462) — `shouldRetryWithReason`: `RetryEmpty` / `RetryPending` checks and `EmptyResultMaxAttempts` cap -- [`erpc/networks.go:374-441`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L374-L441) — selector-scoped served-tip: `requestSelector`, `servedTipPartitionFor`, partition cap +- [`erpc/networks.go:564-655`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L564-L655) — selector-scoped served tip: `EvmRoutableUpstreams`, `requestSelector`, `servedTipLaneFor` - [`upstream/upstream.go:1505-1548`](https://github.com/erpc/erpc/blob/main/upstream/upstream.go#L1505-L1548) — `shouldSkip`: `UseUpstream` selector check via `UpstreamMatchesSelector` - [`common/matcher.go:34-112`](https://github.com/erpc/erpc/blob/main/common/matcher.go#L34-L112) — `WildcardMatch`, `MatchesSelector`, `UpstreamMatchesSelector` - [`erpc/skip_consensus_directive_test.go`](https://github.com/erpc/erpc/blob/main/erpc/skip_consensus_directive_test.go) — end-to-end `SkipConsensus` tests diff --git a/docs/pages/operation/monitoring.mdx b/docs/pages/operation/monitoring.mdx index 3d8af0c09..d626db602 100644 --- a/docs/pages/operation/monitoring.mdx +++ b/docs/pages/operation/monitoring.mdx @@ -377,8 +377,8 @@ and rate-limit events — but **two rules reference stale metric names** (`erpc_ | Metric | What to check | |---|---| -| `erpc_upstream_block_head_lag` | Blocks behind the freshest upstream; alert on persistent lag | -| `erpc_upstream_finalization_lag` | Finalized blocks behind freshest upstream | +| `erpc_upstream_block_head_lag` | Blocks behind the corroborated network head (second-highest non-benched upstream); alert on persistent lag | +| `erpc_upstream_finalization_lag` | Finalized blocks behind the corroborated network finalized head | | `erpc_upstream_request_duration_seconds` | p95 latency per upstream; bundled alert fires when p95 > 1s | | `erpc_upstream_request_errors_total` | Error rate per upstream; bundled alert fires at 5% error rate | | `erpc_selection_position` | `0` = primary, `1+` = runner-up, `-1` = excluded this tick | @@ -416,10 +416,11 @@ and rate-limit events — but **two rules reference stale metric names** (`erpc_ | Metric | What to check | |---|---| | `erpc_network_dynamic_block_time_milliseconds` | EMA block-time estimate (α=0.1, min 3 samples); 0 = startup or halted chain; sanity-clamped to [10ms, 120s] | -| `erpc_network_served_tip_block_number` | Served-tip pick per axis (latest/finalized) and lane; `lane="all"` = network-wide | -| `erpc_network_served_tip_lag_blocks` | Lag of served tip behind freshest velocity-eligible upstream; absent in MAX mode | | `erpc_upstream_stale_latest_block_total` | Upstream returned a block number behind the network leader | | `erpc_upstream_block_head_large_rollback` | Size of the last rollback > 1024 blocks, per `finality` axis (`latest`/`finalized`). Sticky — alert on `changes()`, not on `> 0` | +| `erpc_network_served_tip_advance_age_seconds` | Seconds since the served tip last changed; sustained high on a live chain = stuck tip | +| `erpc_network_served_tip_regression_total` | Non-zero = the regression guard held or gave up holding | +| `erpc_upstream_head_benched_total` | Upstream head benched: it stayed behind peers that kept advancing | **Solana (SVM) upstreams** @@ -493,7 +494,7 @@ Ingestion lag (`maxShredInsertSlot − processedSlot`) has **no gauge** — it i 8. **`erpc_upstream_cordoned` `category` label is the cordon scope (method string), not a request-category.** `"*"` = wholesale cordon; a specific method string = per-method cordon. Do not join this `category` with `category` on request-accounting metrics. () -9. **`erpc_auth_failed_total` always has `strategy="database"`.** Only the database auth strategy records this counter. For other strategies, monitor `erpc_network_failed_request_total{error=~"ErrAuthUnauthorized.*"}` instead. () +9. **`erpc_auth_failed_total` covers client-facing auth denials.** Filter by `strategy` and `reason` (e.g. `strategy="jwt",reason="token_expired"`). () 10. **`erpc_upstream_attempt_outcome_total` `is_hedge` and `is_retry` are `"true"`/`"false"` strings, not `"1"`/`"0"`.** Use `{is_hedge="true"}` in Prometheus queries. () @@ -577,8 +578,8 @@ Top-20 operational metrics: | `erpc_upstream_request_total` | counter | project, vendor, network, upstream, category, attempt, composite, finality, user, agent_name | Each attempt sent to an upstream | | `erpc_upstream_request_errors_total` | counter | project, vendor, network, upstream, category, error, severity, composite, finality, user, agent_name | Upstream attempt returned an error | | `erpc_upstream_request_duration_seconds` | LabeledHistogram | project, vendor, network, upstream, category, composite, finality, user | Duration of each upstream attempt; idle series swept every 30 min | -| `erpc_upstream_block_head_lag` | gauge | project, vendor, network, upstream | Blocks behind the freshest upstream | -| `erpc_upstream_finalization_lag` | gauge | project, vendor, network, upstream | Finalized blocks behind the freshest upstream | +| `erpc_upstream_block_head_lag` | gauge | project, vendor, network, upstream | Blocks behind the corroborated network head (second-highest non-benched upstream) | +| `erpc_upstream_finalization_lag` | gauge | project, vendor, network, upstream | Finalized blocks behind the corroborated network finalized head | | `erpc_upstream_cordoned` | gauge | project, vendor, network, upstream, category, reason | `1` while cordoned; `0` when active | | `erpc_upstream_breaker_state_change_total` | counter | project, upstream, transition | Circuit-breaker state transition | | `erpc_selection_position` | gauge | project, network, method, upstream | `0`=primary, `1+`=runner-up, `-1`=excluded | @@ -602,9 +603,6 @@ Top-20 operational metrics: | `erpc_rate_limiter_budget_max_count` | gauge | budget, method, scope | Budget's configured req/s; `scope` = ScopeString() (comma-joined flags: user/network/ip) | | `erpc_rate_limiter_remote_duration_seconds` | LabeledHistogram | budget, result | Remote rate-limit check (e.g. Redis DoLimit) duration; `result` ∈ ok/over_limit/fail_open | | `erpc_network_dynamic_block_time_milliseconds` | gauge | project, network | EMA block-time (α=0.1, min 3 samples); 0 = startup/halted; clamped [10ms, 120s] | -| `erpc_network_served_tip_block_number` | gauge | project, network, lane, axis | Served-tip pick; `lane="all"` = network-wide; `axis` ∈ latest/finalized | -| `erpc_network_served_tip_lag_blocks` | gauge | project, network, lane, axis | Lag behind freshest velocity-eligible upstream; absent in MAX mode | -| `erpc_network_served_tip_upstream_excluded_total` | counter | project, network, upstream, axis, reason | Upstream excluded from tip pick; `reason` ∈ velocity/outlier | | `erpc_network_evm_block_range_requested_total` | counter | project, network, vendor, upstream, category, user, finality, bucket, size | Block-range heatmap; `bucket` = tip-relative label when tip known (`"TIP"`, `"L100k"`, `"100k-200k"`, …) or static 100k-aligned label | | `erpc_network_evm_get_logs_forced_splits_total` | counter | project, network, dimension, user, agent_name | eth_getLogs forcibly split; `dimension` ∈ block_range/addresses/topics0 | | `erpc_upstream_stale_upper_bound_total` | counter | project, vendor, network, upstream, category, confidence | Request skipped: upstream latest < requested upper bound; `confidence` ∈ blockHead/finalizedBlock | @@ -614,8 +612,13 @@ Top-20 operational metrics: | `erpc_shadow_response_mismatch_total` | counter | project, vendor, network, upstream, category, finality, emptyish, larger | Shadow upstream diverged from primary; `emptyish`/`larger` ∈ `"true"`/`"false"` | | `erpc_selection_probe_errors_total` | counter | network, upstream, method, reason | Probe to excluded upstream errored; `reason` ∈ timeout/throttled/auth/skipped/error | | `erpc_upstream_wrong_empty_response_total` | counter | project, vendor, network, upstream, category, finality, user, agent_name | Upstream returned empty while consensus showed others had data | -| `erpc_auth_failed_total` | counter | project, network, strategy, reason, agent_name | Auth failure; `strategy` is always `"database"` (only database strategy emits this) | +| `erpc_auth_failed_total` | counter | project, network, strategy, reason, agent_name | Client-facing auth denial; `agent_name` always `unknown`; closed `reason` sets | | `erpc_unexpected_panic_total` | counter | scope, extra, error | Recovered panic; `scope` ∈ request-handler/final-error-writer/top-level-handler/timeout-handler/validate-pattern/redis-pubsub/shared-state-registry/matcher | +| `erpc_network_served_tip_block_number` | gauge | project, network, lane, axis | Served tip as served; `lane="all"` = network-wide; `axis` ∈ latest/finalized | +| `erpc_network_served_tip_lag_blocks` | gauge | project, network, lane, axis | Blocks the served tip sits below the second-highest voter, clamped at 0 | +| `erpc_network_served_tip_advance_age_seconds` | gauge | project, network, lane, axis | Seconds since the served tip last changed; absent until the first change | +| `erpc_network_served_tip_regression_total` | counter | project, network, lane, axis, outcome | Regression-guard transitions; `outcome` ∈ held/failed_open | +| `erpc_upstream_head_benched_total` | counter | project, vendor, network, upstream | Upstream head benched: it stayed behind peers that kept advancing | ### Source code entry points diff --git a/docs/pages/operation/tracing.mdx b/docs/pages/operation/tracing.mdx index 297b31769..5499a3ab4 100644 --- a/docs/pages/operation/tracing.mdx +++ b/docs/pages/operation/tracing.mdx @@ -260,7 +260,7 @@ Full struct: . | Field | Type | Default | Behavior / footguns | |---|---|---|---| | `tracing.enabled` | bool | `false` (Go zero value) | Master switch. When `false`, sets `IsTracingEnabled = false` globally; all span calls are no-ops (). | -| `tracing.endpoint` | string | `"localhost:4317"` (gRPC) or `"http://localhost:4318"` (HTTP) — derived from `protocol` in `SetDefaults` () | OTLP collector endpoint. gRPC: `host:port` with no scheme. HTTP: full URL including scheme. **Footgun:** `OTEL_EXPORTER_OTLP_ENDPOINT` is always overridden by this field even when it holds the default value. | +| `tracing.endpoint` | string | `"localhost:4317"` (gRPC) or `"http://localhost:4318"` (HTTP) — derived from `protocol` in `SetDefaults` () | OTLP collector endpoint. gRPC: `host:port` with no scheme. HTTP: `http(s)://host:port[/path]` or `host:port`; the URL scheme decides TLS (`http://` with `tls.enabled` is an init error), and a URL without a path exports to `/v1/traces`. **Footgun:** `OTEL_EXPORTER_OTLP_ENDPOINT` is always overridden by this field even when it holds the default value. | | `tracing.protocol` | TracingProtocol | `"grpc"` () | `"grpc"` uses `otlptracegrpc`; `"http"` uses `otlptracehttp`. Any other value → error at init (). | | `tracing.sampleRate` | float64 | `1.0` () | `<= 0` → `NeverSample()`, `>= 1.0` → `AlwaysSample()`, otherwise → `ParentBased(TraceIDRatioBased(rate))`. **Footgun:** `SetDefaults` replaces `0` with `1.0`, so `sampleRate: 0` in YAML means "always sample", not "never sample". To keep tracing on but silent, use a very small positive value like `0.0000001`. | | `tracing.detailed` | bool | `false` (Go zero value) | When `true`, sets `IsTracingDetailed = true`, enabling `StartDetailSpan` calls. Roughly 2× span count. Includes request params, block numbers, lock spans, hook execution spans. | @@ -575,6 +575,7 @@ only when `tracing.enabled && tracing.detailed`; "direct" = calls `tracer.Start` | `Network.Forward` | normal | `erpc/networks.go:L943` | | `Network.GetFinality` | detail | `erpc/networks.go:L1646` | | `Network.NormalizeResponse` | detail | `erpc/networks.go:L2235` | +| `Network.PostForward.eth_blockNumber` | detail | `architecture/evm/eth_blockNumber.go:L69` | | `Network.PostForward.eth_getBlockByNumber` | detail | `architecture/evm/eth_getBlockByNumber.go:L44` | | `Network.PostForward.eth_sendRawTransaction` | detail | `architecture/evm/eth_sendRawTransaction.go:L280` | | `Network.PostForwardHook` | detail | `architecture/evm/hooks.go:L64` | @@ -596,7 +597,6 @@ only when `tracing.enabled && tracing.detailed`; "direct" = calls `tracer.Start` | `PostgreSQLConnector.getWithWildcard` | detail | `data/postgresql.go:L1006` | | `Project.Forward` | detail | `erpc/projects.go:L103` | | `Project.PreForwardHook` | detail | `architecture/evm/hooks.go:L14` | -| `Project.PreForwardHook.eth_blockNumber` | detail | `architecture/evm/eth_blockNumber.go:L16` | | `Project.PreForwardHook.eth_chainId` | detail | `architecture/evm/eth_chainId.go:L30` | | `Project.executeShadowRequest` | detail | `erpc/shadow.go:L82` | | `Query.Execute` | detail | `erpc/query_executor.go:L45` | diff --git a/docs/pages/operation/websocket.mdx b/docs/pages/operation/websocket.mdx index ef5b677ae..ab0533e44 100644 --- a/docs/pages/operation/websocket.mdx +++ b/docs/pages/operation/websocket.mdx @@ -165,7 +165,8 @@ ws.onmessage = (event) => { - the network has no WebSocket upstream, or none of the selected upstreams is connected and accepts the subscription; - the subscription type is not supported (`-32602`); -- the connection already has `server.webSocket.maxSubscriptionsPerConnection` active subscriptions; +- the connection already has `server.webSocket.maxSubscriptionsPerConnection` active subscriptions (`-32005`); +- the instance already has `server.webSocket.maxSubscriptions` active subscriptions (`-32005`); - the request is rate limited. A failed `eth_subscribe` leaves your WebSocket connection open. You can retry on the same connection. @@ -241,8 +242,11 @@ The WebSocket upstream client features: - **Ping/pong keepalive** (30s interval) to detect dead connections - **Lazy connection** — if the upstream is unavailable at startup, eRPC retries in the background without blocking - **Fail-fast while reconnecting** — requests sent while the connection is down fail immediately with a retryable error so they fail over to other upstreams +- **Heads subscribed at startup** — while the WebSocket server is enabled, eRPC subscribes to `newHeads` on every WebSocket upstream as soon as its network initializes, so state pollers use pushed heads instead of polling even before any client subscribes +- **Silent subscription recovery** — if a `newHeads` subscription stops delivering while the same upstream's polled head keeps advancing, eRPC reconnects to that upstream +- **Unsupported `newHeads` is not retried** — if an upstream rejects `eth_subscribe newHeads` as unsupported, eRPC stops asking until the next reconnect or resubscribe; its state poller keeps tracking the head -`eth_subscribe` and `eth_unsubscribe` are only sent to upstreams by eRPC's own subscription handling. A plain HTTP `eth_subscribe` is never forwarded to a WebSocket upstream, since the resulting subscription would have no owner. +`eth_subscribe` and `eth_unsubscribe` are only sent to upstreams by eRPC's own subscription handling. Over HTTP both are rejected with `-32601` without reaching any upstream, since a subscription needs a client WebSocket connection to deliver to. ## Server configuration @@ -269,8 +273,20 @@ server: maxConcurrentRequestsPerConnection: 250 # Notifications queued per subscription for a slow reader (default: 256) subscriptionBufferSize: 256 + # Instance-wide caps; 0 (the default) means unlimited + maxConnections: 0 + maxConnectionsPerIP: 0 + maxSubscriptions: 0 ``` +`maxSubscriptionsPerConnection` alone does not bound a client: it can open any number of connections. On a public endpoint also set the instance-wide caps: + +- `maxConnections` — concurrent client connections on the instance. The only connection cap that holds when client IPs are unknown or spoofed. +- `maxConnectionsPerIP` — concurrent client connections from one client IP, resolved as for HTTP (see `trustedIPForwarders`). Together with `maxSubscriptionsPerConnection` it bounds one client's subscriptions. +- `maxSubscriptions` — active subscriptions across all connections on the instance. + +A connection over a cap is refused with `429` before the upgrade; a subscription over `maxSubscriptions` gets a JSON-RPC error and the connection stays open. Refusals are counted in `erpc_websocket_rejected_total`. The caps are per instance, so a fleet admits up to N times each. + The upgrade request must name a network (`///`, or a path that [domain aliasing](/operation/url) resolves to one); otherwise it is rejected with `400`. On shutdown eRPC closes every client connection with `1001` (going away) straight away so clients reconnect to another instance; in-flight requests get up to `server.waitBeforeShutdown` to answer first. ## Cost frames @@ -354,7 +370,7 @@ eRPC minimises the gap in three ways: - **Per-block cache keying for moving tags.** The realtime cache entry for a `latest` / `finalized` request is keyed by the block number the tag currently resolves to, so each tip advance is a distinct key rather than pinning every call within the TTL to the same cached payload. - **`enforceHighestBlock` response rewrite.** When the upstream returns a block number below the network's tracked tip, eRPC's [integrity check](/config/failsafe/integrity) fetches the specific higher block from another upstream and returns that instead (falling back to the highest valid block it has if none can serve it). -- **Delivered-head floor.** When an upstream WebSocket delivers a `newHeads` notification, eRPC updates that upstream's tip tracker before fanning the head out to clients, and records it as a floor for the network's `latest`. A head reaches subscribers, and feeds the floor, only once the delivering upstream's tip tracker has accepted it; a head far ahead of the tracker is verified first and is not delivered, so subscribers resume with the next head. Held-back fallback heads are not delivered either. The floor is bounded: a value no live upstream comes close to (a stale or poisoned entry) is ignored rather than trusted, so it cannot pin the served tip. When `servedTip` is enabled for `latest`, it never lifts `latest` past [`servedTip.guaranteedFor`](/config/projects/networks) / `guaranteedMethods`. It does not apply to requests scoped with `use-upstream`. With shared state configured the floor is also propagated to other instances in the background, so they converge within the propagation delay rather than instantly. +- **Delivered-head floor.** When an upstream WebSocket delivers a `newHeads` notification, eRPC updates that upstream's tip tracker before fanning the head out to clients, and records it as a floor for the network's `latest`. A head reaches subscribers, and feeds the floor, only once the delivering upstream's tip tracker has accepted it; a head far ahead of the tracker is verified first and is not delivered, so subscribers resume with the next head. Held-back fallback heads are not delivered either. With shared state configured the floor is also propagated to other instances in the background, so they converge within the propagation delay rather than instantly. How the floor bounds and orders against the served tip: [Block tracking](/reference/evm/block-tracking#websocket-delivered-head-floor). Even so, a transient single-block lag can reach a client when the chain produces a block faster than an in-flight HTTP round-trip completes. If your consumer treats single-block HTTP-vs-WS lag as a hard failure, widen its tolerance to match the chain's block time rather than expect eRPC (or any proxy) to eliminate it. diff --git a/docs/pages/reference/errors.mdx b/docs/pages/reference/errors.mdx index a640b698b..f1465c561 100644 --- a/docs/pages/reference/errors.mdx +++ b/docs/pages/reference/errors.mdx @@ -165,7 +165,7 @@ Legend: | 10 | `ErrNetworkNotFound` | yes | yes | 404 | −32603 | network not configured | | 11 | `ErrUnknownNetworkID` | yes | yes | 200 | −32603 | — | | 12 | `ErrUnknownNetworkArchitecture` | yes | yes | 200 | −32603 | — | -| 13 | `ErrNotImplemented` | yes | yes | 200 | −32603 | — | +| 13 | `ErrNotImplemented` | yes | yes | 200 | −32601 | stateful method under a multi-upstream pool, `eth_accounts`/`eth_sign`, or subscription over HTTP → method not supported | | 14 | `ErrInvalidEvmChainId` | yes | yes | 200 | −32603 | — | | 15 | `ErrFinalizedBlockUnavailable` | yes | yes | 200 | −32603 | finality check | | 16 | `ErrUpstreamClientInitialization` | yes | yes | — | — | not fatal unless wrapped in `TaskFatalError` | @@ -277,6 +277,7 @@ Entry point: "`, while other nodes answer the same call. Rule 7b maps that wording to `ErrEndpointMissingData`, so the sweep moves on to the next upstream. If every upstream lacks the state, the client gets `-32014` with the node's message verbatim. A contract that calls the precompile through a library which catches the failure and reverts with its own error (for example `PrecompileLib__SpotBalancePrecompileFailed()`) produces a plain `execution reverted`, which stays rule 9. `eth_sendRawTransaction` keeps rule 14. Confirmed by `TestNetworkForward_PrecompileStateGap_FailsOverToUpstreamWithState` and `TestHttpServer_PrecompileStateGap_AllUpstreamsLackStateSurfacesNodeMessage`. +26. **`gas required exceeds allowance` is final, not a server fault.** go-ethereum's gas estimator (`eth/gasestimator`) answers `eth_estimateGas` with `-32000 "gas required exceeds allowance (N)"` when the call still runs out of gas at its cap N, most often a sender whose balance buys only N gas. Arbitrum nitro nodes return it (nitro carries the estimator in its go-ethereum fork); other go-ethereum-derived clients may too. Rule 14 maps it to `ErrEndpointExecutionException`, so one upstream answers and the client gets the node's message with code `-32003` (the code rule 14 gives "out of gas" too). Before, it fell through to rule 20: the estimate was sent to every upstream and the client got `ErrUpstreamsExhausted`. A node's own RPC gas cap setting can also produce it; like "out of gas", it is not retried for that reason. Confirmed by `TestExtractJsonRpcError_GasRequiredExceedsAllowance`. ### Observability diff --git a/docs/pages/reference/evm/block-tracking.mdx b/docs/pages/reference/evm/block-tracking.mdx index 93e1c3174..0710b6331 100644 --- a/docs/pages/reference/evm/block-tracking.mdx +++ b/docs/pages/reference/evm/block-tracking.mdx @@ -9,19 +9,20 @@ import { LLMsTxtLink, AISection, ConfigTabs, SourceLink, PromptExample } from ". # Block tracking & served tip -Every upstream sees the chain at a slightly different block. Without coordination, routing a request to the one node that is a block behind causes silent "block not found" failures. eRPC eliminates this by polling every upstream continuously, computing a cluster-consensus **served tip** — the highest block that all eligible nodes actually have — and routing only within that confirmed window. +Every upstream sees the chain at a slightly different block. Without coordination, routing a request to the one node that is a block behind causes silent "block not found" failures. eRPC polls every upstream continuously and advertises one **served tip** per axis (`latest`, `finalized`): the highest block at least `servedTip.quorum` (default 2) eligible upstreams already have. **What you get** - Chain head and finality tracked per upstream, refreshed automatically at the right cadence for each chain's block time -- Consensus-cluster served tip that only advances when the majority of upstreams agree — no phantom future blocks, no stale tips +- One rule for every config: a block is served only once a quorum of upstreams has it — a lone far-ahead or wrong-chain node never defines `latest` +- Stalled heads benched automatically, so one frozen node cannot hold `latest` back - Optional per-group guarantee so the advertised tip stays servable by the upstreams your consensus policy requires - Per-upstream lag gauges that feed selection policies so slow nodes are deprioritized before requests reach them - Redis-backed counters share heads across replicas; each replica samples latest-block timestamps on its own debounce schedule. ## Quick taste -Illustrative, not a tuned production config — enable cluster-min served tip for latest and finalized: +Illustrative, not a tuned production config — require a strict majority of upstreams instead of the default 2: @@ -84,7 +85,7 @@ https://docs.erpc.cloud/reference/evm/block-tracking.llms.txt`} prompt={`My eRPC pool mixes trace-capable archive nodes with cheaper non-archive upstreams. Without a floor, the served tip can advance to a block that only the non-archive nodes have, causing trace_block requests to hit "block not found". Configure guaranteedMethods -in my eRPC config and explain how partition limits interact with the method floor. Reference: +in my eRPC config and explain how the method floor interacts with the quorum. Reference: https://docs.erpc.cloud/reference/evm/block-tracking.llms.txt`} /> @@ -123,27 +124,79 @@ eRPC keeps the `safe` tag verbatim and routes standard block-tagged JSON-RPC req On a 12 s Ethereum Mainnet chain this yields ~8.4 s debounce — fresh enough to catch every block, conservative enough that concurrent pollers share one actual RPC call per window. In Redis-backed multi-pod deployments, the shared counter coalesces fetches across replicas too. +**Finalized debounce.** The finalized fetch widens that debounce to `(latest − finalized) × EMA block time × finalizedDebounceDepthMultiplier` (default 0.05) when this is longer, using the upstream's own latest and finalized blocks. Finalized advances once per finality window, so polling it at the latest-block cadence mostly returns the number already held. With the default the finalized block trails by at most about 5% of the window: ~43 s on Ethereum Mainnet (~72 blocks × 12 s), ~45 s on an Arbitrum-style L2 with a 15 min window. An upstream behind the network's corroborated finalized block (second-highest across upstreams) keeps the regular debounce until it catches up: once a peer sees the next finalized block, the others sample it at the regular cadence. This also covers vendors whose load balancer mixes stale and fresh backends. Chains with near-instant finality (depth of a few blocks) keep the regular debounce. Unknown finalized, latest at or below finalized, or a cold EMA also keep it. Source: + **Dynamic block-time EMA.** The EMA (`alpha = 0.1`, ~19-sample effective window) lives in `health.Tracker` and updates whenever the corroborated network-level latest block advances, using the on-chain timestamp of the upstream report that holds that head (each upstream's latest timestamp is stored beside its head, so the sample comes from whichever poller reported the block). All head publication for a network is serialized under one per-network lock, so a slow re-derivation can never overwrite a newer one. On fast chains where consecutive blocks share the same integer second, block gaps accumulate and samples are normalized. The EMA is not published until ≥3 samples are collected (requiring 4 total advances, because the first stores the baseline without incrementing the sample counter). Values outside `[10 ms, 120 s]` are rejected and reset to the last valid value to prevent runaway spikes after a chain halt. Consumers receive 0 until warm-up completes. **Per-upstream block counters.** Both `latestBlockShared` and `finalizedBlockShared` are forward-only shared counters: `TryUpdate` rejects values ≤ the current value. A value that would regress more than 1024 blocks fires an `OnLargeRollback` callback and records `erpc_upstream_block_head_large_rollback`. -**Network head and `blockHeadLag` / `finalizationLag`.** The tracker's network-level latest and finalized heads are *corroborated*: the second-highest head across the reporting upstreams, or the only head while a single upstream has reported (the same order statistic `PickServedTip` uses for its lag reference). One upstream can therefore never move the reference every other upstream is measured against — a lone far-ahead report (an endpoint answering for another chain, a corrupted response) is not corroborated, and one stale upstream cannot hold the head back either. With exactly two reporters the head is the lower one, so a lone high report and a lone stale report are indistinguishable there; per-upstream block-availability checks still gate block-specific requests. The reporting set is the upstream that polled plus every upstream in the network's request-metrics index (an upstream that has never served a request is not in it). `blockHeadLag = max(0, network_latest − upstream_latest)`; `finalizationLag = max(0, network_finalized − upstream_finalized)`. Both are recomputed on every `SetLatestBlockNumber` / `SetFinalizedBlockNumber` call and exposed as Prometheus gauges; the raw per-upstream `erpc_upstream_latest_block_number` still shows an outlier's own report. These drive selection-policy predicates such as `blockNumberLagAbove`. +**Network head and `blockHeadLag` / `finalizationLag`.** The tracker's network-level latest and finalized heads are *corroborated*: the second-highest head across the non-benched reporting upstreams, or the only head while a single one has reported (the same order statistic as the served tip's default quorum). When every reporter is benched it falls back to all of them. One upstream can therefore never move the reference every other upstream is measured against — a lone far-ahead report (an endpoint answering for another chain, a corrupted response) is not corroborated, and a benched (stalled) upstream cannot hold the head back. With exactly two reporters the head is the lower one, so a lone high report and a lone stale report are indistinguishable there; per-upstream block-availability checks still gate block-specific requests. The reporting set is every upstream that has reported a latest or finalized head to the tracker. `blockHeadLag = max(0, network_latest − presumed_latest)`; `finalizationLag = max(0, network_finalized − presumed_finalized)`. An upstream's presumed head is its last reported head plus what the network head made since that report, counted for at most one poll period (`max(statePollerInterval, statePollerDebounce or the block time)`): within the period its lag stays what it was at the report, because a stored head ages between polls by design. Without this, an upstream polled every 10s on a 0.4s chain read up to 25 blocks behind late in each poll window, and a selection policy ticking at the same period read the same phase on every tick, excluding a healthy upstream for good on some processes. Past the period the credit freezes, so a missing report counts as lag again; a benched head gets no credit. Both are recomputed on every `SetLatestBlockNumber` / `SetFinalizedBlockNumber` call and exposed as Prometheus gauges; the raw per-upstream `erpc_upstream_latest_block_number` still shows an outlier's own report. These drive selection-policy predicates such as `blockNumberLagAbove`. Source: + +#### Served tip + +Both axes, every config. `EvmHighestLatestBlockNumber` / `EvmHighestFinalizedBlockNumber` run one pipeline per evaluation (`servedTip`): + +1. **Candidates** (`EvmRoutableUpstreams`): the selection policy's eligible upstreams in routing order (all registered upstreams until the policy's first decision), narrowed to the request's `use-upstream` selector when present. +2. **Voter rank** (`tipVoterRank`): each candidate with a head on the axis ranks live 0, benched 1, static cap 2, benched static cap 3. A static cap is a `blockAvailability.upper.exactBlock` the chain has passed: a serving range, not a head observation (`latestBlockMinus` upstreams are live). A candidate that is syncing per `eth_syncing`, has no state poller or has no head yet does not vote. +3. **Ballot** (`evmTipBallot`): only the heads of the best voter rank present vote. A benched head votes only when no non-benched live head exists; a static cap only when no live head exists. +4. **Pick** (`PickServedTip`): the highest block at least `servedTip.quorum` voters already have — the q-th highest head, or the lowest when fewer than q vote. +5. The [regression guard](#regression-guard), then the [WebSocket delivered-head floor](#websocket-delivered-head-floor) (network-wide `latest` only), then the [guarantee floors](#guarantee-subsets). + +The result is the value served, recorded on the request's [lane](#selector-scoped-served-tip) and [exported as gauges](/reference/metrics#area-3-block-number-and-tip-tracking). The network-wide lane is also evaluated every second, both axes, so its gauges, advance age and guard memory stay current on a network with little traffic; use-upstream lanes are evaluated only by their requests. Nothing is persisted or predicted. Source: , , , + +**Quorum.** `servedTip.quorum` is an integer ≥ 1 or `"majority"` (`floor(N/2)+1` of N voters), capped at N ([config](/config/projects/networks#config-schema)). Which head is served (heads sorted descending): + +| Voters N | `quorum: 2` (default) | `quorum: majority` | `quorum: 1` | +|---|---|---|---| +| 1 | 1st | 1st | 1st | +| 2 | 2nd | 2nd | 1st | +| 3 | 2nd | 2nd | 1st | +| 4 | 2nd | 3rd | 1st | +| 5 | 2nd | 3rd | 1st | + +`quorum: 1` serves the highest head, so a single upstream defines `latest`. A higher quorum trades freshness for servability: more upstreams already have the advertised block. Source: + +#### Benching + +The health tracker re-evaluates every reporter on each latest-head report (`evaluateHeadBench`). An upstream is *behind* while a non-benched peer proves the chain went on without it (`headIsBehind`): + +- **By height:** a peer that moved after the upstream last moved is higher by no more than a plausible lead: 1024 blocks plus twice the blocks the chain produced since the upstream's head was current (by its last change and its head timestamp). Chain progress uses the network block time, else the peer's own, so a process that starts mid-stall still sees the stall. One such peer is enough. Each move of the upstream resets this rule. +- **By staleness:** a peer that moved within its own or the upstream's stall window is higher within the plausible lead, and its newest block (never counted as newer than now) is at least the stall window younger than the upstream's head block. This holds while the upstream moves, so it catches a node replaying far behind the tip: after a restore, a restart or a deep rollback. One such peer is enough. Only the upstream's own head timestamp proves it stale: a head stored without one (a peer pod's poll through shared state) keeps the last verdict only while a live peer stays a stall window of blocks ahead. A node at the tip has a head block as young as any honest peer's, so a node that trails by less than the stall window of chain time is never benched. +- **By block age:** a peer that moved after the upstream last moved has a newest block at least the stall window younger than the upstream's head block, whatever the heights. This catches a head frozen above a chain that restarted lower (a dead fork), a static far-ahead head, and an endpoint that serves another, slower chain under the same chain id. It takes two such peers while two exist. + +Behind without a break for its stall window (`headStallAfter`), `max(30s, 3 × max(statePollerInterval, statePollerDebounce or the block time))`, it is benched: its voter rank drops to benched and it leaves the lag reference. Its `blockHeadLag` keeps growing so `blockNumberLagAbove` routes around it: while its head has not moved since the bench (`movedSinceBench`), the head counts as no higher than the network head at the bench (a head frozen above a dead fork lags by what the network made since, even after the network passes its height); once it moves again, it lags by its real distance. It rejoins once it has moved since being benched (a move in the evaluation that benched it does not count), no rule above still proves it behind, and its head reached the corroborated non-benched head (second-highest, or the only one); or when no non-benched peer has a head. A voter at or above that head never lowers `latest`, and a lone far-ahead head cannot keep it out. A bench or rejoin also re-derives the finalized lag reference. Stall windows run on the monotonic clock, so a wall-clock step benches nobody. Each bench increments `erpc_upstream_head_benched_total` and logs a WARN. Bench state is per process. Source: -**Served tip: default mode vs. majority mode.** By default, `EvmHighestLatestBlockNumber` returns the corroborated head across all non-syncing, policy-eligible upstreams: the second-highest effective head, or the only one (`PickServedTip(...).Freshest`, the same statistic the tracker's `blockHeadLag` uses). One upstream can never define it. It can still advertise a block only two upstreams have seen. +- **Nobody is benched** for a chain halt itself (no peer moves; a node already a stall window behind when the chain halted is benched like any trailing node), by a moving wrong-chain head above (its lead is implausible), or by one wrong-chain peer whose blocks are younger (the block-age rule needs two). +- **Boundary:** a node that trails by less than the stall window of chain time (for example, 1-3 blocks on a 2s chain) is never benched. It votes with its lower head, which holds the pick back only when it is needed for the quorum (N=2 with the default quorum). `quorum: 1` serves the highest head instead. +- **Boundary:** eRPC compares heights and block times, never block hashes, so it cannot tell a live fork from the canonical chain. During a halt, a node that moves within the plausible lead benches the halted ones after their stall window, and then defines `latest` alone. A benched dead-fork node that starts producing blocks again rejoins once its head reaches the corroborated head. +- **Boundary:** the block-age rule cannot tell two heads frozen above one moving peer (a dead fork the minority left) from two halted honest nodes and one wrong-chain peer. It keeps the frozen pair, so a dead fork that most upstreams stayed on still defines `latest`. +- **Boundary:** two chains under one chain id (an endpoint pointed at the wrong chain) differ by far more than any plausible lead, so only block age tells them apart. Until the slower chain goes twice its stall window without a block, quorum 2 over the pair serves the lower head; then the slower chain is benched, right or wrong. A wrong chain that makes blocks more often than that is never benched. Exclude such an endpoint in config. +- **Boundary:** benching a voter that sat between the pick and a slower voter lowers `latest` to that slower voter's head, as any upstream leaving the ballot does. The regression guard holds only drops beyond `maxRegressionBlocks`. +- **Boundary:** a `latestBlockMinus: N` upstream rejoins on its raw head but votes N below it, so its rejoin can lower `latest` by up to N. +- **Boundary:** rejoin needs the benched head to reach the bar at one report. A node that always trails the second-highest head by a block or two rejoins late; until then it serves traffic while its lag stays under the policy threshold, but does not vote. +- **Boundary:** a bench changes the ballots at once, but routing leaves the benched upstream at the selection policy's next evaluation (`evalInterval`, 15s by default). Until then a request for the new `latest` can reach it; without a configured availability bound nothing gates it, so its missing-data error is retried on another upstream. -When `servedTip.enabledFor` includes `"latest"` or `"finalized"`, the pure function `ComputeServedTipCandidate` runs instead: -1. Drop inputs with `BlockNumber ≤ 0`. -2. **Velocity gate** (requires a prior served anchor AND a known block time): compute `expectedMax = lastServedBlock + ceil(elapsedBlocks × VelocitySlack) + VelocityBufferBlocks` (defaults: `VelocitySlack = 2.0`, `VelocityBufferBlocks = 5`); discard any upstream reporting a block above `expectedMax`. `MaxEligible` is set to the maximum of the remaining (post-gate) inputs, so velocity-dropped upstreams never inflate the lag gauge. -3. Sort remaining inputs ascending. -4. **Greedy clustering**: split into clusters when adjacent inputs differ by more than `clusterDelta` blocks (auto-derived from block time; 2 for Ethereum, up to 10 for sub-100 ms chains). -5. Pick the **dominant cluster** (largest by member count; ties go to the chain-forward cluster). -6. Return the **minimum** of the dominant cluster as the candidate. +#### Regression guard -The result is monotonically clamped via a cross-pod shared counter so the served tip never regresses, even across pods. +When a pick falls more than `maxRegressionBlocks` (0 = 1024) below the corroborated evidence — the second-highest live head, or the last live-corroborated pick — the lane serves its last corroborated pick instead, never above the highest live head (`guardServedTipRegression`). It holds for at most 1 minute, then fails open and serves the pick as computed. In-process only; selectors without a lane get no guard; `-1` disables it. Each transition increments `erpc_network_served_tip_regression_total`. Source: -**Guarantee subsets (`guaranteedMethods`, `guaranteedFor`).** After the pick, the advertised tip is clamped down to the lowest majority of every configured guarantee subset, so it is never a block a required subset cannot serve. `guaranteedMethods` names a subset by method support; `guaranteedFor` names one by upstream id/tag selector — the same vocabulary as `use-upstream` and `consensus.requiredParticipants`. Both run through the same ballot and the same min-across-subsets fold in `guaranteedFloor`; only the membership predicate differs. Each subset is clamped to its OWN MAJORITY, never its minimum, so a single frozen member is outvoted inside its group. An empty or unmatched subset constrains nothing (the tip is never pinned to 0), and the clamp can only lower the tip — a group ahead of the network majority never raises it. Source: [`erpc/networks.go:L1505-L1600`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L1505-L1600) +#### WebSocket delivered-head floor -**Selector-scoped served tip.** When a `use-upstream` selector is present, `tipCandidateUpstreams` narrows to the matching upstreams. A "simple group selector" is a single glob pattern (optionally negated with `!` at position 0), containing no whitespace or `()|&,` characters, of length ≤128, that matches 2–(N−1) upstreams. For such selectors, a per-group `servedTipPartition` is lazily materialized: `partitionKeyFor` resolves the matched upstream ID set, sorts it, SHA-256 hashes the sorted `\0`-joined IDs, truncates to 8 bytes, and encodes as hex (key = `"grp:" + hex`). Equivalent selectors (same matched set) share one partition. Up to 16 partitions are allowed per network; beyond this, extra selectors fall back to stateless cluster-min. Boolean-expression selectors (containing `|`, `&`, `(`, `)`) always use the stateless path. Source: [`erpc/networks.go:L466-L506`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L466-L506) +Network-wide `latest` (no `use-upstream`) is lifted to the highest head already delivered to a `newHeads` subscriber by this instance, or by any instance through the shared counter `network///latestBlock` (`applyDeliveredHeadFloor`). The floor applies only within `maxRetryableBlockDistance` of the ballot's highest head (or when no head is known). The guarantee floors run after it, so it never lifts `latest` past one. Source: + +#### eth_blockNumber + +With `enforceHighestBlock` on (the default) and the served head known, eRPC answers `eth_blockNumber` from the served head without dispatching, scoped to the request's `use-upstream` selector. Each answer triggers `EvmRefreshLatestBlock`: in routing order, quorum-many voters of the best voter rank present — the voters the ballot counts — refresh their pollers in the background, each only when its stored head is older than its poll debounce, one refresh in flight per poller. The answer is the head as of the last refresh, so a client that alone polls every N seconds reads a head up to N seconds old (never older than `statePollerInterval`, and fresher when other traffic refreshes it); a request that follows it sees the refreshed head. With the head unknown (cold pollers) the request is forwarded. Every response that reaches the post-forward hook — an upstream answer or a cache hit — is pinned exactly to the served head when it is known: higher values are lowered, lower values are raised (an upstream answer below it increments `erpc_upstream_stale_latest_block_total`). `enforce-highest-block=false` forwards and returns the upstream's answer unchanged. Source: , + +#### Guarantee subsets + +`guaranteedMethods` and `guaranteedFor` clamp the advertised tip down to the lowest majority pick of every configured guarantee subset, so it is never a block a required subset cannot serve. `guaranteedMethods` names a subset by method support; `guaranteedFor` names one by upstream id/tag selector — the same vocabulary as `use-upstream` and `consensus.requiredParticipants`. Each subset runs the same ballot (`guaranteedSubsetTip` → `evmTipBallot`, same voter ranks) at `quorum: majority` whatever the network quorum, folded to the minimum in `guaranteedFloor`; only the membership predicate differs. A majority, never a minimum, so a single frozen member is outvoted inside its group. A subset with no voters constrains nothing (the tip is never pinned to 0), and the clamp can only lower the tip — a group ahead of the network pick never raises it. A subset whose ballot holds only static caps floors the tip only when no REGISTERED member is live (`anyRegisteredLiveHead`): live members that are merely absent from the eligible set (lag-excluded, cordoned, mid-restart) must not hand back a frozen cap. Source: + +#### Selector-scoped served tip + +With a `use-upstream` selector, `EvmRoutableUpstreams` narrows the candidates to the matching upstreams; the pick is the same rule over that subset. `servedTipLaneFor` returns the request's lane: the network-wide lane (`lane="all"`) without a selector, a group's own lane for a simple group selector, else none. A lane holds no pick state: only its gauge label, advance-age anchor and regression-guard memory. A selector without a lane gets the same pick with no guard and no gauges; only the network-wide lane gets the WebSocket floor. Which selectors form a lane, naming and the 16-lane cap: [Lanes](/reference/lanes). Source: + +#### Syncing, finality and earliest block **Syncing state machine.** `eth_syncing` is called with a 5 s sub-context on each tick. The upstream is declared fully synced only after 4 consecutive "not syncing" responses (`synced counter ≥ 4`); any syncing response resets the counter to 1. The non-standard `{Ok: bool}` response shape is supported: `Ok=false` is treated as still-syncing. After 10 consecutive failures with no prior success, `skipSyncingCheck` is set permanently and the state is fixed at `Unknown` for the process lifetime. If `upstreams[*].evm.skipSyncingCheck: true` is set in config, `eth_syncing` is never called at all and the state is immediately fixed at `NotSyncing` — use this for nodes that always return a syncing object regardless of actual sync status. @@ -174,15 +227,13 @@ The result is monotonically clamped via a cross-pod shared counter so the served | `networks[*].evm.fallbackStatePollerDebounce` | Duration | `5s` | Debounce used when EMA is not yet available (step 3 of `resolveDebounce`). Setting to `0` causes the 1 s hard floor to apply during EMA warm-up. Source: | | `networks[*].evm.fallbackFinalityDepth` | int64 | `1024` | Blocks subtracted from latest to infer finalized when the upstream does not support `eth_getBlockByNumber("finalized")`. Applied only when `finalizedBlock == 0` and `latestBlock > 0`. Source: | | `networks[*].evm.dynamicBlockTimeDebounceMultiplier` | \*float64 | `0.7` | Scales the EMA block time to compute the debounce. **`0` is silently treated as unset** — the default 0.7 is used. Valid useful range: `0.1`–`1.0`; values above `1.0` make the debounce longer than one block time, causing stale data between polls. To eliminate the EMA-derived debounce, use `statePollerDebounce` on the upstream instead. Source: | -| `networks[*].evm.servedTip` | \*EvmServedTipConfig | `nil` (default mode) | When nil, `EvmHighestLatestBlockNumber` returns the corroborated head (second-highest). Set `enabledFor` to enable majority mode. Source: | -| `networks[*].evm.servedTip.enabledFor` | []string | `[]` (default mode for all tags) | Tags for which majority mode is active. Valid: `"latest"`, `"finalized"`, `"safe"` (`"safe"` is an alias for `"finalized"`). Source: | -| `networks[*].evm.servedTip.clusterDelta` | int64 | `0` (auto-derive) | Max block gap between adjacent sorted upstream tips that still groups them into one cluster. Auto-derive: `clamp(ceil(2.0 / blockTimeSec), 2, 10)`. Examples: Ethereum (12 s) → 2; Polygon (2 s) → 1, clamped to 2; Arbitrum (0.25 s) → 8; sub-100 ms → 10 (ceiling). When block time is unknown (EMA not warmed up) the fallback is 2. Source: | -| `networks[*].evm.servedTip.guaranteedMethods` | []string | `[]` | Glob patterns (e.g. `"trace_*"`). For each pattern the served tip is clamped down to the cluster-min of supporting upstreams only, ensuring `"latest"` is always servable for these methods. Source: | -| `networks[*].evm.servedTip.guaranteedFor` | []string | `[]` | Upstream selectors (id or tag glob, e.g. `"type:internal"`, `"int*"`) whose group must be able to serve the advertised tip. For each selector the tip is clamped down to that group's own MAJORITY — not its minimum, so one frozen member cannot pin the network. Several selectors fold to the lowest. An unmatched selector constrains nothing. Source: | +| `networks[*].evm.finalizedDebounceDepthMultiplier` | \*float64 | `0.05` | Finalized polls wait until the upstream's finalized block is older than `(latest − finalized) × EMA block time × multiplier`, when that is longer than the regular debounce. `0` disables it: finalized uses the regular debounce, as before. Also throttles the forced finalized re-check of `blockAvailability` with finalized confidence. Source: | + +`networks[*].evm.servedTip.*` keys (`quorum`, `maxRegressionBlocks`, `guaranteedMethods`, `guaranteedFor`) and the removed keys with their migration: [Networks config](/config/projects/networks#config-schema). ### Worked examples -**1. Enable cluster-min served tip for a multi-provider Mainnet setup.** Cluster mode ensures every provider in the dominant group has the advertised block before eRPC routes to it — eliminates "block not found" on eth_getBlockByNumber after a new block: +**1. Majority quorum for a five-provider Mainnet pool.** The default quorum serves the 2nd-highest head; `majority` serves the 3rd of 5, so three providers already have every advertised block and fewer interpolated requests land on a node without it: -**2. Faster polling for a sub-second chain (Arbitrum).** The EMA auto-tunes the debounce to ~70% of block time, but `fallbackStatePollerDebounce` should be shortened so warm-up doesn't bottleneck fresh starts. Reduce polling interval to keep lag tight: +**2. Faster polling for a sub-second chain (Arbitrum).** The EMA auto-tunes the debounce to ~70% of block time, but `fallbackStatePollerDebounce` should be shortened so warm-up doesn't bottleneck fresh starts: @@ -231,14 +279,12 @@ The result is monotonically clamped via a cross-pod shared counter so the served evm: chainId: 1 servedTip: - enabledFor: ["latest"] guaranteedMethods: ["trace_*"]`} ts={`networks: [{ architecture: "evm", evm: { chainId: 1, servedTip: { - enabledFor: ["latest"], guaranteedMethods: ["trace_*"], }, }, @@ -249,14 +295,13 @@ The result is monotonically clamped via a cross-pod shared counter so the served ] - The `SuggestLatestBlock` fast path (called when a response is observed during request forwarding) is non-blocking and uses no debounce. `SuggestFinalizedBlock` uses `TryLock` and is async — suggestions may be dropped under high concurrency. ### Best practices -- Enable cluster mode (`servedTip.enabledFor: ["latest", "finalized"]`) in any production deployment with multiple upstreams — it eliminates the class of "block not found" failures caused by routing to a lagging node. -- Lower `fallbackStatePollerDebounce` to 200–500 ms on sub-second chains (Arbitrum, Base, Optimism) so warm-up doesn't stall on a 5 s default. - Use `guaranteedMethods: ["trace_*"]` when mixing archive and non-archive upstreams — prevents the served tip from advancing past what trace-capable nodes have. +- Lower `fallbackStatePollerDebounce` to 200–500 ms on sub-second chains (Arbitrum, Base, Optimism) so warm-up doesn't stall on a 5 s default. - Avoid setting `dynamicBlockTimeDebounceMultiplier: 0`; it is silently ignored and the default 0.7 is used. To impose a fixed debounce, use `statePollerDebounce` on the upstream instead. -- Monitor `erpc_upstream_block_head_lag` per upstream. A node consistently lagging more than `clusterDelta` blocks will be excluded from the dominant cluster and effectively deprioritized. -- Alert on `erpc_network_served_tip_lag_blocks` rising with a flat `erpc_network_served_tip_block_number` — this is the diagnostic signature of a monotonic clamp hold-down after fleet regression (all nodes temporarily reported lower blocks but the counter held the old high-water mark). -- Don't alert on `erpc_network_served_tip_lag_blocks == 0` as a health signal in cluster mode — during a total outage all inputs are zero and `MaxEligible - served < 0` clamps to 0, masking the problem. Cross-alert with `erpc_upstream_latest_block_number` being flat instead. +- Monitor `erpc_upstream_block_head_lag` per upstream, and alert on `erpc_network_served_tip_advance_age_seconds` (stuck tip), `erpc_network_served_tip_regression_total` and `erpc_upstream_head_benched_total` ([metrics reference](/reference/metrics#area-3-block-number-and-tip-tracking)). ### Edge cases & gotchas -1. **`statePollerInterval: 0` disables the poller silently.** All block numbers return 0, syncing state is `Unknown`, `IsBlockFinalized` returns `ErrFinalizedBlockUnavailable` immediately, and block availability checks fail-open. The upstream is not excluded from selection but contributes `BlockNumber=0`, which cluster mode drops at step 1. -2. **Velocity gate is inactive on cold start.** If `lastServedBlock == 0` (cold start) or `BlockTimeSeconds == 0` (EMA not warmed up), all candidates pass to clustering. A misbehaving upstream reporting a far-future block during cold start can become the sole cluster and win. -3. **Monotonic clamp survives fleet regression.** Per-upstream shared counters are forward-only. A reorg or mass node reboot causing all upstreams to report lower blocks is silently ignored — the counter holds the previous high-water mark. `erpc_network_served_tip_lag_blocks` grows unboundedly; a flat `served_tip_block_number` alongside rising lag is the diagnostic signature. +1. **`statePollerInterval: 0` disables the poller silently.** All block numbers return 0, syncing state is `Unknown`, `IsBlockFinalized` returns `ErrFinalizedBlockUnavailable` immediately, and block availability checks fail-open. The upstream is not excluded from selection but its head is 0, so it never votes. +2. **Two upstreams with the default quorum serve the lower head.** One stalled node holds `latest` until it is benched (its stall window: 90s at the default 30s `statePollerInterval`); then the other votes alone. A node that is catching up slowly is not benched and holds `latest` at its head until `eth_syncing` reports it syncing. +3. **A rogue head needs `quorum − 1` accomplices.** One far-future or wrong-chain head never moves the pick at quorum ≥ 2. 4. **`dynamicBlockTimeDebounceMultiplier: 0` is silently treated as unset.** The guard `> 0` causes the value to fall through to the default 0.7. -5. **`"safe"` is an alias for `"finalized"` in `enabledFor`.** Listing `"safe"` enables cluster mode on the finalized axis only. -6. **Selector-scoped tip is stateless for boolean-expression selectors.** Selectors containing `|`, `&`, `(`, `)`, or multiple `!` characters fail `isSimpleGroupSelector` and always use stateless cluster-min (no monotonic guarantee, no metrics). -7. **Maximum 16 selector partitions per network.** Beyond the cap, additional selectors fall back to stateless cluster-min. Equivalent selectors (same matched upstream set) dedup to one partition. -8. **`clusterDelta` auto-derive uses the EMA block time.** During warm-up (fewer than 3 EMA samples), `blockTimeSec = 0` and `clusterDelta` defaults to 2. A 2-block spread between an early and late upstream will erroneously split into two clusters until the EMA warms up (typically within the first 3 produced blocks). -9. **`eventLogs` probe requires ≥1 log entry.** A block with no events returns an empty array, treated as "not available" — the binary search may incorrectly mark data-sparse blocks as unavailable. -10. **`updateRate` goroutine is NOT restarted on config hot-reload.** Only a process restart picks up a changed `updateRate` for `earliestBlockPlus` bounds. Setting `updateRate` alongside `latestBlockMinus` produces a validation warning (not an error); the value is silently ignored at runtime. -11. **`ErrFinalizedBlockUnavailable` is retryable.** It carries no explicit opt-out flag, so `IsRetryableTowardNetwork` returns true. On the wire it maps to HTTP 200 with JSON-RPC error code `-32603`. -12. **`erpc_network_served_tip_lag_blocks` reports 0 during a total outage.** When all upstreams return 0 or blocks below the clamp, `MaxEligible = 0` and `lag = MaxEligible − served < 0` is clamped to 0. A lag of 0 with a flat `served_tip_block_number` indicates total outage, not healthy operation. -13. **`OnValue` callback passes `blockTimestamp=0` for cross-pod replications.** Only the replica that actually fetches feeds the EMA with a real timestamp; replicas receiving shared-counter updates use `blockTimestamp=0`, so only one pod per network feeds the EMA per debounce window. -14. **`ErrFinalizedBlockUnavailable` never fires if only `latestBlock == 0`.** The error requires BOTH counters to be 0. If the upstream supports `eth_getBlockByNumber("latest")` but not `"finalized"`, `IsBlockFinalized` falls back to `latestBlock − FallbackFinalityDepth` instead of returning the error. -15. **`SuggestFinalizedBlock` may drop updates under load.** It uses `TryLock`; if an update is already in progress the suggestion is discarded. Finalized block updates may lag under high request volume. -16. **`SuggestLatestBlock` monotonic within a process but not across pod restarts.** It calls `TryUpdate` which the shared counter rejects if the value is ≤ current. Across pod restarts the shared counter (Redis) retains the last value, so a restarted upstream reporting a lower block number after a reorg is silently ignored for up to 1024 blocks of rollback. -17. **`eth_syncing` result `{Ok: false}` is interpreted as SYNCING.** The non-standard `{Ok: bool}` response shape is handled by `!Ok`: `Ok=false` means not-ok = still syncing. Source: [`architecture/evm/evm_state_poller.go:L1087-L1100`](https://github.com/erpc/erpc/blob/main/architecture/evm/evm_state_poller.go#L1087-L1100) -18. **`erpc_network_served_tip_lag_blocks` uses `MaxEligible` (post-velocity-gate), not `MaxObserved`.** A far-future tip from a misbehaving upstream is velocity-dropped and does not inflate the lag gauge. The gauge accurately reflects deliberate lag vs. the highest plausible eligible upstream tip. -19. **`guaranteedFor` on a ONE-member group lets that upstream set the tip.** The clamp is the group's majority, and a group of one has no majority to outvote a stall — a single frozen member holds the network tip at its head. Naming exactly one upstream is declaring it mandatory; name ≥3 for a group that tolerates one stall. Same property `guaranteedMethods` already has for a method only one upstream supports. -20. **`guaranteedFor` constrains nothing when its group is absent from the eligible set.** If every matching upstream is lag-excluded, cordoned or mid-restart, the group ballot has no inputs and the clamp is skipped rather than pinning the tip to 0. A group whose members are all static serving-range caps floors the tip only when no REGISTERED member is live — the same transient-vs-permanent rule `guaranteedMethods` applies, so an eligibility collapse cannot hand back a frozen cap. Source: [`erpc/networks.go:L1654-L1675`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L1654-L1675) -21. **`guaranteedFor` is subject to `enabledFor`.** The clamp lives in the served-tip pipeline, so it is a no-op on any axis not listed in `enabledFor` — setting `guaranteedFor` without `enabledFor` changes nothing. -22. **Finalized block polling is permanently skipped on unsupported chains.** After 10 consecutive `eth_getBlockByNumber("finalized")` failures without any prior success, `skipFinalizedCheck = true`. `IsBlockFinalized` then falls back exclusively to `latestBlock − FallbackFinalityDepth`. On PoW chains or chains without EIP-4895 finality, this is the only reachable path. +5. **`eventLogs` probe requires ≥1 log entry.** A block with no events returns an empty array, treated as "not available" — the binary search may incorrectly mark data-sparse blocks as unavailable. +6. **`updateRate` goroutine is NOT restarted on config hot-reload.** Only a process restart picks up a changed `updateRate` for `earliestBlockPlus` bounds. Setting `updateRate` alongside `latestBlockMinus` produces a validation warning (not an error); the value is silently ignored at runtime. +7. **`ErrFinalizedBlockUnavailable` is retryable.** It carries no explicit opt-out flag, so `IsRetryableTowardNetwork` returns true. On the wire it maps to HTTP 200 with JSON-RPC error code `-32603`. +8. **`OnValue` callback passes `blockTimestamp=0` for cross-pod replications.** Only the replica that actually fetches feeds the EMA with a real timestamp; replicas receiving shared-counter updates use `blockTimestamp=0`, so only one pod per network feeds the EMA per debounce window. +9. **`ErrFinalizedBlockUnavailable` never fires if only `latestBlock == 0`.** The error requires BOTH counters to be 0. If the upstream supports `eth_getBlockByNumber("latest")` but not `"finalized"`, `IsBlockFinalized` falls back to `latestBlock − FallbackFinalityDepth` instead of returning the error. +10. **`SuggestFinalizedBlock` may drop updates under load.** It uses `TryLock`; if an update is already in progress the suggestion is discarded. Finalized block updates may lag under high request volume. +11. **`SuggestLatestBlock` monotonic within a process but not across pod restarts.** It calls `TryUpdate` which the shared counter rejects if the value is ≤ current. Across pod restarts the shared counter (Redis) retains the last value, so a restarted upstream reporting a lower block number after a reorg is silently ignored for up to 1024 blocks of rollback. +12. **`eth_syncing` result `{Ok: false}` is interpreted as SYNCING.** The non-standard `{Ok: bool}` response shape is handled by `!Ok`: `Ok=false` means not-ok = still syncing. Source: [`architecture/evm/evm_state_poller.go:L1087-L1100`](https://github.com/erpc/erpc/blob/main/architecture/evm/evm_state_poller.go#L1087-L1100) +13. **`guaranteedFor` on a ONE-member group lets that upstream set the tip.** The clamp is the group's majority, and a group of one has no majority to outvote a stall — a single frozen member holds the network tip at its head. Naming exactly one upstream is declaring it mandatory; name ≥3 for a group that tolerates one stall. Same property `guaranteedMethods` already has for a method only one upstream supports. +14. **Finalized block polling is permanently skipped on unsupported chains.** After 10 consecutive `eth_getBlockByNumber("finalized")` failures without any prior success, `skipFinalizedCheck = true`. `IsBlockFinalized` then falls back exclusively to `latestBlock − FallbackFinalityDepth`. On PoW chains or chains without EIP-4895 finality, this is the only reachable path. +15. **Finalized trails by up to `finalizedDebounceDepthMultiplier` of the finality window.** Response finality uses the serving upstream's own finalized block and cache hits the lowest across upstreams, so requests for blocks finalized within that span stay classified unfinalized (cache TTL, finality-scoped policies) until the first upstream polls again; the rest then catch up at the regular debounce. A `finalized` response or `SuggestFinalizedBlock` inside the window still advances it at once. ### Observability | Metric | Type | Labels | When it fires | |---|---|---|---| -| `erpc_upstream_latest_block_number` | gauge | project, vendor, network, upstream | Updated on every `SetLatestBlockNumber` call; `upstream="*"` = network-wide max | +| `erpc_upstream_latest_block_number` | gauge | project, vendor, network, upstream | Updated on every `SetLatestBlockNumber` call; `upstream="*"` = corroborated network head (second-highest non-benched) | | `erpc_upstream_finalized_block_number` | gauge | project, vendor, network, upstream | Updated on every `SetFinalizedBlockNumber` call | -| `erpc_upstream_block_head_lag` | gauge | project, vendor, network, upstream | Recomputed on each `SetLatestBlockNumber`; value = `max_network_latest − upstream_latest` | -| `erpc_upstream_finalization_lag` | gauge | project, vendor, network, upstream | Recomputed on each `SetFinalizedBlockNumber`; value = `max_network_finalized − upstream_finalized` | +| `erpc_upstream_block_head_lag` | gauge | project, vendor, network, upstream | Recomputed on each `SetLatestBlockNumber`; value = `network_latest − presumed_latest` (corroborated network head; presumed head = last report plus the network head's progress since, for at most one poll period); a benched head that has not moved since its bench counts as no higher than the network head at the bench | +| `erpc_upstream_finalization_lag` | gauge | project, vendor, network, upstream | Recomputed on each `SetFinalizedBlockNumber`; value = `network_finalized − presumed_finalized` (corroborated network head, same presumed-head credit) | | `erpc_upstream_latest_block_polled_total` | counter | project, vendor, network, upstream | Incremented before each actual `eth_getBlockByNumber("latest")` call (debounce elapsed). Two paths increment this counter: (1) proactive timer-driven `Poll()` on the background goroutine, and (2) forced-fresh polls triggered by block-availability checks (`EvmAssertBlockAvailability` / `EvmIsBlockFinalized`) when a request arrives for a block above the current known latest. Both paths still respect the debounce window — if a pod already fetched within the window, `TryUpdateIfStale` returns the cached value without incrementing. | | `erpc_upstream_finalized_block_polled_total` | counter | project, vendor, network, upstream | Same dual-path semantics as `latest_block_polled_total` for the finalized axis | | `erpc_upstream_block_head_large_rollback` | gauge | project, vendor, network, upstream, finality | Fired when a new value would regress >1024 blocks. `finality` is the head axis (`latest` or `finalized`) — the two pollers would otherwise share one series | | `erpc_network_dynamic_block_time_milliseconds` | gauge | project, network | Current EMA block-time estimate in ms; absent until ≥3 samples | | `erpc_network_latest_block_timestamp_distance_seconds` | gauge | project, network, origin | `(now_ms − block.timestamp × 1000) / 1000`; `origin="evm_state_poller"` | -| `erpc_network_served_tip_block_number` | gauge | project, network, lane, axis | Post-clamp served tip; **cluster mode only** (`lane="all"` or named group) | -| `erpc_network_served_tip_lag_blocks` | gauge | project, network, lane, axis | `MaxEligible − served`; **cluster mode only**; grows during monotonic clamp hold-down | -| `erpc_network_served_tip_upstream_excluded_total` | counter | project, network, upstream, axis, reason | Upstream excluded from pick; `reason="velocity"` or `"outlier"`; **cluster mode only, network-wide pick only** | -**OTel trace spans** (under detailed tracing): `EvmStatePoller.PollLatestBlockNumber`, `EvmStatePoller.PollFinalizedBlockNumber`, `Network.EvmHighestLatestBlockNumber`, `Network.EvmHighestFinalizedBlockNumber`, `Evm.ExtractBlockReferenceFromRequest`, `Evm.ExtractBlockReferenceFromResponse`. The network span includes attributes: `served_tip.axis`, `served_tip.candidate`, `served_tip.max_observed`, `served_tip.cluster_count`, `served_tip.dominant_size`, `served_tip.outliers_count`, `served_tip.velocity_dropped`, `served_tip.lag_vs_max`. Source: +Served-tip and bench metrics (`erpc_network_served_tip_*`, `erpc_upstream_head_benched_total`): [metrics reference](/reference/metrics#area-3-block-number-and-tip-tracking). + +**OTel trace spans** (under detailed tracing): `EvmStatePoller.PollLatestBlockNumber`, `EvmStatePoller.PollFinalizedBlockNumber`, `Network.EvmHighestLatestBlockNumber`, `Network.EvmHighestFinalizedBlockNumber`, `Evm.ExtractBlockReferenceFromRequest`, `Evm.ExtractBlockReferenceFromResponse`. The network span includes attributes: `served_tip.axis`, `served_tip.tip`, `served_tip.freshest`, `served_tip.inputs`. Source: **Notable log messages:** @@ -360,23 +392,34 @@ The result is monotonically clamped via a cross-pod shared counter so the served | `WARN` | `"failed to poll evm state"` | `Poll()` returns a non-cancel error | | `INFO` | `"initial earliest block detection completed for this instance"` | Binary search for `earliestBlockPlus` bound completes | | `INFO` | `"started periodic scheduler for earliest block availability bound"` | `updateRate` configured and scheduler goroutine launched | +| `WARN` | `"benched upstream head: it stayed behind peers that kept advancing; ..."` | Upstream benched | +| `INFO` | `"upstream head rejoined: it caught up with its non-benched peers"` | Benched upstream rejoined | +| `ERROR` | `"served tip regressed far below the corroborated live head; serving the last corroborated tip"` | Regression guard starts holding | +| `ERROR` | `"served tip regression outlasted the guard window; failing open and serving the current pick"` | Hold exceeded 1 minute | +| `WARN` | `"served tip recovered; the regression guard is no longer holding"` | Guard back to healthy | ### Source code entry points - [`architecture/evm/evm_state_poller.go:L145-L204`](https://github.com/erpc/erpc/blob/main/architecture/evm/evm_state_poller.go#L145-L204) — `Bootstrap`, `Poll`, `PollLatestBlockNumber`, `PollFinalizedBlockNumber`; debounce resolution; syncing state machine -- [`architecture/evm/served_tip.go:L114-L246`](https://github.com/erpc/erpc/blob/main/architecture/evm/served_tip.go#L114-L246) — `ComputeServedTipCandidate`: velocity gate, greedy clustering, dominant cluster, `resolveAutoClusterDelta` -- [`erpc/networks.go:L466-L851`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L466-L851) — `clusteredServedTip`, `gatherEvmTipInputsForMethod`, `tipCandidateUpstreams`, `servedTipPartitionFor`, `partitionKeyFor`, `isSimpleGroupSelector`, `guaranteedMethodFloor`, `observeServedTipMetrics` -- [`health/tracker.go:L1306-L1463`](https://github.com/erpc/erpc/blob/main/health/tracker.go#L1306-L1463) — `SetLatestBlockNumber`, EMA update (`updateBlockTimeSample`), `GetNetworkBlockTime`, `BlockHeadLag` computation -- [`health/tracker.go:L1504-L1550`](https://github.com/erpc/erpc/blob/main/health/tracker.go#L1504-L1550) — `SetFinalizedBlockNumber`, `FinalizationLag` computation +- [`common/served_tip.go:L36-L157`](https://github.com/erpc/erpc/blob/main/common/served_tip.go#L36-L157) — `ServedTipQuorum`, `PickServedTip` +- [`erpc/networks.go:L430-L506`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L430-L506) — `tipVoterRank`, `evmTipBallot` +- [`erpc/networks.go:L564-L655`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L564-L655) — `EvmRoutableUpstreams`, `requestSelector`, `servedTipLaneFor` +- [`erpc/networks.go:L748-L795`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L748-L795) — `EvmHighestLatestBlockNumber`, `EvmRefreshLatestBlock` +- [`erpc/networks.go:L870-L1102`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L870-L1102) — `servedTip`, `guardServedTipRegression` +- [`erpc/networks.go:L1369-L1532`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L1369-L1532) — `EvmHighestFinalizedBlockNumber`, `applyDeliveredHeadFloor`, `guaranteedFloor`, `guaranteedSubsetTip`, `anyRegisteredLiveHead`, `observeServedTipMetrics` +- [`architecture/evm/eth_blockNumber.go`](https://github.com/erpc/erpc/blob/main/architecture/evm/eth_blockNumber.go) — `eth_blockNumber` pre-forward answer and post-forward pin +- [`health/tracker.go:L1428-L1664`](https://github.com/erpc/erpc/blob/main/health/tracker.go#L1428-L1664) — `corroboratedNetworkHead`, `headStallAfter`, `evaluateHeadBench`, `movedSinceBench`, `headIsBehind`, `benchPlausibleLead`, `IsHeadBenched` +- [`health/tracker.go:L1666-L1918`](https://github.com/erpc/erpc/blob/main/health/tracker.go#L1666-L1918) — `SetLatestBlockNumber`, EMA update (`updateBlockTimeSample`, `sampleBlockTime`), `BlockHeadLag` computation +- [`health/tracker.go:L1937-L2033`](https://github.com/erpc/erpc/blob/main/health/tracker.go#L1937-L2033) — `SetFinalizedBlockNumber`, `rederiveNetworkFinalized`, `FinalizationLag` computation - [`common/defaults.go:L1718-L1977`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L1718-L1977) — `DefaultEvmFinalityDepth=1024`, `DefaultEvmStatePollerDebounce=5s`, `DefaultDynamicBlockTimeDebounceMultiplier=0.7` -- [`architecture/evm/served_tip_test.go`](https://github.com/erpc/erpc/blob/main/architecture/evm/served_tip_test.go) — unit tests for clustering, velocity gate, `clusterDelta` auto-derive -- [`erpc/networks_served_tip_test.go`](https://github.com/erpc/erpc/blob/main/erpc/networks_served_tip_test.go) — integration tests: cluster-min, monotonic clamp, selector scoping, partition dedup, guaranteed method floor +- [`common/served_tip_test.go`](https://github.com/erpc/erpc/blob/main/common/served_tip_test.go) — unit tests for `PickServedTip` and quorum parsing +- [`erpc/networks_served_tip_test.go`](https://github.com/erpc/erpc/blob/main/erpc/networks_served_tip_test.go) — integration tests: quorum, voter ranks, benching, static caps, selector scoping, guarantee floors ### Related pages - [Selection policies](/config/projects/selection-policies) — uses `blockNumberLagAbove` and `blockHeadLag` produced by this subsystem to deprioritize lagging upstreams. - [Upstream config](/config/projects) — where `evm.statePollerInterval`, `evm.statePollerDebounce`, and `evm.blockAvailability` are set. -- [Network config](/config/projects) — where `evm.servedTip`, `evm.fallbackStatePollerDebounce`, and `evm.dynamicBlockTimeDebounceMultiplier` are set. +- [Network config](/config/projects/networks) — where `evm.servedTip`, `evm.fallbackStatePollerDebounce`, and `evm.dynamicBlockTimeDebounceMultiplier` are set. - [Cache](/config/database) — uses `ExtractBlockReferenceFromRequest` / `ExtractBlockReferenceFromResponse` to build cache keys from block numbers and tags. - [eth\_getLogs splitting](/reference/evm/getlogs-splitting) — calls `CheckBlockRangeAvailability` from this subsystem to gate range queries. diff --git a/docs/pages/reference/evm/method-handlers.mdx b/docs/pages/reference/evm/method-handlers.mdx index 790c32941..f84e5a794 100644 --- a/docs/pages/reference/evm/method-handlers.mdx +++ b/docs/pages/reference/evm/method-handlers.mdx @@ -96,9 +96,9 @@ https://docs.erpc.cloud/reference/evm/method-handlers.llms.txt`} **Hook topology.** Five hook layers fire at specific points in the request lifecycle, ordered around the cache read and the failsafe retry loop: -1. **Project.PreForward** (`HandleProjectPreForward`, `architecture/evm/hooks.go:L10`) — fires before cache read. Handles `eth_blockNumber` (returns highest known, replaces real upstream response), `eth_call` and `eth_estimateGas` (inject missing block param), `eth_chainId` (responds from config), and records `trace_filter`/`arbtrace_filter` range histogram. -2. **Network.PreForward** (`HandleNetworkPreForward`, `architecture/evm/hooks.go:L40`) — fires after upstream selection. Handles `eth_chainId` (responds from config), and proactive `trace_filter`/`arbtrace_filter` auto-splitting when the request range exceeds the per-upstream threshold. -3. **Upstream.PreForward** (`HandleUpstreamPreForward`, `architecture/evm/hooks.go:L86`) — fires once per upstream attempt. Handles `eth_getLogs`/`trace_filter` block-range availability and the `eth_chainId` fallback (upstream config → network ID string → network config). +1. **Project.PreForward** (`HandleProjectPreForward`, `architecture/evm/hooks.go:L10`) — fires before cache read. Answers the [local-answer methods](#local-answers), handles `eth_blockNumber` (returns highest known, replaces real upstream response), `eth_call` and `eth_estimateGas` (inject missing block param), and records `trace_filter`/`arbtrace_filter` range histogram. +2. **Network.PreForward** (`HandleNetworkPreForward`, `architecture/evm/hooks.go:L40`) — fires after upstream selection. Handles proactive `trace_filter`/`arbtrace_filter` auto-splitting when the request range exceeds the per-upstream threshold. +3. **Upstream.PreForward** (`HandleUpstreamPreForward`, `architecture/evm/hooks.go:L86`) — fires once per upstream attempt. Handles `eth_getLogs`/`trace_filter` block-range availability. 4. **Upstream.PostForward** (`HandleUpstreamPostForward`, `architecture/evm/hooks.go:L109`) — fires after each upstream response. Handles `eth_getBlockByNumber`/`eth_getBlockByHash` block validation, `eth_getBlockReceipts` integrity checks, and `eth_sendRawTransaction` nonce-exception interception. Also applies the configurable `markEmptyAsErrorMethods` gate to convert null/empty results to retryable `ErrEndpointMissingData`. Conversion only fires when the `RetryEmpty` directive is `true` on the request; when `RetryEmpty=false`, null/empty passes through unchanged even for methods in the list. 5. **Network.PostForward** (`HandleNetworkPostForward`, `architecture/evm/hooks.go:L62`) — fires once after the failsafe loop. Handles `eth_getBlockByNumber` highest-block enforcement, `eth_sendRawTransaction` exhausted-broadcast recovery, and reactive `trace_filter` splitting. @@ -110,9 +110,22 @@ Resolution is lock-efficient: param values are extracted under a read lock, reso **eth_call / eth_estimateGas missing-block-param injection.** If the request arrives with exactly one parameter (the call object, no block tag), the project pre-forward hook appends `"latest"` as the second param before forwarding; interpolation then pins it to the network's latest block number, so no upstream applies its own default block. `eth_call` is intentionally excluded from `markEmptyAsErrorMethods` and from structured error wrapping — callers expect raw ABI revert data (a hex string), not a structured error object. -**eth_chainId synthetic response.** All three hook levels (project, network, upstream) attempt to return a synthetic response from config without contacting any upstream. The check gates on `ShouldSkipCacheRead("")`: when `x-erpc-skip-cache-read: true` is set, all three layers are bypassed so force-refresh requests reach a real upstream for validation. At the upstream level, the handler prefers `upstreams[].evm.chainId`, then falls back to parsing from the upstream's network ID string (e.g. `"evm:1"`), then falls back to the network config value. +#### Local answers -**eth_blockNumber highest-block enforcement.** The project pre-forward hook forwards the request normally, then compares the upstream's response against `EvmHighestLatestBlockNumber` from the network's block poller. If the poller head is higher, the response is replaced with a synthetic one containing the highest known block number and `erpc_upstream_stale_latest_block_total` is incremented. Cached responses are returned as-is without comparison: the TTL is the correctness control for stale cached data. Controlled by `networks[].evm.integrity.enforceHighestBlock` (default `true`). +eRPC answers these methods itself, before cache and upstream selection, because the true answer does not depend on which upstream serves it, or is about eRPC, the node the client talks to. Routing them made one upstream a single point of failure for a constant (upstreams often reject `net_*` methods) or returned one node's view where eRPC fronts several. + +| Method | eRPC's answer | Routed instead when | +|---|---|---| +| `eth_chainId` | `networks[].evm.chainId` as hex | chain id not configured | +| `net_version` | `networks[].evm.chainId` as a decimal string | chain id not configured | +| `net_listening` | `true` | never | +| `net_peerCount` | the number of upstreams eRPC can route the request to now (the selection policy's eligible set, narrowed by `use-upstream`), as hex | never | +| `web3_clientVersion` | `erpc//-/` | never | +| `eth_syncing` | `false` while one routable upstream is synced | no routable upstream is known synced | + +Every one is on by default. `networks[].evm.localAnswers` turns one off per network: `{net_peerCount: false}` routes `net_peerCount` to the upstreams as any other method, with no eRPC answer. `networkDefaults.evm.localAnswers` applies to each network per method, unless the network sets that method itself. A key that is not in the table fails config validation. A request with `x-erpc-skip-cache-read: true` is routed too, so a force-refresh can read an upstream's own answer. Source: , + +**eth_blockNumber highest-block enforcement.** With the `enforceHighestBlock` directive on (`directiveDefaults.enforceHighestBlock`, default `true`, overridable per request), the network pre-forward hook answers from the served tip without dispatching and the post-forward hook pins every response to it; refresh, freshness and pin rules: [Block tracking](/reference/evm/block-tracking#eth_blocknumber). **eth_getBlockByNumber and eth_getBlockByHash.** Network.PostForward runs two checks: - **`enforceHighestBlock`:** For `"latest"` or `"finalized"` tagged requests, if the response block number is behind the network's known highest, eRPC re-issues the request for that block number, fetches the newer block, and returns whichever of the two is higher via `pickHighestBlock`. The upstream that answered stale stays eligible for the re-fetch: on fast chains it is often the one whose own feed just delivered the tip. Increments `erpc_upstream_stale_latest_block_total` or `erpc_upstream_stale_finalized_block_total`. @@ -300,8 +313,8 @@ upstreams: [ ### Request/response behavior - `eth_call` and `eth_estimateGas` with one param always have `"latest"` appended before forwarding; callers never see this normalization. [] -- `eth_chainId` synthetic response is skipped when `ShouldSkipCacheRead("")` is true, allowing force-refresh requests to reach real upstreams. [] -- `eth_blockNumber` cached responses are returned as-is, bypassing the highest-block comparison. [] +- Local answers are skipped when `ShouldSkipCacheRead("")` is true, allowing force-refresh requests to reach real upstreams. [] +- `eth_blockNumber` is answered from the served head while it is known; any forwarded or cached response is pinned exactly to it. [] - `eth_getBlockByNumber` `enforceHighestBlock` re-forward keeps every upstream eligible, including the one that answered stale, and skips the cache; a re-fetched block that no upstream has comes back empty and `pickHighestBlock` keeps the original block. [] - `emptyResultBeyondConfidence` with head=0 (unknown) fails open — any block is treated as potentially retryable. [] - `eth_sendRawTransaction` network-level probe uses `context.WithoutCancel` + 3-second fixed timeout; probe runs even when the parent context is expired. [] @@ -325,8 +338,8 @@ upstreams: [ ### Edge cases & gotchas 1. **eth_call / eth_estimateGas missing block param auto-injection** — a client sending only the call object (no block tag) gets `"latest"` silently appended. Source: . -2. **eth_chainId skip-cache-read bypasses all synthetic levels** — `x-erpc-skip-cache-read: true` causes all three hook levels to fall through to real upstreams. Source: . -3. **eth_blockNumber from-cache bypass** — cached responses skip the highest-block comparison; TTL is the correctness control for stale cached data. Source: . +2. **Skip-cache-read routes local answers** — `x-erpc-skip-cache-read: true` sends `eth_chainId`, `net_*`, `web3_clientVersion` and `eth_syncing` to real upstreams. Source: . +3. **eth_blockNumber never reaches an upstream while the served head is known** (with `enforceHighestBlock` on) — the network pre-forward hook answers it; a forwarded or cached response is pinned exactly to the served head, never above or below it. Source: . 4. **enforceHighestBlock never excludes the stale upstream** — the re-forward is a fresh request for the higher block number, eligible on every upstream; the stale-block counters increment only when the original response contained a real block number (non-zero). Source: , . 5. **pickHighestBlock safety** — always compares old vs new response rather than blindly returning the re-fetched block, guarding against corrupt poller state. If one response is null/empty, returns the other; if both are null/empty, returns the error from the re-forward rather than a null block. Source: . 6. **eth_getBlockReceipts excluded from default markEmptyAsErrorMethods** — empty arrays are correct for 0-tx blocks; inclusion causes retry storms. Source: . @@ -367,18 +380,17 @@ upstreams: [ **Trace span names** (for distributed tracing / APM): - `Project.PreForwardHook` — parent span for all project-level hooks -- `Project.PreForwardHook.eth_chainId`, `Project.PreForwardHook.eth_blockNumber` -- `Network.PreForwardHook`, `Network.PreForwardHook.eth_chainId` -- `Network.PostForward.eth_getBlockByNumber`, `Network.PostForward.eth_sendRawTransaction` -- `Upstream.PreForwardHook`, `Upstream.PreForwardHook.eth_chainId`, `Upstream.PreForwardHook.trace_filter` +- `Network.PreForwardHook` +- `Network.PostForward.eth_blockNumber`, `Network.PostForward.eth_getBlockByNumber`, `Network.PostForward.eth_sendRawTransaction` +- `Upstream.PreForwardHook`, `Upstream.PreForwardHook.trace_filter` - `Upstream.PostForwardHook.eth_getBlockByNumber`, `Upstream.PostForwardHook.eth_getBlockReceipts`, `Upstream.PostForwardHook.eth_sendRawTransaction` - `Evm.PickHighestBlock`, `extractTxHashFromSendRawTransaction`, `createSyntheticSuccessResponse`, `verifyAndHandleNonceTooLow` **Notable log messages** (for debugging): - `"interpolated block tag to concrete block number"` (debug) — `NormalizeHttpJsonRpc` resolved `"latest"` or `"finalized"` to a concrete hex. - `"passed through block tag"` (trace) — `"safe"`, `"pending"`, etc. left unchanged. -- `"upstream returned older block than we known, falling back to highest known block"` (debug) — `eth_blockNumber` and `eth_getBlockByNumber` stale enforcement. -- `"skipping enforcement of highest block number as response is from cache"` (trace) — cache bypass in `eth_blockNumber` and `eth_getBlockByNumber`. +- `"upstream returned older block than we know, falling back to highest known block"` (debug) — `eth_blockNumber` response below the served head, raised to it. +- `"response is ahead of the served tip, pinning to the served tip"` (debug) — `eth_blockNumber` response above the served head, lowered to it. - `"converting 'already known' error to idempotent success"` (info) — `eth_sendRawTransaction` upstream hook. - `"tx FOUND on-chain - converting 'nonce too low' to idempotent success"` (info) — `eth_sendRawTransaction` nonce-too-low probe success. - `"exhausted error overridden: tx found in network, returning synthetic success"` (info) — `eth_sendRawTransaction` network-level hook. diff --git a/docs/pages/reference/lanes.mdx b/docs/pages/reference/lanes.mdx index 776f6df04..a44371616 100644 --- a/docs/pages/reference/lanes.mdx +++ b/docs/pages/reference/lanes.mdx @@ -1,6 +1,6 @@ --- title: Lanes & concurrency -description: Route a class of requests to a specific provider group and eRPC automatically maintains a separate block-tip counter for that group — eliminating "block not found" churn caused by cross-provider tip pollution. +description: Route a class of requests to a specific provider group and eRPC computes that group's own block tip — eliminating "block not found" churn caused by cross-provider tip pollution. --- import { @@ -19,7 +19,7 @@ Different provider classes see different blocks at different times. When you rou What you get: -- Per-group served-tip counters shared and monotonic across every pod +- Per-group served tip decided among that group's upstreams only - Deterministic lane names on your Prometheus dashboards (`flashblocks`, `systx`, …) - Zero config — lanes emerge automatically from your `use-upstream` selectors @@ -34,8 +34,8 @@ points the agent at this page's machine-readable reference so it can do the work defaultOpen prompt={`I want all requests on my Ethereum mainnet network to route exclusively to flashblocks providers so the block tip is always flashblocks-accurate. Set directivesDefaults.useUpstream -at the network level in my eRPC config, and explain when a selector qualifies for a lane -partition versus falling back to stateless. Reference: +at the network level in my eRPC config, and explain when a selector qualifies for a named lane +versus getting no lane gauges. Reference: https://docs.erpc.cloud/reference/lanes.llms.txt`} /> @@ -54,8 +54,8 @@ https://docs.erpc.cloud/reference/lanes.llms.txt`} title="debug missing lane metrics in Prometheus" prompt={`My Prometheus shows erpc_network_served_tip_block_number with lane=all but never a named lane label even though I'm sending X-ERPC-Use-Upstream headers. Explain the -conditions that prevent a lane partition from forming (complex selector, fewer than 2 -upstreams matched, MAX mode, servedTip not enabled) and check my eRPC config for the +conditions that prevent a lane from forming (complex selector, fewer than 2 upstreams +matched, all upstreams matched, 16-lane cap reached) and check my eRPC config for the likely cause. Reference: https://docs.erpc.cloud/reference/lanes.llms.txt`} /> @@ -63,17 +63,17 @@ likely cause. Reference: https://docs.erpc.cloud/reference/lanes.llms.txt`} ### How it works -A **lane** is not a config object. It is an emergent artifact: when a request carries a `use-upstream` selector (via `X-ERPC-Use-Upstream` header, `?use-upstream=` query parameter, or `directivesDefaults.useUpstream` in config), eRPC resolves which upstreams match. If the matched set is a proper subset of at least two upstreams — and the selector is "simple" (see below) — eRPC derives a stable key by SHA-256-hashing the sorted upstream IDs, then lazily creates a `servedTipPartition` for that key. That partition holds two `CounterInt64SharedVariable` counters (latest and finalized blocks), shared across pods via the shared-state registry. +A **lane** is not a config object. It is an emergent artifact: when a request carries a `use-upstream` selector (via `X-ERPC-Use-Upstream` header, `?use-upstream=` query parameter, or `directivesDefaults.useUpstream` in config), eRPC resolves which upstreams match. If the matched set is a proper subset of at least two upstreams — and the selector is "simple" (see below) — eRPC derives a stable key by SHA-256-hashing the sorted upstream IDs, then lazily creates a `servedTipLane` for that key (`servedTipLaneFor`). A lane holds no pick state: only its label and, per axis, an in-process anchor for the advance-age gauge and the regression guard's memory. The network-wide pick is the lane `servedTipAll`, labelled `"all"`; only it gets the WebSocket delivered-head floor. The tip itself is the [served-tip rule](/reference/evm/block-tracking#served-tip) over the matched subset, computed fresh for every selector, lane or not. Nothing is shared across pods. Source: , **LaneName algorithm.** `LaneName(ids)` sorts the upstream IDs alphabetically, then builds a set of non-empty hyphen-split tokens for each ID (empty strings from consecutive hyphens are discarded). It walks the tokens of the first sorted ID and returns the first token that appears in every other ID's token set. If no universally shared token exists, it falls back to a concatenation of the first two characters of each ID (up to five IDs). The result is deterministic and order-independent — every pod reaches the same name from the same upstream topology. Source: **What counts as a "simple" selector.** `isSimpleGroupSelector` rejects selectors that contain whitespace, `|`, `&`, `(`, `)`, or `,` (no boolean combinators), or more than one `!` not at position 0, or length > 128 characters. Qualifying examples: `flashblocks*`, `!flashblocks*`, `family:systx`, `alchemy-*`. Non-qualifying: `alchemy | quicknode`, `(a | b)`. -**Partition lifecycle and cap.** Partitions are stored in a `sync.Map` keyed by `"grp:"` (the first 8 bytes of the SHA-256 digest encoded as 16 hex characters). On the fast path, `sync.Map.Load` returns an existing partition with no locking. On the slow path, an `atomic.Int32` counter is checked against `maxServedTipPartitions = 16`; if at the cap, the request falls through to the stateless fallback. Because the check-and-create is not atomic, up to `16 + goroutine-concurrency` partitions can transiently exist — `LoadOrStore` guarantees exactly one partition wins per key. +**Lane lifecycle and cap.** Lanes are stored in a `sync.Map` keyed by `"grp:"` (the first 8 bytes of the SHA-256 digest encoded as 16 hex characters). On the fast path, `sync.Map.Load` returns an existing lane with no locking. On the slow path, an `atomic.Int32` counter is checked against `maxServedTipLanes = 16`; if at the cap, the request gets the same subset pick with no lane. Because the check-and-create is not atomic, up to `16 + goroutine-concurrency` lanes can transiently exist — `LoadOrStore` guarantees exactly one lane wins per key. -The partition key is derived from the **matched upstream set**, not the selector text. `flashblocks*` and `fl*` targeting the same three upstreams produce the same key and share one partition. Cardinality is bounded by the actual upstream topology, not selector diversity. +The lane key is derived from the **matched upstream set**, not the selector text. `flashblocks*` and `fl*` targeting the same three upstreams produce the same key and share one lane. Cardinality is bounded by the actual upstream topology, not selector diversity. -**Stateless fallback.** When a selector is complex, matches fewer than two upstreams, or matches all upstreams, `servedTipPartitionFor` returns nil. `observeServedTipMetrics` is called with `lane=servedTipLaneNone` and returns immediately, writing nothing to Prometheus — preventing a subset value from overwriting the `lane="all"` gauge. +**No lane.** When a selector is complex, matches fewer than two upstreams, matches all upstreams, or arrives past the cap, `servedTipLaneFor` returns nil. The subset pick still applies, with no regression guard, no anchor and no gauge write, so a subset value never overwrites the `lane="all"` gauge. **UseUpstream directive precedence** (highest to lowest): @@ -81,15 +81,15 @@ The partition key is derived from the **matched upstream set**, not the selector 2. `?use-upstream=` query parameter. 3. `networks[*].directivesDefaults.useUpstream` config default. -The selector is used in two places: as the partition lookup key for served-tip counters, and inside the upstream selection loop (`UpstreamMatchesSelector`) to filter which upstreams receive the request. Tag-based matching is supported — `family:flashblocks` matches any upstream whose tags include that key — but negated selectors (prefix `!`) never match by tag, only by upstream ID. +The selector is used in two places: to narrow the served-tip ballot (and resolve its lane), and inside the upstream selection loop (`UpstreamMatchesSelector`) to filter which upstreams receive the request. Tag-based matching is supported — `family:flashblocks` matches any upstream whose tags include that key — but negated selectors (prefix `!`) never match by tag, only by upstream ID. ### Config schema | Field | Type | Default | Behavior / footguns | |---|---|---|---| -| `networks[*].directivesDefaults.useUpstream` | `*string` | `nil` (no default selector) | Sets the `use-upstream` selector for all requests on this network when the client sends none. Identical semantics to the `X-ERPC-Use-Upstream` header. **Footgun:** a wildcard matching all upstreams does NOT form a partition — it uses the `lane="all"` path. Sources: , | +| `networks[*].directivesDefaults.useUpstream` | `*string` | `nil` (no default selector) | Sets the `use-upstream` selector for all requests on this network when the client sends none. Identical semantics to the `X-ERPC-Use-Upstream` header. **Footgun:** a selector matching all upstreams forms no lane: its pick runs over the same upstreams as the network-wide one, but without the regression guard, the WebSocket floor or any gauge. Sources: , | -Lanes themselves are not directly configured — they emerge from runtime request selectors and the upstream topology. Lane behavior requires `networks[*].evm.servedTip` cluster mode to be enabled; in MAX mode (the default) `servedTipEnabledFor` returns false, partition lookup never runs, and no `lane`-labelled metrics are emitted. +Lanes themselves are not directly configured — they emerge from runtime request selectors and the upstream topology. They need no `servedTip` block; `servedTip.quorum` applies inside each lane as it does network-wide. ### Worked examples @@ -120,69 +120,59 @@ export default createConfig({ });`} /> -**2. Per-request lane selection via header.** Send `X-ERPC-Use-Upstream: systx*` on individual requests to route them to system-transaction-capable upstreams. eRPC derives the lane name, maintains a `systx`-scoped tip, and serves block numbers visible to that provider class only. +**2. Per-request lane selection via header.** Send `X-ERPC-Use-Upstream: systx*` on individual requests to route them to system-transaction-capable upstreams. eRPC derives the lane name, computes a `systx`-scoped tip, and serves block numbers visible to that provider class only. -**3. Tag-based lane targeting.** If upstreams are tagged with `family:archive`, a selector `family:archive` is simple, qualifies for a partition, and routes archive-depth calls to archive nodes — `LaneName` derives a human-readable label from the matched upstream IDs for your dashboards. +**3. Tag-based lane targeting.** If upstreams are tagged with `family:archive`, a selector `family:archive` is simple, qualifies for a lane, and routes archive-depth calls to archive nodes — `LaneName` derives a human-readable label from the matched upstream IDs for your dashboards. -**4. Complex selector routing without lane tip tracking.** A selector like `alchemy | quicknode` routes to both providers correctly via `UpstreamMatchesSelector` in the forwarding loop — but it is not simple, so no per-group tip partition is created and no `lane=...` gauge appears. The stateless fallback applies; this is expected behavior for multi-provider round-robin. +**4. Complex selector routing without lane tip tracking.** A selector like `alchemy | quicknode` routes to both providers correctly via `UpstreamMatchesSelector` in the forwarding loop — but it is not simple, so no lane is created and no `lane=...` gauge appears. The tip is still computed over the two providers; this is expected behavior for multi-provider round-robin. ### Best practices -- Use simple glob selectors (`flashblocks*`, `family:archive`) rather than boolean expressions when per-group tip accuracy matters; boolean selectors route correctly but never get a lane counter. +- Use simple glob selectors (`flashblocks*`, `family:archive`) rather than boolean expressions when you need per-group gauges or the regression guard; boolean selectors route and scope the tip correctly but never get a lane. - Set `directivesDefaults.useUpstream` at the network level rather than injecting headers per-request — it ensures every request (including retries and hedges) carries the selector without client cooperation. -- Keep the total number of distinct upstream subsets below 16 per network; beyond that, additional groups fall back to stateless tip and lose per-group accuracy. -- Do not rely on `LaneName` label values being unique — two different upstream sets can produce the same label. Use the partition key (internal) as the stable identifier; the label is for human readability only. +- Keep the total number of distinct upstream subsets below 16 per network; beyond that, additional groups get no gauges and no regression guard. +- Do not rely on `LaneName` label values being unique — two different upstream sets can produce the same label. Use the lane key (internal) as the stable identifier; the label is for human readability only. - Negated selectors (`!alchemy-*`) match by upstream ID only, not by tag. If you need tag-based exclusion, restructure as an inclusion selector on the desired tag instead. -- `servedTip` cluster mode must be enabled for lanes to have any effect; in the default MAX mode, no partitions are ever created and all the lane machinery is dormant. ### Edge cases & gotchas -1. **A single matched upstream does not form a lane.** `partitionKeyFor` requires `len(matched) >= 2`; a single-upstream selector gets the stateless fallback and emits no gauge. +1. **A single matched upstream does not form a lane.** `partitionKeyFor` requires `len(matched) >= 2`; a single-upstream selector gets that upstream's head and emits no gauge. 2. **Matching ALL upstreams does not form a lane.** `len(matched) >= len(all)` also falls through, preventing a match-all selector from shadowing `lane="all"`. -3. **Boolean combinator selectors are never partitioned.** `alchemy | quicknode` contains `|`; it routes correctly but never gets a per-group tip counter. -4. **The 16-partition cap is a soft guard.** Concurrent goroutines can transiently exceed 16. Beyond the cap, new groups use the stateless fallback. -5. **Partition key = hash of matched upstream set, not selector text.** `flashblocks*` and `fl*` matching the same upstreams share one partition. Cardinality is bounded by topology, not by selector diversity. -6. **`LaneName` is not guaranteed unique.** Two different upstream sets could produce the same name if both share the same common token. The partition key is the stable identifier; the name is decorative for dashboards. +3. **Boolean combinator selectors are never partitioned.** `alchemy | quicknode` contains `|`; it routes and scopes the tip correctly but never gets a lane. +4. **The 16-lane cap is a soft guard.** Concurrent goroutines can transiently exceed 16. Beyond the cap, new groups get no lane. +5. **Lane key = hash of matched upstream set, not selector text.** `flashblocks*` and `fl*` matching the same upstreams share one lane. Cardinality is bounded by topology, not by selector diversity. +6. **`LaneName` is not guaranteed unique.** Two different upstream sets could produce the same name if both share the same common token. The lane key is the stable identifier; the name is decorative for dashboards. 7. **Negated selectors match by ID only, never by tag.** A pattern containing `!` skips tag evaluation entirely. `!alchemy-*` excludes by ID but will not rescue non-matching IDs via tags. -8. **Stateless fallback emits no gauge.** `servedTipLaneNone` causes `observeServedTipMetrics` to return before any write, ensuring a complex-selector request cannot overwrite `lane="all"` with an incorrect subset value. -9. **`LaneName` sort is alphabetical.** For `(quicknode-systx, systx-quicknode)`, sorted order puts `quicknode-systx` first; `LaneName` returns `"quicknode"`, not `"systx"`. Source: `common/lane_test.go` (`order independent` test). -10. **`servedTip` must be configured for lanes to have effect.** In MAX mode (default), `EvmHighestLatestBlockNumber` takes the non-cluster path and no partition is ever consulted. Because `observeServedTipMetrics` is only called from the cluster path, even the `lane="all"` gauge is absent in MAX mode — no `lane`-labelled metrics of any kind are emitted. -11. **`LaneName` output is written to Prometheus labels as-is, without sanitization.** If upstream IDs contain unusual characters that cause `LaneName` to produce an unexpected string, that string becomes the label value verbatim. The label is never normalized or escaped beyond what `LaneName` itself produces. +8. **`LaneName` sort is alphabetical.** For `(quicknode-systx, systx-quicknode)`, sorted order puts `quicknode-systx` first; `LaneName` returns `"quicknode"`, not `"systx"`. Source: `common/lane_test.go` (`order independent` test). +9. **Lane state is per process.** Anchors and regression-guard memory live in memory; a restarted pod rebuilds them from live heads. +10. **`LaneName` output is written to Prometheus labels as-is, without sanitization.** If upstream IDs contain unusual characters that cause `LaneName` to produce an unexpected string, that string becomes the label value verbatim. The label is never normalized or escaped beyond what `LaneName` itself produces. ### Observability -| Metric | Type | Labels | When it fires | -|---|---|---|---| -| `erpc_network_served_tip_block_number` | gauge | `project`, `network`, `lane`, `axis` | `lane="all"` (network-wide) or `lane=` (per-group); absent when stateless fallback | -| `erpc_network_served_tip_lag_blocks` | gauge | `project`, `network`, `lane`, `axis` | Blocks behind freshest velocity-eligible tip; absent in MAX mode | -| `erpc_network_served_tip_upstream_excluded_total` | counter | `project`, `network`, `upstream`, `axis`, `reason` | Only for `lane="all"` (network-wide pick); `reason` = `velocity` or `outlier` | - -`axis` = `"latest"` or `"finalized"`. The `lane` series for a named group is only present for networks that actually receive targeted `use-upstream` traffic. - -Per-upstream exclusion counters (`erpc_network_served_tip_upstream_excluded_total`) are only emitted for `lane="all"` — the network-wide pick. The internal Go condition checks `lane == ""`, but the Prometheus label written is `"all"` (via `servedTipLaneAll`). Source: +Lanes are the `lane` label of the four `erpc_network_served_tip_*` series ([meanings](/reference/metrics#area-3-block-number-and-tip-tracking)): `lane="all"` is the network-wide pick (`servedTipAll`), `lane=` a group's own pick. A named lane's series exist only for networks that receive targeted `use-upstream` traffic; a request without a lane writes no series. Source: ### Source code entry points - [`common/lane.go`](https://github.com/erpc/erpc/blob/main/common/lane.go) — `LaneName(ids []string) string`: naming algorithm, pure function. - [`common/lane_test.go`](https://github.com/erpc/erpc/blob/main/common/lane_test.go) — unit tests for `LaneName`: shared-token precedence, order-independence, fallback combo, single-id, empty input, determinism. -- [`erpc/networks.go:L80-L105`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L80-L105) — constants, `servedTipPartition` struct, `maxServedTipPartitions = 16`. -- [`erpc/networks.go:L405-L442`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L405-L442) — `servedTipPartitionFor`: lazy partition materialization with cap and `sync.Map`. -- [`erpc/networks.go:L444-L506`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L444-L506) — `partitionKeyFor`: selector → SHA-256 key; `isSimpleGroupSelector` filter. -- [`erpc/networks.go:L517-L540`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L517-L540) — `EvmHighestLatestBlockNumber`: cluster-mode + per-lane branching. -- [`erpc/networks.go:L802-L852`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L802-L852) — `observeServedTipMetrics`: lane label assignment and gauge emission. +- [`erpc/networks.go:L113-L132`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L113-L132) — `maxServedTipLanes = 16`, `servedTipLane` struct. +- [`erpc/networks.go:L628-L655`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L628-L655) — `servedTipLaneFor`: lazy lane materialization with cap and `sync.Map`. +- [`erpc/networks.go:L657-L746`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L657-L746) — `partitionKeyFor`: selector → SHA-256 key; `EvmUpstreamGroupForSelector`; `isSimpleGroupSelector` filter. +- [`erpc/networks.go:L870-L942`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L870-L942) — `servedTip`: guard, floors and gauges per lane. +- [`erpc/networks.go:L1502-L1532`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L1502-L1532) — `observeServedTipMetrics`: gauge emission per lane label. - [`common/matcher.go:L73-L112`](https://github.com/erpc/erpc/blob/main/common/matcher.go#L73-L112) — `MatchesSelector` + `UpstreamMatchesSelector`: ID + tag matching primitive. - [`common/request.go:L739-L741`](https://github.com/erpc/erpc/blob/main/common/request.go#L739-L741) — `X-ERPC-Use-Upstream` header extraction. - [`common/request.go:L811-L812`](https://github.com/erpc/erpc/blob/main/common/request.go#L811-L812) — `?use-upstream=` query-param extraction. - [`common/request.go:L1424-L1445`](https://github.com/erpc/erpc/blob/main/common/request.go#L1424-L1445) — `UpstreamMatchesSelector` call in the upstream selection loop; filters which upstreams receive the forwarded request. -- [`erpc/networks.go:L396-L402`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L396-L402) — `requestSelector(ctx)`: reads `UseUpstream` directive from the bound request context for both served-tip and forwarding use. +- [`erpc/networks.go:L617-L626`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L617-L626) — `requestSelector(ctx)`: reads `UseUpstream` directive from the bound request context for both served-tip and forwarding use. - [`common/config.go:L2109`](https://github.com/erpc/erpc/blob/main/common/config.go#L2109) — `DirectivesConfig.UseUpstream` field. -- [`telemetry/metrics.go:L130-L162`](https://github.com/erpc/erpc/blob/main/telemetry/metrics.go#L130-L162) — metric definitions. +- [`telemetry/metrics.go:L198-L250`](https://github.com/erpc/erpc/blob/main/telemetry/metrics.go#L198-L250) — metric definitions. ### Related pages - [Matcher](/config/matcher) — the `UpstreamMatchesSelector` and tag-matching primitives used to resolve which upstreams a selector targets. - [Tag-aware routing](/config/projects/selection-policies) — how `use-upstream` integrates with the upstream selection loop. -- [Served tip / block number tracking](/reference/evm/block-number) — the cluster-mode served-tip feature that lanes extend. +- [Block tracking & served tip](/reference/evm/block-tracking) — the served-tip rule that lanes scope. - [Rate limiters](/config/rate-limiters) — per-upstream rate limits that interact with lane-scoped routing. diff --git a/docs/pages/reference/metrics.mdx b/docs/pages/reference/metrics.mdx index b5af40bba..54a52035f 100644 --- a/docs/pages/reference/metrics.mdx +++ b/docs/pages/reference/metrics.mdx @@ -267,7 +267,7 @@ metrics: 10. **`erpc_ristretto_cache_current_cost` requires `memory.emitMetrics: true`.** Without this flag the gauge is registered and appears in `/metrics` but always returns 0. It is the only metric in the system requiring an explicit opt-in to be meaningful. () 11. **`erpc_upstream_cordoned` `category` label is the cordon scope (method string or `"*"`), not a request-category label.** Do not join this `category` with `category` from request-accounting metrics. () 12. **`erpc_upstream_cordoned` `vendor` label is `"n/a"` when no vendor is configured.** Dashboard JOINs between cordon metrics and upstream request metrics must handle the `"n/a"` value for unvendored upstreams. () -13. **`erpc_auth_failed_total` always has `strategy="database"`.** Only the database auth strategy emits this counter. For other strategies, monitor failures via `erpc_network_failed_request_total{error=~"ErrAuthUnauthorized.*"}`. () +13. **`erpc_auth_failed_total` increments when `ErrAuthUnauthorized` is sent to the client.** Closed `reason` per strategy via `NewErrAuthUnauthorized`; intermediate multi-strategy rejects are not counted. `agent_name` is always `"unknown"`. () 14. **`erpc_upstream_attempt_outcome_total` `is_hedge`/`is_retry` are literal `"true"`/`"false"` strings.** Use `{is_hedge="true"}` in PromQL, not `{is_hedge="1"}`. () 15. **`erpc_network_evm_block_range_requested_total` `bucket` label is unbounded.** When tip is known, `ComputeBlockHeatmapBucket` produces human-readable relative labels (`"TIP"`, `"L100k"`, `"100k-200k"`, etc.). This counter is not covered by the idle sweep — cardinality grows permanently. () 16. **`erpc_selection_*` metrics carry a `method` label.** Untrusted callers injecting hundreds of distinct method names create unbounded selection-gauge series. Consider a Prometheus relabeling drop rule for the `method` label in untrusted deployments. @@ -313,6 +313,7 @@ All metric names carry the `erpc_` prefix. Full definitions: diff --git a/docs/pages/reference/svm/slot-tracking.mdx b/docs/pages/reference/svm/slot-tracking.mdx index e39eb8f7b..828f4be7b 100644 --- a/docs/pages/reference/svm/slot-tracking.mdx +++ b/docs/pages/reference/svm/slot-tracking.mdx @@ -107,13 +107,13 @@ https://docs.erpc.cloud/reference/svm/slot-tracking.llms.txt`} | `FinalizedSlot()` | `getSlot {"commitment":"finalized"}` | The latest rooted slot | | `ShredInsertSlot()` | `getMaxShredInsertSlot` | Blockstore-ingestion watermark | | `MaxShredInsertSlotLag()` | derived | `maxShredInsertSlot − processedSlot` | -| `FirstAvailableSlot()` | `getBlocksWithLimit(0, 1)` | The oldest block the node serves, from the same storage `getBlock` reads (long-term storage included). **Probed only when the upstream declares a `firstAvailableSlotPlus` bound.** `known=false` until observed (or when not polled); unknown is **not** slot 0. | +| `FirstAvailableSlot()` | `getBlocksWithLimit(0, 1)` | The oldest block the node serves, from the same storage `getBlock` reads (long-term storage included). Probed for every upstream. `known=false` until observed; unknown is **not** slot 0. | **There is deliberately no confirmed tip.** Nothing polls `getSlot {"commitment":"confirmed"}`, and that absence is load-bearing for the `getSlot` correction below. **Tick cadence and the debounce gate.** The ticker fires at a fixed `DefaultPollInterval` of 400 ms — one Solana slot. `svm.statePollerDebounce` (default 400 ms) is a **gate on the fan-out**, not the ticker period: a tick that arrives inside the debounce window returns immediately without issuing any RPC. The ticker is deliberately *not* driven by the gate value, because a ticker firing at exactly the gate interval skips every other tick. -**Per-tick fan-out.** Up to four concurrent calls: `getHealth`, `getSlot(processed)`, `getSlot(finalized)`, `getMaxShredInsertSlot`. Each runs in its own panic-recovered goroutine so a single bad response cannot take down the process. An upstream that declares a `firstAvailableSlotPlus` bound additionally refreshes its floor with `getBlocksWithLimit(0, 1)` after the fan-out joins (staleness-bounded, not on every tick); upstreams without one never issue it. +**Per-tick fan-out.** Up to four concurrent calls: `getHealth`, `getSlot(processed)`, `getSlot(finalized)`, `getMaxShredInsertSlot`. Each runs in its own panic-recovered goroutine so a single bad response cannot take down the process. Every upstream also refreshes its floor with `getBlocksWithLimit(0, 1)` after the fan-out joins: at most once per 30 s, shared across eRPC instances. **Traffic-gated polling.** Solana responses whose result is the `RpcResponse` envelope carry a `context.slot` telling eRPC which slot the node was at when it answered. That observation is harvested on the response path and fed into the poller, routed by the request's **effective commitment**: a finalized-commitment response feeds the finalized view *and* the latest view (a finalized slot is always a valid lower bound for the latest view); weaker commitments feed only the latest view. When both slot views are traffic-fresh within the debounce window, the tick **skips its two `getSlot` calls** — bounded at 4 consecutive skips, so live traffic can never fully starve independent verification. `getHealth` and `getMaxShredInsertSlot` always run. On busy networks this roughly halves background poll quota on metered vendors. @@ -193,9 +193,9 @@ The guard **fails open** whenever the tip it needs is unknown — an indexed fro **Declared slot windows.** Optional `upstreams[*].svm.blockAvailability.{lower,upper}` (`exactSlot` / `latestSlotMinus` / `firstAvailableSlotPlus`) is an operator declaration of the slots an upstream may serve, gated by `svm.enforceBlockAvailability`. An upstream whose declared window excludes the slot a request names is skipped (lower: `-32001`, permanent for the request; upper: `-32004`). The upper bound is checked against the **end** of a `getBlocks` / `getBlocksWithLimit` range, not only its start. Without a declared window every upstream is tried. -**The floor is the oldest block the upstream serves, not `getFirstAvailableBlock`.** `getFirstAvailableBlock` reports only the local ledger, so a node backed by long-term storage reports a floor far above slots it serves (Alchemy devnet: ~509M while serving 116M+). eRPC instead probes `getBlocksWithLimit(0, 1)`, which agave answers from the same storage `getBlock` reads; an empty answer is no observation. There is no pool-level `-32009` short-circuit. The floor is probed — and used — only as the anchor of a declared `firstAvailableSlotPlus` bound, the SVM analogue of EVM's `earliestBlockPlus`. A bound that does not need the floor (`exactSlot`, `latestSlotMinus`) overrides whatever the vendor reports; `exactSlot: 0` declares full history. +**The floor is the oldest block the upstream serves, not `getFirstAvailableBlock`.** `getFirstAvailableBlock` reports only the local ledger, so a node backed by long-term storage reports a floor far above slots it serves (Alchemy devnet: ~509M while serving 116M+). eRPC instead probes `getBlocksWithLimit(0, 1)`, which agave answers from the same storage `getBlock` reads; an empty answer is no observation. There is no pool-level `-32009` short-circuit. The floor is probed for every upstream. It skips nothing by itself: it anchors a declared `firstAvailableSlotPlus` bound (the SVM analogue of EVM's `earliestBlockPlus`) and orders the history sweep below. A bound that does not need the floor (`exactSlot`, `latestSlotMinus`) overrides whatever the vendor reports; `exactSlot: 0` declares full history. -A cold poller, an unobserved floor, or a computed min>max range fail-open. `firstAvailableSlotPlus` with an unobserved floor is unbounded (unlike EVM's `earliestBlockPlus`, which computes `0+N` before detection). +A cold poller, an unobserved floor, or a computed min>max range fail-open. `firstAvailableSlotPlus` with an unobserved floor is unbounded (unlike EVM's `earliestBlockPlus`, which computes `0+N` before detection). The floor is probed every `30s` until observed, then frozen; a `firstAvailableSlotPlus` bound's `updateRate` re-probes it on that cadence, like EVM's `earliestBlockPlus`. **Consensus slot-lag pre-filter.** For a request whose resolved finality is `Finalized` **and** with a consensus policy active, upstreams whose `FinalizedSlot` trails the reference by more than `svm.maxFinalizedSlotLag` are pruned before consensus runs — the same query sent to a stale upstream and a current one returns different answers for data the caller has been promised is immutable. @@ -366,8 +366,12 @@ The filter is defensive throughout: an upstream with no state poller is **includ 10. **Large-rollback events are recorded, not acted on.** A backwards slot jump beyond 1024 is surfaced as `erpc_upstream_block_head_large_rollback`; it does not itself cordon. 11. **`context.slot` harvesting only covers envelope methods.** `getSlot`, `getBlock`, and `getTransaction` return bare values and contribute nothing to the traffic gate. `getProgramAccounts` is included because its envelope form is opt-in via `withContext: true` and the non-envelope miss is cheap. 12. **Declared `svm.blockAvailability` shares `enforceBlockAvailability`.** Setting the switch to `false` disables the declared window and the `getBlock` guard together. There is no second knob. -13. **The observed floor is opt-in, never a default gate.** The floor is probed only for an upstream declaring `firstAvailableSlotPlus`. With no such bound it is neither fetched nor enforced. `exactSlot` / `latestSlotMinus` bounds ignore it entirely. +13. **The observed floor is never a default gate.** The floor is probed for every upstream, but no slot-named read is skipped on it unless the upstream declares `firstAvailableSlotPlus`. `exactSlot` / `latestSlotMinus` bounds ignore it entirely. 14. **`firstAvailableSlotPlus` with an unobserved floor is unbounded.** Unknown floor is not slot 0 — the request forwards until the poller has a real sample. +15. **History reads go deepest floor first, and continue only toward a deeper floor.** Within each failover tier, `getSignatureStatuses` and `getSignaturesForAddress` try upstreams with a known floor first, deepest first; unknown floors follow in score order. `getSignatureStatuses` with `searchTransactionHistory: true` that returns a `null`, and `getSignaturesForAddress` that returns fewer entries than its limit (`limit`, else Solana's default 1000), are also the normal end of history. So the 200 becomes MissingData, and the whole request goes to the next upstream, only when another selected upstream has a known floor strictly below this one. Ties, an unknown floor on either side, and a lone upstream serve the answer as is. Without `searchTransactionHistory: true` a null is a valid recent-status answer. A non-positive or non-integer `limit`, or a non-array result, fails open. There is no cross-upstream merge. +16. **A continued answer stays the fallback.** The verdict is permanent (`WithPermanentMissingData`), so a completed sweep is not re-swept after a block-time delay. If the deeper upstream fails, the client receives the shallower answer through last-valid-response. +17. **The history sweep does not run on consensus participant slots.** Consensus participants (`IsConsensusSlot`) vote on the bodies they returned; turning a legitimate short page into MissingData would make consensus agree on an error. +18. **History-sweep continuation is not wrong-empty misbehavior.** Sweep verdicts carry `historySweepContinuation` in error details; the network layer skips `MetricUpstreamWrongEmptyResponseTotal` and `RecordUpstreamMisbehavior` for them. ### Observability @@ -384,7 +388,7 @@ The SVM poller feeds the shared health tracker, so the standard upstream gauges | `erpc_upstream_cordon_event_total` | counter | Increments only on a cordon **transition** (edge-triggered) | | `erpc_upstream_cordon_duration_seconds` | histogram | Observed on uncordon | | `erpc_upstream_stale_latest_block_total` | counter | A `getSlot` answer was corrected upward or capped | -| `erpc_upstream_svm_first_available_slot` | gauge | The oldest block an upstream serves, from `getBlocksWithLimit(0, 1)`. Exported only for upstreams declaring `firstAvailableSlotPlus`; absent until first observed; set on any accepted value including cross-pod propagation | +| `erpc_upstream_svm_first_available_slot` | gauge | The oldest block an upstream serves, from `getBlocksWithLimit(0, 1)`. Absent until first observed; set on any accepted value including cross-pod propagation | | `erpc_upstream_stale_lower_bound_total{confidence="configured"}` | counter | A slot-named read skipped an upstream below its declared `svm.blockAvailability.lower` | | `erpc_upstream_stale_upper_bound_total{confidence="configured"}` | counter | A slot-named read skipped an upstream above its declared `svm.blockAvailability.upper` | | `erpc_network_retry_attempt_total{reason="missing_data"}` | counter | Includes retries triggered by the `getBlock` availability guard | @@ -404,6 +408,7 @@ The SVM poller feeds the shared health tracker, so the standard upstream gauges - [`architecture/svm/hooks.go`](https://github.com/erpc/erpc/blob/main/architecture/svm/hooks.go) — `networkPostForward_getSlot` (per-commitment correction), `networkPreForward_getBlock` (availability guard), `effectiveCommitment`, `tipStalenessMargin`, `upstreamPostForward_trackContextSlot`. - [`architecture/svm/slot_lag.go`](https://github.com/erpc/erpc/blob/main/architecture/svm/slot_lag.go) — `FilterByFinalizedSlotLag`, `ReferenceFinalizedSlot` (second-highest clamp), `HighestFinalizedSlot`, `FilterByMinContextSlot`. - [`architecture/svm/ledger_floor.go`](https://github.com/erpc/erpc/blob/main/architecture/svm/ledger_floor.go) — slot-named request extraction shared by the declared-window checks (`slot_bounds.go`). +- [`architecture/svm/history_sweep.go`](https://github.com/erpc/erpc/blob/main/architecture/svm/history_sweep.go) — `getSignatureStatuses` null and `getSignaturesForAddress` short-page continuation toward a deeper floor. - [`architecture/svm/slot_bounds.go`](https://github.com/erpc/erpc/blob/main/architecture/svm/slot_bounds.go) — declared `svm.blockAvailability` resolution and per-upstream skip. - - diff --git a/docs/pages/use-cases/how-it-works.mdx b/docs/pages/use-cases/how-it-works.mdx index 1fb77617b..f0de5e9ce 100644 --- a/docs/pages/use-cases/how-it-works.mdx +++ b/docs/pages/use-cases/how-it-works.mdx @@ -89,8 +89,8 @@ authenticates the consumer via `project.AuthenticateConsumer`, resolves `PreparedProject.Forward` acquires a project-level rate-limit permit (no-op when `rateLimitBudget` is unset), fires `HandleProjectPreForward` — the first EVM hook, which -handles `eth_blockNumber` (early cache return), `eth_call` (static call optimisation), -`eth_chainId` (served from config), and proactive `eth_getLogs` / `trace_filter` range +handles `eth_call` / `eth_estimateGas` (default block tag), `eth_chainId` (served from +config), and proactive `eth_getLogs` / `trace_filter` range splitting. If the hook handles the request, `HandleNetworkPostForward` is called on the result before returning. Otherwise the call proceeds to `network.Forward`. @@ -107,9 +107,10 @@ result before returning. Otherwise the call proceeds to `network.Forward`. 4. **Upstream ordering** — `policyEngine.GetOrdered` returns upstreams ranked by selection policy scores; falls back to registry order on cold start. 5. **EVM network pre-forward hook** (`HandleNetworkPreForward`) — upstream-aware - short-circuits for `eth_getLogs` range enforcement, `eth_chainId`, `trace_filter`. -6. **Future-block short-circuit** — requests for a concrete block beyond every eligible - upstream head return `null` immediately without touching any upstream. + short-circuits for `eth_getLogs` range enforcement, `eth_chainId`, `trace_filter`, and + `eth_blockNumber` (answered from the served tip while `enforceHighestBlock` is on). +6. **Future-block short-circuit** — with `evm.shortCircuitFutureBlocks: true`, a numbered + `eth_getBlockByNumber` beyond every upstream's head returns `null` without touching any upstream. 7. **Method guard** — `eth_accounts` and `eth_sign` are hardcoded unsupported; stateful methods require exactly one targeted upstream. 8. **Network rate limiting** — network-level budget checked. @@ -241,9 +242,7 @@ primary control surface. | `evm.blockUnavailableDelayMultiplier` | `*float64` | `1.0` | Multiplies EMA-estimated block time to derive the dynamic retry delay for `ErrUpstreamBlockUnavailable`/`ErrEndpointMissingData`. Returns `0` before EMA warms up — falls back to `failsafe[*].retry.emptyResultDelay`. | | `evm.idempotentTransactionBroadcast` | `*bool` | nil (enabled) | When enabled, `eth_sendRawTransaction` converts "already known" to success and verifies "nonce too low" via `eth_getTransactionByHash`. Set to `false` to return raw upstream errors. | | `evm.emptyResultConfidence` | `AvailbilityConfidence` | `blockHead` | Confidence level for empty-result retries: `blockHead` = retry empties for blocks ≤ latest head; `finalizedBlock` = stricter, only retry for blocks ≤ finalized head. | -| `evm.servedTip.enabledFor` | `[]string` | `[]` | Tags using cluster-min mode; valid values: `"latest"`, `"finalized"`, `"safe"`. | -| `evm.servedTip.clusterDelta` | int64 | `0` (auto-derived, clamped [2,10]) | Max block gap to group upstreams into one cluster for served-tip computation. | -| `evm.servedTip.guaranteedMethods` | `[]string` | `[]` | Glob patterns for methods whose supporting-upstreams subset is used for cluster computation. | +| `evm.servedTip` | object | `nil` (quorum 2) | Tunes how `latest`/`finalized` is chosen: `quorum`, `maxRegressionBlocks`, `guaranteedMethods`, `guaranteedFor`. Keys: [Networks config](/config/projects/networks#config-schema); rule: [Block tracking](/reference/evm/block-tracking#served-tip). | | `evm.integrity.enforceHighestBlock` | `*bool` | `true` | **Deprecated** — migrate to `directiveDefaults.enforceHighestBlock`. | | `evm.integrity.enforceGetLogsBlockRange` | `*bool` | `true` | **Deprecated** — migrate to `directiveDefaults.enforceGetLogsBlockRange`. | | `evm.integrity.enforceNonNullTaggedBlocks` | `*bool` | `true` | **Deprecated** — migrate to `directiveDefaults.enforceNonNullTaggedBlocks`. | @@ -497,11 +496,10 @@ all waiting followers with a response ID patch per follower: | `erpc_network_static_response_served_total` | counter | project, network, category | Static response matched | | `erpc_upstream_wrong_empty_response_total` | counter | project, vendor, network, upstream, category, finality, user, agent_name | Upstream returned empty while another returned data | | `erpc_upstream_request_errors_total` | counter | (upstream error labels) | Upstream skip due to block availability check failure | -| `erpc_network_served_tip_block_number` | gauge | project, network, lane, axis | Served-tip block number computed for the network | -| `erpc_network_served_tip_lag_blocks` | gauge | project, network, lane, axis | Deliberate cushion below the freshest upstream tip | -| `erpc_network_served_tip_upstream_excluded_total` | counter | project, network, upstream, axis, reason | Upstream dropped from served-tip computation (velocity/outlier gate) | | `erpc_upstream_attempt_outcome_total` | counter | project, network, upstream, category, outcome, is_hedge, is_retry, finality | One increment per upstream attempt with its terminal outcome | +Served-tip and bench metrics: see the [metrics reference](/reference/metrics#area-3-block-number-and-tip-tracking). + Key OpenTelemetry spans: - `Network.Forward` — top-level; attributes: `cache.hit`, `multiplexed`, `failsafe.matched_method` diff --git a/docs/pages/use-cases/trust-the-data.mdx b/docs/pages/use-cases/trust-the-data.mdx index a1cda78e2..c07710d2c 100644 --- a/docs/pages/use-cases/trust-the-data.mdx +++ b/docs/pages/use-cases/trust-the-data.mdx @@ -133,9 +133,9 @@ tables and exact defaults live in each page's agent section. **Hook dispatch model.** Four hook entry-points in [`architecture/evm/hooks.go`](https://github.com/erpc/erpc/blob/main/architecture/evm/hooks.go): -- `HandleProjectPreForward` — runs at project layer before cache and upstream selection; handles `eth_blockNumber`, `eth_call`, `eth_chainId`, `eth_getLogs`, `trace_filter`/`arbtrace_filter`. -- `HandleNetworkPreForward` — runs after upstream selection, can short-circuit; handles `eth_getLogs`, `eth_chainId`, `trace_filter`/`arbtrace_filter`. -- `HandleNetworkPostForward` — called after response comes back at network layer; handles `eth_getBlockByNumber`, `eth_getLogs`, `eth_sendRawTransaction`, `trace_filter`/`arbtrace_filter`. +- `HandleProjectPreForward` — runs at project layer before cache and upstream selection; handles `eth_call`, `eth_estimateGas`, `eth_chainId`, `eth_getLogs`, `trace_filter`/`arbtrace_filter`. +- `HandleNetworkPreForward` — runs after upstream selection, can short-circuit; handles `eth_getLogs`, `eth_chainId`, `eth_blockNumber` (answered from the served tip), `trace_filter`/`arbtrace_filter`. +- `HandleNetworkPostForward` — called after response comes back at network layer; handles `eth_blockNumber` (pinned to the served tip), `eth_getBlockByNumber`, `eth_getLogs`, `eth_sendRawTransaction`, `trace_filter`/`arbtrace_filter`. - `HandleUpstreamPostForward` — richest hook, called per-upstream; checks `MarkEmptyAsErrorMethods`, applies per-method validation, marks unexpected empties as `ErrEndpointMissingData`. **Block tag normalization.** [`NormalizeHttpJsonRpc`](https://github.com/erpc/erpc/blob/main/architecture/evm/json_rpc.go#L92) runs on every incoming JSON-RPC request: caches the numeric block number on the request, and when the tag is `"latest"` or `"finalized"` and the network has a known head, replaces the tag with a concrete `0x…` hex. Tags `"safe"` and `"pending"` are intentionally passed through unchanged. The `X-ERPC-Skip-Interpolation` directive or `skipInterpolation` request directive suppresses mutations while still caching the numeric block number. diff --git a/docs/pages/why.mdx b/docs/pages/why.mdx index b8e7dc9e1..3497de5ac 100644 --- a/docs/pages/why.mdx +++ b/docs/pages/why.mdx @@ -21,15 +21,15 @@ The result: provider outages stop reaching your users, historical data is never The request pipeline runs in a fixed order. An HTTP request arrives at `erpc/http_server.go`, is parsed (batch requests fan out concurrently), and forwarded through `RequestProcessor.ProcessUnary` (`erpc/request_processor.go:35-67`). That function resolves the project, validates the JSON-RPC body, authenticates the consumer, resolves the network, applies `DirectiveDefaults`, and calls `project.Forward`. -**Project layer** (`erpc/projects.go`): acquires a project-level rate limit permit, fires `HandleProjectPreForward` (EVM hook for `eth_blockNumber`, `eth_call`, `eth_chainId`, `eth_getLogs`, `trace_filter` — may short-circuit with an early response), then delegates to `network.Forward`. +**Project layer** (`erpc/projects.go`): acquires a project-level rate limit permit, fires `HandleProjectPreForward` (EVM hook for `eth_call`, `eth_estimateGas`, `eth_chainId`, `eth_getLogs`, `trace_filter` — may short-circuit with an early response), then delegates to `network.Forward`. **Network layer** (`erpc/networks.go:931-1603`): in order — 1. Static response check — matched entries return immediately without touching any upstream. 2. Multiplexing — identical in-flight requests are deduplicated; followers wait for the leader's result. 3. Cache read — a non-null hit is returned immediately and closes the multiplexer. 4. Upstream ordering — the policy engine scores upstreams on latency, error rate, throttling, and block lag, returning them in score order. -5. EVM pre-forward hook — range splitting for `eth_getLogs`, shortcircuit for `eth_chainId`. -6. Future-block short-circuit — `eth_getBlockByNumber` requests for blocks beyond all upstream heads return a truthful `null` immediately. +5. EVM pre-forward hook — range splitting for `eth_getLogs`, shortcircuit for `eth_chainId`, and `eth_blockNumber` answered from the served tip. +6. Future-block short-circuit — with `evm.shortCircuitFutureBlocks: true`, `eth_getBlockByNumber` requests for blocks beyond all upstream heads return a truthful `null` immediately. 7. Block-tag normalization — `"latest"` and `"finalized"` are replaced with concrete hex block numbers before the request reaches any upstream. 8. Failsafe execution — the selected executor runs `timeout(retry(hedge(runUpstreamSweep)))`, or `timeout(consensus(retry(hedge(tryOneUpstream))))` when consensus is enabled. 9. Async cache write — on a successful non-null response, the result is written to cache in a background goroutine with a 10-second deadline using the application context (not the request context, so client disconnects do not abort the write). diff --git a/docs/pnpm-lock.yaml b/docs/pnpm-lock.yaml index 010ae01a9..c50e0a361 100644 --- a/docs/pnpm-lock.yaml +++ b/docs/pnpm-lock.yaml @@ -21,14 +21,14 @@ importers: specifier: ^2.1.1 version: 2.1.1 next: - specifier: ^15.5.25 - version: 15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + specifier: ^15.5.28 + version: 15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) nextra: specifier: ^2.13.4 - version: 2.13.4(next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + version: 2.13.4(next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) nextra-theme-docs: specifier: ^2.13.4 - version: 2.13.4(next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(nextra@2.13.4(next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + version: 2.13.4(next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(nextra@2.13.4(next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) prism-react-renderer: specifier: ^2.4.1 version: 2.4.1(react@18.3.1) @@ -895,57 +895,57 @@ packages: resolution: {integrity: sha512-jMxvwzkKzd3cXo2EB9GM2ic0eYo2rP/BS6gJt6HnWbsDO1O8GSD4k7o2Cpr2YERtMpGF/MGcDfsfj2EbQPtrXw==} engines: {node: '>= 10'} - '@next/env@15.5.25': - resolution: {integrity: sha512-42h1lLr07vl4gawALP1hsgRZjHB1xYa58JfUfHwr0f7jG/zhPakh5GHkADHXOC9ZxUvlQFOPIrp7s6qX4DezPQ==} + '@next/env@15.5.28': + resolution: {integrity: sha512-xxW5EBI8fMPuVTPBW7LL980IRbuDXlbkUAwp9DDhtb4Lnich3nJ13uYpvrX4VE74hWPf6AvHHAMmGCidcHjbfg==} - '@next/swc-darwin-arm64@15.5.25': - resolution: {integrity: sha512-w+RR0v/QuApnWEjRGm1z6gcObKwGMb5YPA7V3bzBEVSBpMFUXprer0tS27UxjUcEnqbhL7Zuzohej79B6rYmBg==} + '@next/swc-darwin-arm64@15.5.28': + resolution: {integrity: sha512-dPoQYXcfSIqx5o+TOXVXO7kONK5qbtNRDeiTfII0bT3puGIT6VF5e0QwmxNZfEZ5Zety5f3m9P/p4xqwfiLlPg==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] - '@next/swc-darwin-x64@15.5.25': - resolution: {integrity: sha512-QiGGBUSakt8S1H4Lt9Ehsh6Ja87axiBnQQgysOObvCbI7iUfJnRGntF1P64S4/ijuHFnSB8KLsEddkY3nN26uw==} + '@next/swc-darwin-x64@15.5.28': + resolution: {integrity: sha512-RgCJ7buBivTH+6sddqaVm+/v8kwMHlFkobvdt3DfETutws5HrxnToMIWSN9X3MZ3aAi7BbKrGZZWUDQLEycXqw==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] - '@next/swc-linux-arm64-gnu@15.5.25': - resolution: {integrity: sha512-ehLos/66zo0d/mJCU5u96a/VDcr01aaUrX0o/i16UdInxz8qPTCDSxGtjk/Lps1sIr1RJFdiX3hxc0fxJo+cPA==} + '@next/swc-linux-arm64-gnu@15.5.28': + resolution: {integrity: sha512-OHXQe13olE59aKe++zHavGcJPh5SLHXi6Anret3BBeq1+qhSuOJDNAli/EDDfxs6UTqA4Qddo4uyYry2slyjiQ==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] libc: [glibc] - '@next/swc-linux-arm64-musl@15.5.25': - resolution: {integrity: sha512-ZVMrqLiJ7DiChgmbkQwFtdhAnUkSH/4p7tB29QY+giATb0Q/XGHNRSKAb/B8XGDHRUaA67NepOW5W8u3ZRJBAA==} + '@next/swc-linux-arm64-musl@15.5.28': + resolution: {integrity: sha512-nTB7bB6vfn4ezCJNYP43ukt2o8L3+WxKF/sTcTxepdXsJh3SM9RhlU7wrZYbH7L4HO5M87dqDTwm8587vAkCiw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] libc: [musl] - '@next/swc-linux-x64-gnu@15.5.25': - resolution: {integrity: sha512-UOewtDGkTMJTiODrEdeLZ50yGb59xCZSriNpXkfPMxRRgwDkGc7i8mLWqV5076wEdb+Ca/XN7MhJyM3CupyNyQ==} + '@next/swc-linux-x64-gnu@15.5.28': + resolution: {integrity: sha512-WXCfVI32rW3xCusoUX1RpZKtyChWU2Zwe27JUDmM3KrXwcDNc9vV4zEq0GKFAURnuQ/Ouo/XqPvB0yh445XZCg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] libc: [glibc] - '@next/swc-linux-x64-musl@15.5.25': - resolution: {integrity: sha512-UBHwA8AhkCZgtRfU1aJpunuAJe/6gZv6jDESQe4p5MjTb5V0YEeJBWCdNqx15Vj3x+5jmauRfeMJSjfQj9HGFQ==} + '@next/swc-linux-x64-musl@15.5.28': + resolution: {integrity: sha512-Miq+NDb/HWsXuxseYxMM6fuEOQXgTziarkKIRpVKCn8W69cEfhQaQr+90pO+u4q2CbJR2wzXmUuCFyZaJs/Nng==} engines: {node: '>= 10'} cpu: [x64] os: [linux] libc: [musl] - '@next/swc-win32-arm64-msvc@15.5.25': - resolution: {integrity: sha512-QcFcPRr16djk5IqK5+e8O80eZfgWzIvVBXfitIq0tQ/uc+eyfdoZ0NmKc0cnbIJyfVwREapKuG97YcxWA9gcpA==} + '@next/swc-win32-arm64-msvc@15.5.28': + resolution: {integrity: sha512-5LR79lemgDlgxvG7MQJ+zHCkVcrUoQuO7CiIvq3I58vNW4goSOE2b+19Z6FujRWMouvni4gvudYFuFuNxolGWA==} engines: {node: '>= 10'} cpu: [arm64] os: [win32] - '@next/swc-win32-x64-msvc@15.5.25': - resolution: {integrity: sha512-zREeykps3ndWr9egJgvJKqVkkDuaw6Zrrg23cYBos0ygydFkAWYU4+PaPVwXzP1eAYQJe53ShSK45iDM529BOg==} + '@next/swc-win32-x64-msvc@15.5.28': + resolution: {integrity: sha512-GNipvXUYROSTf4PXEyH95FrWImIB5zdaK6uEOg0RP1/X2drnvoMwiBkJ0VWdzX7w3iiBm3y5g2TxkKDGVTOQcg==} engines: {node: '>= 10'} cpu: [x64] os: [win32] @@ -1205,11 +1205,8 @@ packages: resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} engines: {node: '>=10'} - caniuse-lite@1.0.30001809: - resolution: {integrity: sha512-xxWVywk6a6Arlk+hymeycyn/VgqEfLDxupvhH/xiY5SJ/18kmi9o6MiO320DCUzypORHLtvh0I4i04tUhCNHNQ==} - - caniuse-lite@1.0.30001810: - resolution: {integrity: sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==} + caniuse-lite@1.0.30001819: + resolution: {integrity: sha512-LFUqIIAwpzsIuevnKnsR9qnStBcgzPShTX61lBwN9uFkkf8o8zLiebEjbW8AYtZiF+1YKDWfdt/pWDtMuG8Xwg==} ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} @@ -2024,8 +2021,8 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - nanoid@3.3.18: - resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} + nanoid@3.3.20: + resolution: {integrity: sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -2050,8 +2047,8 @@ packages: react: '*' react-dom: '*' - next@15.5.25: - resolution: {integrity: sha512-OMWNulIIqKM2ykvC2qMjIt0IoavB4UB2SCs4iXJ6z6847FvyH8jBmBWcvrF5iuhTu8Przh20Fo/aoszIdqx4PA==} + next@15.5.28: + resolution: {integrity: sha512-JjhLBBY1PNnuwPtcZ7t8yxVeJVYvD2Jyp633J+gRCNbzSt7N0E+xqCf6uQiff3+3qmUkwcF/ksfSsK1xsZ76Tw==} engines: {node: ^18.18.0 || ^19.8.0 || >= 20.0.0} hasBin: true peerDependencies: @@ -2319,8 +2316,8 @@ packages: resolution: {integrity: sha512-aSbHV0DaBcr7u0PVHXzM6NbZNAtrr9sF6+Qfs9UUVG7Ll3jQ6hHi8F/xqIIcn2rvIVbr0v/2zyjSdwSV47AgLQ==} engines: {node: '>=12'} - source-map-js@1.2.1: - resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + source-map-js@1.2.2: + resolution: {integrity: sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==} engines: {node: '>=0.10.0'} source-map@0.7.4: @@ -3503,30 +3500,30 @@ snapshots: '@napi-rs/simple-git-win32-arm64-msvc': 0.1.19 '@napi-rs/simple-git-win32-x64-msvc': 0.1.19 - '@next/env@15.5.25': {} + '@next/env@15.5.28': {} - '@next/swc-darwin-arm64@15.5.25': + '@next/swc-darwin-arm64@15.5.28': optional: true - '@next/swc-darwin-x64@15.5.25': + '@next/swc-darwin-x64@15.5.28': optional: true - '@next/swc-linux-arm64-gnu@15.5.25': + '@next/swc-linux-arm64-gnu@15.5.28': optional: true - '@next/swc-linux-arm64-musl@15.5.25': + '@next/swc-linux-arm64-musl@15.5.28': optional: true - '@next/swc-linux-x64-gnu@15.5.25': + '@next/swc-linux-x64-gnu@15.5.28': optional: true - '@next/swc-linux-x64-musl@15.5.25': + '@next/swc-linux-x64-musl@15.5.28': optional: true - '@next/swc-win32-arm64-msvc@15.5.25': + '@next/swc-win32-arm64-msvc@15.5.28': optional: true - '@next/swc-win32-x64-msvc@15.5.25': + '@next/swc-win32-x64-msvc@15.5.28': optional: true '@popperjs/core@2.11.8': {} @@ -3781,7 +3778,7 @@ snapshots: browserslist@4.29.0: dependencies: baseline-browser-mapping: 2.11.24 - caniuse-lite: 1.0.30001810 + caniuse-lite: 1.0.30001819 electron-to-chromium: 1.5.430 node-releases: 2.0.55 update-browserslist-db: 1.3.3(browserslist@4.29.0) @@ -3790,9 +3787,7 @@ snapshots: camelcase@6.3.0: {} - caniuse-lite@1.0.30001809: {} - - caniuse-lite@1.0.30001810: {} + caniuse-lite@1.0.30001819: {} ccount@2.0.1: {} @@ -3869,12 +3864,12 @@ snapshots: css-tree@2.2.1: dependencies: mdn-data: 2.0.28 - source-map-js: 1.2.1 + source-map-js: 1.2.2 css-tree@2.3.1: dependencies: mdn-data: 2.0.30 - source-map-js: 1.2.1 + source-map-js: 1.2.2 css-what@6.1.0: {} @@ -4950,7 +4945,7 @@ snapshots: ms@2.1.3: {} - nanoid@3.3.18: {} + nanoid@3.3.20: {} next-mdx-remote@4.4.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: @@ -4963,43 +4958,43 @@ snapshots: transitivePeerDependencies: - supports-color - next-seo@6.8.0(next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + next-seo@6.8.0(next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: - next: 15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + next: 15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) react: 18.3.1 react-dom: 18.3.1(react@18.3.1) - next-themes@0.2.1(next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + next-themes@0.2.1(next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: - next: 15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + next: 15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) react: 18.3.1 react-dom: 18.3.1(react@18.3.1) - next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: - '@next/env': 15.5.25 + '@next/env': 15.5.28 '@swc/helpers': 0.5.15 - caniuse-lite: 1.0.30001809 + caniuse-lite: 1.0.30001819 postcss: 8.5.28 react: 18.3.1 react-dom: 18.3.1(react@18.3.1) styled-jsx: 5.1.6(@babel/core@7.27.7)(react@18.3.1) optionalDependencies: - '@next/swc-darwin-arm64': 15.5.25 - '@next/swc-darwin-x64': 15.5.25 - '@next/swc-linux-arm64-gnu': 15.5.25 - '@next/swc-linux-arm64-musl': 15.5.25 - '@next/swc-linux-x64-gnu': 15.5.25 - '@next/swc-linux-x64-musl': 15.5.25 - '@next/swc-win32-arm64-msvc': 15.5.25 - '@next/swc-win32-x64-msvc': 15.5.25 + '@next/swc-darwin-arm64': 15.5.28 + '@next/swc-darwin-x64': 15.5.28 + '@next/swc-linux-arm64-gnu': 15.5.28 + '@next/swc-linux-arm64-musl': 15.5.28 + '@next/swc-linux-x64-gnu': 15.5.28 + '@next/swc-linux-x64-musl': 15.5.28 + '@next/swc-win32-arm64-msvc': 15.5.28 + '@next/swc-win32-x64-msvc': 15.5.28 sharp: 0.35.5(@types/node@25.9.5) transitivePeerDependencies: - '@babel/core' - '@types/node' - babel-plugin-macros - nextra-theme-docs@2.13.4(next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(nextra@2.13.4(next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + nextra-theme-docs@2.13.4(next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(nextra@2.13.4(next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: '@headlessui/react': 1.7.19(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@popperjs/core': 2.11.8 @@ -5010,16 +5005,16 @@ snapshots: git-url-parse: 13.1.1 intersection-observer: 0.12.2 match-sorter: 6.3.4 - next: 15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - next-seo: 6.8.0(next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - next-themes: 0.2.1(next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - nextra: 2.13.4(next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + next: 15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + next-seo: 6.8.0(next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + next-themes: 0.2.1(next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + nextra: 2.13.4(next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) react: 18.3.1 react-dom: 18.3.1(react@18.3.1) scroll-into-view-if-needed: 3.1.0 zod: 3.25.67 - nextra@2.13.4(next@15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + nextra@2.13.4(next@15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: '@headlessui/react': 1.7.19(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@mdx-js/mdx': 2.3.0 @@ -5033,7 +5028,7 @@ snapshots: gray-matter: 4.0.3 katex: 0.16.22 lodash.get: 4.4.2 - next: 15.5.25(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + next: 15.5.28(@babel/core@7.27.7)(@types/node@25.9.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) next-mdx-remote: 4.4.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1) p-limit: 3.1.0 react: 18.3.1 @@ -5129,9 +5124,9 @@ snapshots: postcss@8.5.28: dependencies: - nanoid: 3.3.18 + nanoid: 3.3.20 picocolors: 1.1.1 - source-map-js: 1.2.1 + source-map-js: 1.2.2 prism-react-renderer@2.4.1(react@18.3.1): dependencies: @@ -5346,7 +5341,7 @@ snapshots: dependencies: is-plain-obj: 4.1.0 - source-map-js@1.2.1: {} + source-map-js@1.2.2: {} source-map@0.7.4: {} diff --git a/docs/public/llms.txt b/docs/public/llms.txt index 4e6ed5269..074e9bfcb 100644 --- a/docs/public/llms.txt +++ b/docs/public/llms.txt @@ -122,7 +122,7 @@ eRPC is a fault-tolerant EVM RPC proxy and permanent caching layer. The bootstra **TypeScript configs** are compiled in-process via esbuild (IIFE bundle) and evaluated with the sobek JS runtime, so closures and module-level helpers in selection-policy `evalFunc` functions are preserved natively — no function serialisation. -**No-config mode**: when `projects` is empty, a synthetic `main` project with public repository + Envio providers is injected, and `/evm/` resolves without a project prefix. +**No-config mode**: when `projects` is empty, a synthetic `main` project with public repository provider is injected, and `/evm/` resolves without a project prefix. ### Config schema @@ -266,8 +266,14 @@ links to its `.llms.txt` companion. - [Metrics reference](https://docs.erpc.cloud/reference/metrics.llms.txt) — Every observable signal eRPC emits — 141 Prometheus metrics across upstreams, cache, rate limiting, consensus, hedging, and more — ready to wire into your dashboards and alerts. - [gRPC & BDS streaming](https://docs.erpc.cloud/reference/grpc-bds.llms.txt) — Use typed protobuf APIs for block, transaction, and log lookups — eRPC routes, caches, and protects every gRPC call exactly like HTTP, with built-in deadlock defenses that keep stuck H2 streams from stalling your traffic. - [HTTP client & proxy pools](https://docs.erpc.cloud/reference/http-client.llms.txt) — eRPC keeps a single pre-warmed, high-throughput connection to every upstream — and can rotate traffic across a fleet of SOCKS5 or HTTP proxies with zero extra latency. -- [Lanes & concurrency](https://docs.erpc.cloud/reference/lanes.llms.txt) — Route a class of requests to a specific provider group and eRPC automatically maintains a separate block-tip counter for that group — eliminating "block not found" churn caused by cross-provider tip pollution. +- [Lanes & concurrency](https://docs.erpc.cloud/reference/lanes.llms.txt) — Route a class of requests to a specific provider group and eRPC computes that group's own block tip — eliminating "block not found" churn caused by cross-provider tip pollution. - [Simulator](https://docs.erpc.cloud/reference/simulator.llms.txt) — A local browser playground that runs a real eRPC instance against synthetic upstreams — explore routing, failsafe, and selection-policy behavior in seconds, no credentials needed. + +### Compare + +- [eRPC vs dshackle](https://docs.erpc.cloud/compare/erpc-vs-dshackle.llms.txt) — eRPC vs dshackle (emeraldpay) feature by feature, with a source link in every cell, and where dshackle is the better fit. +- [eRPC vs web3-proxy](https://docs.erpc.cloud/compare/erpc-vs-web3-proxy.llms.txt) — eRPC vs web3-proxy (llamanodes) feature by feature, with a source link in every cell, and where web3-proxy is the better fit. +- [eRPC vs dRPC NodeCore](https://docs.erpc.cloud/compare/erpc-vs-drpc-nodecore.llms.txt) — eRPC vs drpc nodecore feature by feature, with a source link in every cell, and where NodeCore is the better fit. - **Presets** - [DVN (LayerZero)](https://docs.erpc.cloud/presets/dvn-ready.llms.txt) — A drop-in eRPC config for DVN operators — unanimous consensus on the four source-chain verification methods closes the forged-log attack vector that hit KelpDAO. @@ -278,6 +284,9 @@ links to its `.llms.txt` companion. Convenient when an agent wants to iterate over every page rather than follow the hierarchy: +- [eRPC vs dRPC NodeCore](https://docs.erpc.cloud/compare/erpc-vs-drpc-nodecore.llms.txt) — eRPC vs drpc nodecore feature by feature, with a source link in every cell, and where NodeCore is the better fit. +- [eRPC vs dshackle](https://docs.erpc.cloud/compare/erpc-vs-dshackle.llms.txt) — eRPC vs dshackle (emeraldpay) feature by feature, with a source link in every cell, and where dshackle is the better fit. +- [eRPC vs web3-proxy](https://docs.erpc.cloud/compare/erpc-vs-web3-proxy.llms.txt) — eRPC vs web3-proxy (llamanodes) feature by feature, with a source link in every cell, and where web3-proxy is the better fit. - [Authentication](https://docs.erpc.cloud/config/auth.llms.txt) — Lock down every request with a token, JWT, wallet signature, or IP allowlist — and bind each identity to its own rate-limit budget. - [Storage drivers](https://docs.erpc.cloud/config/database/drivers.llms.txt) — Five interchangeable cache back-ends — memory, Redis, PostgreSQL, DynamoDB, and a read-only gRPC BDS connector — all behind one uniform interface, with optional per-operation failsafe policies that keep transient storage hiccups invisible to your upstreams. - [Cache policies](https://docs.erpc.cloud/config/database/evm-json-rpc-cache.llms.txt) — Stop paying for the same upstream call twice — eRPC caches every EVM JSON-RPC response by finality bucket, fans out reads in parallel, and rejects stale tip-of-chain data before it ever reaches your users. @@ -329,7 +338,7 @@ follow the hierarchy: - [Method Handlers](https://docs.erpc.cloud/reference/evm/method-handlers.llms.txt) — eRPC's per-method EVM hooks eliminate entire classes of provider inconsistency — stale blocks, duplicate broadcasts, range-exceeded traces — before your application ever sees them. - [gRPC & BDS streaming](https://docs.erpc.cloud/reference/grpc-bds.llms.txt) — Use typed protobuf APIs for block, transaction, and log lookups — eRPC routes, caches, and protects every gRPC call exactly like HTTP, with built-in deadlock defenses that keep stuck H2 streams from stalling your traffic. - [HTTP Client & Proxy Pools](https://docs.erpc.cloud/reference/http-client.llms.txt) — eRPC keeps a single pre-warmed, high-throughput connection to every upstream — and can rotate traffic across a fleet of SOCKS5 or HTTP proxies with zero extra latency. -- [Lanes & concurrency](https://docs.erpc.cloud/reference/lanes.llms.txt) — Route a class of requests to a specific provider group and eRPC automatically maintains a separate block-tip counter for that group — eliminating "block not found" churn caused by cross-provider tip pollution. +- [Lanes & concurrency](https://docs.erpc.cloud/reference/lanes.llms.txt) — Route a class of requests to a specific provider group and eRPC computes that group's own block tip — eliminating "block not found" churn caused by cross-provider tip pollution. - [Metrics reference](https://docs.erpc.cloud/reference/metrics.llms.txt) — Every observable signal eRPC emits — 141 Prometheus metrics across upstreams, cache, rate limiting, consensus, hedging, and more — ready to wire into your dashboards and alerts. - [Simulator](https://docs.erpc.cloud/reference/simulator.llms.txt) — A local browser playground that runs a real eRPC instance against synthetic upstreams — explore routing, failsafe, and selection-policy behavior in seconds, no credentials needed. - [Commitment & finality](https://docs.erpc.cloud/reference/svm/commitment.llms.txt) — Pin one Solana commitment level across every upstream so the cache and consensus key on identical data — eRPC injects it shape-aware per method, clamps it where a method refuses it, and never rewrites what your client asked for. diff --git a/erpc/admin_api_test.go b/erpc/admin_api_test.go new file mode 100644 index 000000000..067144560 --- /dev/null +++ b/erpc/admin_api_test.go @@ -0,0 +1,328 @@ +package erpc + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const adminApiSecret = "admin-api-secret" + +// adminApiFixture runs a full eRPC HTTP server whose project "test_project" +// authenticates consumers through a Redis-backed ("keys") and a memory-backed +// ("memkeys") database strategy, next to "open_project" without any auth. +type adminApiFixture struct { + baseURL string + redis *miniredis.Miniredis +} + +func newAdminApiFixture(t *testing.T, admin *common.AdminConfig) *adminApiFixture { + t.Helper() + util.ResetGock() + t.Cleanup(util.ResetGock) + util.SetupMocksForEvmStatePoller() + + redis := miniredis.RunT(t) + cfg := &common.Config{ + Server: &common.ServerConfig{MaxTimeout: common.Duration(5 * time.Second).Ptr()}, + Admin: admin, + Projects: []*common.ProjectConfig{ + { + Id: "test_project", + Networks: []*common.NetworkConfig{{ + Architecture: common.ArchitectureEvm, + Alias: "testnet", + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }}, + Upstreams: []*common.UpstreamConfig{{ + Id: "rpc1", Type: common.UpstreamTypeEvm, Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }}, + Auth: &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{ + {Type: common.AuthTypeDatabase, Database: &common.DatabaseStrategyConfig{ + Connector: &common.ConnectorConfig{Id: "keys", Driver: common.DriverRedis, Redis: &common.RedisConnectorConfig{Addr: redis.Addr()}}, + }}, + {Type: common.AuthTypeDatabase, Database: &common.DatabaseStrategyConfig{ + Connector: &common.ConnectorConfig{Id: "memkeys", Driver: common.DriverMemory, Memory: &common.MemoryConnectorConfig{MaxItems: 1000, MaxTotalSize: "1MB"}}, + }}, + }}, + }, + { + Id: "open_project", + Networks: []*common.NetworkConfig{{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }}, + }, + }, + RateLimiters: &common.RateLimiterConfig{}, + } + _, _, baseURL, shutdown, _ := createServerTestFixtures(cfg, t) + t.Cleanup(shutdown) + return &adminApiFixture{baseURL: baseURL, redis: redis} +} + +func adminSecretConfig() *common.AdminConfig { + return &common.AdminConfig{Auth: &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{ + {Type: common.AuthTypeSecret, Secret: &common.SecretStrategyConfig{Value: adminApiSecret}}, + }}} +} + +// call posts one admin JSON-RPC request and returns the HTTP status with the +// decoded envelope. +func (f *adminApiFixture) call(t *testing.T, secret, method, params string) (int, map[string]interface{}) { + t.Helper() + body := fmt.Sprintf(`{"jsonrpc":"2.0","id":7,"method":%q,"params":%s}`, method, params) + req, err := http.NewRequest(http.MethodPost, f.baseURL+"/admin", strings.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + if secret != "" { + req.Header.Set("X-ERPC-Secret-Token", secret) + } + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + var out map[string]interface{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&out)) + return resp.StatusCode, out +} + +func (f *adminApiFixture) result(t *testing.T, method, params string) map[string]interface{} { + t.Helper() + status, out := f.call(t, adminApiSecret, method, params) + require.Equal(t, http.StatusOK, status, "%v", out) + require.NotContains(t, out, "error", "%v", out) + assert.EqualValues(t, 7, out["id"]) + res, ok := out["result"].(map[string]interface{}) + require.True(t, ok, "result is an object: %v", out) + return res +} + +// errorMessage returns the JSON-RPC error message plus, for admin calls, the +// underlying cause the server exposes under error.data. +func (f *adminApiFixture) errorMessage(t *testing.T, wantStatus int, method, params string) string { + t.Helper() + status, out := f.call(t, adminApiSecret, method, params) + require.Equal(t, wantStatus, status, "%v", out) + e, ok := out["error"].(map[string]interface{}) + require.True(t, ok, "error expected: %v", out) + msg, _ := e["message"].(string) + if d, ok := e["data"]; ok { + b, err := json.Marshal(d) + require.NoError(t, err) + msg += " | " + string(b) + } + return msg +} + +func TestAdminApi_ReadOnlyMethods(t *testing.T) { + f := newAdminApiFixture(t, adminSecretConfig()) + + t.Run("config returns the loaded configuration", func(t *testing.T) { + res := f.result(t, "erpc_config", `[]`) + projects := res["projects"].([]interface{}) + require.Len(t, projects, 2) + assert.Equal(t, "test_project", projects[0].(map[string]interface{})["id"]) + }) + + t.Run("taxonomy lists projects, networks and upstreams", func(t *testing.T) { + res := f.result(t, "erpc_taxonomy", `[]`) + byId := map[string]map[string]interface{}{} + for _, p := range res["projects"].([]interface{}) { + pm := p.(map[string]interface{}) + byId[pm["id"].(string)] = pm + } + require.Contains(t, byId, "test_project") + networks := byId["test_project"]["networks"].([]interface{}) + require.Len(t, networks, 1) + nw := networks[0].(map[string]interface{}) + assert.Equal(t, "evm:123", nw["id"]) + assert.Equal(t, "testnet", nw["alias"]) + ups := nw["upstreams"].([]interface{}) + require.Len(t, ups, 1) + assert.Equal(t, "rpc1", ups[0].(map[string]interface{})["id"]) + }) + + t.Run("project returns config and upstream health", func(t *testing.T) { + res := f.result(t, "erpc_project", `["test_project"]`) + assert.Equal(t, "test_project", res["config"].(map[string]interface{})["id"]) + assert.Contains(t, res, "health") + }) + + cases := []struct { + name, method, params string + status int + msg string + }{ + {"project without params", "erpc_project", `[]`, http.StatusBadRequest, "project id (params[0]) is required"}, + {"project id not a string", "erpc_project", `[42]`, http.StatusBadRequest, "project id (params[0]) must be a string"}, + {"unknown project", "erpc_project", `["nope"]`, http.StatusNotFound, "nope"}, + {"unknown admin method", "erpc_doesNotExist", `[]`, http.StatusOK, "admin method erpc_doesNotExist is not supported"}, + {"cordon without params", "erpc_cordonUpstream", `[]`, http.StatusOK, "cordon admin: params is required"}, + {"cordon without upstream", "erpc_cordonUpstream", `[{"projectId":"test_project"}]`, http.StatusOK, "cordon admin: projectId and upstream are required"}, + {"cordon with malformed params", "erpc_cordonUpstream", `[{"projectId":5}]`, http.StatusOK, "cordon admin: invalid params"}, + {"cordon unknown upstream", "erpc_cordonUpstream", `[{"projectId":"test_project","upstream":"ghost"}]`, http.StatusOK, `cordon admin: upstream "ghost" not found in project "test_project"`}, + {"cordon unknown project", "erpc_uncordonUpstream", `[{"projectId":"nope","upstream":"rpc1"}]`, http.StatusNotFound, "nope"}, + {"list cordoned without project", "erpc_listCordoned", `[]`, http.StatusOK, "cordon admin: projectId is required"}, + {"list cordoned unknown project", "erpc_listCordoned", `[{"projectId":"nope"}]`, http.StatusNotFound, "nope"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Contains(t, f.errorMessage(t, tc.status, tc.method, tc.params), tc.msg) + }) + } + + t.Run("wrong secret is unauthorized", func(t *testing.T) { + status, out := f.call(t, "wrong", "erpc_config", `[]`) + assert.Equal(t, http.StatusUnauthorized, status) + assert.Contains(t, out, "error") + assert.NotContains(t, out, "result") + }) +} + +func TestAdminApi_ApiKeyLifecycle(t *testing.T) { + f := newAdminApiFixture(t, adminSecretConfig()) + const keyParams = `"projectId":"test_project","connectorId":"keys"` + + // The Redis connector initializes asynchronously; the first successful + // write proves it is ready. + require.Eventually(t, func() bool { + _, out := f.call(t, adminApiSecret, "erpc_addApiKey", + `[{`+keyParams+`,"apiKey":"key-1","userId":"alice","rateLimitBudget":"gold"}]`) + return out["error"] == nil + }, 5*time.Second, 50*time.Millisecond) + + res := f.result(t, "erpc_addApiKey", `[{`+keyParams+`,"apiKey":"key-2","userId":"bob","enabled":false}]`) + assert.Equal(t, map[string]interface{}{"success": true, "apiKey": "key-2", "userId": "bob"}, res) + + stored, err := f.redis.Get("key-1:alice") + require.NoError(t, err) + assert.JSONEq(t, `{"userId":"alice","enabled":true,"rateLimitBudget":"gold"}`, stored) + + listed := f.result(t, "erpc_listApiKeys", `[{`+keyParams+`,"limit":100,"paginationToken":"0"}]`) + assert.Equal(t, false, listed["hasMore"]) + assert.EqualValues(t, 2, listed["totalReturned"]) + keys := map[string]map[string]interface{}{} + for _, k := range listed["apiKeys"].([]interface{}) { + km := k.(map[string]interface{}) + keys[km["key"].(string)] = km + } + require.Contains(t, keys, "key-1") + require.Contains(t, keys, "key-2") + assert.Equal(t, "alice", keys["key-1"]["userId"]) + assert.Equal(t, true, keys["key-1"]["enabled"]) + assert.Equal(t, "gold", keys["key-1"]["rateLimitBudget"]) + assert.Equal(t, false, keys["key-2"]["enabled"]) + + // Records that are not JSON are skipped by the listing. + f.redis.Set("broken:entry", "not-json") + listed = f.result(t, "erpc_listApiKeys", `[{`+keyParams+`}]`) + assert.EqualValues(t, 2, listed["totalReturned"]) + + // Update and delete read the current record through the connector's + // wildcard lookup (apiKey, "*"); seed that record as the connector + // returns it. + f.redis.Set("key-3:*", `{"userId":"carol","enabled":true,"rateLimitBudget":"silver"}`) + res = f.result(t, "erpc_updateApiKey", `[{`+keyParams+`,"apiKey":"key-3","updates":{"enabled":false,"rateLimitBudget":null,"tier":"pro"}}]`) + assert.Equal(t, true, res["success"]) + assert.Equal(t, "key-3", res["apiKey"]) + updated, err := f.redis.Get("key-3:carol") + require.NoError(t, err) + assert.JSONEq(t, `{"userId":"carol","enabled":false,"tier":"pro"}`, updated, "null removes a field, values overwrite") + + res = f.result(t, "erpc_deleteApiKey", `[{`+keyParams+`,"apiKey":"key-3"}]`) + assert.Equal(t, map[string]interface{}{"success": true, "apiKey": "key-3", "userId": "carol"}, res) + assert.False(t, f.redis.Exists("key-3:carol")) + + f.redis.Set("key-4:*", `{"enabled":true}`) + f.redis.Set("key-5:*", `not-json`) + for _, method := range []string{"erpc_updateApiKey", "erpc_deleteApiKey"} { + extra := "" + if method == "erpc_updateApiKey" { + extra = `,"updates":{}` + } + assert.Contains(t, f.errorMessage(t, http.StatusOK, method, `[{`+keyParams+`,"apiKey":"key-4"`+extra+`}]`), "missing or invalid userId in current data", method) + assert.Contains(t, f.errorMessage(t, http.StatusOK, method, `[{`+keyParams+`,"apiKey":"key-5"`+extra+`}]`), "failed to parse current data", method) + assert.Contains(t, f.errorMessage(t, http.StatusOK, method, `[{`+keyParams+`,"apiKey":"absent"`+extra+`}]`), "failed to get current API key data", method) + } + + assert.Contains(t, + f.errorMessage(t, http.StatusOK, "erpc_listApiKeys", `[{"projectId":"test_project","connectorId":"memkeys"}]`), + "failed to list API keys") +} + +func TestAdminApi_ApiKeyParamValidation(t *testing.T) { + f := newAdminApiFixture(t, adminSecretConfig()) + + type tc struct { + name, params string + status int + msg string + } + common4 := func(method string) []tc { + return []tc{ + {"no params", `[]`, http.StatusBadRequest, "requires params"}, + {"param not an object", `["x"]`, http.StatusBadRequest, "first parameter must be an object"}, + {"missing projectId", `[{"connectorId":"keys"}]`, http.StatusBadRequest, "projectId is required and must be a string"}, + {"missing connectorId", `[{"projectId":"test_project"}]`, http.StatusBadRequest, "connectorId is required and must be a string"}, + } + } + cases := map[string][]tc{ + "erpc_addApiKey": append(common4("add"), + tc{"missing apiKey", `[{"projectId":"test_project","connectorId":"keys"}]`, http.StatusBadRequest, "apiKey is required and must be a string"}, + tc{"missing userId", `[{"projectId":"test_project","connectorId":"keys","apiKey":"k"}]`, http.StatusBadRequest, "userId is required and must be a string"}, + tc{"budget not a string", `[{"projectId":"test_project","connectorId":"keys","apiKey":"k","userId":"u","rateLimitBudget":1}]`, http.StatusBadRequest, "rateLimitBudget must be a string"}, + tc{"enabled not a bool", `[{"projectId":"test_project","connectorId":"keys","apiKey":"k","userId":"u","enabled":"yes"}]`, http.StatusBadRequest, "enabled must be a boolean"}, + tc{"unknown project", `[{"projectId":"nope","connectorId":"keys","apiKey":"k","userId":"u"}]`, http.StatusOK, "failed to find connector: project 'nope' not found"}, + tc{"project without auth", `[{"projectId":"open_project","connectorId":"keys","apiKey":"k","userId":"u"}]`, http.StatusOK, "project 'open_project' has no auth registry"}, + tc{"unknown connector", `[{"projectId":"test_project","connectorId":"ghost","apiKey":"k","userId":"u"}]`, http.StatusOK, "database connector with ID 'ghost' not found"}, + ), + "erpc_listApiKeys": append(common4("list"), + tc{"unknown connector", `[{"projectId":"test_project","connectorId":"ghost"}]`, http.StatusOK, "database connector with ID 'ghost' not found"}, + ), + "erpc_updateApiKey": append(common4("update"), + tc{"missing apiKey", `[{"projectId":"test_project","connectorId":"keys"}]`, http.StatusBadRequest, "apiKey is required and must be a string"}, + tc{"missing updates", `[{"projectId":"test_project","connectorId":"keys","apiKey":"k"}]`, http.StatusBadRequest, "updates is required and must be an object"}, + tc{"unknown connector", `[{"projectId":"test_project","connectorId":"ghost","apiKey":"k","updates":{}}]`, http.StatusOK, "database connector with ID 'ghost' not found"}, + ), + "erpc_deleteApiKey": append(common4("delete"), + tc{"missing apiKey", `[{"projectId":"test_project","connectorId":"keys"}]`, http.StatusBadRequest, "apiKey is required and must be a string"}, + tc{"unknown connector", `[{"projectId":"test_project","connectorId":"ghost","apiKey":"k"}]`, http.StatusOK, "database connector with ID 'ghost' not found"}, + ), + } + for method, list := range cases { + for _, c := range list { + t.Run(method+"/"+c.name, func(t *testing.T) { + assert.Contains(t, f.errorMessage(t, c.status, method, c.params), c.msg) + }) + } + } +} + +func TestAdminApi_MethodAllowAndDenyLists(t *testing.T) { + admin := adminSecretConfig() + admin.DenyMethods = []string{"erpc_*"} + admin.AllowMethods = []string{"erpc_taxonomy"} + f := newAdminApiFixture(t, admin) + + f.result(t, "erpc_taxonomy", `[]`) + assert.Equal(t, "method not supported: erpc_config", f.errorMessage(t, http.StatusOK, "erpc_config", `[]`)) +} + +func TestAdminApi_DisabledAdminRejectsRequests(t *testing.T) { + f := newAdminApiFixture(t, nil) + status, out := f.call(t, adminApiSecret, "erpc_config", `[]`) + assert.Contains(t, out, "error") + assert.NotContains(t, out, "result", "no configuration is leaked without admin config") + assert.Contains(t, fmt.Sprint(out["error"]), "admin auth not configured") + assert.Equal(t, http.StatusOK, status) +} diff --git a/erpc/auth_failure_metric_test.go b/erpc/auth_failure_metric_test.go new file mode 100644 index 000000000..a31117353 --- /dev/null +++ b/erpc/auth_failure_metric_test.go @@ -0,0 +1,50 @@ +package erpc + +import ( + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func init() { util.ConfigureTestLogger() } + +func authFailureCount(strategy, reason string) float64 { + return testutil.ToFloat64(telemetry.MetricAuthFailedTotal.WithLabelValues( + "n/a", "n/a", strategy, reason, "unknown", + )) +} + +func TestProcessErrorBody_RecordsClientFacingUnauthorized(t *testing.T) { + err := common.NewErrAuthUnauthorizedReason( + "jwt", + common.AuthUnauthorizedReasonTokenExpired, + ) + before := authFailureCount("jwt", "token_expired") + startedAt := time.Now() + logger := zerolog.Nop() + + processErrorBody(&logger, &startedAt, nil, err, &common.FALSE) + + assert.Equal(t, before+1, authFailureCount("jwt", "token_expired")) +} + +func TestGrpcStatusOf_RecordsClientFacingUnauthorized(t *testing.T) { + err := common.NewErrAuthUnauthorizedReason( + "jwt", + common.AuthUnauthorizedReasonInvalidSignature, + ) + before := authFailureCount("jwt", "invalid_signature") + + got := grpcStatusOf(err) + + assert.Equal(t, codes.Unauthenticated, status.Code(got)) + assert.Equal(t, before+1, authFailureCount("jwt", "invalid_signature")) +} diff --git a/erpc/config_analyzer_report_test.go b/erpc/config_analyzer_report_test.go new file mode 100644 index 000000000..a95df05a2 --- /dev/null +++ b/erpc/config_analyzer_report_test.go @@ -0,0 +1,631 @@ +package erpc + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// analyzerNode is a fake EVM node answering the probes the config analyzer +// sends: eth_chainId and eth_getBlockByNumber for "0x0", "latest", +// "finalized" and arbitrary hex heights. An empty field makes the matching +// probe fail with a JSON-RPC error. +type analyzerNode struct { + chainId string + genesis string + latest string + finalized string + // blockHash answers every other hex height. + blockHash string + + mu sync.Mutex + calls map[string]int +} + +func (n *analyzerNode) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var req struct { + ID interface{} `json:"id"` + Method string `json:"method"` + Params []interface{} `json:"params"` + } + if err := json.Unmarshal(body, &req); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + key := req.Method + if len(req.Params) > 0 { + key = fmt.Sprintf("%s(%v)", req.Method, req.Params[0]) + } + n.mu.Lock() + if n.calls == nil { + n.calls = map[string]int{} + } + n.calls[key]++ + n.mu.Unlock() + + var result interface{} + switch req.Method { + case "eth_chainId": + if n.chainId != "" { + result = n.chainId + } + case "eth_getBlockByNumber": + tag, _ := req.Params[0].(string) + switch tag { + case "0x0": + if n.genesis != "" { + result = map[string]interface{}{"number": "0x0", "hash": n.genesis} + } + case "latest": + if n.latest != "" { + result = map[string]interface{}{"number": n.latest, "hash": "0xlatest"} + } + case "finalized": + if n.finalized != "" { + result = map[string]interface{}{"number": n.finalized, "hash": "0xfinal"} + } + default: + if n.blockHash != "" { + result = map[string]interface{}{"number": tag, "hash": n.blockHash} + } + } + } + resp := map[string]interface{}{"jsonrpc": "2.0", "id": req.ID} + if result == nil { + resp["error"] = map[string]interface{}{"code": -32601, "message": "the method is not available"} + } else { + resp["result"] = result + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) +} + +func (n *analyzerNode) callCount(key string) int { + n.mu.Lock() + defer n.mu.Unlock() + return n.calls[key] +} + +func startAnalyzerNode(t *testing.T, n *analyzerNode) string { + t.Helper() + srv := httptest.NewServer(n) + t.Cleanup(srv.Close) + return srv.URL +} + +func analyzerUpstream(id, endpoint string, chainId int64) *common.UpstreamConfig { + return &common.UpstreamConfig{ + Id: id, + Type: common.UpstreamTypeEvm, + Endpoint: endpoint, + Evm: &common.EvmUpstreamConfig{ChainId: chainId}, + } +} + +func analyzerConfig(upstreams ...*common.UpstreamConfig) *common.Config { + return &common.Config{ + Metrics: &common.MetricsConfig{}, + Projects: []*common.ProjectConfig{{ + Id: "main", + Networks: []*common.NetworkConfig{{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 1}, + Failsafe: []*common.FailsafeConfig{{MatchMethod: "*"}}, + }}, + Upstreams: upstreams, + }}, + } +} + +// findingsWith returns the findings that contain every given fragment. +func findingsWith(findings []string, fragments ...string) []string { + var out []string + for _, f := range findings { + all := true + for _, frag := range fragments { + if !strings.Contains(f, frag) { + all = false + break + } + } + if all { + out = append(out, f) + } + } + return out +} + +func resetGockForAnalyzer(t *testing.T) { + t.Helper() + util.ResetGock() + t.Cleanup(util.ResetGock) +} + +func TestConfigAnalyzerReport_StaticFindings(t *testing.T) { + enabled := true + cfg := &common.Config{ + Metrics: &common.MetricsConfig{HistogramBuckets: "0.1,not-a-number"}, + RateLimiters: &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{ + {Id: "project-budget", Rules: []*common.RateLimitRuleConfig{{Method: "*"}, {Method: "eth_call"}}}, + {Id: "upstream-budget"}, + {Id: "tuned-budget"}, + {Id: "provider-budget"}, + {Id: "auth-budget"}, + {Id: "admin-budget"}, + {Id: "unused-budget"}, + }}, + Admin: &common.AdminConfig{Auth: &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{ + {Type: common.AuthTypeSecret, RateLimitBudget: "admin-budget"}, + }}}, + Projects: []*common.ProjectConfig{ + { + Id: "alpha", + RateLimitBudget: "project-budget", + Networks: []*common.NetworkConfig{ + {Architecture: common.ArchitectureEvm, Evm: &common.EvmNetworkConfig{ChainId: 1}, Alias: "mainnet"}, + {Architecture: common.ArchitectureEvm, Evm: &common.EvmNetworkConfig{ChainId: 10}, Failsafe: []*common.FailsafeConfig{{MatchMethod: "*"}}}, + }, + Upstreams: []*common.UpstreamConfig{ + {Id: "on-10", Endpoint: " ", Evm: &common.EvmUpstreamConfig{ChainId: 10}, RateLimitBudget: "upstream-budget"}, + {Id: "unmatched-chain", Endpoint: "", Evm: &common.EvmUpstreamConfig{ChainId: 999}}, + {Id: "no-chain", Endpoint: "", RateLimitBudget: "tuned-budget", RateLimitAutoTune: &common.RateLimitAutoTuneConfig{Enabled: &enabled}}, + }, + Providers: []*common.ProviderConfig{{ + Id: "prov", + Overrides: map[string]*common.UpstreamConfig{"*": {RateLimitBudget: "provider-budget"}}, + }}, + Auth: &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{ + {Type: common.AuthTypeSecret, RateLimitBudget: "auth-budget"}, + }}, + }, + { + Id: "beta", + Upstreams: []*common.UpstreamConfig{ + {Id: "lonely", Endpoint: ""}, + }, + }, + }, + } + + report := GenerateValidationReport(context.Background(), cfg) + + assert.Equal(t, ValidationTotals{ProjectsTotal: 2, NetworksTotal: 2, UpstreamsTotal: 4, RateLimitBudgetsTotal: 7}, report.Resources.Totals) + require.Len(t, report.Resources.Tree.Projects, 2) + alpha := report.Resources.Tree.Projects[0] + require.Len(t, alpha.Networks, 2) + assert.Equal(t, NetworkNode{Id: "evm:1", Alias: "mainnet", Upstreams: []UpstreamNode{{Id: "unmatched-chain"}, {Id: "no-chain"}}, chainId: 1}, alpha.Networks[0], + "upstreams with an unknown chain or no chain attach to the first network") + assert.Equal(t, []UpstreamNode{{Id: "on-10"}}, alpha.Networks[1].Upstreams, "upstream attaches to the network with its chain id") + assert.Empty(t, report.Resources.Tree.Projects[1].Networks, "a project without networks has no place to attach upstreams") + assert.Equal(t, RateLimitBudgetNode{Id: "project-budget", RulesCount: 2}, report.Resources.Tree.RateLimiters.Budgets[0]) + + assert.Equal(t, []string{"orphan rateLimit budget 'unused-budget'"}, findingsWith(report.Notices, "orphan")) + assert.Len(t, findingsWith(report.Notices, "upstream=on-10", "auto-tune disabled"), 1) + assert.Empty(t, findingsWith(report.Notices, "upstream=no-chain", "auto-tune disabled"), "auto-tune enabled is not reported") + assert.Equal(t, []string{"project=alpha network=evm/1 has no failsafe policies (defaults and per-network empty)"}, findingsWith(report.Notices, "no failsafe")) + + assert.Len(t, findingsWith(report.Errors, "invalid metrics histogramBuckets"), 1) + for _, id := range []string{"on-10", "unmatched-chain", "no-chain"} { + assert.Len(t, findingsWith(report.Errors, "project 'alpha'", "upstream '"+id+"' has an empty endpoint URL"), 1, id) + } + assert.Len(t, findingsWith(report.Errors, "project 'beta'", "upstream 'lonely' has an empty endpoint URL"), 1) + assert.Empty(t, report.Warnings) +} + +func TestConfigAnalyzerReport_NetworkDefaultsFailsafeSuppressesNotice(t *testing.T) { + cfg := &common.Config{ + Metrics: &common.MetricsConfig{}, + Projects: []*common.ProjectConfig{{ + Id: "main", + NetworkDefaults: &common.NetworkDefaults{Failsafe: []*common.FailsafeConfig{{MatchMethod: "*"}}}, + Networks: []*common.NetworkConfig{{Architecture: common.ArchitectureEvm, Evm: &common.EvmNetworkConfig{ChainId: 1}}}, + }}, + } + report := GenerateValidationReport(context.Background(), cfg) + assert.Empty(t, findingsWith(report.Notices, "no failsafe")) + assert.Empty(t, report.Errors) + assert.Empty(t, report.Warnings) +} + +func TestConfigAnalyzerReport_ProxyPoolErrorSkipsProject(t *testing.T) { + cfg := analyzerConfig(analyzerUpstream("a", "", 1)) + cfg.ProxyPools = []*common.ProxyPoolConfig{{ID: "empty"}} + + report := GenerateValidationReport(context.Background(), cfg) + + require.Len(t, report.Errors, 1) + assert.Contains(t, report.Errors[0], "project=main failed to create proxy pool registry") + assert.Contains(t, report.Errors[0], "no proxy URLs defined for pool 'empty'") +} + +func TestConfigAnalyzerReport_HealthyUpstreamsProduceNoFindings(t *testing.T) { + resetGockForAnalyzer(t) + healthy := func() *analyzerNode { + return &analyzerNode{chainId: "0x1", genesis: "0xGEN", latest: "0x2000", finalized: "0x1000", blockHash: "0xB"} + } + nodeA, nodeB := healthy(), healthy() + nodeB.genesis = "0xgen" // hash comparison is case-insensitive + cfg := analyzerConfig( + analyzerUpstream("a", startAnalyzerNode(t, nodeA), 1), + analyzerUpstream("b", startAnalyzerNode(t, nodeB), 1), + ) + + report := GenerateValidationReport(context.Background(), cfg) + + assert.Empty(t, report.Errors) + assert.Empty(t, report.Warnings) + // finalized (0x1000) minus the 64-block safety margin + assert.Equal(t, 1, nodeA.callCount("eth_getBlockByNumber(0xfc0)")) + assert.Equal(t, 1, nodeB.callCount("eth_getBlockByNumber(0xfc0)")) +} + +func TestConfigAnalyzerReport_StrongMajorityMismatchIsError(t *testing.T) { + resetGockForAnalyzer(t) + good := func() *analyzerNode { + return &analyzerNode{chainId: "0x1", genesis: "0xgood", latest: "0x2000", finalized: "0x1000", blockHash: "0xblock"} + } + bad := good() + bad.genesis = "0xfork" + bad.blockHash = "0xforkblock" + wrongChain := good() + wrongChain.chainId = "0x5" + cfg := analyzerConfig( + analyzerUpstream("good1", startAnalyzerNode(t, good()), 1), + analyzerUpstream("good2", startAnalyzerNode(t, good()), 1), + analyzerUpstream("bad", startAnalyzerNode(t, bad), 1), + analyzerUpstream("wrong-chain", startAnalyzerNode(t, wrongChain), 1), + ) + + report := GenerateValidationReport(context.Background(), cfg) + + assert.ElementsMatch(t, []string{ + "project=main upstream=bad chain=1 genesis hash mismatch (majority 2/3 agree on 0xgood) got=0xfork", + "project=main upstream=bad chain=1 block=4032 hash mismatch (majority 2/3 agree on 0xblock) got=0xforkblock", + "In project 'main', upstream 'wrong-chain' reported chain ID 5 but the configuration expects 1.", + }, report.Errors) + assert.Empty(t, report.Warnings) +} + +func TestConfigAnalyzerReport_TwoWayMismatchIsWarning(t *testing.T) { + resetGockForAnalyzer(t) + // No finalized support: the comparison falls back to latest-1024, using + // the lowest latest height across upstreams. + a := &analyzerNode{chainId: "0x1", genesis: "0xaaa", latest: "0x900", blockHash: "0xha"} + b := &analyzerNode{chainId: "0x1", genesis: "0xbbb", latest: "0x800", blockHash: "0xhb"} + cfg := analyzerConfig( + analyzerUpstream("a", startAnalyzerNode(t, a), 1), + analyzerUpstream("b", startAnalyzerNode(t, b), 1), + ) + + report := GenerateValidationReport(context.Background(), cfg) + + assert.Empty(t, report.Errors) + assert.Len(t, findingsWith(report.Warnings, "genesis hash differs from other upstream(s)", "only 2 upstreams compared"), 1, + "one of the two disagreeing upstreams is reported: %v", report.Warnings) + assert.Len(t, findingsWith(report.Warnings, "block=1024 hash differs from other upstream(s)", "only 2 upstreams compared"), 1, + "%v", report.Warnings) + assert.Equal(t, 1, a.callCount("eth_getBlockByNumber(0x400)")) +} + +func TestConfigAnalyzerReport_YoungChainSkipsBlockComparison(t *testing.T) { + resetGockForAnalyzer(t) + cases := []struct { + name string + node func() *analyzerNode + }{ + {"latest below 1024", func() *analyzerNode { + return &analyzerNode{chainId: "0x1", genesis: "0xg", latest: "0x10", blockHash: "0xh"} + }}, + {"finalized below 64", func() *analyzerNode { + return &analyzerNode{chainId: "0x1", genesis: "0xg", latest: "0x10", finalized: "0x20", blockHash: "0xh"} + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + a, b := tc.node(), tc.node() + cfg := analyzerConfig( + analyzerUpstream("a", startAnalyzerNode(t, a), 1), + analyzerUpstream("b", startAnalyzerNode(t, b), 1), + ) + report := GenerateValidationReport(context.Background(), cfg) + assert.Empty(t, report.Errors) + assert.Empty(t, report.Warnings) + for _, n := range []*analyzerNode{a, b} { + n.mu.Lock() + for k := range n.calls { + assert.Contains(t, []string{"eth_chainId", "eth_getBlockByNumber(0x0)", "eth_getBlockByNumber(latest)", "eth_getBlockByNumber(finalized)"}, k, + "no historical block probe expected") + } + n.mu.Unlock() + } + }) + } +} + +func TestConfigAnalyzerReport_FullNodeSkipsGenesisAndNonEvmIsNotProbed(t *testing.T) { + resetGockForAnalyzer(t) + full := &analyzerNode{chainId: "0x1", latest: "0x10", blockHash: "0xh"} + svm := &analyzerNode{} + fullCfg := analyzerUpstream("full", startAnalyzerNode(t, full), 1) + fullCfg.Evm.NodeType = common.EvmNodeTypeFull + cfg := analyzerConfig( + fullCfg, + &common.UpstreamConfig{Id: "sol", Type: common.UpstreamTypeSvm, Endpoint: startAnalyzerNode(t, svm), Svm: &common.SvmUpstreamConfig{Chain: "mainnet-beta"}}, + ) + + report := GenerateValidationReport(context.Background(), cfg) + + assert.Empty(t, report.Errors) + assert.Empty(t, report.Warnings) + assert.Equal(t, []string{"project=main upstream=full skipped genesis hash check (full node or maxAvailableRecentBlocks configured)"}, + findingsWith(report.Notices, "skipped genesis")) + assert.Zero(t, full.callCount("eth_getBlockByNumber(0x0)")) + svm.mu.Lock() + assert.Empty(t, svm.calls, "non-EVM upstreams receive no EVM probes") + svm.mu.Unlock() +} + +func TestConfigAnalyzerReport_ZeroChainIdIsWarned(t *testing.T) { + resetGockForAnalyzer(t) + n := &analyzerNode{chainId: "0x0", genesis: "0xg", latest: "0x10"} + cfg := analyzerConfig(analyzerUpstream("zero", startAnalyzerNode(t, n), 0)) + + report := GenerateValidationReport(context.Background(), cfg) + + assert.Equal(t, []string{"In project 'main', upstream 'zero' returned chain ID 0 from eth_chainId, which is invalid."}, report.Warnings) + assert.Empty(t, report.Errors) +} + +// Upstreams whose chain id cannot be fetched are grouped by their configured +// chain id but never cross-compared. +func TestConfigAnalyzerReport_UnknownChainIdSkipsComparison(t *testing.T) { + resetGockForAnalyzer(t) + a := &analyzerNode{genesis: "0xaaa", latest: "0x5000"} + b := &analyzerNode{genesis: "0xbbb", latest: "0x5000"} + cfg := analyzerConfig( + analyzerUpstream("a", startAnalyzerNode(t, a), 1), + analyzerUpstream("b", startAnalyzerNode(t, b), 1), + ) + + report := GenerateValidationReport(context.Background(), cfg) + + assert.Empty(t, report.Errors, "differing genesis hashes on unknown chains are not errors") + assert.Len(t, findingsWith(report.Warnings, "upstream 'a' could not fetch chain ID via eth_chainId"), 1, "%v", report.Warnings) + assert.Len(t, findingsWith(report.Warnings, "upstream 'b' could not fetch chain ID via eth_chainId"), 1, "%v", report.Warnings) + assert.Equal(t, []string{"chain=cfg:1 project=main skipping block hash comparison (chain ID unknown for all upstreams)"}, + findingsWith(report.Warnings, "skipping block hash comparison")) + assert.Len(t, report.Warnings, 3) +} + +func TestConfigAnalyzerReport_MissingBlocksAreWarned(t *testing.T) { + resetGockForAnalyzer(t) + t.Run("no genesis and no heights", func(t *testing.T) { + a := &analyzerNode{chainId: "0x1"} + b := &analyzerNode{chainId: "0x1"} + cfg := analyzerConfig( + analyzerUpstream("a", startAnalyzerNode(t, a), 1), + analyzerUpstream("b", startAnalyzerNode(t, b), 1), + ) + report := GenerateValidationReport(context.Background(), cfg) + assert.Empty(t, report.Errors) + assert.ElementsMatch(t, []string{ + "chain=1 project=main no upstream returned genesis block", + "chain=1 project=main could not resolve latest or finalized block from any upstream", + }, report.Warnings) + }) + t.Run("one upstream lacks genesis and historical block", func(t *testing.T) { + a := &analyzerNode{chainId: "0x1", genesis: "0xg", finalized: "0x1000", blockHash: "0xh"} + b := &analyzerNode{chainId: "0x1", finalized: "0x1000"} + cfg := analyzerConfig( + analyzerUpstream("a", startAnalyzerNode(t, a), 1), + analyzerUpstream("b", startAnalyzerNode(t, b), 1), + ) + report := GenerateValidationReport(context.Background(), cfg) + assert.Empty(t, report.Errors) + assert.Len(t, report.Warnings, 2, "%v", report.Warnings) + assert.Len(t, findingsWith(report.Warnings, "project=main upstream=b chain=1 could not fetch genesis block:"), 1) + assert.Len(t, findingsWith(report.Warnings, "project=main upstream=b chain=1 could not fetch block 4032:"), 1) + }) + t.Run("no upstream has the historical block", func(t *testing.T) { + a := &analyzerNode{chainId: "0x1", genesis: "0xg", finalized: "0x1000"} + b := &analyzerNode{chainId: "0x1", genesis: "0xg", finalized: "0x1000"} + cfg := analyzerConfig( + analyzerUpstream("a", startAnalyzerNode(t, a), 1), + analyzerUpstream("b", startAnalyzerNode(t, b), 1), + ) + report := GenerateValidationReport(context.Background(), cfg) + assert.Empty(t, report.Errors) + assert.Equal(t, []string{"chain=1 project=main could not fetch block 4032 from any upstream"}, report.Warnings) + }) +} + +func TestConfigAnalyzerReport_Renderers(t *testing.T) { + r := &ValidationReport{ + Errors: []string{"broken"}, + Warnings: []string{"careful"}, + Notices: []string{"fyi"}, + Resources: ValidationResources{ + Totals: ValidationTotals{ProjectsTotal: 1, NetworksTotal: 1, UpstreamsTotal: 2, RateLimitBudgetsTotal: 1}, + Tree: ValidationResourcesTree{ + Projects: []ProjectNode{{Id: "main", Networks: []NetworkNode{ + {Id: "evm:1", Alias: "mainnet", Upstreams: []UpstreamNode{{Id: "u1"}}}, + {Id: "evm:10", Upstreams: []UpstreamNode{{Id: "u2"}}}, + }}}, + RateLimiters: RateLimitersNode{Budgets: []RateLimitBudgetNode{{Id: "b1", RulesCount: 3}}}, + }, + }, + } + + md := RenderValidationReportMarkdown(r) + for _, want := range []string{ + "### Errors\n- ❌ broken\n", + "Warnings (1)", "- ⚠️ careful\n", + "Notices (1)", "- 💡 fyi\n", + "- projectsTotal: 1\n", "- upstreamsTotal: 2\n", "- rateLimitBudgetsTotal: 1\n", + "project main\n network evm:1 (mainnet)\n upstream u1\n network evm:10\n upstream u2\n", + "rateLimiters\n budget b1 rules=3\n", + } { + assert.Contains(t, md, want) + } + + empty := RenderValidationReportMarkdown(&ValidationReport{}) + assert.NotContains(t, empty, "Errors") + assert.NotContains(t, empty, "Warnings") + assert.NotContains(t, empty, "Notices") + assert.Contains(t, empty, "- projectsTotal: 0\n") + + compact, err := RenderValidationReportJSON(r, false) + require.NoError(t, err) + pretty, err := RenderValidationReportJSON(r, true) + require.NoError(t, err) + assert.NotContains(t, compact, "\n") + assert.Contains(t, pretty, "\n \"errors\": [") + var a, b map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(compact), &a)) + require.NoError(t, json.Unmarshal([]byte(pretty), &b)) + assert.Equal(t, a, b) + assert.Equal(t, "mainnet", a["resources"].(map[string]interface{})["tree"].(map[string]interface{})["projects"].([]interface{})[0].(map[string]interface{})["networks"].([]interface{})[0].(map[string]interface{})["alias"]) +} + +func TestConfigAnalyzerStats_UsageAndOrphans(t *testing.T) { + cfg := &common.Config{ + RateLimiters: &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{ + {Id: "p"}, {Id: "u"}, {Id: "n"}, {Id: "auth"}, {Id: "admin"}, {Id: "orphan"}, + }}, + Admin: &common.AdminConfig{Auth: &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{ + {Type: common.AuthTypeSecret, RateLimitBudget: "admin"}, + }}}, + Projects: []*common.ProjectConfig{{ + Id: "main", + RateLimitBudget: "p", + Upstreams: []*common.UpstreamConfig{ + {Id: "named", RateLimitBudget: "u"}, + {Type: common.UpstreamTypeEvm, VendorName: "alchemy"}, + {Type: common.UpstreamTypeEvm}, + }, + Networks: []*common.NetworkConfig{{Architecture: common.ArchitectureEvm, RateLimitBudget: "n"}}, + Auth: &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{ + {Type: common.AuthTypeJwt, RateLimitBudget: "auth"}, + }}, + }}, + } + + stats := calculateConfigStats(cfg) + + assert.Equal(t, 1, stats.ProjectCount) + assert.Equal(t, 6, stats.RateLimits.TotalBudgets) + assert.Equal(t, []string{"orphan"}, stats.RateLimits.OrphanBudgets) + assert.Equal(t, map[string][]string{ + "p": {"project:main"}, + "u": {"project:main:upstream:named"}, + "n": {"project:main:network:evm"}, + "auth": {"project:main:auth:jwt"}, + "admin": {"admin:auth:secret"}, + }, stats.RateLimits.UsageBySources) + require.Len(t, stats.Projects, 1) + assert.Equal(t, ProjectStats{ + ProjectID: "main", + UpstreamCount: 3, + Upstreams: []string{"named", "evm-alchemy", "evm-unknown"}, + NetworkCount: 1, + Networks: []string{"evm"}, + }, stats.Projects[0]) +} + +func TestConfigAnalyzerIsLocalEndpoint(t *testing.T) { + cases := []struct { + endpoint string + local bool + wantErr bool + }{ + {"http://localhost:8545", true, false}, + {"http://127.0.0.1", true, false}, + {"http://[::1]:8545", true, false}, + {"http://node.default.svc.cluster.local", true, false}, + {"http://10.1.2.3", true, false}, + {"http://192.168.0.5:80", true, false}, + {"http://127.0.0.2", true, false}, + {"https://rpc.example.com", false, false}, + {"http://8.8.8.8", false, false}, + {"wss://", false, false}, + {"http://bad host/%zz", false, true}, + } + for _, tc := range cases { + t.Run(tc.endpoint, func(t *testing.T) { + local, err := isLocalEndpoint(tc.endpoint) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.local, local) + }) + } +} + +func TestConfigAnalyzerAnalyseConfig(t *testing.T) { + resetGockForAnalyzer(t) + logger := zerolog.New(io.Discard) + nodeURL := func(chainId string) string { + return startAnalyzerNode(t, &analyzerNode{chainId: chainId}) + } + cases := []struct { + name string + mutate func(cfg *common.Config) + ignoreEnv string + wantErr string + }{ + {name: "upstream without chain id is skipped", mutate: func(cfg *common.Config) { + cfg.Projects[0].Upstreams = []*common.UpstreamConfig{{Id: "a", Endpoint: "http://unreachable.invalid"}} + }}, + {name: "empty endpoint", wantErr: `upstream endpoint is empty for project: "main" and upstream id: "a"`, mutate: func(cfg *common.Config) { + cfg.Projects[0].Upstreams = []*common.UpstreamConfig{{Id: "a"}} + }}, + {name: "invalid histogram buckets", wantErr: "failed to set histogram buckets", mutate: func(cfg *common.Config) { + cfg.Metrics.HistogramBuckets = "x" + }}, + {name: "proxy pool without urls", wantErr: `failed to create proxy pool registry for project: "main"`, mutate: func(cfg *common.Config) { + cfg.ProxyPools = []*common.ProxyPoolConfig{{ID: "p"}} + }}, + {name: "chain id unavailable", wantErr: `failed to get chain id for project: "main" and upstream id: "a"`, mutate: func(cfg *common.Config) { + cfg.Projects[0].Upstreams = []*common.UpstreamConfig{analyzerUpstream("a", nodeURL(""), 1)} + }}, + {name: "local endpoint skipped when ignored", ignoreEnv: "true", mutate: func(cfg *common.Config) { + cfg.Projects[0].Upstreams = []*common.UpstreamConfig{analyzerUpstream("a", nodeURL("0x5"), 1)} + }}, + {name: "non-http endpoint skipped when ignored", ignoreEnv: "true", mutate: func(cfg *common.Config) { + cfg.Projects[0].Upstreams = []*common.UpstreamConfig{analyzerUpstream("a", "alchemy://key", 1)} + }}, + {name: "unparseable endpoint when ignoring locals", ignoreEnv: "true", wantErr: `invalid endpoint URL for project "main" upstream "a"`, mutate: func(cfg *common.Config) { + cfg.Projects[0].Upstreams = []*common.UpstreamConfig{analyzerUpstream("a", "http://bad host/%zz", 1)} + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("ERPC_IGNORE_LOCAL_ENDPOINT_VALIDATION", tc.ignoreEnv) + cfg := analyzerConfig() + cfg.RateLimiters = &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{{Id: "unused"}}} + tc.mutate(cfg) + err := AnalyseConfig(context.Background(), cfg, logger) + if tc.wantErr == "" { + require.NoError(t, err) + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } +} diff --git a/erpc/grpc_server.go b/erpc/grpc_server.go index a8daa9af7..8c893ca60 100644 --- a/erpc/grpc_server.go +++ b/erpc/grpc_server.go @@ -504,7 +504,10 @@ func grpcStatusOf(err error) error { switch { case common.HasErrorCode(err, common.ErrCodeEndpointUnsupported): return status.Error(codes.Unimplemented, err.Error()) - case common.HasErrorCode(err, common.ErrCodeEndpointUnauthorized, common.ErrCodeAuthUnauthorized): + case common.HasErrorCode(err, common.ErrCodeAuthUnauthorized): + auth.RecordAuthFailed(nil, err) + return status.Error(codes.Unauthenticated, err.Error()) + case common.HasErrorCode(err, common.ErrCodeEndpointUnauthorized): return status.Error(codes.Unauthenticated, err.Error()) case common.HasErrorCode(err, common.ErrCodeEndpointRequestTimeout, common.ErrCodeNetworkRequestTimeout, common.ErrCodeFailsafeTimeoutExceeded) || errors.Is(err, context.DeadlineExceeded): return status.Error(codes.DeadlineExceeded, err.Error()) diff --git a/erpc/grpc_server_behavior_test.go b/erpc/grpc_server_behavior_test.go new file mode 100644 index 000000000..b848418b8 --- /dev/null +++ b/erpc/grpc_server_behavior_test.go @@ -0,0 +1,971 @@ +package erpc + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/blockchain-data-standards/manifesto/evm" + "github.com/bytedance/sonic" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +// grpcBehaviorHead is the latest (and finalized) block the mocked upstream reports. +const grpcBehaviorHead = 0x100 + +// grpcBehaviorUpstream is a JSON-RPC upstream whose answer per method can be +// replaced while eRPC runs; it records the params of the last call per method. +type grpcBehaviorUpstream struct { + srv *httptest.Server + mu sync.Mutex + replies map[string]string // method -> `"result":...` or `"error":{...}` + params map[string]string +} + +func newGrpcBehaviorUpstream(t *testing.T) *grpcBehaviorUpstream { + t.Helper() + u := &grpcBehaviorUpstream{replies: map[string]string{}, params: map[string]string{}} + u.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var req struct { + Id json.RawMessage `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` + } + if err := sonic.Unmarshal(body, &req); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + id := string(req.Id) + if id == "" { + id = "1" + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":` + id + `,` + u.reply(req.Method, req.Params) + `}`)) + })) + t.Cleanup(u.srv.Close) + return u +} + +func (u *grpcBehaviorUpstream) reply(method string, params json.RawMessage) string { + u.mu.Lock() + defer u.mu.Unlock() + raw := string(params) + isHead := method == "eth_getBlockByNumber" && (strings.Contains(raw, "latest") || strings.Contains(raw, "finalized")) + if !isHead { + u.params[method] = raw + if frag, ok := u.replies[method]; ok { + return frag + } + } + switch method { + case "eth_chainId": + return `"result":"0x7b"` + case "eth_syncing": + return `"result":false` + case "eth_getBlockByNumber": + n := uint64(grpcBehaviorHead) + var p []interface{} + if sonic.Unmarshal(params, &p) == nil && len(p) > 0 { + if s, ok := p[0].(string); ok && strings.HasPrefix(s, "0x") { + n, _ = strconv.ParseUint(s[2:], 16, 64) + } + } + b, _ := sonic.Marshal(qtBlock(n)) + return `"result":` + string(b) + } + return `"error":{"code":-32601,"message":"the method ` + method + ` does not exist/is not available"}` +} + +func (u *grpcBehaviorUpstream) set(method, frag string) { + u.mu.Lock() + defer u.mu.Unlock() + u.replies[method] = frag +} + +func (u *grpcBehaviorUpstream) setResult(method string, result interface{}) { + b, err := sonic.Marshal(result) + if err != nil { + panic(err) + } + u.set(method, `"result":`+string(b)) +} + +func (u *grpcBehaviorUpstream) lastParams(method string) string { + u.mu.Lock() + defer u.mu.Unlock() + return u.params[method] +} + +type grpcBehaviorFixture struct { + up *grpcBehaviorUpstream + erpc *ERPC + gs *GrpcServer + conn *grpc.ClientConn + rpc evm.RPCQueryServiceClient +} + +// newGrpcBehaviorFixture serves project "main" (no auth) and project +// "guarded" (network auth allowing only 203.0.113.9) on one eRPC gRPC server +// that trusts 127.0.0.1 as a forwarder of x-forwarded-for. +func newGrpcBehaviorFixture(t *testing.T) *grpcBehaviorFixture { + t.Helper() + util.ResetGock() + t.Cleanup(util.ResetGock) + + up := newGrpcBehaviorUpstream(t) + upstreamCfg := func() *common.UpstreamConfig { + return &common.UpstreamConfig{ + Id: "rpc1", + Type: common.UpstreamTypeEvm, + Endpoint: up.srv.URL, + JsonRpc: &common.JsonRpcUpstreamConfig{SupportsBatch: util.BoolPtr(false)}, + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + StatePollerInterval: common.Duration(time.Hour), + SkipSyncingCheck: util.BoolPtr(true), + QueryShim: &common.EvmQueryShimConfig{Enabled: util.BoolPtr(true), AllowedMethods: []string{"eth_queryBlocks"}}, + }, + } + } + networkCfg := func() *common.NetworkConfig { + return &common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + Failsafe: []*common.FailsafeConfig{{Retry: &common.RetryPolicyConfig{MaxAttempts: 1}}}, + } + } + cfg := &common.Config{ + Server: &common.ServerConfig{MaxTimeout: common.Duration(5 * time.Second).Ptr()}, + Projects: []*common.ProjectConfig{ + {Id: "main", Networks: []*common.NetworkConfig{networkCfg()}, Upstreams: []*common.UpstreamConfig{upstreamCfg()}}, + { + Id: "guarded", + Auth: &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{{ + Type: common.AuthTypeNetwork, + Network: &common.NetworkStrategyConfig{AllowedIPs: []string{"203.0.113.9"}}, + }}}, + Networks: []*common.NetworkConfig{networkCfg()}, + Upstreams: []*common.UpstreamConfig{upstreamCfg()}, + }, + }, + } + require.NoError(t, cfg.SetDefaults(nil)) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + erpcInstance, err := NewERPC(ctx, &log.Logger, nil, nil, nil, cfg) + require.NoError(t, err) + erpcInstance.Bootstrap(ctx) + + gs, err := NewGrpcServer(ctx, &log.Logger, &common.ServerConfig{ + GrpcMaxRecvMsgSize: util.IntPtr(1 << 20), + GrpcMaxSendMsgSize: util.IntPtr(1 << 20), + GrpcReflection: util.BoolPtr(false), + TrustedIPForwarders: []string{"127.0.0.1"}, + TrustedIPHeaders: []string{"X-Forwarded-For"}, + }, erpcInstance) + require.NoError(t, err) + _, hasReflection := gs.server.GetServiceInfo()["grpc.reflection.v1.ServerReflection"] + require.False(t, hasReflection, "grpcReflection=false must not register reflection") + + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = gs.server.Serve(lis) }() + t.Cleanup(gs.server.Stop) + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + return &grpcBehaviorFixture{up: up, erpc: erpcInstance, gs: gs, conn: conn, rpc: evm.NewRPCQueryServiceClient(conn)} +} + +// grpcBehaviorCtx is an outgoing context carrying project, chain and any extra metadata pairs. +func grpcBehaviorCtx(t *testing.T, project string, kv ...string) context.Context { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) + md := metadata.MD{} + if project != "" { + md.Set("x-erpc-project", project) + md.Set("x-erpc-chain-id", "123") + } + for i := 0; i+1 < len(kv); i += 2 { + md.Set(kv[i], kv[i+1]) + } + return metadata.NewOutgoingContext(ctx, md) +} + +func grpcBehaviorBareServer(t *testing.T, forwarders, headers []string) *GrpcServer { + t.Helper() + gs, err := NewGrpcServer(context.Background(), &log.Logger, &common.ServerConfig{ + GrpcMaxRecvMsgSize: util.IntPtr(1 << 20), + GrpcMaxSendMsgSize: util.IntPtr(1 << 20), + TrustedIPForwarders: forwarders, + TrustedIPHeaders: headers, + }, nil) + require.NoError(t, err) + return gs +} + +func grpcBehaviorReceipt(n, i uint64) map[string]interface{} { + r := qtReceipts(n)[i].(map[string]interface{}) + r["to"] = qtAddrB + r["logs"] = []interface{}{grpcBehaviorLog(n, 0)} + return r +} + +func grpcBehaviorLog(n uint64, idx int) map[string]interface{} { + return map[string]interface{}{ + "address": qtToken, "topics": []interface{}{qtTopic}, "data": "0x01", "blockNumber": fmt.Sprintf("0x%x", n), + "blockHash": qtHash(n), "transactionHash": qtTxHash(n, 0), "transactionIndex": "0x0", "logIndex": fmt.Sprintf("0x%x", idx), + } +} + +func TestGrpcServerBehavior_MissingMetadataRejectedByEveryMethod(t *testing.T) { + fx := newGrpcBehaviorFixture(t) + query := evm.NewQueryServiceClient(fx.conn) + stream := evm.NewStreamServiceClient(fx.conn) + + calls := map[string]func(ctx context.Context) error{ + "ChainId": func(ctx context.Context) error { _, err := fx.rpc.ChainId(ctx, &evm.ChainIdRequest{}); return err }, + "GetBlockByNumber": func(ctx context.Context) error { + _, err := fx.rpc.GetBlockByNumber(ctx, &evm.GetBlockByNumberRequest{BlockNumber: "0x1"}) + return err + }, + "GetBlockByHash": func(ctx context.Context) error { + _, err := fx.rpc.GetBlockByHash(ctx, &evm.GetBlockByHashRequest{BlockHash: []byte{1}}) + return err + }, + "GetLogs": func(ctx context.Context) error { _, err := fx.rpc.GetLogs(ctx, &evm.GetLogsRequest{}); return err }, + "GetTransactionByHash": func(ctx context.Context) error { + _, err := fx.rpc.GetTransactionByHash(ctx, &evm.GetTransactionByHashRequest{TransactionHash: []byte{1}}) + return err + }, + "GetTransactionReceipt": func(ctx context.Context) error { + _, err := fx.rpc.GetTransactionReceipt(ctx, &evm.GetTransactionReceiptRequest{TransactionHash: []byte{1}}) + return err + }, + "GetBlockReceipts": func(ctx context.Context) error { + _, err := fx.rpc.GetBlockReceipts(ctx, &evm.GetBlockReceiptsRequest{BlockHash: []byte{1}}) + return err + }, + "QueryBlocks": func(ctx context.Context) error { + s, err := query.QueryBlocks(ctx, &evm.QueryBlocksRequest{}) + _, code := grpcPages(t, s, err) + return status.Error(code, "") + }, + "StreamBlocks": func(ctx context.Context) error { + s, err := stream.StreamBlocks(ctx, &evm.StreamBlocksRequest{}) + require.NoError(t, err) + _, err = s.Recv() + return err + }, + } + + for name, call := range calls { + t.Run(name+"/no project", func(t *testing.T) { + ctx := grpcBehaviorCtx(t, "") + assert.Equal(t, codes.InvalidArgument, status.Code(call(ctx))) + }) + t.Run(name+"/no chain", func(t *testing.T) { + ctx := grpcBehaviorCtx(t, "", "x-erpc-project", "main") + assert.Equal(t, codes.InvalidArgument, status.Code(call(ctx))) + }) + t.Run(name+"/malformed basic auth", func(t *testing.T) { + ctx := grpcBehaviorCtx(t, "main", "authorization", "Basic !!not-base64!!") + assert.Equal(t, codes.Unauthenticated, status.Code(call(ctx))) + }) + } +} + +func TestGrpcServerBehavior_ExtractRequestInput(t *testing.T) { + gs := grpcBehaviorBareServer(t, nil, nil) + peerCtx := func(md metadata.MD) context.Context { + ctx := peer.NewContext(context.Background(), &peer.Peer{Addr: &net.TCPAddr{IP: net.ParseIP("198.51.100.7"), Port: 4000}}) + return metadata.NewIncomingContext(ctx, md) + } + + t.Run("no incoming metadata", func(t *testing.T) { + _, err := gs.extractRequestInput(context.Background(), "eth_chainId") + assert.Equal(t, codes.InvalidArgument, status.Code(err)) + assert.Contains(t, status.Convert(err).Message(), "missing metadata") + }) + t.Run("project required", func(t *testing.T) { + _, err := gs.extractRequestInput(peerCtx(metadata.Pairs("x-erpc-chain-id", "1")), "eth_chainId") + assert.Equal(t, codes.InvalidArgument, status.Code(err)) + assert.Contains(t, status.Convert(err).Message(), "x-erpc-project") + }) + t.Run("chain required", func(t *testing.T) { + _, err := gs.extractRequestInput(peerCtx(metadata.Pairs("x-erpc-project", "p")), "eth_chainId") + assert.Equal(t, codes.InvalidArgument, status.Code(err)) + assert.Contains(t, status.Convert(err).Message(), "x-erpc-chain-id") + }) + t.Run("basic auth without colon", func(t *testing.T) { + md := metadata.Pairs("x-erpc-project", "p", "x-erpc-chain-id", "1", + "authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("nocolon"))) + _, err := gs.extractRequestInput(peerCtx(md), "eth_chainId") + assert.Equal(t, codes.Unauthenticated, status.Code(err)) + }) + t.Run("defaults and passthrough fields", func(t *testing.T) { + md := metadata.Pairs("x-erpc-project", "p", "x-erpc-chain-id", "42", + "user-agent", "bds-client/1.0", "x-erpc-user-id", "alice", "x-erpc-secret-token", "s3cret") + in, err := gs.extractRequestInput(peerCtx(md), "eth_getLogs") + require.NoError(t, err) + assert.Equal(t, "p", in.ProjectId) + assert.Equal(t, "42", in.ChainId) + assert.Equal(t, "evm", in.Architecture, "architecture defaults to evm") + assert.Equal(t, "198.51.100.7", in.ClientIP) + assert.Contains(t, in.UserAgent, "bds-client/1.0") + assert.Equal(t, "alice", in.TrustedUserId) + require.NotNil(t, in.AuthPayload) + assert.Equal(t, common.AuthTypeSecret, in.AuthPayload.Type) + assert.Equal(t, "eth_getLogs", in.AuthPayload.Method) + }) + t.Run("explicit architecture and bearer", func(t *testing.T) { + md := metadata.Pairs("x-erpc-project", "p", "x-erpc-chain-id", "1", + "x-erpc-architecture", "svm", "authorization", "Bearer tok") + in, err := gs.extractRequestInput(peerCtx(md), "eth_chainId") + require.NoError(t, err) + assert.Equal(t, "svm", in.Architecture) + assert.Equal(t, common.AuthTypeJwt, in.AuthPayload.Type) + assert.Equal(t, "tok", in.AuthPayload.Jwt.Token) + }) +} + +func TestGrpcServerBehavior_ChainId(t *testing.T) { + fx := newGrpcBehaviorFixture(t) + + resp, err := fx.rpc.ChainId(grpcBehaviorCtx(t, "main"), &evm.ChainIdRequest{}) + require.NoError(t, err) + assert.Equal(t, uint64(123), resp.ChainId) + + _, err = fx.rpc.ChainId(grpcBehaviorCtx(t, "no-such-project"), &evm.ChainIdRequest{}) + assert.Equal(t, codes.Internal, status.Code(err)) + assert.Contains(t, status.Convert(err).Message(), "no-such-project") +} + +func TestGrpcServerBehavior_ClientIPAuthThroughTrustedForwarder(t *testing.T) { + fx := newGrpcBehaviorFixture(t) + + cases := []struct { + name string + kv []string + want codes.Code + }{ + // The test client connects from 127.0.0.1, a trusted forwarder. + {"forwarded allowed client", []string{"x-forwarded-for", "203.0.113.9"}, codes.OK}, + {"forwarded chain picks nearest untrusted hop", []string{"x-forwarded-for", "203.0.113.9, 127.0.0.1"}, codes.OK}, + {"forwarded disallowed client", []string{"x-forwarded-for", "198.51.100.1"}, codes.Unauthenticated}, + {"no forwarded header uses the peer", nil, codes.Unauthenticated}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + resp, err := fx.rpc.ChainId(grpcBehaviorCtx(t, "guarded", tc.kv...), &evm.ChainIdRequest{}) + require.Equal(t, tc.want, status.Code(err), "err=%v", err) + if tc.want == codes.OK { + assert.Equal(t, uint64(123), resp.ChainId) + } + }) + } +} + +func TestGrpcServerBehavior_GetBlockByNumber(t *testing.T) { + fx := newGrpcBehaviorFixture(t) + ctx := grpcBehaviorCtx(t, "main") + + t.Run("full transactions decoded", func(t *testing.T) { + resp, err := fx.rpc.GetBlockByNumber(ctx, &evm.GetBlockByNumberRequest{BlockNumber: "0x64", IncludeTransactions: true}) + require.NoError(t, err) + require.NotNil(t, resp.Block) + assert.Equal(t, uint64(0x64), resp.Block.Number) + assert.Equal(t, qtHash(0x64), evm.BytesToHex(resp.Block.Hash)) + assert.Equal(t, qtHash(0x63), evm.BytesToHex(resp.Block.ParentHash)) + assert.Equal(t, uint64(1000+0x64), resp.Block.Timestamp) + require.Len(t, resp.FullTransactions, 2) + assert.Equal(t, qtTxHash(0x64, 0), evm.BytesToHex(resp.FullTransactions[0].Hash)) + assert.Equal(t, [][]byte{resp.FullTransactions[0].Hash, resp.FullTransactions[1].Hash}, resp.Transactions) + assert.Contains(t, fx.up.lastParams("eth_getBlockByNumber"), `"0x64",true`) + }) + t.Run("null block under the head is not found", func(t *testing.T) { + fx.up.set("eth_getBlockByNumber", `"result":null`) + _, err := fx.rpc.GetBlockByNumber(ctx, &evm.GetBlockByNumberRequest{BlockNumber: "0x65"}) + assert.Equal(t, codes.NotFound, status.Code(err), "err=%v", err) + }) + t.Run("null block past the head is an empty response", func(t *testing.T) { + fx.up.set("eth_getBlockByNumber", `"result":null`) + resp, err := fx.rpc.GetBlockByNumber(ctx, &evm.GetBlockByNumberRequest{BlockNumber: "0x100000"}) + require.NoError(t, err) + assert.Nil(t, resp.Block) + assert.Empty(t, resp.Transactions) + }) + t.Run("non-object result is internal", func(t *testing.T) { + fx.up.set("eth_getBlockByNumber", `"result":"0xnotablock"`) + _, err := fx.rpc.GetBlockByNumber(ctx, &evm.GetBlockByNumberRequest{BlockNumber: "0x66"}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("undecodable block field is internal", func(t *testing.T) { + fx.up.set("eth_getBlockByNumber", `"result":{"number":"0x67","hash":"0xzz"}`) + _, err := fx.rpc.GetBlockByNumber(ctx, &evm.GetBlockByNumberRequest{BlockNumber: "0x67"}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("upstream invalid params is invalid argument", func(t *testing.T) { + fx.up.set("eth_getBlockByNumber", `"error":{"code":-32602,"message":"invalid argument 0: hex string without 0x prefix"}`) + _, err := fx.rpc.GetBlockByNumber(ctx, &evm.GetBlockByNumberRequest{BlockNumber: "0x68"}) + assert.Equal(t, codes.InvalidArgument, status.Code(err), "err=%v", err) + }) +} + +func TestGrpcServerBehavior_GetBlockByHash(t *testing.T) { + fx := newGrpcBehaviorFixture(t) + ctx := grpcBehaviorCtx(t, "main") + hash := evm.MustHexToBytes(qtHash(0x64)) + + t.Run("hashes only", func(t *testing.T) { + block := qtBlock(0x64) + block["transactions"] = []interface{}{qtTxHash(0x64, 0), qtTxHash(0x64, 1)} + fx.up.setResult("eth_getBlockByHash", block) + resp, err := fx.rpc.GetBlockByHash(ctx, &evm.GetBlockByHashRequest{BlockHash: hash}) + require.NoError(t, err) + assert.Equal(t, uint64(0x64), resp.Block.Number) + assert.Empty(t, resp.FullTransactions) + require.Len(t, resp.Transactions, 2) + assert.Equal(t, qtTxHash(0x64, 1), evm.BytesToHex(resp.Transactions[1])) + assert.Contains(t, fx.up.lastParams("eth_getBlockByHash"), `"`+qtHash(0x64)+`",false`) + }) + t.Run("null block", func(t *testing.T) { + fx.up.set("eth_getBlockByHash", `"result":null`) + resp, err := fx.rpc.GetBlockByHash(ctx, &evm.GetBlockByHashRequest{BlockHash: evm.MustHexToBytes(qtHash(0x65))}) + require.NoError(t, err) + assert.Nil(t, resp.Block) + }) + t.Run("non-object result", func(t *testing.T) { + fx.up.set("eth_getBlockByHash", `"result":[1,2]`) + _, err := fx.rpc.GetBlockByHash(ctx, &evm.GetBlockByHashRequest{BlockHash: evm.MustHexToBytes(qtHash(0x66))}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("undecodable field", func(t *testing.T) { + fx.up.set("eth_getBlockByHash", `"result":{"number":"zz"}`) + _, err := fx.rpc.GetBlockByHash(ctx, &evm.GetBlockByHashRequest{BlockHash: evm.MustHexToBytes(qtHash(0x67))}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("upstream revert is failed precondition", func(t *testing.T) { + fx.up.set("eth_getBlockByHash", `"error":{"code":3,"message":"execution reverted","data":"0x"}`) + _, err := fx.rpc.GetBlockByHash(ctx, &evm.GetBlockByHashRequest{BlockHash: evm.MustHexToBytes(qtHash(0x68))}) + assert.Equal(t, codes.FailedPrecondition, status.Code(err), "err=%v", err) + }) +} + +func TestGrpcServerBehavior_GetLogs(t *testing.T) { + fx := newGrpcBehaviorFixture(t) + ctx := grpcBehaviorCtx(t, "main") + addrA, addrB := evm.MustHexToBytes(qtAddrA), evm.MustHexToBytes(qtAddrB) + topic := evm.MustHexToBytes(qtTopic) + other := evm.MustHexToBytes(qtHash(9)) + + t.Run("filter translated and logs decoded", func(t *testing.T) { + fx.up.setResult("eth_getLogs", []interface{}{grpcBehaviorLog(0x64, 0), grpcBehaviorLog(0x64, 1)}) + resp, err := fx.rpc.GetLogs(ctx, &evm.GetLogsRequest{ + FromBlock: evm.Uint64Ptr(0x64), ToBlock: evm.Uint64Ptr(0x65), + Addresses: [][]byte{addrA, addrB}, + Topics: []*evm.TopicFilter{{Values: [][]byte{topic}}, nil, {Values: [][]byte{topic, other}}}, + }) + require.NoError(t, err) + require.Len(t, resp.Logs, 2) + assert.Equal(t, qtToken, evm.BytesToHex(resp.Logs[0].Address)) + assert.Equal(t, uint32(1), resp.Logs[1].LogIndex) + assert.Equal(t, uint64(0x64), resp.Logs[0].BlockNumber) + assert.Equal(t, [][]byte{topic}, resp.Logs[0].Topics) + + var sent []map[string]interface{} + require.NoError(t, sonic.UnmarshalString(fx.up.lastParams("eth_getLogs"), &sent)) + require.Len(t, sent, 1) + assert.Equal(t, "0x64", sent[0]["fromBlock"]) + assert.Equal(t, "0x65", sent[0]["toBlock"]) + assert.Equal(t, []interface{}{qtAddrA, qtAddrB}, sent[0]["address"]) + assert.Equal(t, []interface{}{qtTopic, nil, []interface{}{qtTopic, qtHash(9)}}, sent[0]["topics"]) + }) + t.Run("single address sent as a string", func(t *testing.T) { + fx.up.setResult("eth_getLogs", []interface{}{}) + resp, err := fx.rpc.GetLogs(ctx, &evm.GetLogsRequest{FromBlock: evm.Uint64Ptr(0x66), ToBlock: evm.Uint64Ptr(0x66), Addresses: [][]byte{addrA}}) + require.NoError(t, err) + assert.Empty(t, resp.Logs) + var sent []map[string]interface{} + require.NoError(t, sonic.UnmarshalString(fx.up.lastParams("eth_getLogs"), &sent)) + assert.Equal(t, qtAddrA, sent[0]["address"]) + assert.NotContains(t, sent[0], "topics") + }) + t.Run("non-array result", func(t *testing.T) { + fx.up.set("eth_getLogs", `"result":{"logs":[]}`) + _, err := fx.rpc.GetLogs(ctx, &evm.GetLogsRequest{FromBlock: evm.Uint64Ptr(0x67), ToBlock: evm.Uint64Ptr(0x67)}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("undecodable log", func(t *testing.T) { + bad := grpcBehaviorLog(0x68, 0) + bad["address"] = "0xzz" + fx.up.setResult("eth_getLogs", []interface{}{bad}) + _, err := fx.rpc.GetLogs(ctx, &evm.GetLogsRequest{FromBlock: evm.Uint64Ptr(0x68), ToBlock: evm.Uint64Ptr(0x68)}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("upstream invalid params", func(t *testing.T) { + fx.up.set("eth_getLogs", `"error":{"code":-32602,"message":"invalid params"}`) + _, err := fx.rpc.GetLogs(ctx, &evm.GetLogsRequest{FromBlock: evm.Uint64Ptr(0x69), ToBlock: evm.Uint64Ptr(0x69)}) + assert.Equal(t, codes.InvalidArgument, status.Code(err), "err=%v", err) + }) +} + +func TestGrpcServerBehavior_GetTransactionByHash(t *testing.T) { + fx := newGrpcBehaviorFixture(t) + ctx := grpcBehaviorCtx(t, "main") + txHash := func(i uint64) []byte { return evm.MustHexToBytes(qtTxHash(0x64, i)) } + + t.Run("decoded", func(t *testing.T) { + tx := qtBlock(0x64)["transactions"].([]interface{})[0] + fx.up.setResult("eth_getTransactionByHash", tx) + resp, err := fx.rpc.GetTransactionByHash(ctx, &evm.GetTransactionByHashRequest{TransactionHash: txHash(0)}) + require.NoError(t, err) + require.NotNil(t, resp.Transaction) + assert.Equal(t, qtTxHash(0x64, 0), evm.BytesToHex(resp.Transaction.Hash)) + assert.Equal(t, qtAddrA, evm.BytesToHex(resp.Transaction.From)) + assert.Equal(t, uint64(0x5208), resp.Transaction.GasLimit) + assert.Equal(t, uint64(1), resp.Transaction.Nonce) + assert.Contains(t, fx.up.lastParams("eth_getTransactionByHash"), qtTxHash(0x64, 0)) + }) + t.Run("null", func(t *testing.T) { + fx.up.set("eth_getTransactionByHash", `"result":null`) + resp, err := fx.rpc.GetTransactionByHash(ctx, &evm.GetTransactionByHashRequest{TransactionHash: txHash(1)}) + require.NoError(t, err) + assert.Nil(t, resp.Transaction) + }) + t.Run("non-object result", func(t *testing.T) { + fx.up.set("eth_getTransactionByHash", `"result":[1]`) + _, err := fx.rpc.GetTransactionByHash(ctx, &evm.GetTransactionByHashRequest{TransactionHash: txHash(2)}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("undecodable", func(t *testing.T) { + fx.up.set("eth_getTransactionByHash", `"result":{"hash":"0xzz"}`) + _, err := fx.rpc.GetTransactionByHash(ctx, &evm.GetTransactionByHashRequest{TransactionHash: txHash(3)}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("method unsupported upstream", func(t *testing.T) { + fx.up.set("eth_getTransactionByHash", `"error":{"code":-32601,"message":"the method eth_getTransactionByHash does not exist/is not available"}`) + _, err := fx.rpc.GetTransactionByHash(ctx, &evm.GetTransactionByHashRequest{TransactionHash: txHash(4)}) + assert.Equal(t, codes.Unimplemented, status.Code(err), "err=%v", err) + }) +} + +func TestGrpcServerBehavior_GetTransactionReceipt(t *testing.T) { + fx := newGrpcBehaviorFixture(t) + ctx := grpcBehaviorCtx(t, "main") + txHash := func(i uint64) []byte { return evm.MustHexToBytes(qtTxHash(0x65, i)) } + + t.Run("decoded", func(t *testing.T) { + fx.up.setResult("eth_getTransactionReceipt", grpcBehaviorReceipt(0x65, 0)) + resp, err := fx.rpc.GetTransactionReceipt(ctx, &evm.GetTransactionReceiptRequest{TransactionHash: txHash(0)}) + require.NoError(t, err) + require.NotNil(t, resp.Receipt) + assert.Equal(t, uint64(0x65), resp.Receipt.BlockNumber) + assert.Equal(t, uint64(0x5208), resp.Receipt.GasUsed) + require.NotNil(t, resp.Receipt.Status) + assert.Equal(t, uint32(1), *resp.Receipt.Status) + require.Len(t, resp.Receipt.Logs, 1) + assert.Equal(t, qtToken, evm.BytesToHex(resp.Receipt.Logs[0].Address)) + }) + t.Run("null", func(t *testing.T) { + fx.up.set("eth_getTransactionReceipt", `"result":null`) + resp, err := fx.rpc.GetTransactionReceipt(ctx, &evm.GetTransactionReceiptRequest{TransactionHash: txHash(1)}) + require.NoError(t, err, "err=%v", err) + assert.Nil(t, resp.Receipt) + }) + t.Run("non-object result", func(t *testing.T) { + fx.up.set("eth_getTransactionReceipt", `"result":"0x1"`) + _, err := fx.rpc.GetTransactionReceipt(ctx, &evm.GetTransactionReceiptRequest{TransactionHash: txHash(2)}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("undecodable", func(t *testing.T) { + fx.up.set("eth_getTransactionReceipt", `"result":{"blockNumber":"zz"}`) + _, err := fx.rpc.GetTransactionReceipt(ctx, &evm.GetTransactionReceiptRequest{TransactionHash: txHash(3)}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("upstream server error is unavailable", func(t *testing.T) { + fx.up.set("eth_getTransactionReceipt", `"error":{"code":-32603,"message":"internal error"}`) + _, err := fx.rpc.GetTransactionReceipt(ctx, &evm.GetTransactionReceiptRequest{TransactionHash: txHash(4)}) + assert.Equal(t, codes.Unavailable, status.Code(err), "err=%v", err) + }) +} + +func TestGrpcServerBehavior_GetBlockReceipts(t *testing.T) { + fx := newGrpcBehaviorFixture(t) + ctx := grpcBehaviorCtx(t, "main") + + t.Run("by hash", func(t *testing.T) { + fx.up.setResult("eth_getBlockReceipts", []interface{}{grpcBehaviorReceipt(0x64, 0), grpcBehaviorReceipt(0x64, 1)}) + resp, err := fx.rpc.GetBlockReceipts(ctx, &evm.GetBlockReceiptsRequest{BlockHash: evm.MustHexToBytes(qtHash(0x64))}) + require.NoError(t, err) + require.Len(t, resp.Receipts, 2) + assert.Equal(t, uint32(1), resp.Receipts[1].TransactionIndex) + assert.Equal(t, qtTxHash(0x64, 1), evm.BytesToHex(resp.Receipts[1].TransactionHash)) + assert.Contains(t, fx.up.lastParams("eth_getBlockReceipts"), qtHash(0x64)) + }) + t.Run("by number", func(t *testing.T) { + fx.up.setResult("eth_getBlockReceipts", []interface{}{grpcBehaviorReceipt(0x65, 0)}) + resp, err := fx.rpc.GetBlockReceipts(ctx, &evm.GetBlockReceiptsRequest{BlockNumber: util.StringPtr("0x65")}) + require.NoError(t, err) + require.Len(t, resp.Receipts, 1) + assert.Equal(t, uint64(0x65), resp.Receipts[0].BlockNumber) + assert.Contains(t, fx.up.lastParams("eth_getBlockReceipts"), `"0x65"`) + }) + t.Run("neither number nor hash", func(t *testing.T) { + _, err := fx.rpc.GetBlockReceipts(ctx, &evm.GetBlockReceiptsRequest{}) + assert.Equal(t, codes.InvalidArgument, status.Code(err)) + assert.Contains(t, status.Convert(err).Message(), "blockNumber or blockHash required") + }) + t.Run("non-array result", func(t *testing.T) { + fx.up.set("eth_getBlockReceipts", `"result":{"x":1}`) + _, err := fx.rpc.GetBlockReceipts(ctx, &evm.GetBlockReceiptsRequest{BlockNumber: util.StringPtr("0x66")}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("undecodable receipt", func(t *testing.T) { + bad := grpcBehaviorReceipt(0x67, 0) + bad["gasUsed"] = "zz" + fx.up.setResult("eth_getBlockReceipts", []interface{}{bad}) + _, err := fx.rpc.GetBlockReceipts(ctx, &evm.GetBlockReceiptsRequest{BlockNumber: util.StringPtr("0x67")}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("unknown project", func(t *testing.T) { + _, err := fx.rpc.GetBlockReceipts(grpcBehaviorCtx(t, "no-such-project"), &evm.GetBlockReceiptsRequest{BlockNumber: util.StringPtr("0x68")}) + assert.Equal(t, codes.Internal, status.Code(err)) + }) +} + +func TestGrpcServerBehavior_QueryService(t *testing.T) { + fx := newGrpcBehaviorFixture(t) + ctx := grpcBehaviorCtx(t, "main") + client := evm.NewQueryServiceClient(fx.conn) + from, to := util.StringPtr("0x64"), util.StringPtr("0x64") + + t.Run("blocks served by the shim", func(t *testing.T) { + stream, err := client.QueryBlocks(ctx, &evm.QueryBlocksRequest{FromBlock: from, ToBlock: to, BlockFields: &evm.BlockFieldSelection{Number: true, Hash: true}}) + pages, code := grpcPages(t, stream, err) + require.Equal(t, codes.OK, code) + require.Len(t, pages, 1) + require.Len(t, pages[0].Blocks, 1) + assert.Equal(t, uint64(0x64), pages[0].Blocks[0].Number) + assert.Equal(t, qtHash(0x64), evm.BytesToHex(pages[0].Blocks[0].Hash)) + }) + + // The shim allows only eth_queryBlocks and no upstream is a native + // QueryService, so every other query method is unserved. + t.Run("transactions", func(t *testing.T) { + s, err := client.QueryTransactions(ctx, &evm.QueryTransactionsRequest{FromBlock: from, ToBlock: to}) + _, code := grpcPages(t, s, err) + assert.Equal(t, codes.Unimplemented, code) + }) + t.Run("logs", func(t *testing.T) { + s, err := client.QueryLogs(ctx, &evm.QueryLogsRequest{FromBlock: from, ToBlock: to}) + _, code := grpcPages(t, s, err) + assert.Equal(t, codes.Unimplemented, code) + }) + t.Run("traces", func(t *testing.T) { + s, err := client.QueryTraces(ctx, &evm.QueryTracesRequest{FromBlock: from, ToBlock: to}) + _, code := grpcPages(t, s, err) + assert.Equal(t, codes.Unimplemented, code) + }) + t.Run("transfers", func(t *testing.T) { + s, err := client.QueryTransfers(ctx, &evm.QueryTransfersRequest{FromBlock: from, ToBlock: to}) + _, code := grpcPages(t, s, err) + assert.Equal(t, codes.Unimplemented, code) + }) +} + +func TestGrpcServerBehavior_StreamBlocks(t *testing.T) { + fx := newGrpcBehaviorFixture(t) + client := evm.NewStreamServiceClient(fx.conn) + + t.Run("unknown project ends the stream", func(t *testing.T) { + s, err := client.StreamBlocks(grpcBehaviorCtx(t, "no-such-project"), &evm.StreamBlocksRequest{}) + require.NoError(t, err) + _, err = s.Recv() + assert.Equal(t, codes.Internal, status.Code(err)) + }) + + t.Run("pushes each new head in order", func(t *testing.T) { + bctx := grpcBehaviorCtx(t, "main") + ntw, err := fx.erpc.GetNetwork(bctx, "main", util.EvmNetworkId(123)) + require.NoError(t, err) + ups := ntw.AllUpstreams() + require.Len(t, ups, 1) + sp := ups[0].EvmStatePoller() + require.Eventually(t, func() bool { return sp.LatestBlock() > 0 }, 5*time.Second, 10*time.Millisecond) + head := sp.LatestBlock() + + sctx, scancel := context.WithCancel(bctx) + defer scancel() + s, err := client.StreamBlocks(sctx, &evm.StreamBlocksRequest{}) + require.NoError(t, err) + require.Eventually(t, func() bool { + m := fx.gs.processor.blockStream + m.mu.Lock() + defer m.mu.Unlock() + for _, h := range m.hubs { + h.mu.Lock() + n := len(h.subs) + h.mu.Unlock() + if n > 0 { + return true + } + } + return false + }, 5*time.Second, 5*time.Millisecond, "subscriber registered") + + sp.SuggestLatestBlock(head + 2) + for _, want := range []int64{head + 1, head + 2} { + msg, err := s.Recv() + require.NoError(t, err) + assert.Equal(t, uint64(want), msg.Header.Number) + assert.Equal(t, qtHash(uint64(want)), evm.BytesToHex(msg.Header.Hash)) + } + + scancel() + _, err = s.Recv() + assert.Equal(t, codes.Canceled, status.Code(err)) + }) +} + +func TestGrpcServerBehavior_PanicRecovery(t *testing.T) { + gs := grpcBehaviorBareServer(t, nil, nil) + + t.Run("unary panic is internal", func(t *testing.T) { + resp, err := gs.panicRecoveryUnary()(context.Background(), "req", &grpc.UnaryServerInfo{FullMethod: "/x/Y"}, + func(ctx context.Context, req interface{}) (interface{}, error) { panic("boom") }) + assert.Nil(t, resp) + assert.Equal(t, codes.Internal, status.Code(err)) + assert.Equal(t, "internal server error", status.Convert(err).Message()) + }) + t.Run("unary passthrough", func(t *testing.T) { + resp, err := gs.panicRecoveryUnary()(context.Background(), "req", &grpc.UnaryServerInfo{}, + func(ctx context.Context, req interface{}) (interface{}, error) { return req.(string) + "-ok", nil }) + require.NoError(t, err) + assert.Equal(t, "req-ok", resp) + }) + t.Run("stream panic is internal", func(t *testing.T) { + err := gs.panicRecoveryStream()(nil, nil, &grpc.StreamServerInfo{FullMethod: "/x/Z"}, + func(srv interface{}, ss grpc.ServerStream) error { panic(errors.New("boom")) }) + assert.Equal(t, codes.Internal, status.Code(err)) + }) + t.Run("stream passthrough keeps the handler error", func(t *testing.T) { + want := status.Error(codes.Aborted, "stop") + err := gs.panicRecoveryStream()(nil, nil, &grpc.StreamServerInfo{}, + func(srv interface{}, ss grpc.ServerStream) error { return want }) + assert.Equal(t, want, err) + }) +} + +func TestGrpcServerBehavior_GrpcStatusOf(t *testing.T) { + retryableExhausted := common.NewErrUpstreamsExhaustedWithCause(errors.Join(common.NewErrEndpointServerSideException(errors.New("502"), nil, 502))) + revertExhausted := common.NewErrUpstreamsExhaustedWithCause(errors.Join(common.NewErrEndpointExecutionException(errors.New("reverted")))) + noUpstreams := common.NewErrNoUpstreamsLeftToSelect(common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[]}`)), "all cordoned") + + cases := []struct { + name string + err error + want codes.Code + }{ + {"unsupported", common.NewErrEndpointUnsupported(nil), codes.Unimplemented}, + {"auth unauthorized", common.NewErrAuthUnauthorized("secret", "invalid_secret", "bad secret"), codes.Unauthenticated}, + {"endpoint unauthorized", common.NewErrEndpointUnauthorized(nil), codes.Unauthenticated}, + {"endpoint timeout", common.NewErrEndpointRequestTimeout(time.Second, nil), codes.DeadlineExceeded}, + {"network timeout", common.NewErrNetworkRequestTimeout(time.Second, nil), codes.DeadlineExceeded}, + {"failsafe timeout", common.NewErrFailsafeTimeoutExceeded(common.ScopeNetwork, nil, nil), codes.DeadlineExceeded}, + {"context deadline", fmt.Errorf("wrapped: %w", context.DeadlineExceeded), codes.DeadlineExceeded}, + {"capacity", common.NewErrEndpointCapacityExceeded(nil), codes.ResourceExhausted}, + {"too large", common.NewErrEndpointRequestTooLarge(nil, common.EvmBlockRangeTooLarge), codes.ResourceExhausted}, + {"billing", common.NewErrEndpointBillingIssue(nil), codes.ResourceExhausted}, + {"auth rate limit", common.NewErrAuthRateLimitRuleExceeded("p", "secret", "b", "r", "u", "1.2.3.4"), codes.ResourceExhausted}, + {"project rate limit", common.NewErrProjectRateLimitRuleExceeded("p", "b", "r"), codes.ResourceExhausted}, + {"network rate limit", common.NewErrNetworkRateLimitRuleExceeded("p", "n", "b", "r"), codes.ResourceExhausted}, + {"upstream rate limit", common.NewErrUpstreamRateLimitRuleExceeded("u", "b", "r"), codes.ResourceExhausted}, + {"missing data", common.NewErrEndpointMissingData(nil, nil), codes.NotFound}, + {"client side", common.NewErrEndpointClientSideException(nil), codes.InvalidArgument}, + {"invalid request", common.NewErrInvalidRequest(nil), codes.InvalidArgument}, + {"getLogs range", common.NewErrGetLogsExceededMaxAllowedRange(100, 10), codes.InvalidArgument}, + {"execution exception", common.NewErrEndpointExecutionException(nil), codes.FailedPrecondition}, + {"exhausted on a revert", revertExhausted, codes.FailedPrecondition}, + {"no upstreams left", noUpstreams, codes.Unavailable}, + {"exhausted, retryable", retryableExhausted, codes.Unavailable}, + {"exhausted, nothing tried", common.NewErrUpstreamsExhaustedWithCause(nil), codes.Internal}, + {"plain error", errors.New("boom"), codes.Internal}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := grpcStatusOf(tc.err) + st, ok := status.FromError(got) + require.True(t, ok) + assert.Equal(t, tc.want, st.Code()) + // Details are a map, so compare only the message head. + assert.Equal(t, strings.SplitN(tc.err.Error(), " -> ", 2)[0], strings.SplitN(st.Message(), " -> ", 2)[0]) + }) + } + assert.NoError(t, grpcStatusOf(nil)) +} + +func TestGrpcServerBehavior_ClientIP(t *testing.T) { + gs := grpcBehaviorBareServer(t, + []string{"10.0.0.0/8", " 192.0.2.1 ", "", "10.0.0.0/33", "not-an-ip"}, + []string{" X-Real-IP ", "", "x-forwarded-for"}) + peerAt := func(ip string) context.Context { + return peer.NewContext(context.Background(), &peer.Peer{Addr: &net.TCPAddr{IP: net.ParseIP(ip), Port: 1}}) + } + + cases := []struct { + name string + ctx context.Context + md metadata.MD + want string + }{ + {"no peer", context.Background(), metadata.Pairs("x-forwarded-for", "203.0.113.9"), ""}, + {"untrusted peer ignores headers", peerAt("198.51.100.25"), metadata.Pairs("x-forwarded-for", "203.0.113.9"), "198.51.100.25"}, + {"trusted IP peer uses xff", peerAt("192.0.2.1"), metadata.Pairs("x-forwarded-for", "203.0.113.9"), "203.0.113.9"}, + {"trusted CIDR peer trims trusted hops", peerAt("10.1.2.3"), metadata.Pairs("x-forwarded-for", "203.0.113.9, 10.9.9.9, 192.0.2.1"), "203.0.113.9"}, + {"headers tried in configured order", peerAt("10.1.2.3"), metadata.Pairs("x-real-ip", "198.51.100.4", "x-forwarded-for", "203.0.113.9"), "198.51.100.4"}, + {"all-trusted chain falls back to next header", peerAt("10.1.2.3"), metadata.Pairs("x-real-ip", "10.0.0.1", "x-forwarded-for", "203.0.113.9"), "203.0.113.9"}, + {"all-trusted chain falls back to peer", peerAt("10.1.2.3"), metadata.Pairs("x-forwarded-for", "10.0.0.7, 192.0.2.1"), "10.1.2.3"}, + {"trusted peer without headers", peerAt("192.0.2.1"), metadata.MD{}, "192.0.2.1"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, gs.grpcClientIP(tc.ctx, tc.md)) + }) + } + + assert.False(t, gs.isTrustedForwarder(nil)) + assert.False(t, gs.isTrustedForwarder(net.ParseIP("11.0.0.1")), "an invalid /33 CIDR is ignored, not widened") + assert.Nil(t, grpcPeerIP(context.Background())) + assert.Nil(t, grpcPeerIP(peer.NewContext(context.Background(), &peer.Peer{})), "peer without address") +} + +func TestGrpcServerBehavior_SharesHttpV4(t *testing.T) { + host, other := "0.0.0.0", "127.0.0.1" + port, otherPort := 4000, 4001 + cfg := func(mut func(c *common.ServerConfig)) *common.ServerConfig { + c := &common.ServerConfig{ + GrpcEnabled: util.BoolPtr(true), ListenV4: util.BoolPtr(true), + HttpHostV4: &host, HttpPortV4: &port, GrpcHostV4: &host, GrpcPortV4: &port, + } + if mut != nil { + mut(c) + } + return c + } + cases := []struct { + name string + cfg *common.ServerConfig + want bool + }{ + {"nil", nil, false}, + {"grpc unset", cfg(func(c *common.ServerConfig) { c.GrpcEnabled = nil }), false}, + {"grpc disabled", cfg(func(c *common.ServerConfig) { c.GrpcEnabled = util.BoolPtr(false) }), false}, + {"v4 not listening", cfg(func(c *common.ServerConfig) { c.ListenV4 = util.BoolPtr(false) }), false}, + {"v4 unset", cfg(func(c *common.ServerConfig) { c.ListenV4 = nil }), false}, + {"grpc port unset", cfg(func(c *common.ServerConfig) { c.GrpcPortV4 = nil }), false}, + {"same host and port", cfg(nil), true}, + {"different port", cfg(func(c *common.ServerConfig) { c.GrpcPortV4 = &otherPort }), false}, + {"different host", cfg(func(c *common.ServerConfig) { c.GrpcHostV4 = &other }), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, grpcSharesHttpV4(tc.cfg)) + }) + } +} + +func TestGrpcServerBehavior_NewAndStart(t *testing.T) { + t.Run("reflection on by default", func(t *testing.T) { + gs := grpcBehaviorBareServer(t, nil, nil) + info := gs.server.GetServiceInfo() + assert.Contains(t, info, "grpc.reflection.v1.ServerReflection") + assert.Contains(t, info, "bds.evm.RPCQueryService") + assert.Contains(t, info, "bds.evm.QueryService") + assert.Contains(t, info, "bds.evm.StreamService") + }) + + t.Run("unreadable TLS files fail construction", func(t *testing.T) { + _, err := NewGrpcServer(context.Background(), &log.Logger, &common.ServerConfig{ + GrpcMaxRecvMsgSize: util.IntPtr(1 << 20), + GrpcMaxSendMsgSize: util.IntPtr(1 << 20), + TLS: &common.TLSConfig{Enabled: true, CertFile: t.TempDir() + "/missing.crt", KeyFile: t.TempDir() + "/missing.key"}, + }, nil) + assert.Error(t, err) + }) + + t.Run("start fails on an occupied port", func(t *testing.T) { + busy, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer busy.Close() + host, port := "127.0.0.1", busy.Addr().(*net.TCPAddr).Port + gs, err := NewGrpcServer(context.Background(), &log.Logger, &common.ServerConfig{ + GrpcMaxRecvMsgSize: util.IntPtr(1 << 20), GrpcMaxSendMsgSize: util.IntPtr(1 << 20), + GrpcHostV4: &host, GrpcPortV4: &port, + }, nil) + require.NoError(t, err) + err = gs.Start(&log.Logger) + require.Error(t, err) + assert.Contains(t, err.Error(), fmt.Sprintf("failed to listen on 127.0.0.1:%d", port)) + }) + + t.Run("serves until the app context ends", func(t *testing.T) { + probe, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + host, port := "127.0.0.1", probe.Addr().(*net.TCPAddr).Port + require.NoError(t, probe.Close()) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + gs, err := NewGrpcServer(ctx, &log.Logger, &common.ServerConfig{ + GrpcMaxRecvMsgSize: util.IntPtr(1 << 20), GrpcMaxSendMsgSize: util.IntPtr(1 << 20), + GrpcHostV4: &host, GrpcPortV4: &port, + }, nil) + require.NoError(t, err) + done := make(chan error, 1) + go func() { done <- gs.Start(&log.Logger) }() + + addr := fmt.Sprintf("127.0.0.1:%d", port) + require.Eventually(t, func() bool { + c, err := net.DialTimeout("tcp", addr, 100*time.Millisecond) + if err != nil { + return false + } + _ = c.Close() + return true + }, 5*time.Second, 10*time.Millisecond) + + cancel() + select { + case err := <-done: + assert.NoError(t, err, "graceful stop ends Serve cleanly") + case <-time.After(5 * time.Second): + t.Fatal("Start did not return after the app context ended") + } + }) +} diff --git a/erpc/http_server.go b/erpc/http_server.go index 92c7eccef..64f76c024 100644 --- a/erpc/http_server.go +++ b/erpc/http_server.go @@ -59,6 +59,7 @@ type HttpServer struct { resolvedResponseHeaders map[string]string subscriptionManager *SubscriptionManager activeWsConns sync.Map // connId -> *WsConnection + wsConnLimiter *wsConnLimiter reqMaxTimeout time.Duration } @@ -116,6 +117,9 @@ func NewHttpServer( } if cfg != nil { + if ws := cfg.WebSocket; ws != nil { + srv.wsConnLimiter = newWsConnLimiter(ws.MaxConnections, ws.MaxConnectionsPerIP) + } srv.trustedForwarderIPs = make(map[string]struct{}, len(cfg.TrustedIPForwarders)) var forwarders []string forwarders = append(forwarders, cfg.TrustedIPForwarders...) @@ -608,6 +612,18 @@ func (s *HttpServer) createRequestHandler() http.Handler { method, _ := nq.Method() rlg := lg.With().Str("method", method).Logger() + // Subscription methods need a client WebSocket connection to + // deliver notifications to, so they are served only on the + // WebSocket path (handleWebSocket, which returns before this + // loop). Over HTTP they would sweep the upstreams and fail with + // an opaque error; reject them as unsupported (-32601) up front. + if IsSubscriptionMethod(method) { + err := common.NewErrNotImplemented(method + " is only supported over a WebSocket connection") + responses[index] = processErrorBody(&rlg, &startedAt, nq, err, &common.TRUE) + common.EndRequestSpan(requestCtx, nil, err) + return + } + shouldHandleMethod, err := isMethodAllowed(project, method) if err != nil { responses[index] = processErrorBody(&rlg, &startedAt, nq, err, &common.TRUE) @@ -722,6 +738,7 @@ func (s *HttpServer) createRequestHandler() http.Handler { nq, common.NewErrAuthUnauthorized( "", + "unauthorized", "admin is not enabled for this project", ), s.serverCfg.IncludeErrorDetails, @@ -813,7 +830,8 @@ func (s *HttpServer) createRequestHandler() http.Handler { common.InjectHTTPResponseTraceContext(httpCtx, w) // Notifications get no entry; a request of only notifications gets an - // empty body. + // empty body. An empty batch ("[]") still has len(responses)==0 and + // must keep writing "[]" via the isBatch branch below. replies := responses if slices.Contains(notifications, true) { replies = make([]interface{}, 0, len(responses)) @@ -831,7 +849,7 @@ func (s *HttpServer) createRequestHandler() http.Handler { } }() - if len(replies) == 0 { + if len(replies) == 0 && len(responses) > 0 { s.writeBatchExecHeaders(httpCtx, w, responses) s.writeCostHeaders(httpCtx, w, responses) w.WriteHeader(http.StatusOK) @@ -974,7 +992,11 @@ func (s *HttpServer) parseUrlPath( isHealthCheck = true segments = segments[:len(segments)-1] // Remove healthcheck segment } else if len(segments) == 0 || (len(segments) == 1 && segments[0] == "") { - if !(isPost || isOptions) { + // A bare root path is a healthcheck only when it is not a WebSocket + // upgrade. Otherwise wss://host/ (common with domain aliasing) would be + // classified as health and 404'd on the projects listener / mis-served + // as health on the primary, never reaching the WS dispatch. + if !(isPost || isOptions) && !isWebSocketUpgradeRequest(r, s.webSocketEnabled()) { isHealthCheck = true segments = nil } @@ -1744,6 +1766,7 @@ func (r *HttpJsonRpcErrorResponse) MarshalZerologObject(e *zerolog.Event) { func processErrorBody(logger *zerolog.Logger, startedAt *time.Time, nq *common.NormalizedRequest, origErr error, includeErrorDetails *bool, architectureHint ...common.NetworkArchitecture) interface{} { err := origErr + auth.RecordAuthFailed(nq, err) // Build the response first, then log with it. Authentication runs before // network resolution, so its caller passes the URL-parsed architecture as a @@ -1957,6 +1980,76 @@ func handleErrorResponse( ) } +// prewarmInterval is how often the prewarm reconciler re-scans for WS-capable +// networks that still need their newHeads subscription established. Var (not +// const) so tests can compress it. +var prewarmInterval = 15 * time.Second + +// PrewarmUpstreamSubscriptions proactively establishes the upstream newHeads +// subscription for every WS-capable network, without waiting for a client +// eth_subscribe. This keeps the EVM state poller's latest-block tip fed by +// pushed heads (cheap) instead of falling back to RPC-polling `latest` every +// cycle (expensive) — so an instance that never receives a client subscription still +// polls efficiently. +// +// It runs as a reconciler because networks initialize asynchronously (the +// registry's background initializer tasks): each tick it attempts to bootstrap +// any network not yet warmed, until ctx is cancelled. bootstrapNetwork is +// idempotent per network, so re-scans are cheap. Networks with no WS upstreams +// are skipped permanently — the state poller handles head tracking for +// HTTP-only networks via RPC polling. No-op while the WebSocket server is +// disabled. +func (s *HttpServer) PrewarmUpstreamSubscriptions(ctx context.Context) { + if s.subscriptionManager == nil || s.erpc == nil || !s.webSocketEnabled() { + return + } + + // indexer network key (project/network) -> bootstrapped or definitively + // skipped (HTTP-only). + settled := make(map[string]struct{}) + + reconcile := func() { + for _, prj := range s.erpc.GetProjects() { + for _, nw := range prj.GetNetworks() { + id := nw.Id() + key := indexerNetworkKey(nw) + if _, done := settled[key]; done { + continue + } + err := s.subscriptionManager.bootstrapNetwork(ctx, nw) + switch { + case err == nil: + settled[key] = struct{}{} + s.logger.Info().Str("networkId", id). + Msg("prewarmed upstream newHeads subscription") + case common.HasErrorCode(err, common.ErrCodeNoWsUpstreamAvailable): + // HTTP-only network — nothing to prewarm; don't retry. + settled[key] = struct{}{} + s.logger.Debug().Str("networkId", id). + Msg("no WS upstreams; skipping subscription prewarm") + default: + // Transient (network still initializing, upstreams not ready, + // circuit breaker open). Leave unsettled; retried next tick. + s.logger.Debug().Err(err).Str("networkId", id). + Msg("subscription prewarm not ready yet, will retry") + } + } + } + } + + reconcile() // immediate first pass + ticker := time.NewTicker(prewarmInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + reconcile() + } + } +} + func (s *HttpServer) Start(logger *zerolog.Logger) error { // Validate that at least one server is configured if s.serverV4 == nil && s.serverV6 == nil { diff --git a/erpc/http_server_blocknumber_integrity_test.go b/erpc/http_server_blocknumber_integrity_test.go index 35c06ee99..c29505e34 100644 --- a/erpc/http_server_blocknumber_integrity_test.go +++ b/erpc/http_server_blocknumber_integrity_test.go @@ -396,20 +396,23 @@ func TestHttpServer_EthBlockNumberIntegrity(t *testing.T) { send, ntw, shutdown := bniBoot(t, bniConfig(true, integrityOn())) defer shutdown() - first, _ := bniSend(t, send, nil, nil) - require.Equal(t, bniHealthyHead, first, "precondition: fresh-path correction must work") - - // The fix may either write the corrected value back (entry >= tip) or - // skip caching the stale original (no entry). Both satisfy the - // invariant; today the stale 0x800 lands in the cache (bug B1). + // With the head known, an enforced eth_blockNumber is answered from the + // served head before dispatch and writes nothing to the cache. The path + // that still dispatches AND persists is a request that opts out of + // enforcement: the lagging upstream answers first and its raw 0x800 + // lands in the shared cache. + raw, _ := bniSend(t, send, nil, map[string]string{"enforce-highest-block": "false"}) + require.Equal(t, bniLaggingHead, raw, "precondition: the opt-out request is answered by the lagging upstream") v, ok := bniWaitForCachedValue(ntw, 2*time.Second) - if !ok { - t.Log("no eth_blockNumber cache entry written — acceptable (stale value skipped)") - return - } - assert.GreaterOrEqual(t, v, bniHealthyHead, - "cache poisoned: the forward path persisted the stale upstream value (0x%x) instead of the corrected value it served to the client (0x%x)", - v, first) + require.True(t, ok, "precondition: the dispatched opt-out response must reach the cache, or this test exercises nothing") + t.Logf("cache now holds eth_blockNumber=0x%x", v) + + // Every enforced client reading that entry must still see the tip. + got, hdrs := bniSend(t, send, nil, nil) + assert.Equal(t, "HIT", hdrs["X-Erpc-Cache"], "the enforced read must come from the entry the opt-out request wrote") + assert.Equal(t, bniHealthyHead, got, + "cache poisoned: a stale upstream value persisted by the forward path (0x%x) reached an enforced client below the tip (0x%x)", + v, bniHealthyHead) }) // ── B2 isolated: a stale entry already in the (possibly shared) cache ─── @@ -449,7 +452,7 @@ func TestHttpServer_EthBlockNumberIntegrity(t *testing.T) { // The request is explicitly pinned to the lagging upstream. Per the // selector-scoped served-tip semantics (#912, honored by max-mode - // tipCandidateUpstreams when the ctx carries the request), "latest" + // EvmRoutableUpstreams when the ctx carries the request), "latest" // must be decided AMONG the pinned subset — advertising the // network-wide tip promises a block rpc1 cannot serve. got, _ := bniSend(t, send, map[string]string{"X-ERPC-Use-Upstream": "rpc1"}, nil) diff --git a/erpc/http_server_blocknumber_servedtip_test.go b/erpc/http_server_blocknumber_servedtip_test.go index 2022e7df4..293175a0e 100644 --- a/erpc/http_server_blocknumber_servedtip_test.go +++ b/erpc/http_server_blocknumber_servedtip_test.go @@ -2,42 +2,38 @@ package erpc import ( "context" + "fmt" + "net/http" + "strings" + "sync/atomic" "testing" "time" "github.com/erpc/erpc/common" "github.com/erpc/erpc/internal/policy" "github.com/erpc/erpc/util" + "github.com/h2non/gock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// This suite pins eth_blockNumber semantics when the majority served tip is -// enabled for "latest" (EvmServedTipConfig). In that mode "latest" +// This suite pins eth_blockNumber semantics on the default config. "latest" // interpolation (resolveBlockTagToHex) anchors block-tagged methods — eth_call, -// eth_getLogs, ... — to the majority tip before forwarding, so eth_blockNumber -// must be pinned to that SAME tip: floor-only enforcement lets a fresher head +// eth_getLogs, ... — to the served head before forwarding, so eth_blockNumber +// must be pinned to that SAME head: floor-only enforcement lets a fresher head // (or a cache entry written from one) through, and clients observe // eth_blockNumber AHEAD of the block "latest" actually executes at (an // on-chain block.number read via eth_call comes back BELOW eth_blockNumber, // even when the same upstream serves both calls). // // Topology (shared bni* helpers): rpc1 head 0x800, rpc2 head 0x1000. With two -// upstreams the majority tip is the LOWER head (common.PickServedTip: N=2 → the -// 2nd-highest), so the served tip is 0x800 while rpc2 answers 0x1000. - -// bniServedTipConfig is bniConfig (modern, no deprecated integrity block) with -// the majority served tip enabled for "latest". -func bniServedTipConfig(withCache bool) *common.Config { - cfg := bniConfig(withCache, nil) - cfg.Projects[0].Networks[0].Evm.ServedTip = &common.EvmServedTipConfig{EnabledFor: []string{"latest"}} - return cfg -} +// upstreams the default quorum (2) serves the LOWER head, so the served head is +// 0x800 while rpc2 answers 0x1000. // bniServedTipBoot mirrors bniBoot with a caller-chosen upstream order and a -// wait condition for served-tip mode: max==0x1000 requires rpc2's poller and -// majority==0x800 requires BOTH pollers, so together they make the tip -// deterministic before any assertion runs. +// wait condition for the two-upstream topology: every head known with +// max==0x1000, and the served head 0x800, so the head is deterministic before +// any assertion runs. func bniServedTipBoot(t *testing.T, cfg *common.Config, order ...string) ( func(body string, headers map[string]string, queryParams map[string]string) (int, map[string]string, string), *Network, @@ -55,10 +51,11 @@ func bniServedTipBoot(t *testing.T, cfg *common.Config, order ...string) ( require.Eventually(t, func() bool { ctx := context.Background() - return ntw.evmHeadReference(ctx, false).Max == bniHealthyHead && + ceiling, known := ntw.futureBlockCeiling(ctx) + return known && ceiling == bniHealthyHead && ntw.EvmHighestLatestBlockNumber(ctx) == bniLaggingHead }, 5*time.Second, 50*time.Millisecond, - "pollers must learn both heads (majority tip=0x800, max=0x1000) before the scenario starts") + "pollers must learn both heads (served=0x800, max=0x1000) before the scenario starts") return sendRequest, ntw, shutdown } @@ -73,13 +70,13 @@ func TestHttpServer_EthBlockNumberServedTipPin(t *testing.T) { setupBniPollerMocks() setupBniBlockNumberMocks() - // Fresh upstream first: rpc2 serves 0x1000, above the majority tip. - send, _, shutdown := bniServedTipBoot(t, bniServedTipConfig(false), "rpc2", "rpc1") + // Fresh upstream first: rpc2 serves 0x1000, above the served tip. + send, _, shutdown := bniServedTipBoot(t, bniConfig(false, nil), "rpc2", "rpc1") defer shutdown() got, hdrs := bniSend(t, send, nil, nil) assert.Equal(t, bniLaggingHead, got, - "eth_blockNumber must be pinned to the majority tip (0x%x) that \"latest\" interpolation uses, not the fresher upstream head (0x%x)", + "eth_blockNumber must be pinned to the served tip (0x%x) that \"latest\" interpolation uses, not the fresher upstream head (0x%x)", bniLaggingHead, bniHealthyHead) assert.Empty(t, hdrs["X-Erpc-Upstream"], "the pin discards any upstream answer, so the served tip must be answered without dispatching") @@ -91,18 +88,18 @@ func TestHttpServer_EthBlockNumberServedTipPin(t *testing.T) { setupBniPollerMocks() setupBniBlockNumberMocks() - send, ntw, shutdown := bniServedTipBoot(t, bniServedTipConfig(true), "rpc2", "rpc1") + send, ntw, shutdown := bniServedTipBoot(t, bniConfig(true, nil), "rpc2", "rpc1") defer shutdown() // A fresh rpc2 response planted in the (possibly shared) cache — the // dominant path in practice: eth_blockNumber is realtime-cached, so - // most reads are HITs carrying a value above the majority tip. + // most reads are HITs carrying a value above the served tip. bniSeedCache(t, ntw, "0x1000") got, hdrs := bniSend(t, send, nil, nil) t.Logf("response: block=0x%x cacheHeader=%q", got, hdrs["X-Erpc-Cache"]) assert.Equal(t, bniLaggingHead, got, - "a cache-hit response above the majority tip must be pinned down just like a fresh response") + "a cache-hit response above the served tip must be pinned down just like a fresh response") }) t.Run("Baseline_ResponseAtServedTipPassesThrough", func(t *testing.T) { @@ -111,8 +108,8 @@ func TestHttpServer_EthBlockNumberServedTipPin(t *testing.T) { setupBniPollerMocks() setupBniBlockNumberMocks() - // Lagging upstream first: rpc1 serves 0x800 == the majority tip. - send, _, shutdown := bniServedTipBoot(t, bniServedTipConfig(false), "rpc1", "rpc2") + // Lagging upstream first: rpc1 serves 0x800 == the served tip. + send, _, shutdown := bniServedTipBoot(t, bniConfig(false, nil), "rpc1", "rpc2") defer shutdown() got, _ := bniSend(t, send, nil, nil) @@ -125,7 +122,7 @@ func TestHttpServer_EthBlockNumberServedTipPin(t *testing.T) { setupBniPollerMocks() setupBniBlockNumberMocks() - send, _, shutdown := bniServedTipBoot(t, bniServedTipConfig(false), "rpc2", "rpc1") + send, _, shutdown := bniServedTipBoot(t, bniConfig(false, nil), "rpc2", "rpc1") defer shutdown() // enforce-highest-block=false must bypass the pin exactly like it @@ -133,4 +130,56 @@ func TestHttpServer_EthBlockNumberServedTipPin(t *testing.T) { got, _ := bniSend(t, send, nil, map[string]string{"enforce-highest-block": "false"}) assert.Equal(t, bniHealthyHead, got) }) + + t.Run("LocalAnswerRefreshesBallotPollers", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + + // Heads move up 5 blocks once `moved` flips. Registered before the + // static bootstrap mocks so gock matches them first; until the flip + // they decline and the static heads (0x800/0x1000) answer. + var moved atomic.Bool + for host, head := range map[string]string{ + "http://rpc1.localhost": "0x805", + "http://rpc2.localhost": "0x1005", + } { + gock.New(host).Post("").Persist(). + Filter(func(r *http.Request) bool { + b := util.SafeReadBody(r) + return moved.Load() && strings.Contains(b, "eth_getBlockByNumber") && strings.Contains(b, "latest") + }). + Reply(200). + JSON([]byte(fmt.Sprintf(`{"result":{"number":"%s","timestamp":"0x6702a8f0"}}`, head))) + } + setupBniPollerMocks() + setupBniBlockNumberMocks() + + // A 30s ticker cannot poll during the test; a 100ms debounce lets each + // eth_blockNumber answer kick a refresh almost at once. + cfg := bniConfig(false, nil) + for _, u := range cfg.Projects[0].Upstreams { + u.Evm.StatePollerInterval = common.Duration(30 * time.Second) + u.Evm.StatePollerDebounce = common.Duration(100 * time.Millisecond) + } + // rpc3 is a static serving range (upper.exactBlock below its head), first + // in routing order: it never votes, so it must not take a refresh slot + // from the voters rpc2 and rpc1. + rangeEnd := int64(0x700) + cfg.Projects[0].Upstreams[2].Evm.BlockAvailability = &common.EvmBlockAvailabilityConfig{ + Upper: &common.EvmAvailabilityBoundConfig{ExactBlock: &rangeEnd}, + } + send, _, shutdown := bniServedTipBoot(t, cfg, "rpc3", "rpc2", "rpc1") + defer shutdown() + + got, _ := bniSend(t, send, nil, nil) + require.Equal(t, bniLaggingHead, got) + + moved.Store(true) + require.Eventually(t, func() bool { + got, _ := bniSend(t, send, nil, nil) + return got == bniLaggingHead+5 + }, 3*time.Second, 50*time.Millisecond, + "eth_blockNumber must refresh the ballot's pollers and reach the moved served head (0x%x) without waiting for the 30s poll interval", + bniLaggingHead+5) + }) } diff --git a/erpc/http_server_envelope_test.go b/erpc/http_server_envelope_test.go index 69e4715f9..49d218df5 100644 --- a/erpc/http_server_envelope_test.go +++ b/erpc/http_server_envelope_test.go @@ -166,4 +166,23 @@ func TestHttpServer_RequestEnvelope(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode, "body: %s", string(respBody)) assert.Contains(t, string(respBody), "0x1a2b3c") }) + + // Empty batch body must stay "[]". The Solana notification path writes an + // empty body when every reply was filtered out; that branch must not also + // catch len(requests)==0 from "[]", or EVM clients that JSON.parse the + // response will throw on a 200 with no body. + t.Run("EmptyBatchBodyIsEmptyArray", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + defer util.AssertNoPendingMocks(t, 0) + + sendRequest, _, _, shutdown, _ := createServerTestFixtures(envelopeTestConfig(), t) + defer shutdown() + time.Sleep(1500 * time.Millisecond) + + statusCode, _, respBody := sendRequest(`[]`, nil, nil) + assert.Equal(t, http.StatusOK, statusCode, "body: %s", respBody) + assert.Equal(t, `[]`, strings.TrimSpace(respBody)) + }) } diff --git a/erpc/http_server_h2c_test.go b/erpc/http_server_h2c_test.go index c72793562..31b03a0e9 100644 --- a/erpc/http_server_h2c_test.go +++ b/erpc/http_server_h2c_test.go @@ -270,6 +270,9 @@ func TestHttpServer_H2C_ShutdownGoAwaysAndDrainsInFlightStream(t *testing.T) { _, err = bodyW.Write([]byte(reqBody[:20])) require.NoError(t, err) time.Sleep(200 * time.Millisecond) + // CanTakeNewRequest is the GOAWAY signal below; it must hold before Shutdown + // so a full stream limit can't pass for a GOAWAY. + require.True(t, cc.CanTakeNewRequest(), "h2c connection refused new streams before Shutdown started") sigterm() @@ -289,7 +292,9 @@ func TestHttpServer_H2C_ShutdownGoAwaysAndDrainsInFlightStream(t *testing.T) { default: } - require.Eventually(t, func() bool { return cc.State().Closing }, 3*time.Second, 10*time.Millisecond, + // State().Closing misses a received GOAWAY in go1.27's x/net wrapper + // (golang/go#82365); CanTakeNewRequest reports it under both clients. + require.Eventually(t, func() bool { return !cc.CanTakeNewRequest() }, 3*time.Second, 10*time.Millisecond, "h2c connection never received GOAWAY after Shutdown started") newCtx, cancelNew := context.WithTimeout(context.Background(), 2*time.Second) diff --git a/erpc/http_server_headers_test.go b/erpc/http_server_headers_test.go index 5c4172588..08b96d796 100644 --- a/erpc/http_server_headers_test.go +++ b/erpc/http_server_headers_test.go @@ -62,6 +62,16 @@ func TestHttpServer_HeadersCoverage(t *testing.T) { assert.Contains(t, []int{200, 400}, statusCode) assertCounterHeadersPresent(t, headers) }) + + t.Run("null body is a parse error, not a panic", func(t *testing.T) { + sendRequest, _, _, shutdown, _ := createServerTestFixtures(cfg, t) + defer shutdown() + + statusCode, _, body := sendRequest(`null`, nil, nil) + assert.Equal(t, 400, statusCode, "body=%s", body) + assert.Contains(t, body, "-32700") + assert.NotContains(t, body, "panic") + }) } // TestExecState_CounterAggregation verifies that totals (Attempts / diff --git a/erpc/http_server_local_answers_test.go b/erpc/http_server_local_answers_test.go new file mode 100644 index 000000000..5ad64fb53 --- /dev/null +++ b/erpc/http_server_local_answers_test.go @@ -0,0 +1,159 @@ +package erpc + +import ( + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/bytedance/sonic" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The methods eRPC answers itself, over HTTP, against one upstream that +// answers each of them with its own (different) value: on by default with no +// upstream call, routed to the upstream once a network turns them off, and +// routed when a request skips the cache read. +func TestHttpServer_LocalAnswers(t *testing.T) { + upstreamAnswers := map[string]string{ + "eth_chainId": `"0x7b"`, + "net_version": `"7777"`, + "net_listening": `false`, + "net_peerCount": `"0x40"`, + "web3_clientVersion": `"Geth/v1.14.0"`, + "eth_syncing": `{"startingBlock":"0x0","currentBlock":"0x1","highestBlock":"0x2"}`, + } + localAnswers := map[string]any{ + "eth_chainId": "0x7b", + "net_version": "123", + "net_listening": true, + "net_peerCount": "0x1", + "eth_syncing": false, + } + require.ElementsMatch(t, common.LocalAnswerMethods(), keys(upstreamAnswers), "every local answer is covered") + + var mu sync.Mutex + calls := map[string]int{} + callsOf := func(method string) int { + mu.Lock() + defer mu.Unlock() + return calls[method] + } + mockUpstream := func() { + util.ResetGock() + for method, result := range upstreamAnswers { + gock.New("http://rpc1.localhost").Post("").Persist(). + Filter(func(r *http.Request) bool { + // gock filters see every outgoing request, the test client's too. + if r.URL.Host != "rpc1.localhost" || !strings.Contains(util.SafeReadBody(r), `"`+method+`"`) { + return false + } + mu.Lock() + calls[method]++ + mu.Unlock() + return true + }). + Reply(200). + JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":` + result + `}`)) + } + gock.New("http://rpc1.localhost").Post("").Persist(). + Filter(func(r *http.Request) bool { return strings.Contains(util.SafeReadBody(r), "eth_getBlockByNumber") }). + Reply(200). + JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":{"number":"0x100","timestamp":"0x6702a8f0"}}`)) + } + config := func(off map[string]bool) *common.Config { + return &common.Config{ + Server: &common.ServerConfig{MaxTimeout: common.Duration(5 * time.Second).Ptr()}, + Projects: []*common.ProjectConfig{{ + Id: "test_project", + Networks: []*common.NetworkConfig{{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123, LocalAnswers: off}, + }}, + Upstreams: []*common.UpstreamConfig{{ + Id: "rpc1", + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + StatePollerInterval: common.Duration(time.Hour), + SkipSyncingCheck: util.BoolPtr(true), + }, + }}, + }}, + } + } + call := func(t *testing.T, send func(string, map[string]string, map[string]string) (int, map[string]string, string), method string, headers map[string]string) (result any, upstreamCalls int) { + t.Helper() + before := callsOf(method) + status, _, body := send(`{"jsonrpc":"2.0","id":7,"method":"`+method+`","params":[]}`, headers, nil) + require.Equal(t, http.StatusOK, status, "%s: %s", method, body) + var parsed map[string]any + require.NoError(t, sonic.UnmarshalString(body, &parsed), "%s: %s", method, body) + require.Nil(t, parsed["error"], "%s: %s", method, body) + return parsed["result"], callsOf(method) - before + } + upstreamValue := func(t *testing.T, method string) any { + var v any + require.NoError(t, sonic.UnmarshalString(upstreamAnswers[method], &v)) + return v + } + + t.Run("on by default: eRPC answers, no upstream call", func(t *testing.T) { + mockUpstream() + defer util.ResetGock() + send, _, _, shutdown, _ := createServerTestFixtures(config(nil), t) + defer shutdown() + + for method := range upstreamAnswers { + got, n := call(t, send, method, nil) + assert.Zero(t, n, "%s must not reach the upstream", method) + if method == "web3_clientVersion" { + assert.True(t, strings.HasPrefix(got.(string), "erpc/"), "%s = %v", method, got) + continue + } + assert.Equal(t, localAnswers[method], got, method) + } + + got, n := call(t, send, "net_peerCount", map[string]string{"X-ERPC-Skip-Cache-Read": "true"}) + assert.Equal(t, 1, n, "skip-cache-read asks the upstream") + assert.Equal(t, upstreamValue(t, "net_peerCount"), got) + }) + + t.Run("off per method: routed to the upstream", func(t *testing.T) { + mockUpstream() + defer util.ResetGock() + off := map[string]bool{} + for method := range upstreamAnswers { + off[method] = false + } + send, _, _, shutdown, _ := createServerTestFixtures(config(off), t) + defer shutdown() + + for method := range upstreamAnswers { + got, n := call(t, send, method, nil) + assert.Equal(t, 1, n, "%s must reach the upstream", method) + assert.Equal(t, upstreamValue(t, method), got, method) + } + }) + + t.Run("a method without a local answer is refused in config", func(t *testing.T) { + cfg := config(map[string]bool{"eth_getBalance": false}) + require.NoError(t, cfg.SetDefaults(nil)) + err := cfg.Validate() + require.ErrorContains(t, err, "localAnswers.eth_getBalance") + }) +} + +func keys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/erpc/http_server_paths_test.go b/erpc/http_server_paths_test.go new file mode 100644 index 000000000..dded19d83 --- /dev/null +++ b/erpc/http_server_paths_test.go @@ -0,0 +1,1044 @@ +package erpc + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "errors" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const httpPathsAdminSecret = "paths-admin-secret" + +// httpPathsProject is a minimal EVM project served by the mocked rpc1 upstream. +func httpPathsProject(id string) *common.ProjectConfig { + return &common.ProjectConfig{ + Id: id, + Networks: []*common.NetworkConfig{{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + Alias: "main", + }}, + Upstreams: []*common.UpstreamConfig{{ + Id: "rpc1", + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }}, + } +} + +// httpPathsStart builds a full eRPC + HttpServer from cfg and serves the +// server's real (gzip + timeout wrapped) handler over httptest. +func httpPathsStart(t *testing.T, cfg *common.Config) (*HttpServer, string) { + t.Helper() + logger := log.Logger + ctx, cancel := context.WithCancel(context.Background()) + if cfg.Server == nil { + cfg.Server = &common.ServerConfig{} + } + cfg.Server.WaitBeforeShutdown = common.Duration(0).Ptr() + if cfg.RateLimiters == nil { + cfg.RateLimiters = &common.RateLimiterConfig{} + } + require.NoError(t, cfg.SetDefaults(nil)) + cfg.Server.ListenV4 = util.BoolPtr(true) + + ssr, err := data.NewSharedStateRegistry(ctx, &logger, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: "memory", + Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}, + }, + }) + require.NoError(t, err) + instance, err := NewERPC(ctx, &logger, ssr, nil, nil, cfg) + require.NoError(t, err) + instance.Bootstrap(ctx) + + srv, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, instance) + require.NoError(t, err) + ts := httptest.NewServer(srv.serverV4.Handler) + t.Cleanup(func() { + ts.Close() + cancel() + }) + return srv, ts.URL +} + +func httpPathsDo(t *testing.T, method, rawURL string, body []byte, hdr map[string]string) (int, http.Header, string) { + t.Helper() + var rd io.Reader + if body != nil { + rd = bytes.NewReader(body) + } + req, err := http.NewRequest(method, rawURL, rd) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + for k, v := range hdr { + if k == "Host" { + req.Host = v + continue + } + req.Header.Set(k, v) + } + client := &http.Client{Timeout: 10 * time.Second, Transport: &http.Transport{DisableCompression: true}} + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + return resp.StatusCode, resp.Header, string(b) +} + +func httpPathsErrCode(t *testing.T, body string) (int, string) { + t.Helper() + var out struct { + Error struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + require.NoError(t, json.Unmarshal([]byte(body), &out), "body: %s", body) + return out.Error.Code, out.Error.Message +} + +func httpPathsMockBalance() { + gock.New("http://rpc1.localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { return strings.Contains(util.SafeReadBody(r), "eth_getBalance") }). + Reply(200). + JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":"0x2a"}`)) +} + +const httpPathsBalanceBody = `{"jsonrpc":"2.0","id":7,"method":"eth_getBalance","params":["0x0000000000000000000000000000000000000001","latest"]}` + +func TestHttpPaths_MainServerRouting(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + httpPathsMockBalance() + + prj := httpPathsProject("test_project") + prj.CORS = &common.CORSConfig{ + AllowedOrigins: []string{"(", "https://ok.example"}, + AllowedMethods: []string{"POST", "OPTIONS"}, + AllowedHeaders: []string{"Content-Type"}, + ExposedHeaders: []string{"X-ERPC-Version"}, + AllowCredentials: util.BoolPtr(true), + MaxAge: 600, + } + prj.IgnoreMethods = []string{"eth_sign*"} + prj.ForwardHeaders = []string{"X-Fwd-Token"} + prj.TrustUserIdHeader = true + + authPrj := httpPathsProject("auth_project") + authPrj.Auth = &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{ + {Type: common.AuthTypeSecret, Secret: &common.SecretStrategyConfig{Value: "p-secret"}}, + }} + + cfg := &common.Config{ + Server: &common.ServerConfig{ + Aliasing: &common.AliasingConfig{Rules: []*common.AliasingRuleConfig{ + {MatchDomain: "(", ServeProject: "nope"}, + {MatchDomain: "*.alias.test", ServeProject: "test_project", ServeArchitecture: "evm"}, + }}, + }, + Admin: &common.AdminConfig{ + Auth: &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{ + {Type: common.AuthTypeSecret, Secret: &common.SecretStrategyConfig{Value: httpPathsAdminSecret}}, + }}, + CORS: &common.CORSConfig{AllowedOrigins: []string{"https://admin.example"}, AllowedMethods: []string{"POST"}}, + DenyMethods: []string{"erpc_taxonomy"}, + }, + Projects: []*common.ProjectConfig{prj, authPrj}, + } + _, base := httpPathsStart(t, cfg) + post := func(path, body string, hdr map[string]string) (int, http.Header, string) { + return httpPathsDo(t, http.MethodPost, base+path, []byte(body), hdr) + } + + t.Run("UrlPathErrorsAre400WithInvalidArgument", func(t *testing.T) { + cases := []struct { + path string + msg string + }{ + {"/", "project is required"}, + {"/a/b/c/d", "must only provide"}, + {"/test_project/btc/1", "architecture is not valid"}, + } + for _, tc := range cases { + status, _, body := post(tc.path, httpPathsBalanceBody, nil) + assert.Equal(t, http.StatusBadRequest, status, tc.path) + code, _ := httpPathsErrCode(t, body) + assert.Equal(t, int(common.JsonRpcErrorInvalidArgument), code, tc.path) + assert.Contains(t, body, tc.msg, tc.path) + } + }) + + t.Run("UnknownProjectIs404", func(t *testing.T) { + status, _, body := post("/no_such_project/evm/123", httpPathsBalanceBody, nil) + assert.Equal(t, http.StatusNotFound, status) + assert.Contains(t, body, "no_such_project") + }) + + t.Run("ProjectOnlyPathNeedsNetwork", func(t *testing.T) { + status, _, body := post("/test_project", httpPathsBalanceBody, nil) + assert.Equal(t, http.StatusBadRequest, status) + assert.Contains(t, body, "architecture and chain must be provided") + }) + + t.Run("NetworkFromBodyAndAliasAndDomainRouting", func(t *testing.T) { + bodyWithNet := `{"jsonrpc":"2.0","id":7,"networkId":"evm:123","method":"eth_getBalance","params":["0x0000000000000000000000000000000000000001","latest"]}` + for name, call := range map[string]func() (int, http.Header, string){ + "body": func() (int, http.Header, string) { return post("/test_project", bodyWithNet, nil) }, + "alias": func() (int, http.Header, string) { return post("/test_project/main", httpPathsBalanceBody, nil) }, + "domain": func() (int, http.Header, string) { return post("/123", httpPathsBalanceBody, map[string]string{"Host": "rpc.alias.test:8080"}) }, + } { + status, hdr, body := call() + assert.Equal(t, http.StatusOK, status, name) + assert.Contains(t, body, `"result":"0x2a"`, name) + assert.Equal(t, "MISS", hdr.Get("X-ERPC-Cache"), name) + } + }) + + t.Run("IgnoredMethodIsUnsupported", func(t *testing.T) { + status, _, body := post("/test_project/evm/123", `{"jsonrpc":"2.0","id":3,"method":"eth_signTypedData","params":[]}`, nil) + assert.Equal(t, http.StatusOK, status) + code, msg := httpPathsErrCode(t, body) + assert.Equal(t, int(common.JsonRpcErrorUnsupportedException), code) + assert.Equal(t, "method not supported: eth_signTypedData", msg) + assert.Contains(t, body, `"id":3`) + }) + + t.Run("ForwardHeadersReachUpstream", func(t *testing.T) { + gock.New("http://rpc1.localhost"). + Post(""). + MatchHeader("X-Fwd-Token", "tok-1"). + Filter(func(r *http.Request) bool { return strings.Contains(util.SafeReadBody(r), "eth_getCode") }). + Reply(200). + JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":"0xfeed"}`)) + status, _, body := post("/test_project/evm/123", + `{"jsonrpc":"2.0","id":1,"method":"eth_getCode","params":["0x0000000000000000000000000000000000000002","latest"]}`, + map[string]string{"X-Fwd-Token": "tok-1", common.HeaderUserId: "gateway-user"}) + assert.Equal(t, http.StatusOK, status) + assert.Contains(t, body, `"result":"0xfeed"`) + }) + + t.Run("GzipRequestBodies", func(t *testing.T) { + status, _, body := post("/test_project/evm/123", "not-gzip", map[string]string{"Content-Encoding": "gzip"}) + assert.Equal(t, http.StatusBadRequest, status) + assert.Contains(t, body, "invalid gzip body") + + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, _ = zw.Write([]byte(httpPathsBalanceBody)) + require.NoError(t, zw.Close()) + status, _, body = httpPathsDo(t, http.MethodPost, base+"/test_project/evm/123", buf.Bytes(), map[string]string{"Content-Encoding": "gzip"}) + assert.Equal(t, http.StatusOK, status) + assert.Contains(t, body, `"result":"0x2a"`) + }) + + t.Run("BatchBodies", func(t *testing.T) { + status, _, body := post("/test_project/evm/123", `[{"jsonrpc":"2.0",`, nil) + assert.Equal(t, http.StatusBadRequest, status) + code, _ := httpPathsErrCode(t, body) + assert.Equal(t, int(common.JsonRpcErrorParseException), code) + + status, _, body = post("/test_project/evm/123", `[]`, nil) + assert.Equal(t, http.StatusOK, status) + assert.Equal(t, "[]", strings.TrimSpace(body)) + + status, hdr, body := post("/test_project/evm/123", "["+httpPathsBalanceBody+","+strings.Replace(httpPathsBalanceBody, `"id":7`, `"id":8`, 1)+"]", nil) + assert.Equal(t, http.StatusOK, status) + var arr []map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &arr), body) + assert.Len(t, arr, 2) + assert.Equal(t, "MISS", hdr.Get("X-ERPC-Cache")) + assert.NotEmpty(t, hdr.Get("X-ERPC-Attempts")) + }) + + t.Run("ProjectCors", func(t *testing.T) { + status, hdr, _ := httpPathsDo(t, http.MethodOptions, base+"/test_project/evm/123", nil, map[string]string{"Origin": "https://ok.example"}) + assert.Equal(t, http.StatusNoContent, status) + assert.Equal(t, "https://ok.example", hdr.Get("Access-Control-Allow-Origin")) + assert.Equal(t, "true", hdr.Get("Access-Control-Allow-Credentials")) + assert.Equal(t, "600", hdr.Get("Access-Control-Max-Age")) + assert.Equal(t, "POST, OPTIONS", hdr.Get("Access-Control-Allow-Methods")) + + status, hdr, _ = httpPathsDo(t, http.MethodOptions, base+"/test_project/evm/123", nil, map[string]string{"Origin": "https://evil.example"}) + assert.Equal(t, http.StatusNoContent, status) + assert.Empty(t, hdr.Get("Access-Control-Allow-Origin")) + + status, hdr, body := post("/test_project/evm/123", httpPathsBalanceBody, map[string]string{"Origin": "https://evil.example"}) + assert.Equal(t, http.StatusOK, status) + assert.Empty(t, hdr.Get("Access-Control-Allow-Origin")) + assert.Contains(t, body, `"result":"0x2a"`) + }) + + t.Run("ConsumerAuth", func(t *testing.T) { + status, _, body := post("/auth_project/evm/123", httpPathsBalanceBody, nil) + assert.Equal(t, http.StatusUnauthorized, status) + code, _ := httpPathsErrCode(t, body) + assert.Equal(t, int(common.JsonRpcErrorUnauthorized), code) + + status, _, body = post("/auth_project/evm/123", httpPathsBalanceBody, map[string]string{"Authorization": "Basic !!!"}) + assert.Equal(t, http.StatusOK, status) + assert.Contains(t, body, "illegal base64") + + status, _, body = post("/auth_project/evm/123", httpPathsBalanceBody, map[string]string{"X-ERPC-Secret-Token": "p-secret"}) + assert.Equal(t, http.StatusOK, status) + assert.Contains(t, body, `"result":"0x2a"`) + }) + + t.Run("Admin", func(t *testing.T) { + status, hdr, _ := httpPathsDo(t, http.MethodOptions, base+"/admin", nil, map[string]string{"Origin": "https://admin.example"}) + assert.Equal(t, http.StatusNoContent, status) + assert.Equal(t, "https://admin.example", hdr.Get("Access-Control-Allow-Origin")) + + status, _, body := post("/admin", `{"jsonrpc":"2.0","id":1,"method":"erpc_config","params":[]}`, map[string]string{"X-ERPC-Secret-Token": "wrong"}) + assert.Equal(t, http.StatusUnauthorized, status) + code, _ := httpPathsErrCode(t, body) + assert.Equal(t, int(common.JsonRpcErrorUnauthorized), code) + + status, _, body = post("/admin", `{"jsonrpc":"2.0","id":2,"method":"erpc_taxonomy","params":[]}`, map[string]string{"X-ERPC-Secret-Token": httpPathsAdminSecret}) + assert.Equal(t, http.StatusOK, status) + code, msg := httpPathsErrCode(t, body) + assert.Equal(t, int(common.JsonRpcErrorUnsupportedException), code) + assert.Equal(t, "method not supported: erpc_taxonomy", msg) + + status, _, body = post("/admin", `{"jsonrpc":"2.0","id":3,"method":"erpc_doesNotExist","params":[]}`, map[string]string{"X-ERPC-Secret-Token": httpPathsAdminSecret}) + assert.Equal(t, http.StatusOK, status) + _, msg = httpPathsErrCode(t, body) + assert.Contains(t, msg, "erpc_doesNotExist") + }) + + t.Run("HealthcheckThroughHandler", func(t *testing.T) { + status, _, body := httpPathsDo(t, http.MethodGet, base+"/healthcheck", nil, nil) + assert.Equal(t, http.StatusOK, status, body) + status, _, body = httpPathsDo(t, http.MethodGet, base+"/test_project/evm/123/healthcheck", nil, nil) + assert.Equal(t, http.StatusOK, status, body) + }) +} + +func TestHttpPaths_AuxServerOptions(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + httpPathsMockBalance() + + off := common.ExecutionHeadersOff + cfg := &common.Config{ + Server: &common.ServerConfig{ + ListenV6: util.BoolPtr(true), + ExecutionHeaders: &off, + CostHeaders: util.BoolPtr(true), + TrustedIPForwarders: []string{"10.0.0.0/8", " ", "10.0.0.0/99", "not-an-ip", "192.168.1.1"}, + TrustedIPHeaders: []string{" ", "X-Forwarded-For"}, + ResponseHeaders: map[string]string{ + "X-Paths-Static": "static-value", + "X-Paths-Empty": "${HTTP_PATHS_SURELY_UNSET_ENV}", + }, + }, + Indexer: &common.IndexerConfig{DedupWindowSize: 16}, + Projects: []*common.ProjectConfig{httpPathsProject("test_project")}, + } + srv, base := httpPathsStart(t, cfg) + require.NotNil(t, srv.serverV6, "listenV6 builds the IPv6 server") + + t.Run("HeadersOffCostOnCustomHeaders", func(t *testing.T) { + status, hdr, body := httpPathsDo(t, http.MethodPost, base+"/test_project/evm/123", []byte(httpPathsBalanceBody), nil) + assert.Equal(t, http.StatusOK, status) + assert.Contains(t, body, `"result":"0x2a"`) + assert.Empty(t, hdr.Get("X-ERPC-Version")) + assert.Empty(t, hdr.Get("X-ERPC-Attempts")) + assert.Equal(t, "static-value", hdr.Get("X-Paths-Static")) + _, has := hdr["X-Paths-Empty"] + assert.False(t, has) + assert.Equal(t, "1", hdr.Get("X-ERPC-Calls")) + assert.Equal(t, "1", hdr.Get("X-ERPC-Billable")) + assert.Equal(t, "eth_getBalance", hdr.Get("X-ERPC-Methods")) + assert.Equal(t, "evm:123", hdr.Get("X-ERPC-Network-Id")) + assert.Equal(t, "main", hdr.Get("X-ERPC-Network-Alias")) + }) + + t.Run("AdminWithoutConfigFailsAuth", func(t *testing.T) { + status, _, body := httpPathsDo(t, http.MethodPost, base+"/admin", []byte(`{"jsonrpc":"2.0","id":1,"method":"erpc_config","params":[]}`), nil) + // With no admin config, NewHttpServer still routes /admin; ERPC has no + // admin auth registry so authentication fails as an internal error. + assert.Equal(t, http.StatusOK, status) + code, msg := httpPathsErrCode(t, body) + assert.Equal(t, -32603, code) + assert.Equal(t, "admin auth not configured", msg) + }) + + t.Run("TrustedForwardersParsed", func(t *testing.T) { + mk := func(remote, xff string) *http.Request { + r := &http.Request{RemoteAddr: remote, Header: http.Header{}} + if xff != "" { + r.Header.Set("X-Forwarded-For", xff) + } + return r + } + assert.Equal(t, "1.2.3.4", srv.resolveRealClientIP(mk("10.1.2.3:555", "1.2.3.4, 10.0.0.9"))) + assert.Equal(t, "1.2.3.4", srv.resolveRealClientIP(mk("192.168.1.1:555", "1.2.3.4"))) + assert.Equal(t, "10.1.2.3", srv.resolveRealClientIP(mk("10.1.2.3:555", "10.0.0.1, 10.0.0.2"))) + assert.Equal(t, "10.1.2.3", srv.resolveRealClientIP(mk("10.1.2.3:555", ""))) + assert.Equal(t, "8.8.8.8", srv.resolveRealClientIP(mk("8.8.8.8:1", "1.2.3.4"))) + assert.Equal(t, "n/a", srv.resolveRealClientIP(mk("garbage", ""))) + assert.Len(t, srv.trustedForwarderNets, 1) + assert.Len(t, srv.trustedForwarderIPs, 1) + assert.Equal(t, []string{"X-Forwarded-For"}, srv.trustedIPHeaders) + }) +} + +func TestHttpPaths_NewHttpServerHealthAuthError(t *testing.T) { + logger := log.Logger + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cfg := &common.ServerConfig{} + require.NoError(t, cfg.SetDefaults()) + _, err := NewHttpServer(ctx, &logger, cfg, + &common.HealthCheckConfig{Auth: &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{{Type: "bogus"}}}}, + nil, nil, &ERPC{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to create healthcheck auth registry") +} + +func TestHttpPaths_ParseUrlPathAliasCombos(t *testing.T) { + s := &HttpServer{ + draining: &atomic.Bool{}, + erpc: &ERPC{projectsRegistry: &ProjectsRegistry{preparedProjects: map[string]*PreparedProject{ + "p": {networksRegistry: &NetworksRegistry{ + aliasMu: &sync.RWMutex{}, + aliasToNetworkId: map[string]aliasEntry{"arb": {architecture: "evm", chainID: "42161"}}, + }}, + }}}, + } + cases := []struct { + name string + method, path string + pre [3]string + want [3]string + wantHealth bool + errContains string + }{ + {name: "project-only too many", method: "POST", path: "/a/b/c/d", pre: [3]string{"p", "", ""}, errContains: "for project-only alias must only provide"}, + {name: "project-only explicit triple", method: "POST", path: "/q/evm/1", pre: [3]string{"p", "", ""}, want: [3]string{"q", "evm", "1"}}, + {name: "project-only alias", method: "POST", path: "/arb", pre: [3]string{"p", "", ""}, want: [3]string{"p", "evm", "42161"}}, + {name: "project-only healthcheck", method: "GET", path: "/healthcheck", pre: [3]string{"p", "", ""}, want: [3]string{"p", "", ""}, wantHealth: true}, + {name: "project+arch two segments", method: "POST", path: "/a/b", pre: [3]string{"p", "evm", ""}, errContains: "for project-and-architecture alias must only provide"}, + {name: "project+arch triple", method: "POST", path: "/q/svm/mainnet", pre: [3]string{"p", "evm", ""}, want: [3]string{"q", "svm", "mainnet"}}, + {name: "project+arch empty chain", method: "POST", path: "/", pre: [3]string{"p", "evm", ""}, errContains: "for project-and-architecture alias must provide /"}, + {name: "project+arch healthcheck", method: "GET", path: "/healthcheck", pre: [3]string{"p", "evm", ""}, want: [3]string{"p", "evm", ""}, wantHealth: true}, + {name: "all preselected extra path", method: "POST", path: "/a/b", pre: [3]string{"p", "evm", "1"}, errContains: "must not provide anything"}, + {name: "all preselected root", method: "POST", path: "/", pre: [3]string{"p", "evm", "1"}, want: [3]string{"p", "evm", "1"}}, + {name: "arch+chain empty project", method: "POST", path: "/", pre: [3]string{"", "evm", "1"}, errContains: "for architecture-and-chain alias must provide /"}, + {name: "arch+chain project", method: "POST", path: "/q", pre: [3]string{"", "evm", "1"}, want: [3]string{"q", "evm", "1"}}, + {name: "arch+chain triple", method: "POST", path: "/q/evm/5", pre: [3]string{"", "evm", "1"}, want: [3]string{"q", "evm", "5"}}, + {name: "arch+chain two segments", method: "POST", path: "/a/b", pre: [3]string{"", "evm", "1"}, errContains: "for architecture-and-chain alias must only provide"}, + {name: "arch+chain healthcheck", method: "GET", path: "/healthcheck", pre: [3]string{"", "evm", "1"}, want: [3]string{"", "evm", "1"}, wantHealth: true}, + {name: "arch-only project", method: "POST", path: "/q", pre: [3]string{"", "evm", ""}, want: [3]string{"q", "evm", ""}}, + {name: "arch-only project+chain", method: "POST", path: "/q/7", pre: [3]string{"", "evm", ""}, want: [3]string{"q", "evm", "7"}}, + {name: "arch-only triple", method: "POST", path: "/q/svm/7", pre: [3]string{"", "evm", ""}, want: [3]string{"q", "svm", "7"}}, + {name: "arch-only too many", method: "POST", path: "/a/b/c/d", pre: [3]string{"", "evm", ""}, errContains: "for architecture-only alias must only provide"}, + {name: "project+chain without arch", method: "POST", path: "/", pre: [3]string{"p", "", "1"}, errContains: "not possible to alias for project and chain WITHOUT architecture"}, + {name: "chain only", method: "POST", path: "/", pre: [3]string{"", "", "1"}, errContains: "invalid combination"}, + {name: "invalid architecture", method: "POST", path: "/q/btc/1", errContains: "architecture is not valid"}, + {name: "root GET is healthcheck", method: "GET", path: "/", wantHealth: true}, + {name: "network GET is healthcheck", method: "GET", path: "/q/evm/1", want: [3]string{"q", "evm", "1"}, wantHealth: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := &http.Request{Method: tc.method, URL: &url.URL{Path: tc.path}, Header: http.Header{}} + p, a, c, isAdmin, isHealth, err := s.parseUrlPath(r, tc.pre[0], tc.pre[1], tc.pre[2]) + if tc.errContains != "" { + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeInvalidUrlPath)) + assert.Contains(t, err.Error(), tc.errContains) + return + } + require.NoError(t, err) + assert.False(t, isAdmin) + assert.Equal(t, tc.want, [3]string{p, a, c}) + assert.Equal(t, tc.wantHealth, isHealth) + }) + } +} + +func TestHttpPaths_DetermineResponseStatusCode(t *testing.T) { + cases := []struct { + name string + res interface{} + hint []common.NetworkArchitecture + want int + }{ + {"plain response", common.NewNormalizedResponse(), nil, http.StatusOK}, + {"error body without cause", &HttpJsonRpcErrorResponse{}, nil, http.StatusOK}, + {"invalid url path", common.NewErrInvalidUrlPath("x", "/"), nil, http.StatusBadRequest}, + {"invalid request on svm stays 200", common.NewErrInvalidRequest(errors.New("x")), []common.NetworkArchitecture{common.ArchitectureSvm}, http.StatusOK}, + {"unauthorized", common.NewErrAuthUnauthorized("secret", "bad", "bad"), nil, http.StatusUnauthorized}, + {"endpoint unauthorized", common.NewErrEndpointUnauthorized(errors.New("x")), nil, http.StatusUnauthorized}, + {"project not found", common.NewErrProjectNotFound("p"), nil, http.StatusNotFound}, + {"network not found", common.NewErrNetworkNotFound("evm:1"), nil, http.StatusNotFound}, + {"network not supported", common.NewErrNetworkNotSupported("p", "evm:1"), nil, http.StatusNotFound}, + {"auth rate limit", common.NewErrAuthRateLimitRuleExceeded("p", "s", "b", "r", "u", "ip"), nil, http.StatusTooManyRequests}, + {"project rate limit", common.NewErrProjectRateLimitRuleExceeded("p", "b", "r"), nil, http.StatusTooManyRequests}, + {"network rate limit", common.NewErrNetworkRateLimitRuleExceeded("p", "n", "b", "r"), nil, http.StatusTooManyRequests}, + {"capacity exceeded", common.NewErrEndpointCapacityExceeded(errors.New("x")), nil, http.StatusTooManyRequests}, + {"no upstreams available", common.NewErrNetworkNoUpstreamsAvailable("p", "evm:1"), nil, http.StatusNotFound}, + {"wrapped in error body", &HttpJsonRpcErrorResponse{Cause: common.NewErrProjectNotFound("p")}, nil, http.StatusNotFound}, + {"application error", common.NewErrEndpointExecutionException(errors.New("reverted")), nil, http.StatusOK}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, determineResponseStatusCode(tc.res, tc.hint...)) + }) + } +} + +func httpPathsRequest(t *testing.T, body string, nw *Network) *common.NormalizedRequest { + t.Helper() + nq := common.NewNormalizedRequest([]byte(body)) + require.NoError(t, nq.Validate()) + if nw != nil { + nq.SetNetwork(nw) + } + return nq +} + +func httpPathsNetwork(id, label string) *Network { + return &Network{networkId: id, networkLabel: label, cfg: &common.NetworkConfig{Architecture: common.ArchitectureEvm}} +} + +func httpPathsResponse(t *testing.T, nq *common.NormalizedRequest, result interface{}) *common.NormalizedResponse { + t.Helper() + jrr, err := common.NewJsonRpcResponse(1, result, nil) + require.NoError(t, err) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr).WithRequest(nq) +} + +func TestHttpPaths_ProcessErrorBody(t *testing.T) { + logger := zerolog.Nop() + started := time.Now() + + t.Run("SvmLocalRateLimitUsesGenericServerCode", func(t *testing.T) { + nq := httpPathsRequest(t, `{"jsonrpc":"2.0","id":5,"method":"getSlot"}`, nil) + res := processErrorBody(&logger, &started, nq, common.NewErrAuthRateLimitRuleExceeded("p", "s", "b", "r", "u", "ip"), &common.TRUE, common.ArchitectureSvm) + body, ok := res.(*HttpJsonRpcErrorResponse) + require.True(t, ok) + errObj := body.Error.(map[string]interface{}) + assert.Equal(t, common.JsonRpcErrorNumber(-32000), errObj["code"]) + assert.Equal(t, "rate-limit exceeded", errObj["message"]) + assert.Equal(t, http.StatusTooManyRequests, determineResponseStatusCode(body)) + }) + + t.Run("EvmRateLimitKeepsCapacityCode", func(t *testing.T) { + nq := httpPathsRequest(t, `{"jsonrpc":"2.0","id":5,"method":"eth_blockNumber"}`, nil) + body := processErrorBody(&logger, &started, nq, common.NewErrProjectRateLimitRuleExceeded("p", "b", "r"), &common.TRUE).(*HttpJsonRpcErrorResponse) + assert.Equal(t, common.JsonRpcErrorCapacityExceeded, body.Error.(map[string]interface{})["code"]) + }) + + t.Run("ErrorDetailsOmittedForEthCall", func(t *testing.T) { + cause := common.NewErrNetworkNotFound("evm:9") + call := httpPathsRequest(t, `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[]}`, nil) + other := httpPathsRequest(t, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":[]}`, nil) + callObj := processErrorBody(&logger, &started, call, cause, &common.TRUE).(*HttpJsonRpcErrorResponse).Error.(map[string]interface{}) + otherObj := processErrorBody(&logger, &started, other, cause, &common.TRUE).(*HttpJsonRpcErrorResponse).Error.(map[string]interface{}) + hiddenObj := processErrorBody(&logger, &started, other, cause, &common.FALSE).(*HttpJsonRpcErrorResponse).Error.(map[string]interface{}) + _, callHasData := callObj["data"] + _, hiddenHasData := hiddenObj["data"] + assert.False(t, callHasData) + assert.False(t, hiddenHasData) + assert.Equal(t, cause, otherObj["data"]) + }) + + t.Run("SingleExhaustedCauseIsUnwrapped", func(t *testing.T) { + ex := common.NewErrUpstreamsExhaustedWithCause(errors.Join(common.NewErrEndpointUnauthorized(errors.New("bad key")))) + body := processErrorBody(&logger, &started, nil, ex, &common.TRUE).(*HttpJsonRpcErrorResponse) + assert.Equal(t, http.StatusUnauthorized, determineResponseStatusCode(body)) + assert.Equal(t, "2.0", body.Jsonrpc) + assert.Nil(t, body.Id) + }) + + t.Run("PlainAndContextErrorsBecomeServerErrors", func(t *testing.T) { + for _, err := range []error{ + errors.New("boom"), + context.Canceled, + context.DeadlineExceeded, + common.NewErrEndpointServerSideException(errors.New("upstream 500"), nil, 500), + } { + nq := httpPathsRequest(t, `{"jsonrpc":"2.0","id":9,"method":"eth_chainId"}`, nil) + body, ok := processErrorBody(&logger, &started, nq, err, &common.FALSE).(*HttpJsonRpcErrorResponse) + require.True(t, ok, "%v", err) + assert.Equal(t, http.StatusOK, determineResponseStatusCode(body), "%v", err) + assert.NotNil(t, body.Error.(map[string]interface{})["code"], "%v", err) + } + }) +} + +func TestHttpPaths_ErrorResponseLogMarshal(t *testing.T) { + var buf bytes.Buffer + // Package tests disable logging globally; the event must be enabled to + // run the marshaller. + lg := zerolog.New(&buf).Level(zerolog.InfoLevel) + prev := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.InfoLevel) + defer zerolog.SetGlobalLevel(prev) + lg.Info().Object("r", &HttpJsonRpcErrorResponse{ + Jsonrpc: "2.0", Id: 4, + Error: common.NewErrJsonRpcExceptionExternal(-32000, "ext-msg", ""), + }).Msg("") + lg.Info().Object("r", &HttpJsonRpcErrorResponse{ + Jsonrpc: "2.0", Id: 5, + Error: map[string]interface{}{"code": -32601, "message": "map-msg"}, + }).Msg("") + lg.Info().Object("r", (*HttpJsonRpcErrorResponse)(nil)).Msg("") + out := buf.String() + assert.Contains(t, out, "ext-msg") + assert.Contains(t, out, "map-msg") + assert.Contains(t, out, `"id":5`) +} + +func TestHttpPaths_IsSvmRequest(t *testing.T) { + svmBody := httpPathsRequest(t, `{"jsonrpc":"2.0","id":1,"method":"getSlot","networkId":"svm:mainnet"}`, nil) + evmNet := httpPathsRequest(t, `{"jsonrpc":"2.0","id":1,"method":"eth_chainId","networkId":"svm:mainnet"}`, httpPathsNetwork("evm:1", "")) + svmNet := httpPathsRequest(t, `{"jsonrpc":"2.0","id":1,"method":"getSlot"}`, httpPathsNetwork("svm:mainnet", "")) + assert.True(t, isSvmRequest(nil, common.ArchitectureSvm)) + assert.False(t, isSvmRequest(nil)) + assert.True(t, isSvmRequest(svmBody)) + assert.False(t, isSvmRequest(svmBody, common.ArchitectureEvm)) + assert.False(t, isSvmRequest(evmNet)) + assert.True(t, isSvmRequest(svmNet)) +} + +func TestHttpPaths_CostHeaders(t *testing.T) { + ctx := context.Background() + on := &HttpServer{serverCfg: &common.ServerConfig{CostHeaders: util.BoolPtr(true)}} + nwA := httpPathsNetwork("evm:1", "one") + + r1 := httpPathsRequest(t, `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[]}`, nwA) + r1.ExecState().RecordUpstreamAttempt(common.UpstreamAttempt{UpstreamId: "a", VendorName: "alchemy", CreditUnits: 5}) + r1.ExecState().RecordUpstreamAttempt(common.UpstreamAttempt{UpstreamId: "a", VendorName: "alchemy", CreditUnits: 0}) + r2 := httpPathsRequest(t, `{"jsonrpc":"2.0","id":2,"method":"eth_getBalance","params":[]}`, nwA) + r2.ExecState().RecordUpstreamAttempt(common.UpstreamAttempt{UpstreamId: "b", VendorName: "drpc", CreditUnits: 3}) + reverted := &HttpJsonRpcErrorResponse{Cause: common.NewErrEndpointExecutionException(errors.New("reverted")), Request: r2} + transport := &HttpJsonRpcErrorResponse{Cause: errors.New("dial"), Request: nil} + + h := on.costHeaders(ctx, []interface{}{httpPathsResponse(t, r1, "0x1"), reverted, transport}) + require.NotNil(t, h) + assert.Equal(t, "3", h.Get("X-ERPC-Calls")) + assert.Equal(t, "2", h.Get("X-ERPC-Billable")) + assert.Equal(t, "eth_call,eth_getBalance", h.Get("X-ERPC-Methods")) + assert.Equal(t, "alchemy:eth_call=5;drpc:eth_getBalance=3", h.Get("X-ERPC-Credits")) + assert.Equal(t, "8", h.Get("X-ERPC-Credits-Total")) + assert.Equal(t, common.ErpcVersion, h.Get("X-ERPC-Credits-Version")) + assert.Equal(t, "evm:1", h.Get("X-ERPC-Network-Id")) + assert.Equal(t, "one", h.Get("X-ERPC-Network-Alias")) + + rOther := httpPathsRequest(t, `{"jsonrpc":"2.0","id":3,"method":"eth_chainId"}`, httpPathsNetwork("evm:2", "")) + h = on.costHeaders(ctx, []interface{}{httpPathsResponse(t, r1, "0x1"), httpPathsResponse(t, rOther, "0x2")}) + assert.Empty(t, h.Get("X-ERPC-Network-Id"), "disagreeing networks are not reported") + assert.Empty(t, h.Get("X-ERPC-Network-Alias")) + + assert.Nil(t, on.costHeaders(ctx, nil)) + off := &HttpServer{serverCfg: &common.ServerConfig{}} + assert.Nil(t, off.costHeaders(ctx, []interface{}{reverted})) + assert.False(t, isBillableItem(ctx, "not a response")) +} + +func httpPathsExecRequest(t *testing.T, upstream string, n int) *common.NormalizedRequest { + nq := httpPathsRequest(t, `{"jsonrpc":"2.0","id":1,"method":"eth_chainId"}`, nil) + st := nq.ExecState() + st.UpstreamAttempts.Add(int32(n)) + st.CacheAttempts.Add(1) + st.ConsensusSlots.Add(2) + st.ConsensusDisputes.Add(1) + st.ConsensusLowParticipants.Add(1) + for range n { + st.RecordUpstreamAttempt(common.UpstreamAttempt{ + UpstreamId: upstream, Reason: common.SelectionReasonPrimary, + Outcome: common.UpstreamOutcomeSuccess, Duration: 12 * time.Millisecond, + }) + } + st.MarkUpstreamAttemptWon(upstream) + return nq +} + +func TestHttpPaths_BatchExecHeaders(t *testing.T) { + ctx := context.Background() + all := &HttpServer{serverCfg: &common.ServerConfig{}} + r1 := httpPathsExecRequest(t, "up1", 30) + r2 := httpPathsExecRequest(t, "up2", 30) + hit := httpPathsResponse(t, r1, "0x1").SetFromCache(true) + hit.SetDuration(40 * time.Millisecond) + miss := httpPathsResponse(t, r2, "0x2") + miss.SetDuration(90 * time.Millisecond) + + w := httptest.NewRecorder() + all.writeBatchExecHeaders(ctx, w, []interface{}{hit, miss, &HttpJsonRpcErrorResponse{Cause: errors.New("x")}}) + hdr := w.Header() + assert.Equal(t, "62", hdr.Get("X-ERPC-Attempts")) + assert.Equal(t, "60", hdr.Get("X-ERPC-Upstream-Attempts")) + assert.Equal(t, "2", hdr.Get("X-ERPC-Cache-Attempts")) + assert.Equal(t, "4", hdr.Get("X-ERPC-Consensus-Slots")) + assert.Equal(t, "2", hdr.Get("X-ERPC-Consensus-Disputes")) + assert.Equal(t, "2", hdr.Get("X-ERPC-Consensus-Low-Participants")) + assert.Equal(t, "PARTIAL:1", hdr.Get("X-ERPC-Cache")) + assert.Equal(t, "90", hdr.Get("X-ERPC-Duration")) + assert.Equal(t, "10", hdr.Get("X-ERPC-Upstreams-Truncated")) + segs := strings.Split(hdr.Get("X-ERPC-Upstreams"), ";") + assert.Len(t, segs, maxBatchTraceSegments) + assert.Equal(t, "up1=primary:success:12ms", segs[0]) + + w = httptest.NewRecorder() + all.writeBatchExecHeaders(ctx, w, []interface{}{hit}) + assert.Equal(t, "HIT", w.Header().Get("X-ERPC-Cache")) + + summary := common.ExecutionHeadersSummary + w = httptest.NewRecorder() + (&HttpServer{serverCfg: &common.ServerConfig{ExecutionHeaders: &summary}}).writeBatchExecHeaders(ctx, w, []interface{}{miss}) + assert.Equal(t, "MISS", w.Header().Get("X-ERPC-Cache")) + assert.Empty(t, w.Header().Get("X-ERPC-Upstreams")) + assert.Equal(t, "31", w.Header().Get("X-ERPC-Attempts")) + + offMode := common.ExecutionHeadersOff + w = httptest.NewRecorder() + (&HttpServer{serverCfg: &common.ServerConfig{ExecutionHeaders: &offMode}}).writeBatchExecHeaders(ctx, w, []interface{}{miss}) + assert.Empty(t, w.Header()) +} + +func TestHttpPaths_SingleResponseHeaders(t *testing.T) { + ctx := context.Background() + r := httpPathsExecRequest(t, "up1", 2) + resp := httpPathsResponse(t, r, "0x1").SetFromCache(true) + resp.SetDuration(7 * time.Millisecond) + + w := httptest.NewRecorder() + setResponseHeaders(ctx, resp, w, common.ExecutionHeadersAll) + hdr := w.Header() + assert.Equal(t, "HIT", hdr.Get("X-ERPC-Cache")) + assert.Equal(t, "7", hdr.Get("X-ERPC-Duration")) + assert.Equal(t, "3", hdr.Get("X-ERPC-Attempts")) + assert.Equal(t, "1", hdr.Get("X-ERPC-Cache-Attempts")) + assert.Equal(t, "2", hdr.Get("X-ERPC-Consensus-Slots")) + assert.Equal(t, "1", hdr.Get("X-ERPC-Consensus-Disputes")) + assert.Equal(t, "1", hdr.Get("X-ERPC-Consensus-Low-Participants")) + assert.Equal(t, "up1=primary:success:12ms;up1=primary:success:12ms:won", hdr.Get("X-ERPC-Upstreams")) + + w = httptest.NewRecorder() + setResponseHeaders(ctx, resp, w, common.ExecutionHeadersOff) + assert.Empty(t, w.Header()) + + w = httptest.NewRecorder() + setResponseHeaders(ctx, map[string]interface{}{"cause": errors.New("x")}, w, common.ExecutionHeadersAll) + assert.Equal(t, "0", w.Header().Get("X-ERPC-Attempts")) + assert.Empty(t, w.Header().Get("X-ERPC-Cache")) + assert.Nil(t, lookupResponseMetadata(&HttpJsonRpcErrorResponse{Cause: errors.New("x")})) + assert.Nil(t, lookupResponseMetadata(42)) +} + +func TestHttpPaths_ClientIpParsing(t *testing.T) { + assert.Equal(t, "::1", parseRemoteIP("[::1]:80").String()) + assert.Equal(t, "::1", parseRemoteIP("[::1]").String()) + assert.Equal(t, "1.2.3.4", parseRemoteIP("1.2.3.4").String()) + assert.Nil(t, parseRemoteIP("nope")) + + ips := parseXForwardedFor(" 1.1.1.1 , [2001:db8::1], 2.2.2.2:8080, junk") + require.Len(t, ips, 3) + assert.Equal(t, "2001:db8::1", ips[1].String()) + assert.Equal(t, "2.2.2.2", ips[2].String()) + + fwd := parseForwardedFor(`for=192.0.2.60;proto=http;by=203.0.113.43, For="[2001:db8:cafe::17]:4711", for=unknown, proto=https`) + require.Len(t, fwd, 2) + assert.Equal(t, "192.0.2.60", fwd[0].String()) + assert.Equal(t, "2001:db8:cafe::17", fwd[1].String()) + + trusted := func(ip net.IP) bool { return ip.IsPrivate() } + assert.Nil(t, trimRightTrustedAndPick(nil, trusted)) + assert.Nil(t, trimRightTrustedAndPick([]net.IP{net.ParseIP("10.0.0.1")}, trusted)) + assert.Equal(t, "8.8.8.8", trimRightTrustedAndPick([]net.IP{net.ParseIP("1.1.1.1"), net.ParseIP("8.8.8.8"), net.ParseIP("10.0.0.1")}, trusted).String()) + + s := &HttpServer{} + assert.False(t, s.isTrustedForwarder(nil)) + assert.False(t, s.isTrustedForwarder(net.ParseIP("1.1.1.1"))) +} + +func TestHttpPaths_MethodAndHeaderFilters(t *testing.T) { + prj := func(cfg *common.ProjectConfig) *PreparedProject { return &PreparedProject{Config: cfg} } + + ok, err := isMethodAllowed(nil, "eth_call") + require.NoError(t, err) + assert.True(t, ok) + cases := []struct { + ignore, allow []string + method string + want bool + wantErr bool + }{ + {ignore: []string{"eth_*"}, method: "eth_call", want: false}, + {ignore: []string{"eth_*"}, allow: []string{"eth_call"}, method: "eth_call", want: true}, + {ignore: []string{"debug_*"}, method: "eth_call", want: true}, + {ignore: []string{"("}, method: "eth_call", wantErr: true}, + {allow: []string{"("}, method: "eth_call", wantErr: true}, + } + for _, tc := range cases { + got, err := isMethodAllowed(prj(&common.ProjectConfig{IgnoreMethods: tc.ignore, AllowMethods: tc.allow}), tc.method) + if tc.wantErr { + assert.Error(t, err) + continue + } + require.NoError(t, err) + assert.Equal(t, tc.want, got, "%v/%v %s", tc.ignore, tc.allow, tc.method) + } + + nq := common.NewNormalizedRequest(nil) + nq.ForwardHeaders = http.Header{} + require.NoError(t, applyForwardHeaders(nil, nq, http.Header{"X-A": {"1"}})) + assert.Empty(t, nq.ForwardHeaders) + require.NoError(t, applyForwardHeaders(prj(&common.ProjectConfig{ForwardHeaders: []string{"X-Fwd-*"}}), nq, + http.Header{"X-Fwd-One": {"a", "b"}, "X-Other": {"c"}})) + assert.Equal(t, []string{"a", "b"}, nq.ForwardHeaders.Values("X-Fwd-*")) + assert.Empty(t, nq.ForwardHeaders.Values("X-Other")) + assert.Error(t, applyForwardHeaders(prj(&common.ProjectConfig{ForwardHeaders: []string{"("}}), nq, http.Header{"X-A": {"1"}})) + + adminCases := []struct { + deny, allow []string + method string + blocked bool + wantErr bool + }{ + {method: "erpc_config", blocked: false}, + {deny: []string{"erpc_*"}, method: "erpc_config", blocked: true}, + {deny: []string{"erpc_*"}, allow: []string{"erpc_config"}, method: "erpc_config", blocked: false}, + {allow: []string{"erpc_listCordoned"}, method: "erpc_config", blocked: true}, + {allow: []string{"erpc_config"}, method: "erpc_config", blocked: false}, + {deny: []string{"("}, method: "erpc_config", wantErr: true}, + {deny: []string{"erpc_*"}, allow: []string{"("}, method: "erpc_config", wantErr: true}, + {allow: []string{"("}, method: "erpc_config", wantErr: true}, + } + for _, tc := range adminCases { + blocked, err := isAdminMethodBlocked(&common.AdminConfig{DenyMethods: tc.deny, AllowMethods: tc.allow}, tc.method) + if tc.wantErr { + assert.Error(t, err) + continue + } + require.NoError(t, err) + assert.Equal(t, tc.blocked, blocked, "%v/%v", tc.deny, tc.allow) + } +} + +func TestHttpPaths_GzipWriter(t *testing.T) { + t.Run("FlushBelowThresholdCommitsPlain", func(t *testing.T) { + rec := httptest.NewRecorder() + w := &conditionalGzipWriter{ResponseWriter: rec, pool: util.NewGzipWriterPool()} + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte("small")) + w.Flush() + w.WriteHeader(http.StatusTeapot) // already decided: passes through (ignored by recorder after first) + _, _ = w.Write([]byte("-more")) + require.NoError(t, w.Close()) + assert.Equal(t, http.StatusAccepted, rec.Code) + assert.Empty(t, rec.Header().Get("Content-Encoding")) + assert.Equal(t, "small-more", rec.Body.String()) + }) + + t.Run("LargeBodyCompressedAcrossWritesAndFlushes", func(t *testing.T) { + rec := httptest.NewRecorder() + w := &conditionalGzipWriter{ResponseWriter: rec, pool: util.NewGzipWriterPool()} + part := strings.Repeat("a", compressionThreshold) + _, _ = w.Write([]byte(part)) + w.Flush() + _, _ = w.Write([]byte("tail")) + require.NoError(t, w.Close()) + assert.Equal(t, "gzip", rec.Header().Get("Content-Encoding")) + assert.Equal(t, "application/json", rec.Header().Get("Content-Type")) + zr, err := gzip.NewReader(rec.Body) + require.NoError(t, err) + plain, err := io.ReadAll(zr) + require.NoError(t, err) + assert.Equal(t, part+"tail", string(plain)) + }) + + t.Run("HandlerSkipsWithoutAcceptEncodingAndForUpgrades", func(t *testing.T) { + big := strings.Repeat("b", 2*compressionThreshold) + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.WriteString(w, big) }) + h := gzipHandler(next, true) + + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/", nil)) + assert.Equal(t, "Accept-Encoding", rec.Header().Get("Vary")) + assert.Empty(t, rec.Header().Get("Content-Encoding")) + assert.Equal(t, big, rec.Body.String()) + + up := httptest.NewRequest(http.MethodGet, "/", nil) + up.Header.Set("Connection", "Upgrade") + up.Header.Set("Upgrade", "websocket") + up.Header.Set("Accept-Encoding", "gzip") + rec = httptest.NewRecorder() + h.ServeHTTP(rec, up) + assert.Empty(t, rec.Header().Get("Vary")) + assert.Empty(t, rec.Header().Get("Content-Encoding")) + }) +} + +func httpPathsWriteCert(t *testing.T) (certFile, keyFile string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + keyDer, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + dir := t.TempDir() + certFile = filepath.Join(dir, "cert.pem") + keyFile = filepath.Join(dir, "key.pem") + require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600)) + require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDer}), 0o600)) + return certFile, keyFile +} + +func TestHttpPaths_TLSConfig(t *testing.T) { + certFile, keyFile := httpPathsWriteCert(t) + garbage := filepath.Join(t.TempDir(), "garbage.pem") + require.NoError(t, os.WriteFile(garbage, []byte("not a cert"), 0o600)) + + mk := func(tlsCfg *common.TLSConfig) *HttpServer { + return &HttpServer{serverCfg: &common.ServerConfig{TLS: tlsCfg}} + } + cfg, err := mk(&common.TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile, InsecureSkipVerify: true}).createTLSConfig() + require.NoError(t, err) + assert.Len(t, cfg.Certificates, 1) + assert.Equal(t, uint16(tls.VersionTLS12), cfg.MinVersion) + assert.True(t, cfg.InsecureSkipVerify) + assert.Equal(t, tls.NoClientCert, cfg.ClientAuth) + + cfg, err = mk(&common.TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile, CAFile: certFile}).createTLSConfig() + require.NoError(t, err) + assert.Equal(t, tls.RequireAndVerifyClientCert, cfg.ClientAuth) + assert.NotNil(t, cfg.ClientCAs) + + _, err = mk(&common.TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile, CAFile: garbage}).createTLSConfig() + assert.ErrorContains(t, err, "failed to parse CA certificate") + _, err = mk(&common.TLSConfig{Enabled: true, CertFile: certFile, KeyFile: keyFile, CAFile: garbage + ".missing"}).createTLSConfig() + assert.ErrorContains(t, err, "failed to read CA file") + _, err = mk(&common.TLSConfig{Enabled: true, CertFile: garbage, KeyFile: keyFile}).createTLSConfig() + assert.ErrorContains(t, err, "failed to load TLS certificate and key") +} + +func TestHttpPaths_StartAndShutdownErrors(t *testing.T) { + logger := zerolog.Nop() + host := "127.0.0.1" + badPort := -1 + port := 0 + badTLS := &common.TLSConfig{Enabled: true, CertFile: "/nonexistent/cert.pem", KeyFile: "/nonexistent/key.pem"} + cases := []struct { + name string + srv *HttpServer + want string + }{ + {"no servers", &HttpServer{serverCfg: &common.ServerConfig{}}, "you must configure at least one"}, + {"v4 port missing", &HttpServer{serverV4: &http.Server{}, serverCfg: &common.ServerConfig{ListenV4: util.BoolPtr(true)}}, "server.httpPortV4 is not configured"}, + {"v6 port missing", &HttpServer{serverV6: &http.Server{}, serverCfg: &common.ServerConfig{ListenV6: util.BoolPtr(true)}}, "server.httpPortV6 is not configured"}, + {"v4 bad tls", &HttpServer{serverV4: &http.Server{}, serverCfg: &common.ServerConfig{ListenV4: util.BoolPtr(true), HttpHostV4: &host, HttpPortV4: &port, TLS: badTLS}}, "failed to create TLS config"}, + {"v6 bad tls", &HttpServer{serverV6: &http.Server{}, serverCfg: &common.ServerConfig{ListenV6: util.BoolPtr(true), HttpHostV6: &host, HttpPortV6: &port, TLS: badTLS}}, "failed to create TLS config"}, + {"v4 listen fails", &HttpServer{serverV4: &http.Server{}, serverCfg: &common.ServerConfig{ListenV4: util.BoolPtr(true), HttpHostV4: &host, HttpPortV4: &badPort}}, "IPv4 server error"}, + {"v6 listen fails", &HttpServer{serverV6: &http.Server{}, serverCfg: &common.ServerConfig{ListenV6: util.BoolPtr(true), HttpHostV6: &host, HttpPortV6: &badPort}}, "IPv6 server error"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := tc.srv.Start(&logger) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.want) + }) + } + + t.Run("ShutdownBothUnstartedServers", func(t *testing.T) { + s := &HttpServer{serverV4: &http.Server{}, serverV6: &http.Server{}, draining: &atomic.Bool{}} + require.NoError(t, s.Shutdown(&logger)) + assert.True(t, s.draining.Load()) + }) +} + +func TestHttpPaths_PrewarmSettlesHttpOnlyNetworks(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + cfg := &common.Config{ + Server: &common.ServerConfig{WebSocket: &common.WebSocketServerConfig{Enabled: util.BoolPtr(true)}}, + Projects: []*common.ProjectConfig{httpPathsProject("test_project")}, + } + srv, _ := httpPathsStart(t, cfg) + prj, err := srv.erpc.GetProject("test_project") + require.NoError(t, err) + _, err = prj.GetNetwork(context.Background(), "evm:123") + require.NoError(t, err) + + old := prewarmInterval + prewarmInterval = 10 * time.Millisecond + defer func() { prewarmInterval = old }() + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + srv.PrewarmUpstreamSubscriptions(ctx) + close(done) + }() + time.AfterFunc(100*time.Millisecond, cancel) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("prewarm did not stop on context cancel") + } + _, bootstrapped := srv.subscriptionManager.networks.Load("test_project/evm:123") + assert.False(t, bootstrapped, "an HTTP-only network is never bootstrapped for subscriptions") + + // Disabled WebSocket: returns immediately even with a live context. + off := &HttpServer{serverCfg: &common.ServerConfig{}, erpc: srv.erpc, subscriptionManager: srv.subscriptionManager} + finished := make(chan struct{}) + go func() { off.PrewarmUpstreamSubscriptions(context.Background()); close(finished) }() + select { + case <-finished: + case <-time.After(2 * time.Second): + t.Fatal("prewarm with WebSocket disabled must return immediately") + } +} diff --git a/erpc/http_server_prewarm_test.go b/erpc/http_server_prewarm_test.go new file mode 100644 index 000000000..e429f7a2c --- /dev/null +++ b/erpc/http_server_prewarm_test.go @@ -0,0 +1,152 @@ +package erpc + +import ( + "context" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/gorilla/websocket" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestPrewarmUpstreamSubscriptions_Guards pins the reconciler's lifecycle +// contract: it must no-op safely when its dependencies are absent, and it must +// return promptly once the context is cancelled rather than ticking forever. +// The core per-network bootstrap is exercised through the client-subscribe path +// (SubscriptionManager.Subscribe -> bootstrapNetwork). +func TestPrewarmUpstreamSubscriptions_Guards(t *testing.T) { + logger := zerolog.New(zerolog.NewTestWriter(t)) + + t.Run("returns immediately when dependencies are nil", func(t *testing.T) { + s := &HttpServer{logger: &logger} // no erpc, no subscriptionManager + done := make(chan struct{}) + go func() { + s.PrewarmUpstreamSubscriptions(context.Background()) + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("PrewarmUpstreamSubscriptions did not return despite nil dependencies") + } + }) + + t.Run("returns when context is cancelled", func(t *testing.T) { + // A server with an empty ERPC: the first reconcile pass iterates zero + // projects, then the loop blocks on the ticker until ctx is cancelled. + s := &HttpServer{ + logger: &logger, + serverCfg: &common.ServerConfig{WebSocket: &common.WebSocketServerConfig{Enabled: util.BoolPtr(true)}}, + erpc: &ERPC{projectsRegistry: &ProjectsRegistry{}}, + subscriptionManager: NewSubscriptionManager(&logger, nil), + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + s.PrewarmUpstreamSubscriptions(ctx) + close(done) + }() + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("PrewarmUpstreamSubscriptions did not return after context cancellation") + } + }) +} + +// Two projects serving the same chain each own an indexer network, so each +// must be prewarmed on its own upstream. +func TestPrewarmUpstreamSubscriptions_ProjectsSharingANetwork(t *testing.T) { + setupGock() + defer util.ResetGock() + + prev := prewarmInterval + prewarmInterval = 50 * time.Millisecond + defer func() { prewarmInterval = prev }() + + var subscribesA, subscribesB atomic.Int32 + mockWith := func(count *atomic.Int32, subID string) *httptest.Server { + return mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, _ map[string]interface{}) { + result := interface{}("0x1") + if method == "eth_subscribe" { + count.Add(1) + result = subID + } + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": result}) + }) + }) + } + upA, upB := mockWith(&subscribesA, "0xa"), mockWith(&subscribesB, "0xb") + defer upA.Close() + defer upB.Close() + + cfg := standardWsConfig("ws" + strings.TrimPrefix(upA.URL, "http")) + second := *cfg.Projects[0] + second.Id = "test_ws2" + second.Networks = []*common.NetworkConfig{{Architecture: common.ArchitectureEvm, Evm: &common.EvmNetworkConfig{ChainId: 123}}} + second.Upstreams = []*common.UpstreamConfig{ + {Id: "http-upstream", Type: common.UpstreamTypeEvm, Endpoint: "http://rpc1.localhost", Evm: &common.EvmUpstreamConfig{ChainId: 123}}, + {Id: "ws-upstream", Type: common.UpstreamTypeEvm, Endpoint: "ws" + strings.TrimPrefix(upB.URL, "http"), Evm: &common.EvmUpstreamConfig{ChainId: 123}}, + } + cfg.Projects = append(cfg.Projects, &second) + srv, _, cancel := startTestERPCServer(t, cfg) + defer func() { + _ = srv.Shutdown(&log.Logger) + cancel() + }() + + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go srv.PrewarmUpstreamSubscriptions(ctx) + + require.Eventually(t, func() bool { return subscribesA.Load() > 0 }, 5*time.Second, 20*time.Millisecond, + "first project must be prewarmed") + assert.Eventually(t, func() bool { return subscribesB.Load() > 0 }, 5*time.Second, 20*time.Millisecond, + "second project on the same chain must be prewarmed on its own upstream") +} + +// With the WebSocket server disabled nothing consumes upstream subscriptions, +// so prewarm must not open them. +func TestPrewarmUpstreamSubscriptions_SkippedWhenWebSocketDisabled(t *testing.T) { + setupGock() + defer util.ResetGock() + + prev := prewarmInterval + prewarmInterval = 50 * time.Millisecond + defer func() { prewarmInterval = prev }() + + var subscribes atomic.Int32 + up := mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, _ map[string]interface{}) { + if method == "eth_subscribe" { + subscribes.Add(1) + } + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + }) + }) + defer up.Close() + + cfg := standardWsConfig("ws" + strings.TrimPrefix(up.URL, "http")) + cfg.Server.WebSocket = &common.WebSocketServerConfig{Enabled: util.BoolPtr(false)} + srv, _, cancel := startTestERPCServer(t, cfg) + defer func() { + _ = srv.Shutdown(&log.Logger) + cancel() + }() + + ctx, stop := context.WithTimeout(context.Background(), time.Second) + defer stop() + srv.PrewarmUpstreamSubscriptions(ctx) + + assert.Zero(t, subscribes.Load(), "prewarm must not subscribe upstream while the WebSocket server is disabled") +} diff --git a/erpc/http_server_test.go b/erpc/http_server_test.go index ba47c0a00..ef59252ff 100644 --- a/erpc/http_server_test.go +++ b/erpc/http_server_test.go @@ -8584,3 +8584,53 @@ func TestHttpServer_AdminMethodFilter(t *testing.T) { assert.True(t, hasError, "invalid pattern expression should propagate as a request error") }) } + +// eth_subscribe/eth_unsubscribe require a persistent connection and are served +// only on the WebSocket path. Over HTTP eRPC must reject them, not forward: with +// a WS link to the upstream it would otherwise create a real upstream +// subscription and hand the HTTP client a subscription id that is immediately +// orphaned (notifications flow to eRPC's link, not the client) — a fail-open +// where a "did we subscribe?" guard gets a false positive on a dead stream. +func TestHttpServer_RejectsSubscriptionMethodsOverHttp(t *testing.T) { + cfg := &common.Config{ + Server: &common.ServerConfig{ + MaxTimeout: common.Duration(5 * time.Second).Ptr(), + }, + Projects: []*common.ProjectConfig{ + { + Id: "test_project", + Networks: []*common.NetworkConfig{ + { + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }, + }, + Upstreams: []*common.UpstreamConfig{ + { + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + }, + }, + }, + RateLimiters: &common.RateLimiterConfig{}, + } + + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + sendRequest, _, _, shutdown, _ := createServerTestFixtures(cfg, t) + defer shutdown() + + for _, method := range []string{"eth_subscribe", "eth_unsubscribe"} { + statusCode, _, body := sendRequest( + fmt.Sprintf(`{"jsonrpc":"2.0","method":%q,"params":["newHeads"],"id":1}`, method), nil, nil) + + assert.Equal(t, http.StatusOK, statusCode, "%s: JSON-RPC errors ride a 200 for POST", method) + assert.Contains(t, body, "-32601", "%s over HTTP must be method-not-supported (-32601)", method) + assert.Contains(t, body, "WebSocket", "%s rejection must point the client at WebSocket", method) + assert.NotContains(t, body, `"result"`, "%s must not fail open with a subscription id", method) + } +} diff --git a/erpc/http_server_ws_upgrade_test.go b/erpc/http_server_ws_upgrade_test.go new file mode 100644 index 000000000..c6c5b981b --- /dev/null +++ b/erpc/http_server_ws_upgrade_test.go @@ -0,0 +1,92 @@ +package erpc + +import ( + "context" + "net" + "net/http" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/util" + "github.com/gorilla/websocket" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" +) + +// TestHttpServer_WebSocketUpgradeReachesDispatch pins that a WebSocket upgrade +// on the bare aliased root "/" reaches the WS dispatch. The empty-path +// classifier used to mark any non-POST root request as a healthcheck, so +// wss://host/ (common with domain aliasing) was served as health and never +// upgraded. +func TestHttpServer_WebSocketUpgradeReachesDispatch(t *testing.T) { + logger := log.Logger + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cfg := &common.Config{ + Server: &common.ServerConfig{ + ListenV4: util.BoolPtr(true), + MaxTimeout: common.Duration(5 * time.Second).Ptr(), + WebSocket: &common.WebSocketServerConfig{Enabled: util.BoolPtr(true)}, + Aliasing: &common.AliasingConfig{ + Rules: []*common.AliasingRuleConfig{ + {MatchDomain: "*", ServeProject: "test_project", ServeArchitecture: "evm", ServeChain: "123"}, + }, + }, + }, + Projects: []*common.ProjectConfig{ + { + Id: "test_project", + Networks: []*common.NetworkConfig{ + {Architecture: common.ArchitectureEvm, Evm: &common.EvmNetworkConfig{ChainId: 123}}, + }, + Upstreams: []*common.UpstreamConfig{ + {Type: common.UpstreamTypeEvm, Endpoint: "http://rpc1.localhost", Evm: &common.EvmUpstreamConfig{ChainId: 123}}, + }, + }, + }, + RateLimiters: &common.RateLimiterConfig{}, + } + require.NoError(t, cfg.SetDefaults(nil)) + + ssr, err := data.NewSharedStateRegistry(ctx, &logger, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: "memory", + Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}, + }, + }) + require.NoError(t, err) + + erpcInstance, err := NewERPC(ctx, &logger, ssr, nil, nil, cfg) + require.NoError(t, err) + erpcInstance.Bootstrap(ctx) + + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) + require.NoError(t, err) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = httpServer.serverV4.Serve(listener) }() + defer httpServer.serverV4.Shutdown(context.Background()) + base := "ws://" + listener.Addr().String() + + assertReachedWS := func(path string, hdr http.Header) { + dctx, dcancel := context.WithTimeout(ctx, 5*time.Second) + defer dcancel() + c, resp, derr := websocket.DefaultDialer.DialContext(dctx, base+path, hdr) + if resp != nil { + _ = resp.Body.Close() + } + require.NoErrorf(t, derr, "%s must upgrade to WebSocket", path) + _ = c.Close() + } + + assertReachedWS("/", nil) + + // A proxy (e.g. Traefik) that adds Accept-Encoding: gzip to the upgrade + // request must not break the handshake: the gzip writer is not an + // http.Hijacker, which used to 500 the upgrade. + assertReachedWS("/", http.Header{"Accept-Encoding": []string{"gzip"}}) +} diff --git a/erpc/init.go b/erpc/init.go index 59dec553e..276f7ef01 100644 --- a/erpc/init.go +++ b/erpc/init.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "net" "net/http" "time" @@ -165,6 +164,10 @@ func Init( } } }() + // Proactively establish upstream newHeads subscriptions so the state + // poller's latest-block tip is fed by pushed heads instead of RPC + // polling — without waiting for a client eth_subscribe. + go httpServer.PrewarmUpstreamSubscriptions(appCtx) } if cfg.Server != nil && cfg.Server.GrpcEnabled != nil && *cfg.Server.GrpcEnabled && !grpcSharesHttpV4(cfg.Server) { grpcServer, err := NewGrpcServer(appCtx, &logger, cfg.Server, erpcInstance) @@ -178,6 +181,7 @@ func Init( } }() } + var metricsSrv *http.Server if cfg.Metrics != nil && cfg.Metrics.Enabled != nil && *cfg.Metrics.Enabled { if cfg.Metrics.ErrorLabelMode != "" { common.SetErrorLabelMode(cfg.Metrics.ErrorLabelMode) @@ -186,10 +190,9 @@ func Init( return fmt.Errorf("metrics.port is not configured") } logger.Info().Msgf("starting metrics server on port: %d", *cfg.Metrics.Port) - srv := &http.Server{ - BaseContext: func(ln net.Listener) context.Context { - return appCtx - }, + // No appCtx BaseContext: scrapes keep working after SIGTERM, until the + // shutdown below. + metricsSrv = &http.Server{ Addr: fmt.Sprintf(":%d", *cfg.Metrics.Port), // promhttp.Handler() with the gatherer wrapped, so drop customizations // also govern the stock collectors the manager does not own. @@ -200,28 +203,19 @@ func Init( ReadHeaderTimeout: 10 * time.Second, } go func() { - if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + if err := metricsSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { logger.Error().Msgf("error starting metrics server: %s", err) util.OsExit(util.ExitCodeHttpServerFailed) } }() - go func() { - <-appCtx.Done() - logger.Info().Msg("shutting down metrics server...") - shutdownCtx, cancel := context.WithTimeout(appCtx, 5*time.Second) - defer cancel() - if err := srv.Shutdown(shutdownCtx); err != nil { - logger.Error().Msgf("metrics server forced to shutdown: %s", err) - } else { - logger.Info().Msg("metrics server stopped") - } - }() } // Wait until the context is cancelled, then for the http server to finish // draining (waitBeforeShutdown, then Shutdown with its 30s budget) — main // exits as soon as Init returns, which would kill every request still in - // flight — and only then give exporters waitAfterShutdown to flush. + // flight — and only then give exporters waitAfterShutdown to flush. The + // metrics server stays up through all of it, so the scrapes inside + // waitAfterShutdown see every increment made during the drain. <-appCtx.Done() logger.Info().Msg("shutting down gracefully...") if httpDrained != nil { @@ -234,6 +228,16 @@ func Init( if cfg.Server != nil && cfg.Server.WaitAfterShutdown != nil { time.Sleep(cfg.Server.WaitAfterShutdown.Duration()) } + if metricsSrv != nil { + logger.Info().Msg("shutting down metrics server...") + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := metricsSrv.Shutdown(shutdownCtx); err != nil { + logger.Error().Msgf("metrics server forced to shutdown: %s", err) + } else { + logger.Info().Msg("metrics server stopped") + } + } return nil } diff --git a/erpc/multicall3_test.go b/erpc/multicall3_test.go new file mode 100644 index 000000000..1651a5483 --- /dev/null +++ b/erpc/multicall3_test.go @@ -0,0 +1,381 @@ +package erpc + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/ethereum/go-ethereum/accounts/abi" + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + mc3TestEcho = "0x1111111111111111111111111111111111111111" // returns its calldata + mc3TestRevert = "0x2222222222222222222222222222222222222222" // always reverts + mc3TestEmpty = "0x3333333333333333333333333333333333333333" // returns nothing + mc3TestSender = "0x4444444444444444444444444444444444444444" // reverts only when the caller is Multicall3 + mc3TestBypass = "0x6666666666666666666666666666666666666666" // returns its calldata; listed in bypassContracts + mc3TestDeploy = 0x10 // Multicall3 has no code at or below this block +) + +var mc3TestABI = func() abi.ABI { + a, err := abi.JSON(strings.NewReader(`[{"name":"aggregate3","type":"function","stateMutability":"payable", + "inputs":[{"name":"calls","type":"tuple[]","components":[{"name":"target","type":"address"},{"name":"allowFailure","type":"bool"},{"name":"callData","type":"bytes"}]}], + "outputs":[{"name":"returnData","type":"tuple[]","components":[{"name":"success","type":"bool"},{"name":"returnData","type":"bytes"}]}]}]`)) + if err != nil { + panic(err) + } + return a +}() + +// mc3TestNode is a fake EVM vendor: it runs eth_calls (single and through a real +// aggregate3 ABI codec) and counts what it was sent. +type mc3TestNode struct { + mode atomic.Value // "ok" | "error" | "garbage" + mu sync.Mutex + multicalls int + gasOfPacks map[string]int + fromOfPacks map[string]int + packedTargets map[string]int + singles map[string]int +} + +func (f *mc3TestNode) reset(mode string) { + f.mu.Lock() + defer f.mu.Unlock() + f.mode.Store(mode) + f.multicalls = 0 + f.gasOfPacks = map[string]int{} + f.fromOfPacks = map[string]int{} + f.packedTargets = map[string]int{} + f.singles = map[string]int{} +} + +func (f *mc3TestNode) counts() (int, map[string]int) { + f.mu.Lock() + defer f.mu.Unlock() + s := map[string]int{} + for k, v := range f.singles { + s[k] = v + } + return f.multicalls, s +} + +func (f *mc3TestNode) run(to string, data []byte, viaMulticall bool) ([]byte, bool) { + switch strings.ToLower(to) { + case mc3TestEcho, mc3TestBypass: + return data, true + case mc3TestEmpty: + return nil, true + case mc3TestSender: + return data, !viaMulticall + default: + return []byte{0xde, 0xad, 0xbe, 0xef}, false + } +} + +func (f *mc3TestNode) answer(method string, params []interface{}) (interface{}, map[string]interface{}) { + switch method { + case "eth_chainId": + return "0x7b", nil + case "eth_blockNumber": + return "0x100", nil + case "eth_syncing": + return false, nil + case "eth_getBlockByNumber": + return map[string]interface{}{"number": "0x100", "hash": "0x" + strings.Repeat("ab", 32), "timestamp": "0x1"}, nil + case "eth_call": + default: + return nil, nil + } + call := params[0].(map[string]interface{}) + to := call["to"].(string) + data, _ := hex.DecodeString(strings.TrimPrefix(call["data"].(string), "0x")) + block, _ := params[1].(string) + + if !strings.EqualFold(to, "0xcA11bde05977b3631167028862bE2a173976CA11") { + f.mu.Lock() + f.singles[to]++ + f.mu.Unlock() + out, ok := f.run(to, data, false) + if !ok { + return nil, map[string]interface{}{"code": 3, "message": "execution reverted", "data": "0x" + hex.EncodeToString(out)} + } + return "0x" + hex.EncodeToString(out), nil + } + + f.mu.Lock() + f.multicalls++ + gas, _ := call["gas"].(string) + f.gasOfPacks[gas]++ + from, _ := call["from"].(string) + f.fromOfPacks[from]++ + f.mu.Unlock() + switch f.mode.Load().(string) { + case "error": + return nil, map[string]interface{}{"code": -32000, "message": "header not found"} + case "garbage": + return "0x1234", nil + } + if bn, err := common.HexToInt64(block); err == nil && bn <= mc3TestDeploy { + return "0x", nil + } + method3 := mc3TestABI.Methods["aggregate3"] + if len(data) < 4 || hex.EncodeToString(data[:4]) != hex.EncodeToString(method3.ID) { + return nil, map[string]interface{}{"code": -32000, "message": "bad selector"} + } + args, err := method3.Inputs.Unpack(data[4:]) + if err != nil { + return nil, map[string]interface{}{"code": -32000, "message": err.Error()} + } + type call3 struct { + Target ethcommon.Address + AllowFailure bool + CallData []byte + } + type result3 struct { + Success bool + ReturnData []byte + } + calls := *abi.ConvertType(args[0], new([]call3)).(*[]call3) + results := make([]result3, len(calls)) + for i, c := range calls { + target := strings.ToLower(c.Target.Hex()) + f.mu.Lock() + f.packedTargets[target]++ + f.mu.Unlock() + out, ok := f.run(target, c.CallData, true) + results[i] = result3{Success: ok, ReturnData: out} + } + packed, err := method3.Outputs.Pack(results) + if err != nil { + return nil, map[string]interface{}{"code": -32000, "message": err.Error()} + } + return "0x" + hex.EncodeToString(packed), nil +} + +func (f *mc3TestNode) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var req struct { + ID interface{} `json:"id"` + Method string `json:"method"` + Params []interface{} `json:"params"` + } + if err := json.Unmarshal(body, &req); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + result, rpcErr := f.answer(req.Method, req.Params) + resp := map[string]interface{}{"jsonrpc": "2.0", "id": req.ID} + if rpcErr != nil { + resp["error"] = rpcErr + } else { + resp["result"] = result + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) +} + +func TestMulticall3Packing(t *testing.T) { + node := &mc3TestNode{} + node.reset("ok") + srv := httptest.NewServer(node) + defer srv.Close() + + cfg := &common.Config{ + Server: &common.ServerConfig{MaxTimeout: common.Duration(10 * time.Second).Ptr()}, + Projects: []*common.ProjectConfig{{ + Id: "test_project", + Networks: []*common.NetworkConfig{{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }}, + Upstreams: []*common.UpstreamConfig{{ + Id: "vendor", + Type: common.UpstreamTypeEvm, + Endpoint: srv.URL, + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + Multicall3: &common.EvmMulticall3Config{ + MaxWait: common.Duration(300 * time.Millisecond), + MaxCalls: 50, + BypassContracts: []string{mc3TestBypass}, + }, + }, + }}, + }}, + RateLimiters: &common.RateLimiterConfig{}, + } + sendRequest, _, _, shutdown, _ := createServerTestFixtures(cfg, t) + defer shutdown() + + var seq atomic.Int64 + type call struct { + to, block, from, gas string + data string + } + newCall := func(to, block string) call { + return call{to: to, block: block, data: fmt.Sprintf("0x%08x", seq.Add(1))} + } + type answer struct { + Result string `json:"result"` + Error map[string]interface{} `json:"error"` + } + sendAll := func(calls []call) []answer { + out := make([]answer, len(calls)) + var wg sync.WaitGroup + for i, c := range calls { + wg.Add(1) + go func(i int, c call) { + defer wg.Done() + obj := fmt.Sprintf(`"to":"%s","data":"%s"`, c.to, c.data) + if c.from != "" { + obj += fmt.Sprintf(`,"from":"%s"`, c.from) + } + if c.gas != "" { + obj += fmt.Sprintf(`,"gas":"%s"`, c.gas) + } + obj = "{" + obj + "}" + status, _, body := sendRequest(fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"eth_call","params":[%s,"%s"]}`, i+1, obj, c.block), nil, nil) + require.Equal(t, http.StatusOK, status, body) + require.NoError(t, json.Unmarshal([]byte(body), &out[i]), body) + }(i, c) + } + wg.Wait() + return out + } + assertEchoed := func(t *testing.T, calls []call, answers []answer) { + for i, c := range calls { + if c.to == mc3TestEcho || c.to == mc3TestSender { + assert.Equal(t, c.data, answers[i].Result, "call %d to %s", i, c.to) + assert.Nil(t, answers[i].Error, "call %d to %s", i, c.to) + } + } + } + + t.Run("packs concurrent calls and sends alone each call the pack cannot answer exactly", func(t *testing.T) { + node.reset("ok") + calls := []call{ + newCall(mc3TestEcho, "latest"), newCall(mc3TestEcho, "latest"), newCall(mc3TestEcho, "latest"), + newCall(mc3TestEcho, "latest"), newCall(mc3TestEcho, "latest"), + newCall(mc3TestRevert, "latest"), newCall(mc3TestEmpty, "latest"), newCall(mc3TestSender, "latest"), + } + answers := sendAll(calls) + + assertEchoed(t, calls, answers) + assert.Equal(t, "", answers[5].Result) + require.NotNil(t, answers[5].Error) + assert.Contains(t, fmt.Sprint(answers[5].Error["message"]), "execution reverted") + assert.Equal(t, "0x", answers[6].Result) + + multicalls, singles := node.counts() + assert.Equal(t, 1, multicalls) + assert.Equal(t, map[string]int{mc3TestRevert: singles[mc3TestRevert], mc3TestEmpty: 1, mc3TestSender: 1}, singles) + assert.GreaterOrEqual(t, singles[mc3TestRevert], 1) + }) + + t.Run("bypassed contracts are sent alone: configured, and learned from a pack revert", func(t *testing.T) { + // The previous subtest saw mc3TestSender revert inside the pack and + // succeed alone, so it is learned. + node.reset("ok") + calls := []call{newCall(mc3TestEcho, "latest"), newCall(mc3TestEcho, "latest"), newCall(mc3TestSender, "latest"), newCall(mc3TestBypass, "latest")} + answers := sendAll(calls) + + assertEchoed(t, calls, answers) + assert.Equal(t, calls[3].data, answers[3].Result) + multicalls, singles := node.counts() + assert.Equal(t, 1, multicalls) + assert.Equal(t, map[string]int{mc3TestSender: 1, mc3TestBypass: 1}, singles) + node.mu.Lock() + defer node.mu.Unlock() + assert.Equal(t, map[string]int{mc3TestEcho: 2}, node.packedTargets) + }) + + for _, mode := range []string{"error", "garbage"} { + t.Run("multicall "+mode+" sends every call alone", func(t *testing.T) { + node.reset(mode) + calls := []call{newCall(mc3TestEcho, "latest"), newCall(mc3TestEcho, "latest"), newCall(mc3TestEcho, "latest"), newCall(mc3TestEcho, "latest")} + answers := sendAll(calls) + + assertEchoed(t, calls, answers) + multicalls, singles := node.counts() + assert.GreaterOrEqual(t, multicalls, 1) + assert.Equal(t, map[string]int{mc3TestEcho: 4}, singles) + }) + } + + t.Run("calls pack only with calls of the same gas limit and sender", func(t *testing.T) { + node.reset("ok") + withGas := func() call { + c := newCall(mc3TestEcho, "latest") + c.gas = "0x2faf080" + return c + } + zero := "0x" + strings.Repeat("0", 40) + fromZero := func() call { + c := newCall(mc3TestEcho, "latest") + c.from = zero + return c + } + calls := []call{ + withGas(), withGas(), withGas(), + newCall(mc3TestEcho, "latest"), newCall(mc3TestEcho, "latest"), + fromZero(), fromZero(), + } + answers := sendAll(calls) + + assertEchoed(t, calls, answers) + node.mu.Lock() + defer node.mu.Unlock() + assert.Equal(t, map[string]int{"0x2faf080": 1, "": 2}, node.gasOfPacks) + assert.Equal(t, map[string]int{"": 2, zero: 1}, node.fromOfPacks) + assert.Empty(t, node.singles) + }) + + t.Run("lone and unpackable calls are sent alone without a multicall", func(t *testing.T) { + node.reset("ok") + withSender := newCall(mc3TestEcho, "latest") + withSender.from = "0x5555555555555555555555555555555555555555" + calls := []call{withSender, newCall(mc3TestEcho, "latest")} + answers := sendAll(calls) + + assertEchoed(t, calls, answers) + multicalls, singles := node.counts() + assert.Equal(t, 0, multicalls) + assert.Equal(t, map[string]int{mc3TestEcho: 2}, singles) + }) + + t.Run("blocks before Multicall3 existed are learned and then sent alone", func(t *testing.T) { + node.reset("ok") + calls := []call{newCall(mc3TestEcho, "0x10"), newCall(mc3TestEcho, "0x10"), newCall(mc3TestEcho, "0x10")} + assertEchoed(t, calls, sendAll(calls)) + multicalls, singles := node.counts() + assert.Equal(t, 1, multicalls) + assert.Equal(t, map[string]int{mc3TestEcho: 3}, singles) + + node.reset("ok") + calls = []call{newCall(mc3TestEcho, "0xf"), newCall(mc3TestEcho, "0xf"), newCall(mc3TestEcho, "0xf")} + assertEchoed(t, calls, sendAll(calls)) + multicalls, singles = node.counts() + assert.Equal(t, 0, multicalls) + assert.Equal(t, map[string]int{mc3TestEcho: 3}, singles) + + node.reset("ok") + calls = []call{newCall(mc3TestEcho, "0x20"), newCall(mc3TestEcho, "0x20")} + assertEchoed(t, calls, sendAll(calls)) + multicalls, singles = node.counts() + assert.Equal(t, 1, multicalls) + assert.Empty(t, singles) + }) +} diff --git a/erpc/networks.go b/erpc/networks.go index 2b3a8b026..3f8edba67 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "iter" "math" "runtime/debug" "slices" @@ -34,7 +35,7 @@ type Network struct { networkLabel string projectId string logger *zerolog.Logger - bootstrapOnce sync.Once + servedTipRefreshOnce sync.Once appCtx context.Context cfg *common.NetworkConfig inFlightRequests *sync.Map @@ -60,32 +61,21 @@ type Network struct { // a WebSocket subscriber. deliveredLatestBlock atomic.Int64 - // servedLatest / servedFinalized are STRICT-MONOTONIC at the network level: - // once we serve a tip of N to clients, EvmHighestLatest/FinalizedBlockNumber - // servedTipAnchor watchdogs track when this process last SAW the served - // value change — purely for the advance-age stuck-tip gauge. The pick - // itself is stateless (common.PickServedTip); nothing here feeds back into - // what clients receive. - servedLatestAnchor servedTipAnchor - servedFinalizedAnchor servedTipAnchor + // servedTipAll is the network-wide served-tip lane. Lanes hold the + // advance-age watchdog and the regression guard's memory; the pick itself + // is stateless (common.PickServedTip). + servedTipAll servedTipLane - // servedTipPartitions holds lazily-materialized per-group LANES for + // servedTipLanes holds lazily-materialized per-group lanes for // use-upstream selectors that carve out a real sub-group (e.g. - // `flashblocks*` / `family:systx`): a stable gauge label (LaneName) plus - // watchdog anchors. TELEMETRY ONLY — the pick for any selector is the - // same stateless majority over the matched subset. Bounded by - // construction: keyed by the hash of the MATCHED UPSTREAM SET (equivalent - // selectors dedup; garbage selectors can't inflate state beyond the real - // topology) and capped by maxServedTipPartitions. - servedTipPartitions sync.Map // map[string]*servedTipPartition (key = "grp:") - servedTipPartitionCount atomic.Int32 - - // servedTipBlockTimeOverride, when > 0, replaces the tracker's EMA block - // time in buildServedTipConfig. Set ONLY from package-internal tests: the - // EMA needs live timestamped blocks that test fixtures don't produce, and - // the prod-incident invariant tests must arm the velocity gate exactly the - // way prod had it armed. - servedTipBlockTimeOverride float64 + // `flashblocks*` / `family:systx`). The pick for any selector is the same + // stateless quorum over the matched subset; a lane adds only its gauge + // label (LaneName), watchdog and guard memory. Bounded by construction: + // keyed by the hash of the MATCHED UPSTREAM SET (equivalent selectors + // dedup; garbage selectors can't inflate state beyond the real topology) + // and capped by maxServedTipLanes. + servedTipLanes sync.Map // map[string]*servedTipLane (key = "grp:") + servedTipLaneCount atomic.Int32 // queryTraceSource remembers which trace method answered the eth_query* // shim on this network (queryTraceSource* values), so a network whose @@ -120,19 +110,25 @@ func (n *Network) NoteObservedLatestBlock(ctx context.Context, blockNumber int64 } } -// maxServedTipPartitions caps the number of materialized per-tag served-tip -// partitions per network (a backstop on top of the "must be a configured tag" -// gate). Realistic configs have 1–3 groups; beyond the cap, extra tags fall -// back to the stateless subset candidate. -const maxServedTipPartitions = 16 +// maxServedTipLanes caps the materialized group lanes per network, a backstop +// on top of the matched-set dedup. Realistic configs have 1–3 groups; beyond +// the cap, a group's request gets the stateless subset pick with no lane. +const maxServedTipLanes = 16 -// servedTipPartition is one node group's telemetry lane: the stable gauge -// label (common.LaneName of the matched upstream set) and the group's -// watchdog anchors. It holds NO pick state. -type servedTipPartition struct { - lane string - latestAnchor servedTipAnchor - finalizedAnchor servedTipAnchor +// servedTipLane is one served-tip scope with its own telemetry and guard +// memory: the network-wide pick (label "all") or a use-upstream group's pick +// (label common.LaneName of the matched upstream set). It holds NO pick state. +type servedTipLane struct { + label string + latest servedTipAnchor + finalized servedTipAnchor +} + +func (l *servedTipLane) anchor(finalized bool) *servedTipAnchor { + if finalized { + return &l.finalized + } + return &l.latest } // servedTipAnchor tracks, per process, when the served-tip VALUE was last @@ -144,8 +140,8 @@ type servedTipPartition struct { // value actually move. // // IN-PROCESS ONLY, by design: nothing here is persisted or shared between -// pods. A persisted anchor plus a strict clamp is exactly what wedged the -// served tip fleet-wide in 2026-06 (see the picker's doc comment and +// pods. A persisted anchor plus a strict clamp is exactly what produces the +// frozen-tip regression (see the picker's doc comment and // networks_served_tip_invariants_test.go); every field below is rebuilt from // live observations after a restart, and the guard always fails open on its // own after a bounded window. @@ -169,32 +165,8 @@ type servedTipAnchor struct { // on transitions, so a sustained regression reports twice (once when it // starts holding, once if it gives up) instead of once per request. guardState atomic.Int32 - - // trajectory is this lane+axis's long-term head track — the referee that - // lets a corroborated FRESH MINORITY outvote a stalled majority - // (refereeServedTip). It holds no pick either: its samples are the plain - // majority over live heads, its output is always one of the head values the - // live upstreams offer in the same evaluation, and it can only ever raise - // the pick. See evm.TipTrajectory. - trajectory evm.TipTrajectory - - // refereeState is one of the servedTipReferee* constants: like guardState, - // both the referee's counter and its log fire on state ENTRY only. - refereeState atomic.Int32 } -const ( - // servedTipRefereeIdle: the referee is not intervening. - servedTipRefereeIdle int32 = iota - // servedTipRefereeOverriding: a corroborated group is being served in place - // of the majority pick. - servedTipRefereeOverriding - // servedTipRefereeDeclining: a fresher corroborated group sat above the - // majority pick and was refused — too far off the trajectory, or it has not - // held its place long enough to have earned it. - servedTipRefereeDeclining -) - const ( // servedTipGuardHealthy: the pick tracks the live heads. servedTipGuardHealthy int32 = iota @@ -243,24 +215,6 @@ func (a *servedTipAnchor) age() time.Duration { return -1 } -// servedTipLaneAll is the lane label for the network-wide served tip. -const servedTipLaneAll = "all" - -// servedTipLaneNone is the sentinel passed to clusteredServedTip for a -// stateless selector-scoped tip (glob/single-node/match-all that did not form a -// group): observeServedTipMetrics emits nothing for it, so such a subset value -// never overwrites any gauge. -const servedTipLaneNone = "\x00scoped" - -// servedTipLaneLabel maps an internal lane to its metric label: the -// network-wide pick ("") is published as lane="all". -func servedTipLaneLabel(lane string) string { - if lane == "" { - return servedTipLaneAll - } - return lane -} - // Bootstrap registers this network with the policy engine. The engine kicks // off the slot's ticker and runs an initial synchronous eval so request-path // reads through `policyEngine.GetOrdered` always see a populated cache. @@ -473,113 +427,82 @@ func (n *Network) Logger() *zerolog.Logger { return n.logger } -// gatherEvmTipInputsForMethod builds the picker inputs from the upstreams that -// the selection policy considers ELIGIBLE for `method` — so an upstream the -// user (or an integrity guard) excluded/cordoned drops out of head tracking in -// the same place it drops out of routing. `method == "*"` yields the -// network-wide eligible set (the global served tip); a concrete method yields -// the per-method eligible set (a capability lane). Falls back to the full -// registered set on cold start / when the policy has produced no decision yet. -// -// The ballot itself (who votes, and the guard's live reference) is -// evmTipBallot's rule. -func (n *Network) gatherEvmTipInputsForMethod( - ctx context.Context, - useFinalized bool, - method string, -) (common.ServedTipPick, servedTipReference) { - return evmTipBallot(n.tipCandidateUpstreams(ctx, method), useFinalized) -} - -// evmTipBallot turns one candidate set into the served-tip ballot for an axis, -// plus the regression guard's view of its live heads (servedTipReference). +// tipVoterRank is the one rule for who votes on a served-tip ballot. ok is +// false for an upstream with no head on the axis: non-EVM, no state poller, +// syncing, or no head yet. Otherwise rank orders the voters, and a ballot +// counts only the lowest rank present: 0 live, 1 benched, 2 static cap, +// 3 benched static cap. // // STATIC-CAP UPSTREAMS DO NOT VOTE while any live head is present. An upstream // configured with `blockAvailability.upper.exactBlock` below the chain's head is // declaring a SERVING RANGE ("I answer up to block X"), not observing where the -// head is — see evmTipObservation. Letting such a declaration onto the ballot is -// what broke celo (evm:42220) on 2026-08-11: two legacy upstreams pinned at -// `upper.exactBlock: 21615999` outvoted the live heads (74.5M) the moment a -// third live upstream was excluded for lag, "latest" interpolated to a -// September-2023 block, the availability gate then admitted ONLY those two -// upstreams (the live ones start at 21616000), and funded accounts read 0x0. +// head is — see evmTipObservation. Letting such a declaration onto the ballot +// lets two capped legacy upstreams pull the tip down: pinned at an +// `upper.exactBlock` far below the live heads, they outvote them the moment a +// third live upstream is excluded for lag, "latest" interpolates to a +// historical block, the availability gate then admits ONLY those two +// upstreams (the live ones start above the cap), and funded accounts read 0x0. +// +// BENCHED UPSTREAMS DO NOT VOTE while any non-benched one does: the health +// tracker benches a head that stayed behind peers that kept advancing +// (health.Tracker.IsHeadBenched), so one stalled node cannot hold +// "latest" at its height. A live head always outranks a cap, benched or not: +// a stalled live node is still closer to the chain than a serving range. +func tipVoterRank(u common.Upstream, finalized bool, tracker *health.Tracker) (head int64, rank int, ok bool) { + eu, isEvm := u.(common.EvmUpstream) + if !isEvm { + return 0, 0, false + } + if sp := eu.EvmStatePoller(); sp == nil || sp.IsObjectNull() || eu.EvmSyncingState() == common.EvmSyncingStateSyncing { + return 0, 0, false + } + head, capped := evmTipObservation(eu, finalized) + if head <= 0 { + return 0, 0, false + } + if capped { + rank = 2 + } + if tracker != nil && tracker.IsHeadBenched(u) { + rank++ + } + return head, rank, true +} + +// evmTipBallot builds the served-tip pick for an axis under quorum q from the +// request's tip candidates (EvmRoutableUpstreams) that satisfy member (nil = +// all of them): the heads of the best-ranked voters (tipVoterRank). live +// reports whether live, non-capped heads decided it (benched or not): a +// cap-only ballot gives the regression guard no live reference. // // MIXED POOL vs ALL-CAPPED POOL. The rule applies per candidate set, which is // what lets the same function build the network-wide ballot AND each // guaranteed-method's floor ballot: caps are dropped only when the SAME set // contains at least one live head. A set of only historical/frozen upstreams // legitimately serves its cap — that cap is the freshest block anyone in it can -// serve — so it keeps voting. -func evmTipBallot(upstreams []common.Upstream, useFinalized bool) (common.ServedTipPick, servedTipReference) { - live := make([]common.ServedTipInput, 0, len(upstreams)) - var capped []common.ServedTipInput - for _, cu := range upstreams { - u, ok := cu.(common.EvmUpstream) - if !ok || u.EvmStatePoller() == nil { +// serve — so it keeps voting. (No live head means no live reference for the +// regression guard either; it falls back to its own last corroborated pick, +// see guardServedTipRegression.) +func (n *Network) evmTipBallot(ctx context.Context, useFinalized bool, q common.ServedTipQuorum, member func(common.Upstream) bool) (common.ServedTipPick, bool) { + // Typical pools fit the stack buffer, so the ballot stays off the heap; + // append moves a larger one there. + var buf [16]int64 + heads := buf[:0] + best := -1 + for u := range n.EvmRoutableUpstreams(ctx) { + if member != nil && !member(u) { continue } - if u.EvmSyncingState() == common.EvmSyncingStateSyncing { + head, rank, ok := tipVoterRank(u, useFinalized, n.metricsTracker) + if !ok || (best >= 0 && rank > best) { continue } - blk, boundCapped := evmTipObservation(u, useFinalized) - if blk <= 0 { - continue - } - in := common.ServedTipInput{ - UpstreamID: u.Id(), - BlockNumber: blk, - } - if boundCapped { - capped = append(capped, in) - continue + if best < 0 || rank < best { + best, heads = rank, heads[:0] } - live = append(live, in) - } - if len(live) == 0 { - // All-capped set: there is no live head to prefer, so the caps remain - // the only observations available. (No live head means no live reference - // for the regression guard either; it falls back to its own last - // corroborated pick, see guardServedTipRegression.) - return common.PickServedTip(capped), servedTipReference{} - } - pick := common.PickServedTip(live) - return pick, servedTipReference{Corroborated: pick.Freshest, Max: pick.Max} -} - -// servedTipReference is what the LIVE (non-capped) heads of one ballot say, -// in the two distinct roles the regression guard needs. Both are 0 when every -// candidate is capped. -type servedTipReference struct { - // Corroborated is the SECOND-highest live head — the freshest view that two - // live upstreams both reach — or the only live head when there is exactly - // one. It is what a regression is MEASURED against, and it is second-highest - // for the same reason ServedTipPick.Freshest is: measured against the raw - // max, one far-future/wrong-chain upstream made every honest pick look like - // a regression, so the guard never took its healthy branch, never recorded a - // corroborated value, and sat permanently disarmed — emitting no metric and - // no log at all. - // - // BOUNDARY, stated because it is a real one: this tolerates ONE Byzantine - // head, not two. Two upstreams reporting the same far-future block are - // indistinguishable from two upstreams that are simply ahead, so they can - // still raise the reference and leave the guard unarmed (silently — the - // guard only records a corroborated value on its healthy branch). Nothing - // available in a single evaluation separates those cases; what limits the - // damage is elsewhere: the majority pick itself needs a MAJORITY to move, - // rememberServedTipPick refuses to commit an uncorroborated leap into the - // memory, and the trajectory referee's dwell refuses to elect a group that - // has not tracked the chain for half a minute. - Corroborated int64 - - // Max is the highest live head: the ceiling a HELD value may not exceed, so - // the guard can never advertise a block no live upstream has (the invariant - // the old monotonic clamp broke). It is deliberately the raw max and not the - // corroborated view — a ceiling that a single rogue can only RAISE cannot be - // used to make the guard serve anything the rogue chose, while a ceiling of - // the second-highest head would collapse the hold back onto the poisoned - // pick in the very shape the guard exists for (one live witness left, two - // stuck upstreams outvoting it). - Max int64 + heads = append(heads, head) + } + return common.PickServedTip(heads, q), best >= 0 && best <= 1 } // evmTipObservation returns an upstream's effective head for the axis, plus @@ -638,49 +561,58 @@ func configuredUpperExactBlock(u common.Upstream) *int64 { return cfg.Evm.BlockAvailability.Upper.ExactBlock } -// tipCandidateUpstreams returns the eligible upstreams that feed the served-tip -// picker for `method`, sourced from the selection policy so head tracking and -// routing share one notion of "which upstreams count". Falls back to the full -// registered set when the policy is absent or has not yet produced a decision. -func (n *Network) tipCandidateUpstreams(ctx context.Context, method string) []common.Upstream { - var ups []common.Upstream +// EvmRoutableUpstreams yields, in routing order, the upstreams eRPC can route +// the request to, which are also those that may vote on its served tip (and +// eRPC's peers for net_peerCount): the selection policy's eligible set, so an +// upstream the user (or an integrity guard) excluded or cordoned drops out of +// head tracking where it drops out of routing, else the full registered set +// while the policy is absent or has made no decision yet. It walks the +// policy's or registry's own slice in place, so callers on the hot path pay no +// copy. +// +// Selector-scoped served tip: when the request targets a subset of upstreams +// (the use-upstream id/tag selector), the network's `latest`/`finalized` must +// be decided AMONG that subset — otherwise a more-ahead group (e.g. base +// flashblocks vs normal) would define the tip for a request pinned to the +// other group, causing "block not found" churn. The filter is stateless, so an +// arbitrary selector value can never grow per-pod memory. No-op when there is +// no request or no selector in ctx (background/admin callers). +func (n *Network) EvmRoutableUpstreams(ctx context.Context) iter.Seq[common.Upstream] { + // Kept this small so it inlines: the closure then stays on the caller's + // stack instead of costing a heap allocation per walk. + return func(yield func(common.Upstream) bool) { n.walkTipCandidates(ctx, yield) } +} + +func (n *Network) walkTipCandidates(ctx context.Context, yield func(common.Upstream) bool) { + sel := requestSelector(ctx) + visit := func(u common.Upstream) bool { + if sel != "" { + if m, _ := common.UpstreamMatchesSelector(sel, u); !m { + return true + } + } + return yield(u) + } if n.policyEngine != nil { - if eligible := n.policyEngine.GetOrdered(n.networkId, method, "*"); len(eligible) > 0 { - ups = eligible - } - } - if ups == nil { - if n.upstreamsRegistry == nil { - // Defensive: a partially-initialized network (e.g. cache-only paths or tests - // that don't wire a registry) has none — report no candidates so the head - // accessors fail open (return 0) instead of dereferencing a nil registry. - return nil - } - raw := n.upstreamsRegistry.GetNetworkUpstreams(ctx, n.networkId) - ups = make([]common.Upstream, 0, len(raw)) - for _, u := range raw { - ups = append(ups, u) - } - } - - // Selector-scoped served tip: when the request targets a subset of - // upstreams (the use-upstream id/tag selector), the network's - // `latest`/`finalized` must be decided AMONG that subset — otherwise a - // more-ahead group (e.g. base flashblocks vs normal) would define the tip - // for a request pinned to the other group, causing "block not found" - // churn. This is stateless (no shared counter / no materialized state), - // so an arbitrary selector value can never grow per-pod memory. No-op when - // there is no request or no selector in ctx (background/admin callers). - if sel := requestSelector(ctx); sel != "" { - filtered := make([]common.Upstream, 0, len(ups)) - for _, u := range ups { - if m, _ := common.UpstreamMatchesSelector(sel, u); m { - filtered = append(filtered, u) + if eligible := n.policyEngine.GetOrdered(n.networkId, "*", "*"); len(eligible) > 0 { + for _, u := range eligible { + if !visit(u) { + return + } } + return + } + } + // A partially-initialized network (cache-only paths, tests without a + // registry) has no candidates, so the head accessors fail open (0). + if n.upstreamsRegistry == nil { + return + } + for _, u := range n.upstreamsRegistry.GetNetworkUpstreams(ctx, n.networkId) { + if !visit(u) { + return } - return filtered } - return ups } // requestSelector returns the use-upstream selector for the request bound to @@ -694,53 +626,57 @@ func requestSelector(ctx context.Context) string { return "" } -// servedTipPartitionFor returns the lazily-materialized telemetry lane for a -// GROUP-scoped request, or nil to signal the unlabeled stateless path. The group -// identity (key + lane, and its bounded/DDoS-safe gating) comes from -// EvmUpstreamGroupForSelector; the count is capped by maxServedTipPartitions. -func (n *Network) servedTipPartitionFor(ctx context.Context, selector string) *servedTipPartition { - key, lane := n.EvmUpstreamGroupForSelector(ctx, selector) +// servedTipLaneFor returns the served-tip lane of the request bound to ctx: +// the network-wide lane when it carries no use-upstream selector, the group's +// lazily-materialized lane when the selector carves out a real sub-group +// (identity and DDoS-safe gating from EvmUpstreamGroupForSelector, count capped +// by maxServedTipLanes), else nil: the subset pick then runs with no guard, no +// gauges and no anchor, so a subset value never overwrites any gauge. +func (n *Network) servedTipLaneFor(ctx context.Context) *servedTipLane { + sel := requestSelector(ctx) + if sel == "" { + return &n.servedTipAll + } + key, label := n.EvmUpstreamGroupForSelector(ctx, sel) if key == "" { return nil } - // Fast path: already materialized. - if p, ok := n.servedTipPartitions.Load(key); ok { - return p.(*servedTipPartition) + if l, ok := n.servedTipLanes.Load(key); ok { + return l.(*servedTipLane) } // Cap backstop (also bounds a race racing past the cap). - if n.servedTipPartitionCount.Load() >= maxServedTipPartitions { + if n.servedTipLaneCount.Load() >= maxServedTipLanes { return nil } - p := &servedTipPartition{lane: lane} - actual, loaded := n.servedTipPartitions.LoadOrStore(key, p) + actual, loaded := n.servedTipLanes.LoadOrStore(key, &servedTipLane{label: label}) if !loaded { - n.servedTipPartitionCount.Add(1) + n.servedTipLaneCount.Add(1) } - return actual.(*servedTipPartition) + return actual.(*servedTipLane) } -// partitionKeyFor maps a use-upstream selector to a STABLE served-tip partition -// key, or "" when the selector must use the stateless path. +// partitionKeyFor maps a use-upstream selector to a STABLE group key, or "" +// when the selector forms no group. // -// A per-group monotonic tracker is materialized only for a "simple" selector (a -// single glob token, optionally negated — e.g. `flashblocks*`, `!flashblocks*`, -// or an exact tag like `family:systx`) that carves out a real SUB-group of the -// network's upstreams. The key is a hash of the MATCHED UPSTREAM SET, not the -// selector text, which makes the whole thing bounded and cross-pod safe: +// A group forms only for a "simple" selector (a single glob token, optionally +// negated — e.g. `flashblocks*`, `!flashblocks*`, or an exact tag like +// `family:systx`) that carves out a real SUB-group of the network's upstreams. +// The key is a hash of the MATCHED UPSTREAM SET, not the selector text, which +// makes the whole thing bounded and cross-pod safe: // // - `flashblocks*`, `fl*`, `flash*`, and a `family:flashblocks` tag that all -// resolve to the same upstreams collapse to ONE partition (dedup); +// resolve to the same upstreams collapse to ONE group (dedup); // - prefix-enumeration / garbage selectors cannot inflate state beyond the // (small, topology-bounded) number of real groupings — an attacker can't // manufacture more distinct sets than the upstream layout allows; -// - every pod derives the same key from the same config, so the shared -// counter is cross-pod consistent. +// - every pod derives the same key from the same config, so the group's +// identity (and lane label) is consistent across pods. // -// A group must match >=2 upstreams (single-node targeting needs no group -// counter — the per-upstream poller is already monotonic) and fewer than ALL -// (matching everything is just the network-wide tip). A per-network cap -// (maxServedTipPartitions) backstops pathological topologies. Anything else -// returns "" (stateless cluster-min over the subset). +// A group must match >=2 upstreams (single-node targeting has no peers to +// group) and fewer than ALL (matching everything is just the network-wide +// tip). A per-network cap (maxServedTipLanes) backstops pathological +// topologies. Anything else returns "": the request gets the stateless subset +// pick with no lane. func (n *Network) partitionKeyFor(ctx context.Context, selector string) (string, []string) { if !isSimpleGroupSelector(selector) || n.upstreamsRegistry == nil { return "", nil @@ -767,7 +703,7 @@ func (n *Network) partitionKeyFor(ctx context.Context, selector string) (string, // use-upstream selector to a stable group key (for keying per-group state) and a // human-readable lane name, or ("","") when the selector doesn't carve out a real // sub-group. partitionKeyFor provides the dedup/bounds/cross-pod-stable key; this adds -// the lane. Used by served-tip latest-block tracking (servedTipPartitionFor) AND by +// the lane. Used by served-tip latest-block tracking (servedTipLaneFor) AND by // the integrity module, so both corroborate/track within the SAME group a request was // pinned to (whatever node groups the use-upstream selector carves out) — one grouping, not two. // EvmAllUpstreams exposes the network's upstreams to the architecture layer @@ -810,73 +746,53 @@ func isSimpleGroupSelector(selector string) bool { return true } -// EvmHighestLatestBlockNumber returns the served latest block for this network. -// -// By default it is the corroborated effective latest block across eligible -// non-syncing upstreams: the second-highest, or the only one (see -// common.ServedTipPick.Freshest). When the served tip is enabled -// (EvmServedTipConfig), it is instead the freshest block a strict MAJORITY of -// the eligible upstreams already have — so interpolated requests land on -// upstreams that can serve the advertised block. Neither mode advertises a -// block visible on only the single most-ahead upstream: that is the "block not -// found" churn the majority mode avoids, and the wrong-chain head a lone -// upstream can report. +// EvmHighestLatestBlockNumber returns the served latest block for this network: +// the freshest block at least servedTip.quorum (default 2) of the eligible, +// non-syncing, live, non-benched upstreams already have — so interpolated +// requests land on upstreams that can serve the advertised block, and a lone +// far-ahead (or wrong-chain) upstream never defines it — then guarded against a +// deep regression, floored to the head already delivered to WebSocket +// subscribers, and clamped to every configured guarantee. See servedTip. func (n *Network) EvmHighestLatestBlockNumber(ctx context.Context) int64 { ctx, span := common.StartDetailSpan(ctx, "Network.EvmHighestLatestBlockNumber") defer span.End() - - if !n.servedTipEnabledFor("latest") { - // tipCandidateUpstreams already scopes to the request's selector (if - // any), so the head is within-subset. The delivered-head floor is - // network-wide, so it applies to unscoped requests only. - ref := n.evmHeadReference(ctx, false) - if requestSelector(ctx) != "" { - return ref.Corroborated + return n.servedTip(ctx, span, false) +} + +// EvmRefreshLatestBlock asks the pollers that decide the request's latest head +// for a background refresh: in routing order, only as many of the ballot's +// voters (its best tipVoterRank) as the quorum needs, since a fresher head +// anywhere else cannot move the served value. Two passes over the candidates, +// counting then refreshing, keep a hot path such as eth_blockNumber free of +// allocation. +func (n *Network) EvmRefreshLatestBlock(ctx context.Context) { + best, voters := -1, 0 + for u := range n.EvmRoutableUpstreams(ctx) { + if _, rank, ok := tipVoterRank(u, false, n.metricsTracker); ok { + switch { + case best < 0 || rank < best: + best, voters = rank, 1 + case rank == best: + voters++ + } } - return n.applyDeliveredHeadFloor(ref.Corroborated, ref.Max) } - if sel := requestSelector(ctx); sel != "" { - // Targeted request: the gather is already scoped to the selector's - // subset, so the majority is within-group. A selector that names a - // real configured group additionally gets its own lane (gauge labels - // + watchdog anchor); arbitrary selectors emit no gauges. - if p := n.servedTipPartitionFor(ctx, sel); p != nil { - return n.servedTip(ctx, span, false, "latest", &p.latestAnchor, p.lane) - } - return n.servedTip(ctx, span, false, "latest", nil, servedTipLaneNone) + if voters == 0 { + return } - pick := n.servedTip(ctx, span, false, "latest", &n.servedLatestAnchor, "") - floored := n.applyDeliveredHeadFloor(pick, n.evmHeadReference(ctx, false).Max) - if floored > pick { - // The floor may lift past the majority but never past a guarantee - // (servedTip already clamped pick to it). - if g := n.guaranteedFloor(ctx, false); g > 0 && g < floored { - floored = max(pick, g) - } + var q common.ServedTipQuorum + if n.cfg != nil { + q = n.cfg.Evm.ServedTipQuorum() } - return floored -} - -// servedTipEnabledFor reports whether the majority served tip is enabled for -// the given block tag ("latest"/"finalized") on this (EVM) network. Default is -// the corroborated head for every tag — see EvmServedTipConfig. -func (n *Network) servedTipEnabledFor(tag string) bool { - return n.cfg != nil && n.cfg.Evm != nil && n.cfg.Evm.ServedTipEnabledFor(tag) -} - -// evmHeadReference is the corroborated head and the raw live max over the -// eligible, non-syncing upstreams for one axis — the same ballot the served tip -// votes on. It serves the default (non-served-tip) head accessors; the -// future-block short-circuit uses futureBlockCeiling instead. A cap-only set has -// no live reference, so its caps stand in: they are the only heads anyone in it -// can serve. -func (n *Network) evmHeadReference(ctx context.Context, useFinalized bool) servedTipReference { - pick, ref := evmTipBallot(n.tipCandidateUpstreams(ctx, "*"), useFinalized) - if ref.Max <= 0 && pick.Inputs > 0 { - ref.Corroborated = pick.Freshest - ref.Max = pick.Max + need := q.Of(voters) + for u := range n.EvmRoutableUpstreams(ctx) { + if _, rank, ok := tipVoterRank(u, false, n.metricsTracker); ok && rank == best { + u.(common.EvmUpstream).EvmStatePoller().RefreshLatestBlock() + if need--; need == 0 { + return + } + } } - return ref } // tryShortCircuitFutureBlock returns a truthful null response (ok=true) when @@ -891,10 +807,10 @@ func (n *Network) evmHeadReference(ctx context.Context, useFinalized bool) serve // when any of them has no observed head (see futureBlockCeiling), so it never // nulls out a block at or below a head an upstream has reported. A block an // upstream produced after its last observed head can still get null until the -// poller or a head subscription reports it. It runs when the network sets -// evm.shortCircuitFutureBlocks or enables served-tip for the latest axis (see -// EvmNetworkConfig.ShortCircuitFutureBlocksEnabled), and the synthesized -// response is returned directly from Forward, so it is never written to cache +// poller or a head subscription reports it. It runs only when the network sets +// evm.shortCircuitFutureBlocks (EvmNetworkConfig.ShortCircuitFutureBlocksEnabled), +// and the synthesized response is returned directly from Forward, so it is +// never written to cache // (the block will exist later). The post-forward empty guard remains as // defense-in-depth for the in-flight case where an upstream advances mid-request. func (n *Network) tryShortCircuitFutureBlock(ctx context.Context, req *common.NormalizedRequest, method string) (*common.NormalizedResponse, bool) { @@ -952,66 +868,64 @@ func (n *Network) futureBlockCeiling(ctx context.Context) (head int64, known boo return head, true } -// servedTip computes the majority served tip for one axis over the -// selection-policy-eligible upstreams — the freshest block a strict majority of -// them already has (see common.PickServedTip) — lets the trajectory referee prefer -// a corroborated on-trajectory group when that majority has stalled, guards the -// result against a regression far below the live heads, applies the -// guaranteed-method floor, and exports the gauges. +// servedTip computes the served tip for one axis of the request bound to ctx: +// the freshest block at least the configured quorum of the eligible voters +// already has (see evmTipBallot and common.PickServedTip), guarded against a +// regression far below the live heads, lifted to the head already delivered to +// WebSocket subscribers (network-wide latest only), clamped to every +// configured guarantee, then exported on the request's lane. // -// Both history-aware layers are in-process, bounded and one-directional (the -// referee can only raise the pick; the guard can only hold it, and only -// briefly), and NEITHER can produce a block that no live upstream reports in -// the very same evaluation. Nothing is persisted and nothing is predicted, so -// no inherited or rogue value can pin, freeze or poison the result -// (networks_served_tip_invariants_test.go pins those outcomes against the -// 2026-06 production incident). -func (n *Network) servedTip( - ctx context.Context, - span trace.Span, - useFinalized bool, - axis string, - anchor *servedTipAnchor, - lane string, -) int64 { - pick, ref := n.gatherEvmTipInputsForMethod(ctx, useFinalized, "*") - - // Trajectory referee: when the live heads split and the majority is the - // STALLED group, serve the corroborated group that matches where this - // network's head has been going for the last ten minutes. Advisory and - // upward-only — it can only replace the majority pick with a higher block - // that at least two live upstreams already have. - majority := pick.Tip - decision := n.refereeServedTip(anchor, pick.Sorted, pick.Tip, ref) - pick.Tip = decision.Pick +// The guard is in-process, bounded and one-directional (it can only hold the +// tip, and only briefly), and it cannot produce a block that no live upstream +// reports in the very same evaluation. Nothing is persisted and nothing is +// predicted, so no inherited or rogue value can pin, freeze or poison the +// result (networks_served_tip_invariants_test.go pins those outcomes against +// the frozen-tip regression). +func (n *Network) servedTip(ctx context.Context, span trace.Span, finalized bool) int64 { + axis := "latest" + if finalized { + axis = "finalized" + } + lane := n.servedTipLaneFor(ctx) + var q common.ServedTipQuorum + if n.cfg != nil { + q = n.cfg.Evm.ServedTipQuorum() + } + pick, live := n.evmTipBallot(ctx, finalized, q, nil) + tip := pick.Tip // Regression guard: "latest" cannot drop far below the corroborated live // head in one evaluation. Runs BEFORE the guaranteed-method floor, which is // a deliberate operator-configured clamp the guard must not fight. - pick.Tip = n.guardServedTipRegression(axis, lane, anchor, pick.Tip, ref) + var anchor *servedTipAnchor + if lane != nil { + anchor = lane.anchor(finalized) + tip = n.guardServedTipRegression(axis, lane.label, anchor, pick, live) + } + + // The delivered-head floor is network-wide, so it applies to unscoped + // requests only. + if lane == &n.servedTipAll && !finalized { + tip = n.applyDeliveredHeadFloor(tip, pick.Max) + } // Capability guarantee (#855): the served tip must never exceed what any // configured guarantee SUBSET can serve — a method's supporting upstreams // (guaranteedMethods) or an operator-named group (guaranteedFor) — so a // request on such a method (e.g. trace_*), or one consensus requires that // group to answer, never resolves "latest" to a block only upstreams - // outside the subset have. - if pick.Tip > 0 { - if floor := n.guaranteedFloor(ctx, useFinalized); floor > 0 && floor < pick.Tip { - pick.Tip = floor + // outside the subset have. It runs after the delivered-head floor, so the + // floor may lift the tip but never past a guarantee. + if tip > 0 { + if floor := n.guaranteedFloor(ctx, finalized); floor > 0 && floor < tip { + tip = floor } } - // The referee's outcome is reported only now, against the value the network - // actually serves: an override the guaranteed-method floor pulled back to - // (or below) the majority pick changed nothing, and counting or logging it - // would page someone about an intervention that did not happen. - n.observeServedTipRefereeTransition(axis, lane, anchor, majority, pick.Tip, decision) - if common.IsTracingDetailed { span.SetAttributes( attribute.String("served_tip.axis", axis), - attribute.Int64("served_tip.tip", pick.Tip), + attribute.Int64("served_tip.tip", tip), attribute.Int64("served_tip.freshest", pick.Freshest), attribute.Int("served_tip.inputs", pick.Inputs), ) @@ -1019,144 +933,13 @@ func (n *Network) servedTip( // Watchdog anchor: when did this process last see the served value // change. Telemetry only — nothing feeds back into the pick. - advanceAge := time.Duration(-1) - if anchor != nil { - if pick.Tip > 0 { - anchor.observe(pick.Tip) - } - advanceAge = anchor.age() - } - n.observeServedTipMetrics(axis, lane, pick, advanceAge) - return pick.Tip -} - -// refereeServedTip returns the tip to serve for `median` (the plain majority -// pick), letting the network's long-term head trajectory choose between the -// groups the LIVE heads of this same evaluation form. -// -// Every mechanism before this one is a snapshot of one instant, which is what a -// majority stall defeats: if N upstreams freeze while staying inside the -// lag-exclusion threshold — or a fleet-wide shared-counter freeze stalls the -// lag metric itself — the majority IS the stale group, and the two upstreams -// that genuinely hold the chain's head are outvoted. The trajectory is the one -// piece of evidence that separates "the chain stopped" from "those upstreams -// stopped": where this network's head has actually been going for the last ten -// minutes (evm.TipTrajectory). -// -// It is deliberately the weakest intervention that answers that: -// -// - it never runs without an anchor (arbitrary use-upstream selectors -// materialize no state), without a pick, or without a live head. That last -// condition is stricter than the guard's, which measures against its own -// last corroborated pick when the live heads are gone: the referee's -// evidence is a HEAD TRACK, and a ballot of serving ranges must never enter -// one; -// - it never runs when the config disables it (trajectoryWindow: 0), and it -// is a no-op until every confidence condition in evm.TipTrajectory holds, -// so a cold or stale process behaves byte-identically to the plain -// majority; -// - it can only RAISE the pick, and only to the minimum of a group of ≥2 -// live upstreams — a value they can all serve, and one at most as high as -// the freshest live head. It can neither hold the tip back nor invent one; -// - and that group must have EARNED it: half a minute of continuously -// matching the trajectory while advancing its own head (evm.TipTrajectory's -// dwell and velocity-agreement conditions), so neither a static fork swept -// over by a halting chain's expected head nor a routine 15-second -// divergence can trigger an intervention. -// -// Serving a winning group's MINIMUM also keeps the pick ≤ -// common.ServedTipPick.Freshest (the 2nd-highest head): a group has at least two -// members, so at least two heads sit at or above what it serves. -func (n *Network) refereeServedTip(anchor *servedTipAnchor, sorted []common.ServedTipInput, median int64, ref servedTipReference) evm.TipTrajectoryDecision { - stood := evm.TipTrajectoryDecision{Pick: median} - if anchor == nil || median <= 0 || ref.Max <= 0 { - // A live head is also what makes `sorted` safe to sample and cluster: - // gatherEvmTipInputsForMethod only returns bound-capped observations - // when there is no live head at all, and a serving range must never - // enter a head trajectory. - return stood - } - window := n.servedTipTrajectoryWindow() - if window <= 0 { - return stood - } - - return anchor.trajectory.Observe(servedTipClock(), sorted, median, evm.TipTrajectoryParams{ - Window: window, - // maxRegressionBlocks: -1 turns the GUARD off; for the referee it only - // removes the floor under the tolerance, which the window's own residual - // spread then sets on its own. - ToleranceFloor: max(0, n.servedTipMaxRegressionBlocks()), - }) -} - -// observeServedTipRefereeTransition counts and logs the referee's outcome ONCE -// PER STATE CHANGE, against `final` — the value the network actually serves -// after the guaranteed-method floor. -// -// Transitions, not evaluations: EvmHighestLatestBlockNumber runs several times -// per request, so a per-evaluation counter scales with RPS and reads as a storm -// of interventions when there is exactly one. Counting entries is what makes -// "any non-zero rate is alert-worthy" true. -func (n *Network) observeServedTipRefereeTransition(axis string, lane string, anchor *servedTipAnchor, majority int64, final int64, d evm.TipTrajectoryDecision) { - if anchor == nil { - return - } - state := servedTipRefereeIdle - switch { - case d.Overrode && final > majority: - state = servedTipRefereeOverriding - case d.Declined: - state = servedTipRefereeDeclining - } - - // Load first: "still idle" is the answer on essentially every evaluation - // this process ever makes. - if anchor.refereeState.Load() == state { - return - } - previous := anchor.refereeState.Swap(state) - if previous == state { - return - } - switch state { - case servedTipRefereeOverriding: - telemetry.MetricNetworkServedTipTrajectoryTotal. - WithLabelValues(n.projectId, n.Label(), servedTipLaneLabel(lane), axis, "override"). - Inc() - n.logger.Error(). - Str("axis", axis). - Int64("pick", final). - Int64("majorityPick", majority). - Int64("expectedBlock", d.Expected). - Int64("tolerance", d.Tolerance). - Float64("blocksPerSecond", d.VelocityPerSec). - Msg("majority of the eligible upstreams is off the network's long-term head trajectory; serving the corroborated group that is on it") - case servedTipRefereeDeclining: - telemetry.MetricNetworkServedTipTrajectoryTotal. - WithLabelValues(n.projectId, n.Label(), servedTipLaneLabel(lane), axis, "fallback"). - Inc() - default: - // Idle is reachable from Overriding (an actual override just ended) - // and from Declining (the referee never intervened; it fell back to - // the majority pick). Only the former ever overrode anything. - if previous == servedTipRefereeOverriding { - n.logger.Warn(). - Str("axis", axis). - Int64("pick", final). - Msg("served tip is back on the majority pick; the trajectory referee is no longer overriding") + if lane != nil { + if tip > 0 { + anchor.observe(tip) } + n.observeServedTipMetrics(axis, lane.label, tip, pick.Freshest, anchor.age()) } -} - -// servedTipTrajectoryWindow is how much recorded head history the trajectory -// referee needs before it may participate. Unset uses -// DefaultServedTipTrajectoryWindow; an explicit 0 disables the referee. -func (n *Network) servedTipTrajectoryWindow() time.Duration { - if n.cfg != nil && n.cfg.Evm != nil && n.cfg.Evm.ServedTip != nil && n.cfg.Evm.ServedTip.TrajectoryWindow != nil { - return n.cfg.Evm.ServedTip.TrajectoryWindow.Duration() - } - return common.DefaultServedTipTrajectoryWindow + return tip } // guardServedTipRegression returns the tip to serve for `pick`, substituting the @@ -1166,29 +949,45 @@ func (n *Network) servedTipTrajectoryWindow() time.Duration { // This is a physics check, independent of which voters are honest: a chain's // head does not fall a thousand blocks between two evaluations while an upstream // still reports the old one, so a pick that far below corroborated evidence is a -// poisoned ballot — an availability bound voting as a head (the 2026-08 celo -// incident), a bogus vendor head, a rolled-back shared counter. It is -// deliberately independent of the bound-capped filter in -// gatherEvmTipInputsForMethod: that removes ONE known source of low votes, this -// bounds the damage of any other. +// poisoned ballot — an availability bound voting as a head (two capped +// legacy upstreams), a bogus vendor head, a rolled-back shared counter. It is +// deliberately independent of the static-cap rule in tipVoterRank: that +// removes ONE known source of low votes, this bounds the damage of any other. // // TWO SOURCES OF EVIDENCE, ONE CEILING. A regression is measured against the -// higher of (a) `ref.Corroborated`, the second-highest LIVE head of this -// evaluation, and (b) the last pick this process saw live-corroborated. Neither -// alone is enough: (a) alone is defeated by the guard's own headline shape — -// one live upstream left, two stuck ones outvoting it, where the second-highest -// live head IS the poison — and (b) alone would be memory with no present-tense -// check. A HELD value is then capped at `ref.Max`, the highest live head, so the -// guard can never advertise a block no live upstream has. +// higher of (a) the corroborated live head, `ballot.Freshest` of a live ballot +// (the second-highest live head, or the only one), and (b) the last pick this +// process saw live-corroborated. Neither alone is enough: (a) alone is defeated +// by the guard's own headline shape — one live upstream left, two stuck ones +// outvoting it, where the second-highest live head IS the poison — and (b) +// alone would be memory with no present-tense check. (a) is second-highest +// because, measured against the raw max, one far-future/wrong-chain upstream +// made every honest pick look like a regression, so the guard never took its +// healthy branch, never recorded a value, and sat permanently disarmed with no +// metric and no log. That tolerates ONE Byzantine head, not two: two upstreams +// reporting the same far-future block are indistinguishable from two that are +// simply ahead, so they can still raise the reference and leave the guard +// silently unarmed. Nothing in a single evaluation separates those cases; the +// pick's own quorum and rememberServedTipPick's refusal to commit an +// uncorroborated leap limit the damage. +// +// A HELD value is capped at `ballot.Max` of a live ballot, the highest live +// head, so the guard can never advertise a block no live upstream has. It is +// deliberately the raw max: a ceiling a single rogue can only RAISE cannot make +// the guard serve anything the rogue chose, while a second-highest ceiling +// would collapse the hold onto the poisoned pick in the very shape the guard +// exists for (one live witness left, two stuck upstreams outvoting it). // // It never blocks serving and never persists anything: -// - no pick, or no anchor for this lane (arbitrary use-upstream selectors -// deliberately materialize no state), or maxRegressionBlocks: -1 → returns -// the pick untouched; +// - no pick, or maxRegressionBlocks: -1 → returns the pick untouched (a +// request with no lane — an arbitrary use-upstream selector — never reaches +// the guard: it deliberately materializes no state); // - within tolerance → a LIVE-corroborated pick becomes the new remembered // value (a capped one never does); // - below tolerance, for up to servedTipRegressionTTL since that value was -// recorded → serves it instead, never above the highest live head; +// recorded → serves it instead, never above the highest live head, unless +// every live voter is within tolerance of the pick (then only the memory +// disagrees, and the pick is served); // - past the TTL → fails open, serves the pick as computed, and DISARMS: a // value that has outlived the window is no longer evidence, so the guard // re-arms from whatever the live heads corroborate next instead of measuring @@ -1196,9 +995,9 @@ func (n *Network) servedTipTrajectoryWindow() time.Duration { // // NO LIVE HEAD IS NOT A REASON TO STAND DOWN. When every live upstream leaves the // eligible set while bound-capped ones remain, the ballot legitimately falls back -// to the caps — and a guard that requires a live reference would be exactly as -// absent as it was during the celo incident, on the one ballot shape that caused -// it. With no live head the guard measures against its own last corroborated pick +// to the caps — and a guard that requires a live reference would be absent on +// exactly the ballot shape that pulls the tip down to a cap. With no live head +// the guard measures against its own last corroborated pick // and holds it UNCLAMPED (there is no live head to clamp to) for the same bounded // TTL, then fails open to the capped pick. Networks that are genuinely all-capped // never armed the guard in the first place — nothing live ever corroborated a @@ -1215,20 +1014,24 @@ func (n *Network) servedTipTrajectoryWindow() time.Duration { // While it holds, the served-tip lag gauge (Freshest - Tip) would read negative — // the served value is above the corroborated freshest view — so it is clamped at // 0; erpc_network_served_tip_regression_total is what says a hold is in progress. -func (n *Network) guardServedTipRegression(axis string, lane string, anchor *servedTipAnchor, pick int64, ref servedTipReference) int64 { - if anchor == nil || pick <= 0 { +func (n *Network) guardServedTipRegression(axis string, label string, anchor *servedTipAnchor, ballot common.ServedTipPick, live bool) int64 { + pick := ballot.Tip + if pick <= 0 { return pick } + var corroborated, liveMax int64 + if live { + corroborated, liveMax = ballot.Freshest, ballot.Max + } tolerance := n.servedTipMaxRegressionBlocks() if tolerance < 0 { - // maxRegressionBlocks: -1 — the operator turned the guard off, the - // symmetric opposite of trajectoryWindow: 0. + // maxRegressionBlocks: -1 — the operator turned the guard off. return pick } lastGood, lastGoodAt := anchor.lastGoodValue.Load(), anchor.lastGoodAt.Load() armed := lastGood > 0 && lastGoodAt != nil - reference := ref.Corroborated + reference := corroborated if armed && lastGood > reference { reference = lastGood } @@ -1247,7 +1050,7 @@ func (n *Network) guardServedTipRegression(axis string, lane string, anchor *ser } if pick >= minAcceptable { now := servedTipClock() - if n.rememberServedTipPick(anchor, pick, lastGood, lastGoodAt, ref, tolerance, now) { + if n.rememberServedTipPick(anchor, pick, lastGood, lastGoodAt, corroborated, tolerance, now) { // The timestamp is a TTL clock, not an audit log: refreshing it at // most once a second keeps a 60s window exact to within its own // granularity, and keeps the ONLY heap allocation on this path — the @@ -1267,9 +1070,11 @@ func (n *Network) guardServedTipRegression(axis string, lane string, anchor *ser } held := servedTipClock().Sub(*lastGoodAt) if held >= servedTipRegressionTTL { - if n.noteServedTipGuardTransition(anchor, servedTipGuardFailedOpen, axis, pick, reference, held) { + // Failing open is an event only for a guard that was holding; a memory + // that merely aged out while the guard stood aside is dropped quietly. + if anchor.guardState.Load() == servedTipGuardHolding && n.noteServedTipGuardTransition(anchor, servedTipGuardFailedOpen, axis, pick, reference, held) { telemetry.MetricNetworkServedTipRegressionTotal. - WithLabelValues(n.projectId, n.Label(), servedTipLaneLabel(lane), axis, "failed_open"). + WithLabelValues(n.projectId, n.Label(), label, axis, "failed_open"). Inc() } anchor.lastGoodValue.Store(0) @@ -1277,13 +1082,21 @@ func (n *Network) guardServedTipRegression(axis string, lane string, anchor *ser return pick } + // Only the live voters can show that a pick is poisoned. When every one of + // them sits within tolerance of it, the regression is against memory + // alone, left by heads that no longer vote (benched for a dead fork, or + // excluded), and a hold would serve a head fewer than quorum voters have. + if live && liveMax-pick <= tolerance { + return pick + } + // Serve the remembered pick — but never above the highest live head, so the // guard cannot advertise a block no live upstream has (the invariant the old // monotonic clamp broke). With no live head there is nothing to clamp to and // the remembered value stands as recorded. served := lastGood - if ref.Max > 0 && ref.Max < served { - served = ref.Max + if liveMax > 0 && liveMax < served { + served = liveMax } if served <= pick { // A HOLD MAY ONLY EVER RAISE. A memory at or below the current pick has @@ -1295,7 +1108,7 @@ func (n *Network) guardServedTipRegression(axis string, lane string, anchor *ser } if n.noteServedTipGuardTransition(anchor, servedTipGuardHolding, axis, pick, reference, held) { telemetry.MetricNetworkServedTipRegressionTotal. - WithLabelValues(n.projectId, n.Label(), servedTipLaneLabel(lane), axis, "held"). + WithLabelValues(n.projectId, n.Label(), label, axis, "held"). Inc() } return served @@ -1345,8 +1158,8 @@ func (n *Network) guardServedTipRegression(axis string, lane string, anchor *ser // after which the next healthy evaluation adopts the new height — and the // detection reference already takes the HIGHER of the memory and the live // corroborated head, so a memory left behind is ignored rather than harmful. -func (n *Network) rememberServedTipPick(anchor *servedTipAnchor, pick int64, lastGood int64, lastGoodAt *time.Time, ref servedTipReference, tolerance int64, now time.Time) bool { - if ref.Max <= 0 || pick > ref.Corroborated { +func (n *Network) rememberServedTipPick(anchor *servedTipAnchor, pick int64, lastGood int64, lastGoodAt *time.Time, corroborated int64, tolerance int64, now time.Time) bool { + if corroborated <= 0 || pick > corroborated { return false } // The last value this process trusted: the memory while it is still fresh, @@ -1441,8 +1254,7 @@ func (n *Network) maxRetryableBlockDistance() int64 { // pick may fall before the regression guard treats it as poisoned. Defaults to // the rollback tolerance the state poller and health tracker already use, so // every layer agrees on what a genuine deep correction looks like. A configured -// -1 is returned as-is: it disables the guard, the symmetric opposite of -// trajectoryWindow: 0. +// -1 is returned as-is: it disables the guard. func (n *Network) servedTipMaxRegressionBlocks() int64 { if n.cfg != nil && n.cfg.Evm != nil && n.cfg.Evm.ServedTip != nil && n.cfg.Evm.ServedTip.MaxRegressionBlocks != 0 { return n.cfg.Evm.ServedTip.MaxRegressionBlocks @@ -1451,12 +1263,12 @@ func (n *Network) servedTipMaxRegressionBlocks() int64 { } // SvmHighestLatestSlot returns the majority-vote latest slot across SVM -// upstreams. Uses common.PickServedTip (floor(N/2)-th highest) so a single +// upstreams. Uses common.PickServedTip with a strict-majority quorum so a single // fast/rogue upstream cannot inflate the served tip. func (n *Network) SvmHighestLatestSlot(ctx context.Context) int64 { _, span := common.StartDetailSpan(ctx, "Network.SvmHighestLatestSlot") defer span.End() - pick := common.PickServedTip(n.gatherSvmTipInputs(ctx, false)) + pick := common.PickServedTip(n.gatherSvmTipInputs(ctx, false), common.ServedTipQuorumMajority) span.SetAttributes(attribute.Int64("highest_latest_slot", pick.Tip)) return pick.Tip } @@ -1468,7 +1280,7 @@ func (n *Network) SvmHighestLatestSlot(ctx context.Context) int64 { func (n *Network) SvmHighestFinalizedSlot(ctx context.Context) int64 { _, span := common.StartDetailSpan(ctx, "Network.SvmHighestFinalizedSlot") defer span.End() - pick := common.PickServedTip(n.gatherSvmTipInputs(ctx, true)) + pick := common.PickServedTip(n.gatherSvmTipInputs(ctx, true), common.ServedTipQuorumMajority) span.SetAttributes(attribute.Int64("highest_finalized_slot", pick.Tip)) return pick.Tip } @@ -1482,7 +1294,7 @@ func (n *Network) SvmHighestFinalizedSlot(ctx context.Context) int64 { // ~100 slots the third upstream is holding, converting a served request into a // client-visible -32014. // -// Same rationale as SvmHighestIndexedSlot and EVM's evmHeadReference.Max: never +// Same rationale as SvmHighestIndexedSlot and EVM's futureBlockCeiling: never // reject a block the most-ahead upstream actually has. Zero means UNKNOWN (no // poller has reported a root yet), and every caller must treat it as "no bound" // rather than as slot 0. @@ -1545,9 +1357,9 @@ func (n *Network) SvmHighestIndexedSlot(ctx context.Context) int64 { // gatherSvmTipInputs collects slot values from SVM state pollers for // majority-tip computation via common.PickServedTip. -func (n *Network) gatherSvmTipInputs(ctx context.Context, useFinalized bool) []common.ServedTipInput { +func (n *Network) gatherSvmTipInputs(ctx context.Context, useFinalized bool) []int64 { upstreams := n.upstreamsRegistry.GetNetworkUpstreams(ctx, n.networkId) - out := make([]common.ServedTipInput, 0, len(upstreams)) + out := make([]int64, 0, len(upstreams)) for _, u := range upstreams { sp := u.SvmStatePoller() if sp == nil || sp.IsObjectNull() { @@ -1562,33 +1374,21 @@ func (n *Network) gatherSvmTipInputs(ctx context.Context, useFinalized bool) []c if slot <= 0 { continue } - out = append(out, common.ServedTipInput{UpstreamID: u.Id(), BlockNumber: slot}) + out = append(out, slot) } return out } // EvmHighestFinalizedBlockNumber is the finalized-axis sibling of -// EvmHighestLatestBlockNumber. Same majority semantics, applied to each -// upstream's EvmEffectiveFinalizedBlock. +// EvmHighestLatestBlockNumber: the same ballot, quorum, guard and guarantees, +// applied to each upstream's EvmEffectiveFinalizedBlock. func (n *Network) EvmHighestFinalizedBlockNumber(ctx context.Context) int64 { ctx, span := common.StartDetailSpan(ctx, "Network.EvmHighestFinalizedBlockNumber", trace.WithAttributes( attribute.String("network.id", n.networkId), )) defer span.End() - if !n.servedTipEnabledFor("finalized") { - return n.evmHeadReference(ctx, true).Corroborated - } - if sel := requestSelector(ctx); sel != "" { - // See EvmHighestLatestBlockNumber: a configured-tag selector gets its - // own lane (telemetry labels + watchdog anchor); any other selector - // computes the same stateless majority without emitting gauges. - if p := n.servedTipPartitionFor(ctx, sel); p != nil { - return n.servedTip(ctx, span, true, "finalized", &p.finalizedAnchor, p.lane) - } - return n.servedTip(ctx, span, true, "finalized", nil, servedTipLaneNone) - } - return n.servedTip(ctx, span, true, "finalized", &n.servedFinalizedAnchor, "") + return n.servedTip(ctx, span, true) } // applyDeliveredHeadFloor lifts the computed network-wide "latest" head to the @@ -1598,7 +1398,7 @@ func (n *Network) EvmHighestFinalizedBlockNumber(ctx context.Context) int64 { // turn the served tip into a persisted clamp. // // A delivered head always came from some live upstream, so the floor is -// honoured only within maxRetryableBlockDistance of the freshest live head +// honoured only within maxRetryableBlockDistance of the ballot's highest head // (liveMax), or when no head is known at all. A stale shared value is ignored // and ages out as the chain advances. func (n *Network) applyDeliveredHeadFloor(computed, liveMax int64) int64 { @@ -1623,12 +1423,11 @@ func (n *Network) applyDeliveredHeadFloor(computed, liveMax int64) int64 { // guaranteedFloor returns the lowest majority served tip across every // configured guarantee SUBSET, or 0 when none is configured or none constrains -// the tip. Two kinds of subset, one rule: +// the tip. Two kinds of subset, one rule (guaranteedSubsetTip): // -// - GuaranteedMethods — the selection-policy-eligible set for that method -// (which, via autoIgnoreUnsupportedMethods, excludes upstreams that don't -// support it). "latest" never resolves to a block only non-supporting -// upstreams have. +// - GuaranteedMethods — the eligible upstreams that handle the method +// (autoIgnoreUnsupportedMethods drops the rest). "latest" never resolves +// to a block only non-supporting upstreams have. // - GuaranteedFor — the eligible upstreams matching an id/tag selector. The // advertised tip stays servable by an operator-named GROUP, which is what a // mixed internal/external pool needs when consensus requires the internals @@ -1637,145 +1436,70 @@ func (n *Network) applyDeliveredHeadFloor(computed, liveMax int64) int64 { // Each subset is clamped to its OWN MAJORITY, never its minimum, so a single // frozen member is outvoted inside its group and cannot pin the network. An // empty subset constrains nothing. -// -// Every subset's ballot is built by evmTipBallot, i.e. by exactly the rule the -// network-wide ballot uses — including the static-cap filter, applied to THAT -// method's supporting set. Without it the floor was a second door onto the -// incident value: with `guaranteedMethods` configured, the floor ballot took raw -// effective heads and ran AFTER the guard, so the celo pair's 21,615,999 came -// back as a "floor" and was served, on a branch whose whole premise is that a -// serving range never defines the head. -// -// The per-set mixed-pool rule is what keeps that from breaking the floor's -// purpose: capped upstreams drop out of a method's ballot only when some LIVE -// upstream also supports that method. If only capped upstreams support it, they -// set the floor — which is the entire point of the floor, since they are the -// only upstreams that can serve the method at all. -// -// STATIC CONFIG DECIDES THAT, NOT TRANSIENT ELIGIBILITY. A cap-only ballot has -// two very different causes, and only one of them is a floor: -// -// - PERMANENT: no live upstream is configured to serve the method at all -// (the archive-only trace_* shape). The cap is genuinely the freshest block -// anyone can serve, so it floors the tip. -// - TRANSIENT: live upstreams DO serve the method, they are just absent from -// this instant's eligible set — lag-excluded, cordoned, mid-restart. Their -// absence is exactly the condition the regression guard is holding through, -// and flooring to the cap would hand the incident value back the moment the -// eligible set collapsed. That composition (the guard correctly holding -// 74.5M while the floor dragged the served value to 21,615,999) was live at -// the previous head of this branch. -// -// So a cap-only ballot floors only when the REGISTERED set — the operator's -// configuration, which eligibility churn cannot change — has no live upstream -// serving that method. Otherwise the method sets no floor for this evaluation -// and the guard's hold (and its bounded fail-open) governs, exactly as it does -// when the whole ballot goes cap-only. func (n *Network) guaranteedFloor(ctx context.Context, useFinalized bool) int64 { if n.cfg == nil || n.cfg.Evm == nil || n.cfg.Evm.ServedTip == nil { return 0 } st := n.cfg.Evm.ServedTip - methods := st.GuaranteedMethods - if len(methods) == 0 && len(st.GuaranteedFor) == 0 { - return 0 - } - eligible := n.tipCandidateUpstreams(ctx, "*") - supporting := make([]common.Upstream, 0, len(eligible)) var floor int64 - for _, sel := range st.GuaranteedFor { - // Group guarantee: the advertised tip must be servable by this group's - // own MAJORITY. Same ballot, same min-across-subsets fold, same - // transient-absence rule as the method guarantee below — only the - // membership predicate differs (selector match instead of method - // support). - supporting = supporting[:0] - for _, cu := range eligible { - if m, _ := common.UpstreamMatchesSelector(sel, cu); !m { - continue - } - supporting = append(supporting, cu) - } - pick, ref := evmTipBallot(supporting, useFinalized) - if pick.Inputs == 0 { - continue - } - if ref.Max <= 0 && n.liveUpstreamInGroup(ctx, sel, useFinalized) { - continue - } - if pick.Tip > 0 && (floor == 0 || pick.Tip < floor) { - floor = pick.Tip + fold := func(tip int64) { + if tip > 0 && (floor == 0 || tip < floor) { + floor = tip } } - for _, m := range methods { - supporting = supporting[:0] - for _, cu := range eligible { - // Supporting set = eligible upstreams that handle this method. - if handle, _ := cu.ShouldHandleMethod(m); !handle { - continue - } - supporting = append(supporting, cu) - } - pick, ref := evmTipBallot(supporting, useFinalized) - if pick.Inputs == 0 { - // No supporting upstream for this method → no constraint (fall through - // rather than pinning the tip to 0). - continue - } - if ref.Max <= 0 && n.liveUpstreamServesMethod(ctx, m, useFinalized) { - // Cap-only ballot, but the configuration says live upstreams serve - // this method — they are merely absent right now. See the comment - // above: that floor is a transient lie, and serving it is the celo - // value. - continue - } - if pick.Tip > 0 && (floor == 0 || pick.Tip < floor) { - floor = pick.Tip - } + for _, sel := range st.GuaranteedFor { + fold(n.guaranteedSubsetTip(ctx, useFinalized, func(u common.Upstream) bool { + m, _ := common.UpstreamMatchesSelector(sel, u) + return m + })) + } + for _, m := range st.GuaranteedMethods { + fold(n.guaranteedSubsetTip(ctx, useFinalized, func(u common.Upstream) bool { + handle, _ := u.ShouldHandleMethod(m) + return handle + })) } return floor } -// liveUpstreamServesMethod reports whether any REGISTERED upstream that is not a -// static serving-range cap is configured to serve `method` — the static-config -// question guaranteedFloor asks before letting a cap-only ballot set a -// floor. It reads the registered set, not the eligible one, precisely because -// eligibility is the thing that churns. +// guaranteedSubsetTip is the majority tip of the guarantee subset `member` +// selects, or 0 when that subset sets no floor this evaluation. // -// An upstream with no head yet counts as live: "we cannot tell" must fall on the -// side of skipping the floor, since serving a stale cap is the failure this -// whole change exists to prevent. Only reached when a method's ballot came back -// cap-only, i.e. approximately never on the hot path. -func (n *Network) liveUpstreamServesMethod(ctx context.Context, method string, useFinalized bool) bool { - if n.upstreamsRegistry == nil { - return false - } - for _, u := range n.upstreamsRegistry.GetNetworkUpstreams(ctx, n.networkId) { - if handle, _ := u.ShouldHandleMethod(method); !handle { - continue - } - if eu, ok := common.Upstream(u).(common.EvmUpstream); ok { - if _, capped := evmTipObservation(eu, useFinalized); capped { - continue - } - } - return true +// The ballot is evmTipBallot's, so a subset's static caps drop out whenever a +// live member is in it; without that the floor was a second door onto the +// capped value that the network-wide ballot keeps out. A CAP-ONLY ballot has +// two causes, and only one is a floor: +// +// - PERMANENT: no live upstream is configured into the subset at all (the +// archive-only trace_* shape). The cap is the freshest block anyone in it +// can serve, so it floors the tip. +// - TRANSIENT: live members exist but are absent from this instant's eligible +// set — lag-excluded, cordoned, mid-restart. That is exactly the shape the +// regression guard holds through, and flooring to the cap would hand the +// capped value back the moment the eligible set collapsed. +// +// So a cap-only ballot floors only when the REGISTERED set, which eligibility +// churn cannot change, has no live member (anyRegisteredLiveHead). +func (n *Network) guaranteedSubsetTip(ctx context.Context, useFinalized bool, member func(common.Upstream) bool) int64 { + pick, live := n.evmTipBallot(ctx, useFinalized, common.ServedTipQuorumMajority, member) + if pick.Inputs == 0 || (!live && n.anyRegisteredLiveHead(ctx, member, useFinalized)) { + return 0 } - return false + return pick.Tip } -// liveUpstreamInGroup is liveUpstreamServesMethod for a group guarantee: does -// any REGISTERED upstream matching `selector` observe a real head rather than a -// static serving-range cap. Same reason as the method case — a cap-only group -// ballot must floor the tip only when the operator's CONFIG says nobody in that -// group is live, never when live members are merely absent from this instant's -// eligible set. -func (n *Network) liveUpstreamInGroup(ctx context.Context, selector string, useFinalized bool) bool { +// anyRegisteredLiveHead reports whether any REGISTERED upstream matching member +// is not a static serving-range cap. An upstream with no head yet counts as +// live: "we cannot tell" must fall on the side of skipping the floor, since +// serving a stale cap is the failure the floor rule exists to prevent. Only +// reached for a cap-only subset ballot, i.e. approximately never on the hot +// path. +func (n *Network) anyRegisteredLiveHead(ctx context.Context, member func(common.Upstream) bool, useFinalized bool) bool { if n.upstreamsRegistry == nil { return false } for _, u := range n.upstreamsRegistry.GetNetworkUpstreams(ctx, n.networkId) { - if m, _ := common.UpstreamMatchesSelector(selector, u); !m { + if !member(u) { continue } if eu, ok := common.Upstream(u).(common.EvmUpstream); ok { @@ -1788,28 +1512,22 @@ func (n *Network) liveUpstreamInGroup(ctx context.Context, selector string, useF return false } -// observeServedTipMetrics exports the served-tip gauges. axis +// observeServedTipMetrics exports one lane's served-tip gauges. axis // ("latest"|"finalized") is a label rather than part of the metric name to // keep the gauge set small. lane="all" is the network-wide pick; a named lane -// is a use-upstream group's own pick; the stateless lane-none sentinel emits -// nothing (a subset value must never overwrite a gauge). -func (n *Network) observeServedTipMetrics(axis string, lane string, pick common.ServedTipPick, advanceAge time.Duration) { - if lane == servedTipLaneNone { - return - } - laneLabel := servedTipLaneLabel(lane) - if pick.Tip > 0 { +// is a use-upstream group's own pick. +func (n *Network) observeServedTipMetrics(axis string, laneLabel string, tip, freshest int64, advanceAge time.Duration) { + if tip > 0 { telemetry.MetricNetworkServedTipBlockNumber. WithLabelValues(n.projectId, n.Label(), laneLabel, axis). - Set(float64(pick.Tip)) - // Deliberate lag: how far the majority tip sits behind the corroborated + Set(float64(tip)) + // Deliberate lag: how far the served tip sits behind the corroborated // freshest upstream view in the same set. Clamped at 0 because the tip - // can legitimately sit ABOVE that view — while the regression guard - // holds its last corroborated pick, and on the evaluation a referee - // override raises the tip to a fresh group — and a lag gauge that reads - // negative is a broken dashboard, not a signal. Both of those states - // have their own counters (…_regression_total / …_trajectory_total). - lag := pick.Freshest - pick.Tip + // can legitimately sit ABOVE that view while the regression guard + // holds its last corroborated pick, and a lag gauge that reads + // negative is a broken dashboard, not a signal; the hold has its own + // counter (…_regression_total). + lag := freshest - tip if lag < 0 { lag = 0 } @@ -1947,7 +1665,7 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* // Bind the request to ctx so downstream served-tip resolution can decide // `latest`/`finalized` AMONG the use-upstream-selected subset (see - // tipCandidateUpstreams / requestSelector). The failsafe executor re-sets + // EvmRoutableUpstreams / requestSelector). The failsafe executor re-sets // the same value below for its own readers; setting it here makes it // available to the earlier pre-forward / short-circuit paths too. ctx = context.WithValue(ctx, common.RequestContextKey, req) @@ -2115,6 +1833,14 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* upsList = tierUpstreamsByGroup(upsList) } + // SVM history reads go deepest known floor first within each tier, so the + // first answer is the most complete one the pool holds. + if n.Architecture() == common.ArchitectureSvm && svm.IsHistoryMethod(method) { + upsList = svm.OrderByHistoryDepth(upsList, func(a, b common.Upstream) bool { + return isFallbackTier(a) == isFallbackTier(b) + }) + } + // Architecture-specific pruning of the upstream list. Currently only SVM // uses this hook; both filters are gated on ArchitectureSvm so EVM networks // never enter this block. @@ -2553,7 +2279,7 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* selector = d.UseUpstream } var fallbacks []common.Upstream - for _, fb := range n.upstreamsRegistry.GetFallbackEscapeUpstreams(execSpanCtx, n.networkId, method) { + for _, fb := range n.upstreamsRegistry.GetFallbackEscapeUpstreams(execSpanCtx, n.networkId, method, effectiveReq.Finality(execSpanCtx)) { if selector != "" { if matches, _ := common.UpstreamMatchesSelector(selector, fb); !matches { continue @@ -2769,12 +2495,28 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* // If response is not empty, but at least one upstream responded empty we track in a metric. // Derived from ErrorsByUpstream entries with ErrEndpointMissingData code. + winner := resp.Upstream() req.ErrorsByUpstream.Range(func(key, value any) bool { + // The winner's own earlier empty answer was a retry that later succeeded. + if key == winner { + return true + } upstreamErr, ok := value.(error) if !ok || !common.HasErrorCode(upstreamErr, common.ErrCodeEndpointMissingData) { return true } + // SVM history sweep uses MissingData to continue a sweep after an + // incomplete 200 — not evidence this upstream disagreed with the winner. + if common.IsHistorySweepContinuation(upstreamErr) { + return true + } upstream := key.(*upstream.Upstream) + // An upstream whose head was below the winning block when it answered + // was behind, or the data did not exist yet (e.g. a tx not yet mined). + // An unknown head (0) proves neither, so it still counts. + if head := req.HeadAtError(upstream); respBlockNumber > 0 && head > 0 && head < respBlockNumber { + return true + } finality := req.Finality(ctx) telemetry.CounterHandle(telemetry.MetricUpstreamWrongEmptyResponseTotal, n.projectId, @@ -2984,6 +2726,20 @@ func (n *Network) GetFinality(ctx context.Context, req *common.NormalizedRequest } func (n *Network) doForward(execSpanCtx context.Context, u common.Upstream, req *common.NormalizedRequest, skipCacheRead, isHedgeAttempt bool) (*common.NormalizedResponse, error) { + // evm.pinLatestTag: this upstream is sent the network head in place of + // "latest"; the request keeps "latest" for the cache, the response and every + // other upstream. With the head unknown it is never sent "latest" at all. + if params, height, pinned := evm.PinLatestTag(execSpanCtx, n, u, req); pinned { + if height <= 0 { + var latest, finalized int64 + if eu, ok := u.(common.EvmUpstream); ok { + latest, finalized = n.upstreamHeads(eu) + } + return nil, common.NewErrUpstreamBlockUnavailable(u.Id(), 0, latest, finalized) + } + execSpanCtx = common.WithWireParams(execSpanCtx, req, params) + } + h := n.architectureHandler if h != nil { if handled, resp, err := h.HandleUpstreamPreForward(execSpanCtx, n, u, req, skipCacheRead); handled { @@ -3220,11 +2976,15 @@ func (n *Network) checkEvmUpstreamBlockAvailability(ctx context.Context, u commo return nil, false } - // Prefer the cached block number from normalization. Fall back to extracting - // from the request (defensive: handles paths that bypass json_rpc.go's - // normalization, and methods whose params haven't been pre-cached yet). + // An upstream that pins "latest" (evm.pinLatestTag) is sent the network head, + // so that head is the block it must hold. Otherwise prefer the cached block + // number from normalization, falling back to extracting from the request + // (defensive: handles paths that bypass json_rpc.go's normalization, and + // methods whose params haven't been pre-cached yet). var bn int64 - if v := req.EvmBlockNumber(); v != nil { + if _, height, pinned := evm.PinLatestTag(ctx, n, u, req); pinned { + bn = height + } else if v := req.EvmBlockNumber(); v != nil { if n64, ok := v.(int64); ok { bn = n64 } diff --git a/erpc/networks_future_block_test.go b/erpc/networks_future_block_test.go index db5367630..044312340 100644 --- a/erpc/networks_future_block_test.go +++ b/erpc/networks_future_block_test.go @@ -50,6 +50,7 @@ func TestForward_FutureBlock_ShortCircuitsToNull(t *testing.T) { {id: "fb2", chainID: 123, latestBlock: 99}, {id: "fb3", chainID: 123, latestBlock: 98}, }) + network.cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) mockGetBlockByNumberNonNull("fb1", "fb2", "fb3") // max observed head = 100; block 105 (0x69) is beyond every upstream. @@ -115,8 +116,8 @@ func TestForward_FutureBlock_MixedStaticCapDispatchesWithinAvailableRange(t *tes assert.Contains(t, jrr.GetResultString(), "0x270f") } -// The short-circuit is gated on served-tip being enabled for the latest axis -// (so the head is trustworthy). With served-tip disabled it must stay off. +// The short-circuit is opt-in via evm.shortCircuitFutureBlocks; the default +// config (here: nil servedTip, no flag) must dispatch. func TestForward_FutureBlock_ServedTipDisabled_Dispatches(t *testing.T) { util.ResetGock() defer util.ResetGock() diff --git a/erpc/networks_paths_test.go b/erpc/networks_paths_test.go new file mode 100644 index 000000000..825f64a71 --- /dev/null +++ b/erpc/networks_paths_test.go @@ -0,0 +1,1005 @@ +package erpc + +import ( + "context" + "errors" + "fmt" + "math" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/architecture/svm" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/upstream" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// pathsBareNetwork builds a Network with no registry, engine or tracker, the +// shape cache-only paths and partially initialized networks run with. +func pathsBareNetwork(cfg *common.NetworkConfig) *Network { + if cfg == nil { + cfg = &common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + } + } + return &Network{ + networkId: cfg.NetworkId(), + projectId: "paths", + cfg: cfg, + logger: &log.Logger, + inFlightRequests: &sync.Map{}, + } +} + +// pathsFakeUpstream is a FakeUpstream with a fixed polled head. +func pathsFakeUpstream(id string, latest, finalized int64, opts ...func(*common.FakeUpstream)) common.Upstream { + all := append([]func(*common.FakeUpstream){common.WithEvmStatePoller(common.NewFakeEvmStatePoller(latest, finalized))}, opts...) + return common.NewFakeUpstream(id, all...) +} + +func pathsIds(ups []common.Upstream) []string { + out := make([]string, len(ups)) + for i, u := range ups { + out[i] = u.Id() + } + return out +} + +func pathsResult(t *testing.T, resp *common.NormalizedResponse) string { + t.Helper() + require.NotNil(t, resp) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + return jrr.GetResultString() +} + +// pathsCache is a CacheDAL whose Get answer is switched atomically: a nil +// mode is a miss, otherwise a fixed result or error. It is installed once, +// before any Forward, because Forward's async cache writes read the field. +type pathsCache struct { + mode atomic.Pointer[pathsCacheMode] +} + +type pathsCacheMode struct { + result string + err error +} + +func (c *pathsCache) Get(_ context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + m := c.mode.Load() + if m == nil { + return nil, nil + } + if m.err != nil { + return nil, m.err + } + return common.NewNormalizedResponse(). + WithRequest(req). + WithJsonRpcResponse(common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(m.result), nil)), nil +} + +func (*pathsCache) Set(context.Context, *common.NormalizedRequest, *common.NormalizedResponse) error { + return nil +} + +func (*pathsCache) IsObjectNull() bool { return false } + +func TestNetworksPaths_IsSimpleGroupSelector(t *testing.T) { + cases := []struct { + sel string + want bool + }{ + {"flashblocks*", true}, + {"!flashblocks*", true}, + {" primary-* ", true}, + {"", false}, + {" ", false}, + {strings.Repeat("a", 129), false}, + {"a|b", false}, + {"a b", false}, + {"(a)", false}, + {"a&b", false}, + {"a,b", false}, + {"a!b", false}, + {"!!a", false}, + } + for _, tc := range cases { + assert.Equal(t, tc.want, isSimpleGroupSelector(tc.sel), "selector %q", tc.sel) + } +} + +func TestNetworksPaths_EligibleLane(t *testing.T) { + unb := func(id string) upstreamBlockBounds { + return upstreamBlockBounds{id: id, min: math.MinInt64, max: math.MaxInt64} + } + cases := []struct { + name string + bounds []upstreamBlockBounds + bn int64 + want []string + }{ + {"no bounds anywhere", []upstreamBlockBounds{unb("a"), unb("b")}, 100, nil}, + {"all eligible", []upstreamBlockBounds{unb("a"), {id: "b", min: 10, max: 200}}, 100, nil}, + {"below lower excluded", []upstreamBlockBounds{unb("a"), {id: "b", min: 150, max: math.MaxInt64}}, 100, []string{"a"}}, + {"above upper excluded", []upstreamBlockBounds{{id: "a", min: math.MinInt64, max: 50}, unb("b")}, 100, []string{"b"}}, + {"none eligible", []upstreamBlockBounds{{id: "a", min: 500, max: 600}}, 100, []string{}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, eligibleLane(tc.bounds, tc.bn)) + }) + } +} + +func TestNetworksPaths_UpstreamOrderingHelpers(t *testing.T) { + t.Run("TierUpstreamsByGroupMovesFallbacksLast", func(t *testing.T) { + fb := common.WithTags(common.TagTierFallback) + ups := []common.Upstream{ + pathsFakeUpstream("fb1", 1, 1, fb), + pathsFakeUpstream("p1", 1, 1), + pathsFakeUpstream("fb2", 1, 1, fb), + pathsFakeUpstream("p2", 1, 1), + } + assert.Equal(t, []string{"p1", "p2", "fb1", "fb2"}, pathsIds(tierUpstreamsByGroup(ups))) + + plain := []common.Upstream{pathsFakeUpstream("p1", 1, 1), pathsFakeUpstream("p2", 1, 1)} + assert.Equal(t, []string{"p1", "p2"}, pathsIds(tierUpstreamsByGroup(plain))) + }) + + t.Run("PartitionByLatestBlockMovesBehindHeadsLast", func(t *testing.T) { + ups := []common.Upstream{ + pathsFakeUpstream("behind", 50, 1), + pathsFakeUpstream("ahead", 200, 1), + pathsFakeUpstream("unknown", 0, 0), + pathsFakeUpstream("at", 150, 1), + } + assert.Equal(t, []string{"ahead", "unknown", "at", "behind"}, pathsIds(partitionUpstreamsByLatestBlock(ups, 100))) + assert.Equal(t, pathsIds(ups), pathsIds(partitionUpstreamsByLatestBlock(ups, 0)), "no block keeps order") + assert.Equal(t, []string{"behind"}, pathsIds(partitionUpstreamsByLatestBlock(ups[:1], 100))) + noPoller := []common.Upstream{common.NewFakeUpstream("np1"), common.NewFakeUpstream("np2")} + assert.Equal(t, []string{"np1", "np2"}, pathsIds(partitionUpstreamsByLatestBlock(noPoller, 100))) + }) + + t.Run("PreferTipLeaderForNearTipGetBlock", func(t *testing.T) { + ups := []common.Upstream{ + pathsFakeUpstream("a", 100, 1), + pathsFakeUpstream("leader", 105, 1), + pathsFakeUpstream("c", 101, 1), + } + cases := []struct { + name string + ups []common.Upstream + method string + bn int64 + want []string + }{ + {"head block", ups, "eth_getBlockByNumber", 105, []string{"leader", "a", "c"}}, + {"next block", ups, "eth_getBlockByNumber", 106, []string{"leader", "a", "c"}}, + {"beyond next", ups, "eth_getBlockByNumber", 107, []string{"a", "leader", "c"}}, + {"below tip", ups, "eth_getBlockByNumber", 104, []string{"a", "leader", "c"}}, + {"other method", ups, "eth_call", 105, []string{"a", "leader", "c"}}, + {"single upstream", ups[:1], "eth_getBlockByNumber", 100, []string{"a"}}, + {"leader already first", []common.Upstream{ups[1], ups[0]}, "eth_getBlockByNumber", 105, []string{"leader", "a"}}, + {"tie at tip", []common.Upstream{pathsFakeUpstream("x", 105, 1), pathsFakeUpstream("y", 105, 1)}, "eth_getBlockByNumber", 105, []string{"x", "y"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, pathsIds(preferTipLeaderForNearTipGetBlock(tc.ups, tc.method, tc.bn))) + }) + } + }) +} + +func TestNetworksPaths_RequestBlockNumber(t *testing.T) { + ctx := context.Background() + assert.Equal(t, int64(0), requestBlockNumber(ctx, nil)) + + cached := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`)) + cached.SetEvmBlockNumber(int64(42)) + assert.Equal(t, int64(42), requestBlockNumber(ctx, cached)) + + byParam := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x10",false]}`)) + assert.Equal(t, int64(16), requestBlockNumber(ctx, byParam)) +} + +func TestNetworksPaths_TipVoterRank(t *testing.T) { + exactUpper := func(b int64) func(*common.FakeUpstream) { + return func(u *common.FakeUpstream) { + u.Config().Evm = &common.EvmUpstreamConfig{ + BlockAvailability: &common.EvmBlockAvailabilityConfig{ + Upper: &common.EvmAvailabilityBoundConfig{ExactBlock: &b}, + }, + } + } + } + cases := []struct { + name string + u common.Upstream + finalized bool + head int64 + rank int + ok bool + }{ + {"live head votes at rank 0", pathsFakeUpstream("a", 100, 90), false, 100, 0, true}, + {"finalized axis", pathsFakeUpstream("a", 100, 90), true, 90, 0, true}, + {"no poller never votes", common.NewFakeUpstream("np"), false, 0, 0, false}, + {"unknown head never votes", pathsFakeUpstream("z", 0, 0), false, 0, 0, false}, + {"exact cap below head ranks 2", pathsFakeUpstream("cap", 100, 90, exactUpper(50)), false, 100, 2, true}, + {"cap above finalized head does not cap", pathsFakeUpstream("cap", 100, 90, exactUpper(95)), true, 90, 0, true}, + {"cap without exact block", pathsFakeUpstream("cap", 100, 90, func(u *common.FakeUpstream) { + u.Config().Evm = &common.EvmUpstreamConfig{BlockAvailability: &common.EvmBlockAvailabilityConfig{ + Upper: &common.EvmAvailabilityBoundConfig{LatestBlockMinus: i64(0)}, + }} + }), false, 100, 0, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + head, rank, ok := tipVoterRank(tc.u, tc.finalized, nil) + assert.Equal(t, tc.ok, ok) + assert.Equal(t, tc.head, head) + assert.Equal(t, tc.rank, rank) + }) + } +} + +func TestNetworksPaths_BareNetworkAccessors(t *testing.T) { + ctx := context.Background() + + t.Run("LabelPrefersAliasThenId", func(t *testing.T) { + var nilNet *Network + assert.Equal(t, "", nilNet.Label()) + n := pathsBareNetwork(nil) + assert.Equal(t, "evm:123", n.Label()) + n.networkLabel = "mainnet" + assert.Equal(t, "mainnet", n.Label()) + assert.Equal(t, "paths", n.ProjectId()) + assert.Same(t, n.cfg, n.Config()) + assert.Same(t, n.logger, n.Logger()) + }) + + t.Run("ArchitectureDefaultsToEvmWhenEvmConfigured", func(t *testing.T) { + n := pathsBareNetwork(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{ChainId: 1}}) + assert.Equal(t, common.ArchitectureEvm, n.Architecture()) + assert.Equal(t, common.ArchitectureEvm, n.cfg.Architecture, "the default is persisted on the config") + empty := pathsBareNetwork(&common.NetworkConfig{}) + assert.Equal(t, common.NetworkArchitecture(""), empty.Architecture()) + }) + + t.Run("PolicyAccessorsWithoutEngine", func(t *testing.T) { + n := pathsBareNetwork(nil) + assert.Nil(t, n.PolicyScores("")) + assert.Nil(t, n.RecentPolicyDecisions("", 10)) + assert.True(t, n.PolicyLastSwitchAt("").IsZero()) + assert.Nil(t, n.PolicyOrderedUpstreams("")) + n.SetPolicyEnginePaused(true) + n.SetPolicyStepLogEnabled(true) + assert.Nil(t, n.policyEngine) + require.NoError(t, n.Bootstrap(ctx), "bootstrap without an engine is a no-op") + }) + + t.Run("RegistryLessReadsFailOpen", func(t *testing.T) { + n := pathsBareNetwork(nil) + assert.Nil(t, n.MetricsTracker()) + assert.Equal(t, time.Duration(0), n.EvmBlockTime()) + assert.Nil(t, n.AllUpstreams()) + assert.Empty(t, n.EvmAllUpstreams(ctx)) + assert.Nil(t, n.GetMethodMetrics("")) + head, known := n.futureBlockCeiling(ctx) + assert.False(t, known) + assert.Equal(t, int64(0), head) + var visited int + n.walkTipCandidates(ctx, func(common.Upstream) bool { visited++; return true }) + assert.Equal(t, 0, visited) + key, ids := n.partitionKeyFor(ctx, "primary-*") + assert.Equal(t, "", key) + assert.Nil(t, ids) + key, lane := n.EvmUpstreamGroupForSelector(ctx, "a|b") + assert.Equal(t, "", key) + assert.Equal(t, "", lane) + }) + + t.Run("ServedTipLaneForWithoutSelectorIsNetworkWide", func(t *testing.T) { + n := pathsBareNetwork(nil) + assert.Same(t, &n.servedTipAll, n.servedTipLaneFor(ctx)) + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}`)) + req.SetDirectives(&common.RequestDirectives{UseUpstream: "primary-*"}) + rctx := context.WithValue(ctx, common.RequestContextKey, req) + assert.Equal(t, "primary-*", requestSelector(rctx)) + assert.Nil(t, n.servedTipLaneFor(rctx), "a selector that forms no group gets no lane") + }) + + t.Run("NoteObservedLatestBlockIsMonotonic", func(t *testing.T) { + var nilNet *Network + nilNet.NoteObservedLatestBlock(ctx, 10) + + n := pathsBareNetwork(nil) + var nilCtx context.Context + n.NoteObservedLatestBlock(nilCtx, 10) + assert.Equal(t, int64(10), n.deliveredLatestBlock.Load()) + n.NoteObservedLatestBlock(ctx, 5) + assert.Equal(t, int64(10), n.deliveredLatestBlock.Load(), "a lower head never moves it back") + n.NoteObservedLatestBlock(ctx, 0) + n.NoteObservedLatestBlock(ctx, -3) + assert.Equal(t, int64(10), n.deliveredLatestBlock.Load()) + n.appCtx = ctx + n.NoteObservedLatestBlock(nilCtx, 25) + assert.Equal(t, int64(25), n.deliveredLatestBlock.Load()) + }) + + t.Run("RetryDistanceAndRegressionDefaults", func(t *testing.T) { + n := pathsBareNetwork(nil) + assert.Equal(t, defaultMaxRetryableBlockDistance, n.maxRetryableBlockDistance()) + n.cfg.Evm.MaxRetryableBlockDistance = i64(7) + assert.Equal(t, int64(7), n.maxRetryableBlockDistance()) + + assert.Equal(t, int64(common.DefaultToleratedBlockHeadRollback), n.servedTipMaxRegressionBlocks()) + n.cfg.Evm.ServedTip = &common.EvmServedTipConfig{MaxRegressionBlocks: -1} + assert.Equal(t, int64(-1), n.servedTipMaxRegressionBlocks()) + n.cfg.Evm.ServedTip.MaxRegressionBlocks = 0 + assert.Equal(t, int64(common.DefaultToleratedBlockHeadRollback), n.servedTipMaxRegressionBlocks()) + }) + + t.Run("SvmEnforceBlockAvailabilityDefaultsOn", func(t *testing.T) { + n := pathsBareNetwork(nil) + assert.True(t, n.SvmEnforceBlockAvailability()) + n.cfg.Svm = &common.SvmNetworkConfig{} + assert.True(t, n.SvmEnforceBlockAvailability()) + n.cfg.Svm.EnforceBlockAvailability = util.BoolPtr(false) + assert.False(t, n.SvmEnforceBlockAvailability()) + n.cfg.Svm.EnforceBlockAvailability = util.BoolPtr(true) + assert.True(t, n.SvmEnforceBlockAvailability()) + }) +} + +func TestNetworksPaths_ShouldHandleMethod(t *testing.T) { + n := pathsBareNetwork(&common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + Methods: &common.MethodsConfig{Definitions: map[string]*common.CacheMethodConfig{ + "eth_newFilter": {Stateful: true}, + }}, + }) + two := []common.Upstream{common.NewFakeUpstream("a"), common.NewFakeUpstream("b")} + newReq := func(method, useUpstream string) *common.NormalizedRequest { + r := common.NewNormalizedRequest([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":%q,"params":[]}`, method))) + if useUpstream != "" { + r.SetDirectives(&common.RequestDirectives{UseUpstream: useUpstream}) + } + return r + } + cases := []struct { + name string + method string + use string + ups []common.Upstream + wantErr bool + msg string + }{ + {"stateful across two upstreams", "eth_newFilter", "", two, true, "stateful method requires a single targeted upstream"}, + {"stateful pinned to one", "eth_newFilter", "a", two, false, ""}, + {"stateful selector matching both", "eth_newFilter", "*", two, true, "stateful method"}, + {"stateful with one upstream", "eth_newFilter", "", two[:1], false, ""}, + {"eth_accounts rejected", "eth_accounts", "", two[:1], true, "eth_accounts and eth_sign are not supported"}, + {"eth_sign any casing rejected", "ETH_SIGN", "", two[:1], true, "eth_accounts and eth_sign are not supported"}, + {"plain method passes", "eth_call", "", two, false, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := n.shouldHandleMethod(newReq(tc.method, tc.use), tc.method, tc.ups) + if !tc.wantErr { + assert.NoError(t, err) + return + } + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeNotImplemented), "got %v", err) + assert.Contains(t, err.Error(), tc.msg) + }) + } +} + +func TestNetworksPaths_GetFailsafeExecutor(t *testing.T) { + mk := func(cfg *common.FailsafeConfig) *networkExecutor { + ex, err := NewNetworkExecutor(cfg, &log.Logger, nil, nil) + require.NoError(t, err) + return ex + } + logs := mk(&common.FailsafeConfig{MatchMethod: "eth_getLogs"}) + finalizedOnly := mk(&common.FailsafeConfig{MatchMethod: "eth_call", MatchFinality: []common.DataFinalityState{common.DataFinalityStateFinalized}}) + svmFinalized := mk(&common.FailsafeConfig{MatchMethod: "eth_call", MatchCommitment: []string{"finalized"}}) + internal := mk(&common.FailsafeConfig{MatchRequestKind: "internal"}) + catchAll := mk(nil) + + n := pathsBareNetwork(nil) + n.failsafeExecutors = []*networkExecutor{logs, finalizedOnly, svmFinalized, internal, catchAll} + ctx := context.Background() + newReq := func(method string, isInternal bool) *common.NormalizedRequest { + r := common.NewNormalizedRequest([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":%q,"params":[]}`, method))) + if isInternal { + r.SetDirectives(&common.RequestDirectives{IsInternal: true}) + } + return r + } + + assert.Same(t, logs, n.getFailsafeExecutor(ctx, newReq("eth_getLogs", false))) + assert.Same(t, internal, n.getFailsafeExecutor(ctx, newReq("eth_call", true)), "finality and commitment filters skip; internal kind matches") + assert.Same(t, catchAll, n.getFailsafeExecutor(ctx, newReq("eth_call", false))) + + n.failsafeExecutors = []*networkExecutor{logs, internal} + assert.Nil(t, n.getFailsafeExecutor(ctx, newReq("eth_call", false)), "no matching executor") +} + +func TestNetworksPaths_PrepareRequest(t *testing.T) { + ctx := context.Background() + bad := func() *common.NormalizedRequest { return common.NewNormalizedRequest([]byte(`not-json`)) } + good := func() *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSlot","params":[]}`)) + } + + evmNet := pathsBareNetwork(nil) + err := evmNet.prepareRequest(ctx, bad()) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcExceptionInternal), "got %v", err) + assert.Contains(t, err.Error(), "failed to unmarshal json-rpc request") + + svmNet := pathsBareNetwork(&common.NetworkConfig{Architecture: common.ArchitectureSvm, Svm: &common.SvmNetworkConfig{Cluster: "mainnet-beta"}}) + err = svmNet.prepareRequest(ctx, bad()) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to unmarshal json-rpc request") + require.NoError(t, svmNet.prepareRequest(ctx, good())) + + other := pathsBareNetwork(&common.NetworkConfig{Architecture: "cosmos"}) + err = other.prepareRequest(ctx, good()) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcExceptionInternal), "got %v", err) + assert.Contains(t, err.Error(), "unsupported architecture: cosmos") +} + +func TestNetworksPaths_GetFinalityWithoutRegistry(t *testing.T) { + ctx := context.Background() + n := pathsBareNetwork(&common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + Methods: &common.MethodsConfig{Definitions: map[string]*common.CacheMethodConfig{ + "custom_finalized": {Finalized: true}, + "custom_realtime": {Realtime: true}, + }}, + }) + newReq := func(body string) *common.NormalizedRequest { return common.NewNormalizedRequest([]byte(body)) } + + assert.Equal(t, common.DataFinalityStateUnknown, n.GetFinality(ctx, nil, nil)) + assert.Equal(t, common.DataFinalityStateFinalized, + n.GetFinality(ctx, newReq(`{"jsonrpc":"2.0","id":1,"method":"custom_finalized","params":[]}`), nil)) + assert.Equal(t, common.DataFinalityStateRealtime, + n.GetFinality(ctx, newReq(`{"jsonrpc":"2.0","id":1,"method":"custom_realtime","params":[]}`), nil)) + assert.Equal(t, common.DataFinalityStateRealtime, + n.GetFinality(ctx, newReq(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}`), nil), + "a tag reference is realtime") + + numbered := `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x10",false]}` + assert.Equal(t, common.DataFinalityStateUnknown, n.GetFinality(ctx, newReq(numbered), nil), + "no upstream and no registry leaves it unknown") + + viaLast := newReq(numbered) + viaLast.SetLastUpstream(pathsFakeUpstream("last", 100, 90)) + assert.Equal(t, common.DataFinalityStateUnfinalized, n.GetFinality(ctx, viaLast, nil), + "the last upstream's finality answer is used") + + viaResp := newReq(numbered) + resp := common.NewNormalizedResponse().WithRequest(viaResp) + resp.SetUpstream(pathsFakeUpstream("resp", 100, 90)) + assert.Equal(t, common.DataFinalityStateUnfinalized, n.GetFinality(ctx, viaResp, resp)) + + svmNet := pathsBareNetwork(&common.NetworkConfig{Architecture: common.ArchitectureSvm, Svm: &common.SvmNetworkConfig{Cluster: "mainnet-beta"}}) + assert.Equal(t, common.DataFinalityStateRealtime, + svmNet.GetFinality(ctx, newReq(`{"jsonrpc":"2.0","id":1,"method":"getBalance","params":["pk"]}`), nil)) + assert.Equal(t, common.DataFinalityStateFinalized, + svmNet.GetFinality(ctx, newReq(`{"jsonrpc":"2.0","id":1,"method":"getBlockTime","params":[5]}`), nil)) +} + +func TestNetworksPaths_EnrichStatePoller(t *testing.T) { + ctx := context.Background() + n := pathsBareNetwork(nil) + + run := func(t *testing.T, body string, blockRef interface{}, result string) common.EvmStatePoller { + t.Helper() + req := common.NewNormalizedRequest([]byte(body)) + if blockRef != nil { + req.SetEvmBlockRef(blockRef) + } + u := pathsFakeUpstream("u", 100, 90) + resp := common.NewNormalizedResponse().WithRequest(req). + WithJsonRpcResponse(common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(result), nil)) + resp.SetUpstream(u) + method, _ := req.Method() + n.enrichStatePoller(ctx, method, req, resp) + return u.(common.EvmUpstream).EvmStatePoller() + } + + t.Run("LatestTagFromParamsSuggestsLatest", func(t *testing.T) { + sp := run(t, `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}`, nil, `{"number":"0xc8"}`) + assert.Equal(t, int64(200), sp.LatestBlock()) + assert.Equal(t, int64(90), sp.FinalizedBlock()) + }) + t.Run("FinalizedTagFromBlockRefSuggestsFinalized", func(t *testing.T) { + sp := run(t, `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x5f",false]}`, "finalized", `{"number":"0x5f"}`) + assert.Equal(t, int64(95), sp.FinalizedBlock()) + assert.Equal(t, int64(100), sp.LatestBlock()) + }) + t.Run("NumberedBlockIsIgnored", func(t *testing.T) { + sp := run(t, `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x500",false]}`, nil, `{"number":"0x500"}`) + assert.Equal(t, int64(100), sp.LatestBlock()) + }) + t.Run("MissingNumberIsIgnored", func(t *testing.T) { + sp := run(t, `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}`, nil, `{"hash":"0x1"}`) + assert.Equal(t, int64(100), sp.LatestBlock()) + }) + t.Run("BadNumberIsIgnored", func(t *testing.T) { + sp := run(t, `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}`, nil, `{"number":"zz"}`) + assert.Equal(t, int64(100), sp.LatestBlock()) + }) + t.Run("BlockNumberSuggestsLatest", func(t *testing.T) { + sp := run(t, `{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}`, nil, `"0x12c"`) + assert.Equal(t, int64(300), sp.LatestBlock()) + }) +} + +func TestNetworksPaths_StampClientResponseID(t *testing.T) { + ctx := context.Background() + n := pathsBareNetwork(nil) + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":"abc","method":"eth_chainId","params":[]}`)) + require.NoError(t, n.normalizeResponse(ctx, req, nil)) + + resp := common.NewNormalizedResponse().WithRequest(req). + WithJsonRpcResponse(common.MustNewJsonRpcResponseFromBytes([]byte(`999`), []byte(`"0x1"`), nil)) + require.NoError(t, n.normalizeResponse(ctx, req, resp)) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + assert.Equal(t, "abc", jrr.ID(), "the client's id replaces the upstream's") +} + +func TestNetworksPaths_AcquireRateLimitPermit(t *testing.T) { + ctx := context.Background() + budgetId := fmt.Sprintf("paths-net-budget-%d", time.Now().UnixNano()) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ + Budgets: []*common.RateLimitBudgetConfig{{ + Id: budgetId, + Rules: []*common.RateLimitRuleConfig{{ + Method: "eth_getBalance", + MaxCount: 1, + Period: common.RateLimitPeriodMinute, + }}, + }}, + }, &log.Logger) + require.NoError(t, err) + n := pathsBareNetwork(nil) + n.rateLimitersRegistry = rlr + newReq := func(method string) *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":%q,"params":[]}`, method))) + } + + require.NoError(t, n.acquireRateLimitPermit(ctx, newReq("eth_getBalance")), "no budget configured") + + n.cfg.RateLimitBudget = "paths-missing-budget" + err = n.acquireRateLimitPermit(ctx, newReq("eth_getBalance")) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, "ErrRateLimitBudgetNotFound"), "got %v", err) + + n.cfg.RateLimitBudget = budgetId + require.NoError(t, n.acquireRateLimitPermit(ctx, newReq("eth_chainId")), "method without rules passes") +} + +func TestNetworksPaths_MultiplexKey(t *testing.T) { + ctx := context.Background() + lower := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getBalance","params":["dRiFTyPePEHfLqZBUyHFLGVW3d5Fk1AmmZoRbCxDMDy"]}`)) + upper := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":2,"method":"getBalance","params":["DRiFTyPePEHfLqZBUyHFLGVW3d5Fk1AmmZoRbCxDMDy"]}`)) + + svmNet := pathsBareNetwork(&common.NetworkConfig{Architecture: common.ArchitectureSvm, Svm: &common.SvmNetworkConfig{Cluster: "mainnet-beta"}}) + lk, err := svmNet.multiplexKey(ctx, lower) + require.NoError(t, err) + uk, err := svmNet.multiplexKey(ctx, upper) + require.NoError(t, err) + assert.NotEqual(t, lk, uk, "case-distinct base58 keys never share a multiplexer on SVM") + + evmNet := pathsBareNetwork(nil) + ek1, err := evmNet.multiplexKey(ctx, lower) + require.NoError(t, err) + ek2, err := evmNet.multiplexKey(ctx, upper) + require.NoError(t, err) + assert.Equal(t, ek1, ek2, "EVM keeps the case-insensitive cache hash") +} + +func TestNetworksPaths_Multiplexing(t *testing.T) { + lg := log.Logger + body := func(id int, addr string) []byte { + return []byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"eth_getBalance","params":[%q,"latest"]}`, id, addr)) + } + + t.Run("DisabledAndWriteMethodsNeverMultiplex", func(t *testing.T) { + ctx := context.Background() + n := pathsBareNetwork(nil) + n.cfg.Multiplexing = util.BoolPtr(false) + mlx, resp, err := n.handleMultiplexing(ctx, &lg, common.NewNormalizedRequest(body(1, "0xa")), time.Now()) + assert.Nil(t, mlx) + assert.Nil(t, resp) + assert.NoError(t, err) + + n.cfg.Multiplexing = nil + write := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_newFilter","params":[{}]}`)) + mlx, resp, err = n.handleMultiplexing(ctx, &lg, write, time.Now()) + assert.Nil(t, mlx) + assert.Nil(t, resp) + assert.NoError(t, err) + + mlx, resp, err = n.handleMultiplexing(ctx, &lg, common.NewNormalizedRequest([]byte(`not-json`)), time.Now()) + assert.Nil(t, mlx, "no hash means no multiplexer") + assert.Nil(t, resp) + assert.NoError(t, err) + }) + + t.Run("FollowerCopiesLeaderResultThenCleanupReleases", func(t *testing.T) { + ctx := context.Background() + n := pathsBareNetwork(nil) + leaderReq := common.NewNormalizedRequest(body(1, "0xb")) + mlx, resp, err := n.handleMultiplexing(ctx, &lg, leaderReq, time.Now()) + require.NoError(t, err) + require.Nil(t, resp) + require.NotNil(t, mlx, "first request leads") + + leaderResp := common.NewNormalizedResponse().WithRequest(leaderReq). + WithJsonRpcResponse(common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`"0xbeef"`), nil)) + mlx.Close(ctx, leaderResp, nil) + + followerReq := common.NewNormalizedRequest(body(2, "0xb")) + fmlx, fresp, ferr := n.handleMultiplexing(ctx, &lg, followerReq, time.Now()) + require.NoError(t, ferr) + assert.Nil(t, fmlx, "a follower does not lead") + assert.Equal(t, `"0xbeef"`, pathsResult(t, fresp)) + + n.cleanupMultiplexer(mlx) + _, stillThere := n.inFlightRequests.Load(mlx.hash) + assert.False(t, stillThere, "cleanup removes the in-flight entry") + mlx.mu.RLock() + assert.Nil(t, mlx.resp, "cleanup releases the leader response") + mlx.mu.RUnlock() + + again, _, err := n.handleMultiplexing(ctx, &lg, common.NewNormalizedRequest(body(3, "0xb")), time.Now()) + require.NoError(t, err) + require.NotNil(t, again, "after cleanup the next identical request leads") + again.Close(ctx, nil, errors.New("done")) + n.cleanupMultiplexer(again) + }) + + t.Run("FollowerSharesLeaderError", func(t *testing.T) { + ctx := context.Background() + n := pathsBareNetwork(nil) + mlx, _, err := n.handleMultiplexing(ctx, &lg, common.NewNormalizedRequest(body(1, "0xc")), time.Now()) + require.NoError(t, err) + require.NotNil(t, mlx) + leaderErr := common.NewErrNoUpstreamsFound("paths", "evm:123") + mlx.Close(ctx, nil, leaderErr) + + _, resp, ferr := n.handleMultiplexing(ctx, &lg, common.NewNormalizedRequest(body(2, "0xc")), time.Now()) + assert.Nil(t, resp) + require.Error(t, ferr) + assert.True(t, common.HasErrorCode(ferr, "ErrNoUpstreamsFound"), "got %v", ferr) + n.cleanupMultiplexer(mlx) + }) + + t.Run("FollowerDeadlineAndCancel", func(t *testing.T) { + n := pathsBareNetwork(nil) + mlx, _, err := n.handleMultiplexing(context.Background(), &lg, common.NewNormalizedRequest(body(1, "0xd")), time.Now()) + require.NoError(t, err) + require.NotNil(t, mlx) + + tctx, tcancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer tcancel() + _, resp, ferr := n.handleMultiplexing(tctx, &lg, common.NewNormalizedRequest(body(2, "0xd")), time.Now()) + assert.Nil(t, resp) + assert.True(t, common.HasErrorCode(ferr, common.ErrCodeNetworkRequestTimeout), "got %v", ferr) + + cctx, ccancel := context.WithCancel(context.Background()) + ccancel() + _, resp, ferr = n.handleMultiplexing(cctx, &lg, common.NewNormalizedRequest(body(3, "0xd")), time.Now()) + assert.Nil(t, resp) + assert.ErrorIs(t, ferr, context.Canceled) + + mlx.Close(context.Background(), nil, errors.New("done")) + n.cleanupMultiplexer(mlx) + }) +} + +func TestNetworksPaths_SvmDeclaredSlotAvailability(t *testing.T) { + ctx := context.Background() + n := pathsBareNetwork(&common.NetworkConfig{Architecture: common.ArchitectureSvm, Svm: &common.SvmNetworkConfig{Cluster: "mainnet-beta"}}) + withBounds := func(lower, upper *int64) common.Upstream { + return common.NewFakeUpstream("svm-u", func(u *common.FakeUpstream) { + ba := &common.SvmBlockAvailabilityConfig{} + if lower != nil { + ba.Lower = &common.SvmAvailabilityBoundConfig{ExactSlot: lower} + } + if upper != nil { + ba.Upper = &common.SvmAvailabilityBoundConfig{ExactSlot: upper} + } + u.Config().Svm = &common.SvmUpstreamConfig{Cluster: "mainnet-beta", BlockAvailability: ba} + }) + } + getBlock := func(slot int64) *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":"getBlock","params":[%d]}`, slot))) + } + + err, retryable := n.checkUpstreamBlockAvailability(ctx, withBounds(i64(100), nil), getBlock(50), "getBlock") + require.Error(t, err, "below the declared lower bound skips") + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointMissingData), "got %v", err) + assert.False(t, retryable) + assert.False(t, svm.IsDeclaredUpperBoundSkip(err)) + + err, retryable = n.checkUpstreamBlockAvailability(ctx, withBounds(nil, i64(10)), getBlock(50), "getBlock") + require.Error(t, err, "above a fixed upper bound skips") + assert.True(t, svm.IsDeclaredUpperBoundSkip(err)) + assert.False(t, retryable, "a fixed exactSlot upper never becomes retryable") + + err, retryable = n.checkUpstreamBlockAvailability(ctx, withBounds(i64(10), i64(100)), getBlock(50), "getBlock") + assert.NoError(t, err, "inside the window serves") + assert.False(t, retryable) + + n.cfg.Svm.EnforceBlockAvailability = util.BoolPtr(false) + err, _ = n.checkUpstreamBlockAvailability(ctx, withBounds(i64(100), nil), getBlock(50), "getBlock") + assert.NoError(t, err, "the network switch disables declared bounds") + + other := pathsBareNetwork(&common.NetworkConfig{Architecture: "cosmos"}) + err, retryable = other.checkUpstreamBlockAvailability(ctx, withBounds(i64(100), nil), getBlock(50), "getBlock") + assert.NoError(t, err) + assert.False(t, retryable) +} + +func TestNetworksPaths_BoundaryLaneAndFutureBlockGuards(t *testing.T) { + ctx := context.Background() + newReq := func(body string) *common.NormalizedRequest { return common.NewNormalizedRequest([]byte(body)) } + + svmNet := pathsBareNetwork(&common.NetworkConfig{Architecture: common.ArchitectureSvm, Svm: &common.SvmNetworkConfig{Cluster: "mainnet-beta"}}) + assert.Nil(t, svmNet.eligibleUpstreamIDsForBoundary(ctx, "getBlock", newReq(`{"jsonrpc":"2.0","id":1,"method":"getBlock","params":[5]}`))) + + n := pathsBareNetwork(nil) + assert.Nil(t, n.eligibleUpstreamIDsForBoundary(ctx, "eth_getLogs", newReq(`{"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":[{}]}`)), + "range methods own their availability") + + future := newReq(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0xffffff",false]}`) + resp, ok := n.tryShortCircuitFutureBlock(ctx, future, "eth_getBlockByNumber") + assert.False(t, ok, "disabled by default") + assert.Nil(t, resp) + + n.cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) + _, ok = n.tryShortCircuitFutureBlock(ctx, future, "eth_call") + assert.False(t, ok, "only eth_getBlockByNumber") + _, ok = n.tryShortCircuitFutureBlock(ctx, newReq(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["latest",false]}`), "eth_getBlockByNumber") + assert.False(t, ok, "tags never short-circuit") + _, ok = n.tryShortCircuitFutureBlock(ctx, future, "eth_getBlockByNumber") + assert.False(t, ok, "unknown heads fail open") +} + +func TestNetworksPaths_ForwardOnLiveNetwork(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + gock.New("http://rpc1.localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_getBalance") + }). + Reply(200). + JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":"0xupstream"}`)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + network := setupTestNetworkSimple(t, ctx, nil, &common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123, ShortCircuitFutureBlocks: util.BoolPtr(true)}, + }) + + cache := &pathsCache{} + network.cacheDal = cache + + head, known := network.futureBlockCeiling(ctx) + require.True(t, known, "the single upstream's head is known") + require.Greater(t, head, int64(0)) + + forward := func(body string) (*common.NormalizedResponse, error) { + req := common.NewNormalizedRequest([]byte(body)) + req.SetNetwork(network) + return network.Forward(ctx, req) + } + + t.Run("RegistryAccessors", func(t *testing.T) { + all := network.AllUpstreams() + require.Len(t, all, 1) + assert.Equal(t, "test", all[0].Id()) + assert.Equal(t, []string{"test"}, pathsIds(network.EvmAllUpstreams(ctx))) + assert.Empty(t, network.ShadowUpstreams()) + leader := network.EvmLeaderUpstream(ctx) + require.NotNil(t, leader) + assert.Equal(t, "test", leader.Id()) + assert.NotNil(t, network.MetricsTracker()) + network.EvmRefreshLatestBlock(ctx) + assert.Equal(t, head, network.EvmHighestLatestBlockNumber(ctx), "one voter: the served tip is its head") + }) + + t.Run("FinalityHeuristicFromRegistry", func(t *testing.T) { + lowest := network.EvmLowestFinalizedBlockNumber(ctx) + require.Greater(t, lowest, int64(0)) + below := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x1",false]}`)) + assert.Equal(t, common.DataFinalityStateFinalized, network.GetFinality(ctx, below, nil)) + above := common.NewNormalizedRequest([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x%x",false]}`, lowest+1000))) + assert.Equal(t, common.DataFinalityStateUnfinalized, network.GetFinality(ctx, above, nil)) + }) + + t.Run("FutureBlockShortCircuitsToNull", func(t *testing.T) { + resp, err := forward(fmt.Sprintf(`{"jsonrpc":"2.0","id":7,"method":"eth_getBlockByNumber","params":["0x%x",false]}`, head+100000)) + require.NoError(t, err) + assert.Equal(t, "null", pathsResult(t, resp), "a block beyond every head is null without dispatch") + assert.Equal(t, head+100000, resp.EvmBlockNumber()) + + within := common.NewNormalizedRequest([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x%x",false]}`, head))) + _, ok := network.tryShortCircuitFutureBlock(ctx, within, "eth_getBlockByNumber") + assert.False(t, ok, "a block at the head is dispatched") + }) + + t.Run("UnsupportedMethodRejectedBeforeDispatch", func(t *testing.T) { + _, err := forward(`{"jsonrpc":"2.0","id":1,"method":"eth_accounts","params":[]}`) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeNotImplemented), "got %v", err) + }) + + t.Run("MissingRateLimitBudgetFailsForward", func(t *testing.T) { + network.cfg.RateLimitBudget = "paths-no-such-budget" + defer func() { network.cfg.RateLimitBudget = "" }() + _, err := forward(`{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0x1","0x1"]}`) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, "ErrRateLimitBudgetNotFound"), "got %v", err) + }) + + t.Run("CacheHitServedWithoutUpstream", func(t *testing.T) { + cache.mode.Store(&pathsCacheMode{result: `"0xcached"`}) + defer cache.mode.Store(nil) + resp, err := forward(`{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0x2","0x1"]}`) + require.NoError(t, err) + assert.Equal(t, `"0xcached"`, pathsResult(t, resp)) + }) + + t.Run("CacheErrorFallsThroughToUpstream", func(t *testing.T) { + cache.mode.Store(&pathsCacheMode{err: errors.New("cache down")}) + defer cache.mode.Store(nil) + resp, err := forward(`{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0x3","0x1"]}`) + require.NoError(t, err) + assert.Equal(t, `"0xupstream"`, pathsResult(t, resp)) + }) + + t.Run("StaticResponseServedWithoutUpstream", func(t *testing.T) { + network.cfg.StaticResponses = []*common.StaticResponseConfig{{ + Method: "eth_getBalance", + Params: []interface{}{"0x4", "0x1"}, + Response: &common.StaticResponseBodyConfig{Result: "0xstatic"}, + }} + defer func() { network.cfg.StaticResponses = nil }() + resp, err := forward(`{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0x4","0x1"]}`) + require.NoError(t, err) + assert.Equal(t, `"0xstatic"`, pathsResult(t, resp)) + }) +} + +func TestNetworksPaths_PolicyAccessorsAndGroupLanes(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{primaryLatest: "0x3e8", fallbackLatest: "0x3e8"}) + require.NotNil(t, network.policyEngine) + ids := []string{"primary-1", "primary-2", "fallback-1", "fallback-2"} + + t.Run("PolicyReadouts", func(t *testing.T) { + ordered := network.PolicyOrderedUpstreams("") + require.NotEmpty(t, ordered) + assert.Subset(t, ids, ordered) + assert.Contains(t, ordered, "primary-1") + assert.Equal(t, ordered, network.PolicyOrderedUpstreams("*")) + assert.ElementsMatch(t, ordered, network.PolicyOrderedUpstreams("eth_neverTicked"), "an un-ticked method falls back to the wildcard slot") + + decisions := network.RecentPolicyDecisions("", 10) + assert.NotEmpty(t, decisions, "the bootstrap tick records a decision") + assert.Len(t, network.RecentPolicyDecisions("", 1), 1, "the limit caps the history") + + for id := range network.PolicyScores("") { + assert.Contains(t, ids, id) + } + assert.False(t, network.PolicyLastSwitchAt("").After(time.Now())) + }) + + t.Run("PauseAndStepLogToggleEngine", func(t *testing.T) { + network.SetPolicyEnginePaused(true) + assert.True(t, network.policyEngine.IsPaused()) + network.SetPolicyEnginePaused(false) + assert.False(t, network.policyEngine.IsPaused()) + network.SetPolicyStepLogEnabled(true) + assert.True(t, network.policyEngine.IsStepLogEnabled()) + network.SetPolicyStepLogEnabled(false) + assert.False(t, network.policyEngine.IsStepLogEnabled()) + }) + + t.Run("GroupForSelector", func(t *testing.T) { + key, lane := network.EvmUpstreamGroupForSelector(ctx, "primary-*") + assert.True(t, strings.HasPrefix(key, "grp:"), "got %q", key) + assert.Equal(t, "primary", lane) + key2, _ := network.EvmUpstreamGroupForSelector(ctx, "primary-*") + assert.Equal(t, key, key2, "the key is stable") + fkey, flane := network.EvmUpstreamGroupForSelector(ctx, "fallback-*") + assert.NotEqual(t, key, fkey) + assert.Equal(t, "fallback", flane) + + for _, sel := range []string{"*", "primary-1", "nothing-*", "a|b"} { + k, l := network.EvmUpstreamGroupForSelector(ctx, sel) + assert.Equal(t, "", k, "selector %q forms no group", sel) + assert.Equal(t, "", l) + } + k, l := network.EvmUpstreamGroupForSelector(ctx, "*-1") + assert.True(t, strings.HasPrefix(k, "grp:")) + assert.Equal(t, "1", l, "the shared id token names the lane") + }) + + t.Run("ServedTipLanesMaterializeOncePerGroupUpToCap", func(t *testing.T) { + laneCtx := func(sel string) context.Context { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}`)) + req.SetDirectives(&common.RequestDirectives{UseUpstream: sel}) + return context.WithValue(ctx, common.RequestContextKey, req) + } + first := network.servedTipLaneFor(laneCtx("primary-*")) + require.NotNil(t, first) + assert.Equal(t, "primary", first.label) + assert.Same(t, first, network.servedTipLaneFor(laneCtx("primary-*")), "a group reuses its lane") + assert.Nil(t, network.servedTipLaneFor(laneCtx("primary-1")), "a single upstream is no group") + + count := network.servedTipLaneCount.Load() + network.servedTipLaneCount.Store(maxServedTipLanes) + assert.Nil(t, network.servedTipLaneFor(laneCtx("fallback-*")), "beyond the cap a new group gets no lane") + assert.Same(t, first, network.servedTipLaneFor(laneCtx("primary-*")), "existing lanes still resolve at the cap") + network.servedTipLaneCount.Store(count) + }) +} + +func TestNetworksPaths_SvmSlotReadouts(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + const host = "svm-paths-rpc1.localhost" + util.SetupMocksForSvmStatePollerWithShred(host, 1000, 990, 1005) + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return r.URL.Host == host && strings.Contains(util.SafeReadBody(r), `"method":"getAccountInfo"`) + }). + Reply(200). + BodyString(`{"jsonrpc":"2.0","id":1,"result":{"context":{"slot":1000},"value":{"lamports":7}}}`) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + net := setupTestSvmNetwork(t, ctx, []*common.UpstreamConfig{svmUpstreamConfig("rpc1", host)}) + + require.Eventually(t, func() bool { + return net.SvmHighestLatestSlot(ctx) == 1000 && net.SvmHighestIndexedSlot(ctx) == 1005 + }, 5*time.Second, 20*time.Millisecond) + assert.Equal(t, int64(990), net.SvmHighestFinalizedSlot(ctx)) + assert.Equal(t, int64(990), net.SvmHighestFinalizedSlotMax(ctx)) + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["pubkey",{"minContextSlot":500}]}`)) + resp, err := svmProjectForward(ctx, net, req) + require.NoError(t, err) + assert.Contains(t, pathsResult(t, resp), `"lamports":7`, "an upstream past minContextSlot serves") +} diff --git a/erpc/networks_precompile_state_gap_test.go b/erpc/networks_precompile_state_gap_test.go new file mode 100644 index 000000000..207ec689c --- /dev/null +++ b/erpc/networks_precompile_state_gap_test.go @@ -0,0 +1,164 @@ +package erpc + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/bytedance/sonic" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func init() { + util.ConfigureTestLogger() +} + +// hl-node's reply when a HyperEVM read precompile (0x…0800 onwards) runs +// against a block for which the node holds no HyperCore state. Many nodes hold +// that state only for their live tip, so any older block gets this reply, +// while a node that holds the state answers the same call. +const precompileStateGapMessage = "out of gas: gas exhausted during precompiled contract execution: 600000000" + +const oraclePxCall = `{"to":"0x0000000000000000000000000000000000000807","data":"0x0000000000000000000000000000000000000000000000000000000000000000"}` + +func precompileStateGapReply() map[string]interface{} { + return map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "error": map[string]interface{}{"code": -32003, "message": precompileStateGapMessage}, + } +} + +func TestNetworkForward_PrecompileStateGap_FailsOverToUpstreamWithState(t *testing.T) { + cases := []struct { + method string + params string + result string + }{ + {"eth_call", `[` + oraclePxCall + `,"0x11118800"]`, + `"0x0000000000000000000000000000000000000000000000000000000000012345"`}, + {"eth_estimateGas", `[` + oraclePxCall + `,"0x11118800"]`, + `"0x5a3c"`}, + {"eth_createAccessList", `[` + oraclePxCall + `,"0x11118800"]`, + `{"accessList":[],"gasUsed":"0x5a3c"}`}, + {"eth_simulateV1", `[{"blockStateCalls":[{"calls":[` + oraclePxCall + `]}]},"0x11118800"]`, + `[{"number":"0x11118800","calls":[{"returnData":"0x12345","status":"0x1","gasUsed":"0x5a3c","logs":[]}]}]`}, + {"debug_traceCall", `[` + oraclePxCall + `,"0x11118800",{}]`, + `{"gas":23100,"failed":false,"returnValue":"12345","structLogs":[]}`}, + } + + for _, tc := range cases { + t.Run(tc.method, func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + defer util.AssertNoPendingMocks(t, 0) + + gock.New("http://rpc1.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), tc.method) + }). + Times(1). + Reply(200). + JSON(precompileStateGapReply()) + gock.New("http://rpc2.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), tc.method) + }). + Times(1). + Reply(200). + BodyString(`{"jsonrpc":"2.0","id":1,"result":` + tc.result + `}`) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // No network retries: the failover must happen inside a single + // sweep, as it does for any other upstream that lacks the data. + network := setupTestNetworkForMissingDataRetry(t, ctx, nil, + &common.RetryPolicyConfig{MaxAttempts: 1}, + ) + network.PinUpstreamOrderForTest("rpc1", "rpc2") + + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","id":1,"method":"` + tc.method + `","params":` + tc.params + `}`)) + + resp, err := network.Forward(ctx, req) + require.NoError(t, err, "the upstream that holds HyperCore state must answer") + require.NotNil(t, resp) + + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.Nil(t, jrr.Error) + assert.JSONEq(t, tc.result, string(jrr.GetResultBytes())) + assert.Equal(t, "rpc2", resp.UpstreamId()) + }) + } +} + +func TestHttpServer_PrecompileStateGap_AllUpstreamsLackStateSurfacesNodeMessage(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + for _, host := range []string{"http://rpc1.localhost", "http://rpc2.localhost"} { + gock.New(host). + Post(""). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_call") + }). + Persist(). + Reply(200). + JSON(precompileStateGapReply()) + } + + cfg := &common.Config{ + Server: &common.ServerConfig{ + MaxTimeout: common.Duration(10 * time.Second).Ptr(), + }, + Projects: []*common.ProjectConfig{{ + Id: "test_project", + Networks: []*common.NetworkConfig{{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }}, + Upstreams: []*common.UpstreamConfig{ + { + Id: "rpc1", + Endpoint: "http://rpc1.localhost", + Type: common.UpstreamTypeEvm, + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + { + Id: "rpc2", + Endpoint: "http://rpc2.localhost", + Type: common.UpstreamTypeEvm, + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + }, + }}, + } + + sendRequest, _, _, shutdown, _ := createServerTestFixtures(cfg, t) + defer shutdown() + + statusCode, _, body := sendRequest( + `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[`+oraclePxCall+`,"0x11118800"]}`, nil, nil) + assert.Equal(t, http.StatusOK, statusCode) + + var respObject map[string]interface{} + require.NoError(t, sonic.UnmarshalString(body, &respObject)) + errObj, ok := respObject["error"].(map[string]interface{}) + require.True(t, ok, "response must carry a JSON-RPC error: %s", body) + // -32014 tells the client the data is missing on the node, not that its + // call ran out of gas, and the node's own wording is kept verbatim. + assert.Equal(t, float64(common.JsonRpcErrorMissingData), errObj["code"]) + assert.Equal(t, precompileStateGapMessage, errObj["message"]) +} diff --git a/erpc/networks_ratelimit_store_test.go b/erpc/networks_ratelimit_store_test.go new file mode 100644 index 000000000..5fb6e56a9 --- /dev/null +++ b/erpc/networks_ratelimit_store_test.go @@ -0,0 +1,245 @@ +package erpc + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/health" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/thirdparty" + "github.com/erpc/erpc/upstream" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" +) + +type rlStoreArch struct { + name string + host string + mocks func() + network func() *common.NetworkConfig + ups func() *common.UpstreamConfig + netId string + method string + forward func(context.Context, *Network, *common.NormalizedRequest) (*common.NormalizedResponse, error) +} + +var rlStoreArchs = []rlStoreArch{ + { + name: "evm", + host: "rpc1.localhost", + mocks: util.SetupMocksForEvmStatePoller, + network: func() *common.NetworkConfig { + return &common.NetworkConfig{Architecture: common.ArchitectureEvm, Evm: &common.EvmNetworkConfig{ChainId: 123}} + }, + ups: func() *common.UpstreamConfig { + return &common.UpstreamConfig{ + Id: "rpc1", Type: common.UpstreamTypeEvm, Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + } + }, + netId: util.EvmNetworkId(123), + method: "eth_call", // eth_chainId is answered before rate limiting + forward: func(ctx context.Context, n *Network, r *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return n.Forward(ctx, r) + }, + }, + { + name: "svm", + host: "svm-rlstore-rpc1.localhost", + mocks: func() { util.SetupMocksForSvmStatePoller("svm-rlstore-rpc1.localhost", 1000, 990) }, + network: func() *common.NetworkConfig { + return &common.NetworkConfig{ + Architecture: common.ArchitectureSvm, + Svm: &common.SvmNetworkConfig{Cluster: "mainnet-beta", Commitment: "confirmed"}, + } + }, + ups: func() *common.UpstreamConfig { return svmUpstreamConfig("rpc1", "svm-rlstore-rpc1.localhost") }, + netId: util.SvmNetworkId("", "mainnet-beta"), + method: "getAccountInfo", + forward: svmProjectForward, + }, +} + +// A budget of 5/hour on a real Redis client, walked through a healthy, slow, +// dead and restarted store, on each architecture, at network and upstream +// level, under each onFailure mode. Requests run one at a time. The slow and +// dead phases stay under 5 in Redis, so the over-limit cache never decides +// for them. +func TestNetwork_RateLimitStoreFailureFollowsOnFailure(t *testing.T) { + type phases struct{ healthy, slow, down, recovered int } + const healthy, slow, down, recovered = 1, 3, 1, 2 + modes := []struct { + mode common.RateLimitStoreFailureMode + // admitted per phase; the decision label store failures carry + want phases + decision string + }{ + // Failed checks are admitted. Recovered: the slow calls still landed + // in Redis and the restart probe adds one (1+3+1 = 5), so Redis + // denies again. + {common.RateLimitStoreFailureAllow, phases{1, 3, 1, 0}, "allow"}, + {common.RateLimitStoreFailureDeny, phases{1, 0, 0, 0}, "deny"}, + // The local counter (factor 0.6: 3/hour) saw the healthy request: 2 + // more fit. + {common.RateLimitStoreFailureLocal, phases{1, 2, 0, 0}, "local_"}, + } + for _, a := range rlStoreArchs { + for _, level := range []string{"network", "upstream"} { + for _, m := range modes { + t.Run(a.name+"/"+level+"/"+string(m.mode), func(t *testing.T) { + budget := "rl-" + strings.ReplaceAll(t.Name(), "/", "-") + redis := miniredis.RunT(t) + proxy := util.NewSlowTCPProxy(t, redis.Addr()) + ntw := setupRateLimitStoreNetwork(t, a, level, budget, proxy.Addr(), m.mode) + ctx := context.Background() + + failures := func(reason string) float64 { + sum := 0.0 + for _, d := range []string{"allow", "deny", "local_allow", "local_deny"} { + if strings.HasPrefix(d, m.decision) { + sum += testutil.ToFloat64(telemetry.MetricRateLimiterStoreFailureTotal.WithLabelValues(budget, reason, d)) + } + } + return sum + } + // admitted forwards n requests; every rejection must be a + // rate-limit error, never an unrelated failure. + admitted := func(n int) int { + ok := 0 + for range n { + _, err := a.forward(ctx, ntw, common.NewNormalizedRequest( + []byte(`{"jsonrpc":"2.0","id":1,"method":"`+a.method+`","params":[]}`))) + if err == nil { + ok++ + continue + } + require.True(t, common.HasErrorCode(err, + common.ErrCodeNetworkRateLimitRuleExceeded, + common.ErrCodeUpstreamRateLimitRuleExceeded, + ), "rejection must be a rate limit, got %v", err) + } + return ok + } + inflightDrained := func() bool { + return testutil.ToFloat64(telemetry.MetricRateLimiterRemoteInflight.WithLabelValues(budget)) == 0 + } + + require.Equal(t, m.want.healthy, admitted(healthy), "healthy store") + + proxy.SetDelay(200 * time.Millisecond) // > getTimeout 50ms + before := failures("limit_timeout") + require.Equal(t, m.want.slow, admitted(slow), "slow store") + require.Greater(t, failures("limit_timeout"), before, "slow store must count limit_timeout failures") + proxy.SetDelay(0) + require.Eventually(t, inflightDrained, 10*time.Second, 10*time.Millisecond) + + redis.Close() + before = failures("store_error") + require.Equal(t, m.want.down, admitted(down), "dead store") + require.Greater(t, failures("store_error"), before, "dead store must count store_error failures") + + require.NoError(t, redis.Restart()) + // The pool redials on its own; wait until Redis decides again. + // A check slower than getTimeout while it redials fails too, so + // wait out every failure reason, not just store_error. + anyFailure := func() float64 { + sum := 0.0 + for _, r := range []string{"store_error", "limit_timeout", "admission_full", "store_unavailable"} { + sum += failures(r) + } + return sum + } + require.Eventually(t, func() bool { + b := anyFailure() + admitted(1) + return anyFailure() == b + }, 10*time.Second, 50*time.Millisecond, "restarted store must decide again") + require.Equal(t, m.want.recovered, admitted(recovered), "restarted store") + }) + } + } + } +} + +func setupRateLimitStoreNetwork(t *testing.T, a rlStoreArch, level, budget, redisAddr string, mode common.RateLimitStoreFailureMode) *Network { + t.Helper() + util.ResetGock() + t.Cleanup(util.ResetGock) + a.mocks() + gock.New("http://" + a.host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), `"method":"`+a.method+`"`) + }). + Reply(200). + BodyString(`{"jsonrpc":"2.0","id":1,"result":"0x1"}`) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + store := &common.RateLimitStoreConfig{ + Driver: "redis", + Redis: &common.RedisConnectorConfig{ + Addr: redisAddr, ConnPoolSize: 4, + GetTimeout: common.Duration(50 * time.Millisecond), + }, + CacheKeyPrefix: budget + "_", + OnFailure: mode, + } + if mode == common.RateLimitStoreFailureLocal { + store.LocalLimitFactor = 0.6 + } + rlCfg := &common.RateLimiterConfig{ + Store: store, + Budgets: []*common.RateLimitBudgetConfig{{ + Id: budget, + Rules: []*common.RateLimitRuleConfig{{Method: "*", MaxCount: 5, Period: common.RateLimitPeriodHour}}, + }}, + } + require.NoError(t, rlCfg.SetDefaults()) + require.NoError(t, rlCfg.Validate()) + rlr, err := upstream.NewRateLimitersRegistry(ctx, rlCfg, &log.Logger) + require.NoError(t, err) + require.NotNil(t, rlr.GetCache(), "redis limiter must connect") + + vr := thirdparty.NewVendorsRegistry() + pr, err := thirdparty.NewProvidersRegistry(&log.Logger, vr, []*common.ProviderConfig{}, nil) + require.NoError(t, err) + ssCfg := &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: common.DriverMemory, + Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}, + }, + } + ssCfg.SetDefaults("test") + ssr, err := data.NewSharedStateRegistry(ctx, &log.Logger, ssCfg) + require.NoError(t, err) + + ups := a.ups() + netCfg := a.network() + if level == "network" { + netCfg.RateLimitBudget = budget + } else { + ups.RateLimitBudget = budget + } + mt := health.NewTracker(&log.Logger, "test", time.Minute) + upsReg := upstream.NewUpstreamsRegistry(ctx, &log.Logger, "test", + []*common.UpstreamConfig{ups}, ssr, rlr, vr, pr, nil, mt, nil) + ntw, err := NewNetwork(ctx, &log.Logger, "test", netCfg, rlr, upsReg, mt, nil) + require.NoError(t, err) + upsReg.Bootstrap(ctx) + time.Sleep(100 * time.Millisecond) + require.NoError(t, upsReg.PrepareUpstreamsForNetwork(ctx, a.netId)) + time.Sleep(150 * time.Millisecond) + return ntw +} diff --git a/erpc/networks_registry.go b/erpc/networks_registry.go index 5b3f75018..a5ddce422 100644 --- a/erpc/networks_registry.go +++ b/erpc/networks_registry.go @@ -165,11 +165,11 @@ func NewNetwork( rateLimitersRegistry: rateLimitersRegistry, policyEngine: policyEngine, - bootstrapOnce: sync.Once{}, inFlightRequests: &sync.Map{}, failsafeExecutors: failsafeExecutors, initializer: util.NewInitializer(appCtx, &lg, nil), } + network.servedTipAll.label = "all" if nwCfg.Architecture == "" { nwCfg.Architecture = common.ArchitectureEvm @@ -296,6 +296,7 @@ func (nr *NetworksRegistry) buildNetworkBootstrapTask(networkId string) *util.Bo if err != nil { return err } + network.startServedTipRefresh() nr.logger.Debug().Str("networkId", networkId).Msg("network bootstrap completed") return nil }, diff --git a/erpc/networks_retry_missing_data_test.go b/erpc/networks_retry_missing_data_test.go index ce70abbcb..944962c4b 100644 --- a/erpc/networks_retry_missing_data_test.go +++ b/erpc/networks_retry_missing_data_test.go @@ -1693,8 +1693,10 @@ func TestNetworkForward_WrongEmpty_SkipPunishment_BlockAvailabilityBounds(t *tes Type: common.UpstreamTypeEvm, Id: "rpc1", Endpoint: "http://rpc1.localhost", + // Polled, so the head at answer time (0x11118888) is known. Evm: &common.EvmUpstreamConfig{ - ChainId: 123, + ChainId: 123, + StatePollerInterval: common.Duration(time.Minute), }, }, { @@ -1719,7 +1721,8 @@ func TestNetworkForward_WrongEmpty_SkipPunishment_BlockAvailabilityBounds(t *tes require.NoError(t, err) require.NotNil(t, resp) - // rpc1 SHOULD be punished — no block availability config means unbounded + // rpc1 SHOULD be punished — no block availability bounds, and its head + // (0x11118888) was past block 200 when it answered empty ups := network.upstreamsRegistry.GetAllUpstreams() var rpc1 *upstream.Upstream for _, u := range ups { @@ -1729,12 +1732,132 @@ func TestNetworkForward_WrongEmpty_SkipPunishment_BlockAvailabilityBounds(t *tes } } require.NotNil(t, rpc1) + require.Equal(t, int64(0x11118888), req.HeadAtError(rpc1), "rpc1's head must be known when it answered") metrics := network.metricsTracker.GetUpstreamMethodMetrics(rpc1, "eth_getTransactionByHash", common.DataFinalityStateAll) assert.True(t, metrics.MisbehaviorsTotal.Load() > 0, "rpc1 SHOULD have misbehavior recorded — no block availability bounds configured") assert.Equal(t, int64(0), metrics.ErrorsTotal.Load(), "rpc1 should NOT have errors recorded — wrong-empty only records misbehavior, not failure") }) + + findUpstream := func(t *testing.T, network *Network, id string) *upstream.Upstream { + t.Helper() + for _, u := range network.upstreamsRegistry.GetAllUpstreams() { + if u.Id() == id { + return u + } + } + t.Fatalf("upstream %s not found", id) + return nil + } + misbehaviors := func(t *testing.T, network *Network, id string) int64 { + t.Helper() + return network.metricsTracker.GetUpstreamMethodMetrics(findUpstream(t, network, id), "eth_getTransactionByHash", common.DataFinalityStateAll).MisbehaviorsTotal.Load() + } + + t.Run("EmptyBelowItsHead_NoPunishment", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + defer util.AssertNoPendingMocks(t, 0) + + // The tx sits above rpc1's head (0x11118888) and below rpc2's (0x22228888): + // rpc1 was behind, not wrong. + aheadOfRpc1 := map[string]interface{}{} + for k, v := range txReceiptResult { + aheadOfRpc1[k] = v + } + aheadOfRpc1["blockNumber"] = "0x11119999" + + gock.New("http://rpc1.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_getTransactionByHash") + }). + Times(1). + Reply(200). + JSON(map[string]interface{}{"jsonrpc": "2.0", "id": 1, "result": nil}) + gock.New("http://rpc2.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_getTransactionByHash") + }). + Times(1). + Reply(200). + JSON(map[string]interface{}{"jsonrpc": "2.0", "id": 1, "result": aheadOfRpc1}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network := setupTestNetworkWithCustomUpstreams(t, ctx, + []*common.UpstreamConfig{ + {Type: common.UpstreamTypeEvm, Id: "rpc1", Endpoint: "http://rpc1.localhost", Evm: &common.EvmUpstreamConfig{ChainId: 123, StatePollerInterval: common.Duration(time.Minute)}}, + {Type: common.UpstreamTypeEvm, Id: "rpc2", Endpoint: "http://rpc2.localhost", Evm: &common.EvmUpstreamConfig{ChainId: 123, StatePollerInterval: common.Duration(time.Minute)}}, + }, + &common.DirectiveDefaultsConfig{RetryEmpty: util.BoolPtr(true)}, + &common.RetryPolicyConfig{MaxAttempts: 3, EmptyResultAccept: []string{}}, + ) + network.PinUpstreamOrderForTest("rpc1", "rpc2") + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getTransactionByHash","params":["` + txHash + `"]}`)) + req.ApplyDirectiveDefaults(network.cfg.DirectiveDefaults) + + resp, err := network.Forward(ctx, req) + require.NoError(t, err) + require.Equal(t, "rpc2", resp.UpstreamId()) + require.Equal(t, int64(0x11118888), req.HeadAtError(findUpstream(t, network, "rpc1")), "rpc1's head must be known when it answered") + + assert.Equal(t, int64(0), misbehaviors(t, network, "rpc1"), + "rpc1's head was below the winning block when it answered empty") + }) + + t.Run("SameUpstreamRetryWins_NoPunishment", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + defer util.AssertNoPendingMocks(t, 0) + + // rpc1 answers null, then returns the tx on the retry. + gock.New("http://rpc1.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_getTransactionByHash") + }). + Times(1). + Reply(200). + JSON(map[string]interface{}{"jsonrpc": "2.0", "id": 1, "result": nil}) + gock.New("http://rpc1.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_getTransactionByHash") + }). + Times(1). + Reply(200). + JSON(map[string]interface{}{"jsonrpc": "2.0", "id": 1, "result": txReceiptResult}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network := setupTestNetworkWithCustomUpstreams(t, ctx, + []*common.UpstreamConfig{ + {Type: common.UpstreamTypeEvm, Id: "rpc1", Endpoint: "http://rpc1.localhost", Evm: &common.EvmUpstreamConfig{ChainId: 123}}, + }, + &common.DirectiveDefaultsConfig{RetryEmpty: util.BoolPtr(true)}, + &common.RetryPolicyConfig{MaxAttempts: 3, EmptyResultAccept: []string{}}, + ) + network.PinUpstreamOrderForTest("rpc1") + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getTransactionByHash","params":["` + txHash + `"]}`)) + req.ApplyDirectiveDefaults(network.cfg.DirectiveDefaults) + + resp, err := network.Forward(ctx, req) + require.NoError(t, err) + require.Equal(t, "rpc1", resp.UpstreamId()) + require.Equal(t, 2, resp.Attempts(), "rpc1 answers null once, then wins the retry") + + assert.Equal(t, int64(0), misbehaviors(t, network, "rpc1"), + "the winning upstream is never counted for its own earlier empty answer") + }) } // Test: blockUnavailableDelay applied after full round of all upstreams diff --git a/erpc/networks_selection_policy_realpoll_test.go b/erpc/networks_selection_policy_realpoll_test.go index cef35f39e..fe15ff8f5 100644 --- a/erpc/networks_selection_policy_realpoll_test.go +++ b/erpc/networks_selection_policy_realpoll_test.go @@ -32,6 +32,7 @@ import ( "net/http" "sort" "strings" + "sync/atomic" "testing" "time" @@ -54,6 +55,9 @@ import ( type realPollFixture struct { id string latest int64 + // pollEvery is the upstream's statePollerInterval; zero keeps background + // polling off (1h) so tests drive polls explicitly. + pollEvery time.Duration } // mockPollerHostWithLatest mocks one upstream host's state-poller RPCs with a @@ -104,15 +108,17 @@ func setupRealPollLagNetwork(t *testing.T, ctx context.Context, fixtures []realP finalized = 1 } mockPollerHostWithLatest(host, f.latest, finalized) + pollEvery := f.pollEvery + if pollEvery == 0 { + pollEvery = time.Hour + } upstreamConfigs = append(upstreamConfigs, &common.UpstreamConfig{ Type: common.UpstreamTypeEvm, Id: f.id, Endpoint: host, Evm: &common.EvmUpstreamConfig{ - ChainId: 123, - // No background polling — tests drive PollLatestBlockNumber - // explicitly for determinism. - StatePollerInterval: common.Duration(time.Hour), + ChainId: 123, + StatePollerInterval: common.Duration(pollEvery), // Tiny (non-zero) debounce so each explicit re-poll actually // re-fetches instead of returning the cached value. A zero // debounce would fall through resolveDebounce to a ~block-time @@ -262,6 +268,13 @@ func TestNetworkPolicy_RealPoll_LaggingUpstreamExcluded(t *testing.T) { assert.Contains(t, order, "healthy-a") assert.Contains(t, order, "healthy-b") + // No probes either: probe responses never move a head, so probing a + // lag-only exclusion would bill its vendor without re-admitting it. + for _, u := range network.policyEngine.GetExcluded(network.networkId, "*", "*") { + assert.NotEqual(t, "frozen", u.Id(), + "a lag-only exclusion must not be handed to the prober") + } + // 3) End-to-end routing: a real request never reaches the excluded node. mockClean(t, upstreamHostFromID("healthy-a"), "eth_getBalance", "0xaaa") mockClean(t, upstreamHostFromID("healthy-b"), "eth_getBalance", "0xbbb") @@ -303,3 +316,69 @@ func TestNetworkPolicy_RealPoll_WithinTolerance_NoneExcluded(t *testing.T) { assert.ElementsMatch(t, []string{"u1", "u2", "u3"}, order, "all three within-tolerance upstreams must stay in rotation") } + +// TestNetworkPolicy_RealPoll_SlowPolledUpstreamAtTipStaysIn: an upstream +// polled every 10s on a 0.4s chain, always at the tip when asked, read up to +// 25 blocks behind late in each poll window, because peers polled every 500ms +// kept moving the network head past its stored one. A policy ticking every +// 10s too then read the same phase on every tick and excluded it for good on +// some processes. A healthy slow-polled upstream must stay in; one whose +// reports stop must still be excluded once a poll period has passed. +func TestNetworkPolicy_RealPoll_SlowPolledUpstreamAtTipStaysIn(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network := setupRealPollLagNetwork(t, ctx, []realPollFixture{ + {id: "fast-a", latest: 1000}, + {id: "fast-b", latest: 1000}, + {id: "slow", latest: 1000, pollEvery: 10 * time.Second}, + }) + tr := network.metricsTracker + var nowNs atomic.Int64 + nowNs.Store(time.Now().UnixNano()) + tr.SetClockForTest(func() time.Time { return time.Unix(0, nowNs.Load()) }) + fastA, fastB, slow := upstreamByID(t, network, "fast-a"), upstreamByID(t, network, "fast-b"), upstreamByID(t, network, "slow") + + // One 0.4s block per step; the fast pair reports every block, slow every + // 25th (10s). Fast reports first, as their 500ms polls land first. Block + // timestamps continue from the mocked poll's head block. + const ts0 = int64(0x6702a8f0) + step, slowReports := 0, true + runTo := func(last int) { + for ; step < last; step++ { + i := int64(step + 1) + nowNs.Add(int64(400 * time.Millisecond)) + head, ts := 1000+i, ts0+i*2/5 + tr.SetLatestBlockNumber(fastA, head, ts) + tr.SetLatestBlockNumber(fastB, head, ts) + if slowReports && i%25 == 0 { + tr.SetLatestBlockNumber(slow, head, ts) + } + } + } + tick := func() (order, excluded []string) { + policy.TickForTest(network.policyEngine, network.networkId, "*") + return policy.LatestDecisionOutputForTest(network.policyEngine, network.networkId, "*") + } + runTo(70) + + // 8s after slow's report at step 50: the tip is 20 blocks past its stored + // head, all of it made since slow was last asked. + assert.LessOrEqual(t, blockHeadLagOf(t, network, "slow"), int64(1), + "blocks made since the last poll are not lag") + order, excluded := tick() + assert.NotContains(t, excluded, "slow", "a slow-polled upstream at the tip must stay in rotation") + assert.Contains(t, order, "slow") + + // Slow stops reporting. 26s after its last report the chain made 65 + // blocks; one poll period (25 blocks) is credited, the rest is lag. + slowReports = false + runTo(115) + assert.GreaterOrEqual(t, blockHeadLagOf(t, network, "slow"), int64(39), + "a report missing past one poll period counts as lag again") + _, excluded = tick() + assert.Contains(t, excluded, "slow", "an upstream whose reports stopped must be excluded") +} diff --git a/erpc/networks_selection_policy_test.go b/erpc/networks_selection_policy_test.go index 01847a606..e93b3dfe4 100644 --- a/erpc/networks_selection_policy_test.go +++ b/erpc/networks_selection_policy_test.go @@ -25,6 +25,7 @@ import ( "context" "fmt" "net/http" + "sort" "strings" "sync" "testing" @@ -915,6 +916,65 @@ func TestNetworkPolicy_SpreadAcrossTags_PreventsCohortCascade(t *testing.T) { "cohort:cohort-2 (the only one) must be position[1] under spreadAcrossTags('cohort:')") } +// TestNetworkPolicy_BlockMethods_SkipsUpstreamForOneMethodOnly walks +// `blockMethods` end to end: a 9s trace p99 blocks the upstream for traces +// only, eth_call keeps it, the blocked upstream is a probe +// candidate for traces alone, and blocking every upstream keeps them all. +func TestNetworkPolicy_BlockMethods_SkipsUpstreamForOneMethodOnly(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + setupGockObserver(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + const trace = "debug_traceBlockByNumber" + upstreamConfigs := []*common.UpstreamConfig{ + {Type: common.UpstreamTypeEvm, Id: "slowtrace", Endpoint: upstreamHostFromID("slowtrace"), Evm: &common.EvmUpstreamConfig{ChainId: 123}}, + {Type: common.UpstreamTypeEvm, Id: "fasttrace", Endpoint: upstreamHostFromID("fasttrace"), Evm: &common.EvmUpstreamConfig{ChainId: 123}}, + } + network := setupSelectionPolicyNetworkWithEval(t, ctx, upstreamConfigs, + `(upstreams, ctx) => upstreams + .blockMethods({ match: ['debug_trace*', 'trace_*'], when: all(samplesAbove(20), latencyAbove(5000, 99)) }) + .probeExcluded({ sampleRate: 1, minSamples: 0 })`) + + slow, fast := upstreamByID(t, network, "slowtrace"), upstreamByID(t, network, "fasttrace") + for _, u := range []common.Upstream{slow, fast} { + seedDegraded(network.metricsTracker, u, seedSpec{method: "eth_call", successful: 100, successAvgMs: 10}) + } + seedDegraded(network.metricsTracker, slow, seedSpec{method: trace, successful: 100, successAvgMs: 9000}) + seedDegraded(network.metricsTracker, fast, seedSpec{method: trace, successful: 100, successAvgMs: 300}) + policy.TickForTest(network.policyEngine, network.networkId, "*") + + listed := func(method string) []string { + ids := idsFromUpstreams(network.policyEngine.GetOrdered(network.networkId, method, "*")) + sort.Strings(ids) + return ids + } + assert.Equal(t, []string{"fasttrace"}, listed(trace), "trace p99 9s must block slowtrace for traces") + assert.Equal(t, []string{"fasttrace", "slowtrace"}, listed("eth_call"), "the block must not touch eth_call") + assert.Equal(t, []string{"slowtrace"}, idsFromUpstreams(network.policyEngine.GetExcluded(network.networkId, trace, "*")), + "the blocked upstream is a trace probe candidate") + assert.Empty(t, network.policyEngine.GetExcluded(network.networkId, "eth_call", "*"), + "no eth_call probes: nothing is blocked for eth_call") + + // Forward drops slowtrace from every trace request, whatever the order. + mockClean(t, upstreamHostFromID("slowtrace"), trace, "0x1") + mockClean(t, upstreamHostFromID("fasttrace"), trace, "0x2") + for i := 0; i < 5; i++ { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"` + trace + `","params":["0x10",{"tracer":"callTracer"}]}`)) + resp, err := network.Forward(ctx, req) + require.NoError(t, err) + require.Equal(t, "fasttrace", resp.Upstream().Id(), "a blocked upstream never serves the blocked method") + } + + // Both upstreams slow for traces: a method never loses every upstream. + seedDegraded(network.metricsTracker, fast, seedSpec{method: trace, successful: 1000, successAvgMs: 9000}) + policy.TickForTest(network.policyEngine, network.networkId, "*") + assert.Equal(t, []string{"fasttrace", "slowtrace"}, listed(trace), "all blocked: traces keep every upstream") +} + // firstTagWithPrefix returns the first tag in `tags` whose value // begins with `prefix`, or "" if none. Test-only helper used to // inspect tag-encoded labels in network-level assertions. diff --git a/erpc/networks_sendrawtx_test.go b/erpc/networks_sendrawtx_test.go index 8a49665f4..609fbfe3d 100644 --- a/erpc/networks_sendrawtx_test.go +++ b/erpc/networks_sendrawtx_test.go @@ -215,6 +215,119 @@ func TestNetwork_SendRawTransaction_Idempotency(t *testing.T) { _ = resp }) + // HyperEVM returns opaque "Unexpected error (code=10055)" (JSON-RPC -32100) + // for HyperCore↔HyperEVM boundary failures (nonce / duplicate / insufficient + // funds / underpriced). Classify as nonce-too-low so we verify on-chain + // instead of failing over to ErrUpstreamsExhausted or treating as already-known. + t.Run("HyperEVMCode10055WithMatchingTxReturnsSuccess", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + requestBytes := []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_sendRawTransaction","params":["` + sampleSignedTx + `"]}`) + + gock.New("http://rpc1.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + return strings.Contains(body, "eth_sendRawTransaction") + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "error": map[string]interface{}{ + "code": -32100, + "message": "Unexpected error (code=10055)", + }, + }) + + gock.New("http://rpc1.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + return strings.Contains(body, "eth_getTransactionByHash") + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "result": map[string]interface{}{ + "hash": expectedTxHash, + "nonce": "0x9", + "blockHash": "0x1234567890abcdef", + "blockNumber": "0x100", + "from": "0x1234567890123456789012345678901234567890", + "to": "0x3535353535353535353535353535353535353535", + "value": "0xde0b6b3a7640000", + }, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network := setupSendRawTxTestNetworkSingleUpstream(t, ctx) + + req := common.NewNormalizedRequest(requestBytes) + resp, err := network.Forward(ctx, req) + + require.NoError(t, err, "HyperEVM 10055 with matching tx should convert to success") + require.NotNil(t, resp) + + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + assert.Contains(t, jrr.GetResultString(), "0x") + }) + + t.Run("HyperEVMCode10055WithNoMatchingTxReturnsError", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + requestBytes := []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_sendRawTransaction","params":["` + sampleSignedTx + `"]}`) + + gock.New("http://rpc1.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + return strings.Contains(body, "eth_sendRawTransaction") + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "error": map[string]interface{}{ + "code": -32100, + "message": "Unexpected error (code=10055)", + }, + }) + + gock.New("http://rpc1.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + return strings.Contains(body, "eth_getTransactionByHash") + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "result": nil, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network := setupSendRawTxTestNetworkSingleUpstream(t, ctx) + + req := common.NewNormalizedRequest(requestBytes) + _, err := network.Forward(ctx, req) + + require.Error(t, err, "10055 without matching tx must remain an error (e.g. insufficient funds)") + assert.NotContains(t, err.Error(), "ErrUpstreamsExhausted") + assert.Contains(t, err.Error(), "code=10055") + }) + t.Run("ExecutionRevertedIsRetryableAcrossUpstreams", func(t *testing.T) { util.ResetGock() defer util.ResetGock() diff --git a/erpc/networks_served_tip_guaranteedfor_test.go b/erpc/networks_served_tip_guaranteedfor_test.go index 3314e9543..d479445a5 100644 --- a/erpc/networks_served_tip_guaranteedfor_test.go +++ b/erpc/networks_served_tip_guaranteedfor_test.go @@ -55,7 +55,7 @@ func TestGuaranteedFor_UnsetIsNoOp(t *testing.T) { defer cancel() n := guaranteedForNetwork(t, ctx, mixedPool([]int64{1000, 1000, 1000}, []int64{993, 993}), - &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) + &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority}) assert.Equal(t, int64(1000), n.EvmHighestLatestBlockNumber(ctx)) } @@ -67,7 +67,7 @@ func TestGuaranteedFor_ClampsToTheNamedGroupsMajority(t *testing.T) { defer cancel() n := guaranteedForNetwork(t, ctx, mixedPool([]int64{1000, 1000, 1000}, []int64{993, 993}), - &common.EvmServedTipConfig{EnabledFor: []string{"latest"}, GuaranteedFor: []string{tagInternal}}) + &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority, GuaranteedFor: []string{tagInternal}}) assert.Equal(t, int64(993), n.EvmHighestLatestBlockNumber(ctx), "network majority is 1000, but the internals only have 993") } @@ -92,7 +92,7 @@ func TestGuaranteedFor_TracksTheActualLag(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { n := guaranteedForNetwork(t, ctx, mixedPool([]int64{1000, 1000, 1000}, tc.internal), - &common.EvmServedTipConfig{EnabledFor: []string{"latest"}, GuaranteedFor: []string{tagInternal}}) + &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority, GuaranteedFor: []string{tagInternal}}) assert.Equal(t, tc.want, n.EvmHighestLatestBlockNumber(ctx)) }) } @@ -107,7 +107,7 @@ func TestGuaranteedFor_StuckMemberDoesNotPinTheNetwork(t *testing.T) { defer cancel() n := guaranteedForNetwork(t, ctx, mixedPool([]int64{1000, 1000, 1000}, []int64{993, 993, 400}), - &common.EvmServedTipConfig{EnabledFor: []string{"latest"}, GuaranteedFor: []string{tagInternal}}) + &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority, GuaranteedFor: []string{tagInternal}}) assert.Equal(t, int64(993), n.EvmHighestLatestBlockNumber(ctx), "int3 frozen at 400 is outvoted by its own group") } @@ -122,7 +122,7 @@ func TestGuaranteedFor_SingleMemberGroupSetsTheTip(t *testing.T) { defer cancel() n := guaranteedForNetwork(t, ctx, mixedPool([]int64{1000, 1000, 1000}, []int64{400}), - &common.EvmServedTipConfig{EnabledFor: []string{"latest"}, GuaranteedFor: []string{tagInternal}}) + &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority, GuaranteedFor: []string{tagInternal}}) assert.Equal(t, int64(400), n.EvmHighestLatestBlockNumber(ctx)) } @@ -134,7 +134,7 @@ func TestGuaranteedFor_UnmatchedGroupConstrainsNothing(t *testing.T) { defer cancel() n := guaranteedForNetwork(t, ctx, mixedPool([]int64{1000, 1000, 1000}, []int64{993, 993}), - &common.EvmServedTipConfig{EnabledFor: []string{"latest"}, GuaranteedFor: []string{"type:nonexistent"}}) + &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority, GuaranteedFor: []string{"type:nonexistent"}}) assert.Equal(t, int64(1000), n.EvmHighestLatestBlockNumber(ctx)) } @@ -156,7 +156,7 @@ func TestGuaranteedFor_MultipleGroupsTakeTheLowest(t *testing.T) { {id: "arch2", chainID: 123, latestBlock: 980, tags: []string{"role:archive"}}, } n := guaranteedForNetwork(t, ctx, fx, &common.EvmServedTipConfig{ - EnabledFor: []string{"latest"}, + Quorum: common.ServedTipQuorumMajority, GuaranteedFor: []string{tagInternal, "role:archive"}, }) assert.Equal(t, int64(980), n.EvmHighestLatestBlockNumber(ctx), @@ -172,7 +172,7 @@ func TestGuaranteedFor_MatchesByUpstreamIdGlob(t *testing.T) { defer cancel() n := guaranteedForNetwork(t, ctx, mixedPool([]int64{1000, 1000, 1000}, []int64{993, 993}), - &common.EvmServedTipConfig{EnabledFor: []string{"latest"}, GuaranteedFor: []string{"int*"}}) + &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority, GuaranteedFor: []string{"int*"}}) assert.Equal(t, int64(993), n.EvmHighestLatestBlockNumber(ctx)) } @@ -187,7 +187,7 @@ func TestGuaranteedFor_AppliesToTheFinalizedAxis(t *testing.T) { util.SetupMocksForEvmStatePoller() fx := mixedPool([]int64{1000, 1000, 1000}, []int64{993, 993}) network, ups := setupServedTipNetworkWith(t, ctx, fx, &common.EvmServedTipConfig{ - EnabledFor: []string{"finalized"}, + Quorum: common.ServedTipQuorumMajority, GuaranteedFor: []string{tagInternal}, }) @@ -210,7 +210,7 @@ func TestGuaranteedFor_DeliveredHeadFloorDoesNotLiftPastTheGuarantee(t *testing. defer cancel() n := guaranteedForNetwork(t, ctx, mixedPool([]int64{1000, 1000, 1000}, []int64{993, 993}), - &common.EvmServedTipConfig{EnabledFor: []string{"latest"}, GuaranteedFor: []string{tagInternal}}) + &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority, GuaranteedFor: []string{tagInternal}}) n.NoteObservedLatestBlock(ctx, 1000) assert.Equal(t, int64(993), n.EvmHighestLatestBlockNumber(ctx), "the internals only have 993; a delivered 1000 must not override the guarantee") @@ -225,7 +225,7 @@ func TestGuaranteedFor_DeliveredHeadFloorAppliesWhenUnset(t *testing.T) { defer cancel() n := guaranteedForNetwork(t, ctx, mixedPool([]int64{1000, 1000, 1000}, []int64{993, 993}), - &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) + &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority}) n.NoteObservedLatestBlock(ctx, 1001) assert.Equal(t, int64(1001), n.EvmHighestLatestBlockNumber(ctx)) } diff --git a/erpc/networks_served_tip_invariants_test.go b/erpc/networks_served_tip_invariants_test.go index e5e4b111f..1b240d9ae 100644 --- a/erpc/networks_served_tip_invariants_test.go +++ b/erpc/networks_served_tip_invariants_test.go @@ -4,24 +4,25 @@ import ( "context" "testing" + "github.com/erpc/erpc/common" "github.com/erpc/erpc/util" "github.com/stretchr/testify/require" ) // ───────────────────────────────────────────────────────────────────────────── -// PROD-INCIDENT INVARIANTS — DO NOT ADAPT THESE TO NEW INTERNALS. +// FROZEN-TIP INVARIANTS — DO NOT ADAPT THESE TO NEW INTERNALS. // // Each test below reproduces, end-to-end at the Network level, a failure mode -// that actually occurred (or was made possible) in production on 2026-06-10/11, -// when the served tip silently froze hours in the past across 25+ chain/region -// pairs and every eth_call(..., "latest") was rewritten to a stale block. +// of an earlier served-tip design: the served tip silently froze hours in the +// past across many chains and every eth_call(..., "latest") was rewritten to a +// stale block. // // They are BLACK-BOX: they assert client-visible OUTCOMES (the advertised // "latest" tracks live heads, recovers from inherited state, never serves a // rogue counter) — never the mechanism (no gate flags, no cluster shapes, no // metric internals). Any future served-tip implementation, including a full // redesign, MUST keep passing these tests UNCHANGED. If one fails, the new -// logic reintroduces a production outage; fix the logic, not the test. +// logic reintroduces a frozen or poisoned tip; fix the logic, not the test. // ───────────────────────────────────────────────────────────────────────────── // servedTipInvariantFixtures is the standard 3-upstream healthy topology used @@ -34,12 +35,12 @@ func servedTipInvariantFixtures(head int64) []servedTipFixture { } } -// THE incident: a process starts with a served-tip counter inherited from the +// THE frozen tip: a process starts with a served-tip counter inherited from the // persistent shared store that is far BEHIND the live chain (rolling deploys, // region restarts, idle partitions), while the velocity gate is armed (block -// time known). In prod this wedged FOREVER: every pick rejected every live +// time known). That design wedged FOREVER: every pick rejected every live // tip, the counter never advanced, "latest" froze hours in the past, and the -// zeros served to the customer were real state from the frozen block. +// zeros served to clients were real state from the frozen block. // // INVARIANT: within a few picks the advertised "latest" must reach the live // agreed head, and must keep tracking it as the chain advances. No inherited @@ -57,14 +58,14 @@ func TestServedTipInvariant_StaleInheritedCounter_RecoversAndTracks(t *testing.T // PRECONDITION NOTE: the original reproduction seeded the persistent // shared counter to 5_000 (the value a previous process life left in the - // store — prod saw tips inherited 33k–4M blocks behind) with the velocity + // store, far behind the live head) with the velocity // gate armed. The majority design holds NO persistent served-tip state // and NO gate, so there is nothing left to seed or arm: the wedge's // enabling state is gone by construction. The OUTCOME assertions below // are unchanged and must hold for any future implementation, stateful or // not. - // Drive the chain forward one block per pick, exactly like prod traffic. + // Drive the chain forward one block per pick, like live traffic. served := int64(0) for r := int64(1); r <= 20; r++ { for _, u := range ups { @@ -76,11 +77,11 @@ func TestServedTipInvariant_StaleInheritedCounter_RecoversAndTracks(t *testing.T finalHead := liveHead + 20 require.GreaterOrEqual(t, served, finalHead-2, "advertised latest must catch up to the live agreed head and track it; "+ - "got %d while the live head is %d — the inherited counter wedged the tip (the 2026-06 prod incident)", + "got %d while the live head is %d — the inherited counter wedged the tip (the frozen-tip regression)", served, finalHead) } -// Prod follow-up to the same incident: the whole fleet was restarted and the +// Follow-up to the same failure: restarting every process did not heal it; the // freeze SURVIVED, because the wedge state lived in the persistent counter // while every fresh process re-armed the gate against it. // @@ -107,10 +108,10 @@ func TestServedTipInvariant_ProcessRestart_KeepsTracking(t *testing.T) { } // Simulate a process restart: ALL in-process served-tip state is lost; the - // shared counter survives in the backing store (exactly the prod restart - // that failed to heal). The chain meanwhile advanced past the gate bound. - network.servedLatestAnchor.seenValue.Store(0) - network.servedLatestAnchor.changedAtMs.Store(0) + // shared counter survives in the backing store (exactly the restart that + // failed to heal). The chain meanwhile advanced past the gate bound. + network.servedTipAll.latest.seenValue.Store(0) + network.servedTipAll.latest.changedAtMs.Store(0) served := int64(0) for r := int64(100); r <= 110; r++ { // jump: deploy downtime ≫ gate bound @@ -128,7 +129,7 @@ func TestServedTipInvariant_ProcessRestart_KeepsTracking(t *testing.T) { // The inverse failure ("shared state rogueness"): the persistent counter is // poisoned AHEAD of the real chain — a rogue store write, a garbage pick from // a previous buggy build, or corruption. The monotonic clamp would preserve it -// for up to the rollback tolerance, and prod would advertise blocks NO +// for up to the rollback tolerance, and the network would advertise blocks NO // upstream has: every interpolated request would fail or return inconsistent // state. // @@ -151,7 +152,7 @@ func TestServedTipInvariant_PoisonedCounter_NeverServed(t *testing.T) { // self-heal, far beyond cross-pod skew). That state no longer exists; the // only remaining served-tip inputs are the per-upstream poller heads, so // the rogue value enters there instead: one upstream starts reporting a - // fantasy-future tip (the wrong-chain/garbage-endpoint prod scenario). + // fantasy-future tip (a wrong-chain or garbage endpoint). ups[0].EvmStatePoller().SuggestLatestBlock(10_000_000) served := int64(0) @@ -173,8 +174,8 @@ func TestServedTipInvariant_PoisonedCounter_NeverServed(t *testing.T) { // Liveness under sustained operation: no combination of anchor aging, gate // dynamics, or counter state may EVER make the advertised "latest" stop for // long while every upstream keeps advancing. This is the umbrella invariant — -// the three tests above pin the specific prod paths; this one sweeps the -// steady-state loop the same way prod traffic does, with the gate armed and +// the three tests above pin the specific failure paths; this one sweeps the +// steady-state loop the same way live traffic does, with the gate armed and // realistic 1-block jitter between upstreams. func TestServedTipInvariant_SteadyState_NeverStalls(t *testing.T) { util.ResetGock() @@ -208,9 +209,8 @@ func TestServedTipInvariant_SteadyState_NeverStalls(t *testing.T) { } // Sanity guard for the invariants themselves: the fixture really runs the -// majority served-tip mode (not the legacy max fallback) — if someone flips -// the fixture default off, the invariants above would pass vacuously against -// max mode and stop guarding the served-tip path. +// majority quorum — if someone flips the fixture to another quorum, the +// invariants above would no longer guard the shape they were written for. func TestServedTipInvariant_Fixture_ServedTipModeActive(t *testing.T) { util.ResetGock() defer util.ResetGock() @@ -220,8 +220,8 @@ func TestServedTipInvariant_Fixture_ServedTipModeActive(t *testing.T) { defer cancel() network, _ := setupServedTipNetwork(t, ctx, servedTipInvariantFixtures(100)) - require.True(t, network.servedTipEnabledFor("latest"), - "invariant fixtures must exercise the served-tip path, not the max fallback") + require.Equal(t, common.ServedTipQuorumMajority, network.cfg.Evm.ServedTipQuorum(), + "invariant fixtures must exercise the majority quorum") // MAX(heads) != majority(heads) for an uneven topology proves the pick // actually goes through the served-tip algorithm. @@ -234,16 +234,15 @@ func TestServedTipInvariant_Fixture_ServedTipModeActive(t *testing.T) { "served-tip mode must be live (max mode would return 200)") } -// THE 2026-08-11 incident (celo / evm:42220, 12:36–12:41 UTC): two "legacy" -// upstreams configured with `blockAvailability.upper.exactBlock: 21615999` -// polled the live head (74.5M) but reported the BOUND as their effective -// latest. While all five upstreams were eligible the three live heads carried -// the majority. Then the selection policy excluded one live upstream for lag, -// the ballot became [74550978, 74550974, 21615999, 21615999], and the pick — -// the 3rd of 4 — fell to 21,615,999. "latest" was interpolated to that -// September-2023 block, the availability gate then admitted ONLY the two legacy -// upstreams (the live ones declare a lower bound of 21,616,000), and they -// correctly served state from 2023: eth_getBalance returned 0x0 for funded +// Two "legacy" upstreams configured with `blockAvailability.upper.exactBlock` +// far below the chain head polled the live head but reported the BOUND as +// their effective latest. While all five upstreams were eligible the three +// live heads carried the majority. Then the selection policy excluded one live +// upstream for lag, the ballot became [99996, 99968, 50000, 50000], and the +// pick — the 3rd of 4 — fell to the cap. "latest" was interpolated to that +// historical block, the availability gate then admitted ONLY the two legacy +// upstreams (the live ones declare a lower bound above the cap), and they +// correctly served historical state: eth_getBalance returned 0x0 for funded // accounts and eth_call reverted. // // INVARIANT: an upstream that is only DECLARING a serving range must never @@ -257,30 +256,30 @@ func TestServedTipInvariant_AvailabilityBoundNeverDefinesTheHead(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - const legacyCap = int64(21_615_999) - const liveHead = int64(74_550_978) + const legacyCap = int64(50_000) + const liveHead = int64(100_000) network, _ := setupServedTipNetwork(t, ctx, []servedTipFixture{ - {id: "prism-celo", chainID: 123, latestBlock: liveHead}, - {id: "celo-mainnet-nirvana", chainID: 123, latestBlock: 74_550_974}, - {id: "quicknode-celo", chainID: 123, latestBlock: 74_550_946}, - // The legacy pair: pollers at the live tip, serving range frozen in 2023. - {id: "celo-mainnet-nirvana-legacy", chainID: 123, latestBlock: liveHead, upperExactBlock: legacyCap}, - {id: "quicknode-celo-legacy", chainID: 123, latestBlock: liveHead, upperExactBlock: legacyCap}, + {id: "leader", chainID: 123, latestBlock: liveHead}, + {id: "peer-a", chainID: 123, latestBlock: 99_996}, + {id: "peer-b", chainID: 123, latestBlock: 99_968}, + // The legacy pair: pollers at the live tip, serving range frozen far below it. + {id: "legacy-1", chainID: 123, latestBlock: liveHead, upperExactBlock: legacyCap}, + {id: "legacy-2", chainID: 123, latestBlock: liveHead, upperExactBlock: legacyCap}, }) served := network.EvmHighestLatestBlockNumber(ctx) require.Greater(t, served, legacyCap, "with the full fleet eligible the served tip must be a live head, not the legacy serving bound") - // 12:36 UTC: prism-celo's head froze ~30s and the selection policy excluded - // it for lag. This is the exact production ballot that broke. - network.PinUpstreamOrderForTest("celo-mainnet-nirvana", "quicknode-celo", "celo-mainnet-nirvana-legacy", "quicknode-celo-legacy") + // leader's head froze briefly and the selection policy excluded it for + // lag. This is the ballot that pulls the tip down to the cap. + network.PinUpstreamOrderForTest("peer-a", "peer-b", "legacy-1", "legacy-2") served = network.EvmHighestLatestBlockNumber(ctx) require.Greater(t, served, legacyCap, "losing one live upstream must not hand the tip to the availability bounds: "+ - "served %d, the legacy cap is %d — this is the celo outage (eth_getBalance = 0x0 on funded accounts)", + "served %d, the legacy cap is %d — clients would read historical state (eth_getBalance = 0x0 on funded accounts)", served, legacyCap) require.LessOrEqual(t, served, liveHead, "the served tip must still be a block the live upstreams actually have") diff --git a/erpc/networks_served_tip_refresh.go b/erpc/networks_served_tip_refresh.go new file mode 100644 index 000000000..5bf59643d --- /dev/null +++ b/erpc/networks_served_tip_refresh.go @@ -0,0 +1,42 @@ +package erpc + +import ( + "time" + + "github.com/erpc/erpc/common" +) + +// servedTipRefreshInterval is the longest the network-wide served tip goes +// without an evaluation. +const servedTipRefreshInterval = time.Second + +// startServedTipRefresh evaluates an EVM network's network-wide served tip on +// both axes every servedTipRefreshInterval until the app stops. +// +// Only an evaluation writes the served-tip gauges, moves the advance-age +// anchor and refreshes the regression guard's memory. Without this loop a +// network that sees one request a minute exports a tip up to a minute old, and +// its guard measures against a pick that old. The loop evaluates exactly what +// an unscoped request would, so it adds no state of its own. +// +// Once per network: its bootstrap task can run again after it succeeded. +func (n *Network) startServedTipRefresh() { + if n.cfg.Architecture != common.ArchitectureEvm { + return + } + n.servedTipRefreshOnce.Do(func() { + go func() { + ticker := time.NewTicker(servedTipRefreshInterval) + defer ticker.Stop() + for { + select { + case <-n.appCtx.Done(): + return + case <-ticker.C: + n.EvmHighestLatestBlockNumber(n.appCtx) + n.EvmHighestFinalizedBlockNumber(n.appCtx) + } + } + }() + }) +} diff --git a/erpc/networks_served_tip_test.go b/erpc/networks_served_tip_test.go index 328c96962..37483aa10 100644 --- a/erpc/networks_served_tip_test.go +++ b/erpc/networks_served_tip_test.go @@ -3,12 +3,12 @@ package erpc import ( "context" "net/http" - "strconv" "strings" "sync" "testing" "time" + "github.com/erpc/erpc/architecture/evm" "github.com/erpc/erpc/common" "github.com/erpc/erpc/data" "github.com/erpc/erpc/health" @@ -23,10 +23,11 @@ import ( "github.com/stretchr/testify/require" ) -// These tests pin the semantics of Network.EvmHighestLatestBlockNumber in -// served-tip mode: the MAJORITY order statistic — the freshest block a strict -// majority of eligible upstreams already have (common.PickServedTip). Each case -// is designed so MAX(tips) != majority to make regressions visible. +// These tests pin the semantics of Network.EvmHighestLatestBlockNumber: the +// quorum order statistic — the freshest block at least servedTip.quorum +// eligible, non-benched upstreams already have (common.PickServedTip). Most +// cases run quorum "majority" and are designed so MAX(tips) != the pick to +// make regressions visible. // servedTipFixture is a compact description of one upstream's poller state // at test time. Used to drive the lighter setupServedTipNetwork helper. @@ -37,7 +38,7 @@ type servedTipFixture struct { // upperExactBlock, when > 0, configures blockAvailability.upper.exactBlock — // the upstream still polls latestBlock as its head, but only SERVES up to // this block, so its effective latest is the bound rather than a head - // observation (the celo "legacy" upstream shape). + // observation (a "legacy" upstream that serves only a frozen historical range). upperExactBlock int64 // upperLatestMinus, when set, configures blockAvailability.upper.latestBlockMinus // — a bound DERIVED from the upstream's own head, so it tracks the chain at @@ -56,9 +57,10 @@ type servedTipFixture struct { // via gock to answer eth_chainId so initial validation succeeds. After // bootstrap, SuggestLatestBlock and SetSyncingState are applied per fixture. // -// Returns the network and the upstream slice in input order. +// Returns the network and the upstream slice in input order. The fixture runs +// quorum "majority" on both axes. func setupServedTipNetwork(t *testing.T, ctx context.Context, fixtures []servedTipFixture) (*Network, []*upstream.Upstream) { - return setupServedTipNetworkWith(t, ctx, fixtures, &common.EvmServedTipConfig{EnabledFor: []string{"latest", "finalized"}}) + return setupServedTipNetworkWith(t, ctx, fixtures, &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority}) } func setupServedTipNetworkWith(t *testing.T, ctx context.Context, fixtures []servedTipFixture, stCfg *common.EvmServedTipConfig) (*Network, []*upstream.Upstream) { @@ -114,6 +116,9 @@ func setupServedTipNetworkWith(t *testing.T, ctx context.Context, fixtures []ser rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) + // The head bench state machine shares the guard's clock, so + // pinServedTipClock drives both. + metricsTracker.SetClockForTest(func() time.Time { return servedTipClock() }) vr := thirdparty.NewVendorsRegistry() pr, err := thirdparty.NewProvidersRegistry( @@ -353,10 +358,9 @@ func TestServedTip_AllSyncing_ReturnsZero(t *testing.T) { // ----- opt-in gating + eligible-set sourcing ------------------------------- -// With the served-tip feature DISABLED (the default, nil config), the network -// head is the corroborated head across eligible upstreams: the second-highest, -// not the majority pick and not the raw max. -func TestServedTip_DisabledByDefault_ReturnsCorroboratedHead(t *testing.T) { +// The default config (nil servedTip = quorum 2) serves the second-highest head: +// not MAX and not the majority pick. +func TestServedTip_DefaultQuorum_ServesSecondHighest(t *testing.T) { util.ResetGock() defer util.ResetGock() util.SetupMocksForEvmStatePoller() @@ -368,17 +372,17 @@ func TestServedTip_DisabledByDefault_ReturnsCorroboratedHead(t *testing.T) { {id: "u1", chainID: 123, latestBlock: 100}, {id: "u2", chainID: 123, latestBlock: 99}, {id: "u3", chainID: 123, latestBlock: 98}, - }, nil) // nil ServedTip config => feature disabled + }, nil) served := network.EvmHighestLatestBlockNumber(ctx) assert.Equal(t, int64(99), served, - "default mode serves the second-highest head, not MAX(tips)=100") + "the default quorum serves the second-highest head, not MAX(tips)=100") } // A lone upstream reporting another chain's height must not define the network -// head in default mode either: "latest" interpolation, the eth_blockNumber -// floor, and the cache's known-highest-block all read this accessor. -func TestServedTip_DisabledByDefault_LoneFarAheadUpstreamDoesNotDefineHead(t *testing.T) { +// head on the default config: "latest" interpolation, eth_blockNumber and the +// cache's known-highest-block all read this accessor. +func TestServedTip_DefaultQuorum_LoneFarAheadUpstreamDoesNotDefineHead(t *testing.T) { util.ResetGock() defer util.ResetGock() util.SetupMocksForEvmStatePoller() @@ -394,7 +398,9 @@ func TestServedTip_DisabledByDefault_LoneFarAheadUpstreamDoesNotDefineHead(t *te assert.Equal(t, int64(32_610_710), network.EvmHighestLatestBlockNumber(ctx), "the rogue's head is not corroborated") - assert.Equal(t, int64(62_381_379), network.evmHeadReference(ctx, false).Max, + ceiling, known := network.futureBlockCeiling(ctx) + assert.True(t, known) + assert.Equal(t, int64(62_381_379), ceiling, "the raw max stays visible for the future-block short-circuit") } @@ -451,7 +457,7 @@ func TestServedTip_GuaranteedMethodClampsTip(t *testing.T) { "without a guaranteed method, served = dominant cluster min 99 (u3=90 is an outlier)") nwB, _ := setupServedTipNetworkWith(t, ctx, fixtures, &common.EvmServedTipConfig{ - EnabledFor: []string{"latest"}, + Quorum: common.ServedTipQuorumMajority, GuaranteedMethods: []string{"trace_block"}, }) assert.Equal(t, int64(90), nwB.EvmHighestLatestBlockNumber(ctx), @@ -462,8 +468,8 @@ func TestServedTip_GuaranteedMethodClampsTip(t *testing.T) { // served `latest`/`finalized` decision: when a request targets a subset of // upstreams (by tag here), the tip is computed AMONG that subset so a // more-ahead group (e.g. base flashblocks) never defines `latest` for a -// request pinned to the other group. Selector scoping is stateless — it must -// not advance/read the network-wide monotonic counter. +// request pinned to the other group. Selector scoping is stateless — a +// selector that forms no group creates no lane. func TestServedTip_SelectorScoped(t *testing.T) { withSel := func(ctx context.Context, sel string) context.Context { req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}`)) @@ -535,15 +541,15 @@ func TestServedTip_SelectorScoped(t *testing.T) { "fast* -> the fast group's own tip") assert.Equal(t, int64(1000), nw.EvmHighestLatestBlockNumber(withSel(ctx, "!fast*")), "!fast* -> the complement (slow) group's own tip") - assert.Equal(t, int32(2), nw.servedTipPartitionCount.Load(), + assert.Equal(t, int32(2), nw.servedTipLaneCount.Load(), "a pattern and its negation create exactly two lanes") // Each partition is named by common.LaneName of its matched id set: // {fast-1,fast-2} -> "fast", {slow-1,slow-2} -> "slow". A single vendor // prefix is never a lane (the partition needs >=2 upstreams). lanes := map[string]bool{} - nw.servedTipPartitions.Range(func(_, v any) bool { - lanes[v.(*servedTipPartition).lane] = true + nw.servedTipLanes.Range(func(_, v any) bool { + lanes[v.(*servedTipLane).label] = true return true }) assert.True(t, lanes["fast"] && lanes["slow"], @@ -555,7 +561,7 @@ func TestServedTip_SelectorScoped(t *testing.T) { for _, sel := range []string{"fast-*", "family:fast", "slow*", "family:slow", "!fast*"} { nw.EvmHighestLatestBlockNumber(withSel(ctx, sel)) } - assert.Equal(t, int32(2), nw.servedTipPartitionCount.Load(), + assert.Equal(t, int32(2), nw.servedTipLaneCount.Load(), "equivalent selectors (same matched set) must dedup, not create new lanes") // Non-group selectors NEVER materialize: match-all, single-node, @@ -563,7 +569,7 @@ func TestServedTip_SelectorScoped(t *testing.T) { for _, sel := range []string{"*", "fast-1", "slow-1", "ghost*", "does-not-exist", "(fast*|slow*)"} { nw.EvmHighestLatestBlockNumber(withSel(ctx, sel)) } - assert.Equal(t, int32(2), nw.servedTipPartitionCount.Load(), + assert.Equal(t, int32(2), nw.servedTipLaneCount.Load(), "match-all / single-node / no-match / boolean selectors must not create state (DDoS-safe)") }) } @@ -587,14 +593,14 @@ func TestServedTip_AnchorStampsOnServedValueChange(t *testing.T) { }) require.Equal(t, int64(100), network.EvmHighestLatestBlockNumber(ctx)) - require.Negative(t, network.servedLatestAnchor.age(), + require.Negative(t, network.servedTipAll.latest.age(), "first observation alone must not stamp the watchdog") for _, u := range ups { u.EvmStatePoller().SuggestLatestBlock(105) } require.Equal(t, int64(105), network.EvmHighestLatestBlockNumber(ctx)) - age := network.servedLatestAnchor.age() + age := network.servedTipAll.latest.age() require.GreaterOrEqual(t, age, time.Duration(0), "watchdog must stamp once the served value moves") require.Less(t, age, 5*time.Second) @@ -677,7 +683,7 @@ func TestServedTip_NoNetworkLevelStickiness(t *testing.T) { // A halted chain that resumes with a burst (sequencer outage ending, L2 batch // landing) must be served the moment a majority reports the jump. The old // velocity gate had to be tuned (slack/buffer) to permit exactly this, and a -// mis-tuned gate freezing on the jump is what wedged prod; statelessness has +// mis-tuned gate freezing on the jump is the frozen-tip regression; statelessness has // no window to outrun. func TestServedTip_HaltedChainResume_CatchesUpImmediately(t *testing.T) { util.ResetGock() @@ -705,13 +711,13 @@ func TestServedTip_HaltedChainResume_CatchesUpImmediately(t *testing.T) { "the post-halt burst must be served on the first pick that sees it") } -// ─── Availability bounds are not head observations (2026-08 celo) ──────────── +// ─── Availability bounds are not head observations ────────────────────────── // A blockAvailability.upper bound is a SERVING-RANGE declaration, not a view of // the chain head, so it must not appear on the ballot while any live head does. // Two upstreams pinned at `upper.exactBlock` (their pollers sitting at the live // tip) used to outvote the live heads as soon as the eligible set shrank — -// exactly how celo served September-2023 state on 2026-08-11. +// pulling "latest" down to a block years in the past. func TestServedTip_BoundCappedUpstreamsDoNotVote(t *testing.T) { util.ResetGock() defer util.ResetGock() @@ -776,7 +782,7 @@ func TestServedTip_LatestBlockMinusUpstreamsStillVote(t *testing.T) { served, canServe, len(ups)) } -// The guaranteed-method floor was the second door onto the incident value: its +// The guaranteed-method floor was the second door onto the capped value: its // ballot took raw effective heads and ran AFTER the guard, so a static cap that // the network ballot had just refused came back as a "floor" and was served. // The floor now builds its per-method ballot with the same rule. @@ -788,23 +794,23 @@ func TestServedTip_GuaranteedMethodFloorCannotReadmitTheCap(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - const legacyCap = int64(21_615_999) + const legacyCap = int64(50_000) network, _ := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ - {id: "prism-celo", chainID: 123, latestBlock: 74_550_978}, - {id: "celo-mainnet-nirvana", chainID: 123, latestBlock: 74_550_974}, - {id: "quicknode-celo", chainID: 123, latestBlock: 74_550_946}, - {id: "celo-mainnet-nirvana-legacy", chainID: 123, latestBlock: 74_550_978, upperExactBlock: legacyCap}, - {id: "quicknode-celo-legacy", chainID: 123, latestBlock: 74_550_978, upperExactBlock: legacyCap}, + {id: "leader", chainID: 123, latestBlock: 100_000}, + {id: "peer-a", chainID: 123, latestBlock: 99_996}, + {id: "peer-b", chainID: 123, latestBlock: 99_968}, + {id: "legacy-1", chainID: 123, latestBlock: 100_000, upperExactBlock: legacyCap}, + {id: "legacy-2", chainID: 123, latestBlock: 100_000, upperExactBlock: legacyCap}, }, &common.EvmServedTipConfig{ - EnabledFor: []string{"latest"}, + Quorum: common.ServedTipQuorumMajority, GuaranteedMethods: []string{"eth_getLogs"}, }) require.Greater(t, network.EvmHighestLatestBlockNumber(ctx), legacyCap, "precondition: with the full fleet eligible the tip is a live head") - // The incident ballot: one live upstream lag-excluded. - network.PinUpstreamOrderForTest("celo-mainnet-nirvana", "quicknode-celo", "celo-mainnet-nirvana-legacy", "quicknode-celo-legacy") + // The failing ballot: one live upstream lag-excluded. + network.PinUpstreamOrderForTest("peer-a", "peer-b", "legacy-1", "legacy-2") assert.Greater(t, network.EvmHighestLatestBlockNumber(ctx), legacyCap, "the guaranteed-method floor must not re-admit the serving range the ballot filter removed") @@ -814,10 +820,10 @@ func TestServedTip_GuaranteedMethodFloorCannotReadmitTheCap(t *testing.T) { // ballot (no live head → it holds its last corroborated pick), and the // guaranteed-method floor — which runs AFTER the guard — then recomputes that // same collapsed instant and drags the served value back to the cap. Each fix -// was correct alone; together they still served 21,615,999. +// was correct alone; together they still served the cap. // // The live upstreams here go SYNCING rather than being un-pinned: that is the -// production shape (lag-excluded, cordoned, mid-restart upstreams stay +// realistic shape (lag-excluded, cordoned, mid-restart upstreams stay // REGISTERED), and registration is what tells the floor that a cap-only ballot // is transient rather than the operator's configuration. func TestServedTip_GuaranteedMethodFloor_DoesNotUndoTheGuardsHold(t *testing.T) { @@ -829,15 +835,15 @@ func TestServedTip_GuaranteedMethodFloor_DoesNotUndoTheGuardsHold(t *testing.T) defer cancel() advance := pinServedTipClock(t) - const legacyCap = int64(21_615_999) + const legacyCap = int64(50_000) network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ - {id: "prism-celo", chainID: 123, latestBlock: 74_550_978}, - {id: "celo-mainnet-nirvana", chainID: 123, latestBlock: 74_550_974}, - {id: "quicknode-celo", chainID: 123, latestBlock: 74_550_946}, - {id: "celo-mainnet-nirvana-legacy", chainID: 123, latestBlock: 74_550_978, upperExactBlock: legacyCap}, - {id: "quicknode-celo-legacy", chainID: 123, latestBlock: 74_550_978, upperExactBlock: legacyCap}, + {id: "leader", chainID: 123, latestBlock: 100_000}, + {id: "peer-a", chainID: 123, latestBlock: 99_996}, + {id: "peer-b", chainID: 123, latestBlock: 99_968}, + {id: "legacy-1", chainID: 123, latestBlock: 100_000, upperExactBlock: legacyCap}, + {id: "legacy-2", chainID: 123, latestBlock: 100_000, upperExactBlock: legacyCap}, }, &common.EvmServedTipConfig{ - EnabledFor: []string{"latest"}, + Quorum: common.ServedTipQuorumMajority, GuaranteedMethods: []string{"eth_getLogs"}, }) @@ -845,7 +851,7 @@ func TestServedTip_GuaranteedMethodFloor_DoesNotUndoTheGuardsHold(t *testing.T) require.Greater(t, healthy, legacyCap, "precondition: the live heads carry the ballot") // Every live upstream drops out of the ballot at once; only the two - // 2023-capped ones remain servable, and they support eth_getLogs. + // capped ones remain servable, and they support eth_getLogs. for _, u := range ups[:3] { u.EvmStatePoller().SetSyncingState(common.EvmSyncingStateSyncing) } @@ -878,7 +884,7 @@ func TestServedTip_GuaranteedMethodFloor_CapsStillSetTheFloorWhenAloneOnAMethod( {id: "archive-1", chainID: 123, latestBlock: 10_000, upperExactBlock: 5_000}, {id: "archive-2", chainID: 123, latestBlock: 10_000, upperExactBlock: 5_000}, }, &common.EvmServedTipConfig{ - EnabledFor: []string{"latest"}, + Quorum: common.ServedTipQuorumMajority, GuaranteedMethods: []string{"trace_block"}, }) @@ -977,7 +983,7 @@ func TestServedTip_AllInputsFiltered_FailsOpenToZero(t *testing.T) { // servedTipRegressionCount reads the guard's counter for a network/axis/outcome. func servedTipRegressionCount(n *Network, axis, outcome string) float64 { return promUtil.ToFloat64(telemetry.MetricNetworkServedTipRegressionTotal. - WithLabelValues(n.projectId, n.Label(), servedTipLaneAll, axis, outcome)) + WithLabelValues(n.projectId, n.Label(), "all", axis, outcome)) } // pinServedTipClock freezes the guard's clock for the duration of a test and @@ -1003,8 +1009,8 @@ func pinServedTipClock(t *testing.T) func(time.Duration) { // A pick that drops more than the tolerance below the freshest live head is a // poisoned ballot, not a chain that moved: the last corroborated pick is served // instead. Here the two upstreams stuck at block 5 become the majority the -// moment the healthy pair leaves the eligible set — the same shape as celo's -// bound-capped pair, reached without any bounds at all (which is why the guard +// moment the healthy pair leaves the eligible set — the same shape as two +// bound-capped upstreams, reached without any bounds at all (which is why the guard // exists independently of the bounds filter). func TestServedTip_RegressionGuard_HoldsLastCorroboratedPick(t *testing.T) { util.ResetGock() @@ -1040,8 +1046,8 @@ func TestServedTip_RegressionGuard_HoldsLastCorroboratedPick(t *testing.T) { // The guard is a bounded pause, never a wedge: once the regression outlasts // servedTipRegressionTTL, reality is the better explanation and the pick is -// served as computed. This is the property whose absence (a clamp that could -// hold forever) froze the fleet in 2026-06. +// served as computed. Without this property a clamp could hold forever and +// freeze the tip. func TestServedTip_RegressionGuard_FailsOpenAfterTTL(t *testing.T) { util.ResetGock() defer util.ResetGock() @@ -1101,7 +1107,7 @@ func TestServedTip_RegressionGuard_SmallReorgPassesThrough(t *testing.T) { "a regression inside the tolerance is reality and must pass through") assert.Equal(t, before, servedTipRegressionCount(network, "latest", "held"), "a tolerated regression is not a guard trip") - assert.Equal(t, int64(9_500), network.servedLatestAnchor.lastGoodValue.Load(), + assert.Equal(t, int64(9_500), network.servedTipAll.latest.lastGoodValue.Load(), "the tolerated pick becomes the new corroborated value") } @@ -1130,11 +1136,11 @@ func TestServedTip_RegressionGuard_SkippedWithoutLiveHead(t *testing.T) { "no live head → no reference → the guard must not interfere") assert.Equal(t, before, servedTipRegressionCount(network, "latest", "held"), "the guard cannot trip without a live head to compare against") - assert.Zero(t, network.servedLatestAnchor.lastGoodValue.Load(), + assert.Zero(t, network.servedTipAll.latest.lastGoodValue.Load(), "an uncorroborated cap must never be remembered as a good pick") } -// THE remaining door onto the celo incident value, and the reason the guard no +// THE remaining door onto the capped value, and the reason the guard no // longer stands down without a live head: when EVERY live upstream leaves the // eligible set while the bound-capped ones stay, the ballot legitimately falls // back to the caps — and a guard that requires a live reference is absent on the @@ -1150,27 +1156,27 @@ func TestServedTip_RegressionGuard_HoldsWhenEveryLiveUpstreamLeaves(t *testing.T defer cancel() advance := pinServedTipClock(t) - const legacyCap = int64(21_615_999) + const legacyCap = int64(50_000) network, _ := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ - {id: "prism-celo", chainID: 123, latestBlock: 74_550_978}, - {id: "celo-mainnet-nirvana", chainID: 123, latestBlock: 74_550_974}, - {id: "quicknode-celo", chainID: 123, latestBlock: 74_550_946}, - {id: "celo-mainnet-nirvana-legacy", chainID: 123, latestBlock: 74_550_978, upperExactBlock: legacyCap}, - {id: "quicknode-celo-legacy", chainID: 123, latestBlock: 74_550_978, upperExactBlock: legacyCap}, - }, &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) + {id: "leader", chainID: 123, latestBlock: 100_000}, + {id: "peer-a", chainID: 123, latestBlock: 99_996}, + {id: "peer-b", chainID: 123, latestBlock: 99_968}, + {id: "legacy-1", chainID: 123, latestBlock: 100_000, upperExactBlock: legacyCap}, + {id: "legacy-2", chainID: 123, latestBlock: 100_000, upperExactBlock: legacyCap}, + }, &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority}) healthy := network.EvmHighestLatestBlockNumber(ctx) require.Greater(t, healthy, legacyCap, "precondition: the live heads carry the ballot") beforeHeld := servedTipRegressionCount(network, "latest", "held") beforeOpen := servedTipRegressionCount(network, "latest", "failed_open") - // All three live upstreams drop out at once (a vendor incident, a policy - // exclusion sweep): only the two 2023-capped ones remain eligible. - network.PinUpstreamOrderForTest("celo-mainnet-nirvana-legacy", "quicknode-celo-legacy") + // All three live upstreams drop out at once (a provider outage, a policy + // exclusion sweep): only the two capped ones remain eligible. + network.PinUpstreamOrderForTest("legacy-1", "legacy-2") assert.Equal(t, healthy, network.EvmHighestLatestBlockNumber(ctx), "with no live head left the guard must hold its last corroborated pick, "+ - "not hand `latest` to a September-2023 serving range") + "not hand `latest` to a historical serving range") assert.Equal(t, beforeHeld+1, servedTipRegressionCount(network, "latest", "held")) advance(servedTipRegressionTTL + time.Second) @@ -1204,7 +1210,7 @@ func TestServedTip_RegressionGuard_SingleRogueUpstreamDoesNotDisarmIt(t *testing require.Equal(t, int64(10_000), network.EvmHighestLatestBlockNumber(ctx), "the majority pick ignores the rogue, as it always did") - require.Equal(t, int64(10_000), network.servedLatestAnchor.lastGoodValue.Load(), + require.Equal(t, int64(10_000), network.servedTipAll.latest.lastGoodValue.Load(), "and the guard must be ARMED by that honest pick, not silently disabled by the rogue") // Armed means it still protects: lose the honest majority to a stuck pair. @@ -1264,7 +1270,7 @@ func TestServedTip_RegressionGuard_DoesNotCommitAFantasyPick(t *testing.T) { eligible(id, true) } require.Equal(t, int64(10_000), network.EvmHighestLatestBlockNumber(ctx)) - require.Equal(t, int64(10_000), network.servedLatestAnchor.lastGoodValue.Load()) + require.Equal(t, int64(10_000), network.servedTipAll.latest.lastGoodValue.Load()) // Both rogues return, two honest upstreams drop out: the ballot is // [100M, 100M, 10000] and the majority pick IS the fantasy. @@ -1275,7 +1281,7 @@ func TestServedTip_RegressionGuard_DoesNotCommitAFantasyPick(t *testing.T) { advance(time.Second) require.Equal(t, int64(100_000_000), network.EvmHighestLatestBlockNumber(ctx), "precondition: the majority of this eligible set really is far-future") - assert.Equal(t, int64(10_000), network.servedLatestAnchor.lastGoodValue.Load(), + assert.Equal(t, int64(10_000), network.servedTipAll.latest.lastGoodValue.Load(), "an uncorroborated leap must never become the value the guard holds the network to") eligible("h2", true) @@ -1296,7 +1302,7 @@ func TestServedTip_RegressionGuard_DoesNotCommitAFantasyPick(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() network, eligible, advance := fantasyFleet(t, ctx) - anchor := &network.servedLatestAnchor + anchor := &network.servedTipAll.latest // 1. Armed at the honest head. for _, id := range []string{"h1", "h2", "h3"} { @@ -1355,7 +1361,7 @@ func TestServedTip_RegressionGuard_DoesNotCommitAFantasyPick(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() network, eligible, advance := fantasyFleet(t, ctx) - anchor := &network.servedLatestAnchor + anchor := &network.servedTipAll.latest // The very first evaluation this process ever makes is majority-rogue. eligible("rogue-1", true) @@ -1404,8 +1410,8 @@ func TestServedTip_RegressionGuard_AHoldNeverLowersTheTip(t *testing.T) { // The ballot's own corroborated head is 10_000, so a pick of 5_000 trips the // guard — but the memory (100) is far below the pick. - served := network.guardServedTipRegression("latest", "", &anchor, 5_000, - servedTipReference{Corroborated: 10_000, Max: 10_000}) + served := network.guardServedTipRegression("latest", "all", &anchor, + common.ServedTipPick{Tip: 5_000, Freshest: 10_000, Max: 10_000}, true) assert.GreaterOrEqual(t, served, int64(5_000), "a hold must never serve a block LOWER than the pick it was suppressing") @@ -1431,12 +1437,12 @@ func TestServedTip_RegressionGuard_HealthyPathDoesNotAllocate(t *testing.T) { }) var anchor servedTipAnchor - ref := servedTipReference{Corroborated: 10_000, Max: 10_000} - require.Equal(t, int64(10_000), network.guardServedTipRegression("latest", "", &anchor, 10_000, ref), + ballot := common.ServedTipPick{Tip: 10_000, Freshest: 10_000, Max: 10_000} + require.Equal(t, int64(10_000), network.guardServedTipRegression("latest", "all", &anchor, ballot, true), "arm the guard first; that evaluation is the one allowed to allocate") allocs := testing.AllocsPerRun(200, func() { - network.guardServedTipRegression("latest", "", &anchor, 10_000, ref) + network.guardServedTipRegression("latest", "all", &anchor, ballot, true) }) assert.Zero(t, allocs, "the throttled healthy path must not allocate at all") @@ -1444,11 +1450,11 @@ func TestServedTip_RegressionGuard_HealthyPathDoesNotAllocate(t *testing.T) { // refreshed again. advance(2 * time.Second) before := anchor.lastGoodAt.Load() - network.guardServedTipRegression("latest", "", &anchor, 10_000, ref) + network.guardServedTipRegression("latest", "all", &anchor, ballot, true) assert.NotEqual(t, before, anchor.lastGoodAt.Load(), "the TTL clock still advances") } -// The off switch, symmetric with trajectoryWindow: 0. +// The off switch. func TestServedTip_RegressionGuard_DisabledByNegativeOne(t *testing.T) { util.ResetGock() defer util.ResetGock() @@ -1464,7 +1470,7 @@ func TestServedTip_RegressionGuard_DisabledByNegativeOne(t *testing.T) { {id: "live-3", chainID: 123, latestBlock: 10_000}, {id: "stuck-1", chainID: 123, latestBlock: 5}, {id: "stuck-2", chainID: 123, latestBlock: 5}, - }, &common.EvmServedTipConfig{EnabledFor: []string{"latest"}, MaxRegressionBlocks: -1}) + }, &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority, MaxRegressionBlocks: -1}) require.Equal(t, int64(10_000), network.EvmHighestLatestBlockNumber(ctx)) before := servedTipRegressionCount(network, "latest", "held") @@ -1473,7 +1479,7 @@ func TestServedTip_RegressionGuard_DisabledByNegativeOne(t *testing.T) { assert.Equal(t, int64(5), network.EvmHighestLatestBlockNumber(ctx), "maxRegressionBlocks: -1 means the operator gets the raw pick, poisoned or not") assert.Equal(t, before, servedTipRegressionCount(network, "latest", "held")) - assert.Zero(t, network.servedLatestAnchor.lastGoodValue.Load(), + assert.Zero(t, network.servedTipAll.latest.lastGoodValue.Load(), "a disabled guard records nothing either") } @@ -1502,62 +1508,10 @@ func TestServedTip_LagGaugeNeverReadsNegative(t *testing.T) { "precondition: the guard is holding above the ballot's freshest corroborated view") lag := promUtil.ToFloat64(telemetry.MetricNetworkServedTipLagBlocks. - WithLabelValues(network.projectId, network.Label(), servedTipLaneAll, "latest")) + WithLabelValues(network.projectId, network.Label(), "all", "latest")) assert.GreaterOrEqual(t, lag, float64(0), "the deliberate-lag gauge must never read negative") } -// ─── Long-term trajectory referee ──────────────────────────────────────────── -// -// Every mechanism above is a snapshot of one instant, and a majority stall -// defeats all of them at once: when N upstreams freeze while staying eligible, -// the majority IS the stale group and the 2 upstreams holding the real head are -// outvoted. These tests pin the referee that breaks that tie from the network's -// own 10-minute head trajectory — and, just as importantly, pin how narrowly it -// is allowed to act. - -const ( - // servedTipSampleInterval mirrors the tracker's unexported sample throttle - // (architecture/evm: tipSampleInterval) — the cadence a warm-up must tick - // at for the buffer to fill. - servedTipSampleInterval = 5 * time.Second - - // servedTipMaxSampleGap mirrors architecture/evm's tipMaxSampleGap. - servedTipMaxSampleGap = time.Minute - - // The warm-up track: 100 blocks per 5s sample = 20 blocks/s, a fast-L2 - // cadence that makes one stalled sample worth 100 blocks and keeps the - // scenarios legible. 130 samples ≈ 10m45s, just past the default window. - trajectoryStartHead = int64(1_000_000) - trajectoryBlocksPerSample = int64(100) - trajectoryWarmSamples = 130 -) - -// servedTipTrajectoryCount reads the referee's counter for a network/axis/outcome. -func servedTipTrajectoryCount(n *Network, axis, outcome string) float64 { - return promUtil.ToFloat64(telemetry.MetricNetworkServedTipTrajectoryTotal. - WithLabelValues(n.projectId, n.Label(), servedTipLaneAll, axis, outcome)) -} - -// servedTipIsOverriding reports whether the network-wide latest lane is -// CURRENTLY being overridden by the referee — the state the transition log and -// counter are driven from. -func servedTipIsOverriding(n *Network) bool { - return n.servedLatestAnchor.refereeState.Load() == servedTipRefereeOverriding -} - -// setupTrajectoryNetwork builds a `count`-upstream network with the pinned -// served-tip clock, and returns the clock's advance function. -func setupTrajectoryNetwork(t *testing.T, ctx context.Context, count int, stCfg *common.EvmServedTipConfig) (*Network, []*upstream.Upstream, func(time.Duration)) { - t.Helper() - advance := pinServedTipClock(t) - fixtures := make([]servedTipFixture, count) - for i := range fixtures { - fixtures[i] = servedTipFixture{id: "u" + strconv.Itoa(i+1), chainID: 123, latestBlock: trajectoryStartHead} - } - network, ups := setupServedTipNetworkWith(t, ctx, fixtures, stCfg) - return network, ups, advance -} - // suggestServedTipHead moves an upstream's poller head up to `head` in steps no // larger than the poller's major-jump threshold. A single larger suggestion is // deliberately deferred to an asynchronous chain-identity check @@ -1580,123 +1534,16 @@ func suggestServedTipHead(u *upstream.Upstream, head int64) { } } -// warmServedTipTrajectory drives `samples` real evaluations one sample interval -// apart with every upstream advancing together, so the network's tracker sees a -// clean linear head track through the ordinary request path. Returns the head -// every upstream ends on (the clock sits on the last sample). -func warmServedTipTrajectory(t *testing.T, ctx context.Context, network *Network, ups []*upstream.Upstream, advance func(time.Duration), samples int) int64 { - t.Helper() - head := trajectoryStartHead - for i := 0; i < samples; i++ { - for _, u := range ups { - u.EvmStatePoller().SuggestLatestBlock(head) - } - require.Equal(t, head, network.EvmHighestLatestBlockNumber(ctx), - "warm-up sample %d: an agreeing fleet must serve its agreed head", i) - if i < samples-1 { - advance(servedTipSampleInterval) - head += trajectoryBlocksPerSample - } - } - return head -} - -// stallMajority freezes every upstream except the first `fresh` of them, which -// keep the warm-up track for `samples` more evaluations. Returns the head the -// fresh group ends on and the last served value. -func stallMajority(t *testing.T, ctx context.Context, network *Network, ups []*upstream.Upstream, advance func(time.Duration), head int64, fresh int, samples int) (int64, int64) { - t.Helper() - served := int64(0) - for i := 0; i < samples; i++ { - advance(servedTipSampleInterval) - head += trajectoryBlocksPerSample - for _, u := range ups[:fresh] { - u.EvmStatePoller().SuggestLatestBlock(head) - } - served = network.EvmHighestLatestBlockNumber(ctx) - } - return head, served -} - -// THE headline case, and the requirement the referee exists for: in an N-vs-2 -// split where the 2 sit at the higher block, use the 2. Three upstreams freeze -// inside the lag-exclusion threshold (a shared-counter freeze, a stuck vendor -// region) while two keep the chain's head; the strict majority is now the stale -// group, so the plain majority pick — and every instantaneous rule — serves a -// block the chain left minutes ago. -func TestServedTip_TrajectoryReferee_StalledMajorityLosesToFreshMinority(t *testing.T) { - util.ResetGock() - defer util.ResetGock() - util.SetupMocksForEvmStatePoller() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - network, ups, advance := setupTrajectoryNetwork(t, ctx, 5, &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) - frozen := warmServedTipTrajectory(t, ctx, network, ups, advance, trajectoryWarmSamples) - - beforeOverride := servedTipTrajectoryCount(network, "latest", "override") - beforeHeld := servedTipRegressionCount(network, "latest", "held") - - fresh, served := stallMajority(t, ctx, network, ups, advance, frozen, 2, 24) - - // What the plain majority pick WOULD be over the very same ballot: the - // frozen head, held by 3 of the 5 upstreams. - majority := common.PickServedTip([]common.ServedTipInput{ - {UpstreamID: "u1", BlockNumber: fresh}, - {UpstreamID: "u2", BlockNumber: fresh}, - {UpstreamID: "u3", BlockNumber: frozen}, - {UpstreamID: "u4", BlockNumber: frozen}, - {UpstreamID: "u5", BlockNumber: frozen}, - }).Tip - require.Equal(t, frozen, majority, - "precondition: the stalled group holds the majority, so the plain pick is the stale head") - - assert.Equal(t, fresh, served, - "the corroborated pair that is where the chain should be by now must define the tip; "+ - "the plain majority would have served %d, %d blocks behind", majority, fresh-majority) - assert.Greater(t, servedTipTrajectoryCount(network, "latest", "override"), beforeOverride, - "an override is an intervention and must be counted") - assert.Equal(t, beforeHeld, servedTipRegressionCount(network, "latest", "held"), - "raising the tip to the live pair is not a regression; the guard must stay out of it") -} - -// A routine transient: two of four upstreams fall behind for 15 seconds (a -// poller hiccup, a vendor blip) and catch up. Nothing is wrong with the fleet, -// and the value every upstream can serve is the lagged head — an intervention -// here would advertise a block only half the fleet has, for minutes. This shape -// is why corroboration has to be earned over time rather than in one instant. -func TestServedTip_TrajectoryReferee_TransientHalfFleetLagIsNotAnOverride(t *testing.T) { - util.ResetGock() - defer util.ResetGock() - util.SetupMocksForEvmStatePoller() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - network, ups, advance := setupTrajectoryNetwork(t, ctx, 4, &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) - head := warmServedTipTrajectory(t, ctx, network, ups, advance, trajectoryWarmSamples) - lagged := head - - before := servedTipTrajectoryCount(network, "latest", "override") - - // Three ticks in which only u1/u2 advance: u3/u4 end 300 blocks (15s of this - // 20 blocks/s chain) behind — past the 200-block cluster width, so the fleet - // reads as two groups of two. - fresh, served := stallMajority(t, ctx, network, ups, advance, head, 2, 3) - - assert.Equal(t, lagged, served, - "a 15-second divergence must keep serving the value all four upstreams have (fresh pair was at %d)", fresh) - assert.Equal(t, before, servedTipTrajectoryCount(network, "latest", "override"), - "and must not be counted as an intervention") -} - -// A PERMANENTLY split fleet (the flashblocks shape: two upstreams always ~300 -// blocks ahead, both halves advancing at the same velocity) is a normal -// topology, not a stall. The trajectory is fitted to the MEDIAN of the live -// heads, so no permanently-offset group is ever closer to it than the median's -// own — the referee has nothing to correct and stays silent forever. -func TestServedTip_TrajectoryReferee_PermanentlySplitFleetIsSilent(t *testing.T) { +// ─── Stalled heads are benched ─────────────────────────────────────────────── +// +// The erpc#1253 shape end to end, on the default config, through +// real pollers, the real health tracker and the real Network: two upstreams, +// one stops advancing. With quorum 2 the stalled one would hold +// "latest" (and eth_call interpolation and eth_blockNumber) at its height for +// the whole outage, and as the lag reference it would read lag 0 so routing +// never drops it. The tracker benches it once it has sat below a peer that kept +// moving for its stall window. +func TestServedTip_StalledHeadIsBenched(t *testing.T) { util.ResetGock() defer util.ResetGock() util.SetupMocksForEvmStatePoller() @@ -1704,279 +1551,730 @@ func TestServedTip_TrajectoryReferee_PermanentlySplitFleetIsSilent(t *testing.T) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - network, ups, advance := setupTrajectoryNetwork(t, ctx, 4, &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) - before := servedTipTrajectoryCount(network, "latest", "override") - - head := trajectoryStartHead - for i := 0; i < trajectoryWarmSamples+40; i++ { - for j, u := range ups { - target := head - if j < 2 { - target = head + 300 // the always-ahead pair - } - suggestServedTipHead(u, target) + // stallAfter mirrors the tracker's stall window: three poll periods, never + // under 30s. A poll period is the longer of the poll interval and the + // debounce; an unset debounce follows the network block time, which stays 0 + // in subtests that report no block timestamps. + stallAfter := func(u *upstream.Upstream, blockTime time.Duration) time.Duration { + cfg := u.Config().Evm + debounce := blockTime + if d := cfg.StatePollerDebounce.Duration(); d > 0 { + debounce = d } - network.EvmHighestLatestBlockNumber(ctx) - advance(servedTipSampleInterval) - head += trajectoryBlocksPerSample + return max(30*time.Second, 3*max(cfg.StatePollerInterval.Duration(), debounce)) } - - assert.Equal(t, before, servedTipTrajectoryCount(network, "latest", "override"), - "a permanently split fleet must never trigger an override") -} - -// The referee's intervention EXPIRES on its own. Once the group it elected stops -// advancing, that group can no longer have produced the chain progress the fit -// describes, and the override ends without any operator action — the property -// that makes an upward-only referee unable to pin a tip the way the 2026-06 -// clamp did. -func TestServedTip_TrajectoryReferee_OverrideSelfExpires(t *testing.T) { - util.ResetGock() - defer util.ResetGock() - util.SetupMocksForEvmStatePoller() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - network, ups, advance := setupTrajectoryNetwork(t, ctx, 5, &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) - frozen := warmServedTipTrajectory(t, ctx, network, ups, advance, trajectoryWarmSamples) - fresh, served := stallMajority(t, ctx, network, ups, advance, frozen, 2, 24) - require.Equal(t, fresh, served, "precondition: the referee is holding the tip on the corroborated pair") - - require.True(t, servedTipIsOverriding(network), "precondition: the referee reports itself as overriding") - - // Now the whole chain stops: nobody advances, including the elected pair. - for i := 0; i < 240; i++ { - advance(servedTipSampleInterval) - network.EvmHighestLatestBlockNumber(ctx) - if !servedTipIsOverriding(network) { - t.Logf("override expired after %v of a fully halted chain", time.Duration(i+1)*servedTipSampleInterval) - return + // report mirrors one production poll: the tracker gets the block and its + // timestamp first, then the poller's counter (whose OnValue reports ts 0 + // and changes nothing). The counter moves one block at a time so no step + // is a major head move deferred to an asynchronous chain-identity check. + report := func(tr *health.Tracker, u *upstream.Upstream, head, ts int64) { + tr.SetLatestBlockNumber(u, head, ts) + for b := u.EvmStatePoller().LatestBlock() + 1; b <= head; b++ { + u.EvmStatePoller().SuggestLatestBlock(b) } } - t.Fatalf("the override never expired over %v of a halted chain", 240*servedTipSampleInterval) -} - -// The inverse split, and the reason a group is elected by its DISTANCE to the -// trajectory rather than by its freshness: two upstreams running far past where -// the chain can possibly be (wrong chain, garbage endpoint) must lose to the -// healthy group, however corroborated they are with each other. -func TestServedTip_TrajectoryReferee_RunawayMinorityLoses(t *testing.T) { - util.ResetGock() - defer util.ResetGock() - util.SetupMocksForEvmStatePoller() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // maxRegressionBlocks is widened so the branch's regression guard (which - // measures against the RAW max live head, and would hold the tip the moment - // any upstream runs 1024 blocks ahead) stays out of the way: what is under - // test here is the referee's own election. - network, ups, advance := setupTrajectoryNetwork(t, ctx, 5, &common.EvmServedTipConfig{ - EnabledFor: []string{"latest"}, - MaxRegressionBlocks: 100_000, - }) - head := warmServedTipTrajectory(t, ctx, network, ups, advance, trajectoryWarmSamples) - - beforeOverride := servedTipTrajectoryCount(network, "latest", "override") - beforeFallback := servedTipTrajectoryCount(network, "latest", "fallback") - - // Two upstreams run 20k blocks (≈16 minutes of this chain) ahead; the other - // three stay exactly on the track. - advance(servedTipSampleInterval) - head += trajectoryBlocksPerSample - for _, u := range ups { - suggestServedTipHead(u, head) + lagOf := func(n *Network, u *upstream.Upstream) int64 { + return n.metricsTracker.GetUpstreamMethodMetrics(u, "*", common.DataFinalityStateAll).BlockHeadLag.Load() } - for _, u := range ups[:2] { - suggestServedTipHead(u, head+20_000) + interpolatedLatest := func(n *Network) int64 { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0x0000000000000000000000000000000000000001","data":"0x"},"latest"]}`)) + req.SetNetwork(n) + jrq, err := req.JsonRpcRequest() + require.NoError(t, err) + evm.NormalizeHttpJsonRpc(ctx, req, jrq) + blk, ok := jrq.Params[1].(string) + require.True(t, ok, "latest must be interpolated to a hex block, got %v", jrq.Params[1]) + v, err := common.HexToInt64(blk) + require.NoError(t, err) + return v + } + blockNumber := func(n *Network) int64 { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}`)) + req.SetNetwork(n) + req.SetDirectives(&common.RequestDirectives{EnforceHighestBlock: true}) + resp, err := n.Forward(ctx, req) + require.NoError(t, err) + require.Nil(t, resp.Upstream(), "eth_blockNumber must be answered from the served head without dispatching") + _, bn, err := evm.ExtractBlockReferenceFromResponse(ctx, resp) + require.NoError(t, err) + return bn + } + // served asserts every reader of "latest" agrees on `want`. + served := func(t *testing.T, n *Network, want int64, msg string) { + t.Helper() + require.Equal(t, want, n.EvmHighestLatestBlockNumber(ctx), "latest: %s", msg) + require.Equal(t, want, interpolatedLatest(n), "eth_call latest interpolation: %s", msg) + require.Equal(t, want, blockNumber(n), "eth_blockNumber: %s", msg) } - assert.Equal(t, head, network.EvmHighestLatestBlockNumber(ctx), - "the runaway pair is nowhere near the trajectory; the healthy majority keeps the tip") - assert.Equal(t, beforeOverride, servedTipTrajectoryCount(network, "latest", "override"), - "no override happened") - assert.Equal(t, beforeFallback, servedTipTrajectoryCount(network, "latest", "fallback"), - "the healthy group is elected outright, so this is not even a near miss") -} + t.Run("stall, bench, replay, rejoin", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "live", chainID: 123, latestBlock: 1_000}, + {id: "stalled", chainID: 123, latestBlock: 1_000}, + }, nil) + live, stalled := ups[0], ups[1] + tr := network.metricsTracker + finalizedOf := func() int64 { return network.EvmHighestFinalizedBlockNumber(ctx) } + + // Both report finalized heads; stalled's stays frozen with its head. + const stalledFinalized = int64(990) + setServedTipFinalized(t, live, stalledFinalized) + setServedTipFinalized(t, stalled, stalledFinalized) + + // Healthy: both move together. + head := int64(1_000) + for range 5 { + advance(2 * time.Second) + head += 2 + live.EvmStatePoller().SuggestLatestBlock(head) + stalled.EvmStatePoller().SuggestLatestBlock(head) + } + served(t, network, head, "a healthy pair") + stalledAt := head + + // stalled stops advancing; live keeps moving, finalized included. + // Inside the stall window nothing changes: quorum 2 still serves the + // lower heads on both axes. + window := stallAfter(stalled, 0) + for elapsed := time.Duration(0); elapsed+2*time.Second < window; elapsed += 2 * time.Second { + advance(2 * time.Second) + head += 2 + live.EvmStatePoller().SuggestLatestBlock(head) + setServedTipFinalized(t, live, head-10) + } + require.False(t, tr.IsHeadBenched(stalled), "a peer below for less than its stall window is not benched") + served(t, network, stalledAt, "inside the stall window") + require.Equal(t, stalledFinalized, finalizedOf(), "finalized: quorum 2 holds the stalled one's finalized inside the stall window") + require.Zero(t, lagOf(network, stalled), "inside the stall window the lower head is still the lag reference") + + // Past the stall window, the next head live reports benches stalled. + advance(4 * time.Second) + head += 2 + live.EvmStatePoller().SuggestLatestBlock(head) + setServedTipFinalized(t, live, head-10) + require.True(t, tr.IsHeadBenched(stalled), "a head below a moving peer for its stall window is benched") + require.False(t, tr.IsHeadBenched(live)) + served(t, network, head, "a benched head leaves the ballot") + require.Equal(t, head-10, finalizedOf(), "finalized: a benched head leaves the finalized ballot too") + require.Equal(t, head-stalledAt, lagOf(network, stalled), + "the benched head is measured against the live one, so blockNumberLagAbove can route around it") + require.Zero(t, lagOf(network, live)) + + // stalled comes back and replays: moving, but behind. It stays benched + // and "latest" never drops back to it. + last := head + for replay := stalledAt; replay < head-10; replay += 20 { + advance(time.Second) + head++ + live.EvmStatePoller().SuggestLatestBlock(head) + stalled.EvmStatePoller().SuggestLatestBlock(replay) + require.True(t, tr.IsHeadBenched(stalled), "a replaying head stays benched until it catches up (at %d, live %d)", replay, head) + got := network.EvmHighestLatestBlockNumber(ctx) + require.GreaterOrEqual(t, got, last, "latest must never regress while a benched head replays") + last = got + } + served(t, network, head, "while the benched head replays") -// Warm-up inertness: until the tracker has a window's worth of history the -// referee is a no-op and the network behaves exactly as it does without it. -// This is also the boot behaviour of every pod after every deploy. -func TestServedTip_TrajectoryReferee_ColdProcessIsInert(t *testing.T) { - util.ResetGock() - defer util.ResetGock() - util.SetupMocksForEvmStatePoller() + // stalled catches up on both axes: it rejoins and its lag is 0 again. + advance(time.Second) + stalled.EvmStatePoller().SuggestLatestBlock(head) + require.False(t, tr.IsHeadBenched(stalled), "a benched head that caught up rejoins") + require.Zero(t, lagOf(network, stalled)) + served(t, network, head, "after the rejoin") + setServedTipFinalized(t, live, head-10) + setServedTipFinalized(t, stalled, head-10) + require.Equal(t, head-10, finalizedOf(), "finalized after the rejoin") + }) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + t.Run("chain halt benches nobody", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "a", chainID: 123, latestBlock: 2_000}, + {id: "b", chainID: 123, latestBlock: 2_000}, + }, nil) + tr := network.metricsTracker + // Every poll returns the same head, with the same old block timestamp, + // for ten minutes. + haltedTs := servedTipClock().Add(-10 * time.Minute).Unix() + for range 60 { + advance(10 * time.Second) + for _, u := range ups { + report(tr, u, 2_000, haltedTs) + } + } + for _, u := range ups { + require.False(t, tr.IsHeadBenched(u), "%s: nobody moved, so nobody is below a moving peer", u.Id()) + require.Zero(t, lagOf(network, u)) + } + served(t, network, 2_000, "a halted chain") + }) - network, ups, advance := setupTrajectoryNetwork(t, ctx, 5, &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) - // Half a window: enough samples for a fit, nowhere near enough span. - frozen := warmServedTipTrajectory(t, ctx, network, ups, advance, trajectoryWarmSamples/2) + t.Run("static far-future upstream is never served, benches nobody, and never blocks a rejoin", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "honest-a", chainID: 123, latestBlock: 3_000}, + {id: "honest-b", chainID: 123, latestBlock: 3_000}, + {id: "rogue", chainID: 123, latestBlock: 3_000}, + }, nil) + a, b, rogue := ups[0], ups[1], ups[2] + tr := network.metricsTracker + suggestServedTipHead(rogue, 103_000) + require.Equal(t, int64(103_000), rogue.EvmStatePoller().LatestBlock()) + + head := int64(3_000) + for range 60 { + advance(10 * time.Second) + head++ + a.EvmStatePoller().SuggestLatestBlock(head) + b.EvmStatePoller().SuggestLatestBlock(head) + tr.SetLatestBlockNumber(rogue, 103_000, 0) + require.False(t, tr.IsHeadBenched(a), "a static head above never benches the moving ones") + require.False(t, tr.IsHeadBenched(b), "a static head above never benches the moving ones") + } + require.False(t, tr.IsHeadBenched(rogue)) + require.Zero(t, lagOf(network, a), "the rogue is not the lag reference") + served(t, network, head, "a lone far-future head is never served") + + // b stalls past its window and is benched; the rogue stays a voter. + for elapsed := time.Duration(0); elapsed <= stallAfter(b, 0); elapsed += 10 * time.Second { + advance(10 * time.Second) + head++ + a.EvmStatePoller().SuggestLatestBlock(head) + } + require.True(t, tr.IsHeadBenched(b)) + served(t, network, head, "quorum 2 over {a, rogue} still serves the honest head") - before := servedTipTrajectoryCount(network, "latest", "override") - _, served := stallMajority(t, ctx, network, ups, advance, frozen, 2, 24) + // b catches up with a. The rogue's head is out of anyone's reach, so it + // must not be the rejoin bar. + advance(time.Second) + b.EvmStatePoller().SuggestLatestBlock(head) + require.False(t, tr.IsHeadBenched(b), "a caught-up head rejoins even with a far-future voter above it") + require.Zero(t, lagOf(network, b)) + served(t, network, head, "after the rejoin") + }) - assert.Equal(t, frozen, served, - "a cold tracker must leave the plain majority pick untouched") - assert.Equal(t, before, servedTipTrajectoryCount(network, "latest", "override")) -} + t.Run("moving far-future upstream never benches a stalled honest one", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "honest", chainID: 123, latestBlock: 4_000}, + {id: "rogue", chainID: 123, latestBlock: 4_000}, + }, nil) + honest, rogue := ups[0], ups[1] + tr := network.metricsTracker + rogueHead := int64(104_000) + suggestServedTipHead(rogue, rogueHead) + + // honest's poller dies for well past its stall window while the + // wrong-chain head keeps advancing: a lead no chain produced in that + // time is another chain, not a stall. + for range 30 { + advance(10 * time.Second) + rogueHead += 5 + tr.SetLatestBlockNumber(rogue, rogueHead, 0) + } + require.False(t, tr.IsHeadBenched(honest), "an implausible lead must not bench the honest head") + require.Zero(t, lagOf(network, honest), "the rogue is not the lag reference") + served(t, network, 4_000, "a lone moving far-future head is never served") + }) -// Staleness: the fit describes the present only if the track is continuous. A -// gap larger than the tracker's tolerance is exactly where a halt or a burst -// hides, so extrapolating across one is guessing — and the referee stands down. -// The paired sub-tests differ ONLY in the size of that gap. -func TestServedTip_TrajectoryReferee_StaleTrackFailsOpen(t *testing.T) { - // After `gap` of silence the fresh pair returns exactly on the trajectory - // (20 blocks/s) and then HOLDS it for longer than the dwell — i.e. the - // referee's evidence is otherwise perfect, and the only difference between - // the sub-tests is whether the hole in the track is inside the tolerance. - run := func(t *testing.T, gap time.Duration) (served, frozen, fresh int64, overrides float64) { - util.ResetGock() - defer util.ResetGock() - util.SetupMocksForEvmStatePoller() + t.Run("a process that starts mid-stall benches the stalled head", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "live", chainID: 123, latestBlock: 47_880_000}, + {id: "stalled-a", chainID: 123, latestBlock: 1_800_000}, + {id: "stalled-b", chainID: 123, latestBlock: 1_800_000}, + }, nil) + live, stalls := ups[0], ups[1:] + tr := network.metricsTracker + const stalledAt = int64(1_800_000) + // The stalled heads' block was produced two years before this process. + // Each has two non-benched peers and only live is younger, so the time + // rule never fires: only the height rule can bench them, and with the + // network block time unknown (the corroborated head is the stalled + // one) that rule needs live's own block time. + stalledTs := servedTipClock().Add(-2 * 365 * 24 * time.Hour).Unix() + head := int64(47_880_000) + poll := func(d time.Duration) { + advance(d) + for _, u := range stalls { + report(tr, u, stalledAt, stalledTs) + } + head += int64(d / (2 * time.Second)) + report(tr, live, head, servedTipClock().Unix()) + } + for _, u := range stalls { + report(tr, u, stalledAt, stalledTs) + } + served(t, network, stalledAt, "at boot quorum 2 serves the stalled head") - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + // Three timestamped head changes: live's block time is not known yet. + for range 3 { + poll(2 * time.Second) + } + for _, u := range ups { + require.False(t, tr.IsHeadBenched(u), "%s: nobody is benched before live's block time is known", u.Id()) + } + served(t, network, stalledAt, "before live's block time is known") - network, ups, advance := setupTrajectoryNetwork(t, ctx, 5, &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) - frozen = warmServedTipTrajectory(t, ctx, network, ups, advance, trajectoryWarmSamples) - before := servedTipTrajectoryCount(network, "latest", "override") + // The fourth settles it; the stall window runs from there. + poll(2 * time.Second) + window := stallAfter(stalls[0], 0) + for elapsed := time.Duration(0); elapsed+2*time.Second < window; elapsed += 2 * time.Second { + poll(2 * time.Second) + } + for _, u := range stalls { + require.False(t, tr.IsHeadBenched(u), "%s: not benched inside its stall window", u.Id()) + } + served(t, network, stalledAt, "inside the stall window") - advance(gap) - fresh = frozen + trajectoryBlocksPerSample*int64(gap/servedTipSampleInterval) - for _, u := range ups[:2] { - suggestServedTipHead(u, fresh) + poll(4 * time.Second) + for _, u := range stalls { + require.True(t, tr.IsHeadBenched(u), "%s: a head frozen since before the process started is benched after its stall window", u.Id()) } - served = network.EvmHighestLatestBlockNumber(ctx) + require.False(t, tr.IsHeadBenched(live)) + served(t, network, head, "the stalled heads leave the ballot") + require.Equal(t, head-stalledAt, lagOf(network, stalls[0]), "the benched head is measured against the live one") + require.Zero(t, lagOf(network, live)) + }) - // Eight more samples (40s) of the pair tracking the chain: enough dwell - // for a group that is genuinely where the head should be. - for i := 0; i < 8; i++ { - advance(servedTipSampleInterval) - fresh += trajectoryBlocksPerSample - for _, u := range ups[:2] { - suggestServedTipHead(u, fresh) + t.Run("a node replaying far behind the tip is benched while it moves", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "live", chainID: 123, latestBlock: 100_000}, + {id: "replaying", chainID: 123, latestBlock: 99_400}, + }, nil) + live, replaying := ups[0], ups[1] + tr := network.metricsTracker + // A 2s chain. replaying restarted from a snapshot 600 blocks (20 min) + // old and replays 4 blocks per 2s, so it moves on every poll. Once the + // block time is known, every other move of each upstream arrives + // without its block timestamp, as a peer pod's poll does through + // shared state. + start := servedTipClock().Unix() + tsOf := func(n int64) int64 { return start + (n-100_000)*2 } + head, behind := int64(100_000), int64(99_400) + polls := 0 + poll := func(d time.Duration) { + advance(d) + polls++ + ts := func(n int64) int64 { + if polls > 4 && polls%2 == 0 { + return 0 + } + return tsOf(n) } - served = network.EvmHighestLatestBlockNumber(ctx) + behind = min(behind+int64(d/(500*time.Millisecond)), head) + report(tr, replaying, behind, ts(behind)) + head += int64(d / (2 * time.Second)) + report(tr, live, head, ts(head)) } - return served, frozen, fresh, servedTipTrajectoryCount(network, "latest", "override") - before - } + report(tr, replaying, behind, tsOf(behind)) + report(tr, live, head, tsOf(head)) + served(t, network, behind, "at boot quorum 2 serves the replaying head") - t.Run("ContinuousTrackOverrides", func(t *testing.T) { - served, _, fresh, overrides := run(t, servedTipMaxSampleGap-5*time.Second) - assert.Equal(t, fresh, served, "an unbroken track lets the referee act") - assert.Positive(t, overrides) - }) + window := stallAfter(replaying, 2*time.Second) + for elapsed := time.Duration(0); elapsed+2*time.Second < window; elapsed += 2 * time.Second { + poll(2 * time.Second) + } + require.False(t, tr.IsHeadBenched(replaying), "not benched inside its stall window") + served(t, network, behind, "inside the stall window quorum 2 serves the replaying head") + + // Its own moves never reset the window: its head block stays a stall + // window older than live's. + poll(4 * time.Second) + require.True(t, tr.IsHeadBenched(replaying), "a head replaying far behind a live peer for its stall window is benched") + require.False(t, tr.IsHeadBenched(live)) + served(t, network, head, "the replaying head leaves the ballot") + require.Equal(t, head-behind, lagOf(network, replaying), "a benched head that moves lags by its real distance") + require.Greater(t, lagOf(network, replaying), int64(16), "past the default blockNumberLagAbove, so routing leaves it") + + // It gains 3 blocks a poll. It rejoins on the report that reaches live's + // head, not before, and "latest" never drops back to it meanwhile. + last := head + for polls := 0; tr.IsHeadBenched(replaying); polls++ { + require.Less(t, polls, 400, "a replaying head rejoins once it catches up") + poll(2 * time.Second) + got := network.EvmHighestLatestBlockNumber(ctx) + require.GreaterOrEqual(t, got, last, "latest never regresses while it replays") + last = got + } + require.Equal(t, head-1, behind, "it rejoins on reaching live's head; live then moved one on") - t.Run("GappedTrackFailsOpen", func(t *testing.T) { - served, frozen, _, overrides := run(t, servedTipMaxSampleGap+time.Second) - assert.Equal(t, frozen, served, - "one sample past the gap tolerance the same evidence must be refused") - assert.Zero(t, overrides) + // Back at the tip it trails live by one block, far less than its stall + // window of chain time: it votes again and is never benched. + for range 3 * int(window/(2*time.Second)) { + poll(2 * time.Second) + require.False(t, tr.IsHeadBenched(replaying), "a node one block behind is never benched (at %d, live %d)", behind, head) + } + require.Equal(t, head-1, behind) + require.Zero(t, lagOf(network, replaying), "the lower voter is the lag reference again") + served(t, network, behind, "quorum 2 serves the lower of two heads at the tip") }) -} -// A chain that pauses and bursts (an L2 landing batches) has a large residual -// spread of its own, which widens the referee's tolerance automatically — and a -// legitimate fleet-wide jump keeps every head in ONE cluster, where the referee -// has nothing to choose between. Neither may produce a false override. -func TestServedTip_TrajectoryReferee_PauseThenBurstDoesNotFalsePositive(t *testing.T) { - util.ResetGock() - defer util.ResetGock() - util.SetupMocksForEvmStatePoller() + t.Run("a staleness verdict ends when the node catches up, so a plausible rogue cannot keep it", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "live", chainID: 123, latestBlock: 50_000}, + {id: "trailing", chainID: 123, latestBlock: 49_940}, + {id: "rogue", chainID: 123, latestBlock: 50_000}, + }, nil) + live, trailing, rogue := ups[0], ups[1], ups[2] + tr := network.metricsTracker + // A 2s chain. trailing keeps pace 60 blocks (120s) back, more than its + // 90s stall window of chain time, so its own timestamps prove it stale. + start := servedTipClock().Unix() + tsOf := func(n int64) int64 { return start + (n-50_000)*2 } + head := int64(50_000) + for range 5 { + advance(2 * time.Second) + head++ + report(tr, trailing, head-60, tsOf(head-60)) + report(tr, live, head, tsOf(head)) + report(tr, rogue, head, tsOf(head)) + } + require.False(t, tr.IsHeadBenched(trailing), "not benched inside its stall window") + + // It catches up through a peer pod's poll (no timestamp). Then the + // rogue claims a head 60 blocks above the tip and keeps moving: a lead + // a stall window of blocks ahead, but no proof trailing is stale. + advance(2 * time.Second) + head++ + report(tr, trailing, head, 0) + report(tr, live, head, tsOf(head)) + report(tr, rogue, head, tsOf(head)) + for range 3 * int(stallAfter(trailing, 2*time.Second)/(2*time.Second)) { + advance(2 * time.Second) + head++ + report(tr, trailing, head, 0) + report(tr, live, head, tsOf(head)) + report(tr, rogue, head+60, tsOf(head+60)) + require.False(t, tr.IsHeadBenched(trailing), "a node at the tip is never benched by a verdict it disproved (head %d)", head) + } + require.Zero(t, lagOf(network, trailing)) + served(t, network, head, "the rogue's lone head above the tip is never served") + }) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + t.Run("a benched replay that reaches a voter as stale as itself stays benched", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "live", chainID: 123, latestBlock: 100_000}, + {id: "restored", chainID: 123, latestBlock: 99_200}, + {id: "peer-replay", chainID: 123, latestBlock: 99_300}, + }, nil) + live, restored, peerReplay := ups[0], ups[1], ups[2] + tr := network.metricsTracker + // A 2s chain. Two nodes replay from old snapshots. restored gains 4 + // blocks per 2s with its own block timestamps, so it is proven stale + // and benched. peer-replay gains 3, and each of its moves reaches this + // process without a timestamp (a peer pod's poll), so nothing proves + // it stale: it stays a voter and sets the bar restored must reach. + start := servedTipClock().Unix() + tsOf := func(n int64) int64 { return start + (n-100_000)*2 } + head, r, p := int64(100_000), int64(99_200), int64(99_300) + window := stallAfter(restored, 2*time.Second) + last, wasBenched := int64(0), false + for polls := 0; !wasBenched || tr.IsHeadBenched(restored); polls++ { + require.Less(t, polls, 1000, "restored rejoins once it replays to the tip") + advance(2 * time.Second) + r = min(r+4, head) + report(tr, restored, r, tsOf(r)) + p = min(p+3, head) + report(tr, peerReplay, p, 0) + head++ + report(tr, live, head, tsOf(head)) + wasBenched = wasBenched || tr.IsHeadBenched(restored) + if wasBenched && (head-r)*2 >= 2*int64(window/time.Second) { + require.True(t, tr.IsHeadBenched(restored), "a benched head two stall windows behind the tip stays benched, even once it reaches a stale voter (restored %d, peer-replay %d, live %d)", r, p, head) + } + got := network.EvmHighestLatestBlockNumber(ctx) + require.GreaterOrEqual(t, got, last, "latest never regresses (restored %d, peer-replay %d, live %d)", r, p, head) + last = got + } + require.Less(t, head-r, int64(window/(2*time.Second))+2, "it rejoins once within a stall window of chain time of the tip") + served(t, network, r, "after the rejoin quorum 2 serves the restored head") + }) - network, ups, advance := setupTrajectoryNetwork(t, ctx, 5, &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) + t.Run("heads frozen above a chain that restarted lower are benched and routed around", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "fork-a", chainID: 123, latestBlock: 9_060}, + {id: "fork-b", chainID: 123, latestBlock: 9_060}, + {id: "canon-a", chainID: 123, latestBlock: 9_000}, + {id: "canon-b", chainID: 123, latestBlock: 9_000}, + }, nil) + forks, canons := ups[:2], ups[2:] + tr := network.metricsTracker + const forkAt = int64(9_060) + forkTs := servedTipClock().Add(-10 * time.Minute).Unix() + canon := int64(9_000) + poll := func(d time.Duration) { + advance(d) + for _, u := range forks { + report(tr, u, forkAt, forkTs) + } + canon += int64(d / (2 * time.Second)) + for _, u := range canons { + report(tr, u, canon, servedTipClock().Unix()) + } + } - // Same average velocity as the linear warm-up, delivered in batches: nine - // samples of nothing, then 1000 blocks at once. - head := trajectoryStartHead - for i := 0; i < trajectoryWarmSamples; i++ { - if i%10 == 9 { - head += trajectoryBlocksPerSample * 10 + // The corroborated head is a frozen fork's, so the network block time + // stays unknown. + window := stallAfter(forks[0], 0) + for elapsed := time.Duration(0); elapsed+2*time.Second < window; elapsed += 2 * time.Second { + poll(2 * time.Second) } - for _, u := range ups { - u.EvmStatePoller().SuggestLatestBlock(head) + for _, u := range forks { + require.False(t, tr.IsHeadBenched(u), "%s: not benched inside its stall window", u.Id()) } - require.Equal(t, head, network.EvmHighestLatestBlockNumber(ctx)) - advance(servedTipSampleInterval) - } - - beforeOverride := servedTipTrajectoryCount(network, "latest", "override") - beforeFallback := servedTipTrajectoryCount(network, "latest", "fallback") - - // The whole fleet lands the next batch, with ordinary propagation skew. - head += 5_000 - for i, u := range ups { - suggestServedTipHead(u, head-int64(i%2)*50) - } - served := network.EvmHighestLatestBlockNumber(ctx) + served(t, network, forkAt, "inside the stall window quorum 2 still serves the frozen pair") + + // The forks re-report first in each poll, so the bench lands before the + // canonical pair moves: the canonical head at bench time is `canon` now. + benchedAt := canon + poll(4 * time.Second) + for _, u := range forks { + require.True(t, tr.IsHeadBenched(u), "%s: a head whose block is far older than its moving peers' is benched", u.Id()) + } + for _, u := range canons { + require.False(t, tr.IsHeadBenched(u), "%s", u.Id()) + require.Zero(t, lagOf(network, u), "%s: the canonical pair is the lag reference", u.Id()) + } + served(t, network, canon, "both forks benched: the canonical pair defines latest") - assert.Equal(t, head, served, - "a legitimate fleet-wide burst is one cluster; the majority pick stands") - assert.Equal(t, beforeOverride, servedTipTrajectoryCount(network, "latest", "override")) - assert.Equal(t, beforeFallback, servedTipTrajectoryCount(network, "latest", "fallback"), - "a fleet in one cluster does not even register a disagreement") -} + // The canonical chain passes the frozen height. The frozen head still + // lags by what the chain made since its bench, not by the one block it + // now trails: its blocks up to forkAt are the dead fork's. + for canon <= forkAt { + poll(2 * time.Second) + } + for _, u := range forks { + require.True(t, tr.IsHeadBenched(u), "%s: a frozen head never rejoins", u.Id()) + } + lag := lagOf(network, forks[0]) + require.Equal(t, canon-benchedAt, lag, "a frozen benched head lags by what the chain made since its bench") + require.Greater(t, lag, int64(16), "past the default blockNumberLagAbove, so routing leaves the frozen head") + served(t, network, canon, "as the canonical chain moves on") + }) -// Corroboration, not agreement with the model: a single upstream sitting -// exactly where the trajectory says the head is proves nothing — it is the one -// endpoint that could be wrong in the same direction as everyone else being -// stuck. A group of one is never electable. -func TestServedTip_TrajectoryReferee_SingletonIsNotCorroboration(t *testing.T) { - util.ResetGock() - defer util.ResetGock() - util.SetupMocksForEvmStatePoller() + t.Run("benching a pair frozen far above the chain trips no regression hold", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "fork-a", chainID: 123, latestBlock: 20_000}, + {id: "fork-b", chainID: 123, latestBlock: 20_000}, + {id: "canon-a", chainID: 123, latestBlock: 9_000}, + {id: "canon-b", chainID: 123, latestBlock: 8_999}, + }, nil) + forks, canons := ups[:2], ups[2:] + tr := network.metricsTracker + forkTs := servedTipClock().Add(-10 * time.Minute).Unix() + canon := int64(9_000) + poll := func(d time.Duration) { + advance(d) + for _, u := range forks { + report(tr, u, 20_000, forkTs) + } + canon += int64(d / (2 * time.Second)) + // The canonical pair stays one block apart, so its highest head is + // not the quorum pick. + report(tr, canons[0], canon, servedTipClock().Unix()) + report(tr, canons[1], canon-1, servedTipClock().Unix()) + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + window := stallAfter(forks[0], 0) + for elapsed := time.Duration(0); elapsed+2*time.Second < window; elapsed += 2 * time.Second { + poll(2 * time.Second) + } + served(t, network, 20_000, "inside the stall window the frozen pair carries quorum 2, and the guard remembers it") + held := servedTipRegressionCount(network, "latest", "held") + failedOpen := servedTipRegressionCount(network, "latest", "failed_open") - network, ups, advance := setupTrajectoryNetwork(t, ctx, 5, &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) - frozen := warmServedTipTrajectory(t, ctx, network, ups, advance, trajectoryWarmSamples) + poll(4 * time.Second) + for _, u := range forks { + require.True(t, tr.IsHeadBenched(u), "%s", u.Id()) + } + served(t, network, canon-1, "the live voters agree with the pick: the guard must not hold the frozen pair's height, nor serve canon-a's head alone") + for end := servedTipClock().Add(servedTipRegressionTTL + 2*time.Second); servedTipClock().Before(end); { + poll(2 * time.Second) + served(t, network, canon-1, "while the remembered height ages out") + } + assert.Equal(t, held, servedTipRegressionCount(network, "latest", "held"), "a bench is not a regression") + assert.Equal(t, failedOpen, servedTipRegressionCount(network, "latest", "failed_open"), "a guard that never held does not fail open") + }) - before := servedTipTrajectoryCount(network, "latest", "override") - _, served := stallMajority(t, ctx, network, ups, advance, frozen, 1, 24) + t.Run("one moving wrong-chain peer cannot bench a halted fleet by block age", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "a", chainID: 123, latestBlock: 5_000}, + {id: "b", chainID: 123, latestBlock: 5_000}, + {id: "rogue", chainID: 123, latestBlock: 1_000}, + }, nil) + a, b, rogue := ups[0], ups[1], ups[2] + tr := network.metricsTracker + haltedTs := servedTipClock().Add(-10 * time.Minute).Unix() + rogueHead := int64(1_000) + for range 300 { + advance(2 * time.Second) + report(tr, a, 5_000, haltedTs) + report(tr, b, 5_000, haltedTs) + rogueHead++ + report(tr, rogue, rogueHead, servedTipClock().Unix()) + require.False(t, tr.IsHeadBenched(a), "one younger peer is not enough while two peers exist") + require.False(t, tr.IsHeadBenched(b), "one younger peer is not enough while two peers exist") + } + require.Zero(t, lagOf(network, a)) + require.Zero(t, lagOf(network, b)) + served(t, network, 5_000, "a halted fleet beside one moving wrong-chain peer") + }) - assert.Equal(t, frozen, served, - "one on-trajectory witness cannot move the tip, however well it fits") - assert.Equal(t, before, servedTipTrajectoryCount(network, "latest", "override")) -} + t.Run("an endpoint serving another chain with the same chain id is benched by block age", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "chain", chainID: 123, latestBlock: 47_900_000}, + {id: "other", chainID: 123, latestBlock: 1_823_000}, + }, nil) + chain, other := ups[0], ups[1] + tr := network.metricsTracker + // Two chains share the chain id: ours makes a block every 2s, the other + // (an on-demand rollup testnet) one every 5 minutes, both with fresh + // timestamps. 46M blocks apart, no lead is plausible, so only block + // age can tell them apart. + const gap = 5 * time.Minute + window := stallAfter(other, 0) + require.Less(t, 2*window, gap, "the other chain's gap must outlast its bench delay") + head, otherHead := int64(47_900_000), int64(1_823_000) + report(tr, chain, head, servedTipClock().Unix()) + report(tr, other, otherHead, servedTipClock().Unix()) + served(t, network, otherHead, "until block age tells the chains apart, quorum 2 serves the lower head") + + for elapsed := time.Duration(0); elapsed < gap; elapsed += 2 * time.Second { + advance(2 * time.Second) + head++ + report(tr, chain, head, servedTipClock().Unix()) + } + require.True(t, tr.IsHeadBenched(other), "a head whose block stays a stall window older than its peer's is benched") + require.False(t, tr.IsHeadBenched(chain)) + served(t, network, head, "the chain that keeps producing blocks defines latest") + require.Greater(t, lagOf(network, other), int64(16), "past the default blockNumberLagAbove, so routing leaves it") + + // Its next block never brings it back: it cannot reach our head. + advance(2 * time.Second) + head++ + report(tr, other, otherHead+1, servedTipClock().Unix()) + report(tr, chain, head, servedTipClock().Unix()) + require.True(t, tr.IsHeadBenched(other)) + served(t, network, head, "after the other chain's next block") + }) -// Composition with the regression guard: the referee runs first and the guard -// judges whatever comes out of it. Once a stall has handed the tip to the fresh -// pair, losing that pair from the eligible set drops the majority pick far below -// the best live head — and the guard holds the last corroborated value exactly -// as it does for any other poisoned ballot. -func TestServedTip_TrajectoryReferee_ComposesWithRegressionGuard(t *testing.T) { - util.ResetGock() - defer util.ResetGock() - util.SetupMocksForEvmStatePoller() + t.Run("a static head at a slow chain's tip is not benched while peers catch up", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "tip", chainID: 123, latestBlock: 6_000}, + {id: "c1", chainID: 123, latestBlock: 5_000}, + {id: "c2", chainID: 123, latestBlock: 5_000}, + }, nil) + tip, catchers := ups[0], ups[1:] + tr := network.metricsTracker + start := servedTipClock() + tipTs := start.Add(-50 * time.Second).Unix() + // A 60s-block chain: the catchers replay blocks produced ~20h ago. + tsOf := func(n int64) int64 { return start.Add(-20*time.Hour).Unix() + (n-5_000)*60 } + c := int64(5_000) + for range 30 { + advance(10 * time.Second) + report(tr, tip, 6_000, tipTs) + c += 5 + for _, u := range catchers { + report(tr, u, c, tsOf(c)) + } + require.False(t, tr.IsHeadBenched(tip), "peers below with older blocks never prove the tip stalled (at %d)", c) + } + served(t, network, c, "quorum 2 serves the catching-up pair") + }) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + t.Run("quorum majority: three of five stall", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "live-a", chainID: 123, latestBlock: 7_000}, + {id: "live-b", chainID: 123, latestBlock: 7_000}, + {id: "stalled-a", chainID: 123, latestBlock: 7_000}, + {id: "stalled-b", chainID: 123, latestBlock: 7_000}, + {id: "stalled-c", chainID: 123, latestBlock: 7_000}, + }, &common.EvmServedTipConfig{Quorum: common.ServedTipQuorumMajority}) + lives, stalls := ups[:2], ups[2:] + tr := network.metricsTracker + + head := int64(7_000) + for range 5 { + advance(2 * time.Second) + head += 2 + for _, u := range ups { + u.EvmStatePoller().SuggestLatestBlock(head) + } + } + served(t, network, head, "a healthy fleet") + stalledAt := head + + window := stallAfter(stalls[0], 0) + for elapsed := time.Duration(0); elapsed+2*time.Second < window; elapsed += 2 * time.Second { + advance(2 * time.Second) + head += 2 + for _, u := range lives { + u.EvmStatePoller().SuggestLatestBlock(head) + } + } + for _, u := range stalls { + require.False(t, tr.IsHeadBenched(u), "%s: inside its stall window", u.Id()) + } + served(t, network, stalledAt, "inside the stall window the stalled majority (3 of 5) holds latest") - network, ups, advance := setupTrajectoryNetwork(t, ctx, 5, &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) - frozen := warmServedTipTrajectory(t, ctx, network, ups, advance, trajectoryWarmSamples) - fresh, served := stallMajority(t, ctx, network, ups, advance, frozen, 2, 24) - require.Equal(t, fresh, served, "precondition: the referee is holding the tip on the fresh pair") - require.Greater(t, fresh-frozen, int64(common.DefaultToleratedBlockHeadRollback), - "precondition: the stall is deeper than the regression tolerance") + advance(4 * time.Second) + head += 2 + for _, u := range lives { + u.EvmStatePoller().SuggestLatestBlock(head) + } + for _, u := range stalls { + require.True(t, tr.IsHeadBenched(u), "%s: benched after its stall window", u.Id()) + require.Equal(t, head-stalledAt, lagOf(network, u), "%s", u.Id()) + } + for _, u := range lives { + require.False(t, tr.IsHeadBenched(u), "%s", u.Id()) + } + served(t, network, head, "the live pair is the majority of the non-benched voters") - before := servedTipRegressionCount(network, "latest", "held") + advance(time.Second) + stalls[0].EvmStatePoller().SuggestLatestBlock(head) + require.False(t, tr.IsHeadBenched(stalls[0]), "a stalled head that caught up rejoins") + require.Zero(t, lagOf(network, stalls[0])) + for _, u := range stalls[1:] { + require.True(t, tr.IsHeadBenched(u), "%s", u.Id()) + } + served(t, network, head, "after one rejoins") + }) - // One fresh upstream leaves the eligible set: the remaining ballot is - // [fresh, frozen, frozen] — a lone fresh head is no group, so the referee - // stands down and the majority pick collapses onto the stalled pair. - network.PinUpstreamOrderForTest("u1", "u3", "u4") + t.Run("a benched head still votes when every other voter leaves", func(t *testing.T) { + advance := pinServedTipClock(t) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "live", chainID: 123, latestBlock: 8_000}, + {id: "stalled", chainID: 123, latestBlock: 8_000}, + }, nil) + live, stalled := ups[0], ups[1] + tr := network.metricsTracker + const stalledAt = int64(8_000) + head := stalledAt + for elapsed := time.Duration(0); elapsed <= stallAfter(stalled, 0); elapsed += 2 * time.Second { + advance(2 * time.Second) + head += 2 + live.EvmStatePoller().SuggestLatestBlock(head) + } + require.True(t, tr.IsHeadBenched(stalled), "precondition: stalled is benched") + served(t, network, head, "precondition: live defines latest") - assert.Equal(t, fresh, network.EvmHighestLatestBlockNumber(ctx), - "the guard must hold the last corroborated pick, as it does for any poisoned ballot") - assert.Equal(t, before+1, servedTipRegressionCount(network, "latest", "held")) + live.EvmStatePoller().SetSyncingState(common.EvmSyncingStateSyncing) + require.Equal(t, stalledAt, network.EvmHighestLatestBlockNumber(ctx), + "with no non-benched voter left, latest fails open to the benched head instead of 0") + }) } // ─── The finalized axis is not a second-class citizen ──────────────────────── @@ -2026,53 +2324,37 @@ func TestServedTip_Finalized_BoundCappedUpstreamsDoNotVote(t *testing.T) { "the capped pair's effective finalized is their 5000 bound, which is a serving range, not a finalized head") } -// The regression guard and the trajectory referee, both on the finalized axis, -// in one run: warm a finalized head track, stall the majority (the referee must -// serve the corroborated fresh pair), then lose one of that pair from the -// eligible set (the guard must hold what it last saw corroborated). -func TestServedTip_Finalized_RefereeAndGuardEndToEnd(t *testing.T) { +// The regression guard on the finalized axis: one of the fresh pair leaves the +// eligible set, the ballot collapses onto heads far below, and the guard holds +// the last corroborated finalized tip. +func TestServedTip_Finalized_RegressionGuardHolds(t *testing.T) { util.ResetGock() defer util.ResetGock() util.SetupMocksForEvmStatePoller() ctx, cancel := context.WithCancel(context.Background()) defer cancel() + pinServedTipClock(t) - network, ups, advance := setupTrajectoryNetwork(t, ctx, 5, &common.EvmServedTipConfig{EnabledFor: []string{"finalized"}}) - - head := trajectoryStartHead - for i := 0; i < trajectoryWarmSamples; i++ { - for _, u := range ups { - setServedTipFinalized(t, u, head) + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "u1", chainID: 123, latestBlock: 1_010_000}, + {id: "u2", chainID: 123, latestBlock: 1_010_000}, + {id: "u3", chainID: 123, latestBlock: 1_010_000}, + {id: "u4", chainID: 123, latestBlock: 1_010_000}, + }, nil) + for i, u := range ups { + fin := int64(1_000_000) + if i < 2 { + fin = 1_002_400 } - require.Equal(t, head, network.EvmHighestFinalizedBlockNumber(ctx), - "warm-up sample %d: an agreeing fleet serves its agreed finalized head", i) - advance(servedTipSampleInterval) - head += trajectoryBlocksPerSample + setServedTipFinalized(t, u, fin) } - frozen := head - trajectoryBlocksPerSample + require.Equal(t, int64(1_002_400), network.EvmHighestFinalizedBlockNumber(ctx), + "quorum 2: the fresh pair defines the finalized tip") - // The stall: three finalized heads freeze, two keep tracking the chain. - var served int64 - for i := 0; i < 24; i++ { - for _, u := range ups[:2] { - setServedTipFinalized(t, u, head) - } - served = network.EvmHighestFinalizedBlockNumber(ctx) - advance(servedTipSampleInterval) - head += trajectoryBlocksPerSample - } - fresh := head - trajectoryBlocksPerSample - require.Equal(t, fresh, served, - "the corroborated fresh pair must define the finalized tip; the plain majority would serve %d", frozen) - assert.Positive(t, servedTipTrajectoryCount(network, "finalized", "override"), - "the finalized axis gets its own counter series") - - // One of the fresh pair leaves: the ballot collapses onto the stalled trio - // and the guard holds the last corroborated finalized tip. before := servedTipRegressionCount(network, "finalized", "held") network.PinUpstreamOrderForTest("u1", "u3", "u4") - assert.Equal(t, fresh, network.EvmHighestFinalizedBlockNumber(ctx), + assert.Equal(t, int64(1_002_400), network.EvmHighestFinalizedBlockNumber(ctx), "the guard protects the finalized axis exactly as it protects latest") assert.Equal(t, before+1, servedTipRegressionCount(network, "finalized", "held")) } @@ -2099,7 +2381,7 @@ func TestServedTip_RegressionGuard_HeldValueIsClampedToTheLiveHead(t *testing.T) {id: "stuck-2", chainID: 123, latestBlock: 5}, }) require.Equal(t, int64(10_000), network.EvmHighestLatestBlockNumber(ctx)) - require.Equal(t, int64(10_000), network.servedLatestAnchor.lastGoodValue.Load()) + require.Equal(t, int64(10_000), network.servedTipAll.latest.lastGoodValue.Load()) // Every upstream that had 10_000 drops out. The ballot is [9000, 5, 5] → 5, // which the guard suppresses — but it may not serve its remembered 10_000 @@ -2151,67 +2433,3 @@ func TestServedTip_MetricsCountTransitionsNotEvaluations(t *testing.T) { } assert.Equal(t, beforeOpen+1, servedTipRegressionCount(network, "latest", "failed_open")) } - -// An override the guaranteed-method floor immediately nullifies did not happen: -// the referee raised the pick, the floor put it back, the served value is the -// majority pick — counting or logging that would page someone about nothing. -func TestServedTip_OverrideNullifiedByTheFloorIsNotCounted(t *testing.T) { - util.ResetGock() - defer util.ResetGock() - util.SetupMocksForEvmStatePoller() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // u5 is the only trace_block-capable upstream and it is one of the stalled - // trio, so its supporting set floors the tip back at the frozen head. - advance := pinServedTipClock(t) - fixtures := make([]servedTipFixture, 5) - for i := range fixtures { - fixtures[i] = servedTipFixture{id: "u" + strconv.Itoa(i+1), chainID: 123, latestBlock: trajectoryStartHead} - if i < 4 { - fixtures[i].ignoreMethods = []string{"trace_block"} - } - } - network, ups := setupServedTipNetworkWith(t, ctx, fixtures, &common.EvmServedTipConfig{ - EnabledFor: []string{"latest"}, - GuaranteedMethods: []string{"trace_block"}, - }) - - frozen := warmServedTipTrajectory(t, ctx, network, ups, advance, trajectoryWarmSamples) - before := servedTipTrajectoryCount(network, "latest", "override") - - _, served := stallMajority(t, ctx, network, ups, advance, frozen, 2, 24) - - assert.Equal(t, frozen, served, - "the floor holds the tip at what the trace_block upstream can serve") - assert.Equal(t, before, servedTipTrajectoryCount(network, "latest", "override"), - "an override the floor nullified must not be counted as an intervention") -} - -// The off switch is a real off switch: trajectoryWindow: 0 records nothing and -// decides nothing, so the network serves the plain majority pick — the stale -// head — through the exact scenario the referee exists to fix. -func TestServedTip_TrajectoryReferee_DisabledByZeroWindow(t *testing.T) { - util.ResetGock() - defer util.ResetGock() - util.SetupMocksForEvmStatePoller() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - network, ups, advance := setupTrajectoryNetwork(t, ctx, 5, &common.EvmServedTipConfig{ - EnabledFor: []string{"latest"}, - TrajectoryWindow: common.Duration(0).Ptr(), - }) - frozen := warmServedTipTrajectory(t, ctx, network, ups, advance, trajectoryWarmSamples) - - before := servedTipTrajectoryCount(network, "latest", "override") - _, served := stallMajority(t, ctx, network, ups, advance, frozen, 2, 24) - - assert.Equal(t, frozen, served, - "with the referee off the stalled majority defines the tip again") - assert.Equal(t, before, servedTipTrajectoryCount(network, "latest", "override")) - assert.Zero(t, network.servedLatestAnchor.trajectory.SampleCount(), - "a disabled referee must not even record a head track") -} diff --git a/erpc/networks_ws_tip_test.go b/erpc/networks_ws_tip_test.go index fa72fdccb..54bbf03ae 100644 --- a/erpc/networks_ws_tip_test.go +++ b/erpc/networks_ws_tip_test.go @@ -100,9 +100,20 @@ func TestNoteObservedLatestBlock_FloorsEvmHighestLatest(t *testing.T) { assert.Equal(t, int64(1001), network.EvmHighestLatestBlockNumber(ctx)) } -// End-to-end through networkHandle.SuggestLatestBlock, which runs before -// fan-out. -func TestNetworkHandle_SuggestLatestBlock_AdvancesNetworkTipBeforeFanOut(t *testing.T) { +// ingestHead mirrors indexer.Ingest for a head a client subscribed to: the +// head reaches its source's poller and, when delivered, floors "latest" +// before the client sees it. +func ingestHead(h *networkHandle, sourceId string, blockNumber int64) bool { + deliver := h.SuggestLatestBlock(sourceId, blockNumber) + if deliver { + h.DeliveringHead(blockNumber) + } + return deliver +} + +// A WS head always advances its upstream's poller; only a head a client is +// about to receive moves the delivered-head floor. +func TestNetworkHandle_SuggestLatestBlock_AdvancesPollerFloorOnlyOnDelivery(t *testing.T) { util.ResetGock() defer util.ResetGock() ctx, cancel := context.WithCancel(context.Background()) @@ -114,19 +125,23 @@ func TestNetworkHandle_SuggestLatestBlock_AdvancesNetworkTipBeforeFanOut(t *test require.Equal(t, int64(90677358), network.EvmHighestLatestBlockNumber(ctx)) handle := &networkHandle{nw: network} - handle.SuggestLatestBlock("ws:rpc2", 90677359) - + floor := network.deliveredLatestBlock.Load() + require.True(t, handle.SuggestLatestBlock("ws:rpc2", 90677359)) assert.Equal(t, int64(90677359), u.EvmStatePoller().LatestBlock(), "per-upstream poller must advance") - assert.Equal(t, int64(90677359), network.EvmHighestLatestBlockNumber(ctx), - "network tip must advance before any client would see the WS head") + assert.Equal(t, floor, network.deliveredLatestBlock.Load(), + "a head no client received must leave the floor alone") + + handle.DeliveringHead(90677359) assert.GreaterOrEqual(t, network.deliveredLatestBlock.Load(), int64(90677359), - "process-local high-water mark must cover the delivered WS tip") + "the floor must cover a head before a client receives it") } -// Clients receive heads from whichever source delivers first, so a head from -// a cordoned fallback lifts "latest" too; an unknown source does not. -func TestNetworkHandle_SuggestLatestBlock_EveryKnownSourceLiftsLatest(t *testing.T) { +// A head from one WS source must not lift "latest" past the upstreams that +// poll (they would then fail it as ErrUpstreamBlockUnavailable) unless a +// client receives it; then it floors "latest" whichever known source sent it. +// An unknown source never does. +func TestNetworkHandle_SuggestLatestBlock_LiftsLatestOnlyForDeliveredHeads(t *testing.T) { defer util.ResetGock() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -145,14 +160,19 @@ func TestNetworkHandle_SuggestLatestBlock_EveryKnownSourceLiftsLatest(t *testing require.NotNil(t, fallback) handle := &networkHandle{nw: network} - handle.SuggestLatestBlock("ws:unknown", 1020) + assert.False(t, ingestHead(handle, "ws:unknown", 1020)) assert.Equal(t, int64(1000), network.EvmHighestLatestBlockNumber(ctx), "an unknown source must not lift latest") - handle.SuggestLatestBlock("ws:fallback-1", 1010) + // No client subscribed: a prewarmed subscription's head. + require.True(t, handle.SuggestLatestBlock("ws:fallback-1", 1010)) assert.Equal(t, int64(1010), fallback.EvmStatePoller().LatestBlock()) - assert.Equal(t, int64(1010), network.EvmHighestLatestBlockNumber(ctx), - "a head a client can receive from the cordoned fallback floors latest") + assert.Equal(t, int64(1000), network.EvmHighestLatestBlockNumber(ctx), + "a head no client received must not lift latest") + + assert.True(t, ingestHead(handle, "ws:fallback-1", 1011)) + assert.Equal(t, int64(1011), network.EvmHighestLatestBlockNumber(ctx), + "a head a client receives from the cordoned fallback floors latest") } type fakeHeadSource bool @@ -188,13 +208,13 @@ func TestNetworkHandle_FallbackHeadsHeldWhilePrimariesStream(t *testing.T) { "primary-2": fakeHeadSource(true), "fallback-1": fakeHeadSource(true), }} - assert.False(t, handle.SuggestLatestBlock("ws:fallback-1", 1010)) + assert.False(t, ingestHead(handle, "ws:fallback-1", 1010)) assert.Equal(t, int64(1010), ups["fallback-1"].EvmStatePoller().LatestBlock(), "the fallback's poller still sees its head") assert.Equal(t, int64(1000), network.EvmHighestLatestBlockNumber(ctx), "a held head must not lift latest") - assert.True(t, handle.SuggestLatestBlock("ws:primary-2", 1001)) + assert.True(t, ingestHead(handle, "ws:primary-2", 1001)) assert.Equal(t, int64(1001), network.EvmHighestLatestBlockNumber(ctx)) }) @@ -210,7 +230,7 @@ func TestNetworkHandle_FallbackHeadsHeldWhilePrimariesStream(t *testing.T) { {"primary-1": fakeHeadSource(false), "fallback-1": fakeHeadSource(true)}, } { handle := &networkHandle{nw: network, heads: heads} - assert.True(t, handle.SuggestLatestBlock("ws:fallback-1", 1010)) + assert.True(t, ingestHead(handle, "ws:fallback-1", 1010)) } assert.Equal(t, int64(1010), network.EvmHighestLatestBlockNumber(ctx)) }) @@ -226,13 +246,13 @@ func TestNetworkHandle_FallbackHeadsHeldWhilePrimariesStream(t *testing.T) { "primary-2": fakeHeadSource(true), "fallback-1": fakeHeadSource(true), }} - require.False(t, handle.SuggestLatestBlock("ws:fallback-1", 1010)) + require.False(t, ingestHead(handle, "ws:fallback-1", 1010)) ups["primary-1"].Cordon("*", "test") ups["primary-2"].Cordon("*", "test") policy.TickForTest(network.policyEngine, network.networkId, "*") - assert.True(t, handle.SuggestLatestBlock("ws:fallback-1", 1011)) + assert.True(t, ingestHead(handle, "ws:fallback-1", 1011)) assert.Equal(t, int64(1011), network.EvmHighestLatestBlockNumber(ctx)) }) diff --git a/erpc/shadow.go b/erpc/shadow.go index 3a343736b..878430a08 100644 --- a/erpc/shadow.go +++ b/erpc/shadow.go @@ -14,46 +14,6 @@ import ( "github.com/erpc/erpc/upstream" ) -// Tag-selector parameter position for the state methods the mirror rewrites -// when ShadowUpstreamConfig.PinBlockTag is on. Methods absent here are -// forwarded untouched — for eth_getBlockByNumber and friends, pinning the tag -// would change the QUESTION, not just the evaluation height. -var shadowTagParamIndex = map[string]int{ - "eth_call": 1, - "eth_getBalance": 1, - "eth_getCode": 1, - "eth_getTransactionCount": 1, - "eth_getStorageAt": 2, -} - -// pinBlockTagInBody rewrites params[idx] from "latest" to the given concrete -// height in a JSON-RPC request body. Returns the rewritten body and whether a -// rewrite happened; any parse trouble or a non-"latest" selector returns the -// body unchanged (fail-open, like every other shadow-path guard). -func pinBlockTagInBody(body []byte, idx int, height int64) ([]byte, bool) { - if len(body) == 0 || height <= 0 { - return body, false - } - var req map[string]interface{} - if err := common.SonicCfg.Unmarshal(body, &req); err != nil { - return body, false - } - params, ok := req["params"].([]interface{}) - if !ok || len(params) <= idx { - return body, false - } - tag, ok := params[idx].(string) - if !ok || tag != "latest" { - return body, false - } - params[idx] = fmt.Sprintf("0x%x", height) - out, err := common.SonicCfg.Marshal(req) - if err != nil { - return body, false - } - return out, true -} - func (p *PreparedProject) executeShadowRequests(ctx context.Context, network *Network, shadowUpstreams []*upstream.Upstream, resp *common.NormalizedResponse) { defer func() { if r := recover(); r != nil { @@ -217,7 +177,7 @@ func (p *PreparedProject) executeShadowRequests(ctx context.Context, network *Ne // mismatch on every block boundary. No usable height means no pin // (fail-open, like every other guard on this path). if shadowCfg := ups.Config().Shadow; shadowCfg != nil && shadowCfg.PinBlockTag { - if idx, ok := shadowTagParamIndex[method]; ok { + if idx, ok := evm.LatestTagParamIndex(method); ok { height := int64(0) if bn, ok2 := resp.EvmBlockNumber().(int64); ok2 && bn > 0 { height = bn @@ -225,7 +185,7 @@ func (p *PreparedProject) executeShadowRequests(ctx context.Context, network *Ne height = primaryHead } if height > 0 { - if pinned, changed := pinBlockTagInBody(shadowReq.Body(), idx, height); changed { + if pinned, changed := evm.PinBlockTagInBody(shadowReq.Body(), idx, height); changed { repinned := common.NewNormalizedRequest(pinned) if dirs := origReq.Directives(); dirs != nil { repinned.SetDirectives(dirs.Clone()) diff --git a/erpc/shadow_compare_test.go b/erpc/shadow_compare_test.go new file mode 100644 index 000000000..3a2796599 --- /dev/null +++ b/erpc/shadow_compare_test.go @@ -0,0 +1,683 @@ +package erpc + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/thirdparty" + "github.com/erpc/erpc/upstream" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// shadowCmpFixture is a bootstrapped project with primary rpc1 and the given +// shadow upstreams on network evm:123. +type shadowCmpFixture struct { + ctx context.Context + prj *PreparedProject + network *Network +} + +func newShadowCmpFixture(t *testing.T, projectId string, shadows ...*common.UpstreamConfig) *shadowCmpFixture { + t.Helper() + util.ResetGock() + t.Cleanup(util.ResetGock) + util.SetupMocksForEvmStatePoller() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + ssr, err := data.NewSharedStateRegistry(ctx, &log.Logger, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: "memory", + Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}, + }, + }) + require.NoError(t, err) + rlr, err := upstream.NewRateLimitersRegistry(ctx, &common.RateLimiterConfig{}, &log.Logger) + require.NoError(t, err) + + ups := []*common.UpstreamConfig{{ + Id: "rpc1", + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + StatePollerInterval: common.Duration(200 * time.Millisecond), + StatePollerDebounce: common.Duration(50 * time.Millisecond), + }, + }} + for _, s := range shadows { + s.Type = common.UpstreamTypeEvm + if s.Evm == nil { + s.Evm = &common.EvmUpstreamConfig{} + } + s.Evm.ChainId = 123 + s.Evm.StatePollerInterval = common.Duration(200 * time.Millisecond) + s.Evm.StatePollerDebounce = common.Duration(50 * time.Millisecond) + if s.Shadow == nil { + s.Shadow = &common.ShadowUpstreamConfig{} + } + s.Shadow.Enabled = true + ups = append(ups, s) + } + + reg, err := NewProjectsRegistry(ctx, &log.Logger, []*common.ProjectConfig{{ + Id: projectId, + Networks: []*common.NetworkConfig{{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }}, + Upstreams: ups, + }}, ssr, nil, nil, rlr, thirdparty.NewVendorsRegistry(), nil, nil) + require.NoError(t, err) + reg.Bootstrap(ctx) + + prj, err := reg.GetProject(projectId) + require.NoError(t, err) + network, err := prj.GetNetwork(ctx, util.EvmNetworkId(123)) + require.NoError(t, err) + require.Len(t, network.ShadowUpstreams(), len(shadows), "every shadow upstream is registered as shadow") + + return &shadowCmpFixture{ctx: ctx, prj: prj, network: network} +} + +// shadowUpstream returns the registered shadow upstream with the given id. +func (f *shadowCmpFixture) shadowUpstream(t *testing.T, id string) *upstream.Upstream { + t.Helper() + for _, u := range f.network.ShadowUpstreams() { + if u.Id() == id { + return u + } + } + t.Fatalf("shadow upstream %s not found", id) + return nil +} + +// shadowCmpWaitHead blocks until the upstream's state poller reports the given head. +func shadowCmpWaitHead(t *testing.T, u *upstream.Upstream, head int64) { + t.Helper() + require.Eventually(t, func() bool { + sp := u.EvmStatePoller() + return sp != nil && sp.LatestBlock() == head + }, 3*time.Second, 10*time.Millisecond, "poller for %s should settle on %d (poller=%v)", u.Id(), head, func() interface{} { + if sp := u.EvmStatePoller(); sp != nil { + return sp.LatestBlock() + } + return nil + }()) +} + +// shadowCmpHits records the request bodies one mocked host receives for one method. +type shadowCmpHits struct { + mu sync.Mutex + bodies []string +} + +func (h *shadowCmpHits) count() int { + h.mu.Lock() + defer h.mu.Unlock() + return len(h.bodies) +} + +func (h *shadowCmpHits) last() string { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.bodies) == 0 { + return "" + } + return h.bodies[len(h.bodies)-1] +} + +// shadowCmpMock answers every `method` call to `host` with `reply` (a JSON-RPC +// envelope) and records the request bodies. +func shadowCmpMock(host, method, reply string) *shadowCmpHits { + h := &shadowCmpHits{} + gock.New("http://" + host).Post("").Persist(). + Filter(func(r *http.Request) bool { + if r.URL.Host != host { + return false + } + body := util.SafeReadBody(r) + if !strings.Contains(body, `"`+method+`"`) { + return false + } + h.mu.Lock() + h.bodies = append(h.bodies, body) + h.mu.Unlock() + return true + }). + Reply(200).JSON([]byte(reply)) + return h +} + +// shadowCmpCount sums the counter series of c whose labels include every pair in match. +func shadowCmpCount(c prometheus.Collector, match map[string]string) float64 { + ch := make(chan prometheus.Metric, 256) + go func() { + c.Collect(ch) + close(ch) + }() + total := 0.0 + for m := range ch { + var pb dto.Metric + if m.Write(&pb) != nil || pb.Counter == nil { + continue + } + got := make(map[string]string, len(pb.GetLabel())) + for _, lp := range pb.GetLabel() { + got[lp.GetName()] = lp.GetValue() + } + ok := true + for k, v := range match { + if got[k] != v { + ok = false + break + } + } + if ok { + total += pb.Counter.GetValue() + } + } + return total +} + +// shadowCmpOutcome snapshots the three shadow outcome counters for one upstream+method. +type shadowCmpOutcome struct { + identical, mismatch, errors float64 +} + +func shadowCmpOutcomes(upsId, method string) shadowCmpOutcome { + sel := map[string]string{"upstream": upsId, "category": method} + return shadowCmpOutcome{ + identical: shadowCmpCount(telemetry.MetricShadowResponseIdenticalTotal, sel), + mismatch: shadowCmpCount(telemetry.MetricShadowResponseMismatchTotal, sel), + errors: shadowCmpCount(telemetry.MetricShadowResponseErrorTotal, sel), + } +} + +// shadowCmpAwait waits until the outcome counters of upsId/method moved by want +// relative to before, then asserts they moved by exactly want. +func shadowCmpAwait(t *testing.T, upsId, method string, before, want shadowCmpOutcome) { + t.Helper() + diff := func() shadowCmpOutcome { + now := shadowCmpOutcomes(upsId, method) + return shadowCmpOutcome{ + identical: now.identical - before.identical, + mismatch: now.mismatch - before.mismatch, + errors: now.errors - before.errors, + } + } + require.Eventually(t, func() bool { + d := diff() + return d.identical+d.mismatch+d.errors >= want.identical+want.mismatch+want.errors + }, 3*time.Second, 10*time.Millisecond, "shadow outcome for %s/%s never recorded", upsId, method) + assert.Equal(t, want, diff(), "shadow outcome for %s/%s", upsId, method) +} + +// shadowCmpResponse builds a primary response for body with the given raw +// JSON-RPC result or error. +func shadowCmpResponse(network *Network, body, result, rpcErr string) *common.NormalizedResponse { + req := common.NewNormalizedRequest([]byte(body)) + req.SetNetwork(network) + var res, errB []byte + if result != "" { + res = []byte(result) + } + if rpcErr != "" { + errB = []byte(rpcErr) + } + jrr := common.MustNewJsonRpcResponseFromBytes([]byte("1"), res, errB) + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) +} + +// shadowCmpResultOf returns the raw "result" member of a JSON-RPC envelope. +func shadowCmpResultOf(t *testing.T, envelope string) string { + t.Helper() + var env map[string]interface{} + require.NoError(t, common.SonicCfg.Unmarshal([]byte(envelope), &env)) + out, err := common.SonicCfg.Marshal(env["result"]) + require.NoError(t, err) + return string(out) +} + +func shadowCmpForward(t *testing.T, f *shadowCmpFixture, method, params string) string { + t.Helper() + req := common.NewNormalizedRequest([]byte( + fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":"%s","params":%s}`, method, params))) + resp, err := f.prj.Forward(f.ctx, util.EvmNetworkId(123), req) + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse(f.ctx) + require.NoError(t, err) + return jrr.GetResultString() +} + +// TestShadowCompare_ForwardPath drives PreparedProject.Forward with a primary +// and two shadows (one ignoring the "ts" field of shadowcmp_obj) and checks +// the client answer comes from the primary while each shadow's comparison +// outcome lands in the matching counter. +func TestShadowCompare_ForwardPath(t *testing.T) { + f := newShadowCmpFixture(t, "shadowcmp_fwd", + &common.UpstreamConfig{ + Id: "shadowcmp-fwd-ign", + Endpoint: "http://rpc2.localhost", + Shadow: &common.ShadowUpstreamConfig{ + IgnoreFields: map[string][]string{"shadowcmp_obj": {"ts"}}, + }, + }, + &common.UpstreamConfig{ + Id: "shadowcmp-fwd-plain", + Endpoint: "http://rpc3.localhost", + }, + ) + const ign, plain = "shadowcmp-fwd-ign", "shadowcmp-fwd-plain" + + cases := []struct { + name string + method string + params string + primary string + shadow string + wantIgn shadowCmpOutcome + wantPlain shadowCmpOutcome + mismatchLabel map[string]string + }{ + { + name: "IdenticalResult", + method: "shadowcmp_same", + params: `[]`, + primary: `{"jsonrpc":"2.0","id":1,"result":"0x10"}`, + shadow: `{"jsonrpc":"2.0","id":1,"result":"0x10"}`, + wantIgn: shadowCmpOutcome{identical: 1}, + wantPlain: shadowCmpOutcome{identical: 1}, + }, + { + name: "LargerShadowResultMismatches", + method: "shadowcmp_larger", + params: `[]`, + primary: `{"jsonrpc":"2.0","id":1,"result":"0x10"}`, + shadow: `{"jsonrpc":"2.0","id":1,"result":"0x1000"}`, + wantIgn: shadowCmpOutcome{mismatch: 1}, + wantPlain: shadowCmpOutcome{mismatch: 1}, + mismatchLabel: map[string]string{"emptyish": "false", "larger": "true"}, + }, + { + name: "EmptyShadowAgainstNonEmptyPrimaryMismatches", + method: "shadowcmp_emptyshadow", + params: `[]`, + primary: `{"jsonrpc":"2.0","id":1,"result":"0x10"}`, + shadow: `{"jsonrpc":"2.0","id":1,"result":"0x0"}`, + wantIgn: shadowCmpOutcome{mismatch: 1}, + wantPlain: shadowCmpOutcome{mismatch: 1}, + mismatchLabel: map[string]string{"emptyish": "true", "larger": "false"}, + }, + { + name: "BothEmptyishWithDifferentBytesAreIdentical", + method: "shadowcmp_bothempty", + params: `[]`, + primary: `{"jsonrpc":"2.0","id":1,"result":"0x0"}`, + shadow: `{"jsonrpc":"2.0","id":1,"result":"0x"}`, + wantIgn: shadowCmpOutcome{identical: 1}, + wantPlain: shadowCmpOutcome{identical: 1}, + }, + { + name: "IgnoredFieldDifferenceOnlyMatchesWhenConfigured", + method: "shadowcmp_obj", + params: `[]`, + primary: `{"jsonrpc":"2.0","id":1,"result":{"v":"0x1","ts":"0x5"}}`, + shadow: `{"jsonrpc":"2.0","id":1,"result":{"v":"0x1","ts":"0x6"}}`, + wantIgn: shadowCmpOutcome{identical: 1}, + wantPlain: shadowCmpOutcome{mismatch: 1}, + }, + { + name: "ShadowServerErrorCountsAsError", + method: "shadowcmp_err", + params: `[]`, + primary: `{"jsonrpc":"2.0","id":1,"result":"0x10"}`, + shadow: `{"jsonrpc":"2.0","id":1,"error":{"code":-32603,"message":"internal failure"}}`, + wantIgn: shadowCmpOutcome{errors: 1}, + wantPlain: shadowCmpOutcome{errors: 1}, + }, + { + name: "ShadowRevertAgainstPrimaryResultMismatches", + method: "eth_call", + params: `[{"to":"0x0000000000000000000000000000000000000001","data":"0x"},"latest"]`, + primary: `{"jsonrpc":"2.0","id":1,"result":"0x01"}`, + shadow: `{"jsonrpc":"2.0","id":1,"error":{"code":3,"message":"execution reverted","data":"0x"}}`, + wantIgn: shadowCmpOutcome{mismatch: 1}, + wantPlain: shadowCmpOutcome{mismatch: 1}, + mismatchLabel: map[string]string{"finality": "n/a", "emptyish": "false", "larger": "false"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + shadowCmpMock("rpc1.localhost", tc.method, tc.primary) + shadowCmpMock("rpc2.localhost", tc.method, tc.shadow) + shadowCmpMock("rpc3.localhost", tc.method, tc.shadow) + + beforeIgn := shadowCmpOutcomes(ign, tc.method) + beforePlain := shadowCmpOutcomes(plain, tc.method) + var beforeLabeled float64 + if tc.mismatchLabel != nil { + sel := map[string]string{"upstream": plain, "category": tc.method} + for k, v := range tc.mismatchLabel { + sel[k] = v + } + beforeLabeled = shadowCmpCount(telemetry.MetricShadowResponseMismatchTotal, sel) + } + + got := shadowCmpForward(t, f, tc.method, tc.params) + assert.JSONEq(t, shadowCmpResultOf(t, tc.primary), got, "client receives the primary's result") + + shadowCmpAwait(t, ign, tc.method, beforeIgn, tc.wantIgn) + shadowCmpAwait(t, plain, tc.method, beforePlain, tc.wantPlain) + + if tc.mismatchLabel != nil { + sel := map[string]string{"upstream": plain, "category": tc.method} + for k, v := range tc.mismatchLabel { + sel[k] = v + } + assert.Equal(t, 1.0, shadowCmpCount(telemetry.MetricShadowResponseMismatchTotal, sel)-beforeLabeled, + "mismatch carries labels %v", tc.mismatchLabel) + } + }) + } +} + +// TestShadowCompare_SkipConditions calls executeShadowRequests directly with a +// concrete-height eth_getBalance. Only the control shadow may be sent the +// request; the ignored-method, zero-sample-rate, out-of-range and +// invalid-pattern shadows are skipped before any goroutine is spawned. +func TestShadowCompare_SkipConditions(t *testing.T) { + zero := 0.0 + f := newShadowCmpFixture(t, "shadowcmp_skip", + &common.UpstreamConfig{ + Id: "shadowcmp-skip-ignored", + Endpoint: "http://rpc2.localhost", + IgnoreMethods: []string{"eth_getBalance"}, + }, + &common.UpstreamConfig{ + Id: "shadowcmp-skip-sampled", + Endpoint: "http://rpc3.localhost", + Shadow: &common.ShadowUpstreamConfig{SampleRate: &zero}, + }, + &common.UpstreamConfig{ + Id: "shadowcmp-skip-recent", + Endpoint: "http://rpc4.localhost", + Evm: &common.EvmUpstreamConfig{MaxAvailableRecentBlocks: 100}, + }, + &common.UpstreamConfig{ + Id: "shadowcmp-skip-badpattern", + Endpoint: "http://rpc5.localhost", + IgnoreMethods: []string{"(eth_getBalance"}, + }, + &common.UpstreamConfig{ + Id: "shadowcmp-skip-control", + Endpoint: "http://rpc7.localhost", + }, + ) + control := f.shadowUpstream(t, "shadowcmp-skip-control") + shadowCmpWaitHead(t, control, 0x33338888) + shadowCmpWaitHead(t, f.shadowUpstream(t, "shadowcmp-skip-recent"), 0x33338888) + + const method = "eth_getBalance" + reply := `{"jsonrpc":"2.0","id":1,"result":"0x64"}` + skipped := map[string]*shadowCmpHits{ + "shadowcmp-skip-ignored": shadowCmpMock("rpc2.localhost", method, reply), + "shadowcmp-skip-sampled": shadowCmpMock("rpc3.localhost", method, reply), + "shadowcmp-skip-recent": shadowCmpMock("rpc4.localhost", method, reply), + "shadowcmp-skip-badpattern": shadowCmpMock("rpc5.localhost", method, reply), + } + controlHits := shadowCmpMock("rpc7.localhost", method, reply) + + before := map[string]shadowCmpOutcome{} + for id := range skipped { + before[id] = shadowCmpOutcomes(id, method) + } + beforeControl := shadowCmpOutcomes("shadowcmp-skip-control", method) + + resp := shadowCmpResponse(f.network, + `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0x0000000000000000000000000000000000000001","0x1"]}`, + `"0x64"`, "") + f.prj.executeShadowRequests(f.ctx, f.network, f.network.ShadowUpstreams(), resp) + + shadowCmpAwait(t, "shadowcmp-skip-control", method, beforeControl, shadowCmpOutcome{identical: 1}) + assert.Equal(t, 1, controlHits.count(), "control shadow is sent the request once") + for id, hits := range skipped { + assert.Equal(t, 0, hits.count(), "%s must not be sent the request", id) + assert.Equal(t, before[id], shadowCmpOutcomes(id, method), "%s records no outcome", id) + } +} + +// shadowCmpWriter is a result writer whose Size fails or panics. +type shadowCmpWriter struct { + sizeErr error + panicMsg string +} + +func (w *shadowCmpWriter) WriteTo(io.Writer, bool) (int64, error) { return 0, nil } +func (w *shadowCmpWriter) IsResultEmptyish() bool { return false } +func (w *shadowCmpWriter) Size(...context.Context) (int, error) { + if w.panicMsg != "" { + panic(w.panicMsg) + } + return 0, w.sizeErr +} + +// TestShadowCompare_EarlyReturns covers the inputs executeShadowRequests +// refuses before mirroring: no response, no shadows, no request, an +// unsizable primary, and a panic (recovered and counted). +func TestShadowCompare_EarlyReturns(t *testing.T) { + f := newShadowCmpFixture(t, "shadowcmp_early", &common.UpstreamConfig{ + Id: "shadowcmp-early", + Endpoint: "http://rpc2.localhost", + }) + const method = "shadowcmp_early" + hits := shadowCmpMock("rpc2.localhost", method, `{"jsonrpc":"2.0","id":1,"result":"0x1"}`) + shadows := f.network.ShadowUpstreams() + body := `{"jsonrpc":"2.0","id":1,"method":"shadowcmp_early","params":[]}` + + writerResp := func(w *shadowCmpWriter) *common.NormalizedResponse { + req := common.NewNormalizedRequest([]byte(body)) + req.SetNetwork(f.network) + jrr := &common.JsonRpcResponse{} + jrr.SetResultWriter(w) + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) + } + panicSel := map[string]string{"scope": "shadow-upstreams", "extra": "network:" + f.network.Label()} + + cases := []struct { + name string + shadows []*upstream.Upstream + resp *common.NormalizedResponse + wantPanic float64 + }{ + {name: "NilResponse", shadows: shadows, resp: nil}, + {name: "NoShadowUpstreams", shadows: nil, resp: shadowCmpResponse(f.network, body, `"0x1"`, "")}, + { + name: "ResponseWithoutRequest", + shadows: shadows, + resp: common.NewNormalizedResponse().WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte("1"), []byte(`"0x1"`), nil)), + }, + {name: "PrimarySizeError", shadows: shadows, resp: writerResp(&shadowCmpWriter{sizeErr: errors.New("size unavailable")})}, + {name: "PanicIsRecoveredAndCounted", shadows: shadows, resp: writerResp(&shadowCmpWriter{panicMsg: "shadowcmp boom"}), wantPanic: 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + before := shadowCmpOutcomes("shadowcmp-early", method) + beforePanic := shadowCmpCount(telemetry.MetricUnexpectedPanicTotal, panicSel) + + require.NotPanics(t, func() { + f.prj.executeShadowRequests(f.ctx, f.network, tc.shadows, tc.resp) + }) + + assert.Equal(t, 0, hits.count(), "no shadow request is sent") + assert.Equal(t, before, shadowCmpOutcomes("shadowcmp-early", method), "no shadow outcome is recorded") + assert.Equal(t, tc.wantPanic, shadowCmpCount(telemetry.MetricUnexpectedPanicTotal, panicSel)-beforePanic) + }) + } +} + +// TestShadowCompare_DirectVerdicts covers comparison outcomes only reachable +// with a hand-built primary response: both sides reverting, a primary whose +// hash cannot be computed, a body-less primary request, and PinBlockTag. +func TestShadowCompare_DirectVerdicts(t *testing.T) { + f := newShadowCmpFixture(t, "shadowcmp_direct", &common.UpstreamConfig{ + Id: "shadowcmp-direct", + Endpoint: "http://rpc2.localhost", + Shadow: &common.ShadowUpstreamConfig{ + PinBlockTag: true, + IgnoreFields: map[string][]string{"shadowcmp_ignhash": {"ts"}}, + }, + }) + const upsId = "shadowcmp-direct" + shadow := f.shadowUpstream(t, upsId) + shadowCmpWaitHead(t, shadow, 0x22228888) + shadows := f.network.ShadowUpstreams() + + t.Run("BothRevertedIsIdentical", func(t *testing.T) { + const method = "eth_estimateGas" + shadowCmpMock("rpc2.localhost", method, + `{"jsonrpc":"2.0","id":1,"error":{"code":3,"message":"execution reverted","data":"0x"}}`) + before := shadowCmpOutcomes(upsId, method) + resp := shadowCmpResponse(f.network, + `{"jsonrpc":"2.0","id":1,"method":"eth_estimateGas","params":[{"to":"0x0000000000000000000000000000000000000001","data":"0x"}]}`, + "", `{"code":3,"message":"execution reverted"}`) + f.prj.executeShadowRequests(f.ctx, f.network, shadows, resp) + shadowCmpAwait(t, upsId, method, before, shadowCmpOutcome{identical: 1}) + }) + + // A primary without a result cannot be hashed; the comparison proceeds + // against an empty expected hash and reports a mismatch with the + // non-empty shadow, with and without ignore fields. + for _, method := range []string{"shadowcmp_nohash", "shadowcmp_ignhash"} { + t.Run("PrimaryHashFailureMismatches/"+method, func(t *testing.T) { + shadowCmpMock("rpc2.localhost", method, `{"jsonrpc":"2.0","id":1,"result":{"v":"0x1","ts":"0x2"}}`) + before := shadowCmpOutcomes(upsId, method) + resp := shadowCmpResponse(f.network, + fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":"%s","params":[]}`, method), "", "") + f.prj.executeShadowRequests(f.ctx, f.network, shadows, resp) + shadowCmpAwait(t, upsId, method, before, shadowCmpOutcome{mismatch: 1}) + }) + } + + t.Run("BodylessRequestIsMarshalledWithDirectives", func(t *testing.T) { + const method = "shadowcmp_bodyless" + hits := shadowCmpMock("rpc2.localhost", method, `{"jsonrpc":"2.0","id":1,"result":"0x7"}`) + before := shadowCmpOutcomes(upsId, method) + req := common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest(method, []interface{}{"0xabc"})) + req.SetNetwork(f.network) + req.SetDirectives(&common.RequestDirectives{RetryEmpty: true}) + resp := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte("1"), []byte(`"0x7"`), nil)) + f.prj.executeShadowRequests(f.ctx, f.network, shadows, resp) + shadowCmpAwait(t, upsId, method, before, shadowCmpOutcome{identical: 1}) + assert.Contains(t, hits.last(), `"0xabc"`, "shadow receives the marshalled params") + }) + + // The mirror copies the original body only while it is still unparsed; + // a request whose block ref/number are already cached is never parsed + // by the availability check, so its body survives to be pinned. + pinBody := `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0x0000000000000000000000000000000000000001","latest"]}` + pinResp := func(blockNumber int64) *common.NormalizedResponse { + resp := shadowCmpResponse(f.network, pinBody, `"0x5"`, "") + resp.Request().SetEvmBlockRef("latest") + resp.Request().SetEvmBlockNumber(int64(0x1000)) + resp.Request().SetDirectives(&common.RequestDirectives{RetryEmpty: true}) + if blockNumber > 0 { + resp.SetEvmBlockNumber(blockNumber) + } + return resp + } + + // One persistent mock serves both pin subtests: gock answers with the + // first matching mock, so a second registration would never be hit. + pinHits := shadowCmpMock("rpc2.localhost", "eth_getBalance", `{"jsonrpc":"2.0","id":1,"result":"0x5"}`) + + t.Run("PinBlockTagUsesPrimaryBlockNumber", func(t *testing.T) { + before := shadowCmpOutcomes(upsId, "eth_getBalance") + f.prj.executeShadowRequests(f.ctx, f.network, shadows, pinResp(0x1234)) + shadowCmpAwait(t, upsId, "eth_getBalance", before, shadowCmpOutcome{identical: 1}) + assert.Contains(t, pinHits.last(), `"0x1234"`, "latest is pinned to the primary's block number") + assert.NotContains(t, pinHits.last(), `"latest"`) + }) + + t.Run("PinBlockTagFallsBackToServingUpstreamHead", func(t *testing.T) { + var primary common.Upstream + for _, u := range f.prj.upstreamsRegistry.GetNetworkUpstreams(f.ctx, util.EvmNetworkId(123)) { + if u.Id() == "rpc1" { + shadowCmpWaitHead(t, u, 0x11118888) + primary = u + } + } + require.NotNil(t, primary) + before := shadowCmpOutcomes(upsId, "eth_getBalance") + resp := pinResp(0) + resp.SetUpstream(primary) + f.prj.executeShadowRequests(f.ctx, f.network, shadows, resp) + shadowCmpAwait(t, upsId, "eth_getBalance", before, shadowCmpOutcome{identical: 1}) + assert.Contains(t, pinHits.last(), `"0x11118888"`, "latest is pinned to the serving upstream's head") + }) + + t.Run("UnresolvableRequestIsNotMirrored", func(t *testing.T) { + const method = "shadowcmp_unresolvable" + hits := shadowCmpMock("rpc2.localhost", method, `{"jsonrpc":"2.0","id":1,"result":"0x1"}`) + before := shadowCmpOutcomes(upsId, "") + req := common.NewNormalizedRequest(nil) + req.SetNetwork(f.network) + resp := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse( + common.MustNewJsonRpcResponseFromBytes([]byte("1"), []byte(`"0x1"`), nil)) + f.prj.executeShadowRequests(f.ctx, f.network, shadows, resp) + // The clone fails inside the mirror goroutine; give it time to run. + require.Never(t, func() bool { return hits.count() > 0 }, 300*time.Millisecond, 20*time.Millisecond) + assert.Equal(t, before, shadowCmpOutcomes(upsId, ""), "no outcome is recorded") + }) +} + +// TestShadowCompare_ForwardPathPinsLatestTag sends an eth_getBalance at +// "latest" through PreparedProject.Forward to a primary plus a PinBlockTag +// shadow: the client gets the primary's answer and the shadow is sent a +// concrete height in place of the tag. +func TestShadowCompare_ForwardPathPinsLatestTag(t *testing.T) { + f := newShadowCmpFixture(t, "shadowcmp_pinfwd", &common.UpstreamConfig{ + Id: "shadowcmp-pinfwd", + Endpoint: "http://rpc2.localhost", + Shadow: &common.ShadowUpstreamConfig{PinBlockTag: true}, + }) + for _, u := range f.prj.upstreamsRegistry.GetNetworkUpstreams(f.ctx, util.EvmNetworkId(123)) { + shadowCmpWaitHead(t, u, 0x11118888) + } + shadowCmpMock("rpc1.localhost", "eth_getBalance", `{"jsonrpc":"2.0","id":1,"result":"0x5"}`) + hits := shadowCmpMock("rpc2.localhost", "eth_getBalance", `{"jsonrpc":"2.0","id":1,"result":"0x5"}`) + before := shadowCmpOutcomes("shadowcmp-pinfwd", "eth_getBalance") + + got := shadowCmpForward(t, f, "eth_getBalance", `["0x0000000000000000000000000000000000000001","latest"]`) + assert.Equal(t, `"0x5"`, got) + + shadowCmpAwait(t, "shadowcmp-pinfwd", "eth_getBalance", before, shadowCmpOutcome{identical: 1}) + var sent struct { + Params []interface{} `json:"params"` + } + require.NoError(t, common.SonicCfg.Unmarshal([]byte(hits.last()), &sent)) + require.Len(t, sent.Params, 2) + tag, _ := sent.Params[1].(string) + assert.NotEqual(t, "latest", tag, "shadow copy is pinned") + assert.True(t, strings.HasPrefix(tag, "0x"), "shadow copy carries a hex height, got %q", tag) +} diff --git a/erpc/shadow_pin_test.go b/erpc/shadow_pin_test.go index f17a3d04e..0a1993424 100644 --- a/erpc/shadow_pin_test.go +++ b/erpc/shadow_pin_test.go @@ -1,15 +1,31 @@ package erpc import ( + "context" + "fmt" + "net/http" + "strings" + "sync" "testing" + "time" + "github.com/erpc/erpc/architecture/evm" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/health" + "github.com/erpc/erpc/thirdparty" + "github.com/erpc/erpc/upstream" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestPinBlockTagInBody(t *testing.T) { t.Run("PinsLatestForEthCall", func(t *testing.T) { body := []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0xabc","data":"0x1"},"latest"]}`) - out, changed := pinBlockTagInBody(body, 1, 0x189cbe5) + out, changed := evm.PinBlockTagInBody(body, 1, 0x189cbe5) assert.True(t, changed) assert.Contains(t, string(out), `"0x189cbe5"`) assert.NotContains(t, string(out), `"latest"`) @@ -17,34 +33,217 @@ func TestPinBlockTagInBody(t *testing.T) { t.Run("PinsStorageSlotIndexTwo", func(t *testing.T) { body := []byte(`{"jsonrpc":"2.0","id":7,"method":"eth_getStorageAt","params":["0xabc","0x0","latest"]}`) - out, changed := pinBlockTagInBody(body, 2, 100) + out, changed := evm.PinBlockTagInBody(body, 2, 100) assert.True(t, changed) assert.Contains(t, string(out), `"0x64"`) }) t.Run("ConcreteHeightUntouched", func(t *testing.T) { body := []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0xabc"},"0x1234"]}`) - out, changed := pinBlockTagInBody(body, 1, 100) + out, changed := evm.PinBlockTagInBody(body, 1, 100) assert.False(t, changed) assert.Equal(t, body, out) }) t.Run("PendingUntouched", func(t *testing.T) { body := []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0xabc"},"pending"]}`) - _, changed := pinBlockTagInBody(body, 1, 100) + _, changed := evm.PinBlockTagInBody(body, 1, 100) assert.False(t, changed) }) t.Run("MissingParamUntouched", func(t *testing.T) { body := []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0xabc"}]}`) - _, changed := pinBlockTagInBody(body, 1, 100) + _, changed := evm.PinBlockTagInBody(body, 1, 100) assert.False(t, changed) }) t.Run("GarbageUntouched", func(t *testing.T) { body := []byte(`{not json`) - out, changed := pinBlockTagInBody(body, 1, 100) + out, changed := evm.PinBlockTagInBody(body, 1, 100) assert.False(t, changed) assert.Equal(t, []byte(`{not json`), out) }) } + +// ethCallTags records the block selector (params[1]) of every eth_call one +// mocked upstream host receives. gock runs a mock's filters before its host +// matcher, so the filter checks the host itself. +type ethCallTags struct { + host string + mu sync.Mutex + tags []interface{} +} + +func (c *ethCallTags) filter(r *http.Request) bool { + if r.URL.Host != c.host { + return false + } + body := util.SafeReadBody(r) + if !strings.Contains(body, `"eth_call"`) { + return false + } + var rq struct { + Params []interface{} `json:"params"` + } + _ = common.SonicCfg.Unmarshal([]byte(body), &rq) + var tag interface{} + if len(rq.Params) > 1 { + tag = rq.Params[1] + } + c.mu.Lock() + c.tags = append(c.tags, tag) + c.mu.Unlock() + return true +} + +func (c *ethCallTags) get() []interface{} { + c.mu.Lock() + defer c.mu.Unlock() + return append([]interface{}(nil), c.tags...) +} + +// TestNetwork_PinLatestTag walks a network whose eth_call keeps "latest" +// (translateLatestTag: false) over a pinned upstream (evm.pinLatestTag, +// upper.latestBlockMinus: 0) tried first and a vendor tried second. +func TestNetwork_PinLatestTag(t *testing.T) { + const head = int64(0x2000) + headHex := fmt.Sprintf("0x%x", head) + + run := func(t *testing.T, pinnedHead int64, pinnedReply string) (result string, pinnedTags, vendorTags []interface{}) { + util.ResetGock() + defer util.ResetGock() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + pinned, vendor := ðCallTags{host: "pinned.localhost"}, ðCallTags{host: "vendor.localhost"} + heads := map[string]int64{"pinned": pinnedHead, "vendor": head} + for id, h := range heads { + util.SetupMocksForUpstream("http://"+id+".localhost", map[string]interface{}{ + "latestBlock": fmt.Sprintf("0x%x", h), + "finalizedBlock": fmt.Sprintf("0x%x", h-16), + }) + } + gock.New("http://pinned.localhost").Post("").Persist(). + Filter(pinned.filter). + Reply(200).JSON([]byte(pinnedReply)) + gock.New("http://vendor.localhost").Post("").Persist(). + Filter(vendor.filter). + Reply(200).JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":"0x22"}`)) + + upCfgs := []*common.UpstreamConfig{ + { + Id: "pinned", + Type: common.UpstreamTypeEvm, + Endpoint: "http://pinned.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + StatePollerInterval: common.Duration(200 * time.Millisecond), + StatePollerDebounce: common.Duration(50 * time.Millisecond), + BlockAvailability: &common.EvmBlockAvailabilityConfig{ + Upper: &common.EvmAvailabilityBoundConfig{LatestBlockMinus: i64(0)}, + }, + PinLatestTag: b(true), + }, + }, + { + Id: "vendor", + Type: common.UpstreamTypeEvm, + Endpoint: "http://vendor.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + StatePollerInterval: common.Duration(200 * time.Millisecond), + StatePollerDebounce: common.Duration(50 * time.Millisecond), + }, + }, + } + + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) + mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) + vr := thirdparty.NewVendorsRegistry() + pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) + sharedStateCfg := &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: common.DriverMemory, + Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}, + }, + LockMaxWait: common.Duration(200 * time.Millisecond), + UpdateMaxWait: common.Duration(200 * time.Millisecond), + FallbackTimeout: common.Duration(3 * time.Second), + LockTtl: common.Duration(4 * time.Second), + } + sharedStateCfg.SetDefaults("test") + ssr, _ := data.NewSharedStateRegistry(ctx, &log.Logger, sharedStateCfg) + upr := upstream.NewUpstreamsRegistry(ctx, &log.Logger, "prjA", upCfgs, ssr, rlr, vr, pr, nil, mt, nil) + upr.Bootstrap(ctx) + time.Sleep(100 * time.Millisecond) + + ntwCfg := &common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ + ChainId: 123, + // The network head is the highest upstream head, so a lagging + // pinned upstream is behind it. + ServedTip: &common.EvmServedTipConfig{Quorum: 1}, + }, + Methods: &common.MethodsConfig{ + Definitions: map[string]*common.CacheMethodConfig{ + "eth_call": {ReqRefs: [][]interface{}{{1}}, TranslateLatestTag: b(false)}, + }, + }, + } + network, err := NewNetwork(ctx, &log.Logger, "prjA", ntwCfg, rlr, upr, mt, nil) + require.NoError(t, err) + require.NoError(t, upr.PrepareUpstreamsForNetwork(ctx, util.EvmNetworkId(123))) + require.NoError(t, network.Bootstrap(ctx)) + network.PinUpstreamOrderForTest("pinned", "vendor") + + for _, u := range upr.GetNetworkUpstreams(ctx, util.EvmNetworkId(123)) { + want := heads[u.Id()] + require.Eventually(t, func() bool { return u.EvmStatePoller().LatestBlock() == want }, + 3*time.Second, 25*time.Millisecond, "poller for %s should settle on %d", u.Id(), want) + } + + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0x0000000000000000000000000000000000000001","data":"0x"},"latest"]}`)) + req.SetNetwork(network) + jrq, err := req.JsonRpcRequest() + require.NoError(t, err) + evm.NormalizeHttpJsonRpc(ctx, req, jrq) + require.Equal(t, "latest", jrq.Params[1], "translateLatestTag: false keeps latest on the request") + + resp, err := network.Forward(ctx, req) + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse(ctx) + require.NoError(t, err) + result = jrr.GetResultString() + resp.Release() + require.Equal(t, "latest", jrq.Params[1], "the request keeps latest after a pinned attempt") + return result, pinned.get(), vendor.get() + } + + t.Run("PinnedAtHeadIsSentHeadAndAnswers", func(t *testing.T) { + result, pinnedTags, vendorTags := run(t, head, `{"jsonrpc":"2.0","id":1,"result":"0x11"}`) + assert.Equal(t, `"0x11"`, result) + assert.Equal(t, []interface{}{headHex}, pinnedTags) + assert.Empty(t, vendorTags) + }) + + t.Run("PinnedBehindHeadIsSkipped", func(t *testing.T) { + result, pinnedTags, vendorTags := run(t, head-2, `{"jsonrpc":"2.0","id":1,"result":"0x11"}`) + assert.Equal(t, `"0x22"`, result) + assert.Empty(t, pinnedTags) + assert.Equal(t, []interface{}{"latest"}, vendorTags) + }) + + t.Run("PinnedMissingHeadFallsBackToVendor", func(t *testing.T) { + result, pinnedTags, vendorTags := run(t, head, + `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"header not found"}}`) + assert.Equal(t, `"0x22"`, result) + require.NotEmpty(t, pinnedTags) + for _, tag := range pinnedTags { + assert.Equal(t, headHex, tag) + } + assert.Equal(t, []interface{}{"latest"}, vendorTags) + }) +} diff --git a/erpc/subscription_manager.go b/erpc/subscription_manager.go index c7f0fa9e4..1eaa13db1 100644 --- a/erpc/subscription_manager.go +++ b/erpc/subscription_manager.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "time" "github.com/erpc/erpc/common" @@ -40,6 +41,10 @@ type SubscriptionManager struct { networks sync.Map // indexerNetworkKey -> struct{}, once bootstrapped bootstrapMu sync.Mutex + + // activeSubs counts client subscriptions on this instance for + // server.webSocket.maxSubscriptions. + activeSubs atomic.Int64 } type connEntry struct { @@ -98,6 +103,11 @@ func (sm *SubscriptionManager) Subscribe( if conn.adapter.Count() >= maxSubs { return nil, common.NewErrSubscriptionLimitExceeded(maxSubs) } + maxTotal := wsc.server.serverCfg.WebSocket.MaxSubscriptions + if maxTotal > 0 && sm.activeSubs.Load() >= int64(maxTotal) { + telemetry.CounterHandle(telemetry.MetricWebsocketRejectedTotal, project.Config.Id, wsRejectSubGlobal).Inc() + return nil, common.NewErrInstanceSubscriptionLimitExceeded(maxTotal) + } if err := sm.acquireRateLimits(ctx, project, nw, nq); err != nil { return nil, err @@ -130,11 +140,19 @@ func (sm *SubscriptionManager) Subscribe( return nil, err } + if !sm.reserveSub(maxTotal) { + sm.releaseFilter(ctx, key, kind, filterHash) + telemetry.CounterHandle(telemetry.MetricWebsocketRejectedTotal, project.Config.Id, wsRejectSubGlobal).Inc() + err := common.NewErrInstanceSubscriptionLimitExceeded(maxTotal) + sm.recordFailureMetrics(project, nw, method, reqFinality, start, nq, err) + return nil, err + } // Counted before it is visible, so a racing cleanup never takes the // gauge below zero. wsc.subscriptionsGauge(kind).Inc() if err := conn.adapter.AddSubscription(clientSubID, key, kind, filterHash, maxSubs); err != nil { wsc.subscriptionsGauge(kind).Dec() + sm.activeSubs.Add(-1) sm.releaseFilter(ctx, key, kind, filterHash) if errors.Is(err, wsclient.ErrLimitExceeded) { err = common.NewErrSubscriptionLimitExceeded(maxSubs) @@ -210,6 +228,7 @@ func (sm *SubscriptionManager) Unsubscribe( } wsc.subscriptionsGauge(kind).Dec() sm.releaseFilter(ctx, subNetworkID, kind, filterHash) + sm.activeSubs.Add(-1) lg.Info().Str("clientSubId", clientSubID).Str("subType", kind.String()).Msg("subscription removed") @@ -234,12 +253,24 @@ func (sm *SubscriptionManager) CleanupConnection(wsc *WsConnection) { for _, sub := range conn.adapter.Drain() { wsc.subscriptionsGauge(sub.Kind).Dec() sm.releaseFilter(ctx, sub.NetworkID, sub.Kind, sub.FilterHash) + sm.activeSubs.Add(-1) } conn.detach() sm.logger.Debug().Str("connId", wsc.id).Msg("cleaned up all subscriptions for connection") } +// reserveSub takes one slot of the instance-wide subscription cap (0 = +// unlimited); the caller gives it back with activeSubs.Add(-1). +func (sm *SubscriptionManager) reserveSub(maxTotal int) bool { + n := sm.activeSubs.Add(1) + if maxTotal > 0 && n > int64(maxTotal) { + sm.activeSubs.Add(-1) + return false + } + return true +} + // releaseFilter drops a subscription's filter reference; newHeads // subscriptions hold none. func (sm *SubscriptionManager) releaseFilter(ctx context.Context, networkID string, kind indexer.EventKind, filterHash string) { @@ -310,7 +341,7 @@ func (s *subIngressSelector) Select(_, _ string, _ []interface{}) (defaults, fal failoverOn := s.nw.cfg != nil && s.nw.cfg.Failover.Enabled() eligible := make(map[string]struct{}) if failoverOn { - for _, u := range s.nw.tipCandidateUpstreams(ctx, "*") { + for u := range s.nw.EvmRoutableUpstreams(ctx) { eligible[u.Id()] = struct{}{} } } @@ -470,6 +501,22 @@ func (h *networkHandle) DedupEntriesAdded(delta int) { telemetry.GaugeHandle(telemetry.MetricSubscriptionDedupEntries, h.nw.projectId, h.nw.Label()).Add(float64(delta)) } +func (h *networkHandle) HeadIndexed() { + telemetry.GaugeHandle(telemetry.MetricNetworkSubscriptionLastHeadTimestamp, h.nw.projectId, h.nw.Label()).SetToCurrentTime() +} + +func (h *networkHandle) Undelivered(kind indexer.EventKind) { + telemetry.CounterHandle(telemetry.MetricSubscriptionUndeliveredTotal, h.nw.projectId, h.nw.Label(), kind.String()).Inc() +} + +// DeliveringHead floors "latest" to a head a client is about to receive. Only +// such heads count: a head no client subscribed to (a prewarmed subscription) +// would otherwise lift "latest" past the upstreams that poll, and their +// blockAvailability upper bound would then mark them unable to serve it. +func (h *networkHandle) DeliveringHead(blockNumber int64) { + h.nw.NoteObservedLatestBlock(h.nw.appCtx, blockNumber) +} + // indexerNetworkKey names nw in the indexer and its adapters. Projects can // define the same network, each with its own upstreams, so the key includes // the project. @@ -481,9 +528,16 @@ func indexerNetworkKey(nw *Network) string { return nw.projectId + "/" + nw.netw // asynchronously first, so that head is not delivered) and the fallback-tier // gate allows it (see deliversHeadsFrom). A head from an upstream with no // poller (not bootstrapped, e.g. its chain could not be verified) or one the -// network doesn't know is not delivered. A delivered head also advances the -// network's delivered-head floor before clients see it. +// network doesn't know is not delivered. func (h *networkHandle) SuggestLatestBlock(sourceId string, blockNumber int64) bool { + deliver := h.suggestLatestBlock(sourceId, blockNumber) + if !deliver { + telemetry.CounterHandle(telemetry.MetricSubscriptionWithheldHeadsTotal, h.nw.projectId, h.nw.Label()).Inc() + } + return deliver +} + +func (h *networkHandle) suggestLatestBlock(sourceId string, blockNumber int64) bool { upstreamID, ok := strings.CutPrefix(sourceId, "ws:") if !ok { return true @@ -499,11 +553,7 @@ func (h *networkHandle) SuggestLatestBlock(sourceId string, blockNumber int64) b return false } poller.SuggestLatestBlock(blockNumber) - if !deliver || poller.LatestBlock() < blockNumber { - return false - } - h.nw.NoteObservedLatestBlock(h.nw.appCtx, blockNumber) - return true + return deliver && poller.LatestBlock() >= blockNumber } return false } @@ -519,7 +569,7 @@ func (h *networkHandle) deliversHeadsFrom(ctx context.Context, u common.Upstream if h.nw.cfg == nil || !h.nw.cfg.Failover.Enabled() || !isFallbackTier(u) { return true } - for _, c := range h.nw.tipCandidateUpstreams(ctx, "*") { + for c := range h.nw.EvmRoutableUpstreams(ctx) { if s := h.heads[c.Id()]; s != nil && !isFallbackTier(c) && s.HeadsLive() { return false } diff --git a/erpc/svm_e2e_test.go b/erpc/svm_e2e_test.go index 87dfb53af..b3182c66e 100644 --- a/erpc/svm_e2e_test.go +++ b/erpc/svm_e2e_test.go @@ -73,6 +73,78 @@ func setupTestSvmNetwork(t *testing.T, ctx context.Context, upstreams []*common. return network } +// setupTestSvmNetworkWithConsensus is like setupTestSvmNetwork but attaches a +// 2-of-N consensus policy scoped to the given finality states (e.g. Realtime for +// getSignaturesForAddress). +func setupTestSvmNetworkWithConsensus( + t *testing.T, + ctx context.Context, + upstreams []*common.UpstreamConfig, + matchFinality []common.DataFinalityState, +) *Network { + t.Helper() + + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) + metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) + + vr := thirdparty.NewVendorsRegistry() + pr, err := thirdparty.NewProvidersRegistry(&log.Logger, vr, []*common.ProviderConfig{}, nil) + require.NoError(t, err) + + sharedStateCfg := &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: common.DriverMemory, + Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}, + }, + LockMaxWait: common.Duration(200 * time.Millisecond), + UpdateMaxWait: common.Duration(200 * time.Millisecond), + FallbackTimeout: common.Duration(3 * time.Second), + LockTtl: common.Duration(4 * time.Second), + } + sharedStateCfg.SetDefaults("test") + ssr, err := data.NewSharedStateRegistry(ctx, &log.Logger, sharedStateCfg) + require.NoError(t, err) + + upstreamsRegistry := upstream.NewUpstreamsRegistry( + ctx, &log.Logger, "test", + upstreams, ssr, rateLimitersRegistry, vr, pr, + nil, metricsTracker, nil, + ) + + participants := len(upstreams) + if participants < 2 { + participants = 2 + } + netCfg := &common.NetworkConfig{ + Architecture: common.ArchitectureSvm, + Svm: &common.SvmNetworkConfig{ + Cluster: "mainnet-beta", + Commitment: "confirmed", + }, + Failsafe: []*common.FailsafeConfig{{ + MatchMethod: "*", + MatchFinality: matchFinality, + Consensus: &common.ConsensusPolicyConfig{ + MaxParticipants: participants, + AgreementThreshold: 2, + }, + }}, + } + require.NoError(t, netCfg.SetDefaults(upstreams, nil)) + + network, err := NewNetwork( + ctx, &log.Logger, "test", netCfg, + rateLimitersRegistry, upstreamsRegistry, metricsTracker, nil, + ) + require.NoError(t, err) + + upstreamsRegistry.Bootstrap(ctx) + time.Sleep(100 * time.Millisecond) + require.NoError(t, upstreamsRegistry.PrepareUpstreamsForNetwork(ctx, util.SvmNetworkId("", "mainnet-beta"))) + time.Sleep(150 * time.Millisecond) + return network +} + func svmUpstreamConfig(id, host string) *common.UpstreamConfig { return &common.UpstreamConfig{ Id: id, diff --git a/erpc/svm_history_sweep_test.go b/erpc/svm_history_sweep_test.go new file mode 100644 index 000000000..594816c07 --- /dev/null +++ b/erpc/svm_history_sweep_test.go @@ -0,0 +1,381 @@ +package erpc + +import ( + "context" + "fmt" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/upstream" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/stretchr/testify/require" +) + +func init() { + util.ConfigureTestLogger() +} + +// mockSvmMethod registers a persistent reply for `method` on host and returns a +// counter of how many times the upstream actually answered it. +func mockSvmMethod(host, method, body string) *atomic.Int64 { + var calls atomic.Int64 + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return r.URL.Host == host && + strings.Contains(util.SafeReadBody(r), `"method":"`+method+`"`) + }). + Reply(200). + Map(func(res *http.Response) *http.Response { + calls.Add(1) + return res + }). + BodyString(body) + return &calls +} + +func svmSigPage(n int) string { + items := make([]string, n) + for i := range items { + items[i] = fmt.Sprintf(`{"signature":"sig%d","slot":%d,"err":null}`, i, 100+i) + } + return `{"jsonrpc":"2.0","id":1,"result":[` + strings.Join(items, ",") + `]}` +} + +// The archive's deeper known floor puts it first despite the pinned score order, +// so the full page arrives in one call and the pruned validator is never asked. +func TestSvm_GetSignaturesForAddress_DeepestFloorAnswersFirst(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForSvmStatePollerWithFirstAvailable("svm-sigs-rpc1.localhost", 1000, 990, 1000, 500) + util.SetupMocksForSvmStatePollerWithFirstAvailable("svm-sigs-rpc2.localhost", 1000, 990, 1000, 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // rpc1 is a pruned validator (floor 500): its history for the address ends + // after 2. rpc2 is the archive (floor 0). + short := mockSvmMethod("svm-sigs-rpc1.localhost", "getSignaturesForAddress", svmSigPage(2)) + full := mockSvmMethod("svm-sigs-rpc2.localhost", "getSignaturesForAddress", svmSigPage(5)) + + net := setupTestSvmNetwork(t, ctx, []*common.UpstreamConfig{ + svmUpstreamConfig("rpc1", "svm-sigs-rpc1.localhost"), + svmUpstreamConfig("rpc2", "svm-sigs-rpc2.localhost"), + }) + net.PinUpstreamOrderForTest("rpc1", "rpc2") + time.Sleep(300 * time.Millisecond) // let the floor poll land + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSignaturesForAddress","params":["addr",{"limit":5}]}`)) + resp, err := svmProjectForward(ctx, net, req) + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.Contains(t, string(jrr.GetResultBytes()), `"sig4"`, "the full page from the archive must win") + require.Zero(t, short.Load(), "the shallower upstream is not tried before the deepest one") + require.EqualValues(t, 1, full.Load()) +} + +func TestSvm_GetSignaturesForAddress_FullPageStopsSweep(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForSvmStatePoller("svm-sigsfull-rpc1.localhost", 1000, 990) + util.SetupMocksForSvmStatePoller("svm-sigsfull-rpc2.localhost", 1000, 990) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + first := mockSvmMethod("svm-sigsfull-rpc1.localhost", "getSignaturesForAddress", svmSigPage(5)) + second := mockSvmMethod("svm-sigsfull-rpc2.localhost", "getSignaturesForAddress", svmSigPage(5)) + + net := setupTestSvmNetwork(t, ctx, []*common.UpstreamConfig{ + svmUpstreamConfig("rpc1", "svm-sigsfull-rpc1.localhost"), + svmUpstreamConfig("rpc2", "svm-sigsfull-rpc2.localhost"), + }) + net.PinUpstreamOrderForTest("rpc1", "rpc2") + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSignaturesForAddress","params":["addr",{"limit":5}]}`)) + resp, err := svmProjectForward(ctx, net, req) + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, int64(1), first.Load()) + require.Zero(t, second.Load(), "a full page must not trigger the sweep") +} + +// Equal floors: a short page is the normal end of history, so the first +// upstream's answer is final and no other upstream is called. +func TestSvm_GetSignaturesForAddress_ShortPageAtEqualFloorsIsServed(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + hosts := []string{"svm-sigseq-rpc1.localhost", "svm-sigseq-rpc2.localhost", "svm-sigseq-rpc3.localhost"} + for _, h := range hosts { + util.SetupMocksForSvmStatePoller(h, 1000, 990) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + calls := make([]*atomic.Int64, len(hosts)) + cfgs := make([]*common.UpstreamConfig, len(hosts)) + for i, h := range hosts { + calls[i] = mockSvmMethod(h, "getSignaturesForAddress", svmSigPage(2)) + cfgs[i] = svmUpstreamConfig(fmt.Sprintf("rpc%d", i+1), h) + } + net := setupTestSvmNetwork(t, ctx, cfgs) + net.PinUpstreamOrderForTest("rpc1", "rpc2", "rpc3") + time.Sleep(300 * time.Millisecond) // let the floor poll land + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSignaturesForAddress","params":["addr",{"limit":5}]}`)) + resp, err := svmProjectForward(ctx, net, req) + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.Contains(t, string(jrr.GetResultBytes()), `"sig1"`) + require.Equal(t, int64(1), calls[0].Load()) + require.Zero(t, calls[1].Load()+calls[2].Load(), "equal floors give no reason to call another upstream") +} + +func TestSvm_GetSignaturesForAddress_AllShortServesLastValidPage(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForSvmStatePollerWithFirstAvailable("svm-sigsall-rpc1.localhost", 1000, 990, 1000, 500) + util.SetupMocksForSvmStatePollerWithFirstAvailable("svm-sigsall-rpc2.localhost", 1000, 990, 1000, 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // An address with only 2 signatures is legitimately short everywhere: the + // client still gets the answer, not an error. rpc1 is pruned (floor 500) and + // sweeps on; rpc2 holds the deepest history (floor 0), so its short page is + // authoritative. + mockSvmMethod("svm-sigsall-rpc1.localhost", "getSignaturesForAddress", svmSigPage(2)) + mockSvmMethod("svm-sigsall-rpc2.localhost", "getSignaturesForAddress", svmSigPage(2)) + + net := setupTestSvmNetwork(t, ctx, []*common.UpstreamConfig{ + svmUpstreamConfig("rpc1", "svm-sigsall-rpc1.localhost"), + svmUpstreamConfig("rpc2", "svm-sigsall-rpc2.localhost"), + }) + net.PinUpstreamOrderForTest("rpc1", "rpc2") + time.Sleep(300 * time.Millisecond) // let the floor poll land + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSignaturesForAddress","params":["addr"]}`)) + resp, err := svmProjectForward(ctx, net, req) + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.Contains(t, string(jrr.GetResultBytes()), `"sig1"`) +} + +const ( + svmSigStatusFound = `{"jsonrpc":"2.0","id":1,"result":{"context":{"slot":900},"value":[{"slot":5,"confirmations":null,"err":null,"confirmationStatus":"finalized"}]}}` + svmSigStatusNull = `{"jsonrpc":"2.0","id":1,"result":{"context":{"slot":900},"value":[null]}}` +) + +func TestSvm_GetSignatureStatuses_SearchHistoryNullContinuesToNextUpstream(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForSvmStatePollerWithFirstAvailable("svm-status-rpc1.localhost", 1000, 990, 1000, 500) + util.SetupMocksForSvmStatePollerWithFirstAvailable("svm-status-rpc2.localhost", 1000, 990, 1000, 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mockSvmMethod("svm-status-rpc1.localhost", "getSignatureStatuses", svmSigStatusNull) + archive := mockSvmMethod("svm-status-rpc2.localhost", "getSignatureStatuses", svmSigStatusFound) + + net := setupTestSvmNetwork(t, ctx, []*common.UpstreamConfig{ + svmUpstreamConfig("rpc1", "svm-status-rpc1.localhost"), + svmUpstreamConfig("rpc2", "svm-status-rpc2.localhost"), + }) + net.PinUpstreamOrderForTest("rpc1", "rpc2") + time.Sleep(300 * time.Millisecond) // let the floor poll land + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSignatureStatuses","params":[["sig"],{"searchTransactionHistory":true}]}`)) + resp, err := svmProjectForward(ctx, net, req) + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.Contains(t, string(jrr.GetResultBytes()), `"finalized"`) + require.GreaterOrEqual(t, archive.Load(), int64(1)) +} + +func TestSvm_GetSignatureStatuses_WithoutFlagNullIsServed(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForSvmStatePoller("svm-statusoff-rpc1.localhost", 1000, 990) + util.SetupMocksForSvmStatePoller("svm-statusoff-rpc2.localhost", 1000, 990) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mockSvmMethod("svm-statusoff-rpc1.localhost", "getSignatureStatuses", svmSigStatusNull) + second := mockSvmMethod("svm-statusoff-rpc2.localhost", "getSignatureStatuses", svmSigStatusFound) + + net := setupTestSvmNetwork(t, ctx, []*common.UpstreamConfig{ + svmUpstreamConfig("rpc1", "svm-statusoff-rpc1.localhost"), + svmUpstreamConfig("rpc2", "svm-statusoff-rpc2.localhost"), + }) + net.PinUpstreamOrderForTest("rpc1", "rpc2") + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSignatureStatuses","params":[["sig"]]}`)) + resp, err := svmProjectForward(ctx, net, req) + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.Contains(t, string(jrr.GetResultBytes()), `[null]`, "a null is a valid recent-status answer") + require.Zero(t, second.Load(), "no sweep without searchTransactionHistory") +} + +// The shortest page must never beat the most complete one. The archive (floor 0) +// answers first with 4 of limit 5 (short because the address only has 4); the +// pruned validator (floor 500) would answer 2. The archive holds the pool's +// deepest history, so its short page is the answer and the sweep stops there. +func TestSvm_GetSignaturesForAddress_DeepestUpstreamShortPageIsAuthoritative(t *testing.T) { + for _, order := range [][2]string{{"archive", "pruned"}, {"pruned", "archive"}} { + order := order + t.Run(order[0]+"-first", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + archiveHost := "svm-sigsdeep-archive-" + order[0] + ".localhost" + prunedHost := "svm-sigsdeep-pruned-" + order[0] + ".localhost" + util.SetupMocksForSvmStatePollerWithFirstAvailable(archiveHost, 1000, 990, 1000, 0) + util.SetupMocksForSvmStatePollerWithFirstAvailable(prunedHost, 1000, 990, 1000, 500) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + archive := mockSvmMethod(archiveHost, "getSignaturesForAddress", svmSigPage(4)) + pruned := mockSvmMethod(prunedHost, "getSignaturesForAddress", svmSigPage(2)) + + ids := map[string]string{"archive": archiveHost, "pruned": prunedHost} + net := setupTestSvmNetwork(t, ctx, []*common.UpstreamConfig{ + svmUpstreamConfig(order[0], ids[order[0]]), + svmUpstreamConfig(order[1], ids[order[1]]), + }) + net.PinUpstreamOrderForTest(order[0], order[1]) + time.Sleep(300 * time.Millisecond) // let the floor poll land + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSignaturesForAddress","params":["addr",{"limit":5}]}`)) + resp, err := svmProjectForward(ctx, net, req) + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.Contains(t, string(jrr.GetResultBytes()), `"sig3"`, "the archive's 4-entry page must be served") + require.GreaterOrEqual(t, archive.Load(), int64(1)) + if order[0] == "archive" { + require.Zero(t, pruned.Load(), "the deepest upstream's page ends the sweep") + } + }) + } +} + +// Same for getSignatureStatuses: the archive's [found, null] (a signature that +// genuinely never landed) must not be replaced by the pruned node's [null, null]. +func TestSvm_GetSignatureStatuses_DeepestUpstreamNullIsAuthoritative(t *testing.T) { + const archiveBody = `{"jsonrpc":"2.0","id":1,"result":{"context":{"slot":900},"value":[{"slot":5,"confirmations":null,"err":null,"confirmationStatus":"finalized"},null]}}` + const prunedBody = `{"jsonrpc":"2.0","id":1,"result":{"context":{"slot":900},"value":[null,null]}}` + for _, order := range [][2]string{{"archive", "pruned"}, {"pruned", "archive"}} { + order := order + t.Run(order[0]+"-first", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + archiveHost := "svm-statusdeep-archive-" + order[0] + ".localhost" + prunedHost := "svm-statusdeep-pruned-" + order[0] + ".localhost" + util.SetupMocksForSvmStatePollerWithFirstAvailable(archiveHost, 1000, 990, 1000, 0) + util.SetupMocksForSvmStatePollerWithFirstAvailable(prunedHost, 1000, 990, 1000, 500) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mockSvmMethod(archiveHost, "getSignatureStatuses", archiveBody) + mockSvmMethod(prunedHost, "getSignatureStatuses", prunedBody) + + ids := map[string]string{"archive": archiveHost, "pruned": prunedHost} + net := setupTestSvmNetwork(t, ctx, []*common.UpstreamConfig{ + svmUpstreamConfig(order[0], ids[order[0]]), + svmUpstreamConfig(order[1], ids[order[1]]), + }) + net.PinUpstreamOrderForTest(order[0], order[1]) + time.Sleep(300 * time.Millisecond) // let the floor poll land + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSignatureStatuses","params":[["a","b"],{"searchTransactionHistory":true}]}`)) + resp, err := svmProjectForward(ctx, net, req) + require.NoError(t, err) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.Contains(t, string(jrr.GetResultBytes()), `"finalized"`, "the archive's [found, null] must be served") + }) + } +} + +// When rpc1 (shallow) and rpc2 (deepest) return the same legitimately short page, +// rpc1's sweep-continuation MissingData must not record wrong-empty misbehavior. +func TestSvm_HistorySweep_NoMisbehaviorForIdenticalShortPages(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForSvmStatePollerWithFirstAvailable("svm-sigsmb-rpc1.localhost", 1000, 990, 1000, 500) + util.SetupMocksForSvmStatePollerWithFirstAvailable("svm-sigsmb-rpc2.localhost", 1000, 990, 1000, 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mockSvmMethod("svm-sigsmb-rpc1.localhost", "getSignaturesForAddress", svmSigPage(2)) + mockSvmMethod("svm-sigsmb-rpc2.localhost", "getSignaturesForAddress", svmSigPage(2)) + + net := setupTestSvmNetwork(t, ctx, []*common.UpstreamConfig{ + svmUpstreamConfig("rpc1", "svm-sigsmb-rpc1.localhost"), + svmUpstreamConfig("rpc2", "svm-sigsmb-rpc2.localhost"), + }) + net.PinUpstreamOrderForTest("rpc1", "rpc2") + time.Sleep(300 * time.Millisecond) + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSignaturesForAddress","params":["addr",{"limit":5}]}`)) + resp, err := svmProjectForward(ctx, net, req) + require.NoError(t, err) + require.NotNil(t, resp) + + var rpc1 *upstream.Upstream + for _, u := range net.upstreamsRegistry.GetAllUpstreams() { + if u.Id() == "rpc1" { + rpc1 = u + break + } + } + require.NotNil(t, rpc1) + metrics := net.metricsTracker.GetUpstreamMethodMetrics(rpc1, "getSignaturesForAddress", common.DataFinalityStateRealtime) + require.Zero(t, metrics.MisbehaviorsTotal.Load(), "the history sweep must not punish an upstream that returned the same page as the winner") +} + +// Consensus participants vote on the bodies they returned: the shallower +// participant's short page must stay a 200, or consensus agrees on an error. +func TestSvm_Consensus_AllShortSignaturePagesAgreeOnSuccess(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForSvmStatePollerWithFirstAvailable("svm-cons-sigs-a.localhost", 1000, 990, 1000, 500) + util.SetupMocksForSvmStatePollerWithFirstAvailable("svm-cons-sigs-b.localhost", 1000, 990, 1000, 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + shortBody := svmSigPage(2) + mockSvmMethod("svm-cons-sigs-a.localhost", "getSignaturesForAddress", shortBody) + mockSvmMethod("svm-cons-sigs-b.localhost", "getSignaturesForAddress", shortBody) + + net := setupTestSvmNetworkWithConsensus(t, ctx, []*common.UpstreamConfig{ + svmUpstreamConfig("a", "svm-cons-sigs-a.localhost"), + svmUpstreamConfig("b", "svm-cons-sigs-b.localhost"), + }, []common.DataFinalityState{common.DataFinalityStateRealtime}) + time.Sleep(300 * time.Millisecond) // let the floor poll land + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"getSignaturesForAddress","params":["addr",{"limit":5}]}`)) + resp, err := svmProjectForward(ctx, net, req) + require.NoError(t, err, "identical short pages must consensus-agree as success, not MissingData") + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.Contains(t, string(jrr.GetResultBytes()), `"sig1"`) +} diff --git a/erpc/svm_hot_path_test.go b/erpc/svm_hot_path_test.go index f8bff242e..09e25d6b7 100644 --- a/erpc/svm_hot_path_test.go +++ b/erpc/svm_hot_path_test.go @@ -23,8 +23,8 @@ func svmUpstreamCfg(id, host string) *common.UpstreamConfig { } } -// With no declared firstAvailableSlotPlus the floor is neither probed nor -// enforced, so every slot-named read — point or range — reaches the upstream. +// Every upstream's floor is probed, but without a declared firstAvailableSlotPlus +// it is never enforced, so every slot-named read — point or range — reaches the upstream. func TestSvm_ObservedFloor_NeverSkipsWithoutDeclaredBound(t *testing.T) { util.ResetGock() defer util.ResetGock() @@ -46,7 +46,7 @@ func TestSvm_ObservedFloor_NeverSkipsWithoutDeclaredBound(t *testing.T) { for i := range ups { selected[i] = ups[i] _, known := ups[i].SvmStatePoller().FirstAvailableSlot() - require.Falsef(t, known, "%s declared no firstAvailableSlotPlus, so its floor is never polled", ups[i].Id()) + require.Truef(t, known, "%s floor is probed even without a declared bound", ups[i].Id()) } for _, body := range []struct{ method, req string }{ @@ -64,7 +64,7 @@ func TestSvm_ObservedFloor_NeverSkipsWithoutDeclaredBound(t *testing.T) { } } -// firstAvailableSlotPlus is the opt-in: the operator anchors a declared window +// firstAvailableSlotPlus is what enforces the floor: the operator anchors a declared window // on the observed floor, and only that upstream is skipped below it. An // upstream declaring exactSlot: 0 overrides the vendor's higher reported floor. func TestSvm_FirstAvailableSlotPlus_OptInUsesObservedFloor(t *testing.T) { @@ -98,10 +98,11 @@ func TestSvm_FirstAvailableSlotPlus_OptInUsesObservedFloor(t *testing.T) { } floor, known := byId["validator"].(common.SvmUpstream).SvmStatePoller().FirstAvailableSlot() - require.True(t, known, "the opted-in upstream's floor must be polled") + require.True(t, known, "the declaring upstream's floor is observed") require.EqualValues(t, 200000, floor) - _, known = byId["archive"].(common.SvmUpstream).SvmStatePoller().FirstAvailableSlot() - require.False(t, known, "exactSlot needs no floor, so none is polled") + floor, known = byId["archive"].(common.SvmUpstream).SvmStatePoller().FirstAvailableSlot() + require.True(t, known, "every upstream's floor is probed") + require.EqualValues(t, 200000, floor, "the archive's vendor-reported floor is known but overridden by exactSlot") below := common.NewNormalizedRequest( []byte(`{"jsonrpc":"2.0","id":1,"method":"getBlock","params":[1000]}`)) diff --git a/erpc/ws_conn_limiter.go b/erpc/ws_conn_limiter.go new file mode 100644 index 000000000..802197e89 --- /dev/null +++ b/erpc/ws_conn_limiter.go @@ -0,0 +1,72 @@ +package erpc + +import "sync" + +// wsConnLimiter bounds concurrent client WebSocket connections on this +// instance, in total and per client IP; a zero limit is unlimited. A nil +// limiter admits everything. +type wsConnLimiter struct { + mu sync.Mutex + total int + perIP map[string]int + maxTotal int + maxPerIP int +} + +const ( + wsRejectConnGlobal = "conn_global" + wsRejectConnPerIP = "conn_per_ip" + wsRejectSubGlobal = "sub_global" +) + +func newWsConnLimiter(maxTotal, maxPerIP int) *wsConnLimiter { + if maxTotal <= 0 && maxPerIP <= 0 { + return nil + } + return &wsConnLimiter{perIP: make(map[string]int), maxTotal: maxTotal, maxPerIP: maxPerIP} +} + +// acquire reserves a slot for ip, or returns the rejection reason. Every +// successful acquire must be paired with exactly one release(ip). An +// unresolved IP ("" or "n/a") is only subject to the total cap. +func (l *wsConnLimiter) acquire(ip string) (reason string, ok bool) { + if l == nil { + return "", true + } + l.mu.Lock() + defer l.mu.Unlock() + if l.maxTotal > 0 && l.total >= l.maxTotal { + return wsRejectConnGlobal, false + } + tracked := wsIPTracked(ip) + if tracked && l.maxPerIP > 0 && l.perIP[ip] >= l.maxPerIP { + return wsRejectConnPerIP, false + } + l.total++ + if tracked { + l.perIP[ip]++ + } + return "", true +} + +func (l *wsConnLimiter) release(ip string) { + if l == nil { + return + } + l.mu.Lock() + defer l.mu.Unlock() + if l.total > 0 { + l.total-- + } + if !wsIPTracked(ip) { + return + } + // Delete at zero so a stream of distinct IPs can't grow the map. + if n := l.perIP[ip]; n <= 1 { + delete(l.perIP, ip) + } else { + l.perIP[ip] = n - 1 + } +} + +func wsIPTracked(ip string) bool { return ip != "" && ip != "n/a" } diff --git a/erpc/ws_conn_limiter_test.go b/erpc/ws_conn_limiter_test.go new file mode 100644 index 000000000..a83072cd2 --- /dev/null +++ b/erpc/ws_conn_limiter_test.go @@ -0,0 +1,188 @@ +package erpc + +import ( + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" + "github.com/gorilla/websocket" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWsConnLimiter(t *testing.T) { + t.Run("NilAdmitsEverything", func(t *testing.T) { + l := newWsConnLimiter(0, 0) + require.Nil(t, l) + for i := 0; i < 3; i++ { + _, ok := l.acquire("1.2.3.4") + assert.True(t, ok) + } + l.release("1.2.3.4") + }) + + t.Run("GlobalCap", func(t *testing.T) { + l := newWsConnLimiter(2, 0) + _, ok := l.acquire("1.1.1.1") + require.True(t, ok) + _, ok = l.acquire("2.2.2.2") + require.True(t, ok) + reason, ok := l.acquire("3.3.3.3") + assert.False(t, ok) + assert.Equal(t, wsRejectConnGlobal, reason) + l.release("1.1.1.1") + _, ok = l.acquire("3.3.3.3") + assert.True(t, ok) + }) + + t.Run("PerIPCap", func(t *testing.T) { + l := newWsConnLimiter(0, 1) + _, ok := l.acquire("1.1.1.1") + require.True(t, ok) + reason, ok := l.acquire("1.1.1.1") + assert.False(t, ok) + assert.Equal(t, wsRejectConnPerIP, reason) + _, ok = l.acquire("2.2.2.2") + assert.True(t, ok) + }) + + t.Run("UnresolvedIPOnlyCountsTowardsTotal", func(t *testing.T) { + l := newWsConnLimiter(3, 1) + for i := 0; i < 3; i++ { + _, ok := l.acquire("n/a") + require.True(t, ok) + } + reason, ok := l.acquire("n/a") + assert.False(t, ok) + assert.Equal(t, wsRejectConnGlobal, reason) + assert.Empty(t, l.perIP) + }) + + t.Run("ReleaseReclaimsPerIPEntries", func(t *testing.T) { + l := newWsConnLimiter(0, 2) + for i := 0; i < 100; i++ { + ip := fmt.Sprintf("10.0.0.%d", i) + _, ok := l.acquire(ip) + require.True(t, ok) + l.release(ip) + } + assert.Empty(t, l.perIP) + assert.Zero(t, l.total) + }) +} + +func dialWsExpectStatus(t *testing.T, addr string) (*websocket.Conn, int) { + t.Helper() + conn, resp, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/123", addr), nil) + if err != nil { + require.NotNil(t, resp, "dial failed without an HTTP response: %v", err) + return nil, resp.StatusCode + } + return conn, resp.StatusCode +} + +func TestWebSocket_ConnectionCaps(t *testing.T) { + setupGock() + defer util.ResetGock() + + cases := []struct { + name string + ws *common.WebSocketServerConfig + reason string + }{ + {"Global", &common.WebSocketServerConfig{MaxConnections: 1}, wsRejectConnGlobal}, + {"PerIP", &common.WebSocketServerConfig{MaxConnectionsPerIP: 1}, wsRejectConnPerIP}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := httpOnlyConfig() + cfg.Server.WebSocket = tc.ws + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + rejected := telemetry.MetricWebsocketRejectedTotal.WithLabelValues("test_ws", tc.reason) + base := testutil.ToFloat64(rejected) + + first, status := dialWsExpectStatus(t, addr) + require.NotNil(t, first) + assert.Equal(t, http.StatusSwitchingProtocols, status) + + second, status := dialWsExpectStatus(t, addr) + assert.Nil(t, second) + assert.Equal(t, http.StatusTooManyRequests, status) + assert.Equal(t, base+1, testutil.ToFloat64(rejected)) + + // Closing frees the slot. + first.Close() + require.Eventually(t, func() bool { + c, _ := dialWsExpectStatus(t, addr) + if c == nil { + return false + } + c.Close() + return true + }, 5*time.Second, 50*time.Millisecond) + }) + } +} + +func TestWebSocket_InstanceSubscriptionCap(t *testing.T) { + mockUpstream := mockWsUpstream(t, func(conn *websocket.Conn) { + n := 0 + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + switch method { + case "eth_subscribe": + n++ + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": fmt.Sprintf("0xsub%d", n)}) + default: + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": true}) + } + }) + }) + defer mockUpstream.Close() + + setupGock() + defer util.ResetGock() + + cfg := standardWsConfig("ws" + strings.TrimPrefix(mockUpstream.URL, "http")) + cfg.Server.WebSocket = &common.WebSocketServerConfig{MaxSubscriptions: 1} + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + time.Sleep(2 * time.Second) + + rejected := telemetry.MetricWebsocketRejectedTotal.WithLabelValues("test_ws", wsRejectSubGlobal) + base := testutil.ToFloat64(rejected) + + conn1 := dialWs(t, addr) + defer conn1.Close() + conn2 := dialWs(t, addr) + defer conn2.Close() + + resp := sendAndReceive(t, conn1, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.Nil(t, resp["error"]) + subID := resp["result"].(string) + + resp = sendAndReceive(t, conn2, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.NotNil(t, resp["error"], "second subscription on the instance must be refused") + assert.Contains(t, fmt.Sprint(resp["error"]), "per instance") + assert.Equal(t, base+1, testutil.ToFloat64(rejected)) + + resp = sendAndReceive(t, conn1, fmt.Sprintf(`{"jsonrpc":"2.0","id":2,"method":"eth_unsubscribe","params":["%s"]}`, subID)) + require.Equal(t, true, resp["result"]) + + resp = sendAndReceive(t, conn2, `{"jsonrpc":"2.0","id":2,"method":"eth_subscribe","params":["newHeads"]}`) + assert.Nil(t, resp["error"], "unsubscribe must free the slot") + + // A closed connection gives its subscriptions back. + conn2.Close() + require.Eventually(t, func() bool { + resp := sendAndReceive(t, conn1, `{"jsonrpc":"2.0","id":3,"method":"eth_subscribe","params":["newHeads"]}`) + return resp["error"] == nil + }, 5*time.Second, 100*time.Millisecond) +} diff --git a/erpc/ws_indexer_metrics_test.go b/erpc/ws_indexer_metrics_test.go new file mode 100644 index 000000000..1e954500f --- /dev/null +++ b/erpc/ws_indexer_metrics_test.go @@ -0,0 +1,62 @@ +package erpc + +import ( + "fmt" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" + "github.com/gorilla/websocket" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// pushingHeadsUpstream answers eth_subscribe and then pushes one head. +func pushingHeadsUpstream(t *testing.T) *httptest.Server { + return mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + if method != "eth_subscribe" { + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": true}) + return + } + subID := fmt.Sprintf("0xheads%v", id) + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": subID}) + go func() { + time.Sleep(300 * time.Millisecond) + mockWriteJSON(conn, map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_subscription", + "params": map[string]interface{}{ + "subscription": subID, + "result": map[string]interface{}{"number": "0x101", "hash": "0xaaa"}, + }, + }) + }() + }) + }) +} + +func TestWebSocket_LastHeadTimestampCarriesProjectAndNetwork(t *testing.T) { + mockUpstream := pushingHeadsUpstream(t) + defer mockUpstream.Close() + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, standardWsConfig("ws"+strings.TrimPrefix(mockUpstream.URL, "http"))) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.Nil(t, resp["error"]) + + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + _, _, err := conn.ReadMessage() + require.NoError(t, err, "head notification") + + assert.Positive(t, testutil.ToFloat64(telemetry.MetricNetworkSubscriptionLastHeadTimestamp.WithLabelValues("test_ws", "evm:123"))) +} diff --git a/erpc/ws_server.go b/erpc/ws_server.go index 27583f56e..ad8cf575b 100644 --- a/erpc/ws_server.go +++ b/erpc/ws_server.go @@ -85,6 +85,16 @@ func (s *HttpServer) handleWebSocket( ) { wsCfg := s.serverCfg.WebSocket + // Checked before the upgrade so a refused client gets a plain 429. + clientIP := s.resolveRealClientIP(r) + if reason, ok := s.wsConnLimiter.acquire(clientIP); !ok { + lg.Warn().Str("reason", reason).Msg("websocket connection refused: connection limit reached") + telemetry.CounterHandle(telemetry.MetricWebsocketRejectedTotal, project.Config.Id, reason).Inc() + http.Error(w, "too many websocket connections", http.StatusTooManyRequests) + return + } + defer s.wsConnLimiter.release(clientIP) + upgrader := websocket.Upgrader{ ReadBufferSize: wsCfg.ReadBufferSize, WriteBufferSize: wsCfg.WriteBufferSize, diff --git a/failsafe/backoff_test.go b/failsafe/backoff_test.go new file mode 100644 index 000000000..d7f48ef44 --- /dev/null +++ b/failsafe/backoff_test.go @@ -0,0 +1,82 @@ +package failsafe + +import ( + "context" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestComputeBackoff(t *testing.T) { + ms := func(n int) common.Duration { return common.Duration(time.Duration(n) * time.Millisecond) } + cases := []struct { + name string + cfg *common.RetryPolicyConfig + attempt int + want time.Duration + }{ + {"nil config", nil, 3, 0}, + {"zero delay", &common.RetryPolicyConfig{BackoffFactor: 2}, 3, 0}, + {"negative delay", &common.RetryPolicyConfig{Delay: ms(-10)}, 0, 0}, + {"fixed delay without factor", &common.RetryPolicyConfig{Delay: ms(100)}, 5, 100 * time.Millisecond}, + {"first retry ignores factor", &common.RetryPolicyConfig{Delay: ms(100), BackoffFactor: 2}, 0, 100 * time.Millisecond}, + {"exponential growth", &common.RetryPolicyConfig{Delay: ms(100), BackoffFactor: 2}, 3, 800 * time.Millisecond}, + {"fractional factor", &common.RetryPolicyConfig{Delay: ms(100), BackoffFactor: 1.5}, 2, 225 * time.Millisecond}, + {"capped at max delay", &common.RetryPolicyConfig{Delay: ms(100), BackoffFactor: 2, BackoffMaxDelay: ms(300)}, 3, 300 * time.Millisecond}, + {"below cap untouched", &common.RetryPolicyConfig{Delay: ms(100), BackoffFactor: 2, BackoffMaxDelay: ms(300)}, 1, 200 * time.Millisecond}, + {"cap applies to fixed delay", &common.RetryPolicyConfig{Delay: ms(500), BackoffMaxDelay: ms(50)}, 0, 50 * time.Millisecond}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, ComputeBackoff(tc.cfg, tc.attempt)) + }) + } +} + +func TestComputeBackoff_JitterIsAdditiveAndBounded(t *testing.T) { + cfg := &common.RetryPolicyConfig{ + Delay: common.Duration(100 * time.Millisecond), + BackoffFactor: 2, + BackoffMaxDelay: common.Duration(150 * time.Millisecond), + Jitter: common.Duration(20 * time.Millisecond), + } + seen := map[time.Duration]bool{} + for range 200 { + d := ComputeBackoff(cfg, 2) + // Jitter is added after the cap, in [0, Jitter). + require.GreaterOrEqual(t, d, 150*time.Millisecond) + require.Less(t, d, 170*time.Millisecond) + seen[d] = true + } + assert.Greater(t, len(seen), 1, "jitter must vary the delay") +} + +func TestSleepCtx(t *testing.T) { + t.Run("non-positive duration returns immediately even when canceled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + assert.NoError(t, SleepCtx(ctx, 0)) + assert.NoError(t, SleepCtx(ctx, -time.Second)) + }) + + t.Run("elapses", func(t *testing.T) { + start := time.Now() + require.NoError(t, SleepCtx(context.Background(), 5*time.Millisecond)) + assert.GreaterOrEqual(t, time.Since(start), 5*time.Millisecond) + }) + + t.Run("canceled context interrupts", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + assert.ErrorIs(t, SleepCtx(ctx, time.Hour), context.Canceled) + }) + + t.Run("deadline interrupts", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond) + defer cancel() + assert.ErrorIs(t, SleepCtx(ctx, time.Hour), context.DeadlineExceeded) + }) +} diff --git a/failsafe/breaker_states_test.go b/failsafe/breaker_states_test.go new file mode 100644 index 000000000..eebd54034 --- /dev/null +++ b/failsafe/breaker_states_test.go @@ -0,0 +1,350 @@ +package failsafe + +import ( + "bytes" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type transition struct { + from, to State + reason string +} + +// recordTransitions wires OnTransition to a buffered channel; the hook runs +// on its own goroutine, so tests read transitions from the channel. +func recordTransitions(b *Breaker) <-chan transition { + ch := make(chan transition, 16) + b.OnTransition = func(from, to State, reason string) { ch <- transition{from, to, reason} } + return ch +} + +func nextTransition(t *testing.T, ch <-chan transition) transition { + t.Helper() + select { + case tr := <-ch: + return tr + case <-time.After(5 * time.Second): + t.Fatal("expected a state transition") + return transition{} + } +} + +func TestState_String(t *testing.T) { + assert.Equal(t, "closed", StateClosed.String()) + assert.Equal(t, "open", StateOpen.String()) + assert.Equal(t, "half_open", StateHalfOpen.String()) + assert.Equal(t, "unknown", State(42).String()) +} + +func TestErrCircuitOpen(t *testing.T) { + assert.EqualError(t, ErrCircuitOpen, "circuit breaker is open") + wrapped := errors.Join(errors.New("upstream x"), ErrCircuitOpen) + assert.ErrorIs(t, wrapped, ErrCircuitOpen) +} + +func TestBreaker_NilIsPermissive(t *testing.T) { + b := NewBreaker(nil, nil) + require.Nil(t, b) + assert.True(t, b.TryAcquirePermit()) + b.Record(OutcomeFailure) // no-op on nil + assert.Equal(t, StateClosed, b.State()) + f, s, e := b.Metrics() + assert.Zero(t, f) + assert.Zero(t, s) + assert.Zero(t, e) +} + +func TestBreaker_FullCycleWithHooksAndLogs(t *testing.T) { + var buf bytes.Buffer + var bufMu sync.Mutex + logger := zerolog.New(zerolog.SyncWriter(&lockedWriter{w: &buf, mu: &bufMu})) + b := NewBreaker(&common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 2, + FailureThresholdCapacity: 3, + HalfOpenAfter: 0, // Open → HalfOpen on the next permit + SuccessThresholdCount: 1, + SuccessThresholdCapacity: 1, + }, &logger) + trs := recordTransitions(b) + + // Window of 3, threshold of 2: the window must fill before opening. + b.Record(OutcomeFailure) + b.Record(OutcomeFailure) + assert.Equal(t, StateClosed, b.State(), "capacity not met yet") + b.Record(OutcomeSuccess) + assert.Equal(t, StateOpen, b.State(), "2 failures in a full window of 3 opens") + assert.Equal(t, transition{StateClosed, StateOpen, "failure_threshold"}, nextTransition(t, trs)) + + // Outcomes recorded while Open only bump lifetime counters. + b.Record(OutcomeSuccess) + b.Record(OutcomeFailure) + assert.Equal(t, StateOpen, b.State()) + + require.True(t, b.TryAcquirePermit(), "zero HalfOpenAfter permits immediately") + assert.Equal(t, StateHalfOpen, b.State()) + assert.Equal(t, transition{StateOpen, StateHalfOpen, "half_open_delay_elapsed"}, nextTransition(t, trs)) + + b.Record(OutcomeSuccess) + assert.Equal(t, StateClosed, b.State()) + assert.Equal(t, transition{StateHalfOpen, StateClosed, "half_open_success_threshold"}, nextTransition(t, trs)) + + failures, successes, executions := b.Metrics() + assert.Equal(t, uint64(3), failures) + assert.Equal(t, uint64(3), successes) + assert.Equal(t, uint64(6), executions) + + bufMu.Lock() + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + bufMu.Unlock() + require.Len(t, lines, 3) + var first map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &first)) + assert.Equal(t, "warn", first["level"]) + assert.Equal(t, "closed", first["from"]) + assert.Equal(t, "open", first["to"]) + assert.Equal(t, "failure_threshold", first["reason"]) + assert.Equal(t, "circuit breaker state changed", first["message"]) + assert.EqualValues(t, 3, first["executions"]) +} + +type lockedWriter struct { + w *bytes.Buffer + mu *sync.Mutex +} + +func (l *lockedWriter) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.w.Write(p) +} + +func TestBreaker_ClosedWindowEvictsOldest(t *testing.T) { + b := NewBreaker(&common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 2, + FailureThresholdCapacity: 3, + HalfOpenAfter: common.Duration(time.Hour), + }, nil) + + // F S S → 1 failure in window. + b.Record(OutcomeFailure) + b.Record(OutcomeSuccess) + b.Record(OutcomeSuccess) + // Evicts F: S S F → 1 failure. + b.Record(OutcomeFailure) + assert.Equal(t, StateClosed, b.State(), "the evicted failure must not count") + // Evicts S: S F S → still 1. + b.Record(OutcomeSuccess) + assert.Equal(t, StateClosed, b.State()) + // Evicts S: F S F → 2 failures, opens. + b.Record(OutcomeFailure) + assert.Equal(t, StateOpen, b.State()) +} + +func TestBreaker_IgnoreDoesNotMoveClosedWindow(t *testing.T) { + b := NewBreaker(&common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 1, + FailureThresholdCapacity: 1, + }, nil) + for range 5 { + b.Record(OutcomeIgnore) + } + assert.Equal(t, StateClosed, b.State()) + _, _, executions := b.Metrics() + assert.Zero(t, executions, "ignored outcomes are not executions") +} + +func TestBreaker_ThresholdConfigFallbacks(t *testing.T) { + cases := []struct { + name string + cfg common.CircuitBreakerPolicyConfig + failsToOpen int // -1: never opens + }{ + {"capacity falls back to count", common.CircuitBreakerPolicyConfig{FailureThresholdCount: 3}, 3}, + {"zero count never opens", common.CircuitBreakerPolicyConfig{FailureThresholdCapacity: 2}, -1}, + {"nothing configured never opens", common.CircuitBreakerPolicyConfig{}, -1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := tc.cfg + cfg.HalfOpenAfter = common.Duration(time.Hour) + b := NewBreaker(&cfg, nil) + if tc.failsToOpen < 0 { + for range 10 { + b.Record(OutcomeFailure) + } + assert.Equal(t, StateClosed, b.State()) + return + } + for i := 1; i < tc.failsToOpen; i++ { + b.Record(OutcomeFailure) + } + assert.Equal(t, StateClosed, b.State()) + b.Record(OutcomeFailure) + assert.Equal(t, StateOpen, b.State()) + }) + } +} + +func TestBreaker_OpenRejectsUntilDelayElapses(t *testing.T) { + b := NewBreaker(&common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 1, + HalfOpenAfter: common.Duration(time.Hour), + }, nil) + b.Record(OutcomeFailure) + require.Equal(t, StateOpen, b.State()) + for range 3 { + assert.False(t, b.TryAcquirePermit()) + } + assert.Equal(t, StateOpen, b.State()) + + // Pretend the delay elapsed. + b.mu.Lock() + b.openedAt = time.Now().Add(-2 * time.Hour) + b.mu.Unlock() + assert.True(t, b.TryAcquirePermit()) + assert.Equal(t, StateHalfOpen, b.State()) +} + +func TestBreaker_HalfOpenPermitCapacity(t *testing.T) { + cases := []struct { + name string + cfg common.CircuitBreakerPolicyConfig + permits int + }{ + {"capacity", common.CircuitBreakerPolicyConfig{SuccessThresholdCount: 1, SuccessThresholdCapacity: 3}, 3}, + {"count when capacity unset", common.CircuitBreakerPolicyConfig{SuccessThresholdCount: 2}, 2}, + {"one when nothing set", common.CircuitBreakerPolicyConfig{}, 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := tc.cfg + cfg.FailureThresholdCount = 1 + // HalfOpenAfter 0: permits flow immediately and a full trial is + // never considered stale. + b := NewBreaker(&cfg, nil) + b.Record(OutcomeFailure) + require.Equal(t, StateOpen, b.State()) + for i := range tc.permits { + require.True(t, b.TryAcquirePermit(), "permit %d", i+1) + } + assert.Equal(t, StateHalfOpen, b.State()) + assert.False(t, b.TryAcquirePermit(), "permits beyond capacity are refused") + assert.False(t, b.TryAcquirePermit()) + }) + } +} + +func TestBreaker_HalfOpenDefaultsCloseOnFirstSuccessReopenOnFirstFailure(t *testing.T) { + for _, tc := range []struct { + outcome Outcome + want State + reason string + }{ + {OutcomeSuccess, StateClosed, "half_open_success_threshold"}, + {OutcomeFailure, StateOpen, "half_open_failure"}, + } { + b := NewBreaker(&common.CircuitBreakerPolicyConfig{FailureThresholdCount: 1}, nil) + b.Record(OutcomeFailure) + require.True(t, b.TryAcquirePermit()) + require.Equal(t, StateHalfOpen, b.State()) + trs := recordTransitions(b) + b.Record(tc.outcome) + assert.Equal(t, tc.want, b.State()) + assert.Equal(t, transition{StateHalfOpen, tc.want, tc.reason}, nextTransition(t, trs)) + } +} + +func TestBreaker_HalfOpenRecordWithoutPermit(t *testing.T) { + b := NewBreaker(&common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 1, + SuccessThresholdCount: 2, + SuccessThresholdCapacity: 2, + }, nil) + b.Record(OutcomeFailure) + require.True(t, b.TryAcquirePermit()) + b.Record(OutcomeIgnore) // releases the only held permit + b.Record(OutcomeIgnore) // nothing held: must not underflow + // A record with no permit held still counts toward the trial. + b.Record(OutcomeSuccess) + assert.Equal(t, StateHalfOpen, b.State()) + // Two permits are still available: the inflight counter never went negative. + assert.True(t, b.TryAcquirePermit()) + assert.True(t, b.TryAcquirePermit()) + assert.False(t, b.TryAcquirePermit()) + b.Record(OutcomeSuccess) + assert.Equal(t, StateClosed, b.State()) +} + +func TestBreaker_ClosedAfterHalfOpenStartsWithEmptyWindow(t *testing.T) { + b := NewBreaker(&common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 2, + FailureThresholdCapacity: 2, + }, nil) + b.Record(OutcomeFailure) + b.Record(OutcomeFailure) + require.Equal(t, StateOpen, b.State()) + require.True(t, b.TryAcquirePermit()) + b.Record(OutcomeSuccess) + require.Equal(t, StateClosed, b.State()) + + // The window was reset on close: a single failure must not reopen. + b.Record(OutcomeFailure) + assert.Equal(t, StateClosed, b.State()) + b.Record(OutcomeFailure) + assert.Equal(t, StateOpen, b.State()) +} + +func TestBreaker_UnknownStatePermits(t *testing.T) { + b := NewBreaker(&common.CircuitBreakerPolicyConfig{FailureThresholdCount: 1}, nil) + b.state.Store(99) + assert.True(t, b.TryAcquirePermit()) + assert.Equal(t, "unknown", b.State().String()) +} + + +func TestBreaker_StaleHalfOpenTrialIsRearmed(t *testing.T) { + b := NewBreaker(&common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 1, + HalfOpenAfter: common.Duration(time.Hour), + SuccessThresholdCount: 1, + SuccessThresholdCapacity: 1, + }, nil) + b.Record(OutcomeFailure) + b.mu.Lock() + b.openedAt = time.Now().Add(-2 * time.Hour) + b.mu.Unlock() + require.True(t, b.TryAcquirePermit()) + require.False(t, b.TryAcquirePermit(), "trial is fresh and full") + + b.mu.Lock() + b.halfOpenAt = time.Now().Add(-2 * time.Hour) + b.mu.Unlock() + require.True(t, b.TryAcquirePermit(), "stale trial is re-armed") + require.False(t, b.TryAcquirePermit(), "re-armed trial holds exactly one permit") + b.Record(OutcomeSuccess) + assert.Equal(t, StateClosed, b.State()) +} + +func TestBreaker_SuccessCountAboveCapacityToleratesNoFailure(t *testing.T) { + // Count > capacity is a misconfiguration; the trial must not tolerate a + // negative number of failures, it re-opens on the first one. + b := NewBreaker(&common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 1, + SuccessThresholdCount: 3, + SuccessThresholdCapacity: 1, + }, nil) + b.Record(OutcomeFailure) + require.True(t, b.TryAcquirePermit()) + b.Record(OutcomeFailure) + assert.Equal(t, StateOpen, b.State()) +} diff --git a/failsafe/hedge.go b/failsafe/hedge.go index db89f60e5..9981baaff 100644 --- a/failsafe/hedge.go +++ b/failsafe/hedge.go @@ -2,6 +2,8 @@ package failsafe import ( "context" + "fmt" + "runtime/debug" "sync" "time" ) @@ -122,7 +124,16 @@ func RunHedged[R any]( hooks.OnFire(idx, 0) } go func() { - r, err := inner(siblingCtx) + // A panic here would kill the process: no caller frame can recover a + // goroutine. Surface it as this attempt's error instead. + r, err := func() (r R, err error) { + defer func() { + if rec := recover(); rec != nil { + err = fmt.Errorf("hedge attempt panic: %v\n%s", rec, debug.Stack()) + } + }() + return inner(siblingCtx) + }() select { case resultCh <- hedgeResult[R]{r: r, err: err}: default: diff --git a/failsafe/hedge_race_test.go b/failsafe/hedge_race_test.go new file mode 100644 index 000000000..b8d1f15f0 --- /dev/null +++ b/failsafe/hedge_race_test.go @@ -0,0 +1,254 @@ +package failsafe + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// releaseLog records every result handed to release. +type releaseLog struct { + mu sync.Mutex + vals []string +} + +func (l *releaseLog) release(r *string) { + l.mu.Lock() + defer l.mu.Unlock() + l.vals = append(l.vals, *r) +} + +func (l *releaseLog) snapshot() []string { + l.mu.Lock() + defer l.mu.Unlock() + return append([]string(nil), l.vals...) +} + +func strp(s string) *string { return &s } + +func keepNoErr(r *string, err error) bool { return err == nil && r != nil } + +func TestRunHedged_MaxHedgesBoundsAttemptsAndAllFailuresReturnLast(t *testing.T) { + var calls atomic.Int32 + var fired []int + var firedMu sync.Mutex + inner := func(ctx context.Context) (*string, error) { + n := calls.Add(1) + return nil, errors.New("fail-" + string(rune('0'+n))) + } + _, err := RunHedged[*string]( + context.Background(), + 2, + func(idx int) time.Duration { return 0 }, + inner, + keepNoErr, + nil, + HedgeHooks{OnFire: func(idx int, _ time.Duration) { + firedMu.Lock() + fired = append(fired, idx) + firedMu.Unlock() + }}, + ) + require.Error(t, err) + assert.Regexp(t, `^fail-[123]$`, err.Error()) + assert.Equal(t, int32(3), calls.Load(), "primary + 2 hedges, never more") + firedMu.Lock() + assert.Equal(t, []int{1, 2}, fired, "OnFire reports hedges only, not the primary") + firedMu.Unlock() +} + +func TestRunHedged_NegativeMaxHedgesRunsPrimaryOnly(t *testing.T) { + var calls atomic.Int32 + r, err := RunHedged[*string]( + context.Background(), -3, nil, + func(ctx context.Context) (*string, error) { + calls.Add(1) + return nil, errors.New("primary failed") + }, + keepNoErr, nil, HedgeHooks{}, + ) + assert.Nil(t, r) + assert.EqualError(t, err, "primary failed") + assert.Equal(t, int32(1), calls.Load()) +} + +func TestRunHedged_NilKeepAcceptsFirstResult(t *testing.T) { + r, err := RunHedged[*string]( + context.Background(), 0, nil, + func(ctx context.Context) (*string, error) { return strp("partial"), errors.New("soft error") }, + nil, nil, HedgeHooks{}, + ) + require.NotNil(t, r) + assert.Equal(t, "partial", *r) + assert.EqualError(t, err, "soft error", "nil keep returns the first result with its error") +} + +func TestRunHedged_HedgeRecoversAfterFastPrimaryFailure(t *testing.T) { + var calls atomic.Int32 + hedgeGo := make(chan struct{}) + inner := func(ctx context.Context) (*string, error) { + if calls.Add(1) == 1 { + close(hedgeGo) + return nil, errors.New("primary transient") + } + return strp("hedge"), nil + } + r, err := RunHedged[*string]( + context.Background(), 1, + func(idx int) time.Duration { + <-hedgeGo // arm the hedge timer only once the primary is running + return 5 * time.Millisecond + }, + inner, keepNoErr, nil, HedgeHooks{}, + ) + require.NoError(t, err) + assert.Equal(t, "hedge", *r, "a scheduled hedge still fires after the primary failed") +} + +func TestRunHedged_NegativeDelayFiresHedgeImmediately(t *testing.T) { + var calls atomic.Int32 + primaryRelease := make(chan struct{}) + defer close(primaryRelease) + inner := func(ctx context.Context) (*string, error) { + if calls.Add(1) == 1 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-primaryRelease: + return strp("primary"), nil + } + } + return strp("hedge"), nil + } + r, err := RunHedged[*string]( + context.Background(), 1, + func(idx int) time.Duration { return -time.Second }, + inner, keepNoErr, nil, HedgeHooks{}, + ) + require.NoError(t, err) + assert.Equal(t, "hedge", *r) +} + +func TestRunHedged_LosersAreReleasedAndCanceled(t *testing.T) { + var calls atomic.Int32 + losersCanceled := make(chan struct{}, 2) + winnerGo := make(chan struct{}) + started := make(chan struct{}, 3) + inner := func(ctx context.Context) (*string, error) { + n := calls.Add(1) + started <- struct{}{} + if n == 3 { + <-winnerGo + return strp("winner"), nil + } + <-ctx.Done() + losersCanceled <- struct{}{} + return strp("loser"), nil // non-nil: must be released + } + var rel releaseLog + go func() { + for range 3 { + <-started + } + close(winnerGo) + }() + r, err := RunHedged[*string]( + context.Background(), 2, + func(idx int) time.Duration { return 0 }, + inner, keepNoErr, rel.release, HedgeHooks{}, + ) + require.NoError(t, err) + assert.Equal(t, "winner", *r) + for range 2 { + select { + case <-losersCanceled: + case <-time.After(5 * time.Second): + t.Fatal("losers must observe sibling cancellation") + } + } + // RunHedged drains every in-flight attempt before returning. + assert.ElementsMatch(t, []string{"loser", "loser"}, rel.snapshot(), "winner is never released") +} + +func TestRunHedged_NotKeptResultsAreNotReleasedWhenReturned(t *testing.T) { + var rel releaseLog + r, err := RunHedged[*string]( + context.Background(), 0, nil, + func(ctx context.Context) (*string, error) { return strp("rejected"), errors.New("bad") }, + keepNoErr, rel.release, HedgeHooks{}, + ) + require.EqualError(t, err, "bad") + assert.Equal(t, "rejected", *r, "caller owns the returned non-kept result") + assert.Empty(t, rel.snapshot()) +} + +func TestRunHedged_ParentCancelReleasesInFlightResults(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + started := make(chan struct{}, 2) + inner := func(c context.Context) (*string, error) { + started <- struct{}{} + <-c.Done() + return strp("abandoned"), nil + } + go func() { + <-started + <-started + cancel() + }() + var rel releaseLog + r, err := RunHedged[*string]( + ctx, 1, + func(idx int) time.Duration { return 0 }, + inner, keepNoErr, rel.release, HedgeHooks{}, + ) + assert.ErrorIs(t, err, context.Canceled) + assert.Nil(t, r) + assert.Equal(t, []string{"abandoned", "abandoned"}, rel.snapshot()) +} + +func TestRunHedged_ParentCancelSkipsZeroResults(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + started := make(chan struct{}) + go func() { + <-started + cancel() + }() + var rel releaseLog + _, err := RunHedged[*string]( + ctx, 0, nil, + func(c context.Context) (*string, error) { + close(started) + <-c.Done() + return nil, c.Err() + }, + keepNoErr, rel.release, HedgeHooks{}, + ) + assert.ErrorIs(t, err, context.Canceled) + assert.Empty(t, rel.snapshot(), "nil results are never passed to release") +} + +func TestRunHedged_PanicInAttemptBecomesError(t *testing.T) { + var calls atomic.Int32 + r, err := RunHedged[*string]( + context.Background(), 1, + func(idx int) time.Duration { return 0 }, + func(ctx context.Context) (*string, error) { + if calls.Add(1) == 1 { + panic("attempt exploded") + } + <-ctx.Done() + return nil, ctx.Err() + }, + func(r *string, err error) bool { return err != nil }, + nil, HedgeHooks{}, + ) + assert.Nil(t, r) + require.Error(t, err) + assert.Contains(t, err.Error(), "hedge attempt panic: attempt exploded") +} diff --git a/go.mod b/go.mod index 5c8d74657..5d8e03867 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/DataDog/sketches-go v1.4.8 github.com/IGLOU-EU/go-wildcard/v2 v2.1.1 github.com/aws/aws-sdk-go v1.55.8 - github.com/blockchain-data-standards/manifesto v0.0.0-20261008203857-00f4bd1d2991 + github.com/blockchain-data-standards/manifesto v0.0.0-20261027124537-aea005ce6ae7 github.com/bytedance/sonic v1.15.2 github.com/coder/websocket v1.8.15 github.com/dgraph-io/ristretto/v2 v2.4.2 @@ -55,7 +55,10 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) -require github.com/prometheus/client_model v0.6.2 +require ( + github.com/coocood/freecache v1.2.4 + github.com/prometheus/client_model v0.6.2 +) require ( cel.dev/expr v0.25.2 // indirect @@ -78,7 +81,6 @@ require ( github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect - github.com/coocood/freecache v1.2.4 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/crate-crypto/go-eth-kzg v1.5.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index a45323631..c0e39490b 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0= github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/blockchain-data-standards/manifesto v0.0.0-20261008203857-00f4bd1d2991 h1:uHzPykDZNnoUszEYRdD5ykbNZjfXJO1fNP5ISVLvDPo= -github.com/blockchain-data-standards/manifesto v0.0.0-20261008203857-00f4bd1d2991/go.mod h1:BEP+UJDL+dSqF4UddiHmITKlV2l0aaDEagPS9nbbYIc= +github.com/blockchain-data-standards/manifesto v0.0.0-20261027124537-aea005ce6ae7 h1:rglF6yDY+NbUVWjURUqmz2sQbWDhUbqgJCcNaBfpMZM= +github.com/blockchain-data-standards/manifesto v0.0.0-20261027124537-aea005ce6ae7/go.mod h1:BEP+UJDL+dSqF4UddiHmITKlV2l0aaDEagPS9nbbYIc= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= diff --git a/health/tracker.go b/health/tracker.go index ca2337ce7..7710bc37e 100644 --- a/health/tracker.go +++ b/health/tracker.go @@ -60,13 +60,43 @@ type NetworkMetadata struct { // publication for the network so a slow re-derivation can never overwrite a // newer one; reporters is every upstream that has reported a head here. headMu sync.Mutex - reporters []common.Upstream - - // Dynamic block time via EMA on on-chain block timestamps. - // Uses block.timestamp (integer seconds) normalized by block count gap. - // For fast chains where consecutive blocks share the same timestamp, - // samples are skipped until the timestamp advances, then blockGap - // normalization recovers sub-second precision. + reporters []headReporter + + // On upstream entries only; guarded by the network entry's headMu. + // headChangedAt is when this upstream's stored latest head last changed; + // headBehindSince is when a peer was first seen proving the chain went on + // without it (zero = not behind); headBenchedAt is when it was last benched + // and headBenchRef the network head right after. They keep the clock's + // monotonic reading, so a wall-clock step never makes a head look stalled. + // headTsFloor is the newest block timestamp (seconds, never past the + // clock when stored) seen for any of its heads, a lower bound on its + // current head's (evmLatestBlockTimestamp is exact or 0). headStale is + // the staleness verdict (see headIsBehind) of the last evaluation that + // knew its head's timestamp: moves without one (a peer pod's poll through + // shared state, a response suggestion) keep it only while a live peer + // stays a stall window of blocks ahead. + // headBenched is read lock-free by the served-tip ballots (see + // Tracker.IsHeadBenched). + headChangedAt time.Time + headBehindSince time.Time + headBenchedAt time.Time + headBenchRef int64 + headTsFloor int64 + headStale bool + headBenched atomic.Bool + // headReport and finalizedReport are this upstream's last latest and + // finalized reports, changed or not (see headReport). On upstream entries + // only; guarded by the network entry's headMu. + headReport headReport + finalizedReport headReport + + // Dynamic block time via EMA on on-chain block timestamps: on the network + // entry from the corroborated head, on an upstream entry from that + // upstream's own head (the head bench uses it while the network's is + // unknown). Uses block.timestamp (integer seconds) normalized by block + // count gap. For fast chains where consecutive blocks share the same + // timestamp, samples are skipped until the timestamp advances, then + // blockGap normalization recovers sub-second precision. // Callers should use GetNetworkBlockTime which returns 0 until enough // samples have been collected. evmBlockTime atomic.Int64 // computed result in nanoseconds (read by consumers) @@ -299,6 +329,9 @@ type Tracker struct { finalizationLagGaugeCache sync.Map // map[ubKey]prometheus.Gauge cordonedGaugeCache sync.Map // map[cordKey]prometheus.Gauge rollbackGaugeCache sync.Map // map[rbKey]prometheus.Gauge + + // now is the clock of the head bench state machine. + now func() time.Time } // urdoKey uniquely identifies a MetricUpstreamRequestDuration time series. @@ -512,6 +545,7 @@ func NewTracker(logger *zerolog.Logger, projectId string, windowSize time.Durati windowSize: windowSize, upstreamsByNetwork: make(map[string][]upstreamKey), idleEvictionAfter: DefaultIdleEvictionAfter, + now: time.Now, } } @@ -524,6 +558,12 @@ func (t *Tracker) SetIdleEvictionAfter(d time.Duration) { t.idleEvictionAfter = d } +// SetClockForTest replaces the head bench clock. Call it before any head is +// recorded. +func (t *Tracker) SetClockForTest(now func() time.Time) { + t.now = now +} + // Bootstrap starts the goroutine that rotates rolling-window buckets. func (t *Tracker) Bootstrap(ctx context.Context) { go t.rotateMetricsLoop(ctx) @@ -1223,7 +1263,7 @@ func (t *Tracker) GetNetworkMethodMetrics(network, method string) *TrackedMetric func (t *Tracker) updateNetworkLagMetrics( net string, networkValue int64, - getUpstreamValue func(*NetworkMetadata) int64, + getUpstreamValue func(common.Upstream, *NetworkMetadata) int64, setLag func(*TrackedMetrics, int64), getGauge func(string, string, string, string) prometheus.Gauge, lg *zerolog.Logger, @@ -1245,7 +1285,7 @@ func (t *Tracker) updateNetworkLagMetrics( } tm := value.(*TrackedMetrics) upsMeta := t.getMetadata(metadataKey{k.ups, net}) - upsValue := getUpstreamValue(upsMeta) + upsValue := getUpstreamValue(k.ups, upsMeta) if upsValue <= 0 { lg.Debug(). Str("upstreamId", k.ups.Id()). @@ -1272,7 +1312,7 @@ func (t *Tracker) updateNetworkLagMetrics( if v, ok := t.upsMetrics.Load(k); ok { tm := v.(*TrackedMetrics) upsMeta := t.getMetadata(metadataKey{k.ups, net}) - upsValue := getUpstreamValue(upsMeta) + upsValue := getUpstreamValue(k.ups, upsMeta) if upsValue <= 0 { lg.Debug(). Str("upstreamId", k.ups.Id()). @@ -1366,40 +1406,302 @@ func blocksBehind(networkHead, upstreamHead int64) int64 { return networkHead - upstreamHead } -// corroboratedNetworkHead derives the network head for one axis from every -// upstream that has reported on the network: the second-highest once two have -// reported, the only value while one has. It returns the reporter holding that -// head so the caller can carry that block's timestamp. Caller holds -// ntwMeta.headMu. -func (t *Tracker) corroboratedNetworkHead(ntwMeta *NetworkMetadata, net string, reporter common.Upstream, getVal func(*NetworkMetadata) int64) (int64, common.Upstream) { - known := false +// headReporter is one upstream that has reported a head on a network, with its +// per-network metadata resolved once (metadata entries are never evicted) and +// its stall window (headStallAfter), refreshed at the start of each +// evaluateHeadBench. +type headReporter struct { + ups common.Upstream + meta *NetworkMetadata + stallAfter time.Duration +} + +// addHeadReporter registers reporter as a head reporter on the network and +// returns its metadata. Caller holds ntwMeta.headMu. +func (t *Tracker) addHeadReporter(ntwMeta *NetworkMetadata, net string, reporter common.Upstream) *NetworkMetadata { + id := reporter.Id() for _, r := range ntwMeta.reporters { - if r.Id() == reporter.Id() { - known = true - break + if r.ups.Id() == id { + return r.meta } } - if !known { - ntwMeta.reporters = append(ntwMeta.reporters, reporter) + meta := t.getMetadata(metadataKey{reporter, net}) + ntwMeta.reporters = append(ntwMeta.reporters, headReporter{ups: reporter, meta: meta}) + return meta +} + +// corroboratedNetworkHead derives the network head for one axis (the lag +// reference): the second-highest head among the non-benched reporters, the +// only one while a single reporter has a head. When every reporter is benched +// it fails open to all of them. It returns the reporter holding that head so +// the caller can carry that block's timestamp. Caller holds ntwMeta.headMu. +func corroboratedNetworkHead(ntwMeta *NetworkMetadata, getVal func(*NetworkMetadata) int64) (int64, common.Upstream) { + if head, holder := secondHighestHead(ntwMeta, getVal, true); head > 0 { + return head, holder } - heads := make([]common.ServedTipInput, 0, len(ntwMeta.reporters)) + return secondHighestHead(ntwMeta, getVal, false) +} + +func secondHighestHead(ntwMeta *NetworkMetadata, getVal func(*NetworkMetadata) int64, skipBenched bool) (int64, common.Upstream) { + var first, second int64 + var firstUps, secondUps common.Upstream for _, r := range ntwMeta.reporters { - heads = append(heads, common.ServedTipInput{ - UpstreamID: r.Id(), - BlockNumber: getVal(t.getMetadata(metadataKey{r, net})), - }) + if skipBenched && r.meta.headBenched.Load() { + continue + } + v := getVal(r.meta) + switch { + case v <= 0: + case v > first: + second, secondUps = first, firstUps + first, firstUps = v, r.ups + case v > second: + second, secondUps = v, r.ups + } } - pick := common.PickServedTip(heads) - if pick.Inputs == 0 { - return 0, nil + if second > 0 { + return second, secondUps } - corroborated := pick.Sorted[min(1, pick.Inputs-1)] - for _, r := range ntwMeta.reporters { - if r.Id() == corroborated.UpstreamID { - return corroborated.BlockNumber, r + return first, firstUps +} + +// headStallAfter is how long an upstream may stay behind (headIsBehind) +// before its head is benched: three of its poll periods, never under +// 30s, so ordinary poll skew never benches anyone. +func headStallAfter(ups common.Upstream, blockTime time.Duration) time.Duration { + return max(30*time.Second, 3*pollPeriod(ups, blockTime)) +} + +// pollPeriod is how often an upstream's head is fetched: the longer of the +// poll interval and the poll debounce, because a poll inside the debounce +// reuses the stored head (see EvmStatePoller.resolveDebounce); an unset +// debounce follows the block time, which bounds the derived one. +func pollPeriod(ups common.Upstream, blockTime time.Duration) time.Duration { + period, debounce := time.Duration(0), blockTime + if cfg := ups.Config(); cfg != nil && cfg.Evm != nil { + period = cfg.Evm.StatePollerInterval.Duration() + if d := cfg.Evm.StatePollerDebounce.Duration(); d > 0 { + debounce = d + } + } + return max(period, debounce) +} + +// headReport is an upstream's last head report on one axis: when it came and +// the network head right after it. A stored head ages between polls by +// design; counting that age as lag made a healthy upstream polled every 10s +// on a 0.4s chain read 25 blocks behind late in each poll window, and a +// policy ticking at the same period read that same phase on every tick: +// excluded for good on some processes, never on others. So for one poll +// period after a report, what the network head made since counts as made by +// the upstream too: its lag stays what it was at the report. Past the period +// the credit freezes and a missing report counts as lag again. +type headReport struct { + at time.Time + network int64 + credit int64 +} + +func (r *headReport) reported(now time.Time, network int64) { + r.at, r.network, r.credit = now, network, 0 +} + +// presumed is the head the upstream is presumed to hold at now, given the +// network head then. +func (r *headReport) presumed(head, network int64, now time.Time, period time.Duration) int64 { + if r.at.IsZero() || head <= 0 { + return head + } + if now.Sub(r.at) <= period { + r.credit = max(0, network-r.network) + } + return head + r.credit +} + +// evaluateHeadBench runs the head bench state machine over every reporter of +// the network after `changed` (if non-nil) stored a new latest head. A +// reporter is "behind" while a non-benched peer proves the chain went on +// without it (headIsBehind); one behind without a break for its stall window +// is benched. A benched reporter rejoins once it has moved since, no rule +// still proves it behind, and it reached the corroborated non-benched head +// (the second-highest, or the only one): adding a voter at or above the k-th +// highest head never lowers it, and a lone far-ahead rogue cannot hold the +// bar out of reach. It also rejoins when no non-benched peer is left. +// Requiring the benched one to move keeps a stuck head from flapping. +// It reports whether anyone was benched or rejoined. Caller holds +// ntwMeta.headMu. +func (t *Tracker) evaluateHeadBench(ntwMeta *NetworkMetadata, changed *NetworkMetadata) (membershipChanged bool) { + now := t.now() + blockTime := time.Duration(ntwMeta.evmBlockTime.Load()) + reporters := ntwMeta.reporters + for i := range reporters { + reporters[i].stallAfter = headStallAfter(reporters[i].ups, blockTime) + } + if changed != nil { + changed.headChangedAt = now + } + for _, u := range reporters { + behind := u.meta.evmLatestBlockNumber.Load() > 0 && headIsBehind(u, reporters, now, blockTime) + switch { + case !behind: + u.meta.headBehindSince = time.Time{} + case u.meta.headBehindSince.IsZero(): + u.meta.headBehindSince = now + } + } + for _, u := range reporters { + if u.meta.headBenched.Load() || u.meta.headBehindSince.IsZero() { + continue + } + if now.Sub(u.meta.headBehindSince) < u.stallAfter { + continue + } + u.meta.headBenched.Store(true) + u.meta.headBenchedAt = now + u.meta.headBenchRef = 0 // taken below, once the bar excludes it + membershipChanged = true + telemetry.MetricUpstreamHeadBenchedTotal.WithLabelValues(t.projectId, u.ups.VendorName(), u.ups.NetworkLabel(), u.ups.Id()).Inc() + u.ups.Logger().Warn(). + Str("networkId", u.ups.NetworkId()). + Int64("head", u.meta.evmLatestBlockNumber.Load()). + Dur("stallAfter", u.stallAfter). + Msg("benched upstream head: it stayed behind peers that kept advancing; it leaves the served-tip ballots and the lag reference until it catches up") + } + bar, _ := secondHighestHead(ntwMeta, func(m *NetworkMetadata) int64 { return m.evmLatestBlockNumber.Load() }, true) + for _, u := range reporters { + if !u.meta.headBenched.Load() { + continue + } + uHead := u.meta.evmLatestBlockNumber.Load() + if u.meta.headBenchRef == 0 { + u.meta.headBenchRef = uHead + if bar > 0 { + u.meta.headBenchRef = bar + } + } + // A move in the evaluation that benched it does not count, or a node + // benched on its own report would rejoin on the next one without + // moving; a node still proven behind (a replay that reached a voter as + // stale as itself) would rejoin and be benched again a window later. + if bar > 0 && (!u.meta.movedSinceBench() || uHead < bar || !u.meta.headBehindSince.IsZero()) { + continue + } + u.meta.headBenched.Store(false) + u.meta.headBehindSince = time.Time{} + u.meta.headStale = false + membershipChanged = true + u.ups.Logger().Info(). + Str("networkId", u.ups.NetworkId()). + Int64("head", uHead). + Msg("upstream head rejoined: it caught up with its non-benched peers") + } + return membershipChanged +} + +// movedSinceBench reports whether the head moved after the evaluation that +// benched it; a move in that same evaluation does not count. Caller holds the +// network's headMu. +func (m *NetworkMetadata) movedSinceBench() bool { + return m.headChangedAt.After(m.headBenchedAt) +} + +// headIsBehind reports whether a non-benched peer proves the chain went on +// without u: +// - by height: a peer that moved after u last moved is above u within a +// plausible lead (benchPlausibleLead); one such peer is enough. Each move +// of u resets this, so it measures a stall; +// - by staleness: a live peer (one that moved within its or u's stall +// window) is above u within a plausible lead, and its newest block (never +// counted as newer than now) is at least u's stall window younger than +// u's head block. That holds while u moves, so it catches a node +// replaying far behind the tip: after a restore, a restart or a deep +// rollback. A head stored without its timestamp keeps the last verdict +// that had one, while the live peer is still a stall window of blocks +// ahead: only u's own timestamp can prove it stale, so a rogue peer +// claiming a higher head never can; +// - by time: a peer that moved after u last moved has a newest block at +// least u's stall window younger than u's head block, whatever the +// heights. That catches a head frozen above a chain that restarted lower +// (a dead fork) and a static far-ahead one. It takes two such peers while +// two exist, so one wrong-chain peer cannot bench the fleet. +// +// Requiring the peer to move keeps a chain halt (nobody moves) from benching +// anyone; the plausible lead and the two-peer rule keep a moving wrong-chain +// head from benching the honest fleet, and a node at the tip has a head block +// as young as any honest peer's. Caller holds ntwMeta.headMu. +func headIsBehind(u headReporter, reporters []headReporter, now time.Time, blockTime time.Duration) bool { + uHead, uTs := u.meta.evmLatestBlockNumber.Load(), u.meta.evmLatestBlockTimestamp.Load() + stallAfter := u.stallAfter + stallSecs := int64(stallAfter / time.Second) + peers, younger := 0, 0 + byHeight, stale := false, false + for _, v := range reporters { + vHead := v.meta.evmLatestBlockNumber.Load() + if v.meta == u.meta || vHead <= 0 || v.meta.headBenched.Load() { + continue + } + peers++ + bt := blockTime + if bt <= 0 { + // The network block time follows the corroborated head, which + // never advances while it is the stalled one (a process that + // starts mid-outage): measure the chain by the peer's own blocks. + bt = time.Duration(v.meta.evmBlockTime.Load()) + } + lead := vHead - uHead + plausible := lead > 0 && lead <= benchPlausibleLead(u.meta, now, bt) + live := now.Sub(v.meta.headChangedAt) < max(stallAfter, v.stallAfter) + if plausible && live { + if uTs > 0 { + stale = stale || v.meta.headTsFloor-uTs >= stallSecs + } else if u.meta.headStale && lead*bt.Milliseconds() >= stallAfter.Milliseconds() { + stale = true + } + } + if !v.meta.headChangedAt.After(u.meta.headChangedAt) { + continue + } + byHeight = byHeight || plausible + if uTs > 0 && v.meta.headTsFloor-uTs >= stallSecs { + younger++ } } - return corroborated.BlockNumber, nil + u.meta.headStale = stale + return byHeight || stale || (younger > 0 && younger >= min(2, peers)) +} + +// benchPlausibleLead is the most blocks a peer may be ahead of u and still +// count as the chain moving on without it: the rollback tolerance every head +// layer shares, plus twice what the chain produced since u's head was current +// (the earlier of when u last moved and the newest block timestamp seen for +// u, so a process that starts mid-outage still sees the outage). A larger +// lead is another chain, not a stall. Unknown block time leaves only the +// tolerance. +func benchPlausibleLead(u *NetworkMetadata, now time.Time, blockTime time.Duration) int64 { + lead := int64(common.DefaultToleratedBlockHeadRollback) + btMs := blockTime.Milliseconds() + if btMs <= 0 { + return lead + } + stale := now.Sub(u.headChangedAt).Milliseconds() + if ts := u.headTsFloor; ts > 0 { + stale = max(stale, now.UnixMilli()-ts*1000) + } + if stale > 0 { + lead += 2 * stale / btMs + } + return lead +} + +// IsHeadBenched reports whether the upstream's latest head is benched: it +// stayed behind peers that kept advancing, and has not caught up since. +// Benched heads are left out of the network lag reference and of the served-tip +// ballots. +func (t *Tracker) IsHeadBenched(ups common.Upstream) bool { + if v, ok := t.metadata.Load(metadataKey{ups, ups.NetworkId()}); ok { + return v.(*NetworkMetadata).headBenched.Load() + } + return false } func (t *Tracker) SetLatestBlockNumber(upstream common.Upstream, blockNumber int64, blockTimestamp int64) { @@ -1421,11 +1723,13 @@ func (t *Tracker) SetLatestBlockNumber(upstream common.Upstream, blockNumber int defer ntwMeta.headMu.Unlock() oldUpsVal := upsMeta.evmLatestBlockNumber.Load() + headChanged := false if blockNumber > oldUpsVal { upsMeta.evmLatestBlockNumber.Store(blockNumber) upsMeta.evmLatestBlockTimestamp.Store(blockTimestamp) g := t.getLatestBlockGauge(t.projectId, vendor, netLabel, id) g.Set(float64(blockNumber)) + headChanged = true } else if oldUpsVal-blockNumber > common.DefaultToleratedBlockHeadRollback { // The upstream reports a head far behind the one stored for it: treat it // as a correction (a deep reorg, or a previously recorded bogus sample), @@ -1440,12 +1744,31 @@ func (t *Tracker) SetLatestBlockNumber(upstream common.Upstream, blockNumber int Int64("previousValue", oldUpsVal). Int64("newValue", blockNumber). Msg("applied large latest block rollback for upstream in tracker") + upsMeta.headTsFloor = 0 + headChanged = true } else if blockNumber == oldUpsVal && blockTimestamp > 0 && upsMeta.evmLatestBlockTimestamp.Load() == 0 { upsMeta.evmLatestBlockTimestamp.Store(blockTimestamp) } + if blockTimestamp > 0 && blockNumber == upsMeta.evmLatestBlockNumber.Load() { + upsMeta.headTsFloor = max(upsMeta.headTsFloor, min(blockTimestamp, t.now().Unix())) + } + + t.addHeadReporter(ntwMeta, net, upstream) + var changed *NetworkMetadata + if headChanged { + changed = upsMeta + if blockTimestamp > 0 { + upsMeta.sampleBlockTime(blockNumber, blockTimestamp) + } + } + membershipChanged := t.evaluateHeadBench(ntwMeta, changed) + if membershipChanged { + // Bench membership also decides the finalized lag reference. + t.rederiveNetworkFinalized(ntwMeta, net, netLabel, &lg) + } oldNtwVal := ntwMeta.evmLatestBlockNumber.Load() - ntwBn, headReporter := t.corroboratedNetworkHead(ntwMeta, net, upstream, func(meta *NetworkMetadata) int64 { + ntwBn, headReporter := corroboratedNetworkHead(ntwMeta, func(meta *NetworkMetadata) int64 { return meta.evmLatestBlockNumber.Load() }) if ntwBn <= 0 { @@ -1489,18 +1812,36 @@ func (t *Tracker) SetLatestBlockNumber(upstream common.Upstream, blockNumber int } } - // 3) Recompute block head lag for this upstream - upsLag := blocksBehind(ntwBn, upsMeta.evmLatestBlockNumber.Load()) + // 3) Recompute block head lag for this upstream. A benched head that has + // not moved since its bench counts as no higher than the network head was + // then, so a head frozen above a chain that went on below it lags by what + // the network made since, even once the network passes its height (its + // blocks may be a dead fork). A benched head that moves again (replaying) + // lags by its real distance. A head that is not benched lags by what the + // network made beyond its last report (headReport). + now, blockTime := t.now(), time.Duration(ntwMeta.evmBlockTime.Load()) + upsMeta.headReport.reported(now, ntwBn) + lagHead := func(ups common.Upstream, meta *NetworkMetadata) int64 { + h := meta.evmLatestBlockNumber.Load() + if !meta.headBenched.Load() { + return meta.headReport.presumed(h, ntwBn, now, pollPeriod(ups, blockTime)) + } + if meta.headBenchRef > 0 && !meta.movedSinceBench() { + return min(h, meta.headBenchRef) + } + return h + } + upsLag := blocksBehind(ntwBn, lagHead(upstream, upsMeta)) gLag := t.getHeadLagGauge(t.projectId, vendor, netLabel, id) gLag.Set(float64(upsLag)) // 4) Update the TrackedMetrics.BlockHeadLag fields for Upstream(s) - if needsGlobalUpdate { + if needsGlobalUpdate || membershipChanged { // Recompute for every upstream in the network t.updateNetworkLagMetrics( net, ntwBn, - func(meta *NetworkMetadata) int64 { return meta.evmLatestBlockNumber.Load() }, + lagHead, func(tm *TrackedMetrics, lag int64) { tm.BlockHeadLag.Store(lag) }, t.getHeadLagGauge, &lg, @@ -1535,23 +1876,34 @@ const ( blockTimeMinSamples = 3 // minimum EMA samples before emitting (requires 4 observations total) ) -// updateBlockTimeSample feeds a new block into the EMA using on-chain -// block.timestamp (integer seconds). For fast chains where consecutive blocks -// share the same timestamp, we skip the sample and do NOT advance prev — the -// block gap accumulates until the timestamp ticks, then normalization recovers -// sub-second precision (e.g. 1s / 4 blocks = 250ms for Arbitrum). +// updateBlockTimeSample feeds the network head's block into the network block +// time EMA and publishes the result. func (t *Tracker) updateBlockTimeSample(ntwMeta *NetworkMetadata, netLabel string, blockNumber int64, blockTimestamp int64) { - ntwMeta.evmBlockTimeMu.Lock() - defer ntwMeta.evmBlockTimeMu.Unlock() + if ntwMeta.sampleBlockTime(blockNumber, blockTimestamp) { + telemetry.MetricNetworkDynamicBlockTime.WithLabelValues( + t.projectId, netLabel, + ).Set(float64(time.Duration(ntwMeta.evmBlockTime.Load()).Milliseconds())) + } +} - prevBlock := ntwMeta.evmBlockTimePrevBlock - prevTimestamp := ntwMeta.evmBlockTimePrevTimestamp +// sampleBlockTime feeds a new block into m's block time EMA using on-chain +// block.timestamp (integer seconds), and reports whether it stored a new +// evmBlockTime. For fast chains where consecutive blocks share the same +// timestamp, we skip the sample and do NOT advance prev — the block gap +// accumulates until the timestamp ticks, then normalization recovers +// sub-second precision (e.g. 1s / 4 blocks = 250ms for Arbitrum). +func (m *NetworkMetadata) sampleBlockTime(blockNumber int64, blockTimestamp int64) bool { + m.evmBlockTimeMu.Lock() + defer m.evmBlockTimeMu.Unlock() + + prevBlock := m.evmBlockTimePrevBlock + prevTimestamp := m.evmBlockTimePrevTimestamp // First observation: store as previous and return. if prevBlock == 0 { - ntwMeta.evmBlockTimePrevBlock = blockNumber - ntwMeta.evmBlockTimePrevTimestamp = blockTimestamp - return + m.evmBlockTimePrevBlock = blockNumber + m.evmBlockTimePrevTimestamp = blockTimestamp + return false } // Reject out-of-order or duplicate blocks. A block far behind prev means the @@ -1560,57 +1912,54 @@ func (t *Tracker) updateBlockTimeSample(ntwMeta *NetworkMetadata, netLabel strin blockGap := blockNumber - prevBlock if blockGap <= 0 { if prevBlock-blockNumber > common.DefaultToleratedBlockHeadRollback { - ntwMeta.evmBlockTimePrevBlock = blockNumber - ntwMeta.evmBlockTimePrevTimestamp = blockTimestamp + m.evmBlockTimePrevBlock = blockNumber + m.evmBlockTimePrevTimestamp = blockTimestamp } - return + return false } // Skip if timestamp hasn't advanced (fast chains where blocks share the same second). // Do NOT advance prev — let blockGap accumulate until the timestamp ticks. timestampDeltaSec := blockTimestamp - prevTimestamp if timestampDeltaSec <= 0 { - return + return false } // Per-block sample in nanoseconds (seconds → ms → ns, divided by block count). sampleNs := float64(timestampDeltaSec) * 1e9 / float64(blockGap) // Update EMA. - if ntwMeta.evmBlockTimeSamples == 0 { - ntwMeta.evmBlockTimeEmaNs = sampleNs + if m.evmBlockTimeSamples == 0 { + m.evmBlockTimeEmaNs = sampleNs } else { - ntwMeta.evmBlockTimeEmaNs = blockTimeEmaAlpha*sampleNs + (1-blockTimeEmaAlpha)*ntwMeta.evmBlockTimeEmaNs + m.evmBlockTimeEmaNs = blockTimeEmaAlpha*sampleNs + (1-blockTimeEmaAlpha)*m.evmBlockTimeEmaNs } - ntwMeta.evmBlockTimeSamples++ + m.evmBlockTimeSamples++ // Store current as previous. - ntwMeta.evmBlockTimePrevBlock = blockNumber - ntwMeta.evmBlockTimePrevTimestamp = blockTimestamp + m.evmBlockTimePrevBlock = blockNumber + m.evmBlockTimePrevTimestamp = blockTimestamp // Don't emit until we have enough samples. - if ntwMeta.evmBlockTimeSamples < blockTimeMinSamples { - return + if m.evmBlockTimeSamples < blockTimeMinSamples { + return false } - blockTimeNs := int64(ntwMeta.evmBlockTimeEmaNs) + blockTimeNs := int64(m.evmBlockTimeEmaNs) // Sanity bounds: reject absurd values. // Reset the internal EMA to the last published value so that recovery from a // prolonged halt doesn't create a delayed spike when the EMA first crosses // back below the threshold (e.g. jumping from 2s to ~119s). if blockTimeNs < int64(10*time.Millisecond) || blockTimeNs > int64(120*time.Second) { - if lastGood := ntwMeta.evmBlockTime.Load(); lastGood > 0 { - ntwMeta.evmBlockTimeEmaNs = float64(lastGood) + if lastGood := m.evmBlockTime.Load(); lastGood > 0 { + m.evmBlockTimeEmaNs = float64(lastGood) } - return + return false } - ntwMeta.evmBlockTime.Store(blockTimeNs) - - telemetry.MetricNetworkDynamicBlockTime.WithLabelValues( - t.projectId, netLabel, - ).Set(float64(time.Duration(blockTimeNs).Milliseconds())) + m.evmBlockTime.Store(blockTimeNs) + return true } // GetNetworkBlockTime returns the EMA-estimated block time for a network. @@ -1625,6 +1974,13 @@ func (t *Tracker) GetNetworkBlockTime(networkId string) time.Duration { return 0 } +// GetNetworkFinalizedBlock returns the network's corroborated finalized block +// (see corroboratedNetworkHead): the second-highest finalized block across its +// non-benched upstreams, or the only one. 0 until reported. +func (t *Tracker) GetNetworkFinalizedBlock(networkId string) int64 { + return t.getMetadata(metadataKey{nil, networkId}).evmFinalizedBlockNumber.Load() +} + func (t *Tracker) SetFinalizedBlockNumber(upstream common.Upstream, blockNumber int64) { lg := upstream.Logger().With().Str("networkId", upstream.NetworkId()).Logger() @@ -1662,54 +2018,28 @@ func (t *Tracker) SetFinalizedBlockNumber(upstream common.Upstream, blockNumber Msg("applied large finalized block rollback for upstream in tracker") } - oldNtwVal := ntwMeta.evmFinalizedBlockNumber.Load() - ntwVal, _ := t.corroboratedNetworkHead(ntwMeta, net, upstream, func(meta *NetworkMetadata) int64 { - return meta.evmFinalizedBlockNumber.Load() - }) + t.addHeadReporter(ntwMeta, net, upstream) + ntwVal, _ := t.rederiveNetworkFinalized(ntwMeta, net, netLabel, &lg) if ntwVal <= 0 { lg.Warn().Int64("value", ntwVal).Msg("ignoring finalization lag tracking for negative block number in tracker") return } - needsGlobalUpdate := ntwVal != oldNtwVal - if needsGlobalUpdate { - ntwMeta.evmFinalizedBlockNumber.Store(ntwVal) - g := t.getFinalizedBlockGauge(t.projectId, "*", netLabel, "*") - g.Set(float64(ntwVal)) - if ntwVal < oldNtwVal { - lg.Warn(). - Int64("previousValue", oldNtwVal). - Int64("newValue", ntwVal). - Msg("re-derived network finalized block from corroborated upstream heads in tracker") - } - } - // Recompute finalization lag for this upstream + // The re-derivation above credited this upstream against its previous + // report; its lag now is the one at this report. + upsMeta.finalizedReport.reported(t.now(), ntwVal) upsLag := blocksBehind(ntwVal, upsMeta.evmFinalizedBlockNumber.Load()) // Update Prometheus for this upstream gLag := t.getFinalizationLagGauge(t.projectId, vendor, netLabel, id) gLag.Set(float64(upsLag)) - // Update the finalization lag across the network if needed - if needsGlobalUpdate { - // Recompute for every upstream in the network - t.updateNetworkLagMetrics( - net, - ntwVal, - func(meta *NetworkMetadata) int64 { return meta.evmFinalizedBlockNumber.Load() }, - func(tm *TrackedMetrics, lag int64) { tm.FinalizationLag.Store(lag) }, - t.getFinalizationLagGauge, - &lg, - ) - } else { - // Only update finalization lag for this single upstream - t.updateSingleUpstreamLag( - id, - net, - upsLag, - func(tm *TrackedMetrics, lag int64) { tm.FinalizationLag.Store(lag) }, - ) - } + t.updateSingleUpstreamLag( + id, + net, + upsLag, + func(tm *TrackedMetrics, lag int64) { tm.FinalizationLag.Store(lag) }, + ) // Same {*, All} wildcard-aggregate guarantee as SetLatestBlockNumber (see // the comment there): the dedup index may not carry the "*" rollup, so @@ -1717,6 +2047,44 @@ func (t *Tracker) SetFinalizedBlockNumber(upstream common.Upstream, blockNumber t.loadOrStoreUpsMetrics(upstreamKey{upstream, "*", common.DataFinalityStateAll}).FinalizationLag.Store(upsLag) } +// rederiveNetworkFinalized re-derives the network finalized head from the +// non-benched reporters and, when it moved, every upstream's finalization lag. +// It runs on each finalized report and when the latest-axis bench changes who +// counts. It returns the network value (<= 0 while unknown) and whether it +// moved. Caller holds ntwMeta.headMu. +func (t *Tracker) rederiveNetworkFinalized(ntwMeta *NetworkMetadata, net, netLabel string, lg *zerolog.Logger) (int64, bool) { + oldNtwVal := ntwMeta.evmFinalizedBlockNumber.Load() + getVal := func(meta *NetworkMetadata) int64 { return meta.evmFinalizedBlockNumber.Load() } + ntwVal, _ := corroboratedNetworkHead(ntwMeta, getVal) + if ntwVal <= 0 || ntwVal == oldNtwVal { + return ntwVal, false + } + ntwMeta.evmFinalizedBlockNumber.Store(ntwVal) + t.getFinalizedBlockGauge(t.projectId, "*", netLabel, "*").Set(float64(ntwVal)) + if ntwVal < oldNtwVal { + lg.Warn(). + Int64("previousValue", oldNtwVal). + Int64("newValue", ntwVal). + Msg("re-derived network finalized block from corroborated upstream heads in tracker") + } + now, blockTime := t.now(), time.Duration(ntwMeta.evmBlockTime.Load()) + t.updateNetworkLagMetrics( + net, + ntwVal, + func(ups common.Upstream, meta *NetworkMetadata) int64 { + f := meta.evmFinalizedBlockNumber.Load() + if meta.headBenched.Load() { + return f + } + return meta.finalizedReport.presumed(f, ntwVal, now, pollPeriod(ups, blockTime)) + }, + func(tm *TrackedMetrics, lag int64) { tm.FinalizationLag.Store(lag) }, + t.getFinalizationLagGauge, + lg, + ) + return ntwVal, true +} + func (t *Tracker) RecordBlockHeadLargeRollback(upstream common.Upstream, finality string, currentVal, newVal int64) { rollback := currentVal - newVal diff --git a/health/tracker_corroborated_head_test.go b/health/tracker_corroborated_head_test.go index c2de21a2f..5499f976a 100644 --- a/health/tracker_corroborated_head_test.go +++ b/health/tracker_corroborated_head_test.go @@ -56,16 +56,17 @@ func TestTrackerCorroboratedHeadConcurrentUpdatesDoNotRegress(t *testing.T) { upstreams := []common.Upstream{a, b, c, d} ntwMeta := tracker.getMetadata(metadataKey{nil, net}) - ntwMeta.reporters = upstreams for i, head := range []int64{99, 100, 99, 99} { - tracker.getMetadata(metadataKey{upstreams[i], net}).evmLatestBlockNumber.Store(head) + tracker.addHeadReporter(ntwMeta, net, upstreams[i]).evmLatestBlockNumber.Store(head) } ntwMeta.evmLatestBlockNumber.Store(99) + // d is registered after c, so registering d's report walks past c and + // parks inside the network's head critical section. c.armed.Store(true) firstDone := make(chan struct{}) go func() { - tracker.SetLatestBlockNumber(a, 101, 0) + tracker.SetLatestBlockNumber(d, 101, 0) close(firstDone) }() <-c.reached diff --git a/indexer/adapters/wsupstream/adapter.go b/indexer/adapters/wsupstream/adapter.go index d0334ed63..d6d45c8e6 100644 --- a/indexer/adapters/wsupstream/adapter.go +++ b/indexer/adapters/wsupstream/adapter.go @@ -7,10 +7,10 @@ import ( "context" "encoding/json" "errors" - "fmt" "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/erpc/erpc/clients" @@ -37,6 +37,10 @@ var ( resubRetryMax = 30 * time.Second ) +// headsWatchInterval is how long newHeads may stay silent about a block the +// upstream already reports over RPC; a var so tests can shorten it. +var headsWatchInterval = 15 * time.Second + var errNotConnected = errors.New("upstream websocket is not connected") // Adapter is the EventIngress of one WS upstream on one network. @@ -60,6 +64,12 @@ type Adapter struct { stripSubscribeFromBlockZero bool + // polledHead is the upstream's latest block as polled over RPC, 0 if + // unknown; nil disables watchHeads. + polledHead func() int64 + headsSeen atomic.Uint64 + lastHead atomic.Int64 + nw indexer.NetworkHandle sink indexer.Sink @@ -125,6 +135,12 @@ func New(up *upstream.Upstream, networkID string, logger *zerolog.Logger, opts O retryMax: resubRetryMax, attemptTimeout: resubAttemptTimeout, stripSubscribeFromBlockZero: opts.StripSubscribeFromBlockZero, + polledHead: func() int64 { + if p := up.EvmStatePoller(); p != nil && !p.IsObjectNull() { + return p.LatestBlock() + } + return 0 + }, } } @@ -174,9 +190,42 @@ func (a *Adapter) Start(_ context.Context, nw indexer.NetworkHandle, sink indexe }) a.startResubscribe() + if a.polledHead != nil { + go a.watchHeads() + } return nil } +// watchHeads reconnects when the newHeads subscription stays silent for a +// whole interval although the upstream reported a newer block over RPC when +// it began: pings keep such a connection up, so nothing else notices the +// dead subscription. A halted chain or a lagging node trips neither half. +func (a *Adapter) watchHeads() { + ticker := time.NewTicker(headsWatchInterval) + defer ticker.Stop() + var prevID string + var prevEpoch, prevSeen uint64 + var prevPolled int64 + for { + select { + case <-a.wsClient.Done(): + return + case <-ticker.C: + } + a.subsMu.Lock() + id, epoch := a.heads.id, a.heads.epoch + a.subsMu.Unlock() + seen, lastHead := a.headsSeen.Load(), a.lastHead.Load() + if id != "" && id == prevID && epoch == prevEpoch && seen == prevSeen && prevPolled > lastHead { + a.logger.Warn().Str("upstreamSubId", id).Int64("lastHead", lastHead).Int64("polledHead", prevPolled). + Msg("newHeads subscription silent while upstream head advances, reconnecting") + telemetry.CounterHandle(telemetry.MetricUpstreamWebsocketHeadsStallReconnectsTotal, a.projectID, a.networkLabel, a.upstreamID).Inc() + a.wsClient.Reconnect(epoch) + } + prevID, prevEpoch, prevSeen, prevPolled = id, epoch, seen, a.polledHead() + } +} + // startResubscribe launches the retry loop for the current connection // unless one already runs for it. func (a *Adapter) startResubscribe() { @@ -261,19 +310,29 @@ func filterKey(subType, paramsHash string) string { // reconnect. func (a *Adapter) resubscribeWithRetry(ctx context.Context) { backoff := a.retryMin + headsUnsupported := false for { if ctx.Err() != nil { return } done := true - attemptCtx, cancel := context.WithTimeout(ctx, a.attemptTimeout) - a.heads.mu.Lock() - err := a.subscribeLocked(attemptCtx, &a.heads, []interface{}{indexer.SubTypeNewHeads}, a.handleNewHeads) - a.heads.mu.Unlock() - cancel() - if err != nil { - done = false - a.logger.Warn().Err(err).Msg("failed to subscribe newHeads, will retry") + if !headsUnsupported { + attemptCtx, cancel := context.WithTimeout(ctx, a.attemptTimeout) + a.heads.mu.Lock() + err := a.subscribeLocked(attemptCtx, &a.heads, []interface{}{indexer.SubTypeNewHeads}, a.handleNewHeads) + a.heads.mu.Unlock() + cancel() + switch { + case err == nil: + case common.HasErrorCode(err, common.ErrCodeEndpointUnsupported): + // Retrying can't change the answer; the state poller tracks + // the head, and a later resubscribe asks again. + headsUnsupported = true + a.logger.Warn().Err(err).Msg("upstream does not support newHeads, not retrying on this connection") + default: + done = false + a.logger.Warn().Err(err).Msg("failed to subscribe newHeads, will retry") + } } a.subsMu.Lock() @@ -328,13 +387,9 @@ func (a *Adapter) subscribeFilterLocked(ctx context.Context, sub *filterSub) err outParams = cleaned } } - err := a.subscribeLocked(ctx, &sub.upstreamSub, outParams, func(params []byte) { + return a.subscribeLocked(ctx, &sub.upstreamSub, outParams, func(params []byte) { a.handleFilter(sub.subType, sub.paramsHash, params) }) - if err != nil { - return fmt.Errorf("filter subscribe: %w", err) - } - return nil } // subscribeLocked subscribes sub unless it is live or dropped; the caller @@ -411,7 +466,10 @@ func (a *Adapter) release(ctx context.Context, id string, epoch uint64) { } ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout) defer cancel() - _ = a.send(clients.WithWsSubscription(ctx, &clients.WsSubscription{Epoch: epoch}), methodEthUnsubscribe, []interface{}{id}) + if err := a.send(clients.WithWsSubscription(ctx, &clients.WsSubscription{Epoch: epoch}), methodEthUnsubscribe, []interface{}{id}); err != nil { + // A quiet subscription never triggers the client's orphan cancel. + a.wsClient.UnsubscribeOrphan(id, epoch) + } } // send forwards a subscription RPC through the upstream. It is marked @@ -470,6 +528,8 @@ func (a *Adapter) handleNewHeads(raw []byte) { a.logger.Warn().Err(err).Str("number", header.Number).Msg("failed to parse block number") return } + a.lastHead.Store(num) + a.headsSeen.Add(1) a.sink.Ingest(indexer.StreamEvent{ Kind: indexer.KindNewHead, NetworkId: a.networkID, diff --git a/indexer/adapters/wsupstream/adapter_error_test.go b/indexer/adapters/wsupstream/adapter_error_test.go new file mode 100644 index 000000000..b6adbc2ed --- /dev/null +++ b/indexer/adapters/wsupstream/adapter_error_test.go @@ -0,0 +1,54 @@ +package wsupstream + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/indexer" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestAdapterEnsureFilterKeepsStructuredError: an upstream rejection of a +// filter subscribe, as the indexer hands it to the subscription manager +// (errors.Join over ingresses), must grade and label by its error code and +// reach the client with the upstream's JSON-RPC code, as on the HTTP path. +func TestAdapterEnsureFilterKeepsStructuredError(t *testing.T) { + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + nop := zerolog.Nop() + a.logger = &nop + t.Cleanup(func() { + a.subsMu.Lock() + if a.resubCancel != nil { + a.resubCancel() + } + a.subsMu.Unlock() + }) + <-server.newConn + require.Eventually(t, a.wsClient.IsConnected, 3*time.Second, 10*time.Millisecond) + a.sink = sink + + upstreamErr := common.NewErrEndpointClientSideException( + common.NewErrJsonRpcExceptionInternal(-32602, common.JsonRpcErrorInvalidArgument, "invalid filter", nil, nil), + ) + a.forward = func(context.Context, *common.NormalizedRequest, bool) (*common.NormalizedResponse, error) { + return nil, upstreamErr + } + + params := logsParams("0xabc") + ensureErr := a.EnsureFilter(context.Background(), indexer.SubTypeLogs, indexer.BuildParamsKey(params), params) + require.Error(t, ensureErr) + err := errors.Join(ensureErr) // indexer.subscribe's shape + + assert.Equal(t, common.SeverityInfo, common.ClassifySeverity(err)) + assert.Equal(t, common.ErrorFingerprint(upstreamErr), common.ErrorFingerprint(err)) + + jre := &common.ErrJsonRpcExceptionInternal{} + require.True(t, errors.As(common.TranslateToJsonRpcException(err), &jre)) + assert.EqualValues(t, common.JsonRpcErrorInvalidArgument, jre.NormalizedCode()) +} diff --git a/indexer/adapters/wsupstream/adapter_heads_watch_test.go b/indexer/adapters/wsupstream/adapter_heads_watch_test.go new file mode 100644 index 000000000..5a45eaee8 --- /dev/null +++ b/indexer/adapters/wsupstream/adapter_heads_watch_test.go @@ -0,0 +1,104 @@ +package wsupstream + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/clients" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/indexer" + "github.com/erpc/erpc/telemetry" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +func compressHeadsWatch(t *testing.T) { + orig := headsWatchInterval + headsWatchInterval = 50 * time.Millisecond + t.Cleanup(func() { headsWatchInterval = orig }) +} + +// TestAdapterReconnectsSilentNewHeads: a newHeads subscription that goes +// silent on a live connection while the upstream's polled head advances is +// recovered by reconnecting. +func TestAdapterReconnectsSilentNewHeads(t *testing.T) { + compressResubRetry(t) + compressHeadsWatch(t) + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + var polled atomic.Int64 + a.polledHead = polled.Load + conn1 := <-server.newConn + require.NoError(t, a.Start(context.Background(), fakeNetworkHandle{}, sink)) + require.Eventually(t, subscribedHeads(a), 3*time.Second, 10*time.Millisecond) + + stalls := telemetry.MetricUpstreamWebsocketHeadsStallReconnectsTotal.WithLabelValues("test-project", "evm:123", "test-ws-upstream") + stallsBefore := testutil.ToFloat64(stalls) + + conn1.sendNewHead(conn1.nextRequest(t, methodEthSubscribe).SubID, 100) + require.Equal(t, int64(100), nextEvent(t, sink, indexer.KindNewHead).Block.Number) + polled.Store(105) + + var conn2 *notifyConn + select { + case conn2 = <-server.newConn: + case <-time.After(3 * time.Second): + t.Fatal("silent newHeads subscription was never recovered") + } + require.Equal(t, stallsBefore+1, testutil.ToFloat64(stalls), "a forced reconnect is counted") + require.Eventually(t, subscribedHeads(a), 3*time.Second, 10*time.Millisecond) + conn2.sendNewHead(conn2.nextRequest(t, methodEthSubscribe).SubID, 106) + require.Equal(t, int64(106), nextEvent(t, sink, indexer.KindNewHead).Block.Number) +} + +// TestAdapterKeepsSilentNewHeadsWhenChainHalts: silence while the upstream +// reports no newer block is a halted chain, not a dead subscription. +func TestAdapterKeepsSilentNewHeadsWhenChainHalts(t *testing.T) { + compressResubRetry(t) + compressHeadsWatch(t) + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + var polled atomic.Int64 + a.polledHead = polled.Load + conn1 := <-server.newConn + require.NoError(t, a.Start(context.Background(), fakeNetworkHandle{}, sink)) + require.Eventually(t, subscribedHeads(a), 3*time.Second, 10*time.Millisecond) + + conn1.sendNewHead(conn1.nextRequest(t, methodEthSubscribe).SubID, 100) + require.Equal(t, int64(100), nextEvent(t, sink, indexer.KindNewHead).Block.Number) + polled.Store(100) + + select { + case <-server.newConn: + t.Fatal("reconnected although the upstream reported no newer block") + case <-time.After(500 * time.Millisecond): + } +} + +// TestAdapterWatchHeadsStopsWithClient: the watch loop ends with the +// upstream client instead of outliving it. +func TestAdapterWatchHeadsStopsWithClient(t *testing.T) { + compressHeadsWatch(t) + server := newNotifyServer(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logger := zerolog.Nop() + ci, err := clients.NewWsJsonRpcClient(ctx, &logger, "test-project", common.NewFakeUpstream("test-ws-upstream"), server.wsURL(t), nil, nil) + require.NoError(t, err) + a := &Adapter{wsClient: ci.(*clients.WsJsonRpcClient), logger: &logger, polledHead: func() int64 { return 0 }} + + done := make(chan struct{}) + go func() { + a.watchHeads() + close(done) + }() + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watchHeads kept running after the upstream client shut down") + } +} diff --git a/indexer/adapters/wsupstream/adapter_reconnect_test.go b/indexer/adapters/wsupstream/adapter_reconnect_test.go index 9ff85a4a7..51ba314cd 100644 --- a/indexer/adapters/wsupstream/adapter_reconnect_test.go +++ b/indexer/adapters/wsupstream/adapter_reconnect_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/erpc/erpc/architecture/evm" "github.com/erpc/erpc/clients" "github.com/erpc/erpc/common" "github.com/erpc/erpc/indexer" @@ -30,6 +31,9 @@ func (fakeNetworkHandle) Id() string { return "evm:12 func (fakeNetworkHandle) SuggestLatestBlock(string, int64) bool { return true } func (fakeNetworkHandle) DedupDropped(indexer.EventKind) {} func (fakeNetworkHandle) DedupEntriesAdded(int) {} +func (fakeNetworkHandle) HeadIndexed() {} +func (fakeNetworkHandle) Undelivered(indexer.EventKind) {} +func (fakeNetworkHandle) DeliveringHead(int64) {} type fakeSink struct { events chan indexer.StreamEvent @@ -47,6 +51,8 @@ type notifyServer struct { // gate, when set, is received from after each eth_subscribe is // recorded and before it is answered. gate chan struct{} + // rejectNewHeads answers newHeads subscribes with -32601. + rejectNewHeads bool } type notifyConn struct { @@ -72,7 +78,7 @@ func newNotifyServer(t *testing.T) *notifyServer { } nc := ¬ifyConn{conn: conn, requests: make(chan notifyRequest, 64)} n.newConn <- nc - go nc.serve(n.gate) + go nc.serve(n.gate, n.rejectNewHeads) })) t.Cleanup(n.srv.Close) return n @@ -85,7 +91,7 @@ func (n *notifyServer) wsURL(t *testing.T) *url.URL { return u } -func (nc *notifyConn) serve(gate chan struct{}) { +func (nc *notifyConn) serve(gate chan struct{}, rejectNewHeads bool) { subs := 0 for { _, msg, err := nc.conn.ReadMessage() @@ -110,6 +116,10 @@ func (nc *notifyConn) serve(gate chan struct{}) { if gate != nil { <-gate } + if rejectNewHeads && req.Params[0] == indexer.SubTypeNewHeads { + nc.write(`{"jsonrpc":"2.0","id":` + string(req.ID) + `,"error":{"code":-32601,"message":"the method eth_subscribe does not exist/is not available"}}`) + continue + } nc.write(`{"jsonrpc":"2.0","id":` + string(req.ID) + `,"result":"` + req.SubID + `"}`) if req.Params[0] == indexer.SubTypeLogs { nc.sendLog(req.SubID, 1) @@ -164,15 +174,17 @@ func newTestAdapter(t *testing.T, u *url.URL) (*Adapter, *fakeSink) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) logger := zerolog.New(zerolog.NewTestWriter(t)).Level(zerolog.WarnLevel) - ci, err := clients.NewWsJsonRpcClient(ctx, &logger, "test-project", common.NewFakeUpstream("test-ws-upstream"), u, nil, nil) + ci, err := clients.NewWsJsonRpcClient(ctx, &logger, "test-project", common.NewFakeUpstream("test-ws-upstream"), u, nil, evm.NewJsonRpcErrorExtractor()) require.NoError(t, err) wsc := ci.(*clients.WsJsonRpcClient) a := &Adapter{ - upstreamID: "test-ws-upstream", - networkID: "evm:123", - wsClient: wsc, - logger: &logger, - filters: make(map[string]*filterSub), + upstreamID: "test-ws-upstream", + networkID: "evm:123", + projectID: "test-project", + networkLabel: "evm:123", + wsClient: wsc, + logger: &logger, + filters: make(map[string]*filterSub), forward: func(ctx context.Context, nq *common.NormalizedRequest, _ bool) (*common.NormalizedResponse, error) { return wsc.SendRequest(ctx, nq) }, diff --git a/indexer/adapters/wsupstream/adapter_release_fallback_test.go b/indexer/adapters/wsupstream/adapter_release_fallback_test.go new file mode 100644 index 000000000..6a68bd66a --- /dev/null +++ b/indexer/adapters/wsupstream/adapter_release_fallback_test.go @@ -0,0 +1,33 @@ +package wsupstream + +import ( + "context" + "testing" + "time" + + "github.com/erpc/erpc/indexer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Residual of the 6057bccb cap safety net: a release whose eth_unsubscribe +// is rejected before reaching the wire leaves the node-side subscription +// live. A noisy one is reclaimed by the orphan path on its next +// notification; a quiet one holds a per-connection slot until reconnect. +func TestAdapterQuietSubscriptionReleasedDespiteFailedUnsubscribe(t *testing.T) { + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + conn := <-server.newConn + require.Eventually(t, a.wsClient.IsConnected, 3*time.Second, 10*time.Millisecond) + a.sink = sink + failed := failFirstUnsubscribe(a) + + params := []interface{}{indexer.SubTypeNewPendingTransactions} + hash := indexer.BuildParamsKey(params) + require.NoError(t, a.EnsureFilter(context.Background(), indexer.SubTypeNewPendingTransactions, hash, params)) + subID := conn.nextRequest(t, methodEthSubscribe).SubID + + require.NoError(t, a.RemoveFilter(context.Background(), indexer.SubTypeNewPendingTransactions, hash)) + require.True(t, failed.Load()) + assert.Equal(t, []interface{}{subID}, conn.nextRequest(t, methodEthUnsubscribe).Params) +} diff --git a/indexer/adapters/wsupstream/adapter_subscribe_unsupported_test.go b/indexer/adapters/wsupstream/adapter_subscribe_unsupported_test.go new file mode 100644 index 000000000..ed83eab4e --- /dev/null +++ b/indexer/adapters/wsupstream/adapter_subscribe_unsupported_test.go @@ -0,0 +1,55 @@ +package wsupstream + +import ( + "context" + "testing" + "time" + + "github.com/erpc/erpc/indexer" + "github.com/stretchr/testify/require" +) + +// TestAdapterGivesUpNewHeadsWhenUnsupported: an upstream that rejects +// newHeads as an unsupported method is asked once per connection, and its +// filters still settle. +func TestAdapterGivesUpNewHeadsWhenUnsupported(t *testing.T) { + compressResubRetry(t) + server := newNotifyServer(t) + server.rejectNewHeads = true + a, sink := newTestAdapter(t, server.wsURL(t)) + conn1 := <-server.newConn + require.Eventually(t, a.wsClient.IsConnected, 3*time.Second, 10*time.Millisecond) + require.NoError(t, a.Start(context.Background(), fakeNetworkHandle{}, sink)) + + params := logsParams("0xabc") + require.NoError(t, a.EnsureFilter(context.Background(), indexer.SubTypeLogs, indexer.BuildParamsKey(params), params)) + + var headsSubscribes int + deadline := time.After(time.Second) + for done := false; !done; { + select { + case req := <-conn1.requests: + if req.Method == methodEthSubscribe && req.Params[0] == indexer.SubTypeNewHeads { + headsSubscribes++ + } + case <-deadline: + done = true + } + } + require.Equal(t, 1, headsSubscribes, "newHeads must not be retried after an unsupported rejection") + require.False(t, a.HeadsLive()) + a.subsMu.Lock() + settled := a.resubDone + a.subsMu.Unlock() + require.True(t, settled, "resubscribe loop must settle without newHeads") + + _ = conn1.conn.UnderlyingConn().Close() + conn2 := <-server.newConn + require.Equal(t, indexer.SubTypeNewHeads, conn2.nextRequest(t, methodEthSubscribe).Params[0], + "a new connection asks for newHeads again") + require.Eventually(t, func() bool { + a.subsMu.Lock() + defer a.subsMu.Unlock() + return a.resubDone + }, 3*time.Second, 10*time.Millisecond, "resubscribe loop must settle on the new connection") +} diff --git a/indexer/indexer.go b/indexer/indexer.go index 6a00fa644..cfb3de7d5 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -419,8 +419,13 @@ func (i *Indexer) Ingest(ev StreamEvent) { ns.handle.DedupDropped(ev.Kind) return } + if ev.Kind == KindNewHead { + ns.handle.HeadIndexed() + } - i.fanOut(IndexedEvent{StreamEvent: ev, Removed: removed}) + if !i.fanOut(ns, IndexedEvent{StreamEvent: ev, Removed: removed}) { + ns.handle.Undelivered(ev.Kind) + } } // dedupe reports whether ev should be delivered. Heads are compared to the @@ -458,13 +463,20 @@ func (i *Indexer) dedupe(ns *networkState, ev *StreamEvent, removed bool) bool { } } -func (i *Indexer) fanOut(ev IndexedEvent) { +// fanOut delivers ev to every interested egress and reports whether there +// was one. A head is reported to the network before its first delivery. +func (i *Indexer) fanOut(ns *networkState, ev IndexedEvent) (delivered bool) { i.egresses.Range(func(_, v any) bool { eg := v.(EventEgress) if !eg.InterestedIn(ev.Kind, ev.NetworkId, ev.FilterHash) { return true } + if !delivered && ev.Kind == KindNewHead && !ev.Block.Zero() { + ns.handle.DeliveringHead(ev.Block.Number) + } eg.Deliver(ev) + delivered = true return true }) + return delivered } diff --git a/indexer/indexer_test.go b/indexer/indexer_test.go index cf9887895..22dc01657 100644 --- a/indexer/indexer_test.go +++ b/indexer/indexer_test.go @@ -22,6 +22,9 @@ type fakeNetwork struct { held map[string]struct{} // sourceIds whose heads are not delivered dedupDropped int dedupEntries int + headsIndexed int + undelivered int + delivering []int64 // heads reported before reaching a client } func newFakeNetwork(id string) *fakeNetwork { @@ -49,6 +52,26 @@ func (n *fakeNetwork) DedupEntriesAdded(delta int) { defer n.mu.Unlock() n.dedupEntries += delta } +func (n *fakeNetwork) HeadIndexed() { + n.mu.Lock() + defer n.mu.Unlock() + n.headsIndexed++ +} +func (n *fakeNetwork) Undelivered(EventKind) { + n.mu.Lock() + defer n.mu.Unlock() + n.undelivered++ +} +func (n *fakeNetwork) DeliveringHead(block int64) { + n.mu.Lock() + defer n.mu.Unlock() + n.delivering = append(n.delivering, block) +} +func (n *fakeNetwork) deliveryCounts() (headsIndexed, undelivered int, delivering []int64) { + n.mu.Lock() + defer n.mu.Unlock() + return n.headsIndexed, n.undelivered, append([]int64(nil), n.delivering...) +} func (n *fakeNetwork) dedupCounts() (dropped, entries int) { n.mu.Lock() defer n.mu.Unlock() @@ -364,6 +387,43 @@ func TestIndexer_Log_RefcountFanOutAndTeardown(t *testing.T) { } } +// Only heads a client receives are reported as delivering, once each: a head +// no client subscribed to must not floor the network's "latest". +func TestIndexer_ReportsIndexedHeadsAndUndeliveredEvents(t *testing.T) { + idx := newIndexer(t) + nw := newFakeNetwork("evm:1") + nw.held = map[string]struct{}{"ws:fb": {}} + idx.RegisterNetwork(nw) + head := func(src string, num int64, hash string) StreamEvent { + return StreamEvent{Kind: KindNewHead, NetworkId: "evm:1", SourceId: src, Block: BlockRef{Number: num, Hash: hash}} + } + want := func(headsIndexed, undelivered int, delivering ...int64) { + t.Helper() + h, u, d := nw.deliveryCounts() + if h != headsIndexed || u != undelivered || fmt.Sprint(d) != fmt.Sprint(delivering) { + t.Fatalf("want %d indexed heads, %d undelivered, delivering %v; got %d, %d, %v", + headsIndexed, undelivered, delivering, h, u, d) + } + } + + // No client subscribes to heads yet. + idx.Ingest(head("ws:up1", 100, "0xA")) + want(1, 1) + + idx.Attach(&fakeEgress{name: "eg1", acceptAllHeads: true}) + idx.Attach(&fakeEgress{name: "eg2", acceptAllHeads: true}) + idx.Ingest(head("ws:up1", 101, "0xB")) + want(2, 1, 101) + // A duplicate and a withheld head are neither indexed nor undelivered. + idx.Ingest(head("ws:up2", 101, "0xB")) + idx.Ingest(head("ws:fb", 102, "0xC")) + want(2, 1, 101) + + // A log for a filter no client holds. + idx.Ingest(logEvent("orphan", "ws:up1", logAddedJSON)) + want(2, 2, 101) +} + // newLogFilter registers "evm:1", ensures an empty logs filter and attaches // an egress interested in it. Helper for log dedup tests. func newLogFilter(t *testing.T) (*Indexer, string, *fakeEgress) { diff --git a/indexer/ingress.go b/indexer/ingress.go index 8ca08f411..06b9b23e2 100644 --- a/indexer/ingress.go +++ b/indexer/ingress.go @@ -14,14 +14,23 @@ type NetworkHandle interface { Id() string // SuggestLatestBlock reports a head observed by source and whether // clients may receive it. It is called before dedup and fan-out, so - // every source's observation counts and the network tip moves before - // clients see the head; a head it does not deliver is dropped. + // every source's observation reaches its poller; a head it does not + // deliver is dropped. SuggestLatestBlock(sourceId string, blockNumber int64) (deliver bool) + // DeliveringHead reports a head about to reach its first client + // subscription, so "latest" can be floored to it before the client sees + // it. A head no client subscribed to is never reported. Indexer only. + DeliveringHead(blockNumber int64) // DedupDropped reports an event dropped as a duplicate. Indexer only. DedupDropped(kind EventKind) // DedupEntriesAdded reports that the network's filter dedup windows // gained delta keys (negative when a filter is dropped). Indexer only. DedupEntriesAdded(delta int) + // HeadIndexed reports a new head that passed dedup. Indexer only. + HeadIndexed() + // Undelivered reports an event that passed dedup but matched no client + // subscription. Indexer only. + Undelivered(kind EventKind) } // EventIngress turns a transport-specific subscription into StreamEvents diff --git a/indexer/integration_test.go b/indexer/integration_test.go index cf8816f54..b51b04a63 100644 --- a/indexer/integration_test.go +++ b/indexer/integration_test.go @@ -68,6 +68,9 @@ type stubNetwork struct { func (s *stubNetwork) Id() string { return s.id } func (s *stubNetwork) DedupDropped(indexer.EventKind) {} func (s *stubNetwork) DedupEntriesAdded(int) {} +func (s *stubNetwork) HeadIndexed() {} +func (s *stubNetwork) Undelivered(indexer.EventKind) {} +func (s *stubNetwork) DeliveringHead(int64) {} func (s *stubNetwork) SuggestLatestBlock(sourceID string, block int64) bool { s.mu.Lock() if s.suggestions == nil { diff --git a/internal/policy/decision.go b/internal/policy/decision.go index 4fe5fb03d..d148b332a 100644 --- a/internal/policy/decision.go +++ b/internal/policy/decision.go @@ -67,6 +67,10 @@ type DecisionOutput struct { // for real. Always captured (production-default), not gated by the // step-log toggle. ShadowReasons map[string][]string + + // MethodBlocks[upstreamID][method] is what `blockMethods` recorded: + // the upstream stays in Order but is skipped for that method. + MethodBlocks map[string]map[string]MethodBlock } // ExcludedUpstream explains why an upstream was dropped this tick. diff --git a/internal/policy/default_policy.js b/internal/policy/default_policy.js index 0ed07e045..45d59e45d 100644 --- a/internal/policy/default_policy.js +++ b/internal/policy/default_policy.js @@ -39,8 +39,11 @@ // Outer `samplesAbove(20)` gates on aggregate counts so the // predicate doesn't even run on cold-start pods. .excludeIf(any(all(samplesAbove(20), latencyAbove(3000), latencyDeviationAbove(3, { mode: 'majority' })), latencyAbove(10_000))) - // Block-head lag: drop if behind tip by ≥16 blocks or ≥30s. - .excludeIf(any(blockNumberLagAbove(16), blockSecondsLagAbove(30))) + // Block-head lag: drop if behind tip by ≥16 blocks or ≥30s. Not + // probed: lag comes from the state poller and real traffic, never + // from probe responses, so probing a lagging upstream re-admits it + // no sooner and only bills its vendor. + .excludeIf(any(blockNumberLagAbove(16), blockSecondsLagAbove(30)), { probe: false }) // Outage safety net: if everyone failed the health excludes, fall // back to the raw set rather than failing closed. .whenEmpty(() => upstreams) @@ -52,12 +55,12 @@ // Hold the primary stable across ticks unless a meaningfully better // option exists for ≥30s. .stickyPrimary({ hysteresis: 0.30, minSwitchInterval: '30s' }) - // Shadow-mirror sampled real traffic to currently-excluded - // upstreams in the background so they accumulate fresh tracker - // samples without touching real user traffic. sampleRate=0.1 - // bounds the per-excluded-upstream probe RPS on high-RPS - // networks; minSamples=10 ensures even low-traffic networks - // probe enough to clear the chain's `samplesAbove(N)` gates on - // re-admission. maxConcurrent=4 is the absolute concurrent cap + // Shadow-mirror sampled real traffic to upstreams excluded for + // errors, throttling or latency, in the background, so they + // accumulate fresh tracker samples without touching real user + // traffic. sampleRate=0.1 bounds the per-excluded-upstream probe + // RPS on high-RPS networks; minSamples=10 ensures even low-traffic + // networks probe enough to clear the chain's `samplesAbove(N)` gates + // on re-admission. maxConcurrent=4 is the absolute concurrent cap // per upstream. Per-upstream opt-out via `routing.probe: off`. .probeExcluded({ sampleRate: 0.1, minSamples: 10, minSamplesWindow: '60s', maxConcurrent: 4, timeout: '10s' }) diff --git a/internal/policy/dump_test.go b/internal/policy/dump_test.go new file mode 100644 index 000000000..6d5ee9d6e --- /dev/null +++ b/internal/policy/dump_test.go @@ -0,0 +1,96 @@ +package policy + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/grafana/sobek" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func dumpConfig(script *sobek.Program, policies ...*common.SelectionPolicyConfig) *common.Config { + nws := []*common.NetworkConfig{nil} + for _, p := range policies { + nws = append(nws, &common.NetworkConfig{SelectionPolicy: p}) + } + return &common.Config{ + UserScript: script, + Projects: []*common.ProjectConfig{nil, {Networks: nws}}, + } +} + +func TestResolveEffectiveSelectionPolicies(t *testing.T) { + ResolveEffectiveSelectionPolicies(nil) // nil config is a no-op + + script, err := sobek.Compile("user.js", ` +globalThis.__erpcFns['fast'] = function fast(ups) { return ups; }; +globalThis.__erpcFns['nostr'] = Object.create(null); +`, false) + require.NoError(t, err) + + custom := "(ups) => ups.slice(0, 1)" + missing := &common.SelectionPolicyConfig{EvalFunc: "__ts_fn__:missing"} + fast := &common.SelectionPolicyConfig{EvalFunc: "__ts_fn__:fast"} + nostr := &common.SelectionPolicyConfig{EvalFunc: "__ts_fn__:nostr"} + empty := &common.SelectionPolicyConfig{EvalFunc: "__ts_fn__:"} + trivial := &common.SelectionPolicyConfig{EvalFunc: " " + common.DefaultSelectionPolicySource + "\n"} + user := &common.SelectionPolicyConfig{EvalFunc: custom} + cfg := dumpConfig(script, nil, missing, fast, nostr, empty, trivial, user) + + ResolveEffectiveSelectionPolicies(cfg) + nws := cfg.Projects[1].Networks + + rich := DefaultPolicySource() + require.NotNil(t, nws[1].SelectionPolicy, "nil policy gets the rich default") + assert.Equal(t, rich, nws[1].SelectionPolicy.EvalFunc) + assert.NotZero(t, nws[1].SelectionPolicy.EvalInterval, "filled default carries SetDefaults values") + + assert.Equal(t, "__ts_fn__:missing", missing.EvalFunc, "unknown id keeps the sentinel") + assert.Equal(t, "function fast(ups) { return ups; }", fast.EvalFunc) + assert.Equal(t, "__ts_fn__:nostr", nostr.EvalFunc) + assert.Equal(t, "__ts_fn__:", empty.EvalFunc) + assert.Equal(t, rich, trivial.EvalFunc) + assert.Equal(t, rich, trivial.EvalFuncOriginal) + assert.Equal(t, custom, user.EvalFunc, "operator-supplied source is untouched") +} + +func TestResolveEffectiveSelectionPolicies_UnresolvableScripts(t *testing.T) { + broken, err := sobek.Compile("broken.js", `throw new Error("nope")`, false) + require.NoError(t, err) + wipes, err := sobek.Compile("wipe.js", `globalThis.__erpcFns = undefined;`, false) + require.NoError(t, err) + + tests := []struct { + name string + script *sobek.Program + }{ + {"no user script", nil}, + {"user script throws", broken}, + {"registry removed", wipes}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := &common.SelectionPolicyConfig{EvalFunc: "__ts_fn__:a"} + b := &common.SelectionPolicyConfig{EvalFunc: "__ts_fn__:b"} + ResolveEffectiveSelectionPolicies(dumpConfig(tc.script, a, b)) + assert.Equal(t, "__ts_fn__:a", a.EvalFunc) + assert.Equal(t, "__ts_fn__:b", b.EvalFunc, "a failed runtime is not retried for later networks") + }) + } +} + +func TestUpgradeDefaultPolicy(t *testing.T) { + require.NoError(t, upgradeDefaultPolicy(nil)) + + custom := &common.SelectionPolicyConfig{EvalFunc: "(u) => u"} + require.NoError(t, upgradeDefaultPolicy(custom)) + assert.Equal(t, "(u) => u", custom.EvalFunc) + assert.Nil(t, custom.CompiledProgram) + + placeholder := &common.SelectionPolicyConfig{EvalFunc: common.DefaultSelectionPolicySource} + require.NoError(t, upgradeDefaultPolicy(placeholder)) + assert.Equal(t, DefaultPolicySource(), placeholder.EvalFunc) + assert.Equal(t, DefaultPolicySource(), placeholder.EvalFuncOriginal) + assert.NotNil(t, placeholder.CompiledProgram) +} diff --git a/internal/policy/engine.go b/internal/policy/engine.go index 7aa2a35b0..38acfb759 100644 --- a/internal/policy/engine.go +++ b/internal/policy/engine.go @@ -378,13 +378,13 @@ func (e *Engine) getOrdered(networkID, method, finality string, laneIDs []string if slot != nil { slot.lastAccessedAtMs.Store(nowMs) if ordered := slot.cache.Load(); ordered != nil && len(*ordered) > 0 { - return *ordered + return slot.methodBlocks.Load().filter(*ordered, method) } } if wildcard != nil { wildcard.lastAccessedAtMs.Store(nowMs) if ordered := wildcard.cache.Load(); ordered != nil { - return *ordered + return wildcard.methodBlocks.Load().filter(*ordered, method) } } return nil @@ -452,26 +452,48 @@ func (e *Engine) RecentDecisions(networkID, method, finality string, limit int) // candidates." func (e *Engine) GetExcluded(networkID, method, finality string) []common.Upstream { slot, wildcard := e.lookupSlotWithFallback(networkID, method, finality, nil) - if slot != nil { - // Honor the narrow slot whenever it has TICKED (non-nil pointer), - // even when its probe-candidate set is empty: with probe-blocking - // verdicts an empty set is a deliberate outcome ("everything - // excluded here is static"), not a cold start. Falling back to the - // wildcard on len==0 would resurrect probes the narrow eval - // suppressed. Pointer-nil (never ticked) is the only cold-start - // signal that defers to the wildcard slot. - if excluded := slot.excludedCache.Load(); excluded != nil { - return *excluded + // Honor the narrow slot whenever it has TICKED (non-nil pointer), + // even when its probe-candidate set is empty: with probe-blocking + // verdicts an empty set is a deliberate outcome ("everything + // excluded here is static"), not a cold start. Falling back to the + // wildcard on len==0 would resurrect probes the narrow eval + // suppressed. Pointer-nil (never ticked) is the only cold-start + // signal that defers to the wildcard slot. + for _, s := range []*Slot{slot, wildcard} { + if s == nil { + continue } - } - if wildcard != nil { - if excluded := wildcard.excludedCache.Load(); excluded != nil { - return *excluded + if excluded := s.excludedCache.Load(); excluded != nil { + return withMethodBlocked(*excluded, s.methodBlocks.Load().probeCandidates(method)) } } return nil } +// withMethodBlocked appends the upstreams blocked for one method to the +// slot's excluded set, skipping duplicates. Returns `excluded` as-is when +// nothing is added; never mutates it. +func withMethodBlocked(excluded, blocked []common.Upstream) []common.Upstream { + if len(blocked) == 0 { + return excluded + } + out := make([]common.Upstream, 0, len(excluded)+len(blocked)) + out = append(out, excluded...) + for _, b := range blocked { + dup := false + for _, x := range excluded { + if x.Id() == b.Id() { + dup = true + break + } + } + if !dup { + out = append(out, b) + } + } + return out +} + // PublishRequest hands a freshly-served request to the network's // probe-bus (if a Prober exists for this network). Non-blocking — // drops on bus overflow with a metric. Called from the request path diff --git a/internal/policy/engine_eval_paths_test.go b/internal/policy/engine_eval_paths_test.go new file mode 100644 index 000000000..35350f35f --- /dev/null +++ b/internal/policy/engine_eval_paths_test.go @@ -0,0 +1,672 @@ +package policy + +import ( + "context" + "encoding/json" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/health" + "github.com/erpc/erpc/telemetry" + "github.com/grafana/sobek" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// modePolicy is driven tick-by-tick from Go through the `testMode()` global +// the primer installs, so a single registered network can walk through +// every eval outcome without re-registering. +const modePolicy = `(ups, ctx) => { + const m = testMode(); + const byId = {}; + ups.forEach(u => { byId[u.id] = u; }); + switch (m) { + case 'scored': + return ups.map((u, i) => { u.score = ups.length - i; return u; }); + case 'exclude-b': + globalThis.__policyLeafReasons['b'] = ['@step:excludeIf', 'error_rate_above']; + globalThis.__policyProbeVerdicts['b'] = { e: true, b: true }; + globalThis.__policyShadowReasons['a'] = ['latency_p95_above']; + return [byId.a, byId.c]; + case 'swap': + globalThis.__policyStickyHeld = true; + return [byId.b, byId.a, byId.c]; + case 'probe': + globalThis.__probeConfig = { sampleRate: 1, minSamples: 0, maxConcurrent: 2 }; + return [byId.a]; + case 'probe-half': + globalThis.__probeConfig = { sampleRate: 0.5, minSamples: 0 }; + return [byId.a]; + case 'steps': + if (globalThis.__policyStepLogEnabled) { + globalThis.__policyStepLog.push({ step: 'custom', args: { k: 1 }, inIds: ['a', 'b'], outIds: ['b', 'a'], dropped: ['c'], added: ['x'], reordered: true }); + } + return [byId.b, byId.a]; + case 'throw': + throw new Error('boom'); + case 'null': + return null; + default: + return ups; + } +}` + +type modeEngine struct { + *Engine + mode *atomic.Value +} + +func (m *modeEngine) set(mode string) { m.mode.Store(mode) } + +func newModeEngine(t *testing.T, mutate func(*common.SelectionPolicyConfig)) (*modeEngine, *common.SelectionPolicyConfig) { + t.Helper() + mode := &atomic.Value{} + mode.Store("plain") + primer := func(rt *common.Runtime) error { + return rt.VM().Set("testMode", func() string { return mode.Load().(string) }) + } + cfg := &common.SelectionPolicyConfig{EvalFunc: modePolicy, DisableTickerForTest: true} + require.NoError(t, cfg.SetDefaults()) + if mutate != nil { + mutate(cfg) + } + logger := zerolog.Nop() + tracker := health.NewTracker(&logger, "p1", time.Minute) + e := NewEngine(context.Background(), &logger, "p1", tracker, primer, nil) + t.Cleanup(e.Stop) + return &modeEngine{Engine: e, mode: mode}, cfg +} + +func stubUps(ids ...string) ([]common.Upstream, map[string]*stubProbeUpstream) { + ups := make([]common.Upstream, 0, len(ids)) + byID := make(map[string]*stubProbeUpstream, len(ids)) + for _, id := range ids { + s := &stubProbeUpstream{id: id} + ups = append(ups, s) + byID[id] = s + } + return ups, byID +} + +func latestDecision(t *testing.T, e *Engine, network, method string) *Decision { + t.Helper() + ds := e.RecentDecisions(network, method, "*", 1) + require.Len(t, ds, 1) + return ds[0] +} + +func TestEngine_TickOutcomesFeedDecisionsAndMetrics(t *testing.T) { + const label = "cov-tick" + e, cfg := newModeEngine(t, nil) + ups, _ := stubUps("a", "b", "c") + require.NoError(t, e.RegisterNetwork("evm:1", label, func() []common.Upstream { return ups }, cfg)) + + rejections := func(up, step string) float64 { + return testutil.ToFloat64(telemetry.MetricSelectionRejectionTotal.WithLabelValues("p1", label, "*", up, step)) + } + exclusions := func(up, reason string) float64 { + return testutil.ToFloat64(telemetry.MetricSelectionExclusionTotal.WithLabelValues("p1", label, "*", up, reason)) + } + shadows := func(up, reason string) float64 { + return testutil.ToFloat64(telemetry.MetricSelectionShadowExclusionTotal.WithLabelValues("p1", label, "*", up, reason)) + } + readmits := func(up string) float64 { + return testutil.ToFloat64(telemetry.MetricSelectionReadmitTotal.WithLabelValues("p1", label, "*", up)) + } + switches := func(from, to string) float64 { + return testutil.ToFloat64(telemetry.MetricSelectionPrimarySwitchTotal.WithLabelValues("p1", label, "*", from, to)) + } + holds := func(up string) float64 { + return testutil.ToFloat64(telemetry.MetricSelectionStickyHoldTotal.WithLabelValues("p1", label, "*", up)) + } + evalErrors := func(kind string) float64 { + return testutil.ToFloat64(telemetry.MetricSelectionEvalErrorsTotal.WithLabelValues("p1", label, "*", kind)) + } + + // Initial synchronous tick: identity order, first primary counts as a switch. + assert.Equal(t, []string{"a", "b", "c"}, ids(e.GetOrdered("evm:1", "*", "*"))) + assert.False(t, e.LastEvalAt("evm:1", "*", "*").IsZero()) + firstSwitch := e.LastSwitchAt("evm:1", "*", "*") + require.False(t, firstSwitch.IsZero()) + assert.Nil(t, e.GetScores("evm:1", "*", "*"), "policy without scoring publishes no scores") + + // Scored tick keeps the primary, so the switch timestamp is untouched. + e.set("scored") + TickForTest(e.Engine, "evm:1", "*") + assert.Equal(t, map[string]float64{"a": 3, "b": 2, "c": 1}, e.GetScores("evm:1", "*", "*")) + assert.Equal(t, firstSwitch, e.LastSwitchAt("evm:1", "*", "*")) + + // Exclusion with step + leaf attribution and a probe-blocking verdict. + rejBefore, exBefore, shBefore := rejections("b", "excludeIf"), exclusions("b", "error_rate_above"), shadows("a", "latency_p95_above") + e.set("exclude-b") + TickForTest(e.Engine, "evm:1", "*") + assert.Equal(t, []string{"a", "c"}, ids(e.GetOrdered("evm:1", "*", "*"))) + assert.Empty(t, e.GetExcluded("evm:1", "*", "*"), "probe-blocking exclusion must not be offered to the prober") + d := latestDecision(t, e.Engine, "evm:1", "*") + require.Len(t, d.Output.Excluded, 1) + assert.Equal(t, ExcludedUpstream{ID: "b", Reason: "error_rate_above", Step: "excludeIf", ProbeEligible: false, LeafReasons: []string{"error_rate_above"}}, d.Output.Excluded[0]) + assert.Equal(t, map[string][]string{"a": {"latency_p95_above"}}, d.Output.ShadowReasons) + assert.Equal(t, []string{"b"}, d.Diff.Removed) + assert.Equal(t, rejBefore+1, rejections("b", "excludeIf")) + assert.Equal(t, exBefore+1, exclusions("b", "error_rate_above")) + assert.Equal(t, shBefore+1, shadows("a", "latency_p95_above")) + order, excluded := LatestDecisionOutputForTest(e.Engine, "evm:1", "*") + assert.Equal(t, []string{"a", "c"}, order) + assert.Equal(t, []string{"b"}, excluded) + + // Staying excluded for a second tick does not re-admit it. + readmitBefore := readmits("b") + TickForTest(e.Engine, "evm:1", "*") + assert.Equal(t, readmitBefore, readmits("b")) + + // Back in rotation → readmit counted once. + e.set("plain") + TickForTest(e.Engine, "evm:1", "*") + assert.Equal(t, readmitBefore+1, readmits("b")) + assert.Equal(t, []string{"b"}, latestDecision(t, e.Engine, "evm:1", "*").Diff.Added) + + // Primary swap with an active sticky hold. + swBefore, holdBefore := switches("a", "b"), holds("b") + e.set("swap") + TickForTest(e.Engine, "evm:1", "*") + assert.Equal(t, swBefore+1, switches("a", "b")) + assert.Equal(t, holdBefore+1, holds("b")) + d = latestDecision(t, e.Engine, "evm:1", "*") + assert.True(t, d.Diff.PrimaryChanged) + assert.True(t, d.Diff.StickyHeld) + assert.True(t, e.LastSwitchAt("evm:1", "*", "*").After(firstSwitch)) + + // Eval failures keep the previous cache and are classified by kind. + throwBefore, invalidBefore := evalErrors("throw"), evalErrors("invalid_return") + e.set("throw") + TickForTest(e.Engine, "evm:1", "*") + assert.Contains(t, latestDecision(t, e.Engine, "evm:1", "*").Error, "boom") + assert.Equal(t, throwBefore+1, evalErrors("throw")) + e.set("null") + TickForTest(e.Engine, "evm:1", "*") + assert.Contains(t, latestDecision(t, e.Engine, "evm:1", "*").Error, ErrInvalidReturn.Error()) + assert.Equal(t, invalidBefore+1, evalErrors("invalid_return")) + assert.Equal(t, []string{"b", "a", "c"}, ids(e.GetOrdered("evm:1", "*", "*")), "failed evals retain the last good order") +} + +func ids(ups []common.Upstream) []string { + if ups == nil { + return nil + } + out := make([]string, len(ups)) + for i, u := range ups { + out[i] = u.Id() + } + return out +} + +func TestEngine_EvalExceedingTimeoutIsCounted(t *testing.T) { + const label = "cov-timeout" + e, cfg := newModeEngine(t, func(c *common.SelectionPolicyConfig) { c.EvalTimeout = common.Duration(time.Nanosecond) }) + ups, _ := stubUps("a") + require.NoError(t, e.RegisterNetwork("evm:1", label, func() []common.Upstream { return ups }, cfg)) + timeouts := func() float64 { + return testutil.ToFloat64(telemetry.MetricSelectionEvalErrorsTotal.WithLabelValues("p1", label, "*", "timeout")) + } + before := timeouts() + TickForTest(e.Engine, "evm:1", "*") + assert.Equal(t, before+1, timeouts()) + assert.Equal(t, []string{"a"}, ids(e.GetOrdered("evm:1", "*", "*")), "a slow eval's result still replaces the cache") +} + +func TestEngine_ProberLifecycleFollowsProbeConfig(t *testing.T) { + e, cfg := newModeEngine(t, nil) + ups, byID := stubUps("a", "b") + e.set("probe") + require.NoError(t, e.RegisterNetwork("evm:1", "cov-probe", func() []common.Upstream { return ups }, cfg)) + + prober := func() *Prober { + e.proberMu.RLock() + defer e.proberMu.RUnlock() + return e.probers["evm:1"] + } + p := prober() + require.NotNil(t, p, "probeExcluded config must create a prober") + assert.Equal(t, 1.0, p.Config().SampleRate) + assert.Equal(t, 2, p.Config().MaxConcurrent) + assert.Equal(t, []string{"b"}, ids(e.GetExcluded("evm:1", "*", "*"))) + + e.PublishRequest("evm:1", nil) + e.PublishRequest("evm:1", makeProbeReq(t, "eth_blockNumber")) + require.Eventually(t, func() bool { return byID["b"].calls.Load() == 1 }, 5*time.Second, 5*time.Millisecond) + assert.Equal(t, int64(0), byID["a"].calls.Load(), "in-rotation upstreams are never shadow-probed") + + e.set("probe-half") + TickForTest(e.Engine, "evm:1", "*") + assert.Same(t, p, prober(), "a changed config hot-swaps the existing prober") + assert.Equal(t, 0.5, prober().Config().SampleRate) + + e.set("plain") + TickForTest(e.Engine, "evm:1", "*") + assert.Nil(t, prober(), "dropping probeExcluded tears the prober down") + e.PublishRequest("evm:1", makeProbeReq(t, "eth_blockNumber")) + assert.Equal(t, int64(1), byID["b"].calls.Load()) +} + +func TestEngine_StepLogFollowsToggle(t *testing.T) { + e, cfg := newModeEngine(t, nil) + ups, _ := stubUps("a", "b", "c") + e.set("steps") + require.NoError(t, e.RegisterNetwork("evm:1", "cov-steps", func() []common.Upstream { return ups }, cfg)) + assert.False(t, e.IsStepLogEnabled()) + assert.Nil(t, latestDecision(t, e.Engine, "evm:1", "*").Output.StepLog) + + e.SetStepLogEnabled(true) + assert.True(t, e.IsStepLogEnabled()) + TickForTest(e.Engine, "evm:1", "*") + steps := latestDecision(t, e.Engine, "evm:1", "*").Output.StepLog + require.Len(t, steps, 1) + assert.Equal(t, "custom", steps[0].Step) + assert.JSONEq(t, `{"k":1}`, string(steps[0].Args)) + assert.Equal(t, []string{"a", "b"}, steps[0].InIDs) + assert.Equal(t, []string{"c"}, steps[0].Dropped) + assert.Equal(t, []string{"x"}, steps[0].Added) + assert.True(t, steps[0].Reordered) + + e.SetStepLogEnabled(false) + TickForTest(e.Engine, "evm:1", "*") + assert.Nil(t, latestDecision(t, e.Engine, "evm:1", "*").Output.StepLog) +} + +func TestEngine_RecentDecisionsRingIsBounded(t *testing.T) { + e, cfg := newModeEngine(t, nil) + ups, _ := stubUps("a") + require.NoError(t, e.RegisterNetwork("evm:1", "cov-ring", func() []common.Upstream { return ups }, cfg)) + extra := decisionsRingSize + 6 + for range extra { + TickForTest(e.Engine, "evm:1", "*") + } + all := e.RecentDecisions("evm:1", "*", "*", 0) + require.Len(t, all, decisionsRingSize) + last := e.RecentDecisions("evm:1", "*", "*", 3) + require.Len(t, last, 3) + // Register tick has TickCount 0, so the final tick saw TickCount == extra. + for i, d := range last { + assert.Equal(t, uint64(extra-2+i), d.State.TickCount, "oldest-first ordering") + } + assert.Equal(t, uint64(extra-decisionsRingSize+1), all[0].State.TickCount, "oldest entries are evicted") +} + +func TestEngine_PerMethodSlotsAndLookups(t *testing.T) { + const perMethod = `(ups, ctx) => ctx.method === 'eth_call' ? ups.filter(u => u.id === 'a') : ups` + cfg := &common.SelectionPolicyConfig{EvalFunc: perMethod, EvalScope: common.EvalScopeNetworkMethod, DisableTickerForTest: true} + require.NoError(t, cfg.SetDefaults()) + logger := zerolog.Nop() + e := NewEngine(context.Background(), &logger, "p1", health.NewTracker(&logger, "p1", time.Minute), nil, nil) + defer e.Stop() + ups, _ := stubUps("a", "b") + require.NoError(t, e.RegisterNetwork("evm:1", "", func() []common.Upstream { return ups }, cfg)) + + // Narrow slot is lazily created but untick'd: wildcard answers. + assert.Nil(t, e.GetExcluded("evm:1", "eth_call", "*")) + assert.Equal(t, []string{"a", "b"}, ids(e.GetOrdered("evm:1", "eth_call", "*"))) + assert.Equal(t, 2, SlotCountForTest(e)) + assert.True(t, e.LastEvalAt("evm:1", "eth_call", "*").IsZero(), "narrow slot has not evaluated yet") + + TickForTest(e, "evm:1", "eth_call") + assert.Equal(t, []string{"a"}, ids(e.GetOrdered("evm:1", "eth_call", "*"))) + assert.Equal(t, []string{"b"}, ids(e.GetExcluded("evm:1", "eth_call", "*"))) + assert.Equal(t, []string{"a", "b"}, ids(e.GetOrdered("evm:1", "eth_getLogs", "*")), "other methods fall back to wildcard until ticked") + assert.Equal(t, "eth_call", latestDecision(t, e, "evm:1", "eth_call").Method) + + // Unknown networks answer with zero values. + assert.Nil(t, e.GetOrdered("evm:404", "*", "*")) + assert.Nil(t, e.GetExcluded("evm:404", "*", "*")) + assert.True(t, e.LastEvalAt("evm:404", "*", "*").IsZero()) + assert.True(t, e.LastSwitchAt("evm:404", "*", "*").IsZero()) + assert.Nil(t, e.RecentDecisions("evm:404", "*", "*", 0)) + assert.Nil(t, e.GetScores("evm:404", "*", "*")) + assert.False(t, e.PerBoundaryEnabled("evm:404")) + assert.False(t, e.PerBoundaryEnabled("evm:1")) + + on := true + laneCfg := &common.SelectionPolicyConfig{EvalFunc: "(ups) => ups", EvalPerBoundary: &on, DisableTickerForTest: true} + require.NoError(t, laneCfg.SetDefaults()) + require.NoError(t, e.RegisterNetwork("evm:2", "", func() []common.Upstream { return ups }, laneCfg)) + assert.True(t, e.PerBoundaryEnabled("evm:2")) + + before := SlotCountForTest(e) + e.UnregisterNetwork("evm:1") + assert.Equal(t, before-3, SlotCountForTest(e), "every slot of the network is torn down") + assert.Nil(t, e.GetOrdered("evm:1", "*", "*")) + assert.Equal(t, []string{"a", "b"}, ids(e.GetOrdered("evm:2", "*", "*"))) +} + +func TestEngine_PauseGatesTicker(t *testing.T) { + cfg := &common.SelectionPolicyConfig{EvalFunc: "(ups) => ups", EvalInterval: common.Duration(5 * time.Millisecond)} + require.NoError(t, cfg.SetDefaults()) + logger := zerolog.Nop() + e := NewEngine(context.Background(), &logger, "p1", health.NewTracker(&logger, "p1", time.Minute), nil, nil) + defer e.Stop() + var calls atomic.Int64 + ups, _ := stubUps("a") + require.NoError(t, e.RegisterNetwork("evm:1", "", func() []common.Upstream { calls.Add(1); return ups }, cfg)) + require.Eventually(t, func() bool { return calls.Load() >= 3 }, 5*time.Second, time.Millisecond) + + e.SetPaused(true) + assert.True(t, e.IsPaused()) + paused := calls.Load() + // At most one tick that already passed the gate may still land. + require.Never(t, func() bool { return calls.Load() > paused+1 }, 100*time.Millisecond, 5*time.Millisecond) + + e.SetPaused(false) + assert.False(t, e.IsPaused()) + resumed := calls.Load() + require.Eventually(t, func() bool { return calls.Load() >= resumed+2 }, 5*time.Second, time.Millisecond) +} + +func TestEngine_TSFunctionSentinels(t *testing.T) { + userScript, err := sobek.Compile("user.js", ` +globalThis.__erpcFns = globalThis.__erpcFns || {}; +globalThis.__erpcFns['reverse'] = function (ups, ctx) { return ups.slice().reverse(); }; +globalThis.__erpcFns['notfn'] = 42; +`, false) + require.NoError(t, err) + logger := zerolog.Nop() + newCfg := func(id string) *common.SelectionPolicyConfig { + c := &common.SelectionPolicyConfig{EvalFunc: "__ts_fn__:" + id, DisableTickerForTest: true} + require.NoError(t, c.SetDefaults()) + require.Nil(t, c.CompiledProgram) + return c + } + ups, _ := stubUps("a", "b") + upsFn := func() []common.Upstream { return ups } + + e := NewEngine(context.Background(), &logger, "p1", health.NewTracker(&logger, "p1", time.Minute), nil, userScript) + defer e.Stop() + require.NoError(t, e.RegisterNetwork("evm:1", "", upsFn, newCfg("reverse"))) + assert.Equal(t, []string{"b", "a"}, ids(e.GetOrdered("evm:1", "*", "*"))) + + require.NoError(t, e.RegisterNetwork("evm:2", "", upsFn, newCfg("missing"))) + assert.Contains(t, latestDecision(t, e, "evm:2", "*").Error, `id "missing" not found in __erpcFns`) + assert.Nil(t, e.GetOrdered("evm:2", "*", "*")) + + require.NoError(t, e.RegisterNetwork("evm:3", "", upsFn, newCfg("notfn"))) + assert.Contains(t, latestDecision(t, e, "evm:3", "*").Error, ErrInvalidReturn.Error()) + + bare := NewEngine(context.Background(), &logger, "p1", health.NewTracker(&logger, "p1", time.Minute), nil, nil) + defer bare.Stop() + require.NoError(t, bare.RegisterNetwork("evm:1", "", upsFn, newCfg("reverse"))) + assert.Contains(t, latestDecision(t, bare, "evm:1", "*").Error, "__erpcFns registry not populated") +} + +func TestEngine_SharedStickyRegister(t *testing.T) { + const src = `(ups, ctx) => { + __setSharedSticky('network', '', 1); + const s = __getSharedSticky('network'); + if (s) { + return ups.filter(u => u.id === s.primary).concat(ups.filter(u => u.id !== s.primary)); + } + __setSharedSticky('network', 'b', 777); + return ups; +}` + cfg := &common.SelectionPolicyConfig{EvalFunc: src, DisableTickerForTest: true} + require.NoError(t, cfg.SetDefaults()) + logger := zerolog.Nop() + e := NewEngine(context.Background(), &logger, "p1", health.NewTracker(&logger, "p1", time.Minute), nil, nil) + defer e.Stop() + ups, _ := stubUps("a", "b") + require.NoError(t, e.RegisterNetwork("evm:1", "", func() []common.Upstream { return ups }, cfg)) + assert.Equal(t, []string{"a", "b"}, ids(e.GetOrdered("evm:1", "*", "*")), "cold start: nothing shared yet") + assert.Equal(t, map[stickyKey]stickyEntry{{network: "evm:1", method: "*", finality: "*"}: {Primary: "b", LastSwitchAt: 777}}, e.sticky.snapshot(), + "empty primary writes are ignored; the network-scope key collapses method/finality") + + TickForTest(e, "evm:1", "*") + assert.Equal(t, []string{"b", "a"}, ids(e.GetOrdered("evm:1", "*", "*")), "second tick reads the shared primary") + + e.SetIdleEvictionAfter(time.Millisecond) + e.sweepIdleSticky() + assert.Empty(t, e.sticky.snapshot(), "entries older than the idle window are evicted") +} + +func TestResolveStickyKey(t *testing.T) { + tests := []struct { + scope common.EvalScope + want stickyKey + }{ + {common.EvalScopeNetwork, stickyKey{"n", "*", "*"}}, + {common.EvalScopeNetworkMethod, stickyKey{"n", "m", "*"}}, + {common.EvalScopeNetworkFinality, stickyKey{"n", "*", "f"}}, + {common.EvalScopeNetworkMethodFinality, stickyKey{"n", "m", "f"}}, + {common.EvalScope("bogus"), stickyKey{"n", "m", "f"}}, + } + for _, tc := range tests { + t.Run(string(tc.scope), func(t *testing.T) { + assert.Equal(t, tc.want, resolveStickyKey(tc.scope, "n", "m", "f")) + }) + } +} + +func TestStickyStore_SnapshotAndEviction(t *testing.T) { + s := newStickyStore() + old, fresh := stickyKey{"n", "*", "*"}, stickyKey{"n", "m", "*"} + s.Set(old, "a", 100) + s.Set(fresh, "b", 1000) + snap := s.snapshot() + delete(snap, old) + _, _, ok := s.Get(old) + assert.True(t, ok, "snapshot is a copy") + + s.evictIdle(500) + _, _, ok = s.Get(old) + assert.False(t, ok) + primary, at, ok := s.Get(fresh) + assert.True(t, ok) + assert.Equal(t, "b", primary) + assert.Equal(t, int64(1000), at) +} + +func TestRunEval_ErrorPaths(t *testing.T) { + ups, _ := stubUps("a", "b") + evalCtx := EvalContext{Network: "evm:1", Method: "*", Finality: "unknown"} + compiled := func(src string) *common.SelectionPolicyConfig { + p, err := common.CompileProgram(src) + require.NoError(t, err) + return &common.SelectionPolicyConfig{EvalFunc: src, CompiledProgram: p} + } + run := func(pool *runtimePool, cfg *common.SelectionPolicyConfig) (*EvalResult, error) { + return runEval(pool, cfg, ups, nil, nil, nil, evalCtx, false, nil) + } + + _, err := run(newRuntimePool(nil, nil), &common.SelectionPolicyConfig{EvalFunc: "(u) => u"}) + assert.ErrorContains(t, err, "no compiled program") + + _, err = run(newRuntimePool(func(*common.Runtime) error { return assert.AnError }, nil), compiled("(u) => u")) + assert.ErrorContains(t, err, "acquire sobek runtime") + + badScript, err := sobek.Compile("bad.js", `throw new Error("user script broke")`, false) + require.NoError(t, err) + _, err = run(newRuntimePool(nil, badScript), compiled("(u) => u")) + assert.ErrorContains(t, err, "evaluate user TS config in policy runtime") + + _, err = run(newRuntimePool(nil, nil), compiled("42")) + assert.ErrorIs(t, err, ErrInvalidReturn) + + _, err = run(newRuntimePool(nil, nil), compiled("(u) => undefinedFn()")) + assert.ErrorContains(t, err, "call eval") + + returns := []struct { + name string + src string + wantErr string + }{ + {"null", "() => null", "returned null/undefined"}, + {"primitive", "() => 5", "eval returned"}, + {"object without length", "() => ({})", "not an array"}, + {"non-object entry", "() => [5]", "entry 0 is not an object"}, + {"entry without id", "() => [{}]", "entry 0 has no id"}, + {"unknown id", "() => [{id: 'zzz'}]", `unknown id "zzz"`}, + } + for _, tc := range returns { + t.Run(tc.name, func(t *testing.T) { + _, err := run(newRuntimePool(nil, nil), compiled(tc.src)) + require.ErrorIs(t, err, ErrInvalidReturn) + assert.ErrorContains(t, err, tc.wantErr) + }) + } + + t.Run("holes and null scores are skipped; nil sticky store is inert", func(t *testing.T) { + res, err := run(newRuntimePool(nil, nil), compiled(`(ups) => { + __setSharedSticky('network', 'a', 1); + if (__getSharedSticky('network') !== undefined) throw new Error('sticky leaked'); + return [undefined, {id: 'b', score: null}, {id: 'a', score: 2.5}]; +}`)) + require.NoError(t, err) + assert.Equal(t, []string{"b", "a"}, res.OrderedIDs) + assert.Equal(t, map[string]float64{"a": 2.5}, res.Scores) + assert.Nil(t, res.ProbeConfig) + assert.False(t, res.StickyHeld) + }) +} + +func TestRuntimePool_ReusesReleasedRuntime(t *testing.T) { + var primed atomic.Int64 + pool := newRuntimePool(func(*common.Runtime) error { primed.Add(1); return nil }, nil) + rt, err := pool.acquire() + require.NoError(t, err) + pool.release(rt) + pool.release(nil) + again, err := pool.acquire() + require.NoError(t, err) + assert.Same(t, rt, again) + assert.Equal(t, int64(1), primed.Load(), "a pooled runtime is primed once") + fresh, err := pool.acquire() + require.NoError(t, err) + assert.NotSame(t, rt, fresh) + assert.Equal(t, int64(2), primed.Load()) +} + +func vmWithGlobal(t *testing.T, name, js string) *sobek.Runtime { + t.Helper() + vm := sobek.New() + if js != "" { + _, err := vm.RunString("globalThis." + name + " = " + js + ";") + require.NoError(t, err) + } + return vm +} + +func TestReadProbeConfig(t *testing.T) { + defaults := ProbeConfig{SampleRate: 0.1, MinSamples: 10, MinSamplesWindow: 60 * time.Second, MaxConcurrent: 4, Timeout: 10 * time.Second} + tests := []struct { + name string + js string + want *ProbeConfig + }{ + {name: "absent", js: "", want: nil}, + {name: "null", js: "null", want: nil}, + {name: "non-object", js: "5", want: nil}, + {name: "empty object takes defaults", js: "{}", want: &defaults}, + { + name: "values below range clamp, string durations parse", + js: `{sampleRate: -1, minSamples: -3, maxConcurrent: 0, minSamplesWindow: "30s", timeout: "2s"}`, + want: &ProbeConfig{SampleRate: 0, MinSamples: 0, MinSamplesWindow: 30 * time.Second, MaxConcurrent: 1, Timeout: 2 * time.Second}, + }, + { + name: "values above range clamp, numeric durations are ms", + js: `{sampleRate: 5, minSamples: 3, maxConcurrent: 7, minSamplesWindow: 1500, timeout: 250}`, + want: &ProbeConfig{SampleRate: 1, MinSamples: 3, MinSamplesWindow: 1500 * time.Millisecond, MaxConcurrent: 7, Timeout: 250 * time.Millisecond}, + }, + { + name: "invalid durations keep defaults", + js: `{minSamplesWindow: "bogus", timeout: "0s"}`, + want: &defaults, + }, + { + name: "non-positive numeric durations keep defaults", + js: `{minSamplesWindow: 0, timeout: -5}`, + want: &defaults, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := readProbeConfig(vmWithGlobal(t, "__probeConfig", tc.js)) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestReadReasonsMap(t *testing.T) { + tests := []struct { + name string + js string + want map[string][]string + }{ + {"absent", "", nil}, + {"non-object", "7", nil}, + {"empty", "{}", nil}, + { + "drops empty, null and malformed entries", + `{a: ["x", "", null, "y"], b: [], c: 5, d: [null, ""], e: ["z"]}`, + map[string][]string{"a": {"x", "y"}, "e": {"z"}}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, readReasonsMap(vmWithGlobal(t, "__r", tc.js), "__r")) + }) + } +} + +func TestReadProbeVerdicts(t *testing.T) { + assert.Nil(t, readProbeVerdicts(vmWithGlobal(t, "__policyProbeVerdicts", ""))) + assert.Nil(t, readProbeVerdicts(vmWithGlobal(t, "__policyProbeVerdicts", "3"))) + assert.Nil(t, readProbeVerdicts(vmWithGlobal(t, "__policyProbeVerdicts", "{}"))) + + got := readProbeVerdicts(vmWithGlobal(t, "__policyProbeVerdicts", `{a: {e: true}, b: {e: true, b: 1}, c: 5}`)) + assert.Equal(t, map[string]ProbeVerdict{"a": {Eligible: true}, "b": {Eligible: true, Blocking: true}}, got) + assert.True(t, got["a"].ShouldProbe()) + assert.False(t, got["b"].ShouldProbe(), "a blocking step vetoes probing") + assert.True(t, ProbeVerdict{}.ShouldProbe(), "untracked exclusion defaults to probing") +} + +func TestReadStepLog(t *testing.T) { + steps, err := readStepLog(vmWithGlobal(t, "__policyStepLog", "")) + require.NoError(t, err) + assert.Nil(t, steps) + + steps, err = readStepLog(vmWithGlobal(t, "__policyStepLog", `[{step: "take", inIds: ["a","b"], outIds: ["a"], dropped: ["b"]}]`)) + require.NoError(t, err) + assert.Equal(t, []StepEntry{{Step: "take", InIDs: []string{"a", "b"}, OutIDs: []string{"a"}, Dropped: []string{"b"}}}, steps) + + _, err = readStepLog(vmWithGlobal(t, "__policyStepLog", `[{step: "x", inIds: 5}]`)) + var typeErr *json.UnmarshalTypeError + assert.ErrorAs(t, err, &typeErr) +} + +func TestTestHelpers_OverrideAllAndReset(t *testing.T) { + OverrideAllForTest(nil) // nil engine is a documented no-op + + e, cfg := newModeEngine(t, nil) + ups, _ := stubUps("c", "a", "b") + upsFn := func() []common.Upstream { return ups } + require.NoError(t, e.RegisterNetwork("evm:1", "", upsFn, cfg)) + require.NoError(t, e.RegisterNetwork("evm:2", "", upsFn, cfg)) + + OverrideAllForTest(e.Engine) + assert.Equal(t, []string{"a", "b", "c"}, ids(e.GetOrdered("evm:1", "*", "*"))) + assert.Equal(t, []string{"a", "b", "c"}, ids(e.GetOrdered("evm:2", "*", "*"))) + OverrideAllForTest(e.Engine, "b", "nope") + assert.Equal(t, []string{"b"}, ids(e.GetOrdered("evm:2", "*", "*")), "unknown ids are dropped") + + order, _ := LatestDecisionOutputForTest(e.Engine, "evm:1", "*") + require.NotEmpty(t, order) + ResetSlotStateForTest(e.Engine, "evm:1", "*") + ResetSlotStateForTest(e.Engine, "evm:404", "*") + order, excluded := LatestDecisionOutputForTest(e.Engine, "evm:1", "*") + assert.Nil(t, order) + assert.Nil(t, excluded) + assert.True(t, e.LastSwitchAt("evm:1", "*", "*").IsZero()) + o404, x404 := LatestDecisionOutputForTest(e.Engine, "evm:404", "*") + assert.Nil(t, o404) + assert.Nil(t, x404) +} diff --git a/internal/policy/errors.go b/internal/policy/errors.go index dcffb8103..0f1386029 100644 --- a/internal/policy/errors.go +++ b/internal/policy/errors.go @@ -11,10 +11,6 @@ import "errors" // no upstreams, or because no eval has succeeded yet. var ErrNoEligibleUpstream = errors.New("no eligible upstream returned by selection policy") -// ErrEvalTimeout is logged (but not returned to callers) when an eval -// exceeds `selectionPolicy.evalTimeout`. The prior cache is retained. -var ErrEvalTimeout = errors.New("selection policy eval timed out") - // ErrInvalidReturn is logged when the eval returns a non-array or an entry // that does not correspond to a known upstream id. var ErrInvalidReturn = errors.New("selection policy eval returned an invalid value") diff --git a/internal/policy/eval.go b/internal/policy/eval.go index b6653a284..ff060dc64 100644 --- a/internal/policy/eval.go +++ b/internal/policy/eval.go @@ -554,6 +554,10 @@ type EvalResult struct { // default to probing. Always captured (drives prober candidate // filtering), independent of stepLogEnabled. ProbeVerdicts map[string]ProbeVerdict + // MethodBlocks[upstreamID][method] = per-method blocks recorded by + // `blockMethods`. The upstream stays in OrderedIDs; the engine drops + // it per request for the blocked method only. Always captured. + MethodBlocks map[string]map[string]MethodBlock // StickyHeld is true when `stickyPrimary` ACTIVELY held the previous // primary this tick (i.e. challenger would have won under a no-sticky // ordering, but cooldown or hysteresis kept the incumbent). Drives @@ -697,6 +701,10 @@ func runEval( if err := vm.GlobalObject().Set("__policyShadowReasons", vm.NewObject()); err != nil { return nil, err } + // Per-upstream, per-method blocks from `blockMethods`. Reset each tick. + if err := vm.GlobalObject().Set("__policyMethodBlocks", vm.NewObject()); err != nil { + return nil, err + } // Cleared each tick to detect active stickyPrimary holds. if err := vm.GlobalObject().Set("__policyStickyHeld", false); err != nil { return nil, err @@ -748,6 +756,7 @@ func runEval( _ = vm.GlobalObject().Set("__policyLeafReasons", sobek.Undefined()) _ = vm.GlobalObject().Set("__policyShadowReasons", sobek.Undefined()) _ = vm.GlobalObject().Set("__policyProbeVerdicts", sobek.Undefined()) + _ = vm.GlobalObject().Set("__policyMethodBlocks", sobek.Undefined()) _ = vm.GlobalObject().Set("__policyStickyHeld", false) _ = vm.GlobalObject().Set("__probeConfig", sobek.Undefined()) // Stub the sticky helpers back to no-ops so the pooled runtime @@ -785,6 +794,7 @@ func runEval( out.LeafReasons = readReasonsMap(vm, "__policyLeafReasons") out.ShadowReasons = readReasonsMap(vm, "__policyShadowReasons") out.ProbeVerdicts = readProbeVerdicts(vm) + out.MethodBlocks = readMethodBlocks(vm) if heldVal := vm.GlobalObject().Get("__policyStickyHeld"); heldVal != nil && !sobek.IsUndefined(heldVal) && !sobek.IsNull(heldVal) { out.StickyHeld = heldVal.ToBoolean() } @@ -923,6 +933,54 @@ func readProbeVerdicts(vm *sobek.Runtime) map[string]ProbeVerdict { return out } +// MethodBlock is one (upstream, method) block recorded by `blockMethods`. +type MethodBlock struct { + Probe bool `json:"probe"` + Reasons []string `json:"reasons,omitempty"` +} + +// readMethodBlocks drains `__policyMethodBlocks` +// ({ [upstreamId]: { [method]: { probe, reasons } } }). Malformed entries +// are skipped, like the other attribution maps. +func readMethodBlocks(vm *sobek.Runtime) map[string]map[string]MethodBlock { + v := vm.GlobalObject().Get("__policyMethodBlocks") + if v == nil || sobek.IsUndefined(v) || sobek.IsNull(v) { + return nil + } + raw, ok := v.Export().(map[string]interface{}) + if !ok || len(raw) == 0 { + return nil + } + out := make(map[string]map[string]MethodBlock, len(raw)) + for id, perMethodRaw := range raw { + perMethod, ok := perMethodRaw.(map[string]interface{}) + if !ok { + continue + } + blocks := make(map[string]MethodBlock, len(perMethod)) + for method, entryRaw := range perMethod { + entry, ok := entryRaw.(map[string]interface{}) + if !ok { + continue + } + b := MethodBlock{} + b.Probe, _ = entry["probe"].(bool) + if reasons, ok := entry["reasons"].([]interface{}); ok { + for _, r := range reasons { + if s, ok := r.(string); ok && s != "" { + b.Reasons = append(b.Reasons, s) + } + } + } + blocks[method] = b + } + if len(blocks) > 0 { + out[id] = blocks + } + } + return out +} + func readReasonsMap(vm *sobek.Runtime, globalName string) map[string][]string { v := vm.GlobalObject().Get(globalName) if v == nil || sobek.IsUndefined(v) || sobek.IsNull(v) { diff --git a/internal/policy/method_blocks.go b/internal/policy/method_blocks.go new file mode 100644 index 000000000..cd3cd5057 --- /dev/null +++ b/internal/policy/method_blocks.go @@ -0,0 +1,84 @@ +package policy + +import ( + "github.com/erpc/erpc/common" +) + +// methodBlockIndex is one tick's `blockMethods` result, indexed for the +// request path: method -> blocked upstream IDs, plus the blocked upstreams +// the prober may mirror that method to. +type methodBlockIndex struct { + blocked map[string]map[string]struct{} + probe map[string][]common.Upstream +} + +func newMethodBlockIndex(ups []common.Upstream, blocks map[string]map[string]MethodBlock) *methodBlockIndex { + if len(blocks) == 0 { + return nil + } + byID := make(map[string]common.Upstream, len(ups)) + for _, u := range ups { + byID[u.Id()] = u + } + idx := &methodBlockIndex{ + blocked: make(map[string]map[string]struct{}), + probe: make(map[string][]common.Upstream), + } + for id, perMethod := range blocks { + u, ok := byID[id] + if !ok { + continue + } + for method, b := range perMethod { + set := idx.blocked[method] + if set == nil { + set = make(map[string]struct{}) + idx.blocked[method] = set + } + set[id] = struct{}{} + if b.Probe { + idx.probe[method] = append(idx.probe[method], u) + } + } + } + return idx +} + +// filter drops the upstreams blocked for `method`. The input is never +// mutated and is returned as-is when nothing is blocked. When the blocks +// would leave no upstream, the input is returned unchanged: a method keeps +// being served by every upstream rather than by none. +func (idx *methodBlockIndex) filter(ups []common.Upstream, method string) []common.Upstream { + if idx == nil { + return ups + } + set := idx.blocked[method] + if len(set) == 0 { + return ups + } + kept := 0 + for _, u := range ups { + if _, ok := set[u.Id()]; !ok { + kept++ + } + } + if kept == len(ups) || kept == 0 { + return ups + } + out := make([]common.Upstream, 0, kept) + for _, u := range ups { + if _, ok := set[u.Id()]; !ok { + out = append(out, u) + } + } + return out +} + +// probeCandidates returns the upstreams blocked for `method` that the +// prober may mirror it to. +func (idx *methodBlockIndex) probeCandidates(method string) []common.Upstream { + if idx == nil { + return nil + } + return idx.probe[method] +} diff --git a/internal/policy/prober_paths_test.go b/internal/policy/prober_paths_test.go new file mode 100644 index 000000000..56b4cf491 --- /dev/null +++ b/internal/policy/prober_paths_test.go @@ -0,0 +1,127 @@ +package policy + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClassifyProbeErr(t *testing.T) { + cause := errors.New("cause") + tests := []struct { + err error + want string + }{ + {nil, "ok"}, + {common.NewErrUpstreamRequestSkipped(cause, "u"), "skipped"}, + {common.NewErrEndpointRequestTimeout(time.Second, cause), "timeout"}, + {common.NewErrEndpointUnauthorized(cause), "auth"}, + {common.NewErrEndpointCapacityExceeded(cause), "throttled"}, + {cause, "error"}, + } + for _, tc := range tests { + t.Run(tc.want, func(t *testing.T) { + assert.Equal(t, tc.want, classifyProbeErr(tc.err)) + }) + } +} + +// failingProbeUpstream returns a fixed error from every probe. +type failingProbeUpstream struct { + stubProbeUpstream + err error +} + +func (f *failingProbeUpstream) Forward(ctx context.Context, nq *common.NormalizedRequest, byPass, isHedge bool) (*common.NormalizedResponse, error) { + f.calls.Add(1) + return nil, f.err +} + +func TestProber_FailedProbeIsRecordedAsError(t *testing.T) { + const network = "evm:1" + deps := &stubEngine{} + bad := &failingProbeUpstream{stubProbeUpstream: stubProbeUpstream{id: "cov-bad"}, err: common.NewErrEndpointCapacityExceeded(errors.New("429"))} + deps.setExcluded(network, []common.Upstream{bad}) + p, _ := newTestProber(t, deps, &ProbeConfig{SampleRate: 1, MaxConcurrent: 1}) + defer p.Stop() + + errs := func() float64 { + return testutil.ToFloat64(telemetry.MetricSelectionProbeErrors.WithLabelValues(network, "cov-bad", "eth_chainId", "throttled")) + } + before := errs() + p.Publish(makeProbeReq(t, "eth_chainId")) + require.Eventually(t, func() bool { return errs() == before+1 }, 5*time.Second, 5*time.Millisecond) +} + +func TestProber_SkipGates(t *testing.T) { + const network = "evm:1" + skipped := func(reason string) float64 { + return testutil.ToFloat64(telemetry.MetricSelectionProbeSkipped.WithLabelValues(network, reason)) + } + + t.Run("request without method", func(t *testing.T) { + deps := &stubEngine{} + up := &stubProbeUpstream{id: "cov-nomethod"} + deps.setExcluded(network, []common.Upstream{up}) + p, _ := newTestProber(t, deps, &ProbeConfig{SampleRate: 1}) + defer p.Stop() + before := skipped("no_method") + p.Publish(common.NewNormalizedRequest([]byte(`not json`))) + require.Eventually(t, func() bool { return skipped("no_method") == before+1 }, 5*time.Second, 5*time.Millisecond) + assert.Equal(t, int64(0), up.calls.Load()) + }) + + t.Run("zero sample rate drops everything above the floor", func(t *testing.T) { + deps := &stubEngine{} + up := &stubProbeUpstream{id: "cov-sampled"} + deps.setExcluded(network, []common.Upstream{up}) + p, _ := newTestProber(t, deps, &ProbeConfig{SampleRate: 0, MinSamples: 0}) + defer p.Stop() + before := skipped("sampled_out") + p.Publish(makeProbeReq(t, "eth_blockNumber")) + require.Eventually(t, func() bool { return skipped("sampled_out") == before+1 }, 5*time.Second, 5*time.Millisecond) + assert.Equal(t, int64(0), up.calls.Load()) + }) + + t.Run("nil config and empty excluded set are no-ops", func(t *testing.T) { + deps := &stubEngine{} + up := &stubProbeUpstream{id: "cov-noop"} + p, _ := newTestProber(t, deps, nil) + defer p.Stop() + assert.Nil(t, p.Config()) + p.onRequest(context.Background(), makeProbeReq(t, "eth_blockNumber")) + p.UpdateConfig(&ProbeConfig{SampleRate: 1}) + p.onRequest(context.Background(), makeProbeReq(t, "eth_blockNumber")) + assert.False(t, p.tryReserveProbe(nil, p.Config())) + assert.Equal(t, int64(0), up.calls.Load()) + }) +} + +func TestProber_StoppedProberIgnoresPublishAndStopIsIdempotent(t *testing.T) { + deps := &stubEngine{} + up := &stubProbeUpstream{id: "cov-stopped"} + deps.setExcluded("evm:1", []common.Upstream{up}) + p, _ := newTestProber(t, deps, &ProbeConfig{SampleRate: 1}) + p.Stop() + p.Stop() + p.Publish(makeProbeReq(t, "eth_blockNumber")) + assert.Empty(t, p.feed) + assert.Equal(t, int64(0), up.calls.Load()) +} + +func TestProber_WindowCounterDefaultsAndCounts(t *testing.T) { + p, _ := newTestProber(t, &stubEngine{}, &ProbeConfig{}) + defer p.Stop() + assert.Equal(t, int64(0), p.windowCount("u", 0)) + p.recordWindowSample("u", 0) + p.recordWindowSample("u", time.Minute) + assert.Equal(t, int64(2), p.windowCount("u", 0), "non-positive window defaults to one minute") + assert.Equal(t, int64(0), p.windowCount("other", time.Minute)) +} diff --git a/internal/policy/slot.go b/internal/policy/slot.go index 43ae6d6e9..97b82dbd9 100644 --- a/internal/policy/slot.go +++ b/internal/policy/slot.go @@ -46,6 +46,11 @@ type Slot struct { // upstreams are shadow-probe candidates for incoming requests. excludedCache atomic.Pointer[[]common.Upstream] + // methodBlocks holds the most recent tick's `blockMethods` result, + // indexed by method for the request path. Same atomic contract as + // `cache`; nil when the tick recorded no blocks. + methodBlocks atomic.Pointer[methodBlockIndex] + // lastAccessedAtMs is unix-millis of the last tick OR GetOrdered // touching this slot. Drives the engine's idle-slot eviction — // per-(method, finality) slots that haven't seen traffic in a @@ -211,33 +216,28 @@ func (s *Slot) tickOnce() { } s.mu.Unlock() - // 3. Run the eval (with optional timeout). Scores are the per-upstream + // 3. Run the eval. Scores are the per-upstream // `score` values the JS attached during `sortByScore(...)` — we cache // them on the slot so diagnostic tooling (admin endpoint, simulator) // can read the engine's authoritative ranking without re-implementing // the PREFER_FASTEST formula in Go. The chronological step log is // captured only when the engine's debug flag is on (simulator always, // eRPC when the logger is at DEBUG) — see `Engine.SetStepLogEnabled`. - var ( - evalRes *EvalResult - evalErr error - ) + // + // sobek cannot interrupt a running eval, so evalTimeout cannot cut one + // short: an eval past it is reported, and its result still replaces the + // older cache. stepLogEnabled := s.engine.stepLogEnabled.Load() - done := make(chan struct{}) - go func() { - defer close(done) - evalRes, evalErr = runEval(s.engine.pool, s.cfg, ups, metrics, metricsAcrossMethods, metricsByMethod, evalCtx, stepLogEnabled, s.engine.sticky) - }() - - if timeout > 0 { - select { - case <-done: - case <-time.After(timeout): - evalErr = fmt.Errorf("%w after %s", ErrEvalTimeout, timeout) - <-done // let the goroutine finish — sobek doesn't support interrupt mid-call cleanly - } - } else { - <-done + evalStart := time.Now() + evalRes, evalErr := runEval(s.engine.pool, s.cfg, ups, metrics, metricsAcrossMethods, metricsByMethod, evalCtx, stepLogEnabled, s.engine.sticky) + if took := time.Since(evalStart); timeout > 0 && took > timeout { + telemetry.MetricSelectionEvalErrorsTotal.WithLabelValues(s.engine.projectID, s.networkLabel, s.method, "timeout").Inc() + s.engine.logger.Warn(). + Str("network", s.networkID). + Str("method", s.method). + Dur("took", took). + Dur("evalTimeout", timeout). + Msg("selection policy eval exceeded evalTimeout") } decision := &Decision{ @@ -305,6 +305,7 @@ func (s *Slot) tickOnce() { Scores: scores, StepLog: evalRes.StepLog, ShadowReasons: evalRes.ShadowReasons, + MethodBlocks: evalRes.MethodBlocks, } // 5. Compute diff against previous tick. @@ -318,6 +319,7 @@ func (s *Slot) tickOnce() { s.cache.Store(&ordered2) excludedUps := materializeExcluded(ups, excluded) s.excludedCache.Store(&excludedUps) + s.methodBlocks.Store(newMethodBlockIndex(ups, evalRes.MethodBlocks)) // 6b. Reconcile this network's probe subsystem against the // eval-emitted `__probeConfig`. Non-nil → create/update Prober; @@ -468,9 +470,7 @@ func (s *Slot) emitMetrics(d *Decision, prevState DecisionState) { if d.Error != "" { kind := "throw" - if strings.Contains(d.Error, "timed out") { - kind = "timeout" - } else if strings.Contains(d.Error, ErrInvalidReturn.Error()) { + if strings.Contains(d.Error, ErrInvalidReturn.Error()) { kind = "invalid_return" } telemetry.MetricSelectionEvalErrorsTotal.WithLabelValues(project, s.networkLabel, s.method, kind).Inc() @@ -545,6 +545,20 @@ func (s *Slot) emitMetrics(d *Decision, prevState DecisionState) { } } + // Per-method blocks: one increment per (upstream, blocked method, leaf + // slug) per tick. `method` is the blocked method, not the slot's. + for id, perMethod := range d.Output.MethodBlocks { + for method, b := range perMethod { + reasons := b.Reasons + if len(reasons) == 0 { + reasons = []string{"custom"} + } + for _, slug := range reasons { + telemetry.MetricSelectionMethodBlockTotal.WithLabelValues(project, s.networkLabel, method, id, slug).Inc() + } + } + } + // Readmit signal: any ID in this tick's Order that was excluded last // tick — i.e. transitioned from -1 → in-rotation. Increment per-upstream // counter and observe the per-network histogram of "how long was it out". diff --git a/internal/policy/stdlib/stdlib.js b/internal/policy/stdlib/stdlib.js index 6fd12db78..e9b58d69d 100644 --- a/internal/policy/stdlib/stdlib.js +++ b/internal/policy/stdlib/stdlib.js @@ -582,6 +582,55 @@ return out; }); + // ─── 4.3c blockMethods — per-method exclusion ─────────────────────────── + // + // Blocks an upstream for single methods while it stays in the pool for + // every other method. For each upstream in the chain and each method + // it has samples for that matches `match`, `when` runs against a view + // of the upstream whose `u.metrics` is that method's numbers: only + // requestsTotal and latencyP, so samplesAbove / latencyAbove work and + // rate predicates never trip. A trip records the block in + // `__policyMethodBlocks`; the engine applies it per request after the + // eval. The array passes through unchanged. + // + // .blockMethods({ match: ['debug_trace*', 'trace_*'], + // when: all(samplesAbove(20), latencyAbove(5000, 99)) }) + // + // A method stays served by all its upstreams when blocks would leave it + // none. Blocked upstreams are probe candidates for that method unless + // `probe: false`. + const METHOD_METRICS_PROTO = { + latencyP(q) { + if (q > 1) q = q / 100; + if (q <= 0.5) return this.p50ms; + if (q <= 0.7) return this.p70ms; + if (q <= 0.9) return this.p90ms; + if (q <= 0.95) return this.p95ms; + return this.p99ms; + }, + }; + define('blockMethods', function (opts) { + const log = globalThis.__policyMethodBlocks; + if (!log || !opts || typeof opts.when !== 'function') return this.slice(); + const when = opts.when; + const probe = opts.probe !== false; + for (const u of this) { + const byMethod = u.metricsByMethod; + if (!byMethod) continue; + for (const m in byMethod) { + if (!matchAny(opts.match, m)) continue; + const view = Object.create(u); + view.metrics = Object.assign(Object.create(METHOD_METRICS_PROTO), byMethod[m]); + if (!when(view)) continue; + let reasons; + if (typeof when.policyLeaves === 'function') reasons = when.policyLeaves(view); + else reasons = [when.policySlug || 'custom']; + (log[u.id] || (log[u.id] = {}))[m] = { probe: probe, reasons: reasons }; + } + } + return this.slice(); + }); + // ─── 4.4 Generic functional (extends what JS already gives us) ────────── define('reject', function (fn) { return this.filter((u, i, a) => !fn(u, i, a)); }); diff --git a/monitoring/grafana/dashboards/erpc.json b/monitoring/grafana/dashboards/erpc.json index c083c4183..7bc820ae4 100644 --- a/monitoring/grafana/dashboards/erpc.json +++ b/monitoring/grafana/dashboards/erpc.json @@ -1731,7 +1731,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(erpc_network_request_duration_seconds_bucket{\n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n}[$rate_window])) by (le, network, category, finality))", + "expr": "histogram_quantile(0.99, sum(rate(erpc_network_request_duration_seconds_bucket{le!=\"+Inf\", \n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n}[$rate_window])) by (le, network, category, finality))", "hide": false, "legendFormat": "{{network}}, {{category}}, {{finality}}", "range": true, @@ -1833,7 +1833,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(erpc_network_request_duration_seconds_bucket{\n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n}[$rate_window])) by (le, network, category, finality))", + "expr": "histogram_quantile(0.9, sum(rate(erpc_network_request_duration_seconds_bucket{le!=\"+Inf\", \n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n}[$rate_window])) by (le, network, category, finality))", "legendFormat": "{{network}}, {{category}}, {{finality}}", "range": true, "refId": "A" @@ -1938,7 +1938,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(erpc_network_request_duration_seconds_bucket{\n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n}[$rate_window])) by (le, network, category, finality))", + "expr": "histogram_quantile(0.5, sum(rate(erpc_network_request_duration_seconds_bucket{le!=\"+Inf\", \n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n}[$rate_window])) by (le, network, category, finality))", "hide": false, "legendFormat": "{{network}}, {{category}}, {{finality}}", "range": true, @@ -2040,7 +2040,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(erpc_network_request_duration_seconds_bucket{\n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n}[$rate_window])) by (le, network, category, finality, vendor))", + "expr": "histogram_quantile(0.99, sum(rate(erpc_network_request_duration_seconds_bucket{le!=\"+Inf\", \n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n}[$rate_window])) by (le, network, category, finality, vendor))", "hide": false, "legendFormat": "{{network}}, {{category}}, {{finality}}, {{vendor}}", "range": true, @@ -2142,7 +2142,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(erpc_network_request_duration_seconds_bucket{\n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n}[$rate_window])) by (le, network, category, finality, vendor))", + "expr": "histogram_quantile(0.9, sum(rate(erpc_network_request_duration_seconds_bucket{le!=\"+Inf\", \n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n}[$rate_window])) by (le, network, category, finality, vendor))", "legendFormat": "{{network}}, {{category}}, {{vendor}}, {{finality}}", "range": true, "refId": "A" @@ -2243,7 +2243,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(erpc_network_request_duration_seconds_bucket{\n namespace=~\"${namespace:regex}\",\n user=~\"${user:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\"\n}[$rate_window])) by (le, network, category, finality, vendor))", + "expr": "histogram_quantile(0.5, sum(rate(erpc_network_request_duration_seconds_bucket{le!=\"+Inf\", \n namespace=~\"${namespace:regex}\",\n user=~\"${user:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\"\n}[$rate_window])) by (le, network, category, finality, vendor))", "hide": false, "legendFormat": "{{network}}, {{category}}, {{vendor}}, {{finality}}", "range": true, @@ -4318,7 +4318,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "topk(8, histogram_quantile(0.95, sum by (le, network) (rate(erpc_network_request_duration_seconds_bucket{namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\", user=~\"${user:regex}\", upstream=~\"${upstream:regex}\", category=~\"eth_getBlockByNumber|eth_getBlockByHash|eth_blockNumber|eth_getBlockReceipts\", finality=~\"realtime|unfinalized\", network=~\"^${network:regex}$\", network!~\"${ignore_network:regex}\", project=~\"^${project:regex}$\", cluster=~\"^${cluster:regex}$\", category=~\"${category:regex}\", finality=~\"${finality:regex}\"}[$rate_window]))) and on(network) (sum by (network)(rate(erpc_network_request_duration_seconds_count{namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", category=~\"eth_getBlockByNumber|eth_getBlockByHash|eth_blockNumber|eth_getBlockReceipts\", finality=~\"realtime|unfinalized\", network=~\"^${network:regex}$\", network!~\"${ignore_network:regex}\", project=~\"^${project:regex}$\", cluster=~\"^${cluster:regex}$\", vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\", upstream=~\"${upstream:regex}\", user=~\"${user:regex}\", category=~\"${category:regex}\", finality=~\"${finality:regex}\"}[$rate_window])) > 0.2))", + "expr": "topk(8, histogram_quantile(0.95, sum by (le, network) (rate(erpc_network_request_duration_seconds_bucket{le!=\"+Inf\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\", user=~\"${user:regex}\", upstream=~\"${upstream:regex}\", category=~\"eth_getBlockByNumber|eth_getBlockByHash|eth_blockNumber|eth_getBlockReceipts\", finality=~\"realtime|unfinalized\", network=~\"^${network:regex}$\", network!~\"${ignore_network:regex}\", project=~\"^${project:regex}$\", cluster=~\"^${cluster:regex}$\", category=~\"${category:regex}\", finality=~\"${finality:regex}\"}[$rate_window]))) and on(network) (sum by (network)(rate(erpc_network_request_duration_seconds_count{namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", category=~\"eth_getBlockByNumber|eth_getBlockByHash|eth_blockNumber|eth_getBlockReceipts\", finality=~\"realtime|unfinalized\", network=~\"^${network:regex}$\", network!~\"${ignore_network:regex}\", project=~\"^${project:regex}$\", cluster=~\"^${cluster:regex}$\", vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\", upstream=~\"${upstream:regex}\", user=~\"${user:regex}\", category=~\"${category:regex}\", finality=~\"${finality:regex}\"}[$rate_window])) > 0.2))", "format": "time_series", "instant": true, "legendFormat": "{{network}}", @@ -5251,7 +5251,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.95, sum by (le, network, reason) (rate(erpc_network_data_unavailable_wait_seconds_bucket{namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"^${network:regex}$\", network!~\"${ignore_network:regex}\", project=~\"^${project:regex}$\", cluster=~\"^${cluster:regex}$\", category=~\"${category:regex}\", finality=~\"${finality:regex}\"}[$rate_window])))", + "expr": "histogram_quantile(0.95, sum by (le, network, reason) (rate(erpc_network_data_unavailable_wait_seconds_bucket{le!=\"+Inf\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"^${network:regex}$\", network!~\"${ignore_network:regex}\", project=~\"^${project:regex}$\", cluster=~\"^${cluster:regex}$\", category=~\"${category:regex}\", finality=~\"${finality:regex}\"}[$rate_window])))", "format": "time_series", "instant": false, "legendFormat": "{{network}} / {{reason}}", @@ -8066,7 +8066,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by (le, network, category) (rate(erpc_integrity_overhead_seconds_bucket{namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", cluster=~\"${cluster:regex}\", project=~\"${project:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\"}[$rate_window])))", + "expr": "histogram_quantile(0.5, sum by (le, network, category) (rate(erpc_integrity_overhead_seconds_bucket{le!=\"+Inf\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", cluster=~\"${cluster:regex}\", project=~\"${project:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\"}[$rate_window])))", "legendFormat": "p50 {{network}}, {{category}}", "range": true, "instant": false, @@ -8078,7 +8078,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by (le, network, category) (rate(erpc_integrity_overhead_seconds_bucket{namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", cluster=~\"${cluster:regex}\", project=~\"${project:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\"}[$rate_window])))", + "expr": "histogram_quantile(0.9, sum by (le, network, category) (rate(erpc_integrity_overhead_seconds_bucket{le!=\"+Inf\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", cluster=~\"${cluster:regex}\", project=~\"${project:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\"}[$rate_window])))", "legendFormat": "p90 {{network}}, {{category}}", "range": true, "instant": false, @@ -8090,7 +8090,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, network, category) (rate(erpc_integrity_overhead_seconds_bucket{namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", cluster=~\"${cluster:regex}\", project=~\"${project:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\"}[$rate_window])))", + "expr": "histogram_quantile(0.99, sum by (le, network, category) (rate(erpc_integrity_overhead_seconds_bucket{le!=\"+Inf\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", cluster=~\"${cluster:regex}\", project=~\"${project:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\"}[$rate_window])))", "legendFormat": "p99 {{network}}, {{category}}", "range": true, "instant": false, @@ -9133,7 +9133,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(erpc_upstream_request_duration_seconds_bucket{\n namespace=~\"${namespace:regex}\",\n finality=~\"${finality:regex}\",\n upstream=~\"${upstream:regex}\",\n region=~\"${region:regex}\",\n cluster=~\"${cluster:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n user=~\"${user:regex}\",\n category=~\"${category:regex}\",\n project=~\"${project:regex}\"\n}[$rate_window])) by (le, project, vendor, category))", + "expr": "histogram_quantile(0.99, sum(rate(erpc_upstream_request_duration_seconds_bucket{le!=\"+Inf\", \n namespace=~\"${namespace:regex}\",\n finality=~\"${finality:regex}\",\n upstream=~\"${upstream:regex}\",\n region=~\"${region:regex}\",\n cluster=~\"${cluster:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n user=~\"${user:regex}\",\n category=~\"${category:regex}\",\n project=~\"${project:regex}\"\n}[$rate_window])) by (le, project, vendor, category))", "legendFormat": "{{project}}, {{vendor}}, {{category}}", "range": true, "refId": "C" @@ -9240,7 +9240,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(erpc_upstream_request_duration_seconds_bucket{\n cluster=~\"${cluster:regex}\",\n namespace=~\"${namespace:regex}\",\n region=~\"${region:regex}\",\n finality=~\"${finality:regex}\",\n upstream=~\"${upstream:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n user=~\"${user:regex}\",\n category=~\"${category:regex}\",\n project=~\"${project:regex}\"\n}[$rate_window])) by (le, project, vendor, category))", + "expr": "histogram_quantile(0.9, sum(rate(erpc_upstream_request_duration_seconds_bucket{le!=\"+Inf\", \n cluster=~\"${cluster:regex}\",\n namespace=~\"${namespace:regex}\",\n region=~\"${region:regex}\",\n finality=~\"${finality:regex}\",\n upstream=~\"${upstream:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n user=~\"${user:regex}\",\n category=~\"${category:regex}\",\n project=~\"${project:regex}\"\n}[$rate_window])) by (le, project, vendor, category))", "legendFormat": "{{project}}, {{vendor}}, {{category}}", "range": true, "refId": "B" @@ -9347,7 +9347,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(erpc_upstream_request_duration_seconds_bucket{\n cluster=~\"${cluster:regex}\",\n namespace=~\"${namespace:regex}\",\n region=~\"${region:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\",\n upstream=~\"${upstream:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n category=~\"${category:regex}\",\n project=~\"${project:regex}\"\n}[$rate_window])) by (le, project, vendor, category))", + "expr": "histogram_quantile(0.50, sum(rate(erpc_upstream_request_duration_seconds_bucket{le!=\"+Inf\", \n cluster=~\"${cluster:regex}\",\n namespace=~\"${namespace:regex}\",\n region=~\"${region:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\",\n upstream=~\"${upstream:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n category=~\"${category:regex}\",\n project=~\"${project:regex}\"\n}[$rate_window])) by (le, project, vendor, category))", "legendFormat": "{{project}}, {{vendor}}, {{category}}", "range": true, "refId": "A" @@ -9462,7 +9462,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(erpc_upstream_request_duration_seconds_bucket{\n cluster=~\"${cluster:regex}\",\n namespace=~\"${namespace:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n user=~\"${user:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\"\n}[$rate_window])) by (le, project, upstream, category, finality))", + "expr": "histogram_quantile(0.99, sum(rate(erpc_upstream_request_duration_seconds_bucket{le!=\"+Inf\", \n cluster=~\"${cluster:regex}\",\n namespace=~\"${namespace:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n user=~\"${user:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\"\n}[$rate_window])) by (le, project, upstream, category, finality))", "legendFormat": "{{project}}, {{upstream}}, {{category}}, {{finality}}", "range": true, "refId": "B" @@ -9568,7 +9568,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(erpc_upstream_request_duration_seconds_bucket{\n cluster=~\"${cluster:regex}\",\n namespace=~\"${namespace:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n user=~\"${user:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\"\n}[$rate_window])) by (le, project, upstream, category, finality))", + "expr": "histogram_quantile(0.9, sum(rate(erpc_upstream_request_duration_seconds_bucket{le!=\"+Inf\", \n cluster=~\"${cluster:regex}\",\n namespace=~\"${namespace:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n user=~\"${user:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\"\n}[$rate_window])) by (le, project, upstream, category, finality))", "legendFormat": "{{project}}, {{upstream}}, {{category}}, {{finality}}", "range": true, "refId": "B" @@ -9675,7 +9675,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(erpc_upstream_request_duration_seconds_bucket{\n cluster=~\"${cluster:regex}\",\n namespace=~\"${namespace:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n user=~\"${user:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\"\n}[$rate_window])) by (le, project, upstream, category, finality))", + "expr": "histogram_quantile(0.50, sum(rate(erpc_upstream_request_duration_seconds_bucket{le!=\"+Inf\", \n cluster=~\"${cluster:regex}\",\n namespace=~\"${namespace:regex}\",\n region=~\"${region:regex}\",\n network=~\"${network:regex}\", network!~\"${ignore_network:regex}\",\n upstream=~\"${upstream:regex}\",\n category=~\"${category:regex}\",\n vendor=~\"${vendor:regex}\", vendor!~\"${ignore_vendor:regex}\",\n user=~\"${user:regex}\",\n project=~\"${project:regex}\",\n finality=~\"${finality:regex}\"\n}[$rate_window])) by (le, project, upstream, category, finality))", "legendFormat": "{{project}}, {{upstream}}, {{category}}, {{finality}}", "range": true, "refId": "A" @@ -10717,7 +10717,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (le) (rate(erpc_selection_eval_duration_seconds_bucket{\n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n project=~\"${project:regex}\",\n network=~\"${network:regex}\",\n network!~\"${ignore_network:regex}\"\n }[$rate_window])))", + "expr": "histogram_quantile(0.50, sum by (le) (rate(erpc_selection_eval_duration_seconds_bucket{le!=\"+Inf\", \n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n project=~\"${project:regex}\",\n network=~\"${network:regex}\",\n network!~\"${ignore_network:regex}\"\n }[$rate_window])))", "format": "time_series", "instant": false, "legendFormat": "p50", @@ -10730,7 +10730,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le) (rate(erpc_selection_eval_duration_seconds_bucket{\n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n project=~\"${project:regex}\",\n network=~\"${network:regex}\",\n network!~\"${ignore_network:regex}\"\n }[$rate_window])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(erpc_selection_eval_duration_seconds_bucket{le!=\"+Inf\", \n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n project=~\"${project:regex}\",\n network=~\"${network:regex}\",\n network!~\"${ignore_network:regex}\"\n }[$rate_window])))", "format": "time_series", "instant": false, "legendFormat": "p95", @@ -10743,7 +10743,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le) (rate(erpc_selection_eval_duration_seconds_bucket{\n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n project=~\"${project:regex}\",\n network=~\"${network:regex}\",\n network!~\"${ignore_network:regex}\"\n }[$rate_window])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(erpc_selection_eval_duration_seconds_bucket{le!=\"+Inf\", \n namespace=~\"${namespace:regex}\",\n cluster=~\"${cluster:regex}\",\n region=~\"${region:regex}\",\n project=~\"${project:regex}\",\n network=~\"${network:regex}\",\n network!~\"${ignore_network:regex}\"\n }[$rate_window])))", "format": "time_series", "instant": false, "legendFormat": "p99", @@ -13805,7 +13805,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_set_success_duration_seconds_bucket{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector))", + "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_set_success_duration_seconds_bucket{le!=\"+Inf\", cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector))", "hide": false, "legendFormat": "p50 - {{project}}, {{network}}, {{category}}, {{connector}}", "range": true, @@ -13817,7 +13817,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_set_success_duration_seconds_bucket{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector, policy, ttl))", + "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_set_success_duration_seconds_bucket{le!=\"+Inf\", cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector, policy, ttl))", "hide": true, "legendFormat": "p95 - {{project}}, {{network}}, {{category}}, {{connector}}, {{policy}}, TTL={{ttl}}", "range": true, @@ -14020,7 +14020,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_set_error_duration_seconds_bucket{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector, policy, ttl))", + "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_set_error_duration_seconds_bucket{le!=\"+Inf\", cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector, policy, ttl))", "legendFormat": "p50 - {{project}}, {{network}}, {{category}}, {{connector}}, {{policy}}, TTL={{ttl}}", "range": true, "refId": "A" @@ -14031,7 +14031,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_set_error_duration_seconds_bucket{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector, policy, ttl))", + "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_set_error_duration_seconds_bucket{le!=\"+Inf\", cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector, policy, ttl))", "hide": false, "legendFormat": "p95 - {{project}}, {{network}}, {{category}}, {{connector}}, {{policy}}, TTL={{ttl}}", "range": true, @@ -14241,7 +14241,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_get_success_hit_duration_seconds_bucket{namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\", cluster=~\"${cluster:regex}\"}[$rate_window])) by (le, project, network, category, connector))", + "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_get_success_hit_duration_seconds_bucket{le!=\"+Inf\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\", cluster=~\"${cluster:regex}\"}[$rate_window])) by (le, project, network, category, connector))", "legendFormat": "p50 - {{project}}, {{network}}, {{category}}, {{connector}}", "range": true, "refId": "A" @@ -14252,7 +14252,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(erpc_cache_get_success_hit_duration_seconds_bucket{namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\", cluster=~\"${cluster:regex}\"}[$rate_window])) by (le, project, network, category, connector))", + "expr": "histogram_quantile(0.99, sum(rate(erpc_cache_get_success_hit_duration_seconds_bucket{le!=\"+Inf\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\", cluster=~\"${cluster:regex}\"}[$rate_window])) by (le, project, network, category, connector))", "hide": false, "legendFormat": "p99 - {{project}}, {{network}}, {{category}}, {{connector}}", "range": true, @@ -14458,7 +14458,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_get_success_miss_duration_seconds_bucket{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, region, project, network, category, connector))", + "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_get_success_miss_duration_seconds_bucket{le!=\"+Inf\", cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, region, project, network, category, connector))", "hide": true, "legendFormat": "p50 - {{region}}, {{project}}, {{network}}, {{category}}, {{connector}}", "range": true, @@ -14470,7 +14470,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_get_success_miss_duration_seconds_bucket{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, region, project, network, category, connector))", + "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_get_success_miss_duration_seconds_bucket{le!=\"+Inf\", cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\"}[$rate_window])) by (le, region, project, network, category, connector))", "hide": false, "legendFormat": "p95 - {{region}}, {{project}}, {{network}}, {{category}}, {{connector}}", "range": true, @@ -14675,7 +14675,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_get_error_duration_seconds_bucket{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", error!~\".*(ErrRecordNotFound|MissingData).*\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector, error))", + "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_get_error_duration_seconds_bucket{le!=\"+Inf\", cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", error!~\".*(ErrRecordNotFound|MissingData).*\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector, error))", "legendFormat": "p50 - {{project}}, {{network}}, {{category}}, {{connector}}, {{error}}", "range": true, "refId": "A" @@ -14686,7 +14686,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_get_error_duration_seconds_bucket{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", error!~\"ErrRecordNotFound.*\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector, error))", + "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_get_error_duration_seconds_bucket{le!=\"+Inf\", cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", error!~\"ErrRecordNotFound.*\", project=~\"${project:regex}\"}[$rate_window])) by (le, project, network, category, connector, error))", "hide": false, "legendFormat": "p95 - {{project}}, {{network}}, {{category}}, {{connector}}, {{error}}", "range": true, @@ -15503,7 +15503,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_evm_get_logs_range_bucket{namespace=~\"${namespace:regex}\", cluster=~\"${cluster:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", project=~\"${project:regex}\", outcome=\"hit\"}[$rate_window])) by (le, project, network, connector))", + "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_evm_get_logs_range_bucket{le!=\"+Inf\", namespace=~\"${namespace:regex}\", cluster=~\"${cluster:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", project=~\"${project:regex}\", outcome=\"hit\"}[$rate_window])) by (le, project, network, connector))", "legendFormat": "{{project}} · {{connector}} ({{network}})", "range": true, "refId": "A" @@ -15611,7 +15611,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(erpc_cache_evm_get_logs_range_bucket{namespace=~\"${namespace:regex}\", cluster=~\"${cluster:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", project=~\"${project:regex}\", outcome=\"hit\"}[$rate_window])) by (le, project, network, connector))", + "expr": "histogram_quantile(0.99, sum(rate(erpc_cache_evm_get_logs_range_bucket{le!=\"+Inf\", namespace=~\"${namespace:regex}\", cluster=~\"${cluster:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", project=~\"${project:regex}\", outcome=\"hit\"}[$rate_window])) by (le, project, network, connector))", "legendFormat": "{{project}} · {{connector}} ({{network}})", "range": true, "refId": "A" @@ -16026,7 +16026,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(1, sum(rate(erpc_consensus_agreement_count_bucket{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\", finality=~\"${finality:regex}\", user=~\"${user:regex}\"}[$rate_window])) by (le, network, category, finality))", + "expr": "histogram_quantile(1, sum(rate(erpc_consensus_agreement_count_bucket{le!=\"+Inf\", cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\", finality=~\"${finality:regex}\", user=~\"${user:regex}\"}[$rate_window])) by (le, network, category, finality))", "interval": "", "legendFormat": "{{network}}, {{category}}, {{finality}}", "range": true, @@ -16236,7 +16236,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(1, sum(rate(erpc_consensus_responses_collected_bucket{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\", finality=~\"${finality:regex}\", user=~\"${user:regex}\"}[$rate_window])) by (le, network, category, short_circuited, finality))", + "expr": "histogram_quantile(1, sum(rate(erpc_consensus_responses_collected_bucket{le!=\"+Inf\", cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\", region=~\"${region:regex}\", network=~\"${network:regex}\", network!~\"${ignore_network:regex}\", category=~\"${category:regex}\", project=~\"${project:regex}\", finality=~\"${finality:regex}\", user=~\"${user:regex}\"}[$rate_window])) by (le, network, category, short_circuited, finality))", "interval": "", "legendFormat": "{{network}}, {{category}}, {{finality}}, short-circuited={{short_circuited}}", "range": true, diff --git a/package.json b/package.json index 0b49122e2..74b17d796 100644 --- a/package.json +++ b/package.json @@ -15,5 +15,5 @@ "engines": { "node": ">=18.14" }, - "version": "0.4.0" + "version": "0.5.0" } diff --git a/specs/selector-isolation/feature.md b/specs/selector-isolation/feature.md index 6dc3df5a5..d410c0415 100644 --- a/specs/selector-isolation/feature.md +++ b/specs/selector-isolation/feature.md @@ -24,7 +24,7 @@ request pinned to the normal group). Built so far: [erpc/networks.go](../../erpc/networks.go) shouldHandleMethod). Targeting + failover containment. - **Phase 2′ — selector-scoped served tip** *(done)*: the request is bound to - ctx at the top of `Forward`; `tipCandidateUpstreams` filters the served-tip + ctx at the top of `Forward`; `EvmRoutableUpstreams` filters the served-tip input set by the request's selector, so both max-mode (`evmHighestBlockMax`) and cluster-mode (`clusteredServedTip`) resolve `latest`/`finalized` AMONG the selected subset (plus the future-block short-circuit and @@ -121,8 +121,8 @@ to tags is the only change; containment is automatic. ## 5. Lag isolation — request-time fence (the central change) **Today** block-head-lag exclusion is a *background* policy predicate -(`.excludeIf(any(blockNumberLagAbove(16), blockSecondsLagAbove(30)))`, -[internal/policy/default_policy.js:43](../../internal/policy/default_policy.js)) +(`.excludeIf(any(blockNumberLagAbove(16), blockSecondsLagAbove(30)), { probe: false })`, +[internal/policy/default_policy.js:46](../../internal/policy/default_policy.js)) reading the *precomputed network-wide scalar* `BlockHeadLag` ([health/tracker.go:1339](../../health/tracker.go)). That decision is computed once per `(network, method, finality)` tick, request-independent, so a @@ -171,9 +171,9 @@ resolve `latest`/`finalized` and gate block availability. They must reflect the request's selector so a `systx`-targeted request never resolves `latest` to a block only a faster family has. -- `ComputeServedTipCandidate` ([architecture/evm/served_tip.go:102-196](../../architecture/evm/served_tip.go)) - stays a **pure, unchanged** function. `gatherEvmTipInputsForMethod` - ([erpc/networks.go:275-305](../../erpc/networks.go)) gains a selector filter on +- The picker, `common.PickServedTip` ([common/served_tip.go](../../common/served_tip.go)), + stays a **pure** function. The candidate walk, `EvmRoutableUpstreams` + ([erpc/networks.go](../../erpc/networks.go)), applies the selector filter to its inputs. - **Untargeted** → existing network-wide monotonic counter (`servedLatestBlockShared`, [erpc/networks.go:59-62](../../erpc/networks.go)). diff --git a/specs/selector-isolation/plan.md b/specs/selector-isolation/plan.md index 512a2a228..4f04ab806 100644 --- a/specs/selector-isolation/plan.md +++ b/specs/selector-isolation/plan.md @@ -74,8 +74,8 @@ Foundation for the lag fence; no behavior change yet. ### Acceptance - Unit test: `poolMax` over a mixed pool; ignores syncing / zero-head upstreams - (mirror `gatherEvmTipInputsForMethod` filters at - [erpc/networks.go:283-298](../../erpc/networks.go)). + (mirror the voter rule, `tipVoterRank` in + [erpc/networks.go](../../erpc/networks.go)). --- @@ -117,12 +117,12 @@ Foundation for the lag fence; no behavior change yet. ## Phase 3 — Selector-scoped served tip ### 3.1 Pure path (glob / id / untargeted) -- `gatherEvmTipInputsForMethod` ([erpc/networks.go:275-305](../../erpc/networks.go)) +- `EvmRoutableUpstreams` ([erpc/networks.go](../../erpc/networks.go)) gains a selector filter. - Untargeted → existing network counter (unchanged). - Glob/id → stateless cluster-min over filtered inputs; optional per-pod `atomic.Int64` monotonic guard (in-memory, evictable, never shared). -- `ComputeServedTipCandidate` ([architecture/evm/served_tip.go:102-196](../../architecture/evm/served_tip.go)) +- `common.PickServedTip` ([common/served_tip.go](../../common/served_tip.go)) unchanged. ### 3.2 `EvmHighest*` resolve the partition diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 01c7894f2..6007e0b83 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -135,6 +135,16 @@ var ( Help: "Whether the upstream WebSocket connection is currently established (1) or down/wedged (0).", }, []string{"project", "vendor", "network", "upstream"}) + // MetricUpstreamWebsocketHeadsStallReconnectsTotal counts reconnects + // forced because an upstream's newHeads subscription went silent while + // the same upstream's polled head kept advancing. Any increase means a + // node dropped a live subscription without closing it. + MetricUpstreamWebsocketHeadsStallReconnectsTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "upstream_websocket_heads_stall_reconnects_total", + Help: "Upstream WebSocket reconnects forced by a silent newHeads subscription while the upstream head advanced.", + }, []string{"project", "network", "upstream"}) + MetricWebsocketSubscriptionNotificationsDroppedTotal = DefineCounter(prometheus.CounterOpts{ Namespace: "erpc", Name: "websocket_subscription_notifications_dropped_total", @@ -195,36 +205,54 @@ var ( Help: "Event keys held by the network's logs and newPendingTransactions dedup windows; each filter's window holds at most indexer.dedupWindowSize keys (default 8192).", }, []string{"project", "network"}) + MetricSubscriptionWithheldHeadsTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "subscription_withheld_heads_total", + Help: "Upstream newHeads events not delivered to clients because the network did not accept the head.", + }, []string{"project", "network"}) + + MetricSubscriptionUndeliveredTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "subscription_undelivered_total", + Help: "Upstream subscription events that passed dedup but matched no client subscription.", + }, []string{"project", "network", "kind"}) + + MetricNetworkSubscriptionLastHeadTimestamp = DefineGauge(prometheus.GaugeOpts{ + Namespace: "erpc", + Name: "network_subscription_last_head_timestamp_seconds", + Help: "Unix time the subscription indexer last accepted a new head for the network.", + }, []string{"project", "network"}) + // MetricNetworkServedTipBlockNumber is the block number the network actually // advertises/serves as the tip for a block tag (axis=latest|finalized): the - // freshest block a strict MAJORITY of eligible upstreams already have (see - // common.PickServedTip). lane="all" is the network-wide pick; a named lane is a - // use-upstream group's own pick (present only for networks receiving - // targeted traffic). + // freshest block at least servedTip.quorum eligible upstreams already have + // (see common.PickServedTip). lane="all" is the network-wide pick; a named + // lane is a use-upstream group's own pick (present only for networks + // receiving targeted traffic). MetricNetworkServedTipBlockNumber = DefineGauge(prometheus.GaugeOpts{ Namespace: "erpc", Name: "network_served_tip_block_number", - Help: "Block number served/advertised as the tip (freshest block a strict majority of eligible upstreams already have). lane=\"all\" = network-wide; a named lane = a use-upstream group's own tip.", + Help: "Block number served/advertised as the tip (freshest block at least servedTip.quorum eligible upstreams already have). lane=\"all\" = network-wide; a named lane = a use-upstream group's own tip.", }, []string{"project", "network", "lane", "axis"}) // MetricNetworkServedTipLagBlocks is the deliberate, bounded served-tip lag: - // blocks the majority tip sits behind the single freshest upstream view at - // pick time (Freshest - Tip). Computed at the same instant as the pick, so it - // is exact (no cross-metric scrape skew). NOTE: the series is ABSENT (not 0) - // when served-tip is in default MAX mode (feature off) for that axis. + // blocks the served tip sits behind the corroborated freshest upstream view + // at pick time (Freshest - Tip). Computed at the same instant as the pick, + // so it is exact (no cross-metric scrape skew). MetricNetworkServedTipLagBlocks = DefineGauge(prometheus.GaugeOpts{ Namespace: "erpc", Name: "network_served_tip_lag_blocks", - Help: "Blocks the majority served tip sits behind the single freshest upstream view at pick time, per network, lane and axis. lane=\"all\" is the network-wide pick; a named lane is a use-upstream group's own pick.", + Help: "Blocks the served tip sits behind the corroborated freshest upstream view at pick time, per network, lane and axis. lane=\"all\" is the network-wide pick; a named lane is a use-upstream group's own pick.", }, []string{"project", "network", "lane", "axis"}) // MetricNetworkServedTipAdvanceAgeSeconds is the time since this process // last observed the served-tip VALUE change, exported at pick time. This // is the universal stuck-tip detector: regardless of WHICH mechanism would // freeze the tip (a failure mode nobody predicted included), a healthy - // network advances every block or two, so `age >> block time` on a network - // with live traffic is always alert-worthy. Absent until the process - // observes its first change (and absent in default MAX mode). + // network advances every block or two, so `age >> block time` on the + // network-wide lane, which is evaluated every second whatever the + // traffic, is always alert-worthy. Absent until the process + // observes its first change. MetricNetworkServedTipAdvanceAgeSeconds = DefineGauge(prometheus.GaugeOpts{ Namespace: "erpc", Name: "network_served_tip_advance_age_seconds", @@ -234,7 +262,7 @@ var ( // MetricNetworkServedTipRegressionTotal counts the TRANSITIONS into a guard // state: a pick fell further than the configured tolerance below the // corroborated live head, i.e. a ballot poisoned by values that are not head - // observations (the 2026-08 celo incident: two upstreams voting their + // observations (e.g. two capped legacy upstreams voting their // blockAvailability.upper bound). outcome="held" = the network started // serving its last corroborated pick instead; outcome="failed_open" = a hold // outlasted the guard's window and the pick is served as computed. @@ -250,29 +278,25 @@ var ( Help: "Transitions into a served-tip regression-guard state (counts state ENTRIES, not evaluations; any nonzero rate is alert-worthy), by outcome: held = the last corroborated pick is being served instead of a pick far below the corroborated live head; failed_open = that hold outlasted the guard's window and the pick is served as computed.", }, []string{"project", "network", "lane", "axis", "outcome"}) - // MetricNetworkServedTipTrajectoryTotal counts the TRANSITIONS into a - // trajectory-referee state: a FRESHER corroborated group of live heads sat - // above the majority pick. outcome="override" = that group matched the - // network's own head trajectory, earned its dwell, and is being served - // instead of a stalled majority — rare, and the reason to page someone: some - // upstreams froze while remaining eligible. outcome="fallback" = it was - // refused (too far from the trajectory, or it has not held its place long - // enough), so the majority pick stands. - // - // TRANSITIONS, NOT EVALUATIONS (see MetricNetworkServedTipRegressionTotal), - // and an override the guaranteed-method floor pulls back to the majority pick - // is not counted at all — it changed nothing. - // - // Both outcomes are silent in steady state by construction: a fleet whose - // live heads form one cluster elects that cluster, and a fleet permanently - // split around its own median elects the median's group (the trajectory is - // fitted to the median, so no permanently-offset group is ever closer to it). - // Any nonzero rate here means the fleet's groups are genuinely diverging. - MetricNetworkServedTipTrajectoryTotal = DefineCounter(prometheus.CounterOpts{ - Namespace: "erpc", - Name: "network_served_tip_trajectory_total", - Help: "Transitions into a served-tip trajectory-referee state (counts state ENTRIES, not evaluations; any nonzero rate is alert-worthy), by outcome: override = a corroborated group of live heads matched the network's head trajectory and is served in place of a stalled majority; fallback = such a group sat above the majority pick and was refused.", - }, []string{"project", "network", "lane", "axis", "outcome"}) + // MetricUpstreamHeadBenchedTotal counts the times the health tracker benched + // an upstream's head: it stayed behind peers that kept advancing (stalled, + // or replaying far behind the tip) for its stall window. A benched upstream + // votes in no served-tip ballot and is not the network lag reference until + // its head catches up with the live ones. + MetricUpstreamHeadBenchedTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "upstream_head_benched_total", + Help: "Times an upstream's head was benched out of the served-tip ballots and the network lag reference: it stayed behind peers that kept advancing for its stall window.", + }, []string{"project", "vendor", "network", "upstream"}) + + // MetricWebsocketRejectedTotal counts client WebSocket connections and + // subscriptions refused by the instance-wide caps, by reason: + // conn_global, conn_per_ip or sub_global. + MetricWebsocketRejectedTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "websocket_rejected_total", + Help: "Client WebSocket connections and subscriptions refused by server.webSocket caps, by reason (conn_global, conn_per_ip, sub_global).", + }, []string{"project", "reason"}) MetricUpstreamCordoned = DefineGauge(prometheus.GaugeOpts{ Namespace: "erpc", @@ -349,6 +373,12 @@ var ( Help: "Per-upstream WOULD-HAVE-BEEN-EXCLUDED events from `shadowExcludeIf` predicates. Same leaf-slug attribution as `selection_exclusion_total`, but the upstream stays in rotation — meant for safely auditioning a new (or removed) exclusion rule in production before promoting it to a real `excludeIf`.", }, []string{"project", "network", "method", "upstream", "reason"}) + MetricSelectionMethodBlockTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "selection_method_block_total", + Help: "Ticks where `blockMethods` blocked an upstream for one method; it stays in rotation for other methods. `method` is the blocked method, `reason` the leaf-predicate slug.", + }, []string{"project", "network", "method", "upstream", "reason"}) + MetricSelectionExcludedSeconds = DefineGauge(prometheus.GaugeOpts{ Namespace: "erpc", Name: "selection_excluded_seconds", @@ -472,12 +502,13 @@ var ( }, []string{"project", "vendor", "network", "upstream"}) // MetricUpstreamStateProbe counts probe outcomes, by probe kind - // (context = execution-context call, proof = eth_getProof) and outcome - // (match/mismatch/unsupported/error). + // (context = execution-context call at a pinned block, proof = eth_getProof, + // latest = execution-context call at the "latest" tag) and outcome + // (match/stale/pin_ignored/mismatch/unsupported/error). MetricUpstreamStateProbe = DefineCounter(prometheus.CounterOpts{ Namespace: "erpc", Name: "upstream_state_probe_total", - Help: "Total state-trie probes by kind (context/proof) and outcome (match/mismatch/unsupported/error).", + Help: "Total state-trie probes by kind (context/proof/latest) and outcome (match/stale/pin_ignored/mismatch/unsupported/error).", }, []string{"project", "vendor", "network", "upstream", "probe", "outcome"}) // MetricIntegrityFollowHead is the highest block of the CONTIGUOUS, @@ -788,6 +819,20 @@ var ( Help: "Total vendor credit units accrued by upstream attempts, per (project, network, upstream, vendor, method, finality). Vendor-owned units, not money; not comparable across vendors.", }, []string{"project", "network", "upstream", "vendor", "category", "finality"}) + // Vendor requests saved by packing = sum(outcome="packed") - count(batch_size). + MetricUpstreamMulticall3CallsTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "upstream_multicall3_calls_total", + Help: "eth_calls seen by an upstream with multicall3 packing, by outcome: packed (answered by a multicall), alone (no other call to pack with), reverted/empty/failed (multicall could not answer it exactly, sent alone), ineligible (not packable, sent alone), bypassed (target in bypassContracts or learned, sent alone), canceled (attempt ended while waiting).", + }, []string{"project", "network", "upstream", "vendor", "outcome"}) + + MetricUpstreamMulticall3BatchSize = DefineLabeledHistogram(prometheus.HistogramOpts{ + Namespace: "erpc", + Name: "upstream_multicall3_batch_size", + Help: "eth_calls carried by each multicall3 request sent to an upstream.", + Buckets: []float64{2, 3, 5, 8, 13, 20, 50, 100}, + }, []string{"project", "network", "upstream", "vendor"}) + // MetricNetworkRetryAttemptTotal counts retry attempts at the // network scope, labeled by the reason for retry (empty_result / // pending_tx / retryable_error / block_unavailable / missing_data). @@ -886,27 +931,30 @@ var ( Help: "Total number of rate limiter fail-open events (requests allowed due to errors/timeouts).", }, []string{"project", "network", "user", "agent_name", "budget", "category", "reason"}) + // MetricRateLimiterStoreFailureTotal counts requests the remote store could + // not decide, by reason (store_unavailable | store_error | admission_full | + // limit_timeout) and what onFailure decided (allow | deny | local_allow | + // local_deny). rate_limiter_failopen_total keeps counting the allow ones. + MetricRateLimiterStoreFailureTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "rate_limiter_store_failure_total", + Help: "Total number of rate-limit checks the remote store could not decide, by reason and onFailure decision.", + }, []string{"budget", "reason", "decision"}) + // MetricRateLimiterRemoteInflight is a per-budget gauge of concurrent in-flight - // remote (e.g. Redis) DoLimit calls. When a remote rate limiter is overwhelmed - // this gauge climbs without bound — the admission semaphore in - // doLimitWithTimeout uses MetricRateLimiterAdmissionSheddedTotal to indicate - // when the cap is reached, but this gauge is the canary that something is - // queueing on the remote. + // remote (e.g. Redis) DoLimit calls, bounded by the admission semaphore. MetricRateLimiterRemoteInflight = DefineGauge(prometheus.GaugeOpts{ Namespace: "erpc", Name: "rate_limiter_remote_inflight", Help: "Current number of in-flight remote rate-limit checks per budget.", }, []string{"budget"}) - // MetricRateLimiterRemoteAdmissionSheddedTotal is a counter of fail-open events - // caused by the per-budget admission semaphore being full. This is intentionally - // distinct from "limit_timeout" in MetricRateLimiterFailopenTotal because it - // indicates load shedding (we never even attempted the remote call) vs the - // remote being slow. + // MetricRateLimiterRemoteAdmissionSheddedTotal counts checks that never + // reached the remote because the per-budget admission semaphore was full. MetricRateLimiterRemoteAdmissionSheddedTotal = DefineCounter(prometheus.CounterOpts{ Namespace: "erpc", Name: "rate_limiter_remote_admission_shedded_total", - Help: "Total number of remote rate-limit checks fail-opened because the admission semaphore was full.", + Help: "Total number of remote rate-limit checks shed because the admission semaphore was full.", }, []string{"budget"}) MetricCacheSetSuccessTotal = DefineCounter(prometheus.CounterOpts{ diff --git a/thirdparty/alchemy.go b/thirdparty/alchemy.go index 462a7eb21..25061096f 100644 --- a/thirdparty/alchemy.go +++ b/thirdparty/alchemy.go @@ -76,7 +76,6 @@ var defaultAlchemyNetworkSubdomains = map[int64]string{ 34443: "mode-mainnet", 919: "mode-sepolia", 69000: "anime-mainnet", - 6900: "anime-sepolia", 33139: "apechain-mainnet", 33111: "apechain-curtis", 232: "lens-mainnet", @@ -154,6 +153,14 @@ var defaultAlchemyNetworkSubdomains = map[int64]string{ 3448148188: "tron-testnet", } +// alchemyCollidingChainIds holds chain ids under which Alchemy serves a +// different chain than the one that id usually names. eth_chainId cannot tell +// the two apart, so an auto-discovered upstream would serve another chain's +// blocks. +var alchemyCollidingChainIds = map[int64]struct{}{ + 6900: {}, // Alchemy: anime-sepolia testnet; 6900 is Nibiru mainnet +} + const DefaultAlchemyRecheckInterval = 24 * time.Hour // alchemyApiUrl is the tRPC endpoint used to discover Alchemy networks. @@ -624,6 +631,9 @@ func (v *AlchemyVendor) fetchAlchemyNetworks(ctx context.Context, apiUrl string) newData[chainID] = subdomain } } + for chainID := range alchemyCollidingChainIds { + delete(newData, chainID) + } return newData, nil } diff --git a/thirdparty/alchemy_test.go b/thirdparty/alchemy_test.go index 69f81d971..a2567a818 100644 --- a/thirdparty/alchemy_test.go +++ b/thirdparty/alchemy_test.go @@ -69,7 +69,7 @@ func TestAlchemyVendor_SuccessfulFetchPromotesOverFallback(t *testing.T) { fetched := make(chan struct{}, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"result":{"data":[{"networkChainId":424242,"kebabCaseId":"custom-net"}]}}`)) + _, _ = w.Write([]byte(`{"result":{"data":[{"networkChainId":424242,"kebabCaseId":"custom-net"},{"networkChainId":6900,"kebabCaseId":"anime-sepolia"}]}}`)) select { case fetched <- struct{}{}: default: @@ -92,6 +92,9 @@ func TestAlchemyVendor_SuccessfulFetchPromotesOverFallback(t *testing.T) { supported, err := vendor.SupportsNetwork(ctx, &logger, settings, "evm:424242") require.NoError(t, err) assert.False(t, supported, "first call returns built-in static map; async refresh hasn't published yet") + supported, err = vendor.SupportsNetwork(ctx, &logger, settings, "evm:6900") + require.NoError(t, err) + assert.False(t, supported, "chain 6900 is Nibiru mainnet, not Alchemy's anime-sepolia") // Wait for async refresh to hit the mock server, then re-query. select { @@ -103,6 +106,9 @@ func TestAlchemyVendor_SuccessfulFetchPromotesOverFallback(t *testing.T) { ok, err := vendor.SupportsNetwork(ctx, &logger, settings, "evm:424242") return err == nil && ok }, 5*time.Second, 50*time.Millisecond, "async-refreshed snapshot should promote custom chain over fallback") + supported, err = vendor.SupportsNetwork(ctx, &logger, settings, "evm:6900") + require.NoError(t, err) + assert.False(t, supported, "live API's 6900 entry must not survive the merge") // After the refresh, static-default chains are NOT in the snapshot, but // the alchemy resolveNetworks function falls back to the default map @@ -256,3 +262,240 @@ func TestAlchemyVendor_Code3_MissingDataIsRetryable(t *testing.T) { assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointExecutionException), "expected execution exception, got %v", err) assert.False(t, common.IsRetryableTowardNetwork(err)) } + +// alchemyOfflineURL refuses connections instantly so the async refreshes +// kicked off by SupportsNetwork / GenerateConfigs never leave the machine. +const alchemyOfflineURL = "http://127.0.0.1:1/alchemy-offline" + +func TestAlchemyVendor_Name(t *testing.T) { + assert.Equal(t, "alchemy", CreateAlchemyVendor().Name()) +} + +func TestAlchemyVendor_SupportsNetworkBranches(t *testing.T) { + offline := common.VendorSettings{"chainsUrl": alchemyOfflineURL} + svtCheckSupports(t, CreateAlchemyVendor(), offline, []svtSupportCase{ + {"solana:mainnet", false, false}, + {"evm:abc", false, true}, + {"evm:8453", true, false}, // default recheck interval + built-in fallback + }) + + prev := swapAlchemyApiURL(t, alchemyOfflineURL) + defer swapAlchemyApiURL(t, prev) + ok, err := svtSupports(t, CreateAlchemyVendor(), common.VendorSettings{}, "evm:1") + require.NoError(t, err) + assert.True(t, ok, "empty settings use alchemyApiUrl with built-in fallback") +} + +func TestAlchemyVendor_GenerateConfigsBranches(t *testing.T) { + key := common.VendorSettings{"apiKey": "KEY", "chainsUrl": alchemyOfflineURL} + + svtCheckGenErrors(t, CreateAlchemyVendor, []svtErrCase{ + {"missing apiKey", svtEvm(1), common.VendorSettings{"chainsUrl": alchemyOfflineURL}, "apiKey is required in alchemy settings"}, + {"nil evm", &common.UpstreamConfig{}, key, "alchemy vendor requires upstream.evm to be defined"}, + {"zero chain", svtEvm(0), key, "alchemy vendor requires upstream.evm.chainId to be defined"}, + {"invalid chainsUrl", svtEvm(1), common.VendorSettings{"apiKey": "KEY", "chainsUrl": "ftp://host"}, "invalid chainsUrl"}, + {"unsupported chain", svtEvm(999999999999), key, "unsupported network chain ID for Alchemy: 999999999999"}, + {"unparsable url", svtEvm(1), common.VendorSettings{"apiKey": "a\nb", "chainsUrl": alchemyOfflineURL}, "invalid control character"}, + }) + + t.Run("built-in fallback urls", func(t *testing.T) { + svtCheckURLs(t, CreateAlchemyVendor, key, map[int64]string{ + 1: "https://eth-mainnet.g.alchemy.com/v2/KEY", + 42161: "https://arb-mainnet.g.alchemy.com/v2/KEY", + }) + cfg, err := svtGenerate(t, CreateAlchemyVendor(), svtEvm(1), key) + require.NoError(t, err) + assert.Equal(t, "alchemy", cfg.VendorName) + }) + + t.Run("default chainsUrl", func(t *testing.T) { + prev := swapAlchemyApiURL(t, alchemyOfflineURL) + defer swapAlchemyApiURL(t, prev) + cfg, err := svtGenerate(t, CreateAlchemyVendor(), svtEvm(10), common.VendorSettings{"apiKey": "KEY"}) + require.NoError(t, err) + assert.Equal(t, "https://opt-mainnet.g.alchemy.com/v2/KEY", cfg.Endpoint) + }) + + t.Run("fetched snapshot wins over built-in", func(t *testing.T) { + v := CreateAlchemyVendor().(*AlchemyVendor) + primeRemoteCache(t, v.cache, alchemyOfflineURL, map[int64]string{1: "eth-custom"}) + cfg, err := svtGenerate(t, v, svtEvm(1), common.VendorSettings{"apiKey": "KEY", "chainsUrl": alchemyOfflineURL, "recheckInterval": time.Hour}) + require.NoError(t, err) + assert.Equal(t, "https://eth-custom.g.alchemy.com/v2/KEY", cfg.Endpoint) + }) + + t.Run("preset non-evm endpoint passes through without panicking", func(t *testing.T) { + ups := &common.UpstreamConfig{Endpoint: "https://solana-mainnet.g.alchemy.com/v2/KEY", Type: common.UpstreamTypeSvm} + cfg, err := svtGenerate(t, CreateAlchemyVendor(), ups, common.VendorSettings{}) + require.NoError(t, err) + assert.Equal(t, "https://solana-mainnet.g.alchemy.com/v2/KEY", cfg.Endpoint) + assert.Equal(t, common.UpstreamTypeSvm, cfg.Type) + assert.Equal(t, "alchemy", cfg.VendorName) + assert.NotNil(t, cfg.JsonRpc) + }) +} + +func TestAlchemyVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateAlchemyVendor() + + for _, msg := range []string{"Must be authenticated!", "invalid access key"} { + err, details := vendorErr(t, v, "eth_getLogs", 401, -32600, msg, "d") + require.Error(t, err, msg) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointUnauthorized), "%q -> %v", msg, err) + assert.Equal(t, common.JsonRpcErrorUnauthorized, normalizedCode(t, err)) + assert.Equal(t, "d", details["data"]) + } + + t.Run("code 3 missing data", func(t *testing.T) { + err, _ := vendorErr(t, v, "eth_getLogs", 400, 3, "header not found", "") + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointMissingData), "got %v", err) + assert.Equal(t, common.JsonRpcErrorMissingData, normalizedCode(t, err)) + }) + + t.Run("code 3 revert", func(t *testing.T) { + err, details := vendorErr(t, v, "eth_call", 400, 3, "execution reverted", "0x08c379a0") + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointExecutionException), "got %v", err) + assert.Equal(t, common.JsonRpcErrorEvmReverted, normalizedCode(t, err)) + assert.Equal(t, "0x08c379a0", details["data"]) + }) + + err, details := vendorErr(t, v, "eth_getLogs", 400, 0, "Must be authenticated!", "d") + assert.NoError(t, err, "code 0 is not inspected") + assert.NotContains(t, details, "data") + + assert.NoError(t, v.GetVendorSpecificErrorIfAny(svtReq(), nil, "x", map[string]interface{}{})) +} + +func TestAlchemyVendor_OwnsUpstream(t *testing.T) { + v := CreateAlchemyVendor() + svtAssertOwns(t, v, map[string]bool{ + "alchemy://key": true, + "evm+alchemy://key": true, + "https://eth-mainnet.g.alchemy.com/v2/key": true, + "https://eth-mainnet.alchemyapi.io/v2/key": true, + "https://rpc.ankr.com/eth/key": false, + }) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://proxy.example/x", VendorName: "alchemy"}), "vendorName claims ownership") +} + +func TestAlchemyVendor_FetchAlchemyNetworks(t *testing.T) { + v := CreateAlchemyVendor().(*AlchemyVendor) + ctx := context.Background() + + t.Run("api wins, defaults merged, invalid and colliding entries dropped", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `{"result":{"data":[ + {"networkChainId":1,"kebabCaseId":"eth-renamed"}, + {"networkChainId":424242,"kebabCaseId":"custom-net"}, + {"networkChainId":0,"kebabCaseId":"no-chain"}, + {"networkChainId":555,"kebabCaseId":""}, + {"networkChainId":6900,"kebabCaseId":"anime-sepolia"} + ]}}`) + got, err := v.fetchAlchemyNetworks(ctx, srv.URL) + require.NoError(t, err) + assert.Equal(t, "eth-renamed", got[1]) + assert.Equal(t, "custom-net", got[424242]) + assert.Equal(t, "base-mainnet", got[8453], "built-in default merged in") + assert.NotContains(t, got, int64(0)) + assert.NotContains(t, got, int64(555)) + assert.NotContains(t, got, int64(6900)) + assert.Len(t, got, len(defaultAlchemyNetworkSubdomains)+1) + }) + + t.Run("non-2xx status", func(t *testing.T) { + srv := newJSONServer(t, http.StatusBadGateway, `{}`) + _, err := v.fetchAlchemyNetworks(ctx, srv.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "non-200 code: 502") + }) + + t.Run("invalid json", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `{not json`) + _, err := v.fetchAlchemyNetworks(ctx, srv.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse Alchemy API data") + }) + + t.Run("connection refused", func(t *testing.T) { + _, err := v.fetchAlchemyNetworks(ctx, alchemyOfflineURL) + assert.Error(t, err) + }) + + t.Run("malformed url", func(t *testing.T) { + _, err := v.fetchAlchemyNetworks(ctx, "http://bad\nhost") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid control character") + }) +} + +func TestAlchemyVendor_CreditUnitsEdgeCases(t *testing.T) { + t.Run("nil vendor and nil cache use built-in table", func(t *testing.T) { + var nilVendor *AlchemyVendor + assert.Equal(t, int64(26), common.ResolveCreditUnits(nilVendor.creditUnitsTable(), nil, "eth_call")) + bare := &AlchemyVendor{} + assert.Equal(t, int64(26), bare.CreditUnits(cuReq(t, "eth_call"), nil)) + logger := zerolog.Nop() + nilVendor.refreshCreditUnitsAsync(&logger) // must not panic + bare.refreshCreditUnitsAsync(&logger) + }) + + t.Run("fresh snapshot skips refresh", func(t *testing.T) { + hits := make(chan struct{}, 4) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits <- struct{}{} + })) + defer srv.Close() + orig := swapAlchemyCreditUnitsURL(t, srv.URL) + defer swapAlchemyCreditUnitsURL(t, orig) + + v := CreateAlchemyVendor().(*AlchemyVendor) + primeRemoteCache(t, v.cuCache, srv.URL, map[string]int64{"eth_call": 1}) + logger := zerolog.Nop() + v.refreshCreditUnitsAsync(&logger) + assert.Equal(t, int64(1), v.CreditUnits(cuReq(t, "eth_call"), nil)) + assert.Len(t, hits, 0, "fresh table must not be refetched") + }) + + t.Run("parser skips single-cell rows and keeps other sections out", func(t *testing.T) { + md := "# Trace API\n|trace_block|\n| trace_call | 30 |\n| **trace_filter** | 75 |\n|---|---|\n" + assert.Equal(t, map[string]int64{"trace_call": 30, "trace_filter": 75}, parseAlchemyCreditUnits(md)) + assert.Empty(t, parseAlchemyCreditUnits("| eth_call | 1 |\n"), "rows before any allowlisted heading are ignored") + }) + + t.Run("fetch errors", func(t *testing.T) { + orig := swapAlchemyCreditUnitsURL(t, "http://bad\nhost") + defer swapAlchemyCreditUnitsURL(t, orig) + _, err := fetchAlchemyCreditUnits(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid control character") + + swapAlchemyCreditUnitsURL(t, alchemyOfflineURL) + _, err = fetchAlchemyCreditUnits(context.Background()) + assert.Error(t, err, "connection refused") + + truncated := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "1000") + _, _ = w.Write([]byte("# Debug API\n")) + })) + defer truncated.Close() + swapAlchemyCreditUnitsURL(t, truncated.URL) + _, err = fetchAlchemyCreditUnits(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "unexpected EOF") + }) + + t.Run("request sends markdown accept header", func(t *testing.T) { + accept := make(chan string, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + accept <- r.Header.Get("Accept") + _, _ = w.Write([]byte(alchemyCUMarkdownSample)) + })) + defer srv.Close() + orig := swapAlchemyCreditUnitsURL(t, srv.URL) + defer swapAlchemyCreditUnitsURL(t, orig) + _, err := fetchAlchemyCreditUnits(context.Background()) + require.NoError(t, err) + assert.Equal(t, "text/markdown, text/plain", <-accept) + }) +} diff --git a/thirdparty/ankr_test.go b/thirdparty/ankr_test.go new file mode 100644 index 000000000..aba3b72af --- /dev/null +++ b/thirdparty/ankr_test.go @@ -0,0 +1,170 @@ +package thirdparty + +import ( + "context" + "testing" + + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Shared helpers for the static-vendor tests (svt = static vendor test). +// Error normalization uses vendorErr / normalizedCode from helpers_test.go. + +func svtReq() *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":[]}`)) +} + +func svtGenerate(t *testing.T, v common.Vendor, ups *common.UpstreamConfig, settings common.VendorSettings) (*common.UpstreamConfig, error) { + t.Helper() + logger := zerolog.Nop() + cfgs, err := v.GenerateConfigs(context.Background(), &logger, ups, settings) + if err != nil { + assert.Nil(t, cfgs) + return nil, err + } + require.Len(t, cfgs, 1) + return cfgs[0], nil +} + +func svtSupports(t *testing.T, v common.Vendor, settings common.VendorSettings, networkId string) (bool, error) { + t.Helper() + logger := zerolog.Nop() + return v.SupportsNetwork(context.Background(), &logger, settings, networkId) +} + +func svtEvm(chainId int64) *common.UpstreamConfig { + return &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: chainId}} +} + +func svtAssertOwns(t *testing.T, v common.Vendor, cases map[string]bool) { + t.Helper() + for endpoint, want := range cases { + assert.Equal(t, want, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: endpoint}), "OwnsUpstream(%q)", endpoint) + } +} + +type svtSupportCase struct { + id string + want bool + wantErr bool +} + +func svtCheckSupports(t *testing.T, v common.Vendor, settings common.VendorSettings, cases []svtSupportCase) { + t.Helper() + for _, c := range cases { + ok, err := svtSupports(t, v, settings, c.id) + if c.wantErr { + assert.Error(t, err, "SupportsNetwork(%q)", c.id) + } else { + assert.NoError(t, err, "SupportsNetwork(%q)", c.id) + } + assert.Equal(t, c.want, ok, "SupportsNetwork(%q)", c.id) + } +} + +type svtErrCase struct { + name string + ups *common.UpstreamConfig + set common.VendorSettings + msg string +} + +func svtCheckGenErrors(t *testing.T, mk func() common.Vendor, cases []svtErrCase) { + t.Helper() + for _, c := range cases { + _, err := svtGenerate(t, mk(), c.ups, c.set) + if assert.Error(t, err, c.name) { + assert.Contains(t, err.Error(), c.msg, c.name) + } + } +} + +// svtCheckURLs asserts the exact generated endpoint for each chain id, plus +// the defaults every generated config carries (evm type, non-nil JsonRpc). +func svtCheckURLs(t *testing.T, mk func() common.Vendor, settings common.VendorSettings, want map[int64]string) { + t.Helper() + for chainId, u := range want { + cfg, err := svtGenerate(t, mk(), svtEvm(chainId), settings) + if !assert.NoError(t, err, "chain %d", chainId) { + continue + } + assert.Equal(t, u, cfg.Endpoint, "chain %d", chainId) + assert.Equal(t, common.UpstreamTypeEvm, cfg.Type, "chain %d", chainId) + assert.NotNil(t, cfg.JsonRpc, "chain %d", chainId) + } +} + +// svtCheckPreset asserts a preset endpoint is returned untouched even with +// no settings, and that JsonRpc is defaulted. +func svtCheckPreset(t *testing.T, v common.Vendor, endpoint string) *common.UpstreamConfig { + t.Helper() + cfg, err := svtGenerate(t, v, &common.UpstreamConfig{Endpoint: endpoint}, common.VendorSettings{}) + require.NoError(t, err) + assert.Equal(t, endpoint, cfg.Endpoint) + assert.NotNil(t, cfg.JsonRpc) + return cfg +} + +// svtCheckDataOnly asserts a vendor that only forwards error data: no +// normalized error, details["data"] set iff Data is non-empty, and nil for +// a non-JsonRpcResponse input. +func svtCheckDataOnly(t *testing.T, v common.Vendor) { + t.Helper() + err, details := vendorErr(t, v, "eth_getLogs", 400, -32000, "some error", "extra") + assert.NoError(t, err) + assert.Equal(t, "extra", details["data"]) + + err, details = vendorErr(t, v, "eth_getLogs", 400, -32000, "some error", "") + assert.NoError(t, err) + assert.NotContains(t, details, "data") + + assert.NoError(t, v.GetVendorSpecificErrorIfAny(svtReq(), nil, "not a response", map[string]interface{}{})) +} + +func TestAnkrVendor_Name(t *testing.T) { + assert.Equal(t, "ankr", CreateAnkrVendor().Name()) +} + +func TestAnkrVendor_SupportsNetwork(t *testing.T) { + svtCheckSupports(t, CreateAnkrVendor(), common.VendorSettings{}, []svtSupportCase{ + {"evm:1", true, false}, + {"evm:5201", true, false}, + {"evm:999999999", false, false}, + {"solana:mainnet", false, false}, + {"evm:abc", false, true}, + }) +} + +func TestAnkrVendor_GenerateConfigs(t *testing.T) { + key := common.VendorSettings{"apiKey": "k/ey"} + svtCheckURLs(t, CreateAnkrVendor, key, map[int64]string{ + 1: "https://rpc.ankr.com/eth/k%2Fey", + 42161: "https://rpc.ankr.com/arbitrum/k%2Fey", + 16661: "https://rpc.ankr.com/0g_mainnet_evm/k%2Fey", + }) + svtCheckGenErrors(t, CreateAnkrVendor, []svtErrCase{ + {"missing apiKey", svtEvm(1), common.VendorSettings{}, "apiKey is required in ankr settings"}, + {"empty apiKey", svtEvm(1), common.VendorSettings{"apiKey": ""}, "apiKey is required in ankr settings"}, + {"nil evm", &common.UpstreamConfig{}, key, "ankr vendor requires upstream.evm to be defined"}, + {"zero chain", svtEvm(0), key, "ankr vendor requires upstream.evm.chainId to be defined"}, + {"unsupported chain", svtEvm(999999999), key, "unsupported network chain ID for Ankr: 999999999"}, + }) + cfg := svtCheckPreset(t, CreateAnkrVendor(), "https://rpc.ankr.com/eth/own") + assert.Empty(t, cfg.Type, "preset endpoint keeps caller's type") +} + +func TestAnkrVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + svtCheckDataOnly(t, CreateAnkrVendor()) +} + +func TestAnkrVendor_OwnsUpstream(t *testing.T) { + svtAssertOwns(t, CreateAnkrVendor(), map[string]bool{ + "ankr://key": true, + "evm+ankr://key": true, + "https://rpc.ankr.com/eth/key": true, + "https://eth.llamarpc.com/key": false, + }) +} diff --git a/thirdparty/blastapi_test.go b/thirdparty/blastapi_test.go new file mode 100644 index 000000000..3ef9268b5 --- /dev/null +++ b/thirdparty/blastapi_test.go @@ -0,0 +1,54 @@ +package thirdparty + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" +) + +func TestBlastApiVendor_Name(t *testing.T) { + assert.Equal(t, "blastapi", CreateBlastApiVendor().Name()) +} + +func TestBlastApiVendor_SupportsNetwork(t *testing.T) { + svtCheckSupports(t, CreateBlastApiVendor(), common.VendorSettings{}, []svtSupportCase{ + {"evm:1", true, false}, + {"evm:11297108109", true, false}, + {"evm:999999999", false, false}, + {"solana:mainnet", false, false}, + {"evm:abc", false, true}, + }) +} + +func TestBlastApiVendor_GenerateConfigs(t *testing.T) { + key := common.VendorSettings{"apiKey": "KEY"} + svtCheckURLs(t, CreateBlastApiVendor, key, map[int64]string{ + 1: "https://eth-mainnet.blastapi.io/KEY", + 8453: "https://base-mainnet.blastapi.io/KEY", + 43114: "https://ava-mainnet.blastapi.io/KEY/ext/bc/C/rpc", + 43113: "https://ava-testnet.blastapi.io/KEY/ext/bc/C/rpc", + }) + svtCheckGenErrors(t, CreateBlastApiVendor, []svtErrCase{ + {"missing apiKey", svtEvm(1), common.VendorSettings{}, "apiKey is required in blastapi settings"}, + {"non-string apiKey", svtEvm(1), common.VendorSettings{"apiKey": 42}, "apiKey is required in blastapi settings"}, + {"nil evm", &common.UpstreamConfig{}, key, "blastapi vendor requires upstream.evm to be defined"}, + {"zero chain", svtEvm(0), key, "blastapi vendor requires upstream.evm.chainId to be defined"}, + {"unsupported chain", svtEvm(999999999), key, "unsupported network chain ID for BlastAPI: 999999999"}, + {"unparsable url", svtEvm(1), common.VendorSettings{"apiKey": "%zz"}, "invalid URL escape"}, + }) + svtCheckPreset(t, CreateBlastApiVendor(), "https://eth-mainnet.blastapi.io/own") +} + +func TestBlastApiVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + svtCheckDataOnly(t, CreateBlastApiVendor()) +} + +func TestBlastApiVendor_OwnsUpstream(t *testing.T) { + svtAssertOwns(t, CreateBlastApiVendor(), map[string]bool{ + "blastapi://key": true, + "evm+blastapi://key": true, + "https://eth-mainnet.blastapi.io/key": true, + "https://rpc.ankr.com/eth/key": false, + }) +} diff --git a/thirdparty/blockdaemon_test.go b/thirdparty/blockdaemon_test.go index c0b3c314a..40fa2ba4d 100644 --- a/thirdparty/blockdaemon_test.go +++ b/thirdparty/blockdaemon_test.go @@ -7,6 +7,7 @@ import ( "github.com/erpc/erpc/common" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestBlockdaemonVendor_SupportsNetwork(t *testing.T) { @@ -128,3 +129,45 @@ func TestBlockdaemonVendor_OwnsUpstream(t *testing.T) { }) } } + +func TestBlockdaemonVendor_Name(t *testing.T) { + assert.Equal(t, "blockdaemon", CreateBlockdaemonVendor().Name()) +} + +func TestBlockdaemonVendor_SupportsNetworkInvalidChain(t *testing.T) { + svtCheckSupports(t, CreateBlockdaemonVendor(), common.VendorSettings{}, []svtSupportCase{ + {"evm:abc", false, true}, + }) +} + +func TestBlockdaemonVendor_GenerateConfigsBranches(t *testing.T) { + key := common.VendorSettings{"apiKey": "k"} + svtCheckGenErrors(t, CreateBlockdaemonVendor, []svtErrCase{ + {"nil evm", &common.UpstreamConfig{}, key, "blockdaemon vendor requires upstream.evm to be defined"}, + {"zero chain", svtEvm(0), key, "blockdaemon vendor requires upstream.evm.chainId to be defined"}, + }) + + for _, h := range []string{"Authorization", "authorization"} { + ups := svtEvm(1) + ups.JsonRpc = &common.JsonRpcUpstreamConfig{Headers: map[string]string{h: "Bearer mine"}} + cfg, err := svtGenerate(t, CreateBlockdaemonVendor(), ups, key) + require.NoError(t, err) + assert.Equal(t, map[string]string{h: "Bearer mine"}, cfg.JsonRpc.Headers, "caller %s header wins", h) + } +} + +func TestBlockdaemonVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateBlockdaemonVendor() + + err, details := vendorErr(t, v, "eth_getLogs", 401, -32000, "invalid token", "d") + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointUnauthorized), "got %v", err) + assert.Equal(t, common.JsonRpcErrorUnauthorized, normalizedCode(t, err)) + assert.Equal(t, "d", details["data"]) + + svtCheckDataOnly(t, v) + + jrr, jerr := common.NewJsonRpcResponse(1, nil, common.NewErrJsonRpcExceptionExternal(-32000, "x", "")) + require.NoError(t, jerr) + assert.NoError(t, v.GetVendorSpecificErrorIfAny(svtReq(), nil, jrr, map[string]interface{}{}), "nil response is generic") +} diff --git a/thirdparty/blockpi_test.go b/thirdparty/blockpi_test.go new file mode 100644 index 000000000..26c1aabe1 --- /dev/null +++ b/thirdparty/blockpi_test.go @@ -0,0 +1,69 @@ +package thirdparty + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBlockPiVendor_Name(t *testing.T) { + assert.Equal(t, "blockpi", CreateBlockPiVendor().Name()) +} + +func TestBlockPiVendor_SupportsNetwork(t *testing.T) { + svtCheckSupports(t, CreateBlockPiVendor(), common.VendorSettings{}, []svtSupportCase{ + {"evm:1", true, false}, + {"evm:126", true, false}, + {"evm:999999999", false, false}, + {"solana:mainnet", false, false}, + {"evm:abc", false, true}, + }) +} + +func TestBlockPiVendor_GenerateConfigs(t *testing.T) { + key := common.VendorSettings{"apiKey": "k y"} + svtCheckURLs(t, CreateBlockPiVendor, key, map[int64]string{ + 1: "https://ethereum.blockpi.network/v1/rpc/k+y", + 42161: "https://arbitrum.blockpi.network/v1/rpc/k+y", + 126: "https://movement.blockpi.network/rpc/v1/k+y/v1", + }) + svtCheckGenErrors(t, CreateBlockPiVendor, []svtErrCase{ + {"missing apiKey", svtEvm(1), common.VendorSettings{}, "apiKey is required in BlockPi settings"}, + {"nil evm", &common.UpstreamConfig{}, key, "BlockPi vendor requires upstream.evm to be defined"}, + {"zero chain", svtEvm(0), key, "BlockPi vendor requires upstream.evm.chainId to be defined"}, + {"unsupported chain", svtEvm(999999999), key, "unsupported network chain ID for BlockPi: 999999999"}, + }) + svtCheckPreset(t, CreateBlockPiVendor(), "https://ethereum.blockpi.network/v1/rpc/own") +} + +func TestBlockPiVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateBlockPiVendor() + + for _, msg := range []string{"ApiKey is on another chain", "this API key is on another chain"} { + err, details := vendorErr(t, v, "eth_getLogs", 400, -32000, msg, "d") + require.Error(t, err, msg) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointUnauthorized), "%q -> %v", msg, err) + assert.Equal(t, common.JsonRpcErrorUnauthorized, normalizedCode(t, err)) + assert.Contains(t, err.Error(), msg, "original message is preserved") + assert.Equal(t, "d", details["data"]) + } + + // Other messages (and empty) fall through to generic handling. + for _, msg := range []string{"rate limited", ""} { + err, _ := vendorErr(t, v, "eth_getLogs", 400, -32000, msg, "") + assert.NoError(t, err, "%q", msg) + } + + svtCheckDataOnly(t, v) +} + +func TestBlockPiVendor_OwnsUpstream(t *testing.T) { + svtAssertOwns(t, CreateBlockPiVendor(), map[string]bool{ + "blockpi://key": true, + "evm+blockpi://key": true, + "https://ethereum.blockpi.network/v1/rpc/key": true, + "https://rpc.ankr.com/eth/key": false, + }) +} diff --git a/thirdparty/chainstack_test.go b/thirdparty/chainstack_test.go index c85bfb3f3..a72c1c48c 100644 --- a/thirdparty/chainstack_test.go +++ b/thirdparty/chainstack_test.go @@ -34,6 +34,9 @@ func TestChainstackVendor_GenerateConfigs(t *testing.T) { ChainId: 123, }, } + // Hold the refresh slot so the cold path never dials the real API + // (an escaped goroutine would race gock-based tests). + vendor.(*ChainstackVendor).cache.inflight["test-api-key"] = struct{}{} configs, err := vendor.GenerateConfigs(ctx, &logger, upstream, settings) @@ -91,6 +94,7 @@ func TestChainstackVendor_GenerateConfigs(t *testing.T) { settings := common.VendorSettings{ "apiKey": "test-api-key", } + freshVendor.(*ChainstackVendor).cache.inflight["test-api-key"] = struct{}{} supported, err := freshVendor.SupportsNetwork(ctx, &logger, settings, "evm:123") assert.Error(t, err) diff --git a/thirdparty/chainstack_vendor_test.go b/thirdparty/chainstack_vendor_test.go new file mode 100644 index 000000000..a69c693b3 --- /dev/null +++ b/thirdparty/chainstack_vendor_test.go @@ -0,0 +1,236 @@ +package thirdparty + +import ( + "context" + "io" + "net/http" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestChainstackVendor_FetchNodes(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("follows next links, forwards filters and skips invalid nodes", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + + next := "https://api.chainstack.com/v1/nodes/?page=2" + gock.New("https://api.chainstack.com"). + Get("/v1/nodes/"). + MatchHeader("Authorization", "^Bearer cs-key$"). + MatchParam("project", "^p1$"). + MatchParam("organization", "^o1$"). + MatchParam("region", "^r1$"). + MatchParam("provider", "^aws$"). + MatchParam("type", "^dedicated$"). + Reply(200). + JSON(map[string]interface{}{ + "next": next, + "results": []interface{}{ + map[string]interface{}{"id": "n1", "status": "running", "details": map[string]interface{}{"https_endpoint": "https://n1.core.chainstack.com", "auth_key": "a1"}}, + map[string]interface{}{"id": "noendpoint", "status": "running", "details": map[string]interface{}{}}, + map[string]interface{}{"id": 5, "status": "running"}, + }, + }) + gock.New("https://api.chainstack.com"). + Get("/v1/nodes/"). + MatchParam("page", "^2$"). + Reply(200). + JSON(map[string]interface{}{ + "next": nil, + "results": []interface{}{map[string]interface{}{"id": "n2", "status": "stopped", "details": map[string]interface{}{"https_endpoint": "https://n2.core.chainstack.com"}}}, + }) + + v := CreateChainstackVendor().(*ChainstackVendor) + got, err := v.fetchNodes(ctx, &logger, "cs-key", &ChainstackFilterParams{ + Project: "p1", Organization: "o1", Region: "r1", Provider: "aws", Type: "dedicated", + }) + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "n1", got[0].ID) + assert.Equal(t, "a1", got[0].Details.AuthKey) + assert.Equal(t, "n2", got[1].ID) + assert.Equal(t, "stopped", got[1].Status) + assert.True(t, gock.IsDone()) + }) + + errCases := []struct { + name string + status int + body string + wantErr string + }{ + {"non-200 status", http.StatusForbidden, `denied`, "status 403: denied"}, + {"undecodable body", http.StatusOK, `{"results":`, "failed to decode Chainstack nodes response"}, + } + for _, c := range errCases { + t.Run(c.name, func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://api.chainstack.com").Get("/v1/nodes/").Reply(c.status).BodyString(c.body) + + v := CreateChainstackVendor().(*ChainstackVendor) + _, err := v.fetchNodes(ctx, &logger, "k", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), c.wantErr) + }) + } + + t.Run("transport failure is an error", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://api.chainstack.com").Get("/v1/nodes/").ReplyError(io.ErrUnexpectedEOF) + + v := CreateChainstackVendor().(*ChainstackVendor) + _, err := v.fetchNodes(ctx, &logger, "k", &ChainstackFilterParams{}) + require.Error(t, err) + }) + + t.Run("malformed next link is an error", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://api.chainstack.com").Get("/v1/nodes/").Reply(200). + JSON(map[string]interface{}{"next": "http://bad host/", "results": []interface{}{}}) + + v := CreateChainstackVendor().(*ChainstackVendor) + _, err := v.fetchNodes(ctx, &logger, "k", nil) + require.Error(t, err) + }) +} + +func TestChainstackVendor_FetchChainIDs(t *testing.T) { + logger := zerolog.Nop() + srv := newChainIdRouter(t, map[string]string{"/eth/k1": "0x1", "/arb/k2": "0xa4b1", "/bad/k": "0xqq", "/html/k": "!"}) + + nodes := []*ChainstackNode{ + {ID: "eth", Status: "running", Details: ChainstackNodeDetails{HTTPSEndpoint: srv.URL + "/eth", AuthKey: "k1"}}, + {ID: "arb", Status: "running", Details: ChainstackNodeDetails{HTTPSEndpoint: srv.URL + "/arb", AuthKey: "k2"}}, + {ID: "rpcerr", Status: "running", Details: ChainstackNodeDetails{HTTPSEndpoint: srv.URL + "/none", AuthKey: "k"}}, + {ID: "badhex", Status: "running", Details: ChainstackNodeDetails{HTTPSEndpoint: srv.URL + "/bad", AuthKey: "k"}}, + {ID: "html", Status: "running", Details: ChainstackNodeDetails{HTTPSEndpoint: srv.URL + "/html", AuthKey: "k"}}, + {ID: "down", Status: "running", Details: ChainstackNodeDetails{HTTPSEndpoint: "http://127.0.0.1:1"}}, + {ID: "badurl", Status: "running", Details: ChainstackNodeDetails{HTTPSEndpoint: "http://bad host"}}, + {ID: "stopped", Status: "stopped", Details: ChainstackNodeDetails{HTTPSEndpoint: srv.URL + "/eth", AuthKey: "k1"}}, + {ID: "noendpoint", Status: "running"}, + } + v := CreateChainstackVendor().(*ChainstackVendor) + require.NoError(t, v.fetchChainIDs(context.Background(), &logger, nodes)) + + got := map[string]int64{} + for _, n := range nodes { + got[n.ID] = n.ChainID + } + assert.Equal(t, map[string]int64{ + "eth": 1, "arb": 42161, "rpcerr": 0, "badhex": 0, "html": 0, "down": 0, "badurl": 0, "stopped": 0, "noendpoint": 0, + }, got, "only running nodes with an endpoint are probed") + + t.Run("cancelled context fails every acquisition", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + ns := []*ChainstackNode{{ID: "eth", Status: "running", Details: ChainstackNodeDetails{HTTPSEndpoint: srv.URL + "/eth", AuthKey: "k1"}}} + require.NoError(t, v.fetchChainIDs(ctx, &logger, ns)) + assert.Zero(t, ns[0].ChainID) + }) +} + +func TestChainstackVendor_DiscoveryLifecycle(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + + ctx := context.Background() + logger := zerolog.Nop() + probe := newChainIdRouter(t, map[string]string{"/a/ka": "0x1", "/b/kb": "0x1", "/c/kc": "0x89"}) + + gock.New("https://api.chainstack.com"). + Get("/v1/nodes/"). + MatchParam("project", "^proj$"). + Persist(). + Reply(200). + JSON(map[string]interface{}{"next": nil, "results": []interface{}{ + map[string]interface{}{"id": "a", "status": "running", "details": map[string]interface{}{"https_endpoint": probe.URL + "/a", "auth_key": "ka"}}, + map[string]interface{}{"id": "b", "status": "running", "details": map[string]interface{}{"https_endpoint": probe.URL + "/b", "auth_key": "kb"}}, + map[string]interface{}{"id": "c", "status": "running", "details": map[string]interface{}{"https_endpoint": probe.URL + "/c", "auth_key": "kc"}}, + }}) + + v := CreateChainstackVendor() + settings := common.VendorSettings{"apiKey": "cs", "project": "proj", "recheckInterval": time.Hour} + + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:1") + assert.ErrorIs(t, err, ErrRemoteCacheCold) + assert.False(t, ok) + + require.Eventually(t, func() bool { + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:1") + return err == nil && ok + }, 5*time.Second, 10*time.Millisecond) + + ok, err = v.SupportsNetwork(ctx, &logger, settings, "evm:10") + require.NoError(t, err) + assert.False(t, ok) + + _, err = v.SupportsNetwork(ctx, &logger, settings, "evm:zz") + assert.Error(t, err) + + got, err := v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Id: "cs", Evm: &common.EvmUpstreamConfig{ChainId: 1}}, settings) + require.NoError(t, err) + require.Len(t, got, 2) + assert.ElementsMatch(t, []string{"cs-a", "cs-b"}, []string{got[0].Id, got[1].Id}) + assert.ElementsMatch(t, []string{probe.URL + "/a/ka", probe.URL + "/b/kb"}, []string{got[0].Endpoint, got[1].Endpoint}) + for _, c := range got { + assert.Equal(t, common.UpstreamTypeEvm, c.Type) + require.NotNil(t, c.JsonRpc) + assert.NotNil(t, c.JsonRpc.Headers) + } + + got, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 137}}, settings) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "chainstack-137-c", got[0].Id) +} + +func TestChainstackVendor_GenerateConfigs_Validation(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + v := CreateChainstackVendor().(*ChainstackVendor) + + _, err := v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{}, common.VendorSettings{"apiKey": "k"}) + assert.ErrorContains(t, err, "requires upstream.evm to be defined") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{}}, common.VendorSettings{"apiKey": "k"}) + assert.ErrorContains(t, err, "requires upstream.evm.chainId") + + // Mark discovery as in flight so the cold path never dials out. + v.cache.inflight["cold"] = struct{}{} + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}}, common.VendorSettings{"apiKey": "cold", "recheckInterval": time.Minute}) + assert.ErrorIs(t, err, ErrRemoteCacheCold) + + _, err = v.SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "cold", "recheckInterval": time.Minute}, "evm:1") + assert.ErrorIs(t, err, ErrRemoteCacheCold) +} + +func TestChainstackVendor_OwnershipAndErrors(t *testing.T) { + v := CreateChainstackVendor() + assert.Equal(t, "chainstack", v.Name()) + for ep, want := range map[string]bool{ + "chainstack://k": true, + "evm+chainstack://k": true, + "https://nd-1.ethereum-mainnet.core.chainstack.com/abc": true, + "https://rpc.ankr.com/eth": false, + } { + assert.Equal(t, want, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: ep}), ep) + } + + err, details := vendorErr(t, v, "eth_call", 500, -32000, "boom", "0xbeef") + assert.NoError(t, err, "chainstack defers to generic handling") + assert.Equal(t, "0xbeef", details["data"]) + assert.NoError(t, v.GetVendorSpecificErrorIfAny(nil, nil, "raw", map[string]interface{}{})) +} diff --git a/thirdparty/conduit_test.go b/thirdparty/conduit_test.go new file mode 100644 index 000000000..6fe7d600e --- /dev/null +++ b/thirdparty/conduit_test.go @@ -0,0 +1,208 @@ +package thirdparty + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const conduitNetworksBody = `{"endpoints":[ + {"id":"a","name":"Zora","chainId":"7777777","httpEndpoint":"https://rpc.zora.conduit.example"}, + {"id":"b","name":"NoHttp","chainId":"42","httpEndpoint":""}, + {"id":"c","name":"Bad","chainId":"not-a-number","httpEndpoint":"https://bad.example"}, + {"id":"d","name":"Zero","chainId":"0","httpEndpoint":"https://zero.example"} +]}` + +func TestConduitVendor_FetchNetworks(t *testing.T) { + v := CreateConduitVendor().(*ConduitVendor) + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("keeps networks with valid chain id and http endpoint", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, conduitNetworksBody) + got, err := v.fetchConduitNetworks(ctx, &logger, srv.URL) + require.NoError(t, err) + require.Len(t, got, 1) + require.Contains(t, got, int64(7777777)) + assert.Equal(t, "https://rpc.zora.conduit.example", got[7777777].HttpEndpoint) + }) + + t.Run("non-2xx status is an error", func(t *testing.T) { + srv := newJSONServer(t, http.StatusServiceUnavailable, `{}`) + _, err := v.fetchConduitNetworks(ctx, &logger, srv.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "503") + }) + + t.Run("invalid json is an error", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `[1,2`) + _, err := v.fetchConduitNetworks(ctx, &logger, srv.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse Conduit API data") + }) + + t.Run("invalid url and unreachable host are errors", func(t *testing.T) { + _, err := v.fetchConduitNetworks(ctx, &logger, "http://bad host/") + assert.Error(t, err) + _, err = v.fetchConduitNetworks(ctx, &logger, "http://127.0.0.1:1/") + assert.Error(t, err) + }) +} + +func TestConduitVendor_SupportsNetwork(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("rejects non-evm and malformed ids", func(t *testing.T) { + v := CreateConduitVendor() + ok, err := v.SupportsNetwork(ctx, &logger, nil, "solana:mainnet") + assert.NoError(t, err) + assert.False(t, ok) + _, err = v.SupportsNetwork(ctx, &logger, nil, "evm:x") + assert.Error(t, err) + }) + + t.Run("cold cache then populated from networks url", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, conduitNetworksBody) + v := CreateConduitVendor() + settings := common.VendorSettings{"networksUrl": srv.URL, "recheckInterval": time.Hour} + + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:7777777") + assert.ErrorIs(t, err, ErrRemoteCacheCold) + assert.False(t, ok) + + require.Eventually(t, func() bool { + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:7777777") + return err == nil && ok + }, 5*time.Second, 10*time.Millisecond) + + ok, err = v.SupportsNetwork(ctx, &logger, settings, "evm:42") + assert.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("default networks url is used when unset", func(t *testing.T) { + v := CreateConduitVendor().(*ConduitVendor) + primeRemoteCache(t, v.cache, DefaultConduitNetworksUrl, map[int64]*ConduitNetwork{ + 1: {ChainID: "1", HttpEndpoint: "https://one.example"}, + 2: nil, + }) + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:1") + require.NoError(t, err) + assert.True(t, ok) + ok, err = v.SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:2") + require.NoError(t, err) + assert.False(t, ok, "nil network entry is unsupported") + }) +} + +func TestConduitVendor_GenerateConfigs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + const networksUrl = "https://conduit.example/networks" + + newPrimed := func(t *testing.T) *ConduitVendor { + v := CreateConduitVendor().(*ConduitVendor) + primeRemoteCache(t, v.cache, networksUrl, map[int64]*ConduitNetwork{ + 7777777: {ChainID: "7777777", HttpEndpoint: "https://rpc.zora.conduit.example"}, + }) + return v + } + + t.Run("appends api key to the network endpoint", func(t *testing.T) { + v := newPrimed(t) + ups := &common.UpstreamConfig{Id: "zora", Evm: &common.EvmUpstreamConfig{ChainId: 7777777}} + got, err := v.GenerateConfigs(ctx, &logger, ups, common.VendorSettings{ + "apiKey": "k1", "networksUrl": networksUrl, "recheckInterval": time.Hour, + }) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "https://rpc.zora.conduit.example/k1", got[0].Endpoint) + assert.Equal(t, common.UpstreamTypeEvm, got[0].Type) + assert.Equal(t, "conduit", got[0].VendorName) + assert.Equal(t, "zora", got[0].Id) + assert.Empty(t, ups.Endpoint, "input upstream is not mutated with the endpoint") + }) + + t.Run("preset endpoint passes through", func(t *testing.T) { + v := CreateConduitVendor() + ups := &common.UpstreamConfig{Endpoint: "https://rpc.custom.example/key"} + got, err := v.GenerateConfigs(ctx, &logger, ups, nil) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Same(t, ups, got[0]) + assert.NotNil(t, got[0].JsonRpc) + }) + + t.Run("validation errors", func(t *testing.T) { + v := newPrimed(t) + ok := common.VendorSettings{"apiKey": "k", "networksUrl": networksUrl} + + _, err := v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{}, ok) + assert.ErrorContains(t, err, "requires upstream.evm to be defined") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{}}, ok) + assert.ErrorContains(t, err, "requires upstream.evm.chainId") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 7777777}}, nil) + assert.ErrorContains(t, err, "apiKey is required") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 5}}, ok) + assert.ErrorContains(t, err, "chain ID 5 not found") + }) + + t.Run("cold cache is retryable", func(t *testing.T) { + v := CreateConduitVendor() + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}} + _, err := v.GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"apiKey": "k", "networksUrl": "http://127.0.0.1:1/"}) + assert.ErrorIs(t, err, ErrRemoteCacheCold) + }) +} + +func TestConduitVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateConduitVendor() + + cases := []struct { + name string + code int + msg string + want common.ErrorCode + }{ + {"unauthenticated", -32600, "request must be authenticated", common.ErrCodeEndpointUnauthorized}, + {"bad access key", -32600, "invalid access key", common.ErrCodeEndpointUnauthorized}, + {"bad api key", -32600, "missing api key", common.ErrCodeEndpointUnauthorized}, + {"limit exceeded", -32005, "rate limit exceeded", common.ErrCodeEndpointCapacityExceeded}, + {"capacity limit", 429, "capacity limit reached", common.ErrCodeEndpointCapacityExceeded}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err, details := vendorErr(t, v, "eth_call", 500, c.code, c.msg, "extra") + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, c.want), "got %v", err) + assert.Equal(t, "extra", details["data"]) + }) + } + + t.Run("unrelated errors fall through", func(t *testing.T) { + err, _ := vendorErr(t, v, "eth_call", 400, -32600, "invalid request", "") + assert.NoError(t, err) + err, _ = vendorErr(t, v, "eth_call", 400, 3, "execution reverted", "") + assert.NoError(t, err) + assert.NoError(t, v.GetVendorSpecificErrorIfAny(nil, nil, "not a jrr", map[string]interface{}{})) + }) +} + +func TestConduitVendor_OwnsUpstream(t *testing.T) { + v := CreateConduitVendor() + assert.Equal(t, "conduit", v.Name()) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "conduit://x"})) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "evm+conduit://x"})) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://a.example", VendorName: "conduit"})) + assert.False(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://a.example"})) +} diff --git a/thirdparty/drpc_test.go b/thirdparty/drpc_test.go index 537e6e357..d204f9550 100644 --- a/thirdparty/drpc_test.go +++ b/thirdparty/drpc_test.go @@ -125,3 +125,211 @@ func swapDrpcNetworksURL(t *testing.T, newURL string) string { drpcNetworksURL = newURL return prev } + +// drpcOfflineURL refuses connections instantly so the async refresh kicked +// off by SupportsNetwork / GenerateConfigs never leaves the machine. +const drpcOfflineURL = "http://127.0.0.1:1/drpc-offline" + +func TestDrpcVendor_Name(t *testing.T) { + assert.Equal(t, "drpc", CreateDrpcVendor().Name()) +} + +func TestDrpcVendor_CreditUnits(t *testing.T) { + v := CreateDrpcVendor().(*DrpcVendor) + assert.Equal(t, int64(0), v.CreditUnits(cuReq(t, "eth_chainId"), nil), "informational method is free") + assert.Equal(t, int64(20), v.CreditUnits(cuReq(t, "eth_getLogs"), nil), "unlisted method uses '*'") + assert.Equal(t, int64(20), v.CreditUnits(cuReq(t, "eth_getLogs"), &common.UpstreamConfig{}), "no override uses '*'") + + ups := &common.UpstreamConfig{CreditUnits: map[string]int64{"eth_getLogs": 7, "eth_chainId": 3}} + assert.Equal(t, int64(7), v.CreditUnits(cuReq(t, "eth_getLogs"), ups), "override wins over '*'") + assert.Equal(t, int64(3), v.CreditUnits(cuReq(t, "eth_chainId"), ups), "override wins over listed method") + assert.Equal(t, int64(20), v.CreditUnits(cuReq(t, "eth_call"), ups), "non-overridden method keeps '*'") +} + +func TestDrpcVendor_SupportsNetworkBranches(t *testing.T) { + offline := common.VendorSettings{"chainsUrl": drpcOfflineURL} + svtCheckSupports(t, CreateDrpcVendor(), offline, []svtSupportCase{ + {"solana:mainnet", false, false}, + {"evm:abc", false, true}, + {"evm:1", true, false}, // default recheck interval + built-in fallback + }) + + prev := swapDrpcNetworksURL(t, drpcOfflineURL) + defer swapDrpcNetworksURL(t, prev) + ok, err := svtSupports(t, CreateDrpcVendor(), common.VendorSettings{}, "evm:8453") + require.NoError(t, err) + assert.True(t, ok, "empty settings use drpcNetworksURL with built-in fallback") +} + +func TestDrpcVendor_GenerateConfigsBranches(t *testing.T) { + key := common.VendorSettings{"apiKey": "KEY", "chainsUrl": drpcOfflineURL} + + svtCheckGenErrors(t, CreateDrpcVendor, []svtErrCase{ + {"missing apiKey", svtEvm(1), common.VendorSettings{"chainsUrl": drpcOfflineURL}, "apiKey is required in drpc settings"}, + {"nil evm", &common.UpstreamConfig{}, key, "drpc vendor requires upstream.evm to be defined"}, + {"zero chain", svtEvm(0), key, "drpc vendor requires upstream.evm.chainId to be defined"}, + {"invalid chainsUrl", svtEvm(1), common.VendorSettings{"apiKey": "KEY", "chainsUrl": "ftp://host"}, "invalid chainsUrl"}, + {"unsupported chain", svtEvm(999999999999), key, "unsupported network chain ID for DRPC: 999999999999"}, + {"unparsable url", svtEvm(1), common.VendorSettings{"apiKey": "a\nb", "chainsUrl": drpcOfflineURL}, "invalid control character"}, + }) + + t.Run("built-in fallback url", func(t *testing.T) { + cfg, err := svtGenerate(t, CreateDrpcVendor(), svtEvm(42161), key) + require.NoError(t, err) + assert.Equal(t, "https://lb.drpc.org/ogrpc?network=arbitrum&dkey=KEY", cfg.Endpoint) + assert.Equal(t, common.UpstreamTypeEvm, cfg.Type) + require.NotNil(t, cfg.AutoIgnoreUnsupportedMethods) + assert.False(t, *cfg.AutoIgnoreUnsupportedMethods) + }) + + t.Run("default chainsUrl", func(t *testing.T) { + prev := swapDrpcNetworksURL(t, drpcOfflineURL) + defer swapDrpcNetworksURL(t, prev) + cfg, err := svtGenerate(t, CreateDrpcVendor(), svtEvm(1), common.VendorSettings{"apiKey": "KEY"}) + require.NoError(t, err) + assert.Equal(t, "https://lb.drpc.org/ogrpc?network=ethereum&dkey=KEY", cfg.Endpoint) + }) + + t.Run("fetched snapshot wins over built-in", func(t *testing.T) { + v := CreateDrpcVendor().(*DrpcVendor) + primeRemoteCache(t, v.cache, drpcOfflineURL, map[int64]string{1: "eth-custom"}) + cfg, err := svtGenerate(t, v, svtEvm(1), common.VendorSettings{"apiKey": "KEY", "chainsUrl": drpcOfflineURL, "recheckInterval": time.Hour}) + require.NoError(t, err) + assert.Equal(t, "https://lb.drpc.org/ogrpc?network=eth-custom&dkey=KEY", cfg.Endpoint) + }) + + t.Run("preset owned endpoint gets evm type", func(t *testing.T) { + cfg, err := svtGenerate(t, CreateDrpcVendor(), &common.UpstreamConfig{Endpoint: "https://lb.drpc.org/ogrpc?network=base&dkey=x"}, common.VendorSettings{}) + require.NoError(t, err) + assert.Equal(t, "https://lb.drpc.org/ogrpc?network=base&dkey=x", cfg.Endpoint) + assert.Equal(t, common.UpstreamTypeEvm, cfg.Type) + }) + + t.Run("preset owned endpoint keeps explicit type", func(t *testing.T) { + ups := &common.UpstreamConfig{Endpoint: "drpc://KEY", Type: "custom"} + cfg, err := svtGenerate(t, CreateDrpcVendor(), ups, common.VendorSettings{}) + require.NoError(t, err) + assert.Equal(t, common.UpstreamType("custom"), cfg.Type) + }) + + t.Run("preset foreign endpoint stays untyped", func(t *testing.T) { + cfg, err := svtGenerate(t, CreateDrpcVendor(), &common.UpstreamConfig{Endpoint: "https://rpc.ankr.com/eth/x"}, common.VendorSettings{}) + require.NoError(t, err) + assert.Empty(t, cfg.Type) + }) +} + +func TestDrpcVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateDrpcVendor() + + err, details := vendorErr(t, v, "eth_getLogs", 400, -32000, "token is invalid", "d") + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointUnauthorized), "got %v", err) + assert.Equal(t, common.JsonRpcErrorUnauthorized, normalizedCode(t, err)) + assert.Equal(t, "d", details["data"]) + + for _, msg := range []string{"ChainException: Unexpected error (code=40000)", "invalid block range"} { + err, _ := vendorErr(t, v, "eth_getLogs", 400, -32000, msg, "") + require.Error(t, err, msg) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointMissingData), "%q -> %v", msg, err) + assert.Equal(t, common.JsonRpcErrorMissingData, normalizedCode(t, err)) + assert.True(t, common.IsRetryableTowardNetwork(err), "missing data stays network-retryable") + } + + err, details = vendorErr(t, v, "eth_getLogs", 400, -32000, "other", "d") + assert.NoError(t, err) + assert.Equal(t, "d", details["data"]) + + err, details = vendorErr(t, v, "eth_getLogs", 400, 0, "token is invalid", "d") + assert.NoError(t, err, "code 0 is not inspected") + assert.NotContains(t, details, "data") + + assert.NoError(t, v.GetVendorSpecificErrorIfAny(svtReq(), nil, "x", map[string]interface{}{})) +} + +func TestDrpcVendor_OwnsUpstream(t *testing.T) { + svtAssertOwns(t, CreateDrpcVendor(), map[string]bool{ + "drpc://key": true, + "evm+drpc://key": true, + "https://lb.drpc.org/ogrpc?network=ethereum&dkey=key": true, + "https://rpc.ankr.com/eth/key": false, + }) +} + +func TestDrpcVendor_FetchDrpcNetworks(t *testing.T) { + v := CreateDrpcVendor().(*DrpcVendor) + logger := zerolog.Nop() + ctx := context.Background() + + t.Run("filters and picks highest priority", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `[ + {"id":"a","chains":[ + {"name":"eth-low","chain_id":"0x1","priority":1,"api_type":"jsonrpc","blockchain_type":"eth","has_premium":true}, + {"name":"eth-high","chain_id":"0x1","priority":9,"api_type":"jsonrpc","blockchain_type":"eth","has_premium":true}, + {"name":"eth-mid","chain_id":"0x1","priority":5,"api_type":"jsonrpc","blockchain_type":"eth","has_premium":true}, + {"name":"base","chain_id":"2105","priority":1,"api_type":"jsonrpc","blockchain_type":"eth","has_premium":true} + ]}, + {"id":"b","chains":[ + {"name":"bad-id","chain_id":"0xzz","priority":1,"api_type":"jsonrpc","blockchain_type":"eth","has_premium":true}, + {"name":"no-id","chain_id":"","priority":1,"api_type":"jsonrpc","blockchain_type":"eth","has_premium":true}, + {"name":"free","chain_id":"0xa","priority":1,"api_type":"jsonrpc","blockchain_type":"eth","has_premium":false}, + {"name":"rest","chain_id":"0x89","priority":1,"api_type":"rest","blockchain_type":"eth","has_premium":true}, + {"name":"sol","chain_id":"0x65","priority":1,"api_type":"jsonrpc","blockchain_type":"solana","has_premium":true} + ]} + ]`) + got, err := v.fetchDrpcNetworks(ctx, &logger, srv.URL) + require.NoError(t, err) + assert.Equal(t, map[int64]string{1: "eth-high", 0x2105: "base"}, got) + }) + + t.Run("non-2xx status", func(t *testing.T) { + srv := newJSONServer(t, http.StatusServiceUnavailable, `[]`) + _, err := v.fetchDrpcNetworks(ctx, &logger, srv.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "non-200 code: 503") + }) + + t.Run("invalid json", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `{not json`) + _, err := v.fetchDrpcNetworks(ctx, &logger, srv.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse dRPC networks response") + }) + + t.Run("connection refused", func(t *testing.T) { + _, err := v.fetchDrpcNetworks(ctx, &logger, drpcOfflineURL) + assert.Error(t, err) + }) + + t.Run("malformed url", func(t *testing.T) { + _, err := v.fetchDrpcNetworks(ctx, &logger, "http://bad\nhost") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid control character") + }) +} + +func TestDrpcVendor_FetchErrorKeepsFallbackViaChainsUrl(t *testing.T) { + hit := make(chan struct{}, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + select { + case hit <- struct{}{}: + default: + } + })) + defer srv.Close() + + v := CreateDrpcVendor().(*DrpcVendor) + settings := common.VendorSettings{"chainsUrl": srv.URL, "recheckInterval": time.Hour} + ok, err := svtSupports(t, v, settings, "evm:1") + require.NoError(t, err) + assert.True(t, ok) + select { + case <-hit: + case <-time.After(5 * time.Second): + t.Fatal("async refresh never hit the chainsUrl server") + } + ok, err = svtSupports(t, v, settings, "evm:1") + require.NoError(t, err) + assert.True(t, ok, "failed refresh keeps serving the built-in fallback") +} diff --git a/thirdparty/dwellir_test.go b/thirdparty/dwellir_test.go new file mode 100644 index 000000000..2de081c8e --- /dev/null +++ b/thirdparty/dwellir_test.go @@ -0,0 +1,56 @@ +package thirdparty + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" +) + +func TestDwellirVendor_Name(t *testing.T) { + assert.Equal(t, "dwellir", CreateDwellirVendor().Name()) +} + +func TestDwellirVendor_SupportsNetwork(t *testing.T) { + svtCheckSupports(t, CreateDwellirVendor(), common.VendorSettings{}, []svtSupportCase{ + {"evm:1", true, false}, + {"evm:999999999", true, false}, + {"evm:123456789", false, false}, + {"solana:mainnet", false, false}, + {"evm:abc", false, false}, // dwellir swallows parse errors + }) +} + +func TestDwellirVendor_GenerateConfigs(t *testing.T) { + key := common.VendorSettings{"apiKey": "KEY"} + svtCheckURLs(t, CreateDwellirVendor, key, map[int64]string{ + 1: "https://api-ethereum-mainnet.n.dwellir.com/KEY", + 8453: "https://api-base-mainnet-archive.n.dwellir.com/KEY", + 43114: "https://api-avalanche-mainnet-archive.n.dwellir.com/KEY/ext/bc/C/rpc", + }) + svtCheckGenErrors(t, CreateDwellirVendor, []svtErrCase{ + {"missing apiKey", svtEvm(1), common.VendorSettings{}, "apiKey is required in dwellir provider settings"}, + {"nil evm", &common.UpstreamConfig{}, key, "dwellir vendor requires upstream.evm to be defined"}, + {"zero chain", svtEvm(0), key, "dwellir vendor requires upstream.evm.chainId to be defined"}, + {"unsupported chain", svtEvm(123456789), key, "unsupported network chain ID for Dwellir: 123456789"}, + {"unparsable url", svtEvm(1), common.VendorSettings{"apiKey": "%zz"}, "failed to parse generated Dwellir URL"}, + }) + cfg := svtCheckPreset(t, CreateDwellirVendor(), "https://api-ethereum-mainnet.n.dwellir.com/own") + assert.Empty(t, cfg.Type) +} + +func TestDwellirVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateDwellirVendor() + err, _ := vendorErr(t, v, "eth_getLogs", 401, -32000, "unauthorized", "d") + assert.NoError(t, err, "dwellir defers every error to generic handling") + assert.NoError(t, v.GetVendorSpecificErrorIfAny(svtReq(), nil, "x", map[string]interface{}{})) +} + +func TestDwellirVendor_OwnsUpstream(t *testing.T) { + svtAssertOwns(t, CreateDwellirVendor(), map[string]bool{ + "dwellir://key": true, + "https://api-ethereum-mainnet.n.dwellir.com/key": true, + "evm+dwellir://key": false, + "https://rpc.ankr.com/eth/key": false, + }) +} diff --git a/thirdparty/envio_test.go b/thirdparty/envio_test.go new file mode 100644 index 000000000..eedde1523 --- /dev/null +++ b/thirdparty/envio_test.go @@ -0,0 +1,180 @@ +package thirdparty + +import ( + "context" + "errors" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const envioUnknownChain = "evm:999888" // 0xf41d0, not in envioKnownSupportedChains + +func TestEnvioVendor_SupportsNetwork(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("known chain short-circuits without probe", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://1.rpc.hypersync.xyz").Persist().ReplyError(assert.AnError) + ok, err := CreateEnvioVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:1") + require.NoError(t, err) + assert.True(t, ok) + }) + + probeCases := []struct { + name string + settings common.VendorSettings + host string + path string + body string + want bool + wantErr bool + }{ + {"default root domain with api key", common.VendorSettings{"apiKey": "k1"}, "https://999888.rpc.hypersync.xyz", "/k1", `{"jsonrpc":"2.0","id":1,"result":"0xf41d0"}`, true, false}, + {"custom root domain without api key", common.VendorSettings{"rootDomain": "rpc.example.dev"}, "https://999888.rpc.example.dev", "", `{"jsonrpc":"2.0","id":1,"result":"0xf41d0"}`, true, false}, + {"mismatching chain id", common.VendorSettings{"apiKey": "k1"}, "https://999888.rpc.hypersync.xyz", "/k1", `{"jsonrpc":"2.0","id":1,"result":"0x1"}`, false, false}, + {"json-rpc error", common.VendorSettings{"apiKey": "k1"}, "https://999888.rpc.hypersync.xyz", "/k1", `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"boom"}}`, false, true}, + {"non-hex result", common.VendorSettings{"apiKey": "k1"}, "https://999888.rpc.hypersync.xyz", "/k1", `{"jsonrpc":"2.0","id":1,"result":"qq"}`, false, true}, + {"non-string result", common.VendorSettings{"apiKey": "k1"}, "https://999888.rpc.hypersync.xyz", "/k1", `{"jsonrpc":"2.0","id":1,"result":[]}`, false, true}, + } + for _, c := range probeCases { + t.Run(c.name, func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gockProbeReply(c.host, c.path, c.body) + + ok, err := CreateEnvioVendor().SupportsNetwork(ctx, &logger, c.settings, envioUnknownChain) + if c.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, c.want, ok) + }) + } + + t.Run("cached client reused", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gockProbeReply("https://999888.rpc.hypersync.xyz", "/k1", `{"jsonrpc":"2.0","id":1,"result":"0xf41d0"}`) + v := CreateEnvioVendor() + for range 2 { + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1"}, envioUnknownChain) + require.NoError(t, err) + assert.True(t, ok) + } + }) + + t.Run("certificate failure means unsupported", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://999888.rpc.hypersync.xyz").Post("/k1").Persist(). + ReplyError(errors.New("tls: failed to verify certificate: x509: certificate is valid for other.host")) + ok, err := CreateEnvioVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1"}, envioUnknownChain) + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("other transport failure is an error", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://999888.rpc.hypersync.xyz").Post("/k1").Persist().ReplyError(errors.New("connection refused")) + ok, err := CreateEnvioVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1"}, envioUnknownChain) + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("non-evm network", func(t *testing.T) { + ok, err := CreateEnvioVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{}, "solana:mainnet") + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("unparsable chain id", func(t *testing.T) { + ok, err := CreateEnvioVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:z") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("malformed api key", func(t *testing.T) { + ok, err := CreateEnvioVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "%zz"}, envioUnknownChain) + assert.Error(t, err) + assert.False(t, ok) + }) +} + +func TestEnvioVendor_GenerateConfigs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + cases := []struct { + name string + settings common.VendorSettings + expected string + }{ + {"default root domain with api key", common.VendorSettings{"apiKey": "k1"}, "https://42161.rpc.hypersync.xyz/k1"}, + {"default root domain without api key", common.VendorSettings{}, "https://42161.rpc.hypersync.xyz"}, + {"custom root domain", common.VendorSettings{"rootDomain": "rpc.example.dev", "apiKey": "k1"}, "https://42161.rpc.example.dev/k1"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 42161}} + cfgs, err := CreateEnvioVendor().GenerateConfigs(ctx, &logger, ups, c.settings) + require.NoError(t, err) + require.Len(t, cfgs, 1) + assert.Equal(t, c.expected, cfgs[0].Endpoint) + assert.Equal(t, common.UpstreamTypeEvm, cfgs[0].Type) + assert.Equal(t, []string{"*"}, cfgs[0].IgnoreMethods) + assert.Contains(t, cfgs[0].AllowMethods, "eth_getLogs") + assert.NotContains(t, cfgs[0].AllowMethods, "eth_call") + assert.NotNil(t, cfgs[0].JsonRpc) + }) + } + + t.Run("preset endpoint and user method filters are kept", func(t *testing.T) { + ups := &common.UpstreamConfig{ + Endpoint: "https://1.rpc.hypersync.xyz/mine", + Evm: &common.EvmUpstreamConfig{ChainId: 1}, + IgnoreMethods: []string{"eth_getLogs"}, + AllowMethods: []string{"eth_blockNumber"}, + } + cfgs, err := CreateEnvioVendor().GenerateConfigs(ctx, &logger, ups, nil) + require.NoError(t, err) + assert.Equal(t, "https://1.rpc.hypersync.xyz/mine", cfgs[0].Endpoint) + assert.Equal(t, []string{"eth_getLogs"}, cfgs[0].IgnoreMethods) + assert.Equal(t, []string{"eth_blockNumber"}, cfgs[0].AllowMethods) + }) + + t.Run("missing chain id", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{}} + _, err := CreateEnvioVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{}) + assert.ErrorContains(t, err, "requires upstream.evm.chainId") + }) + + t.Run("malformed api key", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}} + _, err := CreateEnvioVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"apiKey": "%zz"}) + assert.ErrorContains(t, err, "invalid URL escape") + }) +} + +func TestEnvioVendor_ErrorsAndOwnership(t *testing.T) { + v := CreateEnvioVendor() + assert.Equal(t, "envio", v.Name()) + + err, details := vendorErr(t, v, "eth_getLogs", 500, -32000, "boom", "d") + assert.NoError(t, err) + assert.Empty(t, details) + + for _, ep := range []string{"envio://x", "evm+envio://x", "https://indexer.envio.dev/x", "https://1.rpc.hypersync.xyz"} { + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: ep}), ep) + } + assert.False(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://rpc.example.com"})) +} diff --git a/thirdparty/erpc_test.go b/thirdparty/erpc_test.go index 45b540b61..3d3ab0806 100644 --- a/thirdparty/erpc_test.go +++ b/thirdparty/erpc_test.go @@ -1,9 +1,14 @@ package thirdparty import ( + "context" + "net/http" "testing" + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestErpcVendor_parseEndpointURL(t *testing.T) { @@ -158,3 +163,166 @@ func TestErpcVendor_parseEndpointURL(t *testing.T) { }) } } + +func TestErpcVendor_SupportsNetwork(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("matching chain id", func(t *testing.T) { + srv := newChainIdServer(t, "0x1") + ok, err := CreateErpcVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"endpoint": srv.URL}, "evm:1") + require.NoError(t, err) + assert.True(t, ok) + }) + + t.Run("cached client is reused across calls", func(t *testing.T) { + srv := newChainIdServer(t, "0x1") + v := CreateErpcVendor() + settings := common.VendorSettings{"endpoint": srv.URL, "secret": "s"} + for range 2 { + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:1") + require.NoError(t, err) + assert.True(t, ok) + } + }) + + t.Run("mismatching chain id", func(t *testing.T) { + srv := newChainIdServer(t, "0x89") + ok, err := CreateErpcVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"endpoint": srv.URL}, "evm:1") + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("json-rpc error response", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"boom"}}`) + ok, err := CreateErpcVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"endpoint": srv.URL}, "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("non-hex result", func(t *testing.T) { + srv := newChainIdServer(t, "not-a-number") + ok, err := CreateErpcVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"endpoint": srv.URL}, "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("non-string result", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `{"jsonrpc":"2.0","id":1,"result":{"a":1}}`) + ok, err := CreateErpcVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"endpoint": srv.URL}, "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("server down", func(t *testing.T) { + srv := newChainIdServer(t, "0x1") + url := srv.URL + srv.Close() + ok, err := CreateErpcVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"endpoint": url}, "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("non-evm network", func(t *testing.T) { + ok, err := CreateErpcVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"endpoint": "http://127.0.0.1:1"}, "solana:mainnet") + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("unparsable chain id", func(t *testing.T) { + ok, err := CreateErpcVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"endpoint": "http://127.0.0.1:1"}, "evm:abc") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("missing endpoint skips probe", func(t *testing.T) { + ok, err := CreateErpcVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:1") + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("malformed endpoint", func(t *testing.T) { + ok, err := CreateErpcVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"endpoint": "http://[::1"}, "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) +} + +func TestErpcVendor_GenerateConfigs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + cases := []struct { + name string + endpoint string + settings common.VendorSettings + expected string + }{ + {"settings endpoint with secret", "", common.VendorSettings{"endpoint": "erpc://edge.example.com", "secret": "s1"}, "https://edge.example.com/42161?secret=s1"}, + {"settings endpoint custom port", "", common.VendorSettings{"endpoint": "edge.example.com:8545"}, "http://edge.example.com:8545/42161"}, + {"preset erpc:// with port 443", "erpc://edge.example.com:443/main?x=1", nil, "https://edge.example.com:443/main/42161?x=1"}, + {"preset evm+erpc:// with custom port", "evm+erpc://edge.example.com:4000", nil, "http://edge.example.com:4000/42161"}, + {"preset https endpoint passes through", "https://rpc.example.com/v1", nil, "https://rpc.example.com/v1"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ups := &common.UpstreamConfig{Endpoint: c.endpoint, Evm: &common.EvmUpstreamConfig{ChainId: 42161}} + cfgs, err := CreateErpcVendor().GenerateConfigs(ctx, &logger, ups, c.settings) + require.NoError(t, err) + require.Len(t, cfgs, 1) + assert.Equal(t, c.expected, cfgs[0].Endpoint) + assert.Equal(t, common.UpstreamTypeEvm, cfgs[0].Type) + assert.NotNil(t, cfgs[0].JsonRpc) + }) + } + + t.Run("missing endpoint setting", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}} + _, err := CreateErpcVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{}) + assert.ErrorContains(t, err, "endpoint is required in erpc settings") + }) + + t.Run("malformed settings endpoint", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}} + _, err := CreateErpcVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"endpoint": "https://[::1"}) + assert.Error(t, err) + }) + + t.Run("malformed preset erpc:// endpoint", func(t *testing.T) { + ups := &common.UpstreamConfig{Endpoint: "erpc://[::1", Evm: &common.EvmUpstreamConfig{ChainId: 1}} + _, err := CreateErpcVendor().GenerateConfigs(ctx, &logger, ups, nil) + assert.Error(t, err) + }) +} + +func TestErpcVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateErpcVendor() + + err, details := vendorErr(t, v, "eth_getLogs", 500, -32000, "boom", "extra-info") + assert.NoError(t, err) + assert.Equal(t, "extra-info", details["data"]) + + err, details = vendorErr(t, v, "eth_getLogs", 500, -32000, "boom", "") + assert.NoError(t, err) + assert.NotContains(t, details, "data") + + assert.NoError(t, v.GetVendorSpecificErrorIfAny(nil, &http.Response{StatusCode: 500}, map[string]interface{}{}, map[string]interface{}{})) +} + +func TestErpcVendor_OwnsUpstream(t *testing.T) { + v := CreateErpcVendor() + assert.Equal(t, "erpc", v.Name()) + cases := []struct { + ups *common.UpstreamConfig + owned bool + }{ + {&common.UpstreamConfig{Endpoint: "erpc://a.example.com"}, true}, + {&common.UpstreamConfig{Endpoint: "evm+erpc://a.example.com"}, true}, + {&common.UpstreamConfig{Endpoint: "https://x.example.com", VendorName: "erpc"}, true}, + {&common.UpstreamConfig{Endpoint: "https://edge.erpc.cloud/1"}, true}, + {&common.UpstreamConfig{Endpoint: "https://rpc.example.com"}, false}, + } + for _, c := range cases { + assert.Equal(t, c.owned, v.OwnsUpstream(c.ups), c.ups.Endpoint) + } +} diff --git a/thirdparty/etherspot_test.go b/thirdparty/etherspot_test.go new file mode 100644 index 000000000..7a865f097 --- /dev/null +++ b/thirdparty/etherspot_test.go @@ -0,0 +1,83 @@ +package thirdparty + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEtherspotVendor_Name(t *testing.T) { + assert.Equal(t, "etherspot", CreateEtherspotVendor().Name()) +} + +func TestEtherspotVendor_SupportsNetwork(t *testing.T) { + svtCheckSupports(t, CreateEtherspotVendor(), common.VendorSettings{}, []svtSupportCase{ + {"evm:1", true, false}, // mainnet + {"evm:11155111", true, false}, // testnet + {"evm:999999999", false, false}, + {"solana:mainnet", false, false}, + {"evm:abc", false, true}, + }) +} + +func TestEtherspotVendor_GenerateConfigs(t *testing.T) { + svtCheckURLs(t, CreateEtherspotVendor, common.VendorSettings{"apiKey": "KEY"}, map[int64]string{ + 1: "https://ethereum-bundler.etherspot.io/?apikey=KEY", + 8453: "https://base-bundler.etherspot.io/?apikey=KEY", + 11155111: "https://testnet-rpc.etherspot.io/v1/11155111?apikey=KEY", + }) + svtCheckURLs(t, CreateEtherspotVendor, common.VendorSettings{"apiKey": "public"}, map[int64]string{ + 1: "https://ethereum-bundler.etherspot.io/", + 84532: "https://testnet-rpc.etherspot.io/v1/84532", + }) + + key := common.VendorSettings{"apiKey": "KEY"} + svtCheckGenErrors(t, CreateEtherspotVendor, []svtErrCase{ + {"missing apiKey", svtEvm(1), common.VendorSettings{}, "apiKey is required in etherspot settings"}, + {"zero chain", svtEvm(0), key, "etherspot vendor requires upstream.evm.chainId to be defined"}, + {"unparsable url", svtEvm(1), common.VendorSettings{"apiKey": "a\nb"}, "invalid control character"}, + }) + + t.Run("unknown chain yields an apikey-only url", func(t *testing.T) { + cfg, err := svtGenerate(t, CreateEtherspotVendor(), svtEvm(999999999), key) + require.NoError(t, err) + assert.Equal(t, "?apikey=KEY", cfg.Endpoint) + }) + + t.Run("method allowlist defaults to bundler methods", func(t *testing.T) { + cfg, err := svtGenerate(t, CreateEtherspotVendor(), svtEvm(1), key) + require.NoError(t, err) + assert.Equal(t, []string{"*"}, cfg.IgnoreMethods) + assert.Contains(t, cfg.AllowMethods, "eth_sendUserOperation") + assert.Contains(t, cfg.AllowMethods, "skandha_config") + }) + + t.Run("caller method lists are kept", func(t *testing.T) { + ups := svtEvm(1) + ups.IgnoreMethods = []string{"eth_call"} + ups.AllowMethods = []string{"eth_chainId"} + cfg, err := svtGenerate(t, CreateEtherspotVendor(), ups, key) + require.NoError(t, err) + assert.Equal(t, []string{"eth_call"}, cfg.IgnoreMethods) + assert.Equal(t, []string{"eth_chainId"}, cfg.AllowMethods) + }) + + svtCheckPreset(t, CreateEtherspotVendor(), "https://ethereum-bundler.etherspot.io/") +} + +func TestEtherspotVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateEtherspotVendor() + err, _ := vendorErr(t, v, "eth_sendUserOperation", 400, -32000, "aa error", "d") + assert.NoError(t, err) +} + +func TestEtherspotVendor_OwnsUpstream(t *testing.T) { + svtAssertOwns(t, CreateEtherspotVendor(), map[string]bool{ + "etherspot://key": true, + "evm+etherspot://key": true, + "https://ethereum-bundler.etherspot.io": true, + "https://rpc.ankr.com/eth/key": false, + }) +} diff --git a/thirdparty/goldsky_test.go b/thirdparty/goldsky_test.go index fd266370c..49fbe4fce 100644 --- a/thirdparty/goldsky_test.go +++ b/thirdparty/goldsky_test.go @@ -2,11 +2,13 @@ package thirdparty import ( "context" + "net/http" "testing" "github.com/erpc/erpc/common" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGoldskyVendor_buildEndpointURL(t *testing.T) { @@ -166,3 +168,158 @@ func TestGoldskyVendor_OwnsUpstream(t *testing.T) { }) } } + +func TestGoldskyVendor_SupportsNetwork_Probe(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + probe := func(t *testing.T, endpoint string, networkId string) (bool, error) { + return CreateGoldskyVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"endpoint": endpoint, "secret": "sec"}, networkId) + } + + t.Run("matching chain id", func(t *testing.T) { + ok, err := probe(t, newChainIdServer(t, "0x1").URL, "evm:1") + require.NoError(t, err) + assert.True(t, ok) + }) + + t.Run("cached client reused", func(t *testing.T) { + srv := newChainIdServer(t, "0x1") + v := CreateGoldskyVendor() + for range 2 { + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{"endpoint": srv.URL, "secret": "sec"}, "evm:1") + require.NoError(t, err) + assert.True(t, ok) + } + }) + + t.Run("secret from goldsky:// shorthand is enough to probe", func(t *testing.T) { + // The shorthand always targets the real Edge host, so assert only that the + // derived secret makes it past the no-secret skip (the URL is built correctly). + u, err := (&GoldskyVendor{}).buildEndpointURL("goldsky://tok", "", "", 1) + require.NoError(t, err) + assert.Equal(t, "https://edge.goldsky.com/standard/evm/1?secret=tok", u.String()) + }) + + t.Run("mismatching chain id", func(t *testing.T) { + ok, err := probe(t, newChainIdServer(t, "0x89").URL, "evm:1") + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("json-rpc error response", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"boom"}}`) + ok, err := probe(t, srv.URL, "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("non-hex result", func(t *testing.T) { + ok, err := probe(t, newChainIdServer(t, "nope").URL, "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("non-string result", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `{"jsonrpc":"2.0","id":1,"result":[1]}`) + ok, err := probe(t, srv.URL, "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("server down", func(t *testing.T) { + srv := newChainIdServer(t, "0x1") + url := srv.URL + srv.Close() + ok, err := probe(t, url, "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("unparsable chain id", func(t *testing.T) { + ok, err := probe(t, "http://127.0.0.1:1", "evm:xyz") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("malformed endpoint", func(t *testing.T) { + ok, err := probe(t, "http://[::1", "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) +} + +func TestGoldskyVendor_GenerateConfigs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + cases := []struct { + name string + endpoint string + settings common.VendorSettings + expected string + }{ + {"settings secret default host", "", common.VendorSettings{"secret": "s1"}, "https://edge.goldsky.com/standard/evm/8453?secret=s1"}, + {"settings tier and custom host", "", common.VendorSettings{"secret": "s1", "tier": "premium", "endpoint": "gw.example.com"}, "https://gw.example.com/premium/evm/8453?secret=s1"}, + {"preset goldsky:// shorthand", "goldsky://tok?tier=custom", nil, "https://edge.goldsky.com/custom/evm/8453?secret=tok"}, + {"preset evm+goldsky:// shorthand", "evm+goldsky://tok", nil, "https://edge.goldsky.com/standard/evm/8453?secret=tok"}, + {"preset https endpoint passes through", "https://edge.goldsky.com/standard/evm/8453?secret=x", nil, "https://edge.goldsky.com/standard/evm/8453?secret=x"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ups := &common.UpstreamConfig{Endpoint: c.endpoint, Evm: &common.EvmUpstreamConfig{ChainId: 8453}} + cfgs, err := CreateGoldskyVendor().GenerateConfigs(ctx, &logger, ups, c.settings) + require.NoError(t, err) + require.Len(t, cfgs, 1) + assert.Equal(t, c.expected, cfgs[0].Endpoint) + assert.Equal(t, common.UpstreamTypeEvm, cfgs[0].Type) + assert.NotNil(t, cfgs[0].JsonRpc) + }) + } + + errCases := []struct { + name string + ups *common.UpstreamConfig + settings common.VendorSettings + contains string + }{ + {"shorthand without evm", &common.UpstreamConfig{Endpoint: "goldsky://tok"}, nil, "requires upstream.evm to be defined"}, + {"settings without evm", &common.UpstreamConfig{}, common.VendorSettings{"secret": "s"}, "requires upstream.evm to be defined"}, + {"settings without chain id", &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{}}, common.VendorSettings{"secret": "s"}, "requires upstream.evm.chainId"}, + {"malformed settings endpoint", &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}}, common.VendorSettings{"endpoint": "https://[::1"}, "missing ']'"}, + } + for _, c := range errCases { + t.Run(c.name, func(t *testing.T) { + _, err := CreateGoldskyVendor().GenerateConfigs(ctx, &logger, c.ups, c.settings) + assert.ErrorContains(t, err, c.contains) + }) + } +} + +func TestGoldskyVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateGoldskyVendor() + assert.Equal(t, "goldsky", v.Name()) + + err, details := vendorErr(t, v, "eth_getLogs", 500, -32000, "boom", "why") + assert.NoError(t, err) + assert.Equal(t, "why", details["data"]) + + err, details = vendorErr(t, v, "eth_getLogs", 500, -32000, "boom", "") + assert.NoError(t, err) + assert.NotContains(t, details, "data") + + assert.NoError(t, v.GetVendorSpecificErrorIfAny(nil, &http.Response{StatusCode: 500}, "not-a-response", map[string]interface{}{})) +} + +func TestGoldskyVendor_buildEndpointURL_UnparsableShorthand(t *testing.T) { + // An unparsable goldsky:// authority yields no derived secret, but still + // targets the default Edge host instead of dialing the shorthand. + u, err := (&GoldskyVendor{}).buildEndpointURL("goldsky://[bad", "", "fromsettings", 1) + require.NoError(t, err) + assert.Equal(t, "https://edge.goldsky.com/standard/evm/1?secret=fromsettings", u.String()) +} + +func TestGoldskyVendor_OwnsUpstream_ByVendorName(t *testing.T) { + v := CreateGoldskyVendor() + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://rpc.example.com", VendorName: "goldsky"})) + assert.False(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://rpc.example.com", VendorName: "alchemy"})) +} diff --git a/thirdparty/helpers_test.go b/thirdparty/helpers_test.go new file mode 100644 index 000000000..eab3dabfd --- /dev/null +++ b/thirdparty/helpers_test.go @@ -0,0 +1,83 @@ +package thirdparty + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +// primeRemoteCache publishes val under key through the cache's real async +// refresh path and waits until that refresh has stored it. A key that is +// already fresh would pass a freshness check before the new value lands. +func primeRemoteCache[T any](t *testing.T, c *RemoteDataCache[T], key string, val T) { + t.Helper() + logger := zerolog.Nop() + ran := make(chan struct{}) + require.Eventually(t, func() bool { + select { + case <-ran: + return true + default: + } + c.TriggerAsyncRefresh(&logger, key, func(ctx context.Context) (T, error) { + close(ran) + return val, nil + }) + return false + }, 5*time.Second, 10*time.Millisecond) + waitRefreshesIdle(t, c) +} + +// waitRefreshesIdle blocks until no async refresh is running on c, so tests +// can restore package globals that the refresh goroutines read. +func waitRefreshesIdle[T any](t *testing.T, c *RemoteDataCache[T]) { + t.Helper() + require.Eventually(t, func() bool { + c.refreshMu.Lock() + defer c.refreshMu.Unlock() + return len(c.inflight) == 0 + }, 10*time.Second, 10*time.Millisecond) +} + +// newJSONServer serves body with the given status code for every request. +func newJSONServer(t *testing.T, status int, body string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return srv +} + +// newChainIdServer answers every JSON-RPC call with the given eth_chainId result. +func newChainIdServer(t *testing.T, hexChainId string) *httptest.Server { + t.Helper() + return newJSONServer(t, http.StatusOK, `{"jsonrpc":"2.0","id":1,"result":"`+hexChainId+`"}`) +} + +// vendorErr runs v's error normalization on a JSON-RPC error response for method. +func vendorErr(t *testing.T, v common.Vendor, method string, status int, code int, msg string, data string) (error, map[string]interface{}) { + t.Helper() + jrr, err := common.NewJsonRpcResponse(1, nil, common.NewErrJsonRpcExceptionExternal(code, msg, data)) + require.NoError(t, err) + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"` + method + `","params":[]}`)) + details := map[string]interface{}{} + return v.GetVendorSpecificErrorIfAny(req, &http.Response{StatusCode: status}, jrr, details), details +} + +// normalizedCode returns the JSON-RPC normalized code carried by a vendor error. +func normalizedCode(t *testing.T, err error) common.JsonRpcErrorNumber { + t.Helper() + se, ok := err.(common.StandardError) + require.True(t, ok, "expected a StandardError, got %T", err) + code, _ := se.DeepSearch("normalizedCode").(common.JsonRpcErrorNumber) + return code +} diff --git a/thirdparty/infura_test.go b/thirdparty/infura_test.go new file mode 100644 index 000000000..18ad75610 --- /dev/null +++ b/thirdparty/infura_test.go @@ -0,0 +1,88 @@ +package thirdparty + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInfuraVendor_Name(t *testing.T) { + assert.Equal(t, "infura", CreateInfuraVendor().Name()) +} + +func TestInfuraVendor_SupportsNetwork(t *testing.T) { + svtCheckSupports(t, CreateInfuraVendor(), common.VendorSettings{}, []svtSupportCase{ + {"evm:1", true, false}, + {"evm:300", true, false}, + {"evm:999999999", false, false}, + {"solana:mainnet", false, false}, + {"evm:abc", false, true}, + }) +} + +func TestInfuraVendor_GenerateConfigs(t *testing.T) { + key := common.VendorSettings{"apiKey": "KEY"} + svtCheckURLs(t, CreateInfuraVendor, key, map[int64]string{ + 1: "https://mainnet.infura.io/v3/KEY", + 137: "https://polygon-mainnet.infura.io/v3/KEY", + }) + svtCheckGenErrors(t, CreateInfuraVendor, []svtErrCase{ + {"missing apiKey", svtEvm(1), common.VendorSettings{}, "apiKey is required in infura settings"}, + {"zero chain", svtEvm(0), key, "infura vendor requires upstream.evm.chainId to be defined"}, + {"unsupported chain", svtEvm(999999999), key, "unsupported network chain ID for Infura: 999999999"}, + {"unparsable url", svtEvm(1), common.VendorSettings{"apiKey": "%zz"}, "invalid URL escape"}, + }) + svtCheckPreset(t, CreateInfuraVendor(), "https://mainnet.infura.io/v3/own") +} + +func TestInfuraVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateInfuraVendor() + + cases := []struct { + name string + code int + msg string + errCode common.ErrorCode + norm common.JsonRpcErrorNumber + }{ + {"auth required", -32600, "project ID must be authenticated", common.ErrCodeEndpointUnauthorized, common.JsonRpcErrorUnauthorized}, + {"bad access key", -32600, "invalid access key", common.ErrCodeEndpointUnauthorized, common.JsonRpcErrorUnauthorized}, + {"method not supported", -32001, "unsupported", common.ErrCodeEndpointUnsupported, common.JsonRpcErrorUnsupportedException}, + {"method not found", -32004, "not found", common.ErrCodeEndpointUnsupported, common.JsonRpcErrorUnsupportedException}, + {"rate limited", -32005, "daily limit", common.ErrCodeEndpointCapacityExceeded, common.JsonRpcErrorCapacityExceeded}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err, details := vendorErr(t, v, "eth_getLogs", 400, c.code, c.msg, "d") + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, c.errCode), "got %v", err) + assert.Equal(t, c.norm, normalizedCode(t, err)) + assert.Equal(t, "d", details["data"]) + }) + } + + t.Run("-32600 without auth wording is generic", func(t *testing.T) { + err, details := vendorErr(t, v, "eth_getLogs", 400, -32600, "invalid request", "") + assert.NoError(t, err) + assert.NotContains(t, details, "data") + }) + t.Run("other code is generic", func(t *testing.T) { + err, details := vendorErr(t, v, "eth_getLogs", 400, -32000, "x", "d") + assert.NoError(t, err) + assert.Equal(t, "d", details["data"]) + }) + t.Run("non-response input", func(t *testing.T) { + assert.NoError(t, v.GetVendorSpecificErrorIfAny(svtReq(), nil, "x", map[string]interface{}{})) + }) +} + +func TestInfuraVendor_OwnsUpstream(t *testing.T) { + svtAssertOwns(t, CreateInfuraVendor(), map[string]bool{ + "infura://key": true, + "evm+infura://key": true, + "https://mainnet.infura.io/v3/key": true, + "https://rpc.ankr.com/eth/key": false, + }) +} diff --git a/thirdparty/llama_test.go b/thirdparty/llama_test.go new file mode 100644 index 000000000..42b17c5aa --- /dev/null +++ b/thirdparty/llama_test.go @@ -0,0 +1,57 @@ +package thirdparty + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLlamaVendor_Name(t *testing.T) { + assert.Equal(t, "llama", CreateLlamaVendor().Name()) +} + +func TestLlamaVendor_SupportsNetwork(t *testing.T) { + svtCheckSupports(t, CreateLlamaVendor(), common.VendorSettings{}, []svtSupportCase{ + {"evm:1", true, false}, + {"evm:56", true, false}, + {"evm:999999999", false, false}, + {"solana:mainnet", false, false}, + {"evm:abc", false, true}, + }) +} + +func TestLlamaVendor_GenerateConfigs(t *testing.T) { + key := common.VendorSettings{"apiKey": "KEY"} + svtCheckURLs(t, CreateLlamaVendor, key, map[int64]string{ + 1: "https://eth.llamarpc.com/KEY", + 56: "https://binance.llamarpc.com/KEY", + }) + svtCheckGenErrors(t, CreateLlamaVendor, []svtErrCase{ + {"missing apiKey", svtEvm(1), common.VendorSettings{}, "apiKey is required in llama settings"}, + {"zero chain", svtEvm(0), key, "llama vendor requires upstream.evm.chainId to be defined"}, + {"unsupported chain", svtEvm(999999999), key, "unsupported network chain ID for Llama: 999999999"}, + }) + svtCheckPreset(t, CreateLlamaVendor(), "https://eth.llamarpc.com/own") +} + +func TestLlamaVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateLlamaVendor() + + err, details := vendorErr(t, v, "eth_getLogs", 429, -32000, "cloudflare error code: 1015 rate limited", "d") + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointCapacityExceeded), "got %v", err) + assert.Equal(t, common.JsonRpcErrorCapacityExceeded, normalizedCode(t, err)) + assert.Equal(t, "d", details["data"]) + + svtCheckDataOnly(t, v) +} + +func TestLlamaVendor_OwnsUpstream(t *testing.T) { + svtAssertOwns(t, CreateLlamaVendor(), map[string]bool{ + "https://eth.llamarpc.com/key": true, + "llama://key": false, + "https://rpc.ankr.com/eth/key": false, + }) +} diff --git a/thirdparty/onfinality_test.go b/thirdparty/onfinality_test.go new file mode 100644 index 000000000..70bedfcf9 --- /dev/null +++ b/thirdparty/onfinality_test.go @@ -0,0 +1,50 @@ +package thirdparty + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" +) + +func TestOnFinalityVendor_Name(t *testing.T) { + assert.Equal(t, "onfinality", CreateOnFinalityVendor().Name()) +} + +func TestOnFinalityVendor_SupportsNetwork(t *testing.T) { + svtCheckSupports(t, CreateOnFinalityVendor(), common.VendorSettings{}, []svtSupportCase{ + {"evm:1", true, false}, + {"evm:534352", true, false}, + {"evm:999999999", false, false}, + {"solana:mainnet", false, false}, + {"evm:abc", false, true}, + }) +} + +func TestOnFinalityVendor_GenerateConfigs(t *testing.T) { + key := common.VendorSettings{"apiKey": "a&b"} + svtCheckURLs(t, CreateOnFinalityVendor, key, map[int64]string{ + 1: "https://eth.api.onfinality.io/rpc?apikey=a%26b", + 56: "https://bnb.api.onfinality.io/rpc?apikey=a%26b", + }) + svtCheckGenErrors(t, CreateOnFinalityVendor, []svtErrCase{ + {"missing apiKey", svtEvm(1), common.VendorSettings{}, "apiKey is required in onfinality settings"}, + {"nil evm", &common.UpstreamConfig{}, key, "onfinality vendor requires upstream.evm to be defined"}, + {"zero chain", svtEvm(0), key, "onfinality vendor requires upstream.evm.chainId to be defined"}, + {"unsupported chain", svtEvm(999999999), key, "unsupported network chain ID for OnFinality: 999999999"}, + }) + svtCheckPreset(t, CreateOnFinalityVendor(), "https://eth.api.onfinality.io/rpc?apikey=own") +} + +func TestOnFinalityVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + svtCheckDataOnly(t, CreateOnFinalityVendor()) +} + +func TestOnFinalityVendor_OwnsUpstream(t *testing.T) { + svtAssertOwns(t, CreateOnFinalityVendor(), map[string]bool{ + "onfinality://key": true, + "evm+onfinality://key": true, + "https://eth.api.onfinality.io/rpc?apikey=": true, + "https://onfinality.io/docs": false, + }) +} diff --git a/thirdparty/phony_test.go b/thirdparty/phony_test.go index 7e1ab4480..55d80ccfe 100644 --- a/thirdparty/phony_test.go +++ b/thirdparty/phony_test.go @@ -26,3 +26,24 @@ func TestPhonyUpstream_Config_NoPanicInEvmExtractor(t *testing.T) { t.Fatal("EVM extractor should classify -32602 for a phony (EVM) upstream") } } + +// The phony upstream is a probe-only identity: it reports its id as both Id and +// Name, always claims EVM, and carries no vendor or vendor settings. +func TestPhonyUpstream_Identity(t *testing.T) { + u := &phonyUpstream{id: "temp-goldsky-8453"} + if u.Id() != "temp-goldsky-8453" || u.Name() != "temp-goldsky-8453" { + t.Fatalf("Id/Name must echo the probe id, got %q/%q", u.Id(), u.Name()) + } + if u.Type() != common.UpstreamTypeEvm { + t.Fatalf("phony upstream must be EVM, got %q", u.Type()) + } + if u.Config().Id != "temp-goldsky-8453" { + t.Fatalf("Config().Id must be the probe id, got %q", u.Config().Id) + } + if u.Vendor() != nil { + t.Fatal("phony upstream must not carry a vendor") + } + if u.VendorSettings() != nil { + t.Fatal("phony upstream must not carry vendor settings") + } +} diff --git a/thirdparty/pimlico_test.go b/thirdparty/pimlico_test.go new file mode 100644 index 000000000..78113e8bf --- /dev/null +++ b/thirdparty/pimlico_test.go @@ -0,0 +1,166 @@ +package thirdparty + +import ( + "context" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const pimlicoUnknownChain = "evm:999888" // 0xf41d0, not in pimlicoSupportedChains + +func TestPimlicoVendor_SupportsNetwork(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("known chain short-circuits without probe or apiKey", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://api.pimlico.io").Persist().ReplyError(assert.AnError) + ok, err := CreatePimlicoVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:8453") + require.NoError(t, err) + assert.True(t, ok) + }) + + probeCases := []struct { + name string + apiKey string + host string + path string + body string + want bool + wantErr bool + }{ + {"api key probe match", "k1", "https://api.pimlico.io", "/v2/999888/rpc", `{"jsonrpc":"2.0","id":1,"result":"0xf41d0"}`, true, false}, + {"public probe match", "public", "https://public.pimlico.io", "/v2/999888/rpc", `{"jsonrpc":"2.0","id":1,"result":"0xf41d0"}`, true, false}, + {"mismatching chain id", "k1", "https://api.pimlico.io", "/v2/999888/rpc", `{"jsonrpc":"2.0","id":1,"result":"0x1"}`, false, false}, + {"json-rpc error", "k1", "https://api.pimlico.io", "/v2/999888/rpc", `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"boom"}}`, false, true}, + {"non-hex result", "k1", "https://api.pimlico.io", "/v2/999888/rpc", `{"jsonrpc":"2.0","id":1,"result":"xyz"}`, false, true}, + {"non-string result", "k1", "https://api.pimlico.io", "/v2/999888/rpc", `{"jsonrpc":"2.0","id":1,"result":{}}`, false, true}, + } + for _, c := range probeCases { + t.Run(c.name, func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gockProbeReply(c.host, c.path, c.body) + + ok, err := CreatePimlicoVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": c.apiKey}, pimlicoUnknownChain) + if c.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, c.want, ok) + }) + } + + t.Run("cached client reused", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gockProbeReply("https://api.pimlico.io", "/v2/999888/rpc", `{"jsonrpc":"2.0","id":1,"result":"0xf41d0"}`) + v := CreatePimlicoVendor() + for range 2 { + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1"}, pimlicoUnknownChain) + require.NoError(t, err) + assert.True(t, ok) + } + }) + + t.Run("unreachable endpoint", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://api.pimlico.io").Post("/v2/999888/rpc").Persist().ReplyError(assert.AnError) + ok, err := CreatePimlicoVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1"}, pimlicoUnknownChain) + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("non-evm network", func(t *testing.T) { + ok, err := CreatePimlicoVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1"}, "solana:mainnet") + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("unparsable chain id", func(t *testing.T) { + ok, err := CreatePimlicoVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1"}, "evm:x") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("unknown chain without apiKey", func(t *testing.T) { + ok, err := CreatePimlicoVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{}, pimlicoUnknownChain) + assert.ErrorContains(t, err, "apiKey is required in pimlico settings") + assert.False(t, ok) + }) +} + +func TestPimlicoVendor_GenerateConfigs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("api key endpoint and default method filters", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 137}} + cfgs, err := CreatePimlicoVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"apiKey": "k1"}) + require.NoError(t, err) + require.Len(t, cfgs, 1) + assert.Equal(t, "https://api.pimlico.io/v2/137/rpc?apikey=k1", cfgs[0].Endpoint) + assert.Equal(t, common.UpstreamTypeEvm, cfgs[0].Type) + assert.Equal(t, []string{"*"}, cfgs[0].IgnoreMethods) + assert.Contains(t, cfgs[0].AllowMethods, "eth_sendUserOperation") + assert.Contains(t, cfgs[0].AllowMethods, "pm_sponsorUserOperation") + assert.NotContains(t, cfgs[0].AllowMethods, "eth_getLogs") + }) + + t.Run("public endpoint", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 10}} + cfgs, err := CreatePimlicoVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"apiKey": "public"}) + require.NoError(t, err) + assert.Equal(t, "https://public.pimlico.io/v2/10/rpc", cfgs[0].Endpoint) + }) + + t.Run("preset endpoint and user method filters are kept", func(t *testing.T) { + ups := &common.UpstreamConfig{ + Endpoint: "https://api.pimlico.io/v2/1/rpc?apikey=mine", + Evm: &common.EvmUpstreamConfig{ChainId: 1}, + IgnoreMethods: []string{"pm_*"}, + AllowMethods: []string{"eth_chainId"}, + } + cfgs, err := CreatePimlicoVendor().GenerateConfigs(ctx, &logger, ups, nil) + require.NoError(t, err) + assert.Equal(t, "https://api.pimlico.io/v2/1/rpc?apikey=mine", cfgs[0].Endpoint) + assert.Equal(t, []string{"pm_*"}, cfgs[0].IgnoreMethods) + assert.Equal(t, []string{"eth_chainId"}, cfgs[0].AllowMethods) + assert.NotNil(t, cfgs[0].JsonRpc) + }) + + t.Run("missing apiKey", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}} + _, err := CreatePimlicoVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{}) + assert.ErrorContains(t, err, "apiKey is required in pimlico settings") + }) + + t.Run("missing chain id", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{}} + _, err := CreatePimlicoVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"apiKey": "k1"}) + assert.ErrorContains(t, err, "requires upstream.evm.chainId") + }) +} + +func TestPimlicoVendor_ErrorsAndOwnership(t *testing.T) { + v := CreatePimlicoVendor() + assert.Equal(t, "pimlico", v.Name()) + + err, details := vendorErr(t, v, "eth_sendUserOperation", 500, -32000, "boom", "d") + assert.NoError(t, err) + assert.Empty(t, details) + + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "pimlico://k"})) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "evm+pimlico://k"})) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://api.pimlico.io/v2/1/rpc"})) + assert.False(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://rpc.example.com"})) +} diff --git a/thirdparty/provider_test.go b/thirdparty/provider_test.go index 9803e8d27..af4a8e004 100644 --- a/thirdparty/provider_test.go +++ b/thirdparty/provider_test.go @@ -9,6 +9,7 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) type MockVendor struct { @@ -151,3 +152,105 @@ func TestProviderOverrideMostSpecificWins(t *testing.T) { assert.Equal(t, "shared", got.RateLimitBudget, "the wildcard still applies when nothing more specific matches") assert.Equal(t, []string{"tier:expensive"}, got.Tags) } + +func TestProvider_GenerateUpstreamConfigs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("builds base config, applies vendor output, credit units and env vars", func(t *testing.T) { + t.Setenv("PROVIDER_TEST_KEY", "sekret") + mv := new(MockVendor) + settings := common.VendorSettings{"creditUnits": map[string]interface{}{"eth_call": 3}} + cfg := &common.ProviderConfig{ + Id: "prov", + Vendor: "mockvendor", + Settings: settings, + UpstreamIdTemplate: "---", + } + defaults := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{NodeType: common.EvmNodeTypeArchive}} + p := NewProvider(&logger, cfg, mv, defaults) + assert.Equal(t, "prov", p.Id()) + + preset := map[string]int64{"eth_getLogs": 9} + mv.On("GenerateConfigs", mock.Anything, mock.Anything, mock.Anything, settings). + Return([]*common.UpstreamConfig{ + {Endpoint: "https://rpc.example.com/${PROVIDER_TEST_KEY}"}, + {Endpoint: "https://rpc2.example.com", CreditUnits: preset}, + }, nil). + Run(func(args mock.Arguments) { + base := args.Get(2).(*common.UpstreamConfig) + assert.Equal(t, "mockvendor-prov-evm:10-10", base.Id) + assert.Equal(t, "mockvendor", base.VendorName) + assert.Equal(t, common.UpstreamTypeEvm, base.Type) + require.NotNil(t, base.Evm) + assert.Equal(t, int64(10), base.Evm.ChainId) + assert.Equal(t, common.EvmNodeTypeArchive, base.Evm.NodeType, "upstream defaults' evm settings apply") + }) + + got, err := p.GenerateUpstreamConfigs(ctx, &logger, "evm:10") + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "https://rpc.example.com/sekret", got[0].Endpoint) + assert.Equal(t, map[string]int64{"eth_call": 3}, got[0].CreditUnits) + assert.Equal(t, preset, got[1].CreditUnits, "an upstream's own credit units are not overwritten") + mv.AssertExpectations(t) + }) + + t.Run("vendor error is returned", func(t *testing.T) { + mv := new(MockVendor) + p := NewProvider(&logger, &common.ProviderConfig{Id: "p", Vendor: "v"}, mv, nil) + mv.On("GenerateConfigs", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]*common.UpstreamConfig(nil), assert.AnError) + _, err := p.GenerateUpstreamConfigs(ctx, &logger, "evm:1") + assert.ErrorIs(t, err, assert.AnError) + }) + + t.Run("bad evm chain id fails before the vendor is called", func(t *testing.T) { + mv := new(MockVendor) + p := NewProvider(&logger, &common.ProviderConfig{Id: "p", Vendor: "v"}, mv, nil) + _, err := p.GenerateUpstreamConfigs(ctx, &logger, "evm:notanumber") + assert.Error(t, err) + mv.AssertNotCalled(t, "GenerateConfigs", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) +} + +func TestProvider_buildBaseUpstreamConfig(t *testing.T) { + t.Run("non-evm network gets no evm block and N/A chain id", func(t *testing.T) { + p := &Provider{config: &common.ProviderConfig{Id: "p", Vendor: "v", UpstreamIdTemplate: "--"}} + got, err := p.buildBaseUpstreamConfig("solana:mainnet") + require.NoError(t, err) + assert.Equal(t, "p-N/A-solana:mainnet", got.Id) + assert.Equal(t, "v", got.VendorName) + }) + + t.Run("override is copied, not shared", func(t *testing.T) { + override := &common.UpstreamConfig{RateLimitBudget: "b"} + p := &Provider{config: &common.ProviderConfig{Id: "p", Vendor: "v", Overrides: map[string]*common.UpstreamConfig{"evm:1": override}}} + got, err := p.buildBaseUpstreamConfig("evm:1") + require.NoError(t, err) + assert.Equal(t, "b", got.RateLimitBudget) + assert.Equal(t, int64(1), got.Evm.ChainId) + assert.Nil(t, override.Evm, "the provider's override must not be mutated") + }) + + t.Run("invalid override pattern is skipped", func(t *testing.T) { + p := &Provider{config: &common.ProviderConfig{Id: "p", Vendor: "v", Overrides: map[string]*common.UpstreamConfig{ + "(evm:1": {RateLimitBudget: "broken"}, + }}} + got, err := p.buildBaseUpstreamConfig("evm:1") + require.NoError(t, err) + assert.Empty(t, got.RateLimitBudget) + }) + + t.Run("equally specific patterns resolve lexicographically", func(t *testing.T) { + p := &Provider{config: &common.ProviderConfig{Id: "p", Vendor: "v", Overrides: map[string]*common.UpstreamConfig{ + "evm:1*": {RateLimitBudget: "one-star"}, + "evm:*1": {RateLimitBudget: "star-one"}, + }}} + for range 50 { + got, err := p.buildBaseUpstreamConfig("evm:1") + require.NoError(t, err) + assert.Equal(t, "star-one", got.RateLimitBudget) + } + }) +} diff --git a/thirdparty/providers_registry_test.go b/thirdparty/providers_registry_test.go new file mode 100644 index 000000000..01a262d32 --- /dev/null +++ b/thirdparty/providers_registry_test.go @@ -0,0 +1,44 @@ +package thirdparty + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewProvidersRegistry(t *testing.T) { + logger := zerolog.Nop() + vr := NewVendorsRegistry() + + t.Run("builds one provider per config", func(t *testing.T) { + cfgs := []*common.ProviderConfig{ + {Id: "gs", Vendor: "goldsky"}, + {Id: "rm", Vendor: "routemesh"}, + } + pr, err := NewProvidersRegistry(&logger, vr, cfgs, nil) + require.NoError(t, err) + all := pr.GetAllProviders() + require.Len(t, all, 2) + assert.Equal(t, "gs", all[0].Id()) + assert.Equal(t, "rm", all[1].Id()) + assert.Equal(t, "goldsky", all[0].vendor.Name()) + assert.Equal(t, "routemesh", all[1].vendor.Name()) + }) + + t.Run("no configs yields empty registry", func(t *testing.T) { + pr, err := NewProvidersRegistry(&logger, vr, nil, nil) + require.NoError(t, err) + assert.Empty(t, pr.GetAllProviders()) + }) + + t.Run("unknown vendor lists supported vendors", func(t *testing.T) { + cfgs := []*common.ProviderConfig{{Id: "x", Vendor: "nope"}} + pr, err := NewProvidersRegistry(&logger, vr, cfgs, nil) + assert.Nil(t, pr) + assert.ErrorContains(t, err, "vendor 'nope' not found for provider 'x'") + assert.ErrorContains(t, err, "goldsky") + }) +} diff --git a/thirdparty/quicknode_vendor_test.go b/thirdparty/quicknode_vendor_test.go new file mode 100644 index 000000000..f0f5994ce --- /dev/null +++ b/thirdparty/quicknode_vendor_test.go @@ -0,0 +1,387 @@ +package thirdparty + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newChainIdRouter answers eth_chainId per request path ("/" → result). +// Paths mapped to "" get a JSON-RPC error; "!" gets an undecodable body. +func newChainIdRouter(t *testing.T, results map[string]string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + if !strings.Contains(string(body), "eth_chainId") { + w.WriteHeader(http.StatusBadRequest) + return + } + res, ok := results[r.URL.Path] + w.Header().Set("Content-Type", "application/json") + switch { + case !ok || res == "": + _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"error":{"code":-32601,"message":"nope"}}`)) + case res == "!": + _, _ = w.Write([]byte(``)) + default: + _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":"` + res + `"}`)) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func TestQuicknodeVendor_FetchEndpoints(t *testing.T) { + ctx := context.Background() + + t.Run("paginates, forwards filters and drops endpoints without http url", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + + page1 := make([]map[string]interface{}, 0, 100) + for i := range 99 { + page1 = append(page1, map[string]interface{}{"id": "e" + strconv.Itoa(i), "http_url": "https://e" + strconv.Itoa(i) + ".quiknode.pro/k/", "chain": "eth"}) + } + page1 = append(page1, map[string]interface{}{"id": "nohttp", "http_url": ""}) + + gock.New("https://api.quicknode.com"). + Get("/v0/endpoints"). + MatchHeader("x-api-key", "qn-key"). + MatchParam("offset", "^0$"). + MatchParam("limit", "^100$"). + MatchParam("tag_ids", "^1,2$"). + MatchParam("tag_labels", "^prod,eu$"). + Reply(200). + JSON(map[string]interface{}{"data": page1}) + gock.New("https://api.quicknode.com"). + Get("/v0/endpoints"). + MatchParam("offset", "^100$"). + Reply(200). + JSON(map[string]interface{}{"data": []map[string]interface{}{{"id": "last", "http_url": "https://last.quiknode.pro/k/", "chain": "matic"}}}) + + v := CreateQuicknodeVendor().(*QuicknodeVendor) + got, err := v.fetchEndpoints(ctx, "qn-key", &QuicknodeFilterParams{TagIDs: []int{1, 2}, TagLabels: []string{"prod", "eu"}}) + require.NoError(t, err) + require.Len(t, got, 100) + assert.Equal(t, "e0", got[0].ID) + assert.Equal(t, "eth", got[0].Chain) + assert.Equal(t, "last", got[99].ID) + assert.Equal(t, "matic", got[99].Chain) + assert.True(t, gock.IsDone(), "both pages requested") + }) + + errCases := []struct { + name string + status int + body string + wantErr string + }{ + {"non-200 status", http.StatusUnauthorized, `{"error":"bad key"}`, "status 401"}, + {"body error field", http.StatusOK, `{"data":[],"error":"forbidden"}`, "quicknode API error: forbidden"}, + {"undecodable body", http.StatusOK, `not-json`, "failed to decode QuickNode endpoints response"}, + } + for _, c := range errCases { + t.Run(c.name, func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://api.quicknode.com").Get("/v0/endpoints").Reply(c.status).BodyString(c.body) + + v := CreateQuicknodeVendor().(*QuicknodeVendor) + _, err := v.fetchEndpoints(ctx, "k", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), c.wantErr) + }) + } + + t.Run("transport failure is an error", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://api.quicknode.com").Get("/v0/endpoints").ReplyError(io.ErrUnexpectedEOF) + + v := CreateQuicknodeVendor().(*QuicknodeVendor) + _, err := v.fetchEndpoints(ctx, "k", &QuicknodeFilterParams{}) + require.Error(t, err) + }) +} + +func TestQuicknodeVendor_FetchChainIDs(t *testing.T) { + logger := zerolog.Nop() + srv := newChainIdRouter(t, map[string]string{"/eth": "0x1", "/base": "0x2105", "/bad": "0xzz", "/html": "!"}) + + endpoints := []*QuicknodeEndpoint{ + {ID: "eth", HttpUrl: srv.URL + "/eth"}, + {ID: "base", HttpUrl: srv.URL + "/base"}, + {ID: "rpcerr", HttpUrl: srv.URL + "/missing"}, + {ID: "badhex", HttpUrl: srv.URL + "/bad"}, + {ID: "html", HttpUrl: srv.URL + "/html"}, + {ID: "down", HttpUrl: "http://127.0.0.1:1/"}, + {ID: "badurl", HttpUrl: "http://bad host/"}, + {ID: "skip"}, + } + v := CreateQuicknodeVendor().(*QuicknodeVendor) + require.NoError(t, v.fetchChainIDs(context.Background(), &logger, endpoints), "per-endpoint failures are tolerated") + + got := map[string]int64{} + for _, e := range endpoints { + got[e.ID] = e.ChainID + } + assert.Equal(t, map[string]int64{ + "eth": 1, "base": 8453, "rpcerr": 0, "badhex": 0, "html": 0, "down": 0, "badurl": 0, "skip": 0, + }, got) + + t.Run("cancelled context fails every acquisition", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + eps := []*QuicknodeEndpoint{{ID: "eth", HttpUrl: srv.URL + "/eth"}} + require.NoError(t, v.fetchChainIDs(ctx, &logger, eps)) + assert.Zero(t, eps[0].ChainID) + }) +} + +// End-to-end discovery: the Admin API lists endpoints, each is probed for its +// chain id, and the snapshot then drives SupportsNetwork/GenerateConfigs. +func TestQuicknodeVendor_DiscoveryLifecycle(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + + ctx := context.Background() + logger := zerolog.Nop() + probe := newChainIdRouter(t, map[string]string{"/eth-a": "0x1", "/eth-b": "0x1", "/base": "0x2105", "/broken": ""}) + + gock.New("https://api.quicknode.com"). + Get("/v0/endpoints"). + MatchHeader("x-api-key", "life-key"). + Persist(). + Reply(200). + JSON(map[string]interface{}{"data": []map[string]interface{}{ + {"id": "a", "http_url": probe.URL + "/eth-a", "chain": "eth"}, + {"id": "b", "http_url": probe.URL + "/eth-b", "chain": "eth"}, + {"id": "c", "http_url": probe.URL + "/base", "chain": "base"}, + {"id": "d", "http_url": probe.URL + "/broken", "chain": "broken"}, + }}) + + v := CreateQuicknodeVendor().(*QuicknodeVendor) + settings := common.VendorSettings{"apiKey": "life-key", "recheckInterval": time.Hour} + + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:1") + assert.ErrorIs(t, err, ErrRemoteCacheCold, "first call is a cold start") + assert.False(t, ok) + + require.Eventually(t, func() bool { + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:1") + return err == nil && ok + }, 5*time.Second, 10*time.Millisecond) + + ok, err = v.SupportsNetwork(ctx, &logger, settings, "evm:10") + require.NoError(t, err) + assert.False(t, ok, "no endpoint probed as chain 10") + + assert.Equal(t, "eth", v.chainSlug("life-key", 1)) + assert.Equal(t, "base", v.chainSlug("life-key", 8453)) + assert.Equal(t, "", v.chainSlug("life-key", 10)) + assert.Equal(t, "", v.chainSlug("other-key", 1)) + + t.Run("generates one upstream per matching endpoint", func(t *testing.T) { + got, err := v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Id: "qn", Evm: &common.EvmUpstreamConfig{ChainId: 1}}, settings) + require.NoError(t, err) + require.Len(t, got, 2) + ids := []string{got[0].Id, got[1].Id} + assert.ElementsMatch(t, []string{"qn-a", "qn-b"}, ids) + endpoints := []string{got[0].Endpoint, got[1].Endpoint} + assert.ElementsMatch(t, []string{probe.URL + "/eth-a", probe.URL + "/eth-b"}, endpoints) + for _, c := range got { + assert.Equal(t, common.UpstreamTypeEvm, c.Type) + } + + got, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 8453}}, settings) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "quicknode-8453-c", got[0].Id) + + got, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 10}}, settings) + require.NoError(t, err) + assert.Empty(t, got) + }) + + t.Run("credit-unit table refresh uses the discovered chain slug", func(t *testing.T) { + gotPath := make(chan string, 1) + cuSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case gotPath <- r.URL.Path: + default: + } + _, _ = w.Write([]byte(`{"data":[{"method":"eth_call","credits":55}],"error":null}`)) + })) + defer cuSrv.Close() + // Earlier SupportsNetwork calls may still be refreshing other chains' + // tables against the disabled URL; let them finish before swapping. + waitRefreshesIdle(t, v.cuCache) + orig := swapQuicknodeApiCreditsBaseURL(t, cuSrv.URL+"/v0/api-credits/") + defer func() { + waitRefreshesIdle(t, v.cuCache) + swapQuicknodeApiCreditsBaseURL(t, orig) + }() + + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 8453}} + _, err := v.SupportsNetwork(ctx, &logger, settings, "evm:8453") + require.NoError(t, err) + require.Eventually(t, func() bool { + return v.CreditUnits(qnCuReq(t, "eth_call"), ups) == 55 + }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, "/v0/api-credits/base", <-gotPath) + assert.Equal(t, int64(20), v.CreditUnits(qnCuReq(t, "eth_call"), &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}}), "other chains keep the built-in table") + }) +} + +func TestQuicknodeVendor_RefreshCreditUnitsAsync_Guards(t *testing.T) { + logger := zerolog.Nop() + + // Nil receiver and missing inputs are no-ops. + var nilVendor *QuicknodeVendor + nilVendor.refreshCreditUnitsAsync(&logger, "k", 1) + + v := CreateQuicknodeVendor().(*QuicknodeVendor) + v.refreshCreditUnitsAsync(&logger, "", 1) + v.refreshCreditUnitsAsync(&logger, "k", 0) + // No endpoints discovered yet: slug unknown, no refresh is started. + v.refreshCreditUnitsAsync(&logger, "k", 1) + assert.False(t, v.cuCache.Has("1")) + + // A fresh table short-circuits before any slug lookup. + primeRemoteCache(t, v.cuCache, "1", map[string]int64{"*": 99}) + v.refreshCreditUnitsAsync(&logger, "k", 1) + assert.Equal(t, int64(99), v.CreditUnits(qnCuReq(t, "eth_getBalance"), &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}})) + assert.Equal(t, int64(20), v.CreditUnits(qnCuReq(t, "eth_getBalance"), nil), "nil upstream uses built-in table") +} + +func TestQuicknodeVendor_SupportsNetworkAndGenerateConfigs_Inputs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("SupportsNetwork input handling", func(t *testing.T) { + v := CreateQuicknodeVendor() + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k"}, "solana:mainnet") + assert.NoError(t, err) + assert.False(t, ok) + + _, err = v.SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k"}, "evm:one") + assert.Error(t, err) + + ok, err = v.SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:1") + assert.NoError(t, err) + assert.False(t, ok, "no apiKey means not supported") + }) + + t.Run("GenerateConfigs validation", func(t *testing.T) { + v := CreateQuicknodeVendor() + _, err := v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{}, common.VendorSettings{}) + assert.ErrorContains(t, err, "apiKey is required") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{}, common.VendorSettings{"apiKey": "k"}) + assert.ErrorContains(t, err, "requires upstream.evm to be defined") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{}}, common.VendorSettings{"apiKey": "k"}) + assert.ErrorContains(t, err, "requires upstream.evm.chainId") + }) + + t.Run("GenerateConfigs cold cache is retryable", func(t *testing.T) { + v := CreateQuicknodeVendor().(*QuicknodeVendor) + // Mark discovery as in flight so the test never dials the Admin API. + v.cache.inflight["cold-key"] = struct{}{} + _, err := v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}}, common.VendorSettings{"apiKey": "cold-key", "recheckInterval": time.Minute}) + assert.ErrorIs(t, err, ErrRemoteCacheCold) + }) + + t.Run("GenerateConfigs keeps a preset endpoint", func(t *testing.T) { + v := CreateQuicknodeVendor() + ups := &common.UpstreamConfig{Endpoint: "https://x.quiknode.pro/abc/"} + got, err := v.GenerateConfigs(ctx, &logger, ups, nil) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Same(t, ups, got[0]) + assert.NotNil(t, got[0].JsonRpc) + }) +} + +func TestQuicknodeFilterParams_SingleValues(t *testing.T) { + v := CreateQuicknodeVendor().(*QuicknodeVendor) + got := v.extractFilterParams(common.VendorSettings{"tagIds": 7, "tagLabels": "prod"}) + assert.Equal(t, []int{7}, got.TagIDs) + assert.Equal(t, []string{"prod"}, got.TagLabels) + + got = v.extractFilterParams(common.VendorSettings{"tagIds": []interface{}{1, "x"}, "tagLabels": []interface{}{"a", 2}}) + assert.Equal(t, []int{1}, got.TagIDs, "non-int entries are dropped") + assert.Equal(t, []string{"a"}, got.TagLabels, "non-string entries are dropped") +} + +func TestQuicknodeVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateQuicknodeVendor() + + cases := []struct { + name string + method string + code int + msg string + wantCode common.ErrorCode + wantNormalized common.JsonRpcErrorNumber + networkRetry bool + }{ + {"-32614 range too large", "eth_call", -32614, "too large", common.ErrCodeEndpointRequestTooLarge, common.JsonRpcErrorEvmLargeRange, true}, + {"getLogs limited to", "eth_getLogs", -32000, "eth_getLogs is limited to a 10000 range", common.ErrCodeEndpointRequestTooLarge, common.JsonRpcErrorEvmLargeRange, true}, + {"-32009 capacity", "eth_call", -32009, "slow down", common.ErrCodeEndpointCapacityExceeded, common.JsonRpcErrorCapacityExceeded, true}, + {"-32007 capacity", "eth_call", -32007, "rate limited", common.ErrCodeEndpointCapacityExceeded, common.JsonRpcErrorCapacityExceeded, true}, + {"-32612 unsupported", "eth_call", -32612, "method not allowed", common.ErrCodeEndpointUnsupported, common.JsonRpcErrorCapacityExceeded, true}, + {"-32613 unsupported", "eth_call", -32613, "not on plan", common.ErrCodeEndpointUnsupported, common.JsonRpcErrorCapacityExceeded, true}, + {"parse failure", "eth_call", -32000, "failed to parse request", common.ErrCodeEndpointClientSideException, common.JsonRpcErrorParseException, false}, + {"-32010 gas limit", "eth_estimateGas", -32010, "transaction cost exceeds gas limit", common.ErrCodeEndpointClientSideException, common.JsonRpcErrorClientSideException, true}, + {"-32602 bad hex", "eth_getBalance", -32602, "cannot unmarshal hex string without 0x", common.ErrCodeEndpointClientSideException, common.JsonRpcErrorInvalidArgument, false}, + {"unauthorized", "eth_call", -32001, "UNAUTHORIZED: invalid token", common.ErrCodeEndpointUnauthorized, common.JsonRpcErrorUnauthorized, true}, + {"code 3 revert", "eth_call", 3, "execution reverted", common.ErrCodeEndpointExecutionException, common.JsonRpcErrorEvmReverted, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err, _ := vendorErr(t, v, c.method, 200, c.code, c.msg, "0xdead") + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, c.wantCode), "got %v", err) + assert.Equal(t, c.wantNormalized, normalizedCode(t, err)) + assert.Equal(t, c.networkRetry, common.IsRetryableTowardNetwork(err)) + se := err.(common.StandardError) + assert.Equal(t, "0xdead", se.DeepSearch("data"), "data travels with the error") + }) + } + + t.Run("unmatched errors and non json-rpc bodies fall through", func(t *testing.T) { + err, _ := vendorErr(t, v, "eth_call", 200, -32602, "invalid params", "") + assert.NoError(t, err) + err, _ = vendorErr(t, v, "eth_call", 200, -32000, "header not found", "") + assert.NoError(t, err) + err, _ = vendorErr(t, v, "eth_call", 200, -32000, "limited to 100", "") + assert.NoError(t, err, "'limited to' only maps for eth_getLogs") + assert.NoError(t, v.GetVendorSpecificErrorIfAny(nil, nil, map[string]string{}, map[string]interface{}{})) + }) +} + +func TestQuicknodeVendor_OwnsUpstream(t *testing.T) { + v := CreateQuicknodeVendor() + assert.Equal(t, "quicknode", v.Name()) + for ep, want := range map[string]bool{ + "quicknode://key": true, + "evm+quicknode://key": true, + "https://foo-bar.quiknode.pro/abc123/": true, + "https://rpc.ankr.com/eth": false, + } { + assert.Equal(t, want, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: ep}), ep) + } +} diff --git a/thirdparty/remote_cache_test.go b/thirdparty/remote_cache_test.go new file mode 100644 index 000000000..77595c130 --- /dev/null +++ b/thirdparty/remote_cache_test.go @@ -0,0 +1,175 @@ +package thirdparty + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func cacheIdle[T any](c *RemoteDataCache[T], key string) bool { + c.refreshMu.Lock() + defer c.refreshMu.Unlock() + _, busy := c.inflight[key] + return !busy +} + +func TestRemoteDataCache_LookupAndHas(t *testing.T) { + c := NewRemoteDataCache[int]("test") + + val, fresh := c.Lookup("a", time.Hour) + assert.Equal(t, 0, val) + assert.False(t, fresh) + assert.False(t, c.Has("a")) + + primeRemoteCache(t, c, "a", 7) + assert.True(t, c.Has("a")) + assert.False(t, c.Has("b"), "other keys stay absent once a snapshot exists") + + val, fresh = c.Lookup("b", time.Hour) + assert.Equal(t, 0, val) + assert.False(t, fresh) + + val, fresh = c.Lookup("a", time.Nanosecond) + assert.Equal(t, 7, val) + assert.False(t, fresh, "an entry older than the recheck interval is stale") +} + +func TestRemoteDataCache_PublishKeepsOtherKeys(t *testing.T) { + c := NewRemoteDataCache[string]("test") + primeRemoteCache(t, c, "a", "one") + require.Eventually(t, func() bool { return cacheIdle(c, "a") }, 5*time.Second, 10*time.Millisecond) + primeRemoteCache(t, c, "b", "two") + + va, _ := c.Lookup("a", time.Hour) + vb, _ := c.Lookup("b", time.Hour) + assert.Equal(t, "one", va) + assert.Equal(t, "two", vb) +} + +func TestRemoteDataCache_EnsureFresh(t *testing.T) { + logger := zerolog.Nop() + + t.Run("cold returns zero and triggers refresh", func(t *testing.T) { + c := NewRemoteDataCache[int]("test") + var calls atomic.Int32 + val, ok := c.EnsureFresh(&logger, "k", time.Hour, func(ctx context.Context) (int, error) { + calls.Add(1) + return 42, nil + }) + assert.Equal(t, 0, val) + assert.False(t, ok) + require.Eventually(t, func() bool { + v, fresh := c.Lookup("k", time.Hour) + return fresh && v == 42 + }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, int32(1), calls.Load()) + }) + + t.Run("fresh value returned without refresh", func(t *testing.T) { + c := NewRemoteDataCache[int]("test") + primeRemoteCache(t, c, "k", 5) + require.Eventually(t, func() bool { return cacheIdle(c, "k") }, 5*time.Second, 10*time.Millisecond) + val, ok := c.EnsureFresh(&logger, "k", time.Hour, func(ctx context.Context) (int, error) { + t.Error("fetcher must not run for a fresh value") + return 0, nil + }) + assert.Equal(t, 5, val) + assert.True(t, ok) + assert.True(t, cacheIdle(c, "k"), "no refresh is started for a fresh value") + }) + + t.Run("stale value returned and refreshed", func(t *testing.T) { + c := NewRemoteDataCache[int]("test") + primeRemoteCache(t, c, "k", 5) + require.Eventually(t, func() bool { return cacheIdle(c, "k") }, 5*time.Second, 10*time.Millisecond) + val, ok := c.EnsureFresh(&logger, "k", time.Nanosecond, func(ctx context.Context) (int, error) { + return 6, nil + }) + assert.Equal(t, 5, val) + assert.True(t, ok) + require.Eventually(t, func() bool { + v, _ := c.Lookup("k", time.Hour) + return v == 6 + }, 5*time.Second, 10*time.Millisecond) + }) +} + +func TestRemoteDataCache_FetcherErrorKeepsSnapshot(t *testing.T) { + logger := zerolog.Nop() + c := NewRemoteDataCache[int]("test") + primeRemoteCache(t, c, "k", 1) + require.Eventually(t, func() bool { return cacheIdle(c, "k") }, 5*time.Second, 10*time.Millisecond) + + done := make(chan struct{}) + c.TriggerAsyncRefresh(&logger, "k", func(ctx context.Context) (int, error) { + defer close(done) + return 99, errors.New("upstream down") + }) + <-done + require.Eventually(t, func() bool { return cacheIdle(c, "k") }, 5*time.Second, 10*time.Millisecond) + + val, fresh := c.Lookup("k", time.Hour) + assert.Equal(t, 1, val) + assert.True(t, fresh, "failed refresh does not touch fetchedAt either") +} + +func TestRemoteDataCache_SingleFlight(t *testing.T) { + logger := zerolog.Nop() + c := NewRemoteDataCache[int]("test") + + release := make(chan struct{}) + started := make(chan struct{}) + var calls atomic.Int32 + c.TriggerAsyncRefresh(&logger, "k", func(ctx context.Context) (int, error) { + calls.Add(1) + close(started) + <-release + return 1, nil + }) + <-started + + c.TriggerAsyncRefresh(&logger, "k", func(ctx context.Context) (int, error) { + calls.Add(1) + return 2, nil + }) + assert.False(t, cacheIdle(c, "k")) + + close(release) + require.Eventually(t, func() bool { + v, fresh := c.Lookup("k", time.Hour) + return fresh && v == 1 && cacheIdle(c, "k") + }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, int32(1), calls.Load(), "second trigger while the first is in flight is a no-op") + + // Once idle, a new trigger runs again. + c.TriggerAsyncRefresh(&logger, "k", func(ctx context.Context) (int, error) { + return 3, nil + }) + require.Eventually(t, func() bool { + v, fresh := c.Lookup("k", time.Hour) + return fresh && v == 3 + }, 5*time.Second, 10*time.Millisecond) +} + +func TestRemoteDataCache_PanicRecovered(t *testing.T) { + logger := zerolog.Nop() + c := NewRemoteDataCache[int]("test") + + c.TriggerAsyncRefresh(&logger, "k", func(ctx context.Context) (int, error) { + panic("fetcher exploded") + }) + require.Eventually(t, func() bool { return cacheIdle(c, "k") }, 5*time.Second, 10*time.Millisecond) + assert.False(t, c.Has("k")) + + // The in-flight marker was cleared, so a later refresh still publishes. + primeRemoteCache(t, c, "k", 8) + v, fresh := c.Lookup("k", time.Hour) + assert.Equal(t, 8, v) + assert.True(t, fresh) +} diff --git a/thirdparty/repository_test.go b/thirdparty/repository_test.go new file mode 100644 index 000000000..6503f2fa1 --- /dev/null +++ b/thirdparty/repository_test.go @@ -0,0 +1,197 @@ +package thirdparty + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRepository_FetchRemoteData(t *testing.T) { + ctx := context.Background() + + t.Run("parses numeric chain keys and skips others", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `{ + "1": {"endpoints": ["https://eth.a", "https://eth.b"]}, + "137": {"endpoints": []}, + "name": {"endpoints": ["https://ignored"]} + }`) + got, err := fetchRemoteData(ctx, srv.URL) + require.NoError(t, err) + assert.Equal(t, map[int64][]string{ + 1: {"https://eth.a", "https://eth.b"}, + 137: {}, + }, got) + }) + + t.Run("non-2xx status is an error", func(t *testing.T) { + srv := newJSONServer(t, http.StatusInternalServerError, `{}`) + _, err := fetchRemoteData(ctx, srv.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "500") + }) + + t.Run("invalid json is an error", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `[`) + _, err := fetchRemoteData(ctx, srv.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse remote repository data") + }) + + t.Run("invalid url and unreachable host are errors", func(t *testing.T) { + _, err := fetchRemoteData(ctx, "http://bad host/") + assert.Error(t, err) + _, err = fetchRemoteData(ctx, "http://127.0.0.1:1/") + assert.Error(t, err) + }) +} + +func TestRepositoryVendor_SupportsNetwork(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("rejects non-evm and malformed ids", func(t *testing.T) { + v := CreateRepositoryVendor() + ok, err := v.SupportsNetwork(ctx, &logger, nil, "solana:mainnet") + assert.NoError(t, err) + assert.False(t, ok) + _, err = v.SupportsNetwork(ctx, &logger, nil, "evm:1x") + assert.Error(t, err) + }) + + t.Run("cold cache then populated from repository url", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `{"1":{"endpoints":["https://eth.a"]},"10":{"endpoints":[]}}`) + v := CreateRepositoryVendor() + settings := common.VendorSettings{"repositoryUrl": srv.URL, "recheckInterval": time.Hour} + + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:1") + assert.ErrorIs(t, err, ErrRemoteCacheCold) + assert.False(t, ok) + + require.Eventually(t, func() bool { + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:1") + return err == nil && ok + }, 5*time.Second, 10*time.Millisecond) + + ok, err = v.SupportsNetwork(ctx, &logger, settings, "evm:10") + assert.NoError(t, err) + assert.False(t, ok, "chain with no endpoints is unsupported") + }) + + t.Run("default repository url is used when unset", func(t *testing.T) { + v := CreateRepositoryVendor().(*RepositoryVendor) + primeRemoteCache(t, v.cache, DefaultRepositoryURL, map[int64][]string{1: {"https://eth.a"}}) + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:1") + require.NoError(t, err) + assert.True(t, ok) + }) +} + +func TestRepositoryVendor_GenerateConfigs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + const repoUrl = "https://repo.example/endpoints" + + newPrimed := func(t *testing.T) *RepositoryVendor { + v := CreateRepositoryVendor().(*RepositoryVendor) + primeRemoteCache(t, v.cache, repoUrl, map[int64][]string{ + 1: {"https://eth.a/key", "wss://eth.ws", "HTTP://eth.b"}, + 10: {}, + }) + return v + } + + t.Run("generates one http upstream per endpoint with copied settings", func(t *testing.T) { + v := newPrimed(t) + budget := "public" + ups := &common.UpstreamConfig{ + Id: "repo", + Tags: []string{"public"}, + Evm: &common.EvmUpstreamConfig{ChainId: 1}, + JsonRpc: &common.JsonRpcUpstreamConfig{BatchMaxSize: 7}, + Failsafe: []*common.FailsafeConfig{{MatchMethod: "eth_call"}}, + RateLimitAutoTune: &common.RateLimitAutoTuneConfig{}, + RateLimitBudget: budget, + IgnoreMethods: []string{"debug_*"}, + } + got, err := v.GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"repositoryUrl": repoUrl, "recheckInterval": time.Hour}) + require.NoError(t, err) + require.Len(t, got, 2, "non-http endpoints are skipped") + + assert.Equal(t, "https://eth.a/key", got[0].Endpoint) + assert.Equal(t, "HTTP://eth.b", got[1].Endpoint) + for _, c := range got { + assert.Equal(t, common.UpstreamTypeEvm, c.Type) + assert.Equal(t, "repo-"+util.RedactEndpoint(c.Endpoint), c.Id) + assert.Equal(t, []string{"public"}, c.Tags) + assert.Equal(t, budget, c.RateLimitBudget) + assert.Equal(t, []string{"debug_*"}, c.IgnoreMethods) + require.NotNil(t, c.AutoIgnoreUnsupportedMethods) + assert.True(t, *c.AutoIgnoreUnsupportedMethods, "auto-ignore defaults to true") + require.NotNil(t, c.Evm) + assert.NotSame(t, ups.Evm, c.Evm, "evm config is deep-copied") + assert.Equal(t, int64(1), c.Evm.ChainId) + require.NotNil(t, c.JsonRpc) + assert.NotSame(t, ups.JsonRpc, c.JsonRpc) + assert.Equal(t, 7, c.JsonRpc.BatchMaxSize) + require.Len(t, c.Failsafe, 1) + assert.NotSame(t, ups.Failsafe[0], c.Failsafe[0]) + assert.Equal(t, "eth_call", c.Failsafe[0].MatchMethod) + require.NotNil(t, c.RateLimitAutoTune) + assert.NotSame(t, ups.RateLimitAutoTune, c.RateLimitAutoTune) + } + }) + + t.Run("explicit auto-ignore false is kept", func(t *testing.T) { + v := newPrimed(t) + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}, AutoIgnoreUnsupportedMethods: util.BoolPtr(false)} + got, err := v.GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"repositoryUrl": repoUrl}) + require.NoError(t, err) + require.NotEmpty(t, got) + assert.False(t, *got[0].AutoIgnoreUnsupportedMethods) + assert.Nil(t, got[0].Tags) + assert.Nil(t, got[0].Failsafe) + }) + + t.Run("validation errors", func(t *testing.T) { + v := newPrimed(t) + s := common.VendorSettings{"repositoryUrl": repoUrl} + + _, err := v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{}, s) + assert.ErrorContains(t, err, "requires upstream.evm to be defined") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{}}, s) + assert.ErrorContains(t, err, "requires upstream.evm.chainId") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 10}}, s) + assert.ErrorContains(t, err, "chain ID 10 not found") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 99}}, s) + assert.ErrorContains(t, err, "chain ID 99 not found") + }) + + t.Run("cold cache on default url is retryable", func(t *testing.T) { + v := CreateRepositoryVendor().(*RepositoryVendor) + // Mark a refresh for the default URL as in flight so nothing dials out. + v.cache.inflight[DefaultRepositoryURL] = struct{}{} + _, err := v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}}, common.VendorSettings{}) + assert.ErrorIs(t, err, ErrRemoteCacheCold) + }) +} + +func TestRepositoryVendor_OwnershipAndErrors(t *testing.T) { + v := CreateRepositoryVendor() + assert.Equal(t, "repository", v.Name()) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "repository://evm-public-endpoints.erpc.cloud"})) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "evm+repository://x"})) + assert.False(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://evm-public-endpoints.erpc.cloud"})) + + err, _ := vendorErr(t, v, "eth_call", 500, -32000, "boom", "") + assert.NoError(t, err) +} diff --git a/thirdparty/routemesh_test.go b/thirdparty/routemesh_test.go new file mode 100644 index 000000000..4dec618db --- /dev/null +++ b/thirdparty/routemesh_test.go @@ -0,0 +1,168 @@ +package thirdparty + +import ( + "context" + "net/http" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// gockProbeReply mocks one JSON-RPC endpoint (host+path) with a raw body for +// the eth_chainId probes issued by vendor SupportsNetwork implementations. +func gockProbeReply(host, path, body string) { + gock.New(host).Post(path).Persist().Reply(http.StatusOK). + SetHeader("Content-Type", "application/json").BodyString(body) +} + +func TestRoutemeshVendor_SupportsNetwork(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + probeCases := []struct { + name string + settings common.VendorSettings + host string + path string + body string + networkId string + want bool + wantErr bool + }{ + {"default base url match", common.VendorSettings{"apiKey": "k1"}, "https://lb.routemes.sh", "/rpc/1/k1", `{"jsonrpc":"2.0","id":1,"result":"0x1"}`, "evm:1", true, false}, + {"custom base url match", common.VendorSettings{"apiKey": "k1", "baseURL": "eu.routemes.sh"}, "https://eu.routemes.sh", "/rpc/10/k1", `{"jsonrpc":"2.0","id":1,"result":"0xa"}`, "evm:10", true, false}, + {"mismatching chain id", common.VendorSettings{"apiKey": "k1"}, "https://lb.routemes.sh", "/rpc/1/k1", `{"jsonrpc":"2.0","id":1,"result":"0x89"}`, "evm:1", false, false}, + {"json-rpc error", common.VendorSettings{"apiKey": "k1"}, "https://lb.routemes.sh", "/rpc/1/k1", `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"boom"}}`, "evm:1", false, true}, + {"non-hex result", common.VendorSettings{"apiKey": "k1"}, "https://lb.routemes.sh", "/rpc/1/k1", `{"jsonrpc":"2.0","id":1,"result":"zz"}`, "evm:1", false, true}, + {"non-string result", common.VendorSettings{"apiKey": "k1"}, "https://lb.routemes.sh", "/rpc/1/k1", `{"jsonrpc":"2.0","id":1,"result":{}}`, "evm:1", false, true}, + } + for _, c := range probeCases { + t.Run(c.name, func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gockProbeReply(c.host, c.path, c.body) + + ok, err := CreateRoutemeshVendor().SupportsNetwork(ctx, &logger, c.settings, c.networkId) + if c.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, c.want, ok) + }) + } + + t.Run("cached client reused", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gockProbeReply("https://lb.routemes.sh", "/rpc/1/k1", `{"jsonrpc":"2.0","id":1,"result":"0x1"}`) + v := CreateRoutemeshVendor() + for range 2 { + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1"}, "evm:1") + require.NoError(t, err) + assert.True(t, ok) + } + }) + + t.Run("unreachable endpoint", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://lb.routemes.sh").Post("/rpc/1/k1").Persist().ReplyError(assert.AnError) + ok, err := CreateRoutemeshVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1"}, "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("non-evm network", func(t *testing.T) { + ok, err := CreateRoutemeshVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1"}, "solana:mainnet") + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("unparsable chain id", func(t *testing.T) { + ok, err := CreateRoutemeshVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1"}, "evm:one") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("missing apiKey", func(t *testing.T) { + ok, err := CreateRoutemeshVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:1") + assert.ErrorContains(t, err, "apiKey is required in routemesh settings") + assert.False(t, ok) + }) + + t.Run("malformed base url", func(t *testing.T) { + ok, err := CreateRoutemeshVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"apiKey": "k1", "baseURL": "bad%zzhost"}, "evm:1") + assert.Error(t, err) + assert.False(t, ok) + }) +} + +func TestRoutemeshVendor_GenerateConfigs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + cases := []struct { + name string + endpoint string + settings common.VendorSettings + expected string + }{ + {"default base url", "", common.VendorSettings{"apiKey": "k1"}, "https://lb.routemes.sh/rpc/137/k1"}, + {"custom base url", "", common.VendorSettings{"apiKey": "k1", "baseURL": "eu.routemes.sh"}, "https://eu.routemes.sh/rpc/137/k1"}, + {"preset endpoint passes through", "https://lb.routemes.sh/rpc/137/other", nil, "https://lb.routemes.sh/rpc/137/other"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ups := &common.UpstreamConfig{Endpoint: c.endpoint, Evm: &common.EvmUpstreamConfig{ChainId: 137}} + cfgs, err := CreateRoutemeshVendor().GenerateConfigs(ctx, &logger, ups, c.settings) + require.NoError(t, err) + require.Len(t, cfgs, 1) + assert.Equal(t, c.expected, cfgs[0].Endpoint) + assert.NotNil(t, cfgs[0].JsonRpc) + }) + } + + t.Run("generated endpoint is typed evm", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 137}} + cfgs, err := CreateRoutemeshVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"apiKey": "k1"}) + require.NoError(t, err) + assert.Equal(t, common.UpstreamTypeEvm, cfgs[0].Type) + }) + + errCases := []struct { + name string + chainId int64 + settings common.VendorSettings + contains string + }{ + {"missing apiKey", 137, common.VendorSettings{}, "apiKey is required in routemesh settings"}, + {"missing chain id", 0, common.VendorSettings{"apiKey": "k1"}, "requires upstream.evm.chainId"}, + {"malformed base url", 137, common.VendorSettings{"apiKey": "k1", "baseURL": "bad%zzhost"}, "invalid URL escape"}, + } + for _, c := range errCases { + t.Run(c.name, func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: c.chainId}} + _, err := CreateRoutemeshVendor().GenerateConfigs(ctx, &logger, ups, c.settings) + assert.ErrorContains(t, err, c.contains) + }) + } +} + +func TestRoutemeshVendor_ErrorsAndOwnership(t *testing.T) { + v := CreateRoutemeshVendor() + assert.Equal(t, "routemesh", v.Name()) + + err, details := vendorErr(t, v, "eth_getLogs", 500, -32000, "boom", "d") + assert.NoError(t, err) + assert.Empty(t, details) + + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{VendorName: "routemesh"})) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://lb.routemes.sh/rpc/1/k"})) + assert.False(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://rpc.example.com"})) +} diff --git a/thirdparty/satelink_test.go b/thirdparty/satelink_test.go index 28d305e98..c21d9cccb 100644 --- a/thirdparty/satelink_test.go +++ b/thirdparty/satelink_test.go @@ -7,6 +7,7 @@ import ( "github.com/erpc/erpc/common" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSatelinkVendor_SupportsNetwork(t *testing.T) { @@ -18,9 +19,9 @@ func TestSatelinkVendor_SupportsNetwork(t *testing.T) { networkId string expected bool }{ - {"evm:137", true}, // Polygon mainnet - {"evm:1", false}, // Ethereum — not supported - {"evm:999999", false}, // unknown + {"evm:137", true}, // Polygon mainnet + {"evm:1", false}, // Ethereum — not supported + {"evm:999999", false}, // unknown {"solana:mainnet", false}, } @@ -64,3 +65,99 @@ func TestSatelinkVendor_GenerateConfigs(t *testing.T) { assert.Empty(t, cfgs) }) } + +func TestSatelinkVendor_Name(t *testing.T) { + assert.Equal(t, "satelink", CreateSatelinkVendor().Name()) +} + +func TestSatelinkVendor_SupportsNetworkInvalidChain(t *testing.T) { + svtCheckSupports(t, CreateSatelinkVendor(), common.VendorSettings{}, []svtSupportCase{ + {"evm:abc", false, true}, + }) +} + +func TestSatelinkVendor_GenerateConfigsBranches(t *testing.T) { + svtCheckGenErrors(t, CreateSatelinkVendor, []svtErrCase{ + {"nil evm", &common.UpstreamConfig{}, common.VendorSettings{}, "satelink vendor requires upstream.evm to be defined"}, + {"zero chain", svtEvm(0), common.VendorSettings{}, "satelink vendor requires upstream.evm.chainId to be defined"}, + }) + + t.Run("explicit free sentinel sends no key header", func(t *testing.T) { + cfg, err := svtGenerate(t, CreateSatelinkVendor(), svtEvm(137), common.VendorSettings{"apiKey": "free"}) + require.NoError(t, err) + assert.Equal(t, "https://rpc.satelink.network/rpc/polygon", cfg.Endpoint) + assert.Equal(t, common.UpstreamTypeEvm, cfg.Type) + assert.NotContains(t, cfg.JsonRpc.Headers, "X-API-Key") + }) + + t.Run("caller-set key header wins", func(t *testing.T) { + for _, h := range []string{"X-API-Key", "x-api-key"} { + ups := svtEvm(137) + ups.JsonRpc = &common.JsonRpcUpstreamConfig{Headers: map[string]string{h: "mine"}} + cfg, err := svtGenerate(t, CreateSatelinkVendor(), ups, common.VendorSettings{"apiKey": "sk"}) + require.NoError(t, err) + assert.Equal(t, map[string]string{h: "mine"}, cfg.JsonRpc.Headers, h) + } + }) + + t.Run("preset endpoint passes through", func(t *testing.T) { + cfg := svtCheckPreset(t, CreateSatelinkVendor(), "https://rpc.satelink.network/rpc/polygon") + assert.Nil(t, cfg.JsonRpc.Headers) + }) +} + +func TestSatelinkVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + v := CreateSatelinkVendor() + + t.Run("402 is a billing issue with payment pointers", func(t *testing.T) { + err, details := vendorErr(t, v, "eth_getLogs", 402, -32000, "no credits", "d") + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointBillingIssue), "got %v", err) + assert.Equal(t, common.JsonRpcErrorCapacityExceeded, normalizedCode(t, err)) + assert.Contains(t, err.Error(), "satelink credits exhausted") + assert.Contains(t, err.Error(), "no credits") + for k, want := range map[string]interface{}{ + "data": "d", + "paymentManifestUrl": "https://rpc.satelink.network/.well-known/satelink.json", + "paymentVaultAddress": "0x577D3716d6Ad5b676d230f5409deF9838FABaCEF", + "paymentTokenAddress": "0xc2132D05D31c914a87C6611C10748AEb04B58e8F", + "paymentChainId": 137, + "paymentCalldataUrl": "https://rpc.satelink.network/credits/deposit/initiate?amount=", + "pricingUrl": "https://rpc.satelink.network/v1/pricing", + } { + assert.Equal(t, want, details[k], k) + } + }) + + t.Run("429 is capacity exceeded", func(t *testing.T) { + err, _ := vendorErr(t, v, "eth_getLogs", 429, -32000, "daily limit", "") + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointCapacityExceeded), "got %v", err) + assert.Equal(t, common.JsonRpcErrorCapacityExceeded, normalizedCode(t, err)) + }) + + t.Run("401 is unauthorized", func(t *testing.T) { + err, _ := vendorErr(t, v, "eth_getLogs", 401, -32000, "bad key", "") + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointUnauthorized), "got %v", err) + assert.Equal(t, common.JsonRpcErrorUnauthorized, normalizedCode(t, err)) + }) + + t.Run("other status and nil response are generic", func(t *testing.T) { + svtCheckDataOnly(t, v) + jrr, err := common.NewJsonRpcResponse(1, nil, common.NewErrJsonRpcExceptionExternal(-32000, "x", "d")) + require.NoError(t, err) + details := map[string]interface{}{} + assert.NoError(t, v.GetVendorSpecificErrorIfAny(svtReq(), nil, jrr, details)) + assert.Equal(t, "d", details["data"]) + }) +} + +func TestSatelinkVendor_OwnsUpstream(t *testing.T) { + svtAssertOwns(t, CreateSatelinkVendor(), map[string]bool{ + "satelink://key": true, + "evm+satelink://key": true, + "https://rpc.satelink.network/rpc/polygon": true, + "https://rpc.ankr.com/eth/key": false, + }) +} diff --git a/thirdparty/spectrum_test.go b/thirdparty/spectrum_test.go index 897494951..c23f2751c 100644 --- a/thirdparty/spectrum_test.go +++ b/thirdparty/spectrum_test.go @@ -71,3 +71,44 @@ func TestSpectrumVendor_GenerateConfigs(t *testing.T) { assert.Error(t, err) }) } + +func TestSpectrumVendor_Name(t *testing.T) { + assert.Equal(t, "spectrum", CreateSpectrumVendor().Name()) +} + +func TestSpectrumVendor_SupportsNetwork(t *testing.T) { + svtCheckSupports(t, CreateSpectrumVendor(), common.VendorSettings{}, []svtSupportCase{ + {"evm:1", true, false}, + {"evm:686868", true, false}, + {"evm:999999999", false, false}, + {"solana:mainnet", false, false}, + {"evm:abc", false, false}, // spectrum swallows parse errors + }) +} + +func TestSpectrumVendor_GenerateConfigsBranches(t *testing.T) { + key := common.VendorSettings{"apiKey": "KEY"} + svtCheckGenErrors(t, CreateSpectrumVendor, []svtErrCase{ + {"nil evm", &common.UpstreamConfig{}, key, "spectrum vendor requires upstream.evm.chainId to be defined"}, + {"zero chain", svtEvm(0), key, "spectrum vendor requires upstream.evm.chainId to be defined"}, + }) + svtCheckURLs(t, CreateSpectrumVendor, key, map[int64]string{ + 1: "https://spectrum-03.simplystaking.xyz/KEY/ethereum/mn/1/shared/archive/rpc/", + }) + svtCheckPreset(t, CreateSpectrumVendor(), "https://spectrum-03.simplystaking.xyz/own/ethereum/mn/1/shared/archive/rpc/") +} + +func TestSpectrumVendor_GetVendorSpecificErrorIfAny(t *testing.T) { + err, _ := vendorErr(t, CreateSpectrumVendor(), "eth_getLogs", 401, -32000, "unauthorized", "d") + assert.NoError(t, err, "spectrum defers every error to generic handling") +} + +func TestSpectrumVendor_OwnsUpstream(t *testing.T) { + svtAssertOwns(t, CreateSpectrumVendor(), map[string]bool{ + "spectrum://key": true, + "evm+spectrum://key": true, + "https://spectrum-03.simplystaking.xyz/KEY/ethereum/mn/1/shared/archive/rpc/": true, + "https://simplystaking.xyz/": false, + "https://rpc.ankr.com/eth/key": false, + }) +} diff --git a/thirdparty/superchain_test.go b/thirdparty/superchain_test.go new file mode 100644 index 000000000..a60898546 --- /dev/null +++ b/thirdparty/superchain_test.go @@ -0,0 +1,257 @@ +package thirdparty + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseSuperchainSpec(t *testing.T) { + cases := []struct { + name string + spec string + want string + }{ + {"shorthand org/repo defaults branch and file", "github.com/acme/registry", "https://raw.githubusercontent.com/acme/registry/main/chainList.json"}, + {"https github org/repo", "https://github.com/acme/registry", "https://raw.githubusercontent.com/acme/registry/main/chainList.json"}, + {"http github org/repo", "http://github.com/acme/registry/", "https://raw.githubusercontent.com/acme/registry/main/chainList.json"}, + {"branch only", "github.com/acme/registry/dev", "https://raw.githubusercontent.com/acme/registry/dev/chainList.json"}, + {"branch and file", "github.com/acme/registry/dev/custom.json", "https://raw.githubusercontent.com/acme/registry/dev/custom.json"}, + {"branch and nested file", "github.com/acme/registry/dev/lists/custom.json", "https://raw.githubusercontent.com/acme/registry/dev/lists/custom.json"}, + {"file without branch implies main", "github.com/acme/registry/custom.json", "https://raw.githubusercontent.com/acme/registry/main/custom.json"}, + {"blob UI url drops blob segment", "https://github.com/acme/registry/blob/release/chainList.json", "https://raw.githubusercontent.com/acme/registry/release/chainList.json"}, + {"tree UI url drops tree segment", "github.com/acme/registry/tree/release", "https://raw.githubusercontent.com/acme/registry/release/chainList.json"}, + {"raw url kept verbatim", "https://raw.githubusercontent.com/a/b/main/x.json", "https://raw.githubusercontent.com/a/b/main/x.json"}, + {"http custom url kept verbatim", "http://registry.local/chains.json", "http://registry.local/chains.json"}, + {"bare host gets https", "mysuperchain.com/chainList.json", "https://mysuperchain.com/chainList.json"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := parseSuperchainSpec(c.spec) + require.NoError(t, err) + assert.Equal(t, c.want, got) + }) + } + + t.Run("github prefix without repo is rejected", func(t *testing.T) { + _, err := parseSuperchainSpec("github.com/acme") + require.Error(t, err) + assert.Contains(t, err.Error(), "org/repo not found") + }) +} + +const superchainRegistryBody = `[ + {"chainId": 10, "rpc": ["https://mainnet.optimism.io", "https://op.backup.io"]}, + {"chainId": 8453, "rpc": ["https://mainnet.base.org"]}, + {"chainId": 999, "rpc": []}, + {"chainId": 0, "rpc": ["https://ignored.example"]} +]` + +func TestSuperchainVendor_FetchNetworks(t *testing.T) { + v := CreateSuperchainVendor().(*SuperchainVendor) + ctx := context.Background() + + t.Run("keeps chains with positive id and at least one rpc", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, superchainRegistryBody) + got, err := v.fetchSuperchainNetworks(ctx, srv.URL) + require.NoError(t, err) + assert.Equal(t, map[int64][]string{ + 10: {"https://mainnet.optimism.io", "https://op.backup.io"}, + 8453: {"https://mainnet.base.org"}, + }, got) + }) + + t.Run("non-2xx status is an error", func(t *testing.T) { + srv := newJSONServer(t, http.StatusNotFound, `not found`) + _, err := v.fetchSuperchainNetworks(ctx, srv.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "404") + }) + + t.Run("invalid json is an error", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, `{"not":"a list"}`) + _, err := v.fetchSuperchainNetworks(ctx, srv.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse Superchain registry data") + }) + + t.Run("invalid url is an error", func(t *testing.T) { + _, err := v.fetchSuperchainNetworks(ctx, "http://bad host/") + require.Error(t, err) + }) + + t.Run("unreachable server is an error", func(t *testing.T) { + _, err := v.fetchSuperchainNetworks(ctx, "http://127.0.0.1:1/chainList.json") + require.Error(t, err) + }) +} + +func TestSuperchainVendor_SupportsNetwork(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("rejects non-evm and malformed ids", func(t *testing.T) { + v := CreateSuperchainVendor() + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{}, "solana:mainnet") + assert.NoError(t, err) + assert.False(t, ok) + + ok, err = v.SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:abc") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("invalid registry spec is an error", func(t *testing.T) { + v := CreateSuperchainVendor() + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{"registryUrl": "github.com/only-org"}, "evm:10") + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse superchain registry URL") + assert.False(t, ok) + }) + + t.Run("cold cache then populated from registry", func(t *testing.T) { + srv := newJSONServer(t, http.StatusOK, superchainRegistryBody) + v := CreateSuperchainVendor() + settings := common.VendorSettings{"registryUrl": srv.URL, "recheckInterval": time.Hour} + + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:10") + assert.ErrorIs(t, err, ErrRemoteCacheCold) + assert.False(t, ok) + + require.Eventually(t, func() bool { + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:10") + return err == nil && ok + }, 5*time.Second, 10*time.Millisecond) + + ok, err = v.SupportsNetwork(ctx, &logger, settings, "evm:999") + assert.NoError(t, err) + assert.False(t, ok, "chain without rpc endpoints is unsupported") + + ok, err = v.SupportsNetwork(ctx, &logger, settings, "evm:12345") + assert.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("default registry url is used when unset", func(t *testing.T) { + v := CreateSuperchainVendor().(*SuperchainVendor) + primeRemoteCache(t, v.cache, DefaultSuperchainRegistryURL, map[int64][]string{10: {"https://op.example"}}) + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:10") + require.NoError(t, err) + assert.True(t, ok) + }) +} + +func TestSuperchainVendor_GenerateConfigs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + const registry = "https://registry.example/chainList.json" + networks := map[int64][]string{10: {"https://rpc-a.example", "https://rpc-b.example"}, 77: {}} + + newPrimed := func(t *testing.T) *SuperchainVendor { + v := CreateSuperchainVendor().(*SuperchainVendor) + primeRemoteCache(t, v.cache, registry, networks) + return v + } + + t.Run("generates one upstream per rpc with indexed ids", func(t *testing.T) { + v := newPrimed(t) + ups := &common.UpstreamConfig{Id: "op", Evm: &common.EvmUpstreamConfig{ChainId: 10}} + got, err := v.GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"registryUrl": registry}) + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "op-0", got[0].Id) + assert.Equal(t, "https://rpc-a.example", got[0].Endpoint) + assert.Equal(t, "op-1", got[1].Id) + assert.Equal(t, "https://rpc-b.example", got[1].Endpoint) + for _, c := range got { + assert.Equal(t, common.UpstreamTypeEvm, c.Type) + assert.Equal(t, "superchain", c.VendorName) + assert.NotNil(t, c.JsonRpc) + } + }) + + t.Run("ids are derived from chain id when upstream id is empty", func(t *testing.T) { + v := newPrimed(t) + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 10}} + got, err := v.GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"registryUrl": registry, "recheckInterval": time.Hour}) + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "superchain-10-0", got[0].Id) + assert.Equal(t, "superchain-10-1", got[1].Id) + }) + + t.Run("superchain:// endpoint selects the registry", func(t *testing.T) { + v := CreateSuperchainVendor().(*SuperchainVendor) + raw := "https://raw.githubusercontent.com/acme/registry/main/chainList.json" + primeRemoteCache(t, v.cache, raw, networks) + for _, ep := range []string{"superchain://github.com/acme/registry", "evm+superchain://github.com/acme/registry"} { + ups := &common.UpstreamConfig{Endpoint: ep, Evm: &common.EvmUpstreamConfig{ChainId: 10}} + got, err := v.GenerateConfigs(ctx, &logger, ups, nil) + require.NoError(t, err, ep) + require.Len(t, got, 2, ep) + assert.Equal(t, "https://rpc-a.example", got[0].Endpoint) + } + }) + + t.Run("superchain:// endpoint with bad spec is an error", func(t *testing.T) { + v := CreateSuperchainVendor() + ups := &common.UpstreamConfig{Endpoint: "superchain://github.com/only-org", Evm: &common.EvmUpstreamConfig{ChainId: 10}} + _, err := v.GenerateConfigs(ctx, &logger, ups, common.VendorSettings{}) + require.Error(t, err) + }) + + t.Run("plain endpoint passes through", func(t *testing.T) { + v := CreateSuperchainVendor() + ups := &common.UpstreamConfig{Endpoint: "https://custom.example", Evm: &common.EvmUpstreamConfig{ChainId: 10}} + got, err := v.GenerateConfigs(ctx, &logger, ups, common.VendorSettings{}) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Same(t, ups, got[0]) + assert.NotNil(t, got[0].JsonRpc) + }) + + t.Run("validation errors", func(t *testing.T) { + v := newPrimed(t) + settings := common.VendorSettings{"registryUrl": registry} + + _, err := v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{}, settings) + assert.ErrorContains(t, err, "requires upstream.evm") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{}}, settings) + assert.ErrorContains(t, err, "requires upstream.evm.chainId") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 10}}, common.VendorSettings{"registryUrl": "github.com/x"}) + assert.ErrorContains(t, err, "failed to parse superchain registry URL") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 77}}, settings) + assert.ErrorContains(t, err, "chain ID 77 not found") + + _, err = v.GenerateConfigs(ctx, &logger, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 5}}, settings) + assert.ErrorContains(t, err, "chain ID 5 not found") + }) + + t.Run("cold cache is retryable", func(t *testing.T) { + v := CreateSuperchainVendor() + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 10}} + _, err := v.GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"registryUrl": "http://127.0.0.1:1/never.json"}) + assert.ErrorIs(t, err, ErrRemoteCacheCold) + }) +} + +func TestSuperchainVendor_OwnershipAndErrors(t *testing.T) { + v := CreateSuperchainVendor() + assert.Equal(t, "superchain", v.Name()) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "superchain://github.com/a/b"})) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "evm+superchain://github.com/a/b"})) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://rpc.example", VendorName: "superchain"})) + assert.False(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://mainnet.optimism.io"})) + + err, _ := vendorErr(t, v, "eth_call", 500, -32000, "boom", "") + assert.NoError(t, err, "superchain defers all errors to generic handling") +} diff --git a/thirdparty/tenderly_test.go b/thirdparty/tenderly_test.go new file mode 100644 index 000000000..51dbc277a --- /dev/null +++ b/thirdparty/tenderly_test.go @@ -0,0 +1,165 @@ +package thirdparty + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newPrimedTenderlyVendor returns a vendor whose supported-networks cache is +// already populated, so no call reaches the (non-configurable) Tenderly API. +func newPrimedTenderlyVendor(t *testing.T, networks map[int64]string) *TenderlyVendor { + t.Helper() + v := CreateTenderlyVendor().(*TenderlyVendor) + primeRemoteCache(t, v.cache, tenderlyApiUrl, networks) + return v +} + +// newColdTenderlyVendor returns a vendor whose cache stays cold: a refresh is +// marked in flight so resolveNetworks' TriggerAsyncRefresh is a no-op and never +// dials the real Tenderly API. +func newColdTenderlyVendor() *TenderlyVendor { + v := CreateTenderlyVendor().(*TenderlyVendor) + v.cache.inflight[tenderlyApiUrl] = struct{}{} + return v +} + +func TestTenderlyVendor_SupportsNetwork(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + v := newPrimedTenderlyVendor(t, map[int64]string{1: "mainnet", 8453: "base"}) + + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:8453") + require.NoError(t, err) + assert.True(t, ok) + + ok, err = v.SupportsNetwork(ctx, &logger, common.VendorSettings{"recheckInterval": time.Hour}, "evm:999") + require.NoError(t, err) + assert.False(t, ok) + + ok, err = v.SupportsNetwork(ctx, &logger, common.VendorSettings{}, "solana:mainnet") + require.NoError(t, err) + assert.False(t, ok) + + ok, err = v.SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:abc") + assert.Error(t, err) + assert.False(t, ok) +} + +func TestTenderlyVendor_SupportsNetwork_ColdStart(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + ok, err := newColdTenderlyVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:1") + assert.ErrorIs(t, err, ErrRemoteCacheCold) + assert.False(t, ok) +} + +func TestTenderlyVendor_SupportsNetwork_StaleUsesCachedData(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + v := newPrimedTenderlyVendor(t, map[int64]string{1: "mainnet"}) + // Block the stale-triggered refresh so it cannot reach the real API. Wait for + // the priming refresh to clear its in-flight marker first, or its deferred + // cleanup would delete ours. + require.Eventually(t, func() bool { + v.cache.refreshMu.Lock() + defer v.cache.refreshMu.Unlock() + if _, busy := v.cache.inflight[tenderlyApiUrl]; busy { + return false + } + v.cache.inflight[tenderlyApiUrl] = struct{}{} + return true + }, 5*time.Second, 10*time.Millisecond) + + // A 1ns recheck interval makes the entry stale: the cached data still answers. + ok, err := v.SupportsNetwork(ctx, &logger, common.VendorSettings{"recheckInterval": time.Nanosecond}, "evm:1") + require.NoError(t, err) + assert.True(t, ok) +} + +func TestTenderlyVendor_GenerateConfigs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + v := newPrimedTenderlyVendor(t, map[int64]string{1: "mainnet", 8453: "base"}) + + t.Run("generated endpoint", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 8453}} + cfgs, err := v.GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"apiKey": "key1"}) + require.NoError(t, err) + require.Len(t, cfgs, 1) + assert.Equal(t, "https://base.gateway.tenderly.co/key1", cfgs[0].Endpoint) + assert.Equal(t, common.UpstreamTypeEvm, cfgs[0].Type) + assert.Equal(t, "tenderly", cfgs[0].VendorName) + assert.NotNil(t, cfgs[0].JsonRpc) + }) + + t.Run("preset endpoint passes through and gets vendor name", func(t *testing.T) { + ups := &common.UpstreamConfig{Endpoint: "https://mainnet.gateway.tenderly.co/mine"} + cfgs, err := v.GenerateConfigs(ctx, &logger, ups, nil) + require.NoError(t, err) + assert.Equal(t, "https://mainnet.gateway.tenderly.co/mine", cfgs[0].Endpoint) + assert.Equal(t, "tenderly", cfgs[0].VendorName) + }) + + errCases := []struct { + name string + vendor *TenderlyVendor + ups *common.UpstreamConfig + settings common.VendorSettings + contains string + is error + }{ + {"missing apiKey", v, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}}, common.VendorSettings{}, "apiKey is required in tenderly settings", nil}, + {"missing evm", v, &common.UpstreamConfig{}, common.VendorSettings{"apiKey": "k"}, "requires upstream.evm to be defined", nil}, + {"missing chain id", v, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{}}, common.VendorSettings{"apiKey": "k"}, "requires upstream.evm.chainId", nil}, + {"unsupported chain", v, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 999}}, common.VendorSettings{"apiKey": "k", "recheckInterval": time.Hour}, "unsupported network chain ID for Tenderly: 999", nil}, + {"malformed api key", v, &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}}, common.VendorSettings{"apiKey": "%zz"}, "invalid URL escape", nil}, + {"cold cache", newColdTenderlyVendor(), &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}}, common.VendorSettings{"apiKey": "k"}, "", ErrRemoteCacheCold}, + } + for _, c := range errCases { + t.Run(c.name, func(t *testing.T) { + _, err := c.vendor.GenerateConfigs(ctx, &logger, c.ups, c.settings) + if c.is != nil { + assert.ErrorIs(t, err, c.is) + } else { + assert.ErrorContains(t, err, c.contains) + } + }) + } +} + +func TestTenderlyVendor_ErrorsAndOwnership(t *testing.T) { + v := CreateTenderlyVendor() + assert.Equal(t, "tenderly", v.Name()) + + err, details := vendorErr(t, v, "eth_call", 500, -32000, "boom", "revert-data") + assert.NoError(t, err) + assert.Equal(t, "revert-data", details["data"]) + + err, details = vendorErr(t, v, "eth_call", 500, -32000, "boom", "") + assert.NoError(t, err) + assert.NotContains(t, details, "data") + + assert.NoError(t, v.GetVendorSpecificErrorIfAny(nil, &http.Response{StatusCode: 500}, nil, map[string]interface{}{})) + + cases := []struct { + ups *common.UpstreamConfig + owned bool + }{ + {&common.UpstreamConfig{Endpoint: "tenderly://k"}, true}, + {&common.UpstreamConfig{Endpoint: "evm+tenderly://k"}, true}, + {&common.UpstreamConfig{Endpoint: "https://x.example.com", VendorName: "tenderly"}, true}, + {&common.UpstreamConfig{Endpoint: "https://mainnet.gateway.tenderly.co/k"}, true}, + {&common.UpstreamConfig{Endpoint: "https://rpc.example.com"}, false}, + } + for _, c := range cases { + assert.Equal(t, c.owned, v.OwnsUpstream(c.ups), c.ups.Endpoint) + } +} diff --git a/thirdparty/thirdweb_test.go b/thirdparty/thirdweb_test.go new file mode 100644 index 000000000..f4acd3c31 --- /dev/null +++ b/thirdparty/thirdweb_test.go @@ -0,0 +1,140 @@ +package thirdparty + +import ( + "context" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestThirdwebVendor_SupportsNetwork(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + settings := common.VendorSettings{"clientId": "cid"} + + probeCases := []struct { + name string + body string + want bool + wantErr bool + }{ + {"matching chain id", `{"jsonrpc":"2.0","id":1,"result":"0x2105"}`, true, false}, + {"mismatching chain id", `{"jsonrpc":"2.0","id":1,"result":"0x1"}`, false, false}, + {"json-rpc error", `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"boom"}}`, false, true}, + {"non-hex result", `{"jsonrpc":"2.0","id":1,"result":"base"}`, false, true}, + {"non-string result", `{"jsonrpc":"2.0","id":1,"result":true}`, false, true}, + } + for _, c := range probeCases { + t.Run(c.name, func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gockProbeReply("https://8453.rpc.thirdweb.com", "/cid", c.body) + + ok, err := CreateThirdwebVendor().SupportsNetwork(ctx, &logger, settings, "evm:8453") + if c.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, c.want, ok) + }) + } + + t.Run("cached client reused", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gockProbeReply("https://8453.rpc.thirdweb.com", "/cid", `{"jsonrpc":"2.0","id":1,"result":"0x2105"}`) + v := CreateThirdwebVendor() + for range 2 { + ok, err := v.SupportsNetwork(ctx, &logger, settings, "evm:8453") + require.NoError(t, err) + assert.True(t, ok) + } + }) + + t.Run("unreachable endpoint", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + gock.New("https://8453.rpc.thirdweb.com").Post("/cid").Persist().ReplyError(assert.AnError) + ok, err := CreateThirdwebVendor().SupportsNetwork(ctx, &logger, settings, "evm:8453") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("non-evm network", func(t *testing.T) { + ok, err := CreateThirdwebVendor().SupportsNetwork(ctx, &logger, settings, "solana:mainnet") + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("unparsable chain id", func(t *testing.T) { + ok, err := CreateThirdwebVendor().SupportsNetwork(ctx, &logger, settings, "evm:base") + assert.Error(t, err) + assert.False(t, ok) + }) + + t.Run("missing clientId", func(t *testing.T) { + ok, err := CreateThirdwebVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{}, "evm:8453") + assert.ErrorContains(t, err, "clientId is required in thirdweb settings") + assert.False(t, ok) + }) + + t.Run("malformed clientId", func(t *testing.T) { + ok, err := CreateThirdwebVendor().SupportsNetwork(ctx, &logger, common.VendorSettings{"clientId": "%zz"}, "evm:8453") + assert.Error(t, err) + assert.False(t, ok) + }) +} + +func TestThirdwebVendor_GenerateConfigs(t *testing.T) { + ctx := context.Background() + logger := zerolog.Nop() + + t.Run("generated endpoint", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 42161}} + cfgs, err := CreateThirdwebVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"clientId": "cid"}) + require.NoError(t, err) + require.Len(t, cfgs, 1) + assert.Equal(t, "https://42161.rpc.thirdweb.com/cid", cfgs[0].Endpoint) + assert.Equal(t, common.UpstreamTypeEvm, cfgs[0].Type) + assert.NotNil(t, cfgs[0].JsonRpc) + }) + + t.Run("preset endpoint passes through", func(t *testing.T) { + ups := &common.UpstreamConfig{Endpoint: "https://1.rpc.thirdweb.com/other", Evm: &common.EvmUpstreamConfig{ChainId: 1}} + cfgs, err := CreateThirdwebVendor().GenerateConfigs(ctx, &logger, ups, nil) + require.NoError(t, err) + assert.Equal(t, "https://1.rpc.thirdweb.com/other", cfgs[0].Endpoint) + }) + + t.Run("missing clientId", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}} + _, err := CreateThirdwebVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{}) + assert.ErrorContains(t, err, "clientId is required in thirdweb settings") + }) + + t.Run("malformed clientId", func(t *testing.T) { + ups := &common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{ChainId: 1}} + _, err := CreateThirdwebVendor().GenerateConfigs(ctx, &logger, ups, common.VendorSettings{"clientId": "%zz"}) + assert.ErrorContains(t, err, "invalid URL escape") + }) +} + +func TestThirdwebVendor_ErrorsAndOwnership(t *testing.T) { + v := CreateThirdwebVendor() + assert.Equal(t, "thirdweb", v.Name()) + + err, details := vendorErr(t, v, "eth_getLogs", 500, -32000, "boom", "d") + assert.NoError(t, err) + assert.Empty(t, details) + + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "thirdweb://cid"})) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "evm+thirdweb://cid"})) + assert.True(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://1.rpc.thirdweb.com/cid"})) + assert.False(t, v.OwnsUpstream(&common.UpstreamConfig{Endpoint: "https://rpc.example.com"})) +} diff --git a/thirdparty/vendor_utils_test.go b/thirdparty/vendor_utils_test.go new file mode 100644 index 000000000..0e1e7f08e --- /dev/null +++ b/thirdparty/vendor_utils_test.go @@ -0,0 +1,31 @@ +package thirdparty + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidateChainsURL(t *testing.T) { + cases := []struct { + url string + contains string + }{ + {"https://chains.example.com/list", ""}, + {"http://127.0.0.1:8080/chains", ""}, + {"ftp://chains.example.com", "scheme must be http or https"}, + {"chains.example.com/list", "scheme must be http or https"}, + {"https:///no-host", "host is empty"}, + {"https://[::1", "invalid chainsUrl"}, + } + for _, c := range cases { + t.Run(c.url, func(t *testing.T) { + err := validateChainsURL(c.url) + if c.contains == "" { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, c.contains) + } + }) + } +} diff --git a/thirdparty/vendors_registry_test.go b/thirdparty/vendors_registry_test.go new file mode 100644 index 000000000..4ca27c80c --- /dev/null +++ b/thirdparty/vendors_registry_test.go @@ -0,0 +1,78 @@ +package thirdparty + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVendorsRegistry_SupportedVendors(t *testing.T) { + names := NewVendorsRegistry().SupportedVendors() + for _, n := range []string{"goldsky", "erpc", "envio", "pimlico", "thirdweb", "tenderly", "routemesh"} { + assert.Contains(t, names, n) + } + seen := map[string]bool{} + for _, n := range names { + assert.False(t, seen[n], "vendor %q registered twice", n) + seen[n] = true + } +} + +func TestVendorsRegistry_LookupByName(t *testing.T) { + r := NewVendorsRegistry() + for _, n := range []string{"goldsky", "erpc", "tenderly", "routemesh"} { + v := r.LookupByName(n) + require.NotNil(t, v, n) + assert.Equal(t, n, v.Name()) + } + assert.Nil(t, r.LookupByName("no-such-vendor")) +} + +func TestVendorsRegistry_LookupByUpstream(t *testing.T) { + r := NewVendorsRegistry() + + byEndpoint := []struct { + endpoint string + vendor string + }{ + {"goldsky://sec", "goldsky"}, + {"erpc://edge.example.com", "erpc"}, + {"https://mainnet.gateway.tenderly.co/k", "tenderly"}, + {"https://lb.routemes.sh/rpc/1/k", "routemesh"}, + {"https://1.rpc.thirdweb.com/cid", "thirdweb"}, + {"https://api.pimlico.io/v2/1/rpc?apikey=k", "pimlico"}, + {"https://1.rpc.hypersync.xyz/k", "envio"}, + } + for _, c := range byEndpoint { + t.Run(c.endpoint, func(t *testing.T) { + v := r.LookupByUpstream(&common.UpstreamConfig{Endpoint: c.endpoint}) + require.NotNil(t, v) + assert.Equal(t, c.vendor, v.Name()) + }) + } + + t.Run("vendor name wins over endpoint ownership", func(t *testing.T) { + v := r.LookupByUpstream(&common.UpstreamConfig{Endpoint: "goldsky://sec", VendorName: "tenderly"}) + require.NotNil(t, v) + assert.Equal(t, "tenderly", v.Name()) + }) + + t.Run("unknown vendor name", func(t *testing.T) { + assert.Nil(t, r.LookupByUpstream(&common.UpstreamConfig{Endpoint: "goldsky://sec", VendorName: "nope"})) + }) + + t.Run("endpoint owned by nobody", func(t *testing.T) { + assert.Nil(t, r.LookupByUpstream(&common.UpstreamConfig{Endpoint: "https://node.unowned-example.net/rpc"})) + }) +} + +func TestVendorsRegistry_Register(t *testing.T) { + r := &VendorsRegistry{} + assert.Empty(t, r.SupportedVendors()) + r.Register(CreateRoutemeshVendor()) + assert.Equal(t, []string{"routemesh"}, r.SupportedVendors()) + assert.NotNil(t, r.LookupByUpstream(&common.UpstreamConfig{Endpoint: "https://lb.routemes.sh/rpc/1/k"})) + assert.Nil(t, r.LookupByUpstream(&common.UpstreamConfig{Endpoint: "goldsky://sec"})) +} diff --git a/typescript/cli/package.json b/typescript/cli/package.json index 2ced4ed8f..f1bad674e 100644 --- a/typescript/cli/package.json +++ b/typescript/cli/package.json @@ -1,6 +1,6 @@ { "name": "@erpc-cloud/cli", - "version": "0.4.0", + "version": "0.5.0", "description": "Library providing the erpc CLI", "bin": "./dist/bin.js", "repository": { diff --git a/typescript/cli/src/generated/checksums.ts b/typescript/cli/src/generated/checksums.ts index fd163e6c5..cdfbc912b 100644 --- a/typescript/cli/src/generated/checksums.ts +++ b/typescript/cli/src/generated/checksums.ts @@ -2,9 +2,9 @@ import type { Checksums } from "../types"; export const CHECKSUMS: Checksums = { - "darwin_x86_64": "7d39e8c045548a0857ff58791029f39e2b28052382ecf250c398b20ea4702926", - "darwin_arm64": "0358f01965f62d9486e0863aa4db94ba94b614e35f860fb695dcc0b78c2f9cc0", - "linux_x86_64": "849b79eee7645bed31f7ec098b1ee623b32e36e48a0744ea7e5275f5274d7aa6", - "linux_arm64": "7b38b2a2205c830ad25df21dc5be67735b9375c615792fd6d44e70b04bfd5df6", - "windows_x86_64": "1d17e2c7ea33ef7c91b6187f39e18aa9b82244ca2c8d6910573cf89da86e54e6" + "darwin_x86_64": "00a012e8272b46669d68a4830ef80133222d33478ed82ad624f4a9c04e74d157", + "darwin_arm64": "33884b98eed48f5bff5d3166c63185b45322c291e0386f646b6e7a10a3c735ba", + "linux_x86_64": "4ff132e81ea07c3af633621e172d60ff6b3ac9d8259cfa17e5621cda707a9904", + "linux_arm64": "f9d4ed43d829f577e57b70babb3a645ad55b4e1524538ae56eed46a4ac4567ae", + "windows_x86_64": "02d0eeb0c25908fb67acb560be749ee7348f658996fb3553723f422185507ad0" }; diff --git a/typescript/cli/src/generated/release.ts b/typescript/cli/src/generated/release.ts index a4b7ceb58..986b68e54 100644 --- a/typescript/cli/src/generated/release.ts +++ b/typescript/cli/src/generated/release.ts @@ -2,6 +2,6 @@ import type { ReleaseInfo } from "../types"; export const RELEASE_INFO: ReleaseInfo = { - version: '0.4.0', - commitSha: 'ad00e1308e71f1ad2053476e779386a404b33ee9', + version: '0.5.0', + commitSha: '8e5a461480543f56e5e577438f4e4762f58795b3', }; diff --git a/typescript/config/lib/generated.d.ts b/typescript/config/lib/generated.d.ts index 3a701a82a..a8ec93e61 100644 --- a/typescript/config/lib/generated.d.ts +++ b/typescript/config/lib/generated.d.ts @@ -136,6 +136,15 @@ export declare const MaxShredInsertSlotLagThreshold: number; * so pre-multi-chain configs keep producing "svm:" IDs. */ export declare const SvmChainSolana = "solana"; +/** + * SvmHistory is the slots from which a cluster's Agave history records block + * fields. Below them Agave renders the field null whatever an archive holds; + * a nil bound means the cluster records the field from its first slot. + */ +export interface SvmHistory { + blocktimefrom?: number; + blockheightfrom?: number; +} /** * BlockTimeAdaptiveDuration is a duration that is either a fixed value or derived from the * network's estimated block time. It's the reusable shape for knobs (e.g. cache @@ -175,9 +184,25 @@ export interface Config { projects?: (ProjectConfig | undefined)[]; rateLimiters?: RateLimiterConfig; metrics?: MetricsConfig; + indexer?: IndexerConfig; proxyPools?: (ProxyPoolConfig | undefined)[]; tracing?: TracingConfig; } +/** + * IndexerConfig tunes the event-stream indexer behind eth_subscribe fan-out. + */ +export interface IndexerConfig { + /** + * Deprecated: ignored; the indexer no longer tracks a canonical chain. + * Kept so configs that still set it load under strict decoding. + */ + canonicalChainDepth?: number; + /** + * DedupWindowSize is the per-filter seen-set capacity for log and + * pending-tx fan-out across upstreams. 0 uses the default (8192). + */ + dedupWindowSize?: number; +} export interface ServerConfig { listenV4?: boolean; httpHostV4?: string; @@ -208,6 +233,7 @@ export interface ServerConfig { responseHeaders?: { [key: string]: string; }; + webSocket?: WebSocketServerConfig; /** * ExecutionHeaders controls the per-request diagnostic headers * (X-ERPC-Attempts, X-ERPC-Upstreams-Tried, etc.) that expose how @@ -225,6 +251,47 @@ export interface ServerConfig { */ costHeaders?: boolean; } +export interface WebSocketServerConfig { + /** + * Enabled accepts WebSocket upgrades on the HTTP port. Off by default: + * with it off an upgrade request is ordinary HTTP. + */ + enabled?: boolean; + readBufferSize?: number; + writeBufferSize?: number; + maxMessageSize?: number; + pingInterval?: Duration; + maxSubscriptionsPerConnection?: number; + /** + * MaxConcurrentRequestsPerConnection bounds the messages a connection has + * in flight; further frames wait (backpressure) instead of being rejected. + * A batch is one message; its requests run concurrently, as over HTTP. + */ + maxConcurrentRequestsPerConnection?: number; + /** + * SubscriptionBufferSize is the number of notifications queued per + * subscription for a client that is behind. On overflow newHeads and + * newPendingTransactions drop their oldest entry; logs close the + * connection instead (1013), so the client knows it missed data. + */ + subscriptionBufferSize?: number; + /** + * MaxConnections caps concurrent client connections on this instance + * (0 = unlimited). Unlike the per-IP cap it holds when client IPs are + * unknown or spoofed. + */ + maxConnections?: number; + /** + * MaxConnectionsPerIP caps concurrent client connections from one + * resolved client IP (0 = unlimited); see trustedIPForwarders. + */ + maxConnectionsPerIP?: number; + /** + * MaxSubscriptions caps active subscriptions across all connections on + * this instance (0 = unlimited). + */ + maxSubscriptions?: number; +} /** * ExecutionHeadersMode controls how much per-request execution detail is * exposed in HTTP response headers. @@ -677,6 +744,19 @@ export interface NetworkDefaults { svm?: TsSvmNetworkConfigForDefaults; multiplexing?: boolean; cacheKeySuffix?: string; + failover?: FailoverConfig; +} +/** + * FailoverConfig controls per-request escalation to fallback-tier upstreams, + * independent of the selection policy. + */ +export interface FailoverConfig { + /** + * OnDefaultsExhausted tries upstreams not tagged `tier:fallback` first and + * advances to the fallback tier within the same request only once every + * other upstream failed with an error retryable toward the network. + */ + onDefaultsExhausted?: boolean; } export interface CORSConfig { allowedOrigins: string[]; @@ -700,6 +780,11 @@ export interface ProviderConfig { [key: string]: UpstreamConfig | undefined; }; } +/** + * TagTierFallback marks an upstream as part of the fallback tier, used only + * when the other upstreams are unavailable. + */ +export declare const TagTierFallback = "tier:fallback"; /** * RateLimitCountMode selects the accounting unit an upstream's rate-limit * budget charges per call. @@ -964,6 +1049,14 @@ export interface EvmUpstreamConfig { * upstream would have answered it. 0 (default) keeps the exact head comparison. */ headLagToleranceBlocks?: number; + /** + * PinLatestTag sends this upstream the network head as a concrete block number wherever a + * state call (eth_call, eth_getBalance, ...) asks for "latest" or omits the block, and skips the + * upstream when its blockAvailability upper bound is below that head or the head is unknown. + * For an upstream whose own head lags the network: it answers only blocks it holds, and a block + * it lacks fails as missing data so the next upstream answers, instead of an older "latest". + */ + pinLatestTag?: boolean; /** * Deprecated: never read at runtime. Configure data integrity via the network * `integrity` block instead. Retained only so existing YAML still parses. @@ -978,6 +1071,30 @@ export interface EvmUpstreamConfig { */ maxAvailableRecentBlocks?: number; queryShim?: EvmQueryShimConfig; + /** + * Multicall3 packs eth_calls bound for this upstream that share a block into one + * Multicall3 aggregate3 eth_call, so a vendor billed per request is billed once + * per pack. Nil (default) sends every eth_call alone. + */ + multicall3?: EvmMulticall3Config; +} +/** + * EvmMulticall3Config bounds one pack. A call joins the open pack for its block, + * and the pack is sent MaxWait after its first call or when it holds MaxCalls. + * A call the pack cannot answer exactly (multicall failed, inner call reverted + * or returned nothing) is sent alone, as it would be without packing. + */ +export interface EvmMulticall3Config { + maxWait?: Duration; + maxCalls?: number; + /** + * BypassContracts are target addresses whose calls are never packed, for + * contracts that answer differently when the caller is Multicall3 instead + * of the zero address (msg.sender or caller-code checks) without reverting. + * Contracts that revert inside a pack but succeed alone are learned at + * runtime and need no entry. + */ + bypassContracts?: string[]; } /** * EvmQueryShimConfig lets an upstream serve the sub-requests (eth_getBlockByNumber, @@ -1366,6 +1483,7 @@ export interface NetworkConfig { methods?: MethodsConfig; multiplexing?: boolean; staticResponses?: (StaticResponseConfig | undefined)[]; + failover?: FailoverConfig; /** * Integrity overrides the project-wide data-integrity configuration for this * network. Merges over the project block (network wins). @@ -1536,9 +1654,9 @@ export interface SvmUpstreamConfig { /** * BlockAvailability is an operator restriction on the slot window this * upstream may serve. Only what the operator declares is enforced; the - * upstream's oldest stored block (getBlocksWithLimit(0, 1)) is probed and - * used as a lower anchor solely when a bound uses firstAvailableSlotPlus, - * mirroring EVM's earliestBlockPlus. Unknown floor = unbounded. + * upstream's oldest stored block (getBlocksWithLimit(0, 1)) is probed for + * every upstream and anchors a firstAvailableSlotPlus bound, mirroring EVM's + * earliestBlockPlus. Unknown floor = unbounded. */ blockAvailability?: SvmBlockAvailabilityConfig; } @@ -1553,12 +1671,15 @@ export interface SvmBlockAvailabilityConfig { } /** * SvmAvailabilityBoundConfig is a single declared slot bound. - * Exactly one field must be set. + * Exactly one of ExactSlot, LatestSlotMinus, FirstAvailableSlotPlus must be set. + * UpdateRate applies only to firstAvailableSlotPlus: how often the floor it + * anchors on is re-probed. 0 freezes it after the first observation, as EVM. */ export interface SvmAvailabilityBoundConfig { exactSlot?: number; latestSlotMinus?: number; firstAvailableSlotPlus?: number; + updateRate?: Duration; } export interface EvmNetworkConfig { chainId: number; @@ -1568,8 +1689,7 @@ export interface EvmNetworkConfig { /** * ServedTip configures how the network derives the "latest"/"finalized" * block it advertises to clients (and enforces via block-availability). - * Nil or disabled selects the default mode (the corroborated latest across eligible - * upstreams, see ServedTipPick.Freshest); set Enabled to opt into the majority tip. See + * Nil uses the defaults: quorum 2, regression guard on, no guarantees. See * EvmServedTipConfig. */ servedTip?: EvmServedTipConfig; @@ -1626,6 +1746,21 @@ export interface EvmNetworkConfig { * Default includes common point-lookup methods like eth_getBlockByNumber, eth_getTransactionByHash, etc. */ markEmptyAsErrorMethods?: string[]; + /** + * LocalAnswers turns off eRPC's own answer for a method on this network: + * `{net_peerCount: false}` routes net_peerCount to the upstreams. Every + * method in LocalAnswerMethods is answered by eRPC unless set false here. + */ + localAnswers?: { + [key: string]: boolean; + }; + /** + * StripSubscribeFromBlockZero removes a zero `fromBlock` from eth_subscribe + * logs filters before forwarding upstream. fromBlock has no meaning for a + * live subscription, and backends that prune history reject it. + * DEFAULT: false. + */ + stripSubscribeFromBlockZero?: boolean; /** * DynamicBlockTimeDebounceMultiplier scales the EMA-estimated block time to derive * the debounce interval for block polling. A value of 0.7 means debounce = 70% of @@ -1634,6 +1769,16 @@ export interface EvmNetworkConfig { * Default: 0.7 (30% under the estimated block time). */ dynamicBlockTimeDebounceMultiplier?: number; + /** + * FinalizedDebounceDepthMultiplier sets how stale an upstream's finalized block may + * get before the state poller fetches it again, as a fraction of the chain's + * finality depth in time: (latest - finalized) x EMA block time x multiplier. The + * result applies only when it is longer than the regular debounce. Finalized + * advances once per finality window, so polling it at the latest-block cadence + * mostly returns the same number. 0 disables it (finalized uses the regular debounce). + * Default: 0.05 (refresh about 20 times per finality window). + */ + finalizedDebounceDepthMultiplier?: number; /** * BlockUnavailableDelayMultiplier scales the EMA-estimated block time to derive the * retry delay when the requested data isn't available yet (ErrUpstreamBlockUnavailable @@ -1662,6 +1807,18 @@ export interface EvmNetworkConfig { * finalized head; an unfinalized block's empty is treated as not-yet-confirmed. */ emptyResultConfidence?: AvailbilityConfidence; + /** + * ShortCircuitFutureBlocks returns null for a numbered eth_getBlockByNumber + * above the highest latest head across every upstream of the network + * (regardless of selection policy, tier or syncing state), without + * dispatching it; it never fires while any upstream's head is unknown. + * Clients that poll the next block by number otherwise walk every routed + * upstream on each poll. + * The head is what the state poller and head subscriptions last reported, so + * a block an upstream produced since then gets null until eRPC sees it: only + * enable it where heads arrive promptly (e.g. over WebSocket). + */ + shortCircuitFutureBlocks?: boolean; /** * SafeBlockSource is an upstream id/tag selector for standard JSON-RPC * requests carrying the `safe` block tag. Matching upstreams define and @@ -1673,34 +1830,28 @@ export interface EvmNetworkConfig { /** * EvmServedTipConfig controls how the network derives the "latest"/"finalized" * block it advertises (and enforces) from its upstreams. - * In the default mode the served tip is the corroborated latest block across eligible - * non-syncing upstreams (second-highest, or the only one) — which can still advertise a block a slightly-ahead - * pair has, causing "block not found" churn when requests route to a - * slightly-behind upstream. When a tag is listed in EnabledFor, that tag's - * served value is instead the freshest block a strict MAJORITY of the eligible - * upstreams already have, so interpolated requests land on upstreams that can - * serve the advertised block. + * Both axes use one rule: among the eligible (selection-policy), non-syncing + * upstreams, only the best voter rank present votes (live, then benched, then + * static-capped, then benched and static-capped), and the served block is the + * highest one at least Quorum of those voters already have. The health + * tracker benches an upstream whose head stopped moving while another kept + * advancing, until it catches up again. */ export interface EvmServedTipConfig { /** - * EnabledFor lists the block tags whose served value uses the cluster-min tip - * instead of the default corroborated head. Valid entries: "latest" and "finalized" (the - * "safe" tag follows "finalized"). Empty selects the default mode for all tags. + * Quorum is how many eligible upstreams must already have a block before it + * is served: an integer >= 1, or "majority" (floor(N/2)+1 of N). Unset + * uses DefaultServedTipQuorum (2). A ballot with fewer voters than the + * quorum serves the block all of them have. */ - enabledFor?: string[]; - /** - * Deprecated: ClusterDelta configured the former cluster-based picker and is - * ignored — the majority order statistic needs no tuning. Kept only so - * existing configs keep parsing. - */ - clusterDelta?: number; + quorum?: number | 'majority'; /** * GuaranteedMethods lists method name patterns (glob; e.g. "trace_*", * "debug_traceBlockByNumber") whose supporting-upstream subset must be able to * serve the advertised latest. For a request on a matching method, "latest" * resolves against the majority of only the upstreams that support it * (membership auto-detected via ShouldHandleMethod — no per-upstream config). - * Empty means only the global (all-eligible) majority is computed. + * Empty means only the global ballot is computed. */ guaranteedMethods?: string[]; /** @@ -1715,35 +1866,17 @@ export interface EvmServedTipConfig { guaranteedFor?: string[]; /** * MaxRegressionBlocks is how far below the corroborated LIVE upstream head - * (the second-highest live head) the majority pick may fall before it is - * treated as a poisoned ballot rather than as reality. While a pick is below - * that bound the network keeps serving its last corroborated pick - * (in-process, for at most one minute, then it fails open and serves the - * pick as computed). 0 uses DefaultToleratedBlockHeadRollback (1024) — the - * same rollback tolerance the state poller and health tracker apply to a + * (the second-highest live head) the pick may fall before it is treated as + * a poisoned ballot rather than as reality. While a pick is below that + * bound the network keeps serving its last corroborated pick (in-process, + * for at most one minute, then it fails open and serves the pick as + * computed). 0 uses DefaultToleratedBlockHeadRollback (1024) — the same + * rollback tolerance the state poller and health tracker apply to a * retreating head, so all three layers agree on what counts as a genuine - * deep correction. -1 disables the regression guard entirely (the symmetric - * opposite of trajectoryWindow: 0); no other negative value is accepted. + * deep correction. -1 disables the regression guard entirely; no other + * negative value is accepted. */ maxRegressionBlocks?: number; - /** - * TrajectoryWindow is how much recorded head history the trajectory referee - * needs before it may participate in the pick. The referee tracks where this - * network's head has been over the window and, when a stalled group holds - * the majority, serves the corroborated group that is actually where the - * chain should be by now (see architecture/evm's TipTrajectory: advisory, - * upward-only, in-process, and a no-op until every confidence condition - * holds). Unset uses DefaultServedTipTrajectoryWindow (10m); an explicit 0 - * disables the referee entirely — nothing is recorded and the pick is the - * plain majority. Any other value must be between - * MinServedTipTrajectoryWindow and MaxServedTipTrajectoryWindow: below the - * minimum the window is self-defeating (it must be long enough that a stall - * cannot look like a trajectory), and above the maximum the sample ring can - * never span it, so the referee would stand down forever while looking - * configured. WRITE IT AS A DURATION STRING — trajectoryWindow: "10m". A - * bare number is parsed as MILLISECONDS. - */ - trajectoryWindow?: Duration; } /** * EvmIntegrityConfig is deprecated. Use DirectiveDefaultsConfig for validation settings. @@ -2103,7 +2236,36 @@ export interface RateLimitStoreConfig { redis?: RedisConnectorConfig; cacheKeyPrefix?: string; nearLimitRatio?: number; + /** + * OnFailure decides a request the redis store could not: not connected, + * erroring, slower than redis.getTimeout, or at its in-flight cap. + */ + onFailure?: RateLimitStoreFailureMode; + /** + * LocalLimitFactor scales every maxCount for the in-process limiter that + * onFailure: local decides with. It counts this replica only, so set it + * near 1/replicas. Required for local; must be > 0 and <= 1. + */ + localLimitFactor?: number; } +/** + * RateLimitStoreFailureMode is what a redis-backed budget does with a request + * its store could not decide. + */ +export type RateLimitStoreFailureMode = string; +/** + * RateLimitStoreFailureAllow (default) admits the request. + */ +export declare const RateLimitStoreFailureAllow: RateLimitStoreFailureMode; +/** + * RateLimitStoreFailureDeny rejects the request as rate limited. + */ +export declare const RateLimitStoreFailureDeny: RateLimitStoreFailureMode; +/** + * RateLimitStoreFailureLocal decides it with an in-process limiter that + * counts every request, at maxCount × localLimitFactor. + */ +export declare const RateLimitStoreFailureLocal: RateLimitStoreFailureMode; /** * IntegrityHeaderModeOff ignores per-request integrity headers entirely. */ @@ -2362,6 +2524,13 @@ export type JsonRpcErrorExtractor = any; * Similar to http.HandlerFunc style adapters. */ export type JsonRpcErrorExtractorFunc = any; +/** + * EvmRoutableUpstreamsProvider is the optional surface a network exposes its + * routable upstreams through: the selection policy's eligible set in routing + * order, narrowed to the request's use-upstream selector. Asserted narrowly, + * like EvmStateProvenWriter, so EvmNetwork implementations need not change. + */ +export type EvmRoutableUpstreamsProvider = any; /** * UpstreamAttemptOutcome enumerates the possible per-attempt outcomes * recorded against an upstream. The set is closed: every attempt ends @@ -2532,41 +2701,23 @@ export type EvmNetwork = Network; export type SvmNetwork = Network; export type QuantileTracker = any; export type TrackedMetrics = any; -/** - * ServedTipInput is a single observation: an upstream's last-known tip block. - * Callers are responsible for excluding syncing or cordoned upstreams BEFORE - * passing observations here; the picker treats every input as a candidate. - */ -export interface ServedTipInput { - /** - * UpstreamID is preserved only for telemetry attribution. It is not used - * in the pick. - */ - upstreamid: string; - /** - * BlockNumber is the upstream's reported tip. Zero or negative values are - * treated as "no data yet" and filtered before picking. - */ - blocknumber: number; -} /** * ServedTipPick is the picker's output. */ export interface ServedTipPick { /** * Tip is the value to advertise as latest/finalized: the highest block - * number that a strict MAJORITY of the inputs have already reached, or 0 - * when there are no valid inputs. + * number that at least quorum of the inputs have already reached (all of + * them when there are fewer inputs than the quorum), or 0 when there are + * no valid inputs. */ tip: number; /** * Freshest is the freshest CORROBORATED view: the 2nd-highest valid input * (or the only input when N=1) — the reference for the deliberate-lag * gauge (Freshest - Tip). Using the 2nd-highest instead of the raw max - * means a single rogue far-future upstream cannot inflate the lag gauge - * (the problem the old velocity gate solved via MaxEligible: one - * wrong-chain endpoint used to make the gauge read hundreds of thousands - * of blocks). The absolute per-upstream maxima remain observable via + * means a single rogue far-future upstream cannot inflate the lag gauge. + * The absolute per-upstream maxima remain observable via * erpc_upstream_latest_block_number. */ freshest: number; @@ -2578,17 +2729,21 @@ export interface ServedTipPick { */ max: number; /** - * Inputs is the number of valid (BlockNumber > 0) observations. + * Inputs is the number of valid (> 0) heads. */ inputs: number; - /** - * Sorted is the valid inputs, DESCENDING by block number — the order - * statistic's own working slice, exposed so the trajectory referee can - * cluster the very same ballot without sorting it a second time. Nil when - * there are no valid inputs; never mutate it. - */ - sorted: ServedTipInput[]; } +/** + * ServedTipQuorum is how many voters must already have a block before it is + * served: a fixed count (>= 1), or ServedTipQuorumMajority for floor(N/2)+1 of + * the N voters. The zero value is DefaultServedTipQuorum. In YAML/JSON it is a + * positive integer or the string "majority". + */ +export type ServedTipQuorum = number; +/** + * ServedTipQuorumMajority requires a strict majority of the voters. + */ +export declare const ServedTipQuorumMajority: ServedTipQuorum; /** * TimeoutFunc computes the timeout for a request. Returns nil when no * timeout applies (caller skips context.WithTimeout). diff --git a/typescript/config/lib/generated.d.ts.map b/typescript/config/lib/generated.d.ts.map index aeddb7f19..a27744b12 100644 --- a/typescript/config/lib/generated.d.ts.map +++ b/typescript/config/lib/generated.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"generated.d.ts","sourceRoot":"","sources":["../src/generated.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,mBAAmB,IAAI,qBAAqB,EAC5C,eAAe,IAAI,iBAAiB,EACpC,YAAY,IAAI,cAAc,EAC9B,mBAAmB,IAAI,qBAAqB,EAC5C,QAAQ,IAAI,UAAU,EACtB,kBAAkB,IAAI,oBAAoB,EAC1C,2BAA2B,IAAI,6BAA6B,EAC5D,2BAA2B,IAAI,6BAA6B,EAC5D,2BAA2B,EAE5B,MAAM,SAAS,CAAA;AAKhB;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAe;IAChC,GAAG,CAAC,EAAE,QAAQ,CAAC;IACf,GAAG,CAAC,EAAE,QAAQ,CAAC;CAChB;AAKD;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAKtC,eAAO,MAAM,eAAe,EAAE,YAAoB,CAAC;AACnD,MAAM,MAAM,WAAW,GACnB,QAAQ,CAAC;AACb;;;;;;;;;GASG;AACH,MAAM,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACvC,MAAM,MAAM,qBAAqB,GAAG,MAAM,CAAW;AACrD,eAAO,MAAM,8BAA8B,EAAE,qBAAyB,CAAC;AACvE,eAAO,MAAM,8BAA8B,EAAE,qBAAyB,CAAC;AACvE,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC;AACjC,eAAO,MAAM,kBAAkB,EAAE,WAAuB,CAAC;AACzD,eAAO,MAAM,eAAe,EAAE,WAAoB,CAAC;AACnD,eAAO,MAAM,kBAAkB,EAAE,WAAuB,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,MAAM,CAAW;AAC/C,eAAO,MAAM,sBAAsB,EAAE,eAAmB,CAAC;AACzD,eAAO,MAAM,sBAAsB,EAAE,eAAmB,CAAC;AACzD,eAAO,MAAM,yBAAyB,EAAE,eAAmB,CAAC;AAC5D,MAAM,MAAM,cAAc,GAAG,GAAG,CAAC;AACjC;;;GAGG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAa;IAChC,cAAc,EAAE,MAAM,CAAa;IACnC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,uBAAuB,CAAC,EAAE,MAAM,CAAW;IAC3C,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,0BAA0B,CAAC,EAAE,MAAM,CAAW;IAC9C,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC;;OAEG;IACH,eAAe,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,wBAAwB,GAAG,oBAAoB,GAAG,SAAS,CAAA;KAAC,CAAC;CACxF;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,wBAAwB,CAAC;IACpC,aAAa,EAAE,MAAM,CAAa;IAClC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAKD,eAAO,MAAM,eAAe,EAAE,YAAoB,CAAC;AACnD;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAAG,GAAG,CAAC;AACrC;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GACnB,QAAQ,CAAC;AACb;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,GAAG,CAAC;AACjC;;;;GAIG;AACH,eAAO,MAAM,8BAA8B,EAAE,MAAwB,CAAC;AACtE;;;;GAIG;AACH,eAAO,MAAM,cAAc,WAAW,CAAC;AAKvC;;;;;;;;;GASG;AACH,MAAM,WAAW,yBAAyB;IACxC;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAe;CAC5C;AAKD,MAAM,MAAM,QAAQ,GAAG,GAAG,CAAC;AAK3B,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,GAAG,CAAiB;CAC3B;AAKD;;GAEG;AACH,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,QAAQ,CAAC,EAAE,CAAC,aAAa,GAAG,SAAS,CAAC,EAAE,CAAC;IACzC,YAAY,CAAC,EAAE,iBAAiB,CAAC;IACjC,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,UAAU,CAAC,EAAE,CAAC,eAAe,GAAG,SAAS,CAAC,EAAE,CAAC;IAC7C,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AACD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAW;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAW;IACtC,kBAAkB,CAAC,EAAE,MAAM,CAAW;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,YAAY,CAAC,EAAE,QAAQ,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,kBAAkB,CAAC,EAAE,QAAQ,CAAC;IAC9B,iBAAiB,CAAC,EAAE,QAAQ,CAAC;IAC7B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,eAAe,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IAC3C;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;IACxC;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AACD;;;GAGG;AACH,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAC1C;;;GAGG;AACH,eAAO,MAAM,mBAAmB,EAAE,oBAA4B,CAAC;AAC/D;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,EAAE,oBAAgC,CAAC;AACvE;;GAEG;AACH,eAAO,MAAM,mBAAmB,EAAE,oBAA4B,CAAC;AAC/D,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,EAAE,eAAe,CAAC;IACvB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC;AACrC,eAAO,MAAM,qBAAqB,EAAE,eAA0B,CAAC;AAC/D,eAAO,MAAM,uBAAuB,EAAE,eAA4B,CAAC;AACnE,eAAO,MAAM,sBAAsB,EAAE,eAA2B,CAAC;AACjE,eAAO,MAAM,2BAA2B,6BAA6B,CAAC;AACtE,eAAO,MAAM,uBAAuB,yBAAyB,CAAC;AAC9D,eAAO,MAAM,uBAAuB,yBAAyB,CAAC;AAC9D,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAChE,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAChE,eAAO,MAAM,iBAAiB,wBAAwB,CAAC;AACvD,eAAO,MAAM,iBAAiB,wBAAwB,CAAC;AACvD,eAAO,MAAM,sBAAsB,wBAAwB,CAAC;AAC5D,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC;AACrC,eAAO,MAAM,mBAAmB,EAAE,eAAwB,CAAC;AAC3D,eAAO,MAAM,mBAAmB,EAAE,eAAwB,CAAC;AAC3D,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,eAAe,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAe;IAClC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IACnC,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,kBAAkB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IAC9C;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,CAAC;CACxD;AACD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AACD,MAAM,WAAW,WAAW;IAC1B,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AACD,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,CAAC,kBAAkB,GAAG,SAAS,CAAC,EAAE,CAAC;CAC3C;AACD,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB;AACD,MAAM,WAAW,cAAc;IAC7B,eAAe,CAAC,EAAE,WAAW,CAAC;IAC9B,eAAe,CAAC,EAAE,WAAW,CAAC;IAC9B,WAAW,CAAC,EAAE,iBAAiB,CAAC;CACjC;AACD,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B;;;OAGG;IACH,eAAe,CAAC,EAAE,QAAQ,CAAC;IAC3B;;;OAGG;IACH,OAAO,CAAC,EAAE,QAAQ,CAAC;IACnB;;;OAGG;IACH,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB;;;OAGG;IACH,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AACD,MAAM,WAAW,WAAW;IAC1B,UAAU,CAAC,EAAE,iBAAiB,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,CAAC;IAC7C,WAAW,CAAC,EAAE,iBAAiB,CAAC;CACjC;AACD,MAAM,WAAW,iBAAiB;IAChC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAW;CAC9B;AACD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;IACjB,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;OAGG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACpC;AACD,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,KAAK,CAAC,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;IACnC,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB;;;;;;OAMG;IACH,GAAG,CAAC,EAAE,QAAQ,GAAG,yBAAyB,CAAC;CAC5C;AACD,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC;AACzC,eAAO,MAAM,YAAY,EAAE,mBAA8B,CAAC;AAC1D,eAAO,MAAM,WAAW,EAAE,mBAA6B,CAAC;AACxD,eAAO,MAAM,gBAAgB,EAAE,mBAAkC,CAAC;AAClE,eAAO,MAAM,cAAc,EAAE,mBAAgC,CAAC;AAC9D,eAAO,MAAM,UAAU,EAAE,mBAA4B,CAAC;AACtD,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,qBAAqB,CAAC;IAC9B;;;;;;;OAOG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,KAAK,CAAC,EAAE,oBAAoB,CAAC;IAC7B,QAAQ,CAAC,EAAE,uBAAuB,CAAC;IACnC,UAAU,CAAC,EAAE,yBAAyB,CAAC;IACvC,IAAI,CAAC,EAAE,mBAAmB,CAAC;IAC3B,eAAe,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IACjD,eAAe,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;CAClD;AACD,MAAM,WAAW,mBAAmB;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IACnC,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAW;IAC5B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AACD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,MAAM,CAAW;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AACD,MAAM,WAAW,mBAAmB;IAClC,qBAAqB,EAAE,qBAAqB,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAA2C;IAC3D,QAAQ,EAAE,MAAM,CAA2C;IAC3D,YAAY,EAAE,MAAM,CAAe;IACnC,YAAY,EAAE,MAAM,CAAe;CACpC;AACD,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AACD,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAW;IACtB,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAW;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,iBAAiB,CAAC,EAAE,QAAQ,CAAC;IAC7B,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AACD,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,aAAa,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B,iBAAiB,CAAC,EAAE,QAAQ,CAAC;IAC7B,iBAAiB,CAAC,EAAE,QAAQ,CAAC;CAC9B;AACD,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAa;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAa;IAC9B,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,OAAO,CAAC,EAAE,uBAAuB,CAAC;IAClC;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AACD,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;CACzB;AACD;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,IAAI,CAAC,EAAE,aAAa,CAAC;CACtB;AACD;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,aAAa,CAAC;CACtB;AACD,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,SAAS,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IAC3C,gBAAgB,CAAC,EAAE,cAAc,CAAC;IAClC,SAAS,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IAC3C,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,EAAE,CAAC,aAAa,GAAG,SAAS,CAAC,EAAE,CAAC;IACzC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,aAAa,CAAC,EAAE,qBAAqB,CAAC;IACtC;;;;;;;;;;OAUG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B;;;;;;;;;;;OAWG;IACH,sBAAsB,CAAC,EAAE,QAAQ,CAAC;CACnC;AACD;;;;;;;;GAQG;AACH,MAAM,WAAW,mBAAmB;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,qBAAqB,CAAC,EAAE,MAAM,CAAe;IAC7C,qBAAqB,CAAC,EAAE,MAAM,CAAe;IAC7C,sBAAsB,CAAC,EAAE,QAAQ,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oBAAoB,CAAC,EAAE,QAAQ,CAAC;CACjC;AACD;;GAEG;AACH,MAAM,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAC3C;;GAEG;AACH,eAAO,MAAM,+BAA+B,EAAE,qBAAoC,CAAC;AACnF;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,qBAA6B,CAAC;AACrE,MAAM,WAAW,eAAe;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IAC1C,eAAe,CAAC,EAAE,qBAAqB,CAAC;IACxC,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C,GAAG,CAAC,EAAE,6BAA6B,CAAC;IACpC,GAAG,CAAC,EAAE,6BAA6B,CAAC;IACpC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AACD,MAAM,WAAW,UAAU;IACzB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAW;CAC1B;AACD,MAAM,MAAM,cAAc,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,CAAC;AACnD,MAAM,WAAW,cAAc;IAC7B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAAA;KAAC,CAAC;CAC1D;AACD;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC;AACxC;;;GAGG;AACH,eAAO,MAAM,yBAAyB,EAAE,kBAA8B,CAAC;AACvE;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB,EAAE,kBAA6B,CAAC;AACrE,MAAM,WAAW,cAAc;IAC7B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,cAAc,CAAC;IACtB;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,iBAAiB,CAAC;IACxB,GAAG,CAAC,EAAE,iBAAiB,CAAC;IACxB,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,IAAI,CAAC,EAAE,kBAAkB,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IAC1C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IACxC;;;;;;OAMG;IACH,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAa,CAAC;IACnD,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AACD;;;;;GAKG;AACH,MAAM,WAAW,qBAAqB;IACpC;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,CAAC,qBAAqB,GAAG,SAAS,CAAC,EAAE,CAAC;IACzD;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAe;IAC5C;;;;;;;;;OASG;IACH,KAAK,CAAC,EAAE,SAAS,GAAG,IAAI,GAAG,KAAK,CAAC;CAClC;AACD;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAC/B;;;;GAIG;AACH,eAAO,MAAM,WAAW,EAAE,SAAgB,CAAC;AAC3C;;;;GAIG;AACH,eAAO,MAAM,YAAY,EAAE,SAAiB,CAAC;AAC7C;;;;;;;;GAQG;AACH,MAAM,WAAW,qBAAqB;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAe;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAe;IACjC,WAAW,CAAC,EAAE,MAAM,CAAe;IACnC,aAAa,CAAC,EAAE,MAAM,CAAe;IACrC,YAAY,CAAC,EAAE,MAAM,CAAe;IACpC,eAAe,CAAC,EAAE,MAAM,CAAe;IACvC,YAAY,CAAC,EAAE,MAAM,CAAe;IACpC;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAe;CACtC;AACD,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAe;IAClC,YAAY,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IAC1C;;;;;;;;OAQG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AACD;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACtC,oBAAoB,CAAC,EAAE,0CAA0C,CAAC;CACnE;AACD,MAAM,WAAW,0CAA0C;IACzD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,6BAA6B,CAAC,EAAE,OAAO,CAAC;IACxC,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AACD,MAAM,MAAM,OAAO,GAAG,cAAc,CAAC;AACrC,MAAM,MAAM,OAAO,GAAG,cAAc,CAAC;AACrC,MAAM,WAAW,uBAAuB;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,gBAAgB,EAAE,QAAQ,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAe;IACzC,cAAc,EAAE,MAAM,CAAe;IACrC,cAAc,EAAE,MAAM,CAAe;IACrC,SAAS,EAAE,MAAM,CAAW;IAC5B,SAAS,EAAE,MAAM,CAAW;CAC7B;AACD,MAAM,WAAW,qBAAqB;IACpC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAW;IAChC,YAAY,CAAC,EAAE,QAAQ,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AACD;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IACnC;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAW;IAC5B;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AACD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAa;IAC5B,mBAAmB,CAAC,EAAE,QAAQ,CAAC;IAC/B;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,QAAQ,CAAC;IAC/B,iBAAiB,CAAC,EAAE,0BAA0B,CAAC;IAC/C,kCAAkC,CAAC,EAAE,MAAM,CAAa;IACxD;;;;;OAKG;IACH,sCAAsC,CAAC,EAAE,MAAM,CAAa;IAC5D,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;OAOG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAa;IAC5C;;;OAGG;IACH,SAAS,CAAC,EAAE,uBAAuB,CAAC;IACpC;;OAEG;IACH,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB;;OAEG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAa;IAC9C,SAAS,CAAC,EAAE,kBAAkB,CAAC;CAChC;AACD;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAa;IACtC;;;;OAIG;IACH,eAAe,CAAC,EAAE,QAAQ,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,0BAA0B,CAAC;IACnC,KAAK,CAAC,EAAE,0BAA0B,CAAC;CACpC;AACD;;;;;GAKG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC;AAC9C,eAAO,MAAM,mBAAmB,EAAE,wBAAwC,CAAC;AAC3E,eAAO,MAAM,iBAAiB,EAAE,wBAAsC,CAAC;AACvE,eAAO,MAAM,iBAAiB,EAAE,wBAAsC,CAAC;AACvE,eAAO,MAAM,iBAAiB,EAAE,wBAAsC,CAAC;AACvE,MAAM,WAAW,0BAA0B;IACzC,UAAU,CAAC,EAAE,MAAM,CAAa;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAa;IACtC,iBAAiB,CAAC,EAAE,MAAM,CAAa;IACvC,KAAK,CAAC,EAAE,wBAAwB,CAAC;IACjC,UAAU,CAAC,EAAE,QAAQ,CAAC;CACvB;AACD,MAAM,WAAW,cAAc;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,iBAAiB,EAAE,CAAC;IACpC;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,WAAW,CAAC,EAAE,CAAC;IACvE;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,GAAG,CAAC;IAC7C,KAAK,CAAC,EAAE,iBAAiB,CAAC;IAC1B,cAAc,CAAC,EAAE,0BAA0B,CAAC;IAC5C,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,KAAK,CAAC,EAAE,iBAAiB,CAAC;IAC1B,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AACD;;;;GAIG;AACH,MAAM,MAAM,qBAAqB,GAAG,cAAc,CAAC;AACnD;;;;;GAKG;AACH,MAAM,MAAM,sBAAsB,GAAG,cAAc,CAAC;AACpD;;;;;;GAMG;AACH,MAAM,MAAM,mBAAmB,GAAG,cAAc,CAAC;AACjD,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAW;IAC9B,KAAK,CAAC,EAAE,QAAQ,CAAC;IACjB,eAAe,CAAC,EAAE,QAAQ,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAe;IACrC,MAAM,CAAC,EAAE,QAAQ,CAAC;IAClB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;OAEG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAW;IAC1C;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,QAAQ,CAAC;CAC7B;AACD,MAAM,WAAW,0BAA0B;IACzC,qBAAqB,EAAE,MAAM,CAAY;IACzC,wBAAwB,EAAE,MAAM,CAAY;IAC5C,aAAa,CAAC,EAAE,QAAQ,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAY;IACzC,wBAAwB,EAAE,MAAM,CAAY;CAC7C;AACD;;;;;;;;GAQG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,QAAQ,GAAG,gBAAgB,CAAC;CACxC;AACD;;;;;;;GAOG;AACH,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,QAAQ,GAAG,gBAAgB,CAAC;IACpC,QAAQ,EAAE,MAAM,CAAW;CAC5B;AACD,MAAM,MAAM,gCAAgC,GAAG,MAAM,CAAC;AACtD,eAAO,MAAM,2CAA2C,EAAE,gCAAgD,CAAC;AAC3G,eAAO,MAAM,2DAA2D,EAAE,gCAAgE,CAAC;AAC3I,eAAO,MAAM,qDAAqD,EAAE,gCAA0D,CAAC;AAC/H,eAAO,MAAM,mDAAmD,EAAE,gCAAwD,CAAC;AAC3H,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC;AAC9C,eAAO,MAAM,mCAAmC,EAAE,wBAAwC,CAAC;AAC3F,eAAO,MAAM,mDAAmD,EAAE,wBAAwD,CAAC;AAC3H,eAAO,MAAM,6CAA6C,EAAE,wBAAkD,CAAC;AAC/G,eAAO,MAAM,2CAA2C,EAAE,wBAAgD,CAAC;AAC3G,MAAM,WAAW,qBAAqB;IACpC,eAAe,EAAE,MAAM,CAAW;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAW;IACtC,eAAe,CAAC,EAAE,wBAAwB,CAAC;IAC3C,uBAAuB,CAAC,EAAE,gCAAgC,CAAC;IAC3D,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IAC1C,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,uBAAuB,CAAC,EAAE,6BAA6B,CAAC;IACxD;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IACnD;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;;;;;;OAUG;IACH,eAAe,CAAC,EAAE,QAAQ,GAAG,gBAAgB,CAAC;IAC9C;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,QAAQ,GAAG,gBAAgB,CAAC;IAC7C;;;;;;;;;;;;;OAaG;IACH,oBAAoB,CAAC,EAAE,CAAC,4BAA4B,GAAG,SAAS,CAAC,EAAE,CAAC;CACrE;AACD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,4BAA4B;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,MAAM,CAAW;IAClC,YAAY,CAAC,EAAE,MAAM,CAAW;CACjC;AACD,MAAM,MAAM,2BAA2B,GAAG,MAAM,CAAC;AACjD,eAAO,MAAM,+BAA+B,EAAE,2BAAoC,CAAC;AACnF,eAAO,MAAM,6BAA6B,EAAE,2BAAkC,CAAC;AAC/E,MAAM,WAAW,6BAA6B;IAC5C;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,EAAE,CAAC,EAAE,aAAa,CAAC;CACpB;AACD,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAa;IAC7B;;OAEG;IACH,aAAa,CAAC,EAAE,QAAQ,CAAC;IACzB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,aAAa,CAAC;IAC5B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,MAAM,WAAW,uBAAuB;IACtC,gBAAgB,EAAE,MAAM,CAAY;IACpC,aAAa,CAAC,EAAE,QAAQ,CAAC;IACzB,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AACD,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,oBAAoB,CAAC;IAC7B,OAAO,EAAE,qBAAqB,EAAE,CAAC;CAClC;AACD,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,mBAAmB,EAAE,CAAC;IAC7B;;;;;OAKG;IACH,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAa,CAAC;CACpD;AACD,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAc;IAC9B;;OAEG;IACH,MAAM,EAAE,eAAe,CAAC;IACxB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAC;CAChC;AACD;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAW;AAC/C,eAAO,MAAM,qBAAqB,EAAE,eAAmB,CAAC;AACxD,eAAO,MAAM,qBAAqB,EAAE,eAAmB,CAAC;AACxD,eAAO,MAAM,mBAAmB,EAAE,eAAmB,CAAC;AACtD,eAAO,MAAM,kBAAkB,EAAE,eAAmB,CAAC;AACrD,eAAO,MAAM,mBAAmB,EAAE,eAAmB,CAAC;AACtD,eAAO,MAAM,oBAAoB,EAAE,eAAmB,CAAC;AACvD,eAAO,MAAM,mBAAmB,EAAE,eAAmB,CAAC;AACtD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AACD,MAAM,WAAW,aAAa;IAC5B,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAAA;KAAC,CAAC;CAC/D;AACD,MAAM,WAAW,aAAa;IAC5B,YAAY,EAAE,qBAAqB,CAAC;IACpC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,gBAAgB,CAAC;IACvB,GAAG,CAAC,EAAE,gBAAgB,CAAC;IACvB,eAAe,CAAC,EAAE,qBAAqB,CAAC;IACxC,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,eAAe,CAAC,EAAE,CAAC,oBAAoB,GAAG,SAAS,CAAC,EAAE,CAAC;IACvD;;;OAGG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AACD;;;;;;;GAOG;AACH,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,QAAQ,CAAC,EAAE,wBAAwB,CAAC;CACrC;AACD;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACvC,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,KAAK,CAAC,EAAE,yBAAyB,CAAC;CACnC;AACD;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,MAAM,CAAW;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,GAAG,CAAC;CACZ;AACD,MAAM,WAAW,uBAAuB;IACtC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,aAAa,CAAC,EAAE,GAAG,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC;;;;;;;;;;OAUG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC,+BAA+B,CAAC,EAAE,OAAO,CAAC;IAC1C,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,+BAA+B,CAAC,EAAE,OAAO,CAAC;IAC1C,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,kBAAkB,CAAC,EAAE,MAAM,CAAa;IACxC,oBAAoB,CAAC,EAAE,MAAM,CAAa;IAC1C,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,6BAA6B,CAAC,EAAE,MAAM,CAAa;CACpD;AACD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;;;;OAQG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,QAAQ,CAAC;IAC/B;;;;;;;;;;OAUG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAa;IACzC;;;;;;;;OAQG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACpC;AACD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;;;OASG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAa;IAC5B,qBAAqB,CAAC,EAAE,MAAM,CAAa;IAC3C,2BAA2B,CAAC,EAAE,QAAQ,CAAC;IACvC,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B,sBAAsB,CAAC,EAAE,MAAM,CAAa;IAC5C,0BAA0B,CAAC,EAAE,MAAM,CAAa;IAChD,uBAAuB,CAAC,EAAE,MAAM,CAAa;IAC7C,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,uBAAuB,CAAC,EAAE,MAAM,CAAW;IAC3C;;;;;;;;;;;;;OAaG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAa;IAC7C;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC;;;OAGG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAW;IAC/C;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;;;;;OAMG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAa;IAC/C;;;;;;OAMG;IACH,uBAAuB,CAAC,EAAE,MAAM,EAAE,CAAC;IACnC;;;;;;OAMG;IACH,kCAAkC,CAAC,EAAE,MAAM,CAAe;IAC1D;;;;;;OAMG;IACH,+BAA+B,CAAC,EAAE,MAAM,CAAe;IACvD;;;;;;OAMG;IACH,8BAA8B,CAAC,EAAE,OAAO,CAAC;IACzC;;;;;;;;;;OAUG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AACD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAa;IAClC;;;;;;;OAOG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAa;IACzC;;;;;;;;;;;;;;;;OAgBG;IACH,gBAAgB,CAAC,EAAE,QAAQ,CAAC;CAC7B;AACD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;OAEG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;OAEG;IACH,0BAA0B,CAAC,EAAE,OAAO,CAAC;CACtC;AACD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAC/B;;;GAGG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAqB,CAAC;AACrD;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,EAAE,SAA4B,CAAC;AAClE;;;;GAIG;AACH,eAAO,MAAM,wBAAwB,EAAE,SAA8B,CAAC;AACtE;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,EAAE,SAAqC,CAAC;AACnF;;;;;;;GAOG;AACH,MAAM,WAAW,qBAAqB;IACpC,YAAY,CAAC,EAAE,QAAQ,CAAC;IACxB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,GAAG,gBAAgB,GAAG,kBAAkB,GAAG,yBAAyB,CAAC;IACtG;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,2BAA2B,GAAG,MAAM,CAAC;CACjD;AACD;;;GAGG;AACH,MAAM,WAAW,2BAA2B;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,gBAAgB,CAAC,EAAE,QAAQ,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAW;CAClC;AACD,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAC9B,eAAO,MAAM,cAAc,EAAE,QAAmB,CAAC;AACjD,eAAO,MAAM,gBAAgB,EAAE,QAAqB,CAAC;AACrD,eAAO,MAAM,WAAW,EAAE,QAAgB,CAAC;AAC3C,eAAO,MAAM,YAAY,EAAE,QAAiB,CAAC;AAC7C,eAAO,MAAM,eAAe,EAAE,QAAoB,CAAC;AACnD,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,oBAAoB,EAAE,CAAC;CACpC;AACD,MAAM,WAAW,kBAAkB;IACjC,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;;;;;;;;OAgBG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,GAAG,CAAC,EAAE,iBAAiB,CAAC;IACxB,IAAI,CAAC,EAAE,kBAAkB,CAAC;CAC3B;AACD,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AACD,MAAM,WAAW,sBAAsB;IACrC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,KAAK,CAAC,EAAE,2BAA2B,CAAC;IACpC,KAAK,CAAC,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,OAAO,CAAC,EAAE,QAAQ,CAAC;CACpB;AACD,MAAM,WAAW,2BAA2B;IAC1C,GAAG,CAAC,EAAE,MAAM,CAA2C;IACvD,OAAO,CAAC,EAAE,MAAM,CAAa;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAa;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAa;CAClC;AACD,MAAM,WAAW,mBAAmB;IAClC,WAAW,CAAC,EAAE,MAAM,CAAW;IAC/B,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AACD,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AACD,MAAM,WAAW,iBAAiB;IAChC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,aAAa,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IAC3C,gBAAgB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+BAA+B,CAAC,EAAE,QAAQ,CAAC;IAC3C;;;OAGG;IACH,qCAAqC,CAAC,EAAE,OAAO,CAAC;IAChD;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACnC;AACD,MAAM,WAAW,kBAAkB;IACjC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AACD,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,EAAE,OAAO,CAAC;IACxB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AACD,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAC/B,eAAO,MAAM,qBAAqB,EAAE,SAAqB,CAAC;AAC1D,eAAO,MAAM,qBAAqB,EAAE,SAAqB,CAAC;AAC1D,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAW;IACxB,cAAc,CAAC,EAAE,SAAS,CAAC;IAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;;;;;;;;;;;;OAiBG;IACH,cAAc,CAAC,EAAE,CAAC,0BAA0B,GAAG,SAAS,CAAC,EAAE,CAAC;IAC5D;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B;;;OAGG;IACH,uBAAuB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IACrD;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;OAGG;IACH,qBAAqB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IACnD;;;;;;;;;;OAUG;IACH,wBAAwB,CAAC,EAAE,QAAQ,CAAC;CACrC;AACD;;;GAGG;AACH,MAAM,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAC/C,eAAO,MAAM,gBAAgB,EAAE,yBAAkC,CAAC;AAClE,eAAO,MAAM,gBAAgB,EAAE,yBAAkC,CAAC;AAClE;;;;;;;;GAQG;AACH,MAAM,WAAW,0BAA0B;IACzC;;;;;;OAMG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;;;;OASG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,MAAM,CAAC,EAAE,CAAC,8BAA8B,GAAG,SAAS,CAAC,EAAE,CAAC;IACxD;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,EAAgB,CAAC;CAClC;AACD;;;GAGG;AACH,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;CACzB;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,oBAAoB,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAe;CACvC;AAKD;;GAEG;AACH,eAAO,MAAM,sBAAsB,QAAQ,CAAC;AAC5C;;GAEG;AACH,eAAO,MAAM,2BAA2B,aAAa,CAAC;AACtD;;GAEG;AACH,eAAO,MAAM,uBAAuB,SAAS,CAAC;AAC9C;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,MAAM,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,oBAAoB,GAAG,SAAS,CAAA;KAAC,CAAC;IAC5D;;OAEG;IACH,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B;;;;;;;;;;;;;OAaG;IACH,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC;;;;;OAKG;IACH,eAAe,CAAC,EAAE,8BAA8B,CAAC;IACjD;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAW;IAC/B;;;;;;;;;;;;;OAaG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;;;;;;;;;;;;OAcG;IACH,UAAU,CAAC,EAAE,yBAAyB,CAAC;IACvC;;;;;OAKG;IACH,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B;;;;;OAKG;IACH,uBAAuB,CAAC,EAAE,6BAA6B,CAAC;CACzD;AACD;;;;;;;GAOG;AACH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAW;CACrC;AACD;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AACD;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,QAAQ,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAAA;KAAC,CAAC;CAC5D;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IAClC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,YAAY,CAAC,EAAE,MAAM,CAAW;IAChC,aAAa,CAAC,EAAE,MAAM,CAAW;CAClC;AACD;;;;;GAKG;AACH,MAAM,WAAW,8BAA8B;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAKD,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAW;AACjD;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,EAAE,iBAAqB,CAAC;AAC/D;;GAEG;AACH,eAAO,MAAM,4BAA4B,EAAE,iBAAqB,CAAC;AACjE;;;GAGG;AACH,eAAO,MAAM,yBAAyB,EAAE,iBAAqB,CAAC;AAC9D;;;GAGG;AACH,eAAO,MAAM,wBAAwB,EAAE,iBAAqB,CAAC;AAC7D;;;;;;;;GAQG;AACH,eAAO,MAAM,oBAAoB,EAAE,iBAAsB,CAAC;AAC1D,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAW;AAClD,eAAO,MAAM,wBAAwB,EAAE,kBAAsB,CAAC;AAC9D,eAAO,MAAM,uBAAuB,EAAE,kBAAsB,CAAC;AAC7D,eAAO,MAAM,sBAAsB,EAAE,kBAAsB,CAAC;AAC5D;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAC1C,eAAO,MAAM,wBAAwB,EAAE,oBAA6B,CAAC;AACrE,eAAO,MAAM,uBAAuB,EAAE,oBAA4B,CAAC;AACnE,eAAO,MAAM,uBAAuB,EAAE,oBAA4B,CAAC;AAKnE;;;;GAIG;AACH,MAAM,MAAM,qBAAqB,GAAG,GAAG,CAAC;AACxC;;;;GAIG;AACH,MAAM,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAK5C;;;;GAIG;AACH,MAAM,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAC5C,eAAO,MAAM,sBAAsB,EAAE,sBAAkC,CAAC;AACxE,eAAO,MAAM,oBAAoB,EAAE,sBAAgC,CAAC;AACpE,eAAO,MAAM,6BAA6B,EAAE,sBAA0C,CAAC;AACvF,eAAO,MAAM,0BAA0B,EAAE,sBAAuC,CAAC;AACjF,eAAO,MAAM,0BAA0B,EAAE,sBAAuC,CAAC;AACjF,eAAO,MAAM,0BAA0B,EAAE,sBAAuC,CAAC;AACjF,eAAO,MAAM,0BAA0B,EAAE,sBAAuC,CAAC;AACjF,eAAO,MAAM,yBAAyB,EAAE,sBAAsC,CAAC;AAC/E,eAAO,MAAM,+BAA+B,EAAE,sBAA4C,CAAC;AAC3F,eAAO,MAAM,0BAA0B,EAAE,sBAAuC,CAAC;AACjF,eAAO,MAAM,wBAAwB,EAAE,sBAAoC,CAAC;AAC5E,eAAO,MAAM,sBAAsB,EAAE,sBAAkC,CAAC;AACxE,eAAO,MAAM,sBAAsB,EAAE,sBAAkC,CAAC;AACxE;;;;;GAKG;AACH,MAAM,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAC7C,eAAO,MAAM,sBAAsB,EAAE,uBAAmC,CAAC;AACzE,eAAO,MAAM,oBAAoB,EAAE,uBAAiC,CAAC;AACrE,eAAO,MAAM,oBAAoB,EAAE,uBAAiC,CAAC;AACrE,eAAO,MAAM,4BAA4B,EAAE,uBAA0C,CAAC;AACtF,eAAO,MAAM,oBAAoB,EAAE,uBAAiC,CAAC;AACrE;;;;;;;;;;GAUG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,GAAG,CAAiB;IAC/B,QAAQ,EAAE,MAAM,CAA2C;IAC3D,OAAO,EAAE,sBAAsB,CAAC;IAChC,MAAM,EAAE,uBAAuB,CAAC;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,MAAM,CAAW;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,WAAW,EAAE,MAAM,CAAa;CACjC;AACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,gBAAgB,EAAE,GAAG,CAAoB;IACzC,eAAe,EAAE,GAAG,CAAoB;IACxC,cAAc,EAAE,GAAG,CAAoB;IACvC,eAAe,EAAE,GAAG,CAAoB;IACxC,cAAc,EAAE,GAAG,CAAoB;IACvC,aAAa,EAAE,GAAG,CAAoB;IACtC,aAAa,EAAE,GAAG,CAAoB;IACtC,YAAY,EAAE,GAAG,CAAoB;IACrC,WAAW,EAAE,GAAG,CAAoB;IACpC;;OAEG;IACH,cAAc,EAAE,GAAG,CAAoB;IACvC;;OAEG;IACH,iBAAiB,EAAE,GAAG,CAAoB;IAC1C;;OAEG;IACH,wBAAwB,EAAE,GAAG,CAAoB;IACjD,SAAS,EAAE,GAAG,CAAiB;CAChC;AACD;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAW;IAC3B,OAAO,EAAE,MAAM,CAAW;IAC1B,MAAM,EAAE,MAAM,CAAW;IACzB;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAW;IACnC,eAAe,EAAE,MAAM,CAAW;IAClC,cAAc,EAAE,MAAM,CAAW;IACjC,eAAe,EAAE,MAAM,CAAW;IAClC,cAAc,EAAE,MAAM,CAAW;IACjC,aAAa,EAAE,MAAM,CAAW;IAChC,aAAa,EAAE,MAAM,CAAW;IAChC,YAAY,EAAE,MAAM,CAAW;IAC/B,WAAW,EAAE,MAAM,CAAW;IAC9B,cAAc,EAAE,MAAM,CAAW;IACjC,iBAAiB,EAAE,MAAM,CAAW;IACpC,wBAAwB,EAAE,MAAM,CAAW;IAC3C,SAAS,EAAE,GAAG,CAAiB;CAChC;AAKD,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC;AACzC,eAAO,MAAM,eAAe,EAAE,mBAA2B,CAAC;AAC1D,eAAO,MAAM,eAAe,EAAE,mBAA2B,CAAC;AAC1D,MAAM,MAAM,OAAO,GAAG,GAAG,CAAC;AAC1B;;;;;GAKG;AACH,MAAM,MAAM,UAAU,GAClB,OAAO,CAAC;AACZ;;;;GAIG;AACH,MAAM,MAAM,UAAU,GAClB,OAAO,CAAC;AACZ,MAAM,MAAM,eAAe,GAAG,GAAG,CAAC;AAClC,MAAM,MAAM,cAAc,GAAG,GAAG,CAAC;AAKjC;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,WAAW,EAAE,MAAM,CAAa;CACjC;AACD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,GAAG,EAAE,MAAM,CAAa;IACxB;;;;;;;;;OASG;IACH,QAAQ,EAAE,MAAM,CAAa;IAC7B;;;;;OAKG;IACH,GAAG,EAAE,MAAM,CAAa;IACxB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAW;IACzB;;;;;OAKG;IACH,MAAM,EAAE,cAAc,EAAE,CAAC;CAC1B;AAKD;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,GAAG,CAAC;AAK9B,MAAM,MAAM,KAAK,GAAG,MAAM,CAAC;AAC3B;;;GAGG;AACH,eAAO,MAAM,YAAY,EAAE,KAAiB,CAAC;AAC7C;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,KAAkB,CAAC;AAC/C,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAClC;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,GAAG,GAAG,CAAC;AAChC,MAAM,MAAM,QAAQ,GAAG,GAAG,CAAC;AAK3B;;;;;;GAMG;AACH,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,eAAe,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAKD;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,MAAM,CAAC"} \ No newline at end of file +{"version":3,"file":"generated.d.ts","sourceRoot":"","sources":["../src/generated.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,mBAAmB,IAAI,qBAAqB,EAC5C,eAAe,IAAI,iBAAiB,EACpC,YAAY,IAAI,cAAc,EAC9B,mBAAmB,IAAI,qBAAqB,EAC5C,QAAQ,IAAI,UAAU,EACtB,kBAAkB,IAAI,oBAAoB,EAC1C,2BAA2B,IAAI,6BAA6B,EAC5D,2BAA2B,IAAI,6BAA6B,EAC5D,2BAA2B,EAE5B,MAAM,SAAS,CAAA;AAKhB;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAe;IAChC,GAAG,CAAC,EAAE,QAAQ,CAAC;IACf,GAAG,CAAC,EAAE,QAAQ,CAAC;CAChB;AAKD;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAKtC,eAAO,MAAM,eAAe,EAAE,YAAoB,CAAC;AACnD,MAAM,MAAM,WAAW,GACnB,QAAQ,CAAC;AACb;;;;;;;;;GASG;AACH,MAAM,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACvC,MAAM,MAAM,qBAAqB,GAAG,MAAM,CAAW;AACrD,eAAO,MAAM,8BAA8B,EAAE,qBAAyB,CAAC;AACvE,eAAO,MAAM,8BAA8B,EAAE,qBAAyB,CAAC;AACvE,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC;AACjC,eAAO,MAAM,kBAAkB,EAAE,WAAuB,CAAC;AACzD,eAAO,MAAM,eAAe,EAAE,WAAoB,CAAC;AACnD,eAAO,MAAM,kBAAkB,EAAE,WAAuB,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,MAAM,CAAW;AAC/C,eAAO,MAAM,sBAAsB,EAAE,eAAmB,CAAC;AACzD,eAAO,MAAM,sBAAsB,EAAE,eAAmB,CAAC;AACzD,eAAO,MAAM,yBAAyB,EAAE,eAAmB,CAAC;AAC5D,MAAM,MAAM,cAAc,GAAG,GAAG,CAAC;AACjC;;;GAGG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAa;IAChC,cAAc,EAAE,MAAM,CAAa;IACnC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,uBAAuB,CAAC,EAAE,MAAM,CAAW;IAC3C,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,0BAA0B,CAAC,EAAE,MAAM,CAAW;IAC9C,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC;;OAEG;IACH,eAAe,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,wBAAwB,GAAG,oBAAoB,GAAG,SAAS,CAAA;KAAC,CAAC;CACxF;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,wBAAwB,CAAC;IACpC,aAAa,EAAE,MAAM,CAAa;IAClC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAKD,eAAO,MAAM,eAAe,EAAE,YAAoB,CAAC;AACnD;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAAG,GAAG,CAAC;AACrC;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GACnB,QAAQ,CAAC;AACb;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,GAAG,CAAC;AACjC;;;;GAIG;AACH,eAAO,MAAM,8BAA8B,EAAE,MAAwB,CAAC;AACtE;;;;GAIG;AACH,eAAO,MAAM,cAAc,WAAW,CAAC;AACvC;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,aAAa,CAAC,EAAE,MAAM,CAAc;IACpC,eAAe,CAAC,EAAE,MAAM,CAAc;CACvC;AAKD;;;;;;;;;GASG;AACH,MAAM,WAAW,yBAAyB;IACxC;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAe;CAC5C;AAKD,MAAM,MAAM,QAAQ,GAAG,GAAG,CAAC;AAK3B,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,GAAG,CAAiB;CAC3B;AAKD;;GAEG;AACH,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,QAAQ,CAAC,EAAE,CAAC,aAAa,GAAG,SAAS,CAAC,EAAE,CAAC;IACzC,YAAY,CAAC,EAAE,iBAAiB,CAAC;IACjC,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,UAAU,CAAC,EAAE,CAAC,eAAe,GAAG,SAAS,CAAC,EAAE,CAAC;IAC7C,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AACD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAW;IACvC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAW;CACpC;AACD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAW;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAW;IACtC,kBAAkB,CAAC,EAAE,MAAM,CAAW;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,YAAY,CAAC,EAAE,QAAQ,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,kBAAkB,CAAC,EAAE,QAAQ,CAAC;IAC9B,iBAAiB,CAAC,EAAE,QAAQ,CAAC;IAC7B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,eAAe,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IAC3C,SAAS,CAAC,EAAE,qBAAqB,CAAC;IAClC;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;IACxC;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AACD,MAAM,WAAW,qBAAqB;IACpC;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAW;IAClC,eAAe,CAAC,EAAE,MAAM,CAAW;IACnC,cAAc,CAAC,EAAE,MAAM,CAAa;IACpC,YAAY,CAAC,EAAE,QAAQ,CAAC;IACxB,6BAA6B,CAAC,EAAE,MAAM,CAAW;IACjD;;;;OAIG;IACH,kCAAkC,CAAC,EAAE,MAAM,CAAW;IACtD;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAW;IAC1C;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAW;IAClC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAW;IACvC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAW;CACrC;AACD;;;GAGG;AACH,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAC1C;;;GAGG;AACH,eAAO,MAAM,mBAAmB,EAAE,oBAA4B,CAAC;AAC/D;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,EAAE,oBAAgC,CAAC;AACvE;;GAEG;AACH,eAAO,MAAM,mBAAmB,EAAE,oBAA4B,CAAC;AAC/D,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,EAAE,eAAe,CAAC;IACvB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC;AACrC,eAAO,MAAM,qBAAqB,EAAE,eAA0B,CAAC;AAC/D,eAAO,MAAM,uBAAuB,EAAE,eAA4B,CAAC;AACnE,eAAO,MAAM,sBAAsB,EAAE,eAA2B,CAAC;AACjE,eAAO,MAAM,2BAA2B,6BAA6B,CAAC;AACtE,eAAO,MAAM,uBAAuB,yBAAyB,CAAC;AAC9D,eAAO,MAAM,uBAAuB,yBAAyB,CAAC;AAC9D,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAChE,eAAO,MAAM,wBAAwB,0BAA0B,CAAC;AAChE,eAAO,MAAM,iBAAiB,wBAAwB,CAAC;AACvD,eAAO,MAAM,iBAAiB,wBAAwB,CAAC;AACvD,eAAO,MAAM,sBAAsB,wBAAwB,CAAC;AAC5D,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC;AACrC,eAAO,MAAM,mBAAmB,EAAE,eAAwB,CAAC;AAC3D,eAAO,MAAM,mBAAmB,EAAE,eAAwB,CAAC;AAC3D,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,eAAe,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAe;IAClC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IACnC,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,kBAAkB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IAC9C;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,CAAC;CACxD;AACD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AACD,MAAM,WAAW,WAAW;IAC1B,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AACD,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,CAAC,kBAAkB,GAAG,SAAS,CAAC,EAAE,CAAC;CAC3C;AACD,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB;AACD,MAAM,WAAW,cAAc;IAC7B,eAAe,CAAC,EAAE,WAAW,CAAC;IAC9B,eAAe,CAAC,EAAE,WAAW,CAAC;IAC9B,WAAW,CAAC,EAAE,iBAAiB,CAAC;CACjC;AACD,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B;;;OAGG;IACH,eAAe,CAAC,EAAE,QAAQ,CAAC;IAC3B;;;OAGG;IACH,OAAO,CAAC,EAAE,QAAQ,CAAC;IACnB;;;OAGG;IACH,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB;;;OAGG;IACH,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AACD,MAAM,WAAW,WAAW;IAC1B,UAAU,CAAC,EAAE,iBAAiB,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,CAAC;IAC7C,WAAW,CAAC,EAAE,iBAAiB,CAAC;CACjC;AACD,MAAM,WAAW,iBAAiB;IAChC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAW;CAC9B;AACD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;IACjB,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;OAGG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACpC;AACD,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,KAAK,CAAC,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;IACnC,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB;;;;;;OAMG;IACH,GAAG,CAAC,EAAE,QAAQ,GAAG,yBAAyB,CAAC;CAC5C;AACD,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC;AACzC,eAAO,MAAM,YAAY,EAAE,mBAA8B,CAAC;AAC1D,eAAO,MAAM,WAAW,EAAE,mBAA6B,CAAC;AACxD,eAAO,MAAM,gBAAgB,EAAE,mBAAkC,CAAC;AAClE,eAAO,MAAM,cAAc,EAAE,mBAAgC,CAAC;AAC9D,eAAO,MAAM,UAAU,EAAE,mBAA4B,CAAC;AACtD,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,qBAAqB,CAAC;IAC9B;;;;;;;OAOG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,KAAK,CAAC,EAAE,oBAAoB,CAAC;IAC7B,QAAQ,CAAC,EAAE,uBAAuB,CAAC;IACnC,UAAU,CAAC,EAAE,yBAAyB,CAAC;IACvC,IAAI,CAAC,EAAE,mBAAmB,CAAC;IAC3B,eAAe,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IACjD,eAAe,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;CAClD;AACD,MAAM,WAAW,mBAAmB;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IACnC,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAW;IAC5B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AACD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,MAAM,CAAW;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AACD,MAAM,WAAW,mBAAmB;IAClC,qBAAqB,EAAE,qBAAqB,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAA2C;IAC3D,QAAQ,EAAE,MAAM,CAA2C;IAC3D,YAAY,EAAE,MAAM,CAAe;IACnC,YAAY,EAAE,MAAM,CAAe;CACpC;AACD,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AACD,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAW;IACtB,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAW;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,iBAAiB,CAAC,EAAE,QAAQ,CAAC;IAC7B,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AACD,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,aAAa,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B,iBAAiB,CAAC,EAAE,QAAQ,CAAC;IAC7B,iBAAiB,CAAC,EAAE,QAAQ,CAAC;CAC9B;AACD,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAa;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAa;IAC9B,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,OAAO,CAAC,EAAE,uBAAuB,CAAC;IAClC;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AACD,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;CACzB;AACD;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,IAAI,CAAC,EAAE,aAAa,CAAC;CACtB;AACD;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,aAAa,CAAC;CACtB;AACD,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,SAAS,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IAC3C,gBAAgB,CAAC,EAAE,cAAc,CAAC;IAClC,SAAS,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IAC3C,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,EAAE,CAAC,aAAa,GAAG,SAAS,CAAC,EAAE,CAAC;IACzC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,aAAa,CAAC,EAAE,qBAAqB,CAAC;IACtC;;;;;;;;;;OAUG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B;;;;;;;;;;;OAWG;IACH,sBAAsB,CAAC,EAAE,QAAQ,CAAC;CACnC;AACD;;;;;;;;GAQG;AACH,MAAM,WAAW,mBAAmB;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,qBAAqB,CAAC,EAAE,MAAM,CAAe;IAC7C,qBAAqB,CAAC,EAAE,MAAM,CAAe;IAC7C,sBAAsB,CAAC,EAAE,QAAQ,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oBAAoB,CAAC,EAAE,QAAQ,CAAC;CACjC;AACD;;GAEG;AACH,MAAM,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAC3C;;GAEG;AACH,eAAO,MAAM,+BAA+B,EAAE,qBAAoC,CAAC;AACnF;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,qBAA6B,CAAC;AACrE,MAAM,WAAW,eAAe;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IAC1C,eAAe,CAAC,EAAE,qBAAqB,CAAC;IACxC,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C,GAAG,CAAC,EAAE,6BAA6B,CAAC;IACpC,GAAG,CAAC,EAAE,6BAA6B,CAAC;IACpC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,cAAc,CAAC;CAC3B;AACD;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AACD,MAAM,WAAW,UAAU;IACzB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAW;CAC1B;AACD,MAAM,MAAM,cAAc,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,CAAC;AACnD,MAAM,WAAW,cAAc;IAC7B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAAA;KAAC,CAAC;CAC1D;AACD;;;GAGG;AACH,eAAO,MAAM,eAAe,kBAAkB,CAAC;AAC/C;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC;AACxC;;;GAGG;AACH,eAAO,MAAM,yBAAyB,EAAE,kBAA8B,CAAC;AACvE;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB,EAAE,kBAA6B,CAAC;AACrE,MAAM,WAAW,cAAc;IAC7B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,cAAc,CAAC;IACtB;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,iBAAiB,CAAC;IACxB,GAAG,CAAC,EAAE,iBAAiB,CAAC;IACxB,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,IAAI,CAAC,EAAE,kBAAkB,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IAC1C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IACxC;;;;;;OAMG;IACH,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAa,CAAC;IACnD,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AACD;;;;;GAKG;AACH,MAAM,WAAW,qBAAqB;IACpC;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,CAAC,qBAAqB,GAAG,SAAS,CAAC,EAAE,CAAC;IACzD;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAe;IAC5C;;;;;;;;;OASG;IACH,KAAK,CAAC,EAAE,SAAS,GAAG,IAAI,GAAG,KAAK,CAAC;CAClC;AACD;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAC/B;;;;GAIG;AACH,eAAO,MAAM,WAAW,EAAE,SAAgB,CAAC;AAC3C;;;;GAIG;AACH,eAAO,MAAM,YAAY,EAAE,SAAiB,CAAC;AAC7C;;;;;;;;GAQG;AACH,MAAM,WAAW,qBAAqB;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAe;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAe;IACjC,WAAW,CAAC,EAAE,MAAM,CAAe;IACnC,aAAa,CAAC,EAAE,MAAM,CAAe;IACrC,YAAY,CAAC,EAAE,MAAM,CAAe;IACpC,eAAe,CAAC,EAAE,MAAM,CAAe;IACvC,YAAY,CAAC,EAAE,MAAM,CAAe;IACpC;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAe;CACtC;AACD,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAe;IAClC,YAAY,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IAC1C;;;;;;;;OAQG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AACD;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACtC,oBAAoB,CAAC,EAAE,0CAA0C,CAAC;CACnE;AACD,MAAM,WAAW,0CAA0C;IACzD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,6BAA6B,CAAC,EAAE,OAAO,CAAC;IACxC,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AACD,MAAM,MAAM,OAAO,GAAG,cAAc,CAAC;AACrC,MAAM,MAAM,OAAO,GAAG,cAAc,CAAC;AACrC,MAAM,WAAW,uBAAuB;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,gBAAgB,EAAE,QAAQ,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAe;IACzC,cAAc,EAAE,MAAM,CAAe;IACrC,cAAc,EAAE,MAAM,CAAe;IACrC,SAAS,EAAE,MAAM,CAAW;IAC5B,SAAS,EAAE,MAAM,CAAW;CAC7B;AACD,MAAM,WAAW,qBAAqB;IACpC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAW;IAChC,YAAY,CAAC,EAAE,QAAQ,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AACD;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IACnC;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAW;IAC5B;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AACD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAa;IAC5B,mBAAmB,CAAC,EAAE,QAAQ,CAAC;IAC/B;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,QAAQ,CAAC;IAC/B,iBAAiB,CAAC,EAAE,0BAA0B,CAAC;IAC/C,kCAAkC,CAAC,EAAE,MAAM,CAAa;IACxD;;;;;OAKG;IACH,sCAAsC,CAAC,EAAE,MAAM,CAAa;IAC5D,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;OAOG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAa;IAC5C;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;OAGG;IACH,SAAS,CAAC,EAAE,uBAAuB,CAAC;IACpC;;OAEG;IACH,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB;;OAEG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAa;IAC9C,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B;;;;OAIG;IACH,UAAU,CAAC,EAAE,mBAAmB,CAAC;CAClC;AACD;;;;;GAKG;AACH,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,QAAQ,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAW;IAC5B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAa;IACtC;;;;OAIG;IACH,eAAe,CAAC,EAAE,QAAQ,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,0BAA0B,CAAC;IACnC,KAAK,CAAC,EAAE,0BAA0B,CAAC;CACpC;AACD;;;;;GAKG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC;AAC9C,eAAO,MAAM,mBAAmB,EAAE,wBAAwC,CAAC;AAC3E,eAAO,MAAM,iBAAiB,EAAE,wBAAsC,CAAC;AACvE,eAAO,MAAM,iBAAiB,EAAE,wBAAsC,CAAC;AACvE,eAAO,MAAM,iBAAiB,EAAE,wBAAsC,CAAC;AACvE,MAAM,WAAW,0BAA0B;IACzC,UAAU,CAAC,EAAE,MAAM,CAAa;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAa;IACtC,iBAAiB,CAAC,EAAE,MAAM,CAAa;IACvC,KAAK,CAAC,EAAE,wBAAwB,CAAC;IACjC,UAAU,CAAC,EAAE,QAAQ,CAAC;CACvB;AACD,MAAM,WAAW,cAAc;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,iBAAiB,EAAE,CAAC;IACpC;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,WAAW,CAAC,EAAE,CAAC;IACvE;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,GAAG,CAAC;IAC7C,KAAK,CAAC,EAAE,iBAAiB,CAAC;IAC1B,cAAc,CAAC,EAAE,0BAA0B,CAAC;IAC5C,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,KAAK,CAAC,EAAE,iBAAiB,CAAC;IAC1B,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AACD;;;;GAIG;AACH,MAAM,MAAM,qBAAqB,GAAG,cAAc,CAAC;AACnD;;;;;GAKG;AACH,MAAM,MAAM,sBAAsB,GAAG,cAAc,CAAC;AACpD;;;;;;GAMG;AACH,MAAM,MAAM,mBAAmB,GAAG,cAAc,CAAC;AACjD,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAW;IAC9B,KAAK,CAAC,EAAE,QAAQ,CAAC;IACjB,eAAe,CAAC,EAAE,QAAQ,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAe;IACrC,MAAM,CAAC,EAAE,QAAQ,CAAC;IAClB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;OAEG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAW;IAC1C;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,QAAQ,CAAC;CAC7B;AACD,MAAM,WAAW,0BAA0B;IACzC,qBAAqB,EAAE,MAAM,CAAY;IACzC,wBAAwB,EAAE,MAAM,CAAY;IAC5C,aAAa,CAAC,EAAE,QAAQ,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAY;IACzC,wBAAwB,EAAE,MAAM,CAAY;CAC7C;AACD;;;;;;;;GAQG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,QAAQ,GAAG,gBAAgB,CAAC;CACxC;AACD;;;;;;;GAOG;AACH,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,QAAQ,GAAG,gBAAgB,CAAC;IACpC,QAAQ,EAAE,MAAM,CAAW;CAC5B;AACD,MAAM,MAAM,gCAAgC,GAAG,MAAM,CAAC;AACtD,eAAO,MAAM,2CAA2C,EAAE,gCAAgD,CAAC;AAC3G,eAAO,MAAM,2DAA2D,EAAE,gCAAgE,CAAC;AAC3I,eAAO,MAAM,qDAAqD,EAAE,gCAA0D,CAAC;AAC/H,eAAO,MAAM,mDAAmD,EAAE,gCAAwD,CAAC;AAC3H,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC;AAC9C,eAAO,MAAM,mCAAmC,EAAE,wBAAwC,CAAC;AAC3F,eAAO,MAAM,mDAAmD,EAAE,wBAAwD,CAAC;AAC3H,eAAO,MAAM,6CAA6C,EAAE,wBAAkD,CAAC;AAC/G,eAAO,MAAM,2CAA2C,EAAE,wBAAgD,CAAC;AAC3G,MAAM,WAAW,qBAAqB;IACpC,eAAe,EAAE,MAAM,CAAW;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAW;IACtC,eAAe,CAAC,EAAE,wBAAwB,CAAC;IAC3C,uBAAuB,CAAC,EAAE,gCAAgC,CAAC;IAC3D,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IAC1C,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,uBAAuB,CAAC,EAAE,6BAA6B,CAAC;IACxD;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IACnD;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;;;;;;OAUG;IACH,eAAe,CAAC,EAAE,QAAQ,GAAG,gBAAgB,CAAC;IAC9C;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,QAAQ,GAAG,gBAAgB,CAAC;IAC7C;;;;;;;;;;;;;OAaG;IACH,oBAAoB,CAAC,EAAE,CAAC,4BAA4B,GAAG,SAAS,CAAC,EAAE,CAAC;CACrE;AACD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,4BAA4B;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,MAAM,CAAW;IAClC,YAAY,CAAC,EAAE,MAAM,CAAW;CACjC;AACD,MAAM,MAAM,2BAA2B,GAAG,MAAM,CAAC;AACjD,eAAO,MAAM,+BAA+B,EAAE,2BAAoC,CAAC;AACnF,eAAO,MAAM,6BAA6B,EAAE,2BAAkC,CAAC;AAC/E,MAAM,WAAW,6BAA6B;IAC5C;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,EAAE,CAAC,EAAE,aAAa,CAAC;CACpB;AACD,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAW;IAC9B;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAa;IAC7B;;OAEG;IACH,aAAa,CAAC,EAAE,QAAQ,CAAC;IACzB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,aAAa,CAAC;IAC5B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,MAAM,WAAW,uBAAuB;IACtC,gBAAgB,EAAE,MAAM,CAAY;IACpC,aAAa,CAAC,EAAE,QAAQ,CAAC;IACzB,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AACD,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,oBAAoB,CAAC;IAC7B,OAAO,EAAE,qBAAqB,EAAE,CAAC;CAClC;AACD,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,mBAAmB,EAAE,CAAC;IAC7B;;;;;OAKG;IACH,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAa,CAAC;CACpD;AACD,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAc;IAC9B;;OAEG;IACH,MAAM,EAAE,eAAe,CAAC;IACxB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAC;CAChC;AACD;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAW;AAC/C,eAAO,MAAM,qBAAqB,EAAE,eAAmB,CAAC;AACxD,eAAO,MAAM,qBAAqB,EAAE,eAAmB,CAAC;AACxD,eAAO,MAAM,mBAAmB,EAAE,eAAmB,CAAC;AACtD,eAAO,MAAM,kBAAkB,EAAE,eAAmB,CAAC;AACrD,eAAO,MAAM,mBAAmB,EAAE,eAAmB,CAAC;AACtD,eAAO,MAAM,oBAAoB,EAAE,eAAmB,CAAC;AACvD,eAAO,MAAM,mBAAmB,EAAE,eAAmB,CAAC;AACtD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AACD,MAAM,WAAW,aAAa;IAC5B,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAAA;KAAC,CAAC;CAC/D;AACD,MAAM,WAAW,aAAa;IAC5B,YAAY,EAAE,qBAAqB,CAAC;IACpC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,gBAAgB,CAAC;IACvB,GAAG,CAAC,EAAE,gBAAgB,CAAC;IACvB,eAAe,CAAC,EAAE,qBAAqB,CAAC;IACxC,iBAAiB,CAAC,EAAE,uBAAuB,CAAC;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,eAAe,CAAC,EAAE,CAAC,oBAAoB,GAAG,SAAS,CAAC,EAAE,CAAC;IACvD,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B;;;OAGG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AACD;;;;;;;GAOG;AACH,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,QAAQ,CAAC,EAAE,wBAAwB,CAAC;CACrC;AACD;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACvC,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,KAAK,CAAC,EAAE,yBAAyB,CAAC;CACnC;AACD;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,MAAM,CAAW;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,GAAG,CAAC;CACZ;AACD,MAAM,WAAW,uBAAuB;IACtC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,aAAa,CAAC,EAAE,GAAG,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC;;;;;;;;;;OAUG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC,+BAA+B,CAAC,EAAE,OAAO,CAAC;IAC1C,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,+BAA+B,CAAC,EAAE,OAAO,CAAC;IAC1C,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,kBAAkB,CAAC,EAAE,MAAM,CAAa;IACxC,oBAAoB,CAAC,EAAE,MAAM,CAAa;IAC1C,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,6BAA6B,CAAC,EAAE,MAAM,CAAa;CACpD;AACD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;;;;OAQG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,QAAQ,CAAC;IAC/B;;;;;;;;;;OAUG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAa;IACzC;;;;;;;;OAQG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACpC;AACD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;;;OASG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,0BAA0B,CAAC;CAChD;AACD;;;;GAIG;AACH,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,0BAA0B,CAAC;IACnC,KAAK,CAAC,EAAE,0BAA0B,CAAC;CACpC;AACD;;;;;GAKG;AACH,MAAM,WAAW,0BAA0B;IACzC,SAAS,CAAC,EAAE,MAAM,CAAa;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAa;IACrC,sBAAsB,CAAC,EAAE,MAAM,CAAa;IAC5C,UAAU,CAAC,EAAE,QAAQ,CAAC;CACvB;AACD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAa;IAC5B,qBAAqB,CAAC,EAAE,MAAM,CAAa;IAC3C,2BAA2B,CAAC,EAAE,QAAQ,CAAC;IACvC,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B,sBAAsB,CAAC,EAAE,MAAM,CAAa;IAC5C,0BAA0B,CAAC,EAAE,MAAM,CAAa;IAChD,uBAAuB,CAAC,EAAE,MAAM,CAAa;IAC7C,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,uBAAuB,CAAC,EAAE,MAAM,CAAW;IAC3C;;;;;;;;;;;;;OAaG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAa;IAC7C;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC;;;OAGG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAW;IAC/C;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;;;;;OAMG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAa;IAC/C;;;;;;OAMG;IACH,uBAAuB,CAAC,EAAE,MAAM,EAAE,CAAC;IACnC;;;;OAIG;IACH,YAAY,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAC,CAAC;IACzC;;;;;OAKG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IACtC;;;;;;OAMG;IACH,kCAAkC,CAAC,EAAE,MAAM,CAAe;IAC1D;;;;;;;;OAQG;IACH,gCAAgC,CAAC,EAAE,MAAM,CAAe;IACxD;;;;;;OAMG;IACH,+BAA+B,CAAC,EAAE,MAAM,CAAe;IACvD;;;;;;OAMG;IACH,8BAA8B,CAAC,EAAE,OAAO,CAAC;IACzC;;;;;;;;;;OAUG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;;;;;;;;;OAUG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AACD;;;;;;;;;GASG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;IAC7B;;;;;;;OAOG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAa;CAC1C;AACD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;OAEG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;OAEG;IACH,0BAA0B,CAAC,EAAE,OAAO,CAAC;CACtC;AACD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAC/B;;;GAGG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAqB,CAAC;AACrD;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,EAAE,SAA4B,CAAC;AAClE;;;;GAIG;AACH,eAAO,MAAM,wBAAwB,EAAE,SAA8B,CAAC;AACtE;;;;;GAKG;AACH,eAAO,MAAM,8BAA8B,EAAE,SAAqC,CAAC;AACnF;;;;;;;GAOG;AACH,MAAM,WAAW,qBAAqB;IACpC,YAAY,CAAC,EAAE,QAAQ,CAAC;IACxB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,GAAG,gBAAgB,GAAG,kBAAkB,GAAG,yBAAyB,CAAC;IACtG;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,2BAA2B,GAAG,MAAM,CAAC;CACjD;AACD;;;GAGG;AACH,MAAM,WAAW,2BAA2B;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,gBAAgB,CAAC,EAAE,QAAQ,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAW;CAClC;AACD,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAC9B,eAAO,MAAM,cAAc,EAAE,QAAmB,CAAC;AACjD,eAAO,MAAM,gBAAgB,EAAE,QAAqB,CAAC;AACrD,eAAO,MAAM,WAAW,EAAE,QAAgB,CAAC;AAC3C,eAAO,MAAM,YAAY,EAAE,QAAiB,CAAC;AAC7C,eAAO,MAAM,eAAe,EAAE,QAAoB,CAAC;AACnD,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,oBAAoB,EAAE,CAAC;CACpC;AACD,MAAM,WAAW,kBAAkB;IACjC,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;;;;;;;;OAgBG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,GAAG,CAAC,EAAE,iBAAiB,CAAC;IACxB,IAAI,CAAC,EAAE,kBAAkB,CAAC;CAC3B;AACD,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AACD,MAAM,WAAW,sBAAsB;IACrC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,KAAK,CAAC,EAAE,2BAA2B,CAAC;IACpC,KAAK,CAAC,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,OAAO,CAAC,EAAE,QAAQ,CAAC;CACpB;AACD,MAAM,WAAW,2BAA2B;IAC1C,GAAG,CAAC,EAAE,MAAM,CAA2C;IACvD,OAAO,CAAC,EAAE,MAAM,CAAa;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAa;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAa;CAClC;AACD,MAAM,WAAW,mBAAmB;IAClC,WAAW,CAAC,EAAE,MAAM,CAAW;IAC/B,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AACD,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AACD,MAAM,WAAW,iBAAiB;IAChC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,aAAa,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IAC3C,gBAAgB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+BAA+B,CAAC,EAAE,QAAQ,CAAC;IAC3C;;;OAGG;IACH,qCAAqC,CAAC,EAAE,OAAO,CAAC;IAChD;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACnC;AACD,MAAM,WAAW,kBAAkB;IACjC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AACD,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,EAAE,OAAO,CAAC;IACxB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AACD,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAC/B,eAAO,MAAM,qBAAqB,EAAE,SAAqB,CAAC;AAC1D,eAAO,MAAM,qBAAqB,EAAE,SAAqB,CAAC;AAC1D,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAW;IACxB,cAAc,CAAC,EAAE,SAAS,CAAC;IAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;;;;;;;;;;;;OAiBG;IACH,cAAc,CAAC,EAAE,CAAC,0BAA0B,GAAG,SAAS,CAAC,EAAE,CAAC;IAC5D;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B;;;OAGG;IACH,uBAAuB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IACrD;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;OAGG;IACH,qBAAqB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;KAAC,CAAC;IACnD;;;;;;;;;;OAUG;IACH,wBAAwB,CAAC,EAAE,QAAQ,CAAC;CACrC;AACD;;;GAGG;AACH,MAAM,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAC/C,eAAO,MAAM,gBAAgB,EAAE,yBAAkC,CAAC;AAClE,eAAO,MAAM,gBAAgB,EAAE,yBAAkC,CAAC;AAClE;;;;;;;;GAQG;AACH,MAAM,WAAW,0BAA0B;IACzC;;;;;;OAMG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;;;;OASG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,MAAM,CAAC,EAAE,CAAC,8BAA8B,GAAG,SAAS,CAAC,EAAE,CAAC;IACxD;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,EAAgB,CAAC;CAClC;AACD;;;GAGG;AACH,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;CACzB;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,oBAAoB,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAe;IACtC;;;OAGG;IACH,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAe;CACzC;AACD;;;GAGG;AACH,MAAM,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAC/C;;GAEG;AACH,eAAO,MAAM,0BAA0B,EAAE,yBAAmC,CAAC;AAC7E;;GAEG;AACH,eAAO,MAAM,yBAAyB,EAAE,yBAAkC,CAAC;AAC3E;;;GAGG;AACH,eAAO,MAAM,0BAA0B,EAAE,yBAAmC,CAAC;AAK7E;;GAEG;AACH,eAAO,MAAM,sBAAsB,QAAQ,CAAC;AAC5C;;GAEG;AACH,eAAO,MAAM,2BAA2B,aAAa,CAAC;AACtD;;GAEG;AACH,eAAO,MAAM,uBAAuB,SAAS,CAAC;AAC9C;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,MAAM,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,oBAAoB,GAAG,SAAS,CAAA;KAAC,CAAC;IAC5D;;OAEG;IACH,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B;;;;;;;;;;;;;OAaG;IACH,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC;;;;;OAKG;IACH,eAAe,CAAC,EAAE,8BAA8B,CAAC;IACjD;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAW;IAC/B;;;;;;;;;;;;;OAaG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;;;;;;;;;;;;OAcG;IACH,UAAU,CAAC,EAAE,yBAAyB,CAAC;IACvC;;;;;OAKG;IACH,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B;;;;;OAKG;IACH,uBAAuB,CAAC,EAAE,6BAA6B,CAAC;CACzD;AACD;;;;;;;GAOG;AACH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAW;CACrC;AACD;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AACD;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,QAAQ,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAAA;KAAC,CAAC;CAC5D;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC;IAClC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,YAAY,CAAC,EAAE,MAAM,CAAW;IAChC,aAAa,CAAC,EAAE,MAAM,CAAW;CAClC;AACD;;;;;GAKG;AACH,MAAM,WAAW,8BAA8B;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAKD,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAW;AACjD;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,EAAE,iBAAqB,CAAC;AAC/D;;GAEG;AACH,eAAO,MAAM,4BAA4B,EAAE,iBAAqB,CAAC;AACjE;;;GAGG;AACH,eAAO,MAAM,yBAAyB,EAAE,iBAAqB,CAAC;AAC9D;;;GAGG;AACH,eAAO,MAAM,wBAAwB,EAAE,iBAAqB,CAAC;AAC7D;;;;;;;;GAQG;AACH,eAAO,MAAM,oBAAoB,EAAE,iBAAsB,CAAC;AAC1D,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAW;AAClD,eAAO,MAAM,wBAAwB,EAAE,kBAAsB,CAAC;AAC9D,eAAO,MAAM,uBAAuB,EAAE,kBAAsB,CAAC;AAC7D,eAAO,MAAM,sBAAsB,EAAE,kBAAsB,CAAC;AAC5D;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAC1C,eAAO,MAAM,wBAAwB,EAAE,oBAA6B,CAAC;AACrE,eAAO,MAAM,uBAAuB,EAAE,oBAA4B,CAAC;AACnE,eAAO,MAAM,uBAAuB,EAAE,oBAA4B,CAAC;AAKnE;;;;GAIG;AACH,MAAM,MAAM,qBAAqB,GAAG,GAAG,CAAC;AACxC;;;;GAIG;AACH,MAAM,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAK5C;;;;;GAKG;AACH,MAAM,MAAM,4BAA4B,GAAG,GAAG,CAAC;AAK/C;;;;GAIG;AACH,MAAM,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAC5C,eAAO,MAAM,sBAAsB,EAAE,sBAAkC,CAAC;AACxE,eAAO,MAAM,oBAAoB,EAAE,sBAAgC,CAAC;AACpE,eAAO,MAAM,6BAA6B,EAAE,sBAA0C,CAAC;AACvF,eAAO,MAAM,0BAA0B,EAAE,sBAAuC,CAAC;AACjF,eAAO,MAAM,0BAA0B,EAAE,sBAAuC,CAAC;AACjF,eAAO,MAAM,0BAA0B,EAAE,sBAAuC,CAAC;AACjF,eAAO,MAAM,0BAA0B,EAAE,sBAAuC,CAAC;AACjF,eAAO,MAAM,yBAAyB,EAAE,sBAAsC,CAAC;AAC/E,eAAO,MAAM,+BAA+B,EAAE,sBAA4C,CAAC;AAC3F,eAAO,MAAM,0BAA0B,EAAE,sBAAuC,CAAC;AACjF,eAAO,MAAM,wBAAwB,EAAE,sBAAoC,CAAC;AAC5E,eAAO,MAAM,sBAAsB,EAAE,sBAAkC,CAAC;AACxE,eAAO,MAAM,sBAAsB,EAAE,sBAAkC,CAAC;AACxE;;;;;GAKG;AACH,MAAM,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAC7C,eAAO,MAAM,sBAAsB,EAAE,uBAAmC,CAAC;AACzE,eAAO,MAAM,oBAAoB,EAAE,uBAAiC,CAAC;AACrE,eAAO,MAAM,oBAAoB,EAAE,uBAAiC,CAAC;AACrE,eAAO,MAAM,4BAA4B,EAAE,uBAA0C,CAAC;AACtF,eAAO,MAAM,oBAAoB,EAAE,uBAAiC,CAAC;AACrE;;;;;;;;;;GAUG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,GAAG,CAAiB;IAC/B,QAAQ,EAAE,MAAM,CAA2C;IAC3D,OAAO,EAAE,sBAAsB,CAAC;IAChC,MAAM,EAAE,uBAAuB,CAAC;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,MAAM,CAAW;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,WAAW,EAAE,MAAM,CAAa;CACjC;AACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,gBAAgB,EAAE,GAAG,CAAoB;IACzC,eAAe,EAAE,GAAG,CAAoB;IACxC,cAAc,EAAE,GAAG,CAAoB;IACvC,eAAe,EAAE,GAAG,CAAoB;IACxC,cAAc,EAAE,GAAG,CAAoB;IACvC,aAAa,EAAE,GAAG,CAAoB;IACtC,aAAa,EAAE,GAAG,CAAoB;IACtC,YAAY,EAAE,GAAG,CAAoB;IACrC,WAAW,EAAE,GAAG,CAAoB;IACpC;;OAEG;IACH,cAAc,EAAE,GAAG,CAAoB;IACvC;;OAEG;IACH,iBAAiB,EAAE,GAAG,CAAoB;IAC1C;;OAEG;IACH,wBAAwB,EAAE,GAAG,CAAoB;IACjD,SAAS,EAAE,GAAG,CAAiB;CAChC;AACD;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAW;IAC3B,OAAO,EAAE,MAAM,CAAW;IAC1B,MAAM,EAAE,MAAM,CAAW;IACzB;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAW;IACnC,eAAe,EAAE,MAAM,CAAW;IAClC,cAAc,EAAE,MAAM,CAAW;IACjC,eAAe,EAAE,MAAM,CAAW;IAClC,cAAc,EAAE,MAAM,CAAW;IACjC,aAAa,EAAE,MAAM,CAAW;IAChC,aAAa,EAAE,MAAM,CAAW;IAChC,YAAY,EAAE,MAAM,CAAW;IAC/B,WAAW,EAAE,MAAM,CAAW;IAC9B,cAAc,EAAE,MAAM,CAAW;IACjC,iBAAiB,EAAE,MAAM,CAAW;IACpC,wBAAwB,EAAE,MAAM,CAAW;IAC3C,SAAS,EAAE,GAAG,CAAiB;CAChC;AAKD,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC;AACzC,eAAO,MAAM,eAAe,EAAE,mBAA2B,CAAC;AAC1D,eAAO,MAAM,eAAe,EAAE,mBAA2B,CAAC;AAC1D,MAAM,MAAM,OAAO,GAAG,GAAG,CAAC;AAC1B;;;;;GAKG;AACH,MAAM,MAAM,UAAU,GAClB,OAAO,CAAC;AACZ;;;;GAIG;AACH,MAAM,MAAM,UAAU,GAClB,OAAO,CAAC;AACZ,MAAM,MAAM,eAAe,GAAG,GAAG,CAAC;AAClC,MAAM,MAAM,cAAc,GAAG,GAAG,CAAC;AAKjC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;;;;OAKG;IACH,GAAG,EAAE,MAAM,CAAa;IACxB;;;;;;;OAOG;IACH,QAAQ,EAAE,MAAM,CAAa;IAC7B;;;;;OAKG;IACH,GAAG,EAAE,MAAM,CAAa;IACxB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAW;CAC1B;AACD;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAW;AAC/C;;GAEG;AACH,eAAO,MAAM,uBAAuB,EAAE,eAAoB,CAAC;AAK3D;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,GAAG,CAAC;AAK9B,MAAM,MAAM,KAAK,GAAG,MAAM,CAAC;AAC3B;;;GAGG;AACH,eAAO,MAAM,YAAY,EAAE,KAAiB,CAAC;AAC7C;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,KAAkB,CAAC;AAC/C,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAClC;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,GAAG,GAAG,CAAC;AAChC,MAAM,MAAM,QAAQ,GAAG,GAAG,CAAC;AAK3B;;;;;;GAMG;AACH,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,eAAe,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAKD;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,MAAM,CAAC"} \ No newline at end of file diff --git a/typescript/config/lib/index.js.map b/typescript/config/lib/index.js.map index b2b2d2ad9..ac8236986 100644 --- a/typescript/config/lib/index.js.map +++ b/typescript/config/lib/index.js.map @@ -1,7 +1,7 @@ { "version": 3, "sources": ["../src/index.ts", "../src/generated.ts", "../src/constants.ts"], - "sourcesContent": ["export type {\n // Tygo generic replacement\n LogLevel,\n Duration,\n ByteSize,\n NetworkArchitecture,\n ConnectorDriverType,\n ConnectorConfig,\n UpstreamType,\n // Policy evaluation\n PolicyEvalUpstreamMetrics,\n PolicyEvalUpstream,\n PolicyEvalUpstreamArray,\n PolicyEvalContext,\n PolicyEvalPredicate,\n EvalContext,\n SelectionPolicyEvalFunction,\n ScoreWeights,\n ScoreBreakdown,\n TagPattern,\n Pattern,\n SortByScoreOptions,\n RemoveByLatencyOptions,\n RemoveByLagOptions,\n KeepHealthyOptions,\n PreferOptions,\n StickyPrimaryOptions,\n ProbeExcludedOptions,\n LatencyDeviationOptions,\n ByFinalityHandlers,\n WhereFilter,\n PolicyEvalArrayCondition,\n IncludeIfTarget,\n EvmNetworkConfigForDefaults,\n SvmNetworkConfigForDefaults,\n} from \"./types\";\nexport {\n // Data finality const exports\n DataFinalityStateUnfinalized,\n DataFinalityStateFinalized,\n DataFinalityStateRealtime,\n DataFinalityStateUnknown,\n // Scope exports\n ScopeNetwork,\n ScopeUpstream,\n // Selection-policy evalScope + stickyPrimary scope constants (canonical)\n EvalScopeNetwork,\n EvalScopeNetworkMethod,\n EvalScopeNetworkFinality,\n EvalScopeNetworkMethodFinality,\n // Cache behavior exports\n CacheEmptyBehaviorIgnore,\n CacheEmptyBehaviorAllow,\n CacheEmptyBehaviorOnly,\n // Evm node type\n EvmNodeTypeFull,\n EvmNodeTypeArchive,\n EvmNodeTypeUnknown,\n // Evm syncing type\n EvmSyncingStateUnknown,\n EvmSyncingStateSyncing,\n EvmSyncingStateNotSyncing,\n // Architecture export\n ArchitectureEvm,\n ArchitectureSvm,\n // Upstream types const exprots\n UpstreamTypeEvm,\n UpstreamTypeSvm,\n // Auth types\n AuthTypeSecret,\n AuthTypeJwt,\n AuthTypeSiwe,\n AuthTypeNetwork,\n // Consensus related\n ConsensusLowParticipantsBehaviorReturnError,\n ConsensusLowParticipantsBehaviorAcceptMostCommonValidResult,\n ConsensusLowParticipantsBehaviorPreferBlockHeadLeader,\n ConsensusLowParticipantsBehaviorOnlyBlockHeadLeader,\n ConsensusDisputeBehaviorReturnError,\n ConsensusDisputeBehaviorAcceptMostCommonValidResult,\n ConsensusDisputeBehaviorPreferBlockHeadLeader,\n ConsensusDisputeBehaviorOnlyBlockHeadLeader,\n // Rate limiter periods\n RateLimitPeriodSecond,\n RateLimitPeriodMinute,\n RateLimitPeriodHour,\n RateLimitPeriodDay,\n RateLimitPeriodWeek,\n RateLimitPeriodMonth,\n RateLimitPeriodYear,\n // Rate limiter count modes\n RateLimitCountModeRequest,\n RateLimitCountModeCredit,\n} from \"./generated\";\n// Short-name re-exports for selection-policy evalScope / stickyPrimary\n// scope (CAPITAL_SNAKE_CASE matching the JS ambient globals installed\n// by the policy stdlib) and the finality bit-flags for `when(mask, ...)`.\nexport {\n NETWORK,\n NETWORK_METHOD,\n NETWORK_FINALITY,\n NETWORK_METHOD_FINALITY,\n REALTIME,\n UNFINALIZED,\n FINALIZED,\n UNKNOWN,\n} from \"./constants\";\nexport type {\n Config,\n ProjectConfig,\n HealthCheckConfig,\n // Provider related\n ProviderConfig,\n VendorSettings,\n // Upstream related\n UpstreamConfig,\n EvmUpstreamConfig,\n EvmQueryShimConfig,\n SvmUpstreamConfig,\n UpstreamIntegrityConfig,\n UpstreamIntegrityEthGetBlockReceiptsConfig,\n RateLimitAutoTuneConfig,\n JsonRpcUpstreamConfig,\n // Failsafe related\n FailsafeConfig,\n RetryPolicyConfig,\n CircuitBreakerPolicyConfig,\n HedgePolicyConfig,\n TimeoutPolicyConfig,\n ConsensusPolicyConfig,\n // Network related\n NetworkConfig,\n EvmNetworkConfig,\n EvmIntegrityConfig,\n SvmNetworkConfig,\n SelectionPolicyConfig,\n EvalScope,\n DirectiveDefaultsConfig,\n // DB related\n DatabaseConfig,\n CacheConfig,\n DataFinalityState,\n CacheEmptyBehavior,\n CachePolicyConfig,\n MemoryConnectorConfig,\n RedisConnectorConfig,\n DynamoDBConnectorConfig,\n AwsAuthConfig,\n PostgreSQLConnectorConfig,\n // Auth related\n AuthStrategyConfig,\n SecretStrategyConfig,\n JwtStrategyConfig,\n SiweStrategyConfig,\n NetworkStrategyConfig,\n // Rate limits related\n RateLimiterConfig,\n RateLimitBudgetConfig,\n RateLimitRuleConfig,\n // Server config related\n ServerConfig,\n CORSConfig,\n MetricsConfig,\n AdminConfig,\n AliasingConfig,\n AliasingRuleConfig,\n TLSConfig,\n // Proxy pools related\n ProxyPoolConfig,\n} from \"./generated\";\n\nimport type { Config } from './generated'\n\nexport const createConfig = (\n cfg: Config\n): Config => {\n return cfg;\n};\n", "// Code generated by tygo. DO NOT EDIT.\nimport type {\n LogLevel,\n Duration,\n ByteSize,\n ConnectorDriverType as TsConnectorDriverType,\n ConnectorConfig as TsConnectorConfig,\n UpstreamType as TsUpstreamType,\n NetworkArchitecture as TsNetworkArchitecture,\n AuthType as TsAuthType,\n AuthStrategyConfig as TsAuthStrategyConfig,\n EvmNetworkConfigForDefaults as TsEvmNetworkConfigForDefaults,\n SvmNetworkConfigForDefaults as TsSvmNetworkConfigForDefaults,\n SelectionPolicyEvalFunction,\n BoolOrString\n} from \"./types\"\n\n//////////\n// source: adaptive_duration.go\n\n/**\n * AdaptiveDuration describes a duration that may be static, derived from a\n * per-method latency quantile, or both. It's the reusable building block\n * for any failsafe knob that wants \"fixed base + adaptive component\n * clamped between min/max\" semantics \u2014 currently consensus wait caps,\n * with timeout/hedge supporting it as an alternative entry-point.\n * Resolution rules:\n * \tfinal = Base + adaptive\n * where `adaptive` is:\n * - `qt.GetQuantile(Quantile)` when Quantile > 0 and quantile data exists\n * - `Min` (the floor) when Quantile > 0 but quantile data is cold (no\n * observations yet) \u2014 this gives a sensible non-zero cap immediately\n * after boot\n * - `0` when Quantile is unset\n * After `Base + adaptive`, the result is clamped to [Min, Max] when those\n * are set. A nil or all-zero AdaptiveDuration returns 0 (the caller treats\n * that as \"no cap\" / \"disabled\").\n * Wire format accepts both shorthand and object form:\n * \tcaps: 500ms # shorthand: Base only\n * \tcaps: { base: 500ms } # explicit Base\n * \tcaps: { quantile: 0.5, min: 5ms, max: 1s } # quantile with bounds\n * \tcaps: { base: 100ms, quantile: 0.9, max: 2s } # combined\n */\nexport interface AdaptiveDuration {\n base?: Duration;\n quantile?: number /* float64 */;\n min?: Duration;\n max?: Duration;\n}\n\n//////////\n// source: architecture.go\n\n/**\n * ArchitectureHandler abstracts all architecture-specific pipeline hooks.\n * Each architecture (evm, svm, \u2026) registers exactly one handler via its package init().\n * Pipeline files dispatch through the registry \u2014 they never need to change for a new chain.\n */\nexport type ArchitectureHandler = any;\n\n//////////\n// source: architecture_evm.go\n\nexport const UpstreamTypeEvm: UpstreamType = \"evm\";\nexport type EvmUpstream = \n Upstream;\n/**\n * EvmStateProvenWriter is the OPTIONAL, separately-asserted surface the\n * integrity state prober records proofs through. The proven head is telemetry\n * (the state-proven block / proven-lag gauges), never a routing input.\n * Deliberately NOT part of EvmUpstream: that interface is implemented outside\n * this repo, and widening it broke every existing implementor \u2014 the chainId\n * suggest-gate silently degraded when its upstream stopped satisfying the\n * assertion. Optional capabilities are asserted narrowly, never added to the\n * core interface.\n */\nexport type EvmStateProvenWriter = any;\nexport type AvailbilityConfidence = number /* int */;\nexport const AvailbilityConfidenceBlockHead: AvailbilityConfidence = 1;\nexport const AvailbilityConfidenceFinalized: AvailbilityConfidence = 2;\nexport type EvmNodeType = string;\nexport const EvmNodeTypeUnknown: EvmNodeType = \"unknown\";\nexport const EvmNodeTypeFull: EvmNodeType = \"full\";\nexport const EvmNodeTypeArchive: EvmNodeType = \"archive\";\nexport type EvmSyncingState = number /* int */;\nexport const EvmSyncingStateUnknown: EvmSyncingState = 0;\nexport const EvmSyncingStateSyncing: EvmSyncingState = 1;\nexport const EvmSyncingStateNotSyncing: EvmSyncingState = 2;\nexport type EvmStatePoller = any;\n/**\n * EvmStatePollerDiagnostics contains diagnostic information about the state poller\n * including block bounds, probe status, and any detection issues.\n */\nexport interface EvmStatePollerDiagnostics {\n enabled: boolean;\n /**\n * Block head information\n */\n latestBlock: number /* int64 */;\n finalizedBlock: number /* int64 */;\n /**\n * Syncing state\n */\n syncingState: string;\n skipSyncingCheck?: boolean;\n syncingCheckError?: string;\n /**\n * Latest block detection status\n */\n skipLatestBlockCheck?: boolean;\n latestBlockFailureCount?: number /* int */;\n latestBlockSuccessfulOnce?: boolean;\n latestBlockDetectionIssue?: string;\n /**\n * Finalized block detection status\n */\n skipFinalizedCheck?: boolean;\n finalizedBlockFailureCount?: number /* int */;\n finalizedBlockSuccessfulOnce?: boolean;\n finalizedBlockDetectionIssue?: string;\n /**\n * Earliest block bounds per probe type\n */\n earliestByProbe?: { [key: EvmAvailabilityProbeType]: EvmProbeEarliestInfo | undefined};\n}\n/**\n * EvmProbeEarliestInfo contains information about earliest block detection for a specific probe type\n */\nexport interface EvmProbeEarliestInfo {\n probeType: EvmAvailabilityProbeType;\n earliestBlock: number /* int64 */;\n schedulerRunning?: boolean;\n}\n\n//////////\n// source: architecture_svm.go\n\nexport const UpstreamTypeSvm: UpstreamType = \"svm\";\n/**\n * SlotSharedVariable mirrors the EVM SharedStateVariable pattern for int64 slot numbers.\n * It lets the state poller publish slot progress to a shared-state backend (memory or Redis)\n * while keeping callers synchronous via the cached local value.\n */\nexport type SlotSharedVariable = any;\n/**\n * SvmUpstream narrows common.Upstream to the surface SVM hooks need to reach\n * into per-upstream state (currently just the state poller). Keeping this a\n * tiny sub-interface avoids committing the full Upstream struct to common/ and\n * parallels the existing EvmUpstream pattern.\n */\nexport type SvmUpstream = \n Upstream;\n/**\n * SvmStatePoller is the per-upstream slot/health tracker. Concrete type lives in\n * architecture/svm to avoid pulling the full implementation into common.\n */\nexport type SvmStatePoller = any;\n/**\n * MaxShredInsertSlotLagThreshold is the cutoff beyond which an upstream is treated\n * as degraded for shred-insert lag. Solana nodes with high `latest - maxShredInsertSlot`\n * are receiving shreds but not processing them; their reads go stale silently.\n */\nexport const MaxShredInsertSlotLagThreshold: number /* int64 */ = 100;\n/**\n * SvmChainSolana is the canonical Solana chain identifier. Empty Chain values\n * in SvmNetworkConfig normalize to this constant at NetworkId derivation time\n * so pre-multi-chain configs keep producing \"svm:\" IDs.\n */\nexport const SvmChainSolana = \"solana\";\n\n//////////\n// source: blocktime_adaptive_duration.go\n\n/**\n * BlockTimeAdaptiveDuration is a duration that is either a fixed value or derived from the\n * network's estimated block time. It's the reusable shape for knobs (e.g. cache\n * TTLs) that should track each chain's cadence rather than a single constant.\n * Wire format accepts a scalar shorthand or an object:\n * \tttl: 2s # fixed\n * \tttl: { blockTimeMultiplier: 1 } # blockTime * 1 (caller default until known)\n * \tttl: { blockTimeMultiplier: 1, fallback: 2s } # with explicit cold-start fallback\n * See Resolve for how the value is computed.\n */\nexport interface BlockTimeAdaptiveDuration {\n /**\n * Fallback is the fixed value. A scalar shorthand sets it directly; in object\n * form it's the cold-start floor used until the block time is known.\n */\n fallback?: Duration;\n /**\n * BlockTimeMultiplier, when > 0, derives the value from the network's\n * estimated block time (blockTime * multiplier).\n */\n blockTimeMultiplier?: number /* float64 */;\n}\n\n//////////\n// source: cache_dal.go\n\nexport type CacheDAL = any;\n\n//////////\n// source: cache_mock.go\n\nexport interface MockCacheDal {\n mock: any /* mock.Mock */;\n}\n\n//////////\n// source: config.go\n\n/**\n * Config represents the configuration of the application.\n */\nexport interface Config {\n logLevel?: LogLevel;\n clusterKey?: string;\n server?: ServerConfig;\n healthCheck?: HealthCheckConfig;\n admin?: AdminConfig;\n database?: DatabaseConfig;\n projects?: (ProjectConfig | undefined)[];\n rateLimiters?: RateLimiterConfig;\n metrics?: MetricsConfig;\n proxyPools?: (ProxyPoolConfig | undefined)[];\n tracing?: TracingConfig;\n}\nexport interface ServerConfig {\n listenV4?: boolean;\n httpHostV4?: string;\n listenV6?: boolean;\n httpHostV6?: string;\n httpPort?: number /* int */; // Deprecated: use HttpPortV4\n httpPortV4?: number /* int */;\n httpPortV6?: number /* int */;\n grpcEnabled?: boolean;\n grpcHostV4?: string;\n grpcPortV4?: number /* int */;\n grpcHostV6?: string;\n grpcPortV6?: number /* int */;\n grpcMaxRecvMsgSize?: number /* int */;\n grpcMaxSendMsgSize?: number /* int */;\n grpcReflection?: boolean;\n maxTimeout?: Duration;\n readTimeout?: Duration;\n writeTimeout?: Duration;\n enableGzip?: boolean;\n tls?: TLSConfig;\n aliasing?: AliasingConfig;\n waitBeforeShutdown?: Duration;\n waitAfterShutdown?: Duration;\n includeErrorDetails?: boolean;\n trustedIPForwarders?: string[];\n trustedIPHeaders?: string[];\n responseHeaders?: { [key: string]: string};\n /**\n * ExecutionHeaders controls the per-request diagnostic headers\n * (X-ERPC-Attempts, X-ERPC-Upstreams-Tried, etc.) that expose how\n * eRPC routed and resolved each request. Defaults to \"all\" \u2014 set\n * \"summary\" to keep only counters, or \"off\" to disable entirely\n * (useful for low-latency / bandwidth-constrained clients).\n */\n executionHeaders?: ExecutionHeadersMode;\n /**\n * CostHeaders opts into the cost/billing response headers\n * (X-ERPC-Calls, X-ERPC-Billable, X-ERPC-Methods, X-ERPC-Credits,\n * X-ERPC-Credits-Version) on single and batch responses. Off by\n * default. Credit-unit pricing is vendor-level configuration \u2014 see\n * CreditUnitsProvider and UpstreamConfig.CreditUnits.\n */\n costHeaders?: boolean;\n}\n/**\n * ExecutionHeadersMode controls how much per-request execution detail is\n * exposed in HTTP response headers.\n */\nexport type ExecutionHeadersMode = string;\n/**\n * ExecutionHeadersAll emits the full set: counters + per-upstream\n * trace (upstream IDs, outcomes, reasons, durations). Default.\n */\nexport const ExecutionHeadersAll: ExecutionHeadersMode = \"all\";\n/**\n * ExecutionHeadersSummary emits only the counter triplet\n * (X-ERPC-Attempts/Retries/Hedges) + the cache-hit / final-upstream\n * markers. Skips the (potentially large) per-attempt slice headers.\n */\nexport const ExecutionHeadersSummary: ExecutionHeadersMode = \"summary\";\n/**\n * ExecutionHeadersOff disables all X-ERPC-* diagnostic headers.\n */\nexport const ExecutionHeadersOff: ExecutionHeadersMode = \"off\";\nexport interface HealthCheckConfig {\n mode?: HealthCheckMode;\n auth?: AuthConfig;\n defaultEval?: string;\n}\nexport type HealthCheckMode = string;\nexport const HealthCheckModeSimple: HealthCheckMode = \"simple\";\nexport const HealthCheckModeNetworks: HealthCheckMode = \"networks\";\nexport const HealthCheckModeVerbose: HealthCheckMode = \"verbose\";\nexport const EvalAnyInitializedUpstreams = \"any:initializedUpstreams\";\nexport const EvalAnyErrorRateBelow90 = \"any:errorRateBelow90\";\nexport const EvalAllErrorRateBelow90 = \"all:errorRateBelow90\";\nexport const EvalAnyErrorRateBelow100 = \"any:errorRateBelow100\";\nexport const EvalAllErrorRateBelow100 = \"all:errorRateBelow100\";\nexport const EvalEvmAnyChainId = \"any:evm:eth_chainId\";\nexport const EvalEvmAllChainId = \"all:evm:eth_chainId\";\nexport const EvalAllActiveUpstreams = \"all:activeUpstreams\";\nexport type TracingProtocol = string;\nexport const TracingProtocolHttp: TracingProtocol = \"http\";\nexport const TracingProtocolGrpc: TracingProtocol = \"grpc\";\nexport interface TracingConfig {\n enabled?: boolean;\n endpoint?: string;\n protocol?: TracingProtocol;\n sampleRate?: number /* float64 */;\n detailed?: boolean;\n serviceName?: string;\n headers?: { [key: string]: string};\n tls?: TLSConfig;\n resourceAttributes?: { [key: string]: string};\n /**\n * ForceTraceMatchers defines conditions for force-tracing requests.\n * Each matcher can specify network and/or method patterns.\n * Multiple patterns can be separated by \"|\" (OR within field).\n * Both network and method must match if both are specified (AND between fields).\n * If only one field is specified, only that field is checked.\n */\n forceTraceMatchers?: (ForceTraceMatcher | undefined)[];\n}\n/**\n * ForceTraceMatcher defines a condition for force-tracing requests.\n */\nexport interface ForceTraceMatcher {\n /**\n * Network patterns to match (e.g., \"evm:1\", \"evm:1|evm:42161\", \"evm:*\")\n * Multiple patterns separated by \"|\" act as OR conditions.\n */\n network?: string;\n /**\n * Method patterns to match (e.g., \"eth_call\", \"debug_*|trace_*\")\n * Multiple patterns separated by \"|\" act as OR conditions.\n */\n method?: string;\n}\nexport interface AdminConfig {\n auth?: AuthConfig;\n cors?: CORSConfig;\n allowMethods?: string[];\n denyMethods?: string[];\n}\nexport interface AliasingConfig {\n rules: (AliasingRuleConfig | undefined)[];\n}\nexport interface AliasingRuleConfig {\n matchDomain: string;\n serveProject: string;\n serveArchitecture: string;\n serveChain: string;\n}\nexport interface DatabaseConfig {\n evmJsonRpcCache?: CacheConfig;\n svmJsonRpcCache?: CacheConfig;\n sharedState?: SharedStateConfig;\n}\nexport interface SharedStateConfig {\n /**\n * ClusterKey identifies the logical group for shared counters across replicas (multi-tenant friendly)\n */\n clusterKey?: string;\n /**\n * Connector contains the storage driver configuration (redis, postgresql, dynamodb, memory)\n */\n connector?: ConnectorConfig;\n /**\n * FallbackTimeout is the timeout for remote storage operations (get/set/publish).\n * It is a seconds-scale network timeout and NOT a foreground latency budget.\n */\n fallbackTimeout?: Duration;\n /**\n * LockTtl is the expiration for the distributed lock key in the backing store.\n * Should comfortably exceed the expected duration of remote writes.\n */\n lockTtl?: Duration;\n /**\n * LockMaxWait caps how long the foreground path will wait to acquire the lock\n * before proceeding locally and deferring the remote write to background.\n */\n lockMaxWait?: Duration;\n /**\n * UpdateMaxWait caps how long the foreground path will spend computing a new value\n * (e.g., polling latest block) before returning the current local value.\n */\n updateMaxWait?: Duration;\n}\nexport interface CacheConfig {\n connectors?: TsConnectorConfig[];\n policies?: (CachePolicyConfig | undefined)[];\n compression?: CompressionConfig;\n}\nexport interface CompressionConfig {\n enabled?: boolean;\n algorithm?: string; // \"zstd\" for now, can be extended\n zstdLevel?: string; // \"fastest\", \"default\", \"better\", \"best\"\n threshold?: number /* int */; // Minimum size in bytes to compress\n}\nexport interface CacheMethodConfig {\n reqRefs: any[][];\n respRefs: any[][];\n finalized: boolean;\n realtime: boolean;\n stateful?: boolean;\n /**\n * TranslateLatestTag controls whether the method-level tag translation should convert \"latest\" to a concrete hex block number.\n * When nil or true, translation is enabled by default.\n */\n translateLatestTag?: boolean;\n /**\n * TranslateFinalizedTag controls whether the method-level tag translation should convert \"finalized\" to a concrete hex block number.\n * When nil or true, translation is enabled by default.\n */\n translateFinalizedTag?: boolean;\n /**\n * EnforceBlockAvailability controls whether per-upstream block availability bounds (upper/lower)\n * are enforced for this method at the network level. When nil or true, enforcement is enabled.\n */\n enforceBlockAvailability?: boolean;\n}\nexport interface CachePolicyConfig {\n connector: string;\n network?: string;\n method?: string;\n params?: any[];\n finality?: DataFinalityState;\n empty?: CacheEmptyBehavior;\n appliesTo?: 'get' | 'set' | 'both';\n minItemSize?: ByteSize;\n maxItemSize?: ByteSize;\n /**\n * TTL is either a fixed duration (\"2s\") or, in object form, derived from the\n * network's estimated block time ({ blockTimeMultiplier: 1, fallback: 2s }).\n * For realtime finality the resolved value is the age limit; the fixed/\n * fallback component is also used as the cache storage expiry. See\n * BlockTimeAdaptiveDuration.\n */\n ttl?: Duration | BlockTimeAdaptiveDuration;\n}\nexport type ConnectorDriverType = string;\nexport const DriverMemory: ConnectorDriverType = \"memory\";\nexport const DriverRedis: ConnectorDriverType = \"redis\";\nexport const DriverPostgreSQL: ConnectorDriverType = \"postgresql\";\nexport const DriverDynamoDB: ConnectorDriverType = \"dynamodb\";\nexport const DriverGrpc: ConnectorDriverType = \"grpc\";\nexport interface ConnectorConfig {\n id?: string;\n driver: TsConnectorDriverType;\n /**\n * Tags label the data source a connector caches, so the `use-upstream`\n * directive can gate which connector is read/written \u2014 e.g. a connector\n * fed by a system-transaction node tagged `systx` is only used when the\n * request's selector admits it. Same glob/`!negation` grammar as upstream\n * tags. Untagged connectors are always eligible (a use-upstream pin meant\n * for upstreams must not disable a normal cache).\n */\n tags?: string[];\n memory?: MemoryConnectorConfig;\n redis?: RedisConnectorConfig;\n dynamodb?: DynamoDBConnectorConfig;\n postgresql?: PostgreSQLConnectorConfig;\n grpc?: GrpcConnectorConfig;\n failsafeForGets?: (FailsafeConfig | undefined)[];\n failsafeForSets?: (FailsafeConfig | undefined)[];\n}\nexport interface GrpcConnectorConfig {\n bootstrap?: string;\n servers?: string[];\n headers?: { [key: string]: string};\n getTimeout?: Duration;\n /**\n * NetworkId pins every server in this connector to one network and skips\n * the eth_chainId bootstrap probe.\n * Required for SVM. Solana has no numeric chain id \u2014 cluster identity is\n * the genesis hash \u2014 and eth_chainId returns Unimplemented, so the probe\n * fails and no client is ever registered. Probing GetGenesisHash instead\n * is not an option either: the BDS reader deliberately does not publish\n * one until it can verify its own, so identity here has to be asserted by\n * configuration rather than discovered.\n * Leave unset for EVM to keep the probe, which also arms the per-request\n * chain-identity assertion that catches an endpoint cross-wired LATER \u2014 a\n * static binding cannot.\n */\n networkId?: string;\n /**\n * PoolSize is the number of independent gRPC connections opened to each\n * backing server, selected round-robin per request. Larger values raise the\n * concurrent-stream ceiling and shrink the blast radius of a single wedged\n * connection, at the cost of more open connections per server. When unset\n * (0) a built-in default is used.\n */\n poolSize?: number /* int */;\n /**\n * HealthCheckService is a grpc.health.v1 service name. When set, each\n * connection watches it on every resolved address and sends requests only\n * to addresses reporting SERVING. Empty (default) disables health checking.\n */\n healthCheckService?: string;\n}\nexport interface MemoryConnectorConfig {\n maxItems: number /* int */;\n maxTotalSize: string;\n emitMetrics?: boolean;\n}\nexport interface MockConnectorConfig {\n memoryconnectorconfig: MemoryConnectorConfig;\n getdelay: number /* time in nanoseconds (time.Duration) */;\n setdelay: number /* time in nanoseconds (time.Duration) */;\n geterrorrate: number /* float64 */;\n seterrorrate: number /* float64 */;\n}\nexport interface TLSConfig {\n enabled: boolean;\n certFile: string;\n keyFile: string;\n caFile?: string;\n insecureSkipVerify?: boolean;\n}\nexport interface RedisConnectorConfig {\n addr?: string;\n username?: string;\n db?: number /* int */;\n tls?: TLSConfig;\n connPoolSize?: number /* int */;\n uri: string;\n initTimeout?: Duration;\n getTimeout?: Duration;\n setTimeout?: Duration;\n lockRetryInterval?: Duration;\n iamAuth?: RedisIAMAuthConfig;\n}\nexport interface DynamoDBConnectorConfig {\n table?: string;\n region?: string;\n endpoint?: string;\n auth?: AwsAuthConfig;\n partitionKeyName?: string;\n rangeKeyName?: string;\n reverseIndexName?: string;\n ttlAttributeName?: string;\n initTimeout?: Duration;\n getTimeout?: Duration;\n setTimeout?: Duration;\n maxRetries?: number /* int */;\n statePollInterval?: Duration;\n lockRetryInterval?: Duration;\n}\nexport interface PostgreSQLConnectorConfig {\n connectionUri: string;\n table: string;\n minConns?: number /* int32 */;\n maxConns?: number /* int32 */;\n initTimeout?: Duration;\n getTimeout?: Duration;\n setTimeout?: Duration;\n iamAuth?: PostgreSQLIAMAuthConfig;\n /**\n * SkipSchemaSetup skips all startup DDL (CREATE TABLE/INDEX, column\n * migrations, pg_cron) and the local expired-row cleanup DELETE loop. Set\n * it for connectors whose ConnectionUri targets a read-only replica (e.g.\n * an Aurora global-database secondary): DDL cannot execute there (SQLSTATE\n * 25006) and is not write-forwarded, so the writer-region connector owns\n * the schema and the replica receives it via storage replication.\n */\n skipSchemaSetup?: boolean;\n}\nexport interface AwsAuthConfig {\n mode: 'file' | 'env' | 'secret'; // \"file\", \"env\", \"secret\"\n credentialsFile: string;\n profile: string;\n accessKeyID: string;\n secretAccessKey: string;\n}\n/**\n * RedisIAMAuthConfig enables AWS IAM authentication for ElastiCache (Valkey \u22657.2\n * or Redis OSS \u22657.0). When enabled, eRPC mints SigV4-presigned auth tokens via\n * go-redis's CredentialsProviderContext on every new connection. TLS is required\n * (auto-enabled by SetDefaults). For IAM-enabled ElastiCache users, the user\n * name and user ID must be identical \u2014 supply that single value as UserID.\n */\nexport interface RedisIAMAuthConfig {\n enabled: boolean;\n /**\n * CacheName is the ElastiCache replication-group ID.\n * Will be lowercased automatically (AWS lowercases cache names at creation time).\n */\n cacheName: string;\n /**\n * Region is optional \u2014 derived from AWS_REGION / instance metadata when omitted.\n */\n region?: string;\n userID: string;\n /**\n * Auth selects the AWS credential source (same shape as DynamoDB's auth).\n * Omit to use the default credential chain (instance role, env vars, \u2026).\n */\n auth?: AwsAuthConfig;\n}\n/**\n * PostgreSQLIAMAuthConfig enables AWS IAM authentication for RDS PostgreSQL.\n * eRPC mints SigV4-presigned tokens via pgxpool.BeforeConnect on each new pool\n * connection. SSL is required (auto-enforced via sslmode=require by SetDefaults).\n */\nexport interface PostgreSQLIAMAuthConfig {\n enabled: boolean;\n /**\n * Endpoint is host:port of the RDS instance. If empty, derived from\n * ConnectionUri at SetDefaults time.\n */\n endpoint?: string;\n /**\n * Region is optional \u2014 derived from AWS_REGION / instance metadata when omitted.\n */\n region?: string;\n /**\n * DBUser is the database user mapped to the IAM role (must be granted rds_iam\n * in PostgreSQL). If empty, derived from the user in ConnectionUri.\n */\n dbUser?: string;\n auth?: AwsAuthConfig;\n}\nexport interface ProjectConfig {\n id: string;\n auth?: AuthConfig;\n cors?: CORSConfig;\n providers?: (ProviderConfig | undefined)[];\n upstreamDefaults?: UpstreamConfig;\n upstreams?: (UpstreamConfig | undefined)[];\n networkDefaults?: NetworkDefaults;\n networks?: (NetworkConfig | undefined)[];\n rateLimitBudget?: string;\n /**\n * Configure user agent tracking at the project level\n */\n userAgentMode?: UserAgentTrackingMode;\n /**\n * TrustUserIdHeader makes erpc read the caller's user identity from the\n * X-ERPC-User-Id request header (see common.HeaderUserId) and use it for the\n * `user` metric/log label \u2014 but only when no auth strategy resolved a user\n * (auth wins) and only for attribution (no rate-limit budget is derived).\n * This is for deployments that authenticate callers in front of erpc (e.g. a\n * gateway) and want per-user erpc telemetry without erpc performing auth.\n * erpc does NOT validate the header, so enable this ONLY when erpc is reachable\n * solely by a trusted proxy that sets the header and strips any client copy \u2014\n * otherwise callers can spoof their own attribution. Default false.\n */\n trustUserIdHeader?: boolean;\n forwardHeaders?: string[];\n allowClientDirectives?: string;\n ignoreMethods?: string[];\n allowMethods?: string[];\n /**\n * Integrity is the project-wide data-integrity configuration. It applies to\n * all networks; each network may override it with its own integrity block.\n */\n integrity?: IntegrityConfig;\n /**\n * ScoreMetricsWindowSize is the tumbling window the per-upstream\n * health tracker uses for its rolling counters (errorRate, p50/p70/\n * p95 latency, throttledRate, misbehaviorRate). At each tick the\n * counters reset and start re-accumulating, so this knob effectively\n * controls how fast a degraded upstream's metrics start reflecting\n * the new reality. Short windows (e.g. 30s) react quickly but give\n * noisier ranking; long windows (5\u201310m) give stable averages but\n * hide spikes for longer. Defaults to 10m when zero \u2014 production\n * systems typically leave it default; the eRPC simulator overrides\n * to 30s so knob changes show up in the UI within seconds.\n */\n scoreMetricsWindowSize?: Duration;\n}\n/**\n * LegacyProjectFields collects the deprecated project-level scoring +\n * routing keys. The translator inspects these to synthesize a\n * `selectionPolicy.eval` for each network and to emit deprecation\n * warnings. All fields are zero-valued for a clean modern config.\n * `scoreMetricsWindowSize` was previously listed here as inert; it is\n * now a first-class field on ProjectConfig (above) \u2014 operators\n * control the health tracker window directly.\n */\nexport interface LegacyProjectFields {\n routingStrategy?: string;\n scoreGranularity?: string;\n scorePenaltyDecayRate?: number /* float64 */;\n scoreSwitchHysteresis?: number /* float64 */;\n scoreMinSwitchInterval?: Duration;\n scoreMetricsMode?: string;\n scoreRefreshInterval?: Duration;\n}\n/**\n * UserAgentTrackingMode controls how user agents are recorded for metrics/labels\n */\nexport type UserAgentTrackingMode = string;\n/**\n * UserAgentTrackingModeSimplified lowers cardinality by bucketing common user agents\n */\nexport const UserAgentTrackingModeSimplified: UserAgentTrackingMode = \"simplified\";\n/**\n * UserAgentTrackingModeRaw records the user agent string as-is (high cardinality)\n */\nexport const UserAgentTrackingModeRaw: UserAgentTrackingMode = \"raw\";\nexport interface NetworkDefaults {\n rateLimitBudget?: string;\n failsafe?: (FailsafeConfig | undefined)[];\n selectionPolicy?: SelectionPolicyConfig;\n directiveDefaults?: DirectiveDefaultsConfig;\n evm?: TsEvmNetworkConfigForDefaults;\n svm?: TsSvmNetworkConfigForDefaults;\n multiplexing?: boolean;\n cacheKeySuffix?: string;\n}\nexport interface CORSConfig {\n allowedOrigins: string[];\n allowedMethods: string[];\n allowedHeaders: string[];\n exposedHeaders: string[];\n allowCredentials?: boolean;\n maxAge: number /* int */;\n}\nexport type VendorSettings = { [key: string]: any};\nexport interface ProviderConfig {\n id?: string;\n vendor: string;\n settings?: VendorSettings;\n onlyNetworks?: string[];\n ignoreNetworks?: string[];\n upstreamIdTemplate?: string;\n overrides?: { [key: string]: UpstreamConfig | undefined};\n}\n/**\n * RateLimitCountMode selects the accounting unit an upstream's rate-limit\n * budget charges per call.\n */\nexport type RateLimitCountMode = string;\n/**\n * RateLimitCountModeRequest (default) charges a flat 1 hit per call,\n * regardless of method \u2014 the historical eRPC behavior.\n */\nexport const RateLimitCountModeRequest: RateLimitCountMode = \"request\";\n/**\n * RateLimitCountModeCredit charges the request's resolved vendor\n * credit-unit cost (the same table used for cost accounting), so a\n * heavy eth_getLogs consumes more budget than a cheap eth_blockNumber.\n * The pre-flight table estimate is used (the real cost is not known\n * until after the call); a 0-CU method consumes nothing.\n */\nexport const RateLimitCountModeCredit: RateLimitCountMode = \"credit\";\nexport interface UpstreamConfig {\n id?: string;\n type?: TsUpstreamType;\n /**\n * Tags is the single canonical user-applied label set. Convention is\n * `:` so a single upstream can carry orthogonal labels:\n * tags:\n * - tier:main # used by .preferTag for tiering\n * - region:us-east # used by .spreadAcrossTags('region:')\n * - sequencer:op-base # shared-fate label\n * Bare strings (no prefix) work too. Patterns supported by every\n * stdlib tag method: glob (`*`, `?`) and `!negation`.\n * Tier convention: `tier:fallback` declares a fallback-tier upstream\n * (matches the long-standing eRPC convention; the default policy\n * looks for `!tier:fallback` as the primary tier).\n * The legacy `group: X` / `cohort: Y` YAML keys are accepted at\n * load time only \u2014 UnmarshalYAML rewrites them as `tier:X` /\n * `cohort:Y` tags and forgets them. There is no Go-level Group or\n * Cohort field; programmatic code uses Tags directly.\n */\n tags?: string[];\n vendorName?: string;\n endpoint?: string;\n evm?: EvmUpstreamConfig;\n svm?: SvmUpstreamConfig;\n jsonRpc?: JsonRpcUpstreamConfig;\n grpc?: GrpcUpstreamConfig;\n ignoreMethods?: string[];\n allowMethods?: string[];\n autoIgnoreUnsupportedMethods?: boolean;\n failsafe?: (FailsafeConfig | undefined)[];\n rateLimitBudget?: string;\n rateLimitAutoTune?: RateLimitAutoTuneConfig;\n /**\n * RateLimitCountMode selects how this upstream's rate-limit budget\n * counts a call: \"request\" (default) charges 1 hit, \"credit\" charges the\n * request's resolved vendor credit-unit cost. Empty resolves to\n * \"request\". Applies to the upstream-level budget only.\n */\n rateLimitCountMode?: RateLimitCountMode;\n /**\n * CreditUnits overrides the vendor's built-in per-method credit table\n * (CreditUnitsProvider) for this upstream, merged per method over the\n * vendor defaults (\"*\" = fallback for unlisted methods). Normally set\n * once per provider via `providers[].settings.creditUnits`, which is\n * copied onto every upstream the provider generates.\n */\n creditUnits?: { [key: string]: number /* int64 */};\n shadow?: ShadowUpstreamConfig;\n /**\n * Routing holds per-upstream routing hints consumed by the selection\n * policy. `scoreMultipliers` bias this upstream's rank inside\n * `sortByScore` (see SelectionPolicyConfig): the engine resolves the\n * matching entry for each (network, method, finality) tick and exposes\n * the resulting weight map to the eval function as `u.scoreMultipliers`.\n * When the upstream omits its own `routing` block, `ApplyDefaults`\n * inherits the project-level `upstreamDefaults.routing` (all-or-nothing,\n * matching the Tags inheritance pattern).\n */\n routing?: UpstreamRoutingConfig;\n}\n/**\n * UpstreamRoutingConfig holds per-upstream routing hints. Today this is\n * the home of `scoreMultipliers` (per-upstream weight overrides folded\n * into `sortByScore`) and `probe` (per-upstream opt-out for the\n * selection policy's `probeExcluded` shadow-mirror traffic).\n */\nexport interface UpstreamRoutingConfig {\n /**\n * ScoreMultipliers biases this upstream's rank. Each entry is a\n * matcher (network/method/finality) plus weight overrides; the engine\n * resolves the first matching entry per tick and hands it to the eval\n * function as `u.scoreMultipliers`. `sortByScore` merges it over the\n * base weights by default (see its `multipliers` option).\n */\n scoreMultipliers?: (ScoreMultiplierConfig | undefined)[];\n /**\n * ScoreLatencyQuantile selects which response-time quantile feeds the\n * score (e.g. 0.9 \u2192 p90). When unset, the policy's own\n * `sortByScore({ latencyQuantile })` (default p70) applies.\n */\n scoreLatencyQuantile?: number /* float64 */;\n /**\n * Probe gates whether the selection policy's `probeExcluded` step\n * may shadow-mirror real requests to THIS upstream when it's\n * currently in the excluded set. `\"on\"` (default) opts in; `\"off\"`\n * disables probing entirely \u2014 the upstream stays excluded\n * permanently once predicates trip until an operator intervenes\n * (manual cordon/uncordon, or until the predicate stops matching\n * via state-poller-driven structural metrics like head lag). Use\n * `\"off\"` for pay-per-call vendors where shadow traffic eats quota.\n */\n probe?: ProbeMode | \"on\" | \"off\";\n}\n/**\n * ProbeMode is the per-upstream `routing.probe` enum.\n */\nexport type ProbeMode = string;\n/**\n * ProbeModeOn \u2014 default. The selection policy may mirror sampled\n * real requests to this upstream while it's excluded so it\n * accumulates fresh tracker samples for natural re-admission.\n */\nexport const ProbeModeOn: ProbeMode = \"on\";\n/**\n * ProbeModeOff \u2014 never mirror. Upstream stays excluded after\n * predicates trip; only structural signals (head lag, etc.) can\n * drive re-admission.\n */\nexport const ProbeModeOff: ProbeMode = \"off\";\n/**\n * ScoreMultiplierConfig is one per-upstream weight override. The matcher\n * fields (network/method/finality) scope the entry \u2014 leave them empty (or\n * `\"*\"`) for an entry that applies to every request. Weight fields are\n * pointers so \"unset\" is distinct from \"zero\": an unset weight inherits\n * from the policy's base weights (merge mode), a zero weight removes that\n * metric's contribution. `overall` scales the upstream's FINAL score \u2014 a\n * preference dial where >1 prefers this upstream and <1 avoids it.\n */\nexport interface ScoreMultiplierConfig {\n network?: string;\n method?: string;\n finality?: DataFinalityState[];\n overall?: number /* float64 */;\n errorRate?: number /* float64 */;\n respLatency?: number /* float64 */;\n throttledRate?: number /* float64 */;\n blockHeadLag?: number /* float64 */;\n finalizationLag?: number /* float64 */;\n misbehaviors?: number /* float64 */;\n /**\n * TotalRequests is accepted for backward compatibility but no longer\n * influences scoring (the score is computed from rolling-window rates,\n * not absolute request counts).\n */\n totalRequests?: number /* float64 */;\n}\nexport interface ShadowUpstreamConfig {\n enabled: boolean;\n sampleRate?: number /* float64 */;\n ignoreFields?: { [key: string]: string[]};\n /**\n * PinBlockTag rewrites a tag block selector (\"latest\"/\"pending\") in the mirrored copy to a\n * concrete height before forwarding: the primary's resolved block number when its response\n * carries one, else the network's tracked head at mirror time. Without it the shadow executes\n * the tag at its OWN head wall-clock-later than the primary did, so any read of volatile state\n * (pool reserves, oracle prices \u2014 most visibly large multicalls) diverges by construction and\n * reports a mismatch that says nothing about correctness. Default false = mirror byte-identical\n * requests, exactly as before.\n */\n pinBlockTag?: boolean;\n}\n/**\n * Deprecated: UpstreamIntegrityConfig is a non-functional legacy stub (never\n * read at runtime). Configure data integrity via the network `integrity` block.\n * Retained only so existing YAML still parses.\n */\nexport interface UpstreamIntegrityConfig {\n eth_getBlockReceipts?: UpstreamIntegrityEthGetBlockReceiptsConfig;\n}\nexport interface UpstreamIntegrityEthGetBlockReceiptsConfig {\n enabled?: boolean;\n checkLogIndexStrictIncrements?: boolean;\n checkLogsBloom?: boolean;\n}\nexport type UJAlias = UpstreamConfig;\nexport type UYAlias = UpstreamConfig;\nexport interface RateLimitAutoTuneConfig {\n enabled?: boolean;\n adjustmentPeriod: Duration;\n errorRateThreshold: number /* float64 */;\n increaseFactor: number /* float64 */;\n decreaseFactor: number /* float64 */;\n minBudget: number /* int */;\n maxBudget: number /* int */;\n}\nexport interface JsonRpcUpstreamConfig {\n supportsBatch?: boolean;\n batchMaxSize?: number /* int */;\n batchMaxWait?: Duration;\n enableGzip?: boolean;\n headers?: { [key: string]: string};\n proxyPool?: string;\n}\n/**\n * GrpcUpstreamConfig tunes a gRPC (grpc:// / grpc+bds:// / grpcs://) upstream. It is the\n * gRPC analogue of JsonRpcUpstreamConfig: JsonRpc holds JSON-RPC/HTTP-specific\n * knobs, this holds gRPC-specific ones. Headers are applied as gRPC metadata on\n * every outbound request (e.g. an edge-api auth key: authorization: Bearer ...).\n */\nexport interface GrpcUpstreamConfig {\n headers?: { [key: string]: string};\n /**\n * PoolSize is the number of independent gRPC connections opened to this\n * upstream, selected round-robin per request. See GrpcConnectorConfig.PoolSize.\n * When unset (0) a built-in default is used.\n */\n poolSize?: number /* int */;\n /**\n * HealthCheckService: see GrpcConnectorConfig.HealthCheckService.\n */\n healthCheckService?: string;\n}\nexport interface EvmUpstreamConfig {\n chainId: number /* int64 */;\n statePollerInterval?: Duration;\n /**\n * StatePollerDebounce overrides the debounce interval for the state poller.\n * When 0 (default), the interval is dynamically inferred from the chain's\n * observed block time, falling back to the network-level\n * FallbackStatePollerDebounce, then to a 1s floor.\n */\n statePollerDebounce?: Duration;\n blockAvailability?: EvmBlockAvailabilityConfig;\n getLogsAutoSplittingRangeThreshold?: number /* int64 */;\n /**\n * TraceFilterAutoSplittingRangeThreshold proactively splits trace_filter and\n * arbtrace_filter requests whose block range exceeds this value into contiguous\n * sub-requests executed concurrently and merged before returning. Zero disables\n * the feature.\n */\n traceFilterAutoSplittingRangeThreshold?: number /* int64 */;\n skipWhenSyncing?: boolean;\n /**\n * SkipSyncingCheck disables eth_syncing polling for this upstream, treating it as always synced.\n * Use for nodes that always return a syncing object (e.g. Pharos/Antora) where the response is\n * misleading and causes circuit breaker false positives.\n */\n skipSyncingCheck?: boolean;\n /**\n * HeadLagToleranceBlocks lets the block-availability gate send requests pinned up to N blocks\n * ABOVE this upstream's observed head instead of rerouting them as stale. Useful for upstreams\n * that follow the chain tightly but become able to serve a new block a moment after it is\n * announced elsewhere (they hold or briefly wait for the block internally): with a hard\n * head comparison every request pinned at head+1 in that moment is rerouted even though the\n * upstream would have answered it. 0 (default) keeps the exact head comparison.\n */\n headLagToleranceBlocks?: number /* int64 */;\n /**\n * Deprecated: never read at runtime. Configure data integrity via the network\n * `integrity` block instead. Retained only so existing YAML still parses.\n */\n integrity?: UpstreamIntegrityConfig;\n /**\n * @deprecated: use blockAvailability bounds instead; kept for config back-compat only\n */\n nodeType?: EvmNodeType;\n /**\n * @deprecated: should be removed in a future release\n */\n maxAvailableRecentBlocks?: number /* int64 */;\n queryShim?: EvmQueryShimConfig;\n}\n/**\n * EvmQueryShimConfig lets an upstream serve the sub-requests (eth_getBlockByNumber,\n * eth_getBlockReceipts, eth_getLogs, trace_block / debug_traceBlockByNumber) the\n * eth_query* executor issues when no native QueryService upstream answers. The\n * sub-requests go through the network like any request; the budget bounds the\n * work of one MIP-16 page.\n */\nexport interface EvmQueryShimConfig {\n enabled?: boolean;\n allowedMethods?: string[];\n /**\n * MaxBlocksPerPage is the budget of blocks one page may scan. A page that\n * reaches it ends at the last scanned block.\n */\n maxBlocksPerPage?: number /* int64 */;\n /**\n * MaxPageDuration is the budget of time one page may scan. A page that\n * reaches it ends at the last complete block, or fails with -32005 when no\n * block completed.\n */\n maxPageDuration?: Duration;\n}\n/**\n * EvmBlockAvailability defines optional lower/upper block availability expressions for an upstream.\n * Presence of lower/upper implies the feature is active. When both are nil, it's effectively off\n */\nexport interface EvmBlockAvailabilityConfig {\n lower?: EvmAvailabilityBoundConfig;\n upper?: EvmAvailabilityBoundConfig;\n}\n/**\n * EvmBound represents a single bound definition.\n * Exactly one of ExactBlock, LatestMinus, EarliestPlus should be set.\n * UpdateRate only applies to earliestBlockPlus bounds: 0 means freeze at first evaluation; >0 means recompute on that cadence.\n * For latestBlockMinus, updateRate is ignored: bounds are computed on-demand using the continuously-updated latest block from evmStatePoller.\n */\nexport type EvmAvailabilityProbeType = string;\nexport const EvmProbeBlockHeader: EvmAvailabilityProbeType = \"blockHeader\";\nexport const EvmProbeEventLogs: EvmAvailabilityProbeType = \"eventLogs\";\nexport const EvmProbeCallState: EvmAvailabilityProbeType = \"callState\";\nexport const EvmProbeTraceData: EvmAvailabilityProbeType = \"traceData\";\nexport interface EvmAvailabilityBoundConfig {\n exactBlock?: number /* int64 */;\n latestBlockMinus?: number /* int64 */;\n earliestBlockPlus?: number /* int64 */;\n probe?: EvmAvailabilityProbeType;\n updateRate?: Duration;\n}\nexport interface FailsafeConfig {\n matchMethod?: string;\n matchFinality?: DataFinalityState[];\n /**\n * MatchCommitment scopes a network-scope policy by the Solana commitment\n * erpc pins on the wire: the caller's value, else the svm.commitment default\n * that injection writes, else \"none\". Empty = any. Values are OR-ed and\n * exact (confirmed does not match processed). Non-SVM requests and SVM\n * write methods (sendTransaction, simulateTransaction, requestAirdrop) are \"none\".\n * Rejected at upstream and connector scopes.\n */\n matchCommitment?: ('none' | 'processed' | 'confirmed' | 'finalized')[];\n /**\n * MatchRequestKind scopes this policy by who issued the request:\n * \"user\" (client traffic), \"internal\" (erpc's own auxiliary fetches, e.g.\n * the integrity module's canonical corroboration), or \"\"/\"*\" for both.\n * This is what lets an operator give INTERNAL canonical fetches a\n * consensus policy (quorum-verified ground truth, deduplicated to ~once\n * per block by the ChainView) while user data methods rely on integrity\n * validation instead of per-request fan-out.\n */\n matchRequestKind?: 'user' | 'internal' | '*';\n retry?: RetryPolicyConfig;\n circuitBreaker?: CircuitBreakerPolicyConfig;\n timeout?: TimeoutPolicyConfig;\n hedge?: HedgePolicyConfig;\n consensus?: ConsensusPolicyConfig;\n}\n/**\n * NetworkFailsafeConfig is the scope-specific alias for network-level\n * failsafe policies. By convention, CircuitBreaker is not used at this\n * scope (use upstream-scope breakers instead); validation enforces this.\n */\nexport type NetworkFailsafeConfig = FailsafeConfig;\n/**\n * UpstreamFailsafeConfig is the scope-specific alias for per-upstream\n * failsafe policies. By convention, Consensus is not used at this\n * scope (consensus is a network-scope concern only); validation\n * enforces this.\n */\nexport type UpstreamFailsafeConfig = FailsafeConfig;\n/**\n * CacheFailsafeConfig is the scope-specific alias for cache-connector\n * failsafe policies. Timeout.Duration and Hedge.Delay support quantile\n * (dynamic) mode at this scope, resolved from a per-executor latency\n * tracker fed by the connector's own operations (see data/cache_executor.go).\n * Consensus is not supported here; validation enforces this.\n */\nexport type CacheFailsafeConfig = FailsafeConfig;\nexport interface RetryPolicyConfig {\n maxAttempts: number /* int */;\n delay?: Duration;\n backoffMaxDelay?: Duration;\n backoffFactor?: number /* float32 */;\n jitter?: Duration;\n /**\n * EmptyResultAccept lists methods for which an empty/null result is considered valid\n * and should NOT be retried (e.g. eth_getLogs, eth_call where empty is a legitimate response).\n */\n emptyResultAccept?: string[];\n /**\n * @deprecated: use EmptyResultAccept instead.\n */\n emptyResultIgnore?: string[];\n /**\n * EmptyResultMaxAttempts limits total attempts when retries are triggered due to empty responses.\n */\n emptyResultMaxAttempts?: number /* int */;\n /**\n * EmptyResultDelay is the fixed fallback delay before retrying when the requested\n * data isn't on the upstream yet \u2014 an empty/missing-data point-lookup OR an\n * ErrUpstreamBlockUnavailable (same root cause: the block/tx isn't produced or\n * indexed yet). Retries prefer the dynamic block-time delay (EMA block time \u00D7\n * Evm.BlockUnavailableDelayMultiplier); this fixed value is used only before that\n * estimate warms up. (Supersedes the now-deprecated BlockUnavailableDelay.)\n */\n emptyResultDelay?: Duration;\n}\nexport interface CircuitBreakerPolicyConfig {\n failureThresholdCount: number /* uint */;\n failureThresholdCapacity: number /* uint */;\n halfOpenAfter?: Duration;\n successThresholdCount: number /* uint */;\n successThresholdCapacity: number /* uint */;\n}\n/**\n * TimeoutPolicyConfig is the timeout policy. Duration is the unified\n * AdaptiveDuration \u2014 a scalar shorthand (\"5s\") or an object form\n * ({base, quantile, min, max}) for adaptive caps driven by per-method\n * latency quantiles.\n * Wire format also accepts the legacy flat form\n * (`duration: 5s, quantile: 0.99, minDuration: 200ms, maxDuration: 10s`)\n * \u2014 siblings get folded into Duration at YAML/JSON unmarshal time.\n */\nexport interface TimeoutPolicyConfig {\n duration?: Duration | AdaptiveDuration;\n}\n/**\n * HedgePolicyConfig is the hedge policy. Delay is the unified\n * AdaptiveDuration \u2014 scalar shorthand (\"100ms\") or object form\n * ({base, quantile, min, max}) for quantile-driven hedge timing.\n * Wire format also accepts the legacy flat form\n * (`delay: 100ms, quantile: 0.95, minDelay: 50ms, maxDelay: 2s`) \u2014\n * siblings get folded into Delay at YAML/JSON unmarshal time.\n */\nexport interface HedgePolicyConfig {\n delay?: Duration | AdaptiveDuration;\n maxCount: number /* int */;\n}\nexport type ConsensusLowParticipantsBehavior = string;\nexport const ConsensusLowParticipantsBehaviorReturnError: ConsensusLowParticipantsBehavior = \"returnError\";\nexport const ConsensusLowParticipantsBehaviorAcceptMostCommonValidResult: ConsensusLowParticipantsBehavior = \"acceptMostCommonValidResult\";\nexport const ConsensusLowParticipantsBehaviorPreferBlockHeadLeader: ConsensusLowParticipantsBehavior = \"preferBlockHeadLeader\";\nexport const ConsensusLowParticipantsBehaviorOnlyBlockHeadLeader: ConsensusLowParticipantsBehavior = \"onlyBlockHeadLeader\";\nexport type ConsensusDisputeBehavior = string;\nexport const ConsensusDisputeBehaviorReturnError: ConsensusDisputeBehavior = \"returnError\";\nexport const ConsensusDisputeBehaviorAcceptMostCommonValidResult: ConsensusDisputeBehavior = \"acceptMostCommonValidResult\";\nexport const ConsensusDisputeBehaviorPreferBlockHeadLeader: ConsensusDisputeBehavior = \"preferBlockHeadLeader\";\nexport const ConsensusDisputeBehaviorOnlyBlockHeadLeader: ConsensusDisputeBehavior = \"onlyBlockHeadLeader\";\nexport interface ConsensusPolicyConfig {\n maxParticipants: number /* int */;\n agreementThreshold?: number /* int */;\n disputeBehavior?: ConsensusDisputeBehavior;\n lowParticipantsBehavior?: ConsensusLowParticipantsBehavior;\n punishMisbehavior?: PunishMisbehaviorConfig;\n disputeLogLevel?: string; // \"trace\", \"debug\", \"info\", \"warn\", \"error\"\n ignoreFields?: { [key: string]: string[]};\n preferNonEmpty?: boolean;\n preferLargerResponses?: boolean;\n misbehaviorsDestination?: MisbehaviorsDestinationConfig;\n /**\n * PreferHighestValueFor specifies methods that should use highest-value comparison\n * instead of hash-based consensus. Map key is method name, value is array of field paths.\n * Field paths: \"result\" for direct result value (e.g., eth_getTransactionCount returns hex),\n * or field name for nested result objects (e.g., \"nonce\" for result.nonce).\n * When multiple fields are specified, they act as tie-breakers in order.\n */\n preferHighestValueFor?: { [key: string]: string[]};\n /**\n * FireAndForget when true, allows consensus to return a response to the client immediately\n * upon short-circuit, but does NOT cancel in-flight requests to other upstreams.\n * This is useful for write operations like eth_sendRawTransaction where you want to\n * broadcast the transaction to as many nodes as possible while still returning quickly.\n * Default is false (normal behavior - cancel remaining requests on short-circuit).\n */\n fireAndForget?: boolean;\n /**\n * MaxWaitOnResult caps how long consensus waits for additional participants\n * AFTER at least one non-empty response has arrived. Use this to bound\n * p99 latency when most upstreams are fast but one is a slow straggler:\n * once a real answer is in hand, give the rest at most this long to\n * confirm or dispute, then resolve with what we have.\n * Accepts a duration scalar (\"200ms\") or an AdaptiveDuration object\n * ({base, quantile, min, max}) for adaptive caps driven by per-method\n * latency quantiles. Defaults are applied when consensus is configured\n * but this field is omitted \u2014 see common/defaults.go.\n */\n maxWaitOnResult?: Duration | AdaptiveDuration;\n /**\n * MaxWaitOnEmpty caps how long consensus waits for additional participants\n * AFTER the first response (of any kind \u2014 empty, error, or non-empty)\n * has arrived. Typically set larger than MaxWaitOnResult because an\n * operator is more patient when no useful data is in hand yet.\n * Same shape as MaxWaitOnResult; defaults applied when consensus is set.\n */\n maxWaitOnEmpty?: Duration | AdaptiveDuration;\n /**\n * RequiredParticipants enforces a minimum number of consensus\n * participants carrying a given tag. Each entry says \"at least\n * `minParticipants` of the upstreams that participate must match\n * `tag`\". The engine front-loads enough tag-matching upstreams into\n * the participant set so the first `maxParticipants` drawn satisfy\n * every entry \u2014 without changing `maxParticipants` itself.\n * Best-effort and governed by the EXISTING consensus behaviors: if a\n * required group has fewer healthy upstreams than requested (or the\n * quotas can't all fit within `maxParticipants`), consensus simply\n * runs with what it can promote and the resulting participation is\n * handled by `lowParticipantsBehavior` / `agreementThreshold` exactly\n * like any other low-participation tick. Empty (default) = disabled.\n */\n requiredParticipants?: (ConsensusRequiredParticipant | undefined)[];\n}\n/**\n * ConsensusRequiredParticipant is one tag-quota entry for\n * `consensus.requiredParticipants`. `Tag` is a glob pattern (`*`, `?`)\n * matched against each upstream's `tags`; `MinParticipants` is the minimum\n * number of matching upstreams that must be in the consensus participant\n * set (pool quota, best-effort). `MinAgreement` is the minimum number of\n * matching upstreams that must be part of the WINNING response group\n * (winner-composition quota, hard-enforced: a winner that does not satisfy\n * it becomes a composition dispute regardless of disputeBehavior). A single\n * upstream can satisfy multiple entries it matches.\n */\nexport interface ConsensusRequiredParticipant {\n tag: string;\n minParticipants: number /* int */;\n minAgreement?: number /* int */;\n}\nexport type MisbehaviorsDestinationType = string;\nexport const MisbehaviorsDestinationTypeFile: MisbehaviorsDestinationType = \"file\";\nexport const MisbehaviorsDestinationTypeS3: MisbehaviorsDestinationType = \"s3\";\nexport interface MisbehaviorsDestinationConfig {\n /**\n * Type of destination: \"file\" or \"s3\"\n */\n type: 'file' | 's3';\n /**\n * Path for file destination, or S3 URI (s3://bucket/prefix/) for S3 destination\n */\n path: string;\n /**\n * Pattern for generating file names. Supports placeholders:\n * {dateByHour} - formatted as 2006-01-02-15\n * {dateByDay} - formatted as 2006-01-02\n * {method} - the RPC method name\n * {networkId} - the network ID with : replaced by _\n * {instanceId} - unique instance identifier\n */\n filePattern?: string;\n /**\n * S3-specific settings for bulk flushing\n */\n s3?: S3FlushConfig;\n}\nexport interface S3FlushConfig {\n /**\n * Maximum number of records to buffer before flushing (default: 100)\n */\n maxRecords?: number /* int */;\n /**\n * Maximum size in bytes to buffer before flushing (default: 1MB)\n */\n maxSize?: number /* int64 */;\n /**\n * Maximum time to wait before flushing buffered records (default: 60s)\n */\n flushInterval?: Duration;\n /**\n * AWS region for S3 bucket (defaults to AWS_REGION env var)\n */\n region?: string;\n /**\n * Custom S3 endpoint URL for S3-compatible providers (Tigris, MinIO, R2, \u2026).\n * Empty = real AWS S3. When set, path-style addressing is used.\n */\n endpoint?: string;\n /**\n * AWS credentials config (optional). If not specified, uses standard AWS credential chain:\n * 1. Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)\n * 2. IAM role (for EC2/ECS/EKS)\n * 3. Shared credentials file (~/.aws/credentials)\n * Supported modes: \"env\", \"file\", \"secret\"\n */\n credentials?: AwsAuthConfig;\n /**\n * Content type for uploaded files (default: \"application/jsonl\")\n */\n contentType?: string;\n}\nexport interface PunishMisbehaviorConfig {\n disputeThreshold: number /* uint */;\n disputeWindow?: Duration;\n sitOutPenalty?: Duration;\n}\nexport interface RateLimiterConfig {\n store?: RateLimitStoreConfig;\n budgets: RateLimitBudgetConfig[];\n}\nexport interface RateLimitBudgetConfig {\n id: string;\n rules: RateLimitRuleConfig[];\n /**\n * CreditUnits prices methods for this budget's countMode: credit rules. \"*\"\n * is the fallback, an unpriced method costs 1, and a method priced 0 is\n * exempt. An upstream on rateLimitCountMode: credit prices from its vendor\n * instead and may not combine the two.\n */\n creditUnits?: { [key: string]: number /* int64 */};\n}\nexport interface RateLimitRuleConfig {\n method: string;\n maxCount: number /* uint32 */;\n /**\n * Period is the canonical period selector. Supported: second, minute, hour, day, week, month, year\n */\n period: RateLimitPeriod;\n waitTime?: Duration;\n perIP?: boolean;\n perUser?: boolean;\n perNetwork?: boolean;\n /**\n * CountMode selects what this rule counts. \"request\" charges 1 per call and\n * counts per method. \"credit\" charges the method's cost from the budget's\n * creditUnits and pools all methods into one counter, making maxCount a\n * wallet. Empty inherits the caller's mode.\n */\n countMode?: RateLimitCountMode;\n}\n/**\n * RateLimitPeriod enumerates supported periods for rate limiting.\n * It is an int enum to enable strong typing in TypeScript generation, while\n * marshaling to JSON/YAML as human-readable strings like \"second\", \"minute\", etc.\n */\nexport type RateLimitPeriod = number /* int */;\nexport const RateLimitPeriodSecond: RateLimitPeriod = 0;\nexport const RateLimitPeriodMinute: RateLimitPeriod = 1;\nexport const RateLimitPeriodHour: RateLimitPeriod = 2;\nexport const RateLimitPeriodDay: RateLimitPeriod = 3;\nexport const RateLimitPeriodWeek: RateLimitPeriod = 4;\nexport const RateLimitPeriodMonth: RateLimitPeriod = 5;\nexport const RateLimitPeriodYear: RateLimitPeriod = 6;\nexport interface ProxyPoolConfig {\n id: string;\n urls: string[];\n}\nexport interface MethodsConfig {\n preserveDefaultMethods?: boolean;\n definitions?: { [key: string]: CacheMethodConfig | undefined};\n}\nexport interface NetworkConfig {\n architecture: TsNetworkArchitecture;\n rateLimitBudget?: string;\n failsafe?: (FailsafeConfig | undefined)[];\n evm?: EvmNetworkConfig;\n svm?: SvmNetworkConfig;\n selectionPolicy?: SelectionPolicyConfig;\n directiveDefaults?: DirectiveDefaultsConfig;\n alias?: string;\n methods?: MethodsConfig;\n multiplexing?: boolean;\n staticResponses?: (StaticResponseConfig | undefined)[];\n /**\n * Integrity overrides the project-wide data-integrity configuration for this\n * network. Merges over the project block (network wins).\n */\n integrity?: IntegrityConfig;\n /**\n * CacheKeySuffix, when set, is inserted into the JSON-RPC cache partition\n * key as {networkId}:{suffix}:{blockRef} so two networks that share a\n * chainId (and a Redis) do not collide. Empty keeps {networkId}:{blockRef}.\n * When {networkId}:{suffix} is itself a valid network id, colons in the\n * network id are escaped; see CachePartitionKey.\n */\n cacheKeySuffix?: string;\n}\n/**\n * StaticResponseConfig declares a canned JSON-RPC response for a specific\n * (method, params) pair on a network. When an inbound request matches, the\n * configured response is returned immediately and no upstream is contacted.\n * Useful for chains that deviate from client assumptions (for example, chains\n * whose genesis block is not 0) where probing upstreams would yield errors\n * or inconsistent data.\n */\nexport interface StaticResponseConfig {\n method: string;\n params?: any[];\n response?: StaticResponseBodyConfig;\n}\n/**\n * StaticResponseBodyConfig holds the JSON-RPC payload to serve. Exactly one\n * of Result or Error must be set.\n */\nexport interface StaticResponseBodyConfig {\n result?: any;\n error?: StaticResponseErrorConfig;\n}\n/**\n * StaticResponseErrorConfig mirrors a JSON-RPC error object.\n */\nexport interface StaticResponseErrorConfig {\n code: number /* int */;\n message: string;\n data?: any;\n}\nexport interface DirectiveDefaultsConfig {\n retryEmpty?: boolean;\n retryPending?: boolean;\n skipCacheRead?: any;\n useUpstream?: string;\n skipInterpolation?: boolean;\n skipConsensus?: boolean;\n /**\n * Validation: Block Integrity\n */\n enforceHighestBlock?: boolean;\n enforceGetLogsBlockRange?: boolean;\n enforceNonNullTaggedBlocks?: boolean;\n /**\n * --- Deprecated data-integrity validation flags ---\n * Deprecated: data integrity is now configured via the `integrity` block\n * (projects[].integrity / networks[].integrity). These per-check flags are\n * retained only so existing YAML still parses. At startup the flags that map\n * to a current check are translated into `integrity.checks` by\n * migrateLegacyIntegrityChecks (an explicit `integrity` block wins per check);\n * they are NOT read at runtime. The receipt-count / expected-block /\n * receipt-to-transaction flags have no equivalent in the new model and are\n * accepted but ignored. See docs/pages/config/failsafe/integrity.mdx.\n */\n validateTransactionsRoot?: boolean;\n validateHeaderFieldLengths?: boolean;\n validateTransactionFields?: boolean;\n validateTransactionBlockInfo?: boolean;\n enforceLogIndexStrictIncrements?: boolean;\n validateTxHashUniqueness?: boolean;\n validateTransactionIndex?: boolean;\n validateLogFields?: boolean;\n validateLogsBloomEmptiness?: boolean;\n validateLogsBloomMatch?: boolean;\n validateReceiptTransactionMatch?: boolean;\n validateContractCreation?: boolean;\n receiptsCountExact?: number /* int64 */;\n receiptsCountAtLeast?: number /* int64 */;\n validationExpectedBlockHash?: string;\n validationExpectedBlockNumber?: number /* int64 */;\n}\n/**\n * SvmNetworkConfig mirrors EvmNetworkConfig for SVM networks.\n * Most fields are Solana-specific and do not have EVM equivalents.\n */\nexport interface SvmNetworkConfig {\n /**\n * Chain identifies which SVM chain this network runs on. Defaults to \"solana\"\n * when empty for backward compatibility. Set explicitly for forks/variants\n * such as \"fogo\" or \"eclipse\" so eRPC can host multiple SVM chains side by\n * side without network-ID or cache-key collisions.\n * When Chain is empty or \"solana\", the derived NetworkId is \"svm:\"\n * \u2014 identical to the pre-multi-chain format. For any other Chain value the\n * NetworkId is \"svm::\".\n */\n chain?: string;\n /**\n * Cluster the upstreams of this network serve (e.g. \"mainnet-beta\", \"devnet\").\n * The NetworkId is derived from this value together with Chain \u2014 see the\n * Chain field above for the exact format.\n */\n cluster?: string;\n /**\n * Commitment is the default commitment level injected into requests whose params\n * omit one. One of \"finalized\", \"confirmed\", \"processed\". No default: when unset,\n * no commitment is injected and each upstream's own server-side default governs\n * (Solana's is \"finalized\"). Set this to pin one commitment across all upstreams\n * so the cache and consensus key on identical data regardless of vendor defaults;\n * note that doing so makes finality classification track the configured level.\n */\n commitment?: string;\n /**\n * StatePollerDebounce sets the minimum interval between polls of an upstream's\n * slot/health view. Default: 400ms (one slot).\n */\n statePollerDebounce?: Duration;\n /**\n * MaxFinalizedSlotLag bounds how many slots an upstream's FinalizedSlot may\n * trail the pool's highest FinalizedSlot before it is excluded from\n * consensus voting on finalized data. Only applied when a consensus policy\n * is active AND the request's resolved finality is Finalized.\n * Pointer so an explicit 0 is distinguishable from \"unset\": nil takes the\n * 100-slot default (filled by SetDefaults, the only place that materializes\n * it), an explicit 0 disables the filter entirely. A plain int64 collapses\n * those two cases and makes the documented disable switch unreachable.\n * Readers downstream of SetDefaults just check `lag != nil && *lag > 0`.\n */\n maxFinalizedSlotLag?: number /* int64 */;\n /**\n * EnforceBlockAvailability controls whether the networkPreForward_getBlock\n * guard is active. When enabled (default), getBlock/getConfirmedBlock\n * requests for slots above the highest indexed slot known to any upstream\n * are short-circuited with ErrEndpointMissingData before hitting providers\n * \u2014 saving quota and triggering the 500ms indexing-lag retry immediately.\n * Set to false to disable the guard when ShredInsertSlot tracking is\n * unavailable or unreliable on a given deployment.\n */\n enforceBlockAvailability?: boolean;\n}\n/**\n * SvmUpstreamConfig carries per-upstream SVM settings.\n */\nexport interface SvmUpstreamConfig {\n /**\n * Chain identifies which SVM chain this upstream serves. Must match the\n * network-level Chain. Empty defaults to \"solana\" for backward compat.\n */\n chain?: string;\n /**\n * Cluster this upstream serves. Must match the network-level cluster for the\n * upstream to be eligible.\n */\n cluster?: string;\n /**\n * CheckGenesisHash opts unknown clusters in to runtime validation via getGenesisHash\n * at bootstrap. Known clusters (mainnet-beta, devnet, testnet) are always validated\n * regardless of this flag: a single getGenesisHash RPC runs at bootstrap and is\n * compared against the hardcoded genesis-hash table \u2014 a mismatch OR a fetch failure\n * fails the upstream, catching nodes mis-pointed at the wrong cluster (and refusing\n * to register one we could not verify). For unknown clusters the same check (with\n * no table comparison) runs only when this flag is set; otherwise it is skipped so\n * private/local clusters with no published genesis hash still work.\n */\n checkGenesisHash?: boolean;\n}\nexport interface EvmNetworkConfig {\n chainId: number /* int64 */;\n fallbackFinalityDepth?: number /* int64 */;\n fallbackStatePollerDebounce?: Duration;\n integrity?: EvmIntegrityConfig;\n /**\n * ServedTip configures how the network derives the \"latest\"/\"finalized\"\n * block it advertises to clients (and enforces via block-availability).\n * Nil or disabled selects the default mode (the corroborated latest across eligible\n * upstreams, see ServedTipPick.Freshest); set Enabled to opt into the majority tip. See\n * EvmServedTipConfig.\n */\n servedTip?: EvmServedTipConfig;\n getLogsMaxAllowedRange?: number /* int64 */;\n getLogsMaxAllowedAddresses?: number /* int64 */;\n getLogsMaxAllowedTopics?: number /* int64 */;\n getLogsSplitOnError?: boolean;\n getLogsSplitConcurrency?: number /* int */;\n /**\n * GetLogsMaxResponseBytes caps the total size of a MERGED eth_getLogs response \u2014\n * the sum of every sub-response a split assembles, not the size of any one piece.\n * Splitting exists to get past an upstream's per-request limits, so once a request\n * is split there is no remaining bound on what the client can make the proxy hold\n * in memory: a range the upstream refuses is halved, fetched concurrently, and\n * merged whole. This is the only bound on that total.\n * Sub-requests are cancelled as soon as the running total crosses the cap, so the\n * in-flight ones do not materialise either, and the client gets\n * ErrGetLogsExceededMaxAllowedResponseSize (HTTP 413) \u2014 a distinct code from\n * ErrEndpointRequestTooLarge precisely so it does NOT feed the splitting path that\n * produced the response.\n * Zero (the default) disables the cap and preserves the historical behaviour.\n */\n getLogsMaxResponseBytes?: number /* int64 */;\n /**\n * TraceFilterSplitOnError controls reactive splitting for trace_filter and\n * arbtrace_filter requests when the upstream returns a range-too-large error.\n * Nil disables the feature.\n */\n traceFilterSplitOnError?: boolean;\n /**\n * TraceFilterSplitConcurrency caps in-flight sub-requests when a trace_filter\n * or arbtrace_filter request is split. Zero falls back to 10.\n */\n traceFilterSplitConcurrency?: number /* int */;\n /**\n * EnforceBlockAvailability controls whether the network should enforce per-upstream\n * block availability bounds (upper/lower) for methods by default. Method-level config may override.\n * When nil or true, enforcement is enabled.\n */\n enforceBlockAvailability?: boolean;\n /**\n * MaxRetryableBlockDistance controls the maximum block distance for which an upstream\n * block unavailability error is considered retryable. If the requested block is within\n * this distance from the upstream's latest block, the error is retryable (upstream may catch up).\n * If the distance is larger, the error is not retryable (upstream is too far behind).\n * Default: 128 blocks.\n */\n maxRetryableBlockDistance?: number /* int64 */;\n /**\n * MarkEmptyAsErrorMethods lists methods for which an empty/null result from an upstream\n * should be treated as a \"missing data\" error, triggering retry on other upstreams.\n * This is useful for point-lookups (blocks, transactions, receipts, traces) where an\n * empty result likely means the upstream hasn't indexed that data yet.\n * Default includes common point-lookup methods like eth_getBlockByNumber, eth_getTransactionByHash, etc.\n */\n markEmptyAsErrorMethods?: string[];\n /**\n * DynamicBlockTimeDebounceMultiplier scales the EMA-estimated block time to derive\n * the debounce interval for block polling. A value of 0.7 means debounce = 70% of\n * the estimated block time, preferring fresher data at the cost of slightly more\n * polling. Lower values reduce staleness risk; higher values reduce RPC calls.\n * Default: 0.7 (30% under the estimated block time).\n */\n dynamicBlockTimeDebounceMultiplier?: number /* float64 */;\n /**\n * BlockUnavailableDelayMultiplier scales the EMA-estimated block time to derive the\n * retry delay when the requested data isn't available yet (ErrUpstreamBlockUnavailable\n * or an empty/missing-data point-lookup). When the dynamic block time is known, the\n * delay is blockTime * this multiplier. Falls back to the static\n * RetryPolicyConfig.EmptyResultDelay when block time is not yet available. Default: 1.0.\n */\n blockUnavailableDelayMultiplier?: number /* float64 */;\n /**\n * IdempotentTransactionBroadcast enables idempotency handling for eth_sendRawTransaction.\n * When enabled (default), \"already known\" and verified \"nonce too low\" errors are converted\n * to success responses with the transaction hash. This allows failsafe policies (retry/hedge)\n * to work safely with transaction broadcasting.\n * Set to false to disable this behavior and return raw upstream errors.\n */\n idempotentTransactionBroadcast?: boolean;\n /**\n * EmptyResultConfidence sets how confirmed a concrete numeric block must be for an\n * empty/null point-lookup result to be treated as retryable missing-data, versus a\n * truthful \"not yet produced/confirmed\" empty returned without retrying. Applies to\n * MarkEmptyAsErrorMethods when the RetryEmpty directive is on; tags and block-hash\n * lookups never qualify, and it fails open when the head is unknown.\n * - blockHead (default): retry empties for blocks at/below the latest head; a\n * block above the head isn't produced yet \u2192 return the empty truthfully.\n * - finalizedBlock: stricter \u2014 only retry empties for blocks at/below the\n * finalized head; an unfinalized block's empty is treated as not-yet-confirmed.\n */\n emptyResultConfidence?: AvailbilityConfidence;\n /**\n * SafeBlockSource is an upstream id/tag selector for standard JSON-RPC\n * requests carrying the `safe` block tag. Matching upstreams define and\n * serve `safe`; empty (without an inherited network default) keeps existing\n * provider-defined routing. This does not affect eth_query* or gRPC Query.\n */\n safeBlockSource?: string;\n}\n/**\n * EvmServedTipConfig controls how the network derives the \"latest\"/\"finalized\"\n * block it advertises (and enforces) from its upstreams.\n * In the default mode the served tip is the corroborated latest block across eligible\n * non-syncing upstreams (second-highest, or the only one) \u2014 which can still advertise a block a slightly-ahead\n * pair has, causing \"block not found\" churn when requests route to a\n * slightly-behind upstream. When a tag is listed in EnabledFor, that tag's\n * served value is instead the freshest block a strict MAJORITY of the eligible\n * upstreams already have, so interpolated requests land on upstreams that can\n * serve the advertised block.\n */\nexport interface EvmServedTipConfig {\n /**\n * EnabledFor lists the block tags whose served value uses the cluster-min tip\n * instead of the default corroborated head. Valid entries: \"latest\" and \"finalized\" (the\n * \"safe\" tag follows \"finalized\"). Empty selects the default mode for all tags.\n */\n enabledFor?: string[];\n /**\n * Deprecated: ClusterDelta configured the former cluster-based picker and is\n * ignored \u2014 the majority order statistic needs no tuning. Kept only so\n * existing configs keep parsing.\n */\n clusterDelta?: number /* int64 */;\n /**\n * GuaranteedMethods lists method name patterns (glob; e.g. \"trace_*\",\n * \"debug_traceBlockByNumber\") whose supporting-upstream subset must be able to\n * serve the advertised latest. For a request on a matching method, \"latest\"\n * resolves against the majority of only the upstreams that support it\n * (membership auto-detected via ShouldHandleMethod \u2014 no per-upstream config).\n * Empty means only the global (all-eligible) majority is computed.\n */\n guaranteedMethods?: string[];\n /**\n * GuaranteedFor lists upstream SELECTORS (id or tag glob \u2014 the same\n * vocabulary as use-upstream and consensus.requiredParticipants, e.g.\n * \"type:internal\") whose group must be able to serve the advertised tip.\n * For each selector the tip is clamped down to that group's OWN majority,\n * exactly as GuaranteedMethods clamps to a method's supporting set. The\n * clamp is a group MAJORITY, not a group minimum, so one stuck member\n * cannot pin the network; an empty group constrains nothing.\n */\n guaranteedFor?: string[];\n /**\n * MaxRegressionBlocks is how far below the corroborated LIVE upstream head\n * (the second-highest live head) the majority pick may fall before it is\n * treated as a poisoned ballot rather than as reality. While a pick is below\n * that bound the network keeps serving its last corroborated pick\n * (in-process, for at most one minute, then it fails open and serves the\n * pick as computed). 0 uses DefaultToleratedBlockHeadRollback (1024) \u2014 the\n * same rollback tolerance the state poller and health tracker apply to a\n * retreating head, so all three layers agree on what counts as a genuine\n * deep correction. -1 disables the regression guard entirely (the symmetric\n * opposite of trajectoryWindow: 0); no other negative value is accepted.\n */\n maxRegressionBlocks?: number /* int64 */;\n /**\n * TrajectoryWindow is how much recorded head history the trajectory referee\n * needs before it may participate in the pick. The referee tracks where this\n * network's head has been over the window and, when a stalled group holds\n * the majority, serves the corroborated group that is actually where the\n * chain should be by now (see architecture/evm's TipTrajectory: advisory,\n * upward-only, in-process, and a no-op until every confidence condition\n * holds). Unset uses DefaultServedTipTrajectoryWindow (10m); an explicit 0\n * disables the referee entirely \u2014 nothing is recorded and the pick is the\n * plain majority. Any other value must be between\n * MinServedTipTrajectoryWindow and MaxServedTipTrajectoryWindow: below the\n * minimum the window is self-defeating (it must be long enough that a stall\n * cannot look like a trajectory), and above the maximum the sample ring can\n * never span it, so the referee would stand down forever while looking\n * configured. WRITE IT AS A DURATION STRING \u2014 trajectoryWindow: \"10m\". A\n * bare number is parsed as MILLISECONDS.\n */\n trajectoryWindow?: Duration;\n}\n/**\n * EvmIntegrityConfig is deprecated. Use DirectiveDefaultsConfig for validation settings.\n */\nexport interface EvmIntegrityConfig {\n /**\n * @deprecated: use DirectiveDefaults.EnforceHighestBlock\n */\n enforceHighestBlock?: boolean;\n /**\n * @deprecated: use DirectiveDefaults.EnforceGetLogsBlockRange\n */\n enforceGetLogsBlockRange?: boolean;\n /**\n * @deprecated: use DirectiveDefaults.EnforceNonNullTaggedBlocks\n */\n enforceNonNullTaggedBlocks?: boolean;\n}\n/**\n * EvalScope picks the grain at which the selection policy evaluates AND\n * at which the health tracker stores per-upstream metrics. One knob\n * covers what `evalPerMethod` + `evalPerFinality` used to cover as two\n * bools \u2014 and pulls the tracker's metric grain in lockstep so a\n * predicate like `errorRateAbove(0.5)` in a (method, finality)-grained\n * slot sees genuinely (method, finality)-specific error rate, not the\n * per-method aggregate.\n * Values are kebab-case strings so they round-trip through YAML / JSON\n * / Go enum literals cleanly. The TS SDK exports the same names in\n * CAPITAL_SNAKE_CASE with these same string values, so `evalScope:\n * NETWORK` in a TS config and `evalScope: network` in a YAML config\n * produce identical Go state.\n */\nexport type EvalScope = string;\n/**\n * EvalScopeNetwork \u2014 single slot per network. Methods + finalities\n * share. Default. Lowest cardinality + lowest tracker memory.\n */\nexport const EvalScopeNetwork: EvalScope = \"network\";\n/**\n * EvalScopeNetworkMethod \u2014 slot per (network, method). Finalities\n * share. Useful when one upstream is fast on `eth_call` but slow on\n * `trace_filter`.\n */\nexport const EvalScopeNetworkMethod: EvalScope = \"network-method\";\n/**\n * EvalScopeNetworkFinality \u2014 slot per (network, finality). Methods\n * share. Useful when realtime reads weight freshness differently\n * from finalized reads.\n */\nexport const EvalScopeNetworkFinality: EvalScope = \"network-finality\";\n/**\n * EvalScopeNetworkMethodFinality \u2014 slot per (network, method,\n * finality). Most granular routing. Cardinality scales linearly;\n * each slot's ticker only spins up after the first request for\n * that bucket lands.\n */\nexport const EvalScopeNetworkMethodFinality: EvalScope = \"network-method-finality\";\n/**\n * SelectionPolicyConfig declares the per-network upstream selection policy.\n * The eval function is JavaScript that receives `upstreams` and `ctx` and\n * returns the ordered list of upstreams that should serve traffic for the\n * network/method scope. The chainable std-lib (see internal/policy/stdlib)\n * provides the building blocks (sortByScore, removeByLag, stickyPrimary,\n * probeExcluded, etc.) \u2014 see specs/selection-policy/feature.md.\n */\nexport interface SelectionPolicyConfig {\n evalInterval?: Duration;\n /**\n * EvalScope picks the slot grain \u2014 and the matching tracker grain\n * (see `EvalScope` doc). Default `network` (one slot per network).\n * Tighter scopes let predicates like `errorRateAbove(0.5)` see\n * genuinely (method, finality)-specific health.\n */\n evalScope?: EvalScope | \"network\" | \"network-method\" | \"network-finality\" | \"network-method-finality\";\n /**\n * EvalPerBoundary scopes selection-policy evaluation by block-availability\n * \"lane\" \u2014 the set of upstreams whose configured block range can actually\n * serve the request's block. Unlike EvalPerMethod / EvalPerFinality this is\n * deliberately NOT folded into EvalScope: it must not change the\n * health-tracker grain (boundary is a decision/pool axis, not a metrics\n * axis), so the engine reads it directly as an orthogonal slot dimension.\n * When on, a request whose block excludes some upstream is evaluated\n * against a lane-scoped pool \u2014 an upstream that cannot serve the range is\n * absent from the pool (a capability fact), distinct from the policy's\n * soft health-based deprioritization. Default off. Pointer-typed so a\n * future SetDefaults can distinguish \"absent\" from \"explicitly false\".\n */\n evalPerBoundary?: boolean;\n evalTimeout?: Duration;\n /**\n * EvalFunc is the per-tick evaluation function. In YAML it's a JS\n * source string; in TS configs it's a real arrow function compiled\n * into `CompiledProgram` at config load.\n * Signature: `(upstreams, ctx) => Upstream[]`.\n * See specs/selection-policy/feature.md for the stdlib reference.\n */\n evalFunc?: SelectionPolicyEvalFunction | string;\n}\n/**\n * LegacySelectionPolicyFields mirrors the deprecated keys that used to\n * live on SelectionPolicyConfig. None survive translation.\n */\nexport interface LegacySelectionPolicyFields {\n evalFunction?: string;\n resampleExcluded?: boolean;\n resampleInterval?: Duration;\n resampleCount?: number /* int */;\n}\nexport type AuthType = string;\nexport const AuthTypeSecret: AuthType = \"secret\";\nexport const AuthTypeDatabase: AuthType = \"database\";\nexport const AuthTypeJwt: AuthType = \"jwt\";\nexport const AuthTypeSiwe: AuthType = \"siwe\";\nexport const AuthTypeNetwork: AuthType = \"network\";\nexport interface AuthConfig {\n strategies: TsAuthStrategyConfig[];\n}\nexport interface AuthStrategyConfig {\n ignoreMethods?: string[];\n allowMethods?: string[];\n rateLimitBudget?: string;\n /**\n * AllowClientDirectives, if set, overrides the project-level\n * `allowClientDirectives` pattern for users authenticated by THIS strategy.\n * Same wildcard syntax as the project-level field (\"*\" = all, \"\" = none).\n * Client directives (`X-ERPC-*` headers) are powerful per-request overrides \u2014\n * e.g. pinning an upstream bypasses the selection policy, and skipping the\n * cache multiplies upstream load \u2014 so operators exposing erpc directly to\n * untrusted callers typically deny them project-wide and re-enable them only\n * for trusted strategies:\n * \tallowClientDirectives: \"\" # project default: nobody\n * \tauth.strategies[0].allowClientDirectives: \"*\" # this strategy: everything\n * Left unset the caller inherits the project-level pattern, so existing\n * configs are unaffected. The capability is attached to the user by the\n * strategy that authenticated them, which means it can never be granted by\n * `trustUserIdHeader` (that path sets only Id \u2014 see\n * NormalizedRequest.SetUserFromTrustedHeader).\n */\n allowClientDirectives?: string;\n type: TsAuthType;\n network?: NetworkStrategyConfig;\n secret?: SecretStrategyConfig;\n database?: DatabaseStrategyConfig;\n jwt?: JwtStrategyConfig;\n siwe?: SiweStrategyConfig;\n}\nexport interface SecretStrategyConfig {\n id: string;\n value: string;\n /**\n * RateLimitBudget, if set, is applied to the authenticated user from this strategy\n */\n rateLimitBudget?: string;\n}\nexport interface DatabaseStrategyConfig {\n connector?: ConnectorConfig;\n cache?: DatabaseStrategyCacheConfig;\n retry?: DatabaseRetryConfig;\n failOpen?: DatabaseFailOpenConfig;\n maxWait?: Duration;\n}\nexport interface DatabaseStrategyCacheConfig {\n ttl?: number /* time in nanoseconds (time.Duration) */;\n maxSize?: number /* int64 */;\n maxCost?: number /* int64 */;\n numCounters?: number /* int64 */;\n}\nexport interface DatabaseRetryConfig {\n maxAttempts?: number /* int */;\n baseBackoff?: Duration;\n}\nexport interface DatabaseFailOpenConfig {\n enabled: boolean;\n userId?: string;\n rateLimitBudget?: string;\n}\nexport interface JwtStrategyConfig {\n allowedIssuers: string[];\n allowedAudiences: string[];\n allowedAlgorithms: string[];\n requiredClaims: string[];\n claimMatchers?: { [key: string]: string[]};\n verificationKeys?: { [key: string]: string};\n verificationJwksUrl?: string;\n verificationJwksRefreshInterval?: Duration;\n /**\n * Skipping TLS verification is an explicit operator opt-in for the JWKS\n * fetch, not a hardcoded bypass.\n */\n verificationJwksTlsInsecureSkipVerify?: boolean;\n /**\n * RateLimitBudgetClaimName is the JWT claim name that, if present,\n * will be used to set the per-user RateLimitBudget override.\n * Defaults to \"rlm\".\n */\n rateLimitBudgetClaimName?: string;\n}\nexport interface SiweStrategyConfig {\n allowedDomains: string[];\n /**\n * RateLimitBudget, if set, is applied to the authenticated user\n */\n rateLimitBudget?: string;\n}\nexport interface NetworkStrategyConfig {\n allowedIPs: string[];\n allowedCIDRs: string[];\n allowLocalhost: boolean;\n trustedProxies: string[];\n /**\n * RateLimitBudget, if set, is applied to the authenticated user (client IP)\n */\n rateLimitBudget?: string;\n ipAsUser?: boolean;\n}\nexport type LabelMode = string;\nexport const ErrorLabelModeVerbose: LabelMode = \"verbose\";\nexport const ErrorLabelModeCompact: LabelMode = \"compact\";\nexport interface MetricsConfig {\n enabled?: boolean;\n listenV4?: boolean;\n hostV4?: string;\n listenV6?: boolean;\n hostV6?: string;\n port?: number /* int */;\n errorLabelMode?: LabelMode;\n histogramBuckets?: string;\n /**\n * Customizations is the single knob for shaping /metrics: which metric\n * families are exposed at all, which of their labels survive, and which\n * buckets a histogram uses. Entries are applied by specificity rather than\n * by list order \u2014 see MetricsCustomizationConfig.\n * \tmetrics:\n * \t customizations:\n * \t - subject: \"consensus_*\"\n * \t action: drop\n * \t - subject: upstream_request_total\n * \t labels:\n * \t - subject: \"agent_*\"\n * \t action: drop\n * \t - subject: agent_name\n * \t action: keep\n * \t - subject: network_request_duration_seconds\n * \t buckets: [0.05, 0.5, 5]\n */\n customizations?: (MetricsCustomizationConfig | undefined)[];\n /**\n * Deprecated: use Customizations with a `labels` list. Kept working so\n * existing configs keep loading; it is desugared onto the same rules as an\n * every-histogram label drop.\n */\n histogramDropLabels?: string[];\n /**\n * Deprecated: use Customizations with an exact `subject` and a `labels` list\n * keeping what this metric needs.\n */\n histogramLabelOverrides?: { [key: string]: string[]};\n /**\n * Deprecated: use Customizations with a `labels` list. Kept working so\n * existing configs keep loading; it is desugared onto the same rules as an\n * every-counter label drop.\n */\n counterDropLabels?: string[];\n /**\n * Deprecated: use Customizations with an exact `subject` and a `labels` list\n * keeping what this metric needs.\n */\n counterLabelOverrides?: { [key: string]: string[]};\n /**\n * CounterIdleEvictionAfter bounds /metrics cardinality for hot-path\n * counters whose label-sets are keyed by caller-controlled inputs\n * (method, user, agentName, ...). Counter series idle for at least this\n * duration are evicted from the Prometheus registry (DeleteLabelValues)\n * by the health tracker's idle sweep; a series that becomes active again\n * restarts at zero \u2014 the same semantics rate()/increase() consumers\n * already handle across process restarts. Defaults to 24h (conservative:\n * only clearly-dead label combinations are released). Set to 0 to\n * disable eviction entirely.\n */\n counterIdleEvictionAfter?: Duration;\n}\n/**\n * MetricCustomizationAction is what a customization entry does to what it\n * selects.\n */\nexport type MetricCustomizationAction = string;\nexport const MetricActionKeep: MetricCustomizationAction = \"keep\";\nexport const MetricActionDrop: MetricCustomizationAction = \"drop\";\n/**\n * MetricsCustomizationConfig is one entry of metrics.customizations: a subject\n * selecting metric families, and what to do with them.\n * Overlapping subjects resolve by specificity, not by list order: an exact\n * family name beats a prefix, a longer prefix beats a shorter one, and equally\n * specific subjects break to the one written later. So \"drop consensus_*, keep\n * consensus_duration_seconds\" means the same thing whichever order it is written\n * in.\n */\nexport interface MetricsCustomizationConfig {\n /**\n * Subject selects metric families: an exact name (\"upstream_request_total\"),\n * a prefix ending in \"*\" (\"consensus_*\"), or \"*\" for every family. The\n * \"erpc_\" namespace prefix is optional. The Go runtime, process and promhttp\n * collectors are named in full (\"go_goroutines\") and are subject to the same\n * rules, so `subject: \"*\", action: drop` drops them too.\n */\n subject: string;\n /**\n * Action drops the matched families from /metrics, or keeps them against a\n * broader drop. Omit it to leave exposure alone and only customize labels or\n * buckets.\n * A dropped eRPC family is never registered, so it costs no series and no\n * collection time \u2014 but that makes it a startup decision, undone only by a\n * restart. Stock collectors are registered outside eRPC and so are filtered\n * out of the scrape response instead, which shrinks the page without saving\n * collection.\n */\n action?: 'keep' | 'drop';\n /**\n * Labels projects the matched families' label sets. Same precedence rules as\n * Subject, applied to label names: `agent_*: drop` then `agent_name: keep`\n * drops the group and spares the one label.\n * Dropping a label collapses every series that differed only in it. Counter\n * sums stay correct, but the dimension stops being queryable \u2014 check what\n * reads it (billing or attribution pipelines, dashboards) first. Gauges have\n * no projection, because collapsing gauge series would report whichever\n * writer wrote last rather than a coarser number.\n */\n labels?: (MetricLabelCustomizationConfig | undefined)[];\n /**\n * Buckets replaces the bucket boundaries of the matched histograms,\n * overriding both metrics.histogramBuckets and what the metric declares in\n * code. Must be strictly increasing.\n */\n buckets?: number /* float64 */[];\n}\n/**\n * MetricLabelCustomizationConfig keeps or drops one label, or a \"*\"-terminated\n * group of them, on the families its parent customization matched.\n */\nexport interface MetricLabelCustomizationConfig {\n subject: string;\n action: 'keep' | 'drop';\n}\n/**\n * RateLimitStoreConfig defines where rate limit counters are stored\n */\nexport interface RateLimitStoreConfig {\n driver: string; // \"redis\" | \"memory\"\n redis?: RedisConnectorConfig;\n cacheKeyPrefix?: string;\n nearLimitRatio?: number /* float32 */;\n}\n\n//////////\n// source: config_integrity.go\n\n/**\n * IntegrityHeaderModeOff ignores per-request integrity headers entirely.\n */\nexport const IntegrityHeaderModeOff = \"off\";\n/**\n * IntegrityHeaderModeProfiles lets a request select only a named profile.\n */\nexport const IntegrityHeaderModeProfiles = \"profiles\";\n/**\n * IntegrityHeaderModeFull lets a request set a level / per-check overrides.\n */\nexport const IntegrityHeaderModeFull = \"full\";\n/**\n * IntegritySettings is the reusable body of an integrity configuration: the\n * front-door level plus the axes (checks / budget) and the per-finality\n * verdict. The top-level/per-network blocks and every named profile share this\n * shape \u2014 only IntegrityConfig adds the header surface and profiles on top.\n */\nexport interface IntegritySettings {\n /**\n * Level is the front-door preset: off | intrinsic | corroborated | authoritative.\n */\n level?: string;\n /**\n * Checks overrides individual checks by their catalog id (enable/disable,\n * params, per-check failure mode).\n */\n checks?: { [key: string]: IntegrityCheckConfig | undefined};\n /**\n * Budget caps the canonical force-fetches the authoritative tier issues.\n */\n budget?: IntegrityBudgetConfig;\n /**\n * AutoCorrectWhenPossible controls whether a violation triggers a hunt for a\n * validated replacement (retry/hedge against other upstreams) BEFORE the\n * InvalidBehavior verdict applies. Default TRUE (nil = true). The two knobs\n * are orthogonal on purpose:\n * autoCorrectWhenPossible | invalidBehavior | on violation\n * ------------------------+-----------------+---------------------------------\n * true | recordOnly | seek replacement; serve it if one\n * | | validates; exhausted -> serve the\n * | | flagged ORIGINAL (never an error)\n * true | hardReject | seek replacement; exhausted -> error\n * false | recordOnly | serve original immediately + record\n * false | hardReject | fail immediately, no alternates\n */\n autoCorrectWhenPossible?: boolean;\n /**\n * InvalidBehavior is the per-finality verdict for reorg-sensitive checks,\n * where invalid data is ambiguously a node bug or a reorg: recordOnly |\n * hardReject | off. Deterministic checks ignore it and always hardReject\n * (override per check via checks..onFailure).\n */\n invalidBehavior?: IntegrityInvalidBehaviorConfig;\n /**\n * ReorgWindow is how many blocks back from the tip the integrity ChainView\n * keeps a number\u2192hash pin + header and tracks reorgs (default 32). Raise for\n * deep-reorg chains (e.g. polygon 256).\n */\n reorgWindow?: number /* int */;\n /**\n * ObserveOnly runs every enabled check but never lets a verdict touch the\n * response: violations that WOULD have been rejected are recorded with the\n * outcome \"would_reject\" and served anyway. This is the safe way to enable\n * integrity on a network for the first time \u2014 it reveals both bad upstream\n * data AND the module's own gaps on that chain at zero request risk, and\n * the would_reject rate is exactly the client-facing cost enforcement\n * would incur.\n * It is ABSOLUTE and overrides everything else, including a per-check\n * onFailure: reject and invalidBehavior \u2014 so a future release adding a new\n * check cannot start rejecting on an observe-only network. Deterministic\n * checks are covered too, which invalidBehavior alone cannot do (they\n * ignore it by design).\n */\n observeOnly?: boolean;\n /**\n * StateProbe proves, per upstream, that the node actually holds the state\n * trie it claims: on each new followed block, probe every upstream with an\n * execution-context call (and eth_getProof where supported) verified\n * against the follower's verified header, and advance that upstream's\n * state-proven head only on success. The probes PUBLISH evidence, they\n * never route: proofs feed the state-proven telemetry, and a sustained\n * streak of wrong-height answers is recorded as upstream misbehavior on\n * the health tracker \u2014 the same ledger consensus disputes and integrity\n * rejects feed \u2014 for the selection policy to act on (e.g. the\n * misbehaviorRateAbove predicate); absence of proof has no effect at all,\n * and ObserveOnly suppresses the misbehavior recording like every other\n * integrity effect. Requires follow to be enabled (the verified header is\n * the trust anchor). Off by default.\n */\n stateProbe?: IntegrityStateProbeConfig;\n /**\n * Follow turns the ChainView into an actual chain follower: it walks the\n * chain forward block by block, requires each block to name its\n * predecessor's hash, and reconciles reorgs by finding the common ancestor.\n * Off by default \u2014 it costs one eth_getBlockByNumber per block per network.\n */\n follow?: IntegrityFollowConfig;\n /**\n * MisbehaviorsDestination durably archives every integrity catch (reject or\n * soft-flag) as a JSONL record \u2014 same file/S3 destination shape as the\n * consensus policy's misbehaviorsDestination. Catches are rare, so the\n * volume is low; the archive is the forensic record metrics can't carry.\n */\n misbehaviorsDestination?: MisbehaviorsDestinationConfig;\n}\n/**\n * IntegrityFollowConfig configures the ChainView chain follower.\n * Following gives the module a CONTIGUOUS, parent-linked view of the chain\n * instead of the sparse scatter of heights that client traffic happens to\n * touch. That is what lets it answer \"is this block canonical\" from a verified\n * chain rather than from whichever source was observed first, and it is the\n * precondition for any check that compares consecutive headers.\n */\nexport interface IntegrityFollowConfig {\n /**\n * Enabled turns the follower on for this network. Default false.\n */\n enabled?: boolean;\n /**\n * Interval is how often to check for new blocks (default 1s). Keep it below\n * the chain's block time or the follower will run a step behind.\n */\n interval?: Duration;\n /**\n * MaxBlocksPerTick bounds catch-up work per tick (default 16), so a\n * follower starting far behind converges steadily instead of bursting.\n */\n maxBlocksPerTick?: number /* int */;\n}\n/**\n * IntegrityStateProbeConfig configures the per-upstream state-trie probes.\n */\nexport interface IntegrityStateProbeConfig {\n /**\n * Enabled turns the probes on. Default false.\n */\n enabled?: boolean;\n /**\n * Interval is the minimum time between probes of the same upstream\n * (default 2s) \u2014 a bound on probe cost, not a schedule; probes fire on\n * followed-block arrival.\n */\n interval?: Duration;\n}\n/**\n * IntegrityConfig is an IntegritySettings plus the per-request header-control\n * surface and the named profiles a request may select. It lives at the project\n * level (applies to all networks) and the network level (overrides).\n */\nexport interface IntegrityConfig {\n integritysettings: IntegritySettings;\n /**\n * HeaderMode controls whether/how per-request X-ERPC-Integrity headers may\n * adjust integrity: off | profiles | full. Defaults to off.\n */\n headerMode?: string;\n /**\n * Profiles are named settings an operator defines; a request may select one\n * by name via header when HeaderMode permits.\n */\n profiles?: { [key: string]: IntegritySettings | undefined};\n}\n/**\n * IntegrityCheckConfig overrides a single check by its catalog id.\n */\nexport interface IntegrityCheckConfig {\n /**\n * Enabled forces the check on/off; nil leaves the level's decision intact.\n */\n enabled?: boolean;\n /**\n * Params are check-specific knobs (e.g. bloom equality-vs-superset).\n */\n params?: { [key: string]: string};\n /**\n * OnFailure overrides this check's failure mode: recordOnly | hardReject.\n */\n onFailure?: string;\n}\n/**\n * IntegrityBudgetConfig caps the authoritative tier's canonical fetches.\n */\nexport interface IntegrityBudgetConfig {\n maxPerSecond?: number /* int */;\n maxConcurrent?: number /* int */;\n}\n/**\n * IntegrityInvalidBehaviorConfig is the per-finality verdict for reorg-sensitive\n * checks: recordOnly | hardReject | off, split by whether the response's block\n * is finalized. The verdict describes what happens when no valid answer is\n * obtainable; whether a replacement is sought first is AutoCorrectWhenPossible.\n */\nexport interface IntegrityInvalidBehaviorConfig {\n finalized?: string;\n unfinalized?: string;\n}\n\n//////////\n// source: data.go\n\nexport type DataFinalityState = number /* int */;\n/**\n * Finalized gets 0 intentionally so that when user has not specified finality,\n * it defaults to finalized, which is safest sane default for caching.\n * This attribute will be calculated based on extracted block number (from request and/or response)\n * and comparing to the upstream (one that returned the response) 'finalized' block (fetch via evm state poller).\n */\nexport const DataFinalityStateFinalized: DataFinalityState = 0;\n/**\n * When we CAN determine the block number, and it's after the upstream 'finalized' block, we consider the data unfinalized.\n */\nexport const DataFinalityStateUnfinalized: DataFinalityState = 1;\n/**\n * Certain methods points are meant to be realtime and updated with every new block (e.g. eth_gasPrice).\n * These can be cached with short TTLs to improve performance.\n */\nexport const DataFinalityStateRealtime: DataFinalityState = 2;\n/**\n * When we CANNOT determine the block number (e.g some trace by hash calls), we consider the data unknown.\n * Most often it is safe to cache this data for longer as they're access when block hash is provided directly.\n */\nexport const DataFinalityStateUnknown: DataFinalityState = 3;\n/**\n * DataFinalityStateAll is the internal wildcard sentinel used by the\n * health tracker to key its cross-finality aggregate rollups. NOT a\n * valid user-facing finality \u2014 never set on a request, never returned\n * from `Finality()`. Lives here so the tracker (health/) and the\n * policy engine (internal/policy/) can both reference it without a\n * cycle. Negative-valued so it sorts before any real finality bucket\n * and doesn't collide with the iota-based enum values.\n */\nexport const DataFinalityStateAll: DataFinalityState = -1;\nexport type CacheEmptyBehavior = number /* int */;\nexport const CacheEmptyBehaviorIgnore: CacheEmptyBehavior = 0;\nexport const CacheEmptyBehaviorAllow: CacheEmptyBehavior = 1;\nexport const CacheEmptyBehaviorOnly: CacheEmptyBehavior = 2;\n/**\n * CachePolicyAppliesTo controls whether a cache policy applies to get, set, or both operations.\n */\nexport type CachePolicyAppliesTo = string;\nexport const CachePolicyAppliesToBoth: CachePolicyAppliesTo = \"both\";\nexport const CachePolicyAppliesToGet: CachePolicyAppliesTo = \"get\";\nexport const CachePolicyAppliesToSet: CachePolicyAppliesTo = \"set\";\n\n//////////\n// source: error_extractor.go\n\n/**\n * JsonRpcErrorExtractor allows callers to inject architecture-specific\n * JSON-RPC error normalization logic into HTTP clients without creating\n * package import cycles.\n */\nexport type JsonRpcErrorExtractor = any;\n/**\n * JsonRpcErrorExtractorFunc is an adapter to allow normal functions to be used\n * as JsonRpcErrorExtractor implementations.\n * Similar to http.HandlerFunc style adapters.\n */\nexport type JsonRpcErrorExtractorFunc = any;\n\n//////////\n// source: exec_state.go\n\n/**\n * UpstreamAttemptOutcome enumerates the possible per-attempt outcomes\n * recorded against an upstream. The set is closed: every attempt ends\n * in exactly one of these.\n */\nexport type UpstreamAttemptOutcome = string;\nexport const UpstreamOutcomeSuccess: UpstreamAttemptOutcome = \"success\";\nexport const UpstreamOutcomeEmpty: UpstreamAttemptOutcome = \"empty\";\nexport const UpstreamOutcomeTransportError: UpstreamAttemptOutcome = \"transport_error\";\nexport const UpstreamOutcomeServerError: UpstreamAttemptOutcome = \"server_error\";\nexport const UpstreamOutcomeClientError: UpstreamAttemptOutcome = \"client_error\";\nexport const UpstreamOutcomeRateLimited: UpstreamAttemptOutcome = \"rate_limited\";\nexport const UpstreamOutcomeMissingData: UpstreamAttemptOutcome = \"missing_data\";\nexport const UpstreamOutcomeExecRevert: UpstreamAttemptOutcome = \"exec_revert\";\nexport const UpstreamOutcomeBlockUnavailable: UpstreamAttemptOutcome = \"block_unavailable\";\nexport const UpstreamOutcomeBreakerOpen: UpstreamAttemptOutcome = \"breaker_open\";\nexport const UpstreamOutcomeCancelled: UpstreamAttemptOutcome = \"cancelled\";\nexport const UpstreamOutcomeTimeout: UpstreamAttemptOutcome = \"timeout\";\nexport const UpstreamOutcomeSkipped: UpstreamAttemptOutcome = \"skipped\";\n/**\n * UpstreamSelectionReason describes WHY a particular upstream was\n * selected for a given attempt. Operators use this to debug skew in\n * upstream-pick distribution (e.g. why is one upstream getting all\n * the hedge fan-out?).\n */\nexport type UpstreamSelectionReason = string;\nexport const SelectionReasonPrimary: UpstreamSelectionReason = \"primary\"; // initial pick\nexport const SelectionReasonRetry: UpstreamSelectionReason = \"retry\"; // network-scope retry\nexport const SelectionReasonHedge: UpstreamSelectionReason = \"hedge\"; // speculative hedge fan-out\nexport const SelectionReasonConsensusSlot: UpstreamSelectionReason = \"consensus_slot\"; // one consensus participant\nexport const SelectionReasonSweep: UpstreamSelectionReason = \"sweep\"; // try-all-upstreams iteration\n/**\n * UpstreamAttempt is one (upstream, attempt) record. The executors\n * append these as participants come and go so operators can answer\n * \"which upstreams were involved in this request, why were they\n * chosen, and what happened to them?\" without parsing trace data.\n * Won is flipped to true by the executor when this attempt's response\n * contributed to the final response returned to the client. For a\n * non-consensus request that's exactly one attempt (the winning one);\n * for consensus it's every participant whose vote landed in the\n * winning agreement group.\n */\nexport interface UpstreamAttempt {\n upstreamid: string;\n vendorname: string;\n startedat: any /* time.Time */;\n duration: number /* time in nanoseconds (time.Duration) */;\n outcome: UpstreamAttemptOutcome;\n reason: UpstreamSelectionReason;\n ishedge: boolean;\n isretry: boolean;\n won: boolean; // true when this attempt contributed to the response\n attemptidx: number /* int */; // 0-based attempt index within the parent loop\n errorcode: string; // ErrorCode string when Outcome is an error variant\n errordetail: string; // free-form short description (truncated)\n /**\n * CreditUnits is the vendor credit-unit cost this attempt accrued\n * (the upstream's resolved table \u2014 vendor defaults merged with config\n * overrides; vendors with no table default to a flat 1 credit per\n * request). 0 when the attempt provably never dialed the vendor\n * (skipped / breaker-open) or the vendor was opted out (\"*\": 0).\n */\n creditunits: number /* int64 */;\n}\n/**\n * ExecState centralizes the per-request execution counters and the\n * per-upstream attempt log. Created lazily on first access via\n * (*NormalizedRequest).ExecState().\n * All counters are atomic; the struct itself is safe for concurrent use.\n * Counter model \u2014 every executor increments its OWN scope only.\n * Snapshot derives the totals so \"forgot to increment the total\" is\n * impossible by construction. The derivation is NOT a flat sum because\n * the scopes are nested: each network rotation triggers exactly one\n * upstream invocation chain, so summing both would double-count\n * physical attempts.\n * \ttotal Attempts = UpstreamAttempts + CacheAttempts\n * \t (every physical call is counted at the deepest scope that\n * \t actually performed it \u2014 upstreams for HTTP, cache for connector\n * \t reads. NetworkAttempts is a separate rotation-count signal,\n * \t exposed as its own counter but NOT summed into the total.)\n * \ttotal Retries = sum of UpstreamRetries + NetworkRetries + CacheRetries\n * \ttotal Hedges = sum of UpstreamHedges + NetworkHedges + CacheHedges\n * \t (retries and hedges ARE different events at each scope \u2014 an\n * \t upstream-scope retry retries the SAME upstream, a network-scope\n * \t retry rotates to a NEW upstream. Summing is correct.)\n * Scope semantics:\n * - UpstreamAttempts: physical Forward calls to a single upstream's\n * transport (primary + retries + hedges within one upstream).\n * - NetworkAttempts: rotations across upstreams driven by the\n * network executor's retry / hedge / consensus loop. Each rotation\n * triggers one upstream invocation chain. Not summed into total.\n * - CacheAttempts: cache-connector reads/writes including\n * within-connector retries and hedges.\n */\nexport interface ExecState {\n /**\n * Per-scope counters. Each executor owns its OWN counter set and\n * MUST NOT touch another scope's counters.\n */\n upstreamattempts: any /* atomic.Int32 */;\n upstreamretries: any /* atomic.Int32 */;\n upstreamhedges: any /* atomic.Int32 */;\n networkattempts: any /* atomic.Int32 */;\n networkretries: any /* atomic.Int32 */;\n networkhedges: any /* atomic.Int32 */;\n cacheattempts: any /* atomic.Int32 */;\n cacheretries: any /* atomic.Int32 */;\n cachehedges: any /* atomic.Int32 */;\n /**\n * ConsensusSlots counts how many consensus participants ran.\n */\n consensusslots: any /* atomic.Int32 */;\n /**\n * ConsensusDisputes counts dispute events.\n */\n consensusdisputes: any /* atomic.Int32 */;\n /**\n * ConsensusLowParticipants counts low-participant events.\n */\n consensuslowparticipants: any /* atomic.Int32 */;\n startedat: any /* time.Time */;\n}\n/**\n * ExecStateSnapshot is a plain-int view of ExecState for log/span\n * labeling \u2014 captured at a point in time. Total Attempts/Retries/Hedges\n * are derived as the sum of per-scope counters at snapshot time.\n */\nexport interface ExecStateSnapshot {\n /**\n * Totals (derived: Upstream + Network + Cache).\n */\n attempts: number /* int */;\n retries: number /* int */;\n hedges: number /* int */;\n /**\n * Per-scope counters (each executor's own bookkeeping).\n */\n upstreamattempts: number /* int */;\n upstreamretries: number /* int */;\n upstreamhedges: number /* int */;\n networkattempts: number /* int */;\n networkretries: number /* int */;\n networkhedges: number /* int */;\n cacheattempts: number /* int */;\n cacheretries: number /* int */;\n cachehedges: number /* int */;\n consensusslots: number /* int */;\n consensusdisputes: number /* int */;\n consensuslowparticipants: number /* int */;\n startedat: any /* time.Time */;\n}\n\n//////////\n// source: network.go\n\nexport type NetworkArchitecture = string;\nexport const ArchitectureEvm: NetworkArchitecture = \"evm\";\nexport const ArchitectureSvm: NetworkArchitecture = \"svm\";\nexport type Network = any;\n/**\n * EvmNetwork is the EVM-specific view of a Network. Callers that need\n * block-number accessors or leader-upstream selection should go through the\n * EvmHighestLatestBlockNumber / EvmHighestFinalizedBlockNumber / EvmLeaderUpstream\n * helpers below, which type-assert and degrade to zero-value on mismatch.\n */\nexport type EvmNetwork = \n Network;\n/**\n * SvmNetwork is the SVM-specific view of a Network. Production Network\n * implementations satisfy this automatically when the underlying network is\n * SVM; EVM networks correctly fail the assertion.\n */\nexport type SvmNetwork = \n Network;\nexport type QuantileTracker = any;\nexport type TrackedMetrics = any;\n\n//////////\n// source: served_tip.go\n\n/**\n * ServedTipInput is a single observation: an upstream's last-known tip block.\n * Callers are responsible for excluding syncing or cordoned upstreams BEFORE\n * passing observations here; the picker treats every input as a candidate.\n */\nexport interface ServedTipInput {\n /**\n * UpstreamID is preserved only for telemetry attribution. It is not used\n * in the pick.\n */\n upstreamid: string;\n /**\n * BlockNumber is the upstream's reported tip. Zero or negative values are\n * treated as \"no data yet\" and filtered before picking.\n */\n blocknumber: number /* int64 */;\n}\n/**\n * ServedTipPick is the picker's output.\n */\nexport interface ServedTipPick {\n /**\n * Tip is the value to advertise as latest/finalized: the highest block\n * number that a strict MAJORITY of the inputs have already reached, or 0\n * when there are no valid inputs.\n */\n tip: number /* int64 */;\n /**\n * Freshest is the freshest CORROBORATED view: the 2nd-highest valid input\n * (or the only input when N=1) \u2014 the reference for the deliberate-lag\n * gauge (Freshest - Tip). Using the 2nd-highest instead of the raw max\n * means a single rogue far-future upstream cannot inflate the lag gauge\n * (the problem the old velocity gate solved via MaxEligible: one\n * wrong-chain endpoint used to make the gauge read hundreds of thousands\n * of blocks). The absolute per-upstream maxima remain observable via\n * erpc_upstream_latest_block_number.\n */\n freshest: number /* int64 */;\n /**\n * Max is the highest valid input: the raw ceiling no pick may exceed. It is\n * deliberately not corroborated \u2014 a ceiling a single rogue can only RAISE\n * cannot be used to make a caller serve anything the rogue chose \u2014 and it\n * is the one place a lone far-ahead head is still visible to callers.\n */\n max: number /* int64 */;\n /**\n * Inputs is the number of valid (BlockNumber > 0) observations.\n */\n inputs: number /* int */;\n /**\n * Sorted is the valid inputs, DESCENDING by block number \u2014 the order\n * statistic's own working slice, exposed so the trajectory referee can\n * cluster the very same ballot without sorting it a second time. Nil when\n * there are no valid inputs; never mutate it.\n */\n sorted: ServedTipInput[];\n}\n\n//////////\n// source: timeout_func.go\n\n/**\n * TimeoutFunc computes the timeout for a request. Returns nil when no\n * timeout applies (caller skips context.WithTimeout).\n */\nexport type TimeoutFunc = any;\n\n//////////\n// source: upstream.go\n\nexport type Scope = string;\n/**\n * Policies must be created with a \"network\" in mind,\n * assuming there will be many upstreams e.g. Retry might endup using a different upstream\n */\nexport const ScopeNetwork: Scope = \"network\";\n/**\n * Policies must be created with one only \"upstream\" in mind\n * e.g. Retry with be towards the same upstream\n */\nexport const ScopeUpstream: Scope = \"upstream\";\nexport type UpstreamType = string;\n/**\n * HealthTracker is an interface for tracking upstream health metrics.\n * `finality` is the DataFinalityState the request resolved to \u2014\n * `DataFinalityStateAll` when the caller can't determine finality\n * (legacy call sites, internal probes, batch retries). Cordon/Uncordon\n * stay finality-agnostic because cordoning is an admin-level decision\n * about an entire (upstream, method) pair, not a per-finality bucket.\n */\nexport type HealthTracker = any;\nexport type Upstream = any;\n\n//////////\n// source: user.go\n\n/**\n * User is the authenticated caller. Beyond identity (Id) it carries the\n * per-caller capabilities resolved at authentication time. Capability fields\n * are populated ONLY by auth strategies; the trusted-header path\n * (NormalizedRequest.SetUserFromTrustedHeader) sets Id alone, so an\n * unvalidated header can never grant a capability.\n */\nexport interface User {\n id: string;\n ratelimitbudget: string;\n /**\n * AllowClientDirectives is the client-directive wildcard pattern granted by\n * the strategy that authenticated this user. Nil means \"no strategy-level\n * override\" \u2014 the project-level pattern applies.\n */\n allowclientdirectives?: string;\n}\n\n//////////\n// source: validation.go\n\n/**\n * MaxGrpcConnPoolSize is the upper bound accepted for a gRPC connection-pool\n * size. It guards against a fat-fingered value opening an absurd number of\n * connections to each backing server; it is not a recommended operating point.\n */\nexport const MaxGrpcConnPoolSize = 256;\n", "// CAPITAL_SNAKE_CASE aliases for selection-policy `evalScope` and\n// `stickyPrimary({scope})`. These are the canonical names a TypeScript\n// config imports:\n//\n// import { NETWORK, NETWORK_FINALITY } from '@erpc/config'\n//\n// The string values are the kebab-case forms the Go side validates\n// against (see `common.EvalScope`). At policy-runtime load time, the\n// stdlib also installs these names as ambient globals in the sobek\n// runtime so they resolve identically inside an `evalFunc` body\n// without an explicit import \u2014 see `internal/policy/stdlib/install.go`.\n\nimport {\n EvalScopeNetwork,\n EvalScopeNetworkMethod,\n EvalScopeNetworkFinality,\n EvalScopeNetworkMethodFinality,\n} from \"./generated\";\n\n/** One primary per network \u2014 max cohesion across methods + finalities. Default. */\nexport const NETWORK = EvalScopeNetwork;\n/** One primary per `(network, method)`; finalities share. */\nexport const NETWORK_METHOD = EvalScopeNetworkMethod;\n/** One primary per `(network, finality)`; methods share. */\nexport const NETWORK_FINALITY = EvalScopeNetworkFinality;\n/** One primary per slot \u2014 independent per `(network, method, finality)`. */\nexport const NETWORK_METHOD_FINALITY = EvalScopeNetworkMethodFinality;\n\n// Finality bit-flag constants \u2014 used INSIDE an `evalFunc` body with\n// the chainable `when(mask, fn)` primitive. Compose with bitwise OR:\n//\n// .when(REALTIME | UNFINALIZED | UNKNOWN, u => u.stickyPrimary({...}))\n//\n// These are also installed as ambient globals at policy-runtime load\n// time, so an `evalFunc` body doesn't need to import them. The\n// importable versions exist for symmetry \u2014 and so a TypeScript config\n// that wants to compute a mask AT CONFIG-LOAD TIME (e.g. into a\n// derived constant outside the eval body) can do so without relying\n// on the runtime ambient.\n/** Bit 0 \u2014 request is reading live tip data (e.g. eth_blockNumber). */\nexport const REALTIME = 1 << 0;\n/** Bit 1 \u2014 request is reading a recent-but-not-yet-finalized block. */\nexport const UNFINALIZED = 1 << 1;\n/** Bit 2 \u2014 request is reading a finalized (no-reorg) block. */\nexport const FINALIZED = 1 << 2;\n/** Bit 3 \u2014 finality couldn't be determined for this request. */\nexport const UNKNOWN = 1 << 3;\n"], - "mappings": ";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;AC+DO,IAAM,kBAAgC;AAkBtC,IAAM,qBAAkC;AACxC,IAAM,kBAA+B;AACrC,IAAM,qBAAkC;AAExC,IAAM,yBAA0C;AAChD,IAAM,yBAA0C;AAChD,IAAM,4BAA6C;AAkDnD,IAAM,kBAAgC;AAimBtC,IAAM,4BAAgD;AAQtD,IAAM,2BAA+C;AA4ZrD,IAAM,8CAAgF;AACtF,IAAM,8DAAgG;AACtG,IAAM,wDAA0F;AAChG,IAAM,sDAAwF;AAE9F,IAAM,sCAAgE;AACtE,IAAM,sDAAgF;AACtF,IAAM,gDAA0E;AAChF,IAAM,8CAAwE;AA0L9E,IAAM,wBAAyC;AAC/C,IAAM,wBAAyC;AAC/C,IAAM,sBAAuC;AAC7C,IAAM,qBAAsC;AAC5C,IAAM,sBAAuC;AAC7C,IAAM,uBAAwC;AAC9C,IAAM,sBAAuC;AA0Z7C,IAAM,mBAA8B;AAMpC,IAAM,yBAAoC;AAM1C,IAAM,2BAAsC;AAO5C,IAAM,iCAA4C;AAqDlD,IAAM,iBAA2B;AAEjC,IAAM,cAAwB;AAC9B,IAAM,eAAyB;AAC/B,IAAM,kBAA4B;AAmclC,IAAM,6BAAgD;AAItD,IAAM,+BAAkD;AAKxD,IAAM,4BAA+C;AAKrD,IAAM,2BAA8C;AAYpD,IAAM,2BAA+C;AACrD,IAAM,0BAA8C;AACpD,IAAM,yBAA6C;AAwLnD,IAAM,kBAAuC;AAC7C,IAAM,kBAAuC;AAkG7C,IAAM,eAAsB;AAK5B,IAAM,gBAAuB;;;ACviF7B,IAAM,UAAU;AAEhB,IAAM,iBAAiB;AAEvB,IAAM,mBAAmB;AAEzB,IAAM,0BAA0B;AAchC,IAAM,WAAW,KAAK;AAEtB,IAAM,cAAc,KAAK;AAEzB,IAAM,YAAY,KAAK;AAEvB,IAAM,UAAU,KAAK;;;AF+HrB,IAAM,eAAe,CAC1B,QACW;AACX,SAAO;AACT;", + "sourcesContent": ["export type {\n // Tygo generic replacement\n LogLevel,\n Duration,\n ByteSize,\n NetworkArchitecture,\n ConnectorDriverType,\n ConnectorConfig,\n UpstreamType,\n // Policy evaluation\n PolicyEvalUpstreamMetrics,\n PolicyEvalUpstream,\n PolicyEvalUpstreamArray,\n PolicyEvalContext,\n PolicyEvalPredicate,\n EvalContext,\n SelectionPolicyEvalFunction,\n ScoreWeights,\n ScoreBreakdown,\n TagPattern,\n Pattern,\n SortByScoreOptions,\n RemoveByLatencyOptions,\n RemoveByLagOptions,\n KeepHealthyOptions,\n PreferOptions,\n StickyPrimaryOptions,\n ProbeExcludedOptions,\n LatencyDeviationOptions,\n ByFinalityHandlers,\n WhereFilter,\n PolicyEvalArrayCondition,\n IncludeIfTarget,\n EvmNetworkConfigForDefaults,\n SvmNetworkConfigForDefaults,\n} from \"./types\";\nexport {\n // Data finality const exports\n DataFinalityStateUnfinalized,\n DataFinalityStateFinalized,\n DataFinalityStateRealtime,\n DataFinalityStateUnknown,\n // Scope exports\n ScopeNetwork,\n ScopeUpstream,\n // Selection-policy evalScope + stickyPrimary scope constants (canonical)\n EvalScopeNetwork,\n EvalScopeNetworkMethod,\n EvalScopeNetworkFinality,\n EvalScopeNetworkMethodFinality,\n // Cache behavior exports\n CacheEmptyBehaviorIgnore,\n CacheEmptyBehaviorAllow,\n CacheEmptyBehaviorOnly,\n // Evm node type\n EvmNodeTypeFull,\n EvmNodeTypeArchive,\n EvmNodeTypeUnknown,\n // Evm syncing type\n EvmSyncingStateUnknown,\n EvmSyncingStateSyncing,\n EvmSyncingStateNotSyncing,\n // Architecture export\n ArchitectureEvm,\n ArchitectureSvm,\n // Upstream types const exprots\n UpstreamTypeEvm,\n UpstreamTypeSvm,\n // Auth types\n AuthTypeSecret,\n AuthTypeJwt,\n AuthTypeSiwe,\n AuthTypeNetwork,\n // Consensus related\n ConsensusLowParticipantsBehaviorReturnError,\n ConsensusLowParticipantsBehaviorAcceptMostCommonValidResult,\n ConsensusLowParticipantsBehaviorPreferBlockHeadLeader,\n ConsensusLowParticipantsBehaviorOnlyBlockHeadLeader,\n ConsensusDisputeBehaviorReturnError,\n ConsensusDisputeBehaviorAcceptMostCommonValidResult,\n ConsensusDisputeBehaviorPreferBlockHeadLeader,\n ConsensusDisputeBehaviorOnlyBlockHeadLeader,\n // Rate limiter periods\n RateLimitPeriodSecond,\n RateLimitPeriodMinute,\n RateLimitPeriodHour,\n RateLimitPeriodDay,\n RateLimitPeriodWeek,\n RateLimitPeriodMonth,\n RateLimitPeriodYear,\n // Rate limiter count modes\n RateLimitCountModeRequest,\n RateLimitCountModeCredit,\n} from \"./generated\";\n// Short-name re-exports for selection-policy evalScope / stickyPrimary\n// scope (CAPITAL_SNAKE_CASE matching the JS ambient globals installed\n// by the policy stdlib) and the finality bit-flags for `when(mask, ...)`.\nexport {\n NETWORK,\n NETWORK_METHOD,\n NETWORK_FINALITY,\n NETWORK_METHOD_FINALITY,\n REALTIME,\n UNFINALIZED,\n FINALIZED,\n UNKNOWN,\n} from \"./constants\";\nexport type {\n Config,\n ProjectConfig,\n HealthCheckConfig,\n // Provider related\n ProviderConfig,\n VendorSettings,\n // Upstream related\n UpstreamConfig,\n EvmUpstreamConfig,\n EvmQueryShimConfig,\n SvmUpstreamConfig,\n UpstreamIntegrityConfig,\n UpstreamIntegrityEthGetBlockReceiptsConfig,\n RateLimitAutoTuneConfig,\n JsonRpcUpstreamConfig,\n // Failsafe related\n FailsafeConfig,\n RetryPolicyConfig,\n CircuitBreakerPolicyConfig,\n HedgePolicyConfig,\n TimeoutPolicyConfig,\n ConsensusPolicyConfig,\n // Network related\n NetworkConfig,\n EvmNetworkConfig,\n EvmIntegrityConfig,\n SvmNetworkConfig,\n SelectionPolicyConfig,\n EvalScope,\n DirectiveDefaultsConfig,\n // DB related\n DatabaseConfig,\n CacheConfig,\n DataFinalityState,\n CacheEmptyBehavior,\n CachePolicyConfig,\n MemoryConnectorConfig,\n RedisConnectorConfig,\n DynamoDBConnectorConfig,\n AwsAuthConfig,\n PostgreSQLConnectorConfig,\n // Auth related\n AuthStrategyConfig,\n SecretStrategyConfig,\n JwtStrategyConfig,\n SiweStrategyConfig,\n NetworkStrategyConfig,\n // Rate limits related\n RateLimiterConfig,\n RateLimitBudgetConfig,\n RateLimitRuleConfig,\n // Server config related\n ServerConfig,\n CORSConfig,\n MetricsConfig,\n AdminConfig,\n AliasingConfig,\n AliasingRuleConfig,\n TLSConfig,\n // Proxy pools related\n ProxyPoolConfig,\n} from \"./generated\";\n\nimport type { Config } from './generated'\n\nexport const createConfig = (\n cfg: Config\n): Config => {\n return cfg;\n};\n", "// Code generated by tygo. DO NOT EDIT.\nimport type {\n LogLevel,\n Duration,\n ByteSize,\n ConnectorDriverType as TsConnectorDriverType,\n ConnectorConfig as TsConnectorConfig,\n UpstreamType as TsUpstreamType,\n NetworkArchitecture as TsNetworkArchitecture,\n AuthType as TsAuthType,\n AuthStrategyConfig as TsAuthStrategyConfig,\n EvmNetworkConfigForDefaults as TsEvmNetworkConfigForDefaults,\n SvmNetworkConfigForDefaults as TsSvmNetworkConfigForDefaults,\n SelectionPolicyEvalFunction,\n BoolOrString\n} from \"./types\"\n\n//////////\n// source: adaptive_duration.go\n\n/**\n * AdaptiveDuration describes a duration that may be static, derived from a\n * per-method latency quantile, or both. It's the reusable building block\n * for any failsafe knob that wants \"fixed base + adaptive component\n * clamped between min/max\" semantics \u2014 currently consensus wait caps,\n * with timeout/hedge supporting it as an alternative entry-point.\n * Resolution rules:\n * \tfinal = Base + adaptive\n * where `adaptive` is:\n * - `qt.GetQuantile(Quantile)` when Quantile > 0 and quantile data exists\n * - `Min` (the floor) when Quantile > 0 but quantile data is cold (no\n * observations yet) \u2014 this gives a sensible non-zero cap immediately\n * after boot\n * - `0` when Quantile is unset\n * After `Base + adaptive`, the result is clamped to [Min, Max] when those\n * are set. A nil or all-zero AdaptiveDuration returns 0 (the caller treats\n * that as \"no cap\" / \"disabled\").\n * Wire format accepts both shorthand and object form:\n * \tcaps: 500ms # shorthand: Base only\n * \tcaps: { base: 500ms } # explicit Base\n * \tcaps: { quantile: 0.5, min: 5ms, max: 1s } # quantile with bounds\n * \tcaps: { base: 100ms, quantile: 0.9, max: 2s } # combined\n */\nexport interface AdaptiveDuration {\n base?: Duration;\n quantile?: number /* float64 */;\n min?: Duration;\n max?: Duration;\n}\n\n//////////\n// source: architecture.go\n\n/**\n * ArchitectureHandler abstracts all architecture-specific pipeline hooks.\n * Each architecture (evm, svm, \u2026) registers exactly one handler via its package init().\n * Pipeline files dispatch through the registry \u2014 they never need to change for a new chain.\n */\nexport type ArchitectureHandler = any;\n\n//////////\n// source: architecture_evm.go\n\nexport const UpstreamTypeEvm: UpstreamType = \"evm\";\nexport type EvmUpstream = \n Upstream;\n/**\n * EvmStateProvenWriter is the OPTIONAL, separately-asserted surface the\n * integrity state prober records proofs through. The proven head is telemetry\n * (the state-proven block / proven-lag gauges), never a routing input.\n * Deliberately NOT part of EvmUpstream: that interface is implemented outside\n * this repo, and widening it broke every existing implementor \u2014 the chainId\n * suggest-gate silently degraded when its upstream stopped satisfying the\n * assertion. Optional capabilities are asserted narrowly, never added to the\n * core interface.\n */\nexport type EvmStateProvenWriter = any;\nexport type AvailbilityConfidence = number /* int */;\nexport const AvailbilityConfidenceBlockHead: AvailbilityConfidence = 1;\nexport const AvailbilityConfidenceFinalized: AvailbilityConfidence = 2;\nexport type EvmNodeType = string;\nexport const EvmNodeTypeUnknown: EvmNodeType = \"unknown\";\nexport const EvmNodeTypeFull: EvmNodeType = \"full\";\nexport const EvmNodeTypeArchive: EvmNodeType = \"archive\";\nexport type EvmSyncingState = number /* int */;\nexport const EvmSyncingStateUnknown: EvmSyncingState = 0;\nexport const EvmSyncingStateSyncing: EvmSyncingState = 1;\nexport const EvmSyncingStateNotSyncing: EvmSyncingState = 2;\nexport type EvmStatePoller = any;\n/**\n * EvmStatePollerDiagnostics contains diagnostic information about the state poller\n * including block bounds, probe status, and any detection issues.\n */\nexport interface EvmStatePollerDiagnostics {\n enabled: boolean;\n /**\n * Block head information\n */\n latestBlock: number /* int64 */;\n finalizedBlock: number /* int64 */;\n /**\n * Syncing state\n */\n syncingState: string;\n skipSyncingCheck?: boolean;\n syncingCheckError?: string;\n /**\n * Latest block detection status\n */\n skipLatestBlockCheck?: boolean;\n latestBlockFailureCount?: number /* int */;\n latestBlockSuccessfulOnce?: boolean;\n latestBlockDetectionIssue?: string;\n /**\n * Finalized block detection status\n */\n skipFinalizedCheck?: boolean;\n finalizedBlockFailureCount?: number /* int */;\n finalizedBlockSuccessfulOnce?: boolean;\n finalizedBlockDetectionIssue?: string;\n /**\n * Earliest block bounds per probe type\n */\n earliestByProbe?: { [key: EvmAvailabilityProbeType]: EvmProbeEarliestInfo | undefined};\n}\n/**\n * EvmProbeEarliestInfo contains information about earliest block detection for a specific probe type\n */\nexport interface EvmProbeEarliestInfo {\n probeType: EvmAvailabilityProbeType;\n earliestBlock: number /* int64 */;\n schedulerRunning?: boolean;\n}\n\n//////////\n// source: architecture_svm.go\n\nexport const UpstreamTypeSvm: UpstreamType = \"svm\";\n/**\n * SlotSharedVariable mirrors the EVM SharedStateVariable pattern for int64 slot numbers.\n * It lets the state poller publish slot progress to a shared-state backend (memory or Redis)\n * while keeping callers synchronous via the cached local value.\n */\nexport type SlotSharedVariable = any;\n/**\n * SvmUpstream narrows common.Upstream to the surface SVM hooks need to reach\n * into per-upstream state (currently just the state poller). Keeping this a\n * tiny sub-interface avoids committing the full Upstream struct to common/ and\n * parallels the existing EvmUpstream pattern.\n */\nexport type SvmUpstream = \n Upstream;\n/**\n * SvmStatePoller is the per-upstream slot/health tracker. Concrete type lives in\n * architecture/svm to avoid pulling the full implementation into common.\n */\nexport type SvmStatePoller = any;\n/**\n * MaxShredInsertSlotLagThreshold is the cutoff beyond which an upstream is treated\n * as degraded for shred-insert lag. Solana nodes with high `latest - maxShredInsertSlot`\n * are receiving shreds but not processing them; their reads go stale silently.\n */\nexport const MaxShredInsertSlotLagThreshold: number /* int64 */ = 100;\n/**\n * SvmChainSolana is the canonical Solana chain identifier. Empty Chain values\n * in SvmNetworkConfig normalize to this constant at NetworkId derivation time\n * so pre-multi-chain configs keep producing \"svm:\" IDs.\n */\nexport const SvmChainSolana = \"solana\";\n/**\n * SvmHistory is the slots from which a cluster's Agave history records block\n * fields. Below them Agave renders the field null whatever an archive holds;\n * a nil bound means the cluster records the field from its first slot.\n */\nexport interface SvmHistory {\n blocktimefrom?: number /* uint64 */;\n blockheightfrom?: number /* uint64 */;\n}\n\n//////////\n// source: blocktime_adaptive_duration.go\n\n/**\n * BlockTimeAdaptiveDuration is a duration that is either a fixed value or derived from the\n * network's estimated block time. It's the reusable shape for knobs (e.g. cache\n * TTLs) that should track each chain's cadence rather than a single constant.\n * Wire format accepts a scalar shorthand or an object:\n * \tttl: 2s # fixed\n * \tttl: { blockTimeMultiplier: 1 } # blockTime * 1 (caller default until known)\n * \tttl: { blockTimeMultiplier: 1, fallback: 2s } # with explicit cold-start fallback\n * See Resolve for how the value is computed.\n */\nexport interface BlockTimeAdaptiveDuration {\n /**\n * Fallback is the fixed value. A scalar shorthand sets it directly; in object\n * form it's the cold-start floor used until the block time is known.\n */\n fallback?: Duration;\n /**\n * BlockTimeMultiplier, when > 0, derives the value from the network's\n * estimated block time (blockTime * multiplier).\n */\n blockTimeMultiplier?: number /* float64 */;\n}\n\n//////////\n// source: cache_dal.go\n\nexport type CacheDAL = any;\n\n//////////\n// source: cache_mock.go\n\nexport interface MockCacheDal {\n mock: any /* mock.Mock */;\n}\n\n//////////\n// source: config.go\n\n/**\n * Config represents the configuration of the application.\n */\nexport interface Config {\n logLevel?: LogLevel;\n clusterKey?: string;\n server?: ServerConfig;\n healthCheck?: HealthCheckConfig;\n admin?: AdminConfig;\n database?: DatabaseConfig;\n projects?: (ProjectConfig | undefined)[];\n rateLimiters?: RateLimiterConfig;\n metrics?: MetricsConfig;\n indexer?: IndexerConfig;\n proxyPools?: (ProxyPoolConfig | undefined)[];\n tracing?: TracingConfig;\n}\n/**\n * IndexerConfig tunes the event-stream indexer behind eth_subscribe fan-out.\n */\nexport interface IndexerConfig {\n /**\n * Deprecated: ignored; the indexer no longer tracks a canonical chain.\n * Kept so configs that still set it load under strict decoding.\n */\n canonicalChainDepth?: number /* int */;\n /**\n * DedupWindowSize is the per-filter seen-set capacity for log and\n * pending-tx fan-out across upstreams. 0 uses the default (8192).\n */\n dedupWindowSize?: number /* int */;\n}\nexport interface ServerConfig {\n listenV4?: boolean;\n httpHostV4?: string;\n listenV6?: boolean;\n httpHostV6?: string;\n httpPort?: number /* int */; // Deprecated: use HttpPortV4\n httpPortV4?: number /* int */;\n httpPortV6?: number /* int */;\n grpcEnabled?: boolean;\n grpcHostV4?: string;\n grpcPortV4?: number /* int */;\n grpcHostV6?: string;\n grpcPortV6?: number /* int */;\n grpcMaxRecvMsgSize?: number /* int */;\n grpcMaxSendMsgSize?: number /* int */;\n grpcReflection?: boolean;\n maxTimeout?: Duration;\n readTimeout?: Duration;\n writeTimeout?: Duration;\n enableGzip?: boolean;\n tls?: TLSConfig;\n aliasing?: AliasingConfig;\n waitBeforeShutdown?: Duration;\n waitAfterShutdown?: Duration;\n includeErrorDetails?: boolean;\n trustedIPForwarders?: string[];\n trustedIPHeaders?: string[];\n responseHeaders?: { [key: string]: string};\n webSocket?: WebSocketServerConfig;\n /**\n * ExecutionHeaders controls the per-request diagnostic headers\n * (X-ERPC-Attempts, X-ERPC-Upstreams-Tried, etc.) that expose how\n * eRPC routed and resolved each request. Defaults to \"all\" \u2014 set\n * \"summary\" to keep only counters, or \"off\" to disable entirely\n * (useful for low-latency / bandwidth-constrained clients).\n */\n executionHeaders?: ExecutionHeadersMode;\n /**\n * CostHeaders opts into the cost/billing response headers\n * (X-ERPC-Calls, X-ERPC-Billable, X-ERPC-Methods, X-ERPC-Credits,\n * X-ERPC-Credits-Version) on single and batch responses. Off by\n * default. Credit-unit pricing is vendor-level configuration \u2014 see\n * CreditUnitsProvider and UpstreamConfig.CreditUnits.\n */\n costHeaders?: boolean;\n}\nexport interface WebSocketServerConfig {\n /**\n * Enabled accepts WebSocket upgrades on the HTTP port. Off by default:\n * with it off an upgrade request is ordinary HTTP.\n */\n enabled?: boolean;\n readBufferSize?: number /* int */;\n writeBufferSize?: number /* int */;\n maxMessageSize?: number /* int64 */;\n pingInterval?: Duration;\n maxSubscriptionsPerConnection?: number /* int */;\n /**\n * MaxConcurrentRequestsPerConnection bounds the messages a connection has\n * in flight; further frames wait (backpressure) instead of being rejected.\n * A batch is one message; its requests run concurrently, as over HTTP.\n */\n maxConcurrentRequestsPerConnection?: number /* int */;\n /**\n * SubscriptionBufferSize is the number of notifications queued per\n * subscription for a client that is behind. On overflow newHeads and\n * newPendingTransactions drop their oldest entry; logs close the\n * connection instead (1013), so the client knows it missed data.\n */\n subscriptionBufferSize?: number /* int */;\n /**\n * MaxConnections caps concurrent client connections on this instance\n * (0 = unlimited). Unlike the per-IP cap it holds when client IPs are\n * unknown or spoofed.\n */\n maxConnections?: number /* int */;\n /**\n * MaxConnectionsPerIP caps concurrent client connections from one\n * resolved client IP (0 = unlimited); see trustedIPForwarders.\n */\n maxConnectionsPerIP?: number /* int */;\n /**\n * MaxSubscriptions caps active subscriptions across all connections on\n * this instance (0 = unlimited).\n */\n maxSubscriptions?: number /* int */;\n}\n/**\n * ExecutionHeadersMode controls how much per-request execution detail is\n * exposed in HTTP response headers.\n */\nexport type ExecutionHeadersMode = string;\n/**\n * ExecutionHeadersAll emits the full set: counters + per-upstream\n * trace (upstream IDs, outcomes, reasons, durations). Default.\n */\nexport const ExecutionHeadersAll: ExecutionHeadersMode = \"all\";\n/**\n * ExecutionHeadersSummary emits only the counter triplet\n * (X-ERPC-Attempts/Retries/Hedges) + the cache-hit / final-upstream\n * markers. Skips the (potentially large) per-attempt slice headers.\n */\nexport const ExecutionHeadersSummary: ExecutionHeadersMode = \"summary\";\n/**\n * ExecutionHeadersOff disables all X-ERPC-* diagnostic headers.\n */\nexport const ExecutionHeadersOff: ExecutionHeadersMode = \"off\";\nexport interface HealthCheckConfig {\n mode?: HealthCheckMode;\n auth?: AuthConfig;\n defaultEval?: string;\n}\nexport type HealthCheckMode = string;\nexport const HealthCheckModeSimple: HealthCheckMode = \"simple\";\nexport const HealthCheckModeNetworks: HealthCheckMode = \"networks\";\nexport const HealthCheckModeVerbose: HealthCheckMode = \"verbose\";\nexport const EvalAnyInitializedUpstreams = \"any:initializedUpstreams\";\nexport const EvalAnyErrorRateBelow90 = \"any:errorRateBelow90\";\nexport const EvalAllErrorRateBelow90 = \"all:errorRateBelow90\";\nexport const EvalAnyErrorRateBelow100 = \"any:errorRateBelow100\";\nexport const EvalAllErrorRateBelow100 = \"all:errorRateBelow100\";\nexport const EvalEvmAnyChainId = \"any:evm:eth_chainId\";\nexport const EvalEvmAllChainId = \"all:evm:eth_chainId\";\nexport const EvalAllActiveUpstreams = \"all:activeUpstreams\";\nexport type TracingProtocol = string;\nexport const TracingProtocolHttp: TracingProtocol = \"http\";\nexport const TracingProtocolGrpc: TracingProtocol = \"grpc\";\nexport interface TracingConfig {\n enabled?: boolean;\n endpoint?: string;\n protocol?: TracingProtocol;\n sampleRate?: number /* float64 */;\n detailed?: boolean;\n serviceName?: string;\n headers?: { [key: string]: string};\n tls?: TLSConfig;\n resourceAttributes?: { [key: string]: string};\n /**\n * ForceTraceMatchers defines conditions for force-tracing requests.\n * Each matcher can specify network and/or method patterns.\n * Multiple patterns can be separated by \"|\" (OR within field).\n * Both network and method must match if both are specified (AND between fields).\n * If only one field is specified, only that field is checked.\n */\n forceTraceMatchers?: (ForceTraceMatcher | undefined)[];\n}\n/**\n * ForceTraceMatcher defines a condition for force-tracing requests.\n */\nexport interface ForceTraceMatcher {\n /**\n * Network patterns to match (e.g., \"evm:1\", \"evm:1|evm:42161\", \"evm:*\")\n * Multiple patterns separated by \"|\" act as OR conditions.\n */\n network?: string;\n /**\n * Method patterns to match (e.g., \"eth_call\", \"debug_*|trace_*\")\n * Multiple patterns separated by \"|\" act as OR conditions.\n */\n method?: string;\n}\nexport interface AdminConfig {\n auth?: AuthConfig;\n cors?: CORSConfig;\n allowMethods?: string[];\n denyMethods?: string[];\n}\nexport interface AliasingConfig {\n rules: (AliasingRuleConfig | undefined)[];\n}\nexport interface AliasingRuleConfig {\n matchDomain: string;\n serveProject: string;\n serveArchitecture: string;\n serveChain: string;\n}\nexport interface DatabaseConfig {\n evmJsonRpcCache?: CacheConfig;\n svmJsonRpcCache?: CacheConfig;\n sharedState?: SharedStateConfig;\n}\nexport interface SharedStateConfig {\n /**\n * ClusterKey identifies the logical group for shared counters across replicas (multi-tenant friendly)\n */\n clusterKey?: string;\n /**\n * Connector contains the storage driver configuration (redis, postgresql, dynamodb, memory)\n */\n connector?: ConnectorConfig;\n /**\n * FallbackTimeout is the timeout for remote storage operations (get/set/publish).\n * It is a seconds-scale network timeout and NOT a foreground latency budget.\n */\n fallbackTimeout?: Duration;\n /**\n * LockTtl is the expiration for the distributed lock key in the backing store.\n * Should comfortably exceed the expected duration of remote writes.\n */\n lockTtl?: Duration;\n /**\n * LockMaxWait caps how long the foreground path will wait to acquire the lock\n * before proceeding locally and deferring the remote write to background.\n */\n lockMaxWait?: Duration;\n /**\n * UpdateMaxWait caps how long the foreground path will spend computing a new value\n * (e.g., polling latest block) before returning the current local value.\n */\n updateMaxWait?: Duration;\n}\nexport interface CacheConfig {\n connectors?: TsConnectorConfig[];\n policies?: (CachePolicyConfig | undefined)[];\n compression?: CompressionConfig;\n}\nexport interface CompressionConfig {\n enabled?: boolean;\n algorithm?: string; // \"zstd\" for now, can be extended\n zstdLevel?: string; // \"fastest\", \"default\", \"better\", \"best\"\n threshold?: number /* int */; // Minimum size in bytes to compress\n}\nexport interface CacheMethodConfig {\n reqRefs: any[][];\n respRefs: any[][];\n finalized: boolean;\n realtime: boolean;\n stateful?: boolean;\n /**\n * TranslateLatestTag controls whether the method-level tag translation should convert \"latest\" to a concrete hex block number.\n * When nil or true, translation is enabled by default.\n */\n translateLatestTag?: boolean;\n /**\n * TranslateFinalizedTag controls whether the method-level tag translation should convert \"finalized\" to a concrete hex block number.\n * When nil or true, translation is enabled by default.\n */\n translateFinalizedTag?: boolean;\n /**\n * EnforceBlockAvailability controls whether per-upstream block availability bounds (upper/lower)\n * are enforced for this method at the network level. When nil or true, enforcement is enabled.\n */\n enforceBlockAvailability?: boolean;\n}\nexport interface CachePolicyConfig {\n connector: string;\n network?: string;\n method?: string;\n params?: any[];\n finality?: DataFinalityState;\n empty?: CacheEmptyBehavior;\n appliesTo?: 'get' | 'set' | 'both';\n minItemSize?: ByteSize;\n maxItemSize?: ByteSize;\n /**\n * TTL is either a fixed duration (\"2s\") or, in object form, derived from the\n * network's estimated block time ({ blockTimeMultiplier: 1, fallback: 2s }).\n * For realtime finality the resolved value is the age limit; the fixed/\n * fallback component is also used as the cache storage expiry. See\n * BlockTimeAdaptiveDuration.\n */\n ttl?: Duration | BlockTimeAdaptiveDuration;\n}\nexport type ConnectorDriverType = string;\nexport const DriverMemory: ConnectorDriverType = \"memory\";\nexport const DriverRedis: ConnectorDriverType = \"redis\";\nexport const DriverPostgreSQL: ConnectorDriverType = \"postgresql\";\nexport const DriverDynamoDB: ConnectorDriverType = \"dynamodb\";\nexport const DriverGrpc: ConnectorDriverType = \"grpc\";\nexport interface ConnectorConfig {\n id?: string;\n driver: TsConnectorDriverType;\n /**\n * Tags label the data source a connector caches, so the `use-upstream`\n * directive can gate which connector is read/written \u2014 e.g. a connector\n * fed by a system-transaction node tagged `systx` is only used when the\n * request's selector admits it. Same glob/`!negation` grammar as upstream\n * tags. Untagged connectors are always eligible (a use-upstream pin meant\n * for upstreams must not disable a normal cache).\n */\n tags?: string[];\n memory?: MemoryConnectorConfig;\n redis?: RedisConnectorConfig;\n dynamodb?: DynamoDBConnectorConfig;\n postgresql?: PostgreSQLConnectorConfig;\n grpc?: GrpcConnectorConfig;\n failsafeForGets?: (FailsafeConfig | undefined)[];\n failsafeForSets?: (FailsafeConfig | undefined)[];\n}\nexport interface GrpcConnectorConfig {\n bootstrap?: string;\n servers?: string[];\n headers?: { [key: string]: string};\n getTimeout?: Duration;\n /**\n * NetworkId pins every server in this connector to one network and skips\n * the eth_chainId bootstrap probe.\n * Required for SVM. Solana has no numeric chain id \u2014 cluster identity is\n * the genesis hash \u2014 and eth_chainId returns Unimplemented, so the probe\n * fails and no client is ever registered. Probing GetGenesisHash instead\n * is not an option either: the BDS reader deliberately does not publish\n * one until it can verify its own, so identity here has to be asserted by\n * configuration rather than discovered.\n * Leave unset for EVM to keep the probe, which also arms the per-request\n * chain-identity assertion that catches an endpoint cross-wired LATER \u2014 a\n * static binding cannot.\n */\n networkId?: string;\n /**\n * PoolSize is the number of independent gRPC connections opened to each\n * backing server, selected round-robin per request. Larger values raise the\n * concurrent-stream ceiling and shrink the blast radius of a single wedged\n * connection, at the cost of more open connections per server. When unset\n * (0) a built-in default is used.\n */\n poolSize?: number /* int */;\n /**\n * HealthCheckService is a grpc.health.v1 service name. When set, each\n * connection watches it on every resolved address and sends requests only\n * to addresses reporting SERVING. Empty (default) disables health checking.\n */\n healthCheckService?: string;\n}\nexport interface MemoryConnectorConfig {\n maxItems: number /* int */;\n maxTotalSize: string;\n emitMetrics?: boolean;\n}\nexport interface MockConnectorConfig {\n memoryconnectorconfig: MemoryConnectorConfig;\n getdelay: number /* time in nanoseconds (time.Duration) */;\n setdelay: number /* time in nanoseconds (time.Duration) */;\n geterrorrate: number /* float64 */;\n seterrorrate: number /* float64 */;\n}\nexport interface TLSConfig {\n enabled: boolean;\n certFile: string;\n keyFile: string;\n caFile?: string;\n insecureSkipVerify?: boolean;\n}\nexport interface RedisConnectorConfig {\n addr?: string;\n username?: string;\n db?: number /* int */;\n tls?: TLSConfig;\n connPoolSize?: number /* int */;\n uri: string;\n initTimeout?: Duration;\n getTimeout?: Duration;\n setTimeout?: Duration;\n lockRetryInterval?: Duration;\n iamAuth?: RedisIAMAuthConfig;\n}\nexport interface DynamoDBConnectorConfig {\n table?: string;\n region?: string;\n endpoint?: string;\n auth?: AwsAuthConfig;\n partitionKeyName?: string;\n rangeKeyName?: string;\n reverseIndexName?: string;\n ttlAttributeName?: string;\n initTimeout?: Duration;\n getTimeout?: Duration;\n setTimeout?: Duration;\n maxRetries?: number /* int */;\n statePollInterval?: Duration;\n lockRetryInterval?: Duration;\n}\nexport interface PostgreSQLConnectorConfig {\n connectionUri: string;\n table: string;\n minConns?: number /* int32 */;\n maxConns?: number /* int32 */;\n initTimeout?: Duration;\n getTimeout?: Duration;\n setTimeout?: Duration;\n iamAuth?: PostgreSQLIAMAuthConfig;\n /**\n * SkipSchemaSetup skips all startup DDL (CREATE TABLE/INDEX, column\n * migrations, pg_cron) and the local expired-row cleanup DELETE loop. Set\n * it for connectors whose ConnectionUri targets a read-only replica (e.g.\n * an Aurora global-database secondary): DDL cannot execute there (SQLSTATE\n * 25006) and is not write-forwarded, so the writer-region connector owns\n * the schema and the replica receives it via storage replication.\n */\n skipSchemaSetup?: boolean;\n}\nexport interface AwsAuthConfig {\n mode: 'file' | 'env' | 'secret'; // \"file\", \"env\", \"secret\"\n credentialsFile: string;\n profile: string;\n accessKeyID: string;\n secretAccessKey: string;\n}\n/**\n * RedisIAMAuthConfig enables AWS IAM authentication for ElastiCache (Valkey \u22657.2\n * or Redis OSS \u22657.0). When enabled, eRPC mints SigV4-presigned auth tokens via\n * go-redis's CredentialsProviderContext on every new connection. TLS is required\n * (auto-enabled by SetDefaults). For IAM-enabled ElastiCache users, the user\n * name and user ID must be identical \u2014 supply that single value as UserID.\n */\nexport interface RedisIAMAuthConfig {\n enabled: boolean;\n /**\n * CacheName is the ElastiCache replication-group ID.\n * Will be lowercased automatically (AWS lowercases cache names at creation time).\n */\n cacheName: string;\n /**\n * Region is optional \u2014 derived from AWS_REGION / instance metadata when omitted.\n */\n region?: string;\n userID: string;\n /**\n * Auth selects the AWS credential source (same shape as DynamoDB's auth).\n * Omit to use the default credential chain (instance role, env vars, \u2026).\n */\n auth?: AwsAuthConfig;\n}\n/**\n * PostgreSQLIAMAuthConfig enables AWS IAM authentication for RDS PostgreSQL.\n * eRPC mints SigV4-presigned tokens via pgxpool.BeforeConnect on each new pool\n * connection. SSL is required (auto-enforced via sslmode=require by SetDefaults).\n */\nexport interface PostgreSQLIAMAuthConfig {\n enabled: boolean;\n /**\n * Endpoint is host:port of the RDS instance. If empty, derived from\n * ConnectionUri at SetDefaults time.\n */\n endpoint?: string;\n /**\n * Region is optional \u2014 derived from AWS_REGION / instance metadata when omitted.\n */\n region?: string;\n /**\n * DBUser is the database user mapped to the IAM role (must be granted rds_iam\n * in PostgreSQL). If empty, derived from the user in ConnectionUri.\n */\n dbUser?: string;\n auth?: AwsAuthConfig;\n}\nexport interface ProjectConfig {\n id: string;\n auth?: AuthConfig;\n cors?: CORSConfig;\n providers?: (ProviderConfig | undefined)[];\n upstreamDefaults?: UpstreamConfig;\n upstreams?: (UpstreamConfig | undefined)[];\n networkDefaults?: NetworkDefaults;\n networks?: (NetworkConfig | undefined)[];\n rateLimitBudget?: string;\n /**\n * Configure user agent tracking at the project level\n */\n userAgentMode?: UserAgentTrackingMode;\n /**\n * TrustUserIdHeader makes erpc read the caller's user identity from the\n * X-ERPC-User-Id request header (see common.HeaderUserId) and use it for the\n * `user` metric/log label \u2014 but only when no auth strategy resolved a user\n * (auth wins) and only for attribution (no rate-limit budget is derived).\n * This is for deployments that authenticate callers in front of erpc (e.g. a\n * gateway) and want per-user erpc telemetry without erpc performing auth.\n * erpc does NOT validate the header, so enable this ONLY when erpc is reachable\n * solely by a trusted proxy that sets the header and strips any client copy \u2014\n * otherwise callers can spoof their own attribution. Default false.\n */\n trustUserIdHeader?: boolean;\n forwardHeaders?: string[];\n allowClientDirectives?: string;\n ignoreMethods?: string[];\n allowMethods?: string[];\n /**\n * Integrity is the project-wide data-integrity configuration. It applies to\n * all networks; each network may override it with its own integrity block.\n */\n integrity?: IntegrityConfig;\n /**\n * ScoreMetricsWindowSize is the tumbling window the per-upstream\n * health tracker uses for its rolling counters (errorRate, p50/p70/\n * p95 latency, throttledRate, misbehaviorRate). At each tick the\n * counters reset and start re-accumulating, so this knob effectively\n * controls how fast a degraded upstream's metrics start reflecting\n * the new reality. Short windows (e.g. 30s) react quickly but give\n * noisier ranking; long windows (5\u201310m) give stable averages but\n * hide spikes for longer. Defaults to 10m when zero \u2014 production\n * systems typically leave it default; the eRPC simulator overrides\n * to 30s so knob changes show up in the UI within seconds.\n */\n scoreMetricsWindowSize?: Duration;\n}\n/**\n * LegacyProjectFields collects the deprecated project-level scoring +\n * routing keys. The translator inspects these to synthesize a\n * `selectionPolicy.eval` for each network and to emit deprecation\n * warnings. All fields are zero-valued for a clean modern config.\n * `scoreMetricsWindowSize` was previously listed here as inert; it is\n * now a first-class field on ProjectConfig (above) \u2014 operators\n * control the health tracker window directly.\n */\nexport interface LegacyProjectFields {\n routingStrategy?: string;\n scoreGranularity?: string;\n scorePenaltyDecayRate?: number /* float64 */;\n scoreSwitchHysteresis?: number /* float64 */;\n scoreMinSwitchInterval?: Duration;\n scoreMetricsMode?: string;\n scoreRefreshInterval?: Duration;\n}\n/**\n * UserAgentTrackingMode controls how user agents are recorded for metrics/labels\n */\nexport type UserAgentTrackingMode = string;\n/**\n * UserAgentTrackingModeSimplified lowers cardinality by bucketing common user agents\n */\nexport const UserAgentTrackingModeSimplified: UserAgentTrackingMode = \"simplified\";\n/**\n * UserAgentTrackingModeRaw records the user agent string as-is (high cardinality)\n */\nexport const UserAgentTrackingModeRaw: UserAgentTrackingMode = \"raw\";\nexport interface NetworkDefaults {\n rateLimitBudget?: string;\n failsafe?: (FailsafeConfig | undefined)[];\n selectionPolicy?: SelectionPolicyConfig;\n directiveDefaults?: DirectiveDefaultsConfig;\n evm?: TsEvmNetworkConfigForDefaults;\n svm?: TsSvmNetworkConfigForDefaults;\n multiplexing?: boolean;\n cacheKeySuffix?: string;\n failover?: FailoverConfig;\n}\n/**\n * FailoverConfig controls per-request escalation to fallback-tier upstreams,\n * independent of the selection policy.\n */\nexport interface FailoverConfig {\n /**\n * OnDefaultsExhausted tries upstreams not tagged `tier:fallback` first and\n * advances to the fallback tier within the same request only once every\n * other upstream failed with an error retryable toward the network.\n */\n onDefaultsExhausted?: boolean;\n}\nexport interface CORSConfig {\n allowedOrigins: string[];\n allowedMethods: string[];\n allowedHeaders: string[];\n exposedHeaders: string[];\n allowCredentials?: boolean;\n maxAge: number /* int */;\n}\nexport type VendorSettings = { [key: string]: any};\nexport interface ProviderConfig {\n id?: string;\n vendor: string;\n settings?: VendorSettings;\n onlyNetworks?: string[];\n ignoreNetworks?: string[];\n upstreamIdTemplate?: string;\n overrides?: { [key: string]: UpstreamConfig | undefined};\n}\n/**\n * TagTierFallback marks an upstream as part of the fallback tier, used only\n * when the other upstreams are unavailable.\n */\nexport const TagTierFallback = \"tier:fallback\";\n/**\n * RateLimitCountMode selects the accounting unit an upstream's rate-limit\n * budget charges per call.\n */\nexport type RateLimitCountMode = string;\n/**\n * RateLimitCountModeRequest (default) charges a flat 1 hit per call,\n * regardless of method \u2014 the historical eRPC behavior.\n */\nexport const RateLimitCountModeRequest: RateLimitCountMode = \"request\";\n/**\n * RateLimitCountModeCredit charges the request's resolved vendor\n * credit-unit cost (the same table used for cost accounting), so a\n * heavy eth_getLogs consumes more budget than a cheap eth_blockNumber.\n * The pre-flight table estimate is used (the real cost is not known\n * until after the call); a 0-CU method consumes nothing.\n */\nexport const RateLimitCountModeCredit: RateLimitCountMode = \"credit\";\nexport interface UpstreamConfig {\n id?: string;\n type?: TsUpstreamType;\n /**\n * Tags is the single canonical user-applied label set. Convention is\n * `:` so a single upstream can carry orthogonal labels:\n * tags:\n * - tier:main # used by .preferTag for tiering\n * - region:us-east # used by .spreadAcrossTags('region:')\n * - sequencer:op-base # shared-fate label\n * Bare strings (no prefix) work too. Patterns supported by every\n * stdlib tag method: glob (`*`, `?`) and `!negation`.\n * Tier convention: `tier:fallback` declares a fallback-tier upstream\n * (matches the long-standing eRPC convention; the default policy\n * looks for `!tier:fallback` as the primary tier).\n * The legacy `group: X` / `cohort: Y` YAML keys are accepted at\n * load time only \u2014 UnmarshalYAML rewrites them as `tier:X` /\n * `cohort:Y` tags and forgets them. There is no Go-level Group or\n * Cohort field; programmatic code uses Tags directly.\n */\n tags?: string[];\n vendorName?: string;\n endpoint?: string;\n evm?: EvmUpstreamConfig;\n svm?: SvmUpstreamConfig;\n jsonRpc?: JsonRpcUpstreamConfig;\n grpc?: GrpcUpstreamConfig;\n ignoreMethods?: string[];\n allowMethods?: string[];\n autoIgnoreUnsupportedMethods?: boolean;\n failsafe?: (FailsafeConfig | undefined)[];\n rateLimitBudget?: string;\n rateLimitAutoTune?: RateLimitAutoTuneConfig;\n /**\n * RateLimitCountMode selects how this upstream's rate-limit budget\n * counts a call: \"request\" (default) charges 1 hit, \"credit\" charges the\n * request's resolved vendor credit-unit cost. Empty resolves to\n * \"request\". Applies to the upstream-level budget only.\n */\n rateLimitCountMode?: RateLimitCountMode;\n /**\n * CreditUnits overrides the vendor's built-in per-method credit table\n * (CreditUnitsProvider) for this upstream, merged per method over the\n * vendor defaults (\"*\" = fallback for unlisted methods). Normally set\n * once per provider via `providers[].settings.creditUnits`, which is\n * copied onto every upstream the provider generates.\n */\n creditUnits?: { [key: string]: number /* int64 */};\n shadow?: ShadowUpstreamConfig;\n /**\n * Routing holds per-upstream routing hints consumed by the selection\n * policy. `scoreMultipliers` bias this upstream's rank inside\n * `sortByScore` (see SelectionPolicyConfig): the engine resolves the\n * matching entry for each (network, method, finality) tick and exposes\n * the resulting weight map to the eval function as `u.scoreMultipliers`.\n * When the upstream omits its own `routing` block, `ApplyDefaults`\n * inherits the project-level `upstreamDefaults.routing` (all-or-nothing,\n * matching the Tags inheritance pattern).\n */\n routing?: UpstreamRoutingConfig;\n}\n/**\n * UpstreamRoutingConfig holds per-upstream routing hints. Today this is\n * the home of `scoreMultipliers` (per-upstream weight overrides folded\n * into `sortByScore`) and `probe` (per-upstream opt-out for the\n * selection policy's `probeExcluded` shadow-mirror traffic).\n */\nexport interface UpstreamRoutingConfig {\n /**\n * ScoreMultipliers biases this upstream's rank. Each entry is a\n * matcher (network/method/finality) plus weight overrides; the engine\n * resolves the first matching entry per tick and hands it to the eval\n * function as `u.scoreMultipliers`. `sortByScore` merges it over the\n * base weights by default (see its `multipliers` option).\n */\n scoreMultipliers?: (ScoreMultiplierConfig | undefined)[];\n /**\n * ScoreLatencyQuantile selects which response-time quantile feeds the\n * score (e.g. 0.9 \u2192 p90). When unset, the policy's own\n * `sortByScore({ latencyQuantile })` (default p70) applies.\n */\n scoreLatencyQuantile?: number /* float64 */;\n /**\n * Probe gates whether the selection policy's `probeExcluded` step\n * may shadow-mirror real requests to THIS upstream when it's\n * currently in the excluded set. `\"on\"` (default) opts in; `\"off\"`\n * disables probing entirely \u2014 the upstream stays excluded\n * permanently once predicates trip until an operator intervenes\n * (manual cordon/uncordon, or until the predicate stops matching\n * via state-poller-driven structural metrics like head lag). Use\n * `\"off\"` for pay-per-call vendors where shadow traffic eats quota.\n */\n probe?: ProbeMode | \"on\" | \"off\";\n}\n/**\n * ProbeMode is the per-upstream `routing.probe` enum.\n */\nexport type ProbeMode = string;\n/**\n * ProbeModeOn \u2014 default. The selection policy may mirror sampled\n * real requests to this upstream while it's excluded so it\n * accumulates fresh tracker samples for natural re-admission.\n */\nexport const ProbeModeOn: ProbeMode = \"on\";\n/**\n * ProbeModeOff \u2014 never mirror. Upstream stays excluded after\n * predicates trip; only structural signals (head lag, etc.) can\n * drive re-admission.\n */\nexport const ProbeModeOff: ProbeMode = \"off\";\n/**\n * ScoreMultiplierConfig is one per-upstream weight override. The matcher\n * fields (network/method/finality) scope the entry \u2014 leave them empty (or\n * `\"*\"`) for an entry that applies to every request. Weight fields are\n * pointers so \"unset\" is distinct from \"zero\": an unset weight inherits\n * from the policy's base weights (merge mode), a zero weight removes that\n * metric's contribution. `overall` scales the upstream's FINAL score \u2014 a\n * preference dial where >1 prefers this upstream and <1 avoids it.\n */\nexport interface ScoreMultiplierConfig {\n network?: string;\n method?: string;\n finality?: DataFinalityState[];\n overall?: number /* float64 */;\n errorRate?: number /* float64 */;\n respLatency?: number /* float64 */;\n throttledRate?: number /* float64 */;\n blockHeadLag?: number /* float64 */;\n finalizationLag?: number /* float64 */;\n misbehaviors?: number /* float64 */;\n /**\n * TotalRequests is accepted for backward compatibility but no longer\n * influences scoring (the score is computed from rolling-window rates,\n * not absolute request counts).\n */\n totalRequests?: number /* float64 */;\n}\nexport interface ShadowUpstreamConfig {\n enabled: boolean;\n sampleRate?: number /* float64 */;\n ignoreFields?: { [key: string]: string[]};\n /**\n * PinBlockTag rewrites a tag block selector (\"latest\"/\"pending\") in the mirrored copy to a\n * concrete height before forwarding: the primary's resolved block number when its response\n * carries one, else the network's tracked head at mirror time. Without it the shadow executes\n * the tag at its OWN head wall-clock-later than the primary did, so any read of volatile state\n * (pool reserves, oracle prices \u2014 most visibly large multicalls) diverges by construction and\n * reports a mismatch that says nothing about correctness. Default false = mirror byte-identical\n * requests, exactly as before.\n */\n pinBlockTag?: boolean;\n}\n/**\n * Deprecated: UpstreamIntegrityConfig is a non-functional legacy stub (never\n * read at runtime). Configure data integrity via the network `integrity` block.\n * Retained only so existing YAML still parses.\n */\nexport interface UpstreamIntegrityConfig {\n eth_getBlockReceipts?: UpstreamIntegrityEthGetBlockReceiptsConfig;\n}\nexport interface UpstreamIntegrityEthGetBlockReceiptsConfig {\n enabled?: boolean;\n checkLogIndexStrictIncrements?: boolean;\n checkLogsBloom?: boolean;\n}\nexport type UJAlias = UpstreamConfig;\nexport type UYAlias = UpstreamConfig;\nexport interface RateLimitAutoTuneConfig {\n enabled?: boolean;\n adjustmentPeriod: Duration;\n errorRateThreshold: number /* float64 */;\n increaseFactor: number /* float64 */;\n decreaseFactor: number /* float64 */;\n minBudget: number /* int */;\n maxBudget: number /* int */;\n}\nexport interface JsonRpcUpstreamConfig {\n supportsBatch?: boolean;\n batchMaxSize?: number /* int */;\n batchMaxWait?: Duration;\n enableGzip?: boolean;\n headers?: { [key: string]: string};\n proxyPool?: string;\n}\n/**\n * GrpcUpstreamConfig tunes a gRPC (grpc:// / grpc+bds:// / grpcs://) upstream. It is the\n * gRPC analogue of JsonRpcUpstreamConfig: JsonRpc holds JSON-RPC/HTTP-specific\n * knobs, this holds gRPC-specific ones. Headers are applied as gRPC metadata on\n * every outbound request (e.g. an edge-api auth key: authorization: Bearer ...).\n */\nexport interface GrpcUpstreamConfig {\n headers?: { [key: string]: string};\n /**\n * PoolSize is the number of independent gRPC connections opened to this\n * upstream, selected round-robin per request. See GrpcConnectorConfig.PoolSize.\n * When unset (0) a built-in default is used.\n */\n poolSize?: number /* int */;\n /**\n * HealthCheckService: see GrpcConnectorConfig.HealthCheckService.\n */\n healthCheckService?: string;\n}\nexport interface EvmUpstreamConfig {\n chainId: number /* int64 */;\n statePollerInterval?: Duration;\n /**\n * StatePollerDebounce overrides the debounce interval for the state poller.\n * When 0 (default), the interval is dynamically inferred from the chain's\n * observed block time, falling back to the network-level\n * FallbackStatePollerDebounce, then to a 1s floor.\n */\n statePollerDebounce?: Duration;\n blockAvailability?: EvmBlockAvailabilityConfig;\n getLogsAutoSplittingRangeThreshold?: number /* int64 */;\n /**\n * TraceFilterAutoSplittingRangeThreshold proactively splits trace_filter and\n * arbtrace_filter requests whose block range exceeds this value into contiguous\n * sub-requests executed concurrently and merged before returning. Zero disables\n * the feature.\n */\n traceFilterAutoSplittingRangeThreshold?: number /* int64 */;\n skipWhenSyncing?: boolean;\n /**\n * SkipSyncingCheck disables eth_syncing polling for this upstream, treating it as always synced.\n * Use for nodes that always return a syncing object (e.g. Pharos/Antora) where the response is\n * misleading and causes circuit breaker false positives.\n */\n skipSyncingCheck?: boolean;\n /**\n * HeadLagToleranceBlocks lets the block-availability gate send requests pinned up to N blocks\n * ABOVE this upstream's observed head instead of rerouting them as stale. Useful for upstreams\n * that follow the chain tightly but become able to serve a new block a moment after it is\n * announced elsewhere (they hold or briefly wait for the block internally): with a hard\n * head comparison every request pinned at head+1 in that moment is rerouted even though the\n * upstream would have answered it. 0 (default) keeps the exact head comparison.\n */\n headLagToleranceBlocks?: number /* int64 */;\n /**\n * PinLatestTag sends this upstream the network head as a concrete block number wherever a\n * state call (eth_call, eth_getBalance, ...) asks for \"latest\" or omits the block, and skips the\n * upstream when its blockAvailability upper bound is below that head or the head is unknown.\n * For an upstream whose own head lags the network: it answers only blocks it holds, and a block\n * it lacks fails as missing data so the next upstream answers, instead of an older \"latest\".\n */\n pinLatestTag?: boolean;\n /**\n * Deprecated: never read at runtime. Configure data integrity via the network\n * `integrity` block instead. Retained only so existing YAML still parses.\n */\n integrity?: UpstreamIntegrityConfig;\n /**\n * @deprecated: use blockAvailability bounds instead; kept for config back-compat only\n */\n nodeType?: EvmNodeType;\n /**\n * @deprecated: should be removed in a future release\n */\n maxAvailableRecentBlocks?: number /* int64 */;\n queryShim?: EvmQueryShimConfig;\n /**\n * Multicall3 packs eth_calls bound for this upstream that share a block into one\n * Multicall3 aggregate3 eth_call, so a vendor billed per request is billed once\n * per pack. Nil (default) sends every eth_call alone.\n */\n multicall3?: EvmMulticall3Config;\n}\n/**\n * EvmMulticall3Config bounds one pack. A call joins the open pack for its block,\n * and the pack is sent MaxWait after its first call or when it holds MaxCalls.\n * A call the pack cannot answer exactly (multicall failed, inner call reverted\n * or returned nothing) is sent alone, as it would be without packing.\n */\nexport interface EvmMulticall3Config {\n maxWait?: Duration;\n maxCalls?: number /* int */;\n /**\n * BypassContracts are target addresses whose calls are never packed, for\n * contracts that answer differently when the caller is Multicall3 instead\n * of the zero address (msg.sender or caller-code checks) without reverting.\n * Contracts that revert inside a pack but succeed alone are learned at\n * runtime and need no entry.\n */\n bypassContracts?: string[];\n}\n/**\n * EvmQueryShimConfig lets an upstream serve the sub-requests (eth_getBlockByNumber,\n * eth_getBlockReceipts, eth_getLogs, trace_block / debug_traceBlockByNumber) the\n * eth_query* executor issues when no native QueryService upstream answers. The\n * sub-requests go through the network like any request; the budget bounds the\n * work of one MIP-16 page.\n */\nexport interface EvmQueryShimConfig {\n enabled?: boolean;\n allowedMethods?: string[];\n /**\n * MaxBlocksPerPage is the budget of blocks one page may scan. A page that\n * reaches it ends at the last scanned block.\n */\n maxBlocksPerPage?: number /* int64 */;\n /**\n * MaxPageDuration is the budget of time one page may scan. A page that\n * reaches it ends at the last complete block, or fails with -32005 when no\n * block completed.\n */\n maxPageDuration?: Duration;\n}\n/**\n * EvmBlockAvailability defines optional lower/upper block availability expressions for an upstream.\n * Presence of lower/upper implies the feature is active. When both are nil, it's effectively off\n */\nexport interface EvmBlockAvailabilityConfig {\n lower?: EvmAvailabilityBoundConfig;\n upper?: EvmAvailabilityBoundConfig;\n}\n/**\n * EvmBound represents a single bound definition.\n * Exactly one of ExactBlock, LatestMinus, EarliestPlus should be set.\n * UpdateRate only applies to earliestBlockPlus bounds: 0 means freeze at first evaluation; >0 means recompute on that cadence.\n * For latestBlockMinus, updateRate is ignored: bounds are computed on-demand using the continuously-updated latest block from evmStatePoller.\n */\nexport type EvmAvailabilityProbeType = string;\nexport const EvmProbeBlockHeader: EvmAvailabilityProbeType = \"blockHeader\";\nexport const EvmProbeEventLogs: EvmAvailabilityProbeType = \"eventLogs\";\nexport const EvmProbeCallState: EvmAvailabilityProbeType = \"callState\";\nexport const EvmProbeTraceData: EvmAvailabilityProbeType = \"traceData\";\nexport interface EvmAvailabilityBoundConfig {\n exactBlock?: number /* int64 */;\n latestBlockMinus?: number /* int64 */;\n earliestBlockPlus?: number /* int64 */;\n probe?: EvmAvailabilityProbeType;\n updateRate?: Duration;\n}\nexport interface FailsafeConfig {\n matchMethod?: string;\n matchFinality?: DataFinalityState[];\n /**\n * MatchCommitment scopes a network-scope policy by the Solana commitment\n * erpc pins on the wire: the caller's value, else the svm.commitment default\n * that injection writes, else \"none\". Empty = any. Values are OR-ed and\n * exact (confirmed does not match processed). Non-SVM requests and SVM\n * write methods (sendTransaction, simulateTransaction, requestAirdrop) are \"none\".\n * Rejected at upstream and connector scopes.\n */\n matchCommitment?: ('none' | 'processed' | 'confirmed' | 'finalized')[];\n /**\n * MatchRequestKind scopes this policy by who issued the request:\n * \"user\" (client traffic), \"internal\" (erpc's own auxiliary fetches, e.g.\n * the integrity module's canonical corroboration), or \"\"/\"*\" for both.\n * This is what lets an operator give INTERNAL canonical fetches a\n * consensus policy (quorum-verified ground truth, deduplicated to ~once\n * per block by the ChainView) while user data methods rely on integrity\n * validation instead of per-request fan-out.\n */\n matchRequestKind?: 'user' | 'internal' | '*';\n retry?: RetryPolicyConfig;\n circuitBreaker?: CircuitBreakerPolicyConfig;\n timeout?: TimeoutPolicyConfig;\n hedge?: HedgePolicyConfig;\n consensus?: ConsensusPolicyConfig;\n}\n/**\n * NetworkFailsafeConfig is the scope-specific alias for network-level\n * failsafe policies. By convention, CircuitBreaker is not used at this\n * scope (use upstream-scope breakers instead); validation enforces this.\n */\nexport type NetworkFailsafeConfig = FailsafeConfig;\n/**\n * UpstreamFailsafeConfig is the scope-specific alias for per-upstream\n * failsafe policies. By convention, Consensus is not used at this\n * scope (consensus is a network-scope concern only); validation\n * enforces this.\n */\nexport type UpstreamFailsafeConfig = FailsafeConfig;\n/**\n * CacheFailsafeConfig is the scope-specific alias for cache-connector\n * failsafe policies. Timeout.Duration and Hedge.Delay support quantile\n * (dynamic) mode at this scope, resolved from a per-executor latency\n * tracker fed by the connector's own operations (see data/cache_executor.go).\n * Consensus is not supported here; validation enforces this.\n */\nexport type CacheFailsafeConfig = FailsafeConfig;\nexport interface RetryPolicyConfig {\n maxAttempts: number /* int */;\n delay?: Duration;\n backoffMaxDelay?: Duration;\n backoffFactor?: number /* float32 */;\n jitter?: Duration;\n /**\n * EmptyResultAccept lists methods for which an empty/null result is considered valid\n * and should NOT be retried (e.g. eth_getLogs, eth_call where empty is a legitimate response).\n */\n emptyResultAccept?: string[];\n /**\n * @deprecated: use EmptyResultAccept instead.\n */\n emptyResultIgnore?: string[];\n /**\n * EmptyResultMaxAttempts limits total attempts when retries are triggered due to empty responses.\n */\n emptyResultMaxAttempts?: number /* int */;\n /**\n * EmptyResultDelay is the fixed fallback delay before retrying when the requested\n * data isn't on the upstream yet \u2014 an empty/missing-data point-lookup OR an\n * ErrUpstreamBlockUnavailable (same root cause: the block/tx isn't produced or\n * indexed yet). Retries prefer the dynamic block-time delay (EMA block time \u00D7\n * Evm.BlockUnavailableDelayMultiplier); this fixed value is used only before that\n * estimate warms up. (Supersedes the now-deprecated BlockUnavailableDelay.)\n */\n emptyResultDelay?: Duration;\n}\nexport interface CircuitBreakerPolicyConfig {\n failureThresholdCount: number /* uint */;\n failureThresholdCapacity: number /* uint */;\n halfOpenAfter?: Duration;\n successThresholdCount: number /* uint */;\n successThresholdCapacity: number /* uint */;\n}\n/**\n * TimeoutPolicyConfig is the timeout policy. Duration is the unified\n * AdaptiveDuration \u2014 a scalar shorthand (\"5s\") or an object form\n * ({base, quantile, min, max}) for adaptive caps driven by per-method\n * latency quantiles.\n * Wire format also accepts the legacy flat form\n * (`duration: 5s, quantile: 0.99, minDuration: 200ms, maxDuration: 10s`)\n * \u2014 siblings get folded into Duration at YAML/JSON unmarshal time.\n */\nexport interface TimeoutPolicyConfig {\n duration?: Duration | AdaptiveDuration;\n}\n/**\n * HedgePolicyConfig is the hedge policy. Delay is the unified\n * AdaptiveDuration \u2014 scalar shorthand (\"100ms\") or object form\n * ({base, quantile, min, max}) for quantile-driven hedge timing.\n * Wire format also accepts the legacy flat form\n * (`delay: 100ms, quantile: 0.95, minDelay: 50ms, maxDelay: 2s`) \u2014\n * siblings get folded into Delay at YAML/JSON unmarshal time.\n */\nexport interface HedgePolicyConfig {\n delay?: Duration | AdaptiveDuration;\n maxCount: number /* int */;\n}\nexport type ConsensusLowParticipantsBehavior = string;\nexport const ConsensusLowParticipantsBehaviorReturnError: ConsensusLowParticipantsBehavior = \"returnError\";\nexport const ConsensusLowParticipantsBehaviorAcceptMostCommonValidResult: ConsensusLowParticipantsBehavior = \"acceptMostCommonValidResult\";\nexport const ConsensusLowParticipantsBehaviorPreferBlockHeadLeader: ConsensusLowParticipantsBehavior = \"preferBlockHeadLeader\";\nexport const ConsensusLowParticipantsBehaviorOnlyBlockHeadLeader: ConsensusLowParticipantsBehavior = \"onlyBlockHeadLeader\";\nexport type ConsensusDisputeBehavior = string;\nexport const ConsensusDisputeBehaviorReturnError: ConsensusDisputeBehavior = \"returnError\";\nexport const ConsensusDisputeBehaviorAcceptMostCommonValidResult: ConsensusDisputeBehavior = \"acceptMostCommonValidResult\";\nexport const ConsensusDisputeBehaviorPreferBlockHeadLeader: ConsensusDisputeBehavior = \"preferBlockHeadLeader\";\nexport const ConsensusDisputeBehaviorOnlyBlockHeadLeader: ConsensusDisputeBehavior = \"onlyBlockHeadLeader\";\nexport interface ConsensusPolicyConfig {\n maxParticipants: number /* int */;\n agreementThreshold?: number /* int */;\n disputeBehavior?: ConsensusDisputeBehavior;\n lowParticipantsBehavior?: ConsensusLowParticipantsBehavior;\n punishMisbehavior?: PunishMisbehaviorConfig;\n disputeLogLevel?: string; // \"trace\", \"debug\", \"info\", \"warn\", \"error\"\n ignoreFields?: { [key: string]: string[]};\n preferNonEmpty?: boolean;\n preferLargerResponses?: boolean;\n misbehaviorsDestination?: MisbehaviorsDestinationConfig;\n /**\n * PreferHighestValueFor specifies methods that should use highest-value comparison\n * instead of hash-based consensus. Map key is method name, value is array of field paths.\n * Field paths: \"result\" for direct result value (e.g., eth_getTransactionCount returns hex),\n * or field name for nested result objects (e.g., \"nonce\" for result.nonce).\n * When multiple fields are specified, they act as tie-breakers in order.\n */\n preferHighestValueFor?: { [key: string]: string[]};\n /**\n * FireAndForget when true, allows consensus to return a response to the client immediately\n * upon short-circuit, but does NOT cancel in-flight requests to other upstreams.\n * This is useful for write operations like eth_sendRawTransaction where you want to\n * broadcast the transaction to as many nodes as possible while still returning quickly.\n * Default is false (normal behavior - cancel remaining requests on short-circuit).\n */\n fireAndForget?: boolean;\n /**\n * MaxWaitOnResult caps how long consensus waits for additional participants\n * AFTER at least one non-empty response has arrived. Use this to bound\n * p99 latency when most upstreams are fast but one is a slow straggler:\n * once a real answer is in hand, give the rest at most this long to\n * confirm or dispute, then resolve with what we have.\n * Accepts a duration scalar (\"200ms\") or an AdaptiveDuration object\n * ({base, quantile, min, max}) for adaptive caps driven by per-method\n * latency quantiles. Defaults are applied when consensus is configured\n * but this field is omitted \u2014 see common/defaults.go.\n */\n maxWaitOnResult?: Duration | AdaptiveDuration;\n /**\n * MaxWaitOnEmpty caps how long consensus waits for additional participants\n * AFTER the first response (of any kind \u2014 empty, error, or non-empty)\n * has arrived. Typically set larger than MaxWaitOnResult because an\n * operator is more patient when no useful data is in hand yet.\n * Same shape as MaxWaitOnResult; defaults applied when consensus is set.\n */\n maxWaitOnEmpty?: Duration | AdaptiveDuration;\n /**\n * RequiredParticipants enforces a minimum number of consensus\n * participants carrying a given tag. Each entry says \"at least\n * `minParticipants` of the upstreams that participate must match\n * `tag`\". The engine front-loads enough tag-matching upstreams into\n * the participant set so the first `maxParticipants` drawn satisfy\n * every entry \u2014 without changing `maxParticipants` itself.\n * Best-effort and governed by the EXISTING consensus behaviors: if a\n * required group has fewer healthy upstreams than requested (or the\n * quotas can't all fit within `maxParticipants`), consensus simply\n * runs with what it can promote and the resulting participation is\n * handled by `lowParticipantsBehavior` / `agreementThreshold` exactly\n * like any other low-participation tick. Empty (default) = disabled.\n */\n requiredParticipants?: (ConsensusRequiredParticipant | undefined)[];\n}\n/**\n * ConsensusRequiredParticipant is one tag-quota entry for\n * `consensus.requiredParticipants`. `Tag` is a glob pattern (`*`, `?`)\n * matched against each upstream's `tags`; `MinParticipants` is the minimum\n * number of matching upstreams that must be in the consensus participant\n * set (pool quota, best-effort). `MinAgreement` is the minimum number of\n * matching upstreams that must be part of the WINNING response group\n * (winner-composition quota, hard-enforced: a winner that does not satisfy\n * it becomes a composition dispute regardless of disputeBehavior). A single\n * upstream can satisfy multiple entries it matches.\n */\nexport interface ConsensusRequiredParticipant {\n tag: string;\n minParticipants: number /* int */;\n minAgreement?: number /* int */;\n}\nexport type MisbehaviorsDestinationType = string;\nexport const MisbehaviorsDestinationTypeFile: MisbehaviorsDestinationType = \"file\";\nexport const MisbehaviorsDestinationTypeS3: MisbehaviorsDestinationType = \"s3\";\nexport interface MisbehaviorsDestinationConfig {\n /**\n * Type of destination: \"file\" or \"s3\"\n */\n type: 'file' | 's3';\n /**\n * Path for file destination, or S3 URI (s3://bucket/prefix/) for S3 destination\n */\n path: string;\n /**\n * Pattern for generating file names. Supports placeholders:\n * {dateByHour} - formatted as 2006-01-02-15\n * {dateByDay} - formatted as 2006-01-02\n * {method} - the RPC method name\n * {networkId} - the network ID with : replaced by _\n * {instanceId} - unique instance identifier\n */\n filePattern?: string;\n /**\n * S3-specific settings for bulk flushing\n */\n s3?: S3FlushConfig;\n}\nexport interface S3FlushConfig {\n /**\n * Maximum number of records to buffer before flushing (default: 100)\n */\n maxRecords?: number /* int */;\n /**\n * Maximum size in bytes to buffer before flushing (default: 1MB)\n */\n maxSize?: number /* int64 */;\n /**\n * Maximum time to wait before flushing buffered records (default: 60s)\n */\n flushInterval?: Duration;\n /**\n * AWS region for S3 bucket (defaults to AWS_REGION env var)\n */\n region?: string;\n /**\n * Custom S3 endpoint URL for S3-compatible providers (Tigris, MinIO, R2, \u2026).\n * Empty = real AWS S3. When set, path-style addressing is used.\n */\n endpoint?: string;\n /**\n * AWS credentials config (optional). If not specified, uses standard AWS credential chain:\n * 1. Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)\n * 2. IAM role (for EC2/ECS/EKS)\n * 3. Shared credentials file (~/.aws/credentials)\n * Supported modes: \"env\", \"file\", \"secret\"\n */\n credentials?: AwsAuthConfig;\n /**\n * Content type for uploaded files (default: \"application/jsonl\")\n */\n contentType?: string;\n}\nexport interface PunishMisbehaviorConfig {\n disputeThreshold: number /* uint */;\n disputeWindow?: Duration;\n sitOutPenalty?: Duration;\n}\nexport interface RateLimiterConfig {\n store?: RateLimitStoreConfig;\n budgets: RateLimitBudgetConfig[];\n}\nexport interface RateLimitBudgetConfig {\n id: string;\n rules: RateLimitRuleConfig[];\n /**\n * CreditUnits prices methods for this budget's countMode: credit rules. \"*\"\n * is the fallback, an unpriced method costs 1, and a method priced 0 is\n * exempt. An upstream on rateLimitCountMode: credit prices from its vendor\n * instead and may not combine the two.\n */\n creditUnits?: { [key: string]: number /* int64 */};\n}\nexport interface RateLimitRuleConfig {\n method: string;\n maxCount: number /* uint32 */;\n /**\n * Period is the canonical period selector. Supported: second, minute, hour, day, week, month, year\n */\n period: RateLimitPeriod;\n waitTime?: Duration;\n perIP?: boolean;\n perUser?: boolean;\n perNetwork?: boolean;\n /**\n * CountMode selects what this rule counts. \"request\" charges 1 per call and\n * counts per method. \"credit\" charges the method's cost from the budget's\n * creditUnits and pools all methods into one counter, making maxCount a\n * wallet. Empty inherits the caller's mode.\n */\n countMode?: RateLimitCountMode;\n}\n/**\n * RateLimitPeriod enumerates supported periods for rate limiting.\n * It is an int enum to enable strong typing in TypeScript generation, while\n * marshaling to JSON/YAML as human-readable strings like \"second\", \"minute\", etc.\n */\nexport type RateLimitPeriod = number /* int */;\nexport const RateLimitPeriodSecond: RateLimitPeriod = 0;\nexport const RateLimitPeriodMinute: RateLimitPeriod = 1;\nexport const RateLimitPeriodHour: RateLimitPeriod = 2;\nexport const RateLimitPeriodDay: RateLimitPeriod = 3;\nexport const RateLimitPeriodWeek: RateLimitPeriod = 4;\nexport const RateLimitPeriodMonth: RateLimitPeriod = 5;\nexport const RateLimitPeriodYear: RateLimitPeriod = 6;\nexport interface ProxyPoolConfig {\n id: string;\n urls: string[];\n}\nexport interface MethodsConfig {\n preserveDefaultMethods?: boolean;\n definitions?: { [key: string]: CacheMethodConfig | undefined};\n}\nexport interface NetworkConfig {\n architecture: TsNetworkArchitecture;\n rateLimitBudget?: string;\n failsafe?: (FailsafeConfig | undefined)[];\n evm?: EvmNetworkConfig;\n svm?: SvmNetworkConfig;\n selectionPolicy?: SelectionPolicyConfig;\n directiveDefaults?: DirectiveDefaultsConfig;\n alias?: string;\n methods?: MethodsConfig;\n multiplexing?: boolean;\n staticResponses?: (StaticResponseConfig | undefined)[];\n failover?: FailoverConfig;\n /**\n * Integrity overrides the project-wide data-integrity configuration for this\n * network. Merges over the project block (network wins).\n */\n integrity?: IntegrityConfig;\n /**\n * CacheKeySuffix, when set, is inserted into the JSON-RPC cache partition\n * key as {networkId}:{suffix}:{blockRef} so two networks that share a\n * chainId (and a Redis) do not collide. Empty keeps {networkId}:{blockRef}.\n * When {networkId}:{suffix} is itself a valid network id, colons in the\n * network id are escaped; see CachePartitionKey.\n */\n cacheKeySuffix?: string;\n}\n/**\n * StaticResponseConfig declares a canned JSON-RPC response for a specific\n * (method, params) pair on a network. When an inbound request matches, the\n * configured response is returned immediately and no upstream is contacted.\n * Useful for chains that deviate from client assumptions (for example, chains\n * whose genesis block is not 0) where probing upstreams would yield errors\n * or inconsistent data.\n */\nexport interface StaticResponseConfig {\n method: string;\n params?: any[];\n response?: StaticResponseBodyConfig;\n}\n/**\n * StaticResponseBodyConfig holds the JSON-RPC payload to serve. Exactly one\n * of Result or Error must be set.\n */\nexport interface StaticResponseBodyConfig {\n result?: any;\n error?: StaticResponseErrorConfig;\n}\n/**\n * StaticResponseErrorConfig mirrors a JSON-RPC error object.\n */\nexport interface StaticResponseErrorConfig {\n code: number /* int */;\n message: string;\n data?: any;\n}\nexport interface DirectiveDefaultsConfig {\n retryEmpty?: boolean;\n retryPending?: boolean;\n skipCacheRead?: any;\n useUpstream?: string;\n skipInterpolation?: boolean;\n skipConsensus?: boolean;\n /**\n * Validation: Block Integrity\n */\n enforceHighestBlock?: boolean;\n enforceGetLogsBlockRange?: boolean;\n enforceNonNullTaggedBlocks?: boolean;\n /**\n * --- Deprecated data-integrity validation flags ---\n * Deprecated: data integrity is now configured via the `integrity` block\n * (projects[].integrity / networks[].integrity). These per-check flags are\n * retained only so existing YAML still parses. At startup the flags that map\n * to a current check are translated into `integrity.checks` by\n * migrateLegacyIntegrityChecks (an explicit `integrity` block wins per check);\n * they are NOT read at runtime. The receipt-count / expected-block /\n * receipt-to-transaction flags have no equivalent in the new model and are\n * accepted but ignored. See docs/pages/config/failsafe/integrity.mdx.\n */\n validateTransactionsRoot?: boolean;\n validateHeaderFieldLengths?: boolean;\n validateTransactionFields?: boolean;\n validateTransactionBlockInfo?: boolean;\n enforceLogIndexStrictIncrements?: boolean;\n validateTxHashUniqueness?: boolean;\n validateTransactionIndex?: boolean;\n validateLogFields?: boolean;\n validateLogsBloomEmptiness?: boolean;\n validateLogsBloomMatch?: boolean;\n validateReceiptTransactionMatch?: boolean;\n validateContractCreation?: boolean;\n receiptsCountExact?: number /* int64 */;\n receiptsCountAtLeast?: number /* int64 */;\n validationExpectedBlockHash?: string;\n validationExpectedBlockNumber?: number /* int64 */;\n}\n/**\n * SvmNetworkConfig mirrors EvmNetworkConfig for SVM networks.\n * Most fields are Solana-specific and do not have EVM equivalents.\n */\nexport interface SvmNetworkConfig {\n /**\n * Chain identifies which SVM chain this network runs on. Defaults to \"solana\"\n * when empty for backward compatibility. Set explicitly for forks/variants\n * such as \"fogo\" or \"eclipse\" so eRPC can host multiple SVM chains side by\n * side without network-ID or cache-key collisions.\n * When Chain is empty or \"solana\", the derived NetworkId is \"svm:\"\n * \u2014 identical to the pre-multi-chain format. For any other Chain value the\n * NetworkId is \"svm::\".\n */\n chain?: string;\n /**\n * Cluster the upstreams of this network serve (e.g. \"mainnet-beta\", \"devnet\").\n * The NetworkId is derived from this value together with Chain \u2014 see the\n * Chain field above for the exact format.\n */\n cluster?: string;\n /**\n * Commitment is the default commitment level injected into requests whose params\n * omit one. One of \"finalized\", \"confirmed\", \"processed\". No default: when unset,\n * no commitment is injected and each upstream's own server-side default governs\n * (Solana's is \"finalized\"). Set this to pin one commitment across all upstreams\n * so the cache and consensus key on identical data regardless of vendor defaults;\n * note that doing so makes finality classification track the configured level.\n */\n commitment?: string;\n /**\n * StatePollerDebounce sets the minimum interval between polls of an upstream's\n * slot/health view. Default: 400ms (one slot).\n */\n statePollerDebounce?: Duration;\n /**\n * MaxFinalizedSlotLag bounds how many slots an upstream's FinalizedSlot may\n * trail the pool's highest FinalizedSlot before it is excluded from\n * consensus voting on finalized data. Only applied when a consensus policy\n * is active AND the request's resolved finality is Finalized.\n * Pointer so an explicit 0 is distinguishable from \"unset\": nil takes the\n * 100-slot default (filled by SetDefaults, the only place that materializes\n * it), an explicit 0 disables the filter entirely. A plain int64 collapses\n * those two cases and makes the documented disable switch unreachable.\n * Readers downstream of SetDefaults just check `lag != nil && *lag > 0`.\n */\n maxFinalizedSlotLag?: number /* int64 */;\n /**\n * EnforceBlockAvailability controls whether the networkPreForward_getBlock\n * guard is active. When enabled (default), getBlock/getConfirmedBlock\n * requests for slots above the highest indexed slot known to any upstream\n * are short-circuited with ErrEndpointMissingData before hitting providers\n * \u2014 saving quota and triggering the 500ms indexing-lag retry immediately.\n * Set to false to disable the guard when ShredInsertSlot tracking is\n * unavailable or unreliable on a given deployment.\n */\n enforceBlockAvailability?: boolean;\n}\n/**\n * SvmUpstreamConfig carries per-upstream SVM settings.\n */\nexport interface SvmUpstreamConfig {\n /**\n * Chain identifies which SVM chain this upstream serves. Must match the\n * network-level Chain. Empty defaults to \"solana\" for backward compat.\n */\n chain?: string;\n /**\n * Cluster this upstream serves. Must match the network-level cluster for the\n * upstream to be eligible.\n */\n cluster?: string;\n /**\n * CheckGenesisHash opts unknown clusters in to runtime validation via getGenesisHash\n * at bootstrap. Known clusters (mainnet-beta, devnet, testnet) are always validated\n * regardless of this flag: a single getGenesisHash RPC runs at bootstrap and is\n * compared against the hardcoded genesis-hash table \u2014 a mismatch OR a fetch failure\n * fails the upstream, catching nodes mis-pointed at the wrong cluster (and refusing\n * to register one we could not verify). For unknown clusters the same check (with\n * no table comparison) runs only when this flag is set; otherwise it is skipped so\n * private/local clusters with no published genesis hash still work.\n */\n checkGenesisHash?: boolean;\n /**\n * BlockAvailability is an operator restriction on the slot window this\n * upstream may serve. Only what the operator declares is enforced; the\n * upstream's oldest stored block (getBlocksWithLimit(0, 1)) is probed for\n * every upstream and anchors a firstAvailableSlotPlus bound, mirroring EVM's\n * earliestBlockPlus. Unknown floor = unbounded.\n */\n blockAvailability?: SvmBlockAvailabilityConfig;\n}\n/**\n * SvmBlockAvailabilityConfig is the per-upstream declared slot window.\n * Presence of lower/upper implies the restriction is active. When both are\n * nil, it is effectively off.\n */\nexport interface SvmBlockAvailabilityConfig {\n lower?: SvmAvailabilityBoundConfig;\n upper?: SvmAvailabilityBoundConfig;\n}\n/**\n * SvmAvailabilityBoundConfig is a single declared slot bound.\n * Exactly one of ExactSlot, LatestSlotMinus, FirstAvailableSlotPlus must be set.\n * UpdateRate applies only to firstAvailableSlotPlus: how often the floor it\n * anchors on is re-probed. 0 freezes it after the first observation, as EVM.\n */\nexport interface SvmAvailabilityBoundConfig {\n exactSlot?: number /* int64 */;\n latestSlotMinus?: number /* int64 */;\n firstAvailableSlotPlus?: number /* int64 */;\n updateRate?: Duration;\n}\nexport interface EvmNetworkConfig {\n chainId: number /* int64 */;\n fallbackFinalityDepth?: number /* int64 */;\n fallbackStatePollerDebounce?: Duration;\n integrity?: EvmIntegrityConfig;\n /**\n * ServedTip configures how the network derives the \"latest\"/\"finalized\"\n * block it advertises to clients (and enforces via block-availability).\n * Nil uses the defaults: quorum 2, regression guard on, no guarantees. See\n * EvmServedTipConfig.\n */\n servedTip?: EvmServedTipConfig;\n getLogsMaxAllowedRange?: number /* int64 */;\n getLogsMaxAllowedAddresses?: number /* int64 */;\n getLogsMaxAllowedTopics?: number /* int64 */;\n getLogsSplitOnError?: boolean;\n getLogsSplitConcurrency?: number /* int */;\n /**\n * GetLogsMaxResponseBytes caps the total size of a MERGED eth_getLogs response \u2014\n * the sum of every sub-response a split assembles, not the size of any one piece.\n * Splitting exists to get past an upstream's per-request limits, so once a request\n * is split there is no remaining bound on what the client can make the proxy hold\n * in memory: a range the upstream refuses is halved, fetched concurrently, and\n * merged whole. This is the only bound on that total.\n * Sub-requests are cancelled as soon as the running total crosses the cap, so the\n * in-flight ones do not materialise either, and the client gets\n * ErrGetLogsExceededMaxAllowedResponseSize (HTTP 413) \u2014 a distinct code from\n * ErrEndpointRequestTooLarge precisely so it does NOT feed the splitting path that\n * produced the response.\n * Zero (the default) disables the cap and preserves the historical behaviour.\n */\n getLogsMaxResponseBytes?: number /* int64 */;\n /**\n * TraceFilterSplitOnError controls reactive splitting for trace_filter and\n * arbtrace_filter requests when the upstream returns a range-too-large error.\n * Nil disables the feature.\n */\n traceFilterSplitOnError?: boolean;\n /**\n * TraceFilterSplitConcurrency caps in-flight sub-requests when a trace_filter\n * or arbtrace_filter request is split. Zero falls back to 10.\n */\n traceFilterSplitConcurrency?: number /* int */;\n /**\n * EnforceBlockAvailability controls whether the network should enforce per-upstream\n * block availability bounds (upper/lower) for methods by default. Method-level config may override.\n * When nil or true, enforcement is enabled.\n */\n enforceBlockAvailability?: boolean;\n /**\n * MaxRetryableBlockDistance controls the maximum block distance for which an upstream\n * block unavailability error is considered retryable. If the requested block is within\n * this distance from the upstream's latest block, the error is retryable (upstream may catch up).\n * If the distance is larger, the error is not retryable (upstream is too far behind).\n * Default: 128 blocks.\n */\n maxRetryableBlockDistance?: number /* int64 */;\n /**\n * MarkEmptyAsErrorMethods lists methods for which an empty/null result from an upstream\n * should be treated as a \"missing data\" error, triggering retry on other upstreams.\n * This is useful for point-lookups (blocks, transactions, receipts, traces) where an\n * empty result likely means the upstream hasn't indexed that data yet.\n * Default includes common point-lookup methods like eth_getBlockByNumber, eth_getTransactionByHash, etc.\n */\n markEmptyAsErrorMethods?: string[];\n /**\n * LocalAnswers turns off eRPC's own answer for a method on this network:\n * `{net_peerCount: false}` routes net_peerCount to the upstreams. Every\n * method in LocalAnswerMethods is answered by eRPC unless set false here.\n */\n localAnswers?: { [key: string]: boolean};\n /**\n * StripSubscribeFromBlockZero removes a zero `fromBlock` from eth_subscribe\n * logs filters before forwarding upstream. fromBlock has no meaning for a\n * live subscription, and backends that prune history reject it.\n * DEFAULT: false.\n */\n stripSubscribeFromBlockZero?: boolean;\n /**\n * DynamicBlockTimeDebounceMultiplier scales the EMA-estimated block time to derive\n * the debounce interval for block polling. A value of 0.7 means debounce = 70% of\n * the estimated block time, preferring fresher data at the cost of slightly more\n * polling. Lower values reduce staleness risk; higher values reduce RPC calls.\n * Default: 0.7 (30% under the estimated block time).\n */\n dynamicBlockTimeDebounceMultiplier?: number /* float64 */;\n /**\n * FinalizedDebounceDepthMultiplier sets how stale an upstream's finalized block may\n * get before the state poller fetches it again, as a fraction of the chain's\n * finality depth in time: (latest - finalized) x EMA block time x multiplier. The\n * result applies only when it is longer than the regular debounce. Finalized\n * advances once per finality window, so polling it at the latest-block cadence\n * mostly returns the same number. 0 disables it (finalized uses the regular debounce).\n * Default: 0.05 (refresh about 20 times per finality window).\n */\n finalizedDebounceDepthMultiplier?: number /* float64 */;\n /**\n * BlockUnavailableDelayMultiplier scales the EMA-estimated block time to derive the\n * retry delay when the requested data isn't available yet (ErrUpstreamBlockUnavailable\n * or an empty/missing-data point-lookup). When the dynamic block time is known, the\n * delay is blockTime * this multiplier. Falls back to the static\n * RetryPolicyConfig.EmptyResultDelay when block time is not yet available. Default: 1.0.\n */\n blockUnavailableDelayMultiplier?: number /* float64 */;\n /**\n * IdempotentTransactionBroadcast enables idempotency handling for eth_sendRawTransaction.\n * When enabled (default), \"already known\" and verified \"nonce too low\" errors are converted\n * to success responses with the transaction hash. This allows failsafe policies (retry/hedge)\n * to work safely with transaction broadcasting.\n * Set to false to disable this behavior and return raw upstream errors.\n */\n idempotentTransactionBroadcast?: boolean;\n /**\n * EmptyResultConfidence sets how confirmed a concrete numeric block must be for an\n * empty/null point-lookup result to be treated as retryable missing-data, versus a\n * truthful \"not yet produced/confirmed\" empty returned without retrying. Applies to\n * MarkEmptyAsErrorMethods when the RetryEmpty directive is on; tags and block-hash\n * lookups never qualify, and it fails open when the head is unknown.\n * - blockHead (default): retry empties for blocks at/below the latest head; a\n * block above the head isn't produced yet \u2192 return the empty truthfully.\n * - finalizedBlock: stricter \u2014 only retry empties for blocks at/below the\n * finalized head; an unfinalized block's empty is treated as not-yet-confirmed.\n */\n emptyResultConfidence?: AvailbilityConfidence;\n /**\n * ShortCircuitFutureBlocks returns null for a numbered eth_getBlockByNumber\n * above the highest latest head across every upstream of the network\n * (regardless of selection policy, tier or syncing state), without\n * dispatching it; it never fires while any upstream's head is unknown.\n * Clients that poll the next block by number otherwise walk every routed\n * upstream on each poll.\n * The head is what the state poller and head subscriptions last reported, so\n * a block an upstream produced since then gets null until eRPC sees it: only\n * enable it where heads arrive promptly (e.g. over WebSocket).\n */\n shortCircuitFutureBlocks?: boolean;\n /**\n * SafeBlockSource is an upstream id/tag selector for standard JSON-RPC\n * requests carrying the `safe` block tag. Matching upstreams define and\n * serve `safe`; empty (without an inherited network default) keeps existing\n * provider-defined routing. This does not affect eth_query* or gRPC Query.\n */\n safeBlockSource?: string;\n}\n/**\n * EvmServedTipConfig controls how the network derives the \"latest\"/\"finalized\"\n * block it advertises (and enforces) from its upstreams.\n * Both axes use one rule: among the eligible (selection-policy), non-syncing\n * upstreams, only the best voter rank present votes (live, then benched, then\n * static-capped, then benched and static-capped), and the served block is the\n * highest one at least Quorum of those voters already have. The health\n * tracker benches an upstream whose head stopped moving while another kept\n * advancing, until it catches up again.\n */\nexport interface EvmServedTipConfig {\n /**\n * Quorum is how many eligible upstreams must already have a block before it\n * is served: an integer >= 1, or \"majority\" (floor(N/2)+1 of N). Unset\n * uses DefaultServedTipQuorum (2). A ballot with fewer voters than the\n * quorum serves the block all of them have.\n */\n quorum?: number | 'majority';\n /**\n * GuaranteedMethods lists method name patterns (glob; e.g. \"trace_*\",\n * \"debug_traceBlockByNumber\") whose supporting-upstream subset must be able to\n * serve the advertised latest. For a request on a matching method, \"latest\"\n * resolves against the majority of only the upstreams that support it\n * (membership auto-detected via ShouldHandleMethod \u2014 no per-upstream config).\n * Empty means only the global ballot is computed.\n */\n guaranteedMethods?: string[];\n /**\n * GuaranteedFor lists upstream SELECTORS (id or tag glob \u2014 the same\n * vocabulary as use-upstream and consensus.requiredParticipants, e.g.\n * \"type:internal\") whose group must be able to serve the advertised tip.\n * For each selector the tip is clamped down to that group's OWN majority,\n * exactly as GuaranteedMethods clamps to a method's supporting set. The\n * clamp is a group MAJORITY, not a group minimum, so one stuck member\n * cannot pin the network; an empty group constrains nothing.\n */\n guaranteedFor?: string[];\n /**\n * MaxRegressionBlocks is how far below the corroborated LIVE upstream head\n * (the second-highest live head) the pick may fall before it is treated as\n * a poisoned ballot rather than as reality. While a pick is below that\n * bound the network keeps serving its last corroborated pick (in-process,\n * for at most one minute, then it fails open and serves the pick as\n * computed). 0 uses DefaultToleratedBlockHeadRollback (1024) \u2014 the same\n * rollback tolerance the state poller and health tracker apply to a\n * retreating head, so all three layers agree on what counts as a genuine\n * deep correction. -1 disables the regression guard entirely; no other\n * negative value is accepted.\n */\n maxRegressionBlocks?: number /* int64 */;\n}\n/**\n * EvmIntegrityConfig is deprecated. Use DirectiveDefaultsConfig for validation settings.\n */\nexport interface EvmIntegrityConfig {\n /**\n * @deprecated: use DirectiveDefaults.EnforceHighestBlock\n */\n enforceHighestBlock?: boolean;\n /**\n * @deprecated: use DirectiveDefaults.EnforceGetLogsBlockRange\n */\n enforceGetLogsBlockRange?: boolean;\n /**\n * @deprecated: use DirectiveDefaults.EnforceNonNullTaggedBlocks\n */\n enforceNonNullTaggedBlocks?: boolean;\n}\n/**\n * EvalScope picks the grain at which the selection policy evaluates AND\n * at which the health tracker stores per-upstream metrics. One knob\n * covers what `evalPerMethod` + `evalPerFinality` used to cover as two\n * bools \u2014 and pulls the tracker's metric grain in lockstep so a\n * predicate like `errorRateAbove(0.5)` in a (method, finality)-grained\n * slot sees genuinely (method, finality)-specific error rate, not the\n * per-method aggregate.\n * Values are kebab-case strings so they round-trip through YAML / JSON\n * / Go enum literals cleanly. The TS SDK exports the same names in\n * CAPITAL_SNAKE_CASE with these same string values, so `evalScope:\n * NETWORK` in a TS config and `evalScope: network` in a YAML config\n * produce identical Go state.\n */\nexport type EvalScope = string;\n/**\n * EvalScopeNetwork \u2014 single slot per network. Methods + finalities\n * share. Default. Lowest cardinality + lowest tracker memory.\n */\nexport const EvalScopeNetwork: EvalScope = \"network\";\n/**\n * EvalScopeNetworkMethod \u2014 slot per (network, method). Finalities\n * share. Useful when one upstream is fast on `eth_call` but slow on\n * `trace_filter`.\n */\nexport const EvalScopeNetworkMethod: EvalScope = \"network-method\";\n/**\n * EvalScopeNetworkFinality \u2014 slot per (network, finality). Methods\n * share. Useful when realtime reads weight freshness differently\n * from finalized reads.\n */\nexport const EvalScopeNetworkFinality: EvalScope = \"network-finality\";\n/**\n * EvalScopeNetworkMethodFinality \u2014 slot per (network, method,\n * finality). Most granular routing. Cardinality scales linearly;\n * each slot's ticker only spins up after the first request for\n * that bucket lands.\n */\nexport const EvalScopeNetworkMethodFinality: EvalScope = \"network-method-finality\";\n/**\n * SelectionPolicyConfig declares the per-network upstream selection policy.\n * The eval function is JavaScript that receives `upstreams` and `ctx` and\n * returns the ordered list of upstreams that should serve traffic for the\n * network/method scope. The chainable std-lib (see internal/policy/stdlib)\n * provides the building blocks (sortByScore, removeByLag, stickyPrimary,\n * probeExcluded, etc.) \u2014 see specs/selection-policy/feature.md.\n */\nexport interface SelectionPolicyConfig {\n evalInterval?: Duration;\n /**\n * EvalScope picks the slot grain \u2014 and the matching tracker grain\n * (see `EvalScope` doc). Default `network` (one slot per network).\n * Tighter scopes let predicates like `errorRateAbove(0.5)` see\n * genuinely (method, finality)-specific health.\n */\n evalScope?: EvalScope | \"network\" | \"network-method\" | \"network-finality\" | \"network-method-finality\";\n /**\n * EvalPerBoundary scopes selection-policy evaluation by block-availability\n * \"lane\" \u2014 the set of upstreams whose configured block range can actually\n * serve the request's block. Unlike EvalPerMethod / EvalPerFinality this is\n * deliberately NOT folded into EvalScope: it must not change the\n * health-tracker grain (boundary is a decision/pool axis, not a metrics\n * axis), so the engine reads it directly as an orthogonal slot dimension.\n * When on, a request whose block excludes some upstream is evaluated\n * against a lane-scoped pool \u2014 an upstream that cannot serve the range is\n * absent from the pool (a capability fact), distinct from the policy's\n * soft health-based deprioritization. Default off. Pointer-typed so a\n * future SetDefaults can distinguish \"absent\" from \"explicitly false\".\n */\n evalPerBoundary?: boolean;\n evalTimeout?: Duration;\n /**\n * EvalFunc is the per-tick evaluation function. In YAML it's a JS\n * source string; in TS configs it's a real arrow function compiled\n * into `CompiledProgram` at config load.\n * Signature: `(upstreams, ctx) => Upstream[]`.\n * See specs/selection-policy/feature.md for the stdlib reference.\n */\n evalFunc?: SelectionPolicyEvalFunction | string;\n}\n/**\n * LegacySelectionPolicyFields mirrors the deprecated keys that used to\n * live on SelectionPolicyConfig. None survive translation.\n */\nexport interface LegacySelectionPolicyFields {\n evalFunction?: string;\n resampleExcluded?: boolean;\n resampleInterval?: Duration;\n resampleCount?: number /* int */;\n}\nexport type AuthType = string;\nexport const AuthTypeSecret: AuthType = \"secret\";\nexport const AuthTypeDatabase: AuthType = \"database\";\nexport const AuthTypeJwt: AuthType = \"jwt\";\nexport const AuthTypeSiwe: AuthType = \"siwe\";\nexport const AuthTypeNetwork: AuthType = \"network\";\nexport interface AuthConfig {\n strategies: TsAuthStrategyConfig[];\n}\nexport interface AuthStrategyConfig {\n ignoreMethods?: string[];\n allowMethods?: string[];\n rateLimitBudget?: string;\n /**\n * AllowClientDirectives, if set, overrides the project-level\n * `allowClientDirectives` pattern for users authenticated by THIS strategy.\n * Same wildcard syntax as the project-level field (\"*\" = all, \"\" = none).\n * Client directives (`X-ERPC-*` headers) are powerful per-request overrides \u2014\n * e.g. pinning an upstream bypasses the selection policy, and skipping the\n * cache multiplies upstream load \u2014 so operators exposing erpc directly to\n * untrusted callers typically deny them project-wide and re-enable them only\n * for trusted strategies:\n * \tallowClientDirectives: \"\" # project default: nobody\n * \tauth.strategies[0].allowClientDirectives: \"*\" # this strategy: everything\n * Left unset the caller inherits the project-level pattern, so existing\n * configs are unaffected. The capability is attached to the user by the\n * strategy that authenticated them, which means it can never be granted by\n * `trustUserIdHeader` (that path sets only Id \u2014 see\n * NormalizedRequest.SetUserFromTrustedHeader).\n */\n allowClientDirectives?: string;\n type: TsAuthType;\n network?: NetworkStrategyConfig;\n secret?: SecretStrategyConfig;\n database?: DatabaseStrategyConfig;\n jwt?: JwtStrategyConfig;\n siwe?: SiweStrategyConfig;\n}\nexport interface SecretStrategyConfig {\n id: string;\n value: string;\n /**\n * RateLimitBudget, if set, is applied to the authenticated user from this strategy\n */\n rateLimitBudget?: string;\n}\nexport interface DatabaseStrategyConfig {\n connector?: ConnectorConfig;\n cache?: DatabaseStrategyCacheConfig;\n retry?: DatabaseRetryConfig;\n failOpen?: DatabaseFailOpenConfig;\n maxWait?: Duration;\n}\nexport interface DatabaseStrategyCacheConfig {\n ttl?: number /* time in nanoseconds (time.Duration) */;\n maxSize?: number /* int64 */;\n maxCost?: number /* int64 */;\n numCounters?: number /* int64 */;\n}\nexport interface DatabaseRetryConfig {\n maxAttempts?: number /* int */;\n baseBackoff?: Duration;\n}\nexport interface DatabaseFailOpenConfig {\n enabled: boolean;\n userId?: string;\n rateLimitBudget?: string;\n}\nexport interface JwtStrategyConfig {\n allowedIssuers: string[];\n allowedAudiences: string[];\n allowedAlgorithms: string[];\n requiredClaims: string[];\n claimMatchers?: { [key: string]: string[]};\n verificationKeys?: { [key: string]: string};\n verificationJwksUrl?: string;\n verificationJwksRefreshInterval?: Duration;\n /**\n * Skipping TLS verification is an explicit operator opt-in for the JWKS\n * fetch, not a hardcoded bypass.\n */\n verificationJwksTlsInsecureSkipVerify?: boolean;\n /**\n * RateLimitBudgetClaimName is the JWT claim name that, if present,\n * will be used to set the per-user RateLimitBudget override.\n * Defaults to \"rlm\".\n */\n rateLimitBudgetClaimName?: string;\n}\nexport interface SiweStrategyConfig {\n allowedDomains: string[];\n /**\n * RateLimitBudget, if set, is applied to the authenticated user\n */\n rateLimitBudget?: string;\n}\nexport interface NetworkStrategyConfig {\n allowedIPs: string[];\n allowedCIDRs: string[];\n allowLocalhost: boolean;\n trustedProxies: string[];\n /**\n * RateLimitBudget, if set, is applied to the authenticated user (client IP)\n */\n rateLimitBudget?: string;\n ipAsUser?: boolean;\n}\nexport type LabelMode = string;\nexport const ErrorLabelModeVerbose: LabelMode = \"verbose\";\nexport const ErrorLabelModeCompact: LabelMode = \"compact\";\nexport interface MetricsConfig {\n enabled?: boolean;\n listenV4?: boolean;\n hostV4?: string;\n listenV6?: boolean;\n hostV6?: string;\n port?: number /* int */;\n errorLabelMode?: LabelMode;\n histogramBuckets?: string;\n /**\n * Customizations is the single knob for shaping /metrics: which metric\n * families are exposed at all, which of their labels survive, and which\n * buckets a histogram uses. Entries are applied by specificity rather than\n * by list order \u2014 see MetricsCustomizationConfig.\n * \tmetrics:\n * \t customizations:\n * \t - subject: \"consensus_*\"\n * \t action: drop\n * \t - subject: upstream_request_total\n * \t labels:\n * \t - subject: \"agent_*\"\n * \t action: drop\n * \t - subject: agent_name\n * \t action: keep\n * \t - subject: network_request_duration_seconds\n * \t buckets: [0.05, 0.5, 5]\n */\n customizations?: (MetricsCustomizationConfig | undefined)[];\n /**\n * Deprecated: use Customizations with a `labels` list. Kept working so\n * existing configs keep loading; it is desugared onto the same rules as an\n * every-histogram label drop.\n */\n histogramDropLabels?: string[];\n /**\n * Deprecated: use Customizations with an exact `subject` and a `labels` list\n * keeping what this metric needs.\n */\n histogramLabelOverrides?: { [key: string]: string[]};\n /**\n * Deprecated: use Customizations with a `labels` list. Kept working so\n * existing configs keep loading; it is desugared onto the same rules as an\n * every-counter label drop.\n */\n counterDropLabels?: string[];\n /**\n * Deprecated: use Customizations with an exact `subject` and a `labels` list\n * keeping what this metric needs.\n */\n counterLabelOverrides?: { [key: string]: string[]};\n /**\n * CounterIdleEvictionAfter bounds /metrics cardinality for hot-path\n * counters whose label-sets are keyed by caller-controlled inputs\n * (method, user, agentName, ...). Counter series idle for at least this\n * duration are evicted from the Prometheus registry (DeleteLabelValues)\n * by the health tracker's idle sweep; a series that becomes active again\n * restarts at zero \u2014 the same semantics rate()/increase() consumers\n * already handle across process restarts. Defaults to 24h (conservative:\n * only clearly-dead label combinations are released). Set to 0 to\n * disable eviction entirely.\n */\n counterIdleEvictionAfter?: Duration;\n}\n/**\n * MetricCustomizationAction is what a customization entry does to what it\n * selects.\n */\nexport type MetricCustomizationAction = string;\nexport const MetricActionKeep: MetricCustomizationAction = \"keep\";\nexport const MetricActionDrop: MetricCustomizationAction = \"drop\";\n/**\n * MetricsCustomizationConfig is one entry of metrics.customizations: a subject\n * selecting metric families, and what to do with them.\n * Overlapping subjects resolve by specificity, not by list order: an exact\n * family name beats a prefix, a longer prefix beats a shorter one, and equally\n * specific subjects break to the one written later. So \"drop consensus_*, keep\n * consensus_duration_seconds\" means the same thing whichever order it is written\n * in.\n */\nexport interface MetricsCustomizationConfig {\n /**\n * Subject selects metric families: an exact name (\"upstream_request_total\"),\n * a prefix ending in \"*\" (\"consensus_*\"), or \"*\" for every family. The\n * \"erpc_\" namespace prefix is optional. The Go runtime, process and promhttp\n * collectors are named in full (\"go_goroutines\") and are subject to the same\n * rules, so `subject: \"*\", action: drop` drops them too.\n */\n subject: string;\n /**\n * Action drops the matched families from /metrics, or keeps them against a\n * broader drop. Omit it to leave exposure alone and only customize labels or\n * buckets.\n * A dropped eRPC family is never registered, so it costs no series and no\n * collection time \u2014 but that makes it a startup decision, undone only by a\n * restart. Stock collectors are registered outside eRPC and so are filtered\n * out of the scrape response instead, which shrinks the page without saving\n * collection.\n */\n action?: 'keep' | 'drop';\n /**\n * Labels projects the matched families' label sets. Same precedence rules as\n * Subject, applied to label names: `agent_*: drop` then `agent_name: keep`\n * drops the group and spares the one label.\n * Dropping a label collapses every series that differed only in it. Counter\n * sums stay correct, but the dimension stops being queryable \u2014 check what\n * reads it (billing or attribution pipelines, dashboards) first. Gauges have\n * no projection, because collapsing gauge series would report whichever\n * writer wrote last rather than a coarser number.\n */\n labels?: (MetricLabelCustomizationConfig | undefined)[];\n /**\n * Buckets replaces the bucket boundaries of the matched histograms,\n * overriding both metrics.histogramBuckets and what the metric declares in\n * code. Must be strictly increasing.\n */\n buckets?: number /* float64 */[];\n}\n/**\n * MetricLabelCustomizationConfig keeps or drops one label, or a \"*\"-terminated\n * group of them, on the families its parent customization matched.\n */\nexport interface MetricLabelCustomizationConfig {\n subject: string;\n action: 'keep' | 'drop';\n}\n/**\n * RateLimitStoreConfig defines where rate limit counters are stored\n */\nexport interface RateLimitStoreConfig {\n driver: string; // \"redis\" | \"memory\"\n redis?: RedisConnectorConfig;\n cacheKeyPrefix?: string;\n nearLimitRatio?: number /* float32 */;\n /**\n * OnFailure decides a request the redis store could not: not connected,\n * erroring, slower than redis.getTimeout, or at its in-flight cap.\n */\n onFailure?: RateLimitStoreFailureMode;\n /**\n * LocalLimitFactor scales every maxCount for the in-process limiter that\n * onFailure: local decides with. It counts this replica only, so set it\n * near 1/replicas. Required for local; must be > 0 and <= 1.\n */\n localLimitFactor?: number /* float64 */;\n}\n/**\n * RateLimitStoreFailureMode is what a redis-backed budget does with a request\n * its store could not decide.\n */\nexport type RateLimitStoreFailureMode = string;\n/**\n * RateLimitStoreFailureAllow (default) admits the request.\n */\nexport const RateLimitStoreFailureAllow: RateLimitStoreFailureMode = \"allow\";\n/**\n * RateLimitStoreFailureDeny rejects the request as rate limited.\n */\nexport const RateLimitStoreFailureDeny: RateLimitStoreFailureMode = \"deny\";\n/**\n * RateLimitStoreFailureLocal decides it with an in-process limiter that\n * counts every request, at maxCount \u00D7 localLimitFactor.\n */\nexport const RateLimitStoreFailureLocal: RateLimitStoreFailureMode = \"local\";\n\n//////////\n// source: config_integrity.go\n\n/**\n * IntegrityHeaderModeOff ignores per-request integrity headers entirely.\n */\nexport const IntegrityHeaderModeOff = \"off\";\n/**\n * IntegrityHeaderModeProfiles lets a request select only a named profile.\n */\nexport const IntegrityHeaderModeProfiles = \"profiles\";\n/**\n * IntegrityHeaderModeFull lets a request set a level / per-check overrides.\n */\nexport const IntegrityHeaderModeFull = \"full\";\n/**\n * IntegritySettings is the reusable body of an integrity configuration: the\n * front-door level plus the axes (checks / budget) and the per-finality\n * verdict. The top-level/per-network blocks and every named profile share this\n * shape \u2014 only IntegrityConfig adds the header surface and profiles on top.\n */\nexport interface IntegritySettings {\n /**\n * Level is the front-door preset: off | intrinsic | corroborated | authoritative.\n */\n level?: string;\n /**\n * Checks overrides individual checks by their catalog id (enable/disable,\n * params, per-check failure mode).\n */\n checks?: { [key: string]: IntegrityCheckConfig | undefined};\n /**\n * Budget caps the canonical force-fetches the authoritative tier issues.\n */\n budget?: IntegrityBudgetConfig;\n /**\n * AutoCorrectWhenPossible controls whether a violation triggers a hunt for a\n * validated replacement (retry/hedge against other upstreams) BEFORE the\n * InvalidBehavior verdict applies. Default TRUE (nil = true). The two knobs\n * are orthogonal on purpose:\n * autoCorrectWhenPossible | invalidBehavior | on violation\n * ------------------------+-----------------+---------------------------------\n * true | recordOnly | seek replacement; serve it if one\n * | | validates; exhausted -> serve the\n * | | flagged ORIGINAL (never an error)\n * true | hardReject | seek replacement; exhausted -> error\n * false | recordOnly | serve original immediately + record\n * false | hardReject | fail immediately, no alternates\n */\n autoCorrectWhenPossible?: boolean;\n /**\n * InvalidBehavior is the per-finality verdict for reorg-sensitive checks,\n * where invalid data is ambiguously a node bug or a reorg: recordOnly |\n * hardReject | off. Deterministic checks ignore it and always hardReject\n * (override per check via checks..onFailure).\n */\n invalidBehavior?: IntegrityInvalidBehaviorConfig;\n /**\n * ReorgWindow is how many blocks back from the tip the integrity ChainView\n * keeps a number\u2192hash pin + header and tracks reorgs (default 32). Raise for\n * deep-reorg chains (e.g. polygon 256).\n */\n reorgWindow?: number /* int */;\n /**\n * ObserveOnly runs every enabled check but never lets a verdict touch the\n * response: violations that WOULD have been rejected are recorded with the\n * outcome \"would_reject\" and served anyway. This is the safe way to enable\n * integrity on a network for the first time \u2014 it reveals both bad upstream\n * data AND the module's own gaps on that chain at zero request risk, and\n * the would_reject rate is exactly the client-facing cost enforcement\n * would incur.\n * It is ABSOLUTE and overrides everything else, including a per-check\n * onFailure: reject and invalidBehavior \u2014 so a future release adding a new\n * check cannot start rejecting on an observe-only network. Deterministic\n * checks are covered too, which invalidBehavior alone cannot do (they\n * ignore it by design).\n */\n observeOnly?: boolean;\n /**\n * StateProbe proves, per upstream, that the node actually holds the state\n * trie it claims: on each new followed block, probe every upstream with an\n * execution-context call (and eth_getProof where supported) verified\n * against the follower's verified header, and advance that upstream's\n * state-proven head only on success. The probes PUBLISH evidence, they\n * never route: proofs feed the state-proven telemetry, and a sustained\n * streak of wrong-height answers is recorded as upstream misbehavior on\n * the health tracker \u2014 the same ledger consensus disputes and integrity\n * rejects feed \u2014 for the selection policy to act on (e.g. the\n * misbehaviorRateAbove predicate); absence of proof has no effect at all,\n * and ObserveOnly suppresses the misbehavior recording like every other\n * integrity effect. Requires follow to be enabled (the verified header is\n * the trust anchor). Off by default.\n */\n stateProbe?: IntegrityStateProbeConfig;\n /**\n * Follow turns the ChainView into an actual chain follower: it walks the\n * chain forward block by block, requires each block to name its\n * predecessor's hash, and reconciles reorgs by finding the common ancestor.\n * Off by default \u2014 it costs one eth_getBlockByNumber per block per network.\n */\n follow?: IntegrityFollowConfig;\n /**\n * MisbehaviorsDestination durably archives every integrity catch (reject or\n * soft-flag) as a JSONL record \u2014 same file/S3 destination shape as the\n * consensus policy's misbehaviorsDestination. Catches are rare, so the\n * volume is low; the archive is the forensic record metrics can't carry.\n */\n misbehaviorsDestination?: MisbehaviorsDestinationConfig;\n}\n/**\n * IntegrityFollowConfig configures the ChainView chain follower.\n * Following gives the module a CONTIGUOUS, parent-linked view of the chain\n * instead of the sparse scatter of heights that client traffic happens to\n * touch. That is what lets it answer \"is this block canonical\" from a verified\n * chain rather than from whichever source was observed first, and it is the\n * precondition for any check that compares consecutive headers.\n */\nexport interface IntegrityFollowConfig {\n /**\n * Enabled turns the follower on for this network. Default false.\n */\n enabled?: boolean;\n /**\n * Interval is how often to check for new blocks (default 1s). Keep it below\n * the chain's block time or the follower will run a step behind.\n */\n interval?: Duration;\n /**\n * MaxBlocksPerTick bounds catch-up work per tick (default 16), so a\n * follower starting far behind converges steadily instead of bursting.\n */\n maxBlocksPerTick?: number /* int */;\n}\n/**\n * IntegrityStateProbeConfig configures the per-upstream state-trie probes.\n */\nexport interface IntegrityStateProbeConfig {\n /**\n * Enabled turns the probes on. Default false.\n */\n enabled?: boolean;\n /**\n * Interval is the minimum time between probes of the same upstream\n * (default 2s) \u2014 a bound on probe cost, not a schedule; probes fire on\n * followed-block arrival.\n */\n interval?: Duration;\n}\n/**\n * IntegrityConfig is an IntegritySettings plus the per-request header-control\n * surface and the named profiles a request may select. It lives at the project\n * level (applies to all networks) and the network level (overrides).\n */\nexport interface IntegrityConfig {\n integritysettings: IntegritySettings;\n /**\n * HeaderMode controls whether/how per-request X-ERPC-Integrity headers may\n * adjust integrity: off | profiles | full. Defaults to off.\n */\n headerMode?: string;\n /**\n * Profiles are named settings an operator defines; a request may select one\n * by name via header when HeaderMode permits.\n */\n profiles?: { [key: string]: IntegritySettings | undefined};\n}\n/**\n * IntegrityCheckConfig overrides a single check by its catalog id.\n */\nexport interface IntegrityCheckConfig {\n /**\n * Enabled forces the check on/off; nil leaves the level's decision intact.\n */\n enabled?: boolean;\n /**\n * Params are check-specific knobs (e.g. bloom equality-vs-superset).\n */\n params?: { [key: string]: string};\n /**\n * OnFailure overrides this check's failure mode: recordOnly | hardReject.\n */\n onFailure?: string;\n}\n/**\n * IntegrityBudgetConfig caps the authoritative tier's canonical fetches.\n */\nexport interface IntegrityBudgetConfig {\n maxPerSecond?: number /* int */;\n maxConcurrent?: number /* int */;\n}\n/**\n * IntegrityInvalidBehaviorConfig is the per-finality verdict for reorg-sensitive\n * checks: recordOnly | hardReject | off, split by whether the response's block\n * is finalized. The verdict describes what happens when no valid answer is\n * obtainable; whether a replacement is sought first is AutoCorrectWhenPossible.\n */\nexport interface IntegrityInvalidBehaviorConfig {\n finalized?: string;\n unfinalized?: string;\n}\n\n//////////\n// source: data.go\n\nexport type DataFinalityState = number /* int */;\n/**\n * Finalized gets 0 intentionally so that when user has not specified finality,\n * it defaults to finalized, which is safest sane default for caching.\n * This attribute will be calculated based on extracted block number (from request and/or response)\n * and comparing to the upstream (one that returned the response) 'finalized' block (fetch via evm state poller).\n */\nexport const DataFinalityStateFinalized: DataFinalityState = 0;\n/**\n * When we CAN determine the block number, and it's after the upstream 'finalized' block, we consider the data unfinalized.\n */\nexport const DataFinalityStateUnfinalized: DataFinalityState = 1;\n/**\n * Certain methods points are meant to be realtime and updated with every new block (e.g. eth_gasPrice).\n * These can be cached with short TTLs to improve performance.\n */\nexport const DataFinalityStateRealtime: DataFinalityState = 2;\n/**\n * When we CANNOT determine the block number (e.g some trace by hash calls), we consider the data unknown.\n * Most often it is safe to cache this data for longer as they're access when block hash is provided directly.\n */\nexport const DataFinalityStateUnknown: DataFinalityState = 3;\n/**\n * DataFinalityStateAll is the internal wildcard sentinel used by the\n * health tracker to key its cross-finality aggregate rollups. NOT a\n * valid user-facing finality \u2014 never set on a request, never returned\n * from `Finality()`. Lives here so the tracker (health/) and the\n * policy engine (internal/policy/) can both reference it without a\n * cycle. Negative-valued so it sorts before any real finality bucket\n * and doesn't collide with the iota-based enum values.\n */\nexport const DataFinalityStateAll: DataFinalityState = -1;\nexport type CacheEmptyBehavior = number /* int */;\nexport const CacheEmptyBehaviorIgnore: CacheEmptyBehavior = 0;\nexport const CacheEmptyBehaviorAllow: CacheEmptyBehavior = 1;\nexport const CacheEmptyBehaviorOnly: CacheEmptyBehavior = 2;\n/**\n * CachePolicyAppliesTo controls whether a cache policy applies to get, set, or both operations.\n */\nexport type CachePolicyAppliesTo = string;\nexport const CachePolicyAppliesToBoth: CachePolicyAppliesTo = \"both\";\nexport const CachePolicyAppliesToGet: CachePolicyAppliesTo = \"get\";\nexport const CachePolicyAppliesToSet: CachePolicyAppliesTo = \"set\";\n\n//////////\n// source: error_extractor.go\n\n/**\n * JsonRpcErrorExtractor allows callers to inject architecture-specific\n * JSON-RPC error normalization logic into HTTP clients without creating\n * package import cycles.\n */\nexport type JsonRpcErrorExtractor = any;\n/**\n * JsonRpcErrorExtractorFunc is an adapter to allow normal functions to be used\n * as JsonRpcErrorExtractor implementations.\n * Similar to http.HandlerFunc style adapters.\n */\nexport type JsonRpcErrorExtractorFunc = any;\n\n//////////\n// source: evm_local_answers.go\n\n/**\n * EvmRoutableUpstreamsProvider is the optional surface a network exposes its\n * routable upstreams through: the selection policy's eligible set in routing\n * order, narrowed to the request's use-upstream selector. Asserted narrowly,\n * like EvmStateProvenWriter, so EvmNetwork implementations need not change.\n */\nexport type EvmRoutableUpstreamsProvider = any;\n\n//////////\n// source: exec_state.go\n\n/**\n * UpstreamAttemptOutcome enumerates the possible per-attempt outcomes\n * recorded against an upstream. The set is closed: every attempt ends\n * in exactly one of these.\n */\nexport type UpstreamAttemptOutcome = string;\nexport const UpstreamOutcomeSuccess: UpstreamAttemptOutcome = \"success\";\nexport const UpstreamOutcomeEmpty: UpstreamAttemptOutcome = \"empty\";\nexport const UpstreamOutcomeTransportError: UpstreamAttemptOutcome = \"transport_error\";\nexport const UpstreamOutcomeServerError: UpstreamAttemptOutcome = \"server_error\";\nexport const UpstreamOutcomeClientError: UpstreamAttemptOutcome = \"client_error\";\nexport const UpstreamOutcomeRateLimited: UpstreamAttemptOutcome = \"rate_limited\";\nexport const UpstreamOutcomeMissingData: UpstreamAttemptOutcome = \"missing_data\";\nexport const UpstreamOutcomeExecRevert: UpstreamAttemptOutcome = \"exec_revert\";\nexport const UpstreamOutcomeBlockUnavailable: UpstreamAttemptOutcome = \"block_unavailable\";\nexport const UpstreamOutcomeBreakerOpen: UpstreamAttemptOutcome = \"breaker_open\";\nexport const UpstreamOutcomeCancelled: UpstreamAttemptOutcome = \"cancelled\";\nexport const UpstreamOutcomeTimeout: UpstreamAttemptOutcome = \"timeout\";\nexport const UpstreamOutcomeSkipped: UpstreamAttemptOutcome = \"skipped\";\n/**\n * UpstreamSelectionReason describes WHY a particular upstream was\n * selected for a given attempt. Operators use this to debug skew in\n * upstream-pick distribution (e.g. why is one upstream getting all\n * the hedge fan-out?).\n */\nexport type UpstreamSelectionReason = string;\nexport const SelectionReasonPrimary: UpstreamSelectionReason = \"primary\"; // initial pick\nexport const SelectionReasonRetry: UpstreamSelectionReason = \"retry\"; // network-scope retry\nexport const SelectionReasonHedge: UpstreamSelectionReason = \"hedge\"; // speculative hedge fan-out\nexport const SelectionReasonConsensusSlot: UpstreamSelectionReason = \"consensus_slot\"; // one consensus participant\nexport const SelectionReasonSweep: UpstreamSelectionReason = \"sweep\"; // try-all-upstreams iteration\n/**\n * UpstreamAttempt is one (upstream, attempt) record. The executors\n * append these as participants come and go so operators can answer\n * \"which upstreams were involved in this request, why were they\n * chosen, and what happened to them?\" without parsing trace data.\n * Won is flipped to true by the executor when this attempt's response\n * contributed to the final response returned to the client. For a\n * non-consensus request that's exactly one attempt (the winning one);\n * for consensus it's every participant whose vote landed in the\n * winning agreement group.\n */\nexport interface UpstreamAttempt {\n upstreamid: string;\n vendorname: string;\n startedat: any /* time.Time */;\n duration: number /* time in nanoseconds (time.Duration) */;\n outcome: UpstreamAttemptOutcome;\n reason: UpstreamSelectionReason;\n ishedge: boolean;\n isretry: boolean;\n won: boolean; // true when this attempt contributed to the response\n attemptidx: number /* int */; // 0-based attempt index within the parent loop\n errorcode: string; // ErrorCode string when Outcome is an error variant\n errordetail: string; // free-form short description (truncated)\n /**\n * CreditUnits is the vendor credit-unit cost this attempt accrued\n * (the upstream's resolved table \u2014 vendor defaults merged with config\n * overrides; vendors with no table default to a flat 1 credit per\n * request). 0 when the attempt provably never dialed the vendor\n * (skipped / breaker-open) or the vendor was opted out (\"*\": 0).\n */\n creditunits: number /* int64 */;\n}\n/**\n * ExecState centralizes the per-request execution counters and the\n * per-upstream attempt log. Created lazily on first access via\n * (*NormalizedRequest).ExecState().\n * All counters are atomic; the struct itself is safe for concurrent use.\n * Counter model \u2014 every executor increments its OWN scope only.\n * Snapshot derives the totals so \"forgot to increment the total\" is\n * impossible by construction. The derivation is NOT a flat sum because\n * the scopes are nested: each network rotation triggers exactly one\n * upstream invocation chain, so summing both would double-count\n * physical attempts.\n * \ttotal Attempts = UpstreamAttempts + CacheAttempts\n * \t (every physical call is counted at the deepest scope that\n * \t actually performed it \u2014 upstreams for HTTP, cache for connector\n * \t reads. NetworkAttempts is a separate rotation-count signal,\n * \t exposed as its own counter but NOT summed into the total.)\n * \ttotal Retries = sum of UpstreamRetries + NetworkRetries + CacheRetries\n * \ttotal Hedges = sum of UpstreamHedges + NetworkHedges + CacheHedges\n * \t (retries and hedges ARE different events at each scope \u2014 an\n * \t upstream-scope retry retries the SAME upstream, a network-scope\n * \t retry rotates to a NEW upstream. Summing is correct.)\n * Scope semantics:\n * - UpstreamAttempts: physical Forward calls to a single upstream's\n * transport (primary + retries + hedges within one upstream).\n * - NetworkAttempts: rotations across upstreams driven by the\n * network executor's retry / hedge / consensus loop. Each rotation\n * triggers one upstream invocation chain. Not summed into total.\n * - CacheAttempts: cache-connector reads/writes including\n * within-connector retries and hedges.\n */\nexport interface ExecState {\n /**\n * Per-scope counters. Each executor owns its OWN counter set and\n * MUST NOT touch another scope's counters.\n */\n upstreamattempts: any /* atomic.Int32 */;\n upstreamretries: any /* atomic.Int32 */;\n upstreamhedges: any /* atomic.Int32 */;\n networkattempts: any /* atomic.Int32 */;\n networkretries: any /* atomic.Int32 */;\n networkhedges: any /* atomic.Int32 */;\n cacheattempts: any /* atomic.Int32 */;\n cacheretries: any /* atomic.Int32 */;\n cachehedges: any /* atomic.Int32 */;\n /**\n * ConsensusSlots counts how many consensus participants ran.\n */\n consensusslots: any /* atomic.Int32 */;\n /**\n * ConsensusDisputes counts dispute events.\n */\n consensusdisputes: any /* atomic.Int32 */;\n /**\n * ConsensusLowParticipants counts low-participant events.\n */\n consensuslowparticipants: any /* atomic.Int32 */;\n startedat: any /* time.Time */;\n}\n/**\n * ExecStateSnapshot is a plain-int view of ExecState for log/span\n * labeling \u2014 captured at a point in time. Total Attempts/Retries/Hedges\n * are derived as the sum of per-scope counters at snapshot time.\n */\nexport interface ExecStateSnapshot {\n /**\n * Totals (derived: Upstream + Network + Cache).\n */\n attempts: number /* int */;\n retries: number /* int */;\n hedges: number /* int */;\n /**\n * Per-scope counters (each executor's own bookkeeping).\n */\n upstreamattempts: number /* int */;\n upstreamretries: number /* int */;\n upstreamhedges: number /* int */;\n networkattempts: number /* int */;\n networkretries: number /* int */;\n networkhedges: number /* int */;\n cacheattempts: number /* int */;\n cacheretries: number /* int */;\n cachehedges: number /* int */;\n consensusslots: number /* int */;\n consensusdisputes: number /* int */;\n consensuslowparticipants: number /* int */;\n startedat: any /* time.Time */;\n}\n\n//////////\n// source: network.go\n\nexport type NetworkArchitecture = string;\nexport const ArchitectureEvm: NetworkArchitecture = \"evm\";\nexport const ArchitectureSvm: NetworkArchitecture = \"svm\";\nexport type Network = any;\n/**\n * EvmNetwork is the EVM-specific view of a Network. Callers that need\n * block-number accessors or leader-upstream selection should go through the\n * EvmHighestLatestBlockNumber / EvmHighestFinalizedBlockNumber / EvmLeaderUpstream\n * helpers below, which type-assert and degrade to zero-value on mismatch.\n */\nexport type EvmNetwork = \n Network;\n/**\n * SvmNetwork is the SVM-specific view of a Network. Production Network\n * implementations satisfy this automatically when the underlying network is\n * SVM; EVM networks correctly fail the assertion.\n */\nexport type SvmNetwork = \n Network;\nexport type QuantileTracker = any;\nexport type TrackedMetrics = any;\n\n//////////\n// source: served_tip.go\n\n/**\n * ServedTipPick is the picker's output.\n */\nexport interface ServedTipPick {\n /**\n * Tip is the value to advertise as latest/finalized: the highest block\n * number that at least quorum of the inputs have already reached (all of\n * them when there are fewer inputs than the quorum), or 0 when there are\n * no valid inputs.\n */\n tip: number /* int64 */;\n /**\n * Freshest is the freshest CORROBORATED view: the 2nd-highest valid input\n * (or the only input when N=1) \u2014 the reference for the deliberate-lag\n * gauge (Freshest - Tip). Using the 2nd-highest instead of the raw max\n * means a single rogue far-future upstream cannot inflate the lag gauge.\n * The absolute per-upstream maxima remain observable via\n * erpc_upstream_latest_block_number.\n */\n freshest: number /* int64 */;\n /**\n * Max is the highest valid input: the raw ceiling no pick may exceed. It is\n * deliberately not corroborated \u2014 a ceiling a single rogue can only RAISE\n * cannot be used to make a caller serve anything the rogue chose \u2014 and it\n * is the one place a lone far-ahead head is still visible to callers.\n */\n max: number /* int64 */;\n /**\n * Inputs is the number of valid (> 0) heads.\n */\n inputs: number /* int */;\n}\n/**\n * ServedTipQuorum is how many voters must already have a block before it is\n * served: a fixed count (>= 1), or ServedTipQuorumMajority for floor(N/2)+1 of\n * the N voters. The zero value is DefaultServedTipQuorum. In YAML/JSON it is a\n * positive integer or the string \"majority\".\n */\nexport type ServedTipQuorum = number /* int */;\n/**\n * ServedTipQuorumMajority requires a strict majority of the voters.\n */\nexport const ServedTipQuorumMajority: ServedTipQuorum = -1;\n\n//////////\n// source: timeout_func.go\n\n/**\n * TimeoutFunc computes the timeout for a request. Returns nil when no\n * timeout applies (caller skips context.WithTimeout).\n */\nexport type TimeoutFunc = any;\n\n//////////\n// source: upstream.go\n\nexport type Scope = string;\n/**\n * Policies must be created with a \"network\" in mind,\n * assuming there will be many upstreams e.g. Retry might endup using a different upstream\n */\nexport const ScopeNetwork: Scope = \"network\";\n/**\n * Policies must be created with one only \"upstream\" in mind\n * e.g. Retry with be towards the same upstream\n */\nexport const ScopeUpstream: Scope = \"upstream\";\nexport type UpstreamType = string;\n/**\n * HealthTracker is an interface for tracking upstream health metrics.\n * `finality` is the DataFinalityState the request resolved to \u2014\n * `DataFinalityStateAll` when the caller can't determine finality\n * (legacy call sites, internal probes, batch retries). Cordon/Uncordon\n * stay finality-agnostic because cordoning is an admin-level decision\n * about an entire (upstream, method) pair, not a per-finality bucket.\n */\nexport type HealthTracker = any;\nexport type Upstream = any;\n\n//////////\n// source: user.go\n\n/**\n * User is the authenticated caller. Beyond identity (Id) it carries the\n * per-caller capabilities resolved at authentication time. Capability fields\n * are populated ONLY by auth strategies; the trusted-header path\n * (NormalizedRequest.SetUserFromTrustedHeader) sets Id alone, so an\n * unvalidated header can never grant a capability.\n */\nexport interface User {\n id: string;\n ratelimitbudget: string;\n /**\n * AllowClientDirectives is the client-directive wildcard pattern granted by\n * the strategy that authenticated this user. Nil means \"no strategy-level\n * override\" \u2014 the project-level pattern applies.\n */\n allowclientdirectives?: string;\n}\n\n//////////\n// source: validation.go\n\n/**\n * MaxGrpcConnPoolSize is the upper bound accepted for a gRPC connection-pool\n * size. It guards against a fat-fingered value opening an absurd number of\n * connections to each backing server; it is not a recommended operating point.\n */\nexport const MaxGrpcConnPoolSize = 256;\n", "// CAPITAL_SNAKE_CASE aliases for selection-policy `evalScope` and\n// `stickyPrimary({scope})`. These are the canonical names a TypeScript\n// config imports:\n//\n// import { NETWORK, NETWORK_FINALITY } from '@erpc/config'\n//\n// The string values are the kebab-case forms the Go side validates\n// against (see `common.EvalScope`). At policy-runtime load time, the\n// stdlib also installs these names as ambient globals in the sobek\n// runtime so they resolve identically inside an `evalFunc` body\n// without an explicit import \u2014 see `internal/policy/stdlib/install.go`.\n\nimport {\n EvalScopeNetwork,\n EvalScopeNetworkMethod,\n EvalScopeNetworkFinality,\n EvalScopeNetworkMethodFinality,\n} from \"./generated\";\n\n/** One primary per network \u2014 max cohesion across methods + finalities. Default. */\nexport const NETWORK = EvalScopeNetwork;\n/** One primary per `(network, method)`; finalities share. */\nexport const NETWORK_METHOD = EvalScopeNetworkMethod;\n/** One primary per `(network, finality)`; methods share. */\nexport const NETWORK_FINALITY = EvalScopeNetworkFinality;\n/** One primary per slot \u2014 independent per `(network, method, finality)`. */\nexport const NETWORK_METHOD_FINALITY = EvalScopeNetworkMethodFinality;\n\n// Finality bit-flag constants \u2014 used INSIDE an `evalFunc` body with\n// the chainable `when(mask, fn)` primitive. Compose with bitwise OR:\n//\n// .when(REALTIME | UNFINALIZED | UNKNOWN, u => u.stickyPrimary({...}))\n//\n// These are also installed as ambient globals at policy-runtime load\n// time, so an `evalFunc` body doesn't need to import them. The\n// importable versions exist for symmetry \u2014 and so a TypeScript config\n// that wants to compute a mask AT CONFIG-LOAD TIME (e.g. into a\n// derived constant outside the eval body) can do so without relying\n// on the runtime ambient.\n/** Bit 0 \u2014 request is reading live tip data (e.g. eth_blockNumber). */\nexport const REALTIME = 1 << 0;\n/** Bit 1 \u2014 request is reading a recent-but-not-yet-finalized block. */\nexport const UNFINALIZED = 1 << 1;\n/** Bit 2 \u2014 request is reading a finalized (no-reorg) block. */\nexport const FINALIZED = 1 << 2;\n/** Bit 3 \u2014 finality couldn't be determined for this request. */\nexport const UNKNOWN = 1 << 3;\n"], + "mappings": ";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;AC+DO,IAAM,kBAAgC;AAkBtC,IAAM,qBAAkC;AACxC,IAAM,kBAA+B;AACrC,IAAM,qBAAkC;AAExC,IAAM,yBAA0C;AAChD,IAAM,yBAA0C;AAChD,IAAM,4BAA6C;AAkDnD,IAAM,kBAAgC;AAsrBtC,IAAM,4BAAgD;AAQtD,IAAM,2BAA+C;AA4brD,IAAM,8CAAgF;AACtF,IAAM,8DAAgG;AACtG,IAAM,wDAA0F;AAChG,IAAM,sDAAwF;AAE9F,IAAM,sCAAgE;AACtE,IAAM,sDAAgF;AACtF,IAAM,gDAA0E;AAChF,IAAM,8CAAwE;AA0L9E,IAAM,wBAAyC;AAC/C,IAAM,wBAAyC;AAC/C,IAAM,sBAAuC;AAC7C,IAAM,qBAAsC;AAC5C,IAAM,sBAAuC;AAC7C,IAAM,uBAAwC;AAC9C,IAAM,sBAAuC;AAkc7C,IAAM,mBAA8B;AAMpC,IAAM,yBAAoC;AAM1C,IAAM,2BAAsC;AAO5C,IAAM,iCAA4C;AAqDlD,IAAM,iBAA2B;AAEjC,IAAM,cAAwB;AAC9B,IAAM,eAAyB;AAC/B,IAAM,kBAA4B;AAgelC,IAAM,6BAAgD;AAItD,IAAM,+BAAkD;AAKxD,IAAM,4BAA+C;AAKrD,IAAM,2BAA8C;AAYpD,IAAM,2BAA+C;AACrD,IAAM,0BAA8C;AACpD,IAAM,yBAA6C;AAmMnD,IAAM,kBAAuC;AAC7C,IAAM,kBAAuC;AAoF7C,IAAM,eAAsB;AAK5B,IAAM,gBAAuB;;;AC9tF7B,IAAM,UAAU;AAEhB,IAAM,iBAAiB;AAEvB,IAAM,mBAAmB;AAEzB,IAAM,0BAA0B;AAchC,IAAM,WAAW,KAAK;AAEtB,IAAM,cAAc,KAAK;AAEzB,IAAM,YAAY,KAAK;AAEvB,IAAM,UAAU,KAAK;;;AF+HrB,IAAM,eAAe,CAC1B,QACW;AACX,SAAO;AACT;", "names": [] } diff --git a/typescript/config/lib/types/policyEval.d.ts b/typescript/config/lib/types/policyEval.d.ts index 2ee535b51..c9f2b3055 100644 --- a/typescript/config/lib/types/policyEval.d.ts +++ b/typescript/config/lib/types/policyEval.d.ts @@ -428,6 +428,21 @@ export interface PolicyEvalUpstreamArray extends ReadonlyArray unknown): PolicyEvalUpstreamArray; partition(fn: (u: PolicyEvalUpstream) => unknown): [PolicyEvalUpstreamArray, PolicyEvalUpstreamArray]; unique(keyFn?: (u: PolicyEvalUpstream) => string): PolicyEvalUpstreamArray; diff --git a/typescript/config/lib/types/policyEval.d.ts.map b/typescript/config/lib/types/policyEval.d.ts.map index 0b2199211..cb757084f 100644 --- a/typescript/config/lib/types/policyEval.d.ts.map +++ b/typescript/config/lib/types/policyEval.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"policyEval.d.ts","sourceRoot":"","sources":["../../src/types/policyEval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C;;;GAGG;AACH,MAAM,MAAM,yBAAyB,GAAG;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;IACnC;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,sBAAsB,EAAE,MAAM,CAAC;IAC/B;;;;OAIG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,YAAY,GAAG;IAClD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,KAAK,GAAG,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IAEjC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;IAE5C;;;;;;;;;OASG;IACH,QAAQ,CAAC,eAAe,EAAE;QACxB,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG;YACzB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;YAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;SACxB,CAAC;KACH,CAAC;IAEF;;;;;OAKG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAEnD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,gBAAgB,CAAC,EAAE,cAAc,CAAC;IAC3C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEhC,2DAA2D;IAC3D,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAC7B,yBAAyB;IACzB,EAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,GAAG,GAAG,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,aAAa,GAAG,WAAW,GAAG,SAAS,CAAC;IACxE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAErB,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,+EAA+E;AAC/E,MAAM,MAAM,WAAW,GAAG,iBAAiB,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,CAAC,EAAE,kBAAkB,KAAK,OAAO,CAAC,GAAG;IACvE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,6DAA6D;AAC7D,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;AAEpD,0DAA0D;AAC1D,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;AAEjD;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,KAAK,CAAC;IAC3C,eAAe,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;IACxD,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,MAAM,CAAC;CAC7C,CAAC;AAEF,2DAA2D;AAC3D,MAAM,MAAM,sBAAsB,GAAG;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,qEAAqE;AACrE,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,kBAAkB,GAAG;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,gDAAgD;AAChD,MAAM,MAAM,aAAa,GAAG;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,UAAU,CAAC;CACvB,CAAC;AAEF,mCAAmC;AACnC,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;;;;;;;OASG;IACH,KAAK,CAAC,EACF,SAAS,GACT,gBAAgB,GAChB,kBAAkB,GAClB,yBAAyB,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,QAAQ,CAAC;CAC9B,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,QAAQ,CAAC;IAC5B;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,QAAQ,CAAC;CACpB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,uBAAuB,GAAG;IACpC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;;OAUG;IACH,IAAI,CAAC,EAAE,SAAS,GAAG,UAAU,GAAG,MAAM,CAAC;IACvC;;;;;;;;;OASG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,+DAA+D;AAC/D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,uBAAuB,KAAK,uBAAuB,CAAC;IACnE,WAAW,CAAC,EAAE,CAAC,CAAC,EAAE,uBAAuB,KAAK,uBAAuB,CAAC;IACtE,SAAS,CAAC,EAAE,CAAC,CAAC,EAAE,uBAAuB,KAAK,uBAAuB,CAAC;IACpE,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,uBAAuB,KAAK,uBAAuB,CAAC;CACnE,CAAC;AAEF,4CAA4C;AAC5C,MAAM,MAAM,WAAW,GAAG;IACxB,EAAE,CAAC,EAAE,OAAO,CAAC;IACb,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,CACrC,SAAS,EAAE,uBAAuB,KAC/B,OAAO,CAAC;AAEb;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,CAAC,EAAE,OAAO,CAAC;IACb,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CAC5B,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,WAAW,uBACf,SAAQ,aAAa,CAAC,kBAAkB,CAAC;IACzC,uDAAuD;IACvD,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAG1B,IAAI,CAAC,EAAE,EAAE,OAAO,GAAG,uBAAuB,CAAC;IAC3C,SAAS,CAAC,EAAE,EAAE,OAAO,GAAG,uBAAuB,CAAC;IAChD,KAAK,CAAC,GAAG,EAAE,UAAU,GAAG,uBAAuB,CAAC;IAChD,UAAU,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,uBAAuB,CAAC;IACjF,QAAQ,CAAC,CAAC,EAAE,OAAO,GAAG,uBAAuB,CAAC;IAC9C,aAAa,CAAC,CAAC,EAAE,OAAO,GAAG,uBAAuB,CAAC;IACnD,MAAM,CAAC,CAAC,EAAE,OAAO,GAAG,uBAAuB,CAAC;IAC5C,KAAK,CAAC,CAAC,EAAE,WAAW,GAAG,uBAAuB,CAAC;IAC/C,QAAQ,CAAC,CAAC,EAAE,WAAW,GAAG,uBAAuB,CAAC;IAGlD,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,CAAC;IACxD,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,CAAC;IACzD,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC1D,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,uBAAuB,CAAC;IAC/D,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC1D,cAAc,CAAC,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,uBAAuB,CAAC;IACpE,eAAe,CAAC,IAAI,EAAE,sBAAsB,GAAG,uBAAuB,CAAC;IACvE,WAAW,CAAC,IAAI,CAAC,EAAE,kBAAkB,GAAG,uBAAuB,CAAC;IAGhE,SAAS,CACP,SAAS,EAAE,mBAAmB,EAC9B,oBAAoB,CAAC,EAAE,MAAM,GAAG;QAAE,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GACnE,uBAAuB,CAAC;IAE3B;;;;;;;;;;;;;;;OAeG;IACH,eAAe,CACb,SAAS,EAAE,mBAAmB,EAC9B,cAAc,CAAC,EAAE,MAAM,GACtB,uBAAuB,CAAC;IAG3B,MAAM,CACJ,EAAE,EAAE,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,kBAAkB,EAAE,KAAK,OAAO,GACzE,uBAAuB,CAAC;IAC3B,SAAS,CACP,EAAE,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,OAAO,GACrC,CAAC,uBAAuB,EAAE,uBAAuB,CAAC,CAAC;IACtD,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,MAAM,GAAG,uBAAuB,CAAC;IAC3E,KAAK,CAAC,KAAK,EAAE,SAAS,kBAAkB,EAAE,GAAG,uBAAuB,CAAC;IACrE,SAAS,CAAC,KAAK,EAAE,SAAS,kBAAkB,EAAE,GAAG,uBAAuB,CAAC;IACzE,UAAU,CAAC,KAAK,EAAE,SAAS,kBAAkB,EAAE,GAAG,uBAAuB,CAAC;IAG1E;;;;;;;;OAQG;IACH,WAAW,CACT,IAAI,CAAC,EACD,YAAY,GACZ,CAAC,CAAC,CAAC,EAAE,kBAAkB,KAAK,YAAY,CAAC,EAC7C,IAAI,CAAC,EAAE,kBAAkB,GACxB,uBAAuB,CAAC;IAC3B,MAAM,CACJ,EAAE,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,MAAM,EACrC,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GACxB,uBAAuB,CAAC;IAC3B,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,MAAM,GAAG,uBAAuB,CAAC;IAC3E,aAAa,CACX,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAC/C,uBAAuB,CAAC;IAC3B,eAAe,IAAI,uBAAuB,CAAC;IAC3C,gBAAgB,IAAI,uBAAuB,CAAC;IAC5C,iBAAiB,IAAI,uBAAuB,CAAC;IAC7C,aAAa,IAAI,uBAAuB,CAAC;IACzC,qBAAqB,IAAI,uBAAuB,CAAC;IAGjD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAChD,QAAQ,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAG7C,aAAa,CAAC,IAAI,CAAC,EAAE,oBAAoB,GAAG,uBAAuB,CAAC;IAGpE,SAAS,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,uBAAuB,CAAC;IAC1E,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,uBAAuB,CAAC;IAC3E,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAG1D,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC5C,UAAU,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC/C,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC5C,UAAU,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC/C,IAAI,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IACzC,IAAI,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IACzC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAAC;IAG1C;;;;;;;;;;;;;;OAcG;IACH,aAAa,CAAC,IAAI,CAAC,EAAE,oBAAoB,GAAG,uBAAuB,CAAC;IACpE,YAAY,CACV,MAAM,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,kBAAkB,KAAK,OAAO,CAAC,EACtD,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GACzB,uBAAuB,CAAC;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,SAAS,CACP,MAAM,EAAE,UAAU,GAAG,eAAe,EACpC,SAAS,EAAE,OAAO,GAAG,wBAAwB,GAC5C,uBAAuB,CAAC;IAG3B,EAAE,CACA,IAAI,EAAE,OAAO,GAAG,CAAC,CAAC,GAAG,EAAE,uBAAuB,KAAK,OAAO,CAAC,EAC3D,MAAM,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,EACjE,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,GACjE,uBAAuB,CAAC;IAC3B,MAAM,CACJ,IAAI,EAAE,OAAO,GAAG,CAAC,CAAC,GAAG,EAAE,uBAAuB,KAAK,OAAO,CAAC,EAC3D,EAAE,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,GAC5D,uBAAuB,CAAC;IAC3B,SAAS,CACP,EAAE,EAAE,MAAM,SAAS,kBAAkB,EAAE,GACtC,uBAAuB,CAAC;IAC3B,YAAY,CACV,EAAE,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,GAC5D,uBAAuB,CAAC;IAC3B;;;;;;;;;;;;OAYG;IACH,IAAI,CACF,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,GAC5D,uBAAuB,CAAC;IAC3B,UAAU,CACR,OAAO,EACH,SAAS,kBAAkB,EAAE,GAC7B,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,SAAS,kBAAkB,EAAE,CAAC,GAC9D,uBAAuB,CAAC;IAC3B,SAAS,CACP,CAAC,EAAE,MAAM,EACT,EAAE,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,GAC5D,uBAAuB,CAAC;IAC3B,UAAU,CAAC,QAAQ,EAAE,kBAAkB,GAAG,uBAAuB,CAAC;IAGlE,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,IAAI,GAAG,uBAAuB,CAAC;IACzE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC7C,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,KAAK,GAAG,uBAAuB,CAAC;CAC9F;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,2BAA2B,GAAG,CACxC,SAAS,EAAE,uBAAuB,EAClC,GAAG,EAAE,iBAAiB,KACnB,SAAS,kBAAkB,EAAE,CAAC;AAWnC,OAAO,CAAC,MAAM,CAAC;IAEb;;;;OAIG;IACH,MAAM,cAAc,EAAE,YAAY,CAAC;IACnC;;;OAGG;IACH,MAAM,eAAe,EAAE,YAAY,CAAC;IACpC;;;OAGG;IACH,MAAM,mBAAmB,EAAE,YAAY,CAAC;IAGxC,SAAS,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC3D,SAAS,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC3D,SAAS,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC9D,SAAS,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC9D,SAAS,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IASjE,SAAS,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAoC1E,SAAS,qBAAqB,CAC5B,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,MAAM,GAAG,uBAAuB,GAChD,mBAAmB,CAAC;IAGvB,SAAS,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAClE,SAAS,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,mBAAmB,CAAC;IACnE,SAAS,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,mBAAmB,CAAC;IACpE,SAAS,2BAA2B,CAAC,OAAO,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAG3E,SAAS,YAAY,CAAC,CAAC,EAAE,MAAM,GAAG,mBAAmB,CAAC;IACtD,SAAS,YAAY,CAAC,CAAC,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAGtD;;;OAGG;IACH,SAAS,GAAG,CAAC,GAAG,KAAK,EAAE,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;IACnE;;;OAGG;IACH,SAAS,GAAG,CAAC,GAAG,KAAK,EAAE,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;IACnE,0BAA0B;IAC1B,SAAS,GAAG,CAAC,IAAI,EAAE,mBAAmB,GAAG,mBAAmB,CAAC;IAG7D,sDAAsD;IACtD,SAAS,aAAa,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC;IAC9C,+CAA+C;IAC/C,SAAS,iBAAiB,IAAI,OAAO,CAAC;IACtC,qEAAqE;IACrE,SAAS,UAAU,CAAC,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;IAOzC,yEAAyE;IACzE,MAAM,OAAO,EAAE,SAAS,CAAC;IACzB,6DAA6D;IAC7D,MAAM,cAAc,EAAE,gBAAgB,CAAC;IACvC,4DAA4D;IAC5D,MAAM,gBAAgB,EAAE,kBAAkB,CAAC;IAC3C,4EAA4E;IAC5E,MAAM,uBAAuB,EAAE,yBAAyB,CAAC;IAMzD,uEAAuE;IACvE,MAAM,QAAQ,EAAE,CAAC,CAAC;IAClB,uEAAuE;IACvE,MAAM,WAAW,EAAE,CAAC,CAAC;IACrB,+DAA+D;IAC/D,MAAM,SAAS,EAAE,CAAC,CAAC;IACnB,gEAAgE;IAChE,MAAM,OAAO,EAAE,CAAC,CAAC;CAClB"} \ No newline at end of file +{"version":3,"file":"policyEval.d.ts","sourceRoot":"","sources":["../../src/types/policyEval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C;;;GAGG;AACH,MAAM,MAAM,yBAAyB,GAAG;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;IACnC;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,sBAAsB,EAAE,MAAM,CAAC;IAC/B;;;;OAIG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,YAAY,GAAG;IAClD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,KAAK,GAAG,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IAEjC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;IAE5C;;;;;;;;;OASG;IACH,QAAQ,CAAC,eAAe,EAAE;QACxB,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG;YACzB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;YAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;SACxB,CAAC;KACH,CAAC;IAEF;;;;;OAKG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAEnD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,gBAAgB,CAAC,EAAE,cAAc,CAAC;IAC3C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEhC,2DAA2D;IAC3D,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAC7B,yBAAyB;IACzB,EAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,GAAG,GAAG,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,aAAa,GAAG,WAAW,GAAG,SAAS,CAAC;IACxE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAErB,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,+EAA+E;AAC/E,MAAM,MAAM,WAAW,GAAG,iBAAiB,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,CAAC,EAAE,kBAAkB,KAAK,OAAO,CAAC,GAAG;IACvE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,6DAA6D;AAC7D,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;AAEpD,0DAA0D;AAC1D,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;AAEjD;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,KAAK,CAAC;IAC3C,eAAe,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;IACxD,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,MAAM,CAAC;CAC7C,CAAC;AAEF,2DAA2D;AAC3D,MAAM,MAAM,sBAAsB,GAAG;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,qEAAqE;AACrE,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,kBAAkB,GAAG;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,gDAAgD;AAChD,MAAM,MAAM,aAAa,GAAG;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,UAAU,CAAC;CACvB,CAAC;AAEF,mCAAmC;AACnC,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;;;;;;;OASG;IACH,KAAK,CAAC,EACF,SAAS,GACT,gBAAgB,GAChB,kBAAkB,GAClB,yBAAyB,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,QAAQ,CAAC;CAC9B,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,QAAQ,CAAC;IAC5B;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,QAAQ,CAAC;CACpB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,uBAAuB,GAAG;IACpC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;;OAUG;IACH,IAAI,CAAC,EAAE,SAAS,GAAG,UAAU,GAAG,MAAM,CAAC;IACvC;;;;;;;;;OASG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,+DAA+D;AAC/D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,uBAAuB,KAAK,uBAAuB,CAAC;IACnE,WAAW,CAAC,EAAE,CAAC,CAAC,EAAE,uBAAuB,KAAK,uBAAuB,CAAC;IACtE,SAAS,CAAC,EAAE,CAAC,CAAC,EAAE,uBAAuB,KAAK,uBAAuB,CAAC;IACpE,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,uBAAuB,KAAK,uBAAuB,CAAC;CACnE,CAAC;AAEF,4CAA4C;AAC5C,MAAM,MAAM,WAAW,GAAG;IACxB,EAAE,CAAC,EAAE,OAAO,CAAC;IACb,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,CACrC,SAAS,EAAE,uBAAuB,KAC/B,OAAO,CAAC;AAEb;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,CAAC,EAAE,OAAO,CAAC;IACb,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CAC5B,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,WAAW,uBACf,SAAQ,aAAa,CAAC,kBAAkB,CAAC;IACzC,uDAAuD;IACvD,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAG1B,IAAI,CAAC,EAAE,EAAE,OAAO,GAAG,uBAAuB,CAAC;IAC3C,SAAS,CAAC,EAAE,EAAE,OAAO,GAAG,uBAAuB,CAAC;IAChD,KAAK,CAAC,GAAG,EAAE,UAAU,GAAG,uBAAuB,CAAC;IAChD,UAAU,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,uBAAuB,CAAC;IACjF,QAAQ,CAAC,CAAC,EAAE,OAAO,GAAG,uBAAuB,CAAC;IAC9C,aAAa,CAAC,CAAC,EAAE,OAAO,GAAG,uBAAuB,CAAC;IACnD,MAAM,CAAC,CAAC,EAAE,OAAO,GAAG,uBAAuB,CAAC;IAC5C,KAAK,CAAC,CAAC,EAAE,WAAW,GAAG,uBAAuB,CAAC;IAC/C,QAAQ,CAAC,CAAC,EAAE,WAAW,GAAG,uBAAuB,CAAC;IAGlD,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,CAAC;IACxD,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,CAAC;IACzD,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC1D,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,uBAAuB,CAAC;IAC/D,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC1D,cAAc,CAAC,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,uBAAuB,CAAC;IACpE,eAAe,CAAC,IAAI,EAAE,sBAAsB,GAAG,uBAAuB,CAAC;IACvE,WAAW,CAAC,IAAI,CAAC,EAAE,kBAAkB,GAAG,uBAAuB,CAAC;IAGhE,SAAS,CACP,SAAS,EAAE,mBAAmB,EAC9B,oBAAoB,CAAC,EAAE,MAAM,GAAG;QAAE,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GACnE,uBAAuB,CAAC;IAE3B;;;;;;;;;;;;;;;OAeG;IACH,eAAe,CACb,SAAS,EAAE,mBAAmB,EAC9B,cAAc,CAAC,EAAE,MAAM,GACtB,uBAAuB,CAAC;IAE3B;;;;;;;;;OASG;IACH,YAAY,CAAC,IAAI,EAAE;QACjB,KAAK,EAAE,OAAO,CAAC;QACf,IAAI,EAAE,mBAAmB,CAAC;QAC1B,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,uBAAuB,CAAC;IAG5B,MAAM,CACJ,EAAE,EAAE,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,kBAAkB,EAAE,KAAK,OAAO,GACzE,uBAAuB,CAAC;IAC3B,SAAS,CACP,EAAE,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,OAAO,GACrC,CAAC,uBAAuB,EAAE,uBAAuB,CAAC,CAAC;IACtD,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,MAAM,GAAG,uBAAuB,CAAC;IAC3E,KAAK,CAAC,KAAK,EAAE,SAAS,kBAAkB,EAAE,GAAG,uBAAuB,CAAC;IACrE,SAAS,CAAC,KAAK,EAAE,SAAS,kBAAkB,EAAE,GAAG,uBAAuB,CAAC;IACzE,UAAU,CAAC,KAAK,EAAE,SAAS,kBAAkB,EAAE,GAAG,uBAAuB,CAAC;IAG1E;;;;;;;;OAQG;IACH,WAAW,CACT,IAAI,CAAC,EACD,YAAY,GACZ,CAAC,CAAC,CAAC,EAAE,kBAAkB,KAAK,YAAY,CAAC,EAC7C,IAAI,CAAC,EAAE,kBAAkB,GACxB,uBAAuB,CAAC;IAC3B,MAAM,CACJ,EAAE,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,MAAM,EACrC,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GACxB,uBAAuB,CAAC;IAC3B,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,MAAM,GAAG,uBAAuB,CAAC;IAC3E,aAAa,CACX,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAC/C,uBAAuB,CAAC;IAC3B,eAAe,IAAI,uBAAuB,CAAC;IAC3C,gBAAgB,IAAI,uBAAuB,CAAC;IAC5C,iBAAiB,IAAI,uBAAuB,CAAC;IAC7C,aAAa,IAAI,uBAAuB,CAAC;IACzC,qBAAqB,IAAI,uBAAuB,CAAC;IAGjD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAChD,QAAQ,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAG7C,aAAa,CAAC,IAAI,CAAC,EAAE,oBAAoB,GAAG,uBAAuB,CAAC;IAGpE,SAAS,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,uBAAuB,CAAC;IAC1E,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,uBAAuB,CAAC;IAC3E,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAG1D,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC5C,UAAU,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC/C,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC5C,UAAU,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC/C,IAAI,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IACzC,IAAI,CAAC,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAAC;IACzC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAAC;IAG1C;;;;;;;;;;;;;;OAcG;IACH,aAAa,CAAC,IAAI,CAAC,EAAE,oBAAoB,GAAG,uBAAuB,CAAC;IACpE,YAAY,CACV,MAAM,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,kBAAkB,KAAK,OAAO,CAAC,EACtD,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GACzB,uBAAuB,CAAC;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,SAAS,CACP,MAAM,EAAE,UAAU,GAAG,eAAe,EACpC,SAAS,EAAE,OAAO,GAAG,wBAAwB,GAC5C,uBAAuB,CAAC;IAG3B,EAAE,CACA,IAAI,EAAE,OAAO,GAAG,CAAC,CAAC,GAAG,EAAE,uBAAuB,KAAK,OAAO,CAAC,EAC3D,MAAM,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,EACjE,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,GACjE,uBAAuB,CAAC;IAC3B,MAAM,CACJ,IAAI,EAAE,OAAO,GAAG,CAAC,CAAC,GAAG,EAAE,uBAAuB,KAAK,OAAO,CAAC,EAC3D,EAAE,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,GAC5D,uBAAuB,CAAC;IAC3B,SAAS,CACP,EAAE,EAAE,MAAM,SAAS,kBAAkB,EAAE,GACtC,uBAAuB,CAAC;IAC3B,YAAY,CACV,EAAE,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,GAC5D,uBAAuB,CAAC;IAC3B;;;;;;;;;;;;OAYG;IACH,IAAI,CACF,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,GAC5D,uBAAuB,CAAC;IAC3B,UAAU,CACR,OAAO,EACH,SAAS,kBAAkB,EAAE,GAC7B,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,SAAS,kBAAkB,EAAE,CAAC,GAC9D,uBAAuB,CAAC;IAC3B,SAAS,CACP,CAAC,EAAE,MAAM,EACT,EAAE,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,uBAAuB,GAC5D,uBAAuB,CAAC;IAC3B,UAAU,CAAC,QAAQ,EAAE,kBAAkB,GAAG,uBAAuB,CAAC;IAGlE,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,uBAAuB,KAAK,IAAI,GAAG,uBAAuB,CAAC;IACzE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC7C,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,KAAK,GAAG,uBAAuB,CAAC;CAC9F;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,2BAA2B,GAAG,CACxC,SAAS,EAAE,uBAAuB,EAClC,GAAG,EAAE,iBAAiB,KACnB,SAAS,kBAAkB,EAAE,CAAC;AAWnC,OAAO,CAAC,MAAM,CAAC;IAEb;;;;OAIG;IACH,MAAM,cAAc,EAAE,YAAY,CAAC;IACnC;;;OAGG;IACH,MAAM,eAAe,EAAE,YAAY,CAAC;IACpC;;;OAGG;IACH,MAAM,mBAAmB,EAAE,YAAY,CAAC;IAGxC,SAAS,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC3D,SAAS,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC3D,SAAS,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC9D,SAAS,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC9D,SAAS,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IASjE,SAAS,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAoC1E,SAAS,qBAAqB,CAC5B,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,MAAM,GAAG,uBAAuB,GAChD,mBAAmB,CAAC;IAGvB,SAAS,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAClE,SAAS,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,mBAAmB,CAAC;IACnE,SAAS,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,mBAAmB,CAAC;IACpE,SAAS,2BAA2B,CAAC,OAAO,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAG3E,SAAS,YAAY,CAAC,CAAC,EAAE,MAAM,GAAG,mBAAmB,CAAC;IACtD,SAAS,YAAY,CAAC,CAAC,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAGtD;;;OAGG;IACH,SAAS,GAAG,CAAC,GAAG,KAAK,EAAE,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;IACnE;;;OAGG;IACH,SAAS,GAAG,CAAC,GAAG,KAAK,EAAE,mBAAmB,EAAE,GAAG,mBAAmB,CAAC;IACnE,0BAA0B;IAC1B,SAAS,GAAG,CAAC,IAAI,EAAE,mBAAmB,GAAG,mBAAmB,CAAC;IAG7D,sDAAsD;IACtD,SAAS,aAAa,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC;IAC9C,+CAA+C;IAC/C,SAAS,iBAAiB,IAAI,OAAO,CAAC;IACtC,qEAAqE;IACrE,SAAS,UAAU,CAAC,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;IAOzC,yEAAyE;IACzE,MAAM,OAAO,EAAE,SAAS,CAAC;IACzB,6DAA6D;IAC7D,MAAM,cAAc,EAAE,gBAAgB,CAAC;IACvC,4DAA4D;IAC5D,MAAM,gBAAgB,EAAE,kBAAkB,CAAC;IAC3C,4EAA4E;IAC5E,MAAM,uBAAuB,EAAE,yBAAyB,CAAC;IAMzD,uEAAuE;IACvE,MAAM,QAAQ,EAAE,CAAC,CAAC;IAClB,uEAAuE;IACvE,MAAM,WAAW,EAAE,CAAC,CAAC;IACrB,+DAA+D;IAC/D,MAAM,SAAS,EAAE,CAAC,CAAC;IACnB,gEAAgE;IAChE,MAAM,OAAO,EAAE,CAAC,CAAC;CAClB"} \ No newline at end of file diff --git a/typescript/config/package.json b/typescript/config/package.json index c692ae7a1..635ffb636 100644 --- a/typescript/config/package.json +++ b/typescript/config/package.json @@ -1,6 +1,6 @@ { "name": "@erpc-cloud/config", - "version": "0.4.0", + "version": "0.5.0", "description": "Library of types for IDE autocompletion of erpc config in Typescript", "repository": { "type": "git", diff --git a/typescript/config/src/generated.ts b/typescript/config/src/generated.ts index 0d97d080f..611e55747 100644 --- a/typescript/config/src/generated.ts +++ b/typescript/config/src/generated.ts @@ -167,6 +167,15 @@ export const MaxShredInsertSlotLagThreshold: number /* int64 */ = 100; * so pre-multi-chain configs keep producing "svm:" IDs. */ export const SvmChainSolana = "solana"; +/** + * SvmHistory is the slots from which a cluster's Agave history records block + * fields. Below them Agave renders the field null whatever an archive holds; + * a nil bound means the cluster records the field from its first slot. + */ +export interface SvmHistory { + blocktimefrom?: number /* uint64 */; + blockheightfrom?: number /* uint64 */; +} ////////// // source: blocktime_adaptive_duration.go @@ -311,6 +320,22 @@ export interface WebSocketServerConfig { * connection instead (1013), so the client knows it missed data. */ subscriptionBufferSize?: number /* int */; + /** + * MaxConnections caps concurrent client connections on this instance + * (0 = unlimited). Unlike the per-IP cap it holds when client IPs are + * unknown or spoofed. + */ + maxConnections?: number /* int */; + /** + * MaxConnectionsPerIP caps concurrent client connections from one + * resolved client IP (0 = unlimited); see trustedIPForwarders. + */ + maxConnectionsPerIP?: number /* int */; + /** + * MaxSubscriptions caps active subscriptions across all connections on + * this instance (0 = unlimited). + */ + maxSubscriptions?: number /* int */; } /** * ExecutionHeadersMode controls how much per-request execution detail is @@ -1051,6 +1076,14 @@ export interface EvmUpstreamConfig { * upstream would have answered it. 0 (default) keeps the exact head comparison. */ headLagToleranceBlocks?: number /* int64 */; + /** + * PinLatestTag sends this upstream the network head as a concrete block number wherever a + * state call (eth_call, eth_getBalance, ...) asks for "latest" or omits the block, and skips the + * upstream when its blockAvailability upper bound is below that head or the head is unknown. + * For an upstream whose own head lags the network: it answers only blocks it holds, and a block + * it lacks fails as missing data so the next upstream answers, instead of an older "latest". + */ + pinLatestTag?: boolean; /** * Deprecated: never read at runtime. Configure data integrity via the network * `integrity` block instead. Retained only so existing YAML still parses. @@ -1065,6 +1098,30 @@ export interface EvmUpstreamConfig { */ maxAvailableRecentBlocks?: number /* int64 */; queryShim?: EvmQueryShimConfig; + /** + * Multicall3 packs eth_calls bound for this upstream that share a block into one + * Multicall3 aggregate3 eth_call, so a vendor billed per request is billed once + * per pack. Nil (default) sends every eth_call alone. + */ + multicall3?: EvmMulticall3Config; +} +/** + * EvmMulticall3Config bounds one pack. A call joins the open pack for its block, + * and the pack is sent MaxWait after its first call or when it holds MaxCalls. + * A call the pack cannot answer exactly (multicall failed, inner call reverted + * or returned nothing) is sent alone, as it would be without packing. + */ +export interface EvmMulticall3Config { + maxWait?: Duration; + maxCalls?: number /* int */; + /** + * BypassContracts are target addresses whose calls are never packed, for + * contracts that answer differently when the caller is Multicall3 instead + * of the zero address (msg.sender or caller-code checks) without reverting. + * Contracts that revert inside a pack but succeed alone are learned at + * runtime and need no entry. + */ + bypassContracts?: string[]; } /** * EvmQueryShimConfig lets an upstream serve the sub-requests (eth_getBlockByNumber, @@ -1616,9 +1673,9 @@ export interface SvmUpstreamConfig { /** * BlockAvailability is an operator restriction on the slot window this * upstream may serve. Only what the operator declares is enforced; the - * upstream's oldest stored block (getBlocksWithLimit(0, 1)) is probed and - * used as a lower anchor solely when a bound uses firstAvailableSlotPlus, - * mirroring EVM's earliestBlockPlus. Unknown floor = unbounded. + * upstream's oldest stored block (getBlocksWithLimit(0, 1)) is probed for + * every upstream and anchors a firstAvailableSlotPlus bound, mirroring EVM's + * earliestBlockPlus. Unknown floor = unbounded. */ blockAvailability?: SvmBlockAvailabilityConfig; } @@ -1633,12 +1690,15 @@ export interface SvmBlockAvailabilityConfig { } /** * SvmAvailabilityBoundConfig is a single declared slot bound. - * Exactly one field must be set. + * Exactly one of ExactSlot, LatestSlotMinus, FirstAvailableSlotPlus must be set. + * UpdateRate applies only to firstAvailableSlotPlus: how often the floor it + * anchors on is re-probed. 0 freezes it after the first observation, as EVM. */ export interface SvmAvailabilityBoundConfig { exactSlot?: number /* int64 */; latestSlotMinus?: number /* int64 */; firstAvailableSlotPlus?: number /* int64 */; + updateRate?: Duration; } export interface EvmNetworkConfig { chainId: number /* int64 */; @@ -1648,8 +1708,7 @@ export interface EvmNetworkConfig { /** * ServedTip configures how the network derives the "latest"/"finalized" * block it advertises to clients (and enforces via block-availability). - * Nil or disabled selects the default mode (the corroborated latest across eligible - * upstreams, see ServedTipPick.Freshest); set Enabled to opt into the majority tip. See + * Nil uses the defaults: quorum 2, regression guard on, no guarantees. See * EvmServedTipConfig. */ servedTip?: EvmServedTipConfig; @@ -1706,6 +1765,12 @@ export interface EvmNetworkConfig { * Default includes common point-lookup methods like eth_getBlockByNumber, eth_getTransactionByHash, etc. */ markEmptyAsErrorMethods?: string[]; + /** + * LocalAnswers turns off eRPC's own answer for a method on this network: + * `{net_peerCount: false}` routes net_peerCount to the upstreams. Every + * method in LocalAnswerMethods is answered by eRPC unless set false here. + */ + localAnswers?: { [key: string]: boolean}; /** * StripSubscribeFromBlockZero removes a zero `fromBlock` from eth_subscribe * logs filters before forwarding upstream. fromBlock has no meaning for a @@ -1721,6 +1786,16 @@ export interface EvmNetworkConfig { * Default: 0.7 (30% under the estimated block time). */ dynamicBlockTimeDebounceMultiplier?: number /* float64 */; + /** + * FinalizedDebounceDepthMultiplier sets how stale an upstream's finalized block may + * get before the state poller fetches it again, as a fraction of the chain's + * finality depth in time: (latest - finalized) x EMA block time x multiplier. The + * result applies only when it is longer than the regular debounce. Finalized + * advances once per finality window, so polling it at the latest-block cadence + * mostly returns the same number. 0 disables it (finalized uses the regular debounce). + * Default: 0.05 (refresh about 20 times per finality window). + */ + finalizedDebounceDepthMultiplier?: number /* float64 */; /** * BlockUnavailableDelayMultiplier scales the EMA-estimated block time to derive the * retry delay when the requested data isn't available yet (ErrUpstreamBlockUnavailable @@ -1755,7 +1830,7 @@ export interface EvmNetworkConfig { * (regardless of selection policy, tier or syncing state), without * dispatching it; it never fires while any upstream's head is unknown. * Clients that poll the next block by number otherwise walk every routed - * upstream on each poll. Always on when servedTip is enabled for "latest". + * upstream on each poll. * The head is what the state poller and head subscriptions last reported, so * a block an upstream produced since then gets null until eRPC sees it: only * enable it where heads arrive promptly (e.g. over WebSocket). @@ -1772,34 +1847,28 @@ export interface EvmNetworkConfig { /** * EvmServedTipConfig controls how the network derives the "latest"/"finalized" * block it advertises (and enforces) from its upstreams. - * In the default mode the served tip is the corroborated latest block across eligible - * non-syncing upstreams (second-highest, or the only one) — which can still advertise a block a slightly-ahead - * pair has, causing "block not found" churn when requests route to a - * slightly-behind upstream. When a tag is listed in EnabledFor, that tag's - * served value is instead the freshest block a strict MAJORITY of the eligible - * upstreams already have, so interpolated requests land on upstreams that can - * serve the advertised block. + * Both axes use one rule: among the eligible (selection-policy), non-syncing + * upstreams, only the best voter rank present votes (live, then benched, then + * static-capped, then benched and static-capped), and the served block is the + * highest one at least Quorum of those voters already have. The health + * tracker benches an upstream whose head stopped moving while another kept + * advancing, until it catches up again. */ export interface EvmServedTipConfig { /** - * EnabledFor lists the block tags whose served value uses the cluster-min tip - * instead of the default corroborated head. Valid entries: "latest" and "finalized" (the - * "safe" tag follows "finalized"). Empty selects the default mode for all tags. - */ - enabledFor?: string[]; - /** - * Deprecated: ClusterDelta configured the former cluster-based picker and is - * ignored — the majority order statistic needs no tuning. Kept only so - * existing configs keep parsing. + * Quorum is how many eligible upstreams must already have a block before it + * is served: an integer >= 1, or "majority" (floor(N/2)+1 of N). Unset + * uses DefaultServedTipQuorum (2). A ballot with fewer voters than the + * quorum serves the block all of them have. */ - clusterDelta?: number /* int64 */; + quorum?: number | 'majority'; /** * GuaranteedMethods lists method name patterns (glob; e.g. "trace_*", * "debug_traceBlockByNumber") whose supporting-upstream subset must be able to * serve the advertised latest. For a request on a matching method, "latest" * resolves against the majority of only the upstreams that support it * (membership auto-detected via ShouldHandleMethod — no per-upstream config). - * Empty means only the global (all-eligible) majority is computed. + * Empty means only the global ballot is computed. */ guaranteedMethods?: string[]; /** @@ -1814,35 +1883,17 @@ export interface EvmServedTipConfig { guaranteedFor?: string[]; /** * MaxRegressionBlocks is how far below the corroborated LIVE upstream head - * (the second-highest live head) the majority pick may fall before it is - * treated as a poisoned ballot rather than as reality. While a pick is below - * that bound the network keeps serving its last corroborated pick - * (in-process, for at most one minute, then it fails open and serves the - * pick as computed). 0 uses DefaultToleratedBlockHeadRollback (1024) — the - * same rollback tolerance the state poller and health tracker apply to a + * (the second-highest live head) the pick may fall before it is treated as + * a poisoned ballot rather than as reality. While a pick is below that + * bound the network keeps serving its last corroborated pick (in-process, + * for at most one minute, then it fails open and serves the pick as + * computed). 0 uses DefaultToleratedBlockHeadRollback (1024) — the same + * rollback tolerance the state poller and health tracker apply to a * retreating head, so all three layers agree on what counts as a genuine - * deep correction. -1 disables the regression guard entirely (the symmetric - * opposite of trajectoryWindow: 0); no other negative value is accepted. + * deep correction. -1 disables the regression guard entirely; no other + * negative value is accepted. */ maxRegressionBlocks?: number /* int64 */; - /** - * TrajectoryWindow is how much recorded head history the trajectory referee - * needs before it may participate in the pick. The referee tracks where this - * network's head has been over the window and, when a stalled group holds - * the majority, serves the corroborated group that is actually where the - * chain should be by now (see architecture/evm's TipTrajectory: advisory, - * upward-only, in-process, and a no-op until every confidence condition - * holds). Unset uses DefaultServedTipTrajectoryWindow (10m); an explicit 0 - * disables the referee entirely — nothing is recorded and the pick is the - * plain majority. Any other value must be between - * MinServedTipTrajectoryWindow and MaxServedTipTrajectoryWindow: below the - * minimum the window is self-defeating (it must be long enough that a stall - * cannot look like a trajectory), and above the maximum the sample ring can - * never span it, so the referee would stand down forever while looking - * configured. WRITE IT AS A DURATION STRING — trajectoryWindow: "10m". A - * bare number is parsed as MILLISECONDS. - */ - trajectoryWindow?: Duration; } /** * EvmIntegrityConfig is deprecated. Use DirectiveDefaultsConfig for validation settings. @@ -2194,7 +2245,36 @@ export interface RateLimitStoreConfig { redis?: RedisConnectorConfig; cacheKeyPrefix?: string; nearLimitRatio?: number /* float32 */; + /** + * OnFailure decides a request the redis store could not: not connected, + * erroring, slower than redis.getTimeout, or at its in-flight cap. + */ + onFailure?: RateLimitStoreFailureMode; + /** + * LocalLimitFactor scales every maxCount for the in-process limiter that + * onFailure: local decides with. It counts this replica only, so set it + * near 1/replicas. Required for local; must be > 0 and <= 1. + */ + localLimitFactor?: number /* float64 */; } +/** + * RateLimitStoreFailureMode is what a redis-backed budget does with a request + * its store could not decide. + */ +export type RateLimitStoreFailureMode = string; +/** + * RateLimitStoreFailureAllow (default) admits the request. + */ +export const RateLimitStoreFailureAllow: RateLimitStoreFailureMode = "allow"; +/** + * RateLimitStoreFailureDeny rejects the request as rate limited. + */ +export const RateLimitStoreFailureDeny: RateLimitStoreFailureMode = "deny"; +/** + * RateLimitStoreFailureLocal decides it with an in-process limiter that + * counts every request, at maxCount × localLimitFactor. + */ +export const RateLimitStoreFailureLocal: RateLimitStoreFailureMode = "local"; ////////// // source: config_integrity.go @@ -2460,6 +2540,17 @@ export type JsonRpcErrorExtractor = any; */ export type JsonRpcErrorExtractorFunc = any; +////////// +// source: evm_local_answers.go + +/** + * EvmRoutableUpstreamsProvider is the optional surface a network exposes its + * routable upstreams through: the selection policy's eligible set in routing + * order, narrowed to the request's use-upstream selector. Asserted narrowly, + * like EvmStateProvenWriter, so EvmNetwork implementations need not change. + */ +export type EvmRoutableUpstreamsProvider = any; + ////////// // source: exec_state.go @@ -2643,41 +2734,23 @@ export type TrackedMetrics = any; ////////// // source: served_tip.go -/** - * ServedTipInput is a single observation: an upstream's last-known tip block. - * Callers are responsible for excluding syncing or cordoned upstreams BEFORE - * passing observations here; the picker treats every input as a candidate. - */ -export interface ServedTipInput { - /** - * UpstreamID is preserved only for telemetry attribution. It is not used - * in the pick. - */ - upstreamid: string; - /** - * BlockNumber is the upstream's reported tip. Zero or negative values are - * treated as "no data yet" and filtered before picking. - */ - blocknumber: number /* int64 */; -} /** * ServedTipPick is the picker's output. */ export interface ServedTipPick { /** * Tip is the value to advertise as latest/finalized: the highest block - * number that a strict MAJORITY of the inputs have already reached, or 0 - * when there are no valid inputs. + * number that at least quorum of the inputs have already reached (all of + * them when there are fewer inputs than the quorum), or 0 when there are + * no valid inputs. */ tip: number /* int64 */; /** * Freshest is the freshest CORROBORATED view: the 2nd-highest valid input * (or the only input when N=1) — the reference for the deliberate-lag * gauge (Freshest - Tip). Using the 2nd-highest instead of the raw max - * means a single rogue far-future upstream cannot inflate the lag gauge - * (the problem the old velocity gate solved via MaxEligible: one - * wrong-chain endpoint used to make the gauge read hundreds of thousands - * of blocks). The absolute per-upstream maxima remain observable via + * means a single rogue far-future upstream cannot inflate the lag gauge. + * The absolute per-upstream maxima remain observable via * erpc_upstream_latest_block_number. */ freshest: number /* int64 */; @@ -2689,17 +2762,21 @@ export interface ServedTipPick { */ max: number /* int64 */; /** - * Inputs is the number of valid (BlockNumber > 0) observations. + * Inputs is the number of valid (> 0) heads. */ inputs: number /* int */; - /** - * Sorted is the valid inputs, DESCENDING by block number — the order - * statistic's own working slice, exposed so the trajectory referee can - * cluster the very same ballot without sorting it a second time. Nil when - * there are no valid inputs; never mutate it. - */ - sorted: ServedTipInput[]; } +/** + * ServedTipQuorum is how many voters must already have a block before it is + * served: a fixed count (>= 1), or ServedTipQuorumMajority for floor(N/2)+1 of + * the N voters. The zero value is DefaultServedTipQuorum. In YAML/JSON it is a + * positive integer or the string "majority". + */ +export type ServedTipQuorum = number /* int */; +/** + * ServedTipQuorumMajority requires a strict majority of the voters. + */ +export const ServedTipQuorumMajority: ServedTipQuorum = -1; ////////// // source: timeout_func.go diff --git a/typescript/config/src/types/policyEval.ts b/typescript/config/src/types/policyEval.ts index 51a9940cf..e1cf9aef7 100644 --- a/typescript/config/src/types/policyEval.ts +++ b/typescript/config/src/types/policyEval.ts @@ -471,6 +471,22 @@ export interface PolicyEvalUpstreamArray reasonOverride?: string, ): PolicyEvalUpstreamArray; + /** + * Per-method exclusion. For each upstream and each method it has samples + * for that matches `match`, `when` runs with `u.metrics` set to that + * method's numbers (`requestsTotal`, `latencyP`). A trip skips the + * upstream for that method only; it stays in rotation for every other + * method. When blocks would leave a method no upstream, all stay. + * Blocked upstreams get probe traffic for that method unless + * `probe: false`. Counted in `erpc_selection_method_block_total`. + * Returns the array unchanged. + */ + blockMethods(opts: { + match: Pattern; + when: PolicyEvalPredicate; + probe?: boolean; + }): PolicyEvalUpstreamArray; + // ─── 4.4 Generic functional ─────────────────────────────────────────── reject( fn: (u: PolicyEvalUpstream, i: number, a: PolicyEvalUpstream[]) => unknown, diff --git a/upstream/evm_upstream_ops.go b/upstream/evm_upstream_ops.go index 38bc2eed8..d1ade4f0d 100644 --- a/upstream/evm_upstream_ops.go +++ b/upstream/evm_upstream_ops.go @@ -10,6 +10,7 @@ import ( "math" "strconv" + "github.com/erpc/erpc/architecture/evm" "github.com/erpc/erpc/common" "github.com/erpc/erpc/telemetry" ) @@ -94,6 +95,11 @@ func (u *Upstream) EvmStatePoller() common.EvmStatePoller { return u.evmStatePoller } +// EvmMulticall3Batcher returns the eth_call packer, nil when evm.multicall3 is off. +func (u *Upstream) EvmMulticall3Batcher() *evm.Multicall3Batcher { + return u.evmMulticall3 +} + // EvmStateProvenBlock returns the highest block at which the integrity state // probe proved this upstream truly holds/executes that block's state. 0 means // nothing proven yet (probing off, warming up, or unsupported) — callers fall diff --git a/upstream/ratelimiter_budget.go b/upstream/ratelimiter_budget.go index 741e86eb3..5dd86dadf 100644 --- a/upstream/ratelimiter_budget.go +++ b/upstream/ratelimiter_budget.go @@ -5,7 +5,6 @@ import ( "fmt" "math" "sync" - "sync/atomic" "time" pb_struct "github.com/envoyproxy/go-control-plane/envoy/extensions/common/ratelimit/v3" @@ -18,6 +17,7 @@ import ( "github.com/rs/zerolog" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "google.golang.org/protobuf/types/known/wrapperspb" ) type RateLimiterBudget struct { @@ -35,24 +35,18 @@ type RateLimiterBudget struct { hasRuleCountMode bool // admission is a buffered semaphore that bounds the number of concurrent - // in-flight remote (Redis) DoLimit calls per budget. It exists because the - // underlying envoyproxy/ratelimit + radix client does not honor context - // cancellation: a goroutine spawned in doLimitWithTimeout that has timed - // out will continue to live until Redis finally answers (which can be - // seconds when the connection pool is contended). - // - // Without this cap we observed 25k+ leaked goroutines per machine during - // a Redis-rate-limiter contention spike (root-caused 2026-05-07 cronos - // receipts incident), which then drove CPU/GC into a death spiral. - // - // When admission is full, doLimitWithTimeout fail-opens immediately - // without spawning a goroutine — increments - // MetricRateLimiterRemoteAdmissionSheddedTotal so we can alert on it. - // - // Nil when no remote cache is in use (e.g. memory cache); only allocated - // when maxTimeout > 0. + // in-flight remote (Redis) DoLimit calls per budget; see doLimit. When it + // is full the request is a store failure ("admission_full") and no + // goroutine is spawned. Nil when no remote cache is in use. admission chan struct{} + // onFailure decides requests the remote store could not (see doLimit). + onFailure common.RateLimitStoreFailureMode + // local counts every request in-process at maxCount × localFactor, so + // onFailure: local decides with a warm counter. Nil unless that mode. + local limiter.RateLimitCache + localFactor float64 + // inflight gauge, kept here for fast hot-path access without a labels // lookup on every call. Refreshed at registration time. inflightGauge prometheus.Gauge @@ -165,12 +159,6 @@ func (b *RateLimiterBudget) AdjustBudgetByFactor(rule *RateLimitRule, factor flo return prev, next, true } -// ruleResult holds the result of evaluating a single rule. -type ruleResult struct { - rule *RateLimitRule - allowed bool -} - // getCache returns the current cache from the registry (thread-safe) func (b *RateLimiterBudget) getCache() limiter.RateLimitCache { return b.registry.GetCache() @@ -188,8 +176,9 @@ type PermitCost struct { // requestPermit is the default cost: one request, request counting. var requestPermit = PermitCost{Hits: 1, Mode: common.RateLimitCountModeRequest} -// TryAcquirePermit evaluates all matching rules for the given method using Envoy's DoLimit. -// Rules are evaluated in parallel for lower latency. Returns true if allowed, false if rate limited. +// TryAcquirePermit evaluates every rule matching method in one cache call, so a +// request costs one Redis pipeline and one admission slot however many rules it +// meets. Returns true if allowed, false if rate limited. // // cost is optional (only the first value is read; default one request hit). In // credit mode it carries the request's estimated vendor credit-unit cost, and @@ -205,24 +194,11 @@ func (b *RateLimiterBudget) TryAcquirePermit(ctx context.Context, projectId stri } // Whole-budget shortcut, valid only when no rule overrides the mode; // otherwise a request-counting rule would be skipped by a 0-credit caller. - // evaluateRule handles that case per rule. + // buildLimitRequest handles that case per rule. if permit.Hits == 0 && !b.hasRuleCountMode { return true, nil } - cache := b.getCache() - if cache == nil { - return true, nil // Fail-open when no cache is available - } - - ctx, span := common.StartDetailSpan(ctx, "RateLimiter.TryAcquirePermit", - trace.WithAttributes( - attribute.String("budget", b.Id), - attribute.String("method", method), - ), - ) - defer span.End() - rules, err := b.GetRulesByMethod(method) if err != nil { return false, err @@ -231,7 +207,6 @@ func (b *RateLimiterBudget) TryAcquirePermit(ctx context.Context, projectId stri return true, nil } - // Extract request metadata once var userLabel, agentName, networkLabel, finality, clientIP string if req != nil { userLabel = req.UserId() @@ -240,66 +215,155 @@ func (b *RateLimiterBudget) TryAcquirePermit(ctx context.Context, projectId stri finality = req.Finality(ctx).String() clientIP = req.ClientIP() } - - // Validate request context upfront for _, rule := range rules { if (rule.Config.PerIP || rule.Config.PerUser || rule.Config.PerNetwork) && req == nil { return false, fmt.Errorf("request cannot be nil when ratelimiter rule has perIP/perUser/perNetwork") } } - // Single rule: evaluate directly without goroutine overhead - if len(rules) == 1 { - allowed := b.evaluateRule(ctx, rules[0], method, clientIP, userLabel, networkLabel, permit) - if !allowed { - telemetry.CounterHandle( - telemetry.MetricRateLimitsTotal, - projectId, networkLabel, vendor, upstreamId, method, finality, - userLabel, agentName, b.Id, rules[0].Config.ScopeString(), authLabel, origin, - ).Inc() - } - return allowed, nil + ctx, span := common.StartDetailSpan(ctx, "RateLimiter.TryAcquirePermit", + trace.WithAttributes( + attribute.String("budget", b.Id), + attribute.String("method", method), + ), + ) + defer span.End() + + rlReq, limits, charged := b.buildLimitRequest(rules, method, clientIP, userLabel, networkLabel, permit) + if len(charged) == 0 { + return true, nil + } + + // The local limiter counts every request, so it is warm when the store fails. + var localStatuses []*pb.RateLimitResponse_DescriptorStatus + if b.local != nil { + localStatuses = b.local.DoLimit(ctx, rlReq, b.scaledLimits(rlReq, limits)) + } + + statuses, failure := b.doLimit(ctx, rlReq, limits) + if failure != "" { + statuses = b.onStoreFailure(failure, localStatuses, method, userLabel, networkLabel) + } + + blocking := overLimitIndex(statuses) + if blocking < 0 { + return true, nil } + scope := "" + if blocking < len(charged) { + scope = charged[blocking].Config.ScopeString() + } + telemetry.CounterHandle( + telemetry.MetricRateLimitsTotal, + projectId, networkLabel, vendor, upstreamId, method, finality, + userLabel, agentName, b.Id, scope, authLabel, origin, + ).Inc() + return false, nil +} - // Multiple rules: evaluate in parallel - resultCh := make(chan ruleResult, len(rules)) - var blocked atomic.Bool +// buildLimitRequest turns the matching rules into one envoy request with a +// descriptor and a limit per charged rule. charged[i] is the rule behind +// descriptor i; a credit rule pricing the call at 0 is left out. +func (b *RateLimiterBudget) buildLimitRequest(rules []*RateLimitRule, method, clientIP, userLabel, networkLabel string, permit PermitCost) (*pb.RateLimitRequest, []*config.RateLimit, []*RateLimitRule) { + descriptors := make([]*pb_struct.RateLimitDescriptor, 0, len(rules)) + limits := make([]*config.RateLimit, 0, len(rules)) + charged := make([]*RateLimitRule, 0, len(rules)) + b.rulesMu.RLock() + defer b.rulesMu.RUnlock() for _, rule := range rules { - go func(r *RateLimitRule) { - // Skip if already blocked - if blocked.Load() { - resultCh <- ruleResult{rule: r, allowed: true} - return - } - allowed := b.evaluateRule(ctx, r, method, clientIP, userLabel, networkLabel, permit) - if !allowed { - blocked.Store(true) - } - resultCh <- ruleResult{rule: r, allowed: allowed} - }(rule) + mode, hits := b.resolveCost(rule, method, permit) + if mode == common.RateLimitCountModeCredit && hits == 0 { + continue + } + // "rule" gives each rule its own counter; without it rules resolving to + // the same {method, scope} at the same period collide. + entries := []*pb_struct.RateLimitDescriptor_Entry{{Key: "rule", Value: rule.key}} + // Request counting is per method. Credit counting is not: the cost is + // already carried by the addend, so every method draws from one pool and + // the budget behaves as the provider wallet it is meant to model. + if mode != common.RateLimitCountModeCredit { + entries = append(entries, &pb_struct.RateLimitDescriptor_Entry{Key: "method", Value: method}) + } + if rule.Config.PerIP && clientIP != "" && clientIP != "n/a" { + entries = append(entries, &pb_struct.RateLimitDescriptor_Entry{Key: "ip", Value: clientIP}) + } + if rule.Config.PerUser && userLabel != "" && userLabel != "n/a" { + entries = append(entries, &pb_struct.RateLimitDescriptor_Entry{Key: "user", Value: userLabel}) + } + if rule.Config.PerNetwork && networkLabel != "" && networkLabel != "n/a" { + entries = append(entries, &pb_struct.RateLimitDescriptor_Entry{Key: "network", Value: networkLabel}) + } + descriptors = append(descriptors, &pb_struct.RateLimitDescriptor{ + Entries: entries, + HitsAddend: wrapperspb.UInt64(uint64(hits)), + }) + + rlStats := b.registry.statsManager.NewStats(b.Id + ".method_" + method + rule.statsKeySuffix()) + limits = append(limits, config.NewRateLimit(rule.Config.MaxCount, rule.Config.Period.Unit(), rlStats, false, false, "", nil, false)) + charged = append(charged, rule) } + return &pb.RateLimitRequest{Domain: b.Id, Descriptors: descriptors}, limits, charged +} + +// scaledLimits returns limits at maxCount × localFactor, raised to the +// request's own hits so a credit rule never scales below one call's cost, and +// capped at maxCount. +func (b *RateLimiterBudget) scaledLimits(rlReq *pb.RateLimitRequest, limits []*config.RateLimit) []*config.RateLimit { + scaled := make([]*config.RateLimit, len(limits)) + for i, l := range limits { + maxCount := l.Limit.RequestsPerUnit + n := uint32(math.Ceil(float64(maxCount) * b.localFactor)) + n = min(max(n, uint32(rlReq.Descriptors[i].GetHitsAddend().GetValue()), 1), maxCount) + scaled[i] = config.NewRateLimit(n, l.Limit.Unit, l.Stats, false, false, "", nil, false) + } + return scaled +} - // Collect results - var blockingRule *RateLimitRule - for i := 0; i < len(rules); i++ { - result := <-resultCh - if !result.allowed && blockingRule == nil { - blockingRule = result.rule +func overLimitIndex(statuses []*pb.RateLimitResponse_DescriptorStatus) int { + for i, s := range statuses { + if s != nil && s.Code == pb.RateLimitResponse_OVER_LIMIT { + return i } } + return -1 +} - if blockingRule != nil { - telemetry.CounterHandle( - telemetry.MetricRateLimitsTotal, - projectId, networkLabel, vendor, upstreamId, method, finality, - userLabel, agentName, b.Id, blockingRule.Config.ScopeString(), authLabel, origin, +// onStoreFailure returns the statuses the budget's onFailure mode decides for a +// request the store could not answer; no OVER_LIMIT status admits it. +func (b *RateLimiterBudget) onStoreFailure(reason string, localStatuses []*pb.RateLimitResponse_DescriptorStatus, method, userLabel, networkLabel string) []*pb.RateLimitResponse_DescriptorStatus { + var statuses []*pb.RateLimitResponse_DescriptorStatus + decision := "allow" + switch b.onFailure { + case common.RateLimitStoreFailureDeny: + statuses, decision = deniedStatus, "deny" + case common.RateLimitStoreFailureLocal: + statuses = localStatuses + if overLimitIndex(localStatuses) >= 0 { + decision = "local_deny" + } else { + decision = "local_allow" + } + default: + telemetry.MetricRateLimiterFailopenTotal.WithLabelValues( + "", networkLabel, userLabel, "", + b.Id, method, reason, ).Inc() - return false, nil } - return true, nil + telemetry.MetricRateLimiterStoreFailureTotal.WithLabelValues(b.Id, reason, decision).Inc() + return statuses } +var deniedStatus = []*pb.RateLimitResponse_DescriptorStatus{{Code: pb.RateLimitResponse_OVER_LIMIT}} + +// Reasons the store could not decide a request. +const ( + storeUnavailable = "store_unavailable" + storeError = "store_error" + admissionFull = "admission_full" + limitTimeout = "limit_timeout" +) + // resolveCost returns the mode and hits this rule charges for one call. The // rule's countMode wins and an unset one inherits the caller's, so a rule that // states its mode is honored even under a credit caller. A credit rule prices @@ -325,91 +389,6 @@ func (b *RateLimiterBudget) resolveCost(rule *RateLimitRule, method string, perm return mode, uint32(units) } -// evaluateRule checks a single rate limit rule against the cache. -// Returns true if allowed, false if over limit. -func (b *RateLimiterBudget) evaluateRule(ctx context.Context, rule *RateLimitRule, method, clientIP, userLabel, networkLabel string, permit PermitCost) bool { - cache := b.getCache() - if cache == nil { - return true // Fail-open when no cache is available - } - - mode, hits := b.resolveCost(rule, method, permit) - if mode == common.RateLimitCountModeCredit && hits == 0 { - // Skip rather than send a 0 addend: envoy GetHitsAddends coerces a - // request-level 0 to 1, which would charge a full hit. - return true - } - - // Build descriptor entries. "rule" gives each rule its own counter; without it - // rules resolving to the same {method, scope} at the same period collide. - entries := []*pb_struct.RateLimitDescriptor_Entry{{Key: "rule", Value: rule.key}} - // Request counting is per method. Credit counting is not: the cost is already - // carried by the addend, so every method draws from one pool and the budget - // behaves as the provider wallet it is meant to model. - if mode != common.RateLimitCountModeCredit { - entries = append(entries, &pb_struct.RateLimitDescriptor_Entry{Key: "method", Value: method}) - } - if rule.Config.PerIP && clientIP != "" && clientIP != "n/a" { - entries = append(entries, &pb_struct.RateLimitDescriptor_Entry{Key: "ip", Value: clientIP}) - } - if rule.Config.PerUser && userLabel != "" && userLabel != "n/a" { - entries = append(entries, &pb_struct.RateLimitDescriptor_Entry{Key: "user", Value: userLabel}) - } - if rule.Config.PerNetwork && networkLabel != "" && networkLabel != "n/a" { - entries = append(entries, &pb_struct.RateLimitDescriptor_Entry{Key: "network", Value: networkLabel}) - } - - rlReq := &pb.RateLimitRequest{ - Domain: b.Id, - Descriptors: []*pb_struct.RateLimitDescriptor{{Entries: entries}}, - HitsAddend: hits, - } - - // Build stats key - statsKey := b.Id + ".method_" + method + rule.statsKeySuffix() - - rlStats := b.registry.statsManager.NewStats(statsKey) - limit := config.NewRateLimit(rule.Config.MaxCount, rule.Config.Period.Unit(), rlStats, false, false, "", nil, false) - limits := []*config.RateLimit{limit} - - _, doSpan := common.StartSpan(ctx, "RateLimiter.DoLimit", - trace.WithSpanKind(trace.SpanKindClient), - trace.WithAttributes( - attribute.String("budget", b.Id), - attribute.String("method", method), - attribute.String("scope", rule.Config.ScopeString()), - ), - ) - - var statuses []*pb.RateLimitResponse_DescriptorStatus - var timedOut bool - var failOpenReason string - if b.maxTimeout > 0 { - statuses, timedOut, failOpenReason = b.doLimitWithTimeout(ctx, cache, rlReq, limits, method, userLabel, networkLabel) - } else { - statuses = cache.DoLimit(ctx, rlReq, limits) - } - - if timedOut { - doSpan.SetAttributes( - attribute.String("result", "fail_open"), - attribute.String("reason", failOpenReason), - ) - doSpan.End() - return true // fail-open - } - - isOverLimit := len(statuses) > 0 && statuses[0].Code == pb.RateLimitResponse_OVER_LIMIT - if isOverLimit { - doSpan.SetAttributes(attribute.String("result", "over_limit")) - } else { - doSpan.SetAttributes(attribute.String("result", "ok")) - } - doSpan.End() - - return !isOverLimit -} - // statsKeySuffix returns the pre-computed suffix for stats key. func (r *RateLimitRule) statsKeySuffix() string { suffix := "" @@ -425,46 +404,57 @@ func (r *RateLimitRule) statsKeySuffix() string { return suffix } -// doLimitWithTimeout executes DoLimit with a timeout AND a per-budget admission cap. -// -// Returns (statuses, failOpen, reason). +// doLimit asks the cache for one request's statuses. failure is empty when the +// cache answered, else the reason it did not; onStoreFailure decides those. // -// The admission semaphore is the critical resilience primitive: the underlying -// envoyproxy/ratelimit cache + radix client do not honor context cancellation -// for the actual Redis I/O. Without a cap, every timed-out call leaks a -// goroutine that lives until Redis answers — which under contention spirals -// into runaway goroutine/FD growth and CPU saturation. With the cap in place, -// the in-flight count is bounded and the worst case is a brief load-shed -// burst. -// -// We DO continue to spawn a goroutine for the actual DoLimit (so the response -// can still complete and update Redis state correctly even after we've -// fail-opened), but only when we have an admission slot available — and we -// release the slot from inside that goroutine, so a slow Redis holds the slot +// A remote (redis) cache runs under a timeout AND a per-budget admission cap. +// The envoyproxy/ratelimit cache + radix client do not honor context +// cancellation for the actual Redis I/O, so every timed-out call keeps a +// goroutine until Redis answers; uncapped, that spiralled into runaway +// goroutine growth (2026-05-07 cronos receipts incident). The goroutine still +// finishes the call so Redis state stays correct, and holds its admission slot // for as long as the call truly takes, not just the local timeout. -func (b *RateLimiterBudget) doLimitWithTimeout( +func (b *RateLimiterBudget) doLimit( ctx context.Context, - cache limiter.RateLimitCache, rlReq *pb.RateLimitRequest, limits []*config.RateLimit, - method, userLabel, networkLabel string, -) ([]*pb.RateLimitResponse_DescriptorStatus, bool, string) { +) (statuses []*pb.RateLimitResponse_DescriptorStatus, failure string) { + cache := b.getCache() + if cache == nil { + return nil, storeUnavailable + } + if b.maxTimeout <= 0 { + return cache.DoLimit(ctx, rlReq, limits), "" + } + + _, span := common.StartSpan(ctx, "RateLimiter.DoLimit", + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes( + attribute.String("budget", b.Id), + attribute.Int("descriptors", len(rlReq.Descriptors)), + ), + ) + defer func() { + switch { + case failure != "": + span.SetAttributes(attribute.String("result", "store_failure"), attribute.String("reason", failure)) + case overLimitIndex(statuses) >= 0: + span.SetAttributes(attribute.String("result", "over_limit")) + default: + span.SetAttributes(attribute.String("result", "ok")) + } + span.End() + }() + if b.admission != nil { select { case b.admission <- struct{}{}: - // Got a slot — proceed. default: - // Admission semaphore is full: too many in-flight Redis calls. - // Fail-open immediately without spawning a goroutine. This is - // the load-shedding path that prevents goroutine accumulation. + // Too many in-flight Redis calls: shed without spawning a goroutine. if b.admissionShedded != nil { b.admissionShedded.Inc() } - telemetry.MetricRateLimiterFailopenTotal.WithLabelValues( - "", networkLabel, userLabel, "", - b.Id, method, "admission_full", - ).Inc() - return nil, true, "admission_full" + return nil, admissionFull } } @@ -477,7 +467,7 @@ func (b *RateLimiterBudget) doLimitWithTimeout( go func() { // Always release the admission slot and decrement the in-flight gauge, // even if cache.DoLimit panics — checkError() in envoy/ratelimit panics - // on Redis errors, so this is a real concern. + // on Redis errors. defer func() { if rec := recover(); rec != nil { telemetry.MetricUnexpectedPanicTotal.WithLabelValues( @@ -485,73 +475,47 @@ func (b *RateLimiterBudget) doLimitWithTimeout( "budget:"+b.Id, common.ErrorFingerprint(rec), ).Inc() - b.logger.Error(). - Interface("panic", rec). - Str("method", method). - Msg("panic recovered during Redis DoLimit (rate limiting fails open for this request)") - // Closed-but-non-nil channel: signal that we got nothing back. - select { - case resultCh <- nil: - default: + if b.logger.GetLevel() <= zerolog.DebugLevel { + b.logger.Debug().Interface("panic", rec).Msg("redis DoLimit failed") } + resultCh <- nil } if b.inflightGauge != nil { b.inflightGauge.Dec() } if b.admission != nil { - select { - case <-b.admission: - default: - } + <-b.admission } }() resultCh <- cache.DoLimit(ctx, rlReq, limits) }() timer := time.NewTimer(b.maxTimeout) + defer timer.Stop() select { - case statuses := <-resultCh: - if !timer.Stop() { - select { - case <-timer.C: - default: - } - } - // Observe duration only for non-timeout cases — timeouts are tracked - // by the timeout branch below to keep buckets clean. + case statuses = <-resultCh: dur := time.Since(start).Seconds() - if statuses != nil { - isOverLimit := len(statuses) > 0 && statuses[0].Code == pb.RateLimitResponse_OVER_LIMIT - if isOverLimit && b.durationOverlimit != nil { + switch { + case statuses == nil: + if b.durationFailopen != nil { + b.durationFailopen.Observe(dur) + } + return nil, storeError + case overLimitIndex(statuses) >= 0: + if b.durationOverlimit != nil { b.durationOverlimit.Observe(dur) - } else if !isOverLimit && b.durationOK != nil { + } + default: + if b.durationOK != nil { b.durationOK.Observe(dur) } - } else if b.durationFailopen != nil { - // Panic path — recovered above, count as fail-open. - b.durationFailopen.Observe(dur) } - return statuses, false, "" + return statuses, "" case <-timer.C: if b.durationFailopen != nil { b.durationFailopen.Observe(time.Since(start).Seconds()) } - // Sample the warn log; under sustained pressure this fires hundreds of - // times per second and dwarfs the rest of the log volume. - if b.logger.GetLevel() <= zerolog.DebugLevel { - b.logger.Debug(). - Str("budget", b.Id). - Str("method", method). - Dur("timeout", b.maxTimeout). - Msg("rate limiter timeout exceeded, failing open") - } - - telemetry.MetricRateLimiterFailopenTotal.WithLabelValues( - "", networkLabel, userLabel, "", - b.Id, method, "limit_timeout", - ).Inc() - - return nil, true, "limit_timeout" + return nil, limitTimeout } } diff --git a/upstream/ratelimiter_registry.go b/upstream/ratelimiter_registry.go index 8cd9bf4fb..bc1019738 100644 --- a/upstream/ratelimiter_registry.go +++ b/upstream/ratelimiter_registry.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/coocood/freecache" gostats "github.com/lyft/gostats" "github.com/rs/zerolog" @@ -24,6 +25,10 @@ import ( "github.com/erpc/erpc/util" ) +// overLimitCacheBytes sizes the per-replica over-limit key cache; freecache's +// floor is 512 KiB and a key is ~100 bytes. +const overLimitCacheBytes = 4 << 20 + type RateLimitersRegistry struct { appCtx context.Context logger *zerolog.Logger @@ -128,8 +133,11 @@ func (r *RateLimitersRegistry) connectRedisTask(ctx context.Context) (err error) "single", url, poolSize, - 5*time.Millisecond, - 32, + // No implicit pipelining: each check is already one explicit + // pipeline, and radix's window would only delay it (5ms window + // measured +23ms per 4-command check). + 0, + 0, nil, false, nil, @@ -140,9 +148,12 @@ func (r *RateLimitersRegistry) connectRedisTask(ctx context.Context) (err error) client, nil, utils.NewTimeSourceImpl(), - rand.New(rand.NewSource(time.Now().UnixNano())), // #nosec G404 + // Locked: concurrent DoLimit calls draw expiry jitter from it. + rand.New(utils.NewLockedSource(time.Now().UnixNano())), // #nosec G404 5, - nil, + // Remembers keys over their limit until the window ends, so denials + // skip Redis. Eviction only costs a Redis call. + freecache.NewCache(overLimitCacheBytes), defaultNearLimitRatio(r.cfg.Store.NearLimitRatio), defaultCacheKeyPrefix(r.cfg.Store.CacheKeyPrefix), mgr, @@ -188,6 +199,20 @@ func (r *RateLimitersRegistry) initializeBudgets() { if admissionCap > 0 { budget.admission = make(chan struct{}, admissionCap) } + if r.cfg.Store != nil && r.cfg.Store.Driver == "redis" { + budget.onFailure = r.cfg.Store.OnFailure + if budget.onFailure == common.RateLimitStoreFailureLocal { + budget.localFactor = r.cfg.Store.LocalLimitFactor + budget.local = NewMemoryRateLimitCache( + utils.NewTimeSourceImpl(), + rand.New(rand.NewSource(time.Now().UnixNano())), // #nosec G404 + 0, + defaultNearLimitRatio(r.cfg.Store.NearLimitRatio), + defaultCacheKeyPrefix(r.cfg.Store.CacheKeyPrefix), + r.statsManager, + ) + } + } // Pre-resolve metric handles for the hot path. budget.inflightGauge = telemetry.MetricRateLimiterRemoteInflight.WithLabelValues(budgetCfg.Id) budget.admissionShedded = telemetry.MetricRateLimiterRemoteAdmissionSheddedTotal.WithLabelValues(budgetCfg.Id) @@ -280,20 +305,13 @@ func defaultCacheKeyPrefix(val string) string { // remoteAdmissionCap derives the per-budget concurrent in-flight cap from the // Redis connection pool size. // -// Sizing rationale (envoyproxy/ratelimit + radix.v3): -// - Each rate-limit check sends a small pipeline (typically INCRBY+EXPIRE). -// - radix collects these via implicit pipelining (window=5ms, limit=32) so a -// single pool connection can have up to 32 commands "in flight" at once, -// i.e. effectively connPoolSize × 16 concurrent rate-limit checks per -// budget when Redis is healthy. -// - We multiply pool size by 32 to leave generous headroom in healthy state. -// Beyond that we'd rather load-shed (fail-open) than queue more goroutines: -// queueing here was the root cause of the 2026-05-07 cronos receipts -// death-spiral. -// - Per-budget cap (not global) so a single hot budget can't starve others. -// -// A floor of 256 ensures a small/misconfigured pool doesn't accidentally pin -// every request into the load-shed path on bursty traffic. +// This is a goroutine ceiling, not Redis capacity: each check is one explicit +// pipeline holding one pool connection for one round trip, so pool checkout is +// the real backpressure. ×32 leaves headroom so healthy traffic never sheds; +// beyond it we would rather let onFailure decide than queue goroutines, the +// root cause of the 2026-05-07 cronos receipts death spiral. Per budget, so +// one hot budget cannot starve others. The 256 floor keeps a small pool from +// shedding bursty traffic. func remoteAdmissionCap(connPoolSize int) int { const minCap = 256 if connPoolSize <= 0 { diff --git a/upstream/ratelimiter_store_failure_test.go b/upstream/ratelimiter_store_failure_test.go new file mode 100644 index 000000000..81625cde6 --- /dev/null +++ b/upstream/ratelimiter_store_failure_test.go @@ -0,0 +1,191 @@ +package upstream + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" +) + +func newRedisBudget(t *testing.T, addr string, store func(*common.RateLimitStoreConfig), rules ...*common.RateLimitRuleConfig) *RateLimiterBudget { + t.Helper() + cfg := &common.RateLimiterConfig{ + Store: &common.RateLimitStoreConfig{ + Driver: "redis", + Redis: &common.RedisConnectorConfig{ + Addr: addr, + ConnPoolSize: 4, + GetTimeout: common.Duration(50 * time.Millisecond), + }, + CacheKeyPrefix: t.Name() + "_", + }, + Budgets: []*common.RateLimitBudgetConfig{{Id: "b", Rules: rules}}, + } + if store != nil { + store(cfg.Store) + } + require.NoError(t, cfg.SetDefaults()) + require.NoError(t, cfg.Validate()) + logger := zerolog.Nop() + reg, err := NewRateLimitersRegistry(context.Background(), cfg, &logger) + require.NoError(t, err) + require.NotNil(t, reg.GetCache(), "redis limiter must connect") + budget, err := reg.GetBudget("b") + require.NoError(t, err) + return budget +} + +func admitted(t *testing.T, b *RateLimiterBudget, n int, concurrent bool) int64 { + t.Helper() + var ok atomic.Int64 + var wg sync.WaitGroup + for range n { + run := func() { + allowed, err := b.TryAcquirePermit(context.Background(), "p", nil, "eth_call", "", "", "", "upstream") + require.NoError(t, err) + if allowed { + ok.Add(1) + } + } + if concurrent { + wg.Add(1) + go func() { defer wg.Done(); run() }() + } else { + run() + } + } + wg.Wait() + return ok.Load() +} + +func perHour(max uint32) *common.RateLimitRuleConfig { + return &common.RateLimitRuleConfig{Method: "*", MaxCount: max, Period: common.RateLimitPeriodHour} +} + +// A redis budget walked through a healthy, slow, saturated and dead store, +// under each onFailure mode. 5 requests land while healthy. The remote limit +// (1000/hour) stays out of reach, so the over-limit cache never decides here; +// local mode counts at factor 0.01, i.e. 10/hour. +func TestRateLimiterRedis_StoreFailureFollowsOnFailure(t *testing.T) { + cases := []struct { + mode common.RateLimitStoreFailureMode + // admitted on top of the 5 healthy ones, per failure phase + slow, saturated, down func(got int64) bool + }{ + { + mode: common.RateLimitStoreFailureAllow, + slow: func(got int64) bool { return got == 20 }, + saturated: func(got int64) bool { return got > 1000 }, + down: func(got int64) bool { return got == 20 }, + }, + { + mode: common.RateLimitStoreFailureDeny, + slow: func(got int64) bool { return got == 0 }, + saturated: func(got int64) bool { return got == 0 }, + down: func(got int64) bool { return got == 0 }, + }, + { + // The local counter saw the 5 healthy requests, so 5 remain in + // the window across every failure phase together. + mode: common.RateLimitStoreFailureLocal, + slow: func(got int64) bool { return got == 5 }, + saturated: func(got int64) bool { return got == 0 }, + down: func(got int64) bool { return got == 0 }, + }, + } + for _, tc := range cases { + t.Run(string(tc.mode), func(t *testing.T) { + m := miniredis.RunT(t) + proxy := util.NewSlowTCPProxy(t, m.Addr()) + budget := newRedisBudget(t, proxy.Addr(), func(s *common.RateLimitStoreConfig) { + s.OnFailure = tc.mode + if tc.mode == common.RateLimitStoreFailureLocal { + s.LocalLimitFactor = 0.01 + } + }, perHour(1000)) + + require.Equal(t, int64(5), admitted(t, budget, 5, false), "healthy store admits within the limit") + + proxy.SetDelay(200 * time.Millisecond) // > getTimeout 50ms + got := admitted(t, budget, 20, false) + require.True(t, tc.slow(got), "slow store: admitted %d", got) + + // Wait for the timed-out calls to drain, then saturate admission. + require.Eventually(t, func() bool { return len(budget.admission) == 0 }, 10*time.Second, 10*time.Millisecond) + got = admitted(t, budget, 2000, true) + require.True(t, tc.saturated(got), "saturated store: admitted %d", got) + + require.Eventually(t, func() bool { return len(budget.admission) == 0 }, 30*time.Second, 10*time.Millisecond) + proxy.SetDelay(0) + m.Close() + got = admitted(t, budget, 20, false) + require.True(t, tc.down(got), "dead store: admitted %d", got) + }) + } +} + +// A request matching several rules holds one admission slot, not one per rule. +func TestRateLimiterRedis_ManyRulesShareOneAdmissionSlot(t *testing.T) { + m := miniredis.RunT(t) + proxy := util.NewSlowTCPProxy(t, m.Addr()) + budget := newRedisBudget(t, proxy.Addr(), func(s *common.RateLimitStoreConfig) { + s.Redis.GetTimeout = common.Duration(5 * time.Second) + }, + perHour(100), + &common.RateLimitRuleConfig{Method: "eth_*", MaxCount: 50, Period: common.RateLimitPeriodHour}, + &common.RateLimitRuleConfig{Method: "eth_call", MaxCount: 2, Period: common.RateLimitPeriodHour}, + ) + + proxy.SetDelay(300 * time.Millisecond) + done := make(chan int64) + go func() { done <- admitted(t, budget, 1, false) }() + require.Eventually(t, func() bool { return len(budget.admission) > 0 }, time.Second, time.Millisecond) + require.Equal(t, 1, len(budget.admission), "one request across 3 rules must hold 1 slot") + require.Equal(t, int64(1), <-done) + + // The tightest rule still decides: eth_call allows 2 per hour. + proxy.SetDelay(0) + require.Equal(t, int64(1), admitted(t, budget, 2, false), "eth_call 2/hour rule enforced in the batch") +} + +// A key Redis reported over its limit is denied from the local cache until +// its window ends: no Redis call, so it holds even with Redis dead under the +// default onFailure allow. +func TestRateLimiterRedis_OverLimitKeyDeniesWithoutRedis(t *testing.T) { + m := miniredis.RunT(t) + budget := newRedisBudget(t, m.Addr(), nil, perHour(2)) + + require.Equal(t, int64(2), admitted(t, budget, 3, false), "third request is over the limit") + m.Close() + require.Equal(t, int64(0), admitted(t, budget, 5, false), "cached over-limit key denies without Redis") +} + +// onFailure: local never scales a credit rule below one call's cost: 600 +// credits × 0.1 is 60, under a 200-credit call, so the local limit is 200. +func TestRateLimiterRedis_LocalCreditLimitFitsOneCall(t *testing.T) { + m := miniredis.RunT(t) + budget := newRedisBudget(t, m.Addr(), func(s *common.RateLimitStoreConfig) { + s.OnFailure = common.RateLimitStoreFailureLocal + s.LocalLimitFactor = 0.1 + }, &common.RateLimitRuleConfig{Method: "*", MaxCount: 600, Period: common.RateLimitPeriodHour, CountMode: common.RateLimitCountModeCredit}) + m.Close() + + call := PermitCost{Hits: 200, Mode: common.RateLimitCountModeCredit} + got := 0 + for range 3 { + ok, err := budget.TryAcquirePermit(context.Background(), "p", nil, "eth_call", "", "", "", "upstream", call) + require.NoError(t, err) + if ok { + got++ + } + } + require.Equal(t, 1, got, "local limit fits exactly one 200-credit call") +} diff --git a/upstream/registry.go b/upstream/registry.go index 5a5202272..79aeb06ee 100644 --- a/upstream/registry.go +++ b/upstream/registry.go @@ -445,10 +445,11 @@ func (u *UpstreamsRegistry) GetNetworkUpstreams(ctx context.Context, networkId s } // GetFallbackEscapeUpstreams returns the network's bootstrapped fallback-tier -// upstreams that are not down for method and allow it. Selection-policy -// cordons are deliberately ignored: the per-request escape exists to get past -// them. -func (u *UpstreamsRegistry) GetFallbackEscapeUpstreams(ctx context.Context, networkId, method string) []*Upstream { +// upstreams that are not down for method at finality and allow it. +// Selection-policy cordons are deliberately ignored: the per-request escape +// exists to get past them. finality must be the request's so a breaker scoped +// to finalized/realtime is not skipped. +func (u *UpstreamsRegistry) GetFallbackEscapeUpstreams(ctx context.Context, networkId, method string, finality common.DataFinalityState) []*Upstream { all := u.GetNetworkUpstreams(ctx, networkId) out := make([]*Upstream, 0, len(all)) for _, up := range all { @@ -456,7 +457,7 @@ func (u *UpstreamsRegistry) GetFallbackEscapeUpstreams(ctx context.Context, netw if cfg == nil || !cfg.HasTag(common.TagTierFallback) { continue } - if up.IsDown(method) { + if up.IsDown(method, finality) { continue } if allowed, err := up.ShouldHandleMethod(method); err != nil || !allowed { diff --git a/upstream/registry_fallback_escape_test.go b/upstream/registry_fallback_escape_test.go index 22bb2b04f..abd6c8f8d 100644 --- a/upstream/registry_fallback_escape_test.go +++ b/upstream/registry_fallback_escape_test.go @@ -93,7 +93,7 @@ func TestGetFallbackEscapeUpstreams_FiltersByGroup(t *testing.T) { defer cancel() registry, _ := setupRegistryWithFallbacks(t, ctx) - out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call") + out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call", common.DataFinalityStateUnknown) ids := make([]string, 0, len(out)) for _, u := range out { ids = append(ids, u.Id()) @@ -120,7 +120,7 @@ func TestGetFallbackEscapeUpstreams_IgnoresCordon(t *testing.T) { require.True(t, mt.IsCordoned(fb1, "eth_call"), "fb1 should be cordoned") require.True(t, mt.IsCordoned(fb2, "eth_call"), "fb2 should be cordoned") - out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call") + out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call", common.DataFinalityStateUnknown) ids := make([]string, 0, len(out)) for _, u := range out { ids = append(ids, u.Id()) @@ -141,7 +141,7 @@ func TestGetFallbackEscapeUpstreams_FiltersIgnoreMethods(t *testing.T) { // Mark eth_call as ignored on fallback-1; AllowMethods left empty. fb1.config.IgnoreMethods = []string{"eth_call"} - out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call") + out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call", common.DataFinalityStateUnknown) ids := make([]string, 0, len(out)) for _, u := range out { ids = append(ids, u.Id()) @@ -150,7 +150,7 @@ func TestGetFallbackEscapeUpstreams_FiltersIgnoreMethods(t *testing.T) { "fallback-1 should be excluded because eth_call is in its IgnoreMethods; got %v", ids) // Sanity: a different method that isn't ignored should include fallback-1. - out2 := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_getLogs") + out2 := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_getLogs", common.DataFinalityStateUnknown) ids2 := make([]string, 0, len(out2)) for _, u := range out2 { ids2 = append(ids2, u.Id()) @@ -189,6 +189,6 @@ func TestGetFallbackEscapeUpstreams_NoFallbacksReturnsEmpty(t *testing.T) { time.Sleep(100 * time.Millisecond) require.NoError(t, registry.PrepareUpstreamsForNetwork(ctx, "evm:123")) - out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call") + out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call", common.DataFinalityStateUnknown) assert.Empty(t, out, "no fallback-group upstreams configured → empty slice") } diff --git a/upstream/upstream.go b/upstream/upstream.go index 251dce30d..bfa59720f 100644 --- a/upstream/upstream.go +++ b/upstream/upstream.go @@ -211,6 +211,7 @@ type Upstream struct { evmStatePoller common.EvmStatePoller svmStatePoller common.SvmStatePoller statePollerOnce sync.Once + evmMulticall3 *evm.Multicall3Batcher // True after successful chainId detection/validation; enables short-circuit in EvmGetChainId. chainIdValidated atomic.Bool // Highest block at which the integrity state probe PROVED this upstream @@ -317,6 +318,10 @@ func NewUpstream( pup.Client = client } + if ec := pup.config.Evm; ec != nil && ec.Multicall3 != nil { + pup.evmMulticall3 = evm.NewMulticall3Batcher(ec.Multicall3) + } + lg.Debug().Msgf("prepared upstream") return pup, nil @@ -421,13 +426,15 @@ func (u *Upstream) Config() *common.UpstreamConfig { } // IsDown reports whether the circuit breaker of the failsafe policy that -// would serve method is open. Breakers scoped to other methods don't count. -func (u *Upstream) IsDown(method string) bool { +// would serve method at finality is open. Breakers scoped to other methods +// or finalities don't count. Pass the request's real finality (not Unknown) +// when checking escape/health paths so a finalized/realtime-scoped open +// breaker is not missed. +func (u *Upstream) IsDown(method string, finality common.DataFinalityState) bool { if u == nil { return true } - req := common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest(method, nil)) - fe := u.getFailsafeExecutor(req) + fe := u.failsafeExecutorFor(method, finality) if fe == nil { return false } @@ -519,8 +526,13 @@ func (u *Upstream) SetNetworkConfig(cfg *common.NetworkConfig) { func (u *Upstream) getFailsafeExecutor(req *common.NormalizedRequest) *upstreamExecutor { method, _ := req.Method() - finality := req.Finality(context.Background()) + return u.failsafeExecutorFor(method, req.Finality(context.Background())) +} +// failsafeExecutorFor picks the policy the same way getFailsafeExecutor does, +// so health checks that only have method+finality (not a full request) still +// see the breaker the real call would hit. +func (u *Upstream) failsafeExecutorFor(method string, finality common.DataFinalityState) *upstreamExecutor { // 4-tier priority: method+finality > method > finality > catch-all. for _, fe := range u.failsafeExecutors { mp, fl := fe.MatchMethod(), fe.MatchFinality() diff --git a/upstream/upstream_block_availability_test.go b/upstream/upstream_block_availability_test.go index b89717cbc..3e64b401c 100644 --- a/upstream/upstream_block_availability_test.go +++ b/upstream/upstream_block_availability_test.go @@ -63,6 +63,7 @@ func (m *mockEvmStatePollerEnhanced) IsBlockFinalized(blockNumber int64) (bool, } func (m *mockEvmStatePollerEnhanced) SuggestFinalizedBlock(blockNumber int64) {} func (m *mockEvmStatePollerEnhanced) SuggestLatestBlock(blockNumber int64) {} +func (m *mockEvmStatePollerEnhanced) RefreshLatestBlock() {} func (m *mockEvmStatePollerEnhanced) SetNetworkConfig(cfg *common.NetworkConfig) {} func (m *mockEvmStatePollerEnhanced) IsObjectNull() bool { return m.isNull } func (m *mockEvmStatePollerEnhanced) GetDiagnostics() *common.EvmStatePollerDiagnostics { @@ -625,6 +626,7 @@ func (m *mockEvmStatePollerWithCustomBehavior) IsBlockFinalized(blockNumber int6 } func (m *mockEvmStatePollerWithCustomBehavior) SuggestFinalizedBlock(blockNumber int64) {} func (m *mockEvmStatePollerWithCustomBehavior) SuggestLatestBlock(blockNumber int64) {} +func (m *mockEvmStatePollerWithCustomBehavior) RefreshLatestBlock() {} func (m *mockEvmStatePollerWithCustomBehavior) SetNetworkConfig(cfg *common.NetworkConfig) {} func (m *mockEvmStatePollerWithCustomBehavior) IsObjectNull() bool { return false } func (m *mockEvmStatePollerWithCustomBehavior) GetDiagnostics() *common.EvmStatePollerDiagnostics { diff --git a/upstream/upstream_executor_internal_test.go b/upstream/upstream_executor_internal_test.go new file mode 100644 index 000000000..bf59b93e3 --- /dev/null +++ b/upstream/upstream_executor_internal_test.go @@ -0,0 +1,44 @@ +package upstream + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/failsafe" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +// Internal requests (e.g. the wsupstream adapter's subscription RPCs) must not +// feed the upstream breaker; the same failure from a client request opens it. +func TestUpstreamExecutor_InternalRequestsSkipBreaker(t *testing.T) { + lg := zerolog.Nop() + serverErr := common.NewErrEndpointServerSideException(errors.New("boom"), nil, 500) + inner := func(context.Context, bool) (*common.NormalizedResponse, error) { return nil, serverErr } + + run := func(internal bool) failsafe.State { + ex, err := NewUpstreamExecutor(&common.FailsafeConfig{ + CircuitBreaker: &common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 1, + FailureThresholdCapacity: 1, + HalfOpenAfter: common.Duration(time.Hour), + SuccessThresholdCount: 1, + SuccessThresholdCapacity: 1, + }, + }, &lg) + require.NoError(t, err) + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`)) + if internal { + req.SetDirectives(&common.RequestDirectives{IsInternal: true}) + } + _, err = ex.Run(context.Background(), req, inner) + require.Error(t, err) + return ex.Breaker().State() + } + + require.Equal(t, failsafe.StateClosed, run(true)) + require.Equal(t, failsafe.StateOpen, run(false)) +} diff --git a/upstream/upstream_forward_test.go b/upstream/upstream_forward_test.go new file mode 100644 index 000000000..57c8527c3 --- /dev/null +++ b/upstream/upstream_forward_test.go @@ -0,0 +1,657 @@ +package upstream + +import ( + "context" + "errors" + "net/url" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/clients" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/failsafe" + "github.com/erpc/erpc/health" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/thirdparty" + "github.com/erpc/erpc/util" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// stubClient stands in for the transport: it answers each SendRequest with +// whatever fn decides, counting calls so tests can tell whether the upstream +// actually dialed. +type stubClient struct { + typ clients.ClientType + calls atomic.Int32 + fn func(ctx context.Context, req *common.NormalizedRequest, call int32) (*common.NormalizedResponse, error) +} + +func (c *stubClient) GetType() clients.ClientType { return c.typ } + +func (c *stubClient) SendRequest(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return c.fn(ctx, req, c.calls.Add(1)) +} + +func newStubClient(fn func(ctx context.Context, req *common.NormalizedRequest, call int32) (*common.NormalizedResponse, error)) *stubClient { + return &stubClient{typ: clients.ClientTypeHttpJsonRpc, fn: fn} +} + +func stubResult(req *common.NormalizedRequest, result string) *common.NormalizedResponse { + jrr := common.MustNewJsonRpcResponseFromBytes([]byte("1"), []byte(result), nil) + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) +} + +func stubRpcError(req *common.NormalizedRequest) *common.NormalizedResponse { + jrr := common.MustNewJsonRpcResponse(1, nil, common.NewErrJsonRpcExceptionExternal(-32000, "boom", "")) + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) +} + +func fwdReq(method string) *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"` + method + `","params":[]}`)) +} + +type fwdEnv struct { + ctx context.Context + logger zerolog.Logger + tracker *health.Tracker + rlr *RateLimitersRegistry + vr *thirdparty.VendorsRegistry + cr *clients.ClientRegistry + ssr data.SharedStateRegistry +} + +func newFwdEnv(t *testing.T, rl *common.RateLimiterConfig) *fwdEnv { + t.Helper() + ctx := t.Context() + lg := zerolog.Nop() + if rl == nil { + rl = &common.RateLimiterConfig{} + } + rlr, err := NewRateLimitersRegistry(ctx, rl, &lg) + require.NoError(t, err) + ssr, err := data.NewSharedStateRegistry(ctx, &lg, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: common.DriverMemory, + Memory: &common.MemoryConnectorConfig{MaxItems: 10_000, MaxTotalSize: "64MB"}, + }, + }) + require.NoError(t, err) + return &fwdEnv{ + ctx: ctx, + logger: lg, + tracker: health.NewTracker(&lg, "prj", time.Minute), + rlr: rlr, + vr: thirdparty.NewVendorsRegistry(), + cr: clients.NewClientRegistry(&lg, "prj", nil, NewCompositeJsonRpcErrorExtractor()), + ssr: ssr, + } +} + +func (e *fwdEnv) newUpstream(t *testing.T, cfg *common.UpstreamConfig) (*Upstream, error) { + t.Helper() + if cfg.Endpoint == "" && cfg.VendorName == "" { + cfg.Endpoint = "http://rpc1.localhost" + } + if cfg.Type == "" { + cfg.Type = common.UpstreamTypeEvm + } + return NewUpstream(e.ctx, "prj", cfg, e.cr, e.rlr, e.vr, &e.logger, e.tracker, e.ssr) +} + +// mustUpstream builds an upstream through NewUpstream and swaps its transport +// for the stub. +func (e *fwdEnv) mustUpstream(t *testing.T, cfg *common.UpstreamConfig, sc *stubClient) *Upstream { + t.Helper() + u, err := e.newUpstream(t, cfg) + require.NoError(t, err) + u.Client = sc + return u +} + +func breakerCfg() *common.CircuitBreakerPolicyConfig { + return &common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 1, + FailureThresholdCapacity: 1, + HalfOpenAfter: common.Duration(time.Hour), + SuccessThresholdCount: 1, + SuccessThresholdCapacity: 1, + } +} + +func TestUpstreamForward_SuccessRecordsAttemptAndLastValidResponse(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(_ context.Context, req *common.NormalizedRequest, _ int32) (*common.NormalizedResponse, error) { + return stubResult(req, `"0xabc"`), nil + }) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "fwd-ok"}, sc) + + req := fwdReq("eth_call") + resp, err := u.Forward(env.ctx, req, false, false) + require.NoError(t, err) + require.NotNil(t, resp) + + assert.Same(t, u, resp.Upstream(), "the response is attributed to the upstream that served it") + assert.Same(t, resp, req.LastValidResponse()) + assert.Same(t, u, req.LastUpstream()) + + log := req.ExecState().UpstreamAttemptLog() + require.Len(t, log, 1) + assert.Equal(t, common.UpstreamOutcomeSuccess, log[0].Outcome) + assert.Equal(t, "fwd-ok", log[0].UpstreamId) + assert.Equal(t, common.SelectionReasonPrimary, log[0].Reason) + assert.Equal(t, int64(1), log[0].CreditUnits, "a vendor without pricing costs one credit per request") + assert.Empty(t, log[0].ErrorCode) + + m := env.tracker.GetUpstreamMethodMetrics(u, "eth_call", common.DataFinalityStateAll) + assert.Equal(t, int64(1), m.RequestsTotal.Load()) + assert.Equal(t, int64(0), m.ErrorsTotal.Load()) +} + +func TestUpstreamForward_ResponseShapesClassifyOutcome(t *testing.T) { + cases := []struct { + name string + resp func(req *common.NormalizedRequest) *common.NormalizedResponse + wantOutcome common.UpstreamAttemptOutcome + wantLastValid bool + }{ + {"empty hex result", func(r *common.NormalizedRequest) *common.NormalizedResponse { return stubResult(r, `"0x"`) }, common.UpstreamOutcomeEmpty, true}, + {"null result", func(r *common.NormalizedRequest) *common.NormalizedResponse { return stubResult(r, `null`) }, common.UpstreamOutcomeEmpty, true}, + // An error body has no result: it reads as empty, and is never kept as the last valid response. + {"json-rpc error body", stubRpcError, common.UpstreamOutcomeEmpty, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(_ context.Context, req *common.NormalizedRequest, _ int32) (*common.NormalizedResponse, error) { + return tc.resp(req), nil + }) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "fwd-shape"}, sc) + req := fwdReq("eth_getLogs") + resp, err := u.Forward(env.ctx, req, false, false) + require.NoError(t, err) + require.NotNil(t, resp) + log := req.ExecState().UpstreamAttemptLog() + require.Len(t, log, 1) + assert.Equal(t, tc.wantOutcome, log[0].Outcome) + assert.Equal(t, tc.wantLastValid, req.LastValidResponse() != nil) + }) + } +} + +func TestUpstreamForward_ClientErrorsAreWrappedAndAccounted(t *testing.T) { + endpoint, _ := url.Parse("http://rpc1.localhost") + cases := []struct { + name string + err error + wantCode common.ErrorCode + wantOutcome common.UpstreamAttemptOutcome + wantErrorCount int64 + wantRemoteRL int64 + wantCredits int64 + }{ + {"capacity exceeded", common.NewErrEndpointCapacityExceeded(errors.New("429")), common.ErrCodeEndpointCapacityExceeded, common.UpstreamOutcomeRateLimited, 0, 1, 1}, + {"missing data", common.NewErrEndpointMissingData(errors.New("no block"), nil), common.ErrCodeEndpointMissingData, common.UpstreamOutcomeMissingData, 0, 0, 1}, + {"request canceled", common.NewErrEndpointRequestCanceled(context.Canceled), common.ErrCodeEndpointRequestCanceled, common.UpstreamOutcomeCancelled, 0, 0, 1}, + {"execution reverted", common.NewErrEndpointExecutionException(errors.New("revert")), common.ErrCodeEndpointExecutionException, common.UpstreamOutcomeExecRevert, 0, 0, 1}, + {"server side", common.NewErrEndpointServerSideException(errors.New("500"), nil, 500), common.ErrCodeEndpointServerSideException, common.UpstreamOutcomeServerError, 1, 0, 1}, + {"transport failure", common.NewErrEndpointTransportFailure(endpoint, errors.New("refused")), common.ErrCodeEndpointTransportFailure, common.UpstreamOutcomeTransportError, 1, 0, 1}, + {"client side", common.NewErrEndpointClientSideException(errors.New("bad params")), common.ErrCodeEndpointClientSideException, common.UpstreamOutcomeClientError, 0, 0, 1}, + {"skipped by client", common.NewErrUpstreamRequestSkipped(errors.New("nope"), "fwd-err"), common.ErrCodeUpstreamRequestSkipped, common.UpstreamOutcomeSkipped, 0, 0, 0}, + {"unclassified error", errors.New("weird"), common.ErrCodeUpstreamRequest, common.UpstreamOutcomeServerError, 1, 0, 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(_ context.Context, req *common.NormalizedRequest, _ int32) (*common.NormalizedResponse, error) { + // A revert carries a body; it must be released and the slot cleared. + return stubResult(req, `"0x1"`), tc.err + }) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "fwd-err"}, sc) + req := fwdReq("eth_call") + // Seed a previous valid response: only a revert must clear it. + prev := stubResult(req, `"0x2"`) + req.SetLastValidResponse(env.ctx, prev) + + resp, err := u.Forward(env.ctx, req, false, false) + require.Error(t, err) + assert.Nil(t, resp) + assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamRequest), "every client error is wrapped as ErrUpstreamRequest: %v", err) + assert.True(t, common.HasErrorCode(err, tc.wantCode), "cause code %s preserved: %v", tc.wantCode, err) + + log := req.ExecState().UpstreamAttemptLog() + require.Len(t, log, 1) + assert.Equal(t, tc.wantOutcome, log[0].Outcome) + assert.NotEmpty(t, log[0].ErrorCode) + assert.Equal(t, tc.wantCredits, log[0].CreditUnits) + + m := env.tracker.GetUpstreamMethodMetrics(u, "eth_call", common.DataFinalityStateAll) + assert.Equal(t, tc.wantErrorCount, m.ErrorsTotal.Load(), "error-rate counter") + assert.Equal(t, tc.wantRemoteRL, m.RemoteRateLimitedTotal.Load(), "remote rate-limit counter") + + if tc.wantCode == common.ErrCodeEndpointExecutionException { + assert.Nil(t, req.LastValidResponse(), "a revert invalidates the last valid response") + } else { + assert.Same(t, prev, req.LastValidResponse()) + } + }) + } +} + +func TestUpstreamForward_AttemptErrorDetailIsTruncated(t *testing.T) { + old := AttemptErrorDetailMaxLen + AttemptErrorDetailMaxLen = 16 + t.Cleanup(func() { AttemptErrorDetailMaxLen = old }) + + env := newFwdEnv(t, nil) + sc := newStubClient(func(context.Context, *common.NormalizedRequest, int32) (*common.NormalizedResponse, error) { + return nil, common.NewErrEndpointServerSideException(errors.New("a very long upstream failure message"), nil, 500) + }) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "fwd-trunc"}, sc) + req := fwdReq("eth_call") + _, err := u.Forward(env.ctx, req, false, false) + require.Error(t, err) + log := req.ExecState().UpstreamAttemptLog() + require.Len(t, log, 1) + assert.Len(t, log[0].ErrorDetail, 16) + assert.Greater(t, len(err.Error()), 16, "the caller still gets the full error") +} + +func TestUpstreamForward_UnresolvableMethodFailsBeforeDialing(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(context.Context, *common.NormalizedRequest, int32) (*common.NormalizedResponse, error) { + t.Fatal("must not dial") + return nil, nil + }) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "fwd-bad"}, sc) + _, err := u.Forward(env.ctx, common.NewNormalizedRequest([]byte(`{not json`)), false, false) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamRequest)) + assert.Equal(t, int32(0), sc.calls.Load()) +} + +type syncingPoller struct{ mockEvmStatePollerEnhanced } + +func (p *syncingPoller) SyncingState() common.EvmSyncingState { return common.EvmSyncingStateSyncing } + +func TestUpstreamForward_SkipRules(t *testing.T) { + okClient := func() *stubClient { + return newStubClient(func(_ context.Context, req *common.NormalizedRequest, _ int32) (*common.NormalizedResponse, error) { + return stubResult(req, `"0x1"`), nil + }) + } + cases := []struct { + name string + cfg *common.UpstreamConfig + poller common.EvmStatePoller + useUps string + bypass bool + wantCause common.ErrorCode // empty = forwarded + }{ + {name: "shadow upstream never serves", cfg: &common.UpstreamConfig{Id: "sk", Shadow: &common.ShadowUpstreamConfig{Enabled: true}}, wantCause: common.ErrCodeUpstreamShadowing}, + {name: "syncing upstream skipped when skipWhenSyncing", cfg: &common.UpstreamConfig{Id: "sk", Evm: &common.EvmUpstreamConfig{SkipWhenSyncing: util.BoolPtr(true)}}, poller: &syncingPoller{}, wantCause: common.ErrCodeUpstreamSyncing}, + {name: "syncing upstream served when skipWhenSyncing off", cfg: &common.UpstreamConfig{Id: "sk", Evm: &common.EvmUpstreamConfig{SkipWhenSyncing: util.BoolPtr(false)}}, poller: &syncingPoller{}}, + {name: "ignored method", cfg: &common.UpstreamConfig{Id: "sk", IgnoreMethods: []string{"eth_call"}}, wantCause: common.ErrCodeUpstreamMethodIgnored}, + {name: "ignored method bypassed", cfg: &common.UpstreamConfig{Id: "sk", IgnoreMethods: []string{"eth_call"}}, bypass: true}, + {name: "invalid ignore pattern skips with its error", cfg: &common.UpstreamConfig{Id: "sk", IgnoreMethods: []string{"(eth_"}}, wantCause: ""}, + {name: "use-upstream directive mismatch", cfg: &common.UpstreamConfig{Id: "sk"}, useUps: "other-*", wantCause: common.ErrCodeUpstreamNotAllowed}, + {name: "use-upstream directive match", cfg: &common.UpstreamConfig{Id: "sk"}, useUps: "s*"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + env := newFwdEnv(t, nil) + sc := okClient() + u := env.mustUpstream(t, tc.cfg, sc) + if tc.poller != nil { + u.evmStatePoller = tc.poller + } + req := fwdReq("eth_call") + if tc.useUps != "" { + req.SetDirectives(&common.RequestDirectives{UseUpstream: tc.useUps}) + } + resp, err := u.Forward(env.ctx, req, tc.bypass, false) + skipped := tc.wantCause != "" || tc.name == "invalid ignore pattern skips with its error" + if !skipped { + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, int32(1), sc.calls.Load()) + return + } + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamRequestSkipped), "%v", err) + if tc.wantCause != "" { + assert.True(t, common.HasErrorCode(err, tc.wantCause), "%v", err) + } + assert.Equal(t, int32(0), sc.calls.Load(), "a skipped request never dials") + }) + } +} + +func TestUpstreamForward_RateLimitBudget(t *testing.T) { + rl := &common.RateLimiterConfig{ + Store: &common.RateLimitStoreConfig{Driver: "memory"}, + Budgets: []*common.RateLimitBudgetConfig{{ + Id: "one-call", + Rules: []*common.RateLimitRuleConfig{{Method: "eth_call", MaxCount: 1, Period: common.RateLimitPeriodMinute}}, + }}, + } + ok := func() *stubClient { + return newStubClient(func(_ context.Context, req *common.NormalizedRequest, _ int32) (*common.NormalizedResponse, error) { + return stubResult(req, `"0x1"`), nil + }) + } + + t.Run("exhausted budget rejects without dialing", func(t *testing.T) { + env := newFwdEnv(t, rl) + sc := ok() + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "rl-a", RateLimitBudget: "one-call"}, sc) + _, err := u.Forward(env.ctx, fwdReq("eth_call"), false, false) + require.NoError(t, err) + _, err = u.Forward(env.ctx, fwdReq("eth_call"), false, false) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamRateLimitRuleExceeded), "%v", err) + assert.Equal(t, int32(1), sc.calls.Load()) + }) + + t.Run("methods without a rule are unlimited", func(t *testing.T) { + env := newFwdEnv(t, rl) + sc := ok() + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "rl-b", RateLimitBudget: "one-call"}, sc) + for range 3 { + _, err := u.Forward(env.ctx, fwdReq("eth_getBalance"), false, false) + require.NoError(t, err) + } + assert.Equal(t, int32(3), sc.calls.Load()) + }) + + t.Run("unknown budget is an error", func(t *testing.T) { + env := newFwdEnv(t, rl) + sc := ok() + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "rl-c", RateLimitBudget: "missing"}, sc) + _, err := u.Forward(env.ctx, fwdReq("eth_call"), false, false) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, "ErrRateLimitBudgetNotFound"), "%v", err) + assert.Equal(t, int32(0), sc.calls.Load()) + }) + + t.Run("credit mode charges the method's credit units", func(t *testing.T) { + env := newFwdEnv(t, &common.RateLimiterConfig{ + Store: &common.RateLimitStoreConfig{Driver: "memory"}, + Budgets: []*common.RateLimitBudgetConfig{{ + Id: "wallet", + Rules: []*common.RateLimitRuleConfig{{Method: "*", MaxCount: 10, Period: common.RateLimitPeriodMinute}}, + }}, + }) + sc := ok() + u := env.mustUpstream(t, &common.UpstreamConfig{ + Id: "rl-credit", + RateLimitBudget: "wallet", + RateLimitCountMode: common.RateLimitCountModeCredit, + CreditUnits: map[string]int64{"eth_call": 5, "eth_chainId": 0}, + }, sc) + for i := range 2 { + _, err := u.Forward(env.ctx, fwdReq("eth_call"), false, false) + require.NoError(t, err, "call %d fits in 10 credits", i) + } + _, err := u.Forward(env.ctx, fwdReq("eth_call"), false, false) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamRateLimitRuleExceeded)) + // A 0-credit method draws nothing from the exhausted wallet. + _, err = u.Forward(env.ctx, fwdReq("eth_chainId"), false, false) + require.NoError(t, err) + }) +} + +func TestUpstream_RateLimitCost(t *testing.T) { + cases := []struct { + name string + cfg *common.UpstreamConfig + want PermitCost + }{ + {"request mode is one hit", &common.UpstreamConfig{}, requestPermit}, + {"credit mode uses the method cost", &common.UpstreamConfig{RateLimitCountMode: common.RateLimitCountModeCredit, CreditUnits: map[string]int64{"eth_call": 26}}, PermitCost{Hits: 26, Mode: common.RateLimitCountModeCredit}}, + {"credit mode zero cost is exempt", &common.UpstreamConfig{RateLimitCountMode: common.RateLimitCountModeCredit, CreditUnits: map[string]int64{"*": 0}}, PermitCost{Hits: 0, Mode: common.RateLimitCountModeCredit}}, + {"credit mode clamps to uint32", &common.UpstreamConfig{RateLimitCountMode: common.RateLimitCountModeCredit, CreditUnits: map[string]int64{"*": 1 << 40}}, PermitCost{Hits: 1<<32 - 1, Mode: common.RateLimitCountModeCredit}}, + {"credit mode without table falls back to one credit", &common.UpstreamConfig{RateLimitCountMode: common.RateLimitCountModeCredit}, PermitCost{Hits: 1, Mode: common.RateLimitCountModeCredit}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + u := &Upstream{config: tc.cfg} + assert.Equal(t, tc.want, u.rateLimitCost(fwdReq("eth_call"))) + }) + } + var nilUps *Upstream + assert.Equal(t, requestPermit, nilUps.rateLimitCost(fwdReq("eth_call"))) + assert.Equal(t, int64(0), nilUps.attemptCreditUnits(fwdReq("eth_call"))) +} + +func TestUpstreamForward_UnsupportedClientType(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(nil) + sc.typ = clients.ClientType("Carrier-Pigeon") + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "fwd-pigeon"}, sc) + _, err := u.Forward(env.ctx, fwdReq("eth_call"), false, false) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, "ErrUpstreamClientInitialization"), "%v", err) + assert.Contains(t, err.Error(), "could not initialize upstream client") +} + +func TestUpstreamForward_RetryPolicy(t *testing.T) { + serverErr := func() error { return common.NewErrEndpointServerSideException(errors.New("503"), nil, 503) } + retry := []*common.FailsafeConfig{{Retry: &common.RetryPolicyConfig{MaxAttempts: 3}}} + + t.Run("transient failures are retried until success", func(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(_ context.Context, req *common.NormalizedRequest, call int32) (*common.NormalizedResponse, error) { + if call < 3 { + return nil, serverErr() + } + return stubResult(req, `"0x1"`), nil + }) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "rt-ok", Failsafe: retry}, sc) + req := fwdReq("eth_call") + resp, err := u.Forward(env.ctx, req, false, false) + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, int32(3), sc.calls.Load()) + snap := req.ExecState().Snapshot() + assert.Equal(t, 3, snap.UpstreamAttempts) + assert.Equal(t, 2, snap.UpstreamRetries) + log := req.ExecState().UpstreamAttemptLog() + require.Len(t, log, 3) + assert.False(t, log[0].IsRetry) + assert.True(t, log[2].IsRetry) + assert.Equal(t, common.SelectionReasonRetry, log[2].Reason) + assert.Equal(t, common.UpstreamOutcomeSuccess, log[2].Outcome) + }) + + t.Run("exhausted retries surface ErrFailsafeRetryExceeded", func(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(context.Context, *common.NormalizedRequest, int32) (*common.NormalizedResponse, error) { + return nil, serverErr() + }) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "rt-fail", Failsafe: retry}, sc) + _, err := u.Forward(env.ctx, fwdReq("eth_call"), false, false) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeFailsafeRetryExceeded), "%v", err) + assert.Equal(t, int32(3), sc.calls.Load()) + }) + + t.Run("non-retryable errors and write methods are attempted once", func(t *testing.T) { + cases := []struct { + method string + err error + }{ + {"eth_sendTransaction", serverErr()}, + {"eth_call", common.NewErrEndpointExecutionException(errors.New("revert"))}, + {"eth_call", common.NewErrEndpointCapacityExceeded(errors.New("429"))}, + } + for _, tc := range cases { + env := newFwdEnv(t, nil) + sc := newStubClient(func(context.Context, *common.NormalizedRequest, int32) (*common.NormalizedResponse, error) { + return nil, tc.err + }) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "rt-once", Failsafe: retry}, sc) + _, err := u.Forward(env.ctx, fwdReq(tc.method), false, false) + require.Error(t, err) + assert.Equal(t, int32(1), sc.calls.Load(), "%s / %v", tc.method, tc.err) + } + }) +} + +func TestUpstreamForward_TimeoutPolicyFires(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(ctx context.Context, _ *common.NormalizedRequest, _ int32) (*common.NormalizedResponse, error) { + <-ctx.Done() + return nil, context.Cause(ctx) + }) + u := env.mustUpstream(t, &common.UpstreamConfig{ + Id: "to-fire", + Failsafe: []*common.FailsafeConfig{{Timeout: &common.TimeoutPolicyConfig{Duration: common.NewStaticDuration(20 * time.Millisecond)}}}, + }, sc) + req := fwdReq("eth_call") + _, err := u.Forward(env.ctx, req, false, false) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeFailsafeTimeoutExceeded), "%v", err) + log := req.ExecState().UpstreamAttemptLog() + require.Len(t, log, 1) + assert.Equal(t, common.UpstreamOutcomeTimeout, log[0].Outcome) +} + +func TestUpstreamForward_CircuitBreakerOpensAndBlocks(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(context.Context, *common.NormalizedRequest, int32) (*common.NormalizedResponse, error) { + return nil, common.NewErrEndpointServerSideException(errors.New("500"), nil, 500) + }) + u := env.mustUpstream(t, &common.UpstreamConfig{ + Id: "cb-forward", + Failsafe: []*common.FailsafeConfig{{CircuitBreaker: breakerCfg()}}, + }, sc) + + transitions := telemetry.MetricUpstreamBreakerStateChange.WithLabelValues("prj", "cb-forward", "closed_to_open") + before := testutil.ToFloat64(transitions) + assert.Equal(t, failsafe.StateClosed, u.CircuitBreakerState()) + _, err := u.Forward(env.ctx, fwdReq("eth_call"), false, false) + require.Error(t, err) + assert.Equal(t, failsafe.StateOpen, u.CircuitBreakerState()) + assert.True(t, u.IsDown("eth_call", common.DataFinalityStateUnknown)) + + req := fwdReq("eth_call") + _, err = u.Forward(env.ctx, req, false, false) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, common.ErrCodeFailsafeCircuitBreakerOpen), "%v", err) + assert.Equal(t, int32(1), sc.calls.Load(), "an open breaker never dials") + _ = req + + // The transition hook wired by NewUpstream feeds the breaker metric (async). + require.Eventually(t, func() bool { + return testutil.ToFloat64(transitions) == before+1 + }, 5*time.Second, 10*time.Millisecond) +} + +func TestUpstreamForward_HedgeWinsOverSlowPrimary(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(ctx context.Context, req *common.NormalizedRequest, call int32) (*common.NormalizedResponse, error) { + if call == 1 { + <-ctx.Done() + return nil, common.NewErrEndpointRequestCanceled(ctx.Err()) + } + return stubResult(req, `"0x1"`), nil + }) + u := env.mustUpstream(t, &common.UpstreamConfig{ + Id: "hedge-win", + Failsafe: []*common.FailsafeConfig{{Hedge: &common.HedgePolicyConfig{MaxCount: 1, Delay: common.NewStaticDuration(10 * time.Millisecond)}}}, + }, sc) + req := fwdReq("eth_getBalance") + resp, err := u.Forward(env.ctx, req, false, false) + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, 1, req.ExecState().Snapshot().UpstreamHedges) + require.Eventually(t, func() bool { return len(req.ExecState().UpstreamAttemptLog()) == 2 }, 5*time.Second, 5*time.Millisecond) + var hedged *common.UpstreamAttempt + for _, a := range req.ExecState().UpstreamAttemptLog() { + if a.IsHedge { + a := a + hedged = &a + } + } + require.NotNil(t, hedged, "the hedge attempt is recorded as such") + assert.Equal(t, common.UpstreamOutcomeSuccess, hedged.Outcome) + assert.Equal(t, common.SelectionReasonHedge, hedged.Reason) + // Hedges are speculative fan-out: they do not inflate the request counter. + m := env.tracker.GetUpstreamMethodMetrics(u, "eth_getBalance", common.DataFinalityStateAll) + assert.Equal(t, int64(1), m.RequestsTotal.Load()) +} + +func TestUpstreamForward_NetworkHedgeFlagSkipsRateCounters(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(context.Context, *common.NormalizedRequest, int32) (*common.NormalizedResponse, error) { + return nil, common.NewErrEndpointServerSideException(errors.New("500"), nil, 500) + }) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "net-hedge"}, sc) + req := fwdReq("eth_call") + _, err := u.Forward(env.ctx, req, false, true) + require.Error(t, err) + m := env.tracker.GetUpstreamMethodMetrics(u, "eth_call", common.DataFinalityStateAll) + assert.Equal(t, int64(0), m.RequestsTotal.Load()) + assert.Equal(t, int64(0), m.ErrorsTotal.Load()) + log := req.ExecState().UpstreamAttemptLog() + require.Len(t, log, 1) + assert.True(t, log[0].IsHedge) +} + +func TestUpstreamForward_AutoTunerShrinksBudgetUnderRemoteThrottling(t *testing.T) { + cases := []struct { + name string + err error + wantMax uint32 + }{ + {"remote 429s shrink the budget", common.NewErrEndpointCapacityExceeded(errors.New("429")), 50}, + {"clean traffic grows the budget", nil, 150}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rule := &common.RateLimitRuleConfig{Method: "eth_call", MaxCount: 100, Period: common.RateLimitPeriodMinute} + env := newFwdEnv(t, &common.RateLimiterConfig{ + Store: &common.RateLimitStoreConfig{Driver: "memory"}, + Budgets: []*common.RateLimitBudgetConfig{{Id: "tuned", Rules: []*common.RateLimitRuleConfig{rule}}}, + }) + sc := newStubClient(func(_ context.Context, req *common.NormalizedRequest, _ int32) (*common.NormalizedResponse, error) { + if tc.err != nil { + return nil, tc.err + } + return stubResult(req, `"0x1"`), nil + }) + u := env.mustUpstream(t, &common.UpstreamConfig{ + Id: "tuned-ups", + RateLimitBudget: "tuned", + RateLimitAutoTune: &common.RateLimitAutoTuneConfig{ + Enabled: util.BoolPtr(true), + AdjustmentPeriod: common.Duration(time.Hour), + ErrorRateThreshold: 0.1, + IncreaseFactor: 1.5, + DecreaseFactor: 0.5, + MinBudget: 10, + MaxBudget: 1000, + }, + }, sc) + require.NotNil(t, u.rateLimiterAutoTuner) + + // The first sample opens the window; the next ten accumulate in it. + for range 11 { + _, _ = u.Forward(env.ctx, fwdReq("eth_call"), false, false) + } + assert.Equal(t, uint32(100), rule.MaxCount, "no adjustment inside the window") + + u.rateLimiterAutoTuner.mu.Lock() + u.rateLimiterAutoTuner.lastAdjustments["eth_call"] = time.Now().Add(-2 * time.Hour) + u.rateLimiterAutoTuner.mu.Unlock() + _, _ = u.Forward(env.ctx, fwdReq("eth_call"), false, false) + assert.Equal(t, tc.wantMax, rule.MaxCount) + }) + } +} diff --git a/upstream/upstream_isdown_test.go b/upstream/upstream_isdown_test.go index 14398d43b..3d84e2728 100644 --- a/upstream/upstream_isdown_test.go +++ b/upstream/upstream_isdown_test.go @@ -34,9 +34,44 @@ func TestUpstream_IsDown_ScopedToTheMethodsPolicy(t *testing.T) { traces.Breaker().Record(failsafe.OutcomeFailure) require.Equal(t, failsafe.StateOpen, traces.Breaker().State()) - require.True(t, u.IsDown("trace_block")) - require.False(t, u.IsDown("eth_subscribe"), "a trace_* breaker must not take eth_subscribe down") + require.True(t, u.IsDown("trace_block", common.DataFinalityStateUnknown)) + require.False(t, u.IsDown("eth_subscribe", common.DataFinalityStateUnknown), "a trace_* breaker must not take eth_subscribe down") catchAll.Breaker().Record(failsafe.OutcomeFailure) - require.True(t, u.IsDown("eth_subscribe")) + require.True(t, u.IsDown("eth_subscribe", common.DataFinalityStateUnknown)) +} + +// An open breaker scoped to finalized must take finalized traffic down +// without affecting realtime. Unknown must not pick the finalized-scoped +// breaker when a catch-all also exists (the old synthetic-request path). +func TestUpstream_IsDown_RespectsFinality(t *testing.T) { + lg := zerolog.Nop() + mk := func(method string, finalities []common.DataFinalityState) *upstreamExecutor { + ex, err := NewUpstreamExecutor(&common.FailsafeConfig{ + MatchMethod: method, + MatchFinality: finalities, + CircuitBreaker: &common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 1, + FailureThresholdCapacity: 1, + HalfOpenAfter: common.Duration(time.Hour), + SuccessThresholdCount: 1, + SuccessThresholdCapacity: 1, + }, + }, &lg) + require.NoError(t, err) + return ex + } + finalized := mk("eth_call", []common.DataFinalityState{common.DataFinalityStateFinalized}) + catchAll := mk("*", nil) + u := &Upstream{failsafeExecutors: []*upstreamExecutor{finalized, catchAll}} + + finalized.Breaker().Record(failsafe.OutcomeFailure) + require.Equal(t, failsafe.StateOpen, finalized.Breaker().State()) + + require.True(t, u.IsDown("eth_call", common.DataFinalityStateFinalized), + "finalized eth_call must see the open finalized breaker") + require.False(t, u.IsDown("eth_call", common.DataFinalityStateRealtime), + "realtime eth_call falls through to the closed catch-all") + require.False(t, u.IsDown("eth_call", common.DataFinalityStateUnknown), + "unknown finality must not pick the finalized-scoped breaker") } diff --git a/upstream/upstream_state_test.go b/upstream/upstream_state_test.go new file mode 100644 index 000000000..f23b1f2fa --- /dev/null +++ b/upstream/upstream_state_test.go @@ -0,0 +1,366 @@ +package upstream + +import ( + "context" + "errors" + "math" + "net/url" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/failsafe" + "github.com/erpc/erpc/util" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const errCodeClientInit = common.ErrorCode("ErrUpstreamClientInitialization") + +func answering(result string) *stubClient { + return newStubClient(func(_ context.Context, req *common.NormalizedRequest, _ int32) (*common.NormalizedResponse, error) { + return stubResult(req, result), nil + }) +} + +func TestUpstreamState_ShouldHandleMethod_IgnoreThenAllowPrecedence(t *testing.T) { + env := newFwdEnv(t, nil) + u := env.mustUpstream(t, &common.UpstreamConfig{ + Id: "rules", + IgnoreMethods: []string{"eth_*"}, + AllowMethods: []string{"eth_getLogs"}, + }, answering(`"0x1"`)) + + cases := map[string]bool{ + "eth_getLogs": true, // allow overrides a matching ignore + "eth_call": false, // ignored by wildcard + "net_version": true, // matched by neither: allowed by default + } + for method, want := range cases { + got, err := u.ShouldHandleMethod(method) + require.NoError(t, err) + assert.Equal(t, want, got, method) + } +} + +func TestUpstreamState_ShouldHandleMethod_InvalidPatternErrors(t *testing.T) { + for _, cfg := range []*common.UpstreamConfig{ + {Id: "bad-ignore", IgnoreMethods: []string{"(eth_"}}, + {Id: "bad-allow", AllowMethods: []string{"(eth_"}}, + } { + env := newFwdEnv(t, nil) + u := env.mustUpstream(t, cfg, answering(`"0x1"`)) + ok, err := u.ShouldHandleMethod("eth_call") + require.Error(t, err, cfg.Id) + assert.False(t, ok, cfg.Id) + } +} + +func TestUpstreamState_IgnoreMethod_OnlyWhenAutoIgnoreEnabled(t *testing.T) { + env := newFwdEnv(t, nil) + off := env.mustUpstream(t, &common.UpstreamConfig{Id: "auto-off"}, answering(`"0x1"`)) + off.IgnoreMethod("trace_block") + ok, err := off.ShouldHandleMethod("trace_block") + require.NoError(t, err) + assert.True(t, ok, "without autoIgnoreUnsupportedMethods the method stays allowed") + assert.Empty(t, off.Config().IgnoreMethods) + + on := env.mustUpstream(t, &common.UpstreamConfig{Id: "auto-on", AutoIgnoreUnsupportedMethods: util.BoolPtr(true)}, answering(`"0x1"`)) + on.IgnoreMethod("trace_block") + ok, err = on.ShouldHandleMethod("trace_block") + require.NoError(t, err) + assert.False(t, ok) + assert.Equal(t, []string{"trace_block"}, on.Config().IgnoreMethods) + + _, err = on.Forward(env.ctx, fwdReq("trace_block"), false, false) + assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamMethodIgnored), "learned ignore is enforced on Forward: %v", err) +} + +func TestUpstreamState_NilUpstreamGetters(t *testing.T) { + var u *Upstream + assert.Equal(t, "", u.Id()) + assert.Equal(t, "n/a", u.NetworkId()) + assert.Equal(t, "n/a", u.NetworkLabel()) + assert.Equal(t, "nil", u.VendorName()) + assert.Nil(t, u.Config()) + assert.True(t, u.IsDown("eth_call", common.DataFinalityStateUnknown)) + assert.Nil(t, u.MetricsTracker()) + assert.Nil(t, u.Tracker()) + assert.Nil(t, u.Vendor()) + assert.Equal(t, failsafe.StateClosed, u.CircuitBreakerState()) +} + +func TestUpstreamState_GettersAndMarshal(t *testing.T) { + env := newFwdEnv(t, nil) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "getters", Endpoint: "https://eth.rpc.example.com"}, answering(`"0x1"`)) + assert.Equal(t, "getters", u.Id()) + assert.Equal(t, "unknown-example.com", u.VendorName(), "vendor name is guessed from the endpoint domain") + assert.Same(t, env.tracker, u.MetricsTracker()) + assert.NotNil(t, u.Tracker()) + assert.Nil(t, u.Vendor()) + assert.Equal(t, failsafe.StateClosed, u.CircuitBreakerState()) + assert.Nil(t, u.EvmMulticall3Batcher(), "multicall3 is off unless configured") + + b, err := u.MarshalJSON() + require.NoError(t, err) + assert.Contains(t, string(b), `"id":"getters"`) + assert.Contains(t, string(b), `"networkId":"n/a"`) + + mc := env.mustUpstream(t, &common.UpstreamConfig{Id: "mc", Evm: &common.EvmUpstreamConfig{Multicall3: &common.EvmMulticall3Config{MaxCalls: 10}}}, answering(`"0x1"`)) + assert.NotNil(t, mc.EvmMulticall3Batcher()) +} + +func TestUpstreamState_CircuitBreakerState_ReflectsCatchAllBreaker(t *testing.T) { + env := newFwdEnv(t, nil) + u := env.mustUpstream(t, &common.UpstreamConfig{ + Id: "cb", + Failsafe: []*common.FailsafeConfig{{MatchMethod: "*", CircuitBreaker: breakerCfg()}}, + }, answering(`"0x1"`)) + assert.Equal(t, failsafe.StateClosed, u.CircuitBreakerState()) + u.failsafeExecutors[0].Breaker().Record(failsafe.OutcomeFailure) + assert.Equal(t, failsafe.StateOpen, u.CircuitBreakerState()) +} + +func TestUpstreamState_SetNetworkConfig(t *testing.T) { + env := newFwdEnv(t, nil) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "netcfg"}, answering(`"0x1"`)) + u.SetNetworkConfig(nil) + assert.Equal(t, "n/a", u.NetworkId(), "nil config is ignored") + + u.SetNetworkConfig(&common.NetworkConfig{Architecture: common.ArchitectureEvm, Evm: &common.EvmNetworkConfig{ChainId: 1}}) + assert.Equal(t, "evm:1", u.NetworkId()) + assert.Equal(t, "evm:1", u.NetworkLabel(), "label defaults to the network id") + + u.SetNetworkConfig(&common.NetworkConfig{Architecture: common.ArchitectureEvm, Evm: &common.EvmNetworkConfig{ChainId: 1}, Alias: "mainnet"}) + assert.Equal(t, "mainnet", u.NetworkLabel(), "alias wins as label") +} + +func TestUpstreamState_DetectFeatures(t *testing.T) { + t.Run("evm chain id is detected and stored", func(t *testing.T) { + env := newFwdEnv(t, nil) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "df-ok"}, answering(`"0x7b"`)) + require.NoError(t, u.detectFeatures(env.ctx)) + assert.Equal(t, int64(123), u.Config().Evm.ChainId) + assert.Equal(t, "evm:123", u.NetworkId()) + }) + t.Run("configured chain id mismatch is task-fatal", func(t *testing.T) { + env := newFwdEnv(t, nil) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "df-mm", Evm: &common.EvmUpstreamConfig{ChainId: 1}}, answering(`"0x7b"`)) + err := u.detectFeatures(env.ctx) + require.Error(t, err) + var fatal *common.TaskFatalError + require.True(t, errors.As(err, &fatal)) + assert.True(t, common.HasErrorCode(fatal.Err, errCodeClientInit)) + assert.Contains(t, err.Error(), "ErrUpstreamChainIdMismatch") + }) + t.Run("rpc failure is retryable init error", func(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(_ context.Context, _ *common.NormalizedRequest, _ int32) (*common.NormalizedResponse, error) { + return nil, common.NewErrEndpointServerSideException(errors.New("down"), nil, 503) + }) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "df-down"}, sc) + err := u.detectFeatures(env.ctx) + require.Error(t, err) + var fatal *common.TaskFatalError + assert.False(t, errors.As(err, &fatal)) + assert.True(t, common.HasErrorCode(err, errCodeClientInit)) + }) + t.Run("json-rpc error and non-hex result fail detection", func(t *testing.T) { + env := newFwdEnv(t, nil) + rpcErr := newStubClient(func(_ context.Context, req *common.NormalizedRequest, _ int32) (*common.NormalizedResponse, error) { + return stubRpcError(req), nil + }) + u := env.mustUpstream(t, &common.UpstreamConfig{Id: "df-rpcerr"}, rpcErr) + _, err := u.EvmGetChainId(env.ctx) + require.Error(t, err) + + u = env.mustUpstream(t, &common.UpstreamConfig{Id: "df-num"}, answering(`123`)) + _, err = u.EvmGetChainId(env.ctx) + require.Error(t, err, "a numeric (non-string) chainId cannot be decoded") + }) + t.Run("type checks", func(t *testing.T) { + lg := zerolog.Nop() + for _, tc := range []struct { + cfg *common.UpstreamConfig + want string + }{ + {&common.UpstreamConfig{Id: "no-type"}, "upstream type not set"}, + {&common.UpstreamConfig{Id: "bad-type", Type: "cosmos"}, "upstream type not supported"}, + {&common.UpstreamConfig{Id: "svm-nocfg", Type: common.UpstreamTypeSvm}, "missing svm config"}, + {&common.UpstreamConfig{Id: "svm-nocluster", Type: common.UpstreamTypeSvm, Svm: &common.SvmUpstreamConfig{}}, "missing svm.cluster"}, + } { + u := &Upstream{config: tc.cfg, logger: &lg} + err := u.detectFeatures(context.Background()) + require.Error(t, err, tc.cfg.Id) + assert.Contains(t, err.Error(), tc.want, tc.cfg.Id) + } + u := &Upstream{config: &common.UpstreamConfig{Id: "svm-ok", Type: common.UpstreamTypeSvm, Svm: &common.SvmUpstreamConfig{Cluster: "devnet"}}, logger: &lg} + require.NoError(t, u.detectFeatures(context.Background())) + assert.Equal(t, "svm:devnet", u.NetworkId()) + }) +} + +func TestUpstreamState_SvmVerifyGenesisHash(t *testing.T) { + const mainnet = "5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d" + svmCfg := func(id string, s *common.SvmUpstreamConfig) *common.UpstreamConfig { + return &common.UpstreamConfig{Id: id, Type: common.UpstreamTypeSvm, Svm: s} + } + t.Run("known cluster matching hash passes", func(t *testing.T) { + env := newFwdEnv(t, nil) + u := env.mustUpstream(t, svmCfg("g-ok", &common.SvmUpstreamConfig{Cluster: "mainnet-beta"}), answering(`"`+mainnet+`"`)) + require.NoError(t, u.svmVerifyGenesisHash(env.ctx)) + assert.Nil(t, u.SvmStatePoller()) + }) + t.Run("known cluster wrong hash is rejected", func(t *testing.T) { + env := newFwdEnv(t, nil) + u := env.mustUpstream(t, svmCfg("g-bad", &common.SvmUpstreamConfig{Cluster: "mainnet-beta"}), answering(`"wronghash"`)) + err := u.svmVerifyGenesisHash(env.ctx) + require.Error(t, err) + assert.True(t, common.HasErrorCode(err, errCodeClientInit)) + assert.Contains(t, err.Error(), "mismatch") + }) + t.Run("fetch failure on a known cluster fails closed", func(t *testing.T) { + env := newFwdEnv(t, nil) + sc := newStubClient(func(_ context.Context, req *common.NormalizedRequest, _ int32) (*common.NormalizedResponse, error) { + return stubRpcError(req), nil + }) + u := env.mustUpstream(t, svmCfg("g-err", &common.SvmUpstreamConfig{Cluster: "devnet"}), sc) + err := u.svmVerifyGenesisHash(env.ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "getGenesisHash failed") + }) + t.Run("unknown cluster with checkGenesisHash accepts any decodable hash", func(t *testing.T) { + env := newFwdEnv(t, nil) + u := env.mustUpstream(t, svmCfg("g-custom", &common.SvmUpstreamConfig{Cluster: "localnet", CheckGenesisHash: true}), answering(`"anyhash"`)) + require.NoError(t, u.svmVerifyGenesisHash(env.ctx)) + + u = env.mustUpstream(t, svmCfg("g-undecodable", &common.SvmUpstreamConfig{Cluster: "localnet", CheckGenesisHash: true}), answering(`42`)) + err := u.svmVerifyGenesisHash(env.ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "decode genesis hash") + }) +} + +func TestUpstreamState_EvmPollerAccessors(t *testing.T) { + lg := zerolog.Nop() + bare := &Upstream{config: &common.UpstreamConfig{Id: "bare"}, logger: &lg} + _, err := bare.EvmLatestBlock() + require.Error(t, err) + _, err = bare.EvmFinalizedBlock() + require.Error(t, err) + _, err = bare.EvmIsBlockFinalized(context.Background(), 1, false) + require.Error(t, err) + assert.Equal(t, common.EvmSyncingStateUnknown, bare.EvmSyncingState()) + + sp := &mockEvmStatePollerEnhanced{latestBlock: 200, finalizedBlock: 100} + u := &Upstream{config: &common.UpstreamConfig{Id: "polled"}, logger: &lg, evmStatePoller: sp} + latest, err := u.EvmLatestBlock() + require.NoError(t, err) + assert.Equal(t, int64(200), latest) + fin, err := u.EvmFinalizedBlock() + require.NoError(t, err) + assert.Equal(t, int64(100), fin) + assert.Equal(t, common.EvmSyncingStateNotSyncing, u.EvmSyncingState()) + + ok, err := u.EvmIsBlockFinalized(context.Background(), 50, false) + require.NoError(t, err) + assert.True(t, ok) + ok, err = u.EvmIsBlockFinalized(context.Background(), 150, true) + require.NoError(t, err) + assert.False(t, ok, "fresh poll still reports finalized=100 < 150") + sp.finalizedBlock = 150 + ok, err = u.EvmIsBlockFinalized(context.Background(), 150, false) + require.NoError(t, err) + assert.True(t, ok) + + sp.pollError = errors.New("poll failed") + _, err = u.EvmIsBlockFinalized(context.Background(), 150, true) + require.Error(t, err) +} + +func TestUpstreamState_ResolveAvailabilityBounds(t *testing.T) { + lg := zerolog.Nop() + i64 := func(v int64) *int64 { return &v } + mk := func(evm *common.EvmUpstreamConfig, latest int64) *Upstream { + return &Upstream{config: &common.UpstreamConfig{Id: "b", Evm: evm}, logger: &lg, evmStatePoller: &mockEvmStatePollerEnhanced{latestBlock: latest}} + } + cases := []struct { + name string + evm *common.EvmUpstreamConfig + latest int64 + min, max int64 + }{ + {"no evm config", nil, 1000, math.MinInt64, math.MaxInt64}, + {"legacy maxAvailableRecentBlocks", &common.EvmUpstreamConfig{MaxAvailableRecentBlocks: 128}, 1000, 872, math.MaxInt64}, + {"legacy without latest is unbounded", &common.EvmUpstreamConfig{MaxAvailableRecentBlocks: 128}, 0, math.MinInt64, math.MaxInt64}, + {"exact lower, latest-minus upper", &common.EvmUpstreamConfig{BlockAvailability: &common.EvmBlockAvailabilityConfig{ + Lower: &common.EvmAvailabilityBoundConfig{ExactBlock: i64(100)}, + Upper: &common.EvmAvailabilityBoundConfig{LatestBlockMinus: i64(5)}, + }}, 1000, 100, 995}, + {"latest-minus without latest is unbounded", &common.EvmUpstreamConfig{BlockAvailability: &common.EvmBlockAvailabilityConfig{ + Lower: &common.EvmAvailabilityBoundConfig{LatestBlockMinus: i64(10)}, + Upper: &common.EvmAvailabilityBoundConfig{LatestBlockMinus: i64(5)}, + }}, 0, math.MinInt64, math.MaxInt64}, + {"earliest-plus lower uses detected earliest", &common.EvmUpstreamConfig{BlockAvailability: &common.EvmBlockAvailabilityConfig{ + Lower: &common.EvmAvailabilityBoundConfig{EarliestBlockPlus: i64(7)}, + }}, 1000, 7, math.MaxInt64}, + {"empty bound config is a no-op", &common.EvmUpstreamConfig{BlockAvailability: &common.EvmBlockAvailabilityConfig{ + Lower: &common.EvmAvailabilityBoundConfig{}, + Upper: &common.EvmAvailabilityBoundConfig{}, + }}, 1000, math.MinInt64, math.MaxInt64}, + {"inverted range fails open", &common.EvmUpstreamConfig{BlockAvailability: &common.EvmBlockAvailabilityConfig{ + Lower: &common.EvmAvailabilityBoundConfig{ExactBlock: i64(500)}, + Upper: &common.EvmAvailabilityBoundConfig{ExactBlock: i64(100)}, + }}, 1000, math.MinInt64, math.MaxInt64}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + lo, hi := mk(tc.evm, tc.latest).EvmBlockAvailabilityBounds() + assert.Equal(t, tc.min, lo) + assert.Equal(t, tc.max, hi) + }) + } +} + +func TestUpstreamState_GuessVendorName(t *testing.T) { + cases := map[string]string{ + "": "", + "http://10.0.0.1:8545": "unknown-10.0.0.1", + "http://rpc1.localhost": "unknown-rpc1.localhost", + "https://eth.rpc.example.com": "unknown-example.com", + "https://x.rpc.b.co": "unknown-rpc.b.co", + "://bad": "", + } + for endpoint, want := range cases { + u := &Upstream{config: &common.UpstreamConfig{Endpoint: endpoint}} + assert.Equal(t, want, u.guessVendorName(), endpoint) + } +} + +func TestUpstreamState_ClassifyUpstreamOutcome(t *testing.T) { + cause := errors.New("x") + req := fwdReq("eth_call") + cases := []struct { + name string + resp *common.NormalizedResponse + err error + want common.UpstreamAttemptOutcome + }{ + {"cancelled code", nil, common.NewErrEndpointRequestCanceled(cause), common.UpstreamOutcomeCancelled}, + {"context canceled", nil, context.Canceled, common.UpstreamOutcomeCancelled}, + {"dynamic timeout", nil, common.ErrDynamicTimeoutExceeded, common.UpstreamOutcomeTimeout}, + {"method ignored", nil, common.NewErrUpstreamMethodIgnored("m", "u"), common.UpstreamOutcomeSkipped}, + {"capacity", nil, common.NewErrEndpointCapacityExceeded(cause), common.UpstreamOutcomeRateLimited}, + {"exec revert", nil, common.NewErrEndpointExecutionException(cause), common.UpstreamOutcomeExecRevert}, + {"block unavailable", nil, common.NewErrUpstreamBlockUnavailable("u", 10, 5, 1), common.UpstreamOutcomeBlockUnavailable}, + {"transport", nil, common.NewErrEndpointTransportFailure(&url.URL{Host: "h"}, cause), common.UpstreamOutcomeTransportError}, + {"server", nil, common.NewErrEndpointServerSideException(cause, nil, 500), common.UpstreamOutcomeServerError}, + {"client", nil, common.NewErrEndpointClientSideException(cause), common.UpstreamOutcomeClientError}, + {"unclassified error", nil, cause, common.UpstreamOutcomeServerError}, + {"empty result", stubResult(req, `null`), nil, common.UpstreamOutcomeEmpty}, + {"success", stubResult(req, `"0x1"`), nil, common.UpstreamOutcomeSuccess}, + } + for _, tc := range cases { + assert.Equal(t, tc.want, classifyUpstreamOutcome(tc.resp, tc.err), tc.name) + } +} diff --git a/upstream/upstream_test.go b/upstream/upstream_test.go index fd1451e6b..a40c061e7 100644 --- a/upstream/upstream_test.go +++ b/upstream/upstream_test.go @@ -297,6 +297,7 @@ func (m *mockEvmStatePoller) IsBlockFinalized(blockNumber int64) (bool, error) { } func (m *mockEvmStatePoller) SuggestFinalizedBlock(blockNumber int64) {} func (m *mockEvmStatePoller) SuggestLatestBlock(blockNumber int64) {} +func (m *mockEvmStatePoller) RefreshLatestBlock() {} func (m *mockEvmStatePoller) SetNetworkConfig(cfg *common.NetworkConfig) {} func (m *mockEvmStatePoller) IsObjectNull() bool { return m.isNull } @@ -616,6 +617,7 @@ func (m *mockEvmStatePollerWithUpdate) IsBlockFinalized(blockNumber int64) (bool } func (m *mockEvmStatePollerWithUpdate) SuggestFinalizedBlock(blockNumber int64) {} func (m *mockEvmStatePollerWithUpdate) SuggestLatestBlock(blockNumber int64) {} +func (m *mockEvmStatePollerWithUpdate) RefreshLatestBlock() {} func (m *mockEvmStatePollerWithUpdate) SetNetworkConfig(cfg *common.NetworkConfig) {} func (m *mockEvmStatePollerWithUpdate) IsObjectNull() bool { return false } diff --git a/util/dsl_test.go b/util/dsl_test.go new file mode 100644 index 000000000..b39e12f13 --- /dev/null +++ b/util/dsl_test.go @@ -0,0 +1,621 @@ +package util + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// dslRoundTripFunc lets tests answer the OpenAI client without a network. +type dslRoundTripFunc func(*http.Request) (*http.Response, error) + +func (f dslRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +// useFakeOpenAI swaps the package OpenAI client's transport for the test. +func useFakeOpenAI(t *testing.T, rt dslRoundTripFunc) { + t.Helper() + prev := openAIHTTPClient + openAIHTTPClient = &http.Client{Transport: rt, Timeout: 5 * time.Second} + t.Cleanup(func() { openAIHTTPClient = prev }) +} + +func openAIResponse(status int, body string, header http.Header) *http.Response { + if header == nil { + header = http.Header{} + } + return &http.Response{ + StatusCode: status, + Header: header, + Body: io.NopCloser(strings.NewReader(body)), + } +} + +// chatCompletion wraps content in the OpenAI chat-completions envelope. +func chatCompletion(t *testing.T, content string) string { + t.Helper() + b, err := json.Marshal(map[string]any{ + "choices": []map[string]any{{"message": map[string]string{"content": content}}}, + }) + require.NoError(t, err) + return string(b) +} + +type failingBody struct{} + +func (failingBody) Read([]byte) (int, error) { return 0, errors.New("body broke") } +func (failingBody) Close() error { return nil } + +func TestSanitizeTestName(t *testing.T) { + cases := []struct { + in, want string + }{ + {"simple", "simple"}, + {"Mixed Case Name", "mixed_case_name"}, + {"a, b; c/d.e(f)-g", "a_b_c_d_e_f_g"}, + {" leading and trailing ", "leading_and_trailing"}, + {"many spaces\tand\nnewlines", "many_spaces_and_newlines"}, + {"__already__underscored__", "already_underscored"}, + {"(only-punctuation)", "only_punctuation"}, + {"", ""}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + assert.Equal(t, tc.want, SanitizeTestName(tc.in)) + }) + } +} + +func TestSafeHash(t *testing.T) { + assert.Equal(t, "", safeHash("")) + assert.Equal(t, "abc", safeHash("abc")) + assert.Equal(t, "01234567", safeHash("01234567")) + assert.Equal(t, "01234567", safeHash("0123456789abcdef")) +} + +func TestChunkScenarios(t *testing.T) { + items := []string{"a", "b", "c", "d", "e", "f", "g"} + cases := []struct { + name string + in []string + size int + want [][]string + }{ + {"even split", items[:6], 3, [][]string{{"a", "b", "c"}, {"d", "e", "f"}}}, + {"remainder in last chunk", items, 3, [][]string{{"a", "b", "c"}, {"d", "e", "f"}, {"g"}}}, + {"size larger than input", items[:2], 10, [][]string{{"a", "b"}}}, + {"zero size defaults to 5", items, 0, [][]string{{"a", "b", "c", "d", "e"}, {"f", "g"}}}, + {"negative size defaults to 5", items[:5], -1, [][]string{{"a", "b", "c", "d", "e"}}}, + {"empty input", nil, 3, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, chunkScenarios(tc.in, tc.size)) + }) + } +} + +func TestParseRetryAfterHeader(t *testing.T) { + cases := []struct { + in string + want time.Duration + }{ + {"1", time.Second}, + {" 30 ", 30 * time.Second}, + {"300", 5 * time.Minute}, + {"301", 5 * time.Minute}, + {"86400", 5 * time.Minute}, + {"0", 0}, + {"-5", 0}, + {"1.5", 0}, + {"Wed, 21 Oct 2015 07:28:00 GMT", 0}, + {"", 0}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + assert.Equal(t, tc.want, parseRetryAfterHeader(tc.in)) + }) + } +} + +func TestBuildOpenAIUserPrompt(t *testing.T) { + p := buildOpenAIUserPrompt("SPEC-BODY", []string{"first scenario", "second scenario"}) + assert.Contains(t, p, "\n\nSPEC-BODY\n\n") + // Scenarios are listed verbatim, one per line, in order, without numbering. + assert.True(t, strings.HasSuffix(p, "Scenarios:\nfirst scenario\nsecond scenario\n"), p) + assert.NotContains(t, p, "1. first") + assert.Less(t, strings.Index(p, "SPEC-BODY"), strings.Index(p, "first scenario")) +} + +func TestParseOpenAIItems(t *testing.T) { + t.Run("items are sanitized and trimmed", func(t *testing.T) { + got, err := parseOpenAIItems(`{"items":[{"nl":"Two Upstreams, One Fails","dsl":" a => b \n"},{"nl":"x","dsl":"y"}]}`) + require.NoError(t, err) + assert.Equal(t, []dslItem{ + {NL: "two_upstreams_one_fails", DSL: "a => b"}, + {NL: "x", DSL: "y"}, + }, got) + }) + for name, content := range map[string]string{ + "empty items": `{"items":[]}`, + "dsl-only shape": `{"dsl":["a => b"]}`, + "not json": `items: nope`, + "wrong items type": `{"items":"nope"}`, + "empty json object": `{}`, + } { + t.Run(name, func(t *testing.T) { + got, err := parseOpenAIItems(content) + require.Error(t, err) + assert.Contains(t, err.Error(), "unrecognized OpenAI content shape") + assert.Nil(t, got) + }) + } +} + +func TestDslCache_WriteReadRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "nested", "dir", "cache.json") + want := &dslCache{Hash: "deadbeef", Items: []dslItem{{NL: "n1", DSL: "d1"}, {NL: "n2", DSL: "d2"}}} + require.NoError(t, writeDslCache(path, want)) + + _, err := os.Stat(path + ".tmp") + assert.True(t, os.IsNotExist(err), "temp file must be renamed away") + + raw, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(raw), "\n \"hash\": \"deadbeef\"", "cache is written indented") + + got := readDslCache(t, path) + require.NotNil(t, got) + assert.Equal(t, want, got) +} + +func TestDslCache_ReadMissingOrCorrupt(t *testing.T) { + dir := t.TempDir() + assert.Nil(t, readDslCache(t, filepath.Join(dir, "missing.json"))) + + corrupt := filepath.Join(dir, "corrupt.json") + require.NoError(t, os.WriteFile(corrupt, []byte("{not json"), 0o600)) + assert.Nil(t, readDslCache(t, corrupt)) + + legacy := filepath.Join(dir, "legacy.json") + require.NoError(t, os.WriteFile(legacy, []byte(`{"hash":"h","dsl":["a => b"]}`), 0o600)) + got := readDslCache(t, legacy) + require.NotNil(t, got) + assert.Equal(t, []string{"a => b"}, got.DSL) + assert.Empty(t, got.Items) +} + +func TestDslCache_WriteErrors(t *testing.T) { + c := &dslCache{Hash: "h"} + + t.Run("parent is a file", func(t *testing.T) { + dir := t.TempDir() + blocker := filepath.Join(dir, "file") + require.NoError(t, os.WriteFile(blocker, nil, 0o600)) + require.Error(t, writeDslCache(filepath.Join(blocker, "cache.json"), c)) + }) + + t.Run("temp path is a directory", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "cache.json") + require.NoError(t, os.Mkdir(path+".tmp", 0o750)) + require.Error(t, writeDslCache(path, c)) + }) + + t.Run("target is a non-empty directory", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "cache.json") + require.NoError(t, os.MkdirAll(filepath.Join(path, "child"), 0o750)) + require.Error(t, writeDslCache(path, c)) + }) +} + +func TestDoOpenAIRequestWithRetries(t *testing.T) { + t.Run("success sends auth and body", func(t *testing.T) { + var gotAuth, gotCT, gotBody, gotURL, gotMethod string + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + gotAuth = r.Header.Get("Authorization") + gotCT = r.Header.Get("Content-Type") + gotURL = r.URL.String() + gotMethod = r.Method + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + return openAIResponse(200, `{"ok":true}`, nil), nil + }) + body, err := doOpenAIRequestWithRetries(t, "sk-test", []byte(`{"q":1}`)) + require.NoError(t, err) + assert.Equal(t, `{"ok":true}`, string(body)) + assert.Equal(t, "Bearer sk-test", gotAuth) + assert.Equal(t, "application/json", gotCT) + assert.Equal(t, `{"q":1}`, gotBody) + assert.Equal(t, "POST", gotMethod) + assert.Equal(t, "https://api.openai.com/v1/chat/completions", gotURL) + }) + + for _, status := range []int{400, 401, 404} { + t.Run(fmt.Sprintf("status %d is not retried", status), func(t *testing.T) { + calls := 0 + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + calls++ + return openAIResponse(status, "denied", nil), nil + }) + _, err := doOpenAIRequestWithRetries(t, "k", []byte(`{}`)) + require.Error(t, err) + assert.Equal(t, fmt.Sprintf("status %d: denied", status), err.Error()) + assert.Equal(t, 1, calls) + }) + } + + t.Run("body read error on non-retryable status", func(t *testing.T) { + calls := 0 + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + calls++ + return &http.Response{StatusCode: 403, Header: http.Header{}, Body: failingBody{}}, nil + }) + _, err := doOpenAIRequestWithRetries(t, "k", []byte(`{}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "read body error (status=403)") + assert.Contains(t, err.Error(), "body broke") + assert.Equal(t, 1, calls) + }) + + t.Run("body read error on 2xx is not success", func(t *testing.T) { + calls := 0 + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return &http.Response{StatusCode: 200, Header: http.Header{}, Body: failingBody{}}, nil + } + return openAIResponse(400, "stop", nil), nil + }) + _, err := doOpenAIRequestWithRetries(t, "k", []byte(`{}`)) + require.Error(t, err) + // A 200 with an unreadable body is not retryable by status, so the + // read error is returned immediately. + assert.Contains(t, err.Error(), "read body error (status=200)") + assert.Equal(t, 1, calls) + }) + + t.Run("transport error then retry-after then success", func(t *testing.T) { + var calls int + var mu sync.Mutex + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + mu.Lock() + defer mu.Unlock() + calls++ + switch calls { + case 1: + return nil, errors.New("connection reset") + case 2: + return openAIResponse(429, "slow down", http.Header{"Retry-After": []string{"1"}}), nil + default: + return openAIResponse(200, "done", nil), nil + } + }) + body, err := doOpenAIRequestWithRetries(t, "k", []byte(`{}`)) + require.NoError(t, err) + assert.Equal(t, "done", string(body)) + assert.Equal(t, 3, calls) + }) + + t.Run("5xx with unusable retry-after backs off then succeeds", func(t *testing.T) { + calls := 0 + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return openAIResponse(503, "busy", http.Header{"Retry-After": []string{"later"}}), nil + } + return openAIResponse(201, "created", nil), nil + }) + body, err := doOpenAIRequestWithRetries(t, "k", []byte(`{}`)) + require.NoError(t, err) + assert.Equal(t, "created", string(body)) + assert.Equal(t, 2, calls) + }) +} + +func TestOpenAIGenerateChunk(t *testing.T) { + cases := []struct { + name string + resp func(t *testing.T) *http.Response + want []dslItem + wantErr string + }{ + { + name: "items parsed", + resp: func(t *testing.T) *http.Response { + return openAIResponse(200, chatCompletion(t, `{"items":[{"nl":"A B","dsl":" x => y "}]}`), nil) + }, + want: []dslItem{{NL: "a_b", DSL: "x => y"}}, + }, + { + name: "http error", + resp: func(t *testing.T) *http.Response { return openAIResponse(400, "bad", nil) }, + wantErr: "status 400: bad", + }, + { + name: "envelope not json", + resp: func(t *testing.T) *http.Response { return openAIResponse(200, "", nil) }, + wantErr: "invalid character", + }, + { + name: "no choices", + resp: func(t *testing.T) *http.Response { return openAIResponse(200, `{"choices":[]}`, nil) }, + wantErr: "no choices", + }, + { + name: "content in wrong shape", + resp: func(t *testing.T) *http.Response { + return openAIResponse(200, chatCompletion(t, `{"dsl":["x"]}`), nil) + }, + wantErr: "unrecognized OpenAI content shape", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var sent map[string]any + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + require.NoError(t, json.NewDecoder(r.Body).Decode(&sent)) + return tc.resp(t), nil + }) + got, err := openAIGenerateChunk(t, "k", "SPEC", []string{"scenario one"}) + if tc.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + assert.Nil(t, got) + } else { + require.NoError(t, err) + assert.Equal(t, tc.want, got) + } + assert.Equal(t, "gpt-5", sent["model"]) + assert.Equal(t, map[string]any{"type": "json_object"}, sent["response_format"]) + msgs := sent["messages"].([]any) + require.Len(t, msgs, 2) + user := msgs[1].(map[string]any) + assert.Equal(t, "user", user["role"]) + assert.Contains(t, user["content"], "scenario one") + assert.Contains(t, user["content"], "SPEC") + }) + } +} + +func TestOpenAIReviewDsl(t *testing.T) { + original := []dslItem{{NL: "n1", DSL: "d1"}, {NL: "n2", DSL: "d2"}} + + t.Run("no items", func(t *testing.T) { + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + t.Error("must not call OpenAI without items") + return nil, errors.New("unexpected") + }) + _, err := openAIReviewDsl(t, "k", "SPEC", nil) + require.EqualError(t, err, "no items to review") + }) + + cases := []struct { + name string + status int + body string + content string + want []dslItem + wantErr string + }{ + { + name: "corrected items returned as-is", + content: `{"approved":false,"items":[{"nl":"n1","dsl":"D1"},{"nl":"n2","dsl":"D2"}],"notes":"fixed"}`, + want: []dslItem{{NL: "n1", DSL: "D1"}, {NL: "n2", DSL: "D2"}}, + }, + { + name: "dsl-only list mapped onto original NLs", + content: `{"approved":true,"dsl":[" X1 ","X2"]}`, + want: []dslItem{{NL: "n1", DSL: "X1"}, {NL: "n2", DSL: "X2"}}, + }, + { + name: "fallback parser keeps original NL when omitted", + // approved is not a bool, so the strict decode fails and the + // generic items parser takes over. + content: `{"approved":"yes","items":[{"nl":"","dsl":"Y1"},{"nl":"Given NL","dsl":"Y2"}]}`, + want: []dslItem{{NL: "n1", DSL: "Y1"}, {NL: "given_nl", DSL: "Y2"}}, + }, + { + name: "item count mismatch", + content: `{"approved":true,"items":[{"nl":"n1","dsl":"d1"}]}`, + wantErr: "review returned unexpected shape", + }, + { + name: "content not json", + content: `looks good to me`, + wantErr: "review returned unexpected shape", + }, + { + name: "no choices", + body: `{"choices":[]}`, + wantErr: "no choices in review", + }, + { + name: "envelope not json", + body: `nope`, + wantErr: "invalid character", + }, + { + name: "http error", + status: 422, + body: "unprocessable", + wantErr: "status 422: unprocessable", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var userPrompt string + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + var sent struct { + Messages []map[string]string `json:"messages"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&sent)) + userPrompt = sent.Messages[1]["content"] + status := tc.status + if status == 0 { + status = 200 + } + body := tc.body + if body == "" { + body = chatCompletion(t, tc.content) + } + return openAIResponse(status, body, nil), nil + }) + in := append([]dslItem(nil), original...) + got, err := openAIReviewDsl(t, "k", "SPEC", in) + if tc.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + assert.Nil(t, got) + } else { + require.NoError(t, err) + assert.Equal(t, tc.want, got) + } + assert.Contains(t, userPrompt, "NL: n1\nDSL: d1\nNL: n2\nDSL: d2\n") + assert.Contains(t, userPrompt, "DSL Specification:\nSPEC") + }) + } +} + +// scenarioHash mirrors the cache key: sha256 of the JSON-encoded scenarios. +func scenarioHash(t *testing.T, scenarios []string) string { + t.Helper() + b, err := json.Marshal(scenarios) + require.NoError(t, err) + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} + +func TestGenerateDslFromScenariosViaLLM_CacheHit(t *testing.T) { + t.Chdir(t.TempDir()) + scenarios := []string{"one", "two"} + cached := []dslItem{{NL: "one", DSL: "a => ok"}, {NL: "two", DSL: "b => ok"}} + require.NoError(t, writeDslCache(filepath.Join("testdata", "ns.json"), &dslCache{Hash: scenarioHash(t, scenarios), Items: cached})) + + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + t.Error("cache hit must not call OpenAI") + return nil, errors.New("unexpected") + }) + assert.Equal(t, cached, GenerateDslFromScenariosViaLLM(t, "ns", "SPEC", scenarios)) +} + +func TestGenerateDslFromScenariosViaLLM_RegeneratesAndCaches(t *testing.T) { + scenarios := make([]string, 12) + for i := range scenarios { + scenarios[i] = fmt.Sprintf("Scenario %02d", i) + } + + cases := []struct { + name string + setup func(t *testing.T) + }{ + {"no cache", func(t *testing.T) {}}, + {"stale hash", func(t *testing.T) { + require.NoError(t, writeDslCache(filepath.Join("testdata", "ns.json"), + &dslCache{Hash: "0000", Items: []dslItem{{NL: "old", DSL: "old"}}})) + }}, + {"matching hash but legacy dsl-only cache", func(t *testing.T) { + require.NoError(t, writeDslCache(filepath.Join("testdata", "ns.json"), + &dslCache{Hash: scenarioHash(t, scenarios), DSL: []string{"legacy"}})) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Chdir(t.TempDir()) + require.NoError(t, os.WriteFile(".env", []byte("UNRELATED_DSL_TEST_VAR=1\n"), 0o600)) + t.Setenv("OPENAI_API_KEY", "sk-from-env") + tc.setup(t) + + var mu sync.Mutex + var chunkSizes []int + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + assert.Equal(t, "Bearer sk-from-env", r.Header.Get("Authorization")) + var sent struct { + Messages []map[string]string `json:"messages"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&sent)) + user := sent.Messages[1]["content"] + var items []map[string]string + for _, s := range scenarios { + if strings.Contains(user, s+"\n") { + items = append(items, map[string]string{"nl": s, "dsl": "dsl for " + s}) + } + } + mu.Lock() + chunkSizes = append(chunkSizes, len(items)) + mu.Unlock() + content, err := json.Marshal(map[string]any{"items": items}) + require.NoError(t, err) + return openAIResponse(200, chatCompletion(t, string(content)), nil), nil + }) + + got := GenerateDslFromScenariosViaLLM(t, "ns", "SPEC", scenarios) + require.Len(t, got, 12) + assert.Equal(t, []int{10, 2}, chunkSizes, "scenarios are generated in chunks of 10") + for i, s := range scenarios { + assert.Equal(t, dslItem{NL: SanitizeTestName(s), DSL: "dsl for " + s}, got[i]) + } + + cache := readDslCache(t, filepath.Join("testdata", "ns.json")) + require.NotNil(t, cache) + assert.Equal(t, scenarioHash(t, scenarios), cache.Hash) + assert.Equal(t, got, cache.Items) + + // Second call is served from the freshly written cache. + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + t.Error("fresh cache must not call OpenAI") + return nil, errors.New("unexpected") + }) + assert.Equal(t, got, GenerateDslFromScenariosViaLLM(t, "ns", "SPEC", scenarios)) + }) + } +} + +func TestGenerateDslFromScenariosViaLLM_CacheWriteFailureStillReturnsItems(t *testing.T) { + t.Chdir(t.TempDir()) + require.NoError(t, os.WriteFile(".env", nil, 0o600)) + // A regular file named testdata makes the cache directory uncreatable. + require.NoError(t, os.WriteFile("testdata", nil, 0o600)) + t.Setenv("OPENAI_API_KEY", "k") + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + return openAIResponse(200, chatCompletion(t, `{"items":[{"nl":"only","dsl":"x => y"}]}`), nil), nil + }) + got := GenerateDslFromScenariosViaLLM(t, "ns", "SPEC", []string{"only"}) + assert.Equal(t, []dslItem{{NL: "only", DSL: "x => y"}}, got) + fi, err := os.Stat("testdata") + require.NoError(t, err) + assert.False(t, fi.IsDir()) +} + +func TestDoOpenAIRequestWithRetries_ExhaustsAttempts(t *testing.T) { + if testing.Short() { + t.Skip("sleeps ~5s honoring Retry-After") + } + t.Run("transport error on final attempt", func(t *testing.T) { + calls := 0 + useFakeOpenAI(t, func(r *http.Request) (*http.Response, error) { + calls++ + if calls < 6 { + return openAIResponse(429, "busy", http.Header{"Retry-After": []string{"1"}}), nil + } + return nil, errors.New("final transport failure") + }) + _, err := doOpenAIRequestWithRetries(t, "k", []byte(`{}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "final transport failure") + assert.Equal(t, 6, calls, "gives up after 6 attempts") + }) +} diff --git a/util/misc_test.go b/util/misc_test.go new file mode 100644 index 000000000..ccb750b12 --- /dev/null +++ b/util/misc_test.go @@ -0,0 +1,390 @@ +package util + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExtractUsefulHeaders(t *testing.T) { + h := http.Header{} + kept := map[string]string{ + "X-Upstream": "u1", + "Traceparent": "00-abc", + "X-Debug-Info": "dbg", + "Correlation-Key": "corr", + "Cf-Ray-Id": "ray", + "Request-Id": "req", + "Span-Context": "span", + "Parent-Ref": "p", + "Upstream-Error": "boom", + "Rate-Limit-Remaining": "7", + "Content-Type": "application/json", + "Content-Length": "42", + "Server": "nginx", + "Date": "Wed, 21 Oct 2026 07:28:00 GMT", + "Etag": "\"v1\"", + "Status": "200", + "Retry-After": "3", + "Cache-Control": "no-cache", + } + for k, v := range kept { + h.Set(k, v) + } + h.Add("X-Multi", "first") + h.Add("X-Multi", "second") + for _, dropped := range []string{"Accept", "Vary", "Connection", "Set-Cookie", "Content-Encoding"} { + h.Set(dropped, "nope") + } + + got := ExtractUsefulHeaders(&http.Response{Header: h}) + + want := map[string]interface{}{"x-multi": "first"} + for k, v := range kept { + want[strings.ToLower(k)] = v + } + assert.Equal(t, want, got) + assert.Empty(t, ExtractUsefulHeaders(&http.Response{Header: http.Header{}})) +} + +func TestIsNativeProtocol(t *testing.T) { + for _, ep := range []string{ + "http://h", "https://h", "ws://h", "wss://h", "grpc://h", "grpc+bds://h", "grpcs://h", + } { + assert.True(t, IsNativeProtocol(ep), ep) + } + for _, ep := range []string{ + "alchemy://key", "envio://", "HTTP://h", "httpx://h", "grpc+other://h", "", "h/http://", + } { + assert.False(t, IsNativeProtocol(ep), ep) + } +} + +func TestRedactEndpoint(t *testing.T) { + cases := []struct { + name string + endpoint string + want string + }{ + {"https keeps host", "https://eth.example.com/v2/SECRETKEY", "https://eth.example.com#redacted="}, + {"wss keeps host", "wss://ws.example.com:8546/key", "wss://ws.example.com:8546#redacted="}, + {"grpc keeps host", "grpc://g.example.com:443", "grpc://g.example.com:443#redacted="}, + {"envio keeps host without hash", "envio://rpc.hypersync.xyz", "envio://rpc.hypersync.xyz"}, + {"evm+envio suffix", "evm+envio://host.io/x", "evm+envio://host.io"}, + {"repository keeps host", "repository://evm-public-endpoints.erpc.cloud", "repository://evm-public-endpoints.erpc.cloud#redacted="}, + {"vendor scheme drops host", "alchemy://SECRETKEY@host", "alchemy#redacted="}, + {"no scheme", "just-a-string", "redacted="}, + {"no host", "mailto:user@example.com", "redacted="}, + {"unparseable", "http://[::1", "redacted="}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := RedactEndpoint(tc.endpoint) + if strings.HasSuffix(tc.want, "=") { + require.True(t, strings.HasPrefix(got, tc.want), "got %q", got) + assert.Len(t, strings.TrimPrefix(got, tc.want), 5, "hash suffix is 5 hex chars") + assert.NotContains(t, got, "SECRETKEY") + } else { + assert.Equal(t, tc.want, got) + } + }) + } + + // Same input → same hash; different secret on same host → different hash. + a := RedactEndpoint("https://h.example/k1") + assert.Equal(t, a, RedactEndpoint("https://h.example/k1")) + assert.NotEqual(t, a, RedactEndpoint("https://h.example/k2")) +} + +func TestParseByteSize(t *testing.T) { + cases := []struct { + in string + want int + wantErr string + }{ + {"0", 0, ""}, + {"512", 512, ""}, + {"512B", 512, ""}, + {"1kb", 1024, ""}, + {" 2 KB ", 2048, ""}, + {"3MB", 3 * 1024 * 1024, ""}, + {"10 mb", 10 * 1024 * 1024, ""}, + {"", 0, "empty size string"}, + {" ", 0, "empty size string"}, + {"-1KB", 0, "size cannot be negative"}, + {"1.5MB", 0, "invalid size format"}, + {"1GB", 0, "invalid size format"}, + {"abc", 0, "invalid size format"}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + got, err := ParseByteSize(tc.in) + if tc.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestPtrHelpers(t *testing.T) { + s := StringPtr("x") + assert.Equal(t, "x", *s) + assert.NotSame(t, s, StringPtr("x"), "each call returns a fresh pointer") + assert.Equal(t, 7, *IntPtr(7)) + assert.Equal(t, true, *BoolPtr(true)) + assert.Equal(t, 1.5, *Float64Ptr(1.5)) +} + +func TestAtomicValue(t *testing.T) { + av := AtomicValue("initial") + assert.Equal(t, "initial", av.Load()) + av.Store("next") + assert.Equal(t, "next", av.Load()) +} + +func TestFastMemConversions(t *testing.T) { + assert.Equal(t, "", B2Str(nil)) + assert.Equal(t, "", B2Str([]byte{})) + assert.Equal(t, "hello", B2Str([]byte("hello"))) + + assert.Nil(t, S2Bytes("")) + assert.Equal(t, []byte("world"), S2Bytes("world")) + + rc := StringToReaderCloser("payload") + b, err := io.ReadAll(rc) + require.NoError(t, err) + assert.Equal(t, "payload", string(b)) + assert.NoError(t, rc.Close()) +} + +func TestDefaultOutboundDialer(t *testing.T) { + d := DefaultOutboundDialer() + assert.Equal(t, 10*time.Second, d.Timeout) + assert.Equal(t, 15*time.Second, d.KeepAlive) + assert.NotSame(t, d, DefaultOutboundDialer(), "callers must get independent dialers") +} + +type errAfterReader struct { + data []byte + err error +} + +func (r *errAfterReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, r.err + } + n := copy(p, r.data) + r.data = r.data[n:] + return n, nil +} + +func TestReadAll(t *testing.T) { + payload := bytes.Repeat([]byte("abc"), 1000) + // maxPreGrowSize+1 would allocate 200MB; the cap branch is not worth that in CI. + for _, expected := range []int{0, -1, 10, len(payload)} { + data, release, err := ReadAll(bytes.NewReader(payload), expected) + require.NoError(t, err) + require.NotNil(t, release) + assert.Equal(t, payload, data, "expected=%d", expected) + release() + } + + data, release, err := ReadAll(&errAfterReader{data: []byte("partial"), err: errors.New("read failed")}, 0) + require.EqualError(t, err, "read failed") + assert.Nil(t, data) + assert.Nil(t, release) +} + +func TestReturnBuf_DropsNilAndOversized(t *testing.T) { + ReturnBuf(nil) // must not reach the pool + + big := bytes.NewBuffer(make([]byte, 0, 4*maxBufCap+1)) + big.WriteString("SENTINEL") + ReturnBuf(big) + for range 16 { + b := BorrowBuf() + assert.Equal(t, 0, b.Len(), "borrowed buffers are reset") + assert.LessOrEqual(t, b.Cap(), 4*maxBufCap, "oversized buffers must not be pooled") + } +} + +func TestBoundedCall(t *testing.T) { + t.Run("returns fn result", func(t *testing.T) { + v, err := BoundedCallT(context.Background(), func(ctx context.Context) (int, error) { return 42, nil }) + require.NoError(t, err) + assert.Equal(t, 42, v) + }) + + t.Run("returns fn error", func(t *testing.T) { + boom := errors.New("boom") + v, err := BoundedCallT(context.Background(), func(ctx context.Context) (string, error) { return "partial", boom }) + assert.ErrorIs(t, err, boom) + assert.Equal(t, "partial", v) + }) + + t.Run("already-done ctx skips fn and returns cause", func(t *testing.T) { + cause := errors.New("budget exhausted") + ctx, cancel := context.WithCancelCause(context.Background()) + cancel(cause) + called := false + v, err := BoundedCallT(ctx, func(ctx context.Context) (int, error) { called = true; return 1, nil }) + assert.ErrorIs(t, err, cause) + assert.Zero(t, v) + assert.False(t, called) + }) + + t.Run("ctx fires while fn is wedged", func(t *testing.T) { + cause := errors.New("dynamic timeout") + ctx, cancel := context.WithCancelCause(context.Background()) + release := make(chan struct{}) + defer close(release) + started := make(chan struct{}) + go func() { + <-started + cancel(cause) + }() + v, err := BoundedCallT(ctx, func(ctx context.Context) (int, error) { + close(started) + <-release // ignores ctx, like a wedged transport + return 7, nil + }) + assert.ErrorIs(t, err, cause) + assert.Zero(t, v) + }) + + t.Run("ctx done by the time fn returns wins over result", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + v, err := BoundedCallT(ctx, func(ctx context.Context) (int, error) { + cancel() + return 9, nil + }) + assert.ErrorIs(t, err, context.Canceled) + assert.Zero(t, v) + }) + + t.Run("panic surfaces as error", func(t *testing.T) { + v, err := BoundedCallT(context.Background(), func(ctx context.Context) (int, error) { panic("kaboom") }) + require.Error(t, err) + assert.Contains(t, err.Error(), "bounded-call goroutine panic: kaboom") + assert.Zero(t, v) + }) + + t.Run("untyped variant", func(t *testing.T) { + boom := errors.New("drain failed") + assert.ErrorIs(t, BoundedCall(context.Background(), func(ctx context.Context) error { return boom }), boom) + assert.NoError(t, BoundedCall(context.Background(), func(ctx context.Context) error { return nil })) + }) +} + +func TestEvmNetworkIdAndIdentifiers(t *testing.T) { + assert.Equal(t, "evm:1", EvmNetworkId(1)) + assert.Equal(t, "evm:42161", EvmNetworkId(int64(42161))) + + for _, s := range []string{"abc", "A_b-9", "x"} { + assert.True(t, IsValidIdentifier(s), s) + } + for _, s := range []string{"", "a b", "a.b", "a:b", "é"} { + assert.False(t, IsValidIdentifier(s), s) + } + + assert.True(t, IsValidNetworkId("evm:1")) + assert.False(t, IsValidNetworkId("evm:abc")) + assert.False(t, IsValidNetworkId("evm:")) + assert.False(t, IsValidNetworkId("cosmos:hub")) + assert.False(t, IsValidNetworkId("")) +} + +func TestIncrementAndGetIndex(t *testing.T) { + // Counters are process-global; a per-run key keeps -count=N runs isolated. + key := t.Name() + time.Now().Format(time.RFC3339Nano) + assert.Equal(t, "1", IncrementAndGetIndex(key, "a")) + assert.Equal(t, "2", IncrementAndGetIndex(key, "a")) + assert.Equal(t, "1", IncrementAndGetIndex(key, "b"), "counters are per key") + // Parts are joined with a separator, so ("ab") and ("a","b") differ. + assert.Equal(t, "1", IncrementAndGetIndex(key+"a", "b")) + + var wg sync.WaitGroup + for range 50 { + wg.Add(1) + go func() { + defer wg.Done() + IncrementAndGetIndex(key, "concurrent") + }() + } + wg.Wait() + assert.Equal(t, "51", IncrementAndGetIndex(key, "concurrent")) +} + +func TestRandomID_Range(t *testing.T) { + for range 1000 { + id := RandomID() + require.GreaterOrEqual(t, id, int64(0)) + require.Less(t, id, int64(1<<31-1)) + } +} + +func TestNormalizeBlockHashHexString_Errors(t *testing.T) { + cases := []struct { + in string + want string + wantErr string + }{ + {"", "", "empty block hash"}, + {"0xzz", "", "invalid hex character"}, + {" 0XAB ", "0x" + strings.Repeat("0", 62) + "ab", ""}, + {"0xabc", "0x" + strings.Repeat("0", 61) + "abc", ""}, + {"0x" + strings.Repeat("0", 3) + strings.Repeat("f", 64), "0x" + strings.Repeat("f", 64), ""}, + {"0x" + strings.Repeat("0", 4) + "1" + strings.Repeat("f", 63), "0x1" + strings.Repeat("f", 63), ""}, + {"0x1" + strings.Repeat("f", 64), "", "block hash too long"}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + got, err := NormalizeBlockHashHexString(tc.in) + if tc.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } + + _, err := ParseBlockHashHexToBytes("0xnothex") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid hex character") +} + +func TestParseBlockParameter_Errors(t *testing.T) { + _, _, err := ParseBlockParameter("0x" + strings.Repeat("g", 64)) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse block hash") + + _, _, err = ParseBlockParameter(map[string]interface{}{"blockHash": "0xnothex"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse blockHash from object") + + // Non-string fields are ignored; with nothing usable left it is an error. + _, _, err = ParseBlockParameter(map[string]interface{}{"blockHash": 1, "blockNumber": 2, "blockTag": true}) + require.Error(t, err) + assert.Contains(t, err.Error(), "must contain blockHash, blockNumber, or blockTag") + + // blockTag takes precedence over blockNumber when both are present. + bn, bh, err := ParseBlockParameter(map[string]interface{}{"blockNumber": "0x10", "blockTag": "safe"}) + require.NoError(t, err) + assert.Equal(t, "safe", bn) + assert.Nil(t, bh) +} diff --git a/util/testing_tcp.go b/util/testing_tcp.go new file mode 100644 index 000000000..d15675272 --- /dev/null +++ b/util/testing_tcp.go @@ -0,0 +1,66 @@ +package util + +import ( + "io" + "net" + "sync/atomic" + "testing" + "time" +) + +// SlowTCPProxy forwards TCP to a target and holds every client->server chunk +// for the set delay, so a real client sees a slow server. +type SlowTCPProxy struct { + ln net.Listener + delay atomic.Int64 +} + +func NewSlowTCPProxy(t testing.TB, target string) *SlowTCPProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + p := &SlowTCPProxy{ln: ln} + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go p.serve(c, target) + } + }() + return p +} + +func (p *SlowTCPProxy) Addr() string { return p.ln.Addr().String() } + +func (p *SlowTCPProxy) SetDelay(d time.Duration) { p.delay.Store(int64(d)) } + +func (p *SlowTCPProxy) serve(c net.Conn, target string) { + defer c.Close() + s, err := net.Dial("tcp", target) + if err != nil { + return + } + defer s.Close() + // Server gone: drop the client too, as a dead server's reset would. + go func() { _, _ = io.Copy(c, s); _ = c.Close() }() + buf := make([]byte, 32*1024) + for { + n, err := c.Read(buf) + if n > 0 { + if d := time.Duration(p.delay.Load()); d > 0 { + time.Sleep(d) + } + if _, werr := s.Write(buf[:n]); werr != nil { + return + } + } + if err != nil { + return + } + } +}