diff --git a/common/request.go b/common/request.go index 17f2b844b..5ca3b8add 100644 --- a/common/request.go +++ b/common/request.go @@ -299,6 +299,10 @@ type NormalizedRequest struct { // Resolved client IP (set by HTTP ingress using trusted forwarders) clientIP atomic.Value + // Client transport that delivered this request ("http" or "ws"). + // Defaults to "http" when unset so HTTP ingress needs no explicit set. + transport atomic.Value + // Per-request execution counters; lazy-init via execStateHolder. execStateHolder execStateHolder } @@ -1415,6 +1419,27 @@ func (r *NormalizedRequest) AgentName() string { return "unknown" } +// SetTransport records the client ingress transport ("http" or "ws"). +func (r *NormalizedRequest) SetTransport(transport string) { + if r == nil || transport == "" { + return + } + r.transport.Store(transport) +} + +// Transport returns the client ingress transport. Defaults to "http". +func (r *NormalizedRequest) Transport() string { + if r == nil { + return "http" + } + if v := r.transport.Load(); v != nil { + if s, ok := v.(string); ok && s != "" { + return s + } + } + return "http" +} + // getUserAgent returns the user agent string, with query parameter taking precedence over header func (r *NormalizedRequest) getUserAgent(headers http.Header, queryArgs url.Values) string { // Query parameter takes precedence diff --git a/common/request_test.go b/common/request_test.go index ffeb4d3bf..f53f43df6 100644 --- a/common/request_test.go +++ b/common/request_test.go @@ -678,3 +678,22 @@ func TestSetUserFromTrustedHeader(t *testing.T) { } }) } + +func TestNormalizedRequest_TransportDefaultsToHttp(t *testing.T) { + var nilReq *NormalizedRequest + if got := nilReq.Transport(); got != "http" { + t.Fatalf("nil request transport = %q, want http", got) + } + r := NewNormalizedRequest(nil) + if got := r.Transport(); got != "http" { + t.Fatalf("default transport = %q, want http", got) + } + r.SetTransport("") + if got := r.Transport(); got != "http" { + t.Fatalf("transport after empty set = %q, want http", got) + } + r.SetTransport("ws") + if got := r.Transport(); got != "ws" { + t.Fatalf("transport = %q, want ws", got) + } +} diff --git a/docs/pages/config/projects.mdx b/docs/pages/config/projects.mdx index a3411a1d7..baed6d5c9 100644 --- a/docs/pages/config/projects.mdx +++ b/docs/pages/config/projects.mdx @@ -206,7 +206,7 @@ All metrics use the `erpc_` namespace. | `erpc_cors_requests_total` | `project`, `origin` | Every request that carries an Origin header, before allow/deny. **NOTE**: the `project` label value is actually `r.URL.Path`, not the project id — cardinality grows with distinct request paths. | | `erpc_cors_preflight_requests_total` | `project`, `origin` | Allowed-origin OPTIONS preflights only. Same path-as-project label caveat. | | `erpc_cors_disallowed_origin_total` | `project`, `origin` | Origin present but no pattern matched. Same path-as-project label caveat. | -| `erpc_network_request_received_total` | `project`, `network`, `category`, `finality`, `user`, `agent_name` | Fired in `Project.Forward` **after** project rate limiting — project-rate-limited requests are not counted here. | +| `erpc_network_request_received_total` | `project`, `network`, `category`, `finality`, `user`, `agent_name`, `transport` | Fired in `Project.Forward` **after** project rate limiting — project-rate-limited requests are not counted here. | | `erpc_network_successful_request_total` | `project`, `network`, `vendor`, `upstream`, `category`, `attempt`, `finality`, `emptyish`, `user`, `agent_name` | Vendor/upstream = `""` for cache hits, `"n/a"` when upstream missing. | | `erpc_network_failed_request_total` | `project`, `network`, `category`, `attempt`, `error`, `severity`, `finality`, `user`, `agent_name` | `error` = `common.ErrorFingerprint(err)`. | | `erpc_network_request_duration_seconds` | `project`, `network`, `vendor`, `upstream`, `category`, `finality`, `user` | Histogram. vendor/upstream = `""` on failure. | diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index 15eabb704..d81440785 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -764,7 +764,7 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} | Metric | Type | Labels | When it fires | |---|---|---|---| -| `erpc_network_request_received_total` | counter | `project`, `network`, `category`, `finality`, `user`, `agent_name` | On project forward entry | +| `erpc_network_request_received_total` | counter | `project`, `network`, `category`, `finality`, `user`, `agent_name`, `transport` | On project forward entry | | `erpc_network_successful_request_total` | counter | `project`, `network`, `vendor`, `upstream`, `category`, `attempt`, `finality`, `emptyish`, `user`, `agent_name` | Successful response (vendor/upstream = `` for cache hits) | | `erpc_network_failed_request_total` | counter | `project`, `network`, `category`, `attempt`, `error`, `severity`, `finality`, `user`, `agent_name` | Failed response | | `erpc_network_request_duration_seconds` | histogram | `project`, `network`, `vendor`, `upstream`, `category`, `finality`, `user` | Request completed (vendor/upstream = `` on failure) | diff --git a/docs/pages/operation/monitoring.mdx b/docs/pages/operation/monitoring.mdx index 6658fca64..c51fda536 100644 --- a/docs/pages/operation/monitoring.mdx +++ b/docs/pages/operation/monitoring.mdx @@ -571,7 +571,7 @@ Top-20 operational metrics: | Metric | Type | Labels | When it fires | |---|---|---|---| -| `erpc_network_request_received_total` | counter | project, network, category, finality, user, agent_name | Request received for a network | +| `erpc_network_request_received_total` | counter | project, network, category, finality, user, agent_name, transport | Request received for a network; `transport` ∈ http/ws | | `erpc_network_failed_request_total` | counter | project, network, category, attempt, error, severity, finality, user, agent_name | Request failed at network level; `severity` ∈ critical/warning/info | | `erpc_network_successful_request_total` | counter | project, network, vendor, upstream, category, attempt, finality, emptyish, user, agent_name | Request succeeded at network level | | `erpc_network_request_duration_seconds` | LabeledHistogram | project, network, vendor, upstream, category, finality, user | End-to-end request duration | diff --git a/docs/pages/operation/websocket.mdx b/docs/pages/operation/websocket.mdx index 78df3c5ef..b9b55aa77 100644 --- a/docs/pages/operation/websocket.mdx +++ b/docs/pages/operation/websocket.mdx @@ -374,6 +374,21 @@ eRPC minimises the gap in three ways: Even so, a transient single-block lag can reach a client when the chain produces a block faster than an in-flight HTTP round-trip completes. If your consumer treats single-block HTTP-vs-WS lag as a hard failure, widen its tolerance to match the chain's block time rather than expect eRPC (or any proxy) to eliminate it. +## Metrics + +Requests that reach the network are counted in the same metrics as HTTP traffic; `erpc_network_request_received_total` carries `transport="ws"` for them (`"http"` otherwise). The metrics below cover the WebSocket connection itself; the [metrics reference](/reference/metrics) also lists the subscription pressure metrics (queue length, bytes, upstream subscriptions, dedup): + +| Metric | Type | Labels | What it counts | +|---|---|---|---| +| `erpc_websocket_connections_active` | gauge | project, network | Client connections currently open. `network` is `n/a` for a chain the project has no network config for. | +| `erpc_websocket_connections_closed_total` | counter | project, network, close_code, initiator | Closed client connections. `initiator="client"` when the client ended the connection (`close_code` is the code it sent, `1006` when it went away without a close frame or stopped answering pings); `initiator="server"` when eRPC did (for example `1001` on shutdown, `1013` on a `logs` buffer overflow). Codes outside `1000`-`1015` are reported as `other`. | +| `erpc_websocket_requests_total` | counter | project, network, category, user, agent_name, outcome | Every JSON-RPC request received on a client connection, batch items counted individually, including requests rejected before they reach the network. `outcome` is one of `ok`, `error`, `rate_limited`, `invalid`, `invalid_batch`, `method_not_allowed`, `unauthorized`, `network_unavailable`. `category` is the method once the request is authenticated and `n/a` before that; `network` is `n/a` until the network resolves, so requests that fail authentication, or name a chain the project cannot serve, add no series of their own. | +| `erpc_websocket_subscriptions_active` | gauge | project, network, kind | Active `eth_subscribe` subscriptions. | +| `erpc_websocket_notifications_sent_total` | counter | project, network, kind, user, agent_name | Subscription notifications written to clients. | +| `erpc_websocket_subscription_notifications_dropped_total` | counter | project, network, kind, user, agent_name | Notifications dropped because the subscription's buffer was full (see [Subscription features](#subscription-features)). | + +`user` and `agent_name` are the values of the request (or of the `eth_subscribe` call, for subscription metrics), and can be dropped with a [`metrics.customizations`](/operation/monitoring) label rule like on every other counter. Gauges carry no `user` label because gauge labels cannot be customized. + ## Directives [Request directives](/operation/directives) can be set via headers or query parameters on the initial WebSocket upgrade request. They apply to every regular RPC request on that connection, batches included, subject to the same `allowClientDirectives` rules as HTTP: diff --git a/docs/pages/reference/metrics.mdx b/docs/pages/reference/metrics.mdx index 2e230dc92..c915400e0 100644 --- a/docs/pages/reference/metrics.mdx +++ b/docs/pages/reference/metrics.mdx @@ -307,17 +307,19 @@ All metric names carry the `erpc_` prefix. Full definitions: = websocket.CloseNormalClosure && code <= websocket.CloseTLSHandshake { + return strconv.Itoa(code) + } + return "other" +} diff --git a/erpc/ws_server_hardening_test.go b/erpc/ws_server_hardening_test.go index 6a20f2a32..2823a39f9 100644 --- a/erpc/ws_server_hardening_test.go +++ b/erpc/ws_server_hardening_test.go @@ -164,7 +164,7 @@ func wsConnCount(s *HttpServer) (active, subscribed int) { // client connections, client subscriptions of kind, and the subscriptions // of kind live on ws-upstream. func wsGauges(kind string) (conns, clientSubs, upstreamSubs float64) { - return testutil.ToFloat64(telemetry.MetricWebsocketConnectionsActive.WithLabelValues("test_ws")), + return testutil.ToFloat64(telemetry.MetricWebsocketConnectionsActive.WithLabelValues("test_ws", "evm:123")), testutil.ToFloat64(telemetry.MetricWebsocketSubscriptionsActive.WithLabelValues("test_ws", "evm:123", kind)), testutil.ToFloat64(telemetry.MetricUpstreamSubscriptionsActive.WithLabelValues("test_ws", "evm:123", "ws-upstream", kind)) } diff --git a/erpc/ws_server_metrics_test.go b/erpc/ws_server_metrics_test.go new file mode 100644 index 000000000..4dc8eda3c --- /dev/null +++ b/erpc/ws_server_metrics_test.go @@ -0,0 +1,154 @@ +package erpc + +import ( + "fmt" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" + "github.com/gorilla/websocket" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// wsRequests reads websocket_requests_total. The network and agent are only +// known once the network resolves (the Go dialer's User-Agent maps to "go"). +func wsRequests(network, category, agent, outcome string) float64 { + return testutil.ToFloat64(telemetry.CounterHandle(telemetry.MetricWebsocketRequestsTotal, + "test_ws", network, category, "n/a", agent, outcome)) +} + +// TestWebSocket_RequestAccounting: every client request over WebSocket is +// counted once in websocket_requests_total, including those rejected before they +// reach the network, and forwarded ones carry transport="ws" on +// network_request_received_total. +func TestWebSocket_RequestAccounting(t *testing.T) { + setupGock() + defer util.ResetGock() + + cfg := httpOnlyConfig() + cfg.Projects[0].IgnoreMethods = []string{"debug_*"} + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + activeConns := func() float64 { + return testutil.ToFloat64(telemetry.GaugeHandle(telemetry.MetricWebsocketConnectionsActive, "test_ws", "evm:123")) + } + closedByClient := func() float64 { + return testutil.ToFloat64(telemetry.CounterHandle(telemetry.MetricWebsocketConnectionsClosedTotal, + "test_ws", "evm:123", "1000", "client")) + } + received := func() float64 { + return testutil.ToFloat64(telemetry.CounterHandle(telemetry.MetricNetworkRequestsReceived, + "test_ws", "evm:123", "eth_getBalance", "realtime", "n/a", "go", "ws")) + } + + okBefore := wsRequests("evm:123", "eth_getBalance", "go", "ok") + notAllowedBefore := wsRequests("n/a", "n/a", "unknown", "method_not_allowed") + invalidBefore := wsRequests("n/a", "n/a", "unknown", "invalid") + invalidBatchBefore := wsRequests("n/a", "n/a", "unknown", "invalid_batch") + receivedBefore := received() + activeBefore := activeConns() + closedBefore := closedByClient() + + conn := dialWs(t, addr) + require.Eventually(t, func() bool { return activeConns() == activeBefore+1 }, 2*time.Second, 10*time.Millisecond) + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0x1234567890abcdef1234567890abcdef12345678","latest"]}`) + assert.Equal(t, "0xabc123", resp["result"]) + + resp = sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":2,"method":"debug_traceTransaction","params":["0xabc"]}`) + assert.NotNil(t, resp["error"]) + + resp = sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":3}`) + assert.NotNil(t, resp["error"]) + + resp = sendAndReceive(t, conn, `[{"jsonrpc":"2.0"`) + assert.NotNil(t, resp["error"]) + + assert.Equal(t, okBefore+1, wsRequests("evm:123", "eth_getBalance", "go", "ok")) + assert.Equal(t, notAllowedBefore+1, wsRequests("n/a", "n/a", "unknown", "method_not_allowed")) + assert.Equal(t, invalidBefore+1, wsRequests("n/a", "n/a", "unknown", "invalid")) + assert.Equal(t, invalidBatchBefore+1, wsRequests("n/a", "n/a", "unknown", "invalid_batch")) + assert.Equal(t, receivedBefore+1, received(), "forwarded WS request must carry transport=ws") + + require.NoError(t, conn.WriteControl(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second))) + conn.Close() + + require.Eventually(t, func() bool { + return activeConns() == activeBefore && closedByClient() == closedBefore+1 + }, 5*time.Second, 20*time.Millisecond) +} + +// A request rejected by the consumer's rate limit budget is counted as +// rate_limited, with category "n/a" since it never authenticated. +func TestWebSocket_RequestAccountingAuthRateLimited(t *testing.T) { + setupGock() + defer util.ResetGock() + + cfg := httpOnlyConfig() + cfg.Projects[0].Auth = &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{{ + Type: common.AuthTypeSecret, + Secret: &common.SecretStrategyConfig{Value: "s3cret"}, + RateLimitBudget: "one-per-minute", + }}} + cfg.RateLimiters = &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{{ + Id: "one-per-minute", + Rules: []*common.RateLimitRuleConfig{{Method: "*", MaxCount: 1, Period: common.RateLimitPeriodMinute}}, + }}} + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + limitedBefore := wsRequests("n/a", "n/a", "unknown", "rate_limited") + + conn, _, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/123?secret=s3cret", addr), nil) + require.NoError(t, err) + defer conn.Close() + + const req = `{"jsonrpc":"2.0","id":%d,"method":"eth_getBalance","params":["0x1234567890abcdef1234567890abcdef12345678","latest"]}` + resp := sendAndReceive(t, conn, fmt.Sprintf(req, 1)) + assert.Equal(t, "0xabc123", resp["result"]) + resp = sendAndReceive(t, conn, fmt.Sprintf(req, 2)) + assert.NotNil(t, resp["error"]) + + assert.Equal(t, limitedBefore+1, wsRequests("n/a", "n/a", "unknown", "rate_limited")) +} + +// A connection to a chain the project has no config for is labeled "n/a", so +// clients cannot add connection series for arbitrary chain ids. +func TestWebSocket_ConnectionNetworkLabelIsBounded(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + active := func(network string) float64 { + return testutil.ToFloat64(telemetry.GaugeHandle(telemetry.MetricWebsocketConnectionsActive, "test_ws", network)) + } + unknownBefore, configuredBefore := active("n/a"), active("evm:123") + + unknown, _, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/987654", addr), nil) + require.NoError(t, err) + defer unknown.Close() + configured, _, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/123", addr), nil) + require.NoError(t, err) + defer configured.Close() + + require.Eventually(t, func() bool { + return active("n/a") == unknownBefore+1 && active("evm:123") == configuredBefore+1 + }, 5*time.Second, 20*time.Millisecond) + assert.Equal(t, 0.0, active("evm:987654")) +} + +func TestWebSocket_CloseCodeLabelIsBounded(t *testing.T) { + assert.Equal(t, "1000", closeCodeLabel(websocket.CloseNormalClosure)) + assert.Equal(t, "1006", closeCodeLabel(websocket.CloseAbnormalClosure)) + assert.Equal(t, "1013", closeCodeLabel(websocket.CloseTryAgainLater)) + assert.Equal(t, "other", closeCodeLabel(4321)) + assert.Equal(t, "other", closeCodeLabel(0)) +} diff --git a/erpc/ws_server_test.go b/erpc/ws_server_test.go index 9973f407f..7774fc54e 100644 --- a/erpc/ws_server_test.go +++ b/erpc/ws_server_test.go @@ -1706,7 +1706,7 @@ func TestWebSocket_LogsSubscribeOnAllUpstreamsAndDedupe(t *testing.T) { } ingressBefore := ingress() dedupBefore := counter(telemetry.MetricSubscriptionDedupDroppedTotal, "test_ws", "evm:123", "logs") - sentBefore := counter(telemetry.MetricWebsocketNotificationsSentTotal, "test_ws", "evm:123", "logs") + sentBefore := testutil.ToFloat64(telemetry.CounterHandle(telemetry.MetricWebsocketNotificationsSentTotal, "test_ws", "evm:123", "logs", "n/a", "go")) bytesBefore := counter(telemetry.MetricWebsocketNotificationBytesTotal, "test_ws", "evm:123", "logs") conn := dialWs(t, addr) @@ -1737,7 +1737,7 @@ func TestWebSocket_LogsSubscribeOnAllUpstreamsAndDedupe(t *testing.T) { in := float64(atomic.LoadInt64(&subscribeCount)) assert.Equal(t, in, ingress()-ingressBefore) assert.Equal(t, in-1, counter(telemetry.MetricSubscriptionDedupDroppedTotal, "test_ws", "evm:123", "logs")-dedupBefore) - assert.Equal(t, 1.0, counter(telemetry.MetricWebsocketNotificationsSentTotal, "test_ws", "evm:123", "logs")-sentBefore) + assert.Equal(t, 1.0, testutil.ToFloat64(telemetry.CounterHandle(telemetry.MetricWebsocketNotificationsSentTotal, "test_ws", "evm:123", "logs", "n/a", "go"))-sentBefore) assert.Equal(t, float64(len(msg)), counter(telemetry.MetricWebsocketNotificationBytesTotal, "test_ws", "evm:123", "logs")-bytesBefore) assert.GreaterOrEqual(t, testutil.ToFloat64(telemetry.MetricSubscriptionDedupEntries.WithLabelValues("test_ws", "evm:123")), 1.0) } diff --git a/indexer/adapters/wsclient/adapter.go b/indexer/adapters/wsclient/adapter.go index 61d3577d6..d5e92efa8 100644 --- a/indexer/adapters/wsclient/adapter.go +++ b/indexer/adapters/wsclient/adapter.go @@ -58,11 +58,20 @@ type routeKey struct { filterHash string } +// SubscriptionLabels identify the client of a subscription on its metrics. +// They are captured at subscribe time so recording a notification needs no +// per-notification lookup. +type SubscriptionLabels struct { + User string + AgentName string +} + type clientSub struct { id string kind indexer.EventKind networkID string filterHash string // "" for newHeads + labels SubscriptionLabels notify chan json.RawMessage done chan struct{} // closed by whoever removes the sub from Adapter.subs @@ -126,12 +135,13 @@ func (a *Adapter) deliverTo(sub *clientSub, payload json.RawMessage) { // AddSubscription registers a subscription and starts its writer. It fails // with ErrLimitExceeded when the connection already holds maxSubs, and with // ErrClosed once the adapter was drained. -func (a *Adapter) AddSubscription(clientSubID, networkID string, kind indexer.EventKind, filterHash string, maxSubs int) error { +func (a *Adapter) AddSubscription(clientSubID, networkID string, kind indexer.EventKind, filterHash string, maxSubs int, labels SubscriptionLabels) error { sub := &clientSub{ id: clientSubID, kind: kind, networkID: networkID, filterHash: filterHash, + labels: labels, notify: make(chan json.RawMessage, a.bufferSize), done: make(chan struct{}), } @@ -207,6 +217,7 @@ func (sub *clientSub) snapshot() Subscription { Kind: sub.kind, NetworkID: sub.networkID, FilterHash: sub.filterHash, + Labels: sub.labels, } } @@ -216,6 +227,7 @@ type Subscription struct { Kind indexer.EventKind NetworkID string FilterHash string + Labels SubscriptionLabels } // Count returns the number of active subscriptions. diff --git a/indexer/adapters/wsclient/adapter_test.go b/indexer/adapters/wsclient/adapter_test.go index 68088da05..6b3e2db8f 100644 --- a/indexer/adapters/wsclient/adapter_test.go +++ b/indexer/adapters/wsclient/adapter_test.go @@ -58,13 +58,13 @@ func deliver(a *Adapter, kind indexer.EventKind, filterHash string, n int) { func TestAdapter_AddAfterDrainIsRefused(t *testing.T) { a := newTestAdapter(&fakeWriter{}, 8) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) removed := a.Drain() require.Len(t, removed, 1) assert.Equal(t, "s1", removed[0].ClientSubID) - assert.ErrorIs(t, a.AddSubscription("s2", "evm:1", indexer.KindLog, "f", 10), ErrClosed) + assert.ErrorIs(t, a.AddSubscription("s2", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{}), ErrClosed) assert.Equal(t, 0, a.Count()) } @@ -72,8 +72,8 @@ func TestAdapter_AddAfterDrainIsRefused(t *testing.T) { // its filter reference is released exactly once. func TestAdapter_RemoveAndDrainAreExclusive(t *testing.T) { a := newTestAdapter(&fakeWriter{}, 8) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) - require.NoError(t, a.AddSubscription("s2", "evm:1", indexer.KindLog, "f", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) + require.NoError(t, a.AddSubscription("s2", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) _, _, _, existed := a.RemoveSubscription("s1") require.True(t, existed) @@ -93,7 +93,7 @@ func TestAdapter_SubscriptionLimitIsAtomic(t *testing.T) { wg.Add(1) go func(i int) { defer wg.Done() - if a.AddSubscription(fmt.Sprintf("s%d", i), "evm:1", indexer.KindNewHead, "", 10) == nil { + if a.AddSubscription(fmt.Sprintf("s%d", i), "evm:1", indexer.KindNewHead, "", 10, SubscriptionLabels{}) == nil { ok.Add(1) } }(i) @@ -108,7 +108,7 @@ func TestAdapter_SubscriptionLimitIsAtomic(t *testing.T) { func TestAdapter_BurstWithinBufferIsDelivered(t *testing.T) { w := &fakeWriter{} a := newTestAdapter(w, 256) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) defer a.Drain() deliver(a, indexer.KindLog, "f", 200) @@ -125,7 +125,7 @@ func TestAdapter_BurstWithinBufferIsDelivered(t *testing.T) { func TestAdapter_NewHeadsOverflowDropsOldest(t *testing.T) { w := &fakeWriter{block: make(chan struct{})} a := newTestAdapter(w, 2) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindNewHead, "", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindNewHead, "", 10, SubscriptionLabels{})) defer a.Drain() // The writer takes one event and blocks on it; two more fill the buffer. @@ -148,7 +148,7 @@ func TestAdapter_NewHeadsOverflowDropsOldest(t *testing.T) { func TestAdapter_LogsOverflowIsReportedNotEvicted(t *testing.T) { w := &fakeWriter{block: make(chan struct{})} a := newTestAdapter(w, 2) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) defer a.Drain() deliver(a, indexer.KindLog, "f", 1) @@ -172,7 +172,7 @@ func TestAdapter_RemovedSubOverflowIsNotReported(t *testing.T) { w := &fakeWriter{block: make(chan struct{})} defer close(w.block) a := newTestAdapter(w, 2) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) defer a.Drain() deliver(a, indexer.KindLog, "f", 1) diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 4a6b62ad0..b5fda0a59 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -145,17 +145,42 @@ var ( Help: "Upstream WebSocket reconnects forced by a silent newHeads subscription while the upstream head advanced.", }, []string{"project", "network", "upstream"}) - MetricWebsocketSubscriptionNotificationsDroppedTotal = DefineCounter(prometheus.CounterOpts{ + // MetricWebsocketConnectionsClosedTotal counts closed client WebSocket + // connections. initiator is "client" when the peer ended the connection + // (close_code is the code it sent, or 1006 when it vanished without a close + // frame) and "server" otherwise (close_code is the code eRPC sent). Codes + // outside the range defined by RFC 6455 are reported as "other" so a client + // cannot mint series by choosing arbitrary close codes. + MetricWebsocketConnectionsClosedTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "websocket_connections_closed_total", + Help: "Total number of client WebSocket connections closed, by close code and initiator.", + }, []string{"project", "network", "close_code", "initiator"}) + + // MetricWebsocketRequestsTotal counts every JSON-RPC request a client sends + // over WebSocket (batch items individually), including those rejected + // before they reach the network and therefore absent from + // network_request_received_total. category is "n/a" until the request is + // authenticated and network is "n/a" until the network resolves, so a + // rejected request cannot mint series with an arbitrary method name or + // chain id. + MetricWebsocketRequestsTotal = DefineLabeledCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "websocket_requests_total", + Help: "Total number of JSON-RPC requests received over client WebSocket connections, by outcome (ok, error, invalid, invalid_batch, method_not_allowed, unauthorized, rate_limited, network_unavailable).", + }, []string{"project", "network", "category", "user", "agent_name", "outcome"}) + + MetricWebsocketSubscriptionNotificationsDroppedTotal = DefineLabeledCounter(prometheus.CounterOpts{ Namespace: "erpc", Name: "websocket_subscription_notifications_dropped_total", Help: "Subscription notifications dropped because a client's per-subscription buffer was full. For logs the client connection is closed with 1013.", - }, []string{"project", "network", "kind"}) + }, []string{"project", "network", "kind", "user", "agent_name"}) MetricWebsocketConnectionsActive = DefineGauge(prometheus.GaugeOpts{ Namespace: "erpc", Name: "websocket_connections_active", Help: "Open client WebSocket connections.", - }, []string{"project"}) + }, []string{"project", "network"}) MetricWebsocketSubscriptionsActive = DefineGauge(prometheus.GaugeOpts{ Namespace: "erpc", @@ -163,11 +188,11 @@ var ( Help: "Open client eth_subscribe subscriptions.", }, []string{"project", "network", "kind"}) - MetricWebsocketNotificationsSentTotal = DefineCounter(prometheus.CounterOpts{ + MetricWebsocketNotificationsSentTotal = DefineLabeledCounter(prometheus.CounterOpts{ Namespace: "erpc", Name: "websocket_notifications_sent_total", Help: "Subscription notifications written to clients.", - }, []string{"project", "network", "kind"}) + }, []string{"project", "network", "kind", "user", "agent_name"}) MetricWebsocketNotificationBytesTotal = DefineCounter(prometheus.CounterOpts{ Namespace: "erpc", @@ -748,7 +773,7 @@ var ( Namespace: "erpc", Name: "network_request_received_total", Help: "Total number of requests received for a network.", - }, []string{"project", "network", "category", "finality", "user", "agent_name"}) + }, []string{"project", "network", "category", "finality", "user", "agent_name", "transport"}) MetricNetworkMultiplexedRequests = DefineLabeledCounter(prometheus.CounterOpts{ Namespace: "erpc",