From 457777b29e64a87420c192abaa1c5780934e5c82 Mon Sep 17 00:00:00 2001 From: snowkide Date: Tue, 3 Nov 2026 11:29:23 +0100 Subject: [PATCH] feat(ws): client websocket request, close and per-client notification metrics Client WebSocket traffic is only partly visible in metrics: requests that reach the network are counted in network_request_received_total exactly like HTTP ones, requests rejected before the network (invalid JSON, disallowed method, failed auth, consumer rate limit, network unavailable) are not counted at all, and nothing records why connections close. The subscription metrics cover connections, subscriptions and notifications per network, but not per client. - network_request_received_total gains a trailing transport label ("http" or "ws"), recorded on NormalizedRequest and defaulting to "http". - erpc_websocket_requests_total{project,network,category,user,agent_name, outcome} counts every JSON-RPC request on a client connection (batch items individually), including pre-network rejections. outcome is one of ok, error, rate_limited, invalid, invalid_batch, method_not_allowed, unauthorized, network_unavailable. category stays "n/a" until the request is authenticated and network until the network resolves, so a rejected request cannot mint series from a client-chosen method name or chain id. - erpc_websocket_connections_closed_total{project,close_code,initiator}. initiator is "client" when the peer ended the connection (its close code, or 1006 when it vanished or stopped answering pings) and "server" otherwise. close_code is bounded to the RFC 6455 range 1000-1015, else "other". - websocket_notifications_sent_total and websocket_subscription_notifications_dropped_total gain trailing user and agent_name labels, captured at eth_subscribe time and carried on the subscription (wsclient.SubscriptionLabels). Both become labeled counters, so metrics.customizations can drop the per-client labels. Docs: Metrics section in operation/websocket, rows in reference/metrics, and the transport label wherever network_request_received_total labels are listed. Co-Authored-By: Claude Opus 5.5 --- common/request.go | 25 ++++ common/request_test.go | 19 +++ docs/pages/config/projects.mdx | 2 +- docs/pages/config/projects/networks.mdx | 2 +- docs/pages/operation/monitoring.mdx | 2 +- docs/pages/operation/websocket.mdx | 15 +++ docs/pages/reference/metrics.mdx | 10 +- erpc/projects.go | 2 +- erpc/subscription_manager.go | 9 +- erpc/ws_server.go | 105 ++++++++++++++- erpc/ws_server_hardening_test.go | 2 +- erpc/ws_server_metrics_test.go | 154 ++++++++++++++++++++++ erpc/ws_server_test.go | 4 +- indexer/adapters/wsclient/adapter.go | 14 +- indexer/adapters/wsclient/adapter_test.go | 18 +-- telemetry/metrics.go | 37 +++++- 16 files changed, 384 insertions(+), 36 deletions(-) create mode 100644 erpc/ws_server_metrics_test.go diff --git a/common/request.go b/common/request.go index 17f2b844b..5ca3b8add 100644 --- a/common/request.go +++ b/common/request.go @@ -299,6 +299,10 @@ type NormalizedRequest struct { // Resolved client IP (set by HTTP ingress using trusted forwarders) clientIP atomic.Value + // Client transport that delivered this request ("http" or "ws"). + // Defaults to "http" when unset so HTTP ingress needs no explicit set. + transport atomic.Value + // Per-request execution counters; lazy-init via execStateHolder. execStateHolder execStateHolder } @@ -1415,6 +1419,27 @@ func (r *NormalizedRequest) AgentName() string { return "unknown" } +// SetTransport records the client ingress transport ("http" or "ws"). +func (r *NormalizedRequest) SetTransport(transport string) { + if r == nil || transport == "" { + return + } + r.transport.Store(transport) +} + +// Transport returns the client ingress transport. Defaults to "http". +func (r *NormalizedRequest) Transport() string { + if r == nil { + return "http" + } + if v := r.transport.Load(); v != nil { + if s, ok := v.(string); ok && s != "" { + return s + } + } + return "http" +} + // getUserAgent returns the user agent string, with query parameter taking precedence over header func (r *NormalizedRequest) getUserAgent(headers http.Header, queryArgs url.Values) string { // Query parameter takes precedence diff --git a/common/request_test.go b/common/request_test.go index ffeb4d3bf..f53f43df6 100644 --- a/common/request_test.go +++ b/common/request_test.go @@ -678,3 +678,22 @@ func TestSetUserFromTrustedHeader(t *testing.T) { } }) } + +func TestNormalizedRequest_TransportDefaultsToHttp(t *testing.T) { + var nilReq *NormalizedRequest + if got := nilReq.Transport(); got != "http" { + t.Fatalf("nil request transport = %q, want http", got) + } + r := NewNormalizedRequest(nil) + if got := r.Transport(); got != "http" { + t.Fatalf("default transport = %q, want http", got) + } + r.SetTransport("") + if got := r.Transport(); got != "http" { + t.Fatalf("transport after empty set = %q, want http", got) + } + r.SetTransport("ws") + if got := r.Transport(); got != "ws" { + t.Fatalf("transport = %q, want ws", got) + } +} diff --git a/docs/pages/config/projects.mdx b/docs/pages/config/projects.mdx index a3411a1d7..baed6d5c9 100644 --- a/docs/pages/config/projects.mdx +++ b/docs/pages/config/projects.mdx @@ -206,7 +206,7 @@ All metrics use the `erpc_` namespace. | `erpc_cors_requests_total` | `project`, `origin` | Every request that carries an Origin header, before allow/deny. **NOTE**: the `project` label value is actually `r.URL.Path`, not the project id — cardinality grows with distinct request paths. | | `erpc_cors_preflight_requests_total` | `project`, `origin` | Allowed-origin OPTIONS preflights only. Same path-as-project label caveat. | | `erpc_cors_disallowed_origin_total` | `project`, `origin` | Origin present but no pattern matched. Same path-as-project label caveat. | -| `erpc_network_request_received_total` | `project`, `network`, `category`, `finality`, `user`, `agent_name` | Fired in `Project.Forward` **after** project rate limiting — project-rate-limited requests are not counted here. | +| `erpc_network_request_received_total` | `project`, `network`, `category`, `finality`, `user`, `agent_name`, `transport` | Fired in `Project.Forward` **after** project rate limiting — project-rate-limited requests are not counted here. | | `erpc_network_successful_request_total` | `project`, `network`, `vendor`, `upstream`, `category`, `attempt`, `finality`, `emptyish`, `user`, `agent_name` | Vendor/upstream = `""` for cache hits, `"n/a"` when upstream missing. | | `erpc_network_failed_request_total` | `project`, `network`, `category`, `attempt`, `error`, `severity`, `finality`, `user`, `agent_name` | `error` = `common.ErrorFingerprint(err)`. | | `erpc_network_request_duration_seconds` | `project`, `network`, `vendor`, `upstream`, `category`, `finality`, `user` | Histogram. vendor/upstream = `""` on failure. | diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index 15eabb704..d81440785 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -764,7 +764,7 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} | Metric | Type | Labels | When it fires | |---|---|---|---| -| `erpc_network_request_received_total` | counter | `project`, `network`, `category`, `finality`, `user`, `agent_name` | On project forward entry | +| `erpc_network_request_received_total` | counter | `project`, `network`, `category`, `finality`, `user`, `agent_name`, `transport` | On project forward entry | | `erpc_network_successful_request_total` | counter | `project`, `network`, `vendor`, `upstream`, `category`, `attempt`, `finality`, `emptyish`, `user`, `agent_name` | Successful response (vendor/upstream = `` for cache hits) | | `erpc_network_failed_request_total` | counter | `project`, `network`, `category`, `attempt`, `error`, `severity`, `finality`, `user`, `agent_name` | Failed response | | `erpc_network_request_duration_seconds` | histogram | `project`, `network`, `vendor`, `upstream`, `category`, `finality`, `user` | Request completed (vendor/upstream = `` on failure) | diff --git a/docs/pages/operation/monitoring.mdx b/docs/pages/operation/monitoring.mdx index 6658fca64..c51fda536 100644 --- a/docs/pages/operation/monitoring.mdx +++ b/docs/pages/operation/monitoring.mdx @@ -571,7 +571,7 @@ Top-20 operational metrics: | Metric | Type | Labels | When it fires | |---|---|---|---| -| `erpc_network_request_received_total` | counter | project, network, category, finality, user, agent_name | Request received for a network | +| `erpc_network_request_received_total` | counter | project, network, category, finality, user, agent_name, transport | Request received for a network; `transport` ∈ http/ws | | `erpc_network_failed_request_total` | counter | project, network, category, attempt, error, severity, finality, user, agent_name | Request failed at network level; `severity` ∈ critical/warning/info | | `erpc_network_successful_request_total` | counter | project, network, vendor, upstream, category, attempt, finality, emptyish, user, agent_name | Request succeeded at network level | | `erpc_network_request_duration_seconds` | LabeledHistogram | project, network, vendor, upstream, category, finality, user | End-to-end request duration | diff --git a/docs/pages/operation/websocket.mdx b/docs/pages/operation/websocket.mdx index 78df3c5ef..b9b55aa77 100644 --- a/docs/pages/operation/websocket.mdx +++ b/docs/pages/operation/websocket.mdx @@ -374,6 +374,21 @@ eRPC minimises the gap in three ways: Even so, a transient single-block lag can reach a client when the chain produces a block faster than an in-flight HTTP round-trip completes. If your consumer treats single-block HTTP-vs-WS lag as a hard failure, widen its tolerance to match the chain's block time rather than expect eRPC (or any proxy) to eliminate it. +## Metrics + +Requests that reach the network are counted in the same metrics as HTTP traffic; `erpc_network_request_received_total` carries `transport="ws"` for them (`"http"` otherwise). The metrics below cover the WebSocket connection itself; the [metrics reference](/reference/metrics) also lists the subscription pressure metrics (queue length, bytes, upstream subscriptions, dedup): + +| Metric | Type | Labels | What it counts | +|---|---|---|---| +| `erpc_websocket_connections_active` | gauge | project, network | Client connections currently open. `network` is `n/a` for a chain the project has no network config for. | +| `erpc_websocket_connections_closed_total` | counter | project, network, close_code, initiator | Closed client connections. `initiator="client"` when the client ended the connection (`close_code` is the code it sent, `1006` when it went away without a close frame or stopped answering pings); `initiator="server"` when eRPC did (for example `1001` on shutdown, `1013` on a `logs` buffer overflow). Codes outside `1000`-`1015` are reported as `other`. | +| `erpc_websocket_requests_total` | counter | project, network, category, user, agent_name, outcome | Every JSON-RPC request received on a client connection, batch items counted individually, including requests rejected before they reach the network. `outcome` is one of `ok`, `error`, `rate_limited`, `invalid`, `invalid_batch`, `method_not_allowed`, `unauthorized`, `network_unavailable`. `category` is the method once the request is authenticated and `n/a` before that; `network` is `n/a` until the network resolves, so requests that fail authentication, or name a chain the project cannot serve, add no series of their own. | +| `erpc_websocket_subscriptions_active` | gauge | project, network, kind | Active `eth_subscribe` subscriptions. | +| `erpc_websocket_notifications_sent_total` | counter | project, network, kind, user, agent_name | Subscription notifications written to clients. | +| `erpc_websocket_subscription_notifications_dropped_total` | counter | project, network, kind, user, agent_name | Notifications dropped because the subscription's buffer was full (see [Subscription features](#subscription-features)). | + +`user` and `agent_name` are the values of the request (or of the `eth_subscribe` call, for subscription metrics), and can be dropped with a [`metrics.customizations`](/operation/monitoring) label rule like on every other counter. Gauges carry no `user` label because gauge labels cannot be customized. + ## Directives [Request directives](/operation/directives) can be set via headers or query parameters on the initial WebSocket upgrade request. They apply to every regular RPC request on that connection, batches included, subject to the same `allowClientDirectives` rules as HTTP: diff --git a/docs/pages/reference/metrics.mdx b/docs/pages/reference/metrics.mdx index 2e230dc92..c915400e0 100644 --- a/docs/pages/reference/metrics.mdx +++ b/docs/pages/reference/metrics.mdx @@ -307,17 +307,19 @@ All metric names carry the `erpc_` prefix. Full definitions: = websocket.CloseNormalClosure && code <= websocket.CloseTLSHandshake { + return strconv.Itoa(code) + } + return "other" +} diff --git a/erpc/ws_server_hardening_test.go b/erpc/ws_server_hardening_test.go index 6a20f2a32..2823a39f9 100644 --- a/erpc/ws_server_hardening_test.go +++ b/erpc/ws_server_hardening_test.go @@ -164,7 +164,7 @@ func wsConnCount(s *HttpServer) (active, subscribed int) { // client connections, client subscriptions of kind, and the subscriptions // of kind live on ws-upstream. func wsGauges(kind string) (conns, clientSubs, upstreamSubs float64) { - return testutil.ToFloat64(telemetry.MetricWebsocketConnectionsActive.WithLabelValues("test_ws")), + return testutil.ToFloat64(telemetry.MetricWebsocketConnectionsActive.WithLabelValues("test_ws", "evm:123")), testutil.ToFloat64(telemetry.MetricWebsocketSubscriptionsActive.WithLabelValues("test_ws", "evm:123", kind)), testutil.ToFloat64(telemetry.MetricUpstreamSubscriptionsActive.WithLabelValues("test_ws", "evm:123", "ws-upstream", kind)) } diff --git a/erpc/ws_server_metrics_test.go b/erpc/ws_server_metrics_test.go new file mode 100644 index 000000000..4dc8eda3c --- /dev/null +++ b/erpc/ws_server_metrics_test.go @@ -0,0 +1,154 @@ +package erpc + +import ( + "fmt" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" + "github.com/gorilla/websocket" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// wsRequests reads websocket_requests_total. The network and agent are only +// known once the network resolves (the Go dialer's User-Agent maps to "go"). +func wsRequests(network, category, agent, outcome string) float64 { + return testutil.ToFloat64(telemetry.CounterHandle(telemetry.MetricWebsocketRequestsTotal, + "test_ws", network, category, "n/a", agent, outcome)) +} + +// TestWebSocket_RequestAccounting: every client request over WebSocket is +// counted once in websocket_requests_total, including those rejected before they +// reach the network, and forwarded ones carry transport="ws" on +// network_request_received_total. +func TestWebSocket_RequestAccounting(t *testing.T) { + setupGock() + defer util.ResetGock() + + cfg := httpOnlyConfig() + cfg.Projects[0].IgnoreMethods = []string{"debug_*"} + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + activeConns := func() float64 { + return testutil.ToFloat64(telemetry.GaugeHandle(telemetry.MetricWebsocketConnectionsActive, "test_ws", "evm:123")) + } + closedByClient := func() float64 { + return testutil.ToFloat64(telemetry.CounterHandle(telemetry.MetricWebsocketConnectionsClosedTotal, + "test_ws", "evm:123", "1000", "client")) + } + received := func() float64 { + return testutil.ToFloat64(telemetry.CounterHandle(telemetry.MetricNetworkRequestsReceived, + "test_ws", "evm:123", "eth_getBalance", "realtime", "n/a", "go", "ws")) + } + + okBefore := wsRequests("evm:123", "eth_getBalance", "go", "ok") + notAllowedBefore := wsRequests("n/a", "n/a", "unknown", "method_not_allowed") + invalidBefore := wsRequests("n/a", "n/a", "unknown", "invalid") + invalidBatchBefore := wsRequests("n/a", "n/a", "unknown", "invalid_batch") + receivedBefore := received() + activeBefore := activeConns() + closedBefore := closedByClient() + + conn := dialWs(t, addr) + require.Eventually(t, func() bool { return activeConns() == activeBefore+1 }, 2*time.Second, 10*time.Millisecond) + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0x1234567890abcdef1234567890abcdef12345678","latest"]}`) + assert.Equal(t, "0xabc123", resp["result"]) + + resp = sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":2,"method":"debug_traceTransaction","params":["0xabc"]}`) + assert.NotNil(t, resp["error"]) + + resp = sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":3}`) + assert.NotNil(t, resp["error"]) + + resp = sendAndReceive(t, conn, `[{"jsonrpc":"2.0"`) + assert.NotNil(t, resp["error"]) + + assert.Equal(t, okBefore+1, wsRequests("evm:123", "eth_getBalance", "go", "ok")) + assert.Equal(t, notAllowedBefore+1, wsRequests("n/a", "n/a", "unknown", "method_not_allowed")) + assert.Equal(t, invalidBefore+1, wsRequests("n/a", "n/a", "unknown", "invalid")) + assert.Equal(t, invalidBatchBefore+1, wsRequests("n/a", "n/a", "unknown", "invalid_batch")) + assert.Equal(t, receivedBefore+1, received(), "forwarded WS request must carry transport=ws") + + require.NoError(t, conn.WriteControl(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second))) + conn.Close() + + require.Eventually(t, func() bool { + return activeConns() == activeBefore && closedByClient() == closedBefore+1 + }, 5*time.Second, 20*time.Millisecond) +} + +// A request rejected by the consumer's rate limit budget is counted as +// rate_limited, with category "n/a" since it never authenticated. +func TestWebSocket_RequestAccountingAuthRateLimited(t *testing.T) { + setupGock() + defer util.ResetGock() + + cfg := httpOnlyConfig() + cfg.Projects[0].Auth = &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{{ + Type: common.AuthTypeSecret, + Secret: &common.SecretStrategyConfig{Value: "s3cret"}, + RateLimitBudget: "one-per-minute", + }}} + cfg.RateLimiters = &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{{ + Id: "one-per-minute", + Rules: []*common.RateLimitRuleConfig{{Method: "*", MaxCount: 1, Period: common.RateLimitPeriodMinute}}, + }}} + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + limitedBefore := wsRequests("n/a", "n/a", "unknown", "rate_limited") + + conn, _, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/123?secret=s3cret", addr), nil) + require.NoError(t, err) + defer conn.Close() + + const req = `{"jsonrpc":"2.0","id":%d,"method":"eth_getBalance","params":["0x1234567890abcdef1234567890abcdef12345678","latest"]}` + resp := sendAndReceive(t, conn, fmt.Sprintf(req, 1)) + assert.Equal(t, "0xabc123", resp["result"]) + resp = sendAndReceive(t, conn, fmt.Sprintf(req, 2)) + assert.NotNil(t, resp["error"]) + + assert.Equal(t, limitedBefore+1, wsRequests("n/a", "n/a", "unknown", "rate_limited")) +} + +// A connection to a chain the project has no config for is labeled "n/a", so +// clients cannot add connection series for arbitrary chain ids. +func TestWebSocket_ConnectionNetworkLabelIsBounded(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + active := func(network string) float64 { + return testutil.ToFloat64(telemetry.GaugeHandle(telemetry.MetricWebsocketConnectionsActive, "test_ws", network)) + } + unknownBefore, configuredBefore := active("n/a"), active("evm:123") + + unknown, _, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/987654", addr), nil) + require.NoError(t, err) + defer unknown.Close() + configured, _, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/123", addr), nil) + require.NoError(t, err) + defer configured.Close() + + require.Eventually(t, func() bool { + return active("n/a") == unknownBefore+1 && active("evm:123") == configuredBefore+1 + }, 5*time.Second, 20*time.Millisecond) + assert.Equal(t, 0.0, active("evm:987654")) +} + +func TestWebSocket_CloseCodeLabelIsBounded(t *testing.T) { + assert.Equal(t, "1000", closeCodeLabel(websocket.CloseNormalClosure)) + assert.Equal(t, "1006", closeCodeLabel(websocket.CloseAbnormalClosure)) + assert.Equal(t, "1013", closeCodeLabel(websocket.CloseTryAgainLater)) + assert.Equal(t, "other", closeCodeLabel(4321)) + assert.Equal(t, "other", closeCodeLabel(0)) +} diff --git a/erpc/ws_server_test.go b/erpc/ws_server_test.go index 9973f407f..7774fc54e 100644 --- a/erpc/ws_server_test.go +++ b/erpc/ws_server_test.go @@ -1706,7 +1706,7 @@ func TestWebSocket_LogsSubscribeOnAllUpstreamsAndDedupe(t *testing.T) { } ingressBefore := ingress() dedupBefore := counter(telemetry.MetricSubscriptionDedupDroppedTotal, "test_ws", "evm:123", "logs") - sentBefore := counter(telemetry.MetricWebsocketNotificationsSentTotal, "test_ws", "evm:123", "logs") + sentBefore := testutil.ToFloat64(telemetry.CounterHandle(telemetry.MetricWebsocketNotificationsSentTotal, "test_ws", "evm:123", "logs", "n/a", "go")) bytesBefore := counter(telemetry.MetricWebsocketNotificationBytesTotal, "test_ws", "evm:123", "logs") conn := dialWs(t, addr) @@ -1737,7 +1737,7 @@ func TestWebSocket_LogsSubscribeOnAllUpstreamsAndDedupe(t *testing.T) { in := float64(atomic.LoadInt64(&subscribeCount)) assert.Equal(t, in, ingress()-ingressBefore) assert.Equal(t, in-1, counter(telemetry.MetricSubscriptionDedupDroppedTotal, "test_ws", "evm:123", "logs")-dedupBefore) - assert.Equal(t, 1.0, counter(telemetry.MetricWebsocketNotificationsSentTotal, "test_ws", "evm:123", "logs")-sentBefore) + assert.Equal(t, 1.0, testutil.ToFloat64(telemetry.CounterHandle(telemetry.MetricWebsocketNotificationsSentTotal, "test_ws", "evm:123", "logs", "n/a", "go"))-sentBefore) assert.Equal(t, float64(len(msg)), counter(telemetry.MetricWebsocketNotificationBytesTotal, "test_ws", "evm:123", "logs")-bytesBefore) assert.GreaterOrEqual(t, testutil.ToFloat64(telemetry.MetricSubscriptionDedupEntries.WithLabelValues("test_ws", "evm:123")), 1.0) } diff --git a/indexer/adapters/wsclient/adapter.go b/indexer/adapters/wsclient/adapter.go index 61d3577d6..d5e92efa8 100644 --- a/indexer/adapters/wsclient/adapter.go +++ b/indexer/adapters/wsclient/adapter.go @@ -58,11 +58,20 @@ type routeKey struct { filterHash string } +// SubscriptionLabels identify the client of a subscription on its metrics. +// They are captured at subscribe time so recording a notification needs no +// per-notification lookup. +type SubscriptionLabels struct { + User string + AgentName string +} + type clientSub struct { id string kind indexer.EventKind networkID string filterHash string // "" for newHeads + labels SubscriptionLabels notify chan json.RawMessage done chan struct{} // closed by whoever removes the sub from Adapter.subs @@ -126,12 +135,13 @@ func (a *Adapter) deliverTo(sub *clientSub, payload json.RawMessage) { // AddSubscription registers a subscription and starts its writer. It fails // with ErrLimitExceeded when the connection already holds maxSubs, and with // ErrClosed once the adapter was drained. -func (a *Adapter) AddSubscription(clientSubID, networkID string, kind indexer.EventKind, filterHash string, maxSubs int) error { +func (a *Adapter) AddSubscription(clientSubID, networkID string, kind indexer.EventKind, filterHash string, maxSubs int, labels SubscriptionLabels) error { sub := &clientSub{ id: clientSubID, kind: kind, networkID: networkID, filterHash: filterHash, + labels: labels, notify: make(chan json.RawMessage, a.bufferSize), done: make(chan struct{}), } @@ -207,6 +217,7 @@ func (sub *clientSub) snapshot() Subscription { Kind: sub.kind, NetworkID: sub.networkID, FilterHash: sub.filterHash, + Labels: sub.labels, } } @@ -216,6 +227,7 @@ type Subscription struct { Kind indexer.EventKind NetworkID string FilterHash string + Labels SubscriptionLabels } // Count returns the number of active subscriptions. diff --git a/indexer/adapters/wsclient/adapter_test.go b/indexer/adapters/wsclient/adapter_test.go index 68088da05..6b3e2db8f 100644 --- a/indexer/adapters/wsclient/adapter_test.go +++ b/indexer/adapters/wsclient/adapter_test.go @@ -58,13 +58,13 @@ func deliver(a *Adapter, kind indexer.EventKind, filterHash string, n int) { func TestAdapter_AddAfterDrainIsRefused(t *testing.T) { a := newTestAdapter(&fakeWriter{}, 8) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) removed := a.Drain() require.Len(t, removed, 1) assert.Equal(t, "s1", removed[0].ClientSubID) - assert.ErrorIs(t, a.AddSubscription("s2", "evm:1", indexer.KindLog, "f", 10), ErrClosed) + assert.ErrorIs(t, a.AddSubscription("s2", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{}), ErrClosed) assert.Equal(t, 0, a.Count()) } @@ -72,8 +72,8 @@ func TestAdapter_AddAfterDrainIsRefused(t *testing.T) { // its filter reference is released exactly once. func TestAdapter_RemoveAndDrainAreExclusive(t *testing.T) { a := newTestAdapter(&fakeWriter{}, 8) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) - require.NoError(t, a.AddSubscription("s2", "evm:1", indexer.KindLog, "f", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) + require.NoError(t, a.AddSubscription("s2", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) _, _, _, existed := a.RemoveSubscription("s1") require.True(t, existed) @@ -93,7 +93,7 @@ func TestAdapter_SubscriptionLimitIsAtomic(t *testing.T) { wg.Add(1) go func(i int) { defer wg.Done() - if a.AddSubscription(fmt.Sprintf("s%d", i), "evm:1", indexer.KindNewHead, "", 10) == nil { + if a.AddSubscription(fmt.Sprintf("s%d", i), "evm:1", indexer.KindNewHead, "", 10, SubscriptionLabels{}) == nil { ok.Add(1) } }(i) @@ -108,7 +108,7 @@ func TestAdapter_SubscriptionLimitIsAtomic(t *testing.T) { func TestAdapter_BurstWithinBufferIsDelivered(t *testing.T) { w := &fakeWriter{} a := newTestAdapter(w, 256) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) defer a.Drain() deliver(a, indexer.KindLog, "f", 200) @@ -125,7 +125,7 @@ func TestAdapter_BurstWithinBufferIsDelivered(t *testing.T) { func TestAdapter_NewHeadsOverflowDropsOldest(t *testing.T) { w := &fakeWriter{block: make(chan struct{})} a := newTestAdapter(w, 2) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindNewHead, "", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindNewHead, "", 10, SubscriptionLabels{})) defer a.Drain() // The writer takes one event and blocks on it; two more fill the buffer. @@ -148,7 +148,7 @@ func TestAdapter_NewHeadsOverflowDropsOldest(t *testing.T) { func TestAdapter_LogsOverflowIsReportedNotEvicted(t *testing.T) { w := &fakeWriter{block: make(chan struct{})} a := newTestAdapter(w, 2) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) defer a.Drain() deliver(a, indexer.KindLog, "f", 1) @@ -172,7 +172,7 @@ func TestAdapter_RemovedSubOverflowIsNotReported(t *testing.T) { w := &fakeWriter{block: make(chan struct{})} defer close(w.block) a := newTestAdapter(w, 2) - require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10, SubscriptionLabels{})) defer a.Drain() deliver(a, indexer.KindLog, "f", 1) diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 4a6b62ad0..b5fda0a59 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -145,17 +145,42 @@ var ( Help: "Upstream WebSocket reconnects forced by a silent newHeads subscription while the upstream head advanced.", }, []string{"project", "network", "upstream"}) - MetricWebsocketSubscriptionNotificationsDroppedTotal = DefineCounter(prometheus.CounterOpts{ + // MetricWebsocketConnectionsClosedTotal counts closed client WebSocket + // connections. initiator is "client" when the peer ended the connection + // (close_code is the code it sent, or 1006 when it vanished without a close + // frame) and "server" otherwise (close_code is the code eRPC sent). Codes + // outside the range defined by RFC 6455 are reported as "other" so a client + // cannot mint series by choosing arbitrary close codes. + MetricWebsocketConnectionsClosedTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "websocket_connections_closed_total", + Help: "Total number of client WebSocket connections closed, by close code and initiator.", + }, []string{"project", "network", "close_code", "initiator"}) + + // MetricWebsocketRequestsTotal counts every JSON-RPC request a client sends + // over WebSocket (batch items individually), including those rejected + // before they reach the network and therefore absent from + // network_request_received_total. category is "n/a" until the request is + // authenticated and network is "n/a" until the network resolves, so a + // rejected request cannot mint series with an arbitrary method name or + // chain id. + MetricWebsocketRequestsTotal = DefineLabeledCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "websocket_requests_total", + Help: "Total number of JSON-RPC requests received over client WebSocket connections, by outcome (ok, error, invalid, invalid_batch, method_not_allowed, unauthorized, rate_limited, network_unavailable).", + }, []string{"project", "network", "category", "user", "agent_name", "outcome"}) + + MetricWebsocketSubscriptionNotificationsDroppedTotal = DefineLabeledCounter(prometheus.CounterOpts{ Namespace: "erpc", Name: "websocket_subscription_notifications_dropped_total", Help: "Subscription notifications dropped because a client's per-subscription buffer was full. For logs the client connection is closed with 1013.", - }, []string{"project", "network", "kind"}) + }, []string{"project", "network", "kind", "user", "agent_name"}) MetricWebsocketConnectionsActive = DefineGauge(prometheus.GaugeOpts{ Namespace: "erpc", Name: "websocket_connections_active", Help: "Open client WebSocket connections.", - }, []string{"project"}) + }, []string{"project", "network"}) MetricWebsocketSubscriptionsActive = DefineGauge(prometheus.GaugeOpts{ Namespace: "erpc", @@ -163,11 +188,11 @@ var ( Help: "Open client eth_subscribe subscriptions.", }, []string{"project", "network", "kind"}) - MetricWebsocketNotificationsSentTotal = DefineCounter(prometheus.CounterOpts{ + MetricWebsocketNotificationsSentTotal = DefineLabeledCounter(prometheus.CounterOpts{ Namespace: "erpc", Name: "websocket_notifications_sent_total", Help: "Subscription notifications written to clients.", - }, []string{"project", "network", "kind"}) + }, []string{"project", "network", "kind", "user", "agent_name"}) MetricWebsocketNotificationBytesTotal = DefineCounter(prometheus.CounterOpts{ Namespace: "erpc", @@ -748,7 +773,7 @@ var ( Namespace: "erpc", Name: "network_request_received_total", Help: "Total number of requests received for a network.", - }, []string{"project", "network", "category", "finality", "user", "agent_name"}) + }, []string{"project", "network", "category", "finality", "user", "agent_name", "transport"}) MetricNetworkMultiplexedRequests = DefineLabeledCounter(prometheus.CounterOpts{ Namespace: "erpc",