Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions common/request.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,10 @@ type NormalizedRequest struct {
// Resolved client IP (set by HTTP ingress using trusted forwarders)
clientIP atomic.Value

// Client transport that delivered this request ("http" or "ws").
// Defaults to "http" when unset so HTTP ingress needs no explicit set.
transport atomic.Value

// Per-request execution counters; lazy-init via execStateHolder.
execStateHolder execStateHolder
}
Expand Down Expand Up @@ -1415,6 +1419,27 @@ func (r *NormalizedRequest) AgentName() string {
return "unknown"
}

// SetTransport records the client ingress transport ("http" or "ws").
func (r *NormalizedRequest) SetTransport(transport string) {
if r == nil || transport == "" {
return
}
r.transport.Store(transport)
}

// Transport returns the client ingress transport. Defaults to "http".
func (r *NormalizedRequest) Transport() string {
if r == nil {
return "http"
}
if v := r.transport.Load(); v != nil {
if s, ok := v.(string); ok && s != "" {
return s
}
}
return "http"
}

// getUserAgent returns the user agent string, with query parameter taking precedence over header
func (r *NormalizedRequest) getUserAgent(headers http.Header, queryArgs url.Values) string {
// Query parameter takes precedence
Expand Down
19 changes: 19 additions & 0 deletions common/request_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -678,3 +678,22 @@ func TestSetUserFromTrustedHeader(t *testing.T) {
}
})
}

func TestNormalizedRequest_TransportDefaultsToHttp(t *testing.T) {
var nilReq *NormalizedRequest
if got := nilReq.Transport(); got != "http" {
t.Fatalf("nil request transport = %q, want http", got)
}
r := NewNormalizedRequest(nil)
if got := r.Transport(); got != "http" {
t.Fatalf("default transport = %q, want http", got)
}
r.SetTransport("")
if got := r.Transport(); got != "http" {
t.Fatalf("transport after empty set = %q, want http", got)
}
r.SetTransport("ws")
if got := r.Transport(); got != "ws" {
t.Fatalf("transport = %q, want ws", got)
}
}
2 changes: 1 addition & 1 deletion docs/pages/config/projects.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ All metrics use the `erpc_` namespace.
| `erpc_cors_requests_total` | `project`, `origin` | Every request that carries an Origin header, before allow/deny. **NOTE**: the `project` label value is actually `r.URL.Path`, not the project id — cardinality grows with distinct request paths. |
| `erpc_cors_preflight_requests_total` | `project`, `origin` | Allowed-origin OPTIONS preflights only. Same path-as-project label caveat. |
| `erpc_cors_disallowed_origin_total` | `project`, `origin` | Origin present but no pattern matched. Same path-as-project label caveat. |
| `erpc_network_request_received_total` | `project`, `network`, `category`, `finality`, `user`, `agent_name` | Fired in `Project.Forward` **after** project rate limiting — project-rate-limited requests are not counted here. |
| `erpc_network_request_received_total` | `project`, `network`, `category`, `finality`, `user`, `agent_name`, `transport` | Fired in `Project.Forward` **after** project rate limiting — project-rate-limited requests are not counted here. |
| `erpc_network_successful_request_total` | `project`, `network`, `vendor`, `upstream`, `category`, `attempt`, `finality`, `emptyish`, `user`, `agent_name` | Vendor/upstream = `"<cache>"` for cache hits, `"n/a"` when upstream missing. |
| `erpc_network_failed_request_total` | `project`, `network`, `category`, `attempt`, `error`, `severity`, `finality`, `user`, `agent_name` | `error` = `common.ErrorFingerprint(err)`. |
| `erpc_network_request_duration_seconds` | `project`, `network`, `vendor`, `upstream`, `category`, `finality`, `user` | Histogram. vendor/upstream = `"<error>"` on failure. |
Expand Down
2 changes: 1 addition & 1 deletion docs/pages/config/projects/networks.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -764,7 +764,7 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...}

| Metric | Type | Labels | When it fires |
|---|---|---|---|
| `erpc_network_request_received_total` | counter | `project`, `network`, `category`, `finality`, `user`, `agent_name` | On project forward entry |
| `erpc_network_request_received_total` | counter | `project`, `network`, `category`, `finality`, `user`, `agent_name`, `transport` | On project forward entry |
| `erpc_network_successful_request_total` | counter | `project`, `network`, `vendor`, `upstream`, `category`, `attempt`, `finality`, `emptyish`, `user`, `agent_name` | Successful response (vendor/upstream = `<cache>` for cache hits) |
| `erpc_network_failed_request_total` | counter | `project`, `network`, `category`, `attempt`, `error`, `severity`, `finality`, `user`, `agent_name` | Failed response |
| `erpc_network_request_duration_seconds` | histogram | `project`, `network`, `vendor`, `upstream`, `category`, `finality`, `user` | Request completed (vendor/upstream = `<error>` on failure) |
Expand Down
2 changes: 1 addition & 1 deletion docs/pages/operation/monitoring.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -571,7 +571,7 @@ Top-20 operational metrics:

| Metric | Type | Labels | When it fires |
|---|---|---|---|
| `erpc_network_request_received_total` | counter | project, network, category, finality, user, agent_name | Request received for a network |
| `erpc_network_request_received_total` | counter | project, network, category, finality, user, agent_name, transport | Request received for a network; `transport` ∈ http/ws |
| `erpc_network_failed_request_total` | counter | project, network, category, attempt, error, severity, finality, user, agent_name | Request failed at network level; `severity` ∈ critical/warning/info |
| `erpc_network_successful_request_total` | counter | project, network, vendor, upstream, category, attempt, finality, emptyish, user, agent_name | Request succeeded at network level |
| `erpc_network_request_duration_seconds` | LabeledHistogram | project, network, vendor, upstream, category, finality, user | End-to-end request duration |
Expand Down
15 changes: 15 additions & 0 deletions docs/pages/operation/websocket.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,21 @@ eRPC minimises the gap in three ways:

Even so, a transient single-block lag can reach a client when the chain produces a block faster than an in-flight HTTP round-trip completes. If your consumer treats single-block HTTP-vs-WS lag as a hard failure, widen its tolerance to match the chain's block time rather than expect eRPC (or any proxy) to eliminate it.

## Metrics

Requests that reach the network are counted in the same metrics as HTTP traffic; `erpc_network_request_received_total` carries `transport="ws"` for them (`"http"` otherwise). The metrics below cover the WebSocket connection itself; the [metrics reference](/reference/metrics) also lists the subscription pressure metrics (queue length, bytes, upstream subscriptions, dedup):

| Metric | Type | Labels | What it counts |
|---|---|---|---|
| `erpc_websocket_connections_active` | gauge | project, network | Client connections currently open. `network` is `n/a` for a chain the project has no network config for. |
| `erpc_websocket_connections_closed_total` | counter | project, network, close_code, initiator | Closed client connections. `initiator="client"` when the client ended the connection (`close_code` is the code it sent, `1006` when it went away without a close frame or stopped answering pings); `initiator="server"` when eRPC did (for example `1001` on shutdown, `1013` on a `logs` buffer overflow). Codes outside `1000`-`1015` are reported as `other`. |
| `erpc_websocket_requests_total` | counter | project, network, category, user, agent_name, outcome | Every JSON-RPC request received on a client connection, batch items counted individually, including requests rejected before they reach the network. `outcome` is one of `ok`, `error`, `rate_limited`, `invalid`, `invalid_batch`, `method_not_allowed`, `unauthorized`, `network_unavailable`. `category` is the method once the request is authenticated and `n/a` before that; `network` is `n/a` until the network resolves, so requests that fail authentication, or name a chain the project cannot serve, add no series of their own. |
| `erpc_websocket_subscriptions_active` | gauge | project, network, kind | Active `eth_subscribe` subscriptions. |
| `erpc_websocket_notifications_sent_total` | counter | project, network, kind, user, agent_name | Subscription notifications written to clients. |
| `erpc_websocket_subscription_notifications_dropped_total` | counter | project, network, kind, user, agent_name | Notifications dropped because the subscription's buffer was full (see [Subscription features](#subscription-features)). |

`user` and `agent_name` are the values of the request (or of the `eth_subscribe` call, for subscription metrics), and can be dropped with a [`metrics.customizations`](/operation/monitoring) label rule like on every other counter. Gauges carry no `user` label because gauge labels cannot be customized.

## Directives

[Request directives](/operation/directives) can be set via headers or query parameters on the initial WebSocket upgrade request. They apply to every regular RPC request on that connection, batches included, subject to the same `allowClientDirectives` rules as HTTP:
Expand Down
10 changes: 6 additions & 4 deletions docs/pages/reference/metrics.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -307,17 +307,19 @@ All metric names carry the `erpc_` prefix. Full definitions: <SourceLink file="t

| Metric | Type | Labels | When it fires |
|---|---|---|---|
| `erpc_network_request_received_total` | counter | project, network, category, finality, user, agent_name | Request received for a network. |
| `erpc_network_request_received_total` | counter | project, network, category, finality, user, agent_name, transport | Request received for a network. `transport` ∈ `"http"`, `"ws"`. |
| `erpc_network_failed_request_total` | counter | project, network, category, attempt, error, severity, finality, user, agent_name | Request failed at network/project level. `severity` ∈ `"critical"`, `"warning"`, `"info"`. Page on critical, ticket on warning, ignore info. |
| `erpc_network_successful_request_total` | counter | project, network, vendor, upstream, category, attempt, finality, emptyish, user, agent_name | Request succeeded. `emptyish` ∈ `"true"`/`"false"` — true means the response was empty-ish (null, empty array). |
| `erpc_network_multiplexed_request_total` | counter | project, network, category, finality, user, agent_name | Request de-duplicated into an identical in-flight request. |
| `erpc_network_fallback_escape_total` | counter | project, network, category | The per-request fallback escape (`failover.onDefaultsExhausted`) retried a request on `tier:fallback` upstreams. Expected zero in steady state. |
| `erpc_upstream_websocket_connected` | gauge | project, vendor, network, upstream | `1` while the upstream's WebSocket connection is established, `0` while it reconnects. |
| `erpc_upstream_websocket_heads_stall_reconnects_total` | counter | project, network, upstream | Reconnects forced because the upstream's newHeads subscription went silent while its polled head kept advancing. Any increase means a node dropped a live subscription without closing it. |
| `erpc_websocket_subscription_notifications_dropped_total` | counter | project, network, kind | Client subscription notifications dropped because the subscription's buffer (`server.webSocket.subscriptionBufferSize`) was full. For `kind="logs"` the connection is also closed with `1013`. |
| `erpc_websocket_connections_active` | gauge | project | Open client WebSocket connections. |
| `erpc_websocket_subscription_notifications_dropped_total` | counter | project, network, kind, user, agent_name | Client subscription notifications dropped because the subscription's buffer (`server.webSocket.subscriptionBufferSize`) was full. For `kind="logs"` the connection is also closed with `1013`. |
| `erpc_websocket_connections_active` | gauge | project, network | Open client WebSocket connections. `network` is `n/a` for a chain the project has no network config for. |
| `erpc_websocket_connections_closed_total` | counter | project, network, close_code, initiator | Client WebSocket connection closed. `initiator` ∈ `"client"`, `"server"`. `close_code` is the RFC 6455 code (`1006` when the client went away without a close frame); codes outside `1000`-`1015` are `"other"`. |
| `erpc_websocket_requests_total` | counter | project, network, category, user, agent_name, outcome | Every JSON-RPC request received over a client WebSocket (batch items individually), including requests rejected before the network. `outcome` ∈ `"ok"`, `"error"`, `"rate_limited"`, `"invalid"`, `"invalid_batch"`, `"method_not_allowed"`, `"unauthorized"`, `"network_unavailable"`. `category` is `"n/a"` until the request is authenticated, `network` is `"n/a"` until the network resolves. |
| `erpc_websocket_subscriptions_active` | gauge | project, network, kind | Open client `eth_subscribe` subscriptions; drops on `eth_unsubscribe` and when the connection closes. |
| `erpc_websocket_notifications_sent_total` | counter | project, network, kind | Subscription notifications written to clients. |
| `erpc_websocket_notifications_sent_total` | counter | project, network, kind, user, agent_name | Subscription notifications written to clients. |
| `erpc_websocket_notification_bytes_total` | counter | project, network, kind | JSON bytes of those notifications, without cost frame headers. |
| `erpc_websocket_send_queue_length` | histogram | project, network, kind | Notifications still queued on a client subscription when its writer takes the next one. Each subscription has its own queue of `server.webSocket.subscriptionBufferSize`. |
| `erpc_upstream_subscriptions_active` | gauge | project, network, upstream, kind | `eth_subscribe` subscriptions live on upstream WebSocket connections: `newHeads` on each upstream, plus one per distinct `logs` / `newPendingTransactions` filter, shared by every client. |
Expand Down
2 changes: 1 addition & 1 deletion erpc/projects.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ func (p *PreparedProject) Forward(ctx context.Context, networkId string, nq *com
reqFinality := nq.Finality(ctx)

telemetry.CounterHandle(telemetry.MetricNetworkRequestsReceived,
p.Config.Id, network.Label(), method, reqFinality.String(), nq.UserId(), nq.AgentName(),
p.Config.Id, network.Label(), method, reqFinality.String(), nq.UserId(), nq.AgentName(), nq.Transport(),
).Inc()
lg := p.Logger.With().
Str("component", "proxy").
Expand Down
9 changes: 6 additions & 3 deletions erpc/subscription_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ func (sm *SubscriptionManager) Subscribe(
reqFinality := nq.Finality(ctx)
telemetry.CounterHandle(telemetry.MetricNetworkRequestsReceived,
project.Config.Id, nw.Label(), method,
reqFinality.String(), nq.UserId(), nq.AgentName(),
reqFinality.String(), nq.UserId(), nq.AgentName(), nq.Transport(),
).Inc()

jrReq, err := nq.JsonRpcRequest()
Expand Down Expand Up @@ -150,7 +150,10 @@ func (sm *SubscriptionManager) Subscribe(
// Counted before it is visible, so a racing cleanup never takes the
// gauge below zero.
wsc.subscriptionsGauge(kind).Inc()
if err := conn.adapter.AddSubscription(clientSubID, key, kind, filterHash, maxSubs); err != nil {
if err := conn.adapter.AddSubscription(clientSubID, key, kind, filterHash, maxSubs, wsclient.SubscriptionLabels{
User: nq.UserId(),
AgentName: nq.AgentName(),
}); err != nil {
wsc.subscriptionsGauge(kind).Dec()
sm.activeSubs.Add(-1)
sm.releaseFilter(ctx, key, kind, filterHash)
Expand Down Expand Up @@ -195,7 +198,7 @@ func (sm *SubscriptionManager) Unsubscribe(
reqFinality := nq.Finality(ctx)
telemetry.CounterHandle(telemetry.MetricNetworkRequestsReceived,
project.Config.Id, nw.Label(), method,
reqFinality.String(), nq.UserId(), nq.AgentName(),
reqFinality.String(), nq.UserId(), nq.AgentName(), nq.Transport(),
).Inc()

jrReq, err := nq.JsonRpcRequest()
Expand Down
Loading
Loading