From 4736c3bba200b21f15ca211df20b0c08b1e9f5e0 Mon Sep 17 00:00:00 2001 From: snowkide Date: Wed, 28 Oct 2026 11:50:43 +0100 Subject: [PATCH] fix(ws): refuse WebSocket upgrades with 503 while shutting down Once shutdown has begun, new client connections completed the handshake and were closed with 1001 straight away. A client sees a successful dial, then a connection that drops before or during its first request; clients that verify a connection before subscribing can report it as healthy and then wait for a subscribe timeout. The docs already describe new WebSocket work as refused from the start of shutdown. Refuse the upgrade with 503 before the handshake instead. The check after the handshake stays for a shutdown that begins mid-handshake. --- docs/pages/operation/websocket.mdx | 2 +- erpc/ws_server.go | 10 +++++++++- erpc/ws_server_hardening_test.go | 18 ++++++++++-------- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/pages/operation/websocket.mdx b/docs/pages/operation/websocket.mdx index ab0533e44..78df3c5ef 100644 --- a/docs/pages/operation/websocket.mdx +++ b/docs/pages/operation/websocket.mdx @@ -287,7 +287,7 @@ server: A connection over a cap is refused with `429` before the upgrade; a subscription over `maxSubscriptions` gets a JSON-RPC error and the connection stays open. Refusals are counted in `erpc_websocket_rejected_total`. The caps are per instance, so a fleet admits up to N times each. -The upgrade request must name a network (`///`, or a path that [domain aliasing](/operation/url) resolves to one); otherwise it is rejected with `400`. On shutdown eRPC closes every client connection with `1001` (going away) straight away so clients reconnect to another instance; in-flight requests get up to `server.waitBeforeShutdown` to answer first. +The upgrade request must name a network (`///`, or a path that [domain aliasing](/operation/url) resolves to one); otherwise it is rejected with `400`. On shutdown eRPC closes every client connection with `1001` (going away) straight away so clients reconnect to another instance; in-flight requests get up to `server.waitBeforeShutdown` to answer first. New connections are refused with `503` before the upgrade from then on. ## Cost frames diff --git a/erpc/ws_server.go b/erpc/ws_server.go index ad8cf575b..3e87c46ac 100644 --- a/erpc/ws_server.go +++ b/erpc/ws_server.go @@ -85,6 +85,14 @@ func (s *HttpServer) handleWebSocket( ) { wsCfg := s.serverCfg.WebSocket + // Refused before the upgrade once shutdown has begun, so the client's dial + // fails instead of completing the handshake and being closed right away. + if s.draining.Load() { + lg.Debug().Msg("websocket connection refused: server shutting down") + http.Error(w, "server shutting down", http.StatusServiceUnavailable) + return + } + // Checked before the upgrade so a refused client gets a plain 429. clientIP := s.resolveRealClientIP(r) if reason, ok := s.wsConnLimiter.acquire(clientIP); !ok { @@ -132,7 +140,7 @@ func (s *HttpServer) handleWebSocket( lg.Info().Str("connId", wsc.id).Str("remoteAddr", r.RemoteAddr).Msg("websocket connection established") // Store before checking draining so shutdownWebSockets either sees this - // connection or we see its flag. + // connection or we see its flag (shutdown began during the handshake). s.activeWsConns.Store(wsc.id, wsc) telemetry.GaugeHandle(telemetry.MetricWebsocketConnectionsActive, project.Config.Id).Inc() if s.draining.Load() { diff --git a/erpc/ws_server_hardening_test.go b/erpc/ws_server_hardening_test.go index 59caf4134..1f93829dd 100644 --- a/erpc/ws_server_hardening_test.go +++ b/erpc/ws_server_hardening_test.go @@ -247,7 +247,8 @@ func TestWebSocket_UnsubscribeIsScopedToConnection(t *testing.T) { } // Shutdown closes client connections with 1001 as soon as it starts, not -// after waitBeforeShutdown, and lets in-flight requests answer first. +// after waitBeforeShutdown, lets in-flight requests answer first, and +// refuses new connections before the upgrade. func TestWebSocket_ShutdownClosesPromptlyAfterInflightRequests(t *testing.T) { util.ResetGock() defer util.ResetGock() @@ -299,13 +300,14 @@ func TestWebSocket_ShutdownClosesPromptlyAfterInflightRequests(t *testing.T) { require.ErrorAs(t, err, &closeErr) assert.Equal(t, websocket.CloseGoingAway, closeErr.Code) - // A draining server closes new connections straight away. - late := dialWs(t, addr) - defer late.Close() - _ = late.SetReadDeadline(time.Now().Add(3 * time.Second)) - _, _, err = late.ReadMessage() - require.ErrorAs(t, err, &closeErr) - assert.Equal(t, websocket.CloseGoingAway, closeErr.Code) + // A draining server refuses new connections with 503 before the upgrade. + late, lateResp, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/123", addr), nil) + if late != nil { + _ = late.Close() + } + require.ErrorIs(t, err, websocket.ErrBadHandshake, "a draining server must not complete the handshake") + require.NotNil(t, lateResp) + assert.Equal(t, http.StatusServiceUnavailable, lateResp.StatusCode) } // A peer that never answers pings is disconnected.