diff --git a/UPGRADING.md b/UPGRADING.md index 78b5313945..99b5f8f224 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -49,17 +49,51 @@ Register owned workers with `BaseService.Go`. `OnStop` must unblock them, not wait for them; move shared-resource cleanup to optional `OnDrain`, which runs after those workers exit. Call `Stop` then `Wait` before releasing resources. Plain goroutines and their existing joins remain the implementation's responsibility. -`TimeoutTicker` implementations must now provide `Wait`. +`TimeoutTicker` implementations must now provide `Wait`. `WSRPCConnection` +implementations must provide `Go(func(context.Context)) bool` so the server owns +subscription workers. + +`WSClient.Stop` is safe to call repeatedly and concurrently, including after +reconnect attempts run out. During a reconnect it now blocks until the context +ends or the attempts run out, instead of returning while a reconnect could still +start workers. Dials now observe that context and fail after ten seconds, including +the one in `Start`, so a peer that never answers the handshake cannot block either +call. Once the context ends, `Stop` waits at most for a write in progress to hit +its ten-second deadline. It also joins `OnReconnect` callbacks, which must return +once the context ends, and `Send` fails once the client has shut down. Stopping the +HTTP RPC client also joins its event listener and resubscriptions. + +`rpc/jsonrpc/server.Serve` and `ServeTLS` always close the listener before +returning, including when TLS setup fails. Cancellation now reaches request +contexts, rejects new requests with 503, closes active connections instead of a +one-second graceful shutdown, and returns only after every handler, including +hijacked WebSocket sessions, has finished. They no longer log a startup line; +one "RPC server stopped" line is logged on return. A stop requested during startup cancels work immediately but defers `OnStop` until successful startup returns. Failed startup must roll back its resources; BaseService joins registered work without calling shutdown hooks. Retry requires recreating any child that already started successfully and then stopped. +A node that fails after reserving its RPC listeners, or a seed node that fails +startup, closes its databases and rejects another `Start`; construct a new node. +In the same process that panics with `duplicate metrics collector registration +attempted` for a seed, or for a node with Prometheus enabled, unless +`Instrumentation.Namespace` differs. An RPC listener bind failure stays retryable +and keeps the stores open, with no `service.Service` call to release them if you +give up. Do not call a Dash Core `RPCClient` after `Close`: requests can block +forever. Its `Close` waits at most five seconds for an in-flight request, but that +does not bound node shutdown: a reactor stuck in a Dash Core call still blocks +`Wait`, because the client has no HTTP timeout. `--shutdown-timeout` limits the +node command's wait, not `Start` or embedders. Direct consensus `State.Stop` drains a commit already in progress using the context originally passed to `State.Start`. Cancel that parent to abort it; reactor/node shutdown still cancels that parent and can require WAL recovery. This is not an unconditional durable-finalization guarantee for node shutdown. +Node shutdown keeps application connections, the event bus and the indexer on a +dependency context until reactors have been joined, then releases them and the +stores. After a node startup failure past RPC listener reservation, construct a +fresh node graph. Once admitted, a consensus handoff belongs to its reactor. Canceling the caller releases its wait; reactor shutdown cancels and joins the admitted handoff. Router disconnect notifications during shutdown are best effort; subscribers diff --git a/abci/client/grpc_client.go b/abci/client/grpc_client.go index 0ffed9b399..bb7981581e 100644 --- a/abci/client/grpc_client.go +++ b/abci/client/grpc_client.go @@ -144,7 +144,13 @@ func (cli *grpcClient) streamClientInterceptor(ctx context.Context, desc *grpc.S return streamer(ctx, desc, cc, method, opts...) } -func (cli *grpcClient) OnStart(ctx context.Context) error { +func (cli *grpcClient) OnStart(ctx context.Context) (startErr error) { + defer func() { + if startErr != nil && cli.conn != nil { + _ = cli.conn.Close() + cli.conn = nil + } + }() timer := time.NewTimer(0) defer timer.Stop() diff --git a/abci/client/routed_client.go b/abci/client/routed_client.go index 492eed2617..780cd08c59 100644 --- a/abci/client/routed_client.go +++ b/abci/client/routed_client.go @@ -17,10 +17,13 @@ import ( ) type routedClient struct { - service.Service + *service.BaseService logger log.Logger routing Routing defaultClient ClientInfo + // owned lists the clients started by the current attempt. Lifecycle hooks + // never overlap, and the drain worker starts after OnStart sets it. + owned []Client } var _ Client = (*routedClient)(nil) @@ -125,46 +128,71 @@ func NewRoutedClient(logger log.Logger, defaultClient Client, routing Routing) ( defaultClient: ClientInfo{defaultClient, defaultClientID}, } - cli.Service = service.NewBaseService(logger, "RoutedClient", cli) + cli.BaseService = service.NewBaseService(logger, "RoutedClient", cli) return cli, nil } +// OnStart starts the clients that are not running yet and owns only those: +// clients the caller already started keep their own lifetime. func (cli *routedClient) OnStart(ctx context.Context) error { + cli.owned = nil + // Client does not require Stop, and BaseService cancels ctx only after OnStart + // returns, so this context is what ends a child before a rollback joins it. + childCtx, cancelChildren := context.WithCancel(ctx) var errs error - for _, clients := range cli.routing { - for _, client := range clients { - if !client.IsRunning() { - if err := client.Start(ctx); err != nil { - errs = multierror.Append(errs, err) - } - } + for _, client := range cli.clients() { + if client.IsRunning() { + continue } - } - - if !cli.defaultClient.IsRunning() { - if err := cli.defaultClient.Start(ctx); err != nil { + if err := client.Start(childCtx); err != nil { errs = multierror.Append(errs, err) + continue } + cli.owned = append(cli.owned, client) + } + // A failed attempt gets no OnStop; release what it started before returning. + if errs != nil { + cancelChildren() + cli.stopOwned() + return errs } - return errs + if !cli.Go(ctx, func(ctx context.Context) { + defer cancelChildren() + <-ctx.Done() + for _, client := range cli.owned { + client.Wait() + } + }) { + cancelChildren() + cli.stopOwned() + } + return nil } func (cli *routedClient) OnStop() { - for _, clients := range cli.routing { - for _, client := range clients { - if client.IsRunning() { - switch c := client.Client.(type) { - case *socketClient: - c.Stop() - case *localClient: - c.Stop() - case *grpcClient: - c.Stop() - } - } + for _, client := range cli.owned { + if stopper, ok := client.(interface{ Stop() }); ok { + stopper.Stop() + } + } +} + +func (cli *routedClient) stopOwned() { + cli.OnStop() + for _, client := range cli.owned { + client.Wait() + } +} + +func (cli *routedClient) clients() []Client { + clients := []Client{cli.defaultClient.Client} + for _, routes := range cli.routing { + for _, route := range routes { + clients = append(clients, route.Client) } } + return clients } // delegate calls the given function on the appropriate client with the given diff --git a/abci/client/routed_lifecycle_test.go b/abci/client/routed_lifecycle_test.go new file mode 100644 index 0000000000..7a1ee706fa --- /dev/null +++ b/abci/client/routed_lifecycle_test.go @@ -0,0 +1,71 @@ +package abciclient_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + + abciclient "github.com/dashpay/tenderdash/abci/client" + "github.com/dashpay/tenderdash/abci/types" + "github.com/dashpay/tenderdash/libs/log" +) + +var errChildStart = errors.New("child start failed") + +type failingStartClient struct{ abciclient.Client } + +func (failingStartClient) Start(context.Context) error { return errChildStart } +func (failingStartClient) IsRunning() bool { return false } +func (failingStartClient) Wait() {} + +// noStopClient satisfies Client while hiding Stop, which Client does not require. +type noStopClient struct{ abciclient.Client } + +// A failed start must stop and join the children it started, and leave alone +// a client the caller had already started. +func TestRoutedClientFailedStartReleasesOnlyItsChildren(t *testing.T) { + testCases := map[string]func(abciclient.Client) abciclient.Client{ + "child exposes Stop": func(c abciclient.Client) abciclient.Client { return c }, + "child hides Stop": func(c abciclient.Client) abciclient.Client { return noStopClient{c} }, + } + for name, wrap := range testCases { + t.Run(name, func(t *testing.T) { + logger := log.NewNopLogger() + callerCtx, cancelCaller := context.WithCancel(context.Background()) + defer cancelCaller() + callerOwned := abciclient.NewLocalClient(logger, types.NewBaseApplication()) + require.NoError(t, callerOwned.Start(callerCtx)) + attemptOwned := abciclient.NewLocalClient(logger, types.NewBaseApplication()) + routing := abciclient.Routing{ + "Info": {{Client: callerOwned, ClientID: "caller"}}, + "Query": {{Client: wrap(attemptOwned), ClientID: "attempt"}}, + } + routed, err := abciclient.NewRoutedClient(logger, failingStartClient{}, routing) + require.NoError(t, err) + + started := make(chan error, 1) + go func() { started <- routed.Start(context.Background()) }() + select { + case err := <-started: + require.ErrorIs(t, err, errChildStart) + case <-time.After(5 * time.Second): + t.Fatal("failed start blocked joining a client") + } + require.False(t, attemptOwned.IsRunning(), "child started by the failed attempt kept running") + joined := make(chan struct{}) + go func() { attemptOwned.Wait(); close(joined) }() + select { + case <-joined: + case <-time.After(5 * time.Second): + t.Fatal("child started by the failed attempt was not joined") + } + // Only a client that started and stopped refuses another start. + require.ErrorContains(t, attemptOwned.Start(callerCtx), "already stopped", + "the failed attempt never started this child") + require.True(t, callerOwned.IsRunning(), "failed start stopped a caller-owned client") + }) + } +} diff --git a/abci/client/socket_client.go b/abci/client/socket_client.go index fb046187d8..6387aeddfa 100644 --- a/abci/client/socket_client.go +++ b/abci/client/socket_client.go @@ -90,21 +90,23 @@ func (cli *socketClient) OnStart(ctx context.Context) error { } cli.conn = conn - go cli.sendRequestsRoutine(ctx, conn) - go cli.recvResponseRoutine(ctx, conn) + cli.Go(ctx, func(ctx context.Context) { cli.sendRequestsRoutine(ctx, conn) }) + cli.Go(ctx, func(ctx context.Context) { cli.recvResponseRoutine(ctx, conn) }) return nil } } -// OnStop implements Service by closing connection and flushing all queues. +// OnStop closes the connection to unblock transport workers. func (cli *socketClient) OnStop() { if cli.conn != nil { cli.conn.Close() } - cli.drainQueue() } +// OnDrain releases pending requests after the transport workers have exited. +func (cli *socketClient) OnDrain() { cli.drainQueue() } + func (cli *socketClient) String() string { if err := cli.Error(); err != nil { return fmt.Sprintf("%T(%s):err=%s", cli, cli.addr, err.Error()) @@ -162,12 +164,12 @@ func (cli *socketClient) sendRequestsRoutine(ctx context.Context, conn io.Writer cli.trackRequest(reqres) if err := types.WriteMessage(reqres.Request, bw); err != nil { - cli.stopForError(fmt.Errorf("write to buffer: %w", err)) + cli.stopForError(ctx, fmt.Errorf("write to buffer: %w", err)) return } if err := bw.Flush(); err != nil { - cli.stopForError(fmt.Errorf("flush buffer: %w", err)) + cli.stopForError(ctx, fmt.Errorf("flush buffer: %w", err)) return } } @@ -181,18 +183,18 @@ func (cli *socketClient) recvResponseRoutine(ctx context.Context, conn io.Reader res := &types.Response{} if err := types.ReadMessage(r, res); err != nil { - cli.stopForError(fmt.Errorf("read message: %w", err)) + cli.stopForError(ctx, fmt.Errorf("read message: %w", err)) return } switch r := res.Value.(type) { case *types.Response_Exception: // app responded with error // XXX After setting cli.err, release waiters (e.g. reqres.Done()) - cli.stopForError(errors.New(r.Exception.Error)) + cli.stopForError(ctx, errors.New(r.Exception.Error)) return default: if err := cli.didRecvResponse(res); err != nil { - cli.stopForError(err) + cli.stopForError(ctx, err) return } } @@ -435,11 +437,10 @@ func resMatchesReq(req *types.Request, res *types.Response) (ok bool) { return ok } -func (cli *socketClient) stopForError(err error) { - if !cli.IsRunning() { +func (cli *socketClient) stopForError(ctx context.Context, err error) { + if ctx.Err() != nil { return } - cli.mtx.Lock() cli.err = err cli.mtx.Unlock() diff --git a/abci/server/grpc_server.go b/abci/server/grpc_server.go index 21bb42b03e..643b2d6bb4 100644 --- a/abci/server/grpc_server.go +++ b/abci/server/grpc_server.go @@ -47,22 +47,22 @@ func (s *GRPCServer) OnStart(ctx context.Context) error { types.RegisterABCIApplicationServer(s.server, &gRPCApplication{Application: s.app}) s.logger.Info("Listening", "proto", s.proto, "addr", s.addr) - go func() { - go func() { - <-ctx.Done() - s.server.GracefulStop() - }() - + if !s.Go(ctx, func(context.Context) { if err := s.server.Serve(ln); err != nil { s.logger.Error("error serving gRPC server", "err", err) } - }() + }) { + return ln.Close() + } return nil } // OnStop stops the gRPC server. func (s *GRPCServer) OnStop() { s.server.Stop() } +// OnDrain joins handlers that were already running when the transport closed. +func (s *GRPCServer) OnDrain() { s.server.GracefulStop() } + //------------------------------------------------------- // gRPCApplication is a gRPC shim for Application diff --git a/abci/server/socket_server.go b/abci/server/socket_server.go index 650d9828f2..83cf8590cd 100644 --- a/abci/server/socket_server.go +++ b/abci/server/socket_server.go @@ -8,6 +8,7 @@ import ( "io" "net" "runtime" + stdsync "sync" sync "github.com/sasha-s/go-deadlock" @@ -55,7 +56,7 @@ func (s *SocketServer) OnStart(ctx context.Context) error { } s.listener = ln - go s.acceptConnectionsRoutine(ctx) + s.Go(ctx, s.acceptConnectionsRoutine) return nil } @@ -103,7 +104,7 @@ func (s *SocketServer) acceptConnectionsRoutine(ctx context.Context) { s.logger.Info("Waiting for new connection...") conn, err := s.listener.Accept() if err != nil { - if !s.IsRunning() { + if ctx.Err() != nil { return // Ignore error from listener closing. } s.logger.Warn("Failed to accept connection", "err", err) @@ -111,7 +112,11 @@ func (s *SocketServer) acceptConnectionsRoutine(ctx context.Context) { } cctx, ccancel := context.WithCancel(ctx) - connID := s.addConn(ccancel) + closeConn := stdsync.OnceValue(conn.Close) + // Cancellation cannot interrupt socket I/O, so shutdown also closes the + // transport. This closer never touches connsMtx: OnStop and rmConn call + // it while holding that lock. + connID := s.addConn(func() { ccancel(); _ = closeConn() }) s.logger.Info("Accepted a new connection", "id", connID) @@ -121,7 +126,7 @@ func (s *SocketServer) acceptConnectionsRoutine(ctx context.Context) { closer := func(err error) { ccancel() once.Do(func() { - if cerr := conn.Close(); err != nil { + if cerr := closeConn(); cerr != nil { s.logger.Error("error closing connection", "id", connID, "close_err", cerr, @@ -153,9 +158,15 @@ func (s *SocketServer) acceptConnectionsRoutine(ctx context.Context) { } // Read requests from conn and deal with them - go s.handleRequests(cctx, closer, conn, responses) + if !s.Go(cctx, func(ctx context.Context) { s.handleRequests(ctx, closer, conn, responses) }) { + closer(context.Canceled) + return + } // Pull responses from 'responses' and write them to conn. - go s.handleResponses(cctx, closer, conn, responses) + if !s.Go(cctx, func(ctx context.Context) { s.handleResponses(ctx, closer, conn, responses) }) { + closer(context.Canceled) + return + } } } diff --git a/abci/server/socket_server_test.go b/abci/server/socket_server_test.go new file mode 100644 index 0000000000..a698e5b9fa --- /dev/null +++ b/abci/server/socket_server_test.go @@ -0,0 +1,69 @@ +package server + +import ( + "bufio" + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/abci/types" + "github.com/dashpay/tenderdash/libs/log" +) + +const ( + stalledQueries = 32 + stalledRespSize = 1 << 20 +) + +// largeQueryApp answers every query with a large value and reports each call. +type largeQueryApp struct { + types.BaseApplication + served chan struct{} +} + +func (a *largeQueryApp) Query(context.Context, *types.RequestQuery) (*types.ResponseQuery, error) { + a.served <- struct{}{} + return &types.ResponseQuery{Value: make([]byte, stalledRespSize)}, nil +} + +// A client that stops reading responses leaves the response writer blocked +// on the socket while the request reader waits for the next request. Stop +// must still unblock both before Wait joins them. +func TestSocketServerStopUnblocksStalledConnection(t *testing.T) { + addr := "unix://" + filepath.Join(t.TempDir(), "abci.sock") + app := &largeQueryApp{served: make(chan struct{}, stalledQueries)} + srv := NewSocketServer(log.NewNopLogger(), addr, app).(*SocketServer) + require.NoError(t, srv.Start(context.Background())) + + conn, err := net.Dial("unix", srv.addr) + require.NoError(t, err) + defer conn.Close() + // Responses (32 MiB) far exceed socket buffers; nothing is ever read back. + w := bufio.NewWriter(conn) + for i := 0; i < stalledQueries; i++ { + require.NoError(t, types.WriteMessage(types.ToRequestQuery(&types.RequestQuery{}), w)) + } + require.NoError(t, w.Flush()) + for i := 0; i < stalledQueries; i++ { + select { + case <-app.served: + case <-time.After(5 * time.Second): + t.Fatalf("server processed only %d of %d queries", i, stalledQueries) + } + } + + srv.Stop() + stopped := make(chan struct{}) + go func() { srv.Wait(); close(stopped) }() + select { + case <-stopped: + case <-time.After(5 * time.Second): + _ = conn.Close() + <-stopped + t.Fatal("Wait blocked on a connection with a stalled response write") + } +} diff --git a/dash/core/client.go b/dash/core/client.go index e012d428ef..5faf8d3f93 100644 --- a/dash/core/client.go +++ b/dash/core/client.go @@ -62,11 +62,16 @@ type Client interface { Ping() error } +// defaultCloseTimeout bounds Close: the endpoint has no HTTP timeout or +// cancellation, so a stuck request would otherwise block it indefinitely. +const defaultCloseTimeout = 5 * time.Second + // RPCClient implements Client // Handles connection to the underlying dashd instance type RPCClient struct { - endpoint *rpc.Client - logger log.Logger + endpoint *rpc.Client + logger log.Logger + closeTimeout time.Duration } // NewRPCClient returns an instance of Client. @@ -96,17 +101,33 @@ func NewRPCClient(host string, username string, password string, logger log.Logg } dashCoreClient := RPCClient{ - endpoint: client, - logger: logger, + endpoint: client, + logger: logger, + closeTimeout: defaultCloseTimeout, } return &dashCoreClient, nil } -// Close closes the underlying connection +// Close closes the underlying connection and joins its dispatcher, up to +// closeTimeout. It is safe to call repeatedly. Do not send requests after Close: +// they can return an error, but they block forever once the endpoint's send +// buffer fills, because nothing drains it any more. func (rpcClient *RPCClient) Close() error { rpcClient.endpoint.Shutdown() - return nil + joined := make(chan struct{}) + go func() { + defer close(joined) + rpcClient.endpoint.WaitForShutdown() + }() + timer := time.NewTimer(rpcClient.closeTimeout) + defer timer.Stop() + select { + case <-joined: + return nil + case <-timer.C: + return fmt.Errorf("dash core request still in flight %s after closing the client", rpcClient.closeTimeout) + } } // Ping sends a ping request to the remote signer diff --git a/dash/core/client_close_test.go b/dash/core/client_close_test.go new file mode 100644 index 0000000000..114843323d --- /dev/null +++ b/dash/core/client_close_test.go @@ -0,0 +1,67 @@ +package core + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" +) + +func newCloseTestClient(t *testing.T, handler http.HandlerFunc) (*RPCClient, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + client, err := NewRPCClient(strings.TrimPrefix(srv.URL, "http://"), "user", "pass", log.NewNopLogger()) + require.NoError(t, err) + return client, srv +} + +// Close joins an idle client, repeatedly, and the first later request gets an error. +func TestRPCClientCloseJoinsIdleClient(t *testing.T) { + client, _ := newCloseTestClient(t, func(http.ResponseWriter, *http.Request) {}) + client.closeTimeout = 5 * time.Second + + require.NoError(t, client.Close()) + require.NoError(t, client.Close()) + require.Error(t, client.Ping()) +} + +// The endpoint cannot cancel a request in flight, so Close must give up on it. +func TestRPCClientCloseBoundedByStuckRequest(t *testing.T) { + entered := make(chan struct{}) + unblock := make(chan struct{}) + release := sync.OnceFunc(func() { close(unblock) }) + client, _ := newCloseTestClient(t, func(http.ResponseWriter, *http.Request) { + close(entered) + <-unblock + }) + t.Cleanup(release) // runs before the server's cleanup, which joins the handler + client.closeTimeout = 50 * time.Millisecond + pinged := make(chan error, 1) + go func() { pinged <- client.Ping() }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("request did not reach the server") + } + + closed := make(chan error, 1) + go func() { closed <- client.Close() }() + select { + case err := <-closed: + require.ErrorContains(t, err, "still in flight") + case <-time.After(5 * time.Second): + t.Fatal("Close blocked on the stuck request") + } + + release() + require.Error(t, <-pinged) + client.closeTimeout = 5 * time.Second + require.NoError(t, client.Close(), "the dispatcher did not exit once the request ended") +} diff --git a/dash/quorum/validator_conn_executor.go b/dash/quorum/validator_conn_executor.go index 4d30cc9fbe..e3768ddfaf 100644 --- a/dash/quorum/validator_conn_executor.go +++ b/dash/quorum/validator_conn_executor.go @@ -167,13 +167,13 @@ func (vc *ValidatorConnExecutor) OnStart(ctx context.Context) error { vc.logger.Error("Warning: ValidatorConnExecutor OnStart failed", "error", err) } - go func() { + vc.Go(ctx, func(ctx context.Context) { var err error for err == nil { err = vc.receiveEvents(ctx) } vc.logger.Error("ValidatorConnExecutor goroutine finished", "reason", err) - }() + }) return nil } @@ -186,7 +186,6 @@ func (vc *ValidatorConnExecutor) OnStop() { if err != nil { vc.logger.Error("cannot unsubscribe from channels", "error", err) } - vc.eventBus = nil } } diff --git a/docs/architecture/service-lifecycle.md b/docs/architecture/service-lifecycle.md index bc52bc099a..70392238e9 100644 --- a/docs/architecture/service-lifecycle.md +++ b/docs/architecture/service-lifecycle.md @@ -1,9 +1,11 @@ -# Service lifecycle core +# Service lifecycle `BaseService.Start` gives `OnStart` an attempt-scoped child context. Parent cancellation and manual `Stop` both cancel it. `Go(ctx, fn)` registers workers using that context or a derived context; registration is synchronized with -shutdown and rejected for a different attempt or after shutdown begins. +shutdown and rejected for a different attempt, a canceled context, or after +shutdown begins. Clean up any resources acquired before a rejected admission. +Plain `go` statements are not tracked. Shutdown closes admission, marks `IsRunning` false, cancels the context and calls `OnStop`. It then joins registered workers, calls optional `OnDrain`, and @@ -25,54 +27,99 @@ The implementation must roll back acquired resources: failed attempts do not invoke `OnStop` or `OnDrain`. Successfully stopped services cannot restart. Composite retries also require recreating children already successfully stopped. -## First migration stage +## Ownership and durability -Consensus State registers its receive loop and joins its queue fan-in and WAL -before returning. The timeout ticker, WAL and autofile group use shared worker -tracking. AutoFile explicitly joins its timer/signal worker. WAL ownership lasts -until the receive loop's final write. Consensus handoff admission is also joined, -so a late blocksync handoff cannot start State after reactor shutdown. The caller -may cancel its wait after admission, but the reactor still owns and joins the -handoff. Startup and the resulting State use the reactor's lifetime. +Registering a wrapper does not automatically join its children. Owners stop and +wait for child services before closing their shared stores, transports or sinks. +A request or peer session can retain its own cancellation and join group when +its lifetime is shorter than the enclosing service. -Node waits for reactors before closing event sinks and stores. The blocked -receive-loop regression and original node teardown tests verify this path -without using leaktest as an extra production-shutdown barrier. - -This is a partial migration. Plain goroutines are not tracked automatically. -RPC/WebSocket handlers, mempool rechecks, reactor gossip and other service -workers retain their existing ownership mechanisms. This change does not claim -that Node.Wait joins every helper or introduce the broader durable-finalization -and dependency-lifetime changes from #1515. +Consensus keeps its WAL available until the receive loop finishes. WAL and +file groups flush after their periodic workers exit; AutoFile closes and joins +its timer/signal worker. The node runs application +connections, the event bus and the indexer on a dependency context that outlives +reactor cancellation, and cancels it only after reactors have been joined. These +explicitly owned lifetimes can outlive cancellation of the caller; their owners +must still join them. A timeout is not a substitute for completion. Three compatibility bridges preserve parent-context lifetimes: blocksync application during handoff, connection error callbacks after I/O shutdown, and an admitted consensus commit during direct State.Stop. Parent cancellation still aborts these operations. In particular, reactor/node shutdown can interrupt -finalization and require WAL recovery. Their existing ownership mechanisms remain -in place; this does not detach application work from node shutdown. +finalization and require WAL recovery; durable finalization across node +shutdown is not guaranteed. Owners still join these operations before releasing +the resources they use. + +## Migration coverage + +The service audit covers consensus (state, reactor, ticker, gossip and WAL), +autofile groups, node and seed owners, event bus and pubsub, indexer, block and +state sync, mempool, evidence, P2P router/PEX/connections, quorum event handling, +ABCI clients/servers, proxy wrappers, signer services, HTTP/WebSocket RPC servers, +and the light RPC wrapper. Worker-free implementations retain their hooks +without artificial workers. + +Service-scoped cancel functions, completion channels and custom `Wait` methods +are replaced by the shared mechanism. Connection/request groups, peer-session +cancellation, restartable RPC broadcast admission and worker-pool synchronization +remain where they represent a separate lifetime. + +This does not claim a repository-wide join for every plain goroutine. RPC +websocket client reconnection/session helpers are outside `BaseService`; generic +consumer middleware can hide context-canceled, in-memory limiter housekeeping. +Those boundaries need their own ownership API before a service can promise to +join every helper. Router connection/request groups and AutoFile workers are +joined explicitly rather than converted into unrelated service lifetimes. + +RPC shutdown closes admission and joins active handlers, including hijacked +WebSocket sessions and their subscription workers. Node shutdown then drains +async broadcasts and reactors, cancels and joins every mempool recheck batch, +and finally releases application connections and stores. + +Arbitrary injected ABCI clients cannot be recreated by the routed wrapper. The +node reserves RPC listeners before starting children, so retry after listener +failure remains supported without restarting successful child lifetimes. That +failure keeps the node's stores open for the retry, and `service.Service` offers +no way to release them if the caller gives up instead. + +Any later node startup failure, and any seed startup failure, is final: rollback +closes the databases, event sinks and other resources registered as node closers +exactly once, later `Stop` and `Wait` calls are no-ops, and another `Start` is +rejected. + +The five-second bound covers only the join in the Dash Core client's `Close`; a +request still stuck there outlives it and an error is logged. It does not bound +node shutdown. The pinned client has no HTTP timeout or cancellation, so a reactor +stuck in a Dash Core call blocks `Wait`, or the rollback inside a failed `Start`, +before closers run. The node command's `--shutdown-timeout` limits only its wait +after the context ends, then exits reporting incomplete cleanup; it does not limit +`Start`, and embedders must bound `Wait` themselves. + +Recover from a final failure by constructing a fresh node graph. Metrics register +on the process-wide default registry, so a second build in the same process panics +with `duplicate metrics collector registration attempted` for a seed, or for a +node with Prometheus enabled, unless `Instrumentation.Namespace` differs. ## Cancellation compatibility audit -Cancellation precedes OnStop for every BaseService implementation, not only the -managed-worker migrations. The in-tree hook/callback audit covers these groups: +Cancellation precedes `OnStop` for every `BaseService` implementation. The +in-tree hook/callback audit covers these groups: | Implementations | Cancellation and cleanup policy | | --- | --- | | Consensus State, reactor, ticker, WAL; autofile Group | Managed work is joined; commit uses the State parent context; file ownership ends with explicit Close. | -| Blocksync reactor and Synchronizer | Handoff preserves application work across Synchronizer.Stop; stopping its parent reactor may cancel it. Reactor OnStop stops and waits for the Synchronizer, so reactor Wait covers in-flight application. | -| P2P Router and MConnection | Connection callbacks retain the parent. Router removes peer bookkeeping even with a canceled context; final PeerStatusDown broadcasts are best effort. | -| PEX, mempool, statesync and evidence reactors | Processing uses the work context; stop hooks close resources/dispatchers or require no extra signal. Existing plain-worker ownership is unchanged. | -| EventBus, pubsub Server and indexer Service | Event processing is cancellation-driven; indexer closes sinks in its existing hook. No guarantee of a live cleanup callback context. | +| Blocksync reactor and Synchronizer | Handoff preserves application work across Synchronizer.Stop; stopping its parent reactor may cancel it. The reactor joins the Synchronizer in OnStop, and again in OnDrain for one started by an admitted handoff. | +| P2P Router and MConnection | Connection callbacks retain the parent context and are joined by `Wait`. Router removes peer bookkeeping even with a canceled context; final PeerStatusDown broadcasts are best effort. | +| PEX, mempool, statesync and evidence reactors | Processing uses the work context; stop hooks close resources/dispatchers or require no extra signal. | +| EventBus, pubsub Server and indexer Service | Event processing is cancellation-driven; the indexer closes its sinks in OnDrain. No guarantee of a live cleanup callback context. | | ABCI local, socket, gRPC and routed clients; proxy; socket/gRPC servers | Stop hooks close transports, drain requests or stop children without requiring a live work context. | | SignerServer, signer listener/dialer endpoints | Hooks close endpoints/listeners; dial/retry work observes cancellation. | -| ValidatorConnExecutor and light RPC Client | Unsubscribe uses a separate bounded cleanup context; client hooks invoke owned closers. | -| Node and seed node | Parent cancellation stops children; hooks wait for existing service completion before closing owned stores. | +| ValidatorConnExecutor and light RPC Client | Unsubscribe uses a separate bounded cleanup context. The light client scopes WebSocket subscriptions to its service and the caller session, and joins (or stops) the wrapped client in OnDrain. | +| Node and seed node | Parent cancellation stops children; the node joins reactors before canceling its dependency context and closing owned stores. | -This is a cancellation compatibility audit, not a claim that unmigrated plain -workers are all joined. Subscribers must stop with their owner, without relying -on a final router broadcast. Avoid detaching broadcasts from cancellation: -unresponsive subscribers could otherwise block shutdown. +Subscribers must stop with their owner, without relying on a final router +broadcast. Avoid detaching broadcasts from cancellation: unresponsive +subscribers could otherwise block shutdown. Group rotation reuses its AutoFile and closes the current descriptor before renaming; it does not create a worker or signal registration per rotation. The diff --git a/internal/blocksync/handover_test.go b/internal/blocksync/handover_test.go index 0d32d13dd6..1baedaef7f 100644 --- a/internal/blocksync/handover_test.go +++ b/internal/blocksync/handover_test.go @@ -55,6 +55,11 @@ func TestSwitchToBlockSyncStartsAfterSnapshot(t *testing.T) { executor: applier, synchronizer: synchronizer, blockSyncFlag: new(atomic.Bool), eventBus: bus, statusUpdateInterval: time.Hour, } + hooks := &handoverStartHook{start: func(ctx context.Context) { reactor.serviceCtx = ctx }} + reactor.BaseService = *service.NewBaseService(log.NewNopLogger(), "blocksync-test", hooks) + require.NoError(t, reactor.Start(ctx)) + defer reactor.Wait() + defer cancel() state := sm.State{InitialHeight: tc.initialHeight, LastBlockHeight: 100} require.NoError(t, reactor.SwitchToBlockSync(ctx, state)) defer synchronizer.Wait() @@ -156,6 +161,22 @@ func (c *handoverCapture) SwitchToConsensus(_ context.Context, state sm.State, s type handoverStopHook struct{ fn func() } +func TestSwitchToBlockSyncRejectsAfterStop(t *testing.T) { + reactor := &Reactor{blockSyncFlag: new(atomic.Bool)} + hooks := &handoverStartHook{start: func(ctx context.Context) { reactor.serviceCtx = ctx }} + reactor.BaseService = *service.NewBaseService(log.NewNopLogger(), "blocksync-test", hooks) + require.NoError(t, reactor.Start(context.Background())) + reactor.Stop() + reactor.Wait() + require.Error(t, reactor.SwitchToBlockSync(context.Background(), sm.State{})) + require.False(t, reactor.blockSyncFlag.Load()) +} + +type handoverStartHook struct{ start func(context.Context) } + +func (h *handoverStartHook) OnStart(ctx context.Context) error { h.start(ctx); return nil } +func (*handoverStartHook) OnStop() {} + func (*handoverStopHook) OnStart(context.Context) error { return nil } func (h *handoverStopHook) OnStop() { h.fn() } @@ -184,13 +205,16 @@ func TestReactorWaitJoinsSynchronizerDrain(t *testing.T) { // The synchronizer's own stop wins and holds its drain open. go synchronizer.Stop() <-draining + poolDone := make(chan struct{}) if handover { // Parent cancellation releases poolRoutine from WaitForSync; its Stop - // returns early and it clears the flag before reactor OnStop runs. + // returns early, so it must join the draining synchronizer before it + // reads the final state and clears the flag. poolCtx, cancelPool := context.WithCancel(ctx) cancelPool() - reactor.poolRoutine(poolCtx, false) - require.False(t, reactor.blockSyncFlag.Load()) + go func() { reactor.poolRoutine(poolCtx, false); close(poolDone) }() + } else { + close(poolDone) } cancel() waited := make(chan struct{}) @@ -200,9 +224,21 @@ func TestReactorWaitJoinsSynchronizerDrain(t *testing.T) { t.Fatal("reactor Wait returned while synchronizer application was still running") case <-time.After(100 * time.Millisecond): } + if handover { + select { + case <-poolDone: + t.Fatal("poolRoutine returned while synchronizer application was still running") + default: + } + require.True(t, reactor.blockSyncFlag.Load()) + } close(releaseApply) <-waited + <-poolDone require.True(t, applyReturned.Load()) + if handover { + require.False(t, reactor.blockSyncFlag.Load()) + } synchronizer.Wait() }) } diff --git a/internal/blocksync/reactor.go b/internal/blocksync/reactor.go index 3da6aa5868..c365975b24 100644 --- a/internal/blocksync/reactor.go +++ b/internal/blocksync/reactor.go @@ -83,7 +83,8 @@ type consensusReactor interface { // Reactor handles long-term catchup syncing. type Reactor struct { service.BaseService - logger log.Logger + serviceCtx context.Context + logger log.Logger // immutable initialState sm.State @@ -185,6 +186,7 @@ func WithSynchronizerOptions(opts ...OptionFunc) ReactorOption { // If blockSyncFlag is enabled, we also start the synchronizer // If the synchronizer fails to start, an error is returned. func (r *Reactor) OnStart(ctx context.Context) error { + r.serviceCtx = ctx state, err := r.stateStore.Load() if err != nil { return err @@ -207,24 +209,29 @@ func (r *Reactor) OnStart(ctx context.Context) error { if err := r.synchronizer.Start(ctx); err != nil { return err } - go r.requestRoutine(ctx, r.p2pClient) - go r.poolRoutine(ctx, false) + r.Go(ctx, func(ctx context.Context) { r.requestRoutine(ctx, r.p2pClient) }) + r.Go(ctx, func(ctx context.Context) { r.poolRoutine(ctx, false) }) } consumer, messageHandler := consumerHandler(ctx, r.logger, r.store, r.synchronizer) r.messageHandler = messageHandler - go func() { + r.Go(ctx, func(ctx context.Context) { err := r.p2pClient.Consume(ctx, consumer) if err != nil { r.logger.Error("failed to consume p2p blocksync messages", "error", err) } - }() - go r.processPeerUpdates(ctx, r.peerEvents(ctx, "blocksync"), r.p2pClient, messageHandler) + }) + updates := r.peerEvents(ctx, "blocksync") + if !r.Go(ctx, func(ctx context.Context) { + defer updates.Close() + r.processPeerUpdates(ctx, updates, r.p2pClient, messageHandler) + }) { + updates.Close() + } return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit. +// OnStop shuts down the request handler and synchronizer. func (r *Reactor) OnStop() { if r.messageHandler != nil { r.messageHandler.stop() @@ -238,6 +245,15 @@ func (r *Reactor) OnStop() { } } +// OnDrain joins a synchronizer started by a handoff worker admitted before +// shutdown: OnStop may run before that worker reaches Synchronizer.Start. +func (r *Reactor) OnDrain() { + if r.synchronizer != nil { + r.synchronizer.Stop() + r.synchronizer.Wait() + } +} + // processPeerUpdate processes a PeerUpdate. func (r *Reactor) processPeerUpdate( ctx context.Context, @@ -300,6 +316,21 @@ func (r *Reactor) processPeerUpdates( // SwitchToBlockSync is called by the state sync reactor when switching to fast // sync. func (r *Reactor) SwitchToBlockSync(ctx context.Context, state sm.State) error { + result := make(chan error, 1) + if !r.Go(r.serviceCtx, func(serviceCtx context.Context) { + result <- r.switchToBlockSync(serviceCtx, state) + }) { + return errors.New("block sync reactor is stopping") + } + select { + case err := <-result: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (r *Reactor) switchToBlockSync(ctx context.Context, state sm.State) error { r.blockSyncFlag.Store(true) r.initialState = state r.executor.UpdateState(state) @@ -311,8 +342,8 @@ func (r *Reactor) SwitchToBlockSync(ctx context.Context, state sm.State) error { r.syncStartTime = time.Now() - go r.requestRoutine(ctx, r.p2pClient) - go r.poolRoutine(ctx, true) + r.Go(ctx, func(ctx context.Context) { r.requestRoutine(ctx, r.p2pClient) }) + r.Go(ctx, func(ctx context.Context) { r.poolRoutine(ctx, true) }) if err := r.PublishStatus(types.EventDataBlockSyncStatus{ Complete: false, @@ -348,6 +379,7 @@ func (r *Reactor) requestRoutine(ctx context.Context, p2pClient *client.Client) func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool) { caughtUp, targetHeight := r.synchronizer.WaitForSync(ctx) r.synchronizer.Stop() + r.synchronizer.Wait() // Wait for an application holding the applier lock and use its final state. state := r.executor.State() r.blockSyncFlag.Store(false) diff --git a/internal/blocksync/synchronizer.go b/internal/blocksync/synchronizer.go index ecf26e9cba..01b411d906 100644 --- a/internal/blocksync/synchronizer.go +++ b/internal/blocksync/synchronizer.go @@ -164,13 +164,7 @@ type ( jobGen *jobGenerator pendingToApply map[int64]BlockResponse - // ctx/cancel scope the handler goroutines' lifetime. Created in OnStart - // (live before the goroutines spawn, so it never observes a start-race) - // and canceled in OnStop so Stop releases the handlers even when the - // caller's context is still live. - ctx context.Context - cancel context.CancelFunc - consumerDone chan struct{} + ctx context.Context } OptionFunc func(v *Synchronizer) ) @@ -262,30 +256,25 @@ func (s *Synchronizer) OnStart(ctx context.Context) error { s.lastBlockTime = state.LastBlockTime s.precision = state.ConsensusParams.Synchrony.SynchronyParamsOrDefaults().Precision } - s.ctx, s.cancel = context.WithCancel(ctx) + s.ctx = ctx applicationCtx, ok := ctx.Value(applicationContextKey{}).(context.Context) if !ok { applicationCtx = ctx } - s.consumerDone = make(chan struct{}) s.workerPool.Run(s.ctx) - go s.runHandler(s.ctx, s.produceJob) - go s.runHandler(s.ctx, s.askEveryPeer) - go func() { - defer close(s.consumerDone) - s.runHandler(s.ctx, func(handlerCtx context.Context) error { + s.Go(ctx, func(ctx context.Context) { s.runHandler(ctx, s.produceJob) }) + s.Go(ctx, func(ctx context.Context) { s.runHandler(ctx, s.askEveryPeer) }) + s.Go(ctx, func(ctx context.Context) { + s.runHandler(ctx, func(handlerCtx context.Context) error { // Handover cancels consumer I/O; parent cancellation also stops application. return s.consumeJobResult(handlerCtx, applicationCtx) }) - }() + }) return nil } func (s *Synchronizer) OnStop() { - s.cancel() s.workerPool.Stop(context.Background()) - // Finish all application work before consensus reads the final state. - <-s.consumerDone } func (s *Synchronizer) produceJob(ctx context.Context) error { diff --git a/internal/blocksync/synchronizer_test.go b/internal/blocksync/synchronizer_test.go index 917acd000c..6367dc58b5 100644 --- a/internal/blocksync/synchronizer_test.go +++ b/internal/blocksync/synchronizer_test.go @@ -117,6 +117,7 @@ func (suite *SynchronizerTestSuite) TestBasic() { return !sync.IsCaughtUp() }, 2*time.Second, 10*time.Millisecond) sync.Stop() + sync.Wait() } func (suite *SynchronizerTestSuite) TestHandoverPreservesInFlightFinalization() { @@ -178,6 +179,7 @@ func (suite *SynchronizerTestSuite) assertHandoverWaitsForConsumer(beforeApply b stopped := make(chan struct{}) go func() { pool.Stop() + pool.Wait() close(stopped) }() select { @@ -259,6 +261,7 @@ func (suite *SynchronizerTestSuite) TestHandoverCancelsBlockedPeerError() { stopped := make(chan struct{}) go func() { pool.Stop() + pool.Wait() close(stopped) }() select { @@ -1127,6 +1130,7 @@ func (suite *SynchronizerTestSuite) TestStopReleasesHandlers() { time.Sleep(50 * time.Millisecond) sync.Stop() + sync.Wait() // With the parent ctx still live, both handler goroutines must have exited. suite.Require().NoError(ctx.Err()) diff --git a/internal/consensus/common_test.go b/internal/consensus/common_test.go index eabc08ae10..6c3b9dc521 100644 --- a/internal/consensus/common_test.go +++ b/internal/consensus/common_test.go @@ -539,6 +539,8 @@ func newStateWithConfigAndBlockStore( eventBus := eventbus.NewDefault(logger.With("module", "events")) require.NoError(t, eventBus.Start(ctx)) + // Join the bus before the test ends: its shutdown logs through t. + t.Cleanup(func() { eventBus.Stop(); eventBus.Wait() }) blockExec := sm.NewBlockExecutor(stateStore, proxyAppConnCon, mpool, evpool, blockStore, eventBus) cs, err := NewState(logger.With("module", "consensus"), diff --git a/internal/consensus/gossip_peer_worker.go b/internal/consensus/gossip_peer_worker.go index e12c298aaa..99896f8a93 100644 --- a/internal/consensus/gossip_peer_worker.go +++ b/internal/consensus/gossip_peer_worker.go @@ -3,13 +3,13 @@ package consensus import ( "context" "runtime/debug" - "sync/atomic" "time" "github.com/jonboulle/clockwork" "github.com/dashpay/tenderdash/config" "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" ) // minGossipSleepDuration is the shortest cadence a gossip handler runs at, @@ -56,24 +56,21 @@ type gossipHandlerFunc func(ctx context.Context, appState StateData) type gossipHandler struct { sleepDuration time.Duration handlerFunc func(ctx context.Context, appState StateData) - stoppedCh chan struct{} } func newGossipHandler(fn gossipHandlerFunc, sleep time.Duration) gossipHandler { return gossipHandler{ sleepDuration: sleep, handlerFunc: fn, - stoppedCh: make(chan struct{}), } } type peerGossipWorker struct { + service.BaseService clock clockwork.Clock logger log.Logger handlers []gossipHandler - running atomic.Bool stateDataStore *StateDataStore - stopCh chan struct{} } func newPeerGossipWorker( @@ -92,10 +89,9 @@ func newPeerGossipWorker( optimistic: true, clock: clock, } - return &peerGossipWorker{ + worker := &peerGossipWorker{ clock: clock, logger: logger, - stopCh: make(chan struct{}), stateDataStore: state.stateDataStore, handlers: []gossipHandler{ newGossipHandler( @@ -133,35 +129,18 @@ func newPeerGossipWorker( ), }, } + worker.BaseService = *service.NewBaseService(logger, "PeerGossipWorker", worker) + return worker } -func (g *peerGossipWorker) IsRunning() bool { - return g.running.Load() -} - -func (g *peerGossipWorker) Start(ctx context.Context) error { - g.running.Store(true) +func (g *peerGossipWorker) OnStart(ctx context.Context) error { for _, handler := range g.handlers { - go g.runHandler(ctx, handler) + g.Go(ctx, func(ctx context.Context) { g.runHandler(ctx, handler) }) } return nil } -func (g *peerGossipWorker) Stop() { - if !g.running.Swap(false) { - return - } - g.logger.Trace("peer gossip worker stopping") - close(g.stopCh) - g.Wait() -} - -func (g *peerGossipWorker) Wait() { - for _, hd := range g.handlers { - <-hd.stoppedCh - g.logger.Trace("peer gossip worker stopped") - } -} +func (g *peerGossipWorker) OnStop() {} // runGossipHandler invokes a gossip handler, converting a panic into a logged // error. Handlers load historical records at heights chosen by the peer they @@ -186,13 +165,8 @@ func (g *peerGossipWorker) runHandler(ctx context.Context, hd gossipHandler) { timer.Reset(hd.sleepDuration) select { case <-timer.Chan(): - case <-g.stopCh: - g.logger.Trace("peer gossip worker got stop signal") - close(hd.stoppedCh) - return case <-ctx.Done(): g.logger.Trace("peer gossip worker got stop signal via context.Done") - close(hd.stoppedCh) return } } diff --git a/internal/consensus/gossip_peer_worker_test.go b/internal/consensus/gossip_peer_worker_test.go index 93b6d2e324..189b0bb879 100644 --- a/internal/consensus/gossip_peer_worker_test.go +++ b/internal/consensus/gossip_peer_worker_test.go @@ -2,7 +2,6 @@ package consensus import ( "context" - "sync/atomic" "testing" "time" @@ -12,6 +11,7 @@ import ( "github.com/dashpay/tenderdash/config" "github.com/dashpay/tenderdash/libs/eventemitter" "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" "github.com/dashpay/tenderdash/types" ) @@ -35,10 +35,9 @@ func TestPeerGossipWorker(t *testing.T) { handlerCalledCh <- struct{}{} }, 1*time.Second), }, - running: atomic.Bool{}, stateDataStore: NewStateDataStore(NopMetrics(), logger, cfg.Consensus, emitter), - stopCh: make(chan struct{}), } + pg.BaseService = *service.NewBaseService(logger, "PeerGossipWorker", &pg) require.False(t, pg.IsRunning()) err := pg.Start(ctx) require.NoError(t, err) @@ -48,10 +47,39 @@ func TestPeerGossipWorker(t *testing.T) { } defer cancel() pg.Stop() + pg.Wait() require.False(t, pg.IsRunning()) close(handlerCalledCh) } +func TestPeerGossipWorkerWaitDrainsHandler(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + entered, release := make(chan struct{}), make(chan struct{}) + cs, _ := makeState(ctx, t, makeStateArgs{}) + pg := &peerGossipWorker{ + clock: clockwork.NewRealClock(), logger: log.NewNopLogger(), stateDataStore: cs.stateDataStore, + handlers: []gossipHandler{newGossipHandler(func(context.Context, StateData) { close(entered); <-release }, time.Hour)}, + } + pg.BaseService = *service.NewBaseService(pg.logger, t.Name(), pg) + require.NoError(t, pg.Start(ctx)) + <-entered + pg.Stop() + done := make(chan struct{}) + go func() { pg.Wait(); close(done) }() + select { + case <-done: + t.Error("Wait returned before gossip handler finished") + case <-time.After(20 * time.Millisecond): + } + close(release) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("gossip handler did not drain") + } +} + // A handler whose sleep is zero runs back to back, spinning a goroutine per // handler per peer, and a config of 0s — or a stray "2" read as 2ns — is not // caught where validation is skipped. The reactor floors the cadence once, for diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index cdac5dca01..83636ad190 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -294,8 +294,8 @@ type Reactor struct { stateOutboxWake chan struct{} // Context for controlling reactor goroutines - ctx context.Context - cancel context.CancelFunc + ctx context.Context + peerUpdates *p2p.PeerUpdates } // NewReactor returns a reference to a new consensus reactor, which implements @@ -343,15 +343,16 @@ type channelBundle struct { voteSet p2p.Channel } -// OnStart starts separate go routines for each p2p Channel and listens for -// envelopes on each. In addition, it also listens for peer updates and handles -// messages on that p2p channel accordingly. The caller must be sure to execute -// OnStop to ensure the outbound p2p Channels are closed. -func (r *Reactor) OnStart(ctx context.Context) error { +// OnStart starts channel, peer-update and gossip workers owned by the reactor. +func (r *Reactor) OnStart(ctx context.Context) (err error) { + defer func() { + if err != nil { + r.closeOwnedHelpers() + } + }() r.logger.Trace("consensus wait sync", "wait_sync", r.WaitSync()) - // Create reactor-specific context that will be canceled in OnStop - r.ctx, r.cancel = context.WithCancel(ctx) + r.ctx = ctx // Per-peer rate limiter for the vote channel. drop=true: over-limit // messages are dropped rather than delayed, so a flood from one peer cannot @@ -386,9 +387,9 @@ func (r *Reactor) OnStart(ctx context.Context) error { r.maj23SurplusLimit = newMaj23SurplusLimiter() peerUpdates := r.peerEvents(r.ctx, "consensus") + r.peerUpdates = peerUpdates var chBundle channelBundle - var err error chans := p2p.ConsensusChannelDescriptors() chBundle.state, err = r.chCreator(r.ctx, chans[p2p.ConsensusStateChannel]) @@ -411,17 +412,13 @@ func (r *Reactor) OnStart(ctx context.Context) error { return err } - // start routine that computes peer statistics for evaluating peer quality - // - // TODO: Evaluate if we need this to be synchronized via WaitGroup as to not - // leak the goroutine when stopping the reactor. - go r.peerStatsRoutine(r.ctx, peerUpdates) + r.Go(ctx, func(ctx context.Context) { r.peerStatsRoutine(ctx, peerUpdates) }) // start routine that handles peer errors from the state machine - go r.peerErrorRoutine(r.ctx, chBundle.state) + r.Go(ctx, func(ctx context.Context) { r.peerErrorRoutine(ctx, chBundle.state) }) - go r.sendLiveness(r.ctx, chBundle.state) - go r.sendState(r.ctx, chBundle.state) + r.Go(ctx, func(ctx context.Context) { r.sendLiveness(ctx, chBundle.state) }) + r.Go(ctx, func(ctx context.Context) { r.sendState(ctx, chBundle.state) }) r.subscribeToBroadcastEvents() if !r.WaitSync() { @@ -436,36 +433,41 @@ func (r *Reactor) OnStart(ctx context.Context) error { // Data, vote and vote set must wait. // We cannot skip waiting messages, as the peers might already have marked them as delivered. // XXX: this can lead to a deadlock, if so - we need additional buffer for (at least) Commits. - go r.processMsgCh(r.ctx, chBundle.state, chBundle) - go func() { + r.Go(ctx, func(ctx context.Context) { r.processMsgCh(ctx, chBundle.state, chBundle) }) + r.Go(ctx, func(ctx context.Context) { select { case <-r.readySignal: - go r.processMsgCh(r.ctx, chBundle.data, chBundle) - go r.processMsgCh(r.ctx, chBundle.vote, chBundle) - go r.processMsgCh(r.ctx, chBundle.voteSet, chBundle) - case <-r.ctx.Done(): + r.Go(ctx, func(ctx context.Context) { r.processMsgCh(ctx, chBundle.data, chBundle) }) + r.Go(ctx, func(ctx context.Context) { r.processMsgCh(ctx, chBundle.vote, chBundle) }) + r.Go(ctx, func(ctx context.Context) { r.processMsgCh(ctx, chBundle.voteSet, chBundle) }) + case <-ctx.Done(): } - }() + }) - go r.processPeerUpdates(r.ctx, peerUpdates, chBundle) + r.Go(ctx, func(ctx context.Context) { r.processPeerUpdates(ctx, peerUpdates, chBundle) }) return nil } -// OnStop requests reactor and consensus shutdown. -func (r *Reactor) OnStop() { - // Cancel the reactor context to signal all goroutines to stop - if r.cancel != nil { - r.cancel() - } - - r.state.Stop() -} +// OnStop requests consensus shutdown. +func (r *Reactor) OnStop() { r.state.Stop() } -// OnDrain joins consensus after any admitted handoff has finished. +// OnDrain joins consensus after channel, peer and handoff workers have finished. func (r *Reactor) OnDrain() { r.state.Stop() r.state.Wait() + r.closeOwnedHelpers() +} + +func (r *Reactor) closeOwnedHelpers() { + if r.peerUpdates != nil { + r.peerUpdates.Close() + } + for _, limiter := range []*client.RateLimit{r.voteRateLimit, r.dataRateLimit, r.stateRateLimit, r.maj23PeerShare} { + if limiter != nil { + limiter.Close() + } + } } // WaitSync returns whether the consensus reactor is waiting for state/block sync. @@ -900,25 +902,17 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda switch peerUpdate.Status { case p2p.PeerStatusUp: - // Do not allow starting new broadcasting goroutines after reactor shutdown - // has been initiated. This can happen after we've manually closed all - // peer goroutines, but the router still sends in-flight peer updates. - if !r.IsRunning() { + if ctx.Err() != nil { return } - r.peerUp(ctx, peerUpdate, 3, chans) + r.peerUp(ctx, peerUpdate, chans) case p2p.PeerStatusDown: r.peerDown(ctx, peerUpdate, chans) } } -// peerUp starts the peer. It recursively retries up to `retries` times if the peer is already closing. -func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries int, chans channelBundle) { - if retries < 1 { - r.logger.Error("peer up failed: max retries exceeded", "peer", peerUpdate.NodeID) - return - } - +// peerUp starts gossip for a live peer connection. +func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, chans channelBundle) { r.mtx.Lock() defer r.mtx.Unlock() @@ -944,31 +938,19 @@ func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries "peer", ps.peerID, "peer_proTxHash", ps.GetProTxHash().ShortString(), ) - // TODO needs to register this gossip worker, to be able to stop it once a peer will be down msgSender := p2pMsgSender{logger: logger, ps: ps, chans: chans} pgw := newPeerGossipWorker(logger, ps, r.state, &msgSender, r.cadence) - select { - case <-ctx.Done(): - // Hmm, someone is closing this peer right now, let's wait and retry - // Note: we run this in a goroutine to not block main goroutine in ps.broadcastWG.Wait() - go func() { - time.Sleep(r.state.config.PeerGossipSleepDuration) - r.peerUp(ctx, peerUpdate, retries-1, chans) - }() + if ctx.Err() != nil { return - default: } if !ps.IsRunning() { - // Set the peer state's closer to signal to all spawned goroutines to exit - // when the peer is removed. We also set the running state to ensure we - // do not spawn multiple instances of the same goroutines and finally we - // set the waitgroup counter so we know when all goroutines have exited. + // Peer removal cancels this session; reactor shutdown also joins it. ps.SetRunning(true) ctx, ps.cancel = context.WithCancel(ctx) - go func() { + if !r.Go(ctx, func(ctx context.Context) { select { case <-ctx.Done(): return @@ -981,8 +963,14 @@ func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries } // start goroutines for this peer; the round-state handler's first // pass announces our height, round and step to the peer at once - _ = pgw.Start(ctx) - }() + if err := pgw.Start(ctx); err != nil { + return + } + defer pgw.Wait() + }) { + ps.SetRunning(false) + ps.cancel() + } } } diff --git a/internal/consensus/round_step_refresh_test.go b/internal/consensus/round_step_refresh_test.go index 093742e3ca..91a99f2374 100644 --- a/internal/consensus/round_step_refresh_test.go +++ b/internal/consensus/round_step_refresh_test.go @@ -29,26 +29,43 @@ func TestReactorRefreshesRoundStepAfterLostAnnouncement(t *testing.T) { const refreshInterval = 20 * time.Millisecond cs, _ := makeState(ctx, t, makeStateArgs{}) cs.config.PeerQueryMaj23SleepDuration = refreshInterval - r := NewReactor(log.NewNopLogger(), cs, nil, nil, cs.eventBus, false, NopMetrics()) announcements := make(chan *tmcons.NewRoundStep, 64) - stateCh := p2pmocks.NewChannel(t) - stateCh.On("Send", mock.Anything, mock.Anything).Maybe(). - Run(func(args mock.Arguments) { - msg, ok := args.Get(1).(p2p.Envelope).Message.(*tmcons.NewRoundStep) - if !ok { - return - } - select { - case announcements <- msg: - default: - } - }). - Return(nil) + newChannel := func(capture bool) p2p.Channel { + ch := p2pmocks.NewChannel(t) + ch.On("Receive", mock.Anything).Maybe().Return(p2p.NewChannelIterator(make(chan p2p.Envelope))) + ch.On("SendError", mock.Anything, mock.Anything).Maybe().Return(nil) + ch.On("String").Maybe().Return("channel") + ch.On("Send", mock.Anything, mock.Anything).Maybe(). + Run(func(args mock.Arguments) { + msg, ok := args.Get(1).(p2p.Envelope).Message.(*tmcons.NewRoundStep) + if !capture || !ok { + return + } + select { + case announcements <- msg: + default: + } + }). + Return(nil) + return ch + } + chCreator := func(_ context.Context, desc *p2p.ChannelDescriptor) (p2p.Channel, error) { + return newChannel(desc.ID == p2p.ConsensusStateChannel), nil + } + peerUpdatesCh := make(chan p2p.PeerUpdate, 1) + peerEvents := func(context.Context, string) *p2p.PeerUpdates { + return p2p.NewPeerUpdates(peerUpdatesCh, 1, "consensus") + } + + // Waiting for sync keeps the consensus state machine stopped, so the round + // state stays put; gossip starts once readySignal closes. + r := NewReactor(log.NewNopLogger(), cs, chCreator, peerEvents, cs.eventBus, true, NopMetrics()) + require.NoError(t, r.Start(ctx)) + t.Cleanup(func() { r.Stop(); r.Wait() }) + close(r.readySignal) - peerUpdate := p2p.PeerUpdate{NodeID: "peer", Status: p2p.PeerStatusUp, ProTxHash: crypto.RandProTxHash()} - r.peerUp(ctx, peerUpdate, 3, channelBundle{state: stateCh}) - defer r.peerDown(ctx, p2p.PeerUpdate{NodeID: "peer", Status: p2p.PeerStatusDown}, channelBundle{}) + peerUpdatesCh <- p2p.PeerUpdate{NodeID: "peer", Status: p2p.PeerStatusUp, ProTxHash: crypto.RandProTxHash()} want := cs.GetRoundState() diff --git a/internal/eventbus/event_bus.go b/internal/eventbus/event_bus.go index 9cffd1475a..6a6d649c02 100644 --- a/internal/eventbus/event_bus.go +++ b/internal/eventbus/event_bus.go @@ -40,7 +40,10 @@ func (b *EventBus) OnStart(ctx context.Context) error { return b.pubsub.Start(ctx) } -func (b *EventBus) OnStop() {} +func (b *EventBus) OnStop() { b.pubsub.Stop() } + +// OnDrain waits for queued events and observers to finish. +func (b *EventBus) OnDrain() { b.pubsub.Wait() } func (b *EventBus) NumClients() int { return b.pubsub.NumClients() @@ -66,6 +69,9 @@ func (b *EventBus) Observe(ctx context.Context, observe func(tmpubsub.Message) e return b.pubsub.Observe(ctx, observe, queries...) } +// RemoveObserver waits for active observation and prevents subsequent callbacks. +func (b *EventBus) RemoveObserver() { b.pubsub.RemoveObserver() } + func (b *EventBus) Publish(eventValue string, eventData types.EventData) error { tokens := strings.Split(types.EventTypeKey, ".") event := abci.Event{ diff --git a/internal/eventbus/lifecycle_test.go b/internal/eventbus/lifecycle_test.go new file mode 100644 index 0000000000..6e81f5f071 --- /dev/null +++ b/internal/eventbus/lifecycle_test.go @@ -0,0 +1,44 @@ +package eventbus_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/internal/eventbus" + "github.com/dashpay/tenderdash/internal/pubsub" + "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/types" +) + +func TestWaitJoinsBlockedObserver(t *testing.T) { + bus := eventbus.NewDefault(log.NewNopLogger()) + require.NoError(t, bus.Start(context.Background())) + entered, release := make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + defer func() { unblock(); bus.Stop(); bus.Wait() }() + require.NoError(t, bus.Observe(context.Background(), func(pubsub.Message) error { + close(entered) + <-release + return nil + })) + require.NoError(t, bus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{})) + <-entered + bus.Stop() + done := make(chan struct{}) + go func() { bus.Wait(); close(done) }() + select { + case <-done: + t.Fatal("Wait returned while the observer was still running") + case <-time.After(20 * time.Millisecond): + } + unblock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("Wait did not finish after the observer returned") + } +} diff --git a/internal/evidence/reactor.go b/internal/evidence/reactor.go index eddca40f94..91aacad2d4 100644 --- a/internal/evidence/reactor.go +++ b/internal/evidence/reactor.go @@ -177,15 +177,23 @@ func (r *Reactor) OnStart(ctx context.Context) (err error) { ctx, peerEvidenceRate, peerEvidenceBurst, true, r.logger, client.WithRateLimitClock(r.clock)) r.nodeBudget = newNodeBudget() - go r.processEvidenceCh(ctx) - go r.processPeerUpdates(ctx, r.peerEvents(ctx, "evidence")) + r.Go(ctx, r.processEvidenceCh) + updates := r.peerEvents(ctx, "evidence") + if !r.Go(ctx, func(ctx context.Context) { + defer updates.Close() + r.processPeerUpdates(ctx, updates) + }) { + updates.Close() + } return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit. -func (r *Reactor) OnStop() { r.evpool.Close() } +// OnStop leaves the shared pool open for its owner to close after consensus drains. +func (r *Reactor) OnStop() {} + +// OnDrain joins the rate limiter's cleanup worker. +func (r *Reactor) OnDrain() { r.peerLimit.Close() } // handleEvidenceMessage handles envelopes sent from peers on the EvidenceChannel. // It returns an error only if the Envelope.Message is unknown for this channel @@ -345,7 +353,7 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda // Do not allow starting new evidence broadcast loops after reactor shutdown // has been initiated. This can happen after we've manually closed all // peer broadcast loops, but the router still sends in-flight peer updates. - if !r.IsRunning() { + if ctx.Err() != nil { return } @@ -360,7 +368,10 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda r.nextSyncID++ entry := peerSyncState{cancel: pcancel, id: r.nextSyncID} r.peerRoutines[peerUpdate.NodeID] = entry - go r.syncEvidence(pctx, peerUpdate.NodeID, entry.id) + if !r.Go(pctx, func(ctx context.Context) { r.syncEvidence(ctx, peerUpdate.NodeID, entry.id) }) { + pcancel() + delete(r.peerRoutines, peerUpdate.NodeID) + } } case p2p.PeerStatusDown: diff --git a/internal/mempool/mempool.go b/internal/mempool/mempool.go index 6dcf7eead6..9bcf7f07f2 100644 --- a/internal/mempool/mempool.go +++ b/internal/mempool/mempool.go @@ -5,6 +5,7 @@ import ( "fmt" "runtime" "sort" + stdsync "sync" "sync/atomic" "time" @@ -61,7 +62,9 @@ type TxMempool struct { txBySender map[string]*clist.CElement // for sender != "" // cancellation function for recheck txs tasks - recheckCancel context.CancelFunc + recheckCancel context.CancelFunc + rechecksStopped bool + recheckWorkers stdsync.WaitGroup } // NewTxMempool constructs a new, empty priority mempool at the specified @@ -96,6 +99,18 @@ func NewTxMempool( return txmp } +// StopRechecks permanently closes recheck admission and joins all admitted batches. +// Call it without holding the mempool lock, before closing the ABCI connection. +func (txmp *TxMempool) StopRechecks() { + txmp.mtx.Lock() + txmp.rechecksStopped = true + if txmp.recheckCancel != nil { + txmp.recheckCancel() + } + txmp.mtx.Unlock() + txmp.recheckWorkers.Wait() +} + // WithPreCheck sets a filter for the mempool to reject a transaction if f(tx) // returns an error. This is executed before CheckTx. It only applies to the // first created block. After that, Update() overwrites the existing value. @@ -785,11 +800,15 @@ func (txmp *TxMempool) handleRecheckResult(tx types.Tx, checkTxRes *abci.Respons // Precondition: The mempool is not empty. // The caller must hold txmp.mtx exclusively. func (txmp *TxMempool) recheckTransactions(ctx context.Context) { + if txmp.rechecksStopped { + return + } // cancel previous recheck if it is still running if txmp.recheckCancel != nil { txmp.recheckCancel() } - ctx, txmp.recheckCancel = context.WithCancel(ctx) + ctx, cancel := context.WithCancel(ctx) + txmp.recheckCancel = cancel if txmp.Size() == 0 { panic("mempool: cannot run recheck on an empty mempool") @@ -808,7 +827,10 @@ func (txmp *TxMempool) recheckTransactions(ctx context.Context) { // Issue CheckTx calls for each remaining transaction, and when all the // rechecks are complete signal watchers that transactions may be available. + txmp.recheckWorkers.Add(1) go func() { + defer txmp.recheckWorkers.Done() + defer cancel() var g errgroup.Group g.SetLimit(2 * runtime.NumCPU()) diff --git a/internal/mempool/reactor.go b/internal/mempool/reactor.go index c00c364359..30c0493518 100644 --- a/internal/mempool/reactor.go +++ b/internal/mempool/reactor.go @@ -73,19 +73,24 @@ func (r *Reactor) OnStart(ctx context.Context) error { if !r.cfg.Broadcast { r.logger.Info("tx broadcasting is disabled") } - go func() { + r.Go(ctx, func(ctx context.Context) { err := r.p2pClient.Consume(ctx, consumerHandler(ctx, r.logger, r.mempool.config, r.mempool, r.ids)) if err != nil { r.logger.Error("failed to consume p2p checker messages", "error", err) } - }() - go r.processPeerUpdates(ctx, r.peerEvents(ctx, "checker")) + }) + updates := r.peerEvents(ctx, "checker") + if !r.Go(ctx, func(ctx context.Context) { + defer updates.Close() + r.processPeerUpdates(ctx, updates) + }) { + updates.Close() + } return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit. +// OnStop needs no additional cleanup after the service context is canceled. func (r *Reactor) OnStop() {} // processPeerUpdate processes a PeerUpdate. For added peers, PeerStatusUp, we @@ -104,7 +109,7 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda // Do not allow starting new tx broadcast loops after reactor shutdown // has been initiated. This can happen after we've manually closed all // peer broadcast, but the router still sends in-flight peer updates. - if !r.IsRunning() { + if ctx.Err() != nil { return } @@ -121,7 +126,11 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda r.ids.ReserveForPeer(peerUpdate.NodeID) // start a broadcast routine ensuring all txs are forwarded to the peer - go r.broadcastTxRoutine(pctx, peerUpdate.NodeID) + if !r.Go(pctx, func(ctx context.Context) { r.broadcastTxRoutine(ctx, peerUpdate.NodeID) }) { + pcancel() + delete(r.peerRoutines, peerUpdate.NodeID) + r.ids.Reclaim(peerUpdate.NodeID) + } } } @@ -185,7 +194,7 @@ func (r *Reactor) broadcastTxRoutine(ctx context.Context, peerID types.NodeID) { }() for { - if !r.IsRunning() || ctx.Err() != nil { + if ctx.Err() != nil { return } diff --git a/internal/mempool/recheck_lifecycle_test.go b/internal/mempool/recheck_lifecycle_test.go new file mode 100644 index 0000000000..96a72837fa --- /dev/null +++ b/internal/mempool/recheck_lifecycle_test.go @@ -0,0 +1,155 @@ +package mempool + +import ( + "context" + stdsync "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + abciclient "github.com/dashpay/tenderdash/abci/client" + abci "github.com/dashpay/tenderdash/abci/types" + "github.com/dashpay/tenderdash/config" + "github.com/dashpay/tenderdash/libs/log" +) + +type recheckLifecycleClient struct { + abciclient.Client + check func(context.Context) (*abci.ResponseCheckTx, error) + flush func(context.Context) error +} + +func (c recheckLifecycleClient) CheckTx(ctx context.Context, _ *abci.RequestCheckTx) (*abci.ResponseCheckTx, error) { + return c.check(ctx) +} + +func (c recheckLifecycleClient) Flush(ctx context.Context) error { return c.flush(ctx) } + +func startLifecycleRecheck(t *testing.T, mp *TxMempool) { + t.Helper() + mp.Lock() + defer mp.Unlock() + require.NoError(t, mp.Update(context.Background(), mp.height+1, nil, nil, nil, nil, true)) +} + +func TestStopRechecksJoinsSupersededBatches(t *testing.T) { + entered := make(chan context.Context, 2) + releases := []chan struct{}{make(chan struct{}), make(chan struct{})} + unblockFirst := stdsync.OnceFunc(func() { close(releases[0]) }) + unblockSecond := stdsync.OnceFunc(func() { close(releases[1]) }) + defer unblockFirst() + defer unblockSecond() + var calls atomic.Int32 + client := recheckLifecycleClient{ + check: func(ctx context.Context) (*abci.ResponseCheckTx, error) { + call := calls.Add(1) + if call <= 2 { + entered <- ctx + <-releases[call-1] + } + return nil, context.Canceled + }, + flush: func(context.Context) error { return nil }, + } + mp := NewTxMempool(log.NewNopLogger(), config.DefaultMempoolConfig(), client) + require.NoError(t, mp.addNewTransaction(randomTx(), &abci.ResponseCheckTx{Code: abci.CodeTypeOK})) + startLifecycleRecheck(t, mp) + first := <-entered + startLifecycleRecheck(t, mp) + second := <-entered + require.ErrorIs(t, first.Err(), context.Canceled) + done := make(chan struct{}) + go func() { mp.StopRechecks(); close(done) }() + select { + case <-done: + t.Fatal("shutdown returned while application rechecks were blocked") + case <-second.Done(): + case <-time.After(time.Second): + t.Fatal("shutdown did not cancel the active recheck") + } + unblockSecond() + select { + case <-done: + t.Fatal("shutdown omitted the superseded recheck") + case <-time.After(20 * time.Millisecond): + } + unblockFirst() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("shutdown did not join completed rechecks") + } + startLifecycleRecheck(t, mp) + mp.StopRechecks() + require.EqualValues(t, 2, calls.Load(), "shutdown must reject subsequent recheck admission") +} + +func TestStopRechecksJoinsFlush(t *testing.T) { + entered := make(chan context.Context, 1) + release := make(chan struct{}) + unblock := stdsync.OnceFunc(func() { close(release) }) + defer unblock() + client := recheckLifecycleClient{ + check: func(context.Context) (*abci.ResponseCheckTx, error) { + return &abci.ResponseCheckTx{Code: abci.CodeTypeOK}, nil + }, + flush: func(ctx context.Context) error { entered <- ctx; <-release; return ctx.Err() }, + } + mp := NewTxMempool(log.NewNopLogger(), config.DefaultMempoolConfig(), client) + require.NoError(t, mp.addNewTransaction(randomTx(), &abci.ResponseCheckTx{Code: abci.CodeTypeOK})) + startLifecycleRecheck(t, mp) + ctx := <-entered + done := make(chan struct{}) + go func() { mp.StopRechecks(); close(done) }() + select { + case <-done: + t.Fatal("shutdown returned while application flush was blocked") + case <-ctx.Done(): + case <-time.After(time.Second): + t.Fatal("shutdown did not cancel application flush") + } + unblock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("shutdown did not join application flush") + } +} + +func TestStopRechecksRacesAdmission(t *testing.T) { + for range 32 { + var active atomic.Int32 + client := recheckLifecycleClient{ + check: func(ctx context.Context) (*abci.ResponseCheckTx, error) { + active.Add(1) + defer active.Add(-1) + <-ctx.Done() + return nil, ctx.Err() + }, + flush: func(context.Context) error { return nil }, + } + mp := NewTxMempool(log.NewNopLogger(), config.DefaultMempoolConfig(), client) + require.NoError(t, mp.addNewTransaction(randomTx(), &abci.ResponseCheckTx{Code: abci.CodeTypeOK})) + start := make(chan struct{}) + updated := make(chan error, 1) + stopped := make(chan struct{}) + go func() { + <-start + mp.Lock() + err := mp.Update(context.Background(), 1, nil, nil, nil, nil, true) + mp.Unlock() + updated <- err + }() + go func() { <-start; mp.StopRechecks(); close(stopped) }() + close(start) + require.NoError(t, <-updated) + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("shutdown raced recheck admission without canceling it") + } + require.Zero(t, active.Load(), "an admitted application call survived shutdown") + } +} diff --git a/internal/p2p/channel.go b/internal/p2p/channel.go index 75936ed9f3..c288a00e26 100644 --- a/internal/p2p/channel.go +++ b/internal/p2p/channel.go @@ -327,25 +327,7 @@ func (ch *legacyChannel) String() string { return fmt.Sprintf("p2p.Channel<%d:%s // Receive returns a new unbuffered iterator to receive messages from ch. // The iterator runs until ctx ends. func (ch *legacyChannel) Receive(ctx context.Context) ChannelIterator { - iter := &channelIterator{ - pipe: make(chan Envelope), // unbuffered - } - go func(pipe chan<- Envelope) { - defer close(iter.pipe) - for { - select { - case <-ctx.Done(): - return - case envelope := <-ch.inCh: - select { - case <-ctx.Done(): - return - case pipe <- envelope: - } - } - } - }(iter.pipe) - return iter + return &channelIterator{pipe: ch.inCh, done: ctx.Done()} } // ChannelIterator is an iterator for receiving messages from a Channel. @@ -370,7 +352,8 @@ type ChannelIterator interface { // MergedChannelIterator makes it possible to combine multiple // channels into a single iterator. type channelIterator struct { - pipe chan Envelope + pipe <-chan Envelope + done <-chan struct{} current *Envelope } @@ -381,6 +364,9 @@ func NewChannelIterator(pipe chan Envelope) ChannelIterator { func (iter *channelIterator) Next(ctx context.Context) bool { select { + case <-iter.done: + iter.current = nil + return false case <-ctx.Done(): iter.current = nil return false @@ -408,8 +394,9 @@ func (iter *channelIterator) Envelope() *Envelope { return iter.current } // This allows the caller to consume messages from multiple channels // without needing to manage the concurrency separately. func MergedChannelIterator(ctx context.Context, chs ...Channel) ChannelIterator { + pipe := make(chan Envelope) iter := &channelIterator{ - pipe: make(chan Envelope), // unbuffered + pipe: pipe, // unbuffered } wg := new(sync.WaitGroup) @@ -425,19 +412,16 @@ func MergedChannelIterator(ctx context.Context, chs ...Channel) ChannelIterator case pipe <- *iter.Envelope(): } } - }(ch, iter.pipe) + }(ch, pipe) } - done := make(chan struct{}) - go func() { defer close(done); wg.Wait() }() - go func() { - defer close(iter.pipe) + defer close(pipe) // we could return early if the context is canceled, // but this is safer because it means the pipe stays // open until all of the ch worker threads end, which // should happen very quickly. - <-done + wg.Wait() }() return iter diff --git a/internal/p2p/client/lifecycle_test.go b/internal/p2p/client/lifecycle_test.go new file mode 100644 index 0000000000..dc2d22272d --- /dev/null +++ b/internal/p2p/client/lifecycle_test.go @@ -0,0 +1,48 @@ +package client + +import ( + "context" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" +) + +type blockedGCClock struct { + clockwork.Clock + entered chan struct{} + release chan struct{} +} + +func (c *blockedGCClock) Now() time.Time { + close(c.entered) + <-c.release + return c.Clock.Now() +} + +func TestRateLimitCloseJoinsGC(t *testing.T) { + fake := clockwork.NewFakeClock() + clock := &blockedGCClock{Clock: fake, entered: make(chan struct{}), release: make(chan struct{})} + limiter := NewRateLimit(context.Background(), 1, true, log.NewNopLogger(), WithRateLimitClock(clock)) + t.Cleanup(limiter.Close) + defer close(clock.release) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, fake.BlockUntilContext(ctx, 1)) + fake.Advance(PeerRateLimitLifetime * time.Second) + select { + case <-clock.entered: + case <-time.After(time.Second): + t.Fatal("garbage collection did not start") + } + done := make(chan struct{}) + go func() { limiter.Close(); close(done) }() + select { + case <-done: + t.Fatal("Close returned before garbage collection finished") + case <-time.After(30 * time.Millisecond): + } +} diff --git a/internal/p2p/client/ratelimit.go b/internal/p2p/client/ratelimit.go index a09904d27b..58284d2481 100644 --- a/internal/p2p/client/ratelimit.go +++ b/internal/p2p/client/ratelimit.go @@ -35,6 +35,8 @@ type RateLimit struct { clock clockwork.Clock logger log.Logger + cancel context.CancelFunc + done chan struct{} } // RateLimitOptionFunc overrides a default parameter of a RateLimit. @@ -86,6 +88,7 @@ func NewRateLimitWithBurst( logger log.Logger, opts ...RateLimitOptionFunc, ) *RateLimit { + ctx, cancel := context.WithCancel(ctx) if limit > 0 && burst < 1 { burst = 1 } @@ -96,6 +99,8 @@ func NewRateLimitWithBurst( drop: drop, clock: clockwork.NewRealClock(), logger: logger, + cancel: cancel, + done: make(chan struct{}), } for _, opt := range opts { opt(h) @@ -107,6 +112,12 @@ func NewRateLimitWithBurst( return h } +// Close stops and joins the limiter's garbage collection worker. +func (h *RateLimit) Close() { + h.cancel() + <-h.done +} + func (h *RateLimit) getLimiter(peerID types.NodeID) *limiter { var limit *limiter if l, ok := h.limiters.Load(peerID); ok { @@ -203,6 +214,7 @@ func (h *RateLimit) waitN(ctx context.Context, l *limiter, nTokens int) error { // gcRoutine is a goroutine that removes unused limiters for peers every `PeerRateLimitLifetime` seconds. func (h *RateLimit) gcRoutine(ctx context.Context) { + defer close(h.done) ticker := h.clock.NewTicker(PeerRateLimitLifetime * time.Second) defer ticker.Stop() diff --git a/internal/p2p/conn/connection.go b/internal/p2p/conn/connection.go index 381b404d17..7319187c7a 100644 --- a/internal/p2p/conn/connection.go +++ b/internal/p2p/conn/connection.go @@ -92,14 +92,7 @@ type MConnection struct { errored uint32 config MConnConfig - // Closing quitSendRoutine will cause the sendRoutine to eventually quit. - // doneSendRoutine is closed when the sendRoutine actually quits. - quitSendRoutine chan struct{} - doneSendRoutine chan struct{} - - // Closing quitRecvRouting will cause the recvRouting to eventually quit. - // doneRecvRoutine is closed when the recvRoutine actually quits. - quitRecvRoutine chan struct{} + // doneRecvRoutine lets the owner collect malformed input after teardown. doneRecvRoutine chan struct{} // peerInputErr is a malformed authenticated frame, recorded before Stop. @@ -107,8 +100,6 @@ type MConnection struct { stopSignal <-chan struct{} - cancel context.CancelFunc - flushTimer *timer.ThrottleTimer // flush writes as necessary but throttled. pingTimer *time.Ticker // send pings periodically @@ -181,7 +172,6 @@ func NewMConnection( onError: onError, config: config, created: time.Now(), - cancel: func() {}, } mconn.BaseService = *service.NewBaseService(logger, "MConnection", mconn) @@ -216,14 +206,13 @@ func (c *MConnection) OnStart(ctx context.Context) error { c.flushTimer = timer.NewThrottleTimer("flush", c.config.FlushThrottle) c.pingTimer = time.NewTicker(c.config.PingInterval) c.chStatsTimer = time.NewTicker(updateStats) - c.quitSendRoutine = make(chan struct{}) - c.doneSendRoutine = make(chan struct{}) - c.quitRecvRoutine = make(chan struct{}) c.doneRecvRoutine = make(chan struct{}) c.stopSignal = ctx.Done() c.setRecvLastMsgAt(time.Now()) - go c.sendRoutine(ctx) - go c.recvRoutine(ctx) + c.Go(ctx, c.sendRoutine) + if !c.Go(ctx, c.recvRoutine) { + close(c.doneRecvRoutine) + } return nil } @@ -239,46 +228,12 @@ func (c *MConnection) getLastMessageAt() time.Time { return c.lastMsgRecv.at } -// stopServices stops the BaseService and timers and closes the quitSendRoutine. -// if the quitSendRoutine was already closed, it returns true, otherwise it returns false. -// It doesn't lock, as we rely on the caller (eg. BaseService) locking -func (c *MConnection) stopServices() (alreadyStopped bool) { - select { - case <-c.quitSendRoutine: - // already quit - return true - default: - } - - select { - case <-c.quitRecvRoutine: - // already quit - return true - default: - } - +// OnStop closes the socket to interrupt pending reads and writes. +func (c *MConnection) OnStop() { c.flushTimer.Stop() c.pingTimer.Stop() c.chStatsTimer.Stop() - - // inform the recvRouting that we are shutting down - close(c.quitRecvRoutine) - close(c.quitSendRoutine) - return false -} - -// OnStop implements BaseService -func (c *MConnection) OnStop() { - if c.stopServices() { - return - } - c.conn.Close() - - // We can't close pong safely here because - // recvRoutine may write to it after we've stopped. - // Though it doesn't need to get closed at all, - // we close it @ recvRoutine. } func (c *MConnection) String() string { @@ -387,8 +342,6 @@ FOR_LOOP: c.flush() case <-ctx.Done(): break FOR_LOOP - case <-c.quitSendRoutine: - break FOR_LOOP case <-pongTimeout.C: // the point of the pong timer is to check to // see if we've seen a message recently, so we @@ -418,13 +371,10 @@ FOR_LOOP: c.stopForError(ctx, err) break FOR_LOOP } - if !c.IsRunning() { + if ctx.Err() != nil { break FOR_LOOP } } - - // Cleanup - close(c.doneSendRoutine) } // Returns true if messages from channels were exhausted. @@ -496,8 +446,6 @@ FOR_LOOP: select { case <-ctx.Done(): break FOR_LOOP - case <-c.doneSendRoutine: - break FOR_LOOP default: } @@ -528,16 +476,11 @@ FOR_LOOP: // Before Stop makes teardown observable; see PeerInputError. c.peerInputErr.Store(&err) } - // stopServices was invoked and we are shutting down - // receiving is excpected to fail since we will close the connection - select { - case <-ctx.Done(): - case <-c.quitRecvRoutine: + if ctx.Err() != nil { break FOR_LOOP - default: } - if c.IsRunning() { + if ctx.Err() == nil { if err == io.EOF { c.logger.Info("Connection is closed @ recvRoutine (likely by the other side)", "conn", c) } else { @@ -587,7 +530,7 @@ FOR_LOOP: msgBytes, err := channel.recvPacketMsg(*pkt.PacketMsg) if err != nil { - if c.IsRunning() { + if ctx.Err() == nil { c.logger.Debug("Connection failed @ recvRoutine", "conn", c, "err", err) c.stopForError(ctx, err) } @@ -606,11 +549,6 @@ FOR_LOOP: } } - // Cleanup - close(c.pong) - for range c.pong { - // Drain - } } // maxPacketMsgSize returns a maximum size of PacketMsg diff --git a/internal/p2p/conn/connection_test.go b/internal/p2p/conn/connection_test.go index 2ce2fda932..82e97ccf67 100644 --- a/internal/p2p/conn/connection_test.go +++ b/internal/p2p/conn/connection_test.go @@ -193,9 +193,10 @@ func TestMConnectionWillEventuallyTimeout(t *testing.T) { } }() - // wait for the send routine to die because it doesn't + done := make(chan struct{}) + go func() { mconn.Wait(); close(done) }() select { - case <-mconn.doneSendRoutine: + case <-done: require.True(t, time.Since(mconn.getLastMessageAt()) > mconn.config.PongTimeout, "the connection state reflects that we've passed the pong timeout") // since we hit the timeout, things should be shutdown diff --git a/internal/p2p/conn/lifecycle_test.go b/internal/p2p/conn/lifecycle_test.go index 4411924758..be6d4fe038 100644 --- a/internal/p2p/conn/lifecycle_test.go +++ b/internal/p2p/conn/lifecycle_test.go @@ -11,6 +11,32 @@ import ( "github.com/dashpay/tenderdash/libs/log" ) +func TestMConnectionWaitDrainsCallback(t *testing.T) { + local, remote := net.Pipe() + entered, release := make(chan struct{}), make(chan struct{}) + receiver := createMConnectionWithCallbacks(log.NewNopLogger(), local, + func(context.Context, ChannelID, []byte) { close(entered); <-release }, nil) + sender := createTestMConnection(log.NewNopLogger(), remote) + require.NoError(t, receiver.Start(context.Background())) + require.NoError(t, sender.Start(context.Background())) + t.Cleanup(func() { sender.Stop(); sender.Wait(); receiver.Stop(); receiver.Wait() }) + defer close(release) + require.True(t, sender.Send(1, []byte("drain"))) + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("receive callback did not start") + } + receiver.Stop() + done := make(chan struct{}) + go func() { receiver.Wait(); close(done) }() + select { + case <-done: + t.Fatal("Wait returned while the receive callback was still running") + case <-time.After(30 * time.Millisecond): + } +} + func TestMConnectionErrorCallbackRetainsParentContext(t *testing.T) { local, remote := net.Pipe() t.Cleanup(func() { _ = local.Close(); _ = remote.Close() }) @@ -37,4 +63,32 @@ func TestMConnectionErrorCallbackRetainsParentContext(t *testing.T) { case <-time.After(time.Second): t.Fatal("error callback did not run") } + done := make(chan struct{}) + go func() { connection.Wait(); close(done) }() + select { + case <-done: + t.Fatal("Wait returned before the error callback finished") + case <-time.After(30 * time.Millisecond): + } +} + +func TestMConnectionPeerInputErrorAfterCanceledStart(t *testing.T) { + local, remote := net.Pipe() + t.Cleanup(func() { _ = local.Close(); _ = remote.Close() }) + connection := createTestMConnection(log.NewNopLogger(), local) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.NoError(t, connection.Start(ctx)) + t.Cleanup(func() { connection.Stop(); connection.Wait() }) + + done := make(chan error, 1) + go func() { done <- connection.PeerInputError() }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(time.Second): + close(connection.doneRecvRoutine) + require.NoError(t, <-done) + t.Fatal("PeerInputError did not return after startup canceled the receive worker") + } } diff --git a/internal/p2p/lifecycle_test.go b/internal/p2p/lifecycle_test.go new file mode 100644 index 0000000000..7ad62f3461 --- /dev/null +++ b/internal/p2p/lifecycle_test.go @@ -0,0 +1,163 @@ +package p2p + +import ( + "context" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" + + tmsync "github.com/dashpay/tenderdash/internal/libs/sync" + "github.com/dashpay/tenderdash/internal/p2p/conn" + dbm "github.com/dashpay/tenderdash/libs/db" + "github.com/dashpay/tenderdash/libs/log" + p2pproto "github.com/dashpay/tenderdash/proto/tendermint/p2p" + "github.com/dashpay/tenderdash/types" +) + +func TestRouterWaitDrainsAcceptedConnection(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db := dbm.NewMemDB() + t.Cleanup(func() { require.NoError(t, db.Close()) }) + manager, err := NewPeerManager(ctx, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", db, PeerManagerOptions{}) + require.NoError(t, err) + transport := NewMConnTransport(log.NewNopLogger(), conn.DefaultMConnConfig(), nil, MConnTransportOptions{}) + entered, release := make(chan struct{}), make(chan struct{}) + router, err := NewRouter(log.NewNopLogger(), NopMetrics(), nil, manager, + func() *types.NodeInfo { return &types.NodeInfo{} }, transport, + &Endpoint{Protocol: MConnProtocol, IP: net.IPv4(127, 0, 0, 1)}, RouterOptions{ + FilterPeerByIP: func(ctx context.Context, _ net.IP, _ uint16) error { + close(entered) + <-ctx.Done() + <-release + return ctx.Err() + }, + }) + require.NoError(t, err) + require.NoError(t, router.Start(ctx)) + t.Cleanup(func() { router.Stop(); router.Wait() }) + defer close(release) + remote, err := net.Dial("tcp", transport.listener.Addr().String()) + require.NoError(t, err) + defer remote.Close() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("connection filter did not start") + } + router.Stop() + done := make(chan struct{}) + go func() { router.Wait(); close(done) }() + select { + case <-done: + t.Fatal("Wait returned while an accepted connection was still being filtered") + case <-time.After(30 * time.Millisecond): + } +} + +func TestSimpleQueueClosedDrainsDeliveryCallback(t *testing.T) { + q := newSimplePriorityQueue(context.Background(), 0) + entered, release := make(chan struct{}), make(chan struct{}) + defer func() { close(release); q.close(); <-q.closed() }() + envelope := Envelope{Message: &p2pproto.PexRequest{}} + envelope.EnableDeliveryNotification() + envelope.delivery.setOnCompleted(func() { close(entered); <-release }) + q.enqueue() <- envelope + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("queue did not drop the envelope") + } + q.close() + select { + case <-q.closed(): + t.Fatal("queue reported completion before delivery callback returned") + case <-time.After(30 * time.Millisecond): + } +} + +func TestPeerUpdatesCloseJoinsUnsubscribe(t *testing.T) { + db := dbm.NewMemDB() + defer db.Close() + manager, err := NewPeerManager(context.Background(), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", db, PeerManagerOptions{}) + require.NoError(t, err) + updates := manager.Subscribe(context.Background(), "lifecycle") + manager.mtx.Lock() + done := make(chan struct{}) + go func() { updates.Close(); close(done) }() + select { + case <-done: + manager.mtx.Unlock() + t.Fatal("Close returned before unsubscribing") + case <-time.After(30 * time.Millisecond): + } + manager.mtx.Unlock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("subscription did not finish cleanup") + } + manager.mtx.Lock() + require.Empty(t, manager.subscriptions) + manager.mtx.Unlock() + updates.Close() +} + +type delayedAcceptListener struct { + net.Listener + entered chan struct{} + release chan struct{} +} + +func (l *delayedAcceptListener) Accept() (net.Conn, error) { + close(l.entered) + <-l.release + return nil, net.ErrClosed +} + +func (*delayedAcceptListener) Close() error { return nil } + +func TestMConnTransportAcceptJoinsRequest(t *testing.T) { + listener := &delayedAcceptListener{entered: make(chan struct{}), release: make(chan struct{})} + transport := NewMConnTransport(log.NewNopLogger(), conn.DefaultMConnConfig(), nil, MConnTransportOptions{}) + transport.listener = listener + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { _, _ = transport.Accept(ctx); close(done) }() + defer func() { close(listener.release); <-done }() + <-listener.entered + cancel() + select { + case <-done: + t.Fatal("Accept returned before its request worker finished") + case <-time.After(30 * time.Millisecond): + } +} + +func TestRouterManualStopClosesIndependentChannel(t *testing.T) { + db := dbm.NewMemDB() + defer db.Close() + manager, err := NewPeerManager(context.Background(), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", db, PeerManagerOptions{}) + require.NoError(t, err) + transport := NewMConnTransport(log.NewNopLogger(), conn.DefaultMConnConfig(), nil, MConnTransportOptions{}) + info := &types.NodeInfo{Channels: tmsync.NewConcurrentSlice[uint16]()} + router, err := NewRouter(log.NewNopLogger(), NopMetrics(), nil, manager, + func() *types.NodeInfo { return info }, transport, + &Endpoint{Protocol: MConnProtocol, IP: net.IPv4(127, 0, 0, 1)}, RouterOptions{}) + require.NoError(t, err) + require.NoError(t, router.Start(context.Background())) + t.Cleanup(func() { router.Stop(); router.Wait() }) + _, err = router.OpenChannel(context.Background(), &ChannelDescriptor{ID: 1, Priority: 1}) + require.NoError(t, err) + router.Stop() + router.Wait() + router.channelMtx.RLock() + count := len(router.channelQueues) + router.channelMtx.RUnlock() + require.Zero(t, count, "Wait must drain channels even when their caller context remains active") + _, err = router.OpenChannel(context.Background(), &ChannelDescriptor{ID: 2, Priority: 1}) + require.Error(t, err) +} diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index 6549a3fb85..b46f83bb15 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -100,6 +100,9 @@ func (pu *PeerUpdate) SetProTxHash(proTxHash types.ProTxHash) { // PeerUpdates is a peer update subscription with notifications about peer // events (currently just status changes). type PeerUpdates struct { + lifecycleMtx sync.Mutex + cancel context.CancelFunc + done chan struct{} routerUpdatesCh chan PeerUpdate reactorUpdatesCh chan PeerUpdate // subscriberName is a label used for debugging @@ -122,6 +125,27 @@ func (pu *PeerUpdates) Updates() <-chan PeerUpdate { return pu.reactorUpdatesCh } +// Close cancels and joins the registered subscription worker. +func (pu *PeerUpdates) Close() { + pu.lifecycleMtx.Lock() + cancel := pu.cancel + pu.lifecycleMtx.Unlock() + if cancel != nil { + cancel() + pu.Wait() + } +} + +// Wait joins the registered subscription worker after its context is canceled. +func (pu *PeerUpdates) Wait() { + pu.lifecycleMtx.Lock() + done := pu.done + pu.lifecycleMtx.Unlock() + if done != nil { + <-done + } +} + // SendUpdate pushes information about a peer into the routing layer, // presumably from a peer. func (pu *PeerUpdates) SendUpdate(ctx context.Context, update PeerUpdate) { @@ -1449,9 +1473,17 @@ func (m *PeerManager) Unsubscribe(sub *PeerUpdates) { func (m *PeerManager) Register(ctx context.Context, peerUpdates *PeerUpdates) { m.mtx.Lock() defer m.mtx.Unlock() + peerUpdates.lifecycleMtx.Lock() + defer peerUpdates.lifecycleMtx.Unlock() + if peerUpdates.done != nil { + return + } + ctx, peerUpdates.cancel = context.WithCancel(ctx) + peerUpdates.done = make(chan struct{}) m.subscriptions[peerUpdates] = peerUpdates go func() { + defer close(peerUpdates.done) defer m.Unsubscribe(peerUpdates) for { select { diff --git a/internal/p2p/pex/lifecycle_test.go b/internal/p2p/pex/lifecycle_test.go new file mode 100644 index 0000000000..4c999b3e88 --- /dev/null +++ b/internal/p2p/pex/lifecycle_test.go @@ -0,0 +1,50 @@ +package pex + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/internal/p2p" + "github.com/dashpay/tenderdash/libs/log" +) + +type blockedReceiveChannel struct { + p2p.Channel + entered chan struct{} + release chan struct{} +} + +func (ch *blockedReceiveChannel) Receive(ctx context.Context) p2p.ChannelIterator { + close(ch.entered) + <-ctx.Done() + <-ch.release + return p2p.NewChannelIterator(make(chan p2p.Envelope)) +} + +func TestReactorWaitDrainsChannelReader(t *testing.T) { + channel := &blockedReceiveChannel{entered: make(chan struct{}), release: make(chan struct{})} + r := NewReactor(log.NewNopLogger(), nil, + func(context.Context, *p2p.ChannelDescriptor) (p2p.Channel, error) { return channel, nil }, + func(context.Context, string) *p2p.PeerUpdates { + return p2p.NewPeerUpdates(make(chan p2p.PeerUpdate), 1, "test") + }) + require.NoError(t, r.Start(context.Background())) + t.Cleanup(func() { r.Stop(); r.Wait() }) + defer close(channel.release) + select { + case <-channel.entered: + case <-time.After(time.Second): + t.Fatal("channel reader did not start") + } + r.Stop() + done := make(chan struct{}) + go func() { r.Wait(); close(done) }() + select { + case <-done: + t.Fatal("Wait returned before the channel reader finished") + case <-time.After(30 * time.Millisecond): + } +} diff --git a/internal/p2p/pex/reactor.go b/internal/p2p/pex/reactor.go index 451bd41e70..ed7660034e 100644 --- a/internal/p2p/pex/reactor.go +++ b/internal/p2p/pex/reactor.go @@ -83,6 +83,7 @@ type Reactor struct { peerManager *p2p.PeerManager chCreator p2p.ChannelCreator peerEvents p2p.PeerEventSubscriber + peerUpdates *p2p.PeerUpdates // list of available peers to loop through and send peer requests to availablePeers map[types.NodeID]struct{} @@ -135,20 +136,27 @@ func (r *Reactor) OnStart(ctx context.Context) error { } peerUpdates := r.peerEvents(ctx, "pex") - go r.processPexCh(ctx, channel) - go r.processPeerUpdates(ctx, peerUpdates) + r.peerUpdates = peerUpdates + r.Go(ctx, func(ctx context.Context) { r.processPexCh(ctx, channel) }) + r.Go(ctx, func(ctx context.Context) { r.processPeerUpdates(ctx, peerUpdates) }) return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit. +// OnStop implements service.Implementation. func (r *Reactor) OnStop() {} +// OnDrain joins the subscription after its consumers have stopped. +func (r *Reactor) OnDrain() { r.peerUpdates.Close() } + // processPexCh implements a blocking event loop where we listen for p2p // Envelope messages from the pexCh. func (r *Reactor) processPexCh(ctx context.Context, pexCh p2p.Channel) { + ctx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + defer func() { cancel(); <-done }() incoming := make(chan *p2p.Envelope) go func() { + defer close(done) defer close(incoming) iter := pexCh.Receive(ctx) for iter.Next(ctx) { diff --git a/internal/p2p/router.go b/internal/p2p/router.go index 6b588482fa..88057906c0 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -173,6 +173,7 @@ type Router struct { // channels on router start. This depends on whether we want to allow // dynamic channels in the future. channelMtx sync.RWMutex + ctx context.Context channelQueues map[ChannelID]queue // inbound messages from all peers to a single channel } @@ -261,8 +262,20 @@ type ChannelCreator func(context.Context, *ChannelDescriptor) (Channel, error) // wrapper message. The caller may provide a size to make the channel buffered, // which internally makes the inbound, outbound, and error channel buffered. func (r *Router) OpenChannel(ctx context.Context, chDesc *ChannelDescriptor) (Channel, error) { + var abandoned queue + // Registered before the unlock, so it runs after channelMtx is released: + // closing a queue joins its delivery callbacks. + defer func() { + if abandoned != nil { + abandoned.close() + <-abandoned.closed() + } + }() r.channelMtx.Lock() defer r.channelMtx.Unlock() + if r.ctx == nil || r.ctx.Err() != nil { + return nil, errors.New("router is not running") + } id := chDesc.ID if _, ok := r.channelQueues[id]; ok { @@ -282,16 +295,27 @@ func (r *Router) OpenChannel(ctx context.Context, chDesc *ChannelDescriptor) (Ch r.transport.AddChannelDescriptors([]*ChannelDescriptor{chDesc}) - go func() { + channelCtx, cancel := context.WithCancel(ctx) + stopCancel := context.AfterFunc(r.ctx, cancel) + if !r.Go(r.ctx, func(context.Context) { + defer stopCancel() + defer cancel() defer func() { r.channelMtx.Lock() delete(r.channelQueues, id) r.channelMtx.Unlock() queue.close() + <-queue.closed() }() - r.routeChannel(ctx, chDesc.ID, outCh, errCh) - }() + r.routeChannel(channelCtx, chDesc.ID, outCh, errCh) + }) { + stopCancel() + cancel() + delete(r.channelQueues, id) + abandoned = queue + return nil, errors.New("router is stopping") + } return channel, nil } @@ -477,13 +501,16 @@ func (r *Router) acceptPeers(ctx context.Context, transport Transport) { } // Spawn a goroutine for the handshake, to avoid head-of-line blocking. - go r.openConnection(ctx, conn) + if !r.Go(ctx, func(ctx context.Context) { r.openConnection(ctx, conn) }) { + r.closeConnection(conn) + r.connTracker.RemoveConn(incomingIP) + } } } func (r *Router) openConnection(ctx context.Context, conn Connection) { - defer conn.Close() + defer r.closeConnection(conn) defer r.connTracker.RemoveConn(conn.RemoteEndpoint().IP) re := conn.RemoteEndpoint() @@ -597,14 +624,14 @@ func (r *Router) connectPeer(ctx context.Context, address NodeAddress) { peerInfo, err := r.handshakePeer(ctx, conn, address.NodeID) switch { case errors.Is(err, context.Canceled): - conn.Close() + r.closeConnection(conn) return case err != nil: r.logger.Error("failed to handshake with peer", "peer", address, "err", err) if err = r.peerManager.DialFailed(ctx, address); err != nil { r.logger.Error("failed to report dial failure", "peer", address, "err", err) } - conn.Close() + r.closeConnection(conn) return } @@ -612,12 +639,25 @@ func (r *Router) connectPeer(ctx context.Context, address NodeAddress) { if err != nil { r.logger.Error("failed to dial peer", "op", "outgoing/dialing", "peer", address.NodeID, "err", err) r.peerManager.dialWaker.Wake() - conn.Close() + r.closeConnection(conn) return } // routePeer (also) calls connection close - go r.routePeer(ctx, address.NodeID, conn, toChannelIDs(peerInfo.Channels.ToSlice())) + if !r.Go(ctx, func(ctx context.Context) { + r.routePeer(ctx, address.NodeID, conn, toChannelIDs(peerInfo.Channels.ToSlice())) + }) { + r.closeConnection(conn) + r.peerManager.Disconnected(ctx, address.NodeID) + } +} + +// closeConnection joins transport workers from their owner, never from a callback. +func (r *Router) closeConnection(conn Connection) { + _ = conn.Close() + if waiter, ok := conn.(interface{ Wait() }); ok { + waiter.Wait() + } } func (r *Router) getOrMakeQueue(peerID types.NodeID, channels ChannelIDSet) queue { @@ -747,8 +787,9 @@ func (r *Router) routePeer(ctx context.Context, peerID types.NodeID, conn Connec r.peerMtx.Unlock() r.completePeerDeliveries(peerID) - _ = conn.Close() + r.closeConnection(conn) sendQueue.close() + <-sendQueue.closed() r.peerManager.Disconnected(ctx, peerID) r.metrics.PeersConnected.Add(-1) @@ -1104,6 +1145,9 @@ func (r *Router) setupQueueFactory(ctx context.Context) error { // OnStart implements service.Service. func (r *Router) OnStart(ctx context.Context) error { + r.channelMtx.Lock() + r.ctx = ctx + r.channelMtx.Unlock() if err := r.setupQueueFactory(ctx); err != nil { return err } @@ -1112,9 +1156,9 @@ func (r *Router) OnStart(ctx context.Context) error { return err } - go r.dialPeers(ctx) - go r.evictPeers(ctx) - go r.acceptPeers(ctx, r.transport) + r.Go(ctx, r.dialPeers) + r.Go(ctx, r.evictPeers) + r.Go(ctx, func(ctx context.Context) { r.acceptPeers(ctx, r.transport) }) return nil } @@ -1131,7 +1175,7 @@ func (r *Router) OnStop() { r.logger.Error("failed to close transport", "err", err) } - // Collect all remaining queues, and wait for them to close. + // Queue owners join their workers after cancellation. queues := []queue{} r.channelMtx.RLock() @@ -1148,7 +1192,6 @@ func (r *Router) OnStop() { for _, q := range queues { q.close() - <-q.closed() } } diff --git a/internal/p2p/rqueue.go b/internal/p2p/rqueue.go index 23b6a1b261..c6a893145a 100644 --- a/internal/p2p/rqueue.go +++ b/internal/p2p/rqueue.go @@ -13,7 +13,7 @@ type simpleQueue struct { input chan Envelope output chan Envelope closeFn func() - closeCh <-chan struct{} + closeCh chan struct{} maxSize int chDescs []*ChannelDescriptor @@ -29,7 +29,7 @@ func newSimplePriorityQueue(ctx context.Context, size int) *simpleQueue { input: make(chan Envelope, size*2), output: make(chan Envelope, size/2), maxSize: size * size, - closeCh: ctx.Done(), + closeCh: make(chan struct{}), closeFn: cancel, } @@ -43,6 +43,7 @@ func (q *simpleQueue) close() { q.closeFn() } func (q *simpleQueue) closed() <-chan struct{} { return q.closeCh } func (q *simpleQueue) run(ctx context.Context) { + defer close(q.closeCh) defer q.closeFn() var chPriorities = make(map[ChannelID]uint, len(q.chDescs)) @@ -54,6 +55,7 @@ func (q *simpleQueue) run(ctx context.Context) { pq := make(priorityQueue, 0, q.maxSize) heap.Init(&pq) ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() // must have a buffer of exactly one because both sides of // this channel are used in this loop, and simply signals adds // to the heap @@ -62,8 +64,6 @@ func (q *simpleQueue) run(ctx context.Context) { select { case <-ctx.Done(): return - case <-q.closeCh: - return case e := <-q.input: // enqueue the incoming Envelope heap.Push(&pq, &pqEnvelope{ @@ -105,8 +105,6 @@ func (q *simpleQueue) run(ctx context.Context) { select { case <-ctx.Done(): return - case <-q.closeCh: - return case q.output <- next: heap.Pop(&pq) continue SEND diff --git a/internal/p2p/transport_mconn.go b/internal/p2p/transport_mconn.go index a5b9ef4a0e..5c6b7296db 100644 --- a/internal/p2p/transport_mconn.go +++ b/internal/p2p/transport_mconn.go @@ -138,7 +138,10 @@ func (m *MConnTransport) Accept(ctx context.Context) (Connection, error) { conCh := make(chan net.Conn) errCh := make(chan error) + done := make(chan struct{}) + defer func() { <-done }() go func() { + defer close(done) tcpConn, err := m.listener.Accept() if err != nil { select { @@ -147,12 +150,15 @@ func (m *MConnTransport) Accept(ctx context.Context) (Connection, error) { m.logger.Trace("MConnTransport Accept: connection closed - doneCh") case <-ctx.Done(): } + return } select { case conCh <- tcpConn: case <-m.doneCh: + _ = tcpConn.Close() m.logger.Trace("MConnTransport Accept: connection closed - doneCh") case <-ctx.Done(): + _ = tcpConn.Close() } }() @@ -273,7 +279,8 @@ type mConnConnection struct { doneCh chan struct{} closeOnce sync.Once - mconn *conn.MConnection // set during Handshake() + mconnMtx sync.Mutex + mconn *conn.MConnection // set during Handshake() // invalidData is the peer-attributable failure that tore the connection // down, kept so ReceiveMessage reports it even after doneCh closes. @@ -326,7 +333,10 @@ func (c *mConnConnection) Handshake( // To handle context cancellation, we need to do the handshake in a // goroutine and abort the blocking network calls by closing the connection // when the context is canceled. + done := make(chan struct{}) + defer func() { <-done }() go func() { + defer close(done) // FIXME: Since the MConnection code panics, we need to recover it and turn it // into an error. We should remove panics instead. defer func() { @@ -337,11 +347,7 @@ func (c *mConnConnection) Handshake( var err error mconn, peerInfo, peerKey, err = c.handshake(handshakeCtx, nodeInfo, privKey) - select { - case errCh <- err: - case <-c.doneCh: - case <-handshakeCtx.Done(): - } + errCh <- err }() @@ -355,6 +361,13 @@ func (c *mConnConnection) Handshake( if err != nil { return types.NodeInfo{}, nil, err } + c.mconnMtx.Lock() + defer c.mconnMtx.Unlock() + select { + case <-c.doneCh: + return types.NodeInfo{}, nil, io.EOF + default: + } c.mconn = mconn // Start must not use the handshakeCtx. The handshakeCtx may have a // timeout set that is intended to terminate only the handshake procedure. @@ -391,28 +404,22 @@ func (c *mConnConnection) handshake( go func() { defer wg.Done() _, err := protoio.NewDelimitedWriter(secretConn).WriteMsg(nodeInfo.ToProto()) - select { - case errCh <- err: - case <-c.doneCh: - case <-ctx.Done(): - } + errCh <- err }() wg.Add(1) go func() { defer wg.Done() _, err := protoio.NewDelimitedReader(secretConn, types.MaxNodeInfoSize()).ReadMsg(&pbPeerInfo) - select { - case errCh <- err: - case <-c.doneCh: - case <-ctx.Done(): - } + errCh <- err }() wg.Wait() - - if err, ok := <-errCh; ok && err != nil { - return nil, types.NodeInfo{}, nil, err + close(errCh) + for err := range errCh { + if err != nil { + return nil, types.NodeInfo{}, nil, err + } } if err := ctx.Err(); err != nil { @@ -576,14 +583,28 @@ func (c *mConnConnection) RemoteEndpoint() Endpoint { func (c *mConnConnection) Close() error { var err error c.closeOnce.Do(func() { + c.mconnMtx.Lock() + defer c.mconnMtx.Unlock() c.logger.Trace("mConnConnection.Close(): closing doneCh") defer close(c.doneCh) - if c.mconn != nil && c.mconn.IsRunning() { + if c.mconn != nil { c.mconn.Stop() - } else { - err = c.conn.Close() + } + err = c.conn.Close() + if errors.Is(err, net.ErrClosed) { + err = nil } }) return err } + +// Wait joins the protocol workers after Close. Call it from the connection owner. +func (c *mConnConnection) Wait() { + c.mconnMtx.Lock() + mconn := c.mconn + c.mconnMtx.Unlock() + if mconn != nil { + mconn.Wait() + } +} diff --git a/internal/proxy/client.go b/internal/proxy/client.go index d1e399ef53..cabbff9590 100644 --- a/internal/proxy/client.go +++ b/internal/proxy/client.go @@ -96,18 +96,12 @@ func tryCallStop(client abciclient.Client) { } func (app *proxyClient) OnStart(ctx context.Context) error { - var err error - defer func() { - if err != nil { - tryCallStop(app.client) - } - }() + if err := app.client.Start(ctx); err != nil { + return err + } // Kill Tendermint if the ABCI application crashes. - go func() { - if !app.client.IsRunning() { - return - } + if !app.Go(ctx, func(ctx context.Context) { app.client.Wait() if ctx.Err() != nil { return @@ -123,9 +117,12 @@ func (app *proxyClient) OnStart(ctx context.Context) error { } } - }() + }) { + tryCallStop(app.client) + app.client.Wait() + } - return app.client.Start(ctx) + return nil } func kill() error { diff --git a/internal/proxy/client_test.go b/internal/proxy/client_test.go index 8821ef7ea5..e4a04afaea 100644 --- a/internal/proxy/client_test.go +++ b/internal/proxy/client_test.go @@ -198,7 +198,6 @@ func TestAppConns_Start_Stop(t *testing.T) { clientMock := &abcimocks.Client{} clientMock.On("Start", mock.Anything).Return(nil) clientMock.On("Error").Return(nil) - clientMock.On("IsRunning").Return(true) clientMock.On("Wait").Return(nil).Times(1) cl := &noopStoppableClientImpl{Client: clientMock} @@ -227,7 +226,6 @@ func TestAppConns_Failure(t *testing.T) { clientMock := &abcimocks.Client{} clientMock.On("SetLogger", mock.Anything).Return() clientMock.On("Start", mock.Anything).Return(nil) - clientMock.On("IsRunning").Return(true) clientMock.On("Wait").Return(nil) clientMock.On("Error").Return(errors.New("EOF")) cl := &noopStoppableClientImpl{Client: clientMock} diff --git a/internal/pubsub/pubsub.go b/internal/pubsub/pubsub.go index 14dc70e694..2e87c4939b 100644 --- a/internal/pubsub/pubsub.go +++ b/internal/pubsub/pubsub.go @@ -93,10 +93,9 @@ type Server struct { service.BaseService logger log.Logger - queue chan item - done <-chan struct{} // closed when server should exit - pubs sync.RWMutex // excl: shutdown; shared: active publisher - exited chan struct{} // server exited + queue chan item + done <-chan struct{} // closed when server should exit + pubs sync.RWMutex // excl: shutdown; shared: active publisher // All subscriptions currently known. // Lock exclusive to add, remove, or cancel subscriptions. @@ -191,6 +190,13 @@ func (s *Server) Observe(_ctx context.Context, observe func(Message) error, quer return nil } +// RemoveObserver detaches the observer after any active invocation finishes. +func (s *Server) RemoveObserver() { + s.subs.Lock() + defer s.subs.Unlock() + s.subs.observe = nil +} + // SubscribeWithArgs creates a subscription for the given arguments. It is an // error if the query is nil, a subscription already exists for the specified // client ID and query, or if the capacity arguments are invalid. @@ -302,9 +308,22 @@ func (s *Server) PublishWithEvents(msg types.EventData, events []abci.Event) err // OnStop implements part of the Service interface. It is a no-op. func (s *Server) OnStop() {} -// Wait implements Service.Wait by blocking until the server has exited, then -// yielding to the base service wait. -func (s *Server) Wait() { <-s.exited; s.BaseService.Wait() } +// OnDrain releases subscriptions even when shutdown rejected sender startup. +func (s *Server) OnDrain() { + s.pubs.Lock() + if s.queue != nil { + close(s.queue) + s.queue = nil + } + s.pubs.Unlock() + s.subs.Lock() + defer s.subs.Unlock() + for si := range s.subs.index.all { + si.sub.stop(ErrTerminated) + } + s.subs.index = nil + s.subs.observe = nil +} // OnStart implements Service.OnStart by starting the server. func (s *Server) OnStart(ctx context.Context) error { s.run(ctx); return nil } @@ -331,17 +350,15 @@ func (s *Server) run(ctx context.Context) { // Shutdown monitor: When the context ends, wait for any active publish // calls to exit, then close the queue to signal the sender to exit. - go func() { + s.Go(ctx, func(ctx context.Context) { <-ctx.Done() s.pubs.Lock() defer s.pubs.Unlock() close(s.queue) s.queue = nil - }() + }) - s.exited = make(chan struct{}) - go func() { - defer close(s.exited) + s.Go(ctx, func(context.Context) { // Sender: Service the queue and forward messages to subscribers. for it := range queue { @@ -349,14 +366,7 @@ func (s *Server) run(ctx context.Context) { s.logger.Error("error sending event", "err", err) } } - // Terminate all subscribers before exit. - s.subs.Lock() - defer s.subs.Unlock() - for si := range s.subs.index.all { - si.sub.stop(ErrTerminated) - } - s.subs.index = nil - }() + }) } // removeSubs cancels and removes all the subscriptions in evict with the given diff --git a/internal/rpc/core/env.go b/internal/rpc/core/env.go index 290034e122..c7a280ec99 100644 --- a/internal/rpc/core/env.go +++ b/internal/rpc/core/env.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "net/http" + "sync" "time" "github.com/rs/cors" @@ -28,6 +29,7 @@ import ( "github.com/dashpay/tenderdash/internal/state/indexer" "github.com/dashpay/tenderdash/internal/statesync" "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" "github.com/dashpay/tenderdash/rpc/coretypes" rpcserver "github.com/dashpay/tenderdash/rpc/jsonrpc/server" "github.com/dashpay/tenderdash/types" @@ -91,6 +93,8 @@ type Environment struct { Config config.RPCConfig asyncBroadcasts asyncBroadcasts + rpcMu sync.Mutex + rpcService *rpcService // cache of chunked genesis data. genChunks []string @@ -227,6 +231,62 @@ func ListenRPC(conf *config.RPCConfig) ([]net.Listener, error) { // StartService serves RPC on already bound listeners until ctx is canceled. // If startup fails, the caller remains responsible for closing the listeners. func (env *Environment) StartService(ctx context.Context, conf *config.Config, listeners []net.Listener) error { + env.rpcMu.Lock() + if env.rpcService != nil { + env.rpcMu.Unlock() + return errors.New("RPC service already started") + } + ctx, cancel := context.WithCancel(ctx) + owner := &rpcService{env: env, conf: conf, listeners: listeners, cancel: cancel, ready: make(chan struct{})} + owner.BaseService = service.NewBaseService(env.Logger, "RPC", owner) + env.rpcService = owner + env.rpcMu.Unlock() + err := owner.Start(ctx) + close(owner.ready) + if err != nil { + cancel() + env.rpcMu.Lock() + if env.rpcService == owner { + env.rpcService = nil + } + env.rpcMu.Unlock() + } + return err +} + +// StopService cancels and joins RPC requests, websocket sessions, and event-log +// forwarding. Call it before releasing the dependencies used by RPC handlers. +func (env *Environment) StopService() { + env.rpcMu.Lock() + owner := env.rpcService + env.rpcMu.Unlock() + if owner == nil { + return + } + owner.cancel() + <-owner.ready + owner.Stop() + owner.Wait() + env.rpcMu.Lock() + if env.rpcService == owner { + env.rpcService = nil + } + env.rpcMu.Unlock() +} + +type rpcService struct { + *service.BaseService + env *Environment + conf *config.Config + listeners []net.Listener + cancel context.CancelFunc + ready chan struct{} +} + +func (*rpcService) OnStop() {} + +func (owner *rpcService) OnStart(ctx context.Context) error { + env, conf, listeners := owner.env, owner.conf, owner.listeners if err := env.InitGenesisChunks(); err != nil { return err } @@ -267,7 +327,7 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config, l if err != nil { return fmt.Errorf("event log subscribe: %w", err) } - go func() { + if !owner.Go(ctx, func(ctx context.Context) { // N.B. Use background for unsubscribe, ctx is already terminated. defer env.EventBus.UnsubscribeAll(context.Background(), subscriberID) //nolint:errcheck for { @@ -281,7 +341,10 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config, l _ = lg.Add(etype, msg.Data()) } } - }() + }) { + _ = env.EventBus.UnsubscribeAll(context.Background(), subscriberID) + return errors.New("RPC service is stopping") + } env.Logger.Info("Event log subscription enabled") } @@ -321,7 +384,7 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config, l rootHandler = corsMiddleware.Handler(mux) } if conf.RPC.IsTLSEnabled() { - go func() { + if !owner.Go(ctx, func(ctx context.Context) { if err := rpcserver.ServeTLS( ctx, listener, @@ -333,9 +396,11 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config, l ); err != nil { env.Logger.Error("error serving server with TLS", "err", err) } - }() + }) { + return errors.New("RPC service is stopping") + } } else { - go func() { + if !owner.Go(ctx, func(ctx context.Context) { if err := rpcserver.Serve( ctx, listener, @@ -345,7 +410,9 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config, l ); err != nil { env.Logger.Error("error serving server", "err", err) } - }() + }) { + return errors.New("RPC service is stopping") + } } } diff --git a/internal/rpc/core/events.go b/internal/rpc/core/events.go index da8e6088ee..58d565530c 100644 --- a/internal/rpc/core/events.go +++ b/internal/rpc/core/events.go @@ -61,9 +61,7 @@ func (env *Environment) Subscribe(ctx context.Context, req *coretypes.RequestSub // Capture the current ID, since it can change in the future. subscriptionID := callInfo.RPCRequest.ID - go func() { - opctx, opcancel := context.WithCancel(context.TODO()) - defer opcancel() + if !callInfo.WSConn.Go(func(opctx context.Context) { for { msg, err := sub.Next(opctx) @@ -81,6 +79,10 @@ func (env *Environment) Subscribe(ctx context.Context, req *coretypes.RequestSub return } + if err != nil { + return + } + // We have a message to deliver to the client. resp := callInfo.RPCRequest.MakeResponse(&coretypes.ResultEvent{ Query: req.Query, @@ -95,7 +97,10 @@ func (env *Environment) Subscribe(ctx context.Context, req *coretypes.RequestSub "to", addr, "subscriptionID", subscriptionID, "err", err) } } - }() + }) { + _ = env.EventBus.UnsubscribeAll(context.Background(), addr) + return nil, errors.New("websocket session is stopping") + } return &coretypes.ResultSubscribe{}, nil } diff --git a/internal/rpc/core/lifecycle_test.go b/internal/rpc/core/lifecycle_test.go new file mode 100644 index 0000000000..5e5fdd11d8 --- /dev/null +++ b/internal/rpc/core/lifecycle_test.go @@ -0,0 +1,60 @@ +package core + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + abci "github.com/dashpay/tenderdash/abci/types" + "github.com/dashpay/tenderdash/config" + "github.com/dashpay/tenderdash/internal/eventbus" + "github.com/dashpay/tenderdash/internal/eventlog" + "github.com/dashpay/tenderdash/internal/pubsub" + "github.com/dashpay/tenderdash/internal/pubsub/query" + "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/types" +) + +type blockedLogEvent struct { + types.EventDataNewBlockHeader + entered chan struct{} + release chan struct{} +} + +func (e blockedLogEvent) ABCIEvents() []abci.Event { + close(e.entered) + <-e.release + return nil +} + +func TestRPCServiceRetryAndEventLogDrain(t *testing.T) { + logger := log.NewNopLogger() + bus := eventbus.NewDefault(logger) + require.NoError(t, bus.Start(context.Background())) + defer func() { bus.Stop(); bus.Wait() }() + eventLog, err := eventlog.New(eventlog.LogSettings{WindowSize: time.Minute}) + require.NoError(t, err) + env := &Environment{Logger: logger, EventBus: bus, EventLog: eventLog} + conf := config.DefaultConfig() + // A failed subscription must leave the RPC owner reusable. + _, err = bus.SubscribeWithArgs(context.Background(), pubsub.SubscribeArgs{ClientID: "event-log-subscriber", Query: query.All}) + require.NoError(t, err) + require.Error(t, env.StartService(context.Background(), conf, nil)) + require.NoError(t, bus.UnsubscribeAll(context.Background(), "event-log-subscriber")) + require.NoError(t, env.StartService(context.Background(), conf, nil)) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + require.NoError(t, bus.Publish(types.EventNewBlockHeaderValue, blockedLogEvent{entered: entered, release: release})) + <-entered + go func() { env.StopService(); close(done) }() + select { + case <-done: + close(release) + t.Fatal("RPC shutdown returned while event-log forwarding was active") + case <-time.After(100 * time.Millisecond): + } + close(release) + <-done + require.Zero(t, bus.NumClientSubscriptions("event-log-subscriber")) +} diff --git a/internal/state/indexer/indexer_service.go b/internal/state/indexer/indexer_service.go index 868a770fdd..fbaa8a851b 100644 --- a/internal/state/indexer/indexer_service.go +++ b/internal/state/indexer/indexer_service.go @@ -139,8 +139,13 @@ func (is *Service) OnStart(ctx context.Context) error { return nil } -// OnStop implements service.Service by closing the event sinks. -func (is *Service) OnStop() { +func (is *Service) OnStop() {} + +// OnDrain detaches the observer before releasing its sinks. +func (is *Service) OnDrain() { + if IndexingEnabled(is.eventSinks) { + is.eventBus.RemoveObserver() + } for _, sink := range is.eventSinks { if err := sink.Stop(); err != nil { is.logger.Error("failed to close eventsink", "eventsink", sink.Type(), "err", err) diff --git a/internal/state/indexer/lifecycle_test.go b/internal/state/indexer/lifecycle_test.go new file mode 100644 index 0000000000..d242572bf2 --- /dev/null +++ b/internal/state/indexer/lifecycle_test.go @@ -0,0 +1,37 @@ +package indexer_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/internal/eventbus" + "github.com/dashpay/tenderdash/internal/pubsub" + "github.com/dashpay/tenderdash/internal/state/indexer" + "github.com/dashpay/tenderdash/internal/state/indexer/sink/kv" + dbm "github.com/dashpay/tenderdash/libs/db" + "github.com/dashpay/tenderdash/libs/log" +) + +func TestIndexerStopsWhileEventBusRemainsLive(t *testing.T) { + bus := eventbus.NewDefault(log.NewNopLogger()) + require.NoError(t, bus.Start(context.Background())) + t.Cleanup(func() { bus.Stop(); bus.Wait() }) + svc := indexer.NewService(indexer.ServiceArgs{ + Logger: log.NewNopLogger(), EventBus: bus, + Sinks: []indexer.EventSink{kv.NewEventSink(dbm.NewMemDB())}, + }) + require.NoError(t, svc.Start(context.Background())) + svc.Stop() + done := make(chan struct{}) + go func() { svc.Wait(); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("indexer shutdown waited for the live event bus") + } + require.True(t, bus.IsRunning()) + require.NoError(t, bus.Observe(context.Background(), func(_ pubsub.Message) error { return nil })) +} diff --git a/internal/statesync/reactor.go b/internal/statesync/reactor.go index bbb8154d91..90c7d000e5 100644 --- a/internal/statesync/reactor.go +++ b/internal/statesync/reactor.go @@ -8,6 +8,7 @@ import ( "fmt" "runtime/debug" "sort" + stdsync "sync" "time" sync "github.com/sasha-s/go-deadlock" @@ -321,13 +322,21 @@ func (r *Reactor) OnStart(ctx context.Context) error { return nil } - go r.processChannels(ctx, map[p2p.ChannelID]p2p.Channel{ - SnapshotChannel: snapshotCh, - ChunkChannel: chunkCh, - LightBlockChannel: blockCh, - ParamsChannel: paramsCh, + r.Go(ctx, func(ctx context.Context) { + r.processChannels(ctx, map[p2p.ChannelID]p2p.Channel{ + SnapshotChannel: snapshotCh, + ChunkChannel: chunkCh, + LightBlockChannel: blockCh, + ParamsChannel: paramsCh, + }) }) - go r.processPeerUpdates(ctx, r.peerEvents(ctx, "statesync")) + updates := r.peerEvents(ctx, "statesync") + if !r.Go(ctx, func(ctx context.Context) { + defer updates.Close() + r.processPeerUpdates(ctx, updates) + }) { + updates.Close() + } if r.needsStateSync { r.logger.Info("starting state sync") @@ -353,8 +362,7 @@ func (r *Reactor) OnStart(ctx context.Context) error { return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit. +// OnStop unblocks outstanding dispatcher requests. func (r *Reactor) OnStop() { // tell the dispatcher to stop sending any more requests r.dispatcher.Close() @@ -639,7 +647,8 @@ func (r *Reactor) backfill( ) ctxWithCancel, cancel := context.WithCancel(ctx) - defer cancel() + var fetchers stdsync.WaitGroup + defer func() { cancel(); fetchers.Wait() }() // Peer-dispatch gate for this run: it quarantines peers that supply light // blocks this node cannot accept and reports when no peer is left to serve. @@ -670,7 +679,7 @@ func (r *Reactor) backfill( dispatch.quarantine(peer, reason) queue.discardPeer(peer) queue.requeue(height) - err := r.sendBlockError(ctx, p2p.PeerError{ + err := r.sendBlockError(ctxWithCancel, p2p.PeerError{ NodeID: peer, Err: reason, Fatal: bool(severity), @@ -686,7 +695,9 @@ func (r *Reactor) backfill( // waiting on blocks. If it takes 4s to retrieve a block and 1s to verify // it, then steady state involves four workers. for i := 0; i < r.cfg.Fetchers; i++ { + fetchers.Add(1) go func() { + defer fetchers.Done() for { select { case <-ctxWithCancel.Done(): @@ -781,7 +792,7 @@ func (r *Reactor) backfill( "height", height, "error", err) queue.retry(height) - if serr := r.sendBlockError(ctx, p2p.PeerError{ + if serr := r.sendBlockError(ctxWithCancel, p2p.PeerError{ NodeID: peer, Err: fmt.Errorf("received invalid light block: %w", err), ConnID: connID, @@ -1331,7 +1342,7 @@ func (r *Reactor) processPeerUp(ctx context.Context, peerUpdate p2p.PeerUpdate) if sp, ok := stateProvider.(*stateProviderP2P); ok { // we do this in a separate routine to not block whilst waiting for the light client to finish // whatever call it's currently executing - go sp.addProvider(newProvider) + r.Go(ctx, func(context.Context) { sp.addProvider(newProvider) }) } } diff --git a/light/proxy/proxy.go b/light/proxy/proxy.go index c5996999f1..b7963bd4c1 100644 --- a/light/proxy/proxy.go +++ b/light/proxy/proxy.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "net/http" + "time" tmpubsub "github.com/dashpay/tenderdash/internal/pubsub" rpccore "github.com/dashpay/tenderdash/internal/rpc/core" @@ -15,6 +16,10 @@ import ( rpcserver "github.com/dashpay/tenderdash/rpc/jsonrpc/server" ) +// disconnectCleanupTimeout bounds unsubscribing a closed WebSocket session from +// the upstream client, which only has to queue the request. +const disconnectCleanupTimeout = time.Second + // A Proxy defines parameters for running an HTTP server proxy. type Proxy struct { Addr string // TCP address to listen on, ":http" if empty @@ -103,8 +108,12 @@ func (p *Proxy) listen(ctx context.Context) (net.Listener, *http.ServeMux, error wmLogger := p.Logger.With("protocol", "websocket") wm := rpcserver.NewWebsocketManager(wmLogger, r, rpcserver.OnDisconnect(func(remoteAddr string) { - err := p.Client.UnsubscribeAll(context.Background(), remoteAddr) - if err != nil && err != tmpubsub.ErrSubscriptionNotFound { + // Serve joins this callback, and closing the listener does not end ctx. + // Bound it: once the upstream client stops, its send has no receiver. + cleanupCtx, cancel := context.WithTimeout(ctx, disconnectCleanupTimeout) + defer cancel() + err := p.Client.UnsubscribeAll(cleanupCtx, remoteAddr) + if err != nil && err != tmpubsub.ErrSubscriptionNotFound && ctx.Err() == nil { wmLogger.Error("Failed to unsubscribe addr from events", "addr", remoteAddr, "err", err) } }), diff --git a/light/proxy/shutdown_test.go b/light/proxy/shutdown_test.go new file mode 100644 index 0000000000..7a6f867560 --- /dev/null +++ b/light/proxy/shutdown_test.go @@ -0,0 +1,111 @@ +package proxy + +import ( + "bytes" + "context" + "net" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" + lrpc "github.com/dashpay/tenderdash/light/rpc" + rpchttp "github.com/dashpay/tenderdash/rpc/client/http" + rpcserver "github.com/dashpay/tenderdash/rpc/jsonrpc/server" +) + +type lockedBuffer struct { + mtx sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mtx.Lock() + defer b.mtx.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mtx.Lock() + defer b.mtx.Unlock() + return b.buf.String() +} + +// Serve joins WebSocket disconnect cleanup. With the upstream client's writer +// already stopped, that cleanup must not block proxy shutdown, whether the proxy +// context ends or only the listener closes and leaves that context live. +func TestProxyShutdownWithStoppedUpstreamClient(t *testing.T) { + testCases := map[string]struct { + shutdown func(t *testing.T, cancel context.CancelFunc, listener net.Listener) + serveErr error + cleanupErrLogged bool + }{ + "proxy context canceled": { + shutdown: func(_ *testing.T, cancel context.CancelFunc, _ net.Listener) { cancel() }, + serveErr: http.ErrServerClosed, + }, + "listener closed": { + shutdown: func(t *testing.T, _ context.CancelFunc, listener net.Listener) { + require.NoError(t, listener.Close()) + }, + serveErr: net.ErrClosed, + cleanupErrLogged: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + logger := log.NewNopLogger() + upstreamWS := rpcserver.NewWebsocketManager(logger, map[string]*rpcserver.RPCFunc{}) + upstreamMux := http.NewServeMux() + upstreamMux.HandleFunc("/websocket", upstreamWS.WebsocketHandler) + upstream := httptest.NewServer(upstreamMux) + t.Cleanup(upstream.Close) + + next, err := rpchttp.New(upstream.URL) + require.NoError(t, err) + client := lrpc.NewClient(logger, next, nil) + clientCtx, stopClient := context.WithCancel(context.Background()) + t.Cleanup(stopClient) + require.NoError(t, client.Start(clientCtx)) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + logs := &lockedBuffer{} + proxyLogger, err := log.NewLogger(log.LogLevelError, logs) + require.NoError(t, err) + p := &Proxy{Addr: "tcp://127.0.0.1:0", Config: rpcserver.DefaultConfig(), Client: client, Logger: proxyLogger} + listener, mux, err := p.listen(ctx) + require.NoError(t, err) + served := make(chan error, 1) + go func() { served <- rpcserver.Serve(ctx, listener, mux, logger, p.Config) }() + + dialer := websocket.Dialer{HandshakeTimeout: 5 * time.Second} + conn, resp, err := dialer.Dial("ws://"+listener.Addr().String()+"/websocket", nil) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + t.Cleanup(func() { _ = conn.Close() }) + + // Stop the upstream client first; Wait returns after its writer exits. + stopClient() + client.Wait() + + tc.shutdown(t, cancel, listener) + select { + case err := <-served: + require.ErrorIs(t, err, tc.serveErr) + case <-time.After(5 * time.Second): + t.Fatal("proxy shutdown blocked on disconnect cleanup through a stopped upstream client") + } + if tc.cleanupErrLogged { + require.Contains(t, logs.String(), "Failed to unsubscribe addr from events") + } else { + require.NotContains(t, logs.String(), "Failed to unsubscribe addr from events") + } + }) + } +} diff --git a/light/rpc/client.go b/light/rpc/client.go index ba0081de85..1ec53a4801 100644 --- a/light/rpc/client.go +++ b/light/rpc/client.go @@ -53,7 +53,10 @@ type Client struct { prt *merkle.ProofRuntime keyPathFn KeyPathFunc - closers []func() + serviceCtx context.Context + // callerOwnsNext records that next was already running when this client + // started. Lifecycle hooks never overlap, so OnStart's write precedes OnDrain. + callerOwnsNext bool } var _ rpcclient.Client = (*Client)(nil) @@ -106,19 +109,29 @@ func NewClient(logger log.Logger, next rpcclient.Client, lc LightClient, opts .. } func (c *Client) OnStart(ctx context.Context) error { - nctx, ncancel := context.WithCancel(ctx) - if err := c.next.Start(nctx); err != nil { - ncancel() - return err + c.serviceCtx = ctx + // A client already running, such as a local client sharing its node's event + // bus, belongs to its caller; other clients are started here. + running, ok := c.next.(interface{ IsRunning() bool }) + c.callerOwnsNext = ok && running.IsRunning() + if c.callerOwnsNext { + return nil } - c.closers = append(c.closers, ncancel) - - return nil + return c.next.Start(ctx) } -func (c *Client) OnStop() { - for _, closer := range c.closers { - closer() +func (c *Client) OnStop() {} + +// OnDrain joins clients exposing transport shutdown in addition to the RPC API, +// unless the caller had already started them and so controls their lifetime. +func (c *Client) OnDrain() { + if c.callerOwnsNext { + return + } + if waiter, ok := c.next.(interface{ Wait() }); ok { + waiter.Wait() + } else if stopper, ok := c.next.(interface{ Stop() error }); ok { + _ = stopper.Stop() } } @@ -648,19 +661,31 @@ func (c *Client) RegisterOpDecoder(typ string, dec merkle.OpDecoder) { // a subscriber, but does not verify responses (UNSAFE)! // TODO: verify data func (c *Client) SubscribeWS(ctx context.Context, query string) (*coretypes.ResultSubscribe, error) { - bctx, bcancel := context.WithCancel(context.Background()) - c.closers = append(c.closers, bcancel) + if !c.IsRunning() { + return nil, fmt.Errorf("light RPC client is not running") + } + bctx, cancel := context.WithCancel(c.serviceCtx) callInfo := rpctypes.GetCallInfo(ctx) + stopCancel := context.AfterFunc(callInfo.WSConn.Context(), cancel) out, err := c.next.Subscribe(bctx, callInfo.RemoteAddr(), query) //nolint:staticcheck if err != nil { + stopCancel() + cancel() return nil, err } - go func() { + done := make(chan struct{}) + if !c.Go(bctx, func(bctx context.Context) { + defer close(done) + defer cancel() + defer stopCancel() for { select { - case resultEvent := <-out: + case resultEvent, ok := <-out: + if !ok { + return + } // We should have a switch here that performs a validation // depending on the event's type. callInfo.WSConn.TryWriteRPCResponse(bctx, callInfo.RPCRequest.MakeResponse(resultEvent)) @@ -668,7 +693,17 @@ func (c *Client) SubscribeWS(ctx context.Context, query string) (*coretypes.Resu return } } - }() + }) { + stopCancel() + cancel() + return nil, fmt.Errorf("light RPC client is stopping") + } + + if !callInfo.WSConn.Go(func(context.Context) { <-done }) { + cancel() + <-done + return nil, fmt.Errorf("websocket session is stopping") + } return &coretypes.ResultSubscribe{}, nil } @@ -676,7 +711,8 @@ func (c *Client) SubscribeWS(ctx context.Context, query string) (*coretypes.Resu // UnsubscribeWS calls original client's Unsubscribe using remote address as a // subscriber. func (c *Client) UnsubscribeWS(ctx context.Context, query string) (*coretypes.ResultUnsubscribe, error) { - err := c.next.Unsubscribe(context.Background(), rpctypes.GetCallInfo(ctx).RemoteAddr(), query) //nolint:staticcheck + // ctx ends with the WebSocket session, which joins this call on shutdown. + err := c.next.Unsubscribe(ctx, rpctypes.GetCallInfo(ctx).RemoteAddr(), query) //nolint:staticcheck if err != nil { return nil, err } @@ -686,7 +722,8 @@ func (c *Client) UnsubscribeWS(ctx context.Context, query string) (*coretypes.Re // UnsubscribeAllWS calls original client's UnsubscribeAll using remote address // as a subscriber. func (c *Client) UnsubscribeAllWS(ctx context.Context) (*coretypes.ResultUnsubscribe, error) { - err := c.next.UnsubscribeAll(context.Background(), rpctypes.GetCallInfo(ctx).RemoteAddr()) //nolint:staticcheck + // ctx ends with the WebSocket session, which joins this call on shutdown. + err := c.next.UnsubscribeAll(ctx, rpctypes.GetCallInfo(ctx).RemoteAddr()) //nolint:staticcheck if err != nil { return nil, err } diff --git a/light/rpc/ownership_test.go b/light/rpc/ownership_test.go new file mode 100644 index 0000000000..58d9723485 --- /dev/null +++ b/light/rpc/ownership_test.go @@ -0,0 +1,87 @@ +package rpc_test + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/internal/eventbus" + rpccore "github.com/dashpay/tenderdash/internal/rpc/core" + "github.com/dashpay/tenderdash/libs/log" + lrpc "github.com/dashpay/tenderdash/light/rpc" + rpcclient "github.com/dashpay/tenderdash/rpc/client" + "github.com/dashpay/tenderdash/rpc/client/local" +) + +type localNode struct{ bus *eventbus.EventBus } + +func (n localNode) RPCEnvironment() *rpccore.Environment { return &rpccore.Environment{} } +func (n localNode) EventBus() *eventbus.EventBus { return n.bus } + +func requireClientWaitReturns(t *testing.T, client *lrpc.Client) { + t.Helper() + waited := make(chan struct{}) + go func() { client.Wait(); close(waited) }() + select { + case <-waited: + case <-time.After(5 * time.Second): + t.Fatal("light client Wait blocked on a service it does not own") + } +} + +// A local client is its node's event bus. Stopping only the wrapper must neither +// wait for that bus nor stop it. +func TestClientWaitLeavesCallerOwnedEventBusRunning(t *testing.T) { + stoppers := map[string]func(*lrpc.Client, context.CancelFunc){ + "cancel": func(_ *lrpc.Client, cancel context.CancelFunc) { cancel() }, + "stop": func(client *lrpc.Client, _ context.CancelFunc) { client.Stop() }, + } + for name, stop := range stoppers { + t.Run(name, func(t *testing.T) { + logger := log.NewNopLogger() + busCtx, stopBus := context.WithCancel(context.Background()) + bus := eventbus.NewDefault(logger) + require.NoError(t, bus.Start(busCtx)) + defer func() { stopBus(); bus.Wait() }() + next, err := local.New(logger, localNode{bus: bus}) + require.NoError(t, err) + client := lrpc.NewClient(logger, next, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, client.Start(ctx)) + + stop(client, cancel) + requireClientWaitReturns(t, client) + require.True(t, bus.IsRunning(), "stopping the wrapper stopped the shared event bus") + }) + } +} + +// lifecycleClient records the join of a wrapped client exposing its running state. +type lifecycleClient struct { + rpcclient.Client + running bool + waited atomic.Bool +} + +func (c *lifecycleClient) Start(context.Context) error { return nil } +func (c *lifecycleClient) IsRunning() bool { return c.running } +func (c *lifecycleClient) Wait() { c.waited.Store(true) } + +// The wrapper joins exactly the client its own start attempt started. +func TestClientJoinsOnlyClientItStarted(t *testing.T) { + for _, alreadyRunning := range []bool{false, true} { + next := &lifecycleClient{running: alreadyRunning} + client := lrpc.NewClient(log.NewNopLogger(), next, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, client.Start(ctx)) + + cancel() + requireClientWaitReturns(t, client) + require.Equal(t, !alreadyRunning, next.waited.Load(), "already running: %t", alreadyRunning) + } +} diff --git a/light/rpc/shutdown_test.go b/light/rpc/shutdown_test.go new file mode 100644 index 0000000000..dba5030bd1 --- /dev/null +++ b/light/rpc/shutdown_test.go @@ -0,0 +1,125 @@ +package rpc + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" + rpchttp "github.com/dashpay/tenderdash/rpc/client/http" + rpcserver "github.com/dashpay/tenderdash/rpc/jsonrpc/server" +) + +// OnDrain stops an HTTP-backed upstream client, which may already have stopped +// itself, as it does when its reconnect attempts run out. +func TestClientDrainToleratesStoppedUpstream(t *testing.T) { + logger := log.NewNopLogger() + ws := rpcserver.NewWebsocketManager(logger, map[string]*rpcserver.RPCFunc{}) + mux := http.NewServeMux() + mux.HandleFunc("/websocket", ws.WebsocketHandler) + upstream := httptest.NewServer(mux) + t.Cleanup(upstream.Close) + next, err := rpchttp.New(upstream.URL) + require.NoError(t, err) + client := NewClient(logger, next, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, client.Start(ctx)) + + cancel() + client.Wait() + require.NoError(t, next.Stop()) +} + +// Cancellation must end the client even when its upstream hangs during a redial. +func TestClientWaitDuringHungUpstreamRedial(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + var accepted atomic.Int32 + upgrader := websocket.Upgrader{} + srv := &http.Server{ + ReadHeaderTimeout: 5 * time.Second, + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if accepted.Add(1) == 1 { + conn, err := upgrader.Upgrade(w, r, nil) + if err == nil { + _ = conn.UnderlyingConn().Close() // the client sees 1006 and redials + } + return + } + <-r.Context().Done() // never answers + }), + } + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + next, err := rpchttp.New("http://" + ln.Addr().String()) + require.NoError(t, err) + client := NewClient(log.NewNopLogger(), next, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, client.Start(ctx)) + require.Eventually(t, func() bool { return accepted.Load() == 2 }, 10*time.Second, time.Millisecond) + + cancel() + waited := make(chan struct{}) + go func() { client.Wait(); close(waited) }() + select { + case <-waited: + case <-time.After(5 * time.Second): + t.Fatal("light client Wait hung after cancellation") + } +} + +// An HTTP client its caller already started belongs to the caller: the wrapper +// must neither start a second transport nor end the caller's one. +func TestClientLeavesCallerStartedHTTPClientRunning(t *testing.T) { + logger := log.NewNopLogger() + ws := rpcserver.NewWebsocketManager(logger, map[string]*rpcserver.RPCFunc{}) + var sessions atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("/websocket", func(w http.ResponseWriter, r *http.Request) { + sessions.Add(1) + ws.WebsocketHandler(w, r) + }) + upstream := httptest.NewServer(mux) + t.Cleanup(upstream.Close) + next, err := rpchttp.New(upstream.URL) + require.NoError(t, err) + nextCtx, stopNext := context.WithCancel(context.Background()) + defer func() { + stopNext() + stopped := make(chan error, 1) + go func() { stopped <- next.Stop() }() + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Error("the caller's client did not stop") + } + }() + require.NoError(t, next.Start(nextCtx)) + + client := NewClient(logger, next, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, client.Start(ctx)) + require.Equal(t, int32(1), sessions.Load(), "the wrapper started the caller's client again") + + cancel() + waited := make(chan struct{}) + go func() { client.Wait(); close(waited) }() + select { + case <-waited: + case <-time.After(5 * time.Second): + t.Fatal("light client Wait blocked on a client it does not own") + } + _, err = next.Subscribe(nextCtx, "caller", "tm.event = 'NewBlock'") + require.NoError(t, err, "stopping the wrapper ended the caller's client") +} diff --git a/light/rpc/unsubscribe_test.go b/light/rpc/unsubscribe_test.go new file mode 100644 index 0000000000..26453a6ff4 --- /dev/null +++ b/light/rpc/unsubscribe_test.go @@ -0,0 +1,53 @@ +package rpc + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" + rpcmock "github.com/dashpay/tenderdash/rpc/client/mocks" + rpctypes "github.com/dashpay/tenderdash/rpc/jsonrpc/types" +) + +// blockUntilCanceled stands in for an upstream WebSocket send with no writer. +func blockUntilCanceled(args mock.Arguments) { + <-args.Get(0).(context.Context).Done() +} + +// Unsubscribe calls run inside a joined WebSocket session, so they must stop +// when the caller's request context is canceled. +func TestUnsubscribeWSObservesCallerCancellation(t *testing.T) { + for name, call := range map[string]func(*Client, context.Context) error{ + "unsubscribe": func(c *Client, ctx context.Context) error { + _, err := c.UnsubscribeWS(ctx, "tm.event = 'NewBlock'") + return err + }, + "unsubscribe_all": func(c *Client, ctx context.Context) error { + _, err := c.UnsubscribeAllWS(ctx) + return err + }, + } { + t.Run(name, func(t *testing.T) { + next := rpcmock.NewClient(t) + next.On("Unsubscribe", mock.Anything, mock.Anything, mock.Anything). + Run(blockUntilCanceled).Return(context.Canceled).Maybe() + next.On("UnsubscribeAll", mock.Anything, mock.Anything). + Run(blockUntilCanceled).Return(context.Canceled).Maybe() + c := NewClient(log.NewNopLogger(), next, nil) + ctx, cancel := context.WithCancel(rpctypes.WithCallInfo(context.Background(), &rpctypes.CallInfo{})) + done := make(chan error, 1) + go func() { done <- call(c, ctx) }() + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("unsubscribe ignored caller cancellation") + } + }) + } +} diff --git a/node/http_server_test.go b/node/http_server_test.go new file mode 100644 index 0000000000..04574a3b38 --- /dev/null +++ b/node/http_server_test.go @@ -0,0 +1,73 @@ +package node + +import ( + "context" + "io" + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" +) + +type httpServerOwner struct { + start func(context.Context) +} + +func (o *httpServerOwner) OnStart(ctx context.Context) error { o.start(ctx); return nil } +func (*httpServerOwner) OnStop() {} + +// A client that never finishes its request (e.g. a long pprof profile) must not +// block the owner's shutdown, and Wait must still join the handler. +func TestOwnedHTTPServerStopEvictsStalledClient(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + entered, handlerDone := make(chan struct{}), make(chan struct{}) + release := make(chan struct{}) + srv := &http.Server{ + ReadHeaderTimeout: httpReadHeaderTimeout, + Handler: http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + defer close(handlerDone) + close(entered) + select { + case <-r.Context().Done(): + case <-release: + } + }), + } + owner := &httpServerOwner{} + bs := service.NewBaseService(log.NewNopLogger(), "http-owner", owner) + owner.start = func(ctx context.Context) { serveOwnedHTTP(ctx, bs, srv, ln) } + require.NoError(t, bs.Start(context.Background())) + + client, err := net.Dial("tcp", ln.Addr().String()) + require.NoError(t, err) + defer client.Close() + _, err = io.WriteString(client, "GET /stalled HTTP/1.1\r\nHost: localhost\r\n\r\n") + require.NoError(t, err) + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("handler was not entered") + } + + bs.Stop() + stopped := make(chan struct{}) + go func() { bs.Wait(); close(stopped) }() + select { + case <-stopped: + case <-time.After(5 * time.Second): + close(release) + <-stopped + t.Fatal("a stalled client blocked HTTP server shutdown") + } + select { + case <-handlerDone: + default: + t.Fatal("Wait returned before the active handler finished") + } +} diff --git a/node/node.go b/node/node.go index ee98fcfabd..45d6d8c6fc 100644 --- a/node/node.go +++ b/node/node.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" "net" - "net/http" "strconv" "strings" + "sync/atomic" "time" abciclient "github.com/dashpay/tenderdash/abci/client" @@ -44,6 +44,10 @@ import ( _ "net/http/pprof" //nolint: gosec // securely exposed on separate, optional port ) +// errResourcesReleased rejects a start after a failed start released the node's +// resources. +var errResourcesReleased = errors.New("failed startup released node resources; construct a new node") + // nodeImpl is the highest level interface to a full Tendermint node. // It includes all configuration information and running services. type nodeImpl struct { @@ -62,17 +66,17 @@ type nodeImpl struct { nodeKey types.NodeKey // our node privkey // services - eventSinks []indexer.EventSink - initialState sm.State - stateStore sm.Store - blockStore *store.BlockStore // store the blockchain to disk - evPool *evidence.Pool - indexerService *indexer.Service - services []service.Service - rpcListeners []net.Listener // rpc servers - shutdownOps closer - rpcEnv *rpccore.Environment - prometheusSrv *http.Server + eventSinks []indexer.EventSink + initialState sm.State + stateStore sm.Store + blockStore *store.BlockStore // store the blockchain to disk + evPool *evidence.Pool + indexerService *indexer.Service + services []service.Service + shutdownOps closer + rpcEnv *rpccore.Environment + dependencyCancel context.CancelFunc + released atomic.Bool // shutdownOps ran; set by OnDrain } // newDefaultNode returns a Tendermint node with default settings for the @@ -211,6 +215,9 @@ func makeNode( if err != nil { return nil, combineCloseError(err, makeCloser(closers)) } + if resource, ok := privValidator.(interface{ Close() error }); ok { + closers = append(closers, resource.Close) + } if dashPrivval, ok := privValidator.(privval.DashPrivValidator); ok { dashCoreRPCClient = dashPrivval.DashRPCClient() } @@ -230,6 +237,10 @@ func makeNode( if err != nil { return nil, fmt.Errorf("failed to create Dash Core RPC client: %w", err) } + // A validator's client belongs to its private validator instead. Close + // bounds only its own join: this closer runs after reactors are joined, so + // a reactor stuck in a Dash Core call blocks shutdown before reaching it. + closers = append(closers, dashCoreRPCClient.Close) } weAreOnlyValidator := onlyValidatorIsUs(state, proTxHash) @@ -291,11 +302,13 @@ func makeNode( makeCloser(closers)) } + mempoolRateLimit := p2pclient.NewRateLimit(ctx, cfg.Mempool.TxSendRateLimit, false, logger) + closers = append(closers, func() error { mempoolRateLimit.Close(); return nil }) p2pClient := p2pclient.New( p2p.ChannelDescriptors(cfg), node.router.OpenChannel, p2pclient.WithLogger(logger), - p2pclient.WithSendRateLimits(p2pclient.NewRateLimit(ctx, cfg.Mempool.TxSendRateLimit, false, logger), p2p.MempoolChannel), + p2pclient.WithSendRateLimits(mempoolRateLimit, p2p.MempoolChannel), ) evReactor, evPool, edbCloser, err := createEvidenceReactor(logger, cfg, dbProvider, @@ -460,12 +473,16 @@ func makeNode( } node.BaseService = *service.NewBaseService(logger, "Node", node) + node.shutdownOps = makeCloser(closers) return node, nil } // OnStart starts the Node. It implements service.Service. func (n *nodeImpl) OnStart(ctx context.Context) (err error) { + if n.released.Load() { + return errResourcesReleased + } // Reserve RPC addresses before starting services; serve only once the node is ready. rpcListeners, err := rpccore.ListenRPC(n.config.RPC) if err != nil { @@ -479,9 +496,35 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { } }() - if err := n.rpcEnv.ProxyApp.Start(ctx); err != nil { + // Application transports and event sinks outlive reactor cancellation so an + // in-flight finalization can finish. Node shutdown joins them explicitly. + dependencyCtx, dependencyCancel := context.WithCancel(context.WithoutCancel(ctx)) + stopStartupCancellation := context.AfterFunc(ctx, dependencyCancel) + defer stopStartupCancellation() + n.dependencyCancel = dependencyCancel + ctx, cancelAttempt := context.WithCancel(ctx) + var startedReactors []service.Service + indexerStarted := false + // Past the retryable RPC reservation, failure is final and BaseService will + // not call OnDrain. Workers it joins after this return serve only pprof and + // metrics, so they do not use the released resources. + defer func() { + if err != nil { + cancelAttempt() + n.joinChildren(startedReactors) + if !indexerStarted { + n.stopEventSinks() + } + n.OnDrain() + } + }() + if err := n.rpcEnv.ProxyApp.Start(dependencyCtx); err != nil { return fmt.Errorf("error starting proxy app connections: %w", err) } + stopStartupCancellation() + if err := ctx.Err(); err != nil { + return err + } var proTxHash tmbytes.HexBytes if n.privValidator != nil { @@ -491,18 +534,20 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { } } ctx = dash.ContextWithProTxHash(ctx, proTxHash) + dependencyCtx = dash.ContextWithProTxHash(dependencyCtx, proTxHash) // EventBus and IndexerService must be started before the handshake because // we might need to index the txs of the replayed block as this might not have happened // when the node stopped last time (i.e. the node stopped or crashed after it saved the block // but before it indexed the txs) - if err := n.rpcEnv.EventBus.Start(ctx); err != nil { + if err := n.rpcEnv.EventBus.Start(dependencyCtx); err != nil { return err } - if err := n.indexerService.Start(ctx); err != nil { + if err := n.indexerService.Start(dependencyCtx); err != nil { return err } + indexerStarted = true // Create the handshaker, which calls RequestInfo, sets the AppVersion on the state, // and replays any blocks as necessary to sync tendermint with the app. @@ -536,7 +581,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { } // Start Internal Services if n.config.RPC.PprofListenAddress != "" { - startPProfServer(ctx, *n.config.RPC) + startPProfServer(ctx, &n.BaseService, n.logger, *n.config.RPC) } now := tmtime.Now() @@ -562,7 +607,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { } if n.config.Instrumentation.Prometheus && n.config.Instrumentation.PrometheusListenAddr != "" { - n.prometheusSrv = startPrometheusServer(ctx, *n.config.Instrumentation) + startPrometheusServer(ctx, &n.BaseService, n.logger, *n.config.Instrumentation) } // Start the transport. @@ -575,6 +620,9 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { if err := reactor.Start(ctx); err != nil { return fmt.Errorf("problem starting service '%T': %w ", reactor, err) } + if reactor != n.rpcEnv.EventBus { + startedReactors = append(startedReactors, reactor) + } } n.rpcEnv.NodeInfo = n.nodeInfo @@ -584,15 +632,9 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { if len(rpcListeners) > 0 { err = n.rpcEnv.StartService(ctx, n.config, rpcListeners) if err != nil { - stopCtx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - if stopErr := n.rpcEnv.StopAsyncBroadcasts(stopCtx); stopErr != nil { - n.logger.Error("Failed to stop async broadcasts after RPC startup failure", "err", stopErr) - } return err } } - n.rpcListeners = rpcListeners return nil } @@ -600,60 +642,69 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { // OnStop stops the Node. It implements service.Service. func (n *nodeImpl) OnStop() { n.logger.Info("Stopping Node") - stopCtx, cancel := context.WithTimeout(context.Background(), time.Second) - if err := n.rpcEnv.StopAsyncBroadcasts(stopCtx); err != nil { - n.logger.Error("Async broadcasts did not finish during shutdown", "err", err) - } - cancel() - // stop the listeners / external services first - for _, l := range n.rpcListeners { - n.logger.Info("Closing rpc listener", "listener", l) - if err := l.Close(); err != nil { - n.logger.Error("error closing listener", "listener", l, "err", err) - } - } - + reactors := make([]service.Service, 0, len(n.services)) for _, reactor := range n.services { - reactor.Wait() - } - - for _, es := range n.eventSinks { - if err := es.Stop(); err != nil { - n.logger.Error("failed to stop event sink", "err", err) + if reactor != n.rpcEnv.EventBus { + reactors = append(reactors, reactor) } } + n.joinChildren(reactors) - n.router.Wait() - n.rpcEnv.IsListening = false - + // The private validator is created with the node, not started by OnStart, + // so only completed shutdown waits for a service implementation of it. if n.privValidator != nil { if pvsc, ok := n.privValidator.(service.Service); ok { pvsc.Wait() } } +} - if n.prometheusSrv != nil { - if err := n.prometheusSrv.Shutdown(context.Background()); err != nil { - // Error from closing listeners, or context timeout: - n.logger.Error("Prometheus HTTP server Shutdown", "err", err) +// joinChildren stops and joins the children started by OnStart after their +// contexts are canceled; both shutdown and failed startup use it. Waiting for a +// child that never started returns immediately. RPC admission closes first; +// application connections, the event bus and the indexer go last because +// reactors may use them until they exit. +func (n *nodeImpl) joinChildren(reactors []service.Service) { + n.rpcEnv.StopService() + if err := n.rpcEnv.StopAsyncBroadcasts(context.Background()); err != nil { + n.logger.Error("Async broadcasts did not finish during shutdown", "err", err) + } + for _, reactor := range reactors { + reactor.Wait() + } + n.router.Wait() + n.rpcEnv.IsListening = false + if mp, ok := n.rpcEnv.Mempool.(*mempool.TxMempool); ok { + mp.StopRechecks() + } + n.rpcEnv.EventBus.Stop() + n.rpcEnv.EventBus.Wait() + n.dependencyCancel() + n.indexerService.Wait() + n.rpcEnv.ProxyApp.Wait() +} + +// stopEventSinks closes sinks of an indexer that never started, whose own OnDrain +// therefore cannot run. +func (n *nodeImpl) stopEventSinks() { + for _, es := range n.eventSinks { + if err := es.Stop(); err != nil { + n.logger.Error("failed to close eventsink", "eventsink", es.Type(), "err", err) } + } +} +// OnDrain releases storage after child services and node workers have finished, +// at most once. shutdownOps also closes the block and state store databases. +func (n *nodeImpl) OnDrain() { + if !n.released.CompareAndSwap(false, true) { + return } if err := n.shutdownOps(); err != nil { if strings.TrimSpace(err.Error()) != "" { n.logger.Error("problem shutting down additional services", "err", err) } } - if n.blockStore != nil { - if err := n.blockStore.Close(); err != nil { - n.logger.Error("problem closing blockstore", "err", err) - } - } - if n.stateStore != sm.Store(nil) { - if err := n.stateStore.Close(); err != nil { - n.logger.Error("problem closing statestore", "err", err) - } - } } func (n *nodeImpl) NodeInfo() *types.NodeInfo { diff --git a/node/node_test.go b/node/node_test.go index eac1c86152..c9e869473f 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -10,6 +10,7 @@ import ( "runtime" "strconv" "strings" + "sync" "testing" "time" @@ -1227,6 +1228,7 @@ func TestNodeStartRetryAfterRPCListenFailure(t *testing.T) { n.Wait() } else { n.OnStop() + n.OnDrain() } }() err = n.Start(ctx) @@ -1264,9 +1266,11 @@ func TestNodeStartFailureReleasesRPCListeners(t *testing.T) { ns, err := newDefaultNode(ctx, cfg, log.NewNopLogger()) require.NoError(t, err) n := ns.(*nodeImpl) + defer n.OnDrain() defer n.OnStop() app := abcimocks.NewClient(t) appErr := errors.New("application unavailable") + app.On("Wait").Return() app.On("Start", mock.Anything).Run(func(mock.Arguments) { probe, bindErr := net.Listen("tcp", address) if bindErr == nil { @@ -1285,12 +1289,14 @@ func TestNodeStartFailureReleasesRPCListeners(t *testing.T) { // finalizePausedClient holds the first commit between block-store and state-store writes. type finalizePausedClient struct { abciclient.Client - entered chan struct{} - release chan struct{} + entered chan struct{} + release chan struct{} + finalizeCtx context.Context } func (c *finalizePausedClient) FinalizeBlock(ctx context.Context, req *abci.RequestFinalizeBlock) (*abci.ResponseFinalizeBlock, error) { if req.Block.Header.Height == 1 { + c.finalizeCtx = ctx close(c.entered) <-c.release } @@ -1343,6 +1349,7 @@ func TestNodeStartDuringFirstCommit(t *testing.T) { n.Wait() } else { n.OnStop() + n.OnDrain() } }() err = n.Start(ctx) @@ -1353,3 +1360,128 @@ func TestNodeStartDuringFirstCommit(t *testing.T) { require.NoError(t, err) require.Zero(t, state.LastBlockHeight, "the application has not completed FinalizeBlock") } + +type shutdownPausedService struct { + entered chan struct{} + release chan struct{} +} + +func (*shutdownPausedService) Start(context.Context) error { return nil } +func (*shutdownPausedService) IsRunning() bool { return true } +func (s *shutdownPausedService) Wait() { close(s.entered); <-s.release } + +// Node shutdown may cancel an in-flight commit (see State.Start), but Wait must +// still join it before later services and stores are released. +func TestNodeWaitJoinsFinalization(t *testing.T) { + for _, manual := range []bool{false, true} { + t.Run(fmt.Sprintf("manual=%t", manual), func(t *testing.T) { + cfg, err := config.ResetTestRoot(t.TempDir(), strings.ReplaceAll(t.Name(), "/", "-")) + require.NoError(t, err) + cfg.Consensus = config.DefaultConsensusConfig() + cfg.SetRoot(cfg.RootDir) + cfg.RPC.ListenAddress = "" + cfg.P2P.ListenAddress = "tcp://127.0.0.1:0" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logger := log.NewNopLogger() + client, _, err := proxy.ClientFactory(logger, *cfg.Abci, cfg.DBDir()) + require.NoError(t, err) + app := &finalizePausedClient{Client: client, entered: make(chan struct{}), release: make(chan struct{})} + unpauseApp := sync.OnceFunc(func() { close(app.release) }) + gate := &shutdownPausedService{entered: make(chan struct{}), release: make(chan struct{})} + unpauseShutdown := sync.OnceFunc(func() { close(gate.release) }) + ns, err := New(ctx, cfg, logger, app, nil) + require.NoError(t, err) + n := ns.(*nodeImpl) + for i, reactor := range n.services { + if reactor == n.rpcEnv.ConsensusReactor { + n.services[i] = startupPausedService{Service: reactor, entered: app.entered} + } + } + n.services = append(n.services, gate) + require.NoError(t, n.Start(ctx)) + t.Cleanup(func() { unpauseApp(); unpauseShutdown(); cancel(); n.Wait() }) + stopped := make(chan struct{}) + if manual { + go n.Stop() + } else { + cancel() + } + go func() { n.Wait(); close(stopped) }() + select { + case <-stopped: + t.Fatal("node Wait returned during finalization") + case <-time.After(25 * time.Millisecond): + } + select { + case <-gate.entered: + t.Fatal("node released later services while finalization was still running") + default: + } + unpauseApp() + select { + case <-gate.entered: + case <-time.After(5 * time.Second): + t.Fatal("consensus did not drain after finalization returned") + } + unpauseShutdown() + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("node shutdown did not finish") + } + }) + } +} + +type startupBlockedClient struct { + abciclient.Client + entered chan struct{} + release chan struct{} +} + +func (c *startupBlockedClient) Start(ctx context.Context) error { + close(c.entered) + select { + case <-ctx.Done(): + return ctx.Err() + case <-c.release: + return errors.New("startup unblocked by test cleanup") + } +} + +func (*startupBlockedClient) Wait() {} + +func TestNodeCancellationInterruptsApplicationStartup(t *testing.T) { + for _, manual := range []bool{false, true} { + t.Run(fmt.Sprintf("manual=%t", manual), func(t *testing.T) { + cfg, err := config.ResetTestRoot(t.TempDir(), strings.ReplaceAll(t.Name(), "/", "-")) + require.NoError(t, err) + cfg.RPC.ListenAddress = "" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ns, err := newDefaultNode(ctx, cfg, log.NewNopLogger()) + require.NoError(t, err) + n := ns.(*nodeImpl) + defer n.OnDrain() + app := &startupBlockedClient{entered: make(chan struct{}), release: make(chan struct{})} + defer close(app.release) + n.rpcEnv.ProxyApp = app + started := make(chan error, 1) + go func() { started <- n.Start(ctx) }() + <-app.entered + if manual { + n.Stop() + } else { + cancel() + } + select { + case err := <-started: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("application startup did not observe cancellation") + } + n.Wait() + }) + } +} diff --git a/node/rollback_test.go b/node/rollback_test.go new file mode 100644 index 0000000000..42a6c78a13 --- /dev/null +++ b/node/rollback_test.go @@ -0,0 +1,368 @@ +package node + +import ( + "context" + "errors" + "maps" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + abciclient "github.com/dashpay/tenderdash/abci/client" + abcimocks "github.com/dashpay/tenderdash/abci/client/mocks" + abci "github.com/dashpay/tenderdash/abci/types" + "github.com/dashpay/tenderdash/config" + "github.com/dashpay/tenderdash/internal/proxy" + "github.com/dashpay/tenderdash/internal/pubsub" + dbm "github.com/dashpay/tenderdash/libs/db" + "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" + "github.com/dashpay/tenderdash/types" +) + +// dbCloseCounter counts Close calls for every database a node opens. +type dbCloseCounter struct { + mtx sync.Mutex + closes map[string]int +} + +type countedDB struct { + dbm.DB + id string + counter *dbCloseCounter +} + +func (d countedDB) Close() error { + d.counter.mtx.Lock() + d.counter.closes[d.id]++ + d.counter.mtx.Unlock() + return d.DB.Close() +} + +func (c *dbCloseCounter) provider(ctx *config.DBContext) (dbm.DB, error) { + db, err := config.DefaultDBProvider(ctx) + if err != nil { + return nil, err + } + c.mtx.Lock() + defer c.mtx.Unlock() + c.closes[ctx.ID] = 0 + return countedDB{DB: db, id: ctx.ID, counter: c}, nil +} + +func (c *dbCloseCounter) closed(id string) int { + c.mtx.Lock() + defer c.mtx.Unlock() + return c.closes[id] +} + +func (c *dbCloseCounter) snapshot() map[string]int { + c.mtx.Lock() + defer c.mtx.Unlock() + return maps.Clone(c.closes) +} + +func (c *dbCloseCounter) requireEachClosedOnce(t *testing.T, ids ...string) { + t.Helper() + c.mtx.Lock() + defer c.mtx.Unlock() + expected := make(map[string]int, len(ids)) + for _, id := range ids { + expected[id] = 1 + } + require.Equal(t, expected, c.closes) +} + +func rollbackTestNode( + ctx context.Context, + t *testing.T, + wrapApp func(abciclient.Client) abciclient.Client, + configure ...func(*config.Config), +) (*nodeImpl, *dbCloseCounter) { + t.Helper() + cfg, err := config.ResetTestRoot(t.TempDir(), strings.ReplaceAll(t.Name(), "/", "-")) + require.NoError(t, err) + for _, apply := range configure { + apply(cfg) + } + cfg.RPC.ListenAddress = "" + cfg.P2P.ListenAddress = "tcp://127.0.0.1:0" + cfg.TxIndex.Indexer = []string{"kv"} + key, err := types.LoadOrGenNodeKey(cfg.NodeKeyFile()) + require.NoError(t, err) + client, _, err := proxy.ClientFactory(log.NewNopLogger(), *cfg.Abci, cfg.DBDir()) + require.NoError(t, err) + if wrapApp != nil { + client = wrapApp(client) + } + counter := &dbCloseCounter{closes: map[string]int{}} + svc, err := makeNode(ctx, cfg, key, client, defaultGenesisDocProviderFunc(cfg), counter.provider, log.NewNopLogger()) + require.NoError(t, err) + return svc.(*nodeImpl), counter +} + +type failingStartService struct{ err error } + +func (s failingStartService) Start(context.Context) error { return s.err } +func (failingStartService) IsRunning() bool { return false } +func (failingStartService) Wait() {} + +type failingInfoClient struct { + abciclient.Client + err error +} + +func (c failingInfoClient) Info(context.Context, *abci.RequestInfo) (*abci.ResponseInfo, error) { + return nil, c.err +} + +var nodeDatabases = []string{"blockstore", "state", "peerstore", "evidence", "tx_index"} + +// A failed start past the RPC reservation is final: it closes every database +// once, whichever children had started, and later lifecycle calls change nothing. +func TestNodeFailedStartReleasesDatabasesOnce(t *testing.T) { + failure := errors.New("startup failed") + testCases := map[string]func(*testing.T, *nodeImpl){ + "application start": func(t *testing.T, n *nodeImpl) { + n.rpcEnv.ProxyApp = failedStartClient{Client: abcimocks.NewClient(t), err: failure} + }, + "reactor start": func(_ *testing.T, n *nodeImpl) { + n.services = append(n.services, failingStartService{err: failure}) + }, + } + for name, inject := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, counter := rollbackTestNode(ctx, t, nil) + inject(t, n) + + require.ErrorIs(t, n.Start(ctx), failure) + counter.requireEachClosedOnce(t, nodeDatabases...) + + n.Stop() + n.Wait() + n.OnDrain() + require.ErrorContains(t, n.Start(ctx), "construct a new node") + counter.requireEachClosedOnce(t, nodeDatabases...) + }) + } + + t.Run("handshake", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, counter := rollbackTestNode(ctx, t, func(app abciclient.Client) abciclient.Client { + return failingInfoClient{Client: app, err: failure} + }) + + require.ErrorContains(t, n.Start(ctx), failure.Error()) + counter.requireEachClosedOnce(t, nodeDatabases...) + n.Wait() + counter.requireEachClosedOnce(t, nodeDatabases...) + }) +} + +// Releasing resources during rollback must not replace the startup error. +func TestNodeFailedStartKeepsErrorWhenReleaseFails(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, _ := rollbackTestNode(ctx, t, nil) + release := n.shutdownOps + n.shutdownOps = func() error { return errors.Join(release(), errors.New("release failed")) } + failure := errors.New("startup failed") + n.rpcEnv.ProxyApp = failedStartClient{Client: abcimocks.NewClient(t), err: failure} + + err := n.Start(ctx) + require.ErrorIs(t, err, failure) + require.NotContains(t, err.Error(), "release failed") +} + +// coreRPCWorkers counts running Dash Core RPC client dispatchers. +func coreRPCWorkers() int { + stacks := make([]byte, 1<<24) + return strings.Count(string(stacks[:runtime.Stack(stacks, true)]), "rpcclient.(*Client).sendPostHandler(") +} + +// A full node owns the Dash Core RPC client it creates, unlike a validator, whose +// client belongs to its private validator. +func TestFullNodeReleasesCoreRPCClient(t *testing.T) { + fullNode := func(cfg *config.Config) { + cfg.Mode = config.ModeFull + cfg.PrivValidator.CoreRPCHost = "127.0.0.1:1" + } + failure := errors.New("startup failed") + testCases := map[string]func(context.Context, context.CancelFunc, *testing.T, *nodeImpl){ + "failed start": func(ctx context.Context, _ context.CancelFunc, t *testing.T, n *nodeImpl) { + n.rpcEnv.ProxyApp = failedStartClient{Client: abcimocks.NewClient(t), err: failure} + require.ErrorIs(t, n.Start(ctx), failure) + }, + "shutdown": func(ctx context.Context, cancel context.CancelFunc, t *testing.T, n *nodeImpl) { + require.NoError(t, n.Start(ctx)) + cancel() + n.Wait() + }, + } + for name, run := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + before := coreRPCWorkers() + n, _ := rollbackTestNode(ctx, t, nil, fullNode) + require.Equal(t, before+1, coreRPCWorkers(), "the full node did not create its client") + run(ctx, cancel, t, n) + require.Equal(t, before, coreRPCWorkers(), "the Dash Core RPC client outlived the node") + }) + } +} + +type failedStartClient struct { + abciclient.Client + err error +} + +func (c failedStartClient) Start(context.Context) error { return c.err } +func (failedStartClient) Wait() {} + +type waitProbeClient struct { + abciclient.Client + onWait func() +} + +func (c waitProbeClient) Wait() { c.onWait(); c.Client.Wait() } + +// The application client is the last child joined, so no node-owned database +// may be closed yet. Only the indexer, already joined, has closed its sink. +func TestNodeFailedStartReleasesDatabasesAfterJoiningChildren(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, counter := rollbackTestNode(ctx, t, nil) + failure := errors.New("startup failed") + n.services = append(n.services, failingStartService{err: failure}) + var atLastJoin map[string]int + n.rpcEnv.ProxyApp = waitProbeClient{ + Client: n.rpcEnv.ProxyApp, + onWait: func() { atLastJoin = counter.snapshot() }, + } + + require.ErrorIs(t, n.Start(ctx), failure) + require.Equal(t, map[string]int{"blockstore": 0, "state": 0, "peerstore": 0, "evidence": 0, "tx_index": 1}, atLastJoin) + counter.requireEachClosedOnce(t, nodeDatabases...) +} + +// An indexer whose own start fails never drains, so the node closes its sinks. +func TestNodeIndexerStartFailureClosesEventSinks(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, counter := rollbackTestNode(ctx, t, nil) + require.NoError(t, n.rpcEnv.EventBus.Observe(ctx, func(pubsub.Message) error { return nil })) + + require.ErrorContains(t, n.Start(ctx), "an observer is already registered") + counter.requireEachClosedOnce(t, nodeDatabases...) +} + +// Normal shutdown has the same exactly-once guarantee. +func TestNodeShutdownClosesEachDatabaseOnce(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, counter := rollbackTestNode(ctx, t, nil) + require.NoError(t, n.Start(ctx)) + cancel() + n.Wait() + counter.requireEachClosedOnce(t, nodeDatabases...) +} + +func TestSeedFailedStartReleasesDatabaseOnce(t *testing.T) { + cfg, err := config.ResetTestRoot(t.TempDir(), t.Name()) + require.NoError(t, err) + cfg.Mode = config.ModeSeed + cfg.P2P.ListenAddress = "tcp://127.0.0.1:0" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + key, err := types.LoadOrGenNodeKey(cfg.NodeKeyFile()) + require.NoError(t, err) + counter := &dbCloseCounter{closes: map[string]int{}} + svc, err := makeSeedNode(ctx, log.NewNopLogger(), cfg, counter.provider, key, defaultGenesisDocProviderFunc(cfg)) + require.NoError(t, err) + n := svc.(*seedNodeImpl) + failure := errors.New("PEX startup failed") + pex := &failedSeedPEX{err: failure} + pex.BaseService = *service.NewBaseService(log.NewNopLogger(), "test-pex", pex) + n.pexReactor = pex + + require.ErrorIs(t, n.Start(ctx), failure) + counter.requireEachClosedOnce(t, "peerstore") + + n.Stop() + n.Wait() + n.OnDrain() + require.ErrorContains(t, n.Start(ctx), "construct a new node") + counter.requireEachClosedOnce(t, "peerstore") +} + +type waitProbeService struct { + service.Service + onWait func() +} + +func (s waitProbeService) Wait() { s.Service.Wait(); s.onWait() } + +// Seed rollback joins PEX, then the router, and only then closes the peer store. +func TestSeedFailedStartReleasesDatabaseAfterJoiningChildren(t *testing.T) { + cfg, err := config.ResetTestRoot(t.TempDir(), t.Name()) + require.NoError(t, err) + cfg.Mode = config.ModeSeed + cfg.P2P.ListenAddress = "tcp://127.0.0.1:0" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + key, err := types.LoadOrGenNodeKey(cfg.NodeKeyFile()) + require.NoError(t, err) + counter := &dbCloseCounter{closes: map[string]int{}} + svc, err := makeSeedNode(ctx, log.NewNopLogger(), cfg, counter.provider, key, defaultGenesisDocProviderFunc(cfg)) + require.NoError(t, err) + n := svc.(*seedNodeImpl) + hooks := &seedRouterDrain{router: n.router, stopping: make(chan struct{}), release: make(chan struct{})} + n.router.BaseService = service.NewBaseService(log.NewNopLogger(), "test-router", hooks) + failure := errors.New("PEX startup failed") + pex := &failedSeedPEX{err: failure} + pex.BaseService = *service.NewBaseService(log.NewNopLogger(), "test-pex", pex) + closesAtPEXJoin := make(chan int, 1) + n.pexReactor = waitProbeService{Service: pex, onWait: func() { closesAtPEXJoin <- counter.closed("peerstore") }} + + returned := make(chan error, 1) + go func() { returned <- n.Start(ctx) }() + releaseRouter := sync.OnceFunc(func() { close(hooks.release) }) + defer func() { releaseRouter(); n.router.Wait(); n.Wait() }() + + select { + case closes := <-closesAtPEXJoin: + require.Zero(t, closes, "peer store closed before PEX was joined") + case <-time.After(5 * time.Second): + t.Fatal("rollback did not join PEX") + } + select { + case <-hooks.stopping: + case <-time.After(5 * time.Second): + t.Fatal("router was not canceled") + } + // The router worker cannot exit until released, so rollback must still be + // blocked joining it. The window only catches a close that arrives early. + select { + case err := <-returned: + t.Fatalf("Start returned before router cleanup: %v", err) + case <-time.After(50 * time.Millisecond): + } + require.Zero(t, counter.closed("peerstore"), "peer store closed before the router was joined") + + releaseRouter() + select { + case err := <-returned: + require.ErrorIs(t, err, failure) + case <-time.After(5 * time.Second): + t.Fatal("Start did not finish after router drained") + } + require.Equal(t, 1, counter.closed("peerstore")) +} diff --git a/node/seed.go b/node/seed.go index 9142fedee2..a1e63d2ce6 100644 --- a/node/seed.go +++ b/node/seed.go @@ -3,8 +3,8 @@ package node import ( "context" "fmt" - "net/http" "strings" + "sync/atomic" "time" "github.com/dashpay/tenderdash/config" @@ -32,10 +32,10 @@ type seedNodeImpl struct { nodeKey types.NodeKey // our node privkey // services - pexReactor service.Service // for exchanging peer addresses - prometheusSrv *http.Server + pexReactor service.Service // for exchanging peer addresses shutdownOps closer + released atomic.Bool // shutdownOps ran; set by OnDrain } // makeSeedNode returns a new seed node, containing only p2p, pex reactor @@ -97,20 +97,40 @@ func makeSeedNode( } // OnStart starts the Seed Node. It implements service.Service. -func (n *seedNodeImpl) OnStart(ctx context.Context) error { +func (n *seedNodeImpl) OnStart(ctx context.Context) (err error) { + if n.released.Load() { + return errResourcesReleased + } + ctx, cancelAttempt := context.WithCancel(ctx) + // Failure is final and BaseService will not call OnDrain. Workers it joins + // after this return serve only pprof and metrics. + defer func() { + if err != nil { + cancelAttempt() + n.pexReactor.Wait() + n.router.Wait() + n.OnDrain() + } + }() if n.config.RPC.PprofListenAddress != "" { - startPProfServer(ctx, *n.config.RPC) + startPProfServer(ctx, &n.BaseService, n.logger, *n.config.RPC) } now := tmtime.Now() genTime := n.genesisDoc.GenesisTime if genTime.After(now) { n.logger.Info("Genesis time is in the future. Sleeping until then...", "genTime", genTime) - time.Sleep(genTime.Sub(now)) + timer := time.NewTimer(genTime.Sub(now)) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } } if n.config.Instrumentation.Prometheus && n.config.Instrumentation.PrometheusListenAddr != "" { - n.prometheusSrv = startPrometheusServer(ctx, *n.config.Instrumentation) + startPrometheusServer(ctx, &n.BaseService, n.logger, *n.config.Instrumentation) } // Start the transport. @@ -127,7 +147,14 @@ func (n *seedNodeImpl) OnStop() { n.pexReactor.Wait() n.router.Wait() +} +// OnDrain releases resources after network and HTTP workers have finished, at +// most once. +func (n *seedNodeImpl) OnDrain() { + if !n.released.CompareAndSwap(false, true) { + return + } if err := n.shutdownOps(); err != nil { if strings.TrimSpace(err.Error()) != "" { n.logger.Error("problem shutting down additional services", "err", err) diff --git a/node/seed_lifecycle_test.go b/node/seed_lifecycle_test.go new file mode 100644 index 0000000000..3613d52fef --- /dev/null +++ b/node/seed_lifecycle_test.go @@ -0,0 +1,85 @@ +package node + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/config" + "github.com/dashpay/tenderdash/internal/p2p" + "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" + "github.com/dashpay/tenderdash/types" +) + +type seedRouterDrain struct { + router *p2p.Router + stopping chan struct{} + release chan struct{} +} + +func (r *seedRouterDrain) OnStart(ctx context.Context) error { + if err := r.router.OnStart(ctx); err != nil { + return err + } + r.router.Go(ctx, func(ctx context.Context) { + <-ctx.Done() + close(r.stopping) + <-r.release + }) + return nil +} +func (r *seedRouterDrain) OnStop() { r.router.OnStop() } + +type failedSeedPEX struct { + service.BaseService + err error +} + +func (p *failedSeedPEX) OnStart(context.Context) error { return p.err } +func (p *failedSeedPEX) OnStop() {} + +func TestSeedFailedStartDrainsRouter(t *testing.T) { + cfg, err := config.ResetTestRoot(t.TempDir(), t.Name()) + require.NoError(t, err) + cfg.Mode = config.ModeSeed + cfg.P2P.ListenAddress = "tcp://127.0.0.1:0" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + key, err := types.LoadOrGenNodeKey(cfg.NodeKeyFile()) + require.NoError(t, err) + svc, err := makeSeedNode(ctx, log.NewNopLogger(), cfg, config.DefaultDBProvider, key, defaultGenesisDocProviderFunc(cfg)) + require.NoError(t, err) + n := svc.(*seedNodeImpl) + hooks := &seedRouterDrain{router: n.router, stopping: make(chan struct{}), release: make(chan struct{})} + n.router.BaseService = service.NewBaseService(log.NewNopLogger(), "test-router", hooks) + failure := errors.New("PEX startup failed") + pex := &failedSeedPEX{err: failure} + pex.BaseService = *service.NewBaseService(log.NewNopLogger(), "test-pex", pex) + n.pexReactor = pex + returned := make(chan error, 1) + go func() { returned <- n.Start(ctx) }() + var release sync.Once + defer func() { release.Do(func() { close(hooks.release) }); n.router.Wait(); n.Wait() }() + select { + case <-hooks.stopping: + case <-time.After(5 * time.Second): + t.Fatal("router was not canceled") + } + select { + case err := <-returned: + t.Fatalf("Start returned before router cleanup: %v", err) + case <-time.After(50 * time.Millisecond): + } + release.Do(func() { close(hooks.release) }) + select { + case err := <-returned: + require.ErrorIs(t, err, failure) + case <-time.After(5 * time.Second): + t.Fatal("Start did not finish after router drained") + } +} diff --git a/node/setup.go b/node/setup.go index 300298c9ed..29f1d9dc79 100644 --- a/node/setup.go +++ b/node/setup.go @@ -5,8 +5,10 @@ import ( "context" "errors" "fmt" + "net" "slices" "strings" + "sync" "time" "github.com/dashpay/dashd-go/btcjson" @@ -650,12 +652,11 @@ func createBlockReplayer(n *nodeImpl) *consensus.BlockReplayer { ) } -// startPrometheusServer starts a Prometheus HTTP server, listening for metrics -// collectors on addr. -func startPrometheusServer(ctx context.Context, cfg config.InstrumentationConfig) *http.Server { - addr := cfg.PrometheusListenAddr +// startPrometheusServer serves Prometheus metrics on the configured address as +// work owned by owner. +func startPrometheusServer(ctx context.Context, owner *service.BaseService, logger log.Logger, cfg config.InstrumentationConfig) { srv := &http.Server{ - Addr: addr, + Addr: cfg.PrometheusListenAddr, ReadHeaderTimeout: httpReadHeaderTimeout, Handler: promhttp.InstrumentMetricHandler( prometheus.DefaultRegisterer, promhttp.HandlerFor( @@ -664,51 +665,67 @@ func startPrometheusServer(ctx context.Context, cfg config.InstrumentationConfig ), ), } - - signal := make(chan struct{}) - go func() { - select { - case <-ctx.Done(): - sctx, scancel := context.WithTimeout(context.Background(), time.Second) - defer scancel() - _ = srv.Shutdown(sctx) - case <-signal: - } - }() - - go func() { - if err := srv.ListenAndServe(); err != nil { - close(signal) - } - }() - - return srv + startHTTPServer(ctx, owner, logger, srv) } -// startPProfServer creates a new pprof server -// FIXME: implement as a Service -func startPProfServer(ctx context.Context, cfg config.RPCConfig) { - signal := make(chan struct{}) +// startPProfServer serves pprof on the configured address as work owned by owner. +func startPProfServer(ctx context.Context, owner *service.BaseService, logger log.Logger, cfg config.RPCConfig) { srv := &http.Server{ Addr: cfg.PprofListenAddress, ReadHeaderTimeout: httpReadHeaderTimeout, - Handler: nil, - } - go func() { - select { - case <-ctx.Done(): - sctx, scancel := context.WithTimeout(context.Background(), time.Second) - defer scancel() - _ = srv.Shutdown(sctx) - case <-signal: - } - }() + Handler: http.DefaultServeMux, + } + startHTTPServer(ctx, owner, logger, srv) +} + +// startHTTPServer binds srv.Addr and serves it until ctx is canceled. A bind +// failure is logged; the auxiliary server is skipped as before. +func startHTTPServer(ctx context.Context, owner *service.BaseService, logger log.Logger, srv *http.Server) { + ln, err := net.Listen("tcp", srv.Addr) + if err != nil { + logger.Error("failed to start HTTP server", "addr", srv.Addr, "err", err) + return + } + serveOwnedHTTP(ctx, owner, srv, ln) +} - go func() { - if err := srv.ListenAndServe(); err != nil { - close(signal) +// serveOwnedHTTP serves srv on ln as work of owner. On cancellation it rejects +// new requests, cancels request contexts and closes connections, so a stalled +// client cannot block shutdown, then joins the active handlers. +func serveOwnedHTTP(ctx context.Context, owner *service.BaseService, srv *http.Server, ln net.Listener) { + var mu sync.Mutex + var handlers sync.WaitGroup + stopping := false + next := srv.Handler + srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Serialize admission with the WaitGroup join below. + mu.Lock() + if stopping { + mu.Unlock() + http.Error(w, "server is stopping", http.StatusServiceUnavailable) + return } - }() + handlers.Add(1) + mu.Unlock() + defer handlers.Done() + next.ServeHTTP(w, r) + }) + srv.BaseContext = func(net.Listener) context.Context { return ctx } + if !owner.Go(ctx, func(ctx context.Context) { + <-ctx.Done() + mu.Lock() + stopping = true + mu.Unlock() + _ = srv.Close() + handlers.Wait() + }) { + _ = ln.Close() + return + } + // Serve closes ln, including when Close has already run. + if !owner.Go(ctx, func(context.Context) { _ = srv.Serve(ln) }) { + _ = ln.Close() + } } // stateSyncMetricsSetter is the consensus-side surface of the state sync handoff. diff --git a/privval/grpc/client.go b/privval/grpc/client.go index 352c8e07d4..bdfe2c09e9 100644 --- a/privval/grpc/client.go +++ b/privval/grpc/client.go @@ -32,20 +32,21 @@ type SignerClient struct { var _ types.PrivValidator = (*SignerClient)(nil) // NewSignerClient returns an instance of SignerClient. -// it will start the endpoint (if not already started) +// It takes ownership of conn, which Close closes. func NewSignerClient(conn *grpc.ClientConn, chainID string, log log.Logger) (*SignerClient, error) { sc := &SignerClient{ logger: log, chainID: chainID, + conn: conn, client: privvalproto.NewPrivValidatorAPIClient(conn), // Create the Private Validator Client } return sc, nil } -// Close closes the underlying connection +// Close closes the connection the client was created with. func (sc *SignerClient) Close() error { sc.logger.Info("Stopping service") if sc.conn != nil { diff --git a/privval/grpc/client_test.go b/privval/grpc/client_test.go index 6894da9224..877caebc56 100644 --- a/privval/grpc/client_test.go +++ b/privval/grpc/client_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/test/bufconn" @@ -184,6 +185,22 @@ func TestSignerClient_SignProposal(t *testing.T) { assert.Equal(t, pbWant.Signature, pbHave.Signature) } +// TestSignerClientCloseShutsDownConnection checks that Close releases the +// connection the client was built on: the node registers Close as the only +// teardown of a grpc:// signer connection. +func TestSignerClientCloseShutsDownConnection(t *testing.T) { + conn, err := grpc.NewClient("localhost", + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + require.NoError(t, err) + + client, err := tmgrpc.NewSignerClient(conn, chainID, log.NewTestingLogger(t)) + require.NoError(t, err) + + require.NoError(t, client.Close()) + assert.Equal(t, connectivity.Shutdown, conn.GetState()) +} + // Heartbeats and skip votes are not part of the remote signer protocol. The // refusal carries code 501 like the socket signer's, which retrying clients // treat as final rather than retrying until they time out. diff --git a/privval/signer_client.go b/privval/signer_client.go index cd79001ac1..73fb5412b0 100644 --- a/privval/signer_client.go +++ b/privval/signer_client.go @@ -46,6 +46,7 @@ func NewSignerClient(ctx context.Context, endpoint *SignerListenerEndpoint, chai // Close closes the underlying connection func (sc *SignerClient) Close() error { sc.endpoint.Stop() + sc.endpoint.Wait() err := sc.endpoint.Close() if err != nil { return err diff --git a/privval/signer_endpoint.go b/privval/signer_endpoint.go index 96ae1f5564..af3fc6737f 100644 --- a/privval/signer_endpoint.go +++ b/privval/signer_endpoint.go @@ -2,6 +2,7 @@ package privval import ( "context" + "errors" "fmt" "net" "time" @@ -24,6 +25,10 @@ type signerEndpoint struct { connMtx sync.Mutex conn net.Conn + // liveConn mirrors conn for interruptConnection, which cannot take connMtx + // while blocked I/O holds it. Lock order: connMtx, then liveMtx. + liveMtx sync.Mutex + liveConn net.Conn timeoutReadWrite time.Duration } @@ -49,6 +54,7 @@ func (se *signerEndpoint) GetAvailableConnection(connectionAvailableCh chan net. // Is there a connection ready? select { case se.conn = <-connectionAvailableCh: + se.publishConn() return true default: } @@ -64,6 +70,7 @@ func (se *signerEndpoint) WaitConnection(ctx context.Context, connectionAvailabl case <-ctx.Done(): return ctx.Err() case se.conn = <-connectionAvailableCh: + se.publishConn() case <-time.After(maxWait): return ErrConnectionTimeout } @@ -76,6 +83,7 @@ func (se *signerEndpoint) SetConnection(newConnection net.Conn) { se.connMtx.Lock() defer se.connMtx.Unlock() se.conn = newConnection + se.publishConn() } // IsConnected indicates if there is an active connection @@ -148,15 +156,34 @@ func (se *signerEndpoint) WriteMessage(msg privvalproto.Message) (err error) { return } +// interruptConnection closes the current connection without connMtx, so a +// read or write blocked while holding that lock fails promptly. +func (se *signerEndpoint) interruptConnection() { + se.liveMtx.Lock() + conn := se.liveConn + se.liveMtx.Unlock() + if conn != nil { + _ = conn.Close() + } +} + +// publishConn must be called with connMtx held after conn changes. +func (se *signerEndpoint) publishConn() { + se.liveMtx.Lock() + se.liveConn = se.conn + se.liveMtx.Unlock() +} + func (se *signerEndpoint) isConnected() bool { return se.conn != nil } func (se *signerEndpoint) dropConnection() { if se.conn != nil { - if err := se.conn.Close(); err != nil { + if err := se.conn.Close(); err != nil && !errors.Is(err, net.ErrClosed) { se.logger.Error("signerEndpoint::dropConnection", "err", err) } se.conn = nil + se.publishConn() } } diff --git a/privval/signer_listener_endpoint.go b/privval/signer_listener_endpoint.go index a66f961e97..ea3679f0cd 100644 --- a/privval/signer_listener_endpoint.go +++ b/privval/signer_listener_endpoint.go @@ -74,8 +74,8 @@ func (sl *SignerListenerEndpoint) OnStart(ctx context.Context) error { sl.pingInterval = time.Duration(sl.signerEndpoint.timeoutReadWrite.Milliseconds()*2/3) * time.Millisecond sl.pingTimer = time.NewTicker(sl.pingInterval) - go sl.serviceLoop(ctx) - go sl.pingLoop(ctx) + sl.Go(ctx, sl.serviceLoop) + sl.Go(ctx, sl.pingLoop) // serviceLoop exits on cancellation (e.g. Stop during startup) and may never // take this request. Returning nil keeps the successful-start path, so OnStop @@ -90,6 +90,8 @@ func (sl *SignerListenerEndpoint) OnStart(ctx context.Context) error { // OnStop implements service.Service func (sl *SignerListenerEndpoint) OnStop() { + // A request can hold instanceMtx across signer I/O; interrupt it first. + sl.interruptConnection() sl.instanceMtx.Lock() defer sl.instanceMtx.Unlock() _ = sl.Close() @@ -98,7 +100,6 @@ func (sl *SignerListenerEndpoint) OnStop() { if sl.listener != nil { if err := sl.listener.Close(); err != nil { sl.logger.Error("Closing Listener", "err", err) - sl.listener = nil } } @@ -155,7 +156,7 @@ func (sl *SignerListenerEndpoint) ensureConnection(ctx context.Context, maxWait } func (sl *SignerListenerEndpoint) acceptNewConnection() (net.Conn, error) { - if !sl.IsRunning() || sl.listener == nil { + if sl.listener == nil { return nil, fmt.Errorf("endpoint is closing") } @@ -194,6 +195,7 @@ func (sl *SignerListenerEndpoint) serviceLoop(ctx context.Context) { select { case sl.connectionAvailableCh <- conn: case <-ctx.Done(): + _ = conn.Close() return } } diff --git a/privval/signer_listener_endpoint_test.go b/privval/signer_listener_endpoint_test.go index be41c82493..d29240adf3 100644 --- a/privval/signer_listener_endpoint_test.go +++ b/privval/signer_listener_endpoint_test.go @@ -2,6 +2,7 @@ package privval import ( "context" + "io" "net" "testing" "time" @@ -14,6 +15,7 @@ import ( "github.com/dashpay/tenderdash/libs/log" tmnet "github.com/dashpay/tenderdash/libs/net" tmrand "github.com/dashpay/tenderdash/libs/rand" + privvalproto "github.com/dashpay/tenderdash/proto/tendermint/privval" "github.com/dashpay/tenderdash/types" ) @@ -29,6 +31,39 @@ type dialerTestCase struct { dialer SocketDialer } +type observedListener struct { + net.Listener + accepted chan struct{} +} + +func (l *observedListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err == nil { + close(l.accepted) + } + return conn, err +} + +func TestListenerStopClosesPendingConnection(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + observed := &observedListener{Listener: listener, accepted: make(chan struct{})} + endpoint := NewSignerListenerEndpoint(log.NewNopLogger(), observed, func(endpoint *SignerListenerEndpoint) { + endpoint.timeoutReadWrite = time.Hour + }) + require.NoError(t, endpoint.Start(context.Background())) + t.Cleanup(func() { endpoint.Stop(); endpoint.Wait() }) + conn, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + defer conn.Close() + <-observed.accepted + endpoint.Stop() + endpoint.Wait() + require.NoError(t, conn.SetReadDeadline(time.Now().Add(time.Second))) + _, err = conn.Read(make([]byte, 1)) + require.ErrorIs(t, err, io.EOF) +} + // TestSignerRemoteRetryTCPOnly will test connection retry attempts over TCP. We // don't need this for Unix sockets because the OS instantly knows the state of // both ends of the socket connection. This basically causes the @@ -244,3 +279,38 @@ func TestSignerListenerEndpointStartupCanceledBeforeConnectRequest(t *testing.T) require.False(t, sl.IsRunning()) } } + +// A request waiting for a signer that never answers holds the endpoint mutex +// across network I/O. Stop must interrupt that I/O instead of waiting for the +// read deadline. +func TestListenerStopInterruptsPendingRequest(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + endpoint := NewSignerListenerEndpoint(log.NewNopLogger(), listener, SignerListenerEndpointTimeoutReadWrite(time.Hour)) + require.NoError(t, endpoint.Start(context.Background())) + t.Cleanup(func() { endpoint.Stop(); endpoint.Wait() }) + signer, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + defer signer.Close() + + requested := make(chan error, 1) + go func() { + _, err := endpoint.SendRequest(context.Background(), mustWrapMsg(&privvalproto.PingRequest{})) + requested <- err + }() + // The request is on the wire, so SendRequest now holds the endpoint mutex. + require.NoError(t, signer.SetReadDeadline(time.Now().Add(5*time.Second))) + _, err = signer.Read(make([]byte, 1)) + require.NoError(t, err) + + stopped := make(chan struct{}) + go func() { endpoint.Stop(); close(stopped) }() + select { + case <-stopped: + case <-time.After(5 * time.Second): + _ = signer.Close() + <-stopped + t.Fatal("Stop waited for a pending signer read") + } + require.Error(t, <-requested) +} diff --git a/privval/signer_server.go b/privval/signer_server.go index 20554332f3..b9909bea7e 100644 --- a/privval/signer_server.go +++ b/privval/signer_server.go @@ -44,7 +44,7 @@ func NewSignerServer(endpoint *SignerDialerEndpoint, chainID string, privVal typ // OnStart implements service.Service. func (ss *SignerServer) OnStart(ctx context.Context) error { - go ss.serviceLoop(ctx) + ss.Go(ctx, ss.serviceLoop) return nil } @@ -62,7 +62,7 @@ func (ss *SignerServer) SetRequestHandler(validationRequestHandler ValidationReq } func (ss *SignerServer) servicePendingRequest(ctx context.Context) { - if !ss.IsRunning() { + if ctx.Err() != nil { return // Ignore error from closing. } diff --git a/rpc/client/http/ws.go b/rpc/client/http/ws.go index 3b43a7ee9b..9d1d3b2459 100644 --- a/rpc/client/http/ws.go +++ b/rpc/client/http/ws.go @@ -3,6 +3,7 @@ package http import ( "context" "encoding/json" + "errors" "fmt" "strings" "time" @@ -21,14 +22,36 @@ type wsEvents struct { Logger log.Logger ws *jsonrpcclient.WSClient - mtx sync.RWMutex + // changes orders upstream (un)subscribes: a subscriber is recorded, and an + // unused query confirmed, while holding it across the send. Taken before mtx. + changes chan struct{} + + mtx sync.RWMutex + // subscriptions maps each query, and its upstream subscription ID once known, + // to the local subscribers sharing its single upstream subscription. subscriptions map[string]*wsSubscription + + // lifetime bounds resubscribing and event delivery. It ends with the context + // passed to Start, or in Stop if the client shut down by itself first. + lifetime context.Context + endLifetime context.CancelFunc + listenerDone chan struct{} + // resubscribeDelay gives Tendermint time to restart after an RPC error. + resubscribeDelay time.Duration } +var errNotRunning = errors.New("websocket client is not running") + type wsSubscription struct { - res chan coretypes.ResultEvent - id string - query string + id string + query string + subscribers map[string]*wsSubscriber +} + +type wsSubscriber struct { + res chan coretypes.ResultEvent + // gone closes on unsubscribing, releasing a delivery its owner no longer reads. + gone chan struct{} } var _ rpcclient.SubscriptionClient = (*wsEvents)(nil) @@ -37,6 +60,9 @@ func newWsEvents(remote string) (*wsEvents, error) { w := &wsEvents{ Logger: log.NewNopLogger(), subscriptions: make(map[string]*wsSubscription), + changes: make(chan struct{}, 1), + + resubscribeDelay: time.Second, } var err error @@ -46,7 +72,7 @@ func newWsEvents(remote string) (*wsEvents, error) { } w.ws.OnReconnect(func() { // resubscribe immediately - w.redoSubscriptionsAfter(0 * time.Second) + w.redoSubscriptionsAfter(w.lifetime, 0) }) w.ws.Logger = w.Logger @@ -55,15 +81,44 @@ func newWsEvents(remote string) (*wsEvents, error) { // Start starts the websocket client and the event loop. func (w *wsEvents) Start(ctx context.Context) error { + w.lifetime, w.endLifetime = context.WithCancel(ctx) if err := w.ws.Start(ctx); err != nil { + w.endLifetime() return err } - go w.eventListener(ctx) + w.listenerDone = make(chan struct{}) + go w.eventListener(w.lifetime, w.listenerDone) return nil } -// Stop shuts down the websocket client. -func (w *wsEvents) Stop() error { return w.ws.Stop() } +// Stop waits for the websocket client, which joins reconnect callbacks, to shut +// down with the context passed to Start, then joins the event loop. +func (w *wsEvents) Stop() error { + err := w.ws.Stop() + w.stopListener() + return err +} + +// IsRunning reports whether the client has started and not yet shut down. +func (w *wsEvents) IsRunning() bool { + if w.listenerDone == nil { + return false + } + select { + case <-w.listenerDone: + return false + default: + return true + } +} + +// stopListener ends the event loop, even if blocked on a subscriber not reading. +func (w *wsEvents) stopListener() { + if w.listenerDone != nil { + w.endLifetime() + <-w.listenerDone + } +} // Subscribe implements SubscriptionClient by using WSClient to subscribe given // subscriber to query. By default, it returns a channel with cap=1. Error is @@ -76,9 +131,16 @@ func (w *wsEvents) Stop() error { return w.ws.Stop() } // The channel is never closed to prevent clients from seeing an erroneous // event. // +// Subscribers share one upstream subscription per query, which Tendermint keys +// by remote IP; subscriber only identifies the owner locally. +// // It returns an error if wsEvents is not running. -func (w *wsEvents) Subscribe(ctx context.Context, _subscriber, query string, +func (w *wsEvents) Subscribe(ctx context.Context, subscriber, query string, outCapacity ...int) (out <-chan coretypes.ResultEvent, err error) { + if err := w.lockChanges(ctx); err != nil { + return nil, err + } + defer w.unlockChanges() if err := w.ws.Subscribe(ctx, query); err != nil { return nil, err } @@ -91,69 +153,156 @@ func (w *wsEvents) Subscribe(ctx context.Context, _subscriber, query string, outc := make(chan coretypes.ResultEvent, outCap) w.mtx.Lock() defer w.mtx.Unlock() - // subscriber param is ignored because Tendermint will override it with - // remote IP anyway. - w.subscriptions[query] = &wsSubscription{res: outc, query: query} + sub, ok := w.subscriptions[query] + if !ok { + sub = &wsSubscription{query: query, subscribers: make(map[string]*wsSubscriber)} + w.subscriptions[query] = sub + } + if old, ok := sub.subscribers[subscriber]; ok { + close(old.gone) + } + sub.subscribers[subscriber] = &wsSubscriber{res: outc, gone: make(chan struct{})} return outc, nil } -// Unsubscribe implements SubscriptionClient by using WSClient to unsubscribe -// given subscriber from query. +// Unsubscribe implements SubscriptionClient by unsubscribing given subscriber +// from query, and using WSClient to unsubscribe from the upstream once no +// subscriber is left. // -// It returns an error if wsEvents is not running. -func (w *wsEvents) Unsubscribe(ctx context.Context, _subscriber, query string) error { - if err := w.ws.Unsubscribe(ctx, query); err != nil { - return err +// It returns an error if wsEvents is not running. The subscription is forgotten +// even then; see UnsubscribeAll. +func (w *wsEvents) Unsubscribe(ctx context.Context, subscriber, query string) error { + w.mtx.Lock() + var unused []string + if sub, ok := w.subscriptions[query]; ok && sub.query == query { + unused = w.removeSubscriber(sub, subscriber, unused) } + w.mtx.Unlock() + + return w.unsubscribeUpstream(ctx, unused) +} +// UnsubscribeAll implements SubscriptionClient by unsubscribing given +// subscriber from all the queries, and using WSClient to unsubscribe from the +// upstream the queries no other subscriber uses. +// +// It returns an error if wsEvents is not running. Subscriptions are forgotten +// even then: a reconnect must not restore one whose channel nobody reads, which +// would stall every other subscription on the single events loop. +func (w *wsEvents) UnsubscribeAll(ctx context.Context, subscriber string) error { w.mtx.Lock() - defer w.mtx.Unlock() - info, ok := w.subscriptions[query] - if ok { - if info.id != "" { - delete(w.subscriptions, info.id) + var unused []string + for key, sub := range w.subscriptions { + if key == sub.query { + unused = w.removeSubscriber(sub, subscriber, unused) } - delete(w.subscriptions, info.query) } + w.mtx.Unlock() - return nil + return w.unsubscribeUpstream(ctx, unused) } -// UnsubscribeAll implements SubscriptionClient by using WSClient to -// unsubscribe given subscriber from all the queries. -// -// It returns an error if wsEvents is not running. -func (w *wsEvents) UnsubscribeAll(ctx context.Context, _subscriber string) error { - if err := w.ws.UnsubscribeAll(ctx); err != nil { - return err +// lockChanges waits for the changes lock as long as ctx allows: its holder's send +// can block until the writer restarts. +func (w *wsEvents) lockChanges(ctx context.Context) error { + select { + case w.changes <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() } +} - w.mtx.Lock() - defer w.mtx.Unlock() - w.subscriptions = make(map[string]*wsSubscription) +func (w *wsEvents) unlockChanges() { <-w.changes } + +// removeSubscriber forgets subscriber's subscription to sub, and sub itself once +// unused, which it then appends to unused. Callers must hold mtx. +func (w *wsEvents) removeSubscriber(sub *wsSubscription, subscriber string, unused []string) []string { + s, ok := sub.subscribers[subscriber] + if !ok { + return unused + } + close(s.gone) + delete(sub.subscribers, subscriber) + if len(sub.subscribers) > 0 { + return unused + } + delete(w.subscriptions, sub.query) + if sub.id != "" { + delete(w.subscriptions, sub.id) + } + return append(unused, sub.query) +} +// unsubscribeUpstream unsubscribes from queries still unused once it holds the +// changes lock. Callers must not hold mtx: sending can block until the writer +// restarts. After an error, eventListener drops events for queries left subscribed. +func (w *wsEvents) unsubscribeUpstream(ctx context.Context, queries []string) error { + if len(queries) == 0 { + if !w.IsRunning() { + return errNotRunning + } + return nil + } + if err := w.lockChanges(ctx); err != nil { + return err + } + defer w.unlockChanges() + for _, query := range queries { + // Lock, not RLock: eventListener writes to the map under RLock. + w.mtx.Lock() + _, used := w.subscriptions[query] + w.mtx.Unlock() + if used { + continue + } + if err := w.ws.Unsubscribe(ctx, query); err != nil { + return err + } + } return nil } // After being reconnected, it is necessary to redo subscription to server // otherwise no data will be automatically received. -func (w *wsEvents) redoSubscriptionsAfter(d time.Duration) { - time.Sleep(d) - - ctx := context.TODO() +func (w *wsEvents) redoSubscriptionsAfter(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + return + } + // Sending can block until the writer restarts, so no lock is held across it. + // eventListener writes to the map under RLock, so reading it here needs Lock. w.mtx.Lock() - defer w.mtx.Unlock() - + queries := make([]string, 0, len(w.subscriptions)) for q, info := range w.subscriptions { if q != "" && q == info.id { continue } - err := w.ws.Subscribe(ctx, q) - if err != nil { + queries = append(queries, q) + } + w.mtx.Unlock() + + for _, q := range queries { + w.mtx.Lock() + _, ok := w.subscriptions[q] + w.mtx.Unlock() + if !ok { + continue + } + // If q is forgotten during this send, it stays subscribed upstream until + // the next reconnect. That is harmless: eventListener drops events for + // unknown queries, and subscribing to q again gets ErrAlreadySubscribed, + // which it ignores. + if err := w.ws.Subscribe(ctx, q); err != nil { w.Logger.Error("failed to resubscribe", "query", q, "err", err) + w.mtx.Lock() delete(w.subscriptions, q) + w.mtx.Unlock() } } } @@ -162,7 +311,8 @@ func isErrAlreadySubscribed(err error) bool { return strings.Contains(err.Error(), pubsub.ErrAlreadySubscribed.Error()) } -func (w *wsEvents) eventListener(ctx context.Context) { +func (w *wsEvents) eventListener(ctx context.Context, done chan<- struct{}) { + defer close(done) for { select { case resp, ok := <-w.ws.ResponsesCh: @@ -177,9 +327,9 @@ func (w *wsEvents) eventListener(ctx context.Context) { // We can ignore ErrAlreadySubscribed, but need to retry in other // cases. if !isErrAlreadySubscribed(resp.Error) { - // Resubscribe after 1 second to give Tendermint time to restart (if + // Resubscribe after a delay to give Tendermint time to restart (if // crashed). - w.redoSubscriptionsAfter(1 * time.Second) + w.redoSubscriptionsAfter(ctx, w.resubscribeDelay) } continue } @@ -192,18 +342,21 @@ func (w *wsEvents) eventListener(ctx context.Context) { } w.mtx.RLock() - out, ok := w.subscriptions[result.Query] - if ok { + var subscribers []*wsSubscriber + if out, ok := w.subscriptions[result.Query]; ok { if _, idOk := w.subscriptions[result.SubscriptionID]; !idOk { out.id = result.SubscriptionID w.subscriptions[result.SubscriptionID] = out } + for _, s := range out.subscribers { + subscribers = append(subscribers, s) + } } - w.mtx.RUnlock() - if ok { + for _, s := range subscribers { select { - case out.res <- *result: + case s.res <- *result: + case <-s.gone: case <-ctx.Done(): return } diff --git a/rpc/client/http/ws_join_leave_stress_test.go b/rpc/client/http/ws_join_leave_stress_test.go new file mode 100644 index 0000000000..ee4eb662e5 --- /dev/null +++ b/rpc/client/http/ws_join_leave_stress_test.go @@ -0,0 +1,101 @@ +package http + +import ( + "context" + "fmt" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// Concurrent joins and leaves of one query must leave the upstream subscribed +// exactly when some local subscriber remains. +func TestJoinLeaveStressKeepsUpstreamConsistent(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &orderedUpstream{} + srv := httptest.NewServer(upstream) + t.Cleanup(srv.Close) + w, err := newWsEvents(srv.URL) + require.NoError(t, err) + require.NoError(t, w.Start(ctx)) + t.Cleanup(func() { cancel(); _ = w.Stop() }) + + for round := 0; round < 300; round++ { + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(2) + name := fmt.Sprintf("s%d-%d", round, i) + go func() { + defer wg.Done() + _, err := w.Subscribe(ctx, name, "q", 100) + require.NoError(t, err) + }() + go func(prev string) { + defer wg.Done() + require.NoError(t, w.UnsubscribeAll(ctx, prev)) + }(fmt.Sprintf("s%d-%d", round-1, i)) + } + wg.Wait() + // Settle: a sync request on a separate query follows every send above. + _, err := w.Subscribe(ctx, "sync", fmt.Sprintf("sync%d", round)) + require.NoError(t, err) + require.Eventually(t, func() bool { + _, s := upstream.state() + return s[fmt.Sprintf("sync%d", round)] + }, 5*time.Second, time.Millisecond) + w.mtx.Lock() + sub, local := w.subscriptions["q"] + n := 0 + if local { + n = len(sub.subscribers) + } + w.mtx.Unlock() + reqs, s := upstream.state() + if local != s["q"] { + t.Fatalf("round %d: local q=%v (%d subscribers) upstream q=%v; last requests %v", + round, local, n, s["q"], reqs[max(0, len(reqs)-12):]) + } + } +} + +// Shutdown while a Subscribe holds the changes lock blocked in its send, and a +// cleanup without deadline waits for that lock, must release both and Stop. +func TestStopWhileSubscribeHoldsChangesLock(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{} + srv := httptest.NewServer(upstream) + t.Cleanup(srv.Close) + w, err := newWsEvents(srv.URL) + require.NoError(t, err) + require.NoError(t, w.Start(ctx)) + upstream.subscribeAs(ctx, t, w, "A", "gone") + require.Eventually(t, func() bool { return len(upstream.subscribesOnCurrentConnection()) == 1 }, + 5*time.Second, time.Millisecond) + reconnecting(t, w, upstream) + srv.Close() // redials keep failing + + subscribed := make(chan error, 1) + go func() { _, err := w.Subscribe(context.Background(), "B", "other"); subscribed <- err }() + require.Eventually(t, func() bool { return len(w.changes) == 1 }, 5*time.Second, time.Millisecond) + cleaned := make(chan error, 1) + go func() { cleaned <- w.UnsubscribeAll(context.Background(), "A") }() + time.Sleep(50 * time.Millisecond) + + cancel() + stopped := make(chan error, 1) + go func() { stopped <- w.Stop() }() + for name, ch := range map[string]chan error{"Stop": stopped, "Subscribe": subscribed, "UnsubscribeAll": cleaned} { + select { + case err := <-ch: + t.Logf("%s returned %v", name, err) + case <-time.After(10 * time.Second): + t.Fatalf("%s hung after shutdown", name) + } + } + require.Empty(t, w.changes, "changes lock leaked") +} diff --git a/rpc/client/http/ws_join_leave_test.go b/rpc/client/http/ws_join_leave_test.go new file mode 100644 index 0000000000..f7876ef8f9 --- /dev/null +++ b/rpc/client/http/ws_join_leave_test.go @@ -0,0 +1,117 @@ +package http + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "sync" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/require" + + rpctypes "github.com/dashpay/tenderdash/rpc/jsonrpc/types" +) + +// orderedUpstream records every request on the current session, in order, and +// models the upstream subscription state per query. +type orderedUpstream struct { + mtx sync.Mutex + conn *websocket.Conn + requests []string + subscribed map[string]bool +} + +func (u *orderedUpstream) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + conn, err := (&websocket.Upgrader{}).Upgrade(rw, r, nil) + if err != nil { + return + } + defer conn.Close() + u.mtx.Lock() + u.conn = conn + u.requests = nil + u.subscribed = map[string]bool{} + u.mtx.Unlock() + for { + var req rpctypes.RPCRequest + if err := conn.ReadJSON(&req); err != nil { + return + } + var params struct { + Query string `json:"query"` + } + _ = json.Unmarshal(req.Params, ¶ms) + u.mtx.Lock() + u.requests = append(u.requests, req.Method+" "+params.Query) + switch req.Method { + case "subscribe": + u.subscribed[params.Query] = true + case "unsubscribe": + delete(u.subscribed, params.Query) + } + u.mtx.Unlock() + } +} + +func (u *orderedUpstream) state() ([]string, map[string]bool) { + u.mtx.Lock() + defer u.mtx.Unlock() + s := map[string]bool{} + for k, v := range u.subscribed { + s[k] = v + } + return append([]string(nil), u.requests...), s +} + +// Subscriber B joins query q while subscriber A, its last other user, leaves it. +// Afterwards B is registered locally, so the upstream must still be subscribed to q. +func TestConcurrentSubscribeAndLastUnsubscribeKeepsUpstream(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &orderedUpstream{} + srv := httptest.NewServer(upstream) + t.Cleanup(srv.Close) + w, err := newWsEvents(srv.URL) + require.NoError(t, err) + require.NoError(t, w.Start(ctx)) + t.Cleanup(func() { cancel(); _ = w.Stop() }) + + _, err = w.Subscribe(ctx, "A", "q") + require.NoError(t, err) + require.Eventually(t, func() bool { _, s := upstream.state(); return s["q"] }, 5*time.Second, time.Millisecond) + + // While the writer is down, sends queue in FIFO order on the unbuffered channel. + upstream.mtx.Lock() + require.NoError(t, upstream.conn.Close()) + upstream.mtx.Unlock() + require.Eventually(t, w.ws.IsReconnecting, 5*time.Second, time.Millisecond) + + bDone := make(chan error, 1) + go func() { + _, err := w.Subscribe(ctx, "B", "q") // blocks in the upstream send, not yet registered + bDone <- err + }() + time.Sleep(100 * time.Millisecond) + aDone := make(chan error, 1) + go func() { aDone <- w.UnsubscribeAll(ctx, "A") }() + + require.NoError(t, <-bDone) + require.NoError(t, <-aDone) + + w.mtx.RLock() + sub, registered := w.subscriptions["q"] + bRegistered := registered && sub.subscribers["B"] != nil + w.mtx.RUnlock() + require.True(t, bRegistered, "B is registered locally") + + // B's subscribe must reach this session; A's unsubscribe, if sent, may follow it. + require.Eventually(t, func() bool { r, _ := upstream.state(); return slices.Contains(r, "subscribe q") }, + 5*time.Second, time.Millisecond) + time.Sleep(200 * time.Millisecond) + reqs, s := upstream.state() + require.True(t, s["q"], "upstream unsubscribed q while B still needs it; requests on session: %v", reqs) +} diff --git a/rpc/client/http/ws_test.go b/rpc/client/http/ws_test.go new file mode 100644 index 0000000000..310f5adbf3 --- /dev/null +++ b/rpc/client/http/ws_test.go @@ -0,0 +1,498 @@ +package http + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/rpc/coretypes" + rpctypes "github.com/dashpay/tenderdash/rpc/jsonrpc/types" + "github.com/dashpay/tenderdash/types" +) + +// fakeUpstream replies to each subscribe with the events scripted for its query. +type fakeUpstream struct { + mtx sync.Mutex + events map[string][]string + failed map[string]bool // queries whose subscribe gets an RPC error + gates map[string]chan struct{} // hold a query's events until closed + conn *websocket.Conn + subscribes []string + others []string // non-subscribe requests on any connection, as "method query" +} + +func (u *fakeUpstream) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + upgrader := websocket.Upgrader{} + conn, err := upgrader.Upgrade(rw, r, nil) + if err != nil { + return + } + defer conn.Close() + u.mtx.Lock() + u.conn = conn + u.subscribes = nil + u.mtx.Unlock() + for { + var req rpctypes.RPCRequest + if err := conn.ReadJSON(&req); err != nil { + return + } + var params struct { + Query string `json:"query"` + } + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + return + } + u.mtx.Lock() + if req.Method != "subscribe" { + u.others = append(u.others, strings.TrimSpace(req.Method+" "+params.Query)) + u.mtx.Unlock() + continue + } + u.subscribes = append(u.subscribes, params.Query) + events, failed, gate := u.events[params.Query], u.failed[params.Query], u.gates[params.Query] + u.mtx.Unlock() + if gate != nil { + <-gate + } + if failed { + if err := conn.WriteJSON(req.MakeErrorf(rpctypes.CodeInternalError, "subscribe failed")); err != nil { + return + } + continue + } + for _, query := range events { + res := req.MakeResponse(coretypes.ResultEvent{ + Query: query, + SubscriptionID: "id-" + query, + Data: types.EventDataNewRound{}, + }) + if err := conn.WriteJSON(res); err != nil { + return + } + } + } +} + +// subscribe subscribes w to query and lets the upstream send the query's events +// only afterwards. wsEvents records a query after sending its request, so it drops +// events that arrive first. +func (u *fakeUpstream) subscribe(ctx context.Context, t *testing.T, w *wsEvents, query string) <-chan coretypes.ResultEvent { + t.Helper() + return u.subscribeAs(ctx, t, w, "", query) +} + +func (u *fakeUpstream) subscribeAs(ctx context.Context, t *testing.T, w *wsEvents, + subscriber, query string) <-chan coretypes.ResultEvent { + t.Helper() + gate := make(chan struct{}) + defer close(gate) + u.mtx.Lock() + if u.gates == nil { + u.gates = make(map[string]chan struct{}) + } + u.gates[query] = gate + u.mtx.Unlock() + out, err := w.Subscribe(ctx, subscriber, query) + require.NoError(t, err) + return out +} + +func (u *fakeUpstream) setEvents(events map[string][]string) { + u.mtx.Lock() + defer u.mtx.Unlock() + u.events = events +} + +// dropConnection ends the session abruptly, which makes the client reconnect. +func (u *fakeUpstream) dropConnection(t *testing.T) { + t.Helper() + u.mtx.Lock() + defer u.mtx.Unlock() + require.NoError(t, u.conn.Close()) +} + +func (u *fakeUpstream) otherRequests() []string { + u.mtx.Lock() + defer u.mtx.Unlock() + return append([]string(nil), u.others...) +} + +func (u *fakeUpstream) subscribesOnCurrentConnection() []string { + u.mtx.Lock() + defer u.mtx.Unlock() + return append([]string(nil), u.subscribes...) +} + +func startWsEvents(ctx context.Context, t *testing.T, upstream *fakeUpstream) *wsEvents { + t.Helper() + srv := httptest.NewServer(upstream) + t.Cleanup(srv.Close) + w, err := newWsEvents(srv.URL) + require.NoError(t, err) + require.NoError(t, w.Start(ctx)) + // The test's deferred cancel has run by then, so no worker outlives its test. + t.Cleanup(func() { requireStopReturns(t, w) }) + return w +} + +// reconnecting drops the session and returns once the writer has stopped. The +// first redial happens at least a second later. +func reconnecting(t *testing.T, w *wsEvents, upstream *fakeUpstream) { + t.Helper() + upstream.dropConnection(t) + require.Eventually(t, w.ws.IsReconnecting, 5*time.Second, time.Millisecond) +} + +func requireEvent(t *testing.T, out <-chan coretypes.ResultEvent, query string) { + t.Helper() + select { + case event := <-out: + require.Equal(t, query, event.Query) + case <-time.After(5 * time.Second): + t.Fatalf("event for %q was not delivered", query) + } +} + +// A cleanup that times out during a reconnect must still forget the subscription, +// or the reconnect restores it and its unread channel stalls every other event. +func TestUnsubscribeAllTimeoutDuringReconnectForgetsSubscriptions(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{events: map[string][]string{ + "other": {"gone", "gone", "gone", "other"}, + }} + w := startWsEvents(ctx, t, upstream) + _, err := w.Subscribe(ctx, "", "gone") + require.NoError(t, err) + require.Eventually(t, func() bool { return len(upstream.subscribesOnCurrentConnection()) == 1 }, + 5*time.Second, time.Millisecond) + + reconnecting(t, w, upstream) + cleanupCtx, cancelCleanup := context.WithTimeout(ctx, 10*time.Millisecond) + defer cancelCleanup() + require.ErrorIs(t, w.UnsubscribeAll(cleanupCtx, ""), context.DeadlineExceeded) + + other := upstream.subscribe(ctx, t, w, "other") + requireEvent(t, other, "other") + require.NotContains(t, upstream.subscribesOnCurrentConnection(), "gone", + "the reconnect restored a subscription whose cleanup timed out") +} + +// One subscriber's cleanup, whether sent upstream or timed out during a reconnect, +// must leave every other subscriber's queries subscribed and delivering. All +// subscribers share one upstream session, so the cleanup may only unsubscribe +// queries nobody else uses. +func TestUnsubscribeAllKeepsOtherSubscribers(t *testing.T) { + for name, tc := range map[string]struct { + bQuery string + timeout bool + sent []string + }{ + "sent": {bQuery: "b", sent: []string{"unsubscribe a"}}, + "timeout during reconnect": {bQuery: "b", timeout: true}, + "shared query": {bQuery: "a"}, + } { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{} + w := startWsEvents(ctx, t, upstream) + a := upstream.subscribeAs(ctx, t, w, "A", "a") + b := upstream.subscribeAs(ctx, t, w, "B", tc.bQuery) + // Dropping a session with unread requests resets it, which the client misses. + require.Eventually(t, func() bool { return len(upstream.subscribesOnCurrentConnection()) == 2 }, + 5*time.Second, time.Millisecond) + + if tc.timeout { + reconnecting(t, w, upstream) + cleanupCtx, cancelCleanup := context.WithTimeout(ctx, 10*time.Millisecond) + defer cancelCleanup() + require.ErrorIs(t, w.UnsubscribeAll(cleanupCtx, "A"), context.DeadlineExceeded) + } else { + require.NoError(t, w.UnsubscribeAll(ctx, "A")) + // The upstream handles requests in order, so this event follows the cleanup. + upstream.setEvents(map[string][]string{"sync": {"sync"}}) + requireEvent(t, upstream.subscribeAs(ctx, t, w, "C", "sync"), "sync") + require.Equal(t, tc.sent, upstream.otherRequests()) + reconnecting(t, w, upstream) + } + // Served only once the reconnect resubscribes. + upstream.setEvents(map[string][]string{"a": {"a"}, "b": {"b"}}) + + requireEvent(t, b, tc.bQuery) + require.Empty(t, a, "a cleaned up subscriber received an event") + if tc.bQuery != "a" { + require.NotContains(t, upstream.subscribesOnCurrentConnection(), "a", + "the reconnect restored a cleaned up subscriber's query") + } + }) + } +} + +// A cleanup timing out behind another subscriber's send during a reconnect must +// still forget its subscriptions, or the reconnect restores an unread channel. +func TestUnsubscribeAllTimeoutBehindSubscribeForgetsSubscriptions(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{} + w := startWsEvents(ctx, t, upstream) + upstream.subscribeAs(ctx, t, w, "A", "gone") + require.Eventually(t, func() bool { return len(upstream.subscribesOnCurrentConnection()) == 1 }, + 5*time.Second, time.Millisecond) + + reconnecting(t, w, upstream) + subscribed := make(chan error, 1) + go func() { + _, err := w.Subscribe(ctx, "B", "other") + subscribed <- err + }() + require.Eventually(t, func() bool { return len(w.changes) == 1 }, 5*time.Second, time.Millisecond) + cleanupCtx, cancelCleanup := context.WithTimeout(ctx, 10*time.Millisecond) + defer cancelCleanup() + require.ErrorIs(t, w.UnsubscribeAll(cleanupCtx, "A"), context.DeadlineExceeded) + + require.NoError(t, <-subscribed) + upstream.setEvents(map[string][]string{"sync": {"sync"}}) + requireEvent(t, upstream.subscribeAs(ctx, t, w, "C", "sync"), "sync") + require.NotContains(t, upstream.subscribesOnCurrentConnection(), "gone", + "the reconnect restored a subscription whose cleanup timed out") +} + +// Forgetting a subscription must also release a delivery blocked on its unread channel. +func TestUnsubscribeAllReleasesBlockedDelivery(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{events: map[string][]string{ + "gone": {"gone", "gone", "gone"}, + "other": {"other"}, + }} + w := startWsEvents(ctx, t, upstream) + gone := upstream.subscribe(ctx, t, w, "gone") + // Taking one event lets the next fill the buffer, so the third has to block. + requireEvent(t, gone, "gone") + require.Eventually(t, func() bool { return len(gone) == 1 }, 5*time.Second, time.Millisecond) + // Nothing observable marks the delivery blocking; this window only lets it get + // there, and cannot fail correct code. + <-time.After(50 * time.Millisecond) + + require.NoError(t, w.UnsubscribeAll(ctx, "")) + other := upstream.subscribe(ctx, t, w, "other") + requireEvent(t, other, "other") +} + +// A clear landing during a resubscribe must not wait for it, and stops it restoring +// more than the one query already being sent. That one stays subscribed upstream, +// but only until the next reconnect, and its events are dropped. +func TestUnsubscribeAllDuringResubscribe(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{} + w := startWsEvents(ctx, t, upstream) + for _, query := range []string{"a", "b"} { + _, err := w.Subscribe(ctx, "", query) + require.NoError(t, err) + } + require.Eventually(t, func() bool { return len(upstream.subscribesOnCurrentConnection()) == 2 }, + 5*time.Second, time.Millisecond) + // Only after the reconnect: unread events would stop the client noticing the drop. + upstream.setEvents(map[string][]string{ + "a": {"a", "a", "a"}, + "b": {"b", "b", "b"}, + "other": {"other"}, + }) + + reconnecting(t, w, upstream) + resubscribed := make(chan struct{}) + go func() { defer close(resubscribed); w.redoSubscriptionsAfter(ctx, 0) }() + // Nothing observable marks the resubscribe blocking on the stopped writer; this + // window only lets it get there, and cannot fail correct code. + <-time.After(50 * time.Millisecond) + + cleanupCtx, cancelCleanup := context.WithTimeout(ctx, 10*time.Millisecond) + defer cancelCleanup() + unsubscribed := make(chan error, 1) + go func() { unsubscribed <- w.UnsubscribeAll(cleanupCtx, "") }() + select { + case err := <-unsubscribed: + require.ErrorIs(t, err, context.DeadlineExceeded) + case <-resubscribed: + t.Fatal("UnsubscribeAll waited for the resubscribe to finish") + } + + select { + case <-resubscribed: + case <-time.After(10 * time.Second): + t.Fatal("resubscribe did not finish after the reconnect") + } + other := upstream.subscribe(ctx, t, w, "other") + requireEvent(t, other, "other") + restored := 0 + for _, query := range upstream.subscribesOnCurrentConnection() { + if query != "other" { + restored++ + } + } + require.LessOrEqual(t, restored, 1, "the resubscribe restored more than the query it was already sending") +} + +// Every UnsubscribeAll gets a fresh signal: repeating it must not panic, and a +// later subscription must still receive all events through its small buffer. +func TestSubscribeAfterRepeatedUnsubscribeAllBlocksNormally(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + events := make([]string, 16) + for i := range events { + events[i] = "later" + } + upstream := &fakeUpstream{events: map[string][]string{"later": events}} + w := startWsEvents(ctx, t, upstream) + require.NoError(t, w.UnsubscribeAll(ctx, "")) + require.NoError(t, w.UnsubscribeAll(ctx, "")) + + later := upstream.subscribe(ctx, t, w, "later") + for range events { + require.Eventually(t, func() bool { return len(later) == 1 }, 5*time.Second, time.Millisecond) + requireEvent(t, later, "later") + } +} + +// goroutinesIn counts running goroutines with a frame in the named function. +func goroutinesIn(function string) int { + stacks := make([]byte, 1<<24) + return strings.Count(string(stacks[:runtime.Stack(stacks, true)]), function+"(") +} + +// requireListenerExited must hold as soon as the join returns, so it does not wait. +func requireListenerExited(t *testing.T, w *wsEvents) { + t.Helper() + select { + case <-w.listenerDone: + default: + t.Fatal("eventListener outlived the join") + } +} + +func requireStopReturns(t *testing.T, w *wsEvents) { + t.Helper() + stopped := make(chan error, 1) + go func() { stopped <- w.Stop() }() + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("Stop did not return") + } +} + +// An upstream RPC error sends eventListener into a delayed resubscribe. Ending the +// context there must end the worker, not strand it on a send nobody receives. +func TestStopJoinsListenerDuringResubscribe(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + srv := httptest.NewServer(&fakeUpstream{failed: map[string]bool{"q": true}}) + t.Cleanup(srv.Close) + w, err := newWsEvents(srv.URL) + require.NoError(t, err) + w.resubscribeDelay = time.Hour // only cancellation can end this delay + require.NoError(t, w.Start(ctx)) + t.Cleanup(func() { requireStopReturns(t, w) }) + _, err = w.Subscribe(ctx, "", "q") + require.NoError(t, err) + require.Eventually(t, func() bool { return goroutinesIn("(*wsEvents).redoSubscriptionsAfter") == 1 }, + 5*time.Second, time.Millisecond) + + cancel() + requireStopReturns(t, w) + requireListenerExited(t, w) +} + +// A subscriber that stopped reading must not turn the join into a shutdown hang. +func TestStopJoinsListenerBlockedOnSubscriber(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{events: map[string][]string{"q": {"q", "q", "q"}}} + w := startWsEvents(ctx, t, upstream) + q := upstream.subscribe(ctx, t, w, "q") + require.Eventually(t, func() bool { return len(q) == 1 }, 5*time.Second, time.Millisecond) + + cancel() + requireStopReturns(t, w) + requireListenerExited(t, w) +} + +// After the client shuts down by itself the context can still be live, and then +// only Stop can release a delivery no subscriber is reading. +func TestStopListenerReleasesDeliveryWithLiveContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{events: map[string][]string{"q": {"q", "q", "q"}}} + w := startWsEvents(ctx, t, upstream) + q := upstream.subscribe(ctx, t, w, "q") + require.Eventually(t, func() bool { return len(q) == 1 }, 5*time.Second, time.Millisecond) + + stopped := make(chan struct{}) + go func() { w.stopListener(); close(stopped) }() + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("stopListener blocked on a subscriber not reading") + } + requireListenerExited(t, w) +} + +// Starting fails cleanly, and a client that never ran has nothing to join. +func TestStopWithoutRunningListener(t *testing.T) { + w, err := newWsEvents("tcp://127.0.0.1:1") + require.NoError(t, err) + require.Error(t, w.Start(context.Background())) + requireStopReturns(t, w) +} + +// Stop must not report completion while the event loop can still run, here held +// on a lock that does not observe cancellation. +func TestStopWaitsForListenerToExit(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{events: map[string][]string{"q": {"q"}}} + w := startWsEvents(ctx, t, upstream) + w.mtx.Lock() + unlock := sync.OnceFunc(w.mtx.Unlock) + defer unlock() + require.NoError(t, w.ws.Subscribe(ctx, "q")) + require.Eventually(t, func() bool { + stacks := make([]byte, 1<<24) + for _, stack := range strings.Split(string(stacks[:runtime.Stack(stacks, true)]), "\n\n") { + if strings.Contains(stack, "(*wsEvents).eventListener(") && strings.Contains(stack, "RLock") { + return true + } + } + return false + }, 5*time.Second, time.Millisecond) + + cancel() + stopped := make(chan error, 1) + go func() { stopped <- w.Stop() }() + // Nothing observable marks Stop blocking; this window cannot fail correct code. + select { + case err := <-stopped: + t.Fatalf("Stop returned while the event loop was still running: %v", err) + case <-time.After(50 * time.Millisecond): + } + unlock() + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("Stop did not return after the event loop could exit") + } +} diff --git a/rpc/jsonrpc/client/http_json_client.go b/rpc/jsonrpc/client/http_json_client.go index d5d6ae8e3e..e94dca3a8f 100644 --- a/rpc/jsonrpc/client/http_json_client.go +++ b/rpc/jsonrpc/client/http_json_client.go @@ -370,7 +370,7 @@ func dialParamsFromURL(pURL *parsedURL) (string, string) { func makeHTTPDialer(protocol, addr string) func(context.Context, string, string) (net.Conn, error) { return func(ctx context.Context, _, _ string) (net.Conn, error) { - return net.DialTimeout(protocol, addr, 10*time.Second) + return (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, protocol, addr) } } diff --git a/rpc/jsonrpc/client/ws_client.go b/rpc/jsonrpc/client/ws_client.go index 8082e9ded3..64927ad9d5 100644 --- a/rpc/jsonrpc/client/ws_client.go +++ b/rpc/jsonrpc/client/ws_client.go @@ -3,6 +3,7 @@ package client import ( "context" "encoding/json" + "errors" "fmt" mrand "math/rand" "net" @@ -20,14 +21,19 @@ import ( // wsOptions carries optional settings for a websocket connection. type wsOptions struct { MaxReconnectAttempts uint // maximum attempts to reconnect + HandshakeTimeout time.Duration // deadline for dialing and the WebSocket handshake ReadWait time.Duration // deadline for any read op WriteWait time.Duration // deadline for any write op PingPeriod time.Duration // frequency with which pings are sent } +// errClientStopped reports a send to a client that has shut down. +var errClientStopped = errors.New("websocket client stopped") + // defaultWSOptions are the default websocket connection settings. var defaultWSOptions = wsOptions{ MaxReconnectAttempts: 10, // first: 2 sec, last: 17 min. + HandshakeTimeout: 10 * time.Second, WriteWait: 10 * time.Second, ReadWait: 0, PingPeriod: 0, @@ -65,12 +71,20 @@ type WSClient struct { //nolint: maligned protocol string wg sync.WaitGroup + // reconnectDone closes when reconnectRoutine exits; no workers start after that. + reconnectDone chan struct{} + closeResponses sync.Once + // callbacks tracks running onReconnect calls. Only reconnectRoutine adds to it. + callbacks sync.WaitGroup mtx sync.RWMutex reconnecting bool nextReqID int // sentIDs map[types.JSONRPCIntID]bool // IDs of the requests currently in flight + // Time allowed to dial the server and finish the WebSocket handshake. + handshakeTimeout time.Duration + // Time allowed to write a message to the server. 0 means block until operation succeeds. writeWait time.Duration @@ -101,6 +115,7 @@ func NewWS(remoteAddr, endpoint string) (*WSClient, error) { DialerContext: dialFn, Endpoint: endpoint, maxReconnectAttempts: opts.MaxReconnectAttempts, + handshakeTimeout: opts.HandshakeTimeout, readWait: opts.ReadWait, writeWait: opts.WriteWait, pingPeriod: opts.PingPeriod, @@ -113,7 +128,8 @@ func NewWS(remoteAddr, endpoint string) (*WSClient, error) { // OnReconnect sets the callback, which will be called every time after // successful reconnect. -// Could only be set before Start. +// Could only be set before Start. The callback must return once the context passed +// to Start ends, and must not call Stop, which joins it and would never return. func (c *WSClient) OnReconnect(cb func()) { c.onReconnect = cb } @@ -127,7 +143,7 @@ func (c *WSClient) String() string { // service routines run until ctx terminates. To wait for the client to exit // after ctx ends, call Stop. func (c *WSClient) Start(ctx context.Context) error { - if err := c.dial(); err != nil { + if err := c.dial(ctx); err != nil { return err } @@ -141,23 +157,42 @@ func (c *WSClient) Start(ctx context.Context) error { // channel is unbuffered. c.backlog = make(chan rpctypes.RPCRequest, 1) + c.reconnectDone = make(chan struct{}) + c.closeResponses = sync.Once{} c.startReadWriteRoutines(ctx) go c.reconnectRoutine(ctx) return nil } -// Stop blocks until the client is shut down and returns nil. +// Stop blocks until the client is shut down and returns nil. It does not end the +// client: shutdown follows the context passed to Start, or running out of +// reconnect attempts. After either, it waits for running OnReconnect callbacks, and +// for a write in progress, which ends at its write deadline. It is safe to call +// repeatedly and concurrently, and returns immediately if the client never started. // // TODO(creachadair): This method exists for compatibility with the original // service plumbing. Give it a better name (e.g., Wait). func (c *WSClient) Stop() error { - // only close user-facing channels when we can't write to them - c.wg.Wait() - close(c.ResponsesCh) + if c.reconnectDone == nil { + return nil + } + <-c.reconnectDone + c.callbacks.Wait() + c.closeResponsesCh() return nil } +// closeResponsesCh joins the workers and closes ResponsesCh once. Only close +// user-facing channels when we can't write to them. Callers must ensure that no +// more workers can start. +func (c *WSClient) closeResponsesCh() { + c.closeResponses.Do(func() { + c.wg.Wait() + close(c.ResponsesCh) + }) +} + // IsReconnecting returns true if the client is reconnecting right now. func (c *WSClient) IsReconnecting() bool { c.mtx.RLock() @@ -166,8 +201,8 @@ func (c *WSClient) IsReconnecting() bool { } // Send the given RPC request to the server. Results will be available on -// ResponsesCh, errors, if any, on ErrorsCh. Will block until send succeeds or -// ctx.Done is closed. +// ResponsesCh, errors, if any, on ErrorsCh. Will block until send succeeds, ctx.Done +// is closed or the client has shut down. func (c *WSClient) Send(ctx context.Context, request rpctypes.RPCRequest) error { select { case c.send <- request: @@ -178,6 +213,8 @@ func (c *WSClient) Send(ctx context.Context, request rpctypes.RPCRequest) error return nil case <-ctx.Done(): return ctx.Err() + case <-c.reconnectDone: + return errClientStopped } } @@ -200,13 +237,24 @@ func (c *WSClient) nextRequestID() int { return id } -func (c *WSClient) dial() error { +// dial connects to the server. The handshake obeys deadlines but not cancellation, +// so ending ctx closes the connection to interrupt it. +func (c *WSClient) dial(ctx context.Context) error { + stopWatching := func() bool { return true } dialer := &websocket.Dialer{ - NetDialContext: c.DialerContext, - Proxy: http.ProxyFromEnvironment, + NetDialContext: func(dialCtx context.Context, network, addr string) (net.Conn, error) { + conn, err := c.DialerContext(dialCtx, network, addr) + if err == nil { + stopWatching = context.AfterFunc(ctx, func() { _ = conn.Close() }) + } + return conn, err + }, + Proxy: http.ProxyFromEnvironment, + HandshakeTimeout: c.handshakeTimeout, } + defer func() { stopWatching() }() rHeader := http.Header{} - conn, _, err := dialer.Dial(c.protocol+"://"+c.Address+c.Endpoint, rHeader) //nolint:nolintlint,bodyclose + conn, _, err := dialer.DialContext(ctx, c.protocol+"://"+c.Address+c.Endpoint, rHeader) //nolint:nolintlint,bodyclose if err != nil { return err } @@ -244,13 +292,17 @@ func (c *WSClient) reconnect(ctx context.Context) error { case <-timer.C: } - err := c.dial() + err := c.dial(ctx) if err != nil { c.Logger.Error("failed to redial", "err", err) } else { c.Logger.Info("reconnected") if c.onReconnect != nil { - go c.onReconnect() + c.callbacks.Add(1) + go func() { + defer c.callbacks.Done() + c.onReconnect() + }() } return nil } @@ -263,6 +315,15 @@ func (c *WSClient) reconnect(ctx context.Context) error { } } +// requestReconnect reports a worker's failure. After ctx ends nothing may be left +// to receive it, and Stop is waiting for the worker. +func (c *WSClient) requestReconnect(ctx context.Context, err error) { + select { + case c.reconnectAfter <- err: + case <-ctx.Done(): + } +} + func (c *WSClient) startReadWriteRoutines(ctx context.Context) { c.wg.Add(2) c.readRoutineQuit = make(chan struct{}) @@ -292,6 +353,7 @@ func (c *WSClient) processBacklog() error { } func (c *WSClient) reconnectRoutine(ctx context.Context) { + defer close(c.reconnectDone) for { select { case <-ctx.Done(): @@ -301,10 +363,7 @@ func (c *WSClient) reconnectRoutine(ctx context.Context) { c.wg.Wait() if err := c.reconnect(ctx); err != nil { c.Logger.Error("failed to reconnect", "err", err, "original_err", originalError) - if err = c.Stop(); err != nil { - c.Logger.Error("failed to stop conn", "error", err) - } - + c.closeResponsesCh() return } // drain reconnectAfter @@ -312,6 +371,7 @@ func (c *WSClient) reconnectRoutine(ctx context.Context) { for { select { case <-ctx.Done(): + _ = c.conn.Close() // no worker will own the new connection return case <-c.reconnectAfter: default: @@ -354,7 +414,7 @@ func (c *WSClient) writeRoutine(ctx context.Context) { } if err := c.conn.WriteJSON(request); err != nil { c.Logger.Error("failed to send request", "err", err) - c.reconnectAfter <- err + c.requestReconnect(ctx, err) // add request to the backlog, so we don't lose it c.backlog <- request return @@ -367,7 +427,7 @@ func (c *WSClient) writeRoutine(ctx context.Context) { } if err := c.conn.WriteMessage(websocket.PingMessage, []byte{}); err != nil { c.Logger.Error("failed to write ping", "err", err) - c.reconnectAfter <- err + c.requestReconnect(ctx, err) return } case <-c.readRoutineQuit: @@ -407,7 +467,7 @@ func (c *WSClient) readRoutine(ctx context.Context) { c.Logger.Error("failed to read response", "err", err) close(c.readRoutineQuit) - c.reconnectAfter <- err + c.requestReconnect(ctx, err) return } diff --git a/rpc/jsonrpc/client/ws_client_test.go b/rpc/jsonrpc/client/ws_client_test.go index 663973cc9f..44d3129670 100644 --- a/rpc/jsonrpc/client/ws_client_test.go +++ b/rpc/jsonrpc/client/ws_client_test.go @@ -3,8 +3,11 @@ package client import ( "context" "encoding/json" + "net" "net/http" "net/http/httptest" + "strings" + "sync/atomic" "testing" "time" @@ -259,3 +262,352 @@ func handleResponses(ctx context.Context, t *testing.T, c *WSClient) { } } } + +// startFailingClient starts a client that gives up after its first failed redial, +// and returns it with a server that drops the connection on any request. +func startFailingClient(ctx context.Context, t *testing.T) (*WSClient, *httptest.Server) { + t.Helper() + s := httptest.NewServer(&myTestHandler{t: t, closeConnAfterRead: true}) + t.Cleanup(s.Close) + c, err := NewWS("//"+s.Listener.Addr().String(), "/websocket") + require.NoError(t, err) + c.maxReconnectAttempts = 0 + require.NoError(t, c.Start(ctx)) + return c, s +} + +func requireResponsesClosed(t *testing.T, c *WSClient) { + t.Helper() + select { + case _, ok := <-c.ResponsesCh: + require.False(t, ok, "unexpected response") + case <-time.After(wsCallTimeout): + t.Fatal("ResponsesCh was not closed") + } +} + +// Running out of reconnect attempts shuts the client down by itself. The owner +// does not know that, and still calls Stop once its context ends. +func TestWSClientStopAfterReconnectExhaustion(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, s := startFailingClient(ctx, t) + require.NoError(t, c.Call(ctx, "drop", make(map[string]interface{}))) + s.Close() + requireResponsesClosed(t, c) + + cancel() + require.NoError(t, c.Stop()) + require.NoError(t, c.Stop()) +} + +// A Stop already blocked when the attempts run out must finish with that shutdown. +func TestWSClientStopRacingReconnectExhaustion(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, s := startFailingClient(ctx, t) + require.NoError(t, c.Call(ctx, "drop", make(map[string]interface{}))) + s.Close() + require.Eventually(t, c.IsReconnecting, wsCallTimeout, time.Millisecond) + + stopped := make(chan error, 2) + for range cap(stopped) { + go func() { stopped <- c.Stop() }() + } + for range cap(stopped) { + select { + case err := <-stopped: + require.NoError(t, err) + requireResponsesClosed(t, c) + case <-time.After(wsCallTimeout): + t.Fatal("Stop did not return after the attempts ran out") + } + } +} + +// A reconnect can start new workers, so Stop must not finish while one may still +// succeed. Every concurrent Stop returns only after the channel closes. +func TestWSClientStopWaitsForReconnect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, _ := startFailingClient(ctx, t) + require.NoError(t, c.Call(ctx, "drop", make(map[string]interface{}))) + require.Eventually(t, c.IsReconnecting, wsCallTimeout, time.Millisecond) + + stopped := make(chan error, 2) + for range cap(stopped) { + go func() { stopped <- c.Stop() }() + } + require.Eventually(t, func() bool { return !c.IsReconnecting() }, wsCallTimeout, time.Millisecond) + select { + case err := <-stopped: + t.Fatalf("Stop returned before the context ended: %v", err) + default: + } + + cancel() + for range cap(stopped) { + select { + case err := <-stopped: + require.NoError(t, err) + requireResponsesClosed(t, c) + case <-time.After(wsCallTimeout): + t.Fatal("Stop did not return after the context ended") + } + } +} + +// A client that never started has nothing to wait for. +func TestWSClientStopWithoutStart(t *testing.T) { + c, err := NewWS("//127.0.0.1:1", "/websocket") + require.NoError(t, err) + stopped := make(chan error, 1) + go func() { stopped <- c.Stop() }() + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(wsCallTimeout): + t.Fatal("Stop blocked on a client that never started") + } +} + +func requireStopReturns(t *testing.T, c *WSClient) { + t.Helper() + stopped := make(chan error, 1) + go func() { stopped <- c.Stop() }() + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(wsCallTimeout): + t.Fatal("Stop did not return") + } +} + +// listenWithoutAnswering returns the address of a peer that never answers the +// upgrade, and a channel signaled once a client has sent its upgrade request. +func listenWithoutAnswering(t *testing.T) (string, <-chan net.Conn) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + accepted := make(chan net.Conn, 8) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + t.Cleanup(func() { _ = conn.Close() }) + if _, err := conn.Read(make([]byte, 1)); err == nil { + accepted <- conn + } + } + }() + return "//" + ln.Addr().String(), accepted +} + +// The first dial observes the context passed to Start. +func TestWSClientStartCanceledDuringHungHandshake(t *testing.T) { + addr, accepted := listenWithoutAnswering(t) + c, err := NewWS(addr, "/websocket") + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + started := make(chan error, 1) + go func() { started <- c.Start(ctx) }() + <-accepted + + cancel() + select { + case err := <-started: + require.Error(t, err) + case <-time.After(wsCallTimeout): + t.Fatal("Start blocked on a hung handshake after its context ended") + } + require.NoError(t, c.Stop()) +} + +// A peer that never answers cannot block a dial forever, even with a live context. +func TestWSClientHandshakeTimesOut(t *testing.T) { + addr, _ := listenWithoutAnswering(t) + c, err := NewWS(addr, "/websocket") + require.NoError(t, err) + c.handshakeTimeout = 50 * time.Millisecond + started := make(chan error, 1) + go func() { started <- c.Start(context.Background()) }() + select { + case err := <-started: + require.Error(t, err) + case <-time.After(wsCallTimeout): + t.Fatal("handshake did not time out") + } +} + +// Stop follows the context even when a redial is stuck in its handshake. +func TestWSClientStopDuringHungRedial(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + var accepted atomic.Int32 + srv := &http.Server{ + ReadHeaderTimeout: wsCallTimeout, + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if accepted.Add(1) == 1 { + conn, err := upgrader.Upgrade(w, r, nil) + if err == nil { + _ = conn.UnderlyingConn().Close() // the client sees 1006 and redials + } + return + } + <-r.Context().Done() // never answers + }), + } + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, err := NewWS("//"+ln.Addr().String(), "/websocket") + require.NoError(t, err) + require.NoError(t, c.Start(ctx)) + require.Eventually(t, func() bool { return accepted.Load() == 2 }, 2*wsCallTimeout, time.Millisecond) + + cancel() + requireStopReturns(t, c) +} + +// A writer stuck in a write outlives its context and the reconnect routine. Stop +// must still join it, and it must not block reporting the error once it fails. +func TestWSClientStopJoinsWorkerFailingAfterReconnectExit(t *testing.T) { + serverConn := make(chan *websocket.Conn, 1) + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err == nil { + serverConn <- conn // never reads, so the client's send path fills + } + })) + t.Cleanup(s.Close) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, err := NewWS("//"+s.Listener.Addr().String(), "/websocket") + require.NoError(t, err) + require.NoError(t, c.Start(ctx)) + peer := <-serverConn + t.Cleanup(func() { _ = peer.Close() }) + // Call returns once the writer has taken the request, too big for socket buffers. + require.NoError(t, c.Call(ctx, "big", map[string]interface{}{"x": strings.Repeat("a", 64<<20)})) + cancel() + select { + case <-c.reconnectDone: + case <-time.After(wsCallTimeout): + t.Fatal("reconnect routine did not exit") + } + + stopped := make(chan error, 1) + go func() { stopped <- c.Stop() }() + // Nothing observable marks Stop blocking; this window cannot fail correct code. + select { + case err := <-stopped: + t.Fatalf("Stop returned while the writer could still use the client: %v", err) + case <-time.After(50 * time.Millisecond): + } + + // The reader reports this and closes the connection, so the write fails too. + require.NoError(t, peer.WriteControl(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseGoingAway, "bye"), time.Now().Add(time.Second))) + // Encoding the request is slow under -race and load, and the writer cannot stop + // until it has finished. This only needs to tell that from a hang. + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(6 * wsCallTimeout): + t.Fatal("Stop did not return after the workers failed") + } +} + +// Each run gets its own shutdown. +func TestWSClientRestartAfterStop(t *testing.T) { + s := httptest.NewServer(&myTestHandler{t: t}) + t.Cleanup(s.Close) + c, err := NewWS("//"+s.Listener.Addr().String(), "/websocket") + require.NoError(t, err) + for range 2 { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, c.Start(ctx)) + cancel() + requireStopReturns(t, c) + requireResponsesClosed(t, c) + } +} + +// The dialer shared with the HTTP client observes its context, so a cancellation +// does not wait out the connect timeout of an unreachable peer. +func TestHTTPDialerObservesContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := makeHTTPDialer("tcp", "127.0.0.1:1")(ctx, "", "") + require.ErrorIs(t, err, context.Canceled) +} + +// Stop joins a reconnect callback already running, which may still use the client. +func TestWSClientStopJoinsReconnectCallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := httptest.NewServer(&myTestHandler{t: t, closeConnAfterRead: true}) + t.Cleanup(s.Close) + c, err := NewWS("//"+s.Listener.Addr().String(), "/websocket") + require.NoError(t, err) + entered := make(chan struct{}) + release := make(chan struct{}) + c.OnReconnect(func() { + close(entered) + <-release + }) + require.NoError(t, c.Start(ctx)) + require.NoError(t, c.Call(ctx, "drop", make(map[string]interface{}))) + select { + case <-entered: + case <-time.After(2 * wsCallTimeout): + close(release) + t.Fatal("reconnect callback did not run") + } + + cancel() + stopped := make(chan error, 1) + go func() { stopped <- c.Stop() }() + // Nothing observable marks Stop blocking; this window cannot fail correct code. + select { + case err := <-stopped: + close(release) + t.Fatalf("Stop returned before the reconnect callback finished: %v", err) + case <-time.After(50 * time.Millisecond): + } + close(release) + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(wsCallTimeout): + t.Fatal("Stop did not return after the reconnect callback finished") + } +} + +// A send needs the writer, so it cannot wait for one that will never return. +func TestWSClientSendFailsAfterShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := httptest.NewServer(&myTestHandler{t: t}) + t.Cleanup(s.Close) + c, err := NewWS("//"+s.Listener.Addr().String(), "/websocket") + require.NoError(t, err) + require.NoError(t, c.Start(ctx)) + cancel() + require.NoError(t, c.Stop()) + + sent := make(chan error, 1) + go func() { sent <- c.Call(context.Background(), "late", make(map[string]interface{})) }() + select { + case err := <-sent: + require.ErrorIs(t, err, errClientStopped) + case <-time.After(wsCallTimeout): + t.Fatal("Send blocked on a client that has shut down") + } +} diff --git a/rpc/jsonrpc/server/http_server.go b/rpc/jsonrpc/server/http_server.go index 4dffa02961..bb02618e6d 100644 --- a/rpc/jsonrpc/server/http_server.go +++ b/rpc/jsonrpc/server/http_server.go @@ -11,6 +11,7 @@ import ( "net/http" "runtime/debug" "strings" + "sync" "time" "golang.org/x/net/netutil" @@ -59,65 +60,65 @@ func DefaultConfig() *Config { // Serve creates a http.Server and calls Serve with the given listener. It // wraps handler to recover panics and limit the request body size. func Serve(ctx context.Context, listener net.Listener, handler http.Handler, logger log.Logger, config *Config) error { - logger.Info("Starting RPC HTTP server on", "addr", listener.Addr()) - h := recoverAndLogHandler(MaxBytesHandler(handler, config.MaxBodyBytes), logger) - s := &http.Server{ - Handler: h, - ReadTimeout: config.ReadTimeout, - WriteTimeout: config.WriteTimeout, - MaxHeaderBytes: config.MaxHeaderBytes, - } - sig := make(chan struct{}) - go func() { - select { - case <-ctx.Done(): - sctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - _ = s.Shutdown(sctx) - case <-sig: - } - }() - - if err := s.Serve(listener); err != nil { - logger.Info("RPC HTTP server stopped", "err", err) - close(sig) - return err - } - return nil + return serve(ctx, listener, handler, logger, config, func(s *http.Server) error { + return s.Serve(listener) + }) } -// Serve creates a http.Server and calls ServeTLS with the given listener, -// certFile and keyFile. It wraps handler to recover panics and limit the -// request body size. +// ServeTLS serves HTTPS and joins all requests, including websocket sessions, +// before returning. Cancellation is propagated to request contexts. func ServeTLS(ctx context.Context, listener net.Listener, handler http.Handler, certFile, keyFile string, logger log.Logger, config *Config) error { - logger.Info("Starting RPC HTTPS server", - "listenterAddr", listener.Addr(), - "certFile", certFile, - "keyFile", keyFile) + return serve(ctx, listener, handler, logger, config, func(s *http.Server) error { + return s.ServeTLS(listener, certFile, keyFile) + }) +} +func serve(ctx context.Context, listener net.Listener, handler http.Handler, logger log.Logger, config *Config, run func(*http.Server) error) error { + // TLS setup can fail before http.Server takes ownership of the listener. + defer func() { _ = listener.Close() }() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + var mu sync.Mutex + var requests sync.WaitGroup + stopping := false + h := recoverAndLogHandler(MaxBytesHandler(handler, config.MaxBodyBytes), logger) s := &http.Server{ - Handler: recoverAndLogHandler(MaxBytesHandler(handler, config.MaxBodyBytes), logger), + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Shutdown does not join hijacked connections. Keep their handlers in + // this group, and serialize admission with shutdown's call to Wait. + mu.Lock() + if stopping { + mu.Unlock() + http.Error(w, "RPC server is stopping", http.StatusServiceUnavailable) + return + } + requests.Add(1) + mu.Unlock() + defer requests.Done() + h.ServeHTTP(w, r) + }), + BaseContext: func(net.Listener) context.Context { return ctx }, ReadTimeout: config.ReadTimeout, WriteTimeout: config.WriteTimeout, MaxHeaderBytes: config.MaxHeaderBytes, } - sig := make(chan struct{}) + drained := make(chan struct{}) go func() { - select { - case <-ctx.Done(): - sctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - _ = s.Shutdown(sctx) - case <-sig: - } + defer close(drained) + <-ctx.Done() + mu.Lock() + stopping = true + mu.Unlock() + // Cancellation alone cannot unblock a handler writing to a stalled + // client. Close its transport before joining the handler itself. + _ = s.Close() + requests.Wait() }() - - if err := s.ServeTLS(listener, certFile, keyFile); err != nil { - logger.Error("RPC HTTPS server stopped", "err", err) - close(sig) - return err - } - return nil + err := run(s) + cancel() + <-drained + logger.Info("RPC server stopped", "err", err) + return err } // writeInternalError writes an internal server error (500) to w with the text diff --git a/rpc/jsonrpc/server/http_server_test.go b/rpc/jsonrpc/server/http_server_test.go index aac3d90d39..9b7216d0df 100644 --- a/rpc/jsonrpc/server/http_server_test.go +++ b/rpc/jsonrpc/server/http_server_test.go @@ -242,3 +242,139 @@ func TestWriteHTTPResponse(t *testing.T) { assert.Equal(t, "application/json", resp.Header.Get("Content-Type")) assert.Equal(t, `{"code":-32603,"message":"Internal error","data":"foo"}`, string(body)) } + +func TestServeJoinsActiveHandler(t *testing.T) { + for _, useTLS := range []bool{false, true} { + t.Run(fmt.Sprintf("TLS=%v", useTLS), func(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + type contextKey struct{} + ctx, cancel := context.WithCancel(context.WithValue(context.Background(), contextKey{}, "inherited")) + defer cancel() + entered, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + handler := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + assert.Equal(t, "inherited", r.Context().Value(contextKey{})) + close(entered) + <-r.Context().Done() + close(canceled) + <-release + }) + go func() { + if useTLS { + done <- ServeTLS(ctx, listener, handler, "test.crt", "test.key", log.NewNopLogger(), DefaultConfig()) + } else { + done <- Serve(ctx, listener, handler, log.NewNopLogger(), DefaultConfig()) + } + }() + transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport} + scheme := "http://" + if useTLS { + scheme = "https://" + } + requestDone := make(chan struct{}) + go func() { + defer close(requestDone) + resp, err := client.Get(scheme + listener.Addr().String()) + if err == nil { + _ = resp.Body.Close() + } + }() + <-entered + cancel() + <-canceled + select { + case err := <-done: + close(release) + <-requestDone + t.Fatalf("Serve returned before active handler completed: %v", err) + case <-time.After(100 * time.Millisecond): + } + close(release) + require.ErrorIs(t, <-done, http.ErrServerClosed) + <-requestDone + }) + } +} + +// blockedWriteListener supplies a connection whose peer never reads responses. +// net.Pipe makes the first response write block without OS buffer assumptions. +type blockedWriteListener struct { + net.Conn + accepted bool + closed chan struct{} + once sync.Once + writing chan struct{} +} + +func (l *blockedWriteListener) Accept() (net.Conn, error) { + if !l.accepted { + l.accepted = true + return &blockedResponseConn{Conn: l.Conn, writing: l.writing}, nil + } + <-l.closed + return nil, net.ErrClosed +} + +func (l *blockedWriteListener) Close() error { + l.once.Do(func() { close(l.closed) }) + return nil +} + +func (l *blockedWriteListener) Addr() net.Addr { return l.LocalAddr() } + +type blockedResponseConn struct { + net.Conn + writing chan struct{} + once sync.Once +} + +func (c *blockedResponseConn) Write(p []byte) (int, error) { + c.once.Do(func() { close(c.writing) }) + return c.Conn.Write(p) +} + +func TestServeCancellationUnblocksResponseWrite(t *testing.T) { + serverConn, clientConn := net.Pipe() + defer serverConn.Close() + defer clientConn.Close() + listener := &blockedWriteListener{Conn: serverConn, closed: make(chan struct{}), writing: make(chan struct{})} + defer listener.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done, handlerDone := make(chan error, 1), make(chan struct{}) + go func() { + done <- Serve(ctx, listener, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + defer close(handlerDone) + _, _ = w.Write(make([]byte, 1<<20)) + }), log.NewNopLogger(), DefaultConfig()) + }() + _, err := io.WriteString(clientConn, "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n") + require.NoError(t, err) + <-listener.writing + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, http.ErrServerClosed) + case <-time.After(time.Second): + _ = clientConn.Close() + err := <-done + t.Fatalf("Serve did not unblock an active response write on cancellation (returned %v after the client closed)", err) + } + <-handlerDone +} + +func TestServeTLSClosesListenerOnCertificateError(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + err = ServeTLS(context.Background(), listener, http.NotFoundHandler(), "missing.crt", "missing.key", log.NewNopLogger(), DefaultConfig()) + require.Error(t, err) + conn, err := net.DialTimeout("tcp", listener.Addr().String(), time.Second) + if conn != nil { + _ = conn.Close() + } + require.Error(t, err, "TLS setup failure must release the bound listener") +} diff --git a/rpc/jsonrpc/server/ws_handler.go b/rpc/jsonrpc/server/ws_handler.go index 9ea70fdabe..da60095526 100644 --- a/rpc/jsonrpc/server/ws_handler.go +++ b/rpc/jsonrpc/server/ws_handler.go @@ -8,6 +8,7 @@ import ( "net/url" "runtime/debug" "strings" + "sync" "time" "github.com/gorilla/websocket" @@ -178,10 +179,6 @@ type wsConnection struct { // writeChan is never closed, to allow WriteRPCResponse() to fail. writeChan chan rpctypes.RPCResponse - // chan, which is closed when/if readRoutine errors - // used to abort writeRoutine - readRoutineQuit chan struct{} - funcMap map[string]*RPCFunc // Connection times out if we haven't received *anything* in this long, not even pings. @@ -196,8 +193,11 @@ type wsConnection struct { // callback which is called upon disconnect onDisconnect func(remoteAddr string) - ctx context.Context - cancel context.CancelFunc + ctx context.Context + cancel context.CancelFunc + workerMu sync.Mutex + workers sync.WaitGroup + stopping bool } // NewWSConnection wraps websocket.Conn. @@ -208,13 +208,12 @@ type wsConnection struct { // disconnect. see https://github.com/gorilla/websocket/issues/97 func newWSConnection(baseConn *websocket.Conn, funcMap map[string]*RPCFunc, logger log.Logger, options ...func(*wsConnection)) *wsConnection { wsc := &wsConnection{ - Logger: logger, - remoteAddr: baseConn.RemoteAddr().String(), - baseConn: baseConn, - funcMap: funcMap, - readWait: defaultWSReadWait, - pingPeriod: defaultWSPingPeriod, - readRoutineQuit: make(chan struct{}), + Logger: logger, + remoteAddr: baseConn.RemoteAddr().String(), + baseConn: baseConn, + funcMap: funcMap, + readWait: defaultWSReadWait, + pingPeriod: defaultWSPingPeriod, } for _, option := range options { option(wsc) @@ -257,27 +256,66 @@ func ReadLimit(readLimit int64) func(*wsConnection) { // Start starts the client service routines and blocks until there is an error. func (wsc *wsConnection) Start(ctx context.Context) error { + wsc.ctx, wsc.cancel = context.WithCancel(ctx) wsc.writeChan = make(chan rpctypes.RPCResponse, defaultWSWriteChanCapacity) + // https://github.com/gorilla/websocket/issues/97 + pongs := make(chan string, 1) + wsc.baseConn.SetPingHandler(func(m string) error { + select { + case pongs <- m: + default: + } + return nil + }) - // Read subscriptions/unsubscriptions to events - go wsc.readRoutine(ctx) - // Write responses, BLOCKING. - wsc.writeRoutine(ctx) - + readDone, closeDone := make(chan struct{}), make(chan struct{}) + go func() { + defer close(readDone) + defer wsc.cancel() + // A handler panic is isolated to its request; retry on this same reader. + for wsc.ctx.Err() == nil && wsc.readRoutine(wsc.ctx) { + } + }() + go func() { + defer close(closeDone) + <-wsc.ctx.Done() + _ = wsc.baseConn.Close() + }() + wsc.writeRoutine(wsc.ctx, pongs) + wsc.cancel() + <-closeDone + <-readDone + wsc.workerMu.Lock() + wsc.stopping = true + wsc.workerMu.Unlock() + wsc.workers.Wait() + if wsc.onDisconnect != nil { + wsc.onDisconnect(wsc.remoteAddr) + } return nil } -// Stop unsubscribes the remote from all subscriptions. +// Stop cancels the session without joining, so it is safe inside a handler. +// Start joins the reader and performs subscription cleanup before returning. func (wsc *wsConnection) Stop() error { - if wsc.onDisconnect != nil { - wsc.onDisconnect(wsc.remoteAddr) - } - if wsc.ctx != nil { + if wsc.cancel != nil { wsc.cancel() } return nil } +// Go starts session work that Start joins before subscription cleanup. +func (wsc *wsConnection) Go(fn func(context.Context)) bool { + wsc.workerMu.Lock() + defer wsc.workerMu.Unlock() + if wsc.stopping || wsc.ctx.Err() != nil { + return false + } + wsc.workers.Add(1) + go func() { defer wsc.workers.Done(); fn(wsc.ctx) }() + return true +} + // GetRemoteAddr returns the remote address of the underlying connection. // It implements WSRPCConnection func (wsc *wsConnection) GetRemoteAddr() string { @@ -289,6 +327,8 @@ func (wsc *wsConnection) GetRemoteAddr() string { // It implements WSRPCConnection. It is Goroutine-safe. func (wsc *wsConnection) WriteRPCResponse(ctx context.Context, resp rpctypes.RPCResponse) error { select { + case <-wsc.ctx.Done(): + return wsc.ctx.Err() case <-ctx.Done(): return ctx.Err() case wsc.writeChan <- resp: @@ -313,17 +353,13 @@ func (wsc *wsConnection) TryWriteRPCResponse(ctx context.Context, resp rpctypes. // Context returns the connection's context. // The context is canceled when the client's connection closes. func (wsc *wsConnection) Context() context.Context { - if wsc.ctx != nil { - return wsc.ctx - } - wsc.ctx, wsc.cancel = context.WithCancel(context.Background()) return wsc.ctx } // Read from the socket and subscribe to or unsubscribe from events -func (wsc *wsConnection) readRoutine(ctx context.Context) { +func (wsc *wsConnection) readRoutine(ctx context.Context) (retry bool) { // readRoutine will block until response is written or WS connection is closed - writeCtx := context.Background() + writeCtx := ctx defer func() { if r := recover(); r != nil { @@ -337,7 +373,7 @@ func (wsc *wsConnection) readRoutine(ctx context.Context) { req.MakeErrorf(rpctypes.CodeInternalError, "Panic in handler: %v", err)); err != nil { wsc.Logger.Error("error writing RPC response", "err", err) } - go wsc.readRoutine(ctx) + retry = true } }() @@ -348,7 +384,7 @@ func (wsc *wsConnection) readRoutine(ctx context.Context) { for { select { case <-ctx.Done(): - return + return false default: // reset deadline for every type of message (control or data) if err := wsc.baseConn.SetReadDeadline(time.Now().Add(wsc.readWait)); err != nil { @@ -365,8 +401,7 @@ func (wsc *wsConnection) readRoutine(ctx context.Context) { if err := wsc.Stop(); err != nil { wsc.Logger.Error("error closing websocket connection", "err", err) } - close(wsc.readRoutineQuit) - return + return false } dec := json.NewDecoder(r) @@ -419,26 +454,14 @@ func (wsc *wsConnection) readRoutine(ctx context.Context) { } // receives on a write channel and writes out on the socket -func (wsc *wsConnection) writeRoutine(ctx context.Context) { +func (wsc *wsConnection) writeRoutine(ctx context.Context, pongs <-chan string) { pingTicker := time.NewTicker(wsc.pingPeriod) defer pingTicker.Stop() - // https://github.com/gorilla/websocket/issues/97 - pongs := make(chan string, 1) - wsc.baseConn.SetPingHandler(func(m string) error { - select { - case pongs <- m: - default: - } - return nil - }) - for { select { case <-ctx.Done(): return - case <-wsc.readRoutineQuit: // error in readRoutine - return case m := <-pongs: err := wsc.writeMessageWithDeadline(websocket.PongMessage, []byte(m)) if err != nil { diff --git a/rpc/jsonrpc/server/ws_handler_test.go b/rpc/jsonrpc/server/ws_handler_test.go index 8bfcfbbc69..d91847a120 100644 --- a/rpc/jsonrpc/server/ws_handler_test.go +++ b/rpc/jsonrpc/server/ws_handler_test.go @@ -3,9 +3,11 @@ package server import ( "context" "encoding/json" + "net" "net/http" "net/http/httptest" "testing" + "time" "github.com/fortytw2/leaktest" "github.com/gorilla/websocket" @@ -63,3 +65,109 @@ func newWSServer(t *testing.T, logger log.Logger) *httptest.Server { return srv } + +func TestWebsocketStartJoinsHandler(t *testing.T) { + type contextKey struct{} + ctx, cancel := context.WithCancel(context.WithValue(context.Background(), contextKey{}, "inherited")) + defer cancel() + entered, canceled, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{}) + disconnected := make(chan struct{}) + functions := map[string]*RPCFunc{"block": NewWSRPCFunc(func(ctx context.Context, _ *struct{}) (string, error) { + if ctx.Value(contextKey{}) != "inherited" { + t.Error("websocket handler lost parent context value") + } + close(entered) + <-ctx.Done() + close(canceled) + <-release + return "done", nil + })} + upgrader := websocket.Upgrader{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Error(err) + return + } + session := newWSConnection(conn, functions, log.NewNopLogger(), OnDisconnect(func(string) { close(disconnected) })) + if err := session.Start(ctx); err != nil { + t.Error(err) + } + close(done) + })) + defer server.Close() + conn, response, err := websocket.DefaultDialer.Dial("ws://"+server.Listener.Addr().String(), nil) + require.NoError(t, err) + defer response.Body.Close() + defer conn.Close() + request := rpctypes.NewRequest(1) + require.NoError(t, request.SetMethodAndParams("block", struct{}{})) + require.NoError(t, conn.WriteJSON(request)) + <-entered + cancel() + <-canceled + select { + case <-done: + close(release) + t.Fatal("Start returned while request handler remained active") + case <-time.After(100 * time.Millisecond): + } + select { + case <-disconnected: + t.Error("subscription cleanup ran before request completed") + default: + } + close(release) + <-done + <-disconnected +} + +func TestServeJoinsWebsocketSessionWork(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + entered, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + functions := map[string]*RPCFunc{ + "panic": NewWSRPCFunc(func(context.Context, *struct{}) (string, error) { panic("request panic") }), + "subscribe": NewWSRPCFunc(func(ctx context.Context, _ *struct{}) (string, error) { + if !rpctypes.GetCallInfo(ctx).WSConn.Go(func(ctx context.Context) { + close(entered) + <-ctx.Done() + close(canceled) + <-release + }) { + return "", context.Canceled + } + return "subscribed", nil + }), + } + manager := NewWebsocketManager(log.NewNopLogger(), functions) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + done := make(chan error, 1) + go func() { + done <- Serve(ctx, listener, http.HandlerFunc(manager.WebsocketHandler), log.NewNopLogger(), DefaultConfig()) + }() + conn, response, err := websocket.DefaultDialer.Dial("ws://"+listener.Addr().String(), nil) + require.NoError(t, err) + defer response.Body.Close() + defer conn.Close() + request := rpctypes.NewRequest(1) + require.NoError(t, request.SetMethodAndParams("panic", struct{}{})) + require.NoError(t, conn.WriteJSON(request)) + var result rpctypes.RPCResponse + require.NoError(t, conn.ReadJSON(&result)) + require.NotNil(t, result.Error) + require.NoError(t, request.SetMethodAndParams("subscribe", struct{}{})) + require.NoError(t, conn.WriteJSON(request)) + <-entered + cancel() + <-canceled + select { + case err := <-done: + close(release) + t.Fatalf("Serve returned while websocket session work remained active: %v", err) + case <-time.After(100 * time.Millisecond): + } + close(release) + require.ErrorIs(t, <-done, http.ErrServerClosed) +} diff --git a/rpc/jsonrpc/types/types.go b/rpc/jsonrpc/types/types.go index 5079184522..7c5963b0f8 100644 --- a/rpc/jsonrpc/types/types.go +++ b/rpc/jsonrpc/types/types.go @@ -316,6 +316,8 @@ func (resp RPCResponse) String() string { // WSRPCConnection represents a websocket connection. type WSRPCConnection interface { + // Go starts work owned and joined by the websocket session. + Go(func(context.Context)) bool // GetRemoteAddr returns a remote address of the connection. GetRemoteAddr() string // WriteRPCResponse writes the response onto connection (BLOCKING).