Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
8a13f0e
fix(service): drain managed workers before releasing resources
lklimek Sep 24, 2026
85f9fae
refactor(p2p): join service-owned workers during shutdown
lklimek Sep 24, 2026
a6ca9e6
refactor(service): join owned workers before releasing resources
lklimek Sep 24, 2026
fad5da3
fix(mempool): cancel and join all recheck batches during shutdown
lklimek Sep 24, 2026
8d16f56
fix(service): close remaining shutdown admission and rollback gaps
lklimek Sep 24, 2026
82d312f
fix(rpc): drain handlers and websocket sessions before shutdown
lklimek Sep 24, 2026
f717b4c
fix(node): drain RPC and mempool before releasing dependencies
lklimek Sep 24, 2026
63a524e
fix(rpc): unblock transport I/O before draining handlers
lklimek Sep 24, 2026
053357f
Merge branch 'v1.8-dev' into fix/base-service-stop
lklimek Sep 25, 2026
0c3e553
Merge v1.8-dev (#1513 lifecycle core) into fix/base-service-stop
lklimek Oct 23, 2026
2cbc4c9
test(consensus): join the test event bus before the test ends
lklimek Oct 23, 2026
aaa3d65
Merge branch 'v1.8-dev' into fix/base-service-stop
lklimek Oct 23, 2026
d97ddbc
test: import libs/db after the database adapter backport
lklimek Oct 23, 2026
e549ab4
test(rpc): report Serve's result when cancellation does not unblock a…
lklimek Oct 23, 2026
687318c
fix(node): close pprof and Prometheus servers instead of waiting on c…
lklimek Oct 23, 2026
915d9bc
refactor(node): share child teardown between shutdown and failed startup
lklimek Oct 23, 2026
1ff0319
docs: describe light RPC client and JSON-RPC server shutdown changes
lklimek Oct 23, 2026
6c0bdf2
fix(abci): close ABCI socket connections on server shutdown
lklimek Oct 23, 2026
b845e6c
fix(light): bound WebSocket unsubscribe cleanup by its owner's lifetime
lklimek Oct 23, 2026
d87e9b2
fix(abci): release only the routed clients a start attempt started
lklimek Oct 23, 2026
878993c
fix(p2p): close an abandoned channel queue outside channelMtx
lklimek Oct 23, 2026
b720244
fix(privval): interrupt pending signer I/O before taking the endpoint…
lklimek Oct 23, 2026
2f4af16
fix(node): release node resources when startup fails
lklimek Oct 31, 2026
6355218
test(node): pin failed-start rollback ordering and document its limits
lklimek Oct 31, 2026
91884d1
fix(abci): cancel routed children before joining them on failed start
lklimek Oct 31, 2026
f68fd08
test(abci): assert the failed routed attempt started its child
lklimek Oct 31, 2026
8beb2f1
fix(light): bound proxy disconnect cleanup when only the listener closes
lklimek Oct 31, 2026
96129a9
fix(rpc): forget subscriptions before UnsubscribeAll sends upstream
lklimek Oct 31, 2026
f13a608
test(rpc): let the delivery block before releasing it
lklimek Oct 31, 2026
eda4da4
fix(node): release a full node's Dash Core RPC client
lklimek Oct 31, 2026
e6f9176
docs(node): state what the Dash Core client's close bound covers
lklimek Oct 31, 2026
c4470c8
fix(rpc): make WSClient.Stop idempotent
lklimek Oct 31, 2026
5c812a1
fix(rpc): bound WSClient dials and worker failure reports after cancel
lklimek Oct 31, 2026
d942aa1
fix(light): do not join a wrapped client the caller already started
lklimek Oct 31, 2026
1b42722
fix(rpc): join HTTP client event workers on stop
lklimek Oct 31, 2026
de50c6a
test(rpc): remove timing assumptions from WebSocket client tests
lklimek Oct 31, 2026
34a3dae
Merge branch 'v1.8-dev' into fix/base-service-stop
lklimek Nov 4, 2026
6fb4a4c
fix(privval): close the gRPC signer connection in SignerClient.Close
lklimek Nov 5, 2026
564a3bc
Merge branch 'dev' into fix/base-service-stop
lklimek Nov 5, 2026
2b0f6c1
Merge remote-tracking branch 'origin/dev' into fix/base-service-stop
lklimek Nov 17, 2026
eff6da2
fix(light): leave an already started HTTP client to its caller
lklimek Nov 17, 2026
4679015
fix(rpc): unsubscribe only the departing subscriber's queries
lklimek Nov 17, 2026
9f412b5
test(rpc): reproduce lost upstream subscription on concurrent join an…
lklimek Nov 17, 2026
fd00190
fix(rpc): order upstream subscribes with unsubscribes of unused queries
lklimek Nov 17, 2026
86a6934
test(rpc): stress concurrent joins and leaves of a shared query
lklimek Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion UPGRADING.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,17 +49,51 @@ Register owned workers with `BaseService.Go`. `OnStop` must unblock them, not
wait for them; move shared-resource cleanup to optional `OnDrain`, which runs
after those workers exit. Call `Stop` then `Wait` before releasing resources.
Plain goroutines and their existing joins remain the implementation's responsibility.
`TimeoutTicker` implementations must now provide `Wait`.
`TimeoutTicker` implementations must now provide `Wait`. `WSRPCConnection`
implementations must provide `Go(func(context.Context)) bool` so the server owns
subscription workers.
Comment thread
lklimek marked this conversation as resolved.
Resolved

`WSClient.Stop` is safe to call repeatedly and concurrently, including after
reconnect attempts run out. During a reconnect it now blocks until the context
ends or the attempts run out, instead of returning while a reconnect could still
start workers. Dials now observe that context and fail after ten seconds, including
the one in `Start`, so a peer that never answers the handshake cannot block either
call. Once the context ends, `Stop` waits at most for a write in progress to hit
its ten-second deadline. It also joins `OnReconnect` callbacks, which must return
once the context ends, and `Send` fails once the client has shut down. Stopping the
HTTP RPC client also joins its event listener and resubscriptions.

`rpc/jsonrpc/server.Serve` and `ServeTLS` always close the listener before
returning, including when TLS setup fails. Cancellation now reaches request
contexts, rejects new requests with 503, closes active connections instead of a
one-second graceful shutdown, and returns only after every handler, including
hijacked WebSocket sessions, has finished. They no longer log a startup line;
one "RPC server stopped" line is logged on return.

A stop requested during startup cancels work immediately but defers `OnStop`
until successful startup returns. Failed startup must roll back its resources;
BaseService joins registered work without calling shutdown hooks. Retry requires
recreating any child that already started successfully and then stopped.
A node that fails after reserving its RPC listeners, or a seed node that fails
startup, closes its databases and rejects another `Start`; construct a new node.
In the same process that panics with `duplicate metrics collector registration
attempted` for a seed, or for a node with Prometheus enabled, unless
`Instrumentation.Namespace` differs. An RPC listener bind failure stays retryable
and keeps the stores open, with no `service.Service` call to release them if you
give up. Do not call a Dash Core `RPCClient` after `Close`: requests can block
forever. Its `Close` waits at most five seconds for an in-flight request, but that
does not bound node shutdown: a reactor stuck in a Dash Core call still blocks
`Wait`, because the client has no HTTP timeout. `--shutdown-timeout` limits the
node command's wait, not `Start` or embedders.

Direct consensus `State.Stop` drains a commit already in progress using the
context originally passed to `State.Start`. Cancel that parent to abort it;
reactor/node shutdown still cancels that parent and can require WAL recovery.
This is not an unconditional durable-finalization guarantee for node shutdown.
Node shutdown keeps application connections, the event bus and the indexer on a
dependency context until reactors have been joined, then releases them and the
stores. After a node startup failure past RPC listener reservation, construct a
fresh node graph.
Once admitted, a consensus handoff belongs to its reactor. Canceling the caller
releases its wait; reactor shutdown cancels and joins the admitted handoff.
Router disconnect notifications during shutdown are best effort; subscribers
Expand Down
8 changes: 7 additions & 1 deletion abci/client/grpc_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,13 @@ func (cli *grpcClient) streamClientInterceptor(ctx context.Context, desc *grpc.S
return streamer(ctx, desc, cc, method, opts...)
}

func (cli *grpcClient) OnStart(ctx context.Context) error {
func (cli *grpcClient) OnStart(ctx context.Context) (startErr error) {
defer func() {
if startErr != nil && cli.conn != nil {
_ = cli.conn.Close()
cli.conn = nil
}
}()
timer := time.NewTimer(0)
defer timer.Stop()

Expand Down
80 changes: 54 additions & 26 deletions abci/client/routed_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@ import (
)

type routedClient struct {
service.Service
*service.BaseService
logger log.Logger
routing Routing
defaultClient ClientInfo
// owned lists the clients started by the current attempt. Lifecycle hooks
// never overlap, and the drain worker starts after OnStart sets it.
owned []Client
}

var _ Client = (*routedClient)(nil)
Expand Down Expand Up @@ -125,46 +128,71 @@ func NewRoutedClient(logger log.Logger, defaultClient Client, routing Routing) (
defaultClient: ClientInfo{defaultClient, defaultClientID},
}

cli.Service = service.NewBaseService(logger, "RoutedClient", cli)
cli.BaseService = service.NewBaseService(logger, "RoutedClient", cli)
return cli, nil
}

// OnStart starts the clients that are not running yet and owns only those:
// clients the caller already started keep their own lifetime.
func (cli *routedClient) OnStart(ctx context.Context) error {
cli.owned = nil
// Client does not require Stop, and BaseService cancels ctx only after OnStart
// returns, so this context is what ends a child before a rollback joins it.
childCtx, cancelChildren := context.WithCancel(ctx)
var errs error
for _, clients := range cli.routing {
for _, client := range clients {
if !client.IsRunning() {
if err := client.Start(ctx); err != nil {
errs = multierror.Append(errs, err)
}
}
for _, client := range cli.clients() {
if client.IsRunning() {
continue
}
}

if !cli.defaultClient.IsRunning() {
if err := cli.defaultClient.Start(ctx); err != nil {
if err := client.Start(childCtx); err != nil {
errs = multierror.Append(errs, err)
continue
}
cli.owned = append(cli.owned, client)
}
// A failed attempt gets no OnStop; release what it started before returning.
if errs != nil {
cancelChildren()
cli.stopOwned()
return errs
}

return errs
if !cli.Go(ctx, func(ctx context.Context) {
defer cancelChildren()
<-ctx.Done()
for _, client := range cli.owned {
client.Wait()
}
}) {
cancelChildren()
cli.stopOwned()
}
return nil
}

func (cli *routedClient) OnStop() {
for _, clients := range cli.routing {
for _, client := range clients {
if client.IsRunning() {
switch c := client.Client.(type) {
case *socketClient:
c.Stop()
case *localClient:
c.Stop()
case *grpcClient:
c.Stop()
}
}
for _, client := range cli.owned {
if stopper, ok := client.(interface{ Stop() }); ok {
stopper.Stop()
}
}
}

func (cli *routedClient) stopOwned() {
cli.OnStop()
for _, client := range cli.owned {
client.Wait()
}
}

func (cli *routedClient) clients() []Client {
clients := []Client{cli.defaultClient.Client}
for _, routes := range cli.routing {
for _, route := range routes {
clients = append(clients, route.Client)
}
}
return clients
}

// delegate calls the given function on the appropriate client with the given
Expand Down
71 changes: 71 additions & 0 deletions abci/client/routed_lifecycle_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package abciclient_test

import (
"context"
"errors"
"testing"
"time"

"github.com/stretchr/testify/require"

abciclient "github.com/dashpay/tenderdash/abci/client"
"github.com/dashpay/tenderdash/abci/types"
"github.com/dashpay/tenderdash/libs/log"
)

var errChildStart = errors.New("child start failed")

type failingStartClient struct{ abciclient.Client }

func (failingStartClient) Start(context.Context) error { return errChildStart }
func (failingStartClient) IsRunning() bool { return false }
func (failingStartClient) Wait() {}

// noStopClient satisfies Client while hiding Stop, which Client does not require.
type noStopClient struct{ abciclient.Client }

// A failed start must stop and join the children it started, and leave alone
// a client the caller had already started.
func TestRoutedClientFailedStartReleasesOnlyItsChildren(t *testing.T) {
testCases := map[string]func(abciclient.Client) abciclient.Client{
"child exposes Stop": func(c abciclient.Client) abciclient.Client { return c },
"child hides Stop": func(c abciclient.Client) abciclient.Client { return noStopClient{c} },
}
for name, wrap := range testCases {
t.Run(name, func(t *testing.T) {
logger := log.NewNopLogger()
callerCtx, cancelCaller := context.WithCancel(context.Background())
defer cancelCaller()
callerOwned := abciclient.NewLocalClient(logger, types.NewBaseApplication())
require.NoError(t, callerOwned.Start(callerCtx))
attemptOwned := abciclient.NewLocalClient(logger, types.NewBaseApplication())
routing := abciclient.Routing{
"Info": {{Client: callerOwned, ClientID: "caller"}},
"Query": {{Client: wrap(attemptOwned), ClientID: "attempt"}},
}
routed, err := abciclient.NewRoutedClient(logger, failingStartClient{}, routing)
require.NoError(t, err)

started := make(chan error, 1)
go func() { started <- routed.Start(context.Background()) }()
select {
case err := <-started:
require.ErrorIs(t, err, errChildStart)
case <-time.After(5 * time.Second):
t.Fatal("failed start blocked joining a client")
}
require.False(t, attemptOwned.IsRunning(), "child started by the failed attempt kept running")
joined := make(chan struct{})
go func() { attemptOwned.Wait(); close(joined) }()
select {
case <-joined:
case <-time.After(5 * time.Second):
t.Fatal("child started by the failed attempt was not joined")
}
// Only a client that started and stopped refuses another start.
require.ErrorContains(t, attemptOwned.Start(callerCtx), "already stopped",
"the failed attempt never started this child")
require.True(t, callerOwned.IsRunning(), "failed start stopped a caller-owned client")
})
}
}
25 changes: 13 additions & 12 deletions abci/client/socket_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,21 +90,23 @@ func (cli *socketClient) OnStart(ctx context.Context) error {
}
cli.conn = conn

go cli.sendRequestsRoutine(ctx, conn)
go cli.recvResponseRoutine(ctx, conn)
cli.Go(ctx, func(ctx context.Context) { cli.sendRequestsRoutine(ctx, conn) })
cli.Go(ctx, func(ctx context.Context) { cli.recvResponseRoutine(ctx, conn) })

return nil
}
}

// OnStop implements Service by closing connection and flushing all queues.
// OnStop closes the connection to unblock transport workers.
func (cli *socketClient) OnStop() {
if cli.conn != nil {
cli.conn.Close()
}
cli.drainQueue()
}

// OnDrain releases pending requests after the transport workers have exited.
func (cli *socketClient) OnDrain() { cli.drainQueue() }

func (cli *socketClient) String() string {
if err := cli.Error(); err != nil {
return fmt.Sprintf("%T(%s):err=%s", cli, cli.addr, err.Error())
Expand Down Expand Up @@ -162,12 +164,12 @@ func (cli *socketClient) sendRequestsRoutine(ctx context.Context, conn io.Writer
cli.trackRequest(reqres)

if err := types.WriteMessage(reqres.Request, bw); err != nil {
cli.stopForError(fmt.Errorf("write to buffer: %w", err))
cli.stopForError(ctx, fmt.Errorf("write to buffer: %w", err))
return
}

if err := bw.Flush(); err != nil {
cli.stopForError(fmt.Errorf("flush buffer: %w", err))
cli.stopForError(ctx, fmt.Errorf("flush buffer: %w", err))
return
}
}
Expand All @@ -181,18 +183,18 @@ func (cli *socketClient) recvResponseRoutine(ctx context.Context, conn io.Reader
res := &types.Response{}

if err := types.ReadMessage(r, res); err != nil {
cli.stopForError(fmt.Errorf("read message: %w", err))
cli.stopForError(ctx, fmt.Errorf("read message: %w", err))
return
}

switch r := res.Value.(type) {
case *types.Response_Exception: // app responded with error
// XXX After setting cli.err, release waiters (e.g. reqres.Done())
cli.stopForError(errors.New(r.Exception.Error))
cli.stopForError(ctx, errors.New(r.Exception.Error))
return
default:
if err := cli.didRecvResponse(res); err != nil {
cli.stopForError(err)
cli.stopForError(ctx, err)
return
}
}
Expand Down Expand Up @@ -435,11 +437,10 @@ func resMatchesReq(req *types.Request, res *types.Response) (ok bool) {
return ok
}

func (cli *socketClient) stopForError(err error) {
if !cli.IsRunning() {
func (cli *socketClient) stopForError(ctx context.Context, err error) {
if ctx.Err() != nil {
return
}

cli.mtx.Lock()
cli.err = err
cli.mtx.Unlock()
Expand Down
14 changes: 7 additions & 7 deletions abci/server/grpc_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,22 +47,22 @@ func (s *GRPCServer) OnStart(ctx context.Context) error {
types.RegisterABCIApplicationServer(s.server, &gRPCApplication{Application: s.app})

s.logger.Info("Listening", "proto", s.proto, "addr", s.addr)
go func() {
go func() {
<-ctx.Done()
s.server.GracefulStop()
}()

if !s.Go(ctx, func(context.Context) {
if err := s.server.Serve(ln); err != nil {
s.logger.Error("error serving gRPC server", "err", err)
}
}()
}) {
return ln.Close()
}
return nil
}

// OnStop stops the gRPC server.
func (s *GRPCServer) OnStop() { s.server.Stop() }

// OnDrain joins handlers that were already running when the transport closed.
func (s *GRPCServer) OnDrain() { s.server.GracefulStop() }

//-------------------------------------------------------

// gRPCApplication is a gRPC shim for Application
Expand Down
Loading
Loading