From 8a13f0e599bdec854d0a84c36738f2e834743e58 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:01:04 +0000 Subject: [PATCH 01/39] fix(service): drain managed workers before releasing resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add attempt-scoped worker admission, cancellation and finalization to BaseService. Migrate consensus, WAL, file groups, ticker and gossip; join child services and preserve durable commit completion. Co-Authored-By: OpenAI GPT-6 🤖 Co-authored by [Claudius the Magnificent](https://github.com/lklimek/claudius) AI Agent --- docs/architecture/service-lifecycle.md | 50 ++++ internal/consensus/common_test.go | 1 + internal/consensus/gossip_peer_worker.go | 42 +-- internal/consensus/gossip_peer_worker_test.go | 34 ++- internal/consensus/reactor.go | 119 ++++---- internal/consensus/reactor_test.go | 14 + internal/consensus/replay_test.go | 4 - internal/consensus/state.go | 71 +++-- internal/consensus/state_apply_commit.go | 3 +- internal/consensus/state_test.go | 49 ++++ internal/consensus/ticker.go | 13 +- internal/consensus/wal.go | 61 +--- internal/consensus/wal_test.go | 39 ++- internal/libs/autofile/autofile.go | 34 +-- internal/libs/autofile/autofile_test.go | 16 ++ internal/libs/autofile/group.go | 62 +--- libs/service/lifecycle_test.go | 238 ++++++++++++++++ libs/service/service.go | 266 +++++++++--------- 18 files changed, 701 insertions(+), 415 deletions(-) create mode 100644 docs/architecture/service-lifecycle.md create mode 100644 libs/service/lifecycle_test.go diff --git a/docs/architecture/service-lifecycle.md b/docs/architecture/service-lifecycle.md new file mode 100644 index 0000000000..b6f8f78aee --- /dev/null +++ b/docs/architecture/service-lifecycle.md @@ -0,0 +1,50 @@ +# Service lifecycle + +`BaseService` owns one successful service lifetime. `Start` passes a child of +its caller's context to `OnStart`, preserving values and deadlines. Both parent +cancellation and `Stop` cancel that child. Services cannot restart after a +successful start; a failed start may be retried after its workers drain. + +Use `Go(ctx, func(context.Context))` for service-owned background work. Pass the +context received by `OnStart`, or a child of it. Admission is synchronized with +shutdown and tied to that startup attempt; it returns false after admission +closes or if the context belongs to a different attempt. Clean up any resources +acquired before rejected admission. Plain `go` statements are not tracked. + +Shutdown has three phases: + +1. Close admission, mark `IsRunning` false, and cancel the service context. +2. Call `OnStop` once to unblock workers and request child shutdown. +3. Join admitted work, call optional `OnDrain` to release resources, and unblock + `Wait`. + +`Stop` does not join workers. Concurrent or recursive `Stop` calls return +immediately. A stop during startup cancels the context and schedules `OnStop` +after successful `OnStart`; hooks never overlap. `OnStop` must not join managed +workers. Workers may call `Stop`; neither workers nor hooks may call their own +service's `Wait`. Hooks run without the lifecycle mutex. + +`IsRunning` is false during startup and from the first shutdown request onward; +it does not imply completion. `Wait` waits for the current attempt, including +startup, registered work, and finalization. It returns immediately before the +first start or after failed startup has drained. Repeated `Start` while running +succeeds; concurrent startup and startup after shutdown return errors. + +On failed startup, the implementation rolls back its acquired resources. +`BaseService` cancels and joins registered work before permitting retry; it does +not call `OnStop` or `OnDrain` for that failed attempt. Avoid waiting for workers +inside startup rollback unless their operation has first been unblocked. + +## Ownership and durability + +Registering a wrapper does not automatically join its children. Owners stop and +wait for child services before closing their shared stores, transports or sinks. +A request or peer session can retain its own cancellation and join group when +its lifetime is shorter than the enclosing service. + +Consensus keeps its WAL available until the receive loop finishes. WAL and +file groups flush after their periodic workers exit; AutoFile closes and joins +its timer/signal worker. An already persisted consensus commit finishes +application before the owner releases stores. These explicitly owned resource +lifetimes can outlive cancellation of the caller; their owners must still join +them. A timeout is not a substitute for completion. diff --git a/internal/consensus/common_test.go b/internal/consensus/common_test.go index d9df4e175e..af2b1a1c74 100644 --- a/internal/consensus/common_test.go +++ b/internal/consensus/common_test.go @@ -1192,6 +1192,7 @@ type mockTicker struct { } func (m *mockTicker) Start(context.Context) error { return nil } +func (m *mockTicker) Wait() {} func (m *mockTicker) Stop() {} func (m *mockTicker) IsRunning() bool { return false } diff --git a/internal/consensus/gossip_peer_worker.go b/internal/consensus/gossip_peer_worker.go index 06667c796d..10ae4a536e 100644 --- a/internal/consensus/gossip_peer_worker.go +++ b/internal/consensus/gossip_peer_worker.go @@ -3,12 +3,12 @@ package consensus import ( "context" "runtime/debug" - "sync/atomic" "time" "github.com/jonboulle/clockwork" "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" ) type gossipHandlerFunc func(ctx context.Context, appState StateData) @@ -16,24 +16,21 @@ type gossipHandlerFunc func(ctx context.Context, appState StateData) type gossipHandler struct { sleepDuration time.Duration handlerFunc func(ctx context.Context, appState StateData) - stoppedCh chan struct{} } func newGossipHandler(fn gossipHandlerFunc, sleep time.Duration) gossipHandler { return gossipHandler{ sleepDuration: sleep, handlerFunc: fn, - stoppedCh: make(chan struct{}), } } type peerGossipWorker struct { + service.BaseService clock clockwork.Clock logger log.Logger handlers []gossipHandler - running atomic.Bool stateDataStore *StateDataStore - stopCh chan struct{} } func newPeerGossipWorker( @@ -51,10 +48,9 @@ func newPeerGossipWorker( optimistic: true, clock: clock, } - return &peerGossipWorker{ + worker := &peerGossipWorker{ clock: clock, logger: logger, - stopCh: make(chan struct{}), stateDataStore: state.stateDataStore, handlers: []gossipHandler{ newGossipHandler( @@ -71,35 +67,18 @@ func newPeerGossipWorker( ), }, } + worker.BaseService = *service.NewBaseService(logger, "PeerGossipWorker", worker) + return worker } -func (g *peerGossipWorker) IsRunning() bool { - return g.running.Load() -} - -func (g *peerGossipWorker) Start(ctx context.Context) error { - g.running.Store(true) +func (g *peerGossipWorker) OnStart(ctx context.Context) error { for _, handler := range g.handlers { - go g.runHandler(ctx, handler) + g.Go(ctx, func(ctx context.Context) { g.runHandler(ctx, handler) }) } return nil } -func (g *peerGossipWorker) Stop() { - if !g.running.Swap(false) { - return - } - g.logger.Trace("peer gossip worker stopping") - close(g.stopCh) - g.Wait() -} - -func (g *peerGossipWorker) Wait() { - for _, hd := range g.handlers { - <-hd.stoppedCh - g.logger.Trace("peer gossip worker stopped") - } -} +func (g *peerGossipWorker) OnStop() {} // runGossipHandler invokes a gossip handler, converting a panic into a logged // error. Handlers load historical records at heights chosen by the peer they @@ -124,13 +103,8 @@ func (g *peerGossipWorker) runHandler(ctx context.Context, hd gossipHandler) { timer.Reset(hd.sleepDuration) select { case <-timer.Chan(): - case <-g.stopCh: - g.logger.Trace("peer gossip worker got stop signal") - close(hd.stoppedCh) - return case <-ctx.Done(): g.logger.Trace("peer gossip worker got stop signal via context.Done") - close(hd.stoppedCh) return } } diff --git a/internal/consensus/gossip_peer_worker_test.go b/internal/consensus/gossip_peer_worker_test.go index dc9a16031a..a7fd678ad8 100644 --- a/internal/consensus/gossip_peer_worker_test.go +++ b/internal/consensus/gossip_peer_worker_test.go @@ -2,7 +2,6 @@ package consensus import ( "context" - "sync/atomic" "testing" "time" @@ -11,6 +10,7 @@ import ( "github.com/dashpay/tenderdash/libs/eventemitter" "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" ) func TestPeerGossipWorker(t *testing.T) { @@ -33,10 +33,9 @@ func TestPeerGossipWorker(t *testing.T) { handlerCalledCh <- struct{}{} }, 1*time.Second), }, - running: atomic.Bool{}, stateDataStore: NewStateDataStore(NopMetrics(), logger, cfg.Consensus, emitter), - stopCh: make(chan struct{}), } + pg.BaseService = *service.NewBaseService(logger, "PeerGossipWorker", &pg) require.False(t, pg.IsRunning()) err := pg.Start(ctx) require.NoError(t, err) @@ -46,6 +45,35 @@ func TestPeerGossipWorker(t *testing.T) { } defer cancel() pg.Stop() + pg.Wait() require.False(t, pg.IsRunning()) close(handlerCalledCh) } + +func TestPeerGossipWorkerWaitDrainsHandler(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + entered, release := make(chan struct{}), make(chan struct{}) + cs, _ := makeState(ctx, t, makeStateArgs{}) + pg := &peerGossipWorker{ + clock: clockwork.NewRealClock(), logger: log.NewNopLogger(), stateDataStore: cs.stateDataStore, + handlers: []gossipHandler{newGossipHandler(func(context.Context, StateData) { close(entered); <-release }, time.Hour)}, + } + pg.BaseService = *service.NewBaseService(pg.logger, t.Name(), pg) + require.NoError(t, pg.Start(ctx)) + <-entered + pg.Stop() + done := make(chan struct{}) + go func() { pg.Wait(); close(done) }() + select { + case <-done: + t.Error("Wait returned before gossip handler finished") + case <-time.After(20 * time.Millisecond): + } + close(release) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("gossip handler did not drain") + } +} diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index 8b6bde1a9f..f31cb735d2 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -261,8 +261,7 @@ type Reactor struct { clock clockwork.Clock // Context for controlling reactor goroutines - ctx context.Context - cancel context.CancelFunc + ctx context.Context } // NewReactor returns a reference to a new consensus reactor, which implements @@ -313,8 +312,7 @@ type channelBundle struct { func (r *Reactor) OnStart(ctx context.Context) error { r.logger.Trace("consensus wait sync", "wait_sync", r.WaitSync()) - // Create reactor-specific context that will be canceled in OnStop - r.ctx, r.cancel = context.WithCancel(ctx) + r.ctx = ctx // Per-peer rate limiter for the vote channel. drop=true: over-limit // messages are dropped rather than delayed, so a flood from one peer cannot @@ -374,14 +372,10 @@ func (r *Reactor) OnStart(ctx context.Context) error { return err } - // start routine that computes peer statistics for evaluating peer quality - // - // TODO: Evaluate if we need this to be synchronized via WaitGroup as to not - // leak the goroutine when stopping the reactor. - go r.peerStatsRoutine(r.ctx, peerUpdates) + r.Go(ctx, func(ctx context.Context) { r.peerStatsRoutine(ctx, peerUpdates) }) // start routine that handles peer errors from the state machine - go r.peerErrorRoutine(r.ctx, chBundle.state) + r.Go(ctx, func(ctx context.Context) { r.peerErrorRoutine(ctx, chBundle.state) }) r.subscribeToBroadcastEvents(r.ctx, chBundle.state) @@ -397,37 +391,27 @@ func (r *Reactor) OnStart(ctx context.Context) error { // Data, vote and vote set must wait. // We cannot skip waiting messages, as the peers might already have marked them as delivered. // XXX: this can lead to a deadlock, if so - we need additional buffer for (at least) Commits. - go r.processMsgCh(r.ctx, chBundle.state, chBundle) - go func() { + r.Go(ctx, func(ctx context.Context) { r.processMsgCh(ctx, chBundle.state, chBundle) }) + r.Go(ctx, func(ctx context.Context) { select { case <-r.readySignal: - go r.processMsgCh(r.ctx, chBundle.data, chBundle) - go r.processMsgCh(r.ctx, chBundle.vote, chBundle) - go r.processMsgCh(r.ctx, chBundle.voteSet, chBundle) - case <-r.ctx.Done(): + r.Go(ctx, func(ctx context.Context) { r.processMsgCh(ctx, chBundle.data, chBundle) }) + r.Go(ctx, func(ctx context.Context) { r.processMsgCh(ctx, chBundle.vote, chBundle) }) + r.Go(ctx, func(ctx context.Context) { r.processMsgCh(ctx, chBundle.voteSet, chBundle) }) + case <-ctx.Done(): } - }() + }) - go r.processPeerUpdates(r.ctx, peerUpdates, chBundle) + r.Go(ctx, func(ctx context.Context) { r.processPeerUpdates(ctx, peerUpdates, chBundle) }) return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit, as well as unsubscribing from events and stopping -// state. -func (r *Reactor) OnStop() { - // Cancel the reactor context to signal all goroutines to stop - if r.cancel != nil { - r.cancel() - } +// OnStop requests consensus shutdown. +func (r *Reactor) OnStop() { r.state.Stop() } - r.state.Stop() - - if !r.WaitSync() { - r.state.Wait() - } -} +// OnDrain joins consensus after channel and peer workers have finished. +func (r *Reactor) OnDrain() { r.state.Stop(); r.state.Wait() } // WaitSync returns whether the consensus reactor is waiting for state/block sync. func (r *Reactor) WaitSync() bool { @@ -447,6 +431,20 @@ func (r *Reactor) WaitSync() bool { // targetHeight is the highest committed block height reported during block sync. // skipWAL says the node needs no WAL catchup. func (r *Reactor) SwitchToConsensus(ctx context.Context, state sm.State, skipWAL bool, targetHeight int64) { + if r.ctx == nil || ctx.Err() != nil { + return + } + done := make(chan struct{}) + if !r.Go(r.ctx, func(ctx context.Context) { + defer close(done) + r.switchToConsensus(ctx, state, skipWAL, targetHeight) + }) { + return + } + <-done +} + +func (r *Reactor) switchToConsensus(ctx context.Context, state sm.State, skipWAL bool, targetHeight int64) { r.logger.Info("switching to consensus", "target_height", targetHeight) if targetHeight > state.LastBlockHeight { @@ -473,6 +471,9 @@ func (r *Reactor) SwitchToConsensus(ctx context.Context, state sm.State, skipWAL r.state.eventPublisher.PublishNewRoundStepEvent(stateData.RoundState) if err := r.state.Start(ctx); err != nil { + if ctx.Err() != nil { + return + } panic(fmt.Sprintf(`failed to start consensus state: %v conS: @@ -520,8 +521,6 @@ func (r *Reactor) GetPeerState(peerID types.NodeID) (*PeerState, bool) { // internal pubsub defined in the consensus state to broadcast them to peers // upon receiving. func (r *Reactor) subscribeToBroadcastEvents(ctx context.Context, stateCh p2p.Channel) { - onStopCh := r.state.getOnStopCh() - r.state.emitter.AddListener( types.EventNewRoundStepValue, func(data eventemitter.EventData) error { @@ -531,14 +530,7 @@ func (r *Reactor) subscribeToBroadcastEvents(ctx context.Context, stateCh p2p.Ch return err } r.logResult(err, r.logger, "broadcasting round step message", "height", rs.Height, "round", rs.Round) - select { - case onStopCh <- data.(*cstypes.RoundState): - return nil - case <-ctx.Done(): - return ctx.Err() - default: - return nil - } + return nil }, ) @@ -612,19 +604,14 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda if !r.IsRunning() { return } - r.peerUp(ctx, peerUpdate, 3, chans) + r.peerUp(ctx, peerUpdate, chans) case p2p.PeerStatusDown: r.peerDown(ctx, peerUpdate, chans) } } -// peerUp starts the peer. It recursively retries up to `retries` times if the peer is already closing. -func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries int, chans channelBundle) { - if retries < 1 { - r.logger.Error("peer up failed: max retries exceeded", "peer", peerUpdate.NodeID) - return - } - +// peerUp starts gossip for a live peer connection. +func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, chans channelBundle) { r.mtx.Lock() defer r.mtx.Unlock() @@ -650,31 +637,19 @@ func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries "peer", ps.peerID, "peer_proTxHash", ps.GetProTxHash().ShortString(), ) - // TODO needs to register this gossip worker, to be able to stop it once a peer will be down msgSender := p2pMsgSender{logger: logger, ps: ps, chans: chans} pgw := newPeerGossipWorker(logger, ps, r.state, &msgSender) - select { - case <-ctx.Done(): - // Hmm, someone is closing this peer right now, let's wait and retry - // Note: we run this in a goroutine to not block main goroutine in ps.broadcastWG.Wait() - go func() { - time.Sleep(r.state.config.PeerGossipSleepDuration) - r.peerUp(ctx, peerUpdate, retries-1, chans) - }() + if ctx.Err() != nil { return - default: } if !ps.IsRunning() { - // Set the peer state's closer to signal to all spawned goroutines to exit - // when the peer is removed. We also set the running state to ensure we - // do not spawn multiple instances of the same goroutines and finally we - // set the waitgroup counter so we know when all goroutines have exited. + // Peer removal cancels this session; reactor shutdown also joins it. ps.SetRunning(true) ctx, ps.cancel = context.WithCancel(ctx) - go func() { + if !r.Go(ctx, func(ctx context.Context) { select { case <-ctx.Done(): return @@ -686,18 +661,24 @@ func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries return } // start goroutines for this peer - _ = pgw.Start(ctx) + if err := pgw.Start(ctx); err != nil { + return + } + defer pgw.Wait() // Send our state to the peer. If we're block-syncing, broadcast a // RoundStepMessage later upon SwitchToConsensus(). if !r.WaitSync() { - go func() { + r.Go(ctx, func(ctx context.Context) { rs := r.state.GetRoundState() err := msgSender.send(ctx, rs.NewRoundStepMessage()) r.logResult(err, r.logger, "sending round step msg", "height", rs.Height, "round", rs.Round) - }() + }) } - }() + }) { + ps.SetRunning(false) + ps.cancel() + } } } diff --git a/internal/consensus/reactor_test.go b/internal/consensus/reactor_test.go index 9787e40d1e..c2e4e5bd55 100644 --- a/internal/consensus/reactor_test.go +++ b/internal/consensus/reactor_test.go @@ -834,3 +834,17 @@ func TestReactorPeerDownDeletesPeerStateSynchronously(t *testing.T) { "peerDown must delete the peer state before returning, so a fast reconnect cannot reuse it") require.False(t, ps.IsRunning(), "peerDown must stop the peer state before returning") } + +func TestReactorRejectsConsensusHandoffAfterShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cs, _ := makeState(ctx, t, makeStateArgs{}) + rts := setup(ctx, t, 1, []*State{cs}, 32) + for _, r := range rts.reactors { + r.Stop() + r.Wait() + r.SwitchToConsensus(ctx, cs.GetStateData().state, false, 0) + require.False(t, cs.IsRunning(), "handoff started consensus after reactor shutdown") + require.True(t, r.WaitSync(), "rejected handoff changed sync state") + } +} diff --git a/internal/consensus/replay_test.go b/internal/consensus/replay_test.go index d2ce9c1332..bb08f1b80e 100644 --- a/internal/consensus/replay_test.go +++ b/internal/consensus/replay_test.go @@ -1521,10 +1521,6 @@ func testWALRoundsSkipper(t *testing.T, slowProposer bool) { <-replayStopped cs.Stop() cs.Wait() - // The stop predicate bypasses receiveRoutine's WAL and queue shutdown. - cs.wal.Stop() - cs.wal.Wait() - cs.msgInfoQueue.stop() }() newBlockSub, err := cs.eventBus.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ diff --git a/internal/consensus/state.go b/internal/consensus/state.go index b1f28890d5..d10a9e2843 100644 --- a/internal/consensus/state.go +++ b/internal/consensus/state.go @@ -192,7 +192,6 @@ type State struct { proposedAppVersion uint64 // wait the channel event happening for shutting down the state gracefully - onStopCh chan *cstypes.RoundState msgInfoQueue *msgInfoQueue msgDispatcher *msgInfoDispatcher @@ -285,7 +284,6 @@ func NewState( evpool: evpool, emitter: eventemitter.New(eventemitter.WithLogger(logger)), metrics: NopMetrics(), - onStopCh: make(chan *cstypes.RoundState), verificationBudget: newVerificationBudget( cfg.VerificationRateLimit, ), @@ -471,7 +469,20 @@ func (cs *State) SetPrivValidator(ctx context.Context, priv types.PrivValidator) // OnStart loads the latest state via the WAL, and starts the timeout and // receive routines. -func (cs *State) OnStart(ctx context.Context) error { +func (cs *State) OnStart(ctx context.Context) (err error) { + defer func() { + if err != nil { + cs.wal.Stop() + cs.wal.Wait() + cs.wal = nilWAL{} + cs.timeoutTicker.Stop() + cs.timeoutTicker.Wait() + if _, ok := cs.timeoutTicker.(*timeoutTicker); ok { + cs.timeoutTicker = NewTimeoutTicker(cs.logger) + cs.roundScheduler.timeoutTicker = cs.timeoutTicker + } + } + }() if err := cs.updateStateFromStore(); err != nil { return err } @@ -518,6 +529,7 @@ func (cs *State) OnStart(ctx context.Context) error { // 1) prep work cs.wal.Stop() + cs.wal.Wait() repairAttempted = true @@ -545,7 +557,9 @@ func (cs *State) OnStart(ctx context.Context) error { } // now start the receiveRoutine - go cs.receiveRoutine(ctx, cs.stopFn) + if !cs.Go(ctx, func(ctx context.Context) { cs.receiveRoutine(ctx, cs.stopFn) }) { + return context.Canceled + } // schedule the first round! // use GetRoundState so we don't race the receiveRoutine for access @@ -556,7 +570,8 @@ func (cs *State) OnStart(ctx context.Context) error { // loadWalFile loads WAL data from file. It overwrites cs.wal. func (cs *State) loadWalFile(ctx context.Context) error { - wal, err := cs.OpenWAL(ctx, cs.config.WalFile()) + // receiveRoutine owns the WAL until its final write and explicit Stop/Wait. + wal, err := cs.OpenWAL(context.WithoutCancel(ctx), cs.config.WalFile()) if err != nil { cs.logger.Error("failed to load state WAL", "err", err) return err @@ -566,30 +581,14 @@ func (cs *State) loadWalFile(ctx context.Context) error { return nil } -func (cs *State) getOnStopCh() chan *cstypes.RoundState { - cs.mtx.RLock() - defer cs.mtx.RUnlock() +// OnStop requests timeout worker shutdown. Wait also joins receiveRoutine. +func (cs *State) OnStop() { cs.timeoutTicker.Stop() } - return cs.onStopCh -} - -// OnStop implements service.Service. -func (cs *State) OnStop() { - stateData := cs.stateDataStore.Get() - // If the node is committing a new block, wait until it is finished! - if cs.GetRoundState().Step == cstypes.RoundStepApplyCommit { - select { - case <-cs.getOnStopCh(): - case <-time.After(stateData.state.ConsensusParams.Timeout.Vote): - // we wait vote timeout, just in case - cs.logger.Error("OnStop: timeout waiting for commit to finish", "time", stateData.state.ConsensusParams.Timeout.Vote) - } - } - - if cs.timeoutTicker.IsRunning() { - cs.timeoutTicker.Stop() - } - // WAL is stopped in receiveRoutine. +// OnDrain joins child services before callers release consensus stores. +func (cs *State) OnDrain() { + cs.timeoutTicker.Wait() + cs.wal.Stop() + cs.wal.Wait() } // OpenWAL opens a file to log all consensus messages and timeouts for @@ -603,6 +602,7 @@ func (cs *State) OpenWAL(ctx context.Context, walFile string) (WAL, error) { if err := wal.Start(ctx); err != nil { cs.logger.Error("failed to start WAL", "err", err) + wal.Group().Close() return nil, err } @@ -700,12 +700,6 @@ func (cs *State) receiveRoutine(ctx context.Context, stopFn func(*State) bool) { if r := recover(); r != nil { cs.logger.Error("CONSENSUS FAILURE!!!", "err", r, "stack", string(debug.Stack())) - // Make a best-effort attempt to close the WAL, but otherwise do not - // attempt to gracefully terminate. Once consensus has irrecoverably - // failed, any additional progress we permit the node to make may - // complicate diagnosing and recovering from the failure. - onExit(cs) - // There are a couple of cases where the we // panic with an error from deeper within the // state machine and in these cases, typically @@ -739,7 +733,12 @@ func (cs *State) receiveRoutine(ctx context.Context, stopFn func(*State) bool) { } }() - go cs.msgInfoQueue.fanIn(ctx) + queueDone := make(chan struct{}) + go func() { + defer close(queueDone) + cs.msgInfoQueue.fanIn(ctx) + }() + defer func() { onExit(cs); <-queueDone }() for { if stopFn != nil && stopFn(cs) { @@ -769,7 +768,6 @@ func (cs *State) receiveRoutine(ctx context.Context, stopFn func(*State) bool) { if err != nil { // Leaving without this would strand the queue's reader // goroutines: nothing else ever tells them to stop. - onExit(cs) return } err = stateData.Save() @@ -790,7 +788,6 @@ func (cs *State) receiveRoutine(ctx context.Context, stopFn func(*State) bool) { cs.logger.Error("failed update state-data", "err", err) } case <-ctx.Done(): - onExit(cs) return } diff --git a/internal/consensus/state_apply_commit.go b/internal/consensus/state_apply_commit.go index 0f0fbc26c2..054aaba054 100644 --- a/internal/consensus/state_apply_commit.go +++ b/internal/consensus/state_apply_commit.go @@ -88,7 +88,8 @@ func (c *ApplyCommitAction) Execute(ctx context.Context, stateEvent StateEvent) } // Create a copy of the state for staging and an event cache for txs. - stateCopy, finalizeResp, err := c.blockExec.finalize(ctx, stateData, commit) + // Once the block is persisted, finish application before shutdown releases stores. + stateCopy, finalizeResp, err := c.blockExec.finalize(context.WithoutCancel(ctx), stateData, commit) if err != nil { c.logger.Error("failed to apply block", "err", err) // If something went wrong within ABCI client, it can stop and we can't recover from it. diff --git a/internal/consensus/state_test.go b/internal/consensus/state_test.go index b4ec0b7435..6b76104a85 100644 --- a/internal/consensus/state_test.go +++ b/internal/consensus/state_test.go @@ -3496,3 +3496,52 @@ func mockProposerApplicationCalls(t *testing.T, m *abcimocks.Application, round Once() } } +func TestStateWaitDrainsReceiveRoutine(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cs, _ := makeState(ctx, t, makeStateArgs{config: configSetup(t)}) + entered, release := make(chan struct{}), make(chan struct{}) + notified := false + cs.stopFn = func(*State) bool { + if !notified { + close(entered) + notified = true + } + <-release + return false + } + require.NoError(t, cs.Start(ctx)) + <-entered + cs.Stop() + + waited := make(chan struct{}) + go func() { + cs.Wait() + close(waited) + }() + select { + case <-waited: + t.Error("Wait returned before the consensus receive routine finished") + case <-time.After(50 * time.Millisecond): + } + close(release) + select { + case <-waited: + case <-time.After(time.Second): + t.Fatal("Wait did not return after the receive routine finished") + } +} + +func TestDirectReceiveRoutineDrainsQueue(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cs, _ := makeState(ctx, t, makeStateArgs{}) + cs.receiveRoutine(ctx, func(*State) bool { return true }) + select { + case _, ok := <-cs.msgInfoQueue.read(): + require.False(t, ok) + default: + t.Fatal("receiveRoutine returned before queue was closed") + } + require.NoError(t, ctx.Err()) +} diff --git a/internal/consensus/ticker.go b/internal/consensus/ticker.go index 30b94d3adb..80b7f56ff4 100644 --- a/internal/consensus/ticker.go +++ b/internal/consensus/ticker.go @@ -18,6 +18,7 @@ var ( type TimeoutTicker interface { Start(context.Context) error Stop() + Wait() IsRunning() bool Chan() <-chan timeoutInfo // on which to receive a timeout ScheduleTimeout(ti timeoutInfo) // reset the timer @@ -52,13 +53,16 @@ func NewTimeoutTicker(logger log.Logger) TimeoutTicker { // OnStart implements service.Service. It starts the timeout routine. func (t *timeoutTicker) OnStart(ctx context.Context) error { - go t.timeoutRoutine(ctx) + t.Go(ctx, t.timeoutRoutine) return nil } // OnStop implements service.Service. It stops the timeout routine. -func (t *timeoutTicker) OnStop() { t.stopTimer() } +func (t *timeoutTicker) OnStop() {} + +// OnDrain stops the timer after its sole scheduling worker exits. +func (t *timeoutTicker) OnDrain() { t.stopTimer() } // Chan returns a channel on which timeouts are sent. func (t *timeoutTicker) Chan() <-chan timeoutInfo { @@ -122,12 +126,13 @@ func (t *timeoutTicker) timeoutRoutine(ctx context.Context) { // Determinism comes from playback in the receiveRoutine. // We can eliminate it by merging the timeoutRoutine into receiveRoutine // and managing the timeouts ourselves with a millisecond ticker - go func(toi timeoutInfo) { + toi := ti + t.Go(ctx, func(ctx context.Context) { select { case t.tockChan <- toi: case <-ctx.Done(): } - }(ti) + }) case <-ctx.Done(): return } diff --git a/internal/consensus/wal.go b/internal/consensus/wal.go index dbf3661dc4..5dd2658d3c 100644 --- a/internal/consensus/wal.go +++ b/internal/consensus/wal.go @@ -8,7 +8,6 @@ import ( "hash/crc32" "io" "path/filepath" - "sync" "time" "github.com/cosmos/gogoproto/proto" @@ -103,17 +102,6 @@ type BaseWAL struct { flushTicker *time.Ticker flushInterval time.Duration - - // procCancel cancels the context supplied to processFlushTicks. It is - // derived from the caller ctx in OnStart (context.WithCancel) so the - // goroutine exits on either procCancel() or the parent ctx. OnStop calls - // procCancel() directly, which makes goroutineWg.Wait() in Wait() safe to - // call immediately after Stop() regardless of whether the caller ctx fired. - procCancel context.CancelFunc - - // goroutineWg tracks the processFlushTicks goroutine so that Wait() can - // block until it has truly exited. - goroutineWg sync.WaitGroup } var _ WAL = &BaseWAL{} @@ -163,17 +151,7 @@ func (wal *BaseWAL) OnStart(ctx context.Context) error { return err } wal.flushTicker = time.NewTicker(wal.flushInterval) - // Derive a child context so that OnStop can cancel the goroutine via - // procCancel() independently of the caller ctx. The child inherits - // cancellation from ctx so the existing "exit when caller ctx cancels" - // behavior is preserved. - procCtx, procCancel := context.WithCancel(ctx) - wal.procCancel = procCancel - wal.goroutineWg.Add(1) - go func() { - defer wal.goroutineWg.Done() - wal.processFlushTicks(procCtx) - }() + wal.Go(ctx, wal.processFlushTicks) return nil } @@ -200,46 +178,19 @@ func (wal *BaseWAL) FlushAndSync() error { // Use Wait() to ensure it's finished shutting down // before cleaning up files. func (wal *BaseWAL) OnStop() { - // Cancel the goroutine's context first so processFlushTicks exits on its - // next select iteration without waiting for the caller ctx to be canceled. - wal.procCancel() wal.flushTicker.Stop() +} + +// OnDrain flushes and closes files after the flush worker has exited. +func (wal *BaseWAL) OnDrain() { if err := wal.FlushAndSync(); err != nil { wal.logger.Error("error on flush data to disk", "error", err) } wal.group.Stop() + wal.group.Wait() wal.group.Close() } -// Wait for the underlying autofile group to finish shutting down -// so it's safe to cleanup files. -// -// This blocks until: -// 1. The WAL BaseService has stopped (OnStop has returned). -// 2. The Group BaseService has stopped. -// 3. The processFlushTicks goroutine has fully exited. -// 4. The Group's processTicks goroutine has fully exited. -// -// Points 3 and 4 are important: those goroutines run with a child context -// derived from the caller ctx. OnStop() cancels that child context via -// procCancel() so goroutineWg.Wait() / WaitForGoroutines() always unblock -// promptly after Stop() — regardless of whether the original caller ctx has -// fired. Without waiting for them, a goroutine still executing a flush tick -// could re-open the WAL file (via AutoFile) after the caller believes the -// WAL is fully stopped, causing a race with t.TempDir() cleanup -// ("directory not empty"). -func (wal *BaseWAL) Wait() { - if wal.IsRunning() { - wal.BaseService.Wait() - } - if wal.group.IsRunning() { - wal.group.Wait() - } - // Wait for context-driven goroutines that are not tracked by BaseService. - wal.group.WaitForGoroutines() - wal.goroutineWg.Wait() -} - // Write is called in newStep and for each receive on the // peerMsgQueue and the timeoutTicker. // NOTE: does not call fsync() diff --git a/internal/consensus/wal_test.go b/internal/consensus/wal_test.go index cdc703da21..d409c12836 100644 --- a/internal/consensus/wal_test.go +++ b/internal/consensus/wal_test.go @@ -43,7 +43,7 @@ func TestWALTruncate(t *testing.T) { require.NoError(t, err) err = wal.Start(ctx) require.NoError(t, err) - t.Cleanup(func() { wal.Stop(); wal.Group().Stop(); wal.Group().Wait(); wal.Wait() }) + t.Cleanup(func() { wal.Stop(); wal.Wait() }) // 60 block's size nearly 70K, greater than group's headBuf size(4096 * 10), // when headBuf is full, truncate content will Flush to the file. at this @@ -111,7 +111,7 @@ func TestWALWrite(t *testing.T) { require.NoError(t, err) err = wal.Start(ctx) require.NoError(t, err) - t.Cleanup(func() { wal.Stop(); wal.Group().Stop(); wal.Group().Wait(); wal.Wait() }) + t.Cleanup(func() { wal.Stop(); wal.Wait() }) // 1) Write returns an error if msg is too big msg := &BlockPartMessage{ @@ -253,7 +253,7 @@ func TestWALPeriodicSync(t *testing.T) { assert.NotZero(t, wal.Group().Buffered()) require.NoError(t, wal.Start(ctx)) - t.Cleanup(func() { wal.Stop(); wal.Group().Stop(); wal.Group().Wait(); wal.Wait() }) + t.Cleanup(func() { wal.Stop(); wal.Wait() }) time.Sleep(walTestFlushInterval + (20 * time.Millisecond)) @@ -271,3 +271,36 @@ func TestWALPeriodicSync(t *testing.T) { t.Cleanup(leaktest.Check(t)) } + +func TestWALShutdownFlushesBufferedWrites(t *testing.T) { + for _, parentStop := range []bool{false, true} { + t.Run(map[bool]string{false: "manual", true: "parent"}[parentStop], func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + path := filepath.Join(t.TempDir(), "wal") + wal, err := NewWAL(ctx, log.NewNopLogger(), path) + require.NoError(t, err) + wal.SetFlushInterval(time.Hour) + require.NoError(t, wal.Start(ctx)) + require.NoError(t, wal.Write(EndHeightMessage{123})) + if parentStop { + cancel() + } else { + wal.Stop() + } + wal.Wait() + file, err := os.Open(path) + require.NoError(t, err) + defer file.Close() + decoder := NewWALDecoder(file) + first, err := decoder.Decode() + require.NoError(t, err) + require.Equal(t, EndHeightMessage{0}, first.Msg) + last, err := decoder.Decode() + require.NoError(t, err) + require.Equal(t, EndHeightMessage{123}, last.Msg) + require.ErrorIs(t, wal.Group().Head.Sync(), autofile.ErrAutoFileClosed) + require.NoError(t, os.Remove(path)) + }) + } +} diff --git a/internal/libs/autofile/autofile.go b/internal/libs/autofile/autofile.go index 7f2dcc0ee8..6f670fe40c 100644 --- a/internal/libs/autofile/autofile.go +++ b/internal/libs/autofile/autofile.go @@ -56,6 +56,7 @@ type AutoFile struct { closeTicker *time.Ticker // signals periodic close cancel func() // cancels the lifecycle context + done chan struct{} mtx sync.Mutex // guards the fields below closed bool // true when the the autofile is no longer usable @@ -78,9 +79,11 @@ func OpenAutoFile(ctx context.Context, path string) (*AutoFile, error) { Path: path, closeTicker: time.NewTicker(autoFileClosePeriod), cancel: cancel, + done: make(chan struct{}), } if err := af.openFile(); err != nil { - af.Close() + cancel() + af.closeTicker.Stop() return nil, err } @@ -89,42 +92,35 @@ func OpenAutoFile(ctx context.Context, path string) (*AutoFile, error) { hupc := make(chan os.Signal, 1) signal.Notify(hupc, syscall.SIGHUP) go func() { - defer close(hupc) + defer close(af.done) + defer signal.Stop(hupc) + defer af.closeTicker.Stop() for { select { case <-hupc: _ = af.closeFile() + case <-af.closeTicker.C: + _ = af.closeFile() case <-ctx.Done(): + _ = af.withLock(func() error { af.closed = true; return af.unsyncCloseFile() }) return } } }() - go af.closeFileRoutine(ctx) - return af, nil } // Close shuts down the service goroutine and marks af as invalid. Operations // on af after Close will report an error. func (af *AutoFile) Close() error { - return af.withLock(func() error { - af.cancel() // signal the close service to stop - af.closed = true // mark the file as invalid + err := af.withLock(func() error { + af.cancel() + af.closed = true return af.unsyncCloseFile() }) -} - -func (af *AutoFile) closeFileRoutine(ctx context.Context) { - for { - select { - case <-ctx.Done(): - _ = af.Close() - return - case <-af.closeTicker.C: - _ = af.closeFile() - } - } + <-af.done + return err } func (af *AutoFile) closeFile() (err error) { diff --git a/internal/libs/autofile/autofile_test.go b/internal/libs/autofile/autofile_test.go index 9dbba276ac..c79eb9719c 100644 --- a/internal/libs/autofile/autofile_test.go +++ b/internal/libs/autofile/autofile_test.go @@ -144,3 +144,19 @@ func mustReadFile(t *testing.T, filePath string) []byte { return fileBytes } + +func TestCloseJoinsAutoFileWorker(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + af, err := OpenAutoFile(ctx, filepath.Join(t.TempDir(), "wal")) + require.NoError(t, err) + require.NoError(t, af.Close()) + select { + case <-af.done: + default: + t.Fatal("Close returned before worker exited") + } + require.NoError(t, af.Close()) + _, err = af.Write([]byte("late write")) + require.ErrorIs(t, err, ErrAutoFileClosed) +} diff --git a/internal/libs/autofile/group.go b/internal/libs/autofile/group.go index 71bb8d04bf..fc0c484040 100644 --- a/internal/libs/autofile/group.go +++ b/internal/libs/autofile/group.go @@ -11,7 +11,6 @@ import ( "regexp" "strconv" "strings" - stdsync "sync" "time" sync "github.com/sasha-s/go-deadlock" @@ -71,19 +70,6 @@ type Group struct { minIndex int // Includes head maxIndex int // Includes head, where Head will move to - // procCancel cancels the context supplied to background goroutines - // (processTicks). It is derived from the caller ctx in OnStart so that - // goroutines also exit when the caller ctx is canceled. OnStop calls - // procCancel() directly, which makes goroutineWg.Wait() safe to call - // immediately after Stop() regardless of whether the caller ctx has fired. - procCancel context.CancelFunc - - // goroutineWg tracks the processTicks goroutine so that WaitForGoroutines - // can block until it has fully exited. - goroutineWg stdsync.WaitGroup - - // TODO: When we start deleting files, we need to start tracking GroupReaders - // and their dependencies. } // OpenGroup creates a new Group with head at headPath. It returns an error if @@ -93,7 +79,8 @@ func OpenGroup(ctx context.Context, logger log.Logger, headPath string, groupOpt if err != nil { return nil, err } - head, err := OpenAutoFile(ctx, headPath) + // The owner closes Head after the group's final flush, not on cancellation. + head, err := OpenAutoFile(context.WithoutCancel(ctx), headPath) if err != nil { return nil, err } @@ -148,56 +135,27 @@ func GroupTotalSizeLimit(limit int64) func(*Group) { // and group limits. func (g *Group) OnStart(ctx context.Context) error { g.ticker = time.NewTicker(g.groupCheckDuration) - // Derive a child context so that OnStop can cancel the goroutine by - // calling g.procCancel() independently of whether the caller ctx fires. - // The child inherits cancellation from ctx, preserving the existing - // "also exit when caller ctx cancels" behavior. - procCtx, procCancel := context.WithCancel(ctx) - g.procCancel = procCancel - g.goroutineWg.Add(1) - go func() { - defer g.goroutineWg.Done() - g.processTicks(procCtx) - }() + g.Go(ctx, g.processTicks) return nil } -// WaitForGoroutines blocks until the background goroutines started by this -// Group (e.g. processTicks) have fully exited. It is safe to call after -// Stop() because OnStop() cancels the goroutine's context via procCancel. -func (g *Group) WaitForGoroutines() { - g.goroutineWg.Wait() -} - -// Wait blocks until the Group service has stopped and its background goroutines -// have fully exited. It overrides BaseService.Wait — which returns as soon as -// OnStop() closes the quit channel, before the procCancel-driven processTicks -// goroutine is joined — so that every direct Group consumer gets the same -// shutdown guarantee BaseWAL.Wait relies on, not just callers that remember to -// chain WaitForGoroutines explicitly. -// -// It calls g.BaseService.Wait() (the embedded method), not g.Wait(), so there -// is no recursion. WaitForGoroutines is idempotent, so chaining it here on top -// of an explicit caller call is harmless. -func (g *Group) Wait() { - g.BaseService.Wait() - g.WaitForGoroutines() -} - // OnStop implements service.Service by stopping the goroutine described above. // NOTE: g.Head must be closed separately using Close. func (g *Group) OnStop() { - // Cancel the goroutine's context first so that processTicks exits on its - // next select iteration without waiting for the caller ctx. - g.procCancel() g.ticker.Stop() +} + +// OnDrain flushes buffered writes after the periodic worker has exited. +func (g *Group) OnDrain() { if err := g.FlushAndSync(); err != nil { g.logger.Error("error flushing to disk", "err", err) } } -// Close closes the head file. The group must be stopped by this moment. +// Close stops and drains the group before closing its head file. func (g *Group) Close() { + g.Stop() + g.Wait() if err := g.FlushAndSync(); err != nil { g.logger.Error("error flushing to disk", "err", err) } diff --git a/libs/service/lifecycle_test.go b/libs/service/lifecycle_test.go new file mode 100644 index 0000000000..2a26047a1c --- /dev/null +++ b/libs/service/lifecycle_test.go @@ -0,0 +1,238 @@ +package service + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" +) + +type lifecycleService struct { + *BaseService + start func(context.Context) error + stop func() +} + +func (s *lifecycleService) OnStart(ctx context.Context) error { return s.start(ctx) } +func (s *lifecycleService) OnStop() { + if s.stop != nil { + s.stop() + } +} +func newLifecycleService(t *testing.T) *lifecycleService { + s := &lifecycleService{} + s.BaseService = NewBaseService(log.NewNopLogger(), t.Name(), s) + return s +} + +func TestManualStopCancelsStartContext(t *testing.T) { + s := newLifecycleService(t) + var workCtx context.Context + s.start = func(ctx context.Context) error { workCtx = ctx; return nil } + require.NoError(t, s.Start(context.Background())) + s.Stop() + s.Wait() + require.ErrorIs(t, workCtx.Err(), context.Canceled) +} + +func assertPending(t *testing.T, done <-chan struct{}) { + t.Helper() + select { + case <-done: + t.Error("Wait returned before worker cleanup completed") + case <-time.After(20 * time.Millisecond): + } +} + +func TestWaitDrainsManagedWork(t *testing.T) { + for _, parentStop := range []bool{false, true} { + t.Run(map[bool]string{false: "manual", true: "parent"}[parentStop], func(t *testing.T) { + s := newLifecycleService(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + entered, release := make(chan struct{}), make(chan struct{}) + s.start = func(ctx context.Context) error { + require.True(t, s.Go(ctx, func(ctx context.Context) { <-ctx.Done(); close(entered); <-release })) + return nil + } + require.NoError(t, s.Start(ctx)) + if parentStop { + cancel() + } else { + s.Stop() + } + <-entered + done := make(chan struct{}) + go func() { s.Wait(); close(done) }() + assertPending(t, done) + close(release) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("worker was not drained") + } + require.False(t, s.IsRunning()) + require.ErrorIs(t, s.Start(ctx), errAlreadyStopped) + }) + } +} + +func TestFailedStartDrainsBeforeRetry(t *testing.T) { + s := newLifecycleService(t) + entered, release := make(chan struct{}), make(chan struct{}) + var old context.Context + s.start = func(ctx context.Context) error { + old = ctx + require.True(t, s.Go(ctx, func(ctx context.Context) { <-ctx.Done(); close(entered); <-release })) + return context.DeadlineExceeded + } + s.Wait() + done := make(chan struct{}) + go func() { defer close(done); require.ErrorIs(t, s.Start(context.Background()), context.DeadlineExceeded) }() + <-entered + assertPending(t, done) + close(release) + <-done + s.Wait() + s.start = func(_ context.Context) error { + require.False(t, s.Go(old, func(context.Context) { t.Error("old attempt admitted") })) + return nil + } + require.NoError(t, s.Start(context.Background())) + require.True(t, s.IsRunning()) + s.Stop() + s.Wait() +} + +func TestWorkerCanStopAndHookCanInspectLifecycle(t *testing.T) { + s := newLifecycleService(t) + s.stop = func() { require.False(t, s.IsRunning()); s.Stop() } + s.start = func(ctx context.Context) error { + require.False(t, s.IsRunning()) + require.True(t, s.Go(ctx, func(context.Context) { s.Stop() })) + return nil + } + require.NoError(t, s.Start(context.Background())) + done := make(chan struct{}) + go func() { s.Wait(); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("lifecycle deadlock") + } +} + +func TestRegistrationRacesStop(t *testing.T) { + for range 100 { + s := newLifecycleService(t) + var ctx context.Context + s.start = func(c context.Context) error { ctx = c; return nil } + require.NoError(t, s.Start(context.Background())) + admittedDone := make(chan struct{}) + require.True(t, s.Go(ctx, func(ctx context.Context) { <-ctx.Done(); close(admittedDone) })) + registered := make(chan struct{}) + go func() { + defer close(registered) + for range 100 { + s.Go(ctx, func(ctx context.Context) { <-ctx.Done() }) + } + }() + s.Stop() + s.Wait() + <-registered + select { + case <-admittedDone: + default: + t.Error("admitted work escaped drain") + } + require.False(t, s.Go(ctx, func(context.Context) { t.Error("work admitted after stop") })) + } +} + +type finalizingService struct { + *lifecycleService + drain func() +} + +func (s *finalizingService) OnDrain() { s.drain() } + +func TestFinalizerFollowsWorkerCleanup(t *testing.T) { + workerDone := make(chan struct{}) + release := make(chan struct{}) + finalizerEntered := make(chan struct{}) + s := &finalizingService{lifecycleService: newLifecycleService(t)} + s.BaseService = NewBaseService(log.NewNopLogger(), t.Name(), s) + s.start = func(ctx context.Context) error { + require.True(t, s.Go(ctx, func(ctx context.Context) { <-ctx.Done(); close(workerDone) })) + return nil + } + s.drain = func() { + select { + case <-workerDone: + default: + t.Error("finalizer ran before worker cleanup") + } + close(finalizerEntered) + <-release + } + require.NoError(t, s.Start(context.Background())) + s.Stop() + <-finalizerEntered + done := make(chan struct{}) + go func() { s.Wait(); close(done) }() + assertPending(t, done) + close(release) + <-done +} + +func TestStopDuringStartup(t *testing.T) { + s := newLifecycleService(t) + entered, release := make(chan struct{}), make(chan struct{}) + stopped := make(chan struct{}) + s.start = func(ctx context.Context) error { + close(entered) + <-ctx.Done() + <-release + return nil + } + s.stop = func() { close(stopped) } + started := make(chan error, 1) + go func() { started <- s.Start(context.Background()) }() + <-entered + require.ErrorIs(t, s.Start(context.Background()), errStarting) + s.Stop() + select { + case <-stopped: + t.Fatal("OnStop ran concurrently with OnStart") + default: + } + close(release) + require.NoError(t, <-started) + s.Wait() + <-stopped + require.False(t, s.IsRunning()) +} + +func TestStartContextPreservesValuesAndDeadline(t *testing.T) { + type key struct{} + deadline := time.Now().Add(time.Hour) + parent, cancel := context.WithDeadline(context.WithValue(context.Background(), key{}, "value"), deadline) + defer cancel() + s := newLifecycleService(t) + s.start = func(ctx context.Context) error { + require.Equal(t, "value", ctx.Value(key{})) + got, ok := ctx.Deadline() + require.True(t, ok) + require.Equal(t, deadline, got) + return nil + } + require.False(t, s.Go(parent, func(context.Context) { t.Error("work before Start") })) + require.NoError(t, s.Start(parent)) + require.NoError(t, s.Start(parent)) + s.Stop() + s.Wait() + require.NoError(t, parent.Err()) +} diff --git a/libs/service/service.go b/libs/service/service.go index 50220d1f53..e80f8d818d 100644 --- a/libs/service/service.go +++ b/libs/service/service.go @@ -3,6 +3,7 @@ package service import ( "context" "errors" + stdsync "sync" "sync/atomic" sync "github.com/sasha-s/go-deadlock" @@ -11,187 +12,184 @@ import ( ) var ( - // errAlreadyStopped is returned when somebody tries to stop an already - // stopped service (without resetting it). - errAlreadyStopped = errors.New("already stopped") - - _ Service = (*BaseService)(nil) + errAlreadyStopped = errors.New("already stopped") + errStarting = errors.New("service is starting") + _ Service = (*BaseService)(nil) ) -// Service defines a service that can be started, stopped, and reset. +// Service defines a service that can be started and stopped. type Service interface { - // Start is called to start the service, which should run until - // the context terminates. If the service is already running, Start - // must report an error. Start(context.Context) error - - // Return true if the service is running IsRunning() bool - - // Wait blocks until the service is stopped. Wait() } -// Implementation describes the implementation that the -// BaseService implementation wraps. +// Implementation supplies startup and shutdown hooks. Hooks run without the +// lifecycle lock. They must not call Wait on their own service. type Implementation interface { - // Called by the Services Start Method OnStart(context.Context) error - - // Called when the service's context is canceled. + // OnStop unblocks workers. It must not wait for managed workers. OnStop() } -/* -Classical-inheritance-style service declarations. Services can be started, then -stopped, but cannot be restarted. - -Users must implement OnStart/OnStop methods. In the absence of errors, these -methods are guaranteed to be called at most once. If OnStart returns an error, -service won't be marked as started, so the user can call Start again. - -The BaseService implementation ensures that the OnStop method is -called after the context passed to Start is canceled. - -Typical usage: - - type FooService struct { - BaseService - // private fields - } +// Finalizer releases resources after OnStop and all managed workers finish. +// It is called only after a successful OnStart, before Wait returns. +type Finalizer interface { + OnDrain() +} - func NewFooService() *FooService { - fs := &FooService{ - // init - } - fs.BaseService = *NewBaseService(log, "FooService", fs) - return fs - } +type attemptKey struct{} - func (fs *FooService) OnStart(ctx context.Context) error { - // initialize private fields - // start subroutines, etc. - } +type serviceAttempt struct { + cancel context.CancelFunc + done chan struct{} + workers stdsync.WaitGroup + starting bool + stopping bool +} - func (fs *FooService) OnStop() { - // close/destroy private fields and releases resources - } -*/ +// BaseService owns a service context and work registered through Go. Stop cancels +// that context and calls OnStop; Wait joins work and optional OnDrain cleanup. +// Plain go statements are not tracked. A successful start cannot be restarted. +// A failed OnStart must release its own resources; BaseService cancels and joins +// its managed work before allowing another attempt. type BaseService struct { - logger log.Logger - name string - mtx sync.Mutex - quit <-chan (struct{}) - cancel context.CancelFunc - - // running is an atomic flag set to 1 when the service is running, - // and 0 otherwise. It allows IsRunning to avoid locking and thus - // prevents recursive lock deadlocks from callers that also hold mtx. + logger log.Logger + name string + mtx sync.Mutex + attempt *serviceAttempt running uint32 - - // The "subclass" of BaseService - impl Implementation + impl Implementation } // NewBaseService creates a new BaseService. func NewBaseService(logger log.Logger, name string, impl Implementation) *BaseService { - return &BaseService{ - logger: logger, - name: name, - impl: impl, - } + return &BaseService{logger: logger, name: name, impl: impl} } -// Start starts the Service and calls its OnStart method. An error -// will be returned if the service is stopped, but not if it is -// already running. +// Start calls OnStart with a context canceled by either the parent or Stop. +// Repeated calls while running succeed; concurrent startup returns an error. func (bs *BaseService) Start(ctx context.Context) error { bs.mtx.Lock() - defer bs.mtx.Unlock() - - if bs.quit != nil { - return nil - } - - select { - case <-bs.quit: - return errAlreadyStopped - default: - bs.logger.Info("starting service", "service", bs.name, "impl", bs.name) - if err := bs.impl.OnStart(ctx); err != nil { - return err + if a := bs.attempt; a != nil { + var err error + switch { + case a.stopping: + err = errAlreadyStopped + case a.starting: + err = errStarting } - - // we need a separate context to ensure that we start - // a thread that will get cleaned up and that the - // Stop/Wait functions work as expected. - srvCtx, cancel := context.WithCancel(context.Background()) - bs.cancel = cancel - bs.quit = srvCtx.Done() + bs.mtx.Unlock() + return err + } + workCtx, cancel := context.WithCancel(ctx) + a := &serviceAttempt{cancel: cancel, done: make(chan struct{}), starting: true} + bs.attempt = a + workCtx = context.WithValue(workCtx, attemptKey{}, a) + bs.mtx.Unlock() + + bs.logger.Info("starting service", "service", bs.name) + err := bs.impl.OnStart(workCtx) + bs.mtx.Lock() + a.starting = false + if err != nil { + a.stopping = true + bs.mtx.Unlock() + cancel() + a.workers.Wait() + bs.mtx.Lock() + close(a.done) + bs.attempt = nil + bs.mtx.Unlock() + return err + } + stopping := a.stopping + if !stopping { atomic.StoreUint32(&bs.running, 1) - - go func(ctx context.Context) { + } + bs.mtx.Unlock() + if stopping { + bs.finishStop(a) + } else { + go func() { select { - case <-srvCtx.Done(): - // this means stop was called manually - return - case <-ctx.Done(): - bs.Stop() + case <-workCtx.Done(): + bs.stopAttempt(a) + case <-a.done: } - - bs.logger.Info("stopped service", - "service", bs.name) - }(ctx) - - return nil + }() } + return nil } -// Stop manually terminates the service by calling OnStop method from -// the implementation and releases all resources related to the -// service. -func (bs *BaseService) Stop() { +// Go registers work belonging to the context received by OnStart (or a child). +// It returns false before startup, after stopping begins, or for an old attempt. +// A worker may call Stop, but must not call Wait on its own service. +func (bs *BaseService) Go(ctx context.Context, fn func(context.Context)) bool { bs.mtx.Lock() defer bs.mtx.Unlock() - - if bs.quit == nil { - return + a := bs.attempt + if a == nil || a.stopping || ctx.Value(attemptKey{}) != a { + return false } + a.workers.Add(1) + go func() { defer a.workers.Done(); fn(ctx) }() + return true +} - select { - case <-bs.quit: - return - default: - bs.logger.Info("stopping service", "service", bs.name) - bs.impl.OnStop() - // Clear running before cancel: cancel() closes quit, which unblocks Wait(). - // Storing first guarantees IsRunning() is false by the time Wait() returns. - atomic.StoreUint32(&bs.running, 0) - bs.cancel() +// Stop cancels work and invokes OnStop once, without waiting for workers. +// During startup it requests shutdown after OnStart returns. Concurrent or +// recursive calls return immediately; use Wait to observe complete shutdown. +func (bs *BaseService) Stop() { + bs.mtx.Lock() + a := bs.attempt + bs.mtx.Unlock() + bs.stopAttempt(a) +} +func (bs *BaseService) stopAttempt(a *serviceAttempt) { + bs.mtx.Lock() + if a == nil || bs.attempt != a || a.stopping { + bs.mtx.Unlock() return } + a.stopping = true + atomic.StoreUint32(&bs.running, 0) + starting := a.starting + bs.mtx.Unlock() + a.cancel() + if !starting { + bs.finishStop(a) + } } -// IsRunning implements Service by returning true or false depending on the -// service's state. +func (bs *BaseService) finishStop(a *serviceAttempt) { + bs.logger.Info("stopping service", "service", bs.name) + bs.impl.OnStop() + go func() { + a.workers.Wait() + if finalizer, ok := bs.impl.(Finalizer); ok { + finalizer.OnDrain() + } + bs.logger.Info("stopped service", "service", bs.name) + close(a.done) + }() +} + +// IsRunning reports successful startup until shutdown is requested. False does +// not imply cleanup has completed; use Wait before releasing owned resources. func (bs *BaseService) IsRunning() bool { return atomic.LoadUint32(&bs.running) == 1 } -func (bs *BaseService) getWait() <-chan struct{} { +// Wait joins the current startup attempt, shutdown hooks and registered work. +// It returns immediately before Start or after a failed Start has drained. +func (bs *BaseService) Wait() { bs.mtx.Lock() - defer bs.mtx.Unlock() - - if bs.quit == nil { - out := make(chan struct{}) - close(out) - return out + a := bs.attempt + bs.mtx.Unlock() + if a != nil { + <-a.done } - - return bs.quit } -// Wait blocks until the service is stopped. -func (bs *BaseService) Wait() { <-bs.getWait() } - // String provides a human-friendly representation of the service. func (bs *BaseService) String() string { return bs.name } From 85f9faed74f9dfe86f07f47534d4ba1e6262f471 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:04:30 +0000 Subject: [PATCH 02/39] refactor(p2p): join service-owned workers during shutdown Track Router, PEX and MConnection work through BaseService; join connection, queue, subscription and request workers at their owners. Preserve the parent context for post-stop error callbacks and expose explicit rate-limit cleanup. Co-Authored-By: Codex GPT-6 --- internal/p2p/channel.go | 38 ++---- internal/p2p/client/lifecycle_test.go | 48 ++++++++ internal/p2p/client/ratelimit.go | 12 ++ internal/p2p/conn/connection.go | 91 +++----------- internal/p2p/conn/connection_test.go | 5 +- internal/p2p/conn/lifecycle_test.go | 73 ++++++++++++ internal/p2p/lifecycle_test.go | 163 ++++++++++++++++++++++++++ internal/p2p/peermanager.go | 32 +++++ internal/p2p/pex/lifecycle_test.go | 50 ++++++++ internal/p2p/pex/reactor.go | 16 ++- internal/p2p/router.go | 65 +++++++--- internal/p2p/rqueue.go | 10 +- internal/p2p/transport_mconn.go | 65 ++++++---- 13 files changed, 519 insertions(+), 149 deletions(-) create mode 100644 internal/p2p/client/lifecycle_test.go create mode 100644 internal/p2p/conn/lifecycle_test.go create mode 100644 internal/p2p/lifecycle_test.go create mode 100644 internal/p2p/pex/lifecycle_test.go diff --git a/internal/p2p/channel.go b/internal/p2p/channel.go index 324ceca65b..233247fd5c 100644 --- a/internal/p2p/channel.go +++ b/internal/p2p/channel.go @@ -321,25 +321,7 @@ func (ch *legacyChannel) String() string { return fmt.Sprintf("p2p.Channel<%d:%s // Receive returns a new unbuffered iterator to receive messages from ch. // The iterator runs until ctx ends. func (ch *legacyChannel) Receive(ctx context.Context) ChannelIterator { - iter := &channelIterator{ - pipe: make(chan Envelope), // unbuffered - } - go func(pipe chan<- Envelope) { - defer close(iter.pipe) - for { - select { - case <-ctx.Done(): - return - case envelope := <-ch.inCh: - select { - case <-ctx.Done(): - return - case pipe <- envelope: - } - } - } - }(iter.pipe) - return iter + return &channelIterator{pipe: ch.inCh, done: ctx.Done()} } // ChannelIterator is an iterator for receiving messages from a Channel. @@ -364,7 +346,8 @@ type ChannelIterator interface { // MergedChannelIterator makes it possible to combine multiple // channels into a single iterator. type channelIterator struct { - pipe chan Envelope + pipe <-chan Envelope + done <-chan struct{} current *Envelope } @@ -375,6 +358,9 @@ func NewChannelIterator(pipe chan Envelope) ChannelIterator { func (iter *channelIterator) Next(ctx context.Context) bool { select { + case <-iter.done: + iter.current = nil + return false case <-ctx.Done(): iter.current = nil return false @@ -402,8 +388,9 @@ func (iter *channelIterator) Envelope() *Envelope { return iter.current } // This allows the caller to consume messages from multiple channels // without needing to manage the concurrency separately. func MergedChannelIterator(ctx context.Context, chs ...Channel) ChannelIterator { + pipe := make(chan Envelope) iter := &channelIterator{ - pipe: make(chan Envelope), // unbuffered + pipe: pipe, // unbuffered } wg := new(sync.WaitGroup) @@ -419,19 +406,16 @@ func MergedChannelIterator(ctx context.Context, chs ...Channel) ChannelIterator case pipe <- *iter.Envelope(): } } - }(ch, iter.pipe) + }(ch, pipe) } - done := make(chan struct{}) - go func() { defer close(done); wg.Wait() }() - go func() { - defer close(iter.pipe) + defer close(pipe) // we could return early if the context is canceled, // but this is safer because it means the pipe stays // open until all of the ch worker threads end, which // should happen very quickly. - <-done + wg.Wait() }() return iter diff --git a/internal/p2p/client/lifecycle_test.go b/internal/p2p/client/lifecycle_test.go new file mode 100644 index 0000000000..dc2d22272d --- /dev/null +++ b/internal/p2p/client/lifecycle_test.go @@ -0,0 +1,48 @@ +package client + +import ( + "context" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" +) + +type blockedGCClock struct { + clockwork.Clock + entered chan struct{} + release chan struct{} +} + +func (c *blockedGCClock) Now() time.Time { + close(c.entered) + <-c.release + return c.Clock.Now() +} + +func TestRateLimitCloseJoinsGC(t *testing.T) { + fake := clockwork.NewFakeClock() + clock := &blockedGCClock{Clock: fake, entered: make(chan struct{}), release: make(chan struct{})} + limiter := NewRateLimit(context.Background(), 1, true, log.NewNopLogger(), WithRateLimitClock(clock)) + t.Cleanup(limiter.Close) + defer close(clock.release) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, fake.BlockUntilContext(ctx, 1)) + fake.Advance(PeerRateLimitLifetime * time.Second) + select { + case <-clock.entered: + case <-time.After(time.Second): + t.Fatal("garbage collection did not start") + } + done := make(chan struct{}) + go func() { limiter.Close(); close(done) }() + select { + case <-done: + t.Fatal("Close returned before garbage collection finished") + case <-time.After(30 * time.Millisecond): + } +} diff --git a/internal/p2p/client/ratelimit.go b/internal/p2p/client/ratelimit.go index a09904d27b..58284d2481 100644 --- a/internal/p2p/client/ratelimit.go +++ b/internal/p2p/client/ratelimit.go @@ -35,6 +35,8 @@ type RateLimit struct { clock clockwork.Clock logger log.Logger + cancel context.CancelFunc + done chan struct{} } // RateLimitOptionFunc overrides a default parameter of a RateLimit. @@ -86,6 +88,7 @@ func NewRateLimitWithBurst( logger log.Logger, opts ...RateLimitOptionFunc, ) *RateLimit { + ctx, cancel := context.WithCancel(ctx) if limit > 0 && burst < 1 { burst = 1 } @@ -96,6 +99,8 @@ func NewRateLimitWithBurst( drop: drop, clock: clockwork.NewRealClock(), logger: logger, + cancel: cancel, + done: make(chan struct{}), } for _, opt := range opts { opt(h) @@ -107,6 +112,12 @@ func NewRateLimitWithBurst( return h } +// Close stops and joins the limiter's garbage collection worker. +func (h *RateLimit) Close() { + h.cancel() + <-h.done +} + func (h *RateLimit) getLimiter(peerID types.NodeID) *limiter { var limit *limiter if l, ok := h.limiters.Load(peerID); ok { @@ -203,6 +214,7 @@ func (h *RateLimit) waitN(ctx context.Context, l *limiter, nTokens int) error { // gcRoutine is a goroutine that removes unused limiters for peers every `PeerRateLimitLifetime` seconds. func (h *RateLimit) gcRoutine(ctx context.Context) { + defer close(h.done) ticker := h.clock.NewTicker(PeerRateLimitLifetime * time.Second) defer ticker.Stop() diff --git a/internal/p2p/conn/connection.go b/internal/p2p/conn/connection.go index 63e7317287..c18cf05190 100644 --- a/internal/p2p/conn/connection.go +++ b/internal/p2p/conn/connection.go @@ -92,18 +92,8 @@ type MConnection struct { errored uint32 config MConnConfig - // Closing quitSendRoutine will cause the sendRoutine to eventually quit. - // doneSendRoutine is closed when the sendRoutine actually quits. - quitSendRoutine chan struct{} - doneSendRoutine chan struct{} - - // Closing quitRecvRouting will cause the recvRouting to eventually quit. - quitRecvRoutine chan struct{} - stopSignal <-chan struct{} - cancel context.CancelFunc - flushTimer *timer.ThrottleTimer // flush writes as necessary but throttled. pingTimer *time.Ticker // send pings periodically @@ -176,7 +166,6 @@ func NewMConnection( onError: onError, config: config, created: time.Now(), - cancel: func() {}, } mconn.BaseService = *service.NewBaseService(logger, "MConnection", mconn) @@ -199,18 +188,22 @@ func NewMConnection( return mconn } +type errorContextKey struct{} + +// Start preserves the caller's context for error notification after I/O stops. +func (c *MConnection) Start(ctx context.Context) error { + return c.BaseService.Start(context.WithValue(ctx, errorContextKey{}, ctx)) +} + // OnStart implements BaseService func (c *MConnection) OnStart(ctx context.Context) error { c.flushTimer = timer.NewThrottleTimer("flush", c.config.FlushThrottle) c.pingTimer = time.NewTicker(c.config.PingInterval) c.chStatsTimer = time.NewTicker(updateStats) - c.quitSendRoutine = make(chan struct{}) - c.doneSendRoutine = make(chan struct{}) - c.quitRecvRoutine = make(chan struct{}) c.stopSignal = ctx.Done() c.setRecvLastMsgAt(time.Now()) - go c.sendRoutine(ctx) - go c.recvRoutine(ctx) + c.Go(ctx, c.sendRoutine) + c.Go(ctx, c.recvRoutine) return nil } @@ -226,46 +219,12 @@ func (c *MConnection) getLastMessageAt() time.Time { return c.lastMsgRecv.at } -// stopServices stops the BaseService and timers and closes the quitSendRoutine. -// if the quitSendRoutine was already closed, it returns true, otherwise it returns false. -// It doesn't lock, as we rely on the caller (eg. BaseService) locking -func (c *MConnection) stopServices() (alreadyStopped bool) { - select { - case <-c.quitSendRoutine: - // already quit - return true - default: - } - - select { - case <-c.quitRecvRoutine: - // already quit - return true - default: - } - +// OnStop closes the socket to interrupt pending reads and writes. +func (c *MConnection) OnStop() { c.flushTimer.Stop() c.pingTimer.Stop() c.chStatsTimer.Stop() - - // inform the recvRouting that we are shutting down - close(c.quitRecvRoutine) - close(c.quitSendRoutine) - return false -} - -// OnStop implements BaseService -func (c *MConnection) OnStop() { - if c.stopServices() { - return - } - c.conn.Close() - - // We can't close pong safely here because - // recvRoutine may write to it after we've stopped. - // Though it doesn't need to get closed at all, - // we close it @ recvRoutine. } func (c *MConnection) String() string { @@ -293,6 +252,9 @@ func (c *MConnection) stopForError(ctx context.Context, r interface{}) { if atomic.CompareAndSwapUint32(&c.errored, 0, 1) { if c.onError != nil { + if parent, ok := ctx.Value(errorContextKey{}).(context.Context); ok { + ctx = parent + } c.onError(ctx, r) } } @@ -371,8 +333,6 @@ FOR_LOOP: c.flush() case <-ctx.Done(): break FOR_LOOP - case <-c.quitSendRoutine: - break FOR_LOOP case <-pongTimeout.C: // the point of the pong timer is to check to // see if we've seen a message recently, so we @@ -402,13 +362,10 @@ FOR_LOOP: c.stopForError(ctx, err) break FOR_LOOP } - if !c.IsRunning() { + if ctx.Err() != nil { break FOR_LOOP } } - - // Cleanup - close(c.doneSendRoutine) } // Returns true if messages from channels were exhausted. @@ -479,8 +436,6 @@ FOR_LOOP: select { case <-ctx.Done(): break FOR_LOOP - case <-c.doneSendRoutine: - break FOR_LOOP default: } @@ -507,16 +462,11 @@ FOR_LOOP: _n, err := protoReader.ReadMsg(&packet) c.recvMonitor.Update(_n) if err != nil { - // stopServices was invoked and we are shutting down - // receiving is excpected to fail since we will close the connection - select { - case <-ctx.Done(): - case <-c.quitRecvRoutine: + if ctx.Err() != nil { break FOR_LOOP - default: } - if c.IsRunning() { + if ctx.Err() == nil { if err == io.EOF { c.logger.Info("Connection is closed @ recvRoutine (likely by the other side)", "conn", c) } else { @@ -556,7 +506,7 @@ FOR_LOOP: msgBytes, err := channel.recvPacketMsg(*pkt.PacketMsg) if err != nil { - if c.IsRunning() { + if ctx.Err() == nil { c.logger.Debug("Connection failed @ recvRoutine", "conn", c, "err", err) c.stopForError(ctx, err) } @@ -575,11 +525,6 @@ FOR_LOOP: } } - // Cleanup - close(c.pong) - for range c.pong { - // Drain - } } // maxPacketMsgSize returns a maximum size of PacketMsg diff --git a/internal/p2p/conn/connection_test.go b/internal/p2p/conn/connection_test.go index c2e34046b6..7beab11600 100644 --- a/internal/p2p/conn/connection_test.go +++ b/internal/p2p/conn/connection_test.go @@ -191,9 +191,10 @@ func TestMConnectionWillEventuallyTimeout(t *testing.T) { } }() - // wait for the send routine to die because it doesn't + done := make(chan struct{}) + go func() { mconn.Wait(); close(done) }() select { - case <-mconn.doneSendRoutine: + case <-done: require.True(t, time.Since(mconn.getLastMessageAt()) > mconn.config.PongTimeout, "the connection state reflects that we've passed the pong timeout") // since we hit the timeout, things should be shutdown diff --git a/internal/p2p/conn/lifecycle_test.go b/internal/p2p/conn/lifecycle_test.go new file mode 100644 index 0000000000..20651a0a56 --- /dev/null +++ b/internal/p2p/conn/lifecycle_test.go @@ -0,0 +1,73 @@ +package conn + +import ( + "context" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" +) + +func TestMConnectionWaitDrainsCallback(t *testing.T) { + local, remote := net.Pipe() + entered, release := make(chan struct{}), make(chan struct{}) + receiver := createMConnectionWithCallbacks(log.NewNopLogger(), local, + func(context.Context, ChannelID, []byte) { close(entered); <-release }, nil) + sender := createTestMConnection(log.NewNopLogger(), remote) + require.NoError(t, receiver.Start(context.Background())) + require.NoError(t, sender.Start(context.Background())) + t.Cleanup(func() { sender.Stop(); sender.Wait(); receiver.Stop(); receiver.Wait() }) + defer close(release) + require.True(t, sender.Send(1, []byte("drain"))) + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("receive callback did not start") + } + receiver.Stop() + done := make(chan struct{}) + go func() { receiver.Wait(); close(done) }() + select { + case <-done: + t.Fatal("Wait returned while the receive callback was still running") + case <-time.After(30 * time.Millisecond): + } +} + +func TestMConnectionErrorCallbackRetainsParentContext(t *testing.T) { + local, remote := net.Pipe() + t.Cleanup(func() { _ = local.Close(); _ = remote.Close() }) + type observation struct { + err error + running bool + } + observed := make(chan observation, 1) + release := make(chan struct{}) + var connection *MConnection + connection = createMConnectionWithCallbacks(log.NewNopLogger(), local, nil, + func(ctx context.Context, _ interface{}) { + observed <- observation{err: ctx.Err(), running: connection.IsRunning()} + <-release + }) + require.NoError(t, connection.Start(context.Background())) + t.Cleanup(func() { connection.Stop(); connection.Wait() }) + defer close(release) + require.NoError(t, remote.Close()) + select { + case result := <-observed: + require.NoError(t, result.err) + require.False(t, result.running) + case <-time.After(time.Second): + t.Fatal("error callback did not run") + } + done := make(chan struct{}) + go func() { connection.Wait(); close(done) }() + select { + case <-done: + t.Fatal("Wait returned before the error callback finished") + case <-time.After(30 * time.Millisecond): + } +} diff --git a/internal/p2p/lifecycle_test.go b/internal/p2p/lifecycle_test.go new file mode 100644 index 0000000000..29bda79b6e --- /dev/null +++ b/internal/p2p/lifecycle_test.go @@ -0,0 +1,163 @@ +package p2p + +import ( + "context" + "net" + "testing" + "time" + + dbm "github.com/cometbft/cometbft-db" + "github.com/stretchr/testify/require" + + tmsync "github.com/dashpay/tenderdash/internal/libs/sync" + "github.com/dashpay/tenderdash/internal/p2p/conn" + "github.com/dashpay/tenderdash/libs/log" + p2pproto "github.com/dashpay/tenderdash/proto/tendermint/p2p" + "github.com/dashpay/tenderdash/types" +) + +func TestRouterWaitDrainsAcceptedConnection(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db := dbm.NewMemDB() + t.Cleanup(func() { require.NoError(t, db.Close()) }) + manager, err := NewPeerManager(ctx, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", db, PeerManagerOptions{}) + require.NoError(t, err) + transport := NewMConnTransport(log.NewNopLogger(), conn.DefaultMConnConfig(), nil, MConnTransportOptions{}) + entered, release := make(chan struct{}), make(chan struct{}) + router, err := NewRouter(log.NewNopLogger(), NopMetrics(), nil, manager, + func() *types.NodeInfo { return &types.NodeInfo{} }, transport, + &Endpoint{Protocol: MConnProtocol, IP: net.IPv4(127, 0, 0, 1)}, RouterOptions{ + FilterPeerByIP: func(ctx context.Context, _ net.IP, _ uint16) error { + close(entered) + <-ctx.Done() + <-release + return ctx.Err() + }, + }) + require.NoError(t, err) + require.NoError(t, router.Start(ctx)) + t.Cleanup(func() { router.Stop(); router.Wait() }) + defer close(release) + remote, err := net.Dial("tcp", transport.listener.Addr().String()) + require.NoError(t, err) + defer remote.Close() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("connection filter did not start") + } + router.Stop() + done := make(chan struct{}) + go func() { router.Wait(); close(done) }() + select { + case <-done: + t.Fatal("Wait returned while an accepted connection was still being filtered") + case <-time.After(30 * time.Millisecond): + } +} + +func TestSimpleQueueClosedDrainsDeliveryCallback(t *testing.T) { + q := newSimplePriorityQueue(context.Background(), 0) + entered, release := make(chan struct{}), make(chan struct{}) + defer func() { close(release); q.close(); <-q.closed() }() + envelope := Envelope{Message: &p2pproto.PexRequest{}} + envelope.EnableDeliveryNotification() + envelope.delivery.setOnCompleted(func() { close(entered); <-release }) + q.enqueue() <- envelope + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("queue did not drop the envelope") + } + q.close() + select { + case <-q.closed(): + t.Fatal("queue reported completion before delivery callback returned") + case <-time.After(30 * time.Millisecond): + } +} + +func TestPeerUpdatesCloseJoinsUnsubscribe(t *testing.T) { + db := dbm.NewMemDB() + defer db.Close() + manager, err := NewPeerManager(context.Background(), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", db, PeerManagerOptions{}) + require.NoError(t, err) + updates := manager.Subscribe(context.Background(), "lifecycle") + manager.mtx.Lock() + done := make(chan struct{}) + go func() { updates.Close(); close(done) }() + select { + case <-done: + manager.mtx.Unlock() + t.Fatal("Close returned before unsubscribing") + case <-time.After(30 * time.Millisecond): + } + manager.mtx.Unlock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("subscription did not finish cleanup") + } + manager.mtx.Lock() + require.Empty(t, manager.subscriptions) + manager.mtx.Unlock() + updates.Close() +} + +type delayedAcceptListener struct { + net.Listener + entered chan struct{} + release chan struct{} +} + +func (l *delayedAcceptListener) Accept() (net.Conn, error) { + close(l.entered) + <-l.release + return nil, net.ErrClosed +} + +func (*delayedAcceptListener) Close() error { return nil } + +func TestMConnTransportAcceptJoinsRequest(t *testing.T) { + listener := &delayedAcceptListener{entered: make(chan struct{}), release: make(chan struct{})} + transport := NewMConnTransport(log.NewNopLogger(), conn.DefaultMConnConfig(), nil, MConnTransportOptions{}) + transport.listener = listener + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { _, _ = transport.Accept(ctx); close(done) }() + defer func() { close(listener.release); <-done }() + <-listener.entered + cancel() + select { + case <-done: + t.Fatal("Accept returned before its request worker finished") + case <-time.After(30 * time.Millisecond): + } +} + +func TestRouterManualStopClosesIndependentChannel(t *testing.T) { + db := dbm.NewMemDB() + defer db.Close() + manager, err := NewPeerManager(context.Background(), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", db, PeerManagerOptions{}) + require.NoError(t, err) + transport := NewMConnTransport(log.NewNopLogger(), conn.DefaultMConnConfig(), nil, MConnTransportOptions{}) + info := &types.NodeInfo{Channels: tmsync.NewConcurrentSlice[uint16]()} + router, err := NewRouter(log.NewNopLogger(), NopMetrics(), nil, manager, + func() *types.NodeInfo { return info }, transport, + &Endpoint{Protocol: MConnProtocol, IP: net.IPv4(127, 0, 0, 1)}, RouterOptions{}) + require.NoError(t, err) + require.NoError(t, router.Start(context.Background())) + t.Cleanup(func() { router.Stop(); router.Wait() }) + _, err = router.OpenChannel(context.Background(), &ChannelDescriptor{ID: 1, Priority: 1}) + require.NoError(t, err) + router.Stop() + router.Wait() + router.channelMtx.RLock() + count := len(router.channelQueues) + router.channelMtx.RUnlock() + require.Zero(t, count, "Wait must drain channels even when their caller context remains active") + _, err = router.OpenChannel(context.Background(), &ChannelDescriptor{ID: 2, Priority: 1}) + require.Error(t, err) +} diff --git a/internal/p2p/peermanager.go b/internal/p2p/peermanager.go index 6aa79b36f0..8404f6306b 100644 --- a/internal/p2p/peermanager.go +++ b/internal/p2p/peermanager.go @@ -99,6 +99,9 @@ func (pu *PeerUpdate) SetProTxHash(proTxHash types.ProTxHash) { // PeerUpdates is a peer update subscription with notifications about peer // events (currently just status changes). type PeerUpdates struct { + lifecycleMtx sync.Mutex + cancel context.CancelFunc + done chan struct{} routerUpdatesCh chan PeerUpdate reactorUpdatesCh chan PeerUpdate // subscriberName is a label used for debugging @@ -121,6 +124,27 @@ func (pu *PeerUpdates) Updates() <-chan PeerUpdate { return pu.reactorUpdatesCh } +// Close cancels and joins the registered subscription worker. +func (pu *PeerUpdates) Close() { + pu.lifecycleMtx.Lock() + cancel := pu.cancel + pu.lifecycleMtx.Unlock() + if cancel != nil { + cancel() + pu.Wait() + } +} + +// Wait joins the registered subscription worker after its context is canceled. +func (pu *PeerUpdates) Wait() { + pu.lifecycleMtx.Lock() + done := pu.done + pu.lifecycleMtx.Unlock() + if done != nil { + <-done + } +} + // SendUpdate pushes information about a peer into the routing layer, // presumably from a peer. func (pu *PeerUpdates) SendUpdate(ctx context.Context, update PeerUpdate) { @@ -1411,9 +1435,17 @@ func (m *PeerManager) Unsubscribe(sub *PeerUpdates) { func (m *PeerManager) Register(ctx context.Context, peerUpdates *PeerUpdates) { m.mtx.Lock() defer m.mtx.Unlock() + peerUpdates.lifecycleMtx.Lock() + defer peerUpdates.lifecycleMtx.Unlock() + if peerUpdates.done != nil { + return + } + ctx, peerUpdates.cancel = context.WithCancel(ctx) + peerUpdates.done = make(chan struct{}) m.subscriptions[peerUpdates] = peerUpdates go func() { + defer close(peerUpdates.done) defer m.Unsubscribe(peerUpdates) for { select { diff --git a/internal/p2p/pex/lifecycle_test.go b/internal/p2p/pex/lifecycle_test.go new file mode 100644 index 0000000000..4c999b3e88 --- /dev/null +++ b/internal/p2p/pex/lifecycle_test.go @@ -0,0 +1,50 @@ +package pex + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/internal/p2p" + "github.com/dashpay/tenderdash/libs/log" +) + +type blockedReceiveChannel struct { + p2p.Channel + entered chan struct{} + release chan struct{} +} + +func (ch *blockedReceiveChannel) Receive(ctx context.Context) p2p.ChannelIterator { + close(ch.entered) + <-ctx.Done() + <-ch.release + return p2p.NewChannelIterator(make(chan p2p.Envelope)) +} + +func TestReactorWaitDrainsChannelReader(t *testing.T) { + channel := &blockedReceiveChannel{entered: make(chan struct{}), release: make(chan struct{})} + r := NewReactor(log.NewNopLogger(), nil, + func(context.Context, *p2p.ChannelDescriptor) (p2p.Channel, error) { return channel, nil }, + func(context.Context, string) *p2p.PeerUpdates { + return p2p.NewPeerUpdates(make(chan p2p.PeerUpdate), 1, "test") + }) + require.NoError(t, r.Start(context.Background())) + t.Cleanup(func() { r.Stop(); r.Wait() }) + defer close(channel.release) + select { + case <-channel.entered: + case <-time.After(time.Second): + t.Fatal("channel reader did not start") + } + r.Stop() + done := make(chan struct{}) + go func() { r.Wait(); close(done) }() + select { + case <-done: + t.Fatal("Wait returned before the channel reader finished") + case <-time.After(30 * time.Millisecond): + } +} diff --git a/internal/p2p/pex/reactor.go b/internal/p2p/pex/reactor.go index 79bfada9bc..af1917fa76 100644 --- a/internal/p2p/pex/reactor.go +++ b/internal/p2p/pex/reactor.go @@ -83,6 +83,7 @@ type Reactor struct { peerManager *p2p.PeerManager chCreator p2p.ChannelCreator peerEvents p2p.PeerEventSubscriber + peerUpdates *p2p.PeerUpdates // list of available peers to loop through and send peer requests to availablePeers map[types.NodeID]struct{} @@ -135,20 +136,27 @@ func (r *Reactor) OnStart(ctx context.Context) error { } peerUpdates := r.peerEvents(ctx, "pex") - go r.processPexCh(ctx, channel) - go r.processPeerUpdates(ctx, peerUpdates) + r.peerUpdates = peerUpdates + r.Go(ctx, func(ctx context.Context) { r.processPexCh(ctx, channel) }) + r.Go(ctx, func(ctx context.Context) { r.processPeerUpdates(ctx, peerUpdates) }) return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit. +// OnStop implements service.Implementation. func (r *Reactor) OnStop() {} +// OnDrain joins the subscription after its consumers have stopped. +func (r *Reactor) OnDrain() { r.peerUpdates.Close() } + // processPexCh implements a blocking event loop where we listen for p2p // Envelope messages from the pexCh. func (r *Reactor) processPexCh(ctx context.Context, pexCh p2p.Channel) { + ctx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + defer func() { cancel(); <-done }() incoming := make(chan *p2p.Envelope) go func() { + defer close(done) defer close(incoming) iter := pexCh.Receive(ctx) for iter.Next(ctx) { diff --git a/internal/p2p/router.go b/internal/p2p/router.go index 375eecab50..ac5a463980 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -170,6 +170,7 @@ type Router struct { // channels on router start. This depends on whether we want to allow // dynamic channels in the future. channelMtx sync.RWMutex + ctx context.Context channelQueues map[ChannelID]queue // inbound messages from all peers to a single channel } @@ -260,6 +261,9 @@ type ChannelCreator func(context.Context, *ChannelDescriptor) (Channel, error) func (r *Router) OpenChannel(ctx context.Context, chDesc *ChannelDescriptor) (Channel, error) { r.channelMtx.Lock() defer r.channelMtx.Unlock() + if r.ctx == nil || r.ctx.Err() != nil { + return nil, errors.New("router is not running") + } id := chDesc.ID if _, ok := r.channelQueues[id]; ok { @@ -279,16 +283,28 @@ func (r *Router) OpenChannel(ctx context.Context, chDesc *ChannelDescriptor) (Ch r.transport.AddChannelDescriptors([]*ChannelDescriptor{chDesc}) - go func() { + channelCtx, cancel := context.WithCancel(ctx) + stopCancel := context.AfterFunc(r.ctx, cancel) + if !r.Go(r.ctx, func(context.Context) { + defer stopCancel() + defer cancel() defer func() { r.channelMtx.Lock() delete(r.channelQueues, id) r.channelMtx.Unlock() queue.close() + <-queue.closed() }() - r.routeChannel(ctx, chDesc.ID, outCh, errCh) - }() + r.routeChannel(channelCtx, chDesc.ID, outCh, errCh) + }) { + stopCancel() + cancel() + delete(r.channelQueues, id) + queue.close() + <-queue.closed() + return nil, errors.New("router is stopping") + } return channel, nil } @@ -463,13 +479,16 @@ func (r *Router) acceptPeers(ctx context.Context, transport Transport) { } // Spawn a goroutine for the handshake, to avoid head-of-line blocking. - go r.openConnection(ctx, conn) + if !r.Go(ctx, func(ctx context.Context) { r.openConnection(ctx, conn) }) { + r.closeConnection(conn) + r.connTracker.RemoveConn(incomingIP) + } } } func (r *Router) openConnection(ctx context.Context, conn Connection) { - defer conn.Close() + defer r.closeConnection(conn) defer r.connTracker.RemoveConn(conn.RemoteEndpoint().IP) re := conn.RemoteEndpoint() @@ -583,14 +602,14 @@ func (r *Router) connectPeer(ctx context.Context, address NodeAddress) { peerInfo, err := r.handshakePeer(ctx, conn, address.NodeID) switch { case errors.Is(err, context.Canceled): - conn.Close() + r.closeConnection(conn) return case err != nil: r.logger.Error("failed to handshake with peer", "peer", address, "err", err) if err = r.peerManager.DialFailed(ctx, address); err != nil { r.logger.Error("failed to report dial failure", "peer", address, "err", err) } - conn.Close() + r.closeConnection(conn) return } @@ -598,12 +617,25 @@ func (r *Router) connectPeer(ctx context.Context, address NodeAddress) { if err != nil { r.logger.Error("failed to dial peer", "op", "outgoing/dialing", "peer", address.NodeID, "err", err) r.peerManager.dialWaker.Wake() - conn.Close() + r.closeConnection(conn) return } // routePeer (also) calls connection close - go r.routePeer(ctx, address.NodeID, conn, toChannelIDs(peerInfo.Channels.ToSlice())) + if !r.Go(ctx, func(ctx context.Context) { + r.routePeer(ctx, address.NodeID, conn, toChannelIDs(peerInfo.Channels.ToSlice())) + }) { + r.closeConnection(conn) + r.peerManager.Disconnected(ctx, address.NodeID) + } +} + +// closeConnection joins transport workers from their owner, never from a callback. +func (r *Router) closeConnection(conn Connection) { + _ = conn.Close() + if waiter, ok := conn.(interface{ Wait() }); ok { + waiter.Wait() + } } func (r *Router) getOrMakeQueue(peerID types.NodeID, channels ChannelIDSet) queue { @@ -733,8 +765,9 @@ func (r *Router) routePeer(ctx context.Context, peerID types.NodeID, conn Connec r.peerMtx.Unlock() r.completePeerDeliveries(peerID) - _ = conn.Close() + r.closeConnection(conn) sendQueue.close() + <-sendQueue.closed() r.peerManager.Disconnected(ctx, peerID) r.metrics.PeersConnected.Add(-1) @@ -1046,6 +1079,9 @@ func (r *Router) setupQueueFactory(ctx context.Context) error { // OnStart implements service.Service. func (r *Router) OnStart(ctx context.Context) error { + r.channelMtx.Lock() + r.ctx = ctx + r.channelMtx.Unlock() if err := r.setupQueueFactory(ctx); err != nil { return err } @@ -1054,9 +1090,9 @@ func (r *Router) OnStart(ctx context.Context) error { return err } - go r.dialPeers(ctx) - go r.evictPeers(ctx) - go r.acceptPeers(ctx, r.transport) + r.Go(ctx, r.dialPeers) + r.Go(ctx, r.evictPeers) + r.Go(ctx, func(ctx context.Context) { r.acceptPeers(ctx, r.transport) }) return nil } @@ -1073,7 +1109,7 @@ func (r *Router) OnStop() { r.logger.Error("failed to close transport", "err", err) } - // Collect all remaining queues, and wait for them to close. + // Queue owners join their workers after cancellation. queues := []queue{} r.channelMtx.RLock() @@ -1090,7 +1126,6 @@ func (r *Router) OnStop() { for _, q := range queues { q.close() - <-q.closed() } } diff --git a/internal/p2p/rqueue.go b/internal/p2p/rqueue.go index 669acfede1..70e40433d8 100644 --- a/internal/p2p/rqueue.go +++ b/internal/p2p/rqueue.go @@ -13,7 +13,7 @@ type simpleQueue struct { input chan Envelope output chan Envelope closeFn func() - closeCh <-chan struct{} + closeCh chan struct{} maxSize int chDescs []*ChannelDescriptor @@ -29,7 +29,7 @@ func newSimplePriorityQueue(ctx context.Context, size int) *simpleQueue { input: make(chan Envelope, size*2), output: make(chan Envelope, size/2), maxSize: size * size, - closeCh: ctx.Done(), + closeCh: make(chan struct{}), closeFn: cancel, } @@ -43,6 +43,7 @@ func (q *simpleQueue) close() { q.closeFn() } func (q *simpleQueue) closed() <-chan struct{} { return q.closeCh } func (q *simpleQueue) run(ctx context.Context) { + defer close(q.closeCh) defer q.closeFn() var chPriorities = make(map[ChannelID]uint, len(q.chDescs)) @@ -54,6 +55,7 @@ func (q *simpleQueue) run(ctx context.Context) { pq := make(priorityQueue, 0, q.maxSize) heap.Init(&pq) ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() // must have a buffer of exactly one because both sides of // this channel are used in this loop, and simply signals adds // to the heap @@ -62,8 +64,6 @@ func (q *simpleQueue) run(ctx context.Context) { select { case <-ctx.Done(): return - case <-q.closeCh: - return case e := <-q.input: // enqueue the incoming Envelope heap.Push(&pq, &pqEnvelope{ @@ -106,8 +106,6 @@ func (q *simpleQueue) run(ctx context.Context) { select { case <-ctx.Done(): return - case <-q.closeCh: - return case q.output <- next: heap.Pop(&pq) continue SEND diff --git a/internal/p2p/transport_mconn.go b/internal/p2p/transport_mconn.go index 4f6890087c..5d8b09c66c 100644 --- a/internal/p2p/transport_mconn.go +++ b/internal/p2p/transport_mconn.go @@ -137,7 +137,10 @@ func (m *MConnTransport) Accept(ctx context.Context) (Connection, error) { conCh := make(chan net.Conn) errCh := make(chan error) + done := make(chan struct{}) + defer func() { <-done }() go func() { + defer close(done) tcpConn, err := m.listener.Accept() if err != nil { select { @@ -146,12 +149,15 @@ func (m *MConnTransport) Accept(ctx context.Context) (Connection, error) { m.logger.Trace("MConnTransport Accept: connection closed - doneCh") case <-ctx.Done(): } + return } select { case conCh <- tcpConn: case <-m.doneCh: + _ = tcpConn.Close() m.logger.Trace("MConnTransport Accept: connection closed - doneCh") case <-ctx.Done(): + _ = tcpConn.Close() } }() @@ -272,7 +278,8 @@ type mConnConnection struct { doneCh chan struct{} closeOnce sync.Once - mconn *conn.MConnection // set during Handshake() + mconnMtx sync.Mutex + mconn *conn.MConnection // set during Handshake() } // mConnMessage passes MConnection messages through internal channels. @@ -321,7 +328,10 @@ func (c *mConnConnection) Handshake( // To handle context cancellation, we need to do the handshake in a // goroutine and abort the blocking network calls by closing the connection // when the context is canceled. + done := make(chan struct{}) + defer func() { <-done }() go func() { + defer close(done) // FIXME: Since the MConnection code panics, we need to recover it and turn it // into an error. We should remove panics instead. defer func() { @@ -332,11 +342,7 @@ func (c *mConnConnection) Handshake( var err error mconn, peerInfo, peerKey, err = c.handshake(handshakeCtx, nodeInfo, privKey) - select { - case errCh <- err: - case <-c.doneCh: - case <-handshakeCtx.Done(): - } + errCh <- err }() @@ -350,6 +356,13 @@ func (c *mConnConnection) Handshake( if err != nil { return types.NodeInfo{}, nil, err } + c.mconnMtx.Lock() + defer c.mconnMtx.Unlock() + select { + case <-c.doneCh: + return types.NodeInfo{}, nil, io.EOF + default: + } c.mconn = mconn // Start must not use the handshakeCtx. The handshakeCtx may have a // timeout set that is intended to terminate only the handshake procedure. @@ -386,28 +399,22 @@ func (c *mConnConnection) handshake( go func() { defer wg.Done() _, err := protoio.NewDelimitedWriter(secretConn).WriteMsg(nodeInfo.ToProto()) - select { - case errCh <- err: - case <-c.doneCh: - case <-ctx.Done(): - } + errCh <- err }() wg.Add(1) go func() { defer wg.Done() _, err := protoio.NewDelimitedReader(secretConn, types.MaxNodeInfoSize()).ReadMsg(&pbPeerInfo) - select { - case errCh <- err: - case <-c.doneCh: - case <-ctx.Done(): - } + errCh <- err }() wg.Wait() - - if err, ok := <-errCh; ok && err != nil { - return nil, types.NodeInfo{}, nil, err + close(errCh) + for err := range errCh { + if err != nil { + return nil, types.NodeInfo{}, nil, err + } } if err := ctx.Err(); err != nil { @@ -550,14 +557,28 @@ func (c *mConnConnection) RemoteEndpoint() Endpoint { func (c *mConnConnection) Close() error { var err error c.closeOnce.Do(func() { + c.mconnMtx.Lock() + defer c.mconnMtx.Unlock() c.logger.Trace("mConnConnection.Close(): closing doneCh") defer close(c.doneCh) - if c.mconn != nil && c.mconn.IsRunning() { + if c.mconn != nil { c.mconn.Stop() - } else { - err = c.conn.Close() + } + err = c.conn.Close() + if errors.Is(err, net.ErrClosed) { + err = nil } }) return err } + +// Wait joins the protocol workers after Close. Call it from the connection owner. +func (c *mConnConnection) Wait() { + c.mconnMtx.Lock() + mconn := c.mconn + c.mconnMtx.Unlock() + if mconn != nil { + mconn.Wait() + } +} From a6ca9e613308a19518b1521ea4804a3ade60c254 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:17:21 +0000 Subject: [PATCH 03/39] refactor(service): join owned workers before releasing resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrate service workers and child ownership to the shared lifecycle. Preserve application finalization during block-sync handover and node shutdown, drain event observers before closing sinks, and keep failed startup interruptible. Co-Authored-By: Codex 🤖 Co-authored by [Claudius the Magnificent](https://github.com/lklimek/claudius) AI Agent --- abci/client/grpc_client.go | 8 +- abci/client/routed_client.go | 40 ++++--- abci/client/socket_client.go | 25 ++-- abci/server/grpc_server.go | 14 +-- abci/server/socket_server.go | 14 ++- dash/quorum/validator_conn_executor.go | 5 +- internal/blocksync/handover_test.go | 21 ++++ internal/blocksync/reactor.go | 49 ++++++-- internal/blocksync/synchronizer.go | 33 +++--- internal/blocksync/synchronizer_test.go | 4 + internal/eventbus/event_bus.go | 8 +- internal/eventbus/lifecycle_test.go | 44 +++++++ internal/evidence/reactor.go | 25 ++-- internal/mempool/reactor.go | 25 ++-- internal/proxy/client.go | 21 ++-- internal/proxy/client_test.go | 2 - internal/pubsub/pubsub.go | 50 ++++---- internal/state/indexer/indexer_service.go | 9 +- internal/state/indexer/lifecycle_test.go | 37 ++++++ internal/statesync/reactor.go | 35 ++++-- light/rpc/client.go | 31 +++-- node/node.go | 101 +++++++++++----- node/node_test.go | 135 +++++++++++++++++++++- node/seed.go | 21 +++- node/setup.go | 45 ++------ privval/signer_client.go | 1 + privval/signer_listener_endpoint.go | 15 ++- privval/signer_listener_endpoint_test.go | 34 ++++++ privval/signer_server.go | 4 +- 29 files changed, 634 insertions(+), 222 deletions(-) create mode 100644 internal/eventbus/lifecycle_test.go create mode 100644 internal/state/indexer/lifecycle_test.go diff --git a/abci/client/grpc_client.go b/abci/client/grpc_client.go index 0ffed9b399..bb7981581e 100644 --- a/abci/client/grpc_client.go +++ b/abci/client/grpc_client.go @@ -144,7 +144,13 @@ func (cli *grpcClient) streamClientInterceptor(ctx context.Context, desc *grpc.S return streamer(ctx, desc, cc, method, opts...) } -func (cli *grpcClient) OnStart(ctx context.Context) error { +func (cli *grpcClient) OnStart(ctx context.Context) (startErr error) { + defer func() { + if startErr != nil && cli.conn != nil { + _ = cli.conn.Close() + cli.conn = nil + } + }() timer := time.NewTimer(0) defer timer.Stop() diff --git a/abci/client/routed_client.go b/abci/client/routed_client.go index fa1afc1bc3..0e104aca0e 100644 --- a/abci/client/routed_client.go +++ b/abci/client/routed_client.go @@ -17,7 +17,7 @@ import ( ) type routedClient struct { - service.Service + *service.BaseService logger log.Logger routing Routing defaultClient ClientInfo @@ -125,7 +125,7 @@ func NewRoutedClient(logger log.Logger, defaultClient Client, routing Routing) ( defaultClient: ClientInfo{defaultClient, defaultClientID}, } - cli.Service = service.NewBaseService(logger, "RoutedClient", cli) + cli.BaseService = service.NewBaseService(logger, "RoutedClient", cli) return cli, nil } @@ -147,24 +147,36 @@ func (cli *routedClient) OnStart(ctx context.Context) error { } } + if !cli.Go(ctx, func(ctx context.Context) { + <-ctx.Done() + for _, client := range cli.clients() { + client.Wait() + } + }) { + cli.OnStop() + for _, client := range cli.clients() { + client.Wait() + } + } return errs } func (cli *routedClient) OnStop() { - for _, clients := range cli.routing { - for _, client := range clients { - if client.IsRunning() { - switch c := client.Client.(type) { - case *socketClient: - c.Stop() - case *localClient: - c.Stop() - case *grpcClient: - c.Stop() - } - } + for _, client := range cli.clients() { + if stopper, ok := client.(interface{ Stop() }); ok { + stopper.Stop() + } + } +} + +func (cli *routedClient) clients() []Client { + clients := []Client{cli.defaultClient.Client} + for _, routes := range cli.routing { + for _, route := range routes { + clients = append(clients, route.Client) } } + return clients } // delegate calls the given function on the appropriate client with the given diff --git a/abci/client/socket_client.go b/abci/client/socket_client.go index fb046187d8..6387aeddfa 100644 --- a/abci/client/socket_client.go +++ b/abci/client/socket_client.go @@ -90,21 +90,23 @@ func (cli *socketClient) OnStart(ctx context.Context) error { } cli.conn = conn - go cli.sendRequestsRoutine(ctx, conn) - go cli.recvResponseRoutine(ctx, conn) + cli.Go(ctx, func(ctx context.Context) { cli.sendRequestsRoutine(ctx, conn) }) + cli.Go(ctx, func(ctx context.Context) { cli.recvResponseRoutine(ctx, conn) }) return nil } } -// OnStop implements Service by closing connection and flushing all queues. +// OnStop closes the connection to unblock transport workers. func (cli *socketClient) OnStop() { if cli.conn != nil { cli.conn.Close() } - cli.drainQueue() } +// OnDrain releases pending requests after the transport workers have exited. +func (cli *socketClient) OnDrain() { cli.drainQueue() } + func (cli *socketClient) String() string { if err := cli.Error(); err != nil { return fmt.Sprintf("%T(%s):err=%s", cli, cli.addr, err.Error()) @@ -162,12 +164,12 @@ func (cli *socketClient) sendRequestsRoutine(ctx context.Context, conn io.Writer cli.trackRequest(reqres) if err := types.WriteMessage(reqres.Request, bw); err != nil { - cli.stopForError(fmt.Errorf("write to buffer: %w", err)) + cli.stopForError(ctx, fmt.Errorf("write to buffer: %w", err)) return } if err := bw.Flush(); err != nil { - cli.stopForError(fmt.Errorf("flush buffer: %w", err)) + cli.stopForError(ctx, fmt.Errorf("flush buffer: %w", err)) return } } @@ -181,18 +183,18 @@ func (cli *socketClient) recvResponseRoutine(ctx context.Context, conn io.Reader res := &types.Response{} if err := types.ReadMessage(r, res); err != nil { - cli.stopForError(fmt.Errorf("read message: %w", err)) + cli.stopForError(ctx, fmt.Errorf("read message: %w", err)) return } switch r := res.Value.(type) { case *types.Response_Exception: // app responded with error // XXX After setting cli.err, release waiters (e.g. reqres.Done()) - cli.stopForError(errors.New(r.Exception.Error)) + cli.stopForError(ctx, errors.New(r.Exception.Error)) return default: if err := cli.didRecvResponse(res); err != nil { - cli.stopForError(err) + cli.stopForError(ctx, err) return } } @@ -435,11 +437,10 @@ func resMatchesReq(req *types.Request, res *types.Response) (ok bool) { return ok } -func (cli *socketClient) stopForError(err error) { - if !cli.IsRunning() { +func (cli *socketClient) stopForError(ctx context.Context, err error) { + if ctx.Err() != nil { return } - cli.mtx.Lock() cli.err = err cli.mtx.Unlock() diff --git a/abci/server/grpc_server.go b/abci/server/grpc_server.go index 21bb42b03e..643b2d6bb4 100644 --- a/abci/server/grpc_server.go +++ b/abci/server/grpc_server.go @@ -47,22 +47,22 @@ func (s *GRPCServer) OnStart(ctx context.Context) error { types.RegisterABCIApplicationServer(s.server, &gRPCApplication{Application: s.app}) s.logger.Info("Listening", "proto", s.proto, "addr", s.addr) - go func() { - go func() { - <-ctx.Done() - s.server.GracefulStop() - }() - + if !s.Go(ctx, func(context.Context) { if err := s.server.Serve(ln); err != nil { s.logger.Error("error serving gRPC server", "err", err) } - }() + }) { + return ln.Close() + } return nil } // OnStop stops the gRPC server. func (s *GRPCServer) OnStop() { s.server.Stop() } +// OnDrain joins handlers that were already running when the transport closed. +func (s *GRPCServer) OnDrain() { s.server.GracefulStop() } + //------------------------------------------------------- // gRPCApplication is a gRPC shim for Application diff --git a/abci/server/socket_server.go b/abci/server/socket_server.go index 650d9828f2..4d6482c1ac 100644 --- a/abci/server/socket_server.go +++ b/abci/server/socket_server.go @@ -55,7 +55,7 @@ func (s *SocketServer) OnStart(ctx context.Context) error { } s.listener = ln - go s.acceptConnectionsRoutine(ctx) + s.Go(ctx, s.acceptConnectionsRoutine) return nil } @@ -103,7 +103,7 @@ func (s *SocketServer) acceptConnectionsRoutine(ctx context.Context) { s.logger.Info("Waiting for new connection...") conn, err := s.listener.Accept() if err != nil { - if !s.IsRunning() { + if ctx.Err() != nil { return // Ignore error from listener closing. } s.logger.Warn("Failed to accept connection", "err", err) @@ -153,9 +153,15 @@ func (s *SocketServer) acceptConnectionsRoutine(ctx context.Context) { } // Read requests from conn and deal with them - go s.handleRequests(cctx, closer, conn, responses) + if !s.Go(cctx, func(ctx context.Context) { s.handleRequests(ctx, closer, conn, responses) }) { + closer(context.Canceled) + return + } // Pull responses from 'responses' and write them to conn. - go s.handleResponses(cctx, closer, conn, responses) + if !s.Go(cctx, func(ctx context.Context) { s.handleResponses(ctx, closer, conn, responses) }) { + closer(context.Canceled) + return + } } } diff --git a/dash/quorum/validator_conn_executor.go b/dash/quorum/validator_conn_executor.go index 4d30cc9fbe..e3768ddfaf 100644 --- a/dash/quorum/validator_conn_executor.go +++ b/dash/quorum/validator_conn_executor.go @@ -167,13 +167,13 @@ func (vc *ValidatorConnExecutor) OnStart(ctx context.Context) error { vc.logger.Error("Warning: ValidatorConnExecutor OnStart failed", "error", err) } - go func() { + vc.Go(ctx, func(ctx context.Context) { var err error for err == nil { err = vc.receiveEvents(ctx) } vc.logger.Error("ValidatorConnExecutor goroutine finished", "reason", err) - }() + }) return nil } @@ -186,7 +186,6 @@ func (vc *ValidatorConnExecutor) OnStop() { if err != nil { vc.logger.Error("cannot unsubscribe from channels", "error", err) } - vc.eventBus = nil } } diff --git a/internal/blocksync/handover_test.go b/internal/blocksync/handover_test.go index 89f2d4b808..86bf6969b3 100644 --- a/internal/blocksync/handover_test.go +++ b/internal/blocksync/handover_test.go @@ -55,6 +55,11 @@ func TestSwitchToBlockSyncStartsAfterSnapshot(t *testing.T) { executor: applier, synchronizer: synchronizer, blockSyncFlag: new(atomic.Bool), eventBus: bus, statusUpdateInterval: time.Hour, } + hooks := &handoverStartHook{start: func(ctx context.Context) { reactor.serviceCtx = ctx }} + reactor.BaseService = *service.NewBaseService(log.NewNopLogger(), "blocksync-test", hooks) + require.NoError(t, reactor.Start(ctx)) + defer reactor.Wait() + defer cancel() state := sm.State{InitialHeight: tc.initialHeight, LastBlockHeight: 100} require.NoError(t, reactor.SwitchToBlockSync(ctx, state)) defer synchronizer.Wait() @@ -156,5 +161,21 @@ func (c *handoverCapture) SwitchToConsensus(_ context.Context, state sm.State, s type handoverStopHook struct{ fn func() } +func TestSwitchToBlockSyncRejectsAfterStop(t *testing.T) { + reactor := &Reactor{blockSyncFlag: new(atomic.Bool)} + hooks := &handoverStartHook{start: func(ctx context.Context) { reactor.serviceCtx = ctx }} + reactor.BaseService = *service.NewBaseService(log.NewNopLogger(), "blocksync-test", hooks) + require.NoError(t, reactor.Start(context.Background())) + reactor.Stop() + reactor.Wait() + require.Error(t, reactor.SwitchToBlockSync(context.Background(), sm.State{})) + require.False(t, reactor.blockSyncFlag.Load()) +} + +type handoverStartHook struct{ start func(context.Context) } + +func (h *handoverStartHook) OnStart(ctx context.Context) error { h.start(ctx); return nil } +func (*handoverStartHook) OnStop() {} + func (*handoverStopHook) OnStart(context.Context) error { return nil } func (h *handoverStopHook) OnStop() { h.fn() } diff --git a/internal/blocksync/reactor.go b/internal/blocksync/reactor.go index 17342ee179..18527448ea 100644 --- a/internal/blocksync/reactor.go +++ b/internal/blocksync/reactor.go @@ -60,7 +60,8 @@ type consensusReactor interface { // Reactor handles long-term catchup syncing. type Reactor struct { service.BaseService - logger log.Logger + serviceCtx context.Context + logger log.Logger // immutable initialState sm.State @@ -162,6 +163,7 @@ func WithSynchronizerOptions(opts ...OptionFunc) ReactorOption { // If blockSyncFlag is enabled, we also start the synchronizer // If the synchronizer fails to start, an error is returned. func (r *Reactor) OnStart(ctx context.Context) error { + r.serviceCtx = ctx state, err := r.stateStore.Load() if err != nil { return err @@ -184,24 +186,29 @@ func (r *Reactor) OnStart(ctx context.Context) error { if err := r.synchronizer.Start(ctx); err != nil { return err } - go r.requestRoutine(ctx, r.p2pClient) - go r.poolRoutine(ctx, false) + r.Go(ctx, func(ctx context.Context) { r.requestRoutine(ctx, r.p2pClient) }) + r.Go(ctx, func(ctx context.Context) { r.poolRoutine(ctx, false) }) } consumer, messageHandler := consumerHandler(ctx, r.logger, r.store, r.synchronizer) r.messageHandler = messageHandler - go func() { + r.Go(ctx, func(ctx context.Context) { err := r.p2pClient.Consume(ctx, consumer) if err != nil { r.logger.Error("failed to consume p2p blocksync messages", "error", err) } - }() - go r.processPeerUpdates(ctx, r.peerEvents(ctx, "blocksync"), r.p2pClient, messageHandler) + }) + updates := r.peerEvents(ctx, "blocksync") + if !r.Go(ctx, func(ctx context.Context) { + defer updates.Close() + r.processPeerUpdates(ctx, updates, r.p2pClient, messageHandler) + }) { + updates.Close() + } return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit. +// OnStop shuts down the request handler and synchronizer. func (r *Reactor) OnStop() { if r.messageHandler != nil { r.messageHandler.stop() @@ -211,6 +218,12 @@ func (r *Reactor) OnStop() { } } +// OnDrain joins the synchronizer before its owner releases block storage. +func (r *Reactor) OnDrain() { + r.synchronizer.Stop() + r.synchronizer.Wait() +} + // processPeerUpdate processes a PeerUpdate. func (r *Reactor) processPeerUpdate( ctx context.Context, @@ -273,6 +286,21 @@ func (r *Reactor) processPeerUpdates( // SwitchToBlockSync is called by the state sync reactor when switching to fast // sync. func (r *Reactor) SwitchToBlockSync(ctx context.Context, state sm.State) error { + result := make(chan error, 1) + if !r.Go(r.serviceCtx, func(serviceCtx context.Context) { + result <- r.switchToBlockSync(serviceCtx, state) + }) { + return errors.New("block sync reactor is stopping") + } + select { + case err := <-result: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (r *Reactor) switchToBlockSync(ctx context.Context, state sm.State) error { r.blockSyncFlag.Store(true) r.initialState = state r.executor.UpdateState(state) @@ -284,8 +312,8 @@ func (r *Reactor) SwitchToBlockSync(ctx context.Context, state sm.State) error { r.syncStartTime = time.Now() - go r.requestRoutine(ctx, r.p2pClient) - go r.poolRoutine(ctx, true) + r.Go(ctx, func(ctx context.Context) { r.requestRoutine(ctx, r.p2pClient) }) + r.Go(ctx, func(ctx context.Context) { r.poolRoutine(ctx, true) }) if err := r.PublishStatus(types.EventDataBlockSyncStatus{ Complete: false, @@ -321,6 +349,7 @@ func (r *Reactor) requestRoutine(ctx context.Context, p2pClient *client.Client) func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool) { caughtUp, targetHeight := r.synchronizer.WaitForSync(ctx) r.synchronizer.Stop() + r.synchronizer.Wait() // Wait for an application holding the applier lock and use its final state. state := r.executor.State() r.blockSyncFlag.Store(false) diff --git a/internal/blocksync/synchronizer.go b/internal/blocksync/synchronizer.go index 6d5317f10a..f7947d1e4a 100644 --- a/internal/blocksync/synchronizer.go +++ b/internal/blocksync/synchronizer.go @@ -161,13 +161,7 @@ type ( jobGen *jobGenerator pendingToApply map[int64]BlockResponse - // ctx/cancel scope the handler goroutines' lifetime. Created in OnStart - // (live before the goroutines spawn, so it never observes a start-race) - // and canceled in OnStop so Stop releases the handlers even when the - // caller's context is still live. - ctx context.Context - cancel context.CancelFunc - consumerDone chan struct{} + ctx context.Context } OptionFunc func(v *Synchronizer) ) @@ -236,6 +230,13 @@ func (s *Synchronizer) setStartHeight(height int64) { s.jobGen.mtx.Unlock() } +type applicationContextKey struct{} + +// Start preserves the caller's application lifetime across a block-sync handover. +func (s *Synchronizer) Start(ctx context.Context) error { + return s.BaseService.Start(context.WithValue(ctx, applicationContextKey{}, ctx)) +} + // OnStart implements service.Service by spawning requesters routine and recording // synchronizer's start time. func (s *Synchronizer) OnStart(ctx context.Context) error { @@ -244,25 +245,21 @@ func (s *Synchronizer) OnStart(ctx context.Context) error { } s.lastAdvance = s.clock.Now() s.lastMonitorUpdate = s.lastAdvance - s.ctx, s.cancel = context.WithCancel(ctx) - s.consumerDone = make(chan struct{}) + s.ctx = ctx + applicationCtx := ctx.Value(applicationContextKey{}).(context.Context) s.workerPool.Run(s.ctx) - go s.runHandler(s.ctx, s.produceJob) - go func() { - defer close(s.consumerDone) - s.runHandler(s.ctx, func(handlerCtx context.Context) error { + s.Go(ctx, func(ctx context.Context) { s.runHandler(ctx, s.produceJob) }) + s.Go(ctx, func(ctx context.Context) { + s.runHandler(ctx, func(handlerCtx context.Context) error { // Handover cancels consumer I/O; only node shutdown may cancel application. - return s.consumeJobResult(handlerCtx, ctx) + return s.consumeJobResult(handlerCtx, applicationCtx) }) - }() + }) return nil } func (s *Synchronizer) OnStop() { - s.cancel() s.workerPool.Stop(context.Background()) - // Finish all application work before consensus reads the final state. - <-s.consumerDone } func (s *Synchronizer) produceJob(ctx context.Context) error { diff --git a/internal/blocksync/synchronizer_test.go b/internal/blocksync/synchronizer_test.go index ac959cf57c..3878c1ee31 100644 --- a/internal/blocksync/synchronizer_test.go +++ b/internal/blocksync/synchronizer_test.go @@ -113,6 +113,7 @@ func (suite *SynchronizerTestSuite) TestBasic() { return !sync.IsCaughtUp() }, 2*time.Second, 10*time.Millisecond) sync.Stop() + sync.Wait() } func (suite *SynchronizerTestSuite) TestHandoverPreservesInFlightFinalization() { @@ -174,6 +175,7 @@ func (suite *SynchronizerTestSuite) assertHandoverWaitsForConsumer(beforeApply b stopped := make(chan struct{}) go func() { pool.Stop() + pool.Wait() close(stopped) }() select { @@ -255,6 +257,7 @@ func (suite *SynchronizerTestSuite) TestHandoverCancelsBlockedPeerError() { stopped := make(chan struct{}) go func() { pool.Stop() + pool.Wait() close(stopped) }() select { @@ -813,6 +816,7 @@ func (suite *SynchronizerTestSuite) TestStopReleasesHandlers() { time.Sleep(50 * time.Millisecond) sync.Stop() + sync.Wait() // With the parent ctx still live, both handler goroutines must have exited. suite.Require().NoError(ctx.Err()) diff --git a/internal/eventbus/event_bus.go b/internal/eventbus/event_bus.go index 9cffd1475a..6a6d649c02 100644 --- a/internal/eventbus/event_bus.go +++ b/internal/eventbus/event_bus.go @@ -40,7 +40,10 @@ func (b *EventBus) OnStart(ctx context.Context) error { return b.pubsub.Start(ctx) } -func (b *EventBus) OnStop() {} +func (b *EventBus) OnStop() { b.pubsub.Stop() } + +// OnDrain waits for queued events and observers to finish. +func (b *EventBus) OnDrain() { b.pubsub.Wait() } func (b *EventBus) NumClients() int { return b.pubsub.NumClients() @@ -66,6 +69,9 @@ func (b *EventBus) Observe(ctx context.Context, observe func(tmpubsub.Message) e return b.pubsub.Observe(ctx, observe, queries...) } +// RemoveObserver waits for active observation and prevents subsequent callbacks. +func (b *EventBus) RemoveObserver() { b.pubsub.RemoveObserver() } + func (b *EventBus) Publish(eventValue string, eventData types.EventData) error { tokens := strings.Split(types.EventTypeKey, ".") event := abci.Event{ diff --git a/internal/eventbus/lifecycle_test.go b/internal/eventbus/lifecycle_test.go new file mode 100644 index 0000000000..6e81f5f071 --- /dev/null +++ b/internal/eventbus/lifecycle_test.go @@ -0,0 +1,44 @@ +package eventbus_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/internal/eventbus" + "github.com/dashpay/tenderdash/internal/pubsub" + "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/types" +) + +func TestWaitJoinsBlockedObserver(t *testing.T) { + bus := eventbus.NewDefault(log.NewNopLogger()) + require.NoError(t, bus.Start(context.Background())) + entered, release := make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + defer func() { unblock(); bus.Stop(); bus.Wait() }() + require.NoError(t, bus.Observe(context.Background(), func(pubsub.Message) error { + close(entered) + <-release + return nil + })) + require.NoError(t, bus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{})) + <-entered + bus.Stop() + done := make(chan struct{}) + go func() { bus.Wait(); close(done) }() + select { + case <-done: + t.Fatal("Wait returned while the observer was still running") + case <-time.After(20 * time.Millisecond): + } + unblock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("Wait did not finish after the observer returned") + } +} diff --git a/internal/evidence/reactor.go b/internal/evidence/reactor.go index ef83f4b264..6f72a9f68f 100644 --- a/internal/evidence/reactor.go +++ b/internal/evidence/reactor.go @@ -177,15 +177,23 @@ func (r *Reactor) OnStart(ctx context.Context) (err error) { ctx, peerEvidenceRate, peerEvidenceBurst, true, r.logger, client.WithRateLimitClock(r.clock)) r.nodeBudget = newNodeBudget() - go r.processEvidenceCh(ctx) - go r.processPeerUpdates(ctx, r.peerEvents(ctx, "evidence")) + r.Go(ctx, r.processEvidenceCh) + updates := r.peerEvents(ctx, "evidence") + if !r.Go(ctx, func(ctx context.Context) { + defer updates.Close() + r.processPeerUpdates(ctx, updates) + }) { + updates.Close() + } return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit. -func (r *Reactor) OnStop() { r.evpool.Close() } +// OnStop leaves the shared pool open for its owner to close after consensus drains. +func (r *Reactor) OnStop() {} + +// OnDrain joins the rate limiter's cleanup worker. +func (r *Reactor) OnDrain() { r.peerLimit.Close() } // handleEvidenceMessage handles envelopes sent from peers on the EvidenceChannel. // It returns an error only if the Envelope.Message is unknown for this channel @@ -341,7 +349,7 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda // Do not allow starting new evidence broadcast loops after reactor shutdown // has been initiated. This can happen after we've manually closed all // peer broadcast loops, but the router still sends in-flight peer updates. - if !r.IsRunning() { + if ctx.Err() != nil { return } @@ -356,7 +364,10 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda r.nextSyncID++ entry := peerSyncState{cancel: pcancel, id: r.nextSyncID} r.peerRoutines[peerUpdate.NodeID] = entry - go r.syncEvidence(pctx, peerUpdate.NodeID, entry.id) + if !r.Go(pctx, func(ctx context.Context) { r.syncEvidence(ctx, peerUpdate.NodeID, entry.id) }) { + pcancel() + delete(r.peerRoutines, peerUpdate.NodeID) + } } case p2p.PeerStatusDown: diff --git a/internal/mempool/reactor.go b/internal/mempool/reactor.go index c00c364359..30c0493518 100644 --- a/internal/mempool/reactor.go +++ b/internal/mempool/reactor.go @@ -73,19 +73,24 @@ func (r *Reactor) OnStart(ctx context.Context) error { if !r.cfg.Broadcast { r.logger.Info("tx broadcasting is disabled") } - go func() { + r.Go(ctx, func(ctx context.Context) { err := r.p2pClient.Consume(ctx, consumerHandler(ctx, r.logger, r.mempool.config, r.mempool, r.ids)) if err != nil { r.logger.Error("failed to consume p2p checker messages", "error", err) } - }() - go r.processPeerUpdates(ctx, r.peerEvents(ctx, "checker")) + }) + updates := r.peerEvents(ctx, "checker") + if !r.Go(ctx, func(ctx context.Context) { + defer updates.Close() + r.processPeerUpdates(ctx, updates) + }) { + updates.Close() + } return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit. +// OnStop needs no additional cleanup after the service context is canceled. func (r *Reactor) OnStop() {} // processPeerUpdate processes a PeerUpdate. For added peers, PeerStatusUp, we @@ -104,7 +109,7 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda // Do not allow starting new tx broadcast loops after reactor shutdown // has been initiated. This can happen after we've manually closed all // peer broadcast, but the router still sends in-flight peer updates. - if !r.IsRunning() { + if ctx.Err() != nil { return } @@ -121,7 +126,11 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda r.ids.ReserveForPeer(peerUpdate.NodeID) // start a broadcast routine ensuring all txs are forwarded to the peer - go r.broadcastTxRoutine(pctx, peerUpdate.NodeID) + if !r.Go(pctx, func(ctx context.Context) { r.broadcastTxRoutine(ctx, peerUpdate.NodeID) }) { + pcancel() + delete(r.peerRoutines, peerUpdate.NodeID) + r.ids.Reclaim(peerUpdate.NodeID) + } } } @@ -185,7 +194,7 @@ func (r *Reactor) broadcastTxRoutine(ctx context.Context, peerID types.NodeID) { }() for { - if !r.IsRunning() || ctx.Err() != nil { + if ctx.Err() != nil { return } diff --git a/internal/proxy/client.go b/internal/proxy/client.go index 9421d6b8ea..e6ea6ed938 100644 --- a/internal/proxy/client.go +++ b/internal/proxy/client.go @@ -96,18 +96,12 @@ func tryCallStop(client abciclient.Client) { } func (app *proxyClient) OnStart(ctx context.Context) error { - var err error - defer func() { - if err != nil { - tryCallStop(app.client) - } - }() + if err := app.client.Start(ctx); err != nil { + return err + } // Kill Tendermint if the ABCI application crashes. - go func() { - if !app.client.IsRunning() { - return - } + if !app.Go(ctx, func(ctx context.Context) { app.client.Wait() if ctx.Err() != nil { return @@ -123,9 +117,12 @@ func (app *proxyClient) OnStart(ctx context.Context) error { } } - }() + }) { + tryCallStop(app.client) + app.client.Wait() + } - return app.client.Start(ctx) + return nil } func kill() error { diff --git a/internal/proxy/client_test.go b/internal/proxy/client_test.go index 8821ef7ea5..e4a04afaea 100644 --- a/internal/proxy/client_test.go +++ b/internal/proxy/client_test.go @@ -198,7 +198,6 @@ func TestAppConns_Start_Stop(t *testing.T) { clientMock := &abcimocks.Client{} clientMock.On("Start", mock.Anything).Return(nil) clientMock.On("Error").Return(nil) - clientMock.On("IsRunning").Return(true) clientMock.On("Wait").Return(nil).Times(1) cl := &noopStoppableClientImpl{Client: clientMock} @@ -227,7 +226,6 @@ func TestAppConns_Failure(t *testing.T) { clientMock := &abcimocks.Client{} clientMock.On("SetLogger", mock.Anything).Return() clientMock.On("Start", mock.Anything).Return(nil) - clientMock.On("IsRunning").Return(true) clientMock.On("Wait").Return(nil) clientMock.On("Error").Return(errors.New("EOF")) cl := &noopStoppableClientImpl{Client: clientMock} diff --git a/internal/pubsub/pubsub.go b/internal/pubsub/pubsub.go index 14dc70e694..2e87c4939b 100644 --- a/internal/pubsub/pubsub.go +++ b/internal/pubsub/pubsub.go @@ -93,10 +93,9 @@ type Server struct { service.BaseService logger log.Logger - queue chan item - done <-chan struct{} // closed when server should exit - pubs sync.RWMutex // excl: shutdown; shared: active publisher - exited chan struct{} // server exited + queue chan item + done <-chan struct{} // closed when server should exit + pubs sync.RWMutex // excl: shutdown; shared: active publisher // All subscriptions currently known. // Lock exclusive to add, remove, or cancel subscriptions. @@ -191,6 +190,13 @@ func (s *Server) Observe(_ctx context.Context, observe func(Message) error, quer return nil } +// RemoveObserver detaches the observer after any active invocation finishes. +func (s *Server) RemoveObserver() { + s.subs.Lock() + defer s.subs.Unlock() + s.subs.observe = nil +} + // SubscribeWithArgs creates a subscription for the given arguments. It is an // error if the query is nil, a subscription already exists for the specified // client ID and query, or if the capacity arguments are invalid. @@ -302,9 +308,22 @@ func (s *Server) PublishWithEvents(msg types.EventData, events []abci.Event) err // OnStop implements part of the Service interface. It is a no-op. func (s *Server) OnStop() {} -// Wait implements Service.Wait by blocking until the server has exited, then -// yielding to the base service wait. -func (s *Server) Wait() { <-s.exited; s.BaseService.Wait() } +// OnDrain releases subscriptions even when shutdown rejected sender startup. +func (s *Server) OnDrain() { + s.pubs.Lock() + if s.queue != nil { + close(s.queue) + s.queue = nil + } + s.pubs.Unlock() + s.subs.Lock() + defer s.subs.Unlock() + for si := range s.subs.index.all { + si.sub.stop(ErrTerminated) + } + s.subs.index = nil + s.subs.observe = nil +} // OnStart implements Service.OnStart by starting the server. func (s *Server) OnStart(ctx context.Context) error { s.run(ctx); return nil } @@ -331,17 +350,15 @@ func (s *Server) run(ctx context.Context) { // Shutdown monitor: When the context ends, wait for any active publish // calls to exit, then close the queue to signal the sender to exit. - go func() { + s.Go(ctx, func(ctx context.Context) { <-ctx.Done() s.pubs.Lock() defer s.pubs.Unlock() close(s.queue) s.queue = nil - }() + }) - s.exited = make(chan struct{}) - go func() { - defer close(s.exited) + s.Go(ctx, func(context.Context) { // Sender: Service the queue and forward messages to subscribers. for it := range queue { @@ -349,14 +366,7 @@ func (s *Server) run(ctx context.Context) { s.logger.Error("error sending event", "err", err) } } - // Terminate all subscribers before exit. - s.subs.Lock() - defer s.subs.Unlock() - for si := range s.subs.index.all { - si.sub.stop(ErrTerminated) - } - s.subs.index = nil - }() + }) } // removeSubs cancels and removes all the subscriptions in evict with the given diff --git a/internal/state/indexer/indexer_service.go b/internal/state/indexer/indexer_service.go index 8a857aba2c..42a61df2d4 100644 --- a/internal/state/indexer/indexer_service.go +++ b/internal/state/indexer/indexer_service.go @@ -139,8 +139,13 @@ func (is *Service) OnStart(ctx context.Context) error { return nil } -// OnStop implements service.Service by closing the event sinks. -func (is *Service) OnStop() { +func (is *Service) OnStop() {} + +// OnDrain detaches the observer before releasing its sinks. +func (is *Service) OnDrain() { + if IndexingEnabled(is.eventSinks) { + is.eventBus.RemoveObserver() + } for _, sink := range is.eventSinks { if err := sink.Stop(); err != nil { is.logger.Error("failed to close eventsink", "eventsink", sink.Type(), "err", err) diff --git a/internal/state/indexer/lifecycle_test.go b/internal/state/indexer/lifecycle_test.go new file mode 100644 index 0000000000..aa3b94d491 --- /dev/null +++ b/internal/state/indexer/lifecycle_test.go @@ -0,0 +1,37 @@ +package indexer_test + +import ( + "context" + "testing" + "time" + + dbm "github.com/cometbft/cometbft-db" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/internal/eventbus" + "github.com/dashpay/tenderdash/internal/pubsub" + "github.com/dashpay/tenderdash/internal/state/indexer" + "github.com/dashpay/tenderdash/internal/state/indexer/sink/kv" + "github.com/dashpay/tenderdash/libs/log" +) + +func TestIndexerStopsWhileEventBusRemainsLive(t *testing.T) { + bus := eventbus.NewDefault(log.NewNopLogger()) + require.NoError(t, bus.Start(context.Background())) + t.Cleanup(func() { bus.Stop(); bus.Wait() }) + svc := indexer.NewService(indexer.ServiceArgs{ + Logger: log.NewNopLogger(), EventBus: bus, + Sinks: []indexer.EventSink{kv.NewEventSink(dbm.NewMemDB())}, + }) + require.NoError(t, svc.Start(context.Background())) + svc.Stop() + done := make(chan struct{}) + go func() { svc.Wait(); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("indexer shutdown waited for the live event bus") + } + require.True(t, bus.IsRunning()) + require.NoError(t, bus.Observe(context.Background(), func(_ pubsub.Message) error { return nil })) +} diff --git a/internal/statesync/reactor.go b/internal/statesync/reactor.go index 6df321458b..eaea35b7f5 100644 --- a/internal/statesync/reactor.go +++ b/internal/statesync/reactor.go @@ -8,6 +8,7 @@ import ( "fmt" "runtime/debug" "sort" + stdsync "sync" "time" sync "github.com/sasha-s/go-deadlock" @@ -321,13 +322,21 @@ func (r *Reactor) OnStart(ctx context.Context) error { return nil } - go r.processChannels(ctx, map[p2p.ChannelID]p2p.Channel{ - SnapshotChannel: snapshotCh, - ChunkChannel: chunkCh, - LightBlockChannel: blockCh, - ParamsChannel: paramsCh, + r.Go(ctx, func(ctx context.Context) { + r.processChannels(ctx, map[p2p.ChannelID]p2p.Channel{ + SnapshotChannel: snapshotCh, + ChunkChannel: chunkCh, + LightBlockChannel: blockCh, + ParamsChannel: paramsCh, + }) }) - go r.processPeerUpdates(ctx, r.peerEvents(ctx, "statesync")) + updates := r.peerEvents(ctx, "statesync") + if !r.Go(ctx, func(ctx context.Context) { + defer updates.Close() + r.processPeerUpdates(ctx, updates) + }) { + updates.Close() + } if r.needsStateSync { r.logger.Info("starting state sync") @@ -353,8 +362,7 @@ func (r *Reactor) OnStart(ctx context.Context) error { return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit. +// OnStop unblocks outstanding dispatcher requests. func (r *Reactor) OnStop() { // tell the dispatcher to stop sending any more requests r.dispatcher.Close() @@ -639,7 +647,8 @@ func (r *Reactor) backfill( ) ctxWithCancel, cancel := context.WithCancel(ctx) - defer cancel() + var fetchers stdsync.WaitGroup + defer func() { cancel(); fetchers.Wait() }() // Peer-dispatch gate for this run: it quarantines peers that supply light // blocks this node cannot accept and reports when no peer is left to serve. @@ -670,7 +679,7 @@ func (r *Reactor) backfill( dispatch.quarantine(peer, reason) queue.discardPeer(peer) queue.requeue(height) - err := r.sendBlockError(ctx, p2p.PeerError{ + err := r.sendBlockError(ctxWithCancel, p2p.PeerError{ NodeID: peer, Err: reason, Fatal: bool(severity), @@ -685,7 +694,9 @@ func (r *Reactor) backfill( // waiting on blocks. If it takes 4s to retrieve a block and 1s to verify // it, then steady state involves four workers. for i := 0; i < r.cfg.Fetchers; i++ { + fetchers.Add(1) go func() { + defer fetchers.Done() for { select { case <-ctxWithCancel.Done(): @@ -780,7 +791,7 @@ func (r *Reactor) backfill( "height", height, "error", err) queue.retry(height) - if serr := r.sendBlockError(ctx, p2p.PeerError{ + if serr := r.sendBlockError(ctxWithCancel, p2p.PeerError{ NodeID: peer, Err: fmt.Errorf("received invalid light block: %w", err), }); serr != nil { @@ -1326,7 +1337,7 @@ func (r *Reactor) processPeerUp(ctx context.Context, peerUpdate p2p.PeerUpdate) if sp, ok := stateProvider.(*stateProviderP2P); ok { // we do this in a separate routine to not block whilst waiting for the light client to finish // whatever call it's currently executing - go sp.addProvider(newProvider) + r.Go(ctx, func(context.Context) { sp.addProvider(newProvider) }) } } diff --git a/light/rpc/client.go b/light/rpc/client.go index ba0081de85..6d949686d9 100644 --- a/light/rpc/client.go +++ b/light/rpc/client.go @@ -53,7 +53,7 @@ type Client struct { prt *merkle.ProofRuntime keyPathFn KeyPathFunc - closers []func() + serviceCtx context.Context } var _ rpcclient.Client = (*Client)(nil) @@ -106,19 +106,22 @@ func NewClient(logger log.Logger, next rpcclient.Client, lc LightClient, opts .. } func (c *Client) OnStart(ctx context.Context) error { - nctx, ncancel := context.WithCancel(ctx) - if err := c.next.Start(nctx); err != nil { - ncancel() + c.serviceCtx = ctx + if err := c.next.Start(ctx); err != nil { return err } - c.closers = append(c.closers, ncancel) return nil } -func (c *Client) OnStop() { - for _, closer := range c.closers { - closer() +func (c *Client) OnStop() {} + +// OnDrain joins clients exposing transport shutdown in addition to the RPC API. +func (c *Client) OnDrain() { + if waiter, ok := c.next.(interface{ Wait() }); ok { + waiter.Wait() + } else if stopper, ok := c.next.(interface{ Stop() error }); ok { + _ = stopper.Stop() } } @@ -648,8 +651,10 @@ func (c *Client) RegisterOpDecoder(typ string, dec merkle.OpDecoder) { // a subscriber, but does not verify responses (UNSAFE)! // TODO: verify data func (c *Client) SubscribeWS(ctx context.Context, query string) (*coretypes.ResultSubscribe, error) { - bctx, bcancel := context.WithCancel(context.Background()) - c.closers = append(c.closers, bcancel) + if !c.IsRunning() { + return nil, fmt.Errorf("light RPC client is not running") + } + bctx := c.serviceCtx callInfo := rpctypes.GetCallInfo(ctx) out, err := c.next.Subscribe(bctx, callInfo.RemoteAddr(), query) //nolint:staticcheck @@ -657,7 +662,7 @@ func (c *Client) SubscribeWS(ctx context.Context, query string) (*coretypes.Resu return nil, err } - go func() { + if !c.Go(bctx, func(bctx context.Context) { for { select { case resultEvent := <-out: @@ -668,7 +673,9 @@ func (c *Client) SubscribeWS(ctx context.Context, query string) (*coretypes.Resu return } } - }() + }) { + return nil, fmt.Errorf("light RPC client is stopping") + } return &coretypes.ResultSubscribe{}, nil } diff --git a/node/node.go b/node/node.go index d967353f21..1fb1838769 100644 --- a/node/node.go +++ b/node/node.go @@ -64,17 +64,18 @@ type nodeImpl struct { nodeKey types.NodeKey // our node privkey // services - eventSinks []indexer.EventSink - initialState sm.State - stateStore sm.Store - blockStore *store.BlockStore // store the blockchain to disk - evPool *evidence.Pool - indexerService *indexer.Service - services []service.Service - rpcListeners []net.Listener // rpc servers - shutdownOps closer - rpcEnv *rpccore.Environment - prometheusSrv *http.Server + eventSinks []indexer.EventSink + initialState sm.State + stateStore sm.Store + blockStore *store.BlockStore // store the blockchain to disk + evPool *evidence.Pool + indexerService *indexer.Service + services []service.Service + rpcListeners []net.Listener // rpc servers + shutdownOps closer + rpcEnv *rpccore.Environment + prometheusSrv *http.Server + dependencyCancel context.CancelFunc } // newDefaultNode returns a Tendermint node with default settings for the @@ -213,6 +214,9 @@ func makeNode( if err != nil { return nil, combineCloseError(err, makeCloser(closers)) } + if resource, ok := privValidator.(interface{ Close() error }); ok { + closers = append(closers, resource.Close) + } if dashPrivval, ok := privValidator.(privval.DashPrivValidator); ok { dashCoreRPCClient = dashPrivval.DashRPCClient() } @@ -293,11 +297,13 @@ func makeNode( makeCloser(closers)) } + mempoolRateLimit := p2pclient.NewRateLimit(ctx, cfg.Mempool.TxSendRateLimit, false, logger) + closers = append(closers, func() error { mempoolRateLimit.Close(); return nil }) p2pClient := p2pclient.New( p2p.ChannelDescriptors(cfg), node.router.OpenChannel, p2pclient.WithLogger(logger), - p2pclient.WithSendRateLimits(p2pclient.NewRateLimit(ctx, cfg.Mempool.TxSendRateLimit, false, logger), p2p.MempoolChannel), + p2pclient.WithSendRateLimits(mempoolRateLimit, p2p.MempoolChannel), ) evReactor, evPool, edbCloser, err := createEvidenceReactor(logger, cfg, dbProvider, @@ -477,6 +483,7 @@ func makeNode( } node.BaseService = *service.NewBaseService(logger, "Node", node) + node.shutdownOps = makeCloser(closers) return node, nil } @@ -496,9 +503,38 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { } }() - if err := n.rpcEnv.ProxyApp.Start(ctx); err != nil { + // Application transports and event sinks outlive reactor cancellation so an + // in-flight finalization can finish. Node shutdown joins them explicitly. + dependencyCtx, dependencyCancel := context.WithCancel(context.WithoutCancel(ctx)) + stopStartupCancellation := context.AfterFunc(ctx, dependencyCancel) + defer stopStartupCancellation() + n.dependencyCancel = dependencyCancel + ctx, cancelAttempt := context.WithCancel(ctx) + var startedReactors []service.Service + routerStarted := false + defer func() { + if err != nil { + cancelAttempt() + for _, reactor := range startedReactors { + reactor.Wait() + } + if routerStarted { + n.router.Wait() + } + n.rpcEnv.EventBus.Stop() + n.rpcEnv.EventBus.Wait() + dependencyCancel() + n.indexerService.Wait() + n.rpcEnv.ProxyApp.Wait() + } + }() + if err := n.rpcEnv.ProxyApp.Start(dependencyCtx); err != nil { return fmt.Errorf("error starting proxy app connections: %w", err) } + stopStartupCancellation() + if err := ctx.Err(); err != nil { + return err + } var proTxHash tmbytes.HexBytes if n.privValidator != nil { @@ -508,16 +544,17 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { } } ctx = dash.ContextWithProTxHash(ctx, proTxHash) + dependencyCtx = dash.ContextWithProTxHash(dependencyCtx, proTxHash) // EventBus and IndexerService must be started before the handshake because // we might need to index the txs of the replayed block as this might not have happened // when the node stopped last time (i.e. the node stopped or crashed after it saved the block // but before it indexed the txs) - if err := n.rpcEnv.EventBus.Start(ctx); err != nil { + if err := n.rpcEnv.EventBus.Start(dependencyCtx); err != nil { return err } - if err := n.indexerService.Start(ctx); err != nil { + if err := n.indexerService.Start(dependencyCtx); err != nil { return err } @@ -553,7 +590,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { } // Start Internal Services if n.config.RPC.PprofListenAddress != "" { - startPProfServer(ctx, *n.config.RPC) + startPProfServer(ctx, &n.BaseService, *n.config.RPC) } now := tmtime.Now() @@ -579,19 +616,23 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { } if n.config.Instrumentation.Prometheus && n.config.Instrumentation.PrometheusListenAddr != "" { - n.prometheusSrv = startPrometheusServer(ctx, *n.config.Instrumentation) + n.prometheusSrv = startPrometheusServer(ctx, &n.BaseService, *n.config.Instrumentation) } // Start the transport. if err := n.router.Start(ctx); err != nil { return err } + routerStarted = true n.rpcEnv.IsListening = true for _, reactor := range n.services { if err := reactor.Start(ctx); err != nil { return fmt.Errorf("problem starting service '%T': %w ", reactor, err) } + if reactor != n.rpcEnv.EventBus { + startedReactors = append(startedReactors, reactor) + } } n.rpcEnv.NodeInfo = n.nodeInfo @@ -601,9 +642,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { if len(rpcListeners) > 0 { err = n.rpcEnv.StartService(ctx, n.config, rpcListeners) if err != nil { - stopCtx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - if stopErr := n.rpcEnv.StopAsyncBroadcasts(stopCtx); stopErr != nil { + if stopErr := n.rpcEnv.StopAsyncBroadcasts(context.Background()); stopErr != nil { n.logger.Error("Failed to stop async broadcasts after RPC startup failure", "err", stopErr) } return err @@ -617,11 +656,9 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { // OnStop stops the Node. It implements service.Service. func (n *nodeImpl) OnStop() { n.logger.Info("Stopping Node") - stopCtx, cancel := context.WithTimeout(context.Background(), time.Second) - if err := n.rpcEnv.StopAsyncBroadcasts(stopCtx); err != nil { + if err := n.rpcEnv.StopAsyncBroadcasts(context.Background()); err != nil { n.logger.Error("Async broadcasts did not finish during shutdown", "err", err) } - cancel() // stop the listeners / external services first for _, l := range n.rpcListeners { n.logger.Info("Closing rpc listener", "listener", l) @@ -630,15 +667,17 @@ func (n *nodeImpl) OnStop() { } } - for _, es := range n.eventSinks { - if err := es.Stop(); err != nil { - n.logger.Error("failed to stop event sink", "err", err) - } - } - for _, reactor := range n.services { + if reactor == n.rpcEnv.EventBus { + continue + } reactor.Wait() } + n.rpcEnv.EventBus.Stop() + n.rpcEnv.EventBus.Wait() + n.dependencyCancel() + n.indexerService.Wait() + n.rpcEnv.ProxyApp.Wait() n.router.Wait() n.rpcEnv.IsListening = false @@ -656,6 +695,10 @@ func (n *nodeImpl) OnStop() { } } +} + +// OnDrain releases storage after child services and node workers have finished. +func (n *nodeImpl) OnDrain() { if err := n.shutdownOps(); err != nil { if strings.TrimSpace(err.Error()) != "" { n.logger.Error("problem shutting down additional services", "err", err) diff --git a/node/node_test.go b/node/node_test.go index 106b2e23f5..3e292ced3a 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -9,6 +9,7 @@ import ( "os" "strconv" "strings" + "sync" "testing" "time" @@ -1126,6 +1127,7 @@ func TestNodeStartRetryAfterRPCListenFailure(t *testing.T) { n.Wait() } else { n.OnStop() + n.OnDrain() } }() err = n.Start(ctx) @@ -1163,9 +1165,11 @@ func TestNodeStartFailureReleasesRPCListeners(t *testing.T) { ns, err := newDefaultNode(ctx, cfg, log.NewNopLogger()) require.NoError(t, err) n := ns.(*nodeImpl) + defer n.OnDrain() defer n.OnStop() app := abcimocks.NewClient(t) appErr := errors.New("application unavailable") + app.On("Wait").Return() app.On("Start", mock.Anything).Run(func(mock.Arguments) { probe, bindErr := net.Listen("tcp", address) if bindErr == nil { @@ -1184,12 +1188,14 @@ func TestNodeStartFailureReleasesRPCListeners(t *testing.T) { // finalizePausedClient holds the first commit between block-store and state-store writes. type finalizePausedClient struct { abciclient.Client - entered chan struct{} - release chan struct{} + entered chan struct{} + release chan struct{} + finalizeCtx context.Context } func (c *finalizePausedClient) FinalizeBlock(ctx context.Context, req *abci.RequestFinalizeBlock) (*abci.ResponseFinalizeBlock, error) { if req.Block.Header.Height == 1 { + c.finalizeCtx = ctx close(c.entered) <-c.release } @@ -1214,8 +1220,6 @@ func (s startupPausedService) Start(ctx context.Context) error { } func TestNodeStartDuringFirstCommit(t *testing.T) { - // Wait for consensus goroutines before TempDir cleanup removes their files. - defer leaktest.CheckTimeout(t, 5*time.Second)() cfg, err := config.ResetTestRoot(t.TempDir(), t.Name()) require.NoError(t, err) cfg.Consensus = config.DefaultConsensusConfig() @@ -1244,6 +1248,7 @@ func TestNodeStartDuringFirstCommit(t *testing.T) { n.Wait() } else { n.OnStop() + n.OnDrain() } }() err = n.Start(ctx) @@ -1254,3 +1259,125 @@ func TestNodeStartDuringFirstCommit(t *testing.T) { require.NoError(t, err) require.Zero(t, state.LastBlockHeight, "the application has not completed FinalizeBlock") } + +type shutdownPausedService struct { + entered chan struct{} + release chan struct{} +} + +func (*shutdownPausedService) Start(context.Context) error { return nil } +func (*shutdownPausedService) IsRunning() bool { return true } +func (s *shutdownPausedService) Wait() { close(s.entered); <-s.release } + +func TestNodeWaitPreservesFinalization(t *testing.T) { + for _, manual := range []bool{false, true} { + t.Run(fmt.Sprintf("manual=%t", manual), func(t *testing.T) { + cfg, err := config.ResetTestRoot(t.TempDir(), strings.ReplaceAll(t.Name(), "/", "-")) + require.NoError(t, err) + cfg.Consensus = config.DefaultConsensusConfig() + cfg.SetRoot(cfg.RootDir) + cfg.RPC.ListenAddress = "" + cfg.P2P.ListenAddress = "tcp://127.0.0.1:0" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logger := log.NewNopLogger() + client, _, err := proxy.ClientFactory(logger, *cfg.Abci, cfg.DBDir()) + require.NoError(t, err) + app := &finalizePausedClient{Client: client, entered: make(chan struct{}), release: make(chan struct{})} + unpauseApp := sync.OnceFunc(func() { close(app.release) }) + gate := &shutdownPausedService{entered: make(chan struct{}), release: make(chan struct{})} + unpauseShutdown := sync.OnceFunc(func() { close(gate.release) }) + ns, err := New(ctx, cfg, logger, app, nil) + require.NoError(t, err) + n := ns.(*nodeImpl) + for i, reactor := range n.services { + if reactor == n.rpcEnv.ConsensusReactor { + n.services[i] = startupPausedService{Service: reactor, entered: app.entered} + } + } + n.services = append(n.services, gate) + require.NoError(t, n.Start(ctx)) + t.Cleanup(func() { unpauseApp(); unpauseShutdown(); cancel(); n.Wait() }) + stopped := make(chan struct{}) + if manual { + go n.Stop() + } else { + cancel() + } + go func() { n.Wait(); close(stopped) }() + select { + case <-stopped: + t.Fatal("node Wait returned during finalization") + case <-time.After(25 * time.Millisecond): + } + require.NoError(t, app.finalizeCtx.Err()) + unpauseApp() + select { + case <-gate.entered: + case <-time.After(5 * time.Second): + t.Fatal("consensus did not finish finalization") + } + state, err := n.stateStore.Load() + require.NoError(t, err) + require.EqualValues(t, 1, state.LastBlockHeight) + unpauseShutdown() + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("node shutdown did not finish") + } + }) + } +} + +type startupBlockedClient struct { + abciclient.Client + entered chan struct{} + release chan struct{} +} + +func (c *startupBlockedClient) Start(ctx context.Context) error { + close(c.entered) + select { + case <-ctx.Done(): + return ctx.Err() + case <-c.release: + return errors.New("startup unblocked by test cleanup") + } +} + +func (*startupBlockedClient) Wait() {} + +func TestNodeCancellationInterruptsApplicationStartup(t *testing.T) { + for _, manual := range []bool{false, true} { + t.Run(fmt.Sprintf("manual=%t", manual), func(t *testing.T) { + cfg, err := config.ResetTestRoot(t.TempDir(), strings.ReplaceAll(t.Name(), "/", "-")) + require.NoError(t, err) + cfg.RPC.ListenAddress = "" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ns, err := newDefaultNode(ctx, cfg, log.NewNopLogger()) + require.NoError(t, err) + n := ns.(*nodeImpl) + defer n.OnDrain() + app := &startupBlockedClient{entered: make(chan struct{}), release: make(chan struct{})} + defer close(app.release) + n.rpcEnv.ProxyApp = app + started := make(chan error, 1) + go func() { started <- n.Start(ctx) }() + <-app.entered + if manual { + n.Stop() + } else { + cancel() + } + select { + case err := <-started: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("application startup did not observe cancellation") + } + n.Wait() + }) + } +} diff --git a/node/seed.go b/node/seed.go index 9142fedee2..450b79cc19 100644 --- a/node/seed.go +++ b/node/seed.go @@ -99,18 +99,24 @@ func makeSeedNode( // OnStart starts the Seed Node. It implements service.Service. func (n *seedNodeImpl) OnStart(ctx context.Context) error { if n.config.RPC.PprofListenAddress != "" { - startPProfServer(ctx, *n.config.RPC) + startPProfServer(ctx, &n.BaseService, *n.config.RPC) } now := tmtime.Now() genTime := n.genesisDoc.GenesisTime if genTime.After(now) { n.logger.Info("Genesis time is in the future. Sleeping until then...", "genTime", genTime) - time.Sleep(genTime.Sub(now)) + timer := time.NewTimer(genTime.Sub(now)) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } } if n.config.Instrumentation.Prometheus && n.config.Instrumentation.PrometheusListenAddr != "" { - n.prometheusSrv = startPrometheusServer(ctx, *n.config.Instrumentation) + n.prometheusSrv = startPrometheusServer(ctx, &n.BaseService, *n.config.Instrumentation) } // Start the transport. @@ -127,7 +133,16 @@ func (n *seedNodeImpl) OnStop() { n.pexReactor.Wait() n.router.Wait() + if n.prometheusSrv != nil { + if err := n.prometheusSrv.Shutdown(context.Background()); err != nil { + n.logger.Error("failed to stop metrics server", "err", err) + } + } + +} +// OnDrain releases resources after network and HTTP workers have finished. +func (n *seedNodeImpl) OnDrain() { if err := n.shutdownOps(); err != nil { if strings.TrimSpace(err.Error()) != "" { n.logger.Error("problem shutting down additional services", "err", err) diff --git a/node/setup.go b/node/setup.go index 3e096d4227..8657b882c3 100644 --- a/node/setup.go +++ b/node/setup.go @@ -654,7 +654,7 @@ func createBlockReplayer(n *nodeImpl) *consensus.BlockReplayer { // startPrometheusServer starts a Prometheus HTTP server, listening for metrics // collectors on addr. -func startPrometheusServer(ctx context.Context, cfg config.InstrumentationConfig) *http.Server { +func startPrometheusServer(ctx context.Context, owner *service.BaseService, cfg config.InstrumentationConfig) *http.Server { addr := cfg.PrometheusListenAddr srv := &http.Server{ Addr: addr, @@ -667,48 +667,25 @@ func startPrometheusServer(ctx context.Context, cfg config.InstrumentationConfig ), } - signal := make(chan struct{}) - go func() { - select { - case <-ctx.Done(): - sctx, scancel := context.WithTimeout(context.Background(), time.Second) - defer scancel() - _ = srv.Shutdown(sctx) - case <-signal: - } - }() - - go func() { - if err := srv.ListenAndServe(); err != nil { - close(signal) - } - }() + startHTTPServer(ctx, owner, srv) return srv } // startPProfServer creates a new pprof server -// FIXME: implement as a Service -func startPProfServer(ctx context.Context, cfg config.RPCConfig) { - signal := make(chan struct{}) +func startPProfServer(ctx context.Context, owner *service.BaseService, cfg config.RPCConfig) { srv := &http.Server{ Addr: cfg.PprofListenAddress, ReadHeaderTimeout: httpReadHeaderTimeout, Handler: nil, } - go func() { - select { - case <-ctx.Done(): - sctx, scancel := context.WithTimeout(context.Background(), time.Second) - defer scancel() - _ = srv.Shutdown(sctx) - case <-signal: - } - }() + startHTTPServer(ctx, owner, srv) +} - go func() { - if err := srv.ListenAndServe(); err != nil { - close(signal) - } - }() +func startHTTPServer(ctx context.Context, owner *service.BaseService, srv *http.Server) { + owner.Go(ctx, func(ctx context.Context) { + <-ctx.Done() + _ = srv.Shutdown(context.Background()) + }) + owner.Go(ctx, func(context.Context) { _ = srv.ListenAndServe() }) } diff --git a/privval/signer_client.go b/privval/signer_client.go index dbc00b6596..e56060866f 100644 --- a/privval/signer_client.go +++ b/privval/signer_client.go @@ -46,6 +46,7 @@ func NewSignerClient(ctx context.Context, endpoint *SignerListenerEndpoint, chai // Close closes the underlying connection func (sc *SignerClient) Close() error { sc.endpoint.Stop() + sc.endpoint.Wait() err := sc.endpoint.Close() if err != nil { return err diff --git a/privval/signer_listener_endpoint.go b/privval/signer_listener_endpoint.go index fd1b40dd62..99c16f6fb8 100644 --- a/privval/signer_listener_endpoint.go +++ b/privval/signer_listener_endpoint.go @@ -74,10 +74,15 @@ func (sl *SignerListenerEndpoint) OnStart(ctx context.Context) error { sl.pingInterval = time.Duration(sl.signerEndpoint.timeoutReadWrite.Milliseconds()*2/3) * time.Millisecond sl.pingTimer = time.NewTicker(sl.pingInterval) - go sl.serviceLoop(ctx) - go sl.pingLoop(ctx) + sl.Go(ctx, sl.serviceLoop) + sl.Go(ctx, sl.pingLoop) - sl.connectRequestCh <- struct{}{} + select { + case sl.connectRequestCh <- struct{}{}: + case <-ctx.Done(): + sl.OnStop() + return ctx.Err() + } return nil } @@ -92,7 +97,6 @@ func (sl *SignerListenerEndpoint) OnStop() { if sl.listener != nil { if err := sl.listener.Close(); err != nil { sl.logger.Error("Closing Listener", "err", err) - sl.listener = nil } } @@ -149,7 +153,7 @@ func (sl *SignerListenerEndpoint) ensureConnection(ctx context.Context, maxWait } func (sl *SignerListenerEndpoint) acceptNewConnection() (net.Conn, error) { - if !sl.IsRunning() || sl.listener == nil { + if sl.listener == nil { return nil, fmt.Errorf("endpoint is closing") } @@ -188,6 +192,7 @@ func (sl *SignerListenerEndpoint) serviceLoop(ctx context.Context) { select { case sl.connectionAvailableCh <- conn: case <-ctx.Done(): + _ = conn.Close() return } } diff --git a/privval/signer_listener_endpoint_test.go b/privval/signer_listener_endpoint_test.go index 42a818528d..2a0f7fb35b 100644 --- a/privval/signer_listener_endpoint_test.go +++ b/privval/signer_listener_endpoint_test.go @@ -2,6 +2,7 @@ package privval import ( "context" + "io" "net" "testing" "time" @@ -29,6 +30,39 @@ type dialerTestCase struct { dialer SocketDialer } +type observedListener struct { + net.Listener + accepted chan struct{} +} + +func (l *observedListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err == nil { + close(l.accepted) + } + return conn, err +} + +func TestListenerStopClosesPendingConnection(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + observed := &observedListener{Listener: listener, accepted: make(chan struct{})} + endpoint := NewSignerListenerEndpoint(log.NewNopLogger(), observed, func(endpoint *SignerListenerEndpoint) { + endpoint.timeoutReadWrite = time.Hour + }) + require.NoError(t, endpoint.Start(context.Background())) + t.Cleanup(func() { endpoint.Stop(); endpoint.Wait() }) + conn, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + defer conn.Close() + <-observed.accepted + endpoint.Stop() + endpoint.Wait() + require.NoError(t, conn.SetReadDeadline(time.Now().Add(time.Second))) + _, err = conn.Read(make([]byte, 1)) + require.ErrorIs(t, err, io.EOF) +} + // TestSignerRemoteRetryTCPOnly will test connection retry attempts over TCP. We // don't need this for Unix sockets because the OS instantly knows the state of // both ends of the socket connection. This basically causes the diff --git a/privval/signer_server.go b/privval/signer_server.go index 20554332f3..b9909bea7e 100644 --- a/privval/signer_server.go +++ b/privval/signer_server.go @@ -44,7 +44,7 @@ func NewSignerServer(endpoint *SignerDialerEndpoint, chainID string, privVal typ // OnStart implements service.Service. func (ss *SignerServer) OnStart(ctx context.Context) error { - go ss.serviceLoop(ctx) + ss.Go(ctx, ss.serviceLoop) return nil } @@ -62,7 +62,7 @@ func (ss *SignerServer) SetRequestHandler(validationRequestHandler ValidationReq } func (ss *SignerServer) servicePendingRequest(ctx context.Context) { - if !ss.IsRunning() { + if ctx.Err() != nil { return // Ignore error from closing. } From fad5da34eb90860d10f612bac566aa617a3d3610 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:24:32 +0000 Subject: [PATCH 04/39] fix(mempool): cancel and join all recheck batches during shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close recheck admission under the mempool mutex and wait outside it for active and superseded batches, including application CheckTx and Flush calls. Co-Authored-By: Codex 🤖 Co-authored by [Claudius the Magnificent](https://github.com/lklimek/claudius) AI Agent --- internal/mempool/mempool.go | 26 +++- internal/mempool/recheck_lifecycle_test.go | 155 +++++++++++++++++++++ 2 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 internal/mempool/recheck_lifecycle_test.go diff --git a/internal/mempool/mempool.go b/internal/mempool/mempool.go index 19a07c8916..3332379bcc 100644 --- a/internal/mempool/mempool.go +++ b/internal/mempool/mempool.go @@ -5,6 +5,7 @@ import ( "fmt" "runtime" "sort" + stdsync "sync" "sync/atomic" "time" @@ -61,7 +62,9 @@ type TxMempool struct { txBySender map[string]*clist.CElement // for sender != "" // cancellation function for recheck txs tasks - recheckCancel context.CancelFunc + recheckCancel context.CancelFunc + rechecksStopped bool + recheckWorkers stdsync.WaitGroup } // NewTxMempool constructs a new, empty priority mempool at the specified @@ -96,6 +99,18 @@ func NewTxMempool( return txmp } +// StopRechecks permanently closes recheck admission and joins all admitted batches. +// Call it without holding the mempool lock, before closing the ABCI connection. +func (txmp *TxMempool) StopRechecks() { + txmp.mtx.Lock() + txmp.rechecksStopped = true + if txmp.recheckCancel != nil { + txmp.recheckCancel() + } + txmp.mtx.Unlock() + txmp.recheckWorkers.Wait() +} + // WithPreCheck sets a filter for the mempool to reject a transaction if f(tx) // returns an error. This is executed before CheckTx. It only applies to the // first created block. After that, Update() overwrites the existing value. @@ -762,11 +777,15 @@ func (txmp *TxMempool) handleRecheckResult(tx types.Tx, checkTxRes *abci.Respons // Precondition: The mempool is not empty. // The caller must hold txmp.mtx exclusively. func (txmp *TxMempool) recheckTransactions(ctx context.Context) { + if txmp.rechecksStopped { + return + } // cancel previous recheck if it is still running if txmp.recheckCancel != nil { txmp.recheckCancel() } - ctx, txmp.recheckCancel = context.WithCancel(ctx) + ctx, cancel := context.WithCancel(ctx) + txmp.recheckCancel = cancel if txmp.Size() == 0 { panic("mempool: cannot run recheck on an empty mempool") @@ -785,7 +804,10 @@ func (txmp *TxMempool) recheckTransactions(ctx context.Context) { // Issue CheckTx calls for each remaining transaction, and when all the // rechecks are complete signal watchers that transactions may be available. + txmp.recheckWorkers.Add(1) go func() { + defer txmp.recheckWorkers.Done() + defer cancel() var g errgroup.Group g.SetLimit(2 * runtime.NumCPU()) diff --git a/internal/mempool/recheck_lifecycle_test.go b/internal/mempool/recheck_lifecycle_test.go new file mode 100644 index 0000000000..96a72837fa --- /dev/null +++ b/internal/mempool/recheck_lifecycle_test.go @@ -0,0 +1,155 @@ +package mempool + +import ( + "context" + stdsync "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + abciclient "github.com/dashpay/tenderdash/abci/client" + abci "github.com/dashpay/tenderdash/abci/types" + "github.com/dashpay/tenderdash/config" + "github.com/dashpay/tenderdash/libs/log" +) + +type recheckLifecycleClient struct { + abciclient.Client + check func(context.Context) (*abci.ResponseCheckTx, error) + flush func(context.Context) error +} + +func (c recheckLifecycleClient) CheckTx(ctx context.Context, _ *abci.RequestCheckTx) (*abci.ResponseCheckTx, error) { + return c.check(ctx) +} + +func (c recheckLifecycleClient) Flush(ctx context.Context) error { return c.flush(ctx) } + +func startLifecycleRecheck(t *testing.T, mp *TxMempool) { + t.Helper() + mp.Lock() + defer mp.Unlock() + require.NoError(t, mp.Update(context.Background(), mp.height+1, nil, nil, nil, nil, true)) +} + +func TestStopRechecksJoinsSupersededBatches(t *testing.T) { + entered := make(chan context.Context, 2) + releases := []chan struct{}{make(chan struct{}), make(chan struct{})} + unblockFirst := stdsync.OnceFunc(func() { close(releases[0]) }) + unblockSecond := stdsync.OnceFunc(func() { close(releases[1]) }) + defer unblockFirst() + defer unblockSecond() + var calls atomic.Int32 + client := recheckLifecycleClient{ + check: func(ctx context.Context) (*abci.ResponseCheckTx, error) { + call := calls.Add(1) + if call <= 2 { + entered <- ctx + <-releases[call-1] + } + return nil, context.Canceled + }, + flush: func(context.Context) error { return nil }, + } + mp := NewTxMempool(log.NewNopLogger(), config.DefaultMempoolConfig(), client) + require.NoError(t, mp.addNewTransaction(randomTx(), &abci.ResponseCheckTx{Code: abci.CodeTypeOK})) + startLifecycleRecheck(t, mp) + first := <-entered + startLifecycleRecheck(t, mp) + second := <-entered + require.ErrorIs(t, first.Err(), context.Canceled) + done := make(chan struct{}) + go func() { mp.StopRechecks(); close(done) }() + select { + case <-done: + t.Fatal("shutdown returned while application rechecks were blocked") + case <-second.Done(): + case <-time.After(time.Second): + t.Fatal("shutdown did not cancel the active recheck") + } + unblockSecond() + select { + case <-done: + t.Fatal("shutdown omitted the superseded recheck") + case <-time.After(20 * time.Millisecond): + } + unblockFirst() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("shutdown did not join completed rechecks") + } + startLifecycleRecheck(t, mp) + mp.StopRechecks() + require.EqualValues(t, 2, calls.Load(), "shutdown must reject subsequent recheck admission") +} + +func TestStopRechecksJoinsFlush(t *testing.T) { + entered := make(chan context.Context, 1) + release := make(chan struct{}) + unblock := stdsync.OnceFunc(func() { close(release) }) + defer unblock() + client := recheckLifecycleClient{ + check: func(context.Context) (*abci.ResponseCheckTx, error) { + return &abci.ResponseCheckTx{Code: abci.CodeTypeOK}, nil + }, + flush: func(ctx context.Context) error { entered <- ctx; <-release; return ctx.Err() }, + } + mp := NewTxMempool(log.NewNopLogger(), config.DefaultMempoolConfig(), client) + require.NoError(t, mp.addNewTransaction(randomTx(), &abci.ResponseCheckTx{Code: abci.CodeTypeOK})) + startLifecycleRecheck(t, mp) + ctx := <-entered + done := make(chan struct{}) + go func() { mp.StopRechecks(); close(done) }() + select { + case <-done: + t.Fatal("shutdown returned while application flush was blocked") + case <-ctx.Done(): + case <-time.After(time.Second): + t.Fatal("shutdown did not cancel application flush") + } + unblock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("shutdown did not join application flush") + } +} + +func TestStopRechecksRacesAdmission(t *testing.T) { + for range 32 { + var active atomic.Int32 + client := recheckLifecycleClient{ + check: func(ctx context.Context) (*abci.ResponseCheckTx, error) { + active.Add(1) + defer active.Add(-1) + <-ctx.Done() + return nil, ctx.Err() + }, + flush: func(context.Context) error { return nil }, + } + mp := NewTxMempool(log.NewNopLogger(), config.DefaultMempoolConfig(), client) + require.NoError(t, mp.addNewTransaction(randomTx(), &abci.ResponseCheckTx{Code: abci.CodeTypeOK})) + start := make(chan struct{}) + updated := make(chan error, 1) + stopped := make(chan struct{}) + go func() { + <-start + mp.Lock() + err := mp.Update(context.Background(), 1, nil, nil, nil, nil, true) + mp.Unlock() + updated <- err + }() + go func() { <-start; mp.StopRechecks(); close(stopped) }() + close(start) + require.NoError(t, <-updated) + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("shutdown raced recheck admission without canceling it") + } + require.Zero(t, active.Load(), "an admitted application call survived shutdown") + } +} From 8d16f5645799f5db4d47bc2a49f329fd6d250c8b Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:26:34 +0000 Subject: [PATCH 05/39] fix(service): close remaining shutdown admission and rollback gaps Co-Authored-By: OpenAI GPT-6 --- docs/architecture/service-lifecycle.md | 28 +++++++++ internal/consensus/reactor.go | 39 ++++++++---- internal/consensus/state.go | 10 +-- internal/consensus/ticker.go | 5 +- internal/consensus/ticker_test.go | 28 +++++++++ internal/consensus/wal.go | 9 +-- libs/service/lifecycle_test.go | 30 +++++++++ libs/service/service.go | 24 +++++-- node/node.go | 6 ++ node/seed.go | 10 ++- node/seed_lifecycle_test.go | 86 ++++++++++++++++++++++++++ 11 files changed, 239 insertions(+), 36 deletions(-) create mode 100644 internal/consensus/ticker_test.go create mode 100644 node/seed_lifecycle_test.go diff --git a/docs/architecture/service-lifecycle.md b/docs/architecture/service-lifecycle.md index b6f8f78aee..62e2e1d1ba 100644 --- a/docs/architecture/service-lifecycle.md +++ b/docs/architecture/service-lifecycle.md @@ -18,6 +18,7 @@ Shutdown has three phases: 3. Join admitted work, call optional `OnDrain` to release resources, and unblock `Wait`. +`Stopping()` exposes context cancellation separately from completion. `Stop` does not join workers. Concurrent or recursive `Stop` calls return immediately. A stop during startup cancels the context and schedules `OnStop` after successful `OnStart`; hooks never overlap. `OnStop` must not join managed @@ -48,3 +49,30 @@ its timer/signal worker. An already persisted consensus commit finishes application before the owner releases stores. These explicitly owned resource lifetimes can outlive cancellation of the caller; their owners must still join them. A timeout is not a substitute for completion. + +## Migration coverage + +The service audit covers consensus (state, reactor, ticker, gossip and WAL), +autofile groups, node and seed owners, event bus and pubsub, indexer, block and +state sync, mempool, evidence, P2P router/PEX/connections, quorum event handling, +ABCI clients/servers, proxy wrappers, signer services, and the light RPC wrapper. +Worker-free implementations retain their hooks without artificial workers. + +Service-scoped cancel functions, completion channels and custom `Wait` methods +are replaced by the shared mechanism. Connection/request groups, peer-session +cancellation, restartable RPC broadcast admission and worker-pool synchronization +remain where they represent a separate lifetime. Explicitly owned dependency +contexts also remain where durable application must finish after I/O stops. + +This does not claim a repository-wide join for every plain goroutine. RPC +websocket client reconnection/session helpers are outside `BaseService`; generic +consumer middleware can hide context-canceled, in-memory limiter housekeeping. +Those boundaries need their own ownership API before a service can promise to +join every helper. Router connection/request groups and AutoFile workers are +joined explicitly rather than converted into unrelated service lifetimes. + +Retry of a composite service also requires recreating any children that already +started successfully and were then stopped during rollback. Arbitrary injected +ABCI clients cannot be recreated by the routed wrapper. The node reserves RPC +listeners before starting children, so retry after listener failure remains +supported without restarting successful child lifetimes. diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index f31cb735d2..c2b56bfdbc 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -261,7 +261,8 @@ type Reactor struct { clock clockwork.Clock // Context for controlling reactor goroutines - ctx context.Context + ctx context.Context + peerUpdates *p2p.PeerUpdates } // NewReactor returns a reference to a new consensus reactor, which implements @@ -305,11 +306,13 @@ type channelBundle struct { voteSet p2p.Channel } -// OnStart starts separate go routines for each p2p Channel and listens for -// envelopes on each. In addition, it also listens for peer updates and handles -// messages on that p2p channel accordingly. The caller must be sure to execute -// OnStop to ensure the outbound p2p Channels are closed. -func (r *Reactor) OnStart(ctx context.Context) error { +// OnStart starts channel, peer-update and gossip workers owned by the reactor. +func (r *Reactor) OnStart(ctx context.Context) (err error) { + defer func() { + if err != nil { + r.closeOwnedHelpers() + } + }() r.logger.Trace("consensus wait sync", "wait_sync", r.WaitSync()) r.ctx = ctx @@ -347,9 +350,9 @@ func (r *Reactor) OnStart(ctx context.Context) error { r.maj23SurplusLimit = newMaj23SurplusLimiter() peerUpdates := r.peerEvents(r.ctx, "consensus") + r.peerUpdates = peerUpdates var chBundle channelBundle - var err error chans := p2p.ConsensusChannelDescriptors() chBundle.state, err = r.chCreator(r.ctx, chans[p2p.ConsensusStateChannel]) @@ -411,7 +414,22 @@ func (r *Reactor) OnStart(ctx context.Context) error { func (r *Reactor) OnStop() { r.state.Stop() } // OnDrain joins consensus after channel and peer workers have finished. -func (r *Reactor) OnDrain() { r.state.Stop(); r.state.Wait() } +func (r *Reactor) OnDrain() { + r.state.Stop() + r.state.Wait() + r.closeOwnedHelpers() +} + +func (r *Reactor) closeOwnedHelpers() { + if r.peerUpdates != nil { + r.peerUpdates.Close() + } + for _, limiter := range []*client.RateLimit{r.voteRateLimit, r.dataRateLimit, r.stateRateLimit, r.maj23PeerShare} { + if limiter != nil { + limiter.Close() + } + } +} // WaitSync returns whether the consensus reactor is waiting for state/block sync. func (r *Reactor) WaitSync() bool { @@ -598,10 +616,7 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda switch peerUpdate.Status { case p2p.PeerStatusUp: - // Do not allow starting new broadcasting goroutines after reactor shutdown - // has been initiated. This can happen after we've manually closed all - // peer goroutines, but the router still sends in-flight peer updates. - if !r.IsRunning() { + if ctx.Err() != nil { return } r.peerUp(ctx, peerUpdate, chans) diff --git a/internal/consensus/state.go b/internal/consensus/state.go index d10a9e2843..9075b67336 100644 --- a/internal/consensus/state.go +++ b/internal/consensus/state.go @@ -191,8 +191,6 @@ type State struct { // proposer's latest available app protocol version that goes to block header proposedAppVersion uint64 - // wait the channel event happening for shutting down the state gracefully - msgInfoQueue *msgInfoQueue msgDispatcher *msgInfoDispatcher blockExecutor *blockExecutor @@ -584,12 +582,8 @@ func (cs *State) loadWalFile(ctx context.Context) error { // OnStop requests timeout worker shutdown. Wait also joins receiveRoutine. func (cs *State) OnStop() { cs.timeoutTicker.Stop() } -// OnDrain joins child services before callers release consensus stores. -func (cs *State) OnDrain() { - cs.timeoutTicker.Wait() - cs.wal.Stop() - cs.wal.Wait() -} +// OnDrain joins the timeout worker before callers release consensus stores. +func (cs *State) OnDrain() { cs.timeoutTicker.Wait() } // OpenWAL opens a file to log all consensus messages and timeouts for // deterministic accountability. diff --git a/internal/consensus/ticker.go b/internal/consensus/ticker.go index 80b7f56ff4..95c42f83fe 100644 --- a/internal/consensus/ticker.go +++ b/internal/consensus/ticker.go @@ -73,7 +73,10 @@ func (t *timeoutTicker) Chan() <-chan timeoutInfo { // The timeoutRoutine is always available to read from tickChan, so this won't block. // The scheduling may fail if the timeoutRoutine has already scheduled a timeout for a later height/round/step. func (t *timeoutTicker) ScheduleTimeout(ti timeoutInfo) { - t.tickChan <- ti + select { + case t.tickChan <- ti: + case <-t.Stopping(): + } } //------------------------------------------------------------- diff --git a/internal/consensus/ticker_test.go b/internal/consensus/ticker_test.go new file mode 100644 index 0000000000..4c44c9da0b --- /dev/null +++ b/internal/consensus/ticker_test.go @@ -0,0 +1,28 @@ +package consensus + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" +) + +func TestScheduleTimeoutDoesNotBlockAfterShutdown(t *testing.T) { + ticker := NewTimeoutTicker(log.NewNopLogger()).(*timeoutTicker) + require.NoError(t, ticker.Start(context.Background())) + ticker.Stop() + ticker.Wait() + for range cap(ticker.tickChan) { + ticker.tickChan <- timeoutInfo{} + } + done := make(chan struct{}) + go func() { ticker.ScheduleTimeout(timeoutInfo{}); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("timeout scheduling blocked after ticker shutdown") + } +} diff --git a/internal/consensus/wal.go b/internal/consensus/wal.go index 5dd2658d3c..31c19dfccc 100644 --- a/internal/consensus/wal.go +++ b/internal/consensus/wal.go @@ -182,14 +182,7 @@ func (wal *BaseWAL) OnStop() { } // OnDrain flushes and closes files after the flush worker has exited. -func (wal *BaseWAL) OnDrain() { - if err := wal.FlushAndSync(); err != nil { - wal.logger.Error("error on flush data to disk", "error", err) - } - wal.group.Stop() - wal.group.Wait() - wal.group.Close() -} +func (wal *BaseWAL) OnDrain() { wal.group.Close() } // Write is called in newStep and for each receive on the // peerMsgQueue and the timeoutTicker. diff --git a/libs/service/lifecycle_test.go b/libs/service/lifecycle_test.go index 2a26047a1c..4a0b2479dc 100644 --- a/libs/service/lifecycle_test.go +++ b/libs/service/lifecycle_test.go @@ -236,3 +236,33 @@ func TestStartContextPreservesValuesAndDeadline(t *testing.T) { s.Wait() require.NoError(t, parent.Err()) } + +func TestFailedStartupRacingStopSkipsShutdownHooks(t *testing.T) { + entered, cleaning, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + s := &finalizingService{lifecycleService: newLifecycleService(t)} + s.BaseService = NewBaseService(log.NewNopLogger(), t.Name(), s) + s.stop = func() { t.Error("OnStop called for failed startup") } + s.drain = func() { t.Error("OnDrain called for failed startup") } + s.start = func(ctx context.Context) error { + s.Go(ctx, func(ctx context.Context) { <-ctx.Done(); close(cleaning); <-release }) + close(entered) + <-ctx.Done() + return context.DeadlineExceeded + } + started := make(chan error, 1) + go func() { started <- s.Start(context.Background()) }() + <-entered + s.Stop() + <-cleaning + waited := make(chan struct{}) + go func() { s.Wait(); close(waited) }() + assertPending(t, waited) + if len(started) != 0 { + t.Error("failed Start returned before cleanup") + } + close(release) + require.ErrorIs(t, <-started, context.DeadlineExceeded) + <-waited + s.Wait() + require.False(t, s.IsRunning()) +} diff --git a/libs/service/service.go b/libs/service/service.go index e80f8d818d..32ca8b99e7 100644 --- a/libs/service/service.go +++ b/libs/service/service.go @@ -41,11 +41,12 @@ type Finalizer interface { type attemptKey struct{} type serviceAttempt struct { - cancel context.CancelFunc - done chan struct{} - workers stdsync.WaitGroup - starting bool - stopping bool + cancel context.CancelFunc + stoppingCh <-chan struct{} + done chan struct{} + workers stdsync.WaitGroup + starting bool + stopping bool } // BaseService owns a service context and work registered through Go. Stop cancels @@ -83,7 +84,7 @@ func (bs *BaseService) Start(ctx context.Context) error { return err } workCtx, cancel := context.WithCancel(ctx) - a := &serviceAttempt{cancel: cancel, done: make(chan struct{}), starting: true} + a := &serviceAttempt{cancel: cancel, stoppingCh: workCtx.Done(), done: make(chan struct{}), starting: true} bs.attempt = a workCtx = context.WithValue(workCtx, attemptKey{}, a) bs.mtx.Unlock() @@ -180,6 +181,17 @@ func (bs *BaseService) finishStop(a *serviceAttempt) { // not imply cleanup has completed; use Wait before releasing owned resources. func (bs *BaseService) IsRunning() bool { return atomic.LoadUint32(&bs.running) == 1 } +// Stopping signals context cancellation, not completed shutdown. It is nil +// before Start and after a failed attempt has drained; use Wait for completion. +func (bs *BaseService) Stopping() <-chan struct{} { + bs.mtx.Lock() + defer bs.mtx.Unlock() + if bs.attempt == nil { + return nil + } + return bs.attempt.stoppingCh +} + // Wait joins the current startup attempt, shutdown hooks and registered work. // It returns immediately before Start or after a failed Start has drained. func (bs *BaseService) Wait() { diff --git a/node/node.go b/node/node.go index 1fb1838769..5103fa9880 100644 --- a/node/node.go +++ b/node/node.go @@ -521,6 +521,9 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { if routerStarted { n.router.Wait() } + if mp, ok := n.rpcEnv.Mempool.(*mempool.TxMempool); ok { + mp.StopRechecks() + } n.rpcEnv.EventBus.Stop() n.rpcEnv.EventBus.Wait() dependencyCancel() @@ -673,6 +676,9 @@ func (n *nodeImpl) OnStop() { } reactor.Wait() } + if mp, ok := n.rpcEnv.Mempool.(*mempool.TxMempool); ok { + mp.StopRechecks() + } n.rpcEnv.EventBus.Stop() n.rpcEnv.EventBus.Wait() n.dependencyCancel() diff --git a/node/seed.go b/node/seed.go index 450b79cc19..b78e494fcf 100644 --- a/node/seed.go +++ b/node/seed.go @@ -97,7 +97,15 @@ func makeSeedNode( } // OnStart starts the Seed Node. It implements service.Service. -func (n *seedNodeImpl) OnStart(ctx context.Context) error { +func (n *seedNodeImpl) OnStart(ctx context.Context) (err error) { + ctx, cancelAttempt := context.WithCancel(ctx) + defer func() { + if err != nil { + cancelAttempt() + n.pexReactor.Wait() + n.router.Wait() + } + }() if n.config.RPC.PprofListenAddress != "" { startPProfServer(ctx, &n.BaseService, *n.config.RPC) } diff --git a/node/seed_lifecycle_test.go b/node/seed_lifecycle_test.go new file mode 100644 index 0000000000..4f9a65ea13 --- /dev/null +++ b/node/seed_lifecycle_test.go @@ -0,0 +1,86 @@ +package node + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/config" + "github.com/dashpay/tenderdash/internal/p2p" + "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" + "github.com/dashpay/tenderdash/types" +) + +type seedRouterDrain struct { + router *p2p.Router + stopping chan struct{} + release chan struct{} +} + +func (r *seedRouterDrain) OnStart(ctx context.Context) error { + if err := r.router.OnStart(ctx); err != nil { + return err + } + r.router.Go(ctx, func(ctx context.Context) { + <-ctx.Done() + close(r.stopping) + <-r.release + }) + return nil +} +func (r *seedRouterDrain) OnStop() { r.router.OnStop() } + +type failedSeedPEX struct { + service.BaseService + err error +} + +func (p *failedSeedPEX) OnStart(context.Context) error { return p.err } +func (p *failedSeedPEX) OnStop() {} + +func TestSeedFailedStartDrainsRouter(t *testing.T) { + cfg, err := config.ResetTestRoot(t.TempDir(), t.Name()) + require.NoError(t, err) + cfg.Mode = config.ModeSeed + cfg.P2P.ListenAddress = "tcp://127.0.0.1:0" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + key, err := types.LoadOrGenNodeKey(cfg.NodeKeyFile()) + require.NoError(t, err) + svc, err := makeSeedNode(ctx, log.NewNopLogger(), cfg, config.DefaultDBProvider, key, defaultGenesisDocProviderFunc(cfg)) + require.NoError(t, err) + n := svc.(*seedNodeImpl) + defer func() { require.NoError(t, n.shutdownOps()) }() + hooks := &seedRouterDrain{router: n.router, stopping: make(chan struct{}), release: make(chan struct{})} + n.router.BaseService = service.NewBaseService(log.NewNopLogger(), "test-router", hooks) + failure := errors.New("PEX startup failed") + pex := &failedSeedPEX{err: failure} + pex.BaseService = *service.NewBaseService(log.NewNopLogger(), "test-pex", pex) + n.pexReactor = pex + returned := make(chan error, 1) + go func() { returned <- n.Start(ctx) }() + var release sync.Once + defer func() { release.Do(func() { close(hooks.release) }); n.router.Wait(); n.Wait() }() + select { + case <-hooks.stopping: + case <-time.After(5 * time.Second): + t.Fatal("router was not canceled") + } + select { + case err := <-returned: + t.Fatalf("Start returned before router cleanup: %v", err) + case <-time.After(50 * time.Millisecond): + } + release.Do(func() { close(hooks.release) }) + select { + case err := <-returned: + require.ErrorIs(t, err, failure) + case <-time.After(5 * time.Second): + t.Fatal("Start did not finish after router drained") + } +} From 82d312f517fa353a4471822e8195c7afa3d3b9d9 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:27:12 +0000 Subject: [PATCH 06/39] fix(rpc): drain handlers and websocket sessions before shutdown --- internal/rpc/core/env.go | 79 +++++++++++++++-- internal/rpc/core/events.go | 13 ++- internal/rpc/core/lifecycle_test.go | 60 +++++++++++++ light/rpc/client.go | 22 ++++- rpc/jsonrpc/server/http_server.go | 93 ++++++++++---------- rpc/jsonrpc/server/http_server_test.go | 56 ++++++++++++ rpc/jsonrpc/server/ws_handler.go | 115 +++++++++++++++---------- rpc/jsonrpc/server/ws_handler_test.go | 108 +++++++++++++++++++++++ rpc/jsonrpc/types/types.go | 2 + 9 files changed, 442 insertions(+), 106 deletions(-) create mode 100644 internal/rpc/core/lifecycle_test.go diff --git a/internal/rpc/core/env.go b/internal/rpc/core/env.go index 290034e122..c7a280ec99 100644 --- a/internal/rpc/core/env.go +++ b/internal/rpc/core/env.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "net/http" + "sync" "time" "github.com/rs/cors" @@ -28,6 +29,7 @@ import ( "github.com/dashpay/tenderdash/internal/state/indexer" "github.com/dashpay/tenderdash/internal/statesync" "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" "github.com/dashpay/tenderdash/rpc/coretypes" rpcserver "github.com/dashpay/tenderdash/rpc/jsonrpc/server" "github.com/dashpay/tenderdash/types" @@ -91,6 +93,8 @@ type Environment struct { Config config.RPCConfig asyncBroadcasts asyncBroadcasts + rpcMu sync.Mutex + rpcService *rpcService // cache of chunked genesis data. genChunks []string @@ -227,6 +231,62 @@ func ListenRPC(conf *config.RPCConfig) ([]net.Listener, error) { // StartService serves RPC on already bound listeners until ctx is canceled. // If startup fails, the caller remains responsible for closing the listeners. func (env *Environment) StartService(ctx context.Context, conf *config.Config, listeners []net.Listener) error { + env.rpcMu.Lock() + if env.rpcService != nil { + env.rpcMu.Unlock() + return errors.New("RPC service already started") + } + ctx, cancel := context.WithCancel(ctx) + owner := &rpcService{env: env, conf: conf, listeners: listeners, cancel: cancel, ready: make(chan struct{})} + owner.BaseService = service.NewBaseService(env.Logger, "RPC", owner) + env.rpcService = owner + env.rpcMu.Unlock() + err := owner.Start(ctx) + close(owner.ready) + if err != nil { + cancel() + env.rpcMu.Lock() + if env.rpcService == owner { + env.rpcService = nil + } + env.rpcMu.Unlock() + } + return err +} + +// StopService cancels and joins RPC requests, websocket sessions, and event-log +// forwarding. Call it before releasing the dependencies used by RPC handlers. +func (env *Environment) StopService() { + env.rpcMu.Lock() + owner := env.rpcService + env.rpcMu.Unlock() + if owner == nil { + return + } + owner.cancel() + <-owner.ready + owner.Stop() + owner.Wait() + env.rpcMu.Lock() + if env.rpcService == owner { + env.rpcService = nil + } + env.rpcMu.Unlock() +} + +type rpcService struct { + *service.BaseService + env *Environment + conf *config.Config + listeners []net.Listener + cancel context.CancelFunc + ready chan struct{} +} + +func (*rpcService) OnStop() {} + +func (owner *rpcService) OnStart(ctx context.Context) error { + env, conf, listeners := owner.env, owner.conf, owner.listeners if err := env.InitGenesisChunks(); err != nil { return err } @@ -267,7 +327,7 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config, l if err != nil { return fmt.Errorf("event log subscribe: %w", err) } - go func() { + if !owner.Go(ctx, func(ctx context.Context) { // N.B. Use background for unsubscribe, ctx is already terminated. defer env.EventBus.UnsubscribeAll(context.Background(), subscriberID) //nolint:errcheck for { @@ -281,7 +341,10 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config, l _ = lg.Add(etype, msg.Data()) } } - }() + }) { + _ = env.EventBus.UnsubscribeAll(context.Background(), subscriberID) + return errors.New("RPC service is stopping") + } env.Logger.Info("Event log subscription enabled") } @@ -321,7 +384,7 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config, l rootHandler = corsMiddleware.Handler(mux) } if conf.RPC.IsTLSEnabled() { - go func() { + if !owner.Go(ctx, func(ctx context.Context) { if err := rpcserver.ServeTLS( ctx, listener, @@ -333,9 +396,11 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config, l ); err != nil { env.Logger.Error("error serving server with TLS", "err", err) } - }() + }) { + return errors.New("RPC service is stopping") + } } else { - go func() { + if !owner.Go(ctx, func(ctx context.Context) { if err := rpcserver.Serve( ctx, listener, @@ -345,7 +410,9 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config, l ); err != nil { env.Logger.Error("error serving server", "err", err) } - }() + }) { + return errors.New("RPC service is stopping") + } } } diff --git a/internal/rpc/core/events.go b/internal/rpc/core/events.go index da8e6088ee..58d565530c 100644 --- a/internal/rpc/core/events.go +++ b/internal/rpc/core/events.go @@ -61,9 +61,7 @@ func (env *Environment) Subscribe(ctx context.Context, req *coretypes.RequestSub // Capture the current ID, since it can change in the future. subscriptionID := callInfo.RPCRequest.ID - go func() { - opctx, opcancel := context.WithCancel(context.TODO()) - defer opcancel() + if !callInfo.WSConn.Go(func(opctx context.Context) { for { msg, err := sub.Next(opctx) @@ -81,6 +79,10 @@ func (env *Environment) Subscribe(ctx context.Context, req *coretypes.RequestSub return } + if err != nil { + return + } + // We have a message to deliver to the client. resp := callInfo.RPCRequest.MakeResponse(&coretypes.ResultEvent{ Query: req.Query, @@ -95,7 +97,10 @@ func (env *Environment) Subscribe(ctx context.Context, req *coretypes.RequestSub "to", addr, "subscriptionID", subscriptionID, "err", err) } } - }() + }) { + _ = env.EventBus.UnsubscribeAll(context.Background(), addr) + return nil, errors.New("websocket session is stopping") + } return &coretypes.ResultSubscribe{}, nil } diff --git a/internal/rpc/core/lifecycle_test.go b/internal/rpc/core/lifecycle_test.go new file mode 100644 index 0000000000..5e5fdd11d8 --- /dev/null +++ b/internal/rpc/core/lifecycle_test.go @@ -0,0 +1,60 @@ +package core + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + abci "github.com/dashpay/tenderdash/abci/types" + "github.com/dashpay/tenderdash/config" + "github.com/dashpay/tenderdash/internal/eventbus" + "github.com/dashpay/tenderdash/internal/eventlog" + "github.com/dashpay/tenderdash/internal/pubsub" + "github.com/dashpay/tenderdash/internal/pubsub/query" + "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/types" +) + +type blockedLogEvent struct { + types.EventDataNewBlockHeader + entered chan struct{} + release chan struct{} +} + +func (e blockedLogEvent) ABCIEvents() []abci.Event { + close(e.entered) + <-e.release + return nil +} + +func TestRPCServiceRetryAndEventLogDrain(t *testing.T) { + logger := log.NewNopLogger() + bus := eventbus.NewDefault(logger) + require.NoError(t, bus.Start(context.Background())) + defer func() { bus.Stop(); bus.Wait() }() + eventLog, err := eventlog.New(eventlog.LogSettings{WindowSize: time.Minute}) + require.NoError(t, err) + env := &Environment{Logger: logger, EventBus: bus, EventLog: eventLog} + conf := config.DefaultConfig() + // A failed subscription must leave the RPC owner reusable. + _, err = bus.SubscribeWithArgs(context.Background(), pubsub.SubscribeArgs{ClientID: "event-log-subscriber", Query: query.All}) + require.NoError(t, err) + require.Error(t, env.StartService(context.Background(), conf, nil)) + require.NoError(t, bus.UnsubscribeAll(context.Background(), "event-log-subscriber")) + require.NoError(t, env.StartService(context.Background(), conf, nil)) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + require.NoError(t, bus.Publish(types.EventNewBlockHeaderValue, blockedLogEvent{entered: entered, release: release})) + <-entered + go func() { env.StopService(); close(done) }() + select { + case <-done: + close(release) + t.Fatal("RPC shutdown returned while event-log forwarding was active") + case <-time.After(100 * time.Millisecond): + } + close(release) + <-done + require.Zero(t, bus.NumClientSubscriptions("event-log-subscriber")) +} diff --git a/light/rpc/client.go b/light/rpc/client.go index 6d949686d9..4c9c454e38 100644 --- a/light/rpc/client.go +++ b/light/rpc/client.go @@ -654,18 +654,28 @@ func (c *Client) SubscribeWS(ctx context.Context, query string) (*coretypes.Resu if !c.IsRunning() { return nil, fmt.Errorf("light RPC client is not running") } - bctx := c.serviceCtx + bctx, cancel := context.WithCancel(c.serviceCtx) callInfo := rpctypes.GetCallInfo(ctx) + stopCancel := context.AfterFunc(callInfo.WSConn.Context(), cancel) out, err := c.next.Subscribe(bctx, callInfo.RemoteAddr(), query) //nolint:staticcheck if err != nil { + stopCancel() + cancel() return nil, err } + done := make(chan struct{}) if !c.Go(bctx, func(bctx context.Context) { + defer close(done) + defer cancel() + defer stopCancel() for { select { - case resultEvent := <-out: + case resultEvent, ok := <-out: + if !ok { + return + } // We should have a switch here that performs a validation // depending on the event's type. callInfo.WSConn.TryWriteRPCResponse(bctx, callInfo.RPCRequest.MakeResponse(resultEvent)) @@ -674,9 +684,17 @@ func (c *Client) SubscribeWS(ctx context.Context, query string) (*coretypes.Resu } } }) { + stopCancel() + cancel() return nil, fmt.Errorf("light RPC client is stopping") } + if !callInfo.WSConn.Go(func(context.Context) { <-done }) { + cancel() + <-done + return nil, fmt.Errorf("websocket session is stopping") + } + return &coretypes.ResultSubscribe{}, nil } diff --git a/rpc/jsonrpc/server/http_server.go b/rpc/jsonrpc/server/http_server.go index 4dffa02961..7fbf1d8ba2 100644 --- a/rpc/jsonrpc/server/http_server.go +++ b/rpc/jsonrpc/server/http_server.go @@ -11,6 +11,7 @@ import ( "net/http" "runtime/debug" "strings" + "sync" "time" "golang.org/x/net/netutil" @@ -59,65 +60,61 @@ func DefaultConfig() *Config { // Serve creates a http.Server and calls Serve with the given listener. It // wraps handler to recover panics and limit the request body size. func Serve(ctx context.Context, listener net.Listener, handler http.Handler, logger log.Logger, config *Config) error { - logger.Info("Starting RPC HTTP server on", "addr", listener.Addr()) - h := recoverAndLogHandler(MaxBytesHandler(handler, config.MaxBodyBytes), logger) - s := &http.Server{ - Handler: h, - ReadTimeout: config.ReadTimeout, - WriteTimeout: config.WriteTimeout, - MaxHeaderBytes: config.MaxHeaderBytes, - } - sig := make(chan struct{}) - go func() { - select { - case <-ctx.Done(): - sctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - _ = s.Shutdown(sctx) - case <-sig: - } - }() - - if err := s.Serve(listener); err != nil { - logger.Info("RPC HTTP server stopped", "err", err) - close(sig) - return err - } - return nil + return serve(ctx, handler, logger, config, func(s *http.Server) error { + return s.Serve(listener) + }) } -// Serve creates a http.Server and calls ServeTLS with the given listener, -// certFile and keyFile. It wraps handler to recover panics and limit the -// request body size. +// ServeTLS serves HTTPS and joins all requests, including websocket sessions, +// before returning. Cancellation is propagated to request contexts. func ServeTLS(ctx context.Context, listener net.Listener, handler http.Handler, certFile, keyFile string, logger log.Logger, config *Config) error { - logger.Info("Starting RPC HTTPS server", - "listenterAddr", listener.Addr(), - "certFile", certFile, - "keyFile", keyFile) + return serve(ctx, handler, logger, config, func(s *http.Server) error { + return s.ServeTLS(listener, certFile, keyFile) + }) +} +func serve(ctx context.Context, handler http.Handler, logger log.Logger, config *Config, run func(*http.Server) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + var mu sync.Mutex + var requests sync.WaitGroup + stopping := false + h := recoverAndLogHandler(MaxBytesHandler(handler, config.MaxBodyBytes), logger) s := &http.Server{ - Handler: recoverAndLogHandler(MaxBytesHandler(handler, config.MaxBodyBytes), logger), + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Shutdown does not join hijacked connections. Keep their handlers in + // this group, and serialize admission with shutdown's call to Wait. + mu.Lock() + if stopping { + mu.Unlock() + http.Error(w, "RPC server is stopping", http.StatusServiceUnavailable) + return + } + requests.Add(1) + mu.Unlock() + defer requests.Done() + h.ServeHTTP(w, r) + }), + BaseContext: func(net.Listener) context.Context { return ctx }, ReadTimeout: config.ReadTimeout, WriteTimeout: config.WriteTimeout, MaxHeaderBytes: config.MaxHeaderBytes, } - sig := make(chan struct{}) + drained := make(chan struct{}) go func() { - select { - case <-ctx.Done(): - sctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - _ = s.Shutdown(sctx) - case <-sig: - } + defer close(drained) + <-ctx.Done() + mu.Lock() + stopping = true + mu.Unlock() + _ = s.Shutdown(context.Background()) + requests.Wait() }() - - if err := s.ServeTLS(listener, certFile, keyFile); err != nil { - logger.Error("RPC HTTPS server stopped", "err", err) - close(sig) - return err - } - return nil + err := run(s) + cancel() + <-drained + logger.Info("RPC server stopped", "err", err) + return err } // writeInternalError writes an internal server error (500) to w with the text diff --git a/rpc/jsonrpc/server/http_server_test.go b/rpc/jsonrpc/server/http_server_test.go index f89fd4786b..0043c97eeb 100644 --- a/rpc/jsonrpc/server/http_server_test.go +++ b/rpc/jsonrpc/server/http_server_test.go @@ -182,3 +182,59 @@ func TestWriteHTTPResponse(t *testing.T) { assert.Equal(t, "application/json", resp.Header.Get("Content-Type")) assert.Equal(t, `{"code":-32603,"message":"Internal error","data":"foo"}`, string(body)) } + +func TestServeJoinsActiveHandler(t *testing.T) { + for _, useTLS := range []bool{false, true} { + t.Run(fmt.Sprintf("TLS=%v", useTLS), func(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + type contextKey struct{} + ctx, cancel := context.WithCancel(context.WithValue(context.Background(), contextKey{}, "inherited")) + defer cancel() + entered, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + handler := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + assert.Equal(t, "inherited", r.Context().Value(contextKey{})) + close(entered) + <-r.Context().Done() + close(canceled) + <-release + }) + go func() { + if useTLS { + done <- ServeTLS(ctx, listener, handler, "test.crt", "test.key", log.NewNopLogger(), DefaultConfig()) + } else { + done <- Serve(ctx, listener, handler, log.NewNopLogger(), DefaultConfig()) + } + }() + transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport} + scheme := "http://" + if useTLS { + scheme = "https://" + } + requestDone := make(chan struct{}) + go func() { + defer close(requestDone) + resp, err := client.Get(scheme + listener.Addr().String()) + if err == nil { + _ = resp.Body.Close() + } + }() + <-entered + cancel() + <-canceled + select { + case err := <-done: + close(release) + <-requestDone + t.Fatalf("Serve returned before active handler completed: %v", err) + case <-time.After(100 * time.Millisecond): + } + close(release) + require.ErrorIs(t, <-done, http.ErrServerClosed) + <-requestDone + }) + } +} diff --git a/rpc/jsonrpc/server/ws_handler.go b/rpc/jsonrpc/server/ws_handler.go index 9ea70fdabe..da60095526 100644 --- a/rpc/jsonrpc/server/ws_handler.go +++ b/rpc/jsonrpc/server/ws_handler.go @@ -8,6 +8,7 @@ import ( "net/url" "runtime/debug" "strings" + "sync" "time" "github.com/gorilla/websocket" @@ -178,10 +179,6 @@ type wsConnection struct { // writeChan is never closed, to allow WriteRPCResponse() to fail. writeChan chan rpctypes.RPCResponse - // chan, which is closed when/if readRoutine errors - // used to abort writeRoutine - readRoutineQuit chan struct{} - funcMap map[string]*RPCFunc // Connection times out if we haven't received *anything* in this long, not even pings. @@ -196,8 +193,11 @@ type wsConnection struct { // callback which is called upon disconnect onDisconnect func(remoteAddr string) - ctx context.Context - cancel context.CancelFunc + ctx context.Context + cancel context.CancelFunc + workerMu sync.Mutex + workers sync.WaitGroup + stopping bool } // NewWSConnection wraps websocket.Conn. @@ -208,13 +208,12 @@ type wsConnection struct { // disconnect. see https://github.com/gorilla/websocket/issues/97 func newWSConnection(baseConn *websocket.Conn, funcMap map[string]*RPCFunc, logger log.Logger, options ...func(*wsConnection)) *wsConnection { wsc := &wsConnection{ - Logger: logger, - remoteAddr: baseConn.RemoteAddr().String(), - baseConn: baseConn, - funcMap: funcMap, - readWait: defaultWSReadWait, - pingPeriod: defaultWSPingPeriod, - readRoutineQuit: make(chan struct{}), + Logger: logger, + remoteAddr: baseConn.RemoteAddr().String(), + baseConn: baseConn, + funcMap: funcMap, + readWait: defaultWSReadWait, + pingPeriod: defaultWSPingPeriod, } for _, option := range options { option(wsc) @@ -257,27 +256,66 @@ func ReadLimit(readLimit int64) func(*wsConnection) { // Start starts the client service routines and blocks until there is an error. func (wsc *wsConnection) Start(ctx context.Context) error { + wsc.ctx, wsc.cancel = context.WithCancel(ctx) wsc.writeChan = make(chan rpctypes.RPCResponse, defaultWSWriteChanCapacity) + // https://github.com/gorilla/websocket/issues/97 + pongs := make(chan string, 1) + wsc.baseConn.SetPingHandler(func(m string) error { + select { + case pongs <- m: + default: + } + return nil + }) - // Read subscriptions/unsubscriptions to events - go wsc.readRoutine(ctx) - // Write responses, BLOCKING. - wsc.writeRoutine(ctx) - + readDone, closeDone := make(chan struct{}), make(chan struct{}) + go func() { + defer close(readDone) + defer wsc.cancel() + // A handler panic is isolated to its request; retry on this same reader. + for wsc.ctx.Err() == nil && wsc.readRoutine(wsc.ctx) { + } + }() + go func() { + defer close(closeDone) + <-wsc.ctx.Done() + _ = wsc.baseConn.Close() + }() + wsc.writeRoutine(wsc.ctx, pongs) + wsc.cancel() + <-closeDone + <-readDone + wsc.workerMu.Lock() + wsc.stopping = true + wsc.workerMu.Unlock() + wsc.workers.Wait() + if wsc.onDisconnect != nil { + wsc.onDisconnect(wsc.remoteAddr) + } return nil } -// Stop unsubscribes the remote from all subscriptions. +// Stop cancels the session without joining, so it is safe inside a handler. +// Start joins the reader and performs subscription cleanup before returning. func (wsc *wsConnection) Stop() error { - if wsc.onDisconnect != nil { - wsc.onDisconnect(wsc.remoteAddr) - } - if wsc.ctx != nil { + if wsc.cancel != nil { wsc.cancel() } return nil } +// Go starts session work that Start joins before subscription cleanup. +func (wsc *wsConnection) Go(fn func(context.Context)) bool { + wsc.workerMu.Lock() + defer wsc.workerMu.Unlock() + if wsc.stopping || wsc.ctx.Err() != nil { + return false + } + wsc.workers.Add(1) + go func() { defer wsc.workers.Done(); fn(wsc.ctx) }() + return true +} + // GetRemoteAddr returns the remote address of the underlying connection. // It implements WSRPCConnection func (wsc *wsConnection) GetRemoteAddr() string { @@ -289,6 +327,8 @@ func (wsc *wsConnection) GetRemoteAddr() string { // It implements WSRPCConnection. It is Goroutine-safe. func (wsc *wsConnection) WriteRPCResponse(ctx context.Context, resp rpctypes.RPCResponse) error { select { + case <-wsc.ctx.Done(): + return wsc.ctx.Err() case <-ctx.Done(): return ctx.Err() case wsc.writeChan <- resp: @@ -313,17 +353,13 @@ func (wsc *wsConnection) TryWriteRPCResponse(ctx context.Context, resp rpctypes. // Context returns the connection's context. // The context is canceled when the client's connection closes. func (wsc *wsConnection) Context() context.Context { - if wsc.ctx != nil { - return wsc.ctx - } - wsc.ctx, wsc.cancel = context.WithCancel(context.Background()) return wsc.ctx } // Read from the socket and subscribe to or unsubscribe from events -func (wsc *wsConnection) readRoutine(ctx context.Context) { +func (wsc *wsConnection) readRoutine(ctx context.Context) (retry bool) { // readRoutine will block until response is written or WS connection is closed - writeCtx := context.Background() + writeCtx := ctx defer func() { if r := recover(); r != nil { @@ -337,7 +373,7 @@ func (wsc *wsConnection) readRoutine(ctx context.Context) { req.MakeErrorf(rpctypes.CodeInternalError, "Panic in handler: %v", err)); err != nil { wsc.Logger.Error("error writing RPC response", "err", err) } - go wsc.readRoutine(ctx) + retry = true } }() @@ -348,7 +384,7 @@ func (wsc *wsConnection) readRoutine(ctx context.Context) { for { select { case <-ctx.Done(): - return + return false default: // reset deadline for every type of message (control or data) if err := wsc.baseConn.SetReadDeadline(time.Now().Add(wsc.readWait)); err != nil { @@ -365,8 +401,7 @@ func (wsc *wsConnection) readRoutine(ctx context.Context) { if err := wsc.Stop(); err != nil { wsc.Logger.Error("error closing websocket connection", "err", err) } - close(wsc.readRoutineQuit) - return + return false } dec := json.NewDecoder(r) @@ -419,26 +454,14 @@ func (wsc *wsConnection) readRoutine(ctx context.Context) { } // receives on a write channel and writes out on the socket -func (wsc *wsConnection) writeRoutine(ctx context.Context) { +func (wsc *wsConnection) writeRoutine(ctx context.Context, pongs <-chan string) { pingTicker := time.NewTicker(wsc.pingPeriod) defer pingTicker.Stop() - // https://github.com/gorilla/websocket/issues/97 - pongs := make(chan string, 1) - wsc.baseConn.SetPingHandler(func(m string) error { - select { - case pongs <- m: - default: - } - return nil - }) - for { select { case <-ctx.Done(): return - case <-wsc.readRoutineQuit: // error in readRoutine - return case m := <-pongs: err := wsc.writeMessageWithDeadline(websocket.PongMessage, []byte(m)) if err != nil { diff --git a/rpc/jsonrpc/server/ws_handler_test.go b/rpc/jsonrpc/server/ws_handler_test.go index 8bfcfbbc69..d91847a120 100644 --- a/rpc/jsonrpc/server/ws_handler_test.go +++ b/rpc/jsonrpc/server/ws_handler_test.go @@ -3,9 +3,11 @@ package server import ( "context" "encoding/json" + "net" "net/http" "net/http/httptest" "testing" + "time" "github.com/fortytw2/leaktest" "github.com/gorilla/websocket" @@ -63,3 +65,109 @@ func newWSServer(t *testing.T, logger log.Logger) *httptest.Server { return srv } + +func TestWebsocketStartJoinsHandler(t *testing.T) { + type contextKey struct{} + ctx, cancel := context.WithCancel(context.WithValue(context.Background(), contextKey{}, "inherited")) + defer cancel() + entered, canceled, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{}) + disconnected := make(chan struct{}) + functions := map[string]*RPCFunc{"block": NewWSRPCFunc(func(ctx context.Context, _ *struct{}) (string, error) { + if ctx.Value(contextKey{}) != "inherited" { + t.Error("websocket handler lost parent context value") + } + close(entered) + <-ctx.Done() + close(canceled) + <-release + return "done", nil + })} + upgrader := websocket.Upgrader{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Error(err) + return + } + session := newWSConnection(conn, functions, log.NewNopLogger(), OnDisconnect(func(string) { close(disconnected) })) + if err := session.Start(ctx); err != nil { + t.Error(err) + } + close(done) + })) + defer server.Close() + conn, response, err := websocket.DefaultDialer.Dial("ws://"+server.Listener.Addr().String(), nil) + require.NoError(t, err) + defer response.Body.Close() + defer conn.Close() + request := rpctypes.NewRequest(1) + require.NoError(t, request.SetMethodAndParams("block", struct{}{})) + require.NoError(t, conn.WriteJSON(request)) + <-entered + cancel() + <-canceled + select { + case <-done: + close(release) + t.Fatal("Start returned while request handler remained active") + case <-time.After(100 * time.Millisecond): + } + select { + case <-disconnected: + t.Error("subscription cleanup ran before request completed") + default: + } + close(release) + <-done + <-disconnected +} + +func TestServeJoinsWebsocketSessionWork(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + entered, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + functions := map[string]*RPCFunc{ + "panic": NewWSRPCFunc(func(context.Context, *struct{}) (string, error) { panic("request panic") }), + "subscribe": NewWSRPCFunc(func(ctx context.Context, _ *struct{}) (string, error) { + if !rpctypes.GetCallInfo(ctx).WSConn.Go(func(ctx context.Context) { + close(entered) + <-ctx.Done() + close(canceled) + <-release + }) { + return "", context.Canceled + } + return "subscribed", nil + }), + } + manager := NewWebsocketManager(log.NewNopLogger(), functions) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + done := make(chan error, 1) + go func() { + done <- Serve(ctx, listener, http.HandlerFunc(manager.WebsocketHandler), log.NewNopLogger(), DefaultConfig()) + }() + conn, response, err := websocket.DefaultDialer.Dial("ws://"+listener.Addr().String(), nil) + require.NoError(t, err) + defer response.Body.Close() + defer conn.Close() + request := rpctypes.NewRequest(1) + require.NoError(t, request.SetMethodAndParams("panic", struct{}{})) + require.NoError(t, conn.WriteJSON(request)) + var result rpctypes.RPCResponse + require.NoError(t, conn.ReadJSON(&result)) + require.NotNil(t, result.Error) + require.NoError(t, request.SetMethodAndParams("subscribe", struct{}{})) + require.NoError(t, conn.WriteJSON(request)) + <-entered + cancel() + <-canceled + select { + case err := <-done: + close(release) + t.Fatalf("Serve returned while websocket session work remained active: %v", err) + case <-time.After(100 * time.Millisecond): + } + close(release) + require.ErrorIs(t, <-done, http.ErrServerClosed) +} diff --git a/rpc/jsonrpc/types/types.go b/rpc/jsonrpc/types/types.go index 5079184522..7c5963b0f8 100644 --- a/rpc/jsonrpc/types/types.go +++ b/rpc/jsonrpc/types/types.go @@ -316,6 +316,8 @@ func (resp RPCResponse) String() string { // WSRPCConnection represents a websocket connection. type WSRPCConnection interface { + // Go starts work owned and joined by the websocket session. + Go(func(context.Context)) bool // GetRemoteAddr returns a remote address of the connection. GetRemoteAddr() string // WriteRPCResponse writes the response onto connection (BLOCKING). From f717b4c52e3d77e791e89a591853132206db2d72 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:30:01 +0000 Subject: [PATCH 07/39] fix(node): drain RPC and mempool before releasing dependencies Co-Authored-By: OpenAI GPT-6 --- docs/architecture/service-lifecycle.md | 15 ++++++++++++--- node/node.go | 17 +++++------------ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/architecture/service-lifecycle.md b/docs/architecture/service-lifecycle.md index 62e2e1d1ba..11cfe15c4f 100644 --- a/docs/architecture/service-lifecycle.md +++ b/docs/architecture/service-lifecycle.md @@ -19,7 +19,8 @@ Shutdown has three phases: `Wait`. `Stopping()` exposes context cancellation separately from completion. -`Stop` does not join workers. Concurrent or recursive `Stop` calls return +`Stop` invokes `OnStop` synchronously but does not join registered workers. +Concurrent or recursive `Stop` calls return immediately. A stop during startup cancels the context and schedules `OnStop` after successful `OnStart`; hooks never overlap. `OnStop` must not join managed workers. Workers may call `Stop`; neither workers nor hooks may call their own @@ -55,7 +56,8 @@ them. A timeout is not a substitute for completion. The service audit covers consensus (state, reactor, ticker, gossip and WAL), autofile groups, node and seed owners, event bus and pubsub, indexer, block and state sync, mempool, evidence, P2P router/PEX/connections, quorum event handling, -ABCI clients/servers, proxy wrappers, signer services, and the light RPC wrapper. +ABCI clients/servers, proxy wrappers, signer services, HTTP/WebSocket RPC servers, +and the light RPC wrapper. Worker-free implementations retain their hooks without artificial workers. Service-scoped cancel functions, completion channels and custom `Wait` methods @@ -71,8 +73,15 @@ Those boundaries need their own ownership API before a service can promise to join every helper. Router connection/request groups and AutoFile workers are joined explicitly rather than converted into unrelated service lifetimes. +RPC shutdown closes admission and joins active handlers, including hijacked +WebSocket sessions and their subscription workers. Node shutdown then drains +async broadcasts and reactors, cancels and joins every mempool recheck batch, +and finally releases application connections and stores. + Retry of a composite service also requires recreating any children that already started successfully and were then stopped during rollback. Arbitrary injected ABCI clients cannot be recreated by the routed wrapper. The node reserves RPC listeners before starting children, so retry after listener failure remains -supported without restarting successful child lifetimes. +supported without restarting successful child lifetimes. After a later node +startup failure, construct a fresh node graph: rollback can also permanently +close non-service resources such as mempool recheck admission. diff --git a/node/node.go b/node/node.go index 5103fa9880..36d0510a53 100644 --- a/node/node.go +++ b/node/node.go @@ -71,7 +71,6 @@ type nodeImpl struct { evPool *evidence.Pool indexerService *indexer.Service services []service.Service - rpcListeners []net.Listener // rpc servers shutdownOps closer rpcEnv *rpccore.Environment prometheusSrv *http.Server @@ -515,6 +514,10 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { defer func() { if err != nil { cancelAttempt() + n.rpcEnv.StopService() + if stopErr := n.rpcEnv.StopAsyncBroadcasts(context.Background()); stopErr != nil { + n.logger.Error("Failed to stop async broadcasts after startup failure", "err", stopErr) + } for _, reactor := range startedReactors { reactor.Wait() } @@ -645,13 +648,9 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { if len(rpcListeners) > 0 { err = n.rpcEnv.StartService(ctx, n.config, rpcListeners) if err != nil { - if stopErr := n.rpcEnv.StopAsyncBroadcasts(context.Background()); stopErr != nil { - n.logger.Error("Failed to stop async broadcasts after RPC startup failure", "err", stopErr) - } return err } } - n.rpcListeners = rpcListeners return nil } @@ -659,16 +658,10 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { // OnStop stops the Node. It implements service.Service. func (n *nodeImpl) OnStop() { n.logger.Info("Stopping Node") + n.rpcEnv.StopService() if err := n.rpcEnv.StopAsyncBroadcasts(context.Background()); err != nil { n.logger.Error("Async broadcasts did not finish during shutdown", "err", err) } - // stop the listeners / external services first - for _, l := range n.rpcListeners { - n.logger.Info("Closing rpc listener", "listener", l) - if err := l.Close(); err != nil { - n.logger.Error("error closing listener", "listener", l, "err", err) - } - } for _, reactor := range n.services { if reactor == n.rpcEnv.EventBus { From 63a524e28e02f9adaf9b0f7c9c458954d51b6b4f Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:31:14 +0000 Subject: [PATCH 08/39] fix(rpc): unblock transport I/O before draining handlers --- rpc/jsonrpc/server/http_server.go | 12 ++-- rpc/jsonrpc/server/http_server_test.go | 80 ++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/rpc/jsonrpc/server/http_server.go b/rpc/jsonrpc/server/http_server.go index 7fbf1d8ba2..bb02618e6d 100644 --- a/rpc/jsonrpc/server/http_server.go +++ b/rpc/jsonrpc/server/http_server.go @@ -60,7 +60,7 @@ func DefaultConfig() *Config { // Serve creates a http.Server and calls Serve with the given listener. It // wraps handler to recover panics and limit the request body size. func Serve(ctx context.Context, listener net.Listener, handler http.Handler, logger log.Logger, config *Config) error { - return serve(ctx, handler, logger, config, func(s *http.Server) error { + return serve(ctx, listener, handler, logger, config, func(s *http.Server) error { return s.Serve(listener) }) } @@ -68,12 +68,14 @@ func Serve(ctx context.Context, listener net.Listener, handler http.Handler, log // ServeTLS serves HTTPS and joins all requests, including websocket sessions, // before returning. Cancellation is propagated to request contexts. func ServeTLS(ctx context.Context, listener net.Listener, handler http.Handler, certFile, keyFile string, logger log.Logger, config *Config) error { - return serve(ctx, handler, logger, config, func(s *http.Server) error { + return serve(ctx, listener, handler, logger, config, func(s *http.Server) error { return s.ServeTLS(listener, certFile, keyFile) }) } -func serve(ctx context.Context, handler http.Handler, logger log.Logger, config *Config, run func(*http.Server) error) error { +func serve(ctx context.Context, listener net.Listener, handler http.Handler, logger log.Logger, config *Config, run func(*http.Server) error) error { + // TLS setup can fail before http.Server takes ownership of the listener. + defer func() { _ = listener.Close() }() ctx, cancel := context.WithCancel(ctx) defer cancel() var mu sync.Mutex @@ -107,7 +109,9 @@ func serve(ctx context.Context, handler http.Handler, logger log.Logger, config mu.Lock() stopping = true mu.Unlock() - _ = s.Shutdown(context.Background()) + // Cancellation alone cannot unblock a handler writing to a stalled + // client. Close its transport before joining the handler itself. + _ = s.Close() requests.Wait() }() err := run(s) diff --git a/rpc/jsonrpc/server/http_server_test.go b/rpc/jsonrpc/server/http_server_test.go index 0043c97eeb..afa5ee153f 100644 --- a/rpc/jsonrpc/server/http_server_test.go +++ b/rpc/jsonrpc/server/http_server_test.go @@ -238,3 +238,83 @@ func TestServeJoinsActiveHandler(t *testing.T) { }) } } + +// blockedWriteListener supplies a connection whose peer never reads responses. +// net.Pipe makes the first response write block without OS buffer assumptions. +type blockedWriteListener struct { + net.Conn + accepted bool + closed chan struct{} + once sync.Once + writing chan struct{} +} + +func (l *blockedWriteListener) Accept() (net.Conn, error) { + if !l.accepted { + l.accepted = true + return &blockedResponseConn{Conn: l.Conn, writing: l.writing}, nil + } + <-l.closed + return nil, net.ErrClosed +} + +func (l *blockedWriteListener) Close() error { + l.once.Do(func() { close(l.closed) }) + return nil +} + +func (l *blockedWriteListener) Addr() net.Addr { return l.LocalAddr() } + +type blockedResponseConn struct { + net.Conn + writing chan struct{} + once sync.Once +} + +func (c *blockedResponseConn) Write(p []byte) (int, error) { + c.once.Do(func() { close(c.writing) }) + return c.Conn.Write(p) +} + +func TestServeCancellationUnblocksResponseWrite(t *testing.T) { + serverConn, clientConn := net.Pipe() + defer serverConn.Close() + defer clientConn.Close() + listener := &blockedWriteListener{Conn: serverConn, closed: make(chan struct{}), writing: make(chan struct{})} + defer listener.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done, handlerDone := make(chan error, 1), make(chan struct{}) + go func() { + done <- Serve(ctx, listener, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + defer close(handlerDone) + _, _ = w.Write(make([]byte, 1<<20)) + }), log.NewNopLogger(), DefaultConfig()) + }() + _, err := io.WriteString(clientConn, "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n") + require.NoError(t, err) + <-listener.writing + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, http.ErrServerClosed) + case <-time.After(time.Second): + _ = clientConn.Close() + <-done + t.Fatal("Serve did not unblock an active response write on cancellation") + } + <-handlerDone +} + +func TestServeTLSClosesListenerOnCertificateError(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + err = ServeTLS(context.Background(), listener, http.NotFoundHandler(), "missing.crt", "missing.key", log.NewNopLogger(), DefaultConfig()) + require.Error(t, err) + conn, err := net.DialTimeout("tcp", listener.Addr().String(), time.Second) + if conn != nil { + _ = conn.Close() + } + require.Error(t, err, "TLS setup failure must release the bound listener") +} From 2cbc4c9d9d75114d076063b4d1070937d3a9abf2 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Oct 2026 08:22:07 +0000 Subject: [PATCH 09/39] test(consensus): join the test event bus before the test ends newStateWithConfigAndBlockStore started an EventBus on the test context and never joined it. Its asynchronous shutdown logged through the testing logger after the test returned, which the race detector reports (pre-existing on v1.8-dev, e.g. under TestPeerVoteVerificationDraws). Co-Authored-By: Claude Opus 5.5 --- internal/consensus/common_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/consensus/common_test.go b/internal/consensus/common_test.go index af2b1a1c74..aff161c4d5 100644 --- a/internal/consensus/common_test.go +++ b/internal/consensus/common_test.go @@ -538,6 +538,8 @@ func newStateWithConfigAndBlockStore( eventBus := eventbus.NewDefault(logger.With("module", "events")) require.NoError(t, eventBus.Start(ctx)) + // Join the bus before the test ends: its shutdown logs through t. + t.Cleanup(func() { eventBus.Stop(); eventBus.Wait() }) blockExec := sm.NewBlockExecutor(stateStore, proxyAppConnCon, mpool, evpool, blockStore, eventBus) cs, err := NewState(logger.With("module", "consensus"), From d97ddbcf244cec7e4b61a6ae0163f45a18fee088 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:13:27 +0000 Subject: [PATCH 10/39] test: import libs/db after the database adapter backport The base dropped cometbft-db (#1566); point the p2p and indexer lifecycle tests at libs/db so they compile against v1.8-dev. Co-Authored-By: Claude Opus 5.5 --- internal/p2p/lifecycle_test.go | 2 +- internal/state/indexer/lifecycle_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/p2p/lifecycle_test.go b/internal/p2p/lifecycle_test.go index 29bda79b6e..7ad62f3461 100644 --- a/internal/p2p/lifecycle_test.go +++ b/internal/p2p/lifecycle_test.go @@ -6,11 +6,11 @@ import ( "testing" "time" - dbm "github.com/cometbft/cometbft-db" "github.com/stretchr/testify/require" tmsync "github.com/dashpay/tenderdash/internal/libs/sync" "github.com/dashpay/tenderdash/internal/p2p/conn" + dbm "github.com/dashpay/tenderdash/libs/db" "github.com/dashpay/tenderdash/libs/log" p2pproto "github.com/dashpay/tenderdash/proto/tendermint/p2p" "github.com/dashpay/tenderdash/types" diff --git a/internal/state/indexer/lifecycle_test.go b/internal/state/indexer/lifecycle_test.go index aa3b94d491..d242572bf2 100644 --- a/internal/state/indexer/lifecycle_test.go +++ b/internal/state/indexer/lifecycle_test.go @@ -5,13 +5,13 @@ import ( "testing" "time" - dbm "github.com/cometbft/cometbft-db" "github.com/stretchr/testify/require" "github.com/dashpay/tenderdash/internal/eventbus" "github.com/dashpay/tenderdash/internal/pubsub" "github.com/dashpay/tenderdash/internal/state/indexer" "github.com/dashpay/tenderdash/internal/state/indexer/sink/kv" + dbm "github.com/dashpay/tenderdash/libs/db" "github.com/dashpay/tenderdash/libs/log" ) From e549ab4f7f2cc35629edf0a5c91d5b211c320aa5 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:39:51 +0000 Subject: [PATCH 11/39] test(rpc): report Serve's result when cancellation does not unblock a write errcheck flags the discarded error received from Serve on the timeout path. Include it in the failure message instead. Co-Authored-By: Claude Opus 5.5 --- rpc/jsonrpc/server/http_server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpc/jsonrpc/server/http_server_test.go b/rpc/jsonrpc/server/http_server_test.go index afa5ee153f..4145fc3c4b 100644 --- a/rpc/jsonrpc/server/http_server_test.go +++ b/rpc/jsonrpc/server/http_server_test.go @@ -300,8 +300,8 @@ func TestServeCancellationUnblocksResponseWrite(t *testing.T) { require.ErrorIs(t, err, http.ErrServerClosed) case <-time.After(time.Second): _ = clientConn.Close() - <-done - t.Fatal("Serve did not unblock an active response write on cancellation") + err := <-done + t.Fatalf("Serve did not unblock an active response write on cancellation (returned %v after the client closed)", err) } <-handlerDone } From 687318c3d425a3df95f703494afb925ab1244b73 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:39:52 +0000 Subject: [PATCH 12/39] fix(node): close pprof and Prometheus servers instead of waiting on clients The auxiliary HTTP servers shut down with an unbounded Shutdown that the node joins, so one stalled client (e.g. a long pprof profile) blocked node Wait forever. Node and seed OnStop also called Shutdown without a deadline. Serve them like the RPC server: on cancellation reject new requests, cancel request contexts, close connections and join active handlers. Drop the redundant OnStop shutdowns; the owned worker already stops the server. Bind failures are now logged instead of silently dropped. Co-Authored-By: Claude Opus 5.5 --- node/http_server_test.go | 73 ++++++++++++++++++++++++++++++++++++++ node/node.go | 14 ++------ node/seed.go | 14 ++------ node/setup.go | 76 ++++++++++++++++++++++++++++++---------- 4 files changed, 136 insertions(+), 41 deletions(-) create mode 100644 node/http_server_test.go diff --git a/node/http_server_test.go b/node/http_server_test.go new file mode 100644 index 0000000000..04574a3b38 --- /dev/null +++ b/node/http_server_test.go @@ -0,0 +1,73 @@ +package node + +import ( + "context" + "io" + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" +) + +type httpServerOwner struct { + start func(context.Context) +} + +func (o *httpServerOwner) OnStart(ctx context.Context) error { o.start(ctx); return nil } +func (*httpServerOwner) OnStop() {} + +// A client that never finishes its request (e.g. a long pprof profile) must not +// block the owner's shutdown, and Wait must still join the handler. +func TestOwnedHTTPServerStopEvictsStalledClient(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + entered, handlerDone := make(chan struct{}), make(chan struct{}) + release := make(chan struct{}) + srv := &http.Server{ + ReadHeaderTimeout: httpReadHeaderTimeout, + Handler: http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + defer close(handlerDone) + close(entered) + select { + case <-r.Context().Done(): + case <-release: + } + }), + } + owner := &httpServerOwner{} + bs := service.NewBaseService(log.NewNopLogger(), "http-owner", owner) + owner.start = func(ctx context.Context) { serveOwnedHTTP(ctx, bs, srv, ln) } + require.NoError(t, bs.Start(context.Background())) + + client, err := net.Dial("tcp", ln.Addr().String()) + require.NoError(t, err) + defer client.Close() + _, err = io.WriteString(client, "GET /stalled HTTP/1.1\r\nHost: localhost\r\n\r\n") + require.NoError(t, err) + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("handler was not entered") + } + + bs.Stop() + stopped := make(chan struct{}) + go func() { bs.Wait(); close(stopped) }() + select { + case <-stopped: + case <-time.After(5 * time.Second): + close(release) + <-stopped + t.Fatal("a stalled client blocked HTTP server shutdown") + } + select { + case <-handlerDone: + default: + t.Fatal("Wait returned before the active handler finished") + } +} diff --git a/node/node.go b/node/node.go index f1400a8a7f..fa16874d29 100644 --- a/node/node.go +++ b/node/node.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net" - "net/http" "strconv" "strings" "time" @@ -71,7 +70,6 @@ type nodeImpl struct { services []service.Service shutdownOps closer rpcEnv *rpccore.Environment - prometheusSrv *http.Server dependencyCancel context.CancelFunc } @@ -579,7 +577,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { } // Start Internal Services if n.config.RPC.PprofListenAddress != "" { - startPProfServer(ctx, &n.BaseService, *n.config.RPC) + startPProfServer(ctx, &n.BaseService, n.logger, *n.config.RPC) } now := tmtime.Now() @@ -605,7 +603,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { } if n.config.Instrumentation.Prometheus && n.config.Instrumentation.PrometheusListenAddr != "" { - n.prometheusSrv = startPrometheusServer(ctx, &n.BaseService, *n.config.Instrumentation) + startPrometheusServer(ctx, &n.BaseService, n.logger, *n.config.Instrumentation) } // Start the transport. @@ -669,14 +667,6 @@ func (n *nodeImpl) OnStop() { pvsc.Wait() } } - - if n.prometheusSrv != nil { - if err := n.prometheusSrv.Shutdown(context.Background()); err != nil { - // Error from closing listeners, or context timeout: - n.logger.Error("Prometheus HTTP server Shutdown", "err", err) - } - - } } // OnDrain releases storage after child services and node workers have finished. diff --git a/node/seed.go b/node/seed.go index b78e494fcf..b24f2c9df3 100644 --- a/node/seed.go +++ b/node/seed.go @@ -3,7 +3,6 @@ package node import ( "context" "fmt" - "net/http" "strings" "time" @@ -32,8 +31,7 @@ type seedNodeImpl struct { nodeKey types.NodeKey // our node privkey // services - pexReactor service.Service // for exchanging peer addresses - prometheusSrv *http.Server + pexReactor service.Service // for exchanging peer addresses shutdownOps closer } @@ -107,7 +105,7 @@ func (n *seedNodeImpl) OnStart(ctx context.Context) (err error) { } }() if n.config.RPC.PprofListenAddress != "" { - startPProfServer(ctx, &n.BaseService, *n.config.RPC) + startPProfServer(ctx, &n.BaseService, n.logger, *n.config.RPC) } now := tmtime.Now() @@ -124,7 +122,7 @@ func (n *seedNodeImpl) OnStart(ctx context.Context) (err error) { } if n.config.Instrumentation.Prometheus && n.config.Instrumentation.PrometheusListenAddr != "" { - n.prometheusSrv = startPrometheusServer(ctx, &n.BaseService, *n.config.Instrumentation) + startPrometheusServer(ctx, &n.BaseService, n.logger, *n.config.Instrumentation) } // Start the transport. @@ -141,12 +139,6 @@ func (n *seedNodeImpl) OnStop() { n.pexReactor.Wait() n.router.Wait() - if n.prometheusSrv != nil { - if err := n.prometheusSrv.Shutdown(context.Background()); err != nil { - n.logger.Error("failed to stop metrics server", "err", err) - } - } - } // OnDrain releases resources after network and HTTP workers have finished. diff --git a/node/setup.go b/node/setup.go index e6f3bfdd68..a144c71973 100644 --- a/node/setup.go +++ b/node/setup.go @@ -5,8 +5,10 @@ import ( "context" "errors" "fmt" + "net" "slices" "strings" + "sync" "time" "github.com/dashpay/dashd-go/btcjson" @@ -653,12 +655,11 @@ func createBlockReplayer(n *nodeImpl) *consensus.BlockReplayer { ) } -// startPrometheusServer starts a Prometheus HTTP server, listening for metrics -// collectors on addr. -func startPrometheusServer(ctx context.Context, owner *service.BaseService, cfg config.InstrumentationConfig) *http.Server { - addr := cfg.PrometheusListenAddr +// startPrometheusServer serves Prometheus metrics on the configured address as +// work owned by owner. +func startPrometheusServer(ctx context.Context, owner *service.BaseService, logger log.Logger, cfg config.InstrumentationConfig) { srv := &http.Server{ - Addr: addr, + Addr: cfg.PrometheusListenAddr, ReadHeaderTimeout: httpReadHeaderTimeout, Handler: promhttp.InstrumentMetricHandler( prometheus.DefaultRegisterer, promhttp.HandlerFor( @@ -667,28 +668,67 @@ func startPrometheusServer(ctx context.Context, owner *service.BaseService, cfg ), ), } - - startHTTPServer(ctx, owner, srv) - - return srv + startHTTPServer(ctx, owner, logger, srv) } -// startPProfServer creates a new pprof server -func startPProfServer(ctx context.Context, owner *service.BaseService, cfg config.RPCConfig) { +// startPProfServer serves pprof on the configured address as work owned by owner. +func startPProfServer(ctx context.Context, owner *service.BaseService, logger log.Logger, cfg config.RPCConfig) { srv := &http.Server{ Addr: cfg.PprofListenAddress, ReadHeaderTimeout: httpReadHeaderTimeout, - Handler: nil, + Handler: http.DefaultServeMux, } - startHTTPServer(ctx, owner, srv) + startHTTPServer(ctx, owner, logger, srv) } -func startHTTPServer(ctx context.Context, owner *service.BaseService, srv *http.Server) { - owner.Go(ctx, func(ctx context.Context) { - <-ctx.Done() - _ = srv.Shutdown(context.Background()) +// startHTTPServer binds srv.Addr and serves it until ctx is canceled. A bind +// failure is logged; the auxiliary server is skipped as before. +func startHTTPServer(ctx context.Context, owner *service.BaseService, logger log.Logger, srv *http.Server) { + ln, err := net.Listen("tcp", srv.Addr) + if err != nil { + logger.Error("failed to start HTTP server", "addr", srv.Addr, "err", err) + return + } + serveOwnedHTTP(ctx, owner, srv, ln) +} + +// serveOwnedHTTP serves srv on ln as work of owner. On cancellation it rejects +// new requests, cancels request contexts and closes connections, so a stalled +// client cannot block shutdown, then joins the active handlers. +func serveOwnedHTTP(ctx context.Context, owner *service.BaseService, srv *http.Server, ln net.Listener) { + var mu sync.Mutex + var handlers sync.WaitGroup + stopping := false + next := srv.Handler + srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Serialize admission with the WaitGroup join below. + mu.Lock() + if stopping { + mu.Unlock() + http.Error(w, "server is stopping", http.StatusServiceUnavailable) + return + } + handlers.Add(1) + mu.Unlock() + defer handlers.Done() + next.ServeHTTP(w, r) }) - owner.Go(ctx, func(context.Context) { _ = srv.ListenAndServe() }) + srv.BaseContext = func(net.Listener) context.Context { return ctx } + if !owner.Go(ctx, func(ctx context.Context) { + <-ctx.Done() + mu.Lock() + stopping = true + mu.Unlock() + _ = srv.Close() + handlers.Wait() + }) { + _ = ln.Close() + return + } + // Serve closes ln, including when Close has already run. + if !owner.Go(ctx, func(context.Context) { _ = srv.Serve(ln) }) { + _ = ln.Close() + } } // stateSyncMetricsSetter is the consensus-side surface of the state sync handoff. From 915d9bcf3a3bbfd8592d2bdbc16a51ba0630b954 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:39:52 +0000 Subject: [PATCH 13/39] refactor(node): share child teardown between shutdown and failed startup OnStart's rollback hand-copied OnStop's teardown and the copies had started to differ in ordering. Both now call joinChildren. Waiting on a child that never started returns immediately, so the rollback no longer needs to track whether the router started. Co-Authored-By: Claude Opus 5.5 --- node/node.go | 61 +++++++++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 34 deletions(-) diff --git a/node/node.go b/node/node.go index fa16874d29..924f810ab8 100644 --- a/node/node.go +++ b/node/node.go @@ -491,28 +491,10 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { n.dependencyCancel = dependencyCancel ctx, cancelAttempt := context.WithCancel(ctx) var startedReactors []service.Service - routerStarted := false defer func() { if err != nil { cancelAttempt() - n.rpcEnv.StopService() - if stopErr := n.rpcEnv.StopAsyncBroadcasts(context.Background()); stopErr != nil { - n.logger.Error("Failed to stop async broadcasts after startup failure", "err", stopErr) - } - for _, reactor := range startedReactors { - reactor.Wait() - } - if routerStarted { - n.router.Wait() - } - if mp, ok := n.rpcEnv.Mempool.(*mempool.TxMempool); ok { - mp.StopRechecks() - } - n.rpcEnv.EventBus.Stop() - n.rpcEnv.EventBus.Wait() - dependencyCancel() - n.indexerService.Wait() - n.rpcEnv.ProxyApp.Wait() + n.joinChildren(startedReactors) } }() if err := n.rpcEnv.ProxyApp.Start(dependencyCtx); err != nil { @@ -610,7 +592,6 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { if err := n.router.Start(ctx); err != nil { return err } - routerStarted = true n.rpcEnv.IsListening = true for _, reactor := range n.services { @@ -639,17 +620,38 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { // OnStop stops the Node. It implements service.Service. func (n *nodeImpl) OnStop() { n.logger.Info("Stopping Node") + reactors := make([]service.Service, 0, len(n.services)) + for _, reactor := range n.services { + if reactor != n.rpcEnv.EventBus { + reactors = append(reactors, reactor) + } + } + n.joinChildren(reactors) + + // The private validator is created with the node, not started by OnStart, + // so only completed shutdown waits for a service implementation of it. + if n.privValidator != nil { + if pvsc, ok := n.privValidator.(service.Service); ok { + pvsc.Wait() + } + } +} + +// joinChildren stops and joins the children started by OnStart after their +// contexts are canceled; both shutdown and failed startup use it. Waiting for a +// child that never started returns immediately. RPC admission closes first; +// application connections, the event bus and the indexer go last because +// reactors may use them until they exit. +func (n *nodeImpl) joinChildren(reactors []service.Service) { n.rpcEnv.StopService() if err := n.rpcEnv.StopAsyncBroadcasts(context.Background()); err != nil { n.logger.Error("Async broadcasts did not finish during shutdown", "err", err) } - - for _, reactor := range n.services { - if reactor == n.rpcEnv.EventBus { - continue - } + for _, reactor := range reactors { reactor.Wait() } + n.router.Wait() + n.rpcEnv.IsListening = false if mp, ok := n.rpcEnv.Mempool.(*mempool.TxMempool); ok { mp.StopRechecks() } @@ -658,15 +660,6 @@ func (n *nodeImpl) OnStop() { n.dependencyCancel() n.indexerService.Wait() n.rpcEnv.ProxyApp.Wait() - - n.router.Wait() - n.rpcEnv.IsListening = false - - if n.privValidator != nil { - if pvsc, ok := n.privValidator.(service.Service); ok { - pvsc.Wait() - } - } } // OnDrain releases storage after child services and node workers have finished. From 1ff0319dc96141691de2884845d4d0ca397038d8 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:39:52 +0000 Subject: [PATCH 14/39] docs: describe light RPC client and JSON-RPC server shutdown changes Co-Authored-By: Claude Opus 5.5 --- UPGRADING.md | 7 +++++++ docs/architecture/service-lifecycle.md | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/UPGRADING.md b/UPGRADING.md index 9d7a56e554..8efef7b297 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -20,6 +20,13 @@ Plain goroutines and their existing joins remain the implementation's responsibi implementations must provide `Go(func(context.Context)) bool` so the server owns subscription workers. +`rpc/jsonrpc/server.Serve` and `ServeTLS` always close the listener before +returning, including when TLS setup fails. Cancellation now reaches request +contexts, rejects new requests with 503, closes active connections instead of a +one-second graceful shutdown, and returns only after every handler, including +hijacked WebSocket sessions, has finished. They no longer log a startup line; +one "RPC server stopped" line is logged on return. + A stop requested during startup cancels work immediately but defers `OnStop` until successful startup returns. Failed startup must roll back its resources; BaseService joins registered work without calling shutdown hooks. Retry requires diff --git a/docs/architecture/service-lifecycle.md b/docs/architecture/service-lifecycle.md index ee4f221dd0..fcd6d12f87 100644 --- a/docs/architecture/service-lifecycle.md +++ b/docs/architecture/service-lifecycle.md @@ -96,7 +96,7 @@ in-tree hook/callback audit covers these groups: | EventBus, pubsub Server and indexer Service | Event processing is cancellation-driven; the indexer closes its sinks in OnDrain. No guarantee of a live cleanup callback context. | | ABCI local, socket, gRPC and routed clients; proxy; socket/gRPC servers | Stop hooks close transports, drain requests or stop children without requiring a live work context. | | SignerServer, signer listener/dialer endpoints | Hooks close endpoints/listeners; dial/retry work observes cancellation. | -| ValidatorConnExecutor and light RPC Client | Unsubscribe uses a separate bounded cleanup context; client hooks invoke owned closers. | +| ValidatorConnExecutor and light RPC Client | Unsubscribe uses a separate bounded cleanup context. The light client scopes WebSocket subscriptions to its service and the caller session, and joins (or stops) the wrapped client in OnDrain. | | Node and seed node | Parent cancellation stops children; the node joins reactors before canceling its dependency context and closing owned stores. | Subscribers must stop with their owner, without relying on a final router From 6c0bdf2115e219931d45f818f3de68453f6bd42b Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:05:08 +0000 Subject: [PATCH 15/39] fix(abci): close ABCI socket connections on server shutdown SocketServer.OnStop only canceled each connection's context. A response writer blocked on a client that stopped reading, plus a reader waiting for the next request, never observe cancellation, so the now-joined Wait hung. Register a closer that cancels and closes the transport; it never takes connsMtx, so OnStop and rmConn can call it under that lock. Also log a close error only when closing actually failed. Co-Authored-By: Claude Opus 5.5 --- abci/server/socket_server.go | 9 +++- abci/server/socket_server_test.go | 69 +++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 abci/server/socket_server_test.go diff --git a/abci/server/socket_server.go b/abci/server/socket_server.go index 4d6482c1ac..83cf8590cd 100644 --- a/abci/server/socket_server.go +++ b/abci/server/socket_server.go @@ -8,6 +8,7 @@ import ( "io" "net" "runtime" + stdsync "sync" sync "github.com/sasha-s/go-deadlock" @@ -111,7 +112,11 @@ func (s *SocketServer) acceptConnectionsRoutine(ctx context.Context) { } cctx, ccancel := context.WithCancel(ctx) - connID := s.addConn(ccancel) + closeConn := stdsync.OnceValue(conn.Close) + // Cancellation cannot interrupt socket I/O, so shutdown also closes the + // transport. This closer never touches connsMtx: OnStop and rmConn call + // it while holding that lock. + connID := s.addConn(func() { ccancel(); _ = closeConn() }) s.logger.Info("Accepted a new connection", "id", connID) @@ -121,7 +126,7 @@ func (s *SocketServer) acceptConnectionsRoutine(ctx context.Context) { closer := func(err error) { ccancel() once.Do(func() { - if cerr := conn.Close(); err != nil { + if cerr := closeConn(); cerr != nil { s.logger.Error("error closing connection", "id", connID, "close_err", cerr, diff --git a/abci/server/socket_server_test.go b/abci/server/socket_server_test.go new file mode 100644 index 0000000000..a698e5b9fa --- /dev/null +++ b/abci/server/socket_server_test.go @@ -0,0 +1,69 @@ +package server + +import ( + "bufio" + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/abci/types" + "github.com/dashpay/tenderdash/libs/log" +) + +const ( + stalledQueries = 32 + stalledRespSize = 1 << 20 +) + +// largeQueryApp answers every query with a large value and reports each call. +type largeQueryApp struct { + types.BaseApplication + served chan struct{} +} + +func (a *largeQueryApp) Query(context.Context, *types.RequestQuery) (*types.ResponseQuery, error) { + a.served <- struct{}{} + return &types.ResponseQuery{Value: make([]byte, stalledRespSize)}, nil +} + +// A client that stops reading responses leaves the response writer blocked +// on the socket while the request reader waits for the next request. Stop +// must still unblock both before Wait joins them. +func TestSocketServerStopUnblocksStalledConnection(t *testing.T) { + addr := "unix://" + filepath.Join(t.TempDir(), "abci.sock") + app := &largeQueryApp{served: make(chan struct{}, stalledQueries)} + srv := NewSocketServer(log.NewNopLogger(), addr, app).(*SocketServer) + require.NoError(t, srv.Start(context.Background())) + + conn, err := net.Dial("unix", srv.addr) + require.NoError(t, err) + defer conn.Close() + // Responses (32 MiB) far exceed socket buffers; nothing is ever read back. + w := bufio.NewWriter(conn) + for i := 0; i < stalledQueries; i++ { + require.NoError(t, types.WriteMessage(types.ToRequestQuery(&types.RequestQuery{}), w)) + } + require.NoError(t, w.Flush()) + for i := 0; i < stalledQueries; i++ { + select { + case <-app.served: + case <-time.After(5 * time.Second): + t.Fatalf("server processed only %d of %d queries", i, stalledQueries) + } + } + + srv.Stop() + stopped := make(chan struct{}) + go func() { srv.Wait(); close(stopped) }() + select { + case <-stopped: + case <-time.After(5 * time.Second): + _ = conn.Close() + <-stopped + t.Fatal("Wait blocked on a connection with a stalled response write") + } +} From b845e6cf964cf39f5c7dd707294c0279cbae97a2 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:05:08 +0000 Subject: [PATCH 16/39] fix(light): bound WebSocket unsubscribe cleanup by its owner's lifetime Serve joins WebSocket sessions and their disconnect callbacks. The light proxy unsubscribed with context.Background(), and the light RPC client's UnsubscribeWS/UnsubscribeAllWS did the same; once the upstream WSClient writer stopped, its unbuffered send never returned and proxy shutdown hung. Use the proxy lifetime for disconnect cleanup and the request context for the RPC unsubscribe calls. Co-Authored-By: Claude Opus 5.5 --- light/proxy/proxy.go | 6 ++-- light/proxy/shutdown_test.go | 61 +++++++++++++++++++++++++++++++++++ light/rpc/client.go | 6 ++-- light/rpc/unsubscribe_test.go | 53 ++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 light/proxy/shutdown_test.go create mode 100644 light/rpc/unsubscribe_test.go diff --git a/light/proxy/proxy.go b/light/proxy/proxy.go index c5996999f1..05fec7afd6 100644 --- a/light/proxy/proxy.go +++ b/light/proxy/proxy.go @@ -103,8 +103,10 @@ func (p *Proxy) listen(ctx context.Context) (net.Listener, *http.ServeMux, error wmLogger := p.Logger.With("protocol", "websocket") wm := rpcserver.NewWebsocketManager(wmLogger, r, rpcserver.OnDisconnect(func(remoteAddr string) { - err := p.Client.UnsubscribeAll(context.Background(), remoteAddr) - if err != nil && err != tmpubsub.ErrSubscriptionNotFound { + // Serve joins this callback. Bound it by the proxy lifetime: once the + // upstream client stops, its send has no receiver. + err := p.Client.UnsubscribeAll(ctx, remoteAddr) + if err != nil && err != tmpubsub.ErrSubscriptionNotFound && ctx.Err() == nil { wmLogger.Error("Failed to unsubscribe addr from events", "addr", remoteAddr, "err", err) } }), diff --git a/light/proxy/shutdown_test.go b/light/proxy/shutdown_test.go new file mode 100644 index 0000000000..e0a7e5f7f9 --- /dev/null +++ b/light/proxy/shutdown_test.go @@ -0,0 +1,61 @@ +package proxy + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" + lrpc "github.com/dashpay/tenderdash/light/rpc" + rpchttp "github.com/dashpay/tenderdash/rpc/client/http" + rpcserver "github.com/dashpay/tenderdash/rpc/jsonrpc/server" +) + +// Serve joins WebSocket disconnect cleanup. With the upstream client's writer +// already stopped, that cleanup must not block proxy shutdown. +func TestProxyShutdownWithStoppedUpstreamClient(t *testing.T) { + logger := log.NewNopLogger() + upstreamWS := rpcserver.NewWebsocketManager(logger, map[string]*rpcserver.RPCFunc{}) + upstreamMux := http.NewServeMux() + upstreamMux.HandleFunc("/websocket", upstreamWS.WebsocketHandler) + upstream := httptest.NewServer(upstreamMux) + t.Cleanup(upstream.Close) + + next, err := rpchttp.New(upstream.URL) + require.NoError(t, err) + client := lrpc.NewClient(logger, next, nil) + clientCtx, stopClient := context.WithCancel(context.Background()) + t.Cleanup(stopClient) + require.NoError(t, client.Start(clientCtx)) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + p := &Proxy{Addr: "tcp://127.0.0.1:0", Config: rpcserver.DefaultConfig(), Client: client, Logger: logger} + listener, mux, err := p.listen(ctx) + require.NoError(t, err) + served := make(chan error, 1) + go func() { served <- rpcserver.Serve(ctx, listener, mux, logger, p.Config) }() + + dialer := websocket.Dialer{HandshakeTimeout: 5 * time.Second} + conn, resp, err := dialer.Dial("ws://"+listener.Addr().String()+"/websocket", nil) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + t.Cleanup(func() { _ = conn.Close() }) + + // Stop the upstream client first; Wait returns after its writer exits. + stopClient() + client.Wait() + + cancel() + select { + case err := <-served: + require.ErrorIs(t, err, http.ErrServerClosed) + case <-time.After(5 * time.Second): + t.Fatal("proxy shutdown blocked on disconnect cleanup through a stopped upstream client") + } +} diff --git a/light/rpc/client.go b/light/rpc/client.go index 4c9c454e38..201df29127 100644 --- a/light/rpc/client.go +++ b/light/rpc/client.go @@ -701,7 +701,8 @@ func (c *Client) SubscribeWS(ctx context.Context, query string) (*coretypes.Resu // UnsubscribeWS calls original client's Unsubscribe using remote address as a // subscriber. func (c *Client) UnsubscribeWS(ctx context.Context, query string) (*coretypes.ResultUnsubscribe, error) { - err := c.next.Unsubscribe(context.Background(), rpctypes.GetCallInfo(ctx).RemoteAddr(), query) //nolint:staticcheck + // ctx ends with the WebSocket session, which joins this call on shutdown. + err := c.next.Unsubscribe(ctx, rpctypes.GetCallInfo(ctx).RemoteAddr(), query) //nolint:staticcheck if err != nil { return nil, err } @@ -711,7 +712,8 @@ func (c *Client) UnsubscribeWS(ctx context.Context, query string) (*coretypes.Re // UnsubscribeAllWS calls original client's UnsubscribeAll using remote address // as a subscriber. func (c *Client) UnsubscribeAllWS(ctx context.Context) (*coretypes.ResultUnsubscribe, error) { - err := c.next.UnsubscribeAll(context.Background(), rpctypes.GetCallInfo(ctx).RemoteAddr()) //nolint:staticcheck + // ctx ends with the WebSocket session, which joins this call on shutdown. + err := c.next.UnsubscribeAll(ctx, rpctypes.GetCallInfo(ctx).RemoteAddr()) //nolint:staticcheck if err != nil { return nil, err } diff --git a/light/rpc/unsubscribe_test.go b/light/rpc/unsubscribe_test.go new file mode 100644 index 0000000000..26453a6ff4 --- /dev/null +++ b/light/rpc/unsubscribe_test.go @@ -0,0 +1,53 @@ +package rpc + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" + rpcmock "github.com/dashpay/tenderdash/rpc/client/mocks" + rpctypes "github.com/dashpay/tenderdash/rpc/jsonrpc/types" +) + +// blockUntilCanceled stands in for an upstream WebSocket send with no writer. +func blockUntilCanceled(args mock.Arguments) { + <-args.Get(0).(context.Context).Done() +} + +// Unsubscribe calls run inside a joined WebSocket session, so they must stop +// when the caller's request context is canceled. +func TestUnsubscribeWSObservesCallerCancellation(t *testing.T) { + for name, call := range map[string]func(*Client, context.Context) error{ + "unsubscribe": func(c *Client, ctx context.Context) error { + _, err := c.UnsubscribeWS(ctx, "tm.event = 'NewBlock'") + return err + }, + "unsubscribe_all": func(c *Client, ctx context.Context) error { + _, err := c.UnsubscribeAllWS(ctx) + return err + }, + } { + t.Run(name, func(t *testing.T) { + next := rpcmock.NewClient(t) + next.On("Unsubscribe", mock.Anything, mock.Anything, mock.Anything). + Run(blockUntilCanceled).Return(context.Canceled).Maybe() + next.On("UnsubscribeAll", mock.Anything, mock.Anything). + Run(blockUntilCanceled).Return(context.Canceled).Maybe() + c := NewClient(log.NewNopLogger(), next, nil) + ctx, cancel := context.WithCancel(rpctypes.WithCallInfo(context.Background(), &rpctypes.CallInfo{})) + done := make(chan error, 1) + go func() { done <- call(c, ctx) }() + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("unsubscribe ignored caller cancellation") + } + }) + } +} From d87e9b25450545053e4eea6c631442e55f65fdba Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:46:08 +0000 Subject: [PATCH 17/39] fix(abci): release only the routed clients a start attempt started routedClient joined and stopped every configured client, including ones the caller had started, so a failed start blocked until a caller-owned client stopped. Track the clients the attempt starts; stop and join only those on startup failure, in OnStop and in the drain worker. Co-Authored-By: Claude Opus 5.5 --- abci/client/routed_client.go | 46 +++++++++++++--------- abci/client/routed_lifecycle_test.go | 57 ++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 18 deletions(-) create mode 100644 abci/client/routed_lifecycle_test.go diff --git a/abci/client/routed_client.go b/abci/client/routed_client.go index 0e104aca0e..f7ace49d03 100644 --- a/abci/client/routed_client.go +++ b/abci/client/routed_client.go @@ -21,6 +21,9 @@ type routedClient struct { logger log.Logger routing Routing defaultClient ClientInfo + // owned lists the clients started by the current attempt. Lifecycle hooks + // never overlap, and the drain worker starts after OnStart sets it. + owned []Client } var _ Client = (*routedClient)(nil) @@ -129,46 +132,53 @@ func NewRoutedClient(logger log.Logger, defaultClient Client, routing Routing) ( return cli, nil } +// OnStart starts the clients that are not running yet and owns only those: +// clients the caller already started keep their own lifetime. func (cli *routedClient) OnStart(ctx context.Context) error { + cli.owned = nil var errs error - for _, clients := range cli.routing { - for _, client := range clients { - if !client.IsRunning() { - if err := client.Start(ctx); err != nil { - errs = multierror.Append(errs, err) - } - } + for _, client := range cli.clients() { + if client.IsRunning() { + continue } - } - - if !cli.defaultClient.IsRunning() { - if err := cli.defaultClient.Start(ctx); err != nil { + if err := client.Start(ctx); err != nil { errs = multierror.Append(errs, err) + continue } + cli.owned = append(cli.owned, client) + } + // A failed attempt gets no OnStop; release what it started before returning. + if errs != nil { + cli.stopOwned() + return errs } if !cli.Go(ctx, func(ctx context.Context) { <-ctx.Done() - for _, client := range cli.clients() { + for _, client := range cli.owned { client.Wait() } }) { - cli.OnStop() - for _, client := range cli.clients() { - client.Wait() - } + cli.stopOwned() } - return errs + return nil } func (cli *routedClient) OnStop() { - for _, client := range cli.clients() { + for _, client := range cli.owned { if stopper, ok := client.(interface{ Stop() }); ok { stopper.Stop() } } } +func (cli *routedClient) stopOwned() { + cli.OnStop() + for _, client := range cli.owned { + client.Wait() + } +} + func (cli *routedClient) clients() []Client { clients := []Client{cli.defaultClient.Client} for _, routes := range cli.routing { diff --git a/abci/client/routed_lifecycle_test.go b/abci/client/routed_lifecycle_test.go new file mode 100644 index 0000000000..e1f94465f5 --- /dev/null +++ b/abci/client/routed_lifecycle_test.go @@ -0,0 +1,57 @@ +package abciclient_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + + abciclient "github.com/dashpay/tenderdash/abci/client" + "github.com/dashpay/tenderdash/abci/types" + "github.com/dashpay/tenderdash/libs/log" +) + +var errChildStart = errors.New("child start failed") + +type failingStartClient struct{ abciclient.Client } + +func (failingStartClient) Start(context.Context) error { return errChildStart } +func (failingStartClient) IsRunning() bool { return false } +func (failingStartClient) Wait() {} + +// A failed start must stop and join the children it started, and leave alone +// a client the caller had already started. +func TestRoutedClientFailedStartReleasesOnlyItsChildren(t *testing.T) { + logger := log.NewNopLogger() + callerCtx, cancelCaller := context.WithCancel(context.Background()) + defer cancelCaller() + callerOwned := abciclient.NewLocalClient(logger, types.NewBaseApplication()) + require.NoError(t, callerOwned.Start(callerCtx)) + attemptOwned := abciclient.NewLocalClient(logger, types.NewBaseApplication()) + routing := abciclient.Routing{ + "Info": {{Client: callerOwned, ClientID: "caller"}}, + "Query": {{Client: attemptOwned, ClientID: "attempt"}}, + } + routed, err := abciclient.NewRoutedClient(logger, failingStartClient{}, routing) + require.NoError(t, err) + + started := make(chan error, 1) + go func() { started <- routed.Start(context.Background()) }() + select { + case err := <-started: + require.ErrorIs(t, err, errChildStart) + case <-time.After(5 * time.Second): + t.Fatal("failed start blocked on a client it does not own") + } + require.False(t, attemptOwned.IsRunning(), "child started by the failed attempt kept running") + joined := make(chan struct{}) + go func() { attemptOwned.Wait(); close(joined) }() + select { + case <-joined: + case <-time.After(5 * time.Second): + t.Fatal("child started by the failed attempt was not joined") + } + require.True(t, callerOwned.IsRunning(), "failed start stopped a caller-owned client") +} From 878993cf6dcf136c1e7efed6832b5ed5dc8c1e17 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:46:08 +0000 Subject: [PATCH 18/39] fix(p2p): close an abandoned channel queue outside channelMtx When OpenChannel cannot register its routing worker, it closed and joined the new queue while still holding the channel write lock. Remove the entry under the lock and close the queue after releasing it. Co-Authored-By: Claude Opus 5.5 --- internal/p2p/router.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/p2p/router.go b/internal/p2p/router.go index 0dfff0ced0..375490a05f 100644 --- a/internal/p2p/router.go +++ b/internal/p2p/router.go @@ -259,6 +259,15 @@ type ChannelCreator func(context.Context, *ChannelDescriptor) (Channel, error) // wrapper message. The caller may provide a size to make the channel buffered, // which internally makes the inbound, outbound, and error channel buffered. func (r *Router) OpenChannel(ctx context.Context, chDesc *ChannelDescriptor) (Channel, error) { + var abandoned queue + // Registered before the unlock, so it runs after channelMtx is released: + // closing a queue joins its delivery callbacks. + defer func() { + if abandoned != nil { + abandoned.close() + <-abandoned.closed() + } + }() r.channelMtx.Lock() defer r.channelMtx.Unlock() if r.ctx == nil || r.ctx.Err() != nil { @@ -301,8 +310,7 @@ func (r *Router) OpenChannel(ctx context.Context, chDesc *ChannelDescriptor) (Ch stopCancel() cancel() delete(r.channelQueues, id) - queue.close() - <-queue.closed() + abandoned = queue return nil, errors.New("router is stopping") } From b720244449611fda58054494c8c4148658d1b31c Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:46:08 +0000 Subject: [PATCH 19/39] fix(privval): interrupt pending signer I/O before taking the endpoint lock SendRequest holds instanceMtx across signer I/O, so OnStop waited up to the read/write deadline before it could close the connection. Mirror the current connection under a separate lock and close it first; the blocked read or write then fails promptly. Co-Authored-By: Claude Opus 5.5 --- privval/signer_endpoint.go | 29 ++++++++++++++++++- privval/signer_listener_endpoint.go | 2 ++ privval/signer_listener_endpoint_test.go | 36 ++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/privval/signer_endpoint.go b/privval/signer_endpoint.go index 96ae1f5564..af3fc6737f 100644 --- a/privval/signer_endpoint.go +++ b/privval/signer_endpoint.go @@ -2,6 +2,7 @@ package privval import ( "context" + "errors" "fmt" "net" "time" @@ -24,6 +25,10 @@ type signerEndpoint struct { connMtx sync.Mutex conn net.Conn + // liveConn mirrors conn for interruptConnection, which cannot take connMtx + // while blocked I/O holds it. Lock order: connMtx, then liveMtx. + liveMtx sync.Mutex + liveConn net.Conn timeoutReadWrite time.Duration } @@ -49,6 +54,7 @@ func (se *signerEndpoint) GetAvailableConnection(connectionAvailableCh chan net. // Is there a connection ready? select { case se.conn = <-connectionAvailableCh: + se.publishConn() return true default: } @@ -64,6 +70,7 @@ func (se *signerEndpoint) WaitConnection(ctx context.Context, connectionAvailabl case <-ctx.Done(): return ctx.Err() case se.conn = <-connectionAvailableCh: + se.publishConn() case <-time.After(maxWait): return ErrConnectionTimeout } @@ -76,6 +83,7 @@ func (se *signerEndpoint) SetConnection(newConnection net.Conn) { se.connMtx.Lock() defer se.connMtx.Unlock() se.conn = newConnection + se.publishConn() } // IsConnected indicates if there is an active connection @@ -148,15 +156,34 @@ func (se *signerEndpoint) WriteMessage(msg privvalproto.Message) (err error) { return } +// interruptConnection closes the current connection without connMtx, so a +// read or write blocked while holding that lock fails promptly. +func (se *signerEndpoint) interruptConnection() { + se.liveMtx.Lock() + conn := se.liveConn + se.liveMtx.Unlock() + if conn != nil { + _ = conn.Close() + } +} + +// publishConn must be called with connMtx held after conn changes. +func (se *signerEndpoint) publishConn() { + se.liveMtx.Lock() + se.liveConn = se.conn + se.liveMtx.Unlock() +} + func (se *signerEndpoint) isConnected() bool { return se.conn != nil } func (se *signerEndpoint) dropConnection() { if se.conn != nil { - if err := se.conn.Close(); err != nil { + if err := se.conn.Close(); err != nil && !errors.Is(err, net.ErrClosed) { se.logger.Error("signerEndpoint::dropConnection", "err", err) } se.conn = nil + se.publishConn() } } diff --git a/privval/signer_listener_endpoint.go b/privval/signer_listener_endpoint.go index 70427af108..ea3679f0cd 100644 --- a/privval/signer_listener_endpoint.go +++ b/privval/signer_listener_endpoint.go @@ -90,6 +90,8 @@ func (sl *SignerListenerEndpoint) OnStart(ctx context.Context) error { // OnStop implements service.Service func (sl *SignerListenerEndpoint) OnStop() { + // A request can hold instanceMtx across signer I/O; interrupt it first. + sl.interruptConnection() sl.instanceMtx.Lock() defer sl.instanceMtx.Unlock() _ = sl.Close() diff --git a/privval/signer_listener_endpoint_test.go b/privval/signer_listener_endpoint_test.go index 74daa60fa0..d29240adf3 100644 --- a/privval/signer_listener_endpoint_test.go +++ b/privval/signer_listener_endpoint_test.go @@ -15,6 +15,7 @@ import ( "github.com/dashpay/tenderdash/libs/log" tmnet "github.com/dashpay/tenderdash/libs/net" tmrand "github.com/dashpay/tenderdash/libs/rand" + privvalproto "github.com/dashpay/tenderdash/proto/tendermint/privval" "github.com/dashpay/tenderdash/types" ) @@ -278,3 +279,38 @@ func TestSignerListenerEndpointStartupCanceledBeforeConnectRequest(t *testing.T) require.False(t, sl.IsRunning()) } } + +// A request waiting for a signer that never answers holds the endpoint mutex +// across network I/O. Stop must interrupt that I/O instead of waiting for the +// read deadline. +func TestListenerStopInterruptsPendingRequest(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + endpoint := NewSignerListenerEndpoint(log.NewNopLogger(), listener, SignerListenerEndpointTimeoutReadWrite(time.Hour)) + require.NoError(t, endpoint.Start(context.Background())) + t.Cleanup(func() { endpoint.Stop(); endpoint.Wait() }) + signer, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + defer signer.Close() + + requested := make(chan error, 1) + go func() { + _, err := endpoint.SendRequest(context.Background(), mustWrapMsg(&privvalproto.PingRequest{})) + requested <- err + }() + // The request is on the wire, so SendRequest now holds the endpoint mutex. + require.NoError(t, signer.SetReadDeadline(time.Now().Add(5*time.Second))) + _, err = signer.Read(make([]byte, 1)) + require.NoError(t, err) + + stopped := make(chan struct{}) + go func() { endpoint.Stop(); close(stopped) }() + select { + case <-stopped: + case <-time.After(5 * time.Second): + _ = signer.Close() + <-stopped + t.Fatal("Stop waited for a pending signer read") + } + require.Error(t, <-requested) +} From 2f4af1621d47b641b475f5720e2e5727b0271da3 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:15:06 +0000 Subject: [PATCH 20/39] fix(node): release node resources when startup fails BaseService never calls OnDrain after a failed OnStart, and node and seed rollback joined their children without releasing anything else. Databases, the peer manager and privval connections stayed open, and event sinks too when the indexer had not started, so a replacement node could not reopen the same on-disk stores. Rollback now drains the node, closing sinks of an indexer that never started. OnDrain runs at most once, and Start after a release returns an error asking for a new node. A failed RPC listener reservation stays retryable and keeps its resources. OnDrain also stops closing the block and state stores a second time, since shutdownOps already closes those databases. Co-Authored-By: Claude Opus 5.5 --- UPGRADING.md | 2 + docs/architecture/service-lifecycle.md | 8 +- node/node.go | 44 ++++-- node/rollback_test.go | 194 +++++++++++++++++++++++++ node/seed.go | 14 +- node/seed_lifecycle_test.go | 1 - 6 files changed, 247 insertions(+), 16 deletions(-) create mode 100644 node/rollback_test.go diff --git a/UPGRADING.md b/UPGRADING.md index 8efef7b297..b30e91b9c8 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -31,6 +31,8 @@ A stop requested during startup cancels work immediately but defers `OnStop` until successful startup returns. Failed startup must roll back its resources; BaseService joins registered work without calling shutdown hooks. Retry requires recreating any child that already started successfully and then stopped. +A node that fails after reserving its RPC listeners, or a seed node that fails +startup, closes its databases and rejects another `Start`; construct a new node. Direct consensus `State.Stop` drains a commit already in progress using the context originally passed to `State.Start`. Cancel that parent to abort it; diff --git a/docs/architecture/service-lifecycle.md b/docs/architecture/service-lifecycle.md index fcd6d12f87..e71e12866f 100644 --- a/docs/architecture/service-lifecycle.md +++ b/docs/architecture/service-lifecycle.md @@ -78,9 +78,11 @@ and finally releases application connections and stores. Arbitrary injected ABCI clients cannot be recreated by the routed wrapper. The node reserves RPC listeners before starting children, so retry after listener -failure remains supported without restarting successful child lifetimes. After -a later node startup failure, construct a fresh node graph: rollback can also -permanently close non-service resources such as mempool recheck admission. +failure remains supported without restarting successful child lifetimes, and +keeps the node's stores open for that retry. Any later node startup failure, and +any seed startup failure, is final: rollback releases stores, event sinks and +other owned resources exactly once, later `Stop` and `Wait` calls are no-ops, and +another `Start` is rejected. Construct a fresh node graph instead. ## Cancellation compatibility audit diff --git a/node/node.go b/node/node.go index 924f810ab8..79e675aaa0 100644 --- a/node/node.go +++ b/node/node.go @@ -7,6 +7,7 @@ import ( "net" "strconv" "strings" + "sync/atomic" "time" abciclient "github.com/dashpay/tenderdash/abci/client" @@ -43,6 +44,10 @@ import ( _ "net/http/pprof" //nolint: gosec // securely exposed on separate, optional port ) +// errResourcesReleased rejects a start after a failed start released the node's +// resources. +var errResourcesReleased = errors.New("failed startup released node resources; construct a new node") + // nodeImpl is the highest level interface to a full Tendermint node. // It includes all configuration information and running services. type nodeImpl struct { @@ -71,6 +76,7 @@ type nodeImpl struct { shutdownOps closer rpcEnv *rpccore.Environment dependencyCancel context.CancelFunc + released atomic.Bool // shutdownOps ran; set by OnDrain } // newDefaultNode returns a Tendermint node with default settings for the @@ -470,6 +476,9 @@ func makeNode( // OnStart starts the Node. It implements service.Service. func (n *nodeImpl) OnStart(ctx context.Context) (err error) { + if n.released.Load() { + return errResourcesReleased + } // Reserve RPC addresses before starting services; serve only once the node is ready. rpcListeners, err := rpccore.ListenRPC(n.config.RPC) if err != nil { @@ -491,10 +500,18 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { n.dependencyCancel = dependencyCancel ctx, cancelAttempt := context.WithCancel(ctx) var startedReactors []service.Service + indexerStarted := false + // Past the retryable RPC reservation, failure is final and BaseService will + // not call OnDrain. Workers it joins after this return serve only pprof and + // metrics, so they do not use the released resources. defer func() { if err != nil { cancelAttempt() n.joinChildren(startedReactors) + if !indexerStarted { + n.stopEventSinks() + } + n.OnDrain() } }() if err := n.rpcEnv.ProxyApp.Start(dependencyCtx); err != nil { @@ -526,6 +543,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) { if err := n.indexerService.Start(dependencyCtx); err != nil { return err } + indexerStarted = true // Create the handshaker, which calls RequestInfo, sets the AppVersion on the state, // and replays any blocks as necessary to sync tendermint with the app. @@ -662,23 +680,27 @@ func (n *nodeImpl) joinChildren(reactors []service.Service) { n.rpcEnv.ProxyApp.Wait() } -// OnDrain releases storage after child services and node workers have finished. +// stopEventSinks closes sinks of an indexer that never started, whose own OnDrain +// therefore cannot run. +func (n *nodeImpl) stopEventSinks() { + for _, es := range n.eventSinks { + if err := es.Stop(); err != nil { + n.logger.Error("failed to close eventsink", "eventsink", es.Type(), "err", err) + } + } +} + +// OnDrain releases storage after child services and node workers have finished, +// at most once. shutdownOps also closes the block and state store databases. func (n *nodeImpl) OnDrain() { + if !n.released.CompareAndSwap(false, true) { + return + } if err := n.shutdownOps(); err != nil { if strings.TrimSpace(err.Error()) != "" { n.logger.Error("problem shutting down additional services", "err", err) } } - if n.blockStore != nil { - if err := n.blockStore.Close(); err != nil { - n.logger.Error("problem closing blockstore", "err", err) - } - } - if n.stateStore != sm.Store(nil) { - if err := n.stateStore.Close(); err != nil { - n.logger.Error("problem closing statestore", "err", err) - } - } } func (n *nodeImpl) NodeInfo() *types.NodeInfo { diff --git a/node/rollback_test.go b/node/rollback_test.go new file mode 100644 index 0000000000..4fa1bdd5a1 --- /dev/null +++ b/node/rollback_test.go @@ -0,0 +1,194 @@ +package node + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + abciclient "github.com/dashpay/tenderdash/abci/client" + abcimocks "github.com/dashpay/tenderdash/abci/client/mocks" + abci "github.com/dashpay/tenderdash/abci/types" + "github.com/dashpay/tenderdash/config" + "github.com/dashpay/tenderdash/internal/proxy" + dbm "github.com/dashpay/tenderdash/libs/db" + "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/libs/service" + "github.com/dashpay/tenderdash/types" +) + +// dbCloseCounter counts Close calls for every database a node opens. +type dbCloseCounter struct { + mtx sync.Mutex + closes map[string]int +} + +type countedDB struct { + dbm.DB + id string + counter *dbCloseCounter +} + +func (d countedDB) Close() error { + d.counter.mtx.Lock() + d.counter.closes[d.id]++ + d.counter.mtx.Unlock() + return d.DB.Close() +} + +func (c *dbCloseCounter) provider(ctx *config.DBContext) (dbm.DB, error) { + db, err := config.DefaultDBProvider(ctx) + if err != nil { + return nil, err + } + c.mtx.Lock() + defer c.mtx.Unlock() + c.closes[ctx.ID] = 0 + return countedDB{DB: db, id: ctx.ID, counter: c}, nil +} + +func (c *dbCloseCounter) requireEachClosedOnce(t *testing.T, ids ...string) { + t.Helper() + c.mtx.Lock() + defer c.mtx.Unlock() + expected := make(map[string]int, len(ids)) + for _, id := range ids { + expected[id] = 1 + } + require.Equal(t, expected, c.closes) +} + +func rollbackTestNode( + ctx context.Context, + t *testing.T, + wrapApp func(abciclient.Client) abciclient.Client, +) (*nodeImpl, *dbCloseCounter) { + t.Helper() + cfg, err := config.ResetTestRoot(t.TempDir(), strings.ReplaceAll(t.Name(), "/", "-")) + require.NoError(t, err) + cfg.RPC.ListenAddress = "" + cfg.P2P.ListenAddress = "tcp://127.0.0.1:0" + cfg.TxIndex.Indexer = []string{"kv"} + key, err := types.LoadOrGenNodeKey(cfg.NodeKeyFile()) + require.NoError(t, err) + client, _, err := proxy.ClientFactory(log.NewNopLogger(), *cfg.Abci, cfg.DBDir()) + require.NoError(t, err) + if wrapApp != nil { + client = wrapApp(client) + } + counter := &dbCloseCounter{closes: map[string]int{}} + svc, err := makeNode(ctx, cfg, key, client, defaultGenesisDocProviderFunc(cfg), counter.provider, log.NewNopLogger()) + require.NoError(t, err) + return svc.(*nodeImpl), counter +} + +type failingStartService struct{ err error } + +func (s failingStartService) Start(context.Context) error { return s.err } +func (failingStartService) IsRunning() bool { return false } +func (failingStartService) Wait() {} + +type failingInfoClient struct { + abciclient.Client + err error +} + +func (c failingInfoClient) Info(context.Context, *abci.RequestInfo) (*abci.ResponseInfo, error) { + return nil, c.err +} + +var nodeDatabases = []string{"blockstore", "state", "peerstore", "evidence", "tx_index"} + +// A failed start past the RPC reservation is final: it closes every database +// once, whichever children had started, and later lifecycle calls change nothing. +func TestNodeFailedStartReleasesDatabasesOnce(t *testing.T) { + failure := errors.New("startup failed") + testCases := map[string]func(*testing.T, *nodeImpl){ + "application start": func(t *testing.T, n *nodeImpl) { + n.rpcEnv.ProxyApp = failedStartClient{Client: abcimocks.NewClient(t), err: failure} + }, + "reactor start": func(_ *testing.T, n *nodeImpl) { + n.services = append(n.services, failingStartService{err: failure}) + }, + } + for name, inject := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, counter := rollbackTestNode(ctx, t, nil) + inject(t, n) + + require.ErrorIs(t, n.Start(ctx), failure) + counter.requireEachClosedOnce(t, nodeDatabases...) + + n.Stop() + n.Wait() + n.OnDrain() + require.ErrorContains(t, n.Start(ctx), "construct a new node") + counter.requireEachClosedOnce(t, nodeDatabases...) + }) + } + + t.Run("handshake", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, counter := rollbackTestNode(ctx, t, func(app abciclient.Client) abciclient.Client { + return failingInfoClient{Client: app, err: failure} + }) + + require.ErrorContains(t, n.Start(ctx), failure.Error()) + counter.requireEachClosedOnce(t, nodeDatabases...) + n.Wait() + counter.requireEachClosedOnce(t, nodeDatabases...) + }) +} + +type failedStartClient struct { + abciclient.Client + err error +} + +func (c failedStartClient) Start(context.Context) error { return c.err } +func (failedStartClient) Wait() {} + +// Normal shutdown has the same exactly-once guarantee. +func TestNodeShutdownClosesEachDatabaseOnce(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, counter := rollbackTestNode(ctx, t, nil) + require.NoError(t, n.Start(ctx)) + cancel() + n.Wait() + counter.requireEachClosedOnce(t, nodeDatabases...) +} + +func TestSeedFailedStartReleasesDatabaseOnce(t *testing.T) { + cfg, err := config.ResetTestRoot(t.TempDir(), t.Name()) + require.NoError(t, err) + cfg.Mode = config.ModeSeed + cfg.P2P.ListenAddress = "tcp://127.0.0.1:0" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + key, err := types.LoadOrGenNodeKey(cfg.NodeKeyFile()) + require.NoError(t, err) + counter := &dbCloseCounter{closes: map[string]int{}} + svc, err := makeSeedNode(ctx, log.NewNopLogger(), cfg, counter.provider, key, defaultGenesisDocProviderFunc(cfg)) + require.NoError(t, err) + n := svc.(*seedNodeImpl) + failure := errors.New("PEX startup failed") + pex := &failedSeedPEX{err: failure} + pex.BaseService = *service.NewBaseService(log.NewNopLogger(), "test-pex", pex) + n.pexReactor = pex + + require.ErrorIs(t, n.Start(ctx), failure) + counter.requireEachClosedOnce(t, "peerstore") + + n.Stop() + n.Wait() + n.OnDrain() + require.ErrorContains(t, n.Start(ctx), "construct a new node") + counter.requireEachClosedOnce(t, "peerstore") +} diff --git a/node/seed.go b/node/seed.go index b24f2c9df3..a1e63d2ce6 100644 --- a/node/seed.go +++ b/node/seed.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "sync/atomic" "time" "github.com/dashpay/tenderdash/config" @@ -34,6 +35,7 @@ type seedNodeImpl struct { pexReactor service.Service // for exchanging peer addresses shutdownOps closer + released atomic.Bool // shutdownOps ran; set by OnDrain } // makeSeedNode returns a new seed node, containing only p2p, pex reactor @@ -96,12 +98,18 @@ func makeSeedNode( // OnStart starts the Seed Node. It implements service.Service. func (n *seedNodeImpl) OnStart(ctx context.Context) (err error) { + if n.released.Load() { + return errResourcesReleased + } ctx, cancelAttempt := context.WithCancel(ctx) + // Failure is final and BaseService will not call OnDrain. Workers it joins + // after this return serve only pprof and metrics. defer func() { if err != nil { cancelAttempt() n.pexReactor.Wait() n.router.Wait() + n.OnDrain() } }() if n.config.RPC.PprofListenAddress != "" { @@ -141,8 +149,12 @@ func (n *seedNodeImpl) OnStop() { n.router.Wait() } -// OnDrain releases resources after network and HTTP workers have finished. +// OnDrain releases resources after network and HTTP workers have finished, at +// most once. func (n *seedNodeImpl) OnDrain() { + if !n.released.CompareAndSwap(false, true) { + return + } if err := n.shutdownOps(); err != nil { if strings.TrimSpace(err.Error()) != "" { n.logger.Error("problem shutting down additional services", "err", err) diff --git a/node/seed_lifecycle_test.go b/node/seed_lifecycle_test.go index 4f9a65ea13..3613d52fef 100644 --- a/node/seed_lifecycle_test.go +++ b/node/seed_lifecycle_test.go @@ -55,7 +55,6 @@ func TestSeedFailedStartDrainsRouter(t *testing.T) { svc, err := makeSeedNode(ctx, log.NewNopLogger(), cfg, config.DefaultDBProvider, key, defaultGenesisDocProviderFunc(cfg)) require.NoError(t, err) n := svc.(*seedNodeImpl) - defer func() { require.NoError(t, n.shutdownOps()) }() hooks := &seedRouterDrain{router: n.router, stopping: make(chan struct{}), release: make(chan struct{})} n.router.BaseService = service.NewBaseService(log.NewNopLogger(), "test-router", hooks) failure := errors.New("PEX startup failed") From 63552188b993505e1fbf2d10ea875642d040eeb4 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:28:04 +0000 Subject: [PATCH 21/39] test(node): pin failed-start rollback ordering and document its limits Node and seed rollback must join their children before closing databases, and an indexer whose own start fails must still have its sinks closed. Each new test fails under the matching mutation of the rollback code. Document what a failed start leaves behind: rebuilding a seed, or a node with Prometheus enabled, in the same process panics on duplicate metrics registration unless the namespace differs; an RPC listener bind failure keeps the stores open with no service call to release them; and a full node's Dash Core RPC client has no closer. Co-Authored-By: Claude Opus 5.5 --- UPGRADING.md | 5 ++ docs/architecture/service-lifecycle.md | 19 ++-- node/rollback_test.go | 116 +++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 5 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index b30e91b9c8..0b87675b6d 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -33,6 +33,11 @@ BaseService joins registered work without calling shutdown hooks. Retry requires recreating any child that already started successfully and then stopped. A node that fails after reserving its RPC listeners, or a seed node that fails startup, closes its databases and rejects another `Start`; construct a new node. +In the same process that panics with `duplicate metrics collector registration +attempted` for a seed, or for a node with Prometheus enabled, unless +`Instrumentation.Namespace` differs. An RPC listener bind failure stays retryable +and keeps the stores open, with no `service.Service` call to release them if you +give up. Rollback and shutdown do not release a full node's Dash Core RPC client. Direct consensus `State.Stop` drains a commit already in progress using the context originally passed to `State.Start`. Cancel that parent to abort it; diff --git a/docs/architecture/service-lifecycle.md b/docs/architecture/service-lifecycle.md index e71e12866f..97d8c31726 100644 --- a/docs/architecture/service-lifecycle.md +++ b/docs/architecture/service-lifecycle.md @@ -78,11 +78,20 @@ and finally releases application connections and stores. Arbitrary injected ABCI clients cannot be recreated by the routed wrapper. The node reserves RPC listeners before starting children, so retry after listener -failure remains supported without restarting successful child lifetimes, and -keeps the node's stores open for that retry. Any later node startup failure, and -any seed startup failure, is final: rollback releases stores, event sinks and -other owned resources exactly once, later `Stop` and `Wait` calls are no-ops, and -another `Start` is rejected. Construct a fresh node graph instead. +failure remains supported without restarting successful child lifetimes. That +failure keeps the node's stores open for the retry, and `service.Service` offers +no way to release them if the caller gives up instead. + +Any later node startup failure, and any seed startup failure, is final: rollback +closes the databases, event sinks and other resources registered as node closers +exactly once, later `Stop` and `Wait` calls are no-ops, and another `Start` is +rejected. The Dash Core RPC client created for a full node has no closer, so it is +not released by rollback or by shutdown. + +Recover from a final failure by constructing a fresh node graph. Metrics register +on the process-wide default registry, so a second build in the same process panics +with `duplicate metrics collector registration attempted` for a seed, or for a +node with Prometheus enabled, unless `Instrumentation.Namespace` differs. ## Cancellation compatibility audit diff --git a/node/rollback_test.go b/node/rollback_test.go index 4fa1bdd5a1..b0b49758e8 100644 --- a/node/rollback_test.go +++ b/node/rollback_test.go @@ -3,9 +3,11 @@ package node import ( "context" "errors" + "maps" "strings" "sync" "testing" + "time" "github.com/stretchr/testify/require" @@ -14,6 +16,7 @@ import ( abci "github.com/dashpay/tenderdash/abci/types" "github.com/dashpay/tenderdash/config" "github.com/dashpay/tenderdash/internal/proxy" + "github.com/dashpay/tenderdash/internal/pubsub" dbm "github.com/dashpay/tenderdash/libs/db" "github.com/dashpay/tenderdash/libs/log" "github.com/dashpay/tenderdash/libs/service" @@ -50,6 +53,18 @@ func (c *dbCloseCounter) provider(ctx *config.DBContext) (dbm.DB, error) { return countedDB{DB: db, id: ctx.ID, counter: c}, nil } +func (c *dbCloseCounter) closed(id string) int { + c.mtx.Lock() + defer c.mtx.Unlock() + return c.closes[id] +} + +func (c *dbCloseCounter) snapshot() map[string]int { + c.mtx.Lock() + defer c.mtx.Unlock() + return maps.Clone(c.closes) +} + func (c *dbCloseCounter) requireEachClosedOnce(t *testing.T, ids ...string) { t.Helper() c.mtx.Lock() @@ -154,6 +169,43 @@ type failedStartClient struct { func (c failedStartClient) Start(context.Context) error { return c.err } func (failedStartClient) Wait() {} +type waitProbeClient struct { + abciclient.Client + onWait func() +} + +func (c waitProbeClient) Wait() { c.onWait(); c.Client.Wait() } + +// The application client is the last child joined, so no node-owned database +// may be closed yet. Only the indexer, already joined, has closed its sink. +func TestNodeFailedStartReleasesDatabasesAfterJoiningChildren(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, counter := rollbackTestNode(ctx, t, nil) + failure := errors.New("startup failed") + n.services = append(n.services, failingStartService{err: failure}) + var atLastJoin map[string]int + n.rpcEnv.ProxyApp = waitProbeClient{ + Client: n.rpcEnv.ProxyApp, + onWait: func() { atLastJoin = counter.snapshot() }, + } + + require.ErrorIs(t, n.Start(ctx), failure) + require.Equal(t, map[string]int{"blockstore": 0, "state": 0, "peerstore": 0, "evidence": 0, "tx_index": 1}, atLastJoin) + counter.requireEachClosedOnce(t, nodeDatabases...) +} + +// An indexer whose own start fails never drains, so the node closes its sinks. +func TestNodeIndexerStartFailureClosesEventSinks(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, counter := rollbackTestNode(ctx, t, nil) + require.NoError(t, n.rpcEnv.EventBus.Observe(ctx, func(pubsub.Message) error { return nil })) + + require.ErrorContains(t, n.Start(ctx), "an observer is already registered") + counter.requireEachClosedOnce(t, nodeDatabases...) +} + // Normal shutdown has the same exactly-once guarantee. func TestNodeShutdownClosesEachDatabaseOnce(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) @@ -192,3 +244,67 @@ func TestSeedFailedStartReleasesDatabaseOnce(t *testing.T) { require.ErrorContains(t, n.Start(ctx), "construct a new node") counter.requireEachClosedOnce(t, "peerstore") } + +type waitProbeService struct { + service.Service + onWait func() +} + +func (s waitProbeService) Wait() { s.Service.Wait(); s.onWait() } + +// Seed rollback joins PEX, then the router, and only then closes the peer store. +func TestSeedFailedStartReleasesDatabaseAfterJoiningChildren(t *testing.T) { + cfg, err := config.ResetTestRoot(t.TempDir(), t.Name()) + require.NoError(t, err) + cfg.Mode = config.ModeSeed + cfg.P2P.ListenAddress = "tcp://127.0.0.1:0" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + key, err := types.LoadOrGenNodeKey(cfg.NodeKeyFile()) + require.NoError(t, err) + counter := &dbCloseCounter{closes: map[string]int{}} + svc, err := makeSeedNode(ctx, log.NewNopLogger(), cfg, counter.provider, key, defaultGenesisDocProviderFunc(cfg)) + require.NoError(t, err) + n := svc.(*seedNodeImpl) + hooks := &seedRouterDrain{router: n.router, stopping: make(chan struct{}), release: make(chan struct{})} + n.router.BaseService = service.NewBaseService(log.NewNopLogger(), "test-router", hooks) + failure := errors.New("PEX startup failed") + pex := &failedSeedPEX{err: failure} + pex.BaseService = *service.NewBaseService(log.NewNopLogger(), "test-pex", pex) + closesAtPEXJoin := make(chan int, 1) + n.pexReactor = waitProbeService{Service: pex, onWait: func() { closesAtPEXJoin <- counter.closed("peerstore") }} + + returned := make(chan error, 1) + go func() { returned <- n.Start(ctx) }() + releaseRouter := sync.OnceFunc(func() { close(hooks.release) }) + defer func() { releaseRouter(); n.router.Wait(); n.Wait() }() + + select { + case closes := <-closesAtPEXJoin: + require.Zero(t, closes, "peer store closed before PEX was joined") + case <-time.After(5 * time.Second): + t.Fatal("rollback did not join PEX") + } + select { + case <-hooks.stopping: + case <-time.After(5 * time.Second): + t.Fatal("router was not canceled") + } + // The router worker cannot exit until released, so rollback must still be + // blocked joining it. The window only catches a close that arrives early. + select { + case err := <-returned: + t.Fatalf("Start returned before router cleanup: %v", err) + case <-time.After(50 * time.Millisecond): + } + require.Zero(t, counter.closed("peerstore"), "peer store closed before the router was joined") + + releaseRouter() + select { + case err := <-returned: + require.ErrorIs(t, err, failure) + case <-time.After(5 * time.Second): + t.Fatal("Start did not finish after router drained") + } + require.Equal(t, 1, counter.closed("peerstore")) +} From 91884d15eba11c275ef1459f3441b5ab554487bc Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:27:41 +0000 Subject: [PATCH 22/39] fix(abci): cancel routed children before joining them on failed start Client does not require Stop, and BaseService cancels the attempt context only after OnStart returns. A routed child that hides Stop, such as a type embedding Client, therefore never stopped during rollback, and joining it blocked the failed start forever. Each start attempt now gives its children a cancellable context, canceled before any rollback join. Caller-owned clients are not started by the attempt, so they keep their own lifetime. Co-Authored-By: Claude Opus 5.5 --- abci/client/routed_client.go | 8 +++- abci/client/routed_lifecycle_test.go | 67 ++++++++++++++++------------ 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/abci/client/routed_client.go b/abci/client/routed_client.go index f7ace49d03..3380e75d99 100644 --- a/abci/client/routed_client.go +++ b/abci/client/routed_client.go @@ -136,12 +136,15 @@ func NewRoutedClient(logger log.Logger, defaultClient Client, routing Routing) ( // clients the caller already started keep their own lifetime. func (cli *routedClient) OnStart(ctx context.Context) error { cli.owned = nil + // Client does not require Stop, and BaseService cancels ctx only after OnStart + // returns, so this context is what ends a child before a rollback joins it. + childCtx, cancelChildren := context.WithCancel(ctx) var errs error for _, client := range cli.clients() { if client.IsRunning() { continue } - if err := client.Start(ctx); err != nil { + if err := client.Start(childCtx); err != nil { errs = multierror.Append(errs, err) continue } @@ -149,16 +152,19 @@ func (cli *routedClient) OnStart(ctx context.Context) error { } // A failed attempt gets no OnStop; release what it started before returning. if errs != nil { + cancelChildren() cli.stopOwned() return errs } if !cli.Go(ctx, func(ctx context.Context) { + defer cancelChildren() <-ctx.Done() for _, client := range cli.owned { client.Wait() } }) { + cancelChildren() cli.stopOwned() } return nil diff --git a/abci/client/routed_lifecycle_test.go b/abci/client/routed_lifecycle_test.go index e1f94465f5..633789310f 100644 --- a/abci/client/routed_lifecycle_test.go +++ b/abci/client/routed_lifecycle_test.go @@ -21,37 +21,48 @@ func (failingStartClient) Start(context.Context) error { return errChildStart } func (failingStartClient) IsRunning() bool { return false } func (failingStartClient) Wait() {} +// noStopClient satisfies Client while hiding Stop, which Client does not require. +type noStopClient struct{ abciclient.Client } + // A failed start must stop and join the children it started, and leave alone // a client the caller had already started. func TestRoutedClientFailedStartReleasesOnlyItsChildren(t *testing.T) { - logger := log.NewNopLogger() - callerCtx, cancelCaller := context.WithCancel(context.Background()) - defer cancelCaller() - callerOwned := abciclient.NewLocalClient(logger, types.NewBaseApplication()) - require.NoError(t, callerOwned.Start(callerCtx)) - attemptOwned := abciclient.NewLocalClient(logger, types.NewBaseApplication()) - routing := abciclient.Routing{ - "Info": {{Client: callerOwned, ClientID: "caller"}}, - "Query": {{Client: attemptOwned, ClientID: "attempt"}}, - } - routed, err := abciclient.NewRoutedClient(logger, failingStartClient{}, routing) - require.NoError(t, err) - - started := make(chan error, 1) - go func() { started <- routed.Start(context.Background()) }() - select { - case err := <-started: - require.ErrorIs(t, err, errChildStart) - case <-time.After(5 * time.Second): - t.Fatal("failed start blocked on a client it does not own") + testCases := map[string]func(abciclient.Client) abciclient.Client{ + "child exposes Stop": func(c abciclient.Client) abciclient.Client { return c }, + "child hides Stop": func(c abciclient.Client) abciclient.Client { return noStopClient{c} }, } - require.False(t, attemptOwned.IsRunning(), "child started by the failed attempt kept running") - joined := make(chan struct{}) - go func() { attemptOwned.Wait(); close(joined) }() - select { - case <-joined: - case <-time.After(5 * time.Second): - t.Fatal("child started by the failed attempt was not joined") + for name, wrap := range testCases { + t.Run(name, func(t *testing.T) { + logger := log.NewNopLogger() + callerCtx, cancelCaller := context.WithCancel(context.Background()) + defer cancelCaller() + callerOwned := abciclient.NewLocalClient(logger, types.NewBaseApplication()) + require.NoError(t, callerOwned.Start(callerCtx)) + attemptOwned := abciclient.NewLocalClient(logger, types.NewBaseApplication()) + routing := abciclient.Routing{ + "Info": {{Client: callerOwned, ClientID: "caller"}}, + "Query": {{Client: wrap(attemptOwned), ClientID: "attempt"}}, + } + routed, err := abciclient.NewRoutedClient(logger, failingStartClient{}, routing) + require.NoError(t, err) + + started := make(chan error, 1) + go func() { started <- routed.Start(context.Background()) }() + select { + case err := <-started: + require.ErrorIs(t, err, errChildStart) + case <-time.After(5 * time.Second): + t.Fatal("failed start blocked joining a client") + } + require.False(t, attemptOwned.IsRunning(), "child started by the failed attempt kept running") + joined := make(chan struct{}) + go func() { attemptOwned.Wait(); close(joined) }() + select { + case <-joined: + case <-time.After(5 * time.Second): + t.Fatal("child started by the failed attempt was not joined") + } + require.True(t, callerOwned.IsRunning(), "failed start stopped a caller-owned client") + }) } - require.True(t, callerOwned.IsRunning(), "failed start stopped a caller-owned client") } From f68fd08590488d48bd82e3ef608b57a6cb4a9974 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:32:52 +0000 Subject: [PATCH 23/39] test(abci): assert the failed routed attempt started its child The rollback test never checked that the attempt-owned client had started, so stopping the start loop at the first failure skipped the join it is meant to exercise and still passed. Co-Authored-By: Claude Opus 5.5 --- abci/client/routed_lifecycle_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/abci/client/routed_lifecycle_test.go b/abci/client/routed_lifecycle_test.go index 633789310f..7a1ee706fa 100644 --- a/abci/client/routed_lifecycle_test.go +++ b/abci/client/routed_lifecycle_test.go @@ -62,6 +62,9 @@ func TestRoutedClientFailedStartReleasesOnlyItsChildren(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("child started by the failed attempt was not joined") } + // Only a client that started and stopped refuses another start. + require.ErrorContains(t, attemptOwned.Start(callerCtx), "already stopped", + "the failed attempt never started this child") require.True(t, callerOwned.IsRunning(), "failed start stopped a caller-owned client") }) } From 8beb2f1d970f39ccece01ab2b0e0450144d2fe9e Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:19:38 +0000 Subject: [PATCH 24/39] fix(light): bound proxy disconnect cleanup when only the listener closes Closing the proxy listener cancels the server's own context, not the one captured by the disconnect callback. With the upstream client's writer stopped, UnsubscribeAll then blocked forever and Serve never finished joining the session. The callback now gives the cleanup a one-second timeout and still logs the error while the proxy context is live, so a timed-out cleanup stays visible. Co-Authored-By: Claude Opus 5.5 --- light/proxy/proxy.go | 13 +++- light/proxy/shutdown_test.go | 128 ++++++++++++++++++++++++----------- 2 files changed, 99 insertions(+), 42 deletions(-) diff --git a/light/proxy/proxy.go b/light/proxy/proxy.go index 05fec7afd6..b7963bd4c1 100644 --- a/light/proxy/proxy.go +++ b/light/proxy/proxy.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "net/http" + "time" tmpubsub "github.com/dashpay/tenderdash/internal/pubsub" rpccore "github.com/dashpay/tenderdash/internal/rpc/core" @@ -15,6 +16,10 @@ import ( rpcserver "github.com/dashpay/tenderdash/rpc/jsonrpc/server" ) +// disconnectCleanupTimeout bounds unsubscribing a closed WebSocket session from +// the upstream client, which only has to queue the request. +const disconnectCleanupTimeout = time.Second + // A Proxy defines parameters for running an HTTP server proxy. type Proxy struct { Addr string // TCP address to listen on, ":http" if empty @@ -103,9 +108,11 @@ func (p *Proxy) listen(ctx context.Context) (net.Listener, *http.ServeMux, error wmLogger := p.Logger.With("protocol", "websocket") wm := rpcserver.NewWebsocketManager(wmLogger, r, rpcserver.OnDisconnect(func(remoteAddr string) { - // Serve joins this callback. Bound it by the proxy lifetime: once the - // upstream client stops, its send has no receiver. - err := p.Client.UnsubscribeAll(ctx, remoteAddr) + // Serve joins this callback, and closing the listener does not end ctx. + // Bound it: once the upstream client stops, its send has no receiver. + cleanupCtx, cancel := context.WithTimeout(ctx, disconnectCleanupTimeout) + defer cancel() + err := p.Client.UnsubscribeAll(cleanupCtx, remoteAddr) if err != nil && err != tmpubsub.ErrSubscriptionNotFound && ctx.Err() == nil { wmLogger.Error("Failed to unsubscribe addr from events", "addr", remoteAddr, "err", err) } diff --git a/light/proxy/shutdown_test.go b/light/proxy/shutdown_test.go index e0a7e5f7f9..7a6f867560 100644 --- a/light/proxy/shutdown_test.go +++ b/light/proxy/shutdown_test.go @@ -1,9 +1,12 @@ package proxy import ( + "bytes" "context" + "net" "net/http" "net/http/httptest" + "sync" "testing" "time" @@ -16,46 +19,93 @@ import ( rpcserver "github.com/dashpay/tenderdash/rpc/jsonrpc/server" ) +type lockedBuffer struct { + mtx sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mtx.Lock() + defer b.mtx.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mtx.Lock() + defer b.mtx.Unlock() + return b.buf.String() +} + // Serve joins WebSocket disconnect cleanup. With the upstream client's writer -// already stopped, that cleanup must not block proxy shutdown. +// already stopped, that cleanup must not block proxy shutdown, whether the proxy +// context ends or only the listener closes and leaves that context live. func TestProxyShutdownWithStoppedUpstreamClient(t *testing.T) { - logger := log.NewNopLogger() - upstreamWS := rpcserver.NewWebsocketManager(logger, map[string]*rpcserver.RPCFunc{}) - upstreamMux := http.NewServeMux() - upstreamMux.HandleFunc("/websocket", upstreamWS.WebsocketHandler) - upstream := httptest.NewServer(upstreamMux) - t.Cleanup(upstream.Close) - - next, err := rpchttp.New(upstream.URL) - require.NoError(t, err) - client := lrpc.NewClient(logger, next, nil) - clientCtx, stopClient := context.WithCancel(context.Background()) - t.Cleanup(stopClient) - require.NoError(t, client.Start(clientCtx)) - - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - p := &Proxy{Addr: "tcp://127.0.0.1:0", Config: rpcserver.DefaultConfig(), Client: client, Logger: logger} - listener, mux, err := p.listen(ctx) - require.NoError(t, err) - served := make(chan error, 1) - go func() { served <- rpcserver.Serve(ctx, listener, mux, logger, p.Config) }() - - dialer := websocket.Dialer{HandshakeTimeout: 5 * time.Second} - conn, resp, err := dialer.Dial("ws://"+listener.Addr().String()+"/websocket", nil) - require.NoError(t, err) - require.NoError(t, resp.Body.Close()) - t.Cleanup(func() { _ = conn.Close() }) - - // Stop the upstream client first; Wait returns after its writer exits. - stopClient() - client.Wait() - - cancel() - select { - case err := <-served: - require.ErrorIs(t, err, http.ErrServerClosed) - case <-time.After(5 * time.Second): - t.Fatal("proxy shutdown blocked on disconnect cleanup through a stopped upstream client") + testCases := map[string]struct { + shutdown func(t *testing.T, cancel context.CancelFunc, listener net.Listener) + serveErr error + cleanupErrLogged bool + }{ + "proxy context canceled": { + shutdown: func(_ *testing.T, cancel context.CancelFunc, _ net.Listener) { cancel() }, + serveErr: http.ErrServerClosed, + }, + "listener closed": { + shutdown: func(t *testing.T, _ context.CancelFunc, listener net.Listener) { + require.NoError(t, listener.Close()) + }, + serveErr: net.ErrClosed, + cleanupErrLogged: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + logger := log.NewNopLogger() + upstreamWS := rpcserver.NewWebsocketManager(logger, map[string]*rpcserver.RPCFunc{}) + upstreamMux := http.NewServeMux() + upstreamMux.HandleFunc("/websocket", upstreamWS.WebsocketHandler) + upstream := httptest.NewServer(upstreamMux) + t.Cleanup(upstream.Close) + + next, err := rpchttp.New(upstream.URL) + require.NoError(t, err) + client := lrpc.NewClient(logger, next, nil) + clientCtx, stopClient := context.WithCancel(context.Background()) + t.Cleanup(stopClient) + require.NoError(t, client.Start(clientCtx)) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + logs := &lockedBuffer{} + proxyLogger, err := log.NewLogger(log.LogLevelError, logs) + require.NoError(t, err) + p := &Proxy{Addr: "tcp://127.0.0.1:0", Config: rpcserver.DefaultConfig(), Client: client, Logger: proxyLogger} + listener, mux, err := p.listen(ctx) + require.NoError(t, err) + served := make(chan error, 1) + go func() { served <- rpcserver.Serve(ctx, listener, mux, logger, p.Config) }() + + dialer := websocket.Dialer{HandshakeTimeout: 5 * time.Second} + conn, resp, err := dialer.Dial("ws://"+listener.Addr().String()+"/websocket", nil) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + t.Cleanup(func() { _ = conn.Close() }) + + // Stop the upstream client first; Wait returns after its writer exits. + stopClient() + client.Wait() + + tc.shutdown(t, cancel, listener) + select { + case err := <-served: + require.ErrorIs(t, err, tc.serveErr) + case <-time.After(5 * time.Second): + t.Fatal("proxy shutdown blocked on disconnect cleanup through a stopped upstream client") + } + if tc.cleanupErrLogged { + require.Contains(t, logs.String(), "Failed to unsubscribe addr from events") + } else { + require.NotContains(t, logs.String(), "Failed to unsubscribe addr from events") + } + }) } } From 96129a961935d847fdd83deb0a1346faa23b04f6 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:31:46 +0000 Subject: [PATCH 25/39] fix(rpc): forget subscriptions before UnsubscribeAll sends upstream A send that timed out returned before the map was cleared. If the upstream client was reconnecting, the reconnect restored the subscription, its unread channel filled and the single events loop stalled every other subscriber. UnsubscribeAll now forgets first, and a replaced signal channel releases a delivery already blocked on a forgotten channel. Resubscribing no longer holds the lock while it sends, so it cannot stall UnsubscribeAll, and it skips queries forgotten meanwhile. A query forgotten during its own send stays subscribed upstream until the next reconnect; its events are dropped. Co-Authored-By: Claude Opus 5.5 --- rpc/client/http/ws.go | 46 +++++-- rpc/client/http/ws_test.go | 241 +++++++++++++++++++++++++++++++++++++ 2 files changed, 275 insertions(+), 12 deletions(-) create mode 100644 rpc/client/http/ws_test.go diff --git a/rpc/client/http/ws.go b/rpc/client/http/ws.go index 3b43a7ee9b..621d145022 100644 --- a/rpc/client/http/ws.go +++ b/rpc/client/http/ws.go @@ -23,6 +23,9 @@ type wsEvents struct { mtx sync.RWMutex subscriptions map[string]*wsSubscription + // cleared closes when UnsubscribeAll forgets the subscriptions read with it, + // and is then replaced. Both fields are guarded by mtx. + cleared chan struct{} } type wsSubscription struct { @@ -37,6 +40,7 @@ func newWsEvents(remote string) (*wsEvents, error) { w := &wsEvents{ Logger: log.NewNopLogger(), subscriptions: make(map[string]*wsSubscription), + cleared: make(chan struct{}), } var err error @@ -123,17 +127,17 @@ func (w *wsEvents) Unsubscribe(ctx context.Context, _subscriber, query string) e // UnsubscribeAll implements SubscriptionClient by using WSClient to // unsubscribe given subscriber from all the queries. // -// It returns an error if wsEvents is not running. +// It returns an error if wsEvents is not running. Subscriptions are forgotten +// even then: a reconnect must not restore one whose channel nobody reads, which +// would stall every other subscription on the single events loop. func (w *wsEvents) UnsubscribeAll(ctx context.Context, _subscriber string) error { - if err := w.ws.UnsubscribeAll(ctx); err != nil { - return err - } - w.mtx.Lock() - defer w.mtx.Unlock() w.subscriptions = make(map[string]*wsSubscription) + close(w.cleared) + w.cleared = make(chan struct{}) + w.mtx.Unlock() - return nil + return w.ws.UnsubscribeAll(ctx) } // After being reconnected, it is necessary to redo subscription to server @@ -143,17 +147,34 @@ func (w *wsEvents) redoSubscriptionsAfter(d time.Duration) { ctx := context.TODO() + // Sending can block until the writer restarts, so no lock is held across it. + // eventListener writes to the map under RLock, so reading it here needs Lock. w.mtx.Lock() - defer w.mtx.Unlock() - + queries := make([]string, 0, len(w.subscriptions)) for q, info := range w.subscriptions { if q != "" && q == info.id { continue } - err := w.ws.Subscribe(ctx, q) - if err != nil { + queries = append(queries, q) + } + w.mtx.Unlock() + + for _, q := range queries { + w.mtx.Lock() + _, ok := w.subscriptions[q] + w.mtx.Unlock() + if !ok { + continue + } + // If q is forgotten during this send, it stays subscribed upstream until + // the next reconnect. That is harmless: eventListener drops events for + // unknown queries, and subscribing to q again gets ErrAlreadySubscribed, + // which it ignores. + if err := w.ws.Subscribe(ctx, q); err != nil { w.Logger.Error("failed to resubscribe", "query", q, "err", err) + w.mtx.Lock() delete(w.subscriptions, q) + w.mtx.Unlock() } } } @@ -199,11 +220,12 @@ func (w *wsEvents) eventListener(ctx context.Context) { w.subscriptions[result.SubscriptionID] = out } } - + cleared := w.cleared w.mtx.RUnlock() if ok { select { case out.res <- *result: + case <-cleared: case <-ctx.Done(): return } diff --git a/rpc/client/http/ws_test.go b/rpc/client/http/ws_test.go new file mode 100644 index 0000000000..9e09002985 --- /dev/null +++ b/rpc/client/http/ws_test.go @@ -0,0 +1,241 @@ +package http + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/rpc/coretypes" + rpctypes "github.com/dashpay/tenderdash/rpc/jsonrpc/types" + "github.com/dashpay/tenderdash/types" +) + +// fakeUpstream replies to each subscribe with the events scripted for its query. +type fakeUpstream struct { + mtx sync.Mutex + events map[string][]string + conn *websocket.Conn + subscribes []string +} + +func (u *fakeUpstream) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + upgrader := websocket.Upgrader{} + conn, err := upgrader.Upgrade(rw, r, nil) + if err != nil { + return + } + defer conn.Close() + u.mtx.Lock() + u.conn = conn + u.subscribes = nil + u.mtx.Unlock() + for { + var req rpctypes.RPCRequest + if err := conn.ReadJSON(&req); err != nil { + return + } + if req.Method != "subscribe" { + continue + } + var params struct { + Query string `json:"query"` + } + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + return + } + u.mtx.Lock() + u.subscribes = append(u.subscribes, params.Query) + events := u.events[params.Query] + u.mtx.Unlock() + for _, query := range events { + res := req.MakeResponse(coretypes.ResultEvent{ + Query: query, + SubscriptionID: "id-" + query, + Data: types.EventDataNewRound{}, + }) + if err := conn.WriteJSON(res); err != nil { + return + } + } + } +} + +func (u *fakeUpstream) setEvents(events map[string][]string) { + u.mtx.Lock() + defer u.mtx.Unlock() + u.events = events +} + +// dropConnection ends the session abruptly, which makes the client reconnect. +func (u *fakeUpstream) dropConnection(t *testing.T) { + t.Helper() + u.mtx.Lock() + defer u.mtx.Unlock() + require.NoError(t, u.conn.Close()) +} + +func (u *fakeUpstream) subscribesOnCurrentConnection() []string { + u.mtx.Lock() + defer u.mtx.Unlock() + return append([]string(nil), u.subscribes...) +} + +func startWsEvents(ctx context.Context, t *testing.T, upstream *fakeUpstream) *wsEvents { + t.Helper() + srv := httptest.NewServer(upstream) + t.Cleanup(srv.Close) + w, err := newWsEvents(srv.URL) + require.NoError(t, err) + require.NoError(t, w.Start(ctx)) + return w +} + +// reconnecting drops the session and returns once the writer has stopped. The +// first redial happens at least a second later. +func reconnecting(t *testing.T, w *wsEvents, upstream *fakeUpstream) { + t.Helper() + upstream.dropConnection(t) + require.Eventually(t, w.ws.IsReconnecting, 5*time.Second, time.Millisecond) +} + +func requireEvent(t *testing.T, out <-chan coretypes.ResultEvent, query string) { + t.Helper() + select { + case event := <-out: + require.Equal(t, query, event.Query) + case <-time.After(5 * time.Second): + t.Fatalf("event for %q was not delivered", query) + } +} + +// A cleanup that times out during a reconnect must still forget the subscription, +// or the reconnect restores it and its unread channel stalls every other event. +func TestUnsubscribeAllTimeoutDuringReconnectForgetsSubscriptions(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{events: map[string][]string{ + "other": {"gone", "gone", "gone", "other"}, + }} + w := startWsEvents(ctx, t, upstream) + _, err := w.Subscribe(ctx, "", "gone") + require.NoError(t, err) + require.Eventually(t, func() bool { return len(upstream.subscribesOnCurrentConnection()) == 1 }, + 5*time.Second, time.Millisecond) + + reconnecting(t, w, upstream) + cleanupCtx, cancelCleanup := context.WithTimeout(ctx, 10*time.Millisecond) + defer cancelCleanup() + require.ErrorIs(t, w.UnsubscribeAll(cleanupCtx, ""), context.DeadlineExceeded) + + other, err := w.Subscribe(ctx, "", "other") + require.NoError(t, err) + requireEvent(t, other, "other") + require.NotContains(t, upstream.subscribesOnCurrentConnection(), "gone", + "the reconnect restored a subscription whose cleanup timed out") +} + +// Forgetting a subscription must also release a delivery blocked on its unread channel. +func TestUnsubscribeAllReleasesBlockedDelivery(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{events: map[string][]string{ + "gone": {"gone", "gone", "gone"}, + "other": {"other"}, + }} + w := startWsEvents(ctx, t, upstream) + gone, err := w.Subscribe(ctx, "", "gone") + require.NoError(t, err) + // Taking one event lets the next fill the buffer, so the third has to block. + requireEvent(t, gone, "gone") + require.Eventually(t, func() bool { return len(gone) == 1 }, 5*time.Second, time.Millisecond) + + require.NoError(t, w.UnsubscribeAll(ctx, "")) + other, err := w.Subscribe(ctx, "", "other") + require.NoError(t, err) + requireEvent(t, other, "other") +} + +// A clear landing during a resubscribe must not wait for it, and stops it restoring +// more than the one query already being sent. That one stays subscribed upstream, +// but only until the next reconnect, and its events are dropped. +func TestUnsubscribeAllDuringResubscribe(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{} + w := startWsEvents(ctx, t, upstream) + for _, query := range []string{"a", "b"} { + _, err := w.Subscribe(ctx, "", query) + require.NoError(t, err) + } + require.Eventually(t, func() bool { return len(upstream.subscribesOnCurrentConnection()) == 2 }, + 5*time.Second, time.Millisecond) + // Only after the reconnect: unread events would stop the client noticing the drop. + upstream.setEvents(map[string][]string{ + "a": {"a", "a", "a"}, + "b": {"b", "b", "b"}, + "other": {"other"}, + }) + + reconnecting(t, w, upstream) + resubscribed := make(chan struct{}) + go func() { defer close(resubscribed); w.redoSubscriptionsAfter(0) }() + // Nothing observable marks the resubscribe blocking on the stopped writer; this + // window only lets it get there, and cannot fail correct code. + <-time.After(50 * time.Millisecond) + + cleanupCtx, cancelCleanup := context.WithTimeout(ctx, 10*time.Millisecond) + defer cancelCleanup() + unsubscribed := make(chan error, 1) + go func() { unsubscribed <- w.UnsubscribeAll(cleanupCtx, "") }() + select { + case err := <-unsubscribed: + require.ErrorIs(t, err, context.DeadlineExceeded) + case <-resubscribed: + t.Fatal("UnsubscribeAll waited for the resubscribe to finish") + } + + select { + case <-resubscribed: + case <-time.After(10 * time.Second): + t.Fatal("resubscribe did not finish after the reconnect") + } + other, err := w.Subscribe(ctx, "", "other") + require.NoError(t, err) + requireEvent(t, other, "other") + restored := 0 + for _, query := range upstream.subscribesOnCurrentConnection() { + if query != "other" { + restored++ + } + } + require.LessOrEqual(t, restored, 1, "the resubscribe restored more than the query it was already sending") +} + +// Every UnsubscribeAll gets a fresh signal: repeating it must not panic, and a +// later subscription must still receive all events through its small buffer. +func TestSubscribeAfterRepeatedUnsubscribeAllBlocksNormally(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + events := make([]string, 16) + for i := range events { + events[i] = "later" + } + upstream := &fakeUpstream{events: map[string][]string{"later": events}} + w := startWsEvents(ctx, t, upstream) + require.NoError(t, w.UnsubscribeAll(ctx, "")) + require.NoError(t, w.UnsubscribeAll(ctx, "")) + + later, err := w.Subscribe(ctx, "", "later") + require.NoError(t, err) + for range events { + require.Eventually(t, func() bool { return len(later) == 1 }, 5*time.Second, time.Millisecond) + requireEvent(t, later, "later") + } +} From f13a6082960a21242317df6071cdd86cf3a37b5f Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:33:00 +0000 Subject: [PATCH 26/39] test(rpc): let the delivery block before releasing it TestUnsubscribeAllReleasesBlockedDelivery could clear the subscription before eventListener reached its blocking send, so removing the release arm still passed. Give the delivery a window to block first. Co-Authored-By: Claude Opus 5.5 --- rpc/client/http/ws_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rpc/client/http/ws_test.go b/rpc/client/http/ws_test.go index 9e09002985..3ab1555ee2 100644 --- a/rpc/client/http/ws_test.go +++ b/rpc/client/http/ws_test.go @@ -155,6 +155,9 @@ func TestUnsubscribeAllReleasesBlockedDelivery(t *testing.T) { // Taking one event lets the next fill the buffer, so the third has to block. requireEvent(t, gone, "gone") require.Eventually(t, func() bool { return len(gone) == 1 }, 5*time.Second, time.Millisecond) + // Nothing observable marks the delivery blocking; this window only lets it get + // there, and cannot fail correct code. + <-time.After(50 * time.Millisecond) require.NoError(t, w.UnsubscribeAll(ctx, "")) other, err := w.Subscribe(ctx, "", "other") From eda4da4c78f108a4d607a38ecd0fec3440188583 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:02:51 +0000 Subject: [PATCH 27/39] fix(node): release a full node's Dash Core RPC client A full node creates this client, but never closed it: its dispatcher goroutine outlived both a failed start and a completed shutdown. Register it with the node's closers, which run after the reactors using it are joined. A validator's client stays owned by its private validator. RPCClient.Close now also joins the dispatcher. The pinned dashd-go client has no HTTP timeout or cancellation, so the join gives up after five seconds with an error rather than hanging node shutdown on a stuck request. Co-Authored-By: Claude Opus 5.5 --- UPGRADING.md | 3 +- dash/core/client.go | 31 +++++++++--- dash/core/client_close_test.go | 67 ++++++++++++++++++++++++++ docs/architecture/service-lifecycle.md | 5 +- node/node.go | 2 + node/rollback_test.go | 58 ++++++++++++++++++++++ 6 files changed, 157 insertions(+), 9 deletions(-) create mode 100644 dash/core/client_close_test.go diff --git a/UPGRADING.md b/UPGRADING.md index 0b87675b6d..629c6cc9db 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -37,7 +37,8 @@ In the same process that panics with `duplicate metrics collector registration attempted` for a seed, or for a node with Prometheus enabled, unless `Instrumentation.Namespace` differs. An RPC listener bind failure stays retryable and keeps the stores open, with no `service.Service` call to release them if you -give up. Rollback and shutdown do not release a full node's Dash Core RPC client. +give up. A Dash Core request stuck at shutdown can outlive the client's `Close`, +whose join gives up after five seconds and logs an error. Direct consensus `State.Stop` drains a commit already in progress using the context originally passed to `State.Start`. Cancel that parent to abort it; diff --git a/dash/core/client.go b/dash/core/client.go index e012d428ef..2e3db71dff 100644 --- a/dash/core/client.go +++ b/dash/core/client.go @@ -62,11 +62,16 @@ type Client interface { Ping() error } +// defaultCloseTimeout bounds Close: the endpoint has no HTTP timeout or +// cancellation, so a stuck request would otherwise block it indefinitely. +const defaultCloseTimeout = 5 * time.Second + // RPCClient implements Client // Handles connection to the underlying dashd instance type RPCClient struct { - endpoint *rpc.Client - logger log.Logger + endpoint *rpc.Client + logger log.Logger + closeTimeout time.Duration } // NewRPCClient returns an instance of Client. @@ -96,17 +101,31 @@ func NewRPCClient(host string, username string, password string, logger log.Logg } dashCoreClient := RPCClient{ - endpoint: client, - logger: logger, + endpoint: client, + logger: logger, + closeTimeout: defaultCloseTimeout, } return &dashCoreClient, nil } -// Close closes the underlying connection +// Close closes the underlying connection and joins its dispatcher, up to +// closeTimeout. It is safe to call repeatedly. Later requests fail. func (rpcClient *RPCClient) Close() error { rpcClient.endpoint.Shutdown() - return nil + joined := make(chan struct{}) + go func() { + defer close(joined) + rpcClient.endpoint.WaitForShutdown() + }() + timer := time.NewTimer(rpcClient.closeTimeout) + defer timer.Stop() + select { + case <-joined: + return nil + case <-timer.C: + return fmt.Errorf("dash core request still in flight %s after closing the client", rpcClient.closeTimeout) + } } // Ping sends a ping request to the remote signer diff --git a/dash/core/client_close_test.go b/dash/core/client_close_test.go new file mode 100644 index 0000000000..440d71e091 --- /dev/null +++ b/dash/core/client_close_test.go @@ -0,0 +1,67 @@ +package core + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" +) + +func newCloseTestClient(t *testing.T, handler http.HandlerFunc) (*RPCClient, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + client, err := NewRPCClient(strings.TrimPrefix(srv.URL, "http://"), "user", "pass", log.NewNopLogger()) + require.NoError(t, err) + return client, srv +} + +// Close joins an idle client, repeatedly, and later requests fail instead of hanging. +func TestRPCClientCloseJoinsIdleClient(t *testing.T) { + client, _ := newCloseTestClient(t, func(http.ResponseWriter, *http.Request) {}) + client.closeTimeout = 5 * time.Second + + require.NoError(t, client.Close()) + require.NoError(t, client.Close()) + require.Error(t, client.Ping()) +} + +// The endpoint cannot cancel a request in flight, so Close must give up on it. +func TestRPCClientCloseBoundedByStuckRequest(t *testing.T) { + entered := make(chan struct{}) + unblock := make(chan struct{}) + release := sync.OnceFunc(func() { close(unblock) }) + client, _ := newCloseTestClient(t, func(http.ResponseWriter, *http.Request) { + close(entered) + <-unblock + }) + t.Cleanup(release) // runs before the server's cleanup, which joins the handler + client.closeTimeout = 50 * time.Millisecond + pinged := make(chan error, 1) + go func() { pinged <- client.Ping() }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("request did not reach the server") + } + + closed := make(chan error, 1) + go func() { closed <- client.Close() }() + select { + case err := <-closed: + require.ErrorContains(t, err, "still in flight") + case <-time.After(5 * time.Second): + t.Fatal("Close blocked on the stuck request") + } + + release() + require.Error(t, <-pinged) + client.closeTimeout = 5 * time.Second + require.NoError(t, client.Close(), "the dispatcher did not exit once the request ended") +} diff --git a/docs/architecture/service-lifecycle.md b/docs/architecture/service-lifecycle.md index 97d8c31726..3c33d331b2 100644 --- a/docs/architecture/service-lifecycle.md +++ b/docs/architecture/service-lifecycle.md @@ -85,8 +85,9 @@ no way to release them if the caller gives up instead. Any later node startup failure, and any seed startup failure, is final: rollback closes the databases, event sinks and other resources registered as node closers exactly once, later `Stop` and `Wait` calls are no-ops, and another `Start` is -rejected. The Dash Core RPC client created for a full node has no closer, so it is -not released by rollback or by shutdown. +rejected. A Dash Core request stuck at shutdown can outlive the client's `Close`: +the pinned client has no HTTP timeout, so the join gives up after five seconds and +logs an error. Recover from a final failure by constructing a fresh node graph. Metrics register on the process-wide default registry, so a second build in the same process panics diff --git a/node/node.go b/node/node.go index 79e675aaa0..c236ba004f 100644 --- a/node/node.go +++ b/node/node.go @@ -237,6 +237,8 @@ func makeNode( if err != nil { return nil, fmt.Errorf("failed to create Dash Core RPC client: %w", err) } + // A validator's client belongs to its private validator instead. + closers = append(closers, dashCoreRPCClient.Close) } weAreOnlyValidator := onlyValidatorIsUs(state, proTxHash) diff --git a/node/rollback_test.go b/node/rollback_test.go index b0b49758e8..42a6c78a13 100644 --- a/node/rollback_test.go +++ b/node/rollback_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "maps" + "runtime" "strings" "sync" "testing" @@ -80,10 +81,14 @@ func rollbackTestNode( ctx context.Context, t *testing.T, wrapApp func(abciclient.Client) abciclient.Client, + configure ...func(*config.Config), ) (*nodeImpl, *dbCloseCounter) { t.Helper() cfg, err := config.ResetTestRoot(t.TempDir(), strings.ReplaceAll(t.Name(), "/", "-")) require.NoError(t, err) + for _, apply := range configure { + apply(cfg) + } cfg.RPC.ListenAddress = "" cfg.P2P.ListenAddress = "tcp://127.0.0.1:0" cfg.TxIndex.Indexer = []string{"kv"} @@ -161,6 +166,59 @@ func TestNodeFailedStartReleasesDatabasesOnce(t *testing.T) { }) } +// Releasing resources during rollback must not replace the startup error. +func TestNodeFailedStartKeepsErrorWhenReleaseFails(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, _ := rollbackTestNode(ctx, t, nil) + release := n.shutdownOps + n.shutdownOps = func() error { return errors.Join(release(), errors.New("release failed")) } + failure := errors.New("startup failed") + n.rpcEnv.ProxyApp = failedStartClient{Client: abcimocks.NewClient(t), err: failure} + + err := n.Start(ctx) + require.ErrorIs(t, err, failure) + require.NotContains(t, err.Error(), "release failed") +} + +// coreRPCWorkers counts running Dash Core RPC client dispatchers. +func coreRPCWorkers() int { + stacks := make([]byte, 1<<24) + return strings.Count(string(stacks[:runtime.Stack(stacks, true)]), "rpcclient.(*Client).sendPostHandler(") +} + +// A full node owns the Dash Core RPC client it creates, unlike a validator, whose +// client belongs to its private validator. +func TestFullNodeReleasesCoreRPCClient(t *testing.T) { + fullNode := func(cfg *config.Config) { + cfg.Mode = config.ModeFull + cfg.PrivValidator.CoreRPCHost = "127.0.0.1:1" + } + failure := errors.New("startup failed") + testCases := map[string]func(context.Context, context.CancelFunc, *testing.T, *nodeImpl){ + "failed start": func(ctx context.Context, _ context.CancelFunc, t *testing.T, n *nodeImpl) { + n.rpcEnv.ProxyApp = failedStartClient{Client: abcimocks.NewClient(t), err: failure} + require.ErrorIs(t, n.Start(ctx), failure) + }, + "shutdown": func(ctx context.Context, cancel context.CancelFunc, t *testing.T, n *nodeImpl) { + require.NoError(t, n.Start(ctx)) + cancel() + n.Wait() + }, + } + for name, run := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + before := coreRPCWorkers() + n, _ := rollbackTestNode(ctx, t, nil, fullNode) + require.Equal(t, before+1, coreRPCWorkers(), "the full node did not create its client") + run(ctx, cancel, t, n) + require.Equal(t, before, coreRPCWorkers(), "the Dash Core RPC client outlived the node") + }) + } +} + type failedStartClient struct { abciclient.Client err error From e6f917616b56676d6d276b6266992c6ba7a989a8 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:06:12 +0000 Subject: [PATCH 28/39] docs(node): state what the Dash Core client's close bound covers The five-second bound limits only the join inside Close, not node shutdown: a reactor stuck in a Dash Core call blocks Wait before closers run, which the previous commit's message overstated. Requests after Close can also block once the endpoint's send buffer fills, so the contract is not to send them. Co-Authored-By: Claude Opus 5.5 --- UPGRADING.md | 7 +++++-- dash/core/client.go | 4 +++- dash/core/client_close_test.go | 2 +- docs/architecture/service-lifecycle.md | 12 +++++++++--- node/node.go | 4 +++- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index 629c6cc9db..86f8e662d7 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -37,8 +37,11 @@ In the same process that panics with `duplicate metrics collector registration attempted` for a seed, or for a node with Prometheus enabled, unless `Instrumentation.Namespace` differs. An RPC listener bind failure stays retryable and keeps the stores open, with no `service.Service` call to release them if you -give up. A Dash Core request stuck at shutdown can outlive the client's `Close`, -whose join gives up after five seconds and logs an error. +give up. Do not call a Dash Core `RPCClient` after `Close`: requests can block +forever. Its `Close` waits at most five seconds for an in-flight request, but that +does not bound node shutdown: a reactor stuck in a Dash Core call still blocks +`Wait`, because the client has no HTTP timeout. `--shutdown-timeout` limits the +node command's wait, not `Start` or embedders. Direct consensus `State.Stop` drains a commit already in progress using the context originally passed to `State.Start`. Cancel that parent to abort it; diff --git a/dash/core/client.go b/dash/core/client.go index 2e3db71dff..5faf8d3f93 100644 --- a/dash/core/client.go +++ b/dash/core/client.go @@ -110,7 +110,9 @@ func NewRPCClient(host string, username string, password string, logger log.Logg } // Close closes the underlying connection and joins its dispatcher, up to -// closeTimeout. It is safe to call repeatedly. Later requests fail. +// closeTimeout. It is safe to call repeatedly. Do not send requests after Close: +// they can return an error, but they block forever once the endpoint's send +// buffer fills, because nothing drains it any more. func (rpcClient *RPCClient) Close() error { rpcClient.endpoint.Shutdown() joined := make(chan struct{}) diff --git a/dash/core/client_close_test.go b/dash/core/client_close_test.go index 440d71e091..114843323d 100644 --- a/dash/core/client_close_test.go +++ b/dash/core/client_close_test.go @@ -22,7 +22,7 @@ func newCloseTestClient(t *testing.T, handler http.HandlerFunc) (*RPCClient, *ht return client, srv } -// Close joins an idle client, repeatedly, and later requests fail instead of hanging. +// Close joins an idle client, repeatedly, and the first later request gets an error. func TestRPCClientCloseJoinsIdleClient(t *testing.T) { client, _ := newCloseTestClient(t, func(http.ResponseWriter, *http.Request) {}) client.closeTimeout = 5 * time.Second diff --git a/docs/architecture/service-lifecycle.md b/docs/architecture/service-lifecycle.md index 3c33d331b2..70392238e9 100644 --- a/docs/architecture/service-lifecycle.md +++ b/docs/architecture/service-lifecycle.md @@ -85,9 +85,15 @@ no way to release them if the caller gives up instead. Any later node startup failure, and any seed startup failure, is final: rollback closes the databases, event sinks and other resources registered as node closers exactly once, later `Stop` and `Wait` calls are no-ops, and another `Start` is -rejected. A Dash Core request stuck at shutdown can outlive the client's `Close`: -the pinned client has no HTTP timeout, so the join gives up after five seconds and -logs an error. +rejected. + +The five-second bound covers only the join in the Dash Core client's `Close`; a +request still stuck there outlives it and an error is logged. It does not bound +node shutdown. The pinned client has no HTTP timeout or cancellation, so a reactor +stuck in a Dash Core call blocks `Wait`, or the rollback inside a failed `Start`, +before closers run. The node command's `--shutdown-timeout` limits only its wait +after the context ends, then exits reporting incomplete cleanup; it does not limit +`Start`, and embedders must bound `Wait` themselves. Recover from a final failure by constructing a fresh node graph. Metrics register on the process-wide default registry, so a second build in the same process panics diff --git a/node/node.go b/node/node.go index c236ba004f..45d6d8c6fc 100644 --- a/node/node.go +++ b/node/node.go @@ -237,7 +237,9 @@ func makeNode( if err != nil { return nil, fmt.Errorf("failed to create Dash Core RPC client: %w", err) } - // A validator's client belongs to its private validator instead. + // A validator's client belongs to its private validator instead. Close + // bounds only its own join: this closer runs after reactors are joined, so + // a reactor stuck in a Dash Core call blocks shutdown before reaching it. closers = append(closers, dashCoreRPCClient.Close) } From c4470c859b8d0d01d6de4f6f8d521e13ead825ce Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 21:31:30 +0000 Subject: [PATCH 29/39] fix(rpc): make WSClient.Stop idempotent Running out of reconnect attempts made reconnectRoutine call Stop, which closes ResponsesCh. The owner's later Stop closed it again, so a light client whose upstream had given up panicked with "close of closed channel" in its OnDrain goroutine and crashed the process on shutdown. ResponsesCh now closes exactly once. Stop first waits for reconnectRoutine to exit, because a Stop returning during a reconnect could close the channel just before new workers started writing to it. Stop on a client that never started returns immediately. Co-Authored-By: Claude Opus 5.5 --- UPGRADING.md | 5 ++ light/rpc/shutdown_test.go | 35 +++++++++ rpc/jsonrpc/client/ws_client.go | 33 ++++++-- rpc/jsonrpc/client/ws_client_test.go | 108 +++++++++++++++++++++++++++ 4 files changed, 173 insertions(+), 8 deletions(-) create mode 100644 light/rpc/shutdown_test.go diff --git a/UPGRADING.md b/UPGRADING.md index 86f8e662d7..c2de0eb9ac 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -20,6 +20,11 @@ Plain goroutines and their existing joins remain the implementation's responsibi implementations must provide `Go(func(context.Context)) bool` so the server owns subscription workers. +`WSClient.Stop` is safe to call repeatedly and concurrently, including after +reconnect attempts run out. During a reconnect it now blocks until the context +ends or the attempts run out, instead of returning while a reconnect could still +start workers. + `rpc/jsonrpc/server.Serve` and `ServeTLS` always close the listener before returning, including when TLS setup fails. Cancellation now reaches request contexts, rejects new requests with 503, closes active connections instead of a diff --git a/light/rpc/shutdown_test.go b/light/rpc/shutdown_test.go new file mode 100644 index 0000000000..0299b4747b --- /dev/null +++ b/light/rpc/shutdown_test.go @@ -0,0 +1,35 @@ +package rpc + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" + rpchttp "github.com/dashpay/tenderdash/rpc/client/http" + rpcserver "github.com/dashpay/tenderdash/rpc/jsonrpc/server" +) + +// OnDrain stops an HTTP-backed upstream client, which may already have stopped +// itself, as it does when its reconnect attempts run out. +func TestClientDrainToleratesStoppedUpstream(t *testing.T) { + logger := log.NewNopLogger() + ws := rpcserver.NewWebsocketManager(logger, map[string]*rpcserver.RPCFunc{}) + mux := http.NewServeMux() + mux.HandleFunc("/websocket", ws.WebsocketHandler) + upstream := httptest.NewServer(mux) + t.Cleanup(upstream.Close) + next, err := rpchttp.New(upstream.URL) + require.NoError(t, err) + client := NewClient(logger, next, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, client.Start(ctx)) + + cancel() + client.Wait() + require.NoError(t, next.Stop()) +} diff --git a/rpc/jsonrpc/client/ws_client.go b/rpc/jsonrpc/client/ws_client.go index 8082e9ded3..6c135d8fcf 100644 --- a/rpc/jsonrpc/client/ws_client.go +++ b/rpc/jsonrpc/client/ws_client.go @@ -65,6 +65,9 @@ type WSClient struct { //nolint: maligned protocol string wg sync.WaitGroup + // reconnectDone closes when reconnectRoutine exits; no workers start after that. + reconnectDone chan struct{} + closeResponses sync.Once mtx sync.RWMutex reconnecting bool @@ -141,23 +144,39 @@ func (c *WSClient) Start(ctx context.Context) error { // channel is unbuffered. c.backlog = make(chan rpctypes.RPCRequest, 1) + c.reconnectDone = make(chan struct{}) c.startReadWriteRoutines(ctx) go c.reconnectRoutine(ctx) return nil } -// Stop blocks until the client is shut down and returns nil. +// Stop blocks until the client is shut down and returns nil. It does not end the +// client: shutdown follows the context passed to Start, or running out of +// reconnect attempts. It is safe to call repeatedly and concurrently, and returns +// immediately if the client never started. // // TODO(creachadair): This method exists for compatibility with the original // service plumbing. Give it a better name (e.g., Wait). func (c *WSClient) Stop() error { - // only close user-facing channels when we can't write to them - c.wg.Wait() - close(c.ResponsesCh) + if c.reconnectDone == nil { + return nil + } + <-c.reconnectDone + c.closeResponsesCh() return nil } +// closeResponsesCh joins the workers and closes ResponsesCh once. Only close +// user-facing channels when we can't write to them. Callers must ensure that no +// more workers can start. +func (c *WSClient) closeResponsesCh() { + c.closeResponses.Do(func() { + c.wg.Wait() + close(c.ResponsesCh) + }) +} + // IsReconnecting returns true if the client is reconnecting right now. func (c *WSClient) IsReconnecting() bool { c.mtx.RLock() @@ -292,6 +311,7 @@ func (c *WSClient) processBacklog() error { } func (c *WSClient) reconnectRoutine(ctx context.Context) { + defer close(c.reconnectDone) for { select { case <-ctx.Done(): @@ -301,10 +321,7 @@ func (c *WSClient) reconnectRoutine(ctx context.Context) { c.wg.Wait() if err := c.reconnect(ctx); err != nil { c.Logger.Error("failed to reconnect", "err", err, "original_err", originalError) - if err = c.Stop(); err != nil { - c.Logger.Error("failed to stop conn", "error", err) - } - + c.closeResponsesCh() return } // drain reconnectAfter diff --git a/rpc/jsonrpc/client/ws_client_test.go b/rpc/jsonrpc/client/ws_client_test.go index 663973cc9f..c3df2e39ce 100644 --- a/rpc/jsonrpc/client/ws_client_test.go +++ b/rpc/jsonrpc/client/ws_client_test.go @@ -259,3 +259,111 @@ func handleResponses(ctx context.Context, t *testing.T, c *WSClient) { } } } + +// startFailingClient starts a client that gives up after its first failed redial, +// and returns it with a server that drops the connection on any request. +func startFailingClient(ctx context.Context, t *testing.T) (*WSClient, *httptest.Server) { + t.Helper() + s := httptest.NewServer(&myTestHandler{t: t, closeConnAfterRead: true}) + t.Cleanup(s.Close) + c, err := NewWS("//"+s.Listener.Addr().String(), "/websocket") + require.NoError(t, err) + c.maxReconnectAttempts = 0 + require.NoError(t, c.Start(ctx)) + return c, s +} + +func requireResponsesClosed(t *testing.T, c *WSClient) { + t.Helper() + select { + case _, ok := <-c.ResponsesCh: + require.False(t, ok, "unexpected response") + case <-time.After(wsCallTimeout): + t.Fatal("ResponsesCh was not closed") + } +} + +// Running out of reconnect attempts shuts the client down by itself. The owner +// does not know that, and still calls Stop once its context ends. +func TestWSClientStopAfterReconnectExhaustion(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, s := startFailingClient(ctx, t) + require.NoError(t, c.Call(ctx, "drop", make(map[string]interface{}))) + s.Close() + requireResponsesClosed(t, c) + + cancel() + require.NoError(t, c.Stop()) + require.NoError(t, c.Stop()) +} + +// A Stop already blocked when the attempts run out must finish with that shutdown. +func TestWSClientStopRacingReconnectExhaustion(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, s := startFailingClient(ctx, t) + require.NoError(t, c.Call(ctx, "drop", make(map[string]interface{}))) + s.Close() + require.Eventually(t, c.IsReconnecting, wsCallTimeout, time.Millisecond) + + stopped := make(chan error, 2) + for range cap(stopped) { + go func() { stopped <- c.Stop() }() + } + for range cap(stopped) { + select { + case err := <-stopped: + require.NoError(t, err) + requireResponsesClosed(t, c) + case <-time.After(wsCallTimeout): + t.Fatal("Stop did not return after the attempts ran out") + } + } +} + +// A reconnect can start new workers, so Stop must not finish while one may still +// succeed. Every concurrent Stop returns only after the channel closes. +func TestWSClientStopWaitsForReconnect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, _ := startFailingClient(ctx, t) + require.NoError(t, c.Call(ctx, "drop", make(map[string]interface{}))) + require.Eventually(t, c.IsReconnecting, wsCallTimeout, time.Millisecond) + + stopped := make(chan error, 2) + for range cap(stopped) { + go func() { stopped <- c.Stop() }() + } + require.Eventually(t, func() bool { return !c.IsReconnecting() }, wsCallTimeout, time.Millisecond) + select { + case err := <-stopped: + t.Fatalf("Stop returned before the context ended: %v", err) + default: + } + + cancel() + for range cap(stopped) { + select { + case err := <-stopped: + require.NoError(t, err) + requireResponsesClosed(t, c) + case <-time.After(wsCallTimeout): + t.Fatal("Stop did not return after the context ended") + } + } +} + +// A client that never started has nothing to wait for. +func TestWSClientStopWithoutStart(t *testing.T) { + c, err := NewWS("//127.0.0.1:1", "/websocket") + require.NoError(t, err) + stopped := make(chan error, 1) + go func() { stopped <- c.Stop() }() + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(wsCallTimeout): + t.Fatal("Stop blocked on a client that never started") + } +} From 5c812a1ba1bbcb5a7c26fcccd7a3f46f392ec00f Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 21:47:16 +0000 Subject: [PATCH 30/39] fix(rpc): bound WSClient dials and worker failure reports after cancel Waiting for the reconnect routine made Stop depend on the redial, which had no context or timeout: a peer that completes TCP but never answers the upgrade blocked Stop, and a light client's Wait, forever. Dials now fail after ten seconds and observe the context. The handshake obeys deadlines but not cancellation, so ending the context closes the connection to interrupt it. A writer stuck in a write could also outlive the reconnect routine, then block Stop reporting its failure to a channel nobody reads. Reports now give up once the context ends. ResponsesCh also closes on each run of a restarted client. Co-Authored-By: Claude Opus 5.5 --- UPGRADING.md | 5 +- light/rpc/shutdown_test.go | 43 ++++++ rpc/jsonrpc/client/http_json_client.go | 2 +- rpc/jsonrpc/client/ws_client.go | 49 +++++-- rpc/jsonrpc/client/ws_client_test.go | 178 +++++++++++++++++++++++++ 5 files changed, 265 insertions(+), 12 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index c2de0eb9ac..dde49e2527 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -23,7 +23,10 @@ subscription workers. `WSClient.Stop` is safe to call repeatedly and concurrently, including after reconnect attempts run out. During a reconnect it now blocks until the context ends or the attempts run out, instead of returning while a reconnect could still -start workers. +start workers. Dials now observe that context and fail after ten seconds, including +the one in `Start`, so a peer that never answers the handshake cannot block either +call. Once the context ends, `Stop` waits at most for a write in progress to hit +its ten-second deadline. `rpc/jsonrpc/server.Serve` and `ServeTLS` always close the listener before returning, including when TLS setup fails. Cancellation now reaches request diff --git a/light/rpc/shutdown_test.go b/light/rpc/shutdown_test.go index 0299b4747b..f5891d7a7c 100644 --- a/light/rpc/shutdown_test.go +++ b/light/rpc/shutdown_test.go @@ -2,10 +2,14 @@ package rpc import ( "context" + "net" "net/http" "net/http/httptest" + "sync/atomic" "testing" + "time" + "github.com/gorilla/websocket" "github.com/stretchr/testify/require" "github.com/dashpay/tenderdash/libs/log" @@ -33,3 +37,42 @@ func TestClientDrainToleratesStoppedUpstream(t *testing.T) { client.Wait() require.NoError(t, next.Stop()) } + +// Cancellation must end the client even when its upstream hangs during a redial. +func TestClientWaitDuringHungUpstreamRedial(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + var accepted atomic.Int32 + upgrader := websocket.Upgrader{} + srv := &http.Server{ + ReadHeaderTimeout: 5 * time.Second, + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if accepted.Add(1) == 1 { + conn, err := upgrader.Upgrade(w, r, nil) + if err == nil { + _ = conn.UnderlyingConn().Close() // the client sees 1006 and redials + } + return + } + <-r.Context().Done() // never answers + }), + } + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + next, err := rpchttp.New("http://" + ln.Addr().String()) + require.NoError(t, err) + client := NewClient(log.NewNopLogger(), next, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, client.Start(ctx)) + require.Eventually(t, func() bool { return accepted.Load() == 2 }, 10*time.Second, time.Millisecond) + + cancel() + waited := make(chan struct{}) + go func() { client.Wait(); close(waited) }() + select { + case <-waited: + case <-time.After(5 * time.Second): + t.Fatal("light client Wait hung after cancellation") + } +} diff --git a/rpc/jsonrpc/client/http_json_client.go b/rpc/jsonrpc/client/http_json_client.go index d5d6ae8e3e..e94dca3a8f 100644 --- a/rpc/jsonrpc/client/http_json_client.go +++ b/rpc/jsonrpc/client/http_json_client.go @@ -370,7 +370,7 @@ func dialParamsFromURL(pURL *parsedURL) (string, string) { func makeHTTPDialer(protocol, addr string) func(context.Context, string, string) (net.Conn, error) { return func(ctx context.Context, _, _ string) (net.Conn, error) { - return net.DialTimeout(protocol, addr, 10*time.Second) + return (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, protocol, addr) } } diff --git a/rpc/jsonrpc/client/ws_client.go b/rpc/jsonrpc/client/ws_client.go index 6c135d8fcf..f79c73ca00 100644 --- a/rpc/jsonrpc/client/ws_client.go +++ b/rpc/jsonrpc/client/ws_client.go @@ -20,6 +20,7 @@ import ( // wsOptions carries optional settings for a websocket connection. type wsOptions struct { MaxReconnectAttempts uint // maximum attempts to reconnect + HandshakeTimeout time.Duration // deadline for dialing and the WebSocket handshake ReadWait time.Duration // deadline for any read op WriteWait time.Duration // deadline for any write op PingPeriod time.Duration // frequency with which pings are sent @@ -28,6 +29,7 @@ type wsOptions struct { // defaultWSOptions are the default websocket connection settings. var defaultWSOptions = wsOptions{ MaxReconnectAttempts: 10, // first: 2 sec, last: 17 min. + HandshakeTimeout: 10 * time.Second, WriteWait: 10 * time.Second, ReadWait: 0, PingPeriod: 0, @@ -74,6 +76,9 @@ type WSClient struct { //nolint: maligned nextReqID int // sentIDs map[types.JSONRPCIntID]bool // IDs of the requests currently in flight + // Time allowed to dial the server and finish the WebSocket handshake. + handshakeTimeout time.Duration + // Time allowed to write a message to the server. 0 means block until operation succeeds. writeWait time.Duration @@ -104,6 +109,7 @@ func NewWS(remoteAddr, endpoint string) (*WSClient, error) { DialerContext: dialFn, Endpoint: endpoint, maxReconnectAttempts: opts.MaxReconnectAttempts, + handshakeTimeout: opts.HandshakeTimeout, readWait: opts.ReadWait, writeWait: opts.WriteWait, pingPeriod: opts.PingPeriod, @@ -130,7 +136,7 @@ func (c *WSClient) String() string { // service routines run until ctx terminates. To wait for the client to exit // after ctx ends, call Stop. func (c *WSClient) Start(ctx context.Context) error { - if err := c.dial(); err != nil { + if err := c.dial(ctx); err != nil { return err } @@ -145,6 +151,7 @@ func (c *WSClient) Start(ctx context.Context) error { c.backlog = make(chan rpctypes.RPCRequest, 1) c.reconnectDone = make(chan struct{}) + c.closeResponses = sync.Once{} c.startReadWriteRoutines(ctx) go c.reconnectRoutine(ctx) @@ -153,7 +160,8 @@ func (c *WSClient) Start(ctx context.Context) error { // Stop blocks until the client is shut down and returns nil. It does not end the // client: shutdown follows the context passed to Start, or running out of -// reconnect attempts. It is safe to call repeatedly and concurrently, and returns +// reconnect attempts. After either, the wait is bounded by the write deadline of a +// write in progress. It is safe to call repeatedly and concurrently, and returns // immediately if the client never started. // // TODO(creachadair): This method exists for compatibility with the original @@ -219,13 +227,24 @@ func (c *WSClient) nextRequestID() int { return id } -func (c *WSClient) dial() error { +// dial connects to the server. The handshake obeys deadlines but not cancellation, +// so ending ctx closes the connection to interrupt it. +func (c *WSClient) dial(ctx context.Context) error { + stopWatching := func() bool { return true } dialer := &websocket.Dialer{ - NetDialContext: c.DialerContext, - Proxy: http.ProxyFromEnvironment, + NetDialContext: func(dialCtx context.Context, network, addr string) (net.Conn, error) { + conn, err := c.DialerContext(dialCtx, network, addr) + if err == nil { + stopWatching = context.AfterFunc(ctx, func() { _ = conn.Close() }) + } + return conn, err + }, + Proxy: http.ProxyFromEnvironment, + HandshakeTimeout: c.handshakeTimeout, } + defer func() { stopWatching() }() rHeader := http.Header{} - conn, _, err := dialer.Dial(c.protocol+"://"+c.Address+c.Endpoint, rHeader) //nolint:nolintlint,bodyclose + conn, _, err := dialer.DialContext(ctx, c.protocol+"://"+c.Address+c.Endpoint, rHeader) //nolint:nolintlint,bodyclose if err != nil { return err } @@ -263,7 +282,7 @@ func (c *WSClient) reconnect(ctx context.Context) error { case <-timer.C: } - err := c.dial() + err := c.dial(ctx) if err != nil { c.Logger.Error("failed to redial", "err", err) } else { @@ -282,6 +301,15 @@ func (c *WSClient) reconnect(ctx context.Context) error { } } +// requestReconnect reports a worker's failure. After ctx ends nothing may be left +// to receive it, and Stop is waiting for the worker. +func (c *WSClient) requestReconnect(ctx context.Context, err error) { + select { + case c.reconnectAfter <- err: + case <-ctx.Done(): + } +} + func (c *WSClient) startReadWriteRoutines(ctx context.Context) { c.wg.Add(2) c.readRoutineQuit = make(chan struct{}) @@ -329,6 +357,7 @@ func (c *WSClient) reconnectRoutine(ctx context.Context) { for { select { case <-ctx.Done(): + _ = c.conn.Close() // no worker will own the new connection return case <-c.reconnectAfter: default: @@ -371,7 +400,7 @@ func (c *WSClient) writeRoutine(ctx context.Context) { } if err := c.conn.WriteJSON(request); err != nil { c.Logger.Error("failed to send request", "err", err) - c.reconnectAfter <- err + c.requestReconnect(ctx, err) // add request to the backlog, so we don't lose it c.backlog <- request return @@ -384,7 +413,7 @@ func (c *WSClient) writeRoutine(ctx context.Context) { } if err := c.conn.WriteMessage(websocket.PingMessage, []byte{}); err != nil { c.Logger.Error("failed to write ping", "err", err) - c.reconnectAfter <- err + c.requestReconnect(ctx, err) return } case <-c.readRoutineQuit: @@ -424,7 +453,7 @@ func (c *WSClient) readRoutine(ctx context.Context) { c.Logger.Error("failed to read response", "err", err) close(c.readRoutineQuit) - c.reconnectAfter <- err + c.requestReconnect(ctx, err) return } diff --git a/rpc/jsonrpc/client/ws_client_test.go b/rpc/jsonrpc/client/ws_client_test.go index c3df2e39ce..e65546e91b 100644 --- a/rpc/jsonrpc/client/ws_client_test.go +++ b/rpc/jsonrpc/client/ws_client_test.go @@ -3,8 +3,11 @@ package client import ( "context" "encoding/json" + "net" "net/http" "net/http/httptest" + "strings" + "sync/atomic" "testing" "time" @@ -367,3 +370,178 @@ func TestWSClientStopWithoutStart(t *testing.T) { t.Fatal("Stop blocked on a client that never started") } } + +func requireStopReturns(t *testing.T, c *WSClient) { + t.Helper() + stopped := make(chan error, 1) + go func() { stopped <- c.Stop() }() + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(wsCallTimeout): + t.Fatal("Stop did not return") + } +} + +// listenWithoutAnswering returns the address of a peer that never answers the +// upgrade, and a channel signaled once a client has sent its upgrade request. +func listenWithoutAnswering(t *testing.T) (string, <-chan net.Conn) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + accepted := make(chan net.Conn, 8) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + t.Cleanup(func() { _ = conn.Close() }) + if _, err := conn.Read(make([]byte, 1)); err == nil { + accepted <- conn + } + } + }() + return "//" + ln.Addr().String(), accepted +} + +// The first dial observes the context passed to Start. +func TestWSClientStartCanceledDuringHungHandshake(t *testing.T) { + addr, accepted := listenWithoutAnswering(t) + c, err := NewWS(addr, "/websocket") + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + started := make(chan error, 1) + go func() { started <- c.Start(ctx) }() + <-accepted + + cancel() + select { + case err := <-started: + require.Error(t, err) + case <-time.After(wsCallTimeout): + t.Fatal("Start blocked on a hung handshake after its context ended") + } + require.NoError(t, c.Stop()) +} + +// A peer that never answers cannot block a dial forever, even with a live context. +func TestWSClientHandshakeTimesOut(t *testing.T) { + addr, _ := listenWithoutAnswering(t) + c, err := NewWS(addr, "/websocket") + require.NoError(t, err) + c.handshakeTimeout = 50 * time.Millisecond + started := make(chan error, 1) + go func() { started <- c.Start(context.Background()) }() + select { + case err := <-started: + require.Error(t, err) + case <-time.After(wsCallTimeout): + t.Fatal("handshake did not time out") + } +} + +// Stop follows the context even when a redial is stuck in its handshake. +func TestWSClientStopDuringHungRedial(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + var accepted atomic.Int32 + srv := &http.Server{ + ReadHeaderTimeout: wsCallTimeout, + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if accepted.Add(1) == 1 { + conn, err := upgrader.Upgrade(w, r, nil) + if err == nil { + _ = conn.UnderlyingConn().Close() // the client sees 1006 and redials + } + return + } + <-r.Context().Done() // never answers + }), + } + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, err := NewWS("//"+ln.Addr().String(), "/websocket") + require.NoError(t, err) + require.NoError(t, c.Start(ctx)) + require.Eventually(t, func() bool { return accepted.Load() == 2 }, 2*wsCallTimeout, time.Millisecond) + + cancel() + requireStopReturns(t, c) +} + +// A writer stuck in a write outlives its context and the reconnect routine. Stop +// must still join it, and it must not block reporting the error once it fails. +func TestWSClientStopJoinsWorkerFailingAfterReconnectExit(t *testing.T) { + serverConn := make(chan *websocket.Conn, 1) + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err == nil { + serverConn <- conn // never reads, so the client's send path fills + } + })) + t.Cleanup(s.Close) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c, err := NewWS("//"+s.Listener.Addr().String(), "/websocket") + require.NoError(t, err) + require.NoError(t, c.Start(ctx)) + peer := <-serverConn + t.Cleanup(func() { _ = peer.Close() }) + // Call returns once the writer has taken the request, too big for socket buffers. + require.NoError(t, c.Call(ctx, "big", map[string]interface{}{"x": strings.Repeat("a", 64<<20)})) + cancel() + select { + case <-c.reconnectDone: + case <-time.After(wsCallTimeout): + t.Fatal("reconnect routine did not exit") + } + + stopped := make(chan error, 1) + go func() { stopped <- c.Stop() }() + // Nothing observable marks Stop blocking; this window cannot fail correct code. + select { + case err := <-stopped: + t.Fatalf("Stop returned while the writer could still use the client: %v", err) + case <-time.After(50 * time.Millisecond): + } + + // The reader reports this and closes the connection, so the write fails too. + require.NoError(t, peer.WriteControl(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseGoingAway, "bye"), time.Now().Add(time.Second))) + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(wsCallTimeout): + t.Fatal("Stop did not return after the workers failed") + } +} + +// Each run gets its own shutdown. +func TestWSClientRestartAfterStop(t *testing.T) { + s := httptest.NewServer(&myTestHandler{t: t}) + t.Cleanup(s.Close) + c, err := NewWS("//"+s.Listener.Addr().String(), "/websocket") + require.NoError(t, err) + for range 2 { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, c.Start(ctx)) + cancel() + requireStopReturns(t, c) + requireResponsesClosed(t, c) + } +} + +// The dialer shared with the HTTP client observes its context, so a cancellation +// does not wait out the connect timeout of an unreachable peer. +func TestHTTPDialerObservesContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := makeHTTPDialer("tcp", "127.0.0.1:1")(ctx, "", "") + require.ErrorIs(t, err, context.Canceled) +} From d942aa1e67fb6bdfbdcf941263f5885ed9aa1920 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:38:08 +0000 Subject: [PATCH 31/39] fix(light): do not join a wrapped client the caller already started A local RPC client is its node's event bus. Starting a light client around it succeeds without tying the bus to the light client's context, so OnDrain then waited for a bus only the node could stop, and Wait never returned. OnDrain now skips a client that was already running when the light client started, as the routed ABCI client does. Clients that do not report their state, such as the HTTP client, are still joined. Co-Authored-By: Claude Opus 5.5 --- light/rpc/client.go | 13 +++++- light/rpc/ownership_test.go | 87 +++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 light/rpc/ownership_test.go diff --git a/light/rpc/client.go b/light/rpc/client.go index 201df29127..95673d5e9d 100644 --- a/light/rpc/client.go +++ b/light/rpc/client.go @@ -54,6 +54,9 @@ type Client struct { keyPathFn KeyPathFunc serviceCtx context.Context + // callerOwnsNext records that next was already running when this client + // started. Lifecycle hooks never overlap, so OnStart's write precedes OnDrain. + callerOwnsNext bool } var _ rpcclient.Client = (*Client)(nil) @@ -107,6 +110,10 @@ func NewClient(logger log.Logger, next rpcclient.Client, lc LightClient, opts .. func (c *Client) OnStart(ctx context.Context) error { c.serviceCtx = ctx + // A local client shares its node's event bus, whose Start ignores ctx if it is + // already running. Clients not reporting their state are started here. + running, ok := c.next.(interface{ IsRunning() bool }) + c.callerOwnsNext = ok && running.IsRunning() if err := c.next.Start(ctx); err != nil { return err } @@ -116,8 +123,12 @@ func (c *Client) OnStart(ctx context.Context) error { func (c *Client) OnStop() {} -// OnDrain joins clients exposing transport shutdown in addition to the RPC API. +// OnDrain joins clients exposing transport shutdown in addition to the RPC API, +// unless the caller had already started them and so controls their lifetime. func (c *Client) OnDrain() { + if c.callerOwnsNext { + return + } if waiter, ok := c.next.(interface{ Wait() }); ok { waiter.Wait() } else if stopper, ok := c.next.(interface{ Stop() error }); ok { diff --git a/light/rpc/ownership_test.go b/light/rpc/ownership_test.go new file mode 100644 index 0000000000..58d9723485 --- /dev/null +++ b/light/rpc/ownership_test.go @@ -0,0 +1,87 @@ +package rpc_test + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/internal/eventbus" + rpccore "github.com/dashpay/tenderdash/internal/rpc/core" + "github.com/dashpay/tenderdash/libs/log" + lrpc "github.com/dashpay/tenderdash/light/rpc" + rpcclient "github.com/dashpay/tenderdash/rpc/client" + "github.com/dashpay/tenderdash/rpc/client/local" +) + +type localNode struct{ bus *eventbus.EventBus } + +func (n localNode) RPCEnvironment() *rpccore.Environment { return &rpccore.Environment{} } +func (n localNode) EventBus() *eventbus.EventBus { return n.bus } + +func requireClientWaitReturns(t *testing.T, client *lrpc.Client) { + t.Helper() + waited := make(chan struct{}) + go func() { client.Wait(); close(waited) }() + select { + case <-waited: + case <-time.After(5 * time.Second): + t.Fatal("light client Wait blocked on a service it does not own") + } +} + +// A local client is its node's event bus. Stopping only the wrapper must neither +// wait for that bus nor stop it. +func TestClientWaitLeavesCallerOwnedEventBusRunning(t *testing.T) { + stoppers := map[string]func(*lrpc.Client, context.CancelFunc){ + "cancel": func(_ *lrpc.Client, cancel context.CancelFunc) { cancel() }, + "stop": func(client *lrpc.Client, _ context.CancelFunc) { client.Stop() }, + } + for name, stop := range stoppers { + t.Run(name, func(t *testing.T) { + logger := log.NewNopLogger() + busCtx, stopBus := context.WithCancel(context.Background()) + bus := eventbus.NewDefault(logger) + require.NoError(t, bus.Start(busCtx)) + defer func() { stopBus(); bus.Wait() }() + next, err := local.New(logger, localNode{bus: bus}) + require.NoError(t, err) + client := lrpc.NewClient(logger, next, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, client.Start(ctx)) + + stop(client, cancel) + requireClientWaitReturns(t, client) + require.True(t, bus.IsRunning(), "stopping the wrapper stopped the shared event bus") + }) + } +} + +// lifecycleClient records the join of a wrapped client exposing its running state. +type lifecycleClient struct { + rpcclient.Client + running bool + waited atomic.Bool +} + +func (c *lifecycleClient) Start(context.Context) error { return nil } +func (c *lifecycleClient) IsRunning() bool { return c.running } +func (c *lifecycleClient) Wait() { c.waited.Store(true) } + +// The wrapper joins exactly the client its own start attempt started. +func TestClientJoinsOnlyClientItStarted(t *testing.T) { + for _, alreadyRunning := range []bool{false, true} { + next := &lifecycleClient{running: alreadyRunning} + client := lrpc.NewClient(log.NewNopLogger(), next, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, client.Start(ctx)) + + cancel() + requireClientWaitReturns(t, client) + require.Equal(t, !alreadyRunning, next.waited.Load(), "already running: %t", alreadyRunning) + } +} From 1b427226d0b137c3f990de08b207c260701381ad Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:15:20 +0000 Subject: [PATCH 32/39] fix(rpc): join HTTP client event workers on stop Resubscribing used context.TODO() and wsEvents.Stop joined only the transport. After cancellation, eventListener or a reconnect callback could block forever on a send that no writer would ever receive, outliving Stop and a light client's Wait. wsEvents now has a lifetime derived from the Start context, observed by the resubscribe delay, the send and event delivery. Stop waits for the transport, ends that lifetime and joins eventListener, so a subscriber that stopped reading cannot block it. WSClient.Stop joins reconnect callbacks, which must return once the context ends, and Send fails once the client has shut down. Co-Authored-By: Claude Opus 5.5 --- UPGRADING.md | 4 +- rpc/client/http/ws.go | 53 +++++++--- rpc/client/http/ws_test.go | 138 ++++++++++++++++++++++++++- rpc/jsonrpc/client/ws_client.go | 19 +++- rpc/jsonrpc/client/ws_client_test.go | 64 +++++++++++++ 5 files changed, 261 insertions(+), 17 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index dde49e2527..e3928009c8 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -26,7 +26,9 @@ ends or the attempts run out, instead of returning while a reconnect could still start workers. Dials now observe that context and fail after ten seconds, including the one in `Start`, so a peer that never answers the handshake cannot block either call. Once the context ends, `Stop` waits at most for a write in progress to hit -its ten-second deadline. +its ten-second deadline. It also joins `OnReconnect` callbacks, which must return +once the context ends, and `Send` fails once the client has shut down. Stopping the +HTTP RPC client also joins its event listener and resubscriptions. `rpc/jsonrpc/server.Serve` and `ServeTLS` always close the listener before returning, including when TLS setup fails. Cancellation now reaches request diff --git a/rpc/client/http/ws.go b/rpc/client/http/ws.go index 621d145022..1f82d8b18a 100644 --- a/rpc/client/http/ws.go +++ b/rpc/client/http/ws.go @@ -26,6 +26,14 @@ type wsEvents struct { // cleared closes when UnsubscribeAll forgets the subscriptions read with it, // and is then replaced. Both fields are guarded by mtx. cleared chan struct{} + + // lifetime bounds resubscribing and event delivery. It ends with the context + // passed to Start, or in Stop if the client shut down by itself first. + lifetime context.Context + endLifetime context.CancelFunc + listenerDone chan struct{} + // resubscribeDelay gives Tendermint time to restart after an RPC error. + resubscribeDelay time.Duration } type wsSubscription struct { @@ -41,6 +49,8 @@ func newWsEvents(remote string) (*wsEvents, error) { Logger: log.NewNopLogger(), subscriptions: make(map[string]*wsSubscription), cleared: make(chan struct{}), + + resubscribeDelay: time.Second, } var err error @@ -50,7 +60,7 @@ func newWsEvents(remote string) (*wsEvents, error) { } w.ws.OnReconnect(func() { // resubscribe immediately - w.redoSubscriptionsAfter(0 * time.Second) + w.redoSubscriptionsAfter(w.lifetime, 0) }) w.ws.Logger = w.Logger @@ -59,15 +69,31 @@ func newWsEvents(remote string) (*wsEvents, error) { // Start starts the websocket client and the event loop. func (w *wsEvents) Start(ctx context.Context) error { + w.lifetime, w.endLifetime = context.WithCancel(ctx) if err := w.ws.Start(ctx); err != nil { + w.endLifetime() return err } - go w.eventListener(ctx) + w.listenerDone = make(chan struct{}) + go w.eventListener(w.lifetime, w.listenerDone) return nil } -// Stop shuts down the websocket client. -func (w *wsEvents) Stop() error { return w.ws.Stop() } +// Stop waits for the websocket client, which joins reconnect callbacks, to shut +// down with the context passed to Start, then joins the event loop. +func (w *wsEvents) Stop() error { + err := w.ws.Stop() + w.stopListener() + return err +} + +// stopListener ends the event loop, even if blocked on a subscriber not reading. +func (w *wsEvents) stopListener() { + if w.listenerDone != nil { + w.endLifetime() + <-w.listenerDone + } +} // Subscribe implements SubscriptionClient by using WSClient to subscribe given // subscriber to query. By default, it returns a channel with cap=1. Error is @@ -142,10 +168,14 @@ func (w *wsEvents) UnsubscribeAll(ctx context.Context, _subscriber string) error // After being reconnected, it is necessary to redo subscription to server // otherwise no data will be automatically received. -func (w *wsEvents) redoSubscriptionsAfter(d time.Duration) { - time.Sleep(d) - - ctx := context.TODO() +func (w *wsEvents) redoSubscriptionsAfter(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + return + } // Sending can block until the writer restarts, so no lock is held across it. // eventListener writes to the map under RLock, so reading it here needs Lock. @@ -183,7 +213,8 @@ func isErrAlreadySubscribed(err error) bool { return strings.Contains(err.Error(), pubsub.ErrAlreadySubscribed.Error()) } -func (w *wsEvents) eventListener(ctx context.Context) { +func (w *wsEvents) eventListener(ctx context.Context, done chan<- struct{}) { + defer close(done) for { select { case resp, ok := <-w.ws.ResponsesCh: @@ -198,9 +229,9 @@ func (w *wsEvents) eventListener(ctx context.Context) { // We can ignore ErrAlreadySubscribed, but need to retry in other // cases. if !isErrAlreadySubscribed(resp.Error) { - // Resubscribe after 1 second to give Tendermint time to restart (if + // Resubscribe after a delay to give Tendermint time to restart (if // crashed). - w.redoSubscriptionsAfter(1 * time.Second) + w.redoSubscriptionsAfter(ctx, w.resubscribeDelay) } continue } diff --git a/rpc/client/http/ws_test.go b/rpc/client/http/ws_test.go index 3ab1555ee2..798c8392c1 100644 --- a/rpc/client/http/ws_test.go +++ b/rpc/client/http/ws_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "runtime" + "strings" "sync" "testing" "time" @@ -21,6 +23,7 @@ import ( type fakeUpstream struct { mtx sync.Mutex events map[string][]string + failed map[string]bool // queries whose subscribe gets an RPC error conn *websocket.Conn subscribes []string } @@ -52,8 +55,14 @@ func (u *fakeUpstream) ServeHTTP(rw http.ResponseWriter, r *http.Request) { } u.mtx.Lock() u.subscribes = append(u.subscribes, params.Query) - events := u.events[params.Query] + events, failed := u.events[params.Query], u.failed[params.Query] u.mtx.Unlock() + if failed { + if err := conn.WriteJSON(req.MakeErrorf(rpctypes.CodeInternalError, "subscribe failed")); err != nil { + return + } + continue + } for _, query := range events { res := req.MakeResponse(coretypes.ResultEvent{ Query: query, @@ -188,7 +197,7 @@ func TestUnsubscribeAllDuringResubscribe(t *testing.T) { reconnecting(t, w, upstream) resubscribed := make(chan struct{}) - go func() { defer close(resubscribed); w.redoSubscriptionsAfter(0) }() + go func() { defer close(resubscribed); w.redoSubscriptionsAfter(ctx, 0) }() // Nothing observable marks the resubscribe blocking on the stopped writer; this // window only lets it get there, and cannot fail correct code. <-time.After(50 * time.Millisecond) @@ -242,3 +251,128 @@ func TestSubscribeAfterRepeatedUnsubscribeAllBlocksNormally(t *testing.T) { requireEvent(t, later, "later") } } + +// goroutinesIn counts running goroutines with a frame in the named function. +func goroutinesIn(function string) int { + stacks := make([]byte, 1<<24) + return strings.Count(string(stacks[:runtime.Stack(stacks, true)]), function+"(") +} + +func requireStopReturns(t *testing.T, w *wsEvents) { + t.Helper() + stopped := make(chan error, 1) + go func() { stopped <- w.Stop() }() + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("Stop did not return") + } +} + +// An upstream RPC error sends eventListener into a delayed resubscribe. Ending the +// context there must end the worker, not strand it on a send nobody receives. +func TestStopJoinsListenerDuringResubscribe(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + listeners := goroutinesIn("(*wsEvents).eventListener") + srv := httptest.NewServer(&fakeUpstream{failed: map[string]bool{"q": true}}) + t.Cleanup(srv.Close) + w, err := newWsEvents(srv.URL) + require.NoError(t, err) + w.resubscribeDelay = time.Hour // only cancellation can end this delay + require.NoError(t, w.Start(ctx)) + _, err = w.Subscribe(ctx, "", "q") + require.NoError(t, err) + require.Eventually(t, func() bool { return goroutinesIn("(*wsEvents).redoSubscriptionsAfter") == 1 }, + 5*time.Second, time.Millisecond) + + cancel() + requireStopReturns(t, w) + require.Equal(t, listeners, goroutinesIn("(*wsEvents).eventListener"), "eventListener outlived Stop") +} + +// A subscriber that stopped reading must not turn the join into a shutdown hang. +func TestStopJoinsListenerBlockedOnSubscriber(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + listeners := goroutinesIn("(*wsEvents).eventListener") + upstream := &fakeUpstream{events: map[string][]string{"q": {"q", "q", "q"}}} + w := startWsEvents(ctx, t, upstream) + q, err := w.Subscribe(ctx, "", "q") + require.NoError(t, err) + require.Eventually(t, func() bool { return len(q) == 1 }, 5*time.Second, time.Millisecond) + + cancel() + requireStopReturns(t, w) + require.Equal(t, listeners, goroutinesIn("(*wsEvents).eventListener"), "eventListener outlived Stop") +} + +// After the client shuts down by itself the context can still be live, and then +// only Stop can release a delivery no subscriber is reading. +func TestStopListenerReleasesDeliveryWithLiveContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + listeners := goroutinesIn("(*wsEvents).eventListener") + upstream := &fakeUpstream{events: map[string][]string{"q": {"q", "q", "q"}}} + w := startWsEvents(ctx, t, upstream) + q, err := w.Subscribe(ctx, "", "q") + require.NoError(t, err) + require.Eventually(t, func() bool { return len(q) == 1 }, 5*time.Second, time.Millisecond) + + stopped := make(chan struct{}) + go func() { w.stopListener(); close(stopped) }() + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("stopListener blocked on a subscriber not reading") + } + require.Equal(t, listeners, goroutinesIn("(*wsEvents).eventListener"), "eventListener outlived stopListener") +} + +// Starting fails cleanly, and a client that never ran has nothing to join. +func TestStopWithoutRunningListener(t *testing.T) { + w, err := newWsEvents("tcp://127.0.0.1:1") + require.NoError(t, err) + require.Error(t, w.Start(context.Background())) + requireStopReturns(t, w) +} + +// Stop must not report completion while the event loop can still run, here held +// on a lock that does not observe cancellation. +func TestStopWaitsForListenerToExit(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{events: map[string][]string{"q": {"q"}}} + w := startWsEvents(ctx, t, upstream) + w.mtx.Lock() + unlock := sync.OnceFunc(w.mtx.Unlock) + defer unlock() + require.NoError(t, w.ws.Subscribe(ctx, "q")) + require.Eventually(t, func() bool { + stacks := make([]byte, 1<<24) + for _, stack := range strings.Split(string(stacks[:runtime.Stack(stacks, true)]), "\n\n") { + if strings.Contains(stack, "(*wsEvents).eventListener(") && strings.Contains(stack, "RLock") { + return true + } + } + return false + }, 5*time.Second, time.Millisecond) + + cancel() + stopped := make(chan error, 1) + go func() { stopped <- w.Stop() }() + // Nothing observable marks Stop blocking; this window cannot fail correct code. + select { + case err := <-stopped: + t.Fatalf("Stop returned while the event loop was still running: %v", err) + case <-time.After(50 * time.Millisecond): + } + unlock() + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("Stop did not return after the event loop could exit") + } +} diff --git a/rpc/jsonrpc/client/ws_client.go b/rpc/jsonrpc/client/ws_client.go index f79c73ca00..edc6c5adc3 100644 --- a/rpc/jsonrpc/client/ws_client.go +++ b/rpc/jsonrpc/client/ws_client.go @@ -3,6 +3,7 @@ package client import ( "context" "encoding/json" + "errors" "fmt" mrand "math/rand" "net" @@ -26,6 +27,9 @@ type wsOptions struct { PingPeriod time.Duration // frequency with which pings are sent } +// errClientStopped reports a send to a client that has shut down. +var errClientStopped = errors.New("websocket client stopped") + // defaultWSOptions are the default websocket connection settings. var defaultWSOptions = wsOptions{ MaxReconnectAttempts: 10, // first: 2 sec, last: 17 min. @@ -70,6 +74,8 @@ type WSClient struct { //nolint: maligned // reconnectDone closes when reconnectRoutine exits; no workers start after that. reconnectDone chan struct{} closeResponses sync.Once + // callbacks tracks running onReconnect calls. Only reconnectRoutine adds to it. + callbacks sync.WaitGroup mtx sync.RWMutex reconnecting bool @@ -171,6 +177,7 @@ func (c *WSClient) Stop() error { return nil } <-c.reconnectDone + c.callbacks.Wait() c.closeResponsesCh() return nil } @@ -193,8 +200,8 @@ func (c *WSClient) IsReconnecting() bool { } // Send the given RPC request to the server. Results will be available on -// ResponsesCh, errors, if any, on ErrorsCh. Will block until send succeeds or -// ctx.Done is closed. +// ResponsesCh, errors, if any, on ErrorsCh. Will block until send succeeds, ctx.Done +// is closed or the client has shut down. func (c *WSClient) Send(ctx context.Context, request rpctypes.RPCRequest) error { select { case c.send <- request: @@ -205,6 +212,8 @@ func (c *WSClient) Send(ctx context.Context, request rpctypes.RPCRequest) error return nil case <-ctx.Done(): return ctx.Err() + case <-c.reconnectDone: + return errClientStopped } } @@ -288,7 +297,11 @@ func (c *WSClient) reconnect(ctx context.Context) error { } else { c.Logger.Info("reconnected") if c.onReconnect != nil { - go c.onReconnect() + c.callbacks.Add(1) + go func() { + defer c.callbacks.Done() + c.onReconnect() + }() } return nil } diff --git a/rpc/jsonrpc/client/ws_client_test.go b/rpc/jsonrpc/client/ws_client_test.go index e65546e91b..14e7fec447 100644 --- a/rpc/jsonrpc/client/ws_client_test.go +++ b/rpc/jsonrpc/client/ws_client_test.go @@ -545,3 +545,67 @@ func TestHTTPDialerObservesContext(t *testing.T) { _, err := makeHTTPDialer("tcp", "127.0.0.1:1")(ctx, "", "") require.ErrorIs(t, err, context.Canceled) } + +// Stop joins a reconnect callback already running, which may still use the client. +func TestWSClientStopJoinsReconnectCallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := httptest.NewServer(&myTestHandler{t: t, closeConnAfterRead: true}) + t.Cleanup(s.Close) + c, err := NewWS("//"+s.Listener.Addr().String(), "/websocket") + require.NoError(t, err) + entered := make(chan struct{}) + release := make(chan struct{}) + c.OnReconnect(func() { + close(entered) + <-release + }) + require.NoError(t, c.Start(ctx)) + require.NoError(t, c.Call(ctx, "drop", make(map[string]interface{}))) + select { + case <-entered: + case <-time.After(2 * wsCallTimeout): + close(release) + t.Fatal("reconnect callback did not run") + } + + cancel() + stopped := make(chan error, 1) + go func() { stopped <- c.Stop() }() + // Nothing observable marks Stop blocking; this window cannot fail correct code. + select { + case err := <-stopped: + close(release) + t.Fatalf("Stop returned before the reconnect callback finished: %v", err) + case <-time.After(50 * time.Millisecond): + } + close(release) + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(wsCallTimeout): + t.Fatal("Stop did not return after the reconnect callback finished") + } +} + +// A send needs the writer, so it cannot wait for one that will never return. +func TestWSClientSendFailsAfterShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := httptest.NewServer(&myTestHandler{t: t}) + t.Cleanup(s.Close) + c, err := NewWS("//"+s.Listener.Addr().String(), "/websocket") + require.NoError(t, err) + require.NoError(t, c.Start(ctx)) + cancel() + require.NoError(t, c.Stop()) + + sent := make(chan error, 1) + go func() { sent <- c.Call(context.Background(), "late", make(map[string]interface{})) }() + select { + case err := <-sent: + require.ErrorIs(t, err, errClientStopped) + case <-time.After(wsCallTimeout): + t.Fatal("Send blocked on a client that has shut down") + } +} From de50c6ac6070c2e9d749ca54ef5dbc9927f662ac Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:58:53 +0000 Subject: [PATCH 33/39] test(rpc): remove timing assumptions from WebSocket client tests The eventListener goroutine count included a previous test's listener still finishing, so the HTTP client tests now check the join they just made and stop their clients on cleanup. wsEvents records a query after sending its request, so a fast event could be dropped; the fake upstream now holds a query's events until Subscribe has returned. Stop can take most of five seconds to encode a large request under -race and load, so that test's hang detector allows thirty. Correct two WSClient comments: an OnReconnect callback must not call Stop, and Stop also waits for running callbacks, not only a write in progress. Co-Authored-By: Claude Opus 5.5 --- rpc/client/http/ws_test.go | 69 +++++++++++++++++++--------- rpc/jsonrpc/client/ws_client.go | 9 ++-- rpc/jsonrpc/client/ws_client_test.go | 4 +- 3 files changed, 55 insertions(+), 27 deletions(-) diff --git a/rpc/client/http/ws_test.go b/rpc/client/http/ws_test.go index 798c8392c1..c59863cbd0 100644 --- a/rpc/client/http/ws_test.go +++ b/rpc/client/http/ws_test.go @@ -23,7 +23,8 @@ import ( type fakeUpstream struct { mtx sync.Mutex events map[string][]string - failed map[string]bool // queries whose subscribe gets an RPC error + failed map[string]bool // queries whose subscribe gets an RPC error + gates map[string]chan struct{} // hold a query's events until closed conn *websocket.Conn subscribes []string } @@ -55,8 +56,11 @@ func (u *fakeUpstream) ServeHTTP(rw http.ResponseWriter, r *http.Request) { } u.mtx.Lock() u.subscribes = append(u.subscribes, params.Query) - events, failed := u.events[params.Query], u.failed[params.Query] + events, failed, gate := u.events[params.Query], u.failed[params.Query], u.gates[params.Query] u.mtx.Unlock() + if gate != nil { + <-gate + } if failed { if err := conn.WriteJSON(req.MakeErrorf(rpctypes.CodeInternalError, "subscribe failed")); err != nil { return @@ -76,6 +80,24 @@ func (u *fakeUpstream) ServeHTTP(rw http.ResponseWriter, r *http.Request) { } } +// subscribe subscribes w to query and lets the upstream send the query's events +// only afterwards. wsEvents records a query after sending its request, so it drops +// events that arrive first. +func (u *fakeUpstream) subscribe(ctx context.Context, t *testing.T, w *wsEvents, query string) <-chan coretypes.ResultEvent { + t.Helper() + gate := make(chan struct{}) + defer close(gate) + u.mtx.Lock() + if u.gates == nil { + u.gates = make(map[string]chan struct{}) + } + u.gates[query] = gate + u.mtx.Unlock() + out, err := w.Subscribe(ctx, "", query) + require.NoError(t, err) + return out +} + func (u *fakeUpstream) setEvents(events map[string][]string) { u.mtx.Lock() defer u.mtx.Unlock() @@ -103,6 +125,8 @@ func startWsEvents(ctx context.Context, t *testing.T, upstream *fakeUpstream) *w w, err := newWsEvents(srv.URL) require.NoError(t, err) require.NoError(t, w.Start(ctx)) + // The test's deferred cancel has run by then, so no worker outlives its test. + t.Cleanup(func() { requireStopReturns(t, w) }) return w } @@ -143,8 +167,7 @@ func TestUnsubscribeAllTimeoutDuringReconnectForgetsSubscriptions(t *testing.T) defer cancelCleanup() require.ErrorIs(t, w.UnsubscribeAll(cleanupCtx, ""), context.DeadlineExceeded) - other, err := w.Subscribe(ctx, "", "other") - require.NoError(t, err) + other := upstream.subscribe(ctx, t, w, "other") requireEvent(t, other, "other") require.NotContains(t, upstream.subscribesOnCurrentConnection(), "gone", "the reconnect restored a subscription whose cleanup timed out") @@ -159,8 +182,7 @@ func TestUnsubscribeAllReleasesBlockedDelivery(t *testing.T) { "other": {"other"}, }} w := startWsEvents(ctx, t, upstream) - gone, err := w.Subscribe(ctx, "", "gone") - require.NoError(t, err) + gone := upstream.subscribe(ctx, t, w, "gone") // Taking one event lets the next fill the buffer, so the third has to block. requireEvent(t, gone, "gone") require.Eventually(t, func() bool { return len(gone) == 1 }, 5*time.Second, time.Millisecond) @@ -169,8 +191,7 @@ func TestUnsubscribeAllReleasesBlockedDelivery(t *testing.T) { <-time.After(50 * time.Millisecond) require.NoError(t, w.UnsubscribeAll(ctx, "")) - other, err := w.Subscribe(ctx, "", "other") - require.NoError(t, err) + other := upstream.subscribe(ctx, t, w, "other") requireEvent(t, other, "other") } @@ -218,8 +239,7 @@ func TestUnsubscribeAllDuringResubscribe(t *testing.T) { case <-time.After(10 * time.Second): t.Fatal("resubscribe did not finish after the reconnect") } - other, err := w.Subscribe(ctx, "", "other") - require.NoError(t, err) + other := upstream.subscribe(ctx, t, w, "other") requireEvent(t, other, "other") restored := 0 for _, query := range upstream.subscribesOnCurrentConnection() { @@ -244,8 +264,7 @@ func TestSubscribeAfterRepeatedUnsubscribeAllBlocksNormally(t *testing.T) { require.NoError(t, w.UnsubscribeAll(ctx, "")) require.NoError(t, w.UnsubscribeAll(ctx, "")) - later, err := w.Subscribe(ctx, "", "later") - require.NoError(t, err) + later := upstream.subscribe(ctx, t, w, "later") for range events { require.Eventually(t, func() bool { return len(later) == 1 }, 5*time.Second, time.Millisecond) requireEvent(t, later, "later") @@ -258,6 +277,16 @@ func goroutinesIn(function string) int { return strings.Count(string(stacks[:runtime.Stack(stacks, true)]), function+"(") } +// requireListenerExited must hold as soon as the join returns, so it does not wait. +func requireListenerExited(t *testing.T, w *wsEvents) { + t.Helper() + select { + case <-w.listenerDone: + default: + t.Fatal("eventListener outlived the join") + } +} + func requireStopReturns(t *testing.T, w *wsEvents) { t.Helper() stopped := make(chan error, 1) @@ -275,13 +304,13 @@ func requireStopReturns(t *testing.T, w *wsEvents) { func TestStopJoinsListenerDuringResubscribe(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - listeners := goroutinesIn("(*wsEvents).eventListener") srv := httptest.NewServer(&fakeUpstream{failed: map[string]bool{"q": true}}) t.Cleanup(srv.Close) w, err := newWsEvents(srv.URL) require.NoError(t, err) w.resubscribeDelay = time.Hour // only cancellation can end this delay require.NoError(t, w.Start(ctx)) + t.Cleanup(func() { requireStopReturns(t, w) }) _, err = w.Subscribe(ctx, "", "q") require.NoError(t, err) require.Eventually(t, func() bool { return goroutinesIn("(*wsEvents).redoSubscriptionsAfter") == 1 }, @@ -289,23 +318,21 @@ func TestStopJoinsListenerDuringResubscribe(t *testing.T) { cancel() requireStopReturns(t, w) - require.Equal(t, listeners, goroutinesIn("(*wsEvents).eventListener"), "eventListener outlived Stop") + requireListenerExited(t, w) } // A subscriber that stopped reading must not turn the join into a shutdown hang. func TestStopJoinsListenerBlockedOnSubscriber(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - listeners := goroutinesIn("(*wsEvents).eventListener") upstream := &fakeUpstream{events: map[string][]string{"q": {"q", "q", "q"}}} w := startWsEvents(ctx, t, upstream) - q, err := w.Subscribe(ctx, "", "q") - require.NoError(t, err) + q := upstream.subscribe(ctx, t, w, "q") require.Eventually(t, func() bool { return len(q) == 1 }, 5*time.Second, time.Millisecond) cancel() requireStopReturns(t, w) - require.Equal(t, listeners, goroutinesIn("(*wsEvents).eventListener"), "eventListener outlived Stop") + requireListenerExited(t, w) } // After the client shuts down by itself the context can still be live, and then @@ -313,11 +340,9 @@ func TestStopJoinsListenerBlockedOnSubscriber(t *testing.T) { func TestStopListenerReleasesDeliveryWithLiveContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - listeners := goroutinesIn("(*wsEvents).eventListener") upstream := &fakeUpstream{events: map[string][]string{"q": {"q", "q", "q"}}} w := startWsEvents(ctx, t, upstream) - q, err := w.Subscribe(ctx, "", "q") - require.NoError(t, err) + q := upstream.subscribe(ctx, t, w, "q") require.Eventually(t, func() bool { return len(q) == 1 }, 5*time.Second, time.Millisecond) stopped := make(chan struct{}) @@ -327,7 +352,7 @@ func TestStopListenerReleasesDeliveryWithLiveContext(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("stopListener blocked on a subscriber not reading") } - require.Equal(t, listeners, goroutinesIn("(*wsEvents).eventListener"), "eventListener outlived stopListener") + requireListenerExited(t, w) } // Starting fails cleanly, and a client that never ran has nothing to join. diff --git a/rpc/jsonrpc/client/ws_client.go b/rpc/jsonrpc/client/ws_client.go index edc6c5adc3..64927ad9d5 100644 --- a/rpc/jsonrpc/client/ws_client.go +++ b/rpc/jsonrpc/client/ws_client.go @@ -128,7 +128,8 @@ func NewWS(remoteAddr, endpoint string) (*WSClient, error) { // OnReconnect sets the callback, which will be called every time after // successful reconnect. -// Could only be set before Start. +// Could only be set before Start. The callback must return once the context passed +// to Start ends, and must not call Stop, which joins it and would never return. func (c *WSClient) OnReconnect(cb func()) { c.onReconnect = cb } @@ -166,9 +167,9 @@ func (c *WSClient) Start(ctx context.Context) error { // Stop blocks until the client is shut down and returns nil. It does not end the // client: shutdown follows the context passed to Start, or running out of -// reconnect attempts. After either, the wait is bounded by the write deadline of a -// write in progress. It is safe to call repeatedly and concurrently, and returns -// immediately if the client never started. +// reconnect attempts. After either, it waits for running OnReconnect callbacks, and +// for a write in progress, which ends at its write deadline. It is safe to call +// repeatedly and concurrently, and returns immediately if the client never started. // // TODO(creachadair): This method exists for compatibility with the original // service plumbing. Give it a better name (e.g., Wait). diff --git a/rpc/jsonrpc/client/ws_client_test.go b/rpc/jsonrpc/client/ws_client_test.go index 14e7fec447..44d3129670 100644 --- a/rpc/jsonrpc/client/ws_client_test.go +++ b/rpc/jsonrpc/client/ws_client_test.go @@ -513,10 +513,12 @@ func TestWSClientStopJoinsWorkerFailingAfterReconnectExit(t *testing.T) { // The reader reports this and closes the connection, so the write fails too. require.NoError(t, peer.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseGoingAway, "bye"), time.Now().Add(time.Second))) + // Encoding the request is slow under -race and load, and the writer cannot stop + // until it has finished. This only needs to tell that from a hang. select { case err := <-stopped: require.NoError(t, err) - case <-time.After(wsCallTimeout): + case <-time.After(6 * wsCallTimeout): t.Fatal("Stop did not return after the workers failed") } } From 6fb4a4c06ea8b485051c359b81681b01b1569d4b Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Thu, 5 Nov 2026 08:02:05 +0000 Subject: [PATCH 34/39] fix(privval): close the gRPC signer connection in SignerClient.Close NewSignerClient never stored the connection it was given, so Close always saw a nil conn and left the node-owned grpc.ClientConn (and its workers) open after shutdown or failed-start rollback. The client now takes ownership of conn and Close releases it. Co-Authored-By: Claude Opus 5.5 --- privval/grpc/client.go | 5 +++-- privval/grpc/client_test.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/privval/grpc/client.go b/privval/grpc/client.go index 99ba6506d7..9a004f25e8 100644 --- a/privval/grpc/client.go +++ b/privval/grpc/client.go @@ -31,20 +31,21 @@ type SignerClient struct { var _ types.PrivValidator = (*SignerClient)(nil) // NewSignerClient returns an instance of SignerClient. -// it will start the endpoint (if not already started) +// It takes ownership of conn, which Close closes. func NewSignerClient(conn *grpc.ClientConn, chainID string, log log.Logger) (*SignerClient, error) { sc := &SignerClient{ logger: log, chainID: chainID, + conn: conn, client: privvalproto.NewPrivValidatorAPIClient(conn), // Create the Private Validator Client } return sc, nil } -// Close closes the underlying connection +// Close closes the connection the client was created with. func (sc *SignerClient) Close() error { sc.logger.Info("Stopping service") if sc.conn != nil { diff --git a/privval/grpc/client_test.go b/privval/grpc/client_test.go index 18df566cd8..d0aa40eb05 100644 --- a/privval/grpc/client_test.go +++ b/privval/grpc/client_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/test/bufconn" @@ -182,3 +183,19 @@ func TestSignerClient_SignProposal(t *testing.T) { assert.Equal(t, pbWant.Signature, pbHave.Signature) } + +// TestSignerClientCloseShutsDownConnection checks that Close releases the +// connection the client was built on: the node registers Close as the only +// teardown of a grpc:// signer connection. +func TestSignerClientCloseShutsDownConnection(t *testing.T) { + conn, err := grpc.NewClient("localhost", + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + require.NoError(t, err) + + client, err := tmgrpc.NewSignerClient(conn, chainID, log.NewTestingLogger(t)) + require.NoError(t, err) + + require.NoError(t, client.Close()) + assert.Equal(t, connectivity.Shutdown, conn.GetState()) +} From eff6da2c0f42230c4473e140152765494954fec0 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:33:01 +0000 Subject: [PATCH 35/39] fix(light): leave an already started HTTP client to its caller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The light RPC client recognized caller-owned clients through IsRunning, which the HTTP client did not expose, so it started a running HTTP client again (redialing and replacing its transport) and later stopped it on drain. Expose IsRunning on the HTTP client and skip both start and drain for clients already running. 🤖 Co-authored by [Claudius the Magnificent](https://github.com/lklimek/claudius) AI Agent Co-Authored-By: Claude Opus 5.5 --- light/rpc/client.go | 11 ++++----- light/rpc/shutdown_test.go | 47 ++++++++++++++++++++++++++++++++++++++ rpc/client/http/ws.go | 13 +++++++++++ 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/light/rpc/client.go b/light/rpc/client.go index 95673d5e9d..1ec53a4801 100644 --- a/light/rpc/client.go +++ b/light/rpc/client.go @@ -110,15 +110,14 @@ func NewClient(logger log.Logger, next rpcclient.Client, lc LightClient, opts .. func (c *Client) OnStart(ctx context.Context) error { c.serviceCtx = ctx - // A local client shares its node's event bus, whose Start ignores ctx if it is - // already running. Clients not reporting their state are started here. + // A client already running, such as a local client sharing its node's event + // bus, belongs to its caller; other clients are started here. running, ok := c.next.(interface{ IsRunning() bool }) c.callerOwnsNext = ok && running.IsRunning() - if err := c.next.Start(ctx); err != nil { - return err + if c.callerOwnsNext { + return nil } - - return nil + return c.next.Start(ctx) } func (c *Client) OnStop() {} diff --git a/light/rpc/shutdown_test.go b/light/rpc/shutdown_test.go index f5891d7a7c..dba5030bd1 100644 --- a/light/rpc/shutdown_test.go +++ b/light/rpc/shutdown_test.go @@ -76,3 +76,50 @@ func TestClientWaitDuringHungUpstreamRedial(t *testing.T) { t.Fatal("light client Wait hung after cancellation") } } + +// An HTTP client its caller already started belongs to the caller: the wrapper +// must neither start a second transport nor end the caller's one. +func TestClientLeavesCallerStartedHTTPClientRunning(t *testing.T) { + logger := log.NewNopLogger() + ws := rpcserver.NewWebsocketManager(logger, map[string]*rpcserver.RPCFunc{}) + var sessions atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("/websocket", func(w http.ResponseWriter, r *http.Request) { + sessions.Add(1) + ws.WebsocketHandler(w, r) + }) + upstream := httptest.NewServer(mux) + t.Cleanup(upstream.Close) + next, err := rpchttp.New(upstream.URL) + require.NoError(t, err) + nextCtx, stopNext := context.WithCancel(context.Background()) + defer func() { + stopNext() + stopped := make(chan error, 1) + go func() { stopped <- next.Stop() }() + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Error("the caller's client did not stop") + } + }() + require.NoError(t, next.Start(nextCtx)) + + client := NewClient(logger, next, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, client.Start(ctx)) + require.Equal(t, int32(1), sessions.Load(), "the wrapper started the caller's client again") + + cancel() + waited := make(chan struct{}) + go func() { client.Wait(); close(waited) }() + select { + case <-waited: + case <-time.After(5 * time.Second): + t.Fatal("light client Wait blocked on a client it does not own") + } + _, err = next.Subscribe(nextCtx, "caller", "tm.event = 'NewBlock'") + require.NoError(t, err, "stopping the wrapper ended the caller's client") +} diff --git a/rpc/client/http/ws.go b/rpc/client/http/ws.go index 1f82d8b18a..2b0b45c0b0 100644 --- a/rpc/client/http/ws.go +++ b/rpc/client/http/ws.go @@ -87,6 +87,19 @@ func (w *wsEvents) Stop() error { return err } +// IsRunning reports whether the client has started and not yet shut down. +func (w *wsEvents) IsRunning() bool { + if w.listenerDone == nil { + return false + } + select { + case <-w.listenerDone: + return false + default: + return true + } +} + // stopListener ends the event loop, even if blocked on a subscriber not reading. func (w *wsEvents) stopListener() { if w.listenerDone != nil { From 46790158a19e08c50c7c5e17872d7c22fe3bdbf2 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:33:01 +0000 Subject: [PATCH 36/39] fix(rpc): unsubscribe only the departing subscriber's queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wsEvents ignored the subscriber: UnsubscribeAll forgot every local subscription and sent unsubscribe_all upstream, so one downstream session's cleanup removed the queries of all others sharing the connection, and a second subscriber to a query replaced the first. Track local subscribers per query over a single upstream subscription. Unsubscribe and UnsubscribeAll forget only the given subscriber's subscriptions, releasing deliveries blocked on its channels, and unsubscribe upstream only queries no subscriber uses, outside the lock. 🤖 Co-authored by [Claudius the Magnificent](https://github.com/lklimek/claudius) AI Agent Co-Authored-By: Claude Opus 5.5 --- rpc/client/http/ws.go | 132 +++++++++++++++++++++++++------------ rpc/client/http/ws_test.go | 74 +++++++++++++++++++-- 2 files changed, 161 insertions(+), 45 deletions(-) diff --git a/rpc/client/http/ws.go b/rpc/client/http/ws.go index 2b0b45c0b0..1fe0772f6b 100644 --- a/rpc/client/http/ws.go +++ b/rpc/client/http/ws.go @@ -3,6 +3,7 @@ package http import ( "context" "encoding/json" + "errors" "fmt" "strings" "time" @@ -21,11 +22,10 @@ type wsEvents struct { Logger log.Logger ws *jsonrpcclient.WSClient - mtx sync.RWMutex + mtx sync.RWMutex + // subscriptions maps each query, and its upstream subscription ID once known, + // to the local subscribers sharing its single upstream subscription. subscriptions map[string]*wsSubscription - // cleared closes when UnsubscribeAll forgets the subscriptions read with it, - // and is then replaced. Both fields are guarded by mtx. - cleared chan struct{} // lifetime bounds resubscribing and event delivery. It ends with the context // passed to Start, or in Stop if the client shut down by itself first. @@ -36,10 +36,18 @@ type wsEvents struct { resubscribeDelay time.Duration } +var errNotRunning = errors.New("websocket client is not running") + type wsSubscription struct { - res chan coretypes.ResultEvent - id string - query string + id string + query string + subscribers map[string]*wsSubscriber +} + +type wsSubscriber struct { + res chan coretypes.ResultEvent + // gone closes on unsubscribing, releasing a delivery its owner no longer reads. + gone chan struct{} } var _ rpcclient.SubscriptionClient = (*wsEvents)(nil) @@ -48,7 +56,6 @@ func newWsEvents(remote string) (*wsEvents, error) { w := &wsEvents{ Logger: log.NewNopLogger(), subscriptions: make(map[string]*wsSubscription), - cleared: make(chan struct{}), resubscribeDelay: time.Second, } @@ -119,8 +126,11 @@ func (w *wsEvents) stopListener() { // The channel is never closed to prevent clients from seeing an erroneous // event. // +// Subscribers share one upstream subscription per query, which Tendermint keys +// by remote IP; subscriber only identifies the owner locally. +// // It returns an error if wsEvents is not running. -func (w *wsEvents) Subscribe(ctx context.Context, _subscriber, query string, +func (w *wsEvents) Subscribe(ctx context.Context, subscriber, query string, outCapacity ...int) (out <-chan coretypes.ResultEvent, err error) { if err := w.ws.Subscribe(ctx, query); err != nil { return nil, err @@ -134,49 +144,87 @@ func (w *wsEvents) Subscribe(ctx context.Context, _subscriber, query string, outc := make(chan coretypes.ResultEvent, outCap) w.mtx.Lock() defer w.mtx.Unlock() - // subscriber param is ignored because Tendermint will override it with - // remote IP anyway. - w.subscriptions[query] = &wsSubscription{res: outc, query: query} + sub, ok := w.subscriptions[query] + if !ok { + sub = &wsSubscription{query: query, subscribers: make(map[string]*wsSubscriber)} + w.subscriptions[query] = sub + } + if old, ok := sub.subscribers[subscriber]; ok { + close(old.gone) + } + sub.subscribers[subscriber] = &wsSubscriber{res: outc, gone: make(chan struct{})} return outc, nil } -// Unsubscribe implements SubscriptionClient by using WSClient to unsubscribe -// given subscriber from query. +// Unsubscribe implements SubscriptionClient by unsubscribing given subscriber +// from query, and using WSClient to unsubscribe from the upstream once no +// subscriber is left. // -// It returns an error if wsEvents is not running. -func (w *wsEvents) Unsubscribe(ctx context.Context, _subscriber, query string) error { - if err := w.ws.Unsubscribe(ctx, query); err != nil { - return err - } - +// It returns an error if wsEvents is not running. The subscription is forgotten +// even then; see UnsubscribeAll. +func (w *wsEvents) Unsubscribe(ctx context.Context, subscriber, query string) error { w.mtx.Lock() - defer w.mtx.Unlock() - info, ok := w.subscriptions[query] - if ok { - if info.id != "" { - delete(w.subscriptions, info.id) - } - delete(w.subscriptions, info.query) + var unused []string + if sub, ok := w.subscriptions[query]; ok && sub.query == query { + unused = w.removeSubscriber(sub, subscriber, unused) } + w.mtx.Unlock() - return nil + return w.unsubscribeUpstream(ctx, unused) } -// UnsubscribeAll implements SubscriptionClient by using WSClient to -// unsubscribe given subscriber from all the queries. +// UnsubscribeAll implements SubscriptionClient by unsubscribing given +// subscriber from all the queries, and using WSClient to unsubscribe from the +// upstream the queries no other subscriber uses. // // It returns an error if wsEvents is not running. Subscriptions are forgotten // even then: a reconnect must not restore one whose channel nobody reads, which // would stall every other subscription on the single events loop. -func (w *wsEvents) UnsubscribeAll(ctx context.Context, _subscriber string) error { +func (w *wsEvents) UnsubscribeAll(ctx context.Context, subscriber string) error { w.mtx.Lock() - w.subscriptions = make(map[string]*wsSubscription) - close(w.cleared) - w.cleared = make(chan struct{}) + var unused []string + for key, sub := range w.subscriptions { + if key == sub.query { + unused = w.removeSubscriber(sub, subscriber, unused) + } + } w.mtx.Unlock() - return w.ws.UnsubscribeAll(ctx) + return w.unsubscribeUpstream(ctx, unused) +} + +// removeSubscriber forgets subscriber's subscription to sub, and sub itself once +// unused, which it then appends to unused. Callers must hold mtx. +func (w *wsEvents) removeSubscriber(sub *wsSubscription, subscriber string, unused []string) []string { + s, ok := sub.subscribers[subscriber] + if !ok { + return unused + } + close(s.gone) + delete(sub.subscribers, subscriber) + if len(sub.subscribers) > 0 { + return unused + } + delete(w.subscriptions, sub.query) + if sub.id != "" { + delete(w.subscriptions, sub.id) + } + return append(unused, sub.query) +} + +// unsubscribeUpstream must not hold mtx: sending can block until the writer restarts. +// After an error, eventListener drops events for the queries left subscribed. +func (w *wsEvents) unsubscribeUpstream(ctx context.Context, queries []string) error { + if len(queries) == 0 && !w.IsRunning() { + return errNotRunning + } + for _, query := range queries { + if err := w.ws.Unsubscribe(ctx, query); err != nil { + return err + } + } + return nil } // After being reconnected, it is necessary to redo subscription to server @@ -257,19 +305,21 @@ func (w *wsEvents) eventListener(ctx context.Context, done chan<- struct{}) { } w.mtx.RLock() - out, ok := w.subscriptions[result.Query] - if ok { + var subscribers []*wsSubscriber + if out, ok := w.subscriptions[result.Query]; ok { if _, idOk := w.subscriptions[result.SubscriptionID]; !idOk { out.id = result.SubscriptionID w.subscriptions[result.SubscriptionID] = out } + for _, s := range out.subscribers { + subscribers = append(subscribers, s) + } } - cleared := w.cleared w.mtx.RUnlock() - if ok { + for _, s := range subscribers { select { - case out.res <- *result: - case <-cleared: + case s.res <- *result: + case <-s.gone: case <-ctx.Done(): return } diff --git a/rpc/client/http/ws_test.go b/rpc/client/http/ws_test.go index c59863cbd0..0ed66db04f 100644 --- a/rpc/client/http/ws_test.go +++ b/rpc/client/http/ws_test.go @@ -27,6 +27,7 @@ type fakeUpstream struct { gates map[string]chan struct{} // hold a query's events until closed conn *websocket.Conn subscribes []string + others []string // non-subscribe requests on any connection, as "method query" } func (u *fakeUpstream) ServeHTTP(rw http.ResponseWriter, r *http.Request) { @@ -45,9 +46,6 @@ func (u *fakeUpstream) ServeHTTP(rw http.ResponseWriter, r *http.Request) { if err := conn.ReadJSON(&req); err != nil { return } - if req.Method != "subscribe" { - continue - } var params struct { Query string `json:"query"` } @@ -55,6 +53,11 @@ func (u *fakeUpstream) ServeHTTP(rw http.ResponseWriter, r *http.Request) { return } u.mtx.Lock() + if req.Method != "subscribe" { + u.others = append(u.others, strings.TrimSpace(req.Method+" "+params.Query)) + u.mtx.Unlock() + continue + } u.subscribes = append(u.subscribes, params.Query) events, failed, gate := u.events[params.Query], u.failed[params.Query], u.gates[params.Query] u.mtx.Unlock() @@ -84,6 +87,12 @@ func (u *fakeUpstream) ServeHTTP(rw http.ResponseWriter, r *http.Request) { // only afterwards. wsEvents records a query after sending its request, so it drops // events that arrive first. func (u *fakeUpstream) subscribe(ctx context.Context, t *testing.T, w *wsEvents, query string) <-chan coretypes.ResultEvent { + t.Helper() + return u.subscribeAs(ctx, t, w, "", query) +} + +func (u *fakeUpstream) subscribeAs(ctx context.Context, t *testing.T, w *wsEvents, + subscriber, query string) <-chan coretypes.ResultEvent { t.Helper() gate := make(chan struct{}) defer close(gate) @@ -93,7 +102,7 @@ func (u *fakeUpstream) subscribe(ctx context.Context, t *testing.T, w *wsEvents, } u.gates[query] = gate u.mtx.Unlock() - out, err := w.Subscribe(ctx, "", query) + out, err := w.Subscribe(ctx, subscriber, query) require.NoError(t, err) return out } @@ -112,6 +121,12 @@ func (u *fakeUpstream) dropConnection(t *testing.T) { require.NoError(t, u.conn.Close()) } +func (u *fakeUpstream) otherRequests() []string { + u.mtx.Lock() + defer u.mtx.Unlock() + return append([]string(nil), u.others...) +} + func (u *fakeUpstream) subscribesOnCurrentConnection() []string { u.mtx.Lock() defer u.mtx.Unlock() @@ -173,6 +188,57 @@ func TestUnsubscribeAllTimeoutDuringReconnectForgetsSubscriptions(t *testing.T) "the reconnect restored a subscription whose cleanup timed out") } +// One subscriber's cleanup, whether sent upstream or timed out during a reconnect, +// must leave every other subscriber's queries subscribed and delivering. All +// subscribers share one upstream session, so the cleanup may only unsubscribe +// queries nobody else uses. +func TestUnsubscribeAllKeepsOtherSubscribers(t *testing.T) { + for name, tc := range map[string]struct { + bQuery string + timeout bool + sent []string + }{ + "sent": {bQuery: "b", sent: []string{"unsubscribe a"}}, + "timeout during reconnect": {bQuery: "b", timeout: true}, + "shared query": {bQuery: "a"}, + } { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{} + w := startWsEvents(ctx, t, upstream) + a := upstream.subscribeAs(ctx, t, w, "A", "a") + b := upstream.subscribeAs(ctx, t, w, "B", tc.bQuery) + // Dropping a session with unread requests resets it, which the client misses. + require.Eventually(t, func() bool { return len(upstream.subscribesOnCurrentConnection()) == 2 }, + 5*time.Second, time.Millisecond) + + if tc.timeout { + reconnecting(t, w, upstream) + cleanupCtx, cancelCleanup := context.WithTimeout(ctx, 10*time.Millisecond) + defer cancelCleanup() + require.ErrorIs(t, w.UnsubscribeAll(cleanupCtx, "A"), context.DeadlineExceeded) + } else { + require.NoError(t, w.UnsubscribeAll(ctx, "A")) + // The upstream handles requests in order, so this event follows the cleanup. + upstream.setEvents(map[string][]string{"sync": {"sync"}}) + requireEvent(t, upstream.subscribeAs(ctx, t, w, "C", "sync"), "sync") + require.Equal(t, tc.sent, upstream.otherRequests()) + reconnecting(t, w, upstream) + } + // Served only once the reconnect resubscribes. + upstream.setEvents(map[string][]string{"a": {"a"}, "b": {"b"}}) + + requireEvent(t, b, tc.bQuery) + require.Empty(t, a, "a cleaned up subscriber received an event") + if tc.bQuery != "a" { + require.NotContains(t, upstream.subscribesOnCurrentConnection(), "a", + "the reconnect restored a cleaned up subscriber's query") + } + }) + } +} + // Forgetting a subscription must also release a delivery blocked on its unread channel. func TestUnsubscribeAllReleasesBlockedDelivery(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) From 9f412b5ada022ac2f110a2a75eaef8e6231c234f Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:43:55 +0000 Subject: [PATCH 37/39] test(rpc): reproduce lost upstream subscription on concurrent join and leave A subscriber joining a query while its last other subscriber leaves can end up registered locally while the upstream session is unsubscribed. This test fails on the reviewed head and documents the interleaving. Co-Authored-By: Claude Opus 5.5 --- rpc/client/http/ws_join_leave_test.go | 114 ++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 rpc/client/http/ws_join_leave_test.go diff --git a/rpc/client/http/ws_join_leave_test.go b/rpc/client/http/ws_join_leave_test.go new file mode 100644 index 0000000000..4c4cd851f1 --- /dev/null +++ b/rpc/client/http/ws_join_leave_test.go @@ -0,0 +1,114 @@ +package http + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/require" + + rpctypes "github.com/dashpay/tenderdash/rpc/jsonrpc/types" +) + +// orderedUpstream records every request on the current session, in order, and +// models the upstream subscription state per query. +type orderedUpstream struct { + mtx sync.Mutex + conn *websocket.Conn + requests []string + subscribed map[string]bool +} + +func (u *orderedUpstream) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + conn, err := (&websocket.Upgrader{}).Upgrade(rw, r, nil) + if err != nil { + return + } + defer conn.Close() + u.mtx.Lock() + u.conn = conn + u.requests = nil + u.subscribed = map[string]bool{} + u.mtx.Unlock() + for { + var req rpctypes.RPCRequest + if err := conn.ReadJSON(&req); err != nil { + return + } + var params struct { + Query string `json:"query"` + } + _ = json.Unmarshal(req.Params, ¶ms) + u.mtx.Lock() + u.requests = append(u.requests, req.Method+" "+params.Query) + switch req.Method { + case "subscribe": + u.subscribed[params.Query] = true + case "unsubscribe": + delete(u.subscribed, params.Query) + } + u.mtx.Unlock() + } +} + +func (u *orderedUpstream) state() ([]string, map[string]bool) { + u.mtx.Lock() + defer u.mtx.Unlock() + s := map[string]bool{} + for k, v := range u.subscribed { + s[k] = v + } + return append([]string(nil), u.requests...), s +} + +// Subscriber B joins query q while subscriber A, its last other user, leaves it. +// Afterwards B is registered locally, so the upstream must still be subscribed to q. +func TestConcurrentSubscribeAndLastUnsubscribeKeepsUpstream(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &orderedUpstream{} + srv := httptest.NewServer(upstream) + t.Cleanup(srv.Close) + w, err := newWsEvents(srv.URL) + require.NoError(t, err) + require.NoError(t, w.Start(ctx)) + t.Cleanup(func() { cancel(); _ = w.Stop() }) + + _, err = w.Subscribe(ctx, "A", "q") + require.NoError(t, err) + require.Eventually(t, func() bool { _, s := upstream.state(); return s["q"] }, 5*time.Second, time.Millisecond) + + // While the writer is down, sends queue in FIFO order on the unbuffered channel. + upstream.mtx.Lock() + require.NoError(t, upstream.conn.Close()) + upstream.mtx.Unlock() + require.Eventually(t, w.ws.IsReconnecting, 5*time.Second, time.Millisecond) + + bDone := make(chan error, 1) + go func() { + _, err := w.Subscribe(ctx, "B", "q") // blocks in the upstream send, not yet registered + bDone <- err + }() + time.Sleep(100 * time.Millisecond) + aDone := make(chan error, 1) + go func() { aDone <- w.UnsubscribeAll(ctx, "A") }() + + require.NoError(t, <-bDone) + require.NoError(t, <-aDone) + + w.mtx.RLock() + sub, registered := w.subscriptions["q"] + bRegistered := registered && sub.subscribers["B"] != nil + w.mtx.RUnlock() + require.True(t, bRegistered, "B is registered locally") + + require.Eventually(t, func() bool { r, _ := upstream.state(); return len(r) >= 2 }, 5*time.Second, time.Millisecond) + time.Sleep(200 * time.Millisecond) + reqs, s := upstream.state() + require.True(t, s["q"], "upstream unsubscribed q while B still needs it; requests on session: %v", reqs) +} From fd0019028d451e7457667d0370d3e91f9c738c2f Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:52:07 +0000 Subject: [PATCH 38/39] fix(rpc): order upstream subscribes with unsubscribes of unused queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A subscriber joining a query while its last other subscriber left could reach the upstream before the leaver's unsubscribe, leaving the joiner registered locally but unsubscribed upstream until the next reconnect. Serialize upstream sends behind a context-bounded lock: Subscribe records its subscriber while holding it, and an unsubscribe is sent only if the query is still unused once the lock is held. Local cleanup does not wait for the lock, so a cleanup timing out behind a blocked send still forgets its subscriptions. 🤖 Co-authored by [Claudius the Magnificent](https://github.com/lklimek/claudius) AI Agent Co-Authored-By: Claude Opus 5.5 --- rpc/client/http/ws.go | 45 ++++++++++++++++++++++++--- rpc/client/http/ws_join_leave_test.go | 5 ++- rpc/client/http/ws_test.go | 29 +++++++++++++++++ 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/rpc/client/http/ws.go b/rpc/client/http/ws.go index 1fe0772f6b..9d1d3b2459 100644 --- a/rpc/client/http/ws.go +++ b/rpc/client/http/ws.go @@ -22,6 +22,10 @@ type wsEvents struct { Logger log.Logger ws *jsonrpcclient.WSClient + // changes orders upstream (un)subscribes: a subscriber is recorded, and an + // unused query confirmed, while holding it across the send. Taken before mtx. + changes chan struct{} + mtx sync.RWMutex // subscriptions maps each query, and its upstream subscription ID once known, // to the local subscribers sharing its single upstream subscription. @@ -56,6 +60,7 @@ func newWsEvents(remote string) (*wsEvents, error) { w := &wsEvents{ Logger: log.NewNopLogger(), subscriptions: make(map[string]*wsSubscription), + changes: make(chan struct{}, 1), resubscribeDelay: time.Second, } @@ -132,6 +137,10 @@ func (w *wsEvents) stopListener() { // It returns an error if wsEvents is not running. func (w *wsEvents) Subscribe(ctx context.Context, subscriber, query string, outCapacity ...int) (out <-chan coretypes.ResultEvent, err error) { + if err := w.lockChanges(ctx); err != nil { + return nil, err + } + defer w.unlockChanges() if err := w.ws.Subscribe(ctx, query); err != nil { return nil, err } @@ -194,6 +203,19 @@ func (w *wsEvents) UnsubscribeAll(ctx context.Context, subscriber string) error return w.unsubscribeUpstream(ctx, unused) } +// lockChanges waits for the changes lock as long as ctx allows: its holder's send +// can block until the writer restarts. +func (w *wsEvents) lockChanges(ctx context.Context) error { + select { + case w.changes <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (w *wsEvents) unlockChanges() { <-w.changes } + // removeSubscriber forgets subscriber's subscription to sub, and sub itself once // unused, which it then appends to unused. Callers must hold mtx. func (w *wsEvents) removeSubscriber(sub *wsSubscription, subscriber string, unused []string) []string { @@ -213,13 +235,28 @@ func (w *wsEvents) removeSubscriber(sub *wsSubscription, subscriber string, unus return append(unused, sub.query) } -// unsubscribeUpstream must not hold mtx: sending can block until the writer restarts. -// After an error, eventListener drops events for the queries left subscribed. +// unsubscribeUpstream unsubscribes from queries still unused once it holds the +// changes lock. Callers must not hold mtx: sending can block until the writer +// restarts. After an error, eventListener drops events for queries left subscribed. func (w *wsEvents) unsubscribeUpstream(ctx context.Context, queries []string) error { - if len(queries) == 0 && !w.IsRunning() { - return errNotRunning + if len(queries) == 0 { + if !w.IsRunning() { + return errNotRunning + } + return nil + } + if err := w.lockChanges(ctx); err != nil { + return err } + defer w.unlockChanges() for _, query := range queries { + // Lock, not RLock: eventListener writes to the map under RLock. + w.mtx.Lock() + _, used := w.subscriptions[query] + w.mtx.Unlock() + if used { + continue + } if err := w.ws.Unsubscribe(ctx, query); err != nil { return err } diff --git a/rpc/client/http/ws_join_leave_test.go b/rpc/client/http/ws_join_leave_test.go index 4c4cd851f1..f7876ef8f9 100644 --- a/rpc/client/http/ws_join_leave_test.go +++ b/rpc/client/http/ws_join_leave_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "slices" "sync" "testing" "time" @@ -107,7 +108,9 @@ func TestConcurrentSubscribeAndLastUnsubscribeKeepsUpstream(t *testing.T) { w.mtx.RUnlock() require.True(t, bRegistered, "B is registered locally") - require.Eventually(t, func() bool { r, _ := upstream.state(); return len(r) >= 2 }, 5*time.Second, time.Millisecond) + // B's subscribe must reach this session; A's unsubscribe, if sent, may follow it. + require.Eventually(t, func() bool { r, _ := upstream.state(); return slices.Contains(r, "subscribe q") }, + 5*time.Second, time.Millisecond) time.Sleep(200 * time.Millisecond) reqs, s := upstream.state() require.True(t, s["q"], "upstream unsubscribed q while B still needs it; requests on session: %v", reqs) diff --git a/rpc/client/http/ws_test.go b/rpc/client/http/ws_test.go index 0ed66db04f..310f5adbf3 100644 --- a/rpc/client/http/ws_test.go +++ b/rpc/client/http/ws_test.go @@ -239,6 +239,35 @@ func TestUnsubscribeAllKeepsOtherSubscribers(t *testing.T) { } } +// A cleanup timing out behind another subscriber's send during a reconnect must +// still forget its subscriptions, or the reconnect restores an unread channel. +func TestUnsubscribeAllTimeoutBehindSubscribeForgetsSubscriptions(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{} + w := startWsEvents(ctx, t, upstream) + upstream.subscribeAs(ctx, t, w, "A", "gone") + require.Eventually(t, func() bool { return len(upstream.subscribesOnCurrentConnection()) == 1 }, + 5*time.Second, time.Millisecond) + + reconnecting(t, w, upstream) + subscribed := make(chan error, 1) + go func() { + _, err := w.Subscribe(ctx, "B", "other") + subscribed <- err + }() + require.Eventually(t, func() bool { return len(w.changes) == 1 }, 5*time.Second, time.Millisecond) + cleanupCtx, cancelCleanup := context.WithTimeout(ctx, 10*time.Millisecond) + defer cancelCleanup() + require.ErrorIs(t, w.UnsubscribeAll(cleanupCtx, "A"), context.DeadlineExceeded) + + require.NoError(t, <-subscribed) + upstream.setEvents(map[string][]string{"sync": {"sync"}}) + requireEvent(t, upstream.subscribeAs(ctx, t, w, "C", "sync"), "sync") + require.NotContains(t, upstream.subscribesOnCurrentConnection(), "gone", + "the reconnect restored a subscription whose cleanup timed out") +} + // Forgetting a subscription must also release a delivery blocked on its unread channel. func TestUnsubscribeAllReleasesBlockedDelivery(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) From 86a6934b6d1079edd016971f90c8768f1a26caff Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:57:07 +0000 Subject: [PATCH 39/39] test(rpc): stress concurrent joins and leaves of a shared query Checks that the upstream stays subscribed exactly while a local subscriber remains, and that shutdown releases a Subscribe holding the changes lock and a cleanup waiting for it. Co-Authored-By: Claude Opus 5.5 --- rpc/client/http/ws_join_leave_stress_test.go | 101 +++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 rpc/client/http/ws_join_leave_stress_test.go diff --git a/rpc/client/http/ws_join_leave_stress_test.go b/rpc/client/http/ws_join_leave_stress_test.go new file mode 100644 index 0000000000..ee4eb662e5 --- /dev/null +++ b/rpc/client/http/ws_join_leave_stress_test.go @@ -0,0 +1,101 @@ +package http + +import ( + "context" + "fmt" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// Concurrent joins and leaves of one query must leave the upstream subscribed +// exactly when some local subscriber remains. +func TestJoinLeaveStressKeepsUpstreamConsistent(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &orderedUpstream{} + srv := httptest.NewServer(upstream) + t.Cleanup(srv.Close) + w, err := newWsEvents(srv.URL) + require.NoError(t, err) + require.NoError(t, w.Start(ctx)) + t.Cleanup(func() { cancel(); _ = w.Stop() }) + + for round := 0; round < 300; round++ { + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(2) + name := fmt.Sprintf("s%d-%d", round, i) + go func() { + defer wg.Done() + _, err := w.Subscribe(ctx, name, "q", 100) + require.NoError(t, err) + }() + go func(prev string) { + defer wg.Done() + require.NoError(t, w.UnsubscribeAll(ctx, prev)) + }(fmt.Sprintf("s%d-%d", round-1, i)) + } + wg.Wait() + // Settle: a sync request on a separate query follows every send above. + _, err := w.Subscribe(ctx, "sync", fmt.Sprintf("sync%d", round)) + require.NoError(t, err) + require.Eventually(t, func() bool { + _, s := upstream.state() + return s[fmt.Sprintf("sync%d", round)] + }, 5*time.Second, time.Millisecond) + w.mtx.Lock() + sub, local := w.subscriptions["q"] + n := 0 + if local { + n = len(sub.subscribers) + } + w.mtx.Unlock() + reqs, s := upstream.state() + if local != s["q"] { + t.Fatalf("round %d: local q=%v (%d subscribers) upstream q=%v; last requests %v", + round, local, n, s["q"], reqs[max(0, len(reqs)-12):]) + } + } +} + +// Shutdown while a Subscribe holds the changes lock blocked in its send, and a +// cleanup without deadline waits for that lock, must release both and Stop. +func TestStopWhileSubscribeHoldsChangesLock(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := &fakeUpstream{} + srv := httptest.NewServer(upstream) + t.Cleanup(srv.Close) + w, err := newWsEvents(srv.URL) + require.NoError(t, err) + require.NoError(t, w.Start(ctx)) + upstream.subscribeAs(ctx, t, w, "A", "gone") + require.Eventually(t, func() bool { return len(upstream.subscribesOnCurrentConnection()) == 1 }, + 5*time.Second, time.Millisecond) + reconnecting(t, w, upstream) + srv.Close() // redials keep failing + + subscribed := make(chan error, 1) + go func() { _, err := w.Subscribe(context.Background(), "B", "other"); subscribed <- err }() + require.Eventually(t, func() bool { return len(w.changes) == 1 }, 5*time.Second, time.Millisecond) + cleaned := make(chan error, 1) + go func() { cleaned <- w.UnsubscribeAll(context.Background(), "A") }() + time.Sleep(50 * time.Millisecond) + + cancel() + stopped := make(chan error, 1) + go func() { stopped <- w.Stop() }() + for name, ch := range map[string]chan error{"Stop": stopped, "Subscribe": subscribed, "UnsubscribeAll": cleaned} { + select { + case err := <-ch: + t.Logf("%s returned %v", name, err) + case <-time.After(10 * time.Second): + t.Fatalf("%s hung after shutdown", name) + } + } + require.Empty(t, w.changes, "changes lock leaked") +}