feat: add metric for sequencer mode

Makefile

@@ -1,7 +1,7 @@
 ################## update dependencies ####################
 ETHEREUM_SUBMODULE_COMMIT_OR_TAG := bb0b3067eb62eeffa3b73a7b9daf34a870d80493
 ETHEREUM_TARGET_VERSION := v1.10.14-0.20260623033834-bb0b3067eb62
-TENDERMINT_TARGET_VERSION := v0.3.8-0.20260623033856-a213c3ffdb4c
+TENDERMINT_TARGET_VERSION := v0.3.8-0.20260625105428-a5de063445a3
 
 
 ETHEREUM_MODULE_NAME := github.com/morph-l2/go-ethereum

bindings/go.mod

@@ -2,7 +2,7 @@ module morph-l2/bindings
 
 go 1.24.0
 
-replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c
+replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3
 
 require github.com/morph-l2/go-ethereum v1.10.14-0.20260623033834-bb0b3067eb62
 

common/go.mod

@@ -2,7 +2,7 @@ module morph-l2/common
 
 go 1.24.0
 
-replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c
+replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3
 
 require (
 	github.com/holiman/uint256 v1.2.4

contracts/go.mod

@@ -2,7 +2,7 @@ module morph-l2/contract
 
 go 1.24.0
 
-replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c
+replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3
 
 require (
 	github.com/iden3/go-iden3-crypto v0.0.16

node/go.mod

@@ -2,7 +2,7 @@ module morph-l2/node
 
 go 1.24.0
 
-replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c
+replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3
 
 require (
 	github.com/cenkalti/backoff/v4 v4.1.3

node/go.sum

@@ -416,8 +416,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/morph-l2/go-ethereum v1.10.14-0.20260623033834-bb0b3067eb62 h1:6/K1+ABmJmvxdf39otAks02Uukx+dxzE8J4BDwdL3xM=
 github.com/morph-l2/go-ethereum v1.10.14-0.20260623033834-bb0b3067eb62/go.mod h1:nkVzHjQWCOjvukQW8ittlwX+Xz9gmVHrP7mUi7zoHTs=
-github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c h1:fQbDN8w5S2XdWs04lKUboldXsp4qlQg8ubUeCmIeLfA=
-github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c/go.mod h1:qpiwqfcCB89dBYfqVJOc/HjGxDp3OdDlthgttJJYyRs=
+github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3 h1:ZpUi8rHb4KOcW8Eua5VSQksWBI+8F25mxdtzjJl4h/M=
+github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3/go.mod h1:qpiwqfcCB89dBYfqVJOc/HjGxDp3OdDlthgttJJYyRs=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=

node/hakeeper/block_fsm.go

@@ -39,8 +39,9 @@ var _ raft.FSM = (*BlockFSM)(nil)
 // it stores only the applied block height (for log compaction) and delivers decoded
 // blocks to subscribers via a buffered channel.
 type BlockFSM struct {
-	logger tmlog.Logger
-	mu     sync.RWMutex
+	logger  tmlog.Logger
+	metrics *Metrics
+	mu      sync.RWMutex
 
 	// appliedHeight is the block number of the most recently applied log entry.
 	// Used exclusively by Snapshot for log compaction; NOT a full block reference.
@@ -55,9 +56,13 @@ type BlockFSM struct {
 }
 
 // NewBlockFSM creates a new BlockFSM.
-func NewBlockFSM(logger tmlog.Logger) *BlockFSM {
+func NewBlockFSM(logger tmlog.Logger, metrics *Metrics) *BlockFSM {
+	if metrics == nil {
+		metrics = NopMetrics()
+	}
 	return &BlockFSM{
 		logger:  logger,
+		metrics: metrics,
 		blockCh: make(chan *types.BlockV2, 1000),
 	}
 }
@@ -117,6 +122,7 @@ func (f *BlockFSM) Apply(l *raft.Log) interface{} {
 			panic(&FSMApplyError{Height: block.Number, Err: err})
 		}
 		onAppliedDur = time.Since(t1)
+		f.metrics.ObserveFSMApplyDuration(onAppliedDur)
 	} else {
 		panic(fmt.Sprintf("BlockFSM.Apply: onApplied is nil at height %d, "+
 			"this is a programmer error", block.Number))
@@ -136,6 +142,7 @@ func (f *BlockFSM) Apply(l *raft.Log) interface{} {
 	select {
 	case f.blockCh <- block:
 	default:
+		f.metrics.IncFSMBlockChannelDrops()
 		f.logger.Error("BlockFSM: blockCh full, subscriber too slow", "height", block.Number)
 	}
 

node/hakeeper/ha_service.go

@@ -41,10 +41,12 @@ type HAService struct {
 	advertisedAddr string // resolved once in New(), used throughout
 	fsm            *BlockFSM
 	rpcServer      *hakeeperrpc.Server
+	metrics        *Metrics
 
 	// Raft internals (initialized in Start)
-	r         *raft.Raft
-	transport *raft.NetworkTransport
+	r            *raft.Raft
+	transport    *raft.NetworkTransport
+	raftObserver *raft.Observer
 
 	leaderReady int32 // atomic: 1 = can produce blocks
 	stopCh      chan struct{}
@@ -58,11 +60,15 @@ var _ hakeeperrpc.ConsensusAdapter = (*HAService)(nil)
 // Expects cfg to be fully resolved (Resolve + Validate already called).
 // Call SetOnBlockApplied before Start().
 func New(cfg *Config, logger tmlog.Logger) (*HAService, error) {
+	// Register HA metrics on the default registry, tagged with this node's id
+	// (matches the morphnode_* namespace used by the executor/syncer metrics).
+	m := PrometheusMetrics("morphnode", "server_id", cfg.ServerID)
 	return &HAService{
 		logger:         logger,
 		cfg:            cfg,
 		advertisedAddr: cfg.Consensus.AdvertisedAddr, // already resolved
-		fsm:            NewBlockFSM(logger),
+		fsm:            NewBlockFSM(logger, m),
+		metrics:        m,
 		stopCh:         make(chan struct{}),
 	}, nil
 }
@@ -93,6 +99,13 @@ func (h *HAService) Start() error {
 	}
 	h.rpcServer = rpcSrv
 
+	// Subscribe before taking the initial snapshot: any state change that
+	// happens between the two is then captured by the observer rather than
+	// lost in the gap (which would leave raft_state stale until the next
+	// event or the minute-interval safety net).
+	h.startRaftMetricsObserver()
+	h.refreshRaftMetrics()
+
 	h.wg.Add(1)
 	go h.leaderMonitor()
 
@@ -109,6 +122,9 @@ func (h *HAService) Start() error {
 // Order: close stopCh → shutdown Raft (unblocks Barrier) → wg.Wait → stop RPC.
 func (h *HAService) Stop() {
 	close(h.stopCh)
+	if h.raftObserver != nil && h.r != nil {
+		h.r.DeregisterObserver(h.raftObserver)
+	}
 	h.shutdownRaft()
 	h.wg.Wait()
 	if h.rpcServer != nil {
@@ -163,7 +179,11 @@ func (h *HAService) tryJoin(addr string) error {
 		}
 	}
 
-	return client.AddServerAsVoter(ctx, h.cfg.ServerID, h.advertisedAddr, membership.Version)
+	if err := client.AddServerAsVoter(ctx, h.cfg.ServerID, h.advertisedAddr, membership.Version); err != nil {
+		return err
+	}
+	h.refreshClusterMembers()
+	return nil
 }
 
 // Commit replicates a signed block via Raft.
@@ -176,13 +196,15 @@ func (h *HAService) Commit(block *types.BlockV2) error {
 		return fmt.Errorf("Commit: encode: %w", err)
 	}
 	encodeDur := time.Since(t0)
+	h.metrics.ObserveCommitDuration("encode", encodeDur)
 
 	t1 := time.Now()
 	f := h.r.Apply(data, raftInfiniteTimeout)
 	if err := f.Error(); err != nil {
 		return err
 	}
 	raftDur := time.Since(t1)
+	h.metrics.ObserveCommitDuration("raft", raftDur)
 
 	if resp := f.Response(); resp != nil {
 		if err, ok := resp.(error); ok {
@@ -226,19 +248,35 @@ func (h *HAService) LeaderWithID() *hakeeperrpc.ServerInfo {
 }
 
 func (h *HAService) AddVoter(id, addr string, version uint64) error {
-	return h.r.AddVoter(raft.ServerID(id), raft.ServerAddress(addr), version, raftTimeout).Error()
+	if err := h.r.AddVoter(raft.ServerID(id), raft.ServerAddress(addr), version, raftTimeout).Error(); err != nil {
+		return err
+	}
+	h.refreshClusterMembers()
+	return nil
 }
 
 func (h *HAService) AddNonVoter(id, addr string, version uint64) error {
-	return h.r.AddNonvoter(raft.ServerID(id), raft.ServerAddress(addr), version, raftTimeout).Error()
+	if err := h.r.AddNonvoter(raft.ServerID(id), raft.ServerAddress(addr), version, raftTimeout).Error(); err != nil {
+		return err
+	}
+	h.refreshClusterMembers()
+	return nil
 }
 
 func (h *HAService) DemoteVoter(id string, version uint64) error {
-	return h.r.DemoteVoter(raft.ServerID(id), version, raftTimeout).Error()
+	if err := h.r.DemoteVoter(raft.ServerID(id), version, raftTimeout).Error(); err != nil {
+		return err
+	}
+	h.refreshClusterMembers()
+	return nil
 }
 
 func (h *HAService) RemoveServer(id string, version uint64) error {
-	return h.r.RemoveServer(raft.ServerID(id), version, raftTimeout).Error()
+	if err := h.r.RemoveServer(raft.ServerID(id), version, raftTimeout).Error(); err != nil {
+		return err
+	}
+	h.refreshClusterMembers()
+	return nil
 }
 
 func (h *HAService) TransferLeader() error {
@@ -417,6 +455,7 @@ func (h *HAService) joinLoop() {
 				continue
 			}
 			h.logger.Info("hakeeper: joined cluster")
+			h.refreshClusterMembers()
 			return
 		}
 	}

node/hakeeper/ha_service_test.go

@@ -0,0 +1,46 @@
+package hakeeper
+
+import (
+	"testing"
+
+	"github.com/go-kit/kit/metrics"
+	"github.com/hashicorp/raft"
+)
+
+type recordingGauge struct {
+	value float64
+}
+
+func (g *recordingGauge) With(labelValues ...string) metrics.Gauge { return g }
+func (g *recordingGauge) Set(value float64)                        { g.value = value }
+func (g *recordingGauge) Add(delta float64)                        { g.value += delta }
+
+func TestRaftMetricsObservationFilter(t *testing.T) {
+	if !raftMetricsObservationFilter(&raft.Observation{Data: raft.Leader}) {
+		t.Fatal("RaftState observations should be included")
+	}
+	if !raftMetricsObservationFilter(&raft.Observation{Data: raft.PeerObservation{}}) {
+		t.Fatal("PeerObservation observations should be included")
+	}
+	if raftMetricsObservationFilter(&raft.Observation{Data: "ignored"}) {
+		t.Fatal("unrelated observations should be filtered out")
+	}
+}
+
+func TestHandleRaftObservationUpdatesRaftStateImmediately(t *testing.T) {
+	stateGauge := &recordingGauge{}
+	h := &HAService{
+		metrics: NopMetrics(),
+	}
+	h.metrics.RaftState = stateGauge
+
+	h.handleRaftObservation(raft.Observation{Data: raft.Candidate})
+	if stateGauge.value != float64(raft.Candidate) {
+		t.Fatalf("raft_state metric = %v, want %v", stateGauge.value, raft.Candidate)
+	}
+
+	h.handleRaftObservation(raft.Observation{Data: raft.Leader})
+	if stateGauge.value != float64(raft.Leader) {
+		t.Fatalf("raft_state metric = %v, want %v", stateGauge.value, raft.Leader)
+	}
+}