From 5a2e7d07631830c053fd197aee7a17563c040736 Mon Sep 17 00:00:00 2001 From: "allen.wu" Date: Thu, 25 Jun 2026 12:09:36 +0800 Subject: [PATCH 1/2] feat: add metric for sequencer mode --- node/hakeeper/block_fsm.go | 13 +- node/hakeeper/ha_service.go | 55 +- node/hakeeper/ha_service_test.go | 46 + node/hakeeper/metrics.go | 127 ++ node/hakeeper/raft_metrics.go | 74 + node/l1sequencer/metrics.go | 52 + node/l1sequencer/tracker.go | 8 +- node/l1sequencer/tracker_test.go | 1 + ops/devnet-morph/devnet/setup_nodes.py | 3 + ops/docker/grafana/dashboards/dashboards.yml | 9 + .../grafana/dashboards/json/morph-node.json | 1279 +++++++++++++++++ ops/docker/grafana/datasources/prometheus.yml | 9 + ops/docker/prometheus/prometheus.yml | 40 + 13 files changed, 1704 insertions(+), 12 deletions(-) create mode 100644 node/hakeeper/ha_service_test.go create mode 100644 node/hakeeper/metrics.go create mode 100644 node/hakeeper/raft_metrics.go create mode 100644 node/l1sequencer/metrics.go create mode 100644 ops/docker/grafana/dashboards/dashboards.yml create mode 100644 ops/docker/grafana/dashboards/json/morph-node.json create mode 100644 ops/docker/grafana/datasources/prometheus.yml create mode 100644 ops/docker/prometheus/prometheus.yml diff --git a/node/hakeeper/block_fsm.go b/node/hakeeper/block_fsm.go index 08bb68d53..67a5c86a1 100644 --- a/node/hakeeper/block_fsm.go +++ b/node/hakeeper/block_fsm.go @@ -39,8 +39,9 @@ var _ raft.FSM = (*BlockFSM)(nil) // it stores only the applied block height (for log compaction) and delivers decoded // blocks to subscribers via a buffered channel. type BlockFSM struct { - logger tmlog.Logger - mu sync.RWMutex + logger tmlog.Logger + metrics *Metrics + mu sync.RWMutex // appliedHeight is the block number of the most recently applied log entry. // Used exclusively by Snapshot for log compaction; NOT a full block reference. @@ -55,9 +56,13 @@ type BlockFSM struct { } // NewBlockFSM creates a new BlockFSM. -func NewBlockFSM(logger tmlog.Logger) *BlockFSM { +func NewBlockFSM(logger tmlog.Logger, metrics *Metrics) *BlockFSM { + if metrics == nil { + metrics = NopMetrics() + } return &BlockFSM{ logger: logger, + metrics: metrics, blockCh: make(chan *types.BlockV2, 1000), } } @@ -117,6 +122,7 @@ func (f *BlockFSM) Apply(l *raft.Log) interface{} { panic(&FSMApplyError{Height: block.Number, Err: err}) } onAppliedDur = time.Since(t1) + f.metrics.ObserveFSMApplyDuration(onAppliedDur) } else { panic(fmt.Sprintf("BlockFSM.Apply: onApplied is nil at height %d, "+ "this is a programmer error", block.Number)) @@ -136,6 +142,7 @@ func (f *BlockFSM) Apply(l *raft.Log) interface{} { select { case f.blockCh <- block: default: + f.metrics.IncFSMBlockChannelDrops() f.logger.Error("BlockFSM: blockCh full, subscriber too slow", "height", block.Number) } diff --git a/node/hakeeper/ha_service.go b/node/hakeeper/ha_service.go index cf2e2901a..b5c03a8b8 100644 --- a/node/hakeeper/ha_service.go +++ b/node/hakeeper/ha_service.go @@ -41,10 +41,12 @@ type HAService struct { advertisedAddr string // resolved once in New(), used throughout fsm *BlockFSM rpcServer *hakeeperrpc.Server + metrics *Metrics // Raft internals (initialized in Start) - r *raft.Raft - transport *raft.NetworkTransport + r *raft.Raft + transport *raft.NetworkTransport + raftObserver *raft.Observer leaderReady int32 // atomic: 1 = can produce blocks stopCh chan struct{} @@ -58,11 +60,15 @@ var _ hakeeperrpc.ConsensusAdapter = (*HAService)(nil) // Expects cfg to be fully resolved (Resolve + Validate already called). // Call SetOnBlockApplied before Start(). func New(cfg *Config, logger tmlog.Logger) (*HAService, error) { + // Register HA metrics on the default registry, tagged with this node's id + // (matches the morphnode_* namespace used by the executor/syncer metrics). + m := PrometheusMetrics("morphnode", "server_id", cfg.ServerID) return &HAService{ logger: logger, cfg: cfg, advertisedAddr: cfg.Consensus.AdvertisedAddr, // already resolved - fsm: NewBlockFSM(logger), + fsm: NewBlockFSM(logger, m), + metrics: m, stopCh: make(chan struct{}), }, nil } @@ -93,6 +99,13 @@ func (h *HAService) Start() error { } h.rpcServer = rpcSrv + // Subscribe before taking the initial snapshot: any state change that + // happens between the two is then captured by the observer rather than + // lost in the gap (which would leave raft_state stale until the next + // event or the minute-interval safety net). + h.startRaftMetricsObserver() + h.refreshRaftMetrics() + h.wg.Add(1) go h.leaderMonitor() @@ -109,6 +122,9 @@ func (h *HAService) Start() error { // Order: close stopCh → shutdown Raft (unblocks Barrier) → wg.Wait → stop RPC. func (h *HAService) Stop() { close(h.stopCh) + if h.raftObserver != nil && h.r != nil { + h.r.DeregisterObserver(h.raftObserver) + } h.shutdownRaft() h.wg.Wait() if h.rpcServer != nil { @@ -163,7 +179,11 @@ func (h *HAService) tryJoin(addr string) error { } } - return client.AddServerAsVoter(ctx, h.cfg.ServerID, h.advertisedAddr, membership.Version) + if err := client.AddServerAsVoter(ctx, h.cfg.ServerID, h.advertisedAddr, membership.Version); err != nil { + return err + } + h.refreshClusterMembers() + return nil } // Commit replicates a signed block via Raft. @@ -176,6 +196,7 @@ func (h *HAService) Commit(block *types.BlockV2) error { return fmt.Errorf("Commit: encode: %w", err) } encodeDur := time.Since(t0) + h.metrics.ObserveCommitDuration("encode", encodeDur) t1 := time.Now() f := h.r.Apply(data, raftInfiniteTimeout) @@ -183,6 +204,7 @@ func (h *HAService) Commit(block *types.BlockV2) error { return err } raftDur := time.Since(t1) + h.metrics.ObserveCommitDuration("raft", raftDur) if resp := f.Response(); resp != nil { if err, ok := resp.(error); ok { @@ -226,19 +248,35 @@ func (h *HAService) LeaderWithID() *hakeeperrpc.ServerInfo { } func (h *HAService) AddVoter(id, addr string, version uint64) error { - return h.r.AddVoter(raft.ServerID(id), raft.ServerAddress(addr), version, raftTimeout).Error() + if err := h.r.AddVoter(raft.ServerID(id), raft.ServerAddress(addr), version, raftTimeout).Error(); err != nil { + return err + } + h.refreshClusterMembers() + return nil } func (h *HAService) AddNonVoter(id, addr string, version uint64) error { - return h.r.AddNonvoter(raft.ServerID(id), raft.ServerAddress(addr), version, raftTimeout).Error() + if err := h.r.AddNonvoter(raft.ServerID(id), raft.ServerAddress(addr), version, raftTimeout).Error(); err != nil { + return err + } + h.refreshClusterMembers() + return nil } func (h *HAService) DemoteVoter(id string, version uint64) error { - return h.r.DemoteVoter(raft.ServerID(id), version, raftTimeout).Error() + if err := h.r.DemoteVoter(raft.ServerID(id), version, raftTimeout).Error(); err != nil { + return err + } + h.refreshClusterMembers() + return nil } func (h *HAService) RemoveServer(id string, version uint64) error { - return h.r.RemoveServer(raft.ServerID(id), version, raftTimeout).Error() + if err := h.r.RemoveServer(raft.ServerID(id), version, raftTimeout).Error(); err != nil { + return err + } + h.refreshClusterMembers() + return nil } func (h *HAService) TransferLeader() error { @@ -417,6 +455,7 @@ func (h *HAService) joinLoop() { continue } h.logger.Info("hakeeper: joined cluster") + h.refreshClusterMembers() return } } diff --git a/node/hakeeper/ha_service_test.go b/node/hakeeper/ha_service_test.go new file mode 100644 index 000000000..ca1150bc1 --- /dev/null +++ b/node/hakeeper/ha_service_test.go @@ -0,0 +1,46 @@ +package hakeeper + +import ( + "testing" + + "github.com/go-kit/kit/metrics" + "github.com/hashicorp/raft" +) + +type recordingGauge struct { + value float64 +} + +func (g *recordingGauge) With(labelValues ...string) metrics.Gauge { return g } +func (g *recordingGauge) Set(value float64) { g.value = value } +func (g *recordingGauge) Add(delta float64) { g.value += delta } + +func TestRaftMetricsObservationFilter(t *testing.T) { + if !raftMetricsObservationFilter(&raft.Observation{Data: raft.Leader}) { + t.Fatal("RaftState observations should be included") + } + if !raftMetricsObservationFilter(&raft.Observation{Data: raft.PeerObservation{}}) { + t.Fatal("PeerObservation observations should be included") + } + if raftMetricsObservationFilter(&raft.Observation{Data: "ignored"}) { + t.Fatal("unrelated observations should be filtered out") + } +} + +func TestHandleRaftObservationUpdatesRaftStateImmediately(t *testing.T) { + stateGauge := &recordingGauge{} + h := &HAService{ + metrics: NopMetrics(), + } + h.metrics.RaftState = stateGauge + + h.handleRaftObservation(raft.Observation{Data: raft.Candidate}) + if stateGauge.value != float64(raft.Candidate) { + t.Fatalf("raft_state metric = %v, want %v", stateGauge.value, raft.Candidate) + } + + h.handleRaftObservation(raft.Observation{Data: raft.Leader}) + if stateGauge.value != float64(raft.Leader) { + t.Fatalf("raft_state metric = %v, want %v", stateGauge.value, raft.Leader) + } +} diff --git a/node/hakeeper/metrics.go b/node/hakeeper/metrics.go new file mode 100644 index 000000000..2061c4984 --- /dev/null +++ b/node/hakeeper/metrics.go @@ -0,0 +1,127 @@ +package hakeeper + +import ( + "time" + + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/discard" + "github.com/go-kit/kit/metrics/prometheus" + "github.com/hashicorp/raft" + stdprometheus "github.com/prometheus/client_golang/prometheus" +) + +// metricsSubsystem groups all HA-keeper metrics under morphnode_hakeeper_*. +const metricsSubsystem = "hakeeper" + +// commitBuckets / fsmApplyBuckets are millisecond exponential buckets +// (1ms .. ~8.2s) suitable for the sub-second-to-seconds commit / apply paths. +var ( + commitBuckets = stdprometheus.ExponentialBuckets(1, 2, 14) + fsmApplyBuckets = stdprometheus.ExponentialBuckets(1, 2, 14) +) + +// Metrics holds the Raft HA-cluster metrics. Every series carries a server_id +// label (the local Raft node id) so a cluster dashboard can aggregate across +// members. State gauges are sampled periodically by HAService; the duration +// and counter series are updated inline on the commit / FSM-apply paths. +// +// All values are integer-valued: durations are recorded in whole milliseconds +// (seconds would round sub-second work to 0). Use the typed helper methods so +// call sites never deal with float64 directly. +type Metrics struct { + // RaftState is the local Raft role as an enum: + // Follower=0, Candidate=1, Leader=2, Shutdown=3. Across the cluster exactly + // one member should report Leader(2). + RaftState metrics.Gauge + + // ClusterMembers is the number of servers in the current Raft configuration. + ClusterMembers metrics.Gauge + + // CommitDurationMilliseconds measures the leader-side block commit latency + // in milliseconds, labeled by step (encode / raft). + CommitDurationMilliseconds metrics.Histogram + + // FSMApplyDurationMilliseconds measures the FSM business-callback (geth apply + // + signature persist) duration in milliseconds per committed log entry. + FSMApplyDurationMilliseconds metrics.Histogram + + // FSMBlockChannelDropsTotal counts blocks dropped because the FSM->broadcast + // channel was full (subscriber too slow). + FSMBlockChannelDropsTotal metrics.Counter +} + +// PrometheusMetrics registers the HA metrics on the default Prometheus +// registry under the given namespace. Pass "server_id", in +// labelsAndValues so every series is tagged with the local node id. +func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { + var labels []string + for i := 0; i < len(labelsAndValues); i += 2 { + labels = append(labels, labelsAndValues[i]) + } + return &Metrics{ + RaftState: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: metricsSubsystem, + Name: "raft_state", + Help: "Local Raft role: Follower=0, Candidate=1, Leader=2, Shutdown=3.", + }, labels).With(labelsAndValues...), + ClusterMembers: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: metricsSubsystem, + Name: "cluster_members", + Help: "Number of servers in the current Raft configuration.", + }, labels).With(labelsAndValues...), + CommitDurationMilliseconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: metricsSubsystem, + Name: "commit_duration_milliseconds", + Help: "Leader-side block commit latency in milliseconds, labeled by step (encode / raft).", + Buckets: commitBuckets, + }, append(labels, "step")).With(labelsAndValues...), + FSMApplyDurationMilliseconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: metricsSubsystem, + Name: "fsm_apply_duration_milliseconds", + Help: "FSM business-callback (geth apply + signature persist) duration in milliseconds per committed log entry.", + Buckets: fsmApplyBuckets, + }, labels).With(labelsAndValues...), + FSMBlockChannelDropsTotal: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ + Namespace: namespace, + Subsystem: metricsSubsystem, + Name: "fsm_block_channel_drops_total", + Help: "Blocks dropped because the FSM->broadcast channel was full.", + }, labels).With(labelsAndValues...), + } +} + +// NopMetrics returns metrics that discard all observations (no registration). +func NopMetrics() *Metrics { + return &Metrics{ + RaftState: discard.NewGauge(), + ClusterMembers: discard.NewGauge(), + CommitDurationMilliseconds: discard.NewHistogram(), + FSMApplyDurationMilliseconds: discard.NewHistogram(), + FSMBlockChannelDropsTotal: discard.NewCounter(), + } +} + +// ---- Typed helpers (keep float64 conversions out of call sites) ---- + +// SetRaftState records the local Raft role enum. +func (m *Metrics) SetRaftState(s raft.RaftState) { m.RaftState.Set(float64(s)) } + +// SetClusterMembers records the current Raft configuration size. +func (m *Metrics) SetClusterMembers(n int) { m.ClusterMembers.Set(float64(n)) } + +// ObserveCommitDuration records a leader-side commit step latency in ms. +func (m *Metrics) ObserveCommitDuration(step string, d time.Duration) { + m.CommitDurationMilliseconds.With("step", step).Observe(float64(d.Milliseconds())) +} + +// ObserveFSMApplyDuration records the FSM business-callback latency in ms. +func (m *Metrics) ObserveFSMApplyDuration(d time.Duration) { + m.FSMApplyDurationMilliseconds.Observe(float64(d.Milliseconds())) +} + +// IncFSMBlockChannelDrops counts one dropped block (FSM->broadcast full). +func (m *Metrics) IncFSMBlockChannelDrops() { m.FSMBlockChannelDropsTotal.Add(1) } diff --git a/node/hakeeper/raft_metrics.go b/node/hakeeper/raft_metrics.go new file mode 100644 index 000000000..ce22fe107 --- /dev/null +++ b/node/hakeeper/raft_metrics.go @@ -0,0 +1,74 @@ +package hakeeper + +import ( + "github.com/hashicorp/raft" +) + +// raftMetricsObservationFilter selects only the observations the metrics layer +// cares about: local role transitions (RaftState) and peer add/remove +// (PeerObservation). +func raftMetricsObservationFilter(o *raft.Observation) bool { + switch o.Data.(type) { + case raft.RaftState, raft.PeerObservation: + return true + default: + return false + } +} + +// startRaftMetricsObserver subscribes to Raft observations and drains them in a +// dedicated goroutine. The channel is non-blocking (best-effort): if the buffer +// fills, an observation may be dropped, leaving its gauge stale until the next +// event of that kind (raft_state self-corrects on the next role change). +func (h *HAService) startRaftMetricsObserver() { + ch := make(chan raft.Observation, 16) + observer := raft.NewObserver(ch, false, raftMetricsObservationFilter) + h.raftObserver = observer + h.r.RegisterObserver(observer) + + h.wg.Add(1) + go h.raftMetricsObserverLoop(ch) +} + +func (h *HAService) raftMetricsObserverLoop(ch <-chan raft.Observation) { + defer h.wg.Done() + for { + select { + case <-h.stopCh: + return + case observation := <-ch: + h.handleRaftObservation(observation) + } + } +} + +func (h *HAService) handleRaftObservation(observation raft.Observation) { + switch state := observation.Data.(type) { + case raft.RaftState: + h.metrics.SetRaftState(state) + case raft.PeerObservation: + h.refreshClusterMembers() + } +} + +// refreshRaftMetrics seeds both gauges with a one-shot snapshot. Called once at +// startup (right after the observer is registered) to set initial values. +func (h *HAService) refreshRaftMetrics() { + if h.r == nil { + return + } + h.metrics.SetRaftState(h.r.State()) + h.refreshClusterMembers() +} + +func (h *HAService) refreshClusterMembers() { + if h.r == nil { + return + } + future := h.r.GetConfiguration() + if err := future.Error(); err != nil { + h.logger.Error("hakeeper: refresh cluster members metric failed", "err", err) + return + } + h.metrics.SetClusterMembers(len(future.Configuration().Servers)) +} diff --git a/node/l1sequencer/metrics.go b/node/l1sequencer/metrics.go new file mode 100644 index 000000000..7b2ca112d --- /dev/null +++ b/node/l1sequencer/metrics.go @@ -0,0 +1,52 @@ +package l1sequencer + +import ( + "time" + + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/discard" + "github.com/go-kit/kit/metrics/prometheus" + stdprometheus "github.com/prometheus/client_golang/prometheus" +) + +// metricsSubsystem groups the L1 tracker metrics under morphnode_l1tracker_*. +const metricsSubsystem = "l1tracker" + +// Metrics holds the L1 health-tracker metrics. The tracker owns and reports +// these itself from its poll loop, so no other component needs to read its +// state. We only care about lag at second granularity (the halt/warn +// thresholds are minute-scale), so the value is whole seconds. +type Metrics struct { + // LagSeconds is how far behind wall-clock the node's view of L1 is: now + // minus the latest observed L1 head time (or since the first failed poll + // during an RPC outage). Crossing the halt threshold stops block + // production and sync. + LagSeconds metrics.Gauge +} + +// PrometheusMetrics registers the L1 tracker metrics on the default Prometheus +// registry under the given namespace (use "morphnode"). +func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { + var labels []string + for i := 0; i < len(labelsAndValues); i += 2 { + labels = append(labels, labelsAndValues[i]) + } + return &Metrics{ + LagSeconds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: metricsSubsystem, + Name: "lag_seconds", + Help: "Seconds the node's view of L1 has fallen behind wall-clock.", + }, labels).With(labelsAndValues...), + } +} + +// NopMetrics returns metrics that discard all observations (no registration). +func NopMetrics() *Metrics { + return &Metrics{ + LagSeconds: discard.NewGauge(), + } +} + +// SetLag records the L1 lag, truncated to whole seconds. +func (m *Metrics) SetLag(d time.Duration) { m.LagSeconds.Set(float64(d / time.Second)) } diff --git a/node/l1sequencer/tracker.go b/node/l1sequencer/tracker.go index dd810cb53..42e7ad2f3 100644 --- a/node/l1sequencer/tracker.go +++ b/node/l1sequencer/tracker.go @@ -39,6 +39,7 @@ type L1Tracker struct { haltLag time.Duration // halt threshold logger tmlog.Logger stop chan struct{} + metrics *Metrics // State below is only mutated from the single loop goroutine (and tests). healthy atomic.Bool // read concurrently by gate consumers @@ -64,6 +65,7 @@ func NewL1Tracker( haltLag: defaultHaltLag, logger: logger.With("module", "l1tracker"), stop: make(chan struct{}), + metrics: PrometheusMetrics("morphnode"), } t.healthy.Store(true) // start allowed t.lastSeen = time.Now() // grace: tolerate initial RPC failures for haltLag @@ -133,7 +135,11 @@ func (t *L1Tracker) update(headTime time.Time, ok bool, now time.Time) { t.lastSeen = now } - if now.Sub(t.lastSeen) > t.haltLag { + // Report lag every poll (single goroutine, so lastSeen reads are safe). + lag := now.Sub(t.lastSeen) + t.metrics.SetLag(lag) + + if lag > t.haltLag { if t.healthy.CompareAndSwap(true, false) { t.logger.Error("L1 health gate TRIPPED: L1 too stale, halting block production and sync", "haltLag", t.haltLag) } diff --git a/node/l1sequencer/tracker_test.go b/node/l1sequencer/tracker_test.go index fd787debe..54e87170d 100644 --- a/node/l1sequencer/tracker_test.go +++ b/node/l1sequencer/tracker_test.go @@ -22,6 +22,7 @@ func newTrackerForTest() (*L1Tracker, *fakeRefresher) { lagThreshold: 5 * time.Minute, haltLag: 30 * time.Minute, logger: tmlog.NewNopLogger(), + metrics: NopMetrics(), } t.healthy.Store(true) return t, f diff --git a/ops/devnet-morph/devnet/setup_nodes.py b/ops/devnet-morph/devnet/setup_nodes.py index 667e23f24..f4b925fe9 100644 --- a/ops/devnet-morph/devnet/setup_nodes.py +++ b/ops/devnet-morph/devnet/setup_nodes.py @@ -88,6 +88,9 @@ def setup_devnet_nodes(): if i < 4: content = content.replace('pex = true', 'pex = false') + # Enable prometheus metrics for all nodes + content = content.replace('prometheus = false', 'prometheus = true') + with open(config_file, "w") as f: f.write(content) diff --git a/ops/docker/grafana/dashboards/dashboards.yml b/ops/docker/grafana/dashboards/dashboards.yml new file mode 100644 index 000000000..6a51bf1e9 --- /dev/null +++ b/ops/docker/grafana/dashboards/dashboards.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: 'Morph Node Metrics' + folder: 'Morph' + type: file + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/ops/docker/grafana/dashboards/json/morph-node.json b/ops/docker/grafana/dashboards/json/morph-node.json new file mode 100644 index 000000000..6e2e50905 --- /dev/null +++ b/ops/docker/grafana/dashboards/json/morph-node.json @@ -0,0 +1,1279 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + "id": 100, + "panels": [], + "title": "🔴 Core Health (P0)", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 1}, + "hiddenSeries": false, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "morphnode_executor_height", + "interval": "", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block Height", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 6, "x": 12, "y": 1}, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "morphnode_sequencer_is_active_sequencer", + "interval": "", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "thresholds": [ + {"colorMode": "critical", "fill": true, "line": true, "op": "lt", "value": 1, "visible": true} + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Sequencer (1=active)", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": 1.5, "min": -0.5, "show": true, "decimals": 0}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 6, "x": 18, "y": 1}, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "morphnode_sequencer_bcast_sync_gap", + "interval": "", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "thresholds": [ + {"colorMode": "warning", "fill": true, "line": true, "op": "gt", "value": 50, "visible": true} + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sync Gap (blocks behind, negative=ahead)", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true, "decimals": 0}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 9}, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "morphnode_hakeeper_raft_state", + "interval": "", + "legendFormat": "{{server_id}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft State (0=Follower,1=Candidate,2=Leader,3=Shutdown)", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": 3.5, "min": -0.5, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 6, "x": 12, "y": 9}, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "morphnode_hakeeper_cluster_members", + "interval": "", + "legendFormat": "{{server_id}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster Members", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true, "decimals": 0}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 6, "x": 18, "y": 9}, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "morphnode_l1tracker_lag_seconds", + "interval": "", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "thresholds": [ + {"colorMode": "warning", "fill": true, "line": true, "op": "gt", "value": 300, "visible": true}, + {"colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1800, "visible": true} + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "L1 Lag (seconds)", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 17}, + "id": 200, + "panels": [], + "title": "🟡 Throughput & Performance (P1)", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 18}, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(morphnode_sequencer_blocks_produced_total[1m])", + "interval": "", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blocks Produced Rate (per second)", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 6, "x": 12, "y": 18}, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(morphnode_sequencer_block_interval_seconds_bucket[5m]))", + "interval": "", + "legendFormat": "p50 {{service}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(morphnode_sequencer_block_interval_seconds_bucket[5m]))", + "interval": "", + "legendFormat": "p95 {{service}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block Interval (seconds)", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 6, "x": 18, "y": 18}, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(morphnode_sequencer_block_size_bytes_bucket[5m]))", + "interval": "", + "legendFormat": "p50 {{service}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(morphnode_sequencer_block_size_bytes_bucket[5m]))", + "interval": "", + "legendFormat": "p95 {{service}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block Size (bytes)", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "decbytes", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 26}, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(morphnode_sequencer_block_txs_sum[1m])", + "interval": "", + "legendFormat": "tx/s {{service}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transactions Per Second", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 26}, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(morphnode_sequencer_apply_duration_milliseconds_bucket{step=\"geth\"}[5m]))", + "interval": "", + "legendFormat": "p50 {{service}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(morphnode_sequencer_apply_duration_milliseconds_bucket{step=\"geth\"}[5m]))", + "interval": "", + "legendFormat": "p95 {{service}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Apply Duration (geth step, ms)", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 34}, + "id": 300, + "panels": [], + "title": "🟢 Broadcast & Sync (P2)", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 35}, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(morphnode_sequencer_bcast_blocks_applied_total[1m])", + "interval": "", + "legendFormat": "{{source}} {{service}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Broadcast Blocks Applied Rate", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 35}, + "hiddenSeries": false, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(morphnode_sequencer_bcast_peers_banned_total[1m])", + "interval": "", + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [ + {"colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0, "visible": true} + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peers Banned (by reason)", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 35}, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(morphnode_sequencer_sync_v2_blocks_total[1m])", + "interval": "", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sync V2 Blocks Rate", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 43}, + "id": 400, + "panels": [], + "title": "🔵 Go Runtime", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 44}, + "hiddenSeries": false, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines", + "interval": "", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Goroutines", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 44}, + "hiddenSeries": false, + "id": 31, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_alloc_bytes", + "interval": "", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Allocated", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "decbytes", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 44}, + "hiddenSeries": false, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "tendermint_p2p_peers", + "interval": "", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "P2P Peers", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 52}, + "id": 500, + "panels": [], + "title": "🟣 HA Metrics (hakeeper)", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 53}, + "hiddenSeries": false, + "id": 40, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(morphnode_hakeeper_commit_duration_milliseconds_bucket[5m]))", + "interval": "", + "legendFormat": "p50 {{step}} {{server_id}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(morphnode_hakeeper_commit_duration_milliseconds_bucket[5m]))", + "interval": "", + "legendFormat": "p95 {{step}} {{server_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HA Commit Duration (ms)", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 6, "x": 12, "y": 53}, + "hiddenSeries": false, + "id": 41, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(morphnode_hakeeper_fsm_block_channel_drops_total[1m])", + "interval": "", + "legendFormat": "{{server_id}}", + "refId": "A" + } + ], + "thresholds": [ + {"colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0, "visible": true} + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "FSM Block Channel Drops", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {}}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 6, "x": 18, "y": 53}, + "hiddenSeries": false, + "id": 42, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": true}, + "percentage": false, + "pluginVersion": "7.5.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(morphnode_hakeeper_fsm_apply_duration_milliseconds_bucket[5m]))", + "interval": "", + "legendFormat": "p50 {{server_id}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(morphnode_hakeeper_fsm_apply_duration_milliseconds_bucket[5m]))", + "interval": "", + "legendFormat": "p95 {{server_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "FSM Apply Duration (ms)", + "tooltip": {"shared": true, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": {"buckets": null, "mode": "time", "name": null, "show": true, "values": []}, + "yaxes": [ + {"format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true}, + {"format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false} + ], + "yaxis": {"align": false, "alignLevel": null} + } + ], + "refresh": "10s", + "schemaVersion": 27, + "style": "dark", + "tags": ["morph", "node", "metrics"], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h"]}, + "timezone": "", + "title": "Morph Node Metrics", + "uid": "morph-node-metrics", + "version": 1 +} diff --git a/ops/docker/grafana/datasources/prometheus.yml b/ops/docker/grafana/datasources/prometheus.yml new file mode 100644 index 000000000..1a57b69c8 --- /dev/null +++ b/ops/docker/grafana/datasources/prometheus.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/ops/docker/prometheus/prometheus.yml b/ops/docker/prometheus/prometheus.yml new file mode 100644 index 000000000..471eec362 --- /dev/null +++ b/ops/docker/prometheus/prometheus.yml @@ -0,0 +1,40 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + # L1 execution layer + - job_name: 'l1-el' + static_configs: + - targets: ['layer1-el:9001'] + + # L1 consensus layer + - job_name: 'l1-cl' + static_configs: + - targets: ['layer1-cl:5054'] + + # L2 morph nodes (tendermint prometheus metrics on port 26660) + - job_name: 'morph-node' + static_configs: + - targets: + - 'node-0:26660' + - 'node-1:26660' + - 'node-2:26660' + - 'node-3:26660' + - 'sentry-node-0:26660' + - 'sentry-node-1:26660' + labels: + service: 'morph-node' + + # L2 execution engines (geth metrics on port 6060) + - job_name: 'morph-geth' + static_configs: + - targets: + - 'morph-el-0:6060' + - 'morph-el-1:6060' + - 'morph-el-2:6060' + - 'morph-el-3:6060' + - 'sentry-el-0:6060' + - 'sentry-el-1:6060' + labels: + service: 'morph-geth' From c04dd02f6ed87ea54f2682a375ff9262e27c23aa Mon Sep 17 00:00:00 2001 From: "allen.wu" Date: Thu, 25 Jun 2026 19:03:28 +0800 Subject: [PATCH 2/2] feat: update go mod --- Makefile | 2 +- bindings/go.mod | 2 +- common/go.mod | 2 +- contracts/go.mod | 2 +- node/go.mod | 2 +- node/go.sum | 4 ++-- ops/l2-genesis/go.mod | 2 +- ops/tools/go.mod | 2 +- ops/tools/go.sum | 4 ++-- tx-submitter/go.mod | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index d0128c615..d03834712 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ ################## update dependencies #################### ETHEREUM_SUBMODULE_COMMIT_OR_TAG := bb0b3067eb62eeffa3b73a7b9daf34a870d80493 ETHEREUM_TARGET_VERSION := v1.10.14-0.20260623033834-bb0b3067eb62 -TENDERMINT_TARGET_VERSION := v0.3.8-0.20260623033856-a213c3ffdb4c +TENDERMINT_TARGET_VERSION := v0.3.8-0.20260625105428-a5de063445a3 ETHEREUM_MODULE_NAME := github.com/morph-l2/go-ethereum diff --git a/bindings/go.mod b/bindings/go.mod index 2152befc1..fc8aab001 100644 --- a/bindings/go.mod +++ b/bindings/go.mod @@ -2,7 +2,7 @@ module morph-l2/bindings go 1.24.0 -replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c +replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3 require github.com/morph-l2/go-ethereum v1.10.14-0.20260623033834-bb0b3067eb62 diff --git a/common/go.mod b/common/go.mod index 50d267ec1..2567b445e 100644 --- a/common/go.mod +++ b/common/go.mod @@ -2,7 +2,7 @@ module morph-l2/common go 1.24.0 -replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c +replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3 require ( github.com/holiman/uint256 v1.2.4 diff --git a/contracts/go.mod b/contracts/go.mod index 6c4ac1ec1..00ece6456 100644 --- a/contracts/go.mod +++ b/contracts/go.mod @@ -2,7 +2,7 @@ module morph-l2/contract go 1.24.0 -replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c +replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3 require ( github.com/iden3/go-iden3-crypto v0.0.16 diff --git a/node/go.mod b/node/go.mod index 95732cdaf..5eac9a27d 100644 --- a/node/go.mod +++ b/node/go.mod @@ -2,7 +2,7 @@ module morph-l2/node go 1.24.0 -replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c +replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3 require ( github.com/cenkalti/backoff/v4 v4.1.3 diff --git a/node/go.sum b/node/go.sum index 3050a2488..ba6c1e171 100644 --- a/node/go.sum +++ b/node/go.sum @@ -416,8 +416,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/morph-l2/go-ethereum v1.10.14-0.20260623033834-bb0b3067eb62 h1:6/K1+ABmJmvxdf39otAks02Uukx+dxzE8J4BDwdL3xM= github.com/morph-l2/go-ethereum v1.10.14-0.20260623033834-bb0b3067eb62/go.mod h1:nkVzHjQWCOjvukQW8ittlwX+Xz9gmVHrP7mUi7zoHTs= -github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c h1:fQbDN8w5S2XdWs04lKUboldXsp4qlQg8ubUeCmIeLfA= -github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c/go.mod h1:qpiwqfcCB89dBYfqVJOc/HjGxDp3OdDlthgttJJYyRs= +github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3 h1:ZpUi8rHb4KOcW8Eua5VSQksWBI+8F25mxdtzjJl4h/M= +github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3/go.mod h1:qpiwqfcCB89dBYfqVJOc/HjGxDp3OdDlthgttJJYyRs= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= diff --git a/ops/l2-genesis/go.mod b/ops/l2-genesis/go.mod index b2dc5149a..77a3899b1 100644 --- a/ops/l2-genesis/go.mod +++ b/ops/l2-genesis/go.mod @@ -2,7 +2,7 @@ module morph-l2/morph-deployer go 1.24.0 -replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c +replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3 require ( github.com/holiman/uint256 v1.2.4 diff --git a/ops/tools/go.mod b/ops/tools/go.mod index df4e023e3..25604ca26 100644 --- a/ops/tools/go.mod +++ b/ops/tools/go.mod @@ -2,7 +2,7 @@ module morph-l2/tools go 1.24.0 -replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c +replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3 require ( github.com/morph-l2/go-ethereum v1.10.14-0.20260623033834-bb0b3067eb62 diff --git a/ops/tools/go.sum b/ops/tools/go.sum index d19742373..b8eff3e32 100644 --- a/ops/tools/go.sum +++ b/ops/tools/go.sum @@ -163,8 +163,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/morph-l2/go-ethereum v1.10.14-0.20260623033834-bb0b3067eb62 h1:6/K1+ABmJmvxdf39otAks02Uukx+dxzE8J4BDwdL3xM= github.com/morph-l2/go-ethereum v1.10.14-0.20260623033834-bb0b3067eb62/go.mod h1:nkVzHjQWCOjvukQW8ittlwX+Xz9gmVHrP7mUi7zoHTs= -github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c h1:fQbDN8w5S2XdWs04lKUboldXsp4qlQg8ubUeCmIeLfA= -github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c/go.mod h1:qpiwqfcCB89dBYfqVJOc/HjGxDp3OdDlthgttJJYyRs= +github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3 h1:ZpUi8rHb4KOcW8Eua5VSQksWBI+8F25mxdtzjJl4h/M= +github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3/go.mod h1:qpiwqfcCB89dBYfqVJOc/HjGxDp3OdDlthgttJJYyRs= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/tx-submitter/go.mod b/tx-submitter/go.mod index e9e06c1ca..e6c9edc07 100644 --- a/tx-submitter/go.mod +++ b/tx-submitter/go.mod @@ -2,7 +2,7 @@ module morph-l2/tx-submitter go 1.24.0 -replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260623033856-a213c3ffdb4c +replace github.com/tendermint/tendermint => github.com/morph-l2/tendermint v0.3.8-0.20260625105428-a5de063445a3 require ( github.com/consensys/gnark-crypto v0.16.0