Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
002aa34
[INFOPLAT-19706] Send node telemetry through Beholder domain emitters
4of9 Oct 28, 2026
b219095
Re-pin chainlink-common to chainlink-common#2504 head (b3de7aa7)
4of9 Oct 28, 2026
3446bfc
DomainManager: harden fates, metrics, chains and supported types
4of9 Oct 28, 2026
e0d8f28
Extract telemetry manager selection; parallel tests
4of9 Oct 28, 2026
35f173f
docs: BeholderEnabled ignores TelemetryIngress sending settings; need…
4of9 Oct 28, 2026
59e32e5
Fix golangci-lint v2.15.0 findings (gci, shadow, testifylint, whitesp…
4of9 Oct 28, 2026
8eedadc
Fix CI: test fake implements PublishBatchNoAck; go mod tidy
4of9 Oct 29, 2026
6796378
DomainManager: explicit Beholder client, loud refusals, core metric n…
4of9 Oct 29, 2026
0b58cf0
Tests: fast real-client tests, buffer-full end to end, exact counts a…
4of9 Oct 29, 2026
bd92375
docs: BeholderEnabled sends for every chain; list the supported telem…
4of9 Oct 29, 2026
10cb619
Fix gci import grouping
4of9 Oct 29, 2026
730ab99
Re-pin chainlink-common#2504 (2cd02358): ChipDomainEmittersEnabled
4of9 Oct 29, 2026
9e03648
Re-pin chainlink-common to main a3881fed (per-domain chip ingress emi…
thomaska Nov 13, 2026
d3df5d0
Merge remote-tracking branch 'origin/develop' into repin-19706-common
thomaska Nov 13, 2026
cac894a
Merge remote-tracking branch 'origin/develop' into repin-19706-common
thomaska Nov 16, 2026
07f1924
Merge remote-tracking branch 'origin/develop' into INFOPLAT-19706-tel…
4of9 Nov 16, 2026
f5c1858
Add chain_selector to the domain manager OTel counters
4of9 Nov 16, 2026
a2d24a1
Use a struct key for the domain manager metric option cache
4of9 Nov 16, 2026
28a269e
telemetry: bound unsupported-type metric cardinality and record the d…
thomaska Nov 17, 2026
64a94d4
Amend @pkcll's comments
thomaska Nov 17, 2026
d2c7e75
Merge remote-tracking branch 'origin/develop' into repin-19706-common
thomaska Nov 17, 2026
425f455
Merge origin/develop: take develop's chainlink-common
4of9 Nov 17, 2026
445f7b7
telemetry: route ocr3_1-mercury to data-streams
4of9 Nov 17, 2026
8f734c8
telemetry: send recordedtime in UTC; fix countUnsupported doc comment…
4of9 Nov 17, 2026
7035964
telemetry: derive send and unsupported-count contexts from the servic…
thomaska Nov 18, 2026
70f4b29
telemetry: promote the telemetry manager interface to telemetry.Manager
4of9 Nov 18, 2026
2a9bd9f
telemetry: reuse one lifecycle context for DomainManager sends
4of9 Nov 18, 2026
5cac6d0
telemetry: allow DomainManager's lifecycle context field (containedctx)
4of9 Nov 18, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/telemetry-domain-manager.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"chainlink": minor
---

#added `TelemetryIngress.BeholderEnabled` (off by default): sends node telemetry through Beholder's per-domain emitters instead of the WSRPC `TelemetryIngress.Endpoints`. Requires `Telemetry.Enabled` and `Telemetry.ChipIngressEndpoint`; cannot be combined with `TelemetryIngress.ChipIngressEnabled`.
69 changes: 69 additions & 0 deletions core/cmd/beholder_client_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package cmd

import (
"fmt"
"net"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"

"github.com/smartcontractkit/chainlink-common/pkg/beholder"
"github.com/smartcontractkit/chainlink/v2/core/logger"
"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
"github.com/smartcontractkit/chainlink/v2/core/services/keystore/mocks"
)

func TestNewBeholderClient_DomainEmittersFollowTelemetryIngressBeholderEnabled(t *testing.T) {
t.Parallel()
// A local gRPC server with no services answers OTel exports at once, so Close is quick.
lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0")
require.NoError(t, err)
srv := grpc.NewServer()
go func() { _ = srv.Serve(lis) }()
t.Cleanup(srv.Stop)
beholderTOML := fmt.Sprintf(`
[Telemetry]
Enabled = true
Endpoint = '%s'
InsecureConnection = true
ChipIngressEndpoint = '%s'
ChipIngressInsecureConnection = true
`, lis.Addr(), lis.Addr())
for _, tt := range []struct {
name string
toml string
enabled bool
}{
{name: "off", toml: beholderTOML},
{name: "on", toml: beholderTOML + "\n[TelemetryIngress]\nBeholderEnabled = true\n", enabled: true},
} {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
cfg, err := chainlink.GeneralConfigOpts{ConfigStrings: []string{tt.toml}}.New()
require.NoError(t, err)

ks := mocks.NewMaster(t)
ks.On("CSA").Return(mocks.NewCSA(t)).Maybe()

client, err := newBeholderClient(logger.TestLogger(t), ks, cfg.Tracing(), cfg, cfg.Telemetry(), "abc123", nil)
require.NoError(t, err)
require.NoError(t, client.Start(t.Context()))
t.Cleanup(func() {
// OTel shutdown may fail to flush to the absent collector; the domain emitters
// must close cleanly.
if closeErr := client.Close(); closeErr != nil {
assert.NotContains(t, closeErr.Error(), "client connection is closing")
}
})

_, err = client.ForDomain(beholder.DomainDataFeeds)
if tt.enabled {
require.NoError(t, err, "BeholderEnabled builds the domain emitters")
} else {
require.Error(t, err, "no domain emitters unless BeholderEnabled")
}
})
}
}
37 changes: 20 additions & 17 deletions core/cmd/shell.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,21 +160,23 @@ func newBeholderClient(
ChipIngressEmitterGRPCEndpoint: cfgTelemetry.ChipIngressEndpoint(),
ChipIngressInsecureConnection: cfgTelemetry.ChipIngressInsecureConnection(),
ChipIngressBatchEmitterEnabled: cfgTelemetry.ChipIngressBatchEmitterEnabled(),
ChipIngressBufferSize: cfgTelemetry.ChipIngressBufferSize(),
ChipIngressMaxBatchSize: cfgTelemetry.ChipIngressMaxBatchSize(),
ChipIngressMaxConcurrentSends: cfgTelemetry.ChipIngressMaxConcurrentSends(),
ChipIngressSendInterval: cfgTelemetry.ChipIngressSendInterval(),
ChipIngressSendTimeout: cfgTelemetry.ChipIngressSendTimeout(),
ChipIngressDrainTimeout: cfgTelemetry.ChipIngressDrainTimeout(),
ChipIngressMaxGRPCRequestSize: cfgTelemetry.ChipIngressMaxGRPCRequestSize(),
ChipIngressLogger: lggr,
LogStreamingEnabled: cfgTelemetry.LogStreamingEnabled(),
LogLevel: cfgTelemetry.LogLevel(),
LogBatchProcessor: cfgTelemetry.LogBatchProcessor(),
LogExportTimeout: cfgTelemetry.LogExportTimeout(),
LogExportMaxBatchSize: cfgTelemetry.LogExportMaxBatchSize(),
LogExportInterval: cfgTelemetry.LogExportInterval(),
LogMaxQueueSize: cfgTelemetry.LogMaxQueueSize(),
// TelemetryIngress.BeholderEnabled sends node telemetry through the per-domain emitters.
ChipDomainEmittersEnabled: cfgApp.TelemetryIngress().BeholderEnabled(),
ChipIngressBufferSize: cfgTelemetry.ChipIngressBufferSize(),
ChipIngressMaxBatchSize: cfgTelemetry.ChipIngressMaxBatchSize(),
ChipIngressMaxConcurrentSends: cfgTelemetry.ChipIngressMaxConcurrentSends(),
ChipIngressSendInterval: cfgTelemetry.ChipIngressSendInterval(),
ChipIngressSendTimeout: cfgTelemetry.ChipIngressSendTimeout(),
ChipIngressDrainTimeout: cfgTelemetry.ChipIngressDrainTimeout(),
ChipIngressMaxGRPCRequestSize: cfgTelemetry.ChipIngressMaxGRPCRequestSize(),
ChipIngressLogger: lggr,
LogStreamingEnabled: cfgTelemetry.LogStreamingEnabled(),
LogLevel: cfgTelemetry.LogLevel(),
LogBatchProcessor: cfgTelemetry.LogBatchProcessor(),
LogExportTimeout: cfgTelemetry.LogExportTimeout(),
LogExportMaxBatchSize: cfgTelemetry.LogExportMaxBatchSize(),
LogExportInterval: cfgTelemetry.LogExportInterval(),
LogMaxQueueSize: cfgTelemetry.LogMaxQueueSize(),
// Due to OpenTelemetry semantics, histogram bucket boundaries must be set
// when the Beholder client is constructed.
MetricViews: metricViews(),
Expand Down Expand Up @@ -269,14 +271,14 @@ func (s *Shell) configExitErr(validateFn func() error) cli.ExitCoder {

// AppFactory implements the NewApplication method.
type AppFactory interface {
NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master) (chainlink.Application, error)
NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master, beholderClient *beholder.Client) (chainlink.Application, error)
}

// ChainlinkAppFactory is used to create a new Application.
type ChainlinkAppFactory struct{}

// NewApplication returns a new instance of the node with the given config.
func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master) (app chainlink.Application, err error) {
func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master, beholderClient *beholder.Client) (app chainlink.Application, err error) {
err = migrate.SetMigrationENVVars(cfg.EVMConfigs())
if err != nil {
return nil, err
Expand Down Expand Up @@ -315,6 +317,7 @@ func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.G

return chainlink.NewApplication(ctx, chainlink.ApplicationOpts{
Opts: creOpts,
BeholderClient: beholderClient,
Config: cfg,
DS: ds,
KeyStore: keyStore,
Expand Down
6 changes: 3 additions & 3 deletions core/cmd/shell_local.go
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ func (s *Shell) runNode(c *cli.Context) error {
os.Exit(-1)
})

app, err := s.AppFactory.NewApplication(rootCtx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore)
app, err := s.AppFactory.NewApplication(rootCtx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore, s.BeholderClient)
if err != nil {
return s.errorOut(fmt.Errorf("fatal error instantiating application: %w", err))
}
Expand Down Expand Up @@ -780,7 +780,7 @@ func (s *Shell) RebroadcastTransactions(c *cli.Context) (err error) {

lggr := logger.Sugared(s.Logger.Named("RebroadcastTransactions"))

app, err := s.AppFactory.NewApplication(ctx, s.Config, lggr, s.Registerer, s.DS, s.KeyStore)
app, err := s.AppFactory.NewApplication(ctx, s.Config, lggr, s.Registerer, s.DS, s.KeyStore, s.BeholderClient)
if err != nil {
return s.errorOut(fmt.Errorf("fatal error instantiating application: %w", err))
}
Expand Down Expand Up @@ -1145,7 +1145,7 @@ func (s *Shell) RemoveBlocks(c *cli.Context) error {
s.afterNode(lggr)
})

app, err := s.AppFactory.NewApplication(ctx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore)
app, err := s.AppFactory.NewApplication(ctx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore, s.BeholderClient)
if err != nil {
return s.errorOut(fmt.Errorf("fatal error instantiating application: %w", err))
}
Expand Down
9 changes: 9 additions & 0 deletions core/config/docs/core.toml
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,15 @@ SendTimeout = '10s' # Default
UseBatchSend = true # Default
# ChipIngressEnabled enables sending telemetry to CHIP Ingress.
ChipIngressEnabled = false # Default
# BeholderEnabled sends telemetry through Beholder, one emitter per product domain, instead of the Endpoints below.
# When set, Endpoints and the settings above (BufferSize, MaxBatchSize, SendInterval, SendTimeout, UseBatchSend, UniConn, Logging)
# are ignored: the Telemetry.ChipIngress* settings size one buffer per domain, shared by all of that domain's jobs.
# Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint, and chip ingress that serves PublishBatchNoAck for every domain
# (there is no fallback). Cannot be combined with ChipIngressEnabled.
# Telemetry is sent for every chain the node's jobs run on: Endpoints no longer select chains. Supported types are ocr,
# ocr2-median, ocr3-data-feeds, enhanced-ea, ocr3-mercury, ocr3_1-mercury, enhanced-ea-mercury, llo-observation, llo-outcome,
# pipeline-bridge, ocr2-ccip-commit, ocr2-ccip-exec, ocr3-ccip-commit, ocr3-ccip-exec and head-report; other types are dropped.
BeholderEnabled = false # Default

[[TelemetryIngress.Endpoints]] # Example
# Network aka EVM, Solana, Starknet
Expand Down
45 changes: 45 additions & 0 deletions core/config/mocks/telemetry_ingress.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions core/config/telemetry_ingress_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ type TelemetryIngress interface {
UseBatchSend() bool
Endpoints() []TelemetryIngressEndpoint
ChipIngressEnabled() bool
BeholderEnabled() bool
}

type TelemetryIngressEndpoint interface {
Expand Down
13 changes: 13 additions & 0 deletions core/config/toml/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,15 @@ func (c *Core) ValidateConfig() (err error) {
}
}

if *c.TelemetryIngress.BeholderEnabled {
if *c.TelemetryIngress.ChipIngressEnabled {
err = errors.Join(err, configutils.InvalidError{Name: "TelemetryIngress.BeholderEnabled", Value: true, Msg: "cannot be combined with TelemetryIngress.ChipIngressEnabled"})
}
if !*c.Telemetry.Enabled || *c.Telemetry.ChipIngressEndpoint == "" {
err = errors.Join(err, configutils.InvalidError{Name: "TelemetryIngress.BeholderEnabled", Value: true, Msg: "requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint"})
}
}

return err
}

Expand Down Expand Up @@ -982,6 +991,7 @@ type TelemetryIngress struct {
UseBatchSend *bool
Endpoints []TelemetryIngressEndpoint `toml:",omitempty"`
ChipIngressEnabled *bool
BeholderEnabled *bool
}

type TelemetryIngressEndpoint struct {
Expand Down Expand Up @@ -1019,6 +1029,9 @@ func (t *TelemetryIngress) setFrom(f *TelemetryIngress) {
if v := f.ChipIngressEnabled; v != nil {
t.ChipIngressEnabled = v
}
if v := f.BeholderEnabled; v != nil {
t.BeholderEnabled = v
}
}

type AuditLogger struct {
Expand Down
5 changes: 3 additions & 2 deletions core/internal/cltest/mocks.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/smartcontractkit/chainlink-common/pkg/beholder"
"github.com/smartcontractkit/chainlink-common/pkg/sqlutil"
"github.com/smartcontractkit/chainlink-common/pkg/types"
"github.com/smartcontractkit/chainlink-evm/pkg/chains/legacyevm"
Expand Down Expand Up @@ -90,15 +91,15 @@ type InstanceAppFactory struct {
}

// NewApplication creates a new application with specified config
func (f InstanceAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master) (chainlink.Application, error) {
func (f InstanceAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master, *beholder.Client) (chainlink.Application, error) {
return f.App, nil
}

type seededAppFactory struct {
Application chainlink.Application
}

func (s seededAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master) (chainlink.Application, error) {
func (s seededAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master, *beholder.Client) (chainlink.Application, error) {
return noopStopApplication{s.Application}, nil
}

Expand Down
8 changes: 6 additions & 2 deletions core/services/chainlink/application.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ import (
"github.com/smartcontractkit/chainlink/v2/core/services/shardorchestrator"
"github.com/smartcontractkit/chainlink/v2/core/services/standardcapabilities"
"github.com/smartcontractkit/chainlink/v2/core/services/streams"
"github.com/smartcontractkit/chainlink/v2/core/services/telemetry"
"github.com/smartcontractkit/chainlink/v2/core/services/vrf"
workflowstore "github.com/smartcontractkit/chainlink/v2/core/services/workflows/store"
"github.com/smartcontractkit/chainlink/v2/core/sessions"
Expand Down Expand Up @@ -214,6 +213,8 @@ type ApplicationOpts struct {
NewOracleFactoryFn standardcapabilities.NewOracleFactoryFn
EVMFactoryConfigFn func(*EVMFactoryConfig)
DonTimeStore *dontime.Store
// BeholderClient is the node's Beholder client. Required when TelemetryIngress.BeholderEnabled is set.
BeholderClient *beholder.Client
// LLOKeyValueDatabaseFactory replaces the on-disk store LLO v31 jobs use.
// Tests only.
LLOKeyValueDatabaseFactory ocr3_1types.KeyValueDatabaseFactory
Expand Down Expand Up @@ -567,7 +568,10 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err
globalLogger.Info("Nurse service (automatic pprof profiling) is disabled")
}

telemetryManager := telemetry.NewManager(cfg.TelemetryIngress(), csaKeystore, globalLogger)
telemetryManager, err := newTelemetryManager(cfg.TelemetryIngress(), opts.BeholderClient, csaKeystore, csaPubKeyHex, globalLogger)
if err != nil {
return nil, err
}
srvcs = append(srvcs, telemetryManager)

backupCfg := cfg.Database().Backup()
Expand Down
4 changes: 4 additions & 0 deletions core/services/chainlink/config_telemetry_ingress.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,3 +78,7 @@ func (t *telemetryIngressEndpointConfig) URL() *url.URL {
func (t *telemetryIngressEndpointConfig) ServerPubKey() string {
return *t.c.ServerPubKey
}

func (t *telemetryIngressConfig) BeholderEnabled() bool {
return *t.c.BeholderEnabled
}
Loading
Loading