From 002aa34f0e35f6545a2ee397aaf60b2ad6a933ce Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:35:00 -0400 Subject: [PATCH 01/23] [INFOPLAT-19706] Send node telemetry through Beholder domain emitters Co-Authored-By: Claude Opus 5.5 --- .changeset/telemetry-domain-manager.md | 5 + core/cmd/beholder_client_test.go | 50 +++ core/cmd/shell.go | 36 +- core/config/docs/core.toml | 3 + core/config/mocks/telemetry_ingress.go | 45 +++ core/config/telemetry_ingress_config.go | 1 + core/config/toml/types.go | 13 + core/scripts/go.mod | 6 +- core/scripts/go.sum | 12 +- core/services/chainlink/application.go | 20 +- .../chainlink/config_telemetry_ingress.go | 4 + .../config_telemetry_ingress_test.go | 50 +++ core/services/chainlink/config_test.go | 2 + .../testdata/config-empty-effective.toml | 1 + .../chainlink/testdata/config-full.toml | 1 + .../config-multi-chain-effective.toml | 1 + core/services/telemetry/domain_manager.go | 306 +++++++++++++++++ .../services/telemetry/domain_manager_test.go | 310 ++++++++++++++++++ .../testdata/config-empty-effective.toml | 1 + core/web/resolver/testdata/config-full.toml | 1 + .../config-multi-chain-effective.toml | 1 + deployment/go.mod | 6 +- deployment/go.sum | 12 +- docs/CONFIG.md | 8 + go.mod | 6 +- go.sum | 12 +- integration-tests/go.mod | 6 +- integration-tests/go.sum | 12 +- integration-tests/load/go.mod | 6 +- integration-tests/load/go.sum | 12 +- system-tests/lib/go.mod | 6 +- system-tests/lib/go.sum | 12 +- system-tests/tests/go.mod | 6 +- system-tests/tests/go.sum | 12 +- .../scripts/config/merge_raw_configs.txtar | 1 + testdata/scripts/node/validate/default.txtar | 1 + .../node/validate/defaults-override.txtar | 1 + .../disk-based-logging-disabled.txtar | 1 + .../validate/disk-based-logging-no-dir.txtar | 1 + .../node/validate/disk-based-logging.txtar | 1 + .../node/validate/fallback-override.txtar | 1 + .../node/validate/invalid-ocr-p2p.txtar | 1 + testdata/scripts/node/validate/invalid.txtar | 1 + testdata/scripts/node/validate/valid.txtar | 1 + testdata/scripts/node/validate/warnings.txtar | 1 + 45 files changed, 916 insertions(+), 80 deletions(-) create mode 100644 .changeset/telemetry-domain-manager.md create mode 100644 core/cmd/beholder_client_test.go create mode 100644 core/services/telemetry/domain_manager.go create mode 100644 core/services/telemetry/domain_manager_test.go diff --git a/.changeset/telemetry-domain-manager.md b/.changeset/telemetry-domain-manager.md new file mode 100644 index 00000000000..5eac5be4806 --- /dev/null +++ b/.changeset/telemetry-domain-manager.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +#added `TelemetryIngress.BeholderEnabled` (off by default): sends node telemetry through Beholder's per-domain emitters instead of the WSRPC `TelemetryIngress.Endpoints`. Requires `Telemetry.Enabled` and `Telemetry.ChipIngressEndpoint`; cannot be combined with `TelemetryIngress.ChipIngressEnabled`. diff --git a/core/cmd/beholder_client_test.go b/core/cmd/beholder_client_test.go new file mode 100644 index 00000000000..4e97c3ce017 --- /dev/null +++ b/core/cmd/beholder_client_test.go @@ -0,0 +1,50 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/mocks" +) + +func TestNewBeholderClient_DomainEmittersFollowTelemetryIngressBeholderEnabled(t *testing.T) { + const beholderTOML = ` +[Telemetry] +Enabled = true +Endpoint = 'localhost:4317' +InsecureConnection = true +ChipIngressEndpoint = 'localhost:9999' +ChipIngressInsecureConnection = true +` + for _, tt := range []struct { + name string + toml string + enabled bool + }{ + {name: "off", toml: beholderTOML}, + {name: "on", toml: beholderTOML + "\n[TelemetryIngress]\nBeholderEnabled = true\n", enabled: true}, + } { + t.Run(tt.name, func(t *testing.T) { + cfg, err := chainlink.GeneralConfigOpts{ConfigStrings: []string{tt.toml}}.New() + require.NoError(t, err) + + ks := mocks.NewMaster(t) + ks.On("CSA").Return(mocks.NewCSA(t)).Maybe() + + client, err := newBeholderClient(logger.TestLogger(t), ks, cfg.Tracing(), cfg, cfg.Telemetry(), "abc123", nil) + require.NoError(t, err) + + _, err = client.ForDomain(beholder.DomainDataFeeds) + if tt.enabled { + require.NoError(t, err, "BeholderEnabled builds the domain emitters") + } else { + require.Error(t, err, "no domain emitters unless BeholderEnabled") + } + }) + } +} diff --git a/core/cmd/shell.go b/core/cmd/shell.go index 9f2f36fed74..f96a14e9953 100644 --- a/core/cmd/shell.go +++ b/core/cmd/shell.go @@ -160,21 +160,23 @@ func newBeholderClient( ChipIngressEmitterGRPCEndpoint: cfgTelemetry.ChipIngressEndpoint(), ChipIngressInsecureConnection: cfgTelemetry.ChipIngressInsecureConnection(), ChipIngressBatchEmitterEnabled: cfgTelemetry.ChipIngressBatchEmitterEnabled(), - ChipIngressBufferSize: cfgTelemetry.ChipIngressBufferSize(), - ChipIngressMaxBatchSize: cfgTelemetry.ChipIngressMaxBatchSize(), - ChipIngressMaxConcurrentSends: cfgTelemetry.ChipIngressMaxConcurrentSends(), - ChipIngressSendInterval: cfgTelemetry.ChipIngressSendInterval(), - ChipIngressSendTimeout: cfgTelemetry.ChipIngressSendTimeout(), - ChipIngressDrainTimeout: cfgTelemetry.ChipIngressDrainTimeout(), - ChipIngressMaxGRPCRequestSize: cfgTelemetry.ChipIngressMaxGRPCRequestSize(), - ChipIngressLogger: lggr, - LogStreamingEnabled: cfgTelemetry.LogStreamingEnabled(), - LogLevel: cfgTelemetry.LogLevel(), - LogBatchProcessor: cfgTelemetry.LogBatchProcessor(), - LogExportTimeout: cfgTelemetry.LogExportTimeout(), - LogExportMaxBatchSize: cfgTelemetry.LogExportMaxBatchSize(), - LogExportInterval: cfgTelemetry.LogExportInterval(), - LogMaxQueueSize: cfgTelemetry.LogMaxQueueSize(), + // TelemetryIngress.BeholderEnabled sends node telemetry through the per-domain emitters. + ChipIngressDomainEmittersEnabled: cfgApp.TelemetryIngress().BeholderEnabled(), + ChipIngressBufferSize: cfgTelemetry.ChipIngressBufferSize(), + ChipIngressMaxBatchSize: cfgTelemetry.ChipIngressMaxBatchSize(), + ChipIngressMaxConcurrentSends: cfgTelemetry.ChipIngressMaxConcurrentSends(), + ChipIngressSendInterval: cfgTelemetry.ChipIngressSendInterval(), + ChipIngressSendTimeout: cfgTelemetry.ChipIngressSendTimeout(), + ChipIngressDrainTimeout: cfgTelemetry.ChipIngressDrainTimeout(), + ChipIngressMaxGRPCRequestSize: cfgTelemetry.ChipIngressMaxGRPCRequestSize(), + ChipIngressLogger: lggr, + LogStreamingEnabled: cfgTelemetry.LogStreamingEnabled(), + LogLevel: cfgTelemetry.LogLevel(), + LogBatchProcessor: cfgTelemetry.LogBatchProcessor(), + LogExportTimeout: cfgTelemetry.LogExportTimeout(), + LogExportMaxBatchSize: cfgTelemetry.LogExportMaxBatchSize(), + LogExportInterval: cfgTelemetry.LogExportInterval(), + LogMaxQueueSize: cfgTelemetry.LogMaxQueueSize(), // Due to OpenTelemetry semantics, histogram bucket boundaries must be set // when the Beholder client is constructed. MetricViews: metricViews(), @@ -314,7 +316,9 @@ func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.G } return chainlink.NewApplication(ctx, chainlink.ApplicationOpts{ - Opts: creOpts, + Opts: creOpts, + // Set as the global client in BeforeNode; NewApplication takes it explicitly. + BeholderClient: beholder.GetClient(), Config: cfg, DS: ds, KeyStore: keyStore, diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 5273289460b..513bb15a658 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -116,6 +116,9 @@ SendTimeout = '10s' # Default UseBatchSend = true # Default # ChipIngressEnabled enables sending telemetry to CHIP Ingress. ChipIngressEnabled = false # Default +# BeholderEnabled sends telemetry through Beholder instead of the Endpoints below, which are then ignored. +# Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint. Cannot be combined with ChipIngressEnabled. +BeholderEnabled = false # Default [[TelemetryIngress.Endpoints]] # Example # Network aka EVM, Solana, Starknet diff --git a/core/config/mocks/telemetry_ingress.go b/core/config/mocks/telemetry_ingress.go index 94e83549e51..2cc9532131c 100644 --- a/core/config/mocks/telemetry_ingress.go +++ b/core/config/mocks/telemetry_ingress.go @@ -22,6 +22,51 @@ func (_m *TelemetryIngress) EXPECT() *TelemetryIngress_Expecter { return &TelemetryIngress_Expecter{mock: &_m.Mock} } +// BeholderEnabled provides a mock function with no fields +func (_m *TelemetryIngress) BeholderEnabled() bool { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for BeholderEnabled") + } + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// TelemetryIngress_BeholderEnabled_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BeholderEnabled' +type TelemetryIngress_BeholderEnabled_Call struct { + *mock.Call +} + +// BeholderEnabled is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) BeholderEnabled() *TelemetryIngress_BeholderEnabled_Call { + return &TelemetryIngress_BeholderEnabled_Call{Call: _e.mock.On("BeholderEnabled")} +} + +func (_c *TelemetryIngress_BeholderEnabled_Call) Run(run func()) *TelemetryIngress_BeholderEnabled_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_BeholderEnabled_Call) Return(_a0 bool) *TelemetryIngress_BeholderEnabled_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_BeholderEnabled_Call) RunAndReturn(run func() bool) *TelemetryIngress_BeholderEnabled_Call { + _c.Call.Return(run) + return _c +} + // BufferSize provides a mock function with no fields func (_m *TelemetryIngress) BufferSize() uint { ret := _m.Called() diff --git a/core/config/telemetry_ingress_config.go b/core/config/telemetry_ingress_config.go index 918af269601..867e7fe2497 100644 --- a/core/config/telemetry_ingress_config.go +++ b/core/config/telemetry_ingress_config.go @@ -15,6 +15,7 @@ type TelemetryIngress interface { UseBatchSend() bool Endpoints() []TelemetryIngressEndpoint ChipIngressEnabled() bool + BeholderEnabled() bool } type TelemetryIngressEndpoint interface { diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 53b2677f2e8..99f886b0295 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -141,6 +141,15 @@ func (c *Core) ValidateConfig() (err error) { } } + if *c.TelemetryIngress.BeholderEnabled { + if *c.TelemetryIngress.ChipIngressEnabled { + err = errors.Join(err, configutils.InvalidError{Name: "TelemetryIngress.BeholderEnabled", Value: true, Msg: "cannot be combined with TelemetryIngress.ChipIngressEnabled"}) + } + if !*c.Telemetry.Enabled || *c.Telemetry.ChipIngressEndpoint == "" { + err = errors.Join(err, configutils.InvalidError{Name: "TelemetryIngress.BeholderEnabled", Value: true, Msg: "requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint"}) + } + } + return err } @@ -982,6 +991,7 @@ type TelemetryIngress struct { UseBatchSend *bool Endpoints []TelemetryIngressEndpoint `toml:",omitempty"` ChipIngressEnabled *bool + BeholderEnabled *bool } type TelemetryIngressEndpoint struct { @@ -1019,6 +1029,9 @@ func (t *TelemetryIngress) setFrom(f *TelemetryIngress) { if v := f.ChipIngressEnabled; v != nil { t.ChipIngressEnabled = v } + if v := f.BeholderEnabled; v != nil { + t.BeholderEnabled = v + } } type AuditLogger struct { diff --git a/core/scripts/go.mod b/core/scripts/go.mod index c5e2d10b62d..aa07e00dd23 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -29,7 +29,7 @@ require ( github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a @@ -133,7 +133,7 @@ require ( github.com/buger/goterm v1.0.4 // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect + github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -500,7 +500,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca // indirect github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index ec3d805ca43..2dfebf50672 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -316,8 +316,8 @@ github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40 github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= github.com/bytecodealliance/wasmtime-go/v39 v39.0.1 h1:RibaT47yiyCRxMOj/l2cvL8cWiWBSqDXHyqsa9sGcCE= github.com/bytecodealliance/wasmtime-go/v39 v39.0.1/go.mod h1:miR4NYIEBXeDNamZIzpskhJ0z/p8al+lwMWylQ/ZJb4= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 h1:oC7otiLbHEIDiL+7otXozGw+hnwD4WihronoKYU3drM= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0/go.mod h1:xWTHNOKHkN8YVWrTRZpmb1eUOkWPMEXTaVtm3AOEWWE= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1686,12 +1686,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 h1:mpb2dy2Cr3/J3d09u5L0vjOeA4SVPvk2VZcuOEWUopY= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index 3b3f4cadaa3..d926281049d 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -212,6 +212,8 @@ type ApplicationOpts struct { NewOracleFactoryFn standardcapabilities.NewOracleFactoryFn EVMFactoryConfigFn func(*EVMFactoryConfig) DonTimeStore *dontime.Store + // BeholderClient is the node's Beholder client. Required when TelemetryIngress.BeholderEnabled is set. + BeholderClient *beholder.Client } // safeDefaultBootstrappers returns the configured default bootstrappers from @@ -551,7 +553,23 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err globalLogger.Info("Nurse service (automatic pprof profiling) is disabled") } - telemetryManager := telemetry.NewManager(cfg.TelemetryIngress(), csaKeystore, globalLogger) + var telemetryManager interface { + commonsrv.Service + telemetry.MonitoringEndpointGenerator + } + if cfg.TelemetryIngress().BeholderEnabled() { + if opts.BeholderClient == nil { + return nil, errors.New("TelemetryIngress.BeholderEnabled requires a Beholder client") + } + domainManager, dmErr := telemetry.NewDomainManager(opts.BeholderClient, csaPubKeyHex, globalLogger) + if dmErr != nil { + return nil, fmt.Errorf("failed to create telemetry domain manager: %w", dmErr) + } + globalLogger.Info("Telemetry is sent through Beholder; TelemetryIngress.Endpoints are ignored") + telemetryManager = domainManager + } else { + telemetryManager = telemetry.NewManager(cfg.TelemetryIngress(), csaKeystore, globalLogger) + } srvcs = append(srvcs, telemetryManager) backupCfg := cfg.Database().Backup() diff --git a/core/services/chainlink/config_telemetry_ingress.go b/core/services/chainlink/config_telemetry_ingress.go index 21c50d967a0..c633fa10e66 100644 --- a/core/services/chainlink/config_telemetry_ingress.go +++ b/core/services/chainlink/config_telemetry_ingress.go @@ -78,3 +78,7 @@ func (t *telemetryIngressEndpointConfig) URL() *url.URL { func (t *telemetryIngressEndpointConfig) ServerPubKey() string { return *t.c.ServerPubKey } + +func (t *telemetryIngressConfig) BeholderEnabled() bool { + return *t.c.BeholderEnabled +} diff --git a/core/services/chainlink/config_telemetry_ingress_test.go b/core/services/chainlink/config_telemetry_ingress_test.go index 2bcdf3c80b1..4a50d8c9bfa 100644 --- a/core/services/chainlink/config_telemetry_ingress_test.go +++ b/core/services/chainlink/config_telemetry_ingress_test.go @@ -1,12 +1,15 @@ package chainlink import ( + "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink/v2/core/config/toml" ) @@ -56,3 +59,50 @@ func TestTelemetryIngressConfig_ChipIngressEnabled(t *testing.T) { assert.True(t, config.ChipIngressEnabled()) }) } + +func TestTelemetryIngressConfig_BeholderEnabled(t *testing.T) { + trueVal, falseVal := true, false + assert.True(t, (&telemetryIngressConfig{c: toml.TelemetryIngress{BeholderEnabled: &trueVal}}).BeholderEnabled()) + assert.False(t, (&telemetryIngressConfig{c: toml.TelemetryIngress{BeholderEnabled: &falseVal}}).BeholderEnabled()) +} + +func TestCoreValidateConfig_TelemetryIngressBeholderEnabled(t *testing.T) { + const enabled = ` +[TelemetryIngress] +BeholderEnabled = true +` + const beholderChip = ` +[Telemetry] +Enabled = true +ChipIngressEndpoint = 'chip.example.com:443' +` + for _, tt := range []struct { + name string + toml string + wantErr string // empty: the BeholderEnabled rule must not fire + }{ + {name: "off by default", toml: ``}, + {name: "requires Beholder telemetry", toml: enabled, + wantErr: "TelemetryIngress.BeholderEnabled: invalid value (true): requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint"}, + {name: "requires a chip ingress endpoint", toml: enabled + "\n[Telemetry]\nEnabled = true\n", + wantErr: "TelemetryIngress.BeholderEnabled: invalid value (true): requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint"}, + {name: "cannot combine with ChipIngressEnabled", toml: "[TelemetryIngress]\nBeholderEnabled = true\nChipIngressEnabled = true\n" + beholderChip, + wantErr: "TelemetryIngress.BeholderEnabled: invalid value (true): cannot be combined with TelemetryIngress.ChipIngressEnabled"}, + {name: "valid with Beholder chip ingress", toml: enabled + beholderChip}, + } { + t.Run(tt.name, func(t *testing.T) { + var c Config + require.NoError(t, commonconfig.DecodeTOML(strings.NewReader(tt.toml), &c)) + c.setDefaults() + err := c.Core.ValidateConfig() + if tt.wantErr == "" { + if err != nil { + assert.NotContains(t, err.Error(), "TelemetryIngress.BeholderEnabled") + } + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index e97a05898ff..25882dfc8af 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -254,6 +254,7 @@ func TestConfig_Marshal(t *testing.T) { SendTimeout: commonconfig.MustNewDuration(5 * time.Second), UseBatchSend: new(true), ChipIngressEnabled: new(false), + BeholderEnabled: new(false), Endpoints: []toml.TelemetryIngressEndpoint{ { Network: new("EVM"), @@ -914,6 +915,7 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index 3e6fb0dd366..0da4e4c6a87 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -44,6 +44,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index 144b3c07271..b149b1e71fd 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -44,6 +44,7 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 98600eeaa73..a025d3febf0 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -44,6 +44,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = true diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go new file mode 100644 index 00000000000..ef16fc61b47 --- /dev/null +++ b/core/services/telemetry/domain_manager.go @@ -0,0 +1,306 @@ +package telemetry + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + chainselector "github.com/smartcontractkit/chain-selectors" + "github.com/smartcontractkit/libocr/commontypes" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/metric" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + + "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" +) + +// DomainEmitterSource provides Beholder's per-domain emitters. *beholder.Client implements it. +type DomainEmitterSource interface { + ForDomain(domain beholder.Domain) (beholder.DomainEmitter, error) +} + +// telemetryRoute is where one telemetry type goes: the Beholder domain whose emitter carries it, +// and the CloudEvent source and type it is sent with. +type telemetryRoute struct { + domain beholder.Domain + source string + entity string +} + +const ( + ocr2Entity = "offchainreporting2.TelemetryWrapper" + ocr3Entity = "offchainreporting3.TelemetryWrapper" +) + +// telemetryRoutes maps every supported telemetry type to its route. Types not listed (functions, +// llo-report, threshold, s4, rebalancer, CRE plugin types, ...) are not supported and are dropped. +var telemetryRoutes = map[synchronization.TelemetryType]telemetryRoute{ + synchronization.OCR: {beholder.DomainDataFeeds, "data-feeds.telemetry.ocr", "offchainreporting.TelemetryWrapper"}, + synchronization.OCR2Median: {beholder.DomainDataFeeds, "data-feeds.telemetry.ocr2-median", ocr2Entity}, + synchronization.OCR3DataFeeds: {beholder.DomainDataFeeds, "data-feeds.telemetry.ocr3-data-feeds", ocr3Entity}, + synchronization.EnhancedEA: {beholder.DomainDataFeeds, "data-feeds.telemetry.enhanced-ea", "telem.EnhancedEA"}, + synchronization.OCR3Mercury: {beholder.DomainDataStreams, "data-streams.telemetry.ocr3-mercury", ocr3Entity}, + synchronization.EnhancedEAMercury: {beholder.DomainDataStreams, "data-streams.telemetry.enhanced-ea-mercury", "telem.EnhancedEAMercury"}, + synchronization.LLOObservation: {beholder.DomainDataStreams, "data-streams.telemetry.llo-observation", "telem.LLOObservationTelemetry"}, + synchronization.LLOOutcome: {beholder.DomainDataStreams, "data-streams.telemetry.llo-outcome", "telem.LLOOutcomeTelemetry"}, + synchronization.LLOAttributedObservation: {beholder.DomainDataStreams, "data-streams.telemetry.llo-attributed-observation", "telem.LLOAttributedObservationTelemetry"}, + synchronization.PipelineBridge: {beholder.DomainDataStreams, "data-streams.telemetry.pipeline-bridge", "telem.LLOBridgeTelemetry"}, + synchronization.OCR2CCIPCommit: {beholder.DomainCCIP, "ccip.telemetry.ocr2", ocr2Entity}, + synchronization.OCR2CCIPExec: {beholder.DomainCCIP, "ccip.telemetry.ocr2", ocr2Entity}, + synchronization.OCR3CCIPCommit: {beholder.DomainCCIP, "ccip.telemetry.ocr3", ocr3Entity}, + synchronization.OCR3CCIPExec: {beholder.DomainCCIP, "ccip.telemetry.ocr3", ocr3Entity}, + // TODO(INFOPLAT-19709): head reports are emitted for every chain on every node; confirm ccip + // is their domain. + synchronization.HeadReport: {beholder.DomainCCIP, "ccip.telemetry.head-report", "telem.HeadReportRequest"}, +} + +// Metric names and attribute values. The manager's metrics are OTel (Beholder) metrics, so they +// reach us from every node rather than only from NOPs' own Prometheus. +const ( + metricMessagesSent = "telemetry.messages_sent" + metricMessagesSendErrors = "telemetry.messages_send_errors" + metricMessagesDropped = "telemetry.messages_dropped" + metricMessagesUnsupported = "telemetry.messages_unsupported" + + unsupportedReasonTelemetryType = "telemetry_type" + unsupportedReasonChain = "chain" +) + +var _ MonitoringEndpointGenerator = (*DomainManager)(nil) + +// DomainManager is a MonitoringEndpointGenerator that sends telemetry through Beholder's +// per-domain emitters. Each telemetry type maps to a domain; endpoints bind their domain's +// emitter when created. Network and chain only describe the event: nothing is configured or +// routed per network. +type DomainManager struct { + services.Service + eng *services.Engine + + emitters map[beholder.Domain]beholder.DomainEmitter + csaPubKeyHex string + metrics domainManagerMetrics + now func() time.Time +} + +type domainManagerMetrics struct { + sent otelmetric.Int64Counter + sendErrors otelmetric.Int64Counter + dropped otelmetric.Int64Counter + unsupported otelmetric.Int64Counter +} + +// NewDomainManager resolves an emitter for every domain a telemetry route uses; a missing one is +// an error. csaPubKeyHex is the node's CSA public key, which legacy-mode chip ingress binds to +// the authenticated key. +func NewDomainManager(src DomainEmitterSource, csaPubKeyHex string, lggr logger.Logger) (*DomainManager, error) { + if csaPubKeyHex == "" { + return nil, errors.New("telemetry: CSA public key is required") + } + emitters := make(map[beholder.Domain]beholder.DomainEmitter) + for _, route := range telemetryRoutes { + if _, ok := emitters[route.domain]; ok { + continue + } + em, err := src.ForDomain(route.domain) + if err != nil { + return nil, fmt.Errorf("telemetry: no Beholder emitter for domain %q: %w", route.domain, err) + } + emitters[route.domain] = em + } + metrics, err := newDomainManagerMetrics() + if err != nil { + return nil, err + } + + m := &DomainManager{ + emitters: emitters, + csaPubKeyHex: csaPubKeyHex, + metrics: metrics, + now: time.Now, + } + m.Service, m.eng = services.Config{Name: "DomainManager"}.NewServiceEngine(lggr) + return m, nil +} + +func newDomainManagerMetrics() (m domainManagerMetrics, err error) { + meter := otel.Meter("github.com/smartcontractkit/chainlink/v2/core/services/telemetry") + if m.sent, err = meter.Int64Counter(metricMessagesSent, + otelmetric.WithDescription("Telemetry messages accepted by chip ingress")); err != nil { + return m, err + } + if m.sendErrors, err = meter.Int64Counter(metricMessagesSendErrors, + otelmetric.WithDescription("Telemetry messages that failed to publish")); err != nil { + return m, err + } + if m.dropped, err = meter.Int64Counter(metricMessagesDropped, + otelmetric.WithDescription("Telemetry messages dropped before publishing")); err != nil { + return m, err + } + if m.unsupported, err = meter.Int64Counter(metricMessagesUnsupported, + otelmetric.WithDescription("Telemetry messages dropped because their type or chain is not supported")); err != nil { + return m, err + } + return m, nil +} + +// GenMonitoringEndpoint returns an endpoint for one telemetry type. Unsupported types and chains +// get an endpoint that drops and counts what it is given. +func (m *DomainManager) GenMonitoringEndpoint(network string, chainID string, contractID string, telemType synchronization.TelemetryType) commontypes.MonitoringEndpoint { + route, ok := telemetryRoutes[telemType] + if !ok { + m.eng.Warnw("Unsupported telemetry type: its telemetry will be dropped", "telemType", telemType, "network", network, "chainID", chainID, "contractID", contractID) + return m.unsupportedAgent(telemType, unsupportedReasonTelemetryType) + } + selector, err := chainSelector(network, chainID) + if err != nil { + m.eng.Warnw("Unsupported chain: its telemetry will be dropped", "err", err, "telemType", telemType, "network", network, "chainID", chainID, "contractID", contractID) + return m.unsupportedAgent(telemType, unsupportedReasonChain) + } + return &domainAgent{ + m: m, + route: route, + emitter: m.emitters[route.domain], + telemType: telemType, + network: network, + contractID: contractID, + selector: selector, + attrs: routeAttrs(route, telemType), + } +} + +// GenMultitypeMonitoringEndpoint returns an endpoint whose messages carry their own type. +func (m *DomainManager) GenMultitypeMonitoringEndpoint(network string, chainID string, contractID string) MultitypeMonitoringEndpoint { + selector, err := chainSelector(network, chainID) + if err != nil { + m.eng.Warnw("Unsupported chain: its telemetry will be dropped", "err", err, "network", network, "chainID", chainID, "contractID", contractID) + return m.unsupportedAgent("", unsupportedReasonChain) + } + return &multitypeDomainAgent{m: m, network: network, contractID: contractID, selector: selector} +} + +func chainSelector(network, chainID string) (uint64, error) { + details, err := chainselector.GetChainDetailsByChainIDAndFamily(chainID, strings.ToLower(network)) + if err != nil { + return 0, err + } + return details.ChainSelector, nil +} + +func routeAttrs(route telemetryRoute, telemType synchronization.TelemetryType) attribute.Set { + return attribute.NewSet( + attribute.String("domain", string(route.domain)), + attribute.String("telemetry_type", string(telemType)), + ) +} + +// send builds the event and hands it to emitter. Fates are counted, never logged per message. +func (m *DomainManager) send(emitter beholder.DomainEmitter, route telemetryRoute, telemType synchronization.TelemetryType, + network, contractID string, selector uint64, attrs attribute.Set, telemetry []byte) { + // SendLog has no context (libocr's MonitoringEndpoint); the emitter only queues. + ctx := context.Background() + ev, err := m.newEvent(route, telemType, network, contractID, selector, telemetry) + if err != nil { + m.metrics.dropped.Add(ctx, 1, otelmetric.WithAttributeSet(attrs)) + return + } + err = emitter.EmitEvent(ctx, ev, func(sendErr error) { + if sendErr == nil { + m.metrics.sent.Add(ctx, 1, otelmetric.WithAttributeSet(attrs)) + return + } + errAttrs := append(attrs.ToSlice(), attribute.String("error_code", batch.ErrorCodeFor(sendErr))) + m.metrics.sendErrors.Add(ctx, 1, otelmetric.WithAttributes(errAttrs...)) + }) + if err != nil { + m.metrics.dropped.Add(ctx, 1, otelmetric.WithAttributeSet(attrs)) + } +} + +// newEvent builds the CloudEvent legacy-mode chip ingress expects. Each extension becomes a +// "ce_" Kafka header; the comments name the header OTI used for the same value. Chip +// ingress derives nodeoperatorname/nodename (from the CSA key's NOP info) and receivedat itself. +func (m *DomainManager) newEvent(route telemetryRoute, telemType synchronization.TelemetryType, + network, contractID string, selector uint64, telemetry []byte) (chipingress.CloudEvent, error) { + now := m.now() + event, err := chipingress.NewEvent(route.source, route.entity, telemetry, map[string]any{"time": now.UTC()}) + if err != nil { + return event, fmt.Errorf("failed creating CloudEvent: %w", err) + } + event.SetExtension("legacytelemetry", "true") // marks node-sent legacy telemetry + event.SetExtension("telemetrytype", string(telemType)) // "telemetry-type" + event.SetExtension("chainselector", strconv.FormatUint(selector, 10)) // chain-selectors ID of the job's chain + event.SetExtension("networkname", network) // e.g. "EVM" + event.SetExtension("contractid", contractID) // "contract-address" + event.SetExtension("csapublickey", m.csaPubKeyHex) // "node-operator-csa-public-key" + event.SetExtension("partitionkey", contractID+"-"+m.csaPubKeyHex) // Kafka record key + event.SetExtension("sentat", strconv.FormatInt(now.UnixNano(), 10)) // "sent-at", unix ns + return event, nil +} + +func (m *DomainManager) unsupportedAgent(telemType synchronization.TelemetryType, reason string) *unsupportedAgent { + return &unsupportedAgent{m: m, telemType: telemType, reason: reason} +} + +// domainAgent sends one telemetry type through its domain's emitter. +type domainAgent struct { + m *DomainManager + route telemetryRoute + emitter beholder.DomainEmitter + telemType synchronization.TelemetryType + network string + contractID string + selector uint64 + attrs attribute.Set +} + +func (a *domainAgent) SendLog(telemetry []byte) { + a.m.send(a.emitter, a.route, a.telemType, a.network, a.contractID, a.selector, a.attrs, telemetry) +} + +// multitypeDomainAgent routes each message by the type it carries. +type multitypeDomainAgent struct { + m *DomainManager + network string + contractID string + selector uint64 +} + +func (a *multitypeDomainAgent) SendTypedLog(telemType synchronization.TelemetryType, telemetry []byte) { + route, ok := telemetryRoutes[telemType] + if !ok { + a.m.unsupportedAgent(telemType, unsupportedReasonTelemetryType).SendLog(telemetry) + return + } + a.m.send(a.m.emitters[route.domain], route, telemType, a.network, a.contractID, a.selector, routeAttrs(route, telemType), telemetry) +} + +// unsupportedAgent drops everything it is given and counts it. +type unsupportedAgent struct { + m *DomainManager + telemType synchronization.TelemetryType + reason string +} + +func (a *unsupportedAgent) SendLog([]byte) { + a.m.metrics.unsupported.Add(context.Background(), 1, otelmetric.WithAttributes( + attribute.String("telemetry_type", string(a.telemType)), + attribute.String("reason", a.reason), + )) +} + +func (a *unsupportedAgent) SendTypedLog(telemType synchronization.TelemetryType, telemetry []byte) { + if a.telemType == "" { + a.m.unsupportedAgent(telemType, a.reason).SendLog(telemetry) + return + } + a.SendLog(telemetry) +} diff --git a/core/services/telemetry/domain_manager_test.go b/core/services/telemetry/domain_manager_test.go new file mode 100644 index 00000000000..306811f804f --- /dev/null +++ b/core/services/telemetry/domain_manager_test.go @@ -0,0 +1,310 @@ +package telemetry + +import ( + "context" + "errors" + "strconv" + "strings" + "sync" + "testing" + + chainselectors "github.com/smartcontractkit/chain-selectors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + + "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" +) + +const testCSAPubKeyHex = "755294c8a1f0" + +// fakeDomainEmitter records EmitEvent calls and reports callbackErr as each event's fate. +type fakeDomainEmitter struct { + mu sync.Mutex + events []chipingress.CloudEvent + emitErr error // returned by EmitEvent (callback not invoked) + callbackErr error // reported to the callback +} + +func (f *fakeDomainEmitter) Emit(context.Context, []byte, ...any) error { return nil } +func (f *fakeDomainEmitter) Close() error { return nil } + +func (f *fakeDomainEmitter) EmitEvent(_ context.Context, event chipingress.CloudEvent, callback func(error)) error { + if f.emitErr != nil { + return f.emitErr + } + f.mu.Lock() + f.events = append(f.events, event) + f.mu.Unlock() + if callback != nil { + callback(f.callbackErr) + } + return nil +} + +func (f *fakeDomainEmitter) sent() []chipingress.CloudEvent { + f.mu.Lock() + defer f.mu.Unlock() + return append([]chipingress.CloudEvent(nil), f.events...) +} + +// fakeEmitterSource stands in for *beholder.Client. +type fakeEmitterSource map[beholder.Domain]*fakeDomainEmitter + +func (s fakeEmitterSource) ForDomain(d beholder.Domain) (beholder.DomainEmitter, error) { + em, ok := s[d] + if !ok { + return nil, errors.New("no emitter for domain " + string(d)) + } + return em, nil +} + +func newFakeEmitterSource() fakeEmitterSource { + return fakeEmitterSource{ + beholder.DomainDataFeeds: {}, + beholder.DomainDataStreams: {}, + beholder.DomainCCIP: {}, + } +} + +// installTestMeter routes otel.Meter to a manual reader for the test's duration. +func installTestMeter(t *testing.T) *sdkmetric.ManualReader { + t.Helper() + reader := sdkmetric.NewManualReader() + prev := otel.GetMeterProvider() + otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))) + t.Cleanup(func() { otel.SetMeterProvider(prev) }) + return reader +} + +// counterValue sums the int64 counter name over data points whose attributes include want. +func counterValue(t *testing.T, reader *sdkmetric.ManualReader, name string, want ...attribute.KeyValue) int64 { + t.Helper() + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + var total int64 + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != name { + continue + } + sum, ok := m.Data.(metricdata.Sum[int64]) + require.True(t, ok, "%s is an int64 counter", name) + for _, dp := range sum.DataPoints { + matches := true + for _, kv := range want { + if v, ok := dp.Attributes.Value(kv.Key); !ok || v != kv.Value { + matches = false + break + } + } + if matches { + total += dp.Value + } + } + } + } + return total +} + +func newTestDomainManager(t *testing.T, src fakeEmitterSource) *DomainManager { + t.Helper() + m, err := NewDomainManager(src, testCSAPubKeyHex, logger.Test(t)) + require.NoError(t, err) + return m +} + +var ethMainnetChainID = strconv.FormatUint(chainselectors.ETHEREUM_MAINNET.EvmChainID, 10) + +func TestNewDomainManager_RequiresEveryDomain(t *testing.T) { + src := newFakeEmitterSource() + delete(src, beholder.DomainCCIP) + + _, err := NewDomainManager(src, testCSAPubKeyHex, logger.Test(t)) + require.Error(t, err, "a missing domain emitter fails construction") +} + +func TestNewDomainManager_RequiresCSAPublicKey(t *testing.T) { + _, err := NewDomainManager(newFakeEmitterSource(), "", logger.Test(t)) + require.Error(t, err) +} + +func TestDomainManager_RoutesEachTypeToItsDomain(t *testing.T) { + for _, tc := range []struct { + telemType synchronization.TelemetryType + domain beholder.Domain + source string + }{ + {synchronization.OCR, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr"}, + {synchronization.OCR2Median, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr2-median"}, + {synchronization.OCR3DataFeeds, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr3-data-feeds"}, + {synchronization.EnhancedEA, beholder.DomainDataFeeds, "data-feeds.telemetry.enhanced-ea"}, + {synchronization.OCR3Mercury, beholder.DomainDataStreams, "data-streams.telemetry.ocr3-mercury"}, + {synchronization.EnhancedEAMercury, beholder.DomainDataStreams, "data-streams.telemetry.enhanced-ea-mercury"}, + {synchronization.LLOObservation, beholder.DomainDataStreams, "data-streams.telemetry.llo-observation"}, + {synchronization.LLOOutcome, beholder.DomainDataStreams, "data-streams.telemetry.llo-outcome"}, + {synchronization.LLOAttributedObservation, beholder.DomainDataStreams, "data-streams.telemetry.llo-attributed-observation"}, + {synchronization.PipelineBridge, beholder.DomainDataStreams, "data-streams.telemetry.pipeline-bridge"}, + {synchronization.OCR2CCIPCommit, beholder.DomainCCIP, "ccip.telemetry.ocr2"}, + {synchronization.OCR2CCIPExec, beholder.DomainCCIP, "ccip.telemetry.ocr2"}, + {synchronization.OCR3CCIPCommit, beholder.DomainCCIP, "ccip.telemetry.ocr3"}, + {synchronization.OCR3CCIPExec, beholder.DomainCCIP, "ccip.telemetry.ocr3"}, + {synchronization.HeadReport, beholder.DomainCCIP, "ccip.telemetry.head-report"}, + } { + t.Run(string(tc.telemType), func(t *testing.T) { + src := newFakeEmitterSource() + m := newTestDomainManager(t, src) + + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", tc.telemType).SendLog([]byte("payload")) + + for d, em := range src { + if d == tc.domain { + require.Len(t, em.sent(), 1, "sent on %s", d) + assert.Equal(t, tc.source, em.sent()[0].Source()) + } else { + assert.Empty(t, em.sent(), "nothing on %s", d) + } + } + }) + } +} + +func TestDomainManager_EventCarriesLegacyTelemetryExtensions(t *testing.T) { + src := newFakeEmitterSource() + m := newTestDomainManager(t, src) + + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("payload")) + + events := src[beholder.DomainDataFeeds].sent() + require.Len(t, events, 1) + ev := events[0] + + require.NoError(t, ev.Validate()) + assert.Equal(t, "offchainreporting2.TelemetryWrapper", ev.Type()) + assert.Equal(t, []byte("payload"), ev.Data()) + assert.False(t, ev.Time().IsZero()) + + ext := ev.Extensions() + assert.Equal(t, "true", ext["legacytelemetry"]) + assert.Equal(t, "ocr2-median", ext["telemetrytype"]) + assert.Equal(t, strconv.FormatUint(chainselectors.ETHEREUM_MAINNET.Selector, 10), ext["chainselector"]) + assert.Equal(t, "EVM", ext["networkname"]) + assert.Equal(t, "0xabc", ext["contractid"]) + assert.Equal(t, testCSAPubKeyHex, ext["csapublickey"]) + assert.Equal(t, "0xabc-"+testCSAPubKeyHex, ext["partitionkey"]) + sentAt, ok := ext["sentat"].(string) + require.True(t, ok) + _, err := strconv.ParseInt(sentAt, 10, 64) + assert.NoError(t, err, "sentat is unix nanoseconds") +} + +func TestDomainManager_SendsForAnyNetworkWithoutPerNetworkConfig(t *testing.T) { + src := newFakeEmitterSource() + m := newTestDomainManager(t, src) + + for _, chain := range []chainselectors.Chain{chainselectors.ETHEREUM_MAINNET, chainselectors.POLYGON_MAINNET} { + m.GenMonitoringEndpoint("EVM", strconv.FormatUint(chain.EvmChainID, 10), "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + } + assert.Len(t, src[beholder.DomainDataFeeds].sent(), 2) +} + +func TestDomainManager_UnsupportedTypesAreDroppedAndCounted(t *testing.T) { + reader := installTestMeter(t) + src := newFakeEmitterSource() + m := newTestDomainManager(t, src) + + unsupported := []synchronization.TelemetryType{ + synchronization.OCR2Functions, + synchronization.FunctionsRequests, + synchronization.OCR2Threshold, + synchronization.OCR2S4, + synchronization.OCR3Rebalancer, + synchronization.LLOReport, + "vault-plugin", + } + for _, tt := range unsupported { + ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", tt) + require.NotNil(t, ep) + ep.SendLog([]byte("p")) + } + + for d, em := range src { + assert.Empty(t, em.sent(), "nothing sent on %s", d) + } + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, + attribute.String("telemetry_type", string(synchronization.OCR2Functions)), + attribute.String("reason", unsupportedReasonTelemetryType))) + assert.Equal(t, int64(len(unsupported)), counterValue(t, reader, metricMessagesUnsupported)) +} + +func TestDomainManager_UnresolvableChainIsDroppedAndCounted(t *testing.T) { + reader := installTestMeter(t) + src := newFakeEmitterSource() + m := newTestDomainManager(t, src) + + m.GenMonitoringEndpoint("EVM", "999999999999", "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + + assert.Empty(t, src[beholder.DomainDataFeeds].sent()) + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, + attribute.String("reason", unsupportedReasonChain))) +} + +func TestDomainManager_MultitypeEndpointRoutesPerMessage(t *testing.T) { + src := newFakeEmitterSource() + m := newTestDomainManager(t, src) + + ep := m.GenMultitypeMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc/1") + ep.SendTypedLog(synchronization.LLOOutcome, []byte("p")) + ep.SendTypedLog(synchronization.LLOReport, []byte("p")) // unmapped: dropped + + events := src[beholder.DomainDataStreams].sent() + require.Len(t, events, 1) + assert.Equal(t, "data-streams.telemetry.llo-outcome", events[0].Source()) + assert.Equal(t, "llo-outcome", events[0].Extensions()["telemetrytype"]) +} + +func TestDomainManager_MetricsFollowEachEventsFate(t *testing.T) { + dataFeeds := attribute.String("domain", string(beholder.DomainDataFeeds)) + ocr2 := attribute.String("telemetry_type", string(synchronization.OCR2Median)) + + t.Run("sent", func(t *testing.T) { + reader := installTestMeter(t) + m := newTestDomainManager(t, newFakeEmitterSource()) + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesSent, dataFeeds, ocr2)) + }) + t.Run("send error from the per-message result", func(t *testing.T) { + reader := installTestMeter(t) + src := newFakeEmitterSource() + src[beholder.DomainDataFeeds].callbackErr = errors.New("rejected") + m := newTestDomainManager(t, src) + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesSendErrors, dataFeeds, ocr2)) + assert.Zero(t, counterValue(t, reader, metricMessagesSent)) + }) + t.Run("dropped when the emitter refuses the event", func(t *testing.T) { + reader := installTestMeter(t) + src := newFakeEmitterSource() + src[beholder.DomainDataFeeds].emitErr = errors.New("stopped") + m := newTestDomainManager(t, src) + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesDropped, dataFeeds, ocr2)) + }) +} + +// Beholder domain emitters reject events whose source is not under their domain, so every +// route's source must be the domain or a subdomain of it. +func TestTelemetryRoutes_SourcesSitUnderTheirDomain(t *testing.T) { + for telemType, route := range telemetryRoutes { + d := string(route.domain) + assert.True(t, route.source == d || strings.HasPrefix(route.source, d+"."), + "%s: source %q is not under domain %q", telemType, route.source, d) + } +} diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index 3e6fb0dd366..0da4e4c6a87 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -44,6 +44,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 8735192c3ca..b74a0caa5cf 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -44,6 +44,7 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index d031ee10283..86abfb1d0d4 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -44,6 +44,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = true diff --git a/deployment/go.mod b/deployment/go.mod index aad81685f37..30edaf830c6 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -41,7 +41,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/deployment v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 @@ -163,7 +163,7 @@ require ( github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect + github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -430,7 +430,7 @@ require ( github.com/smartcontractkit/chainlink-canton/authentication v1.0.0 // indirect github.com/smartcontractkit/chainlink-canton/contracts/v2 v2.1.0 // indirect github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20261009212658-df7dcfcb7a77 // indirect diff --git a/deployment/go.sum b/deployment/go.sum index 84a162998f0..8e88467b9be 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -282,8 +282,8 @@ github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 h1:oC7otiLbHEIDiL+7otXozGw+hnwD4WihronoKYU3drM= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0/go.mod h1:xWTHNOKHkN8YVWrTRZpmb1eUOkWPMEXTaVtm3AOEWWE= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1411,12 +1411,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 h1:mpb2dy2Cr3/J3d09u5L0vjOeA4SVPvk2VZcuOEWUopY= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 h1:HeQlHO9MrUXTJmxYCNX4xPX14glLAdwIN3igc9Am/g8= diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 9c75807c44e..cce65db70cb 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -281,6 +281,7 @@ SendInterval = '500ms' # Default SendTimeout = '10s' # Default UseBatchSend = true # Default ChipIngressEnabled = false # Default +BeholderEnabled = false # Default ``` @@ -332,6 +333,13 @@ ChipIngressEnabled = false # Default ``` ChipIngressEnabled enables sending telemetry to CHIP Ingress. +### BeholderEnabled +```toml +BeholderEnabled = false # Default +``` +BeholderEnabled sends telemetry through Beholder instead of the Endpoints below, which are then ignored. +Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint. Cannot be combined with ChipIngressEnabled. + ## TelemetryIngress.Endpoints ```toml [[TelemetryIngress.Endpoints]] # Example diff --git a/go.mod b/go.mod index 7e8aca189bc..c91b8f75003 100644 --- a/go.mod +++ b/go.mod @@ -82,9 +82,9 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 @@ -179,7 +179,7 @@ require ( github.com/blendle/zapdriver v1.3.1 // indirect github.com/block-vision/sui-go-sdk v1.2.1 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect - github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect + github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 // indirect github.com/bytedance/sonic v1.12.3 // indirect github.com/bytedance/sonic/loader v0.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect diff --git a/go.sum b/go.sum index 349c3b414ac..d8912ae8c9d 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,8 @@ github.com/buger/jsonparser v1.2.0 h1:4EFcvK1kD4jyj6YqNK6skK6w+y7FHHBR+XBCtxwu/6 github.com/buger/jsonparser v1.2.0/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU= github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 h1:oC7otiLbHEIDiL+7otXozGw+hnwD4WihronoKYU3drM= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0/go.mod h1:xWTHNOKHkN8YVWrTRZpmb1eUOkWPMEXTaVtm3AOEWWE= github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -1104,12 +1104,12 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62/go.mod h1:c++Ed0jeYn4w8WIfiAboRwN6ClRRVGglL0qwB/QjJvM= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 h1:mpb2dy2Cr3/J3d09u5L0vjOeA4SVPvk2VZcuOEWUopY= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 h1:HeQlHO9MrUXTJmxYCNX4xPX14glLAdwIN3igc9Am/g8= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 9a70720706c..f0fe15462e2 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -30,7 +30,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a @@ -129,7 +129,7 @@ require ( github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect + github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -405,7 +405,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/deployment v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca // indirect github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 2aa7365670c..8750d1909ff 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -269,8 +269,8 @@ github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 h1:oC7otiLbHEIDiL+7otXozGw+hnwD4WihronoKYU3drM= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0/go.mod h1:xWTHNOKHkN8YVWrTRZpmb1eUOkWPMEXTaVtm3AOEWWE= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1396,12 +1396,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 h1:mpb2dy2Cr3/J3d09u5L0vjOeA4SVPvk2VZcuOEWUopY= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 h1:HeQlHO9MrUXTJmxYCNX4xPX14glLAdwIN3igc9Am/g8= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index fc6d035f840..a94406e877e 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -20,7 +20,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.9 @@ -120,7 +120,7 @@ require ( github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect + github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -476,7 +476,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca // indirect github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 // indirect github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 6bb93b929ad..4ec2a214771 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -324,8 +324,8 @@ github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 h1:oC7otiLbHEIDiL+7otXozGw+hnwD4WihronoKYU3drM= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0/go.mod h1:xWTHNOKHkN8YVWrTRZpmb1eUOkWPMEXTaVtm3AOEWWE= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1636,12 +1636,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 h1:mpb2dy2Cr3/J3d09u5L0vjOeA4SVPvk2VZcuOEWUopY= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 h1:HeQlHO9MrUXTJmxYCNX4xPX14glLAdwIN3igc9Am/g8= diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 4c6466ddef2..e2fbccdda32 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -32,7 +32,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 @@ -146,7 +146,7 @@ require ( github.com/buger/goterm v1.0.4 // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect + github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -486,7 +486,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca // indirect github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 957b3b0f077..31fdbd63e4f 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -313,8 +313,8 @@ github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40 github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= github.com/bytecodealliance/wasmtime-go/v39 v39.0.1 h1:RibaT47yiyCRxMOj/l2cvL8cWiWBSqDXHyqsa9sGcCE= github.com/bytecodealliance/wasmtime-go/v39 v39.0.1/go.mod h1:miR4NYIEBXeDNamZIzpskhJ0z/p8al+lwMWylQ/ZJb4= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 h1:oC7otiLbHEIDiL+7otXozGw+hnwD4WihronoKYU3drM= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0/go.mod h1:xWTHNOKHkN8YVWrTRZpmb1eUOkWPMEXTaVtm3AOEWWE= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1657,12 +1657,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 h1:mpb2dy2Cr3/J3d09u5L0vjOeA4SVPvk2VZcuOEWUopY= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index 391187922bb..9eeb9423edd 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -73,9 +73,9 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers v0.0.0-20260812145307-d77342c53d7d github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 @@ -218,7 +218,7 @@ require ( github.com/buger/goterm v1.0.4 // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect + github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index 6eb14ec54a0..caa1a53f642 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -357,8 +357,8 @@ github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40 github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= github.com/bytecodealliance/wasmtime-go/v39 v39.0.1 h1:RibaT47yiyCRxMOj/l2cvL8cWiWBSqDXHyqsa9sGcCE= github.com/bytecodealliance/wasmtime-go/v39 v39.0.1/go.mod h1:miR4NYIEBXeDNamZIzpskhJ0z/p8al+lwMWylQ/ZJb4= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= -github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0 h1:oC7otiLbHEIDiL+7otXozGw+hnwD4WihronoKYU3drM= +github.com/bytecodealliance/wasmtime-go/v50 v50.0.0/go.mod h1:xWTHNOKHkN8YVWrTRZpmb1eUOkWPMEXTaVtm3AOEWWE= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1836,12 +1836,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5 h1:mpb2dy2Cr3/J3d09u5L0vjOeA4SVPvk2VZcuOEWUopY= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261020093734-0127cb4d6ae5/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= diff --git a/testdata/scripts/config/merge_raw_configs.txtar b/testdata/scripts/config/merge_raw_configs.txtar index 1463970ca35..a069ca9926e 100644 --- a/testdata/scripts/config/merge_raw_configs.txtar +++ b/testdata/scripts/config/merge_raw_configs.txtar @@ -191,6 +191,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index e423ec523f9..bd384f9d7d8 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -56,6 +56,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 20f3b565be0..1076fa70894 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -117,6 +117,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 721d1701c82..f1a32080b9c 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -100,6 +100,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index d5337a40c9a..6940039a950 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -100,6 +100,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index 7ff4cbf59b5..6abf9de76a0 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -100,6 +100,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index e34c251e4de..a966635764b 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -199,6 +199,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar index 3ed1117dd42..6af411c6ee5 100644 --- a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar +++ b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar @@ -85,6 +85,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 1c27e505de9..7f76893794a 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -96,6 +96,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index cfdb2cae79b..7179d2de3f5 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -97,6 +97,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index 9e15d037a5b..63febc37d50 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -79,6 +79,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false From b219095520ded8b80ad8988b31b195df4143a23f Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:37:14 -0400 Subject: [PATCH 02/23] Re-pin chainlink-common to chainlink-common#2504 head (b3de7aa7) Picks up the domain emitter review fixes: connections closed once, rate-limited per-event failure logs, outage tests. Co-Authored-By: Claude Opus 5.5 --- core/scripts/go.mod | 2 +- core/scripts/go.sum | 4 ++-- deployment/go.mod | 2 +- deployment/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 ++-- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 ++-- system-tests/lib/go.mod | 2 +- system-tests/lib/go.sum | 4 ++-- system-tests/tests/go.mod | 2 +- system-tests/tests/go.sum | 4 ++-- 14 files changed, 21 insertions(+), 21 deletions(-) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index aa07e00dd23..18b8ebf8a68 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -29,7 +29,7 @@ require ( github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 2dfebf50672..ab0349c6f4c 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1686,8 +1686,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/deployment/go.mod b/deployment/go.mod index 30edaf830c6..3ffc85f4a89 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -41,7 +41,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/deployment v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 diff --git a/deployment/go.sum b/deployment/go.sum index 8e88467b9be..37967adca33 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1411,8 +1411,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/go.mod b/go.mod index c91b8f75003..f445a0e6353 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 diff --git a/go.sum b/go.sum index d8912ae8c9d..44ffba99adb 100644 --- a/go.sum +++ b/go.sum @@ -1104,8 +1104,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62/go.mod h1:c++Ed0jeYn4w8WIfiAboRwN6ClRRVGglL0qwB/QjJvM= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index f0fe15462e2..c4c0b8b1797 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -30,7 +30,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 8750d1909ff..a1569b2f373 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1396,8 +1396,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index a94406e877e..3bdcbbc36b6 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -20,7 +20,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.9 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 4ec2a214771..cea9e662efb 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1636,8 +1636,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index e2fbccdda32..852c6ff9913 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -32,7 +32,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 31fdbd63e4f..ee861bea0d4 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1657,8 +1657,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index 9eeb9423edd..70a1c6bf715 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -73,7 +73,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index caa1a53f642..c1df9c39139 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1836,8 +1836,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730 h1:o9iDSgnfjQ5n1tXB+4k1t2hlOUK5Zlnze+adnTiu6t4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261027203133-c2f5a8a88730/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= From 3446bfc5e357ffeb7673eb1a2a6ba6ef39674498 Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:46:02 -0400 Subject: [PATCH 03/23] DomainManager: harden fates, metrics, chains and supported types - Count buffer-full and shutdown drops as dropped, not send errors - Also increment the Prometheus telemetry_client_messages_* counters - Meter provider option; precomputed per-route metric options - Log unsupported types (Warn) and unresolvable chains (Error) once each - Lower-case networkname; recordedtime from the manager clock - Supported types are the ones ingested in production; source/entity come from synchronization.TelemetryTypeToDomainAndEntity - Drop llo-attributed-observation; deterministic missing-domain error - Tests: real Beholder client end to end (no-ack, header, clean Close), failure injection, -race concurrency, chain families, multitype Co-Authored-By: Claude Opus 5.5 --- core/services/telemetry/domain_manager.go | 355 ++++++++++------- .../telemetry/domain_manager_beholder_test.go | 143 +++++++ .../services/telemetry/domain_manager_test.go | 363 ++++++++++++------ 3 files changed, 601 insertions(+), 260 deletions(-) create mode 100644 core/services/telemetry/domain_manager_beholder_test.go diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index ef16fc61b47..c9bdfe6d2e2 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -4,10 +4,13 @@ import ( "context" "errors" "fmt" + "slices" "strconv" "strings" + "sync" "time" + "github.com/prometheus/client_golang/prometheus" chainselector "github.com/smartcontractkit/chain-selectors" "github.com/smartcontractkit/libocr/commontypes" "go.opentelemetry.io/otel" @@ -28,43 +31,33 @@ type DomainEmitterSource interface { ForDomain(domain beholder.Domain) (beholder.DomainEmitter, error) } -// telemetryRoute is where one telemetry type goes: the Beholder domain whose emitter carries it, -// and the CloudEvent source and type it is sent with. -type telemetryRoute struct { - domain beholder.Domain - source string - entity string -} - -const ( - ocr2Entity = "offchainreporting2.TelemetryWrapper" - ocr3Entity = "offchainreporting3.TelemetryWrapper" -) - -// telemetryRoutes maps every supported telemetry type to its route. Types not listed (functions, -// llo-report, threshold, s4, rebalancer, CRE plugin types, ...) are not supported and are dropped. -var telemetryRoutes = map[synchronization.TelemetryType]telemetryRoute{ - synchronization.OCR: {beholder.DomainDataFeeds, "data-feeds.telemetry.ocr", "offchainreporting.TelemetryWrapper"}, - synchronization.OCR2Median: {beholder.DomainDataFeeds, "data-feeds.telemetry.ocr2-median", ocr2Entity}, - synchronization.OCR3DataFeeds: {beholder.DomainDataFeeds, "data-feeds.telemetry.ocr3-data-feeds", ocr3Entity}, - synchronization.EnhancedEA: {beholder.DomainDataFeeds, "data-feeds.telemetry.enhanced-ea", "telem.EnhancedEA"}, - synchronization.OCR3Mercury: {beholder.DomainDataStreams, "data-streams.telemetry.ocr3-mercury", ocr3Entity}, - synchronization.EnhancedEAMercury: {beholder.DomainDataStreams, "data-streams.telemetry.enhanced-ea-mercury", "telem.EnhancedEAMercury"}, - synchronization.LLOObservation: {beholder.DomainDataStreams, "data-streams.telemetry.llo-observation", "telem.LLOObservationTelemetry"}, - synchronization.LLOOutcome: {beholder.DomainDataStreams, "data-streams.telemetry.llo-outcome", "telem.LLOOutcomeTelemetry"}, - synchronization.LLOAttributedObservation: {beholder.DomainDataStreams, "data-streams.telemetry.llo-attributed-observation", "telem.LLOAttributedObservationTelemetry"}, - synchronization.PipelineBridge: {beholder.DomainDataStreams, "data-streams.telemetry.pipeline-bridge", "telem.LLOBridgeTelemetry"}, - synchronization.OCR2CCIPCommit: {beholder.DomainCCIP, "ccip.telemetry.ocr2", ocr2Entity}, - synchronization.OCR2CCIPExec: {beholder.DomainCCIP, "ccip.telemetry.ocr2", ocr2Entity}, - synchronization.OCR3CCIPCommit: {beholder.DomainCCIP, "ccip.telemetry.ocr3", ocr3Entity}, - synchronization.OCR3CCIPExec: {beholder.DomainCCIP, "ccip.telemetry.ocr3", ocr3Entity}, - // TODO(INFOPLAT-19709): head reports are emitted for every chain on every node; confirm ccip - // is their domain. - synchronization.HeadReport: {beholder.DomainCCIP, "ccip.telemetry.head-report", "telem.HeadReportRequest"}, +// supportedTelemetryDomains lists the telemetry types DomainManager sends and the Beholder domain +// whose emitter carries each: the types ingested in production. Their CloudEvent source and type +// come from synchronization.TelemetryTypeToDomainAndEntity. Every other type is dropped and counted. +func supportedTelemetryDomains() map[synchronization.TelemetryType]beholder.Domain { + return map[synchronization.TelemetryType]beholder.Domain{ + synchronization.OCR: beholder.DomainDataFeeds, + synchronization.OCR2Median: beholder.DomainDataFeeds, + synchronization.OCR3DataFeeds: beholder.DomainDataFeeds, + synchronization.EnhancedEA: beholder.DomainDataFeeds, + synchronization.OCR3Mercury: beholder.DomainDataStreams, + synchronization.EnhancedEAMercury: beholder.DomainDataStreams, + synchronization.LLOObservation: beholder.DomainDataStreams, + synchronization.LLOOutcome: beholder.DomainDataStreams, + synchronization.PipelineBridge: beholder.DomainDataStreams, + synchronization.OCR2CCIPCommit: beholder.DomainCCIP, + synchronization.OCR2CCIPExec: beholder.DomainCCIP, + synchronization.OCR3CCIPCommit: beholder.DomainCCIP, + synchronization.OCR3CCIPExec: beholder.DomainCCIP, + // TODO(INFOPLAT-19709): head reports are emitted for every chain on every node; confirm + // ccip is their domain. + synchronization.HeadReport: beholder.DomainCCIP, + } } // Metric names and attribute values. The manager's metrics are OTel (Beholder) metrics, so they -// reach us from every node rather than only from NOPs' own Prometheus. +// reach us from every node; the existing Prometheus telemetry counters are incremented too, so +// node-local dashboards keep working. const ( metricMessagesSent = "telemetry.messages_sent" metricMessagesSendErrors = "telemetry.messages_send_errors" @@ -73,22 +66,44 @@ const ( unsupportedReasonTelemetryType = "telemetry_type" unsupportedReasonChain = "chain" + + // errorCodeEmitRejected is the error_code of events the emitter refused outright (not + // started, invalid, or outside its domain); other codes come from batch.ErrorCodeFor. + errorCodeEmitRejected = "emit_rejected" + errorCodeEventBuild = "event_build" ) var _ MonitoringEndpointGenerator = (*DomainManager)(nil) // DomainManager is a MonitoringEndpointGenerator that sends telemetry through Beholder's -// per-domain emitters. Each telemetry type maps to a domain; endpoints bind their domain's -// emitter when created. Network and chain only describe the event: nothing is configured or -// routed per network. +// per-domain emitters. Each supported telemetry type maps to a domain; endpoints bind their +// route when created, so sending does no lookups. Network and chain only describe the event: +// nothing is configured or routed per network. type DomainManager struct { services.Service eng *services.Engine - emitters map[beholder.Domain]beholder.DomainEmitter - csaPubKeyHex string - metrics domainManagerMetrics - now func() time.Time + routes map[synchronization.TelemetryType]*telemetryRoute + csaPubKeyHex string + meterProvider otelmetric.MeterProvider + metrics domainManagerMetrics + now func() time.Time + reported sync.Map // keys of unsupported types and chains already logged +} + +// telemetryRoute is where one telemetry type goes, with its metric handles precomputed. +type telemetryRoute struct { + telemType synchronization.TelemetryType + domain beholder.Domain + source string + entity string + emitter beholder.DomainEmitter + + attrs attribute.Set + okOpt otelmetric.AddOption + errorOpts sync.Map // error code -> otelmetric.AddOption + + promSent, promSendErrors, promDropped prometheus.Counter } type domainManagerMetrics struct { @@ -98,41 +113,88 @@ type domainManagerMetrics struct { unsupported otelmetric.Int64Counter } -// NewDomainManager resolves an emitter for every domain a telemetry route uses; a missing one is -// an error. csaPubKeyHex is the node's CSA public key, which legacy-mode chip ingress binds to -// the authenticated key. -func NewDomainManager(src DomainEmitterSource, csaPubKeyHex string, lggr logger.Logger) (*DomainManager, error) { +// DomainManagerOpt configures NewDomainManager. +type DomainManagerOpt func(*DomainManager) + +// WithMeterProvider sets the OTel meter provider for the manager's metrics. The default is the +// global provider, which is Beholder's once the Beholder client is set globally. +func WithMeterProvider(mp otelmetric.MeterProvider) DomainManagerOpt { + return func(m *DomainManager) { m.meterProvider = mp } +} + +// NewDomainManager resolves an emitter for every domain the supported telemetry types use; a +// missing one is an error. csaPubKeyHex is the node's CSA public key, which legacy-mode chip +// ingress binds to the authenticated key. +func NewDomainManager(src DomainEmitterSource, csaPubKeyHex string, lggr logger.Logger, opts ...DomainManagerOpt) (*DomainManager, error) { + if src == nil { + return nil, errors.New("telemetry: a Beholder domain emitter source is required") + } if csaPubKeyHex == "" { return nil, errors.New("telemetry: CSA public key is required") } - emitters := make(map[beholder.Domain]beholder.DomainEmitter) - for _, route := range telemetryRoutes { - if _, ok := emitters[route.domain]; ok { - continue - } - em, err := src.ForDomain(route.domain) - if err != nil { - return nil, fmt.Errorf("telemetry: no Beholder emitter for domain %q: %w", route.domain, err) - } - emitters[route.domain] = em + m := &DomainManager{ + csaPubKeyHex: csaPubKeyHex, + meterProvider: otel.GetMeterProvider(), + now: time.Now, } - metrics, err := newDomainManagerMetrics() + for _, opt := range opts { + opt(m) + } + metrics, err := newDomainManagerMetrics(m.meterProvider) if err != nil { return nil, err } + m.metrics = metrics - m := &DomainManager{ - emitters: emitters, - csaPubKeyHex: csaPubKeyHex, - metrics: metrics, - now: time.Now, + supported := supportedTelemetryDomains() + // Resolve domains in a stable order so a missing one is reported the same way every time. + var domains []beholder.Domain + for _, d := range supported { + if !slices.Contains(domains, d) { + domains = append(domains, d) + } + } + slices.Sort(domains) + emitters := make(map[beholder.Domain]beholder.DomainEmitter, len(domains)) + for _, d := range domains { + em, err := src.ForDomain(d) + if err != nil { + return nil, fmt.Errorf("telemetry: no Beholder emitter for domain %q: %w", d, err) + } + emitters[d] = em } + + m.routes = make(map[synchronization.TelemetryType]*telemetryRoute, len(supported)) + for telemType, d := range supported { + source, entity, err := synchronization.TelemetryTypeToDomainAndEntity(telemType) + if err != nil { + return nil, fmt.Errorf("telemetry: no CloudEvent source for %q: %w", telemType, err) + } + attrs := attribute.NewSet( + attribute.String("domain", string(d)), + attribute.String("telemetry_type", string(telemType)), + ) + endpoint := "beholder/" + string(d) + m.routes[telemType] = &telemetryRoute{ + telemType: telemType, + domain: d, + source: source, + entity: entity, + emitter: emitters[d], + attrs: attrs, + okOpt: otelmetric.WithAttributeSet(attrs), + promSent: synchronization.TelemetryClientMessagesSent.WithLabelValues(endpoint, string(telemType)), + promSendErrors: synchronization.TelemetryClientMessagesSendErrors.WithLabelValues(endpoint, string(telemType)), + promDropped: synchronization.TelemetryClientMessagesDropped.WithLabelValues(endpoint, string(telemType)), + } + } + m.Service, m.eng = services.Config{Name: "DomainManager"}.NewServiceEngine(lggr) return m, nil } -func newDomainManagerMetrics() (m domainManagerMetrics, err error) { - meter := otel.Meter("github.com/smartcontractkit/chainlink/v2/core/services/telemetry") +func newDomainManagerMetrics(mp otelmetric.MeterProvider) (m domainManagerMetrics, err error) { + meter := mp.Meter("github.com/smartcontractkit/chainlink/v2/core/services/telemetry") if m.sent, err = meter.Int64Counter(metricMessagesSent, otelmetric.WithDescription("Telemetry messages accepted by chip ingress")); err != nil { return m, err @@ -142,7 +204,7 @@ func newDomainManagerMetrics() (m domainManagerMetrics, err error) { return m, err } if m.dropped, err = meter.Int64Counter(metricMessagesDropped, - otelmetric.WithDescription("Telemetry messages dropped before publishing")); err != nil { + otelmetric.WithDescription("Telemetry messages dropped before publishing (buffer full, shutdown, refused)")); err != nil { return m, err } if m.unsupported, err = meter.Int64Counter(metricMessagesUnsupported, @@ -155,115 +217,135 @@ func newDomainManagerMetrics() (m domainManagerMetrics, err error) { // GenMonitoringEndpoint returns an endpoint for one telemetry type. Unsupported types and chains // get an endpoint that drops and counts what it is given. func (m *DomainManager) GenMonitoringEndpoint(network string, chainID string, contractID string, telemType synchronization.TelemetryType) commontypes.MonitoringEndpoint { - route, ok := telemetryRoutes[telemType] + route, ok := m.routes[telemType] if !ok { - m.eng.Warnw("Unsupported telemetry type: its telemetry will be dropped", "telemType", telemType, "network", network, "chainID", chainID, "contractID", contractID) - return m.unsupportedAgent(telemType, unsupportedReasonTelemetryType) - } - selector, err := chainSelector(network, chainID) - if err != nil { - m.eng.Warnw("Unsupported chain: its telemetry will be dropped", "err", err, "telemType", telemType, "network", network, "chainID", chainID, "contractID", contractID) - return m.unsupportedAgent(telemType, unsupportedReasonChain) + m.reportOnce("type:"+string(telemType), func() { + m.eng.Warnw("Telemetry type is not supported: its telemetry will be dropped", "telemType", telemType) + }) + return &unsupportedAgent{m: m, telemType: telemType, reason: unsupportedReasonTelemetryType} } - return &domainAgent{ - m: m, - route: route, - emitter: m.emitters[route.domain], - telemType: telemType, - network: network, - contractID: contractID, - selector: selector, - attrs: routeAttrs(route, telemType), + selector, ok := m.chainSelector(network, chainID) + if !ok { + return &unsupportedAgent{m: m, telemType: telemType, reason: unsupportedReasonChain} } + return &domainAgent{m: m, route: route, network: strings.ToLower(network), contractID: contractID, selector: selector} } // GenMultitypeMonitoringEndpoint returns an endpoint whose messages carry their own type. func (m *DomainManager) GenMultitypeMonitoringEndpoint(network string, chainID string, contractID string) MultitypeMonitoringEndpoint { - selector, err := chainSelector(network, chainID) - if err != nil { - m.eng.Warnw("Unsupported chain: its telemetry will be dropped", "err", err, "network", network, "chainID", chainID, "contractID", contractID) - return m.unsupportedAgent("", unsupportedReasonChain) + selector, ok := m.chainSelector(network, chainID) + if !ok { + return &unsupportedAgent{m: m, reason: unsupportedReasonChain} } - return &multitypeDomainAgent{m: m, network: network, contractID: contractID, selector: selector} + return &multitypeDomainAgent{m: m, network: strings.ToLower(network), contractID: contractID, selector: selector} } -func chainSelector(network, chainID string) (uint64, error) { +// chainSelector resolves the job's chain to its chain-selectors selector, which legacy-mode chip +// ingress maps to the network's topics. A chain that does not resolve (e.g. one newer than the +// pinned chain-selectors) loses its telemetry, so it is logged at Error, once per chain. +func (m *DomainManager) chainSelector(network, chainID string) (uint64, bool) { details, err := chainselector.GetChainDetailsByChainIDAndFamily(chainID, strings.ToLower(network)) if err != nil { - return 0, err + m.reportOnce("chain:"+strings.ToLower(network)+"/"+chainID, func() { + m.eng.Errorw("Chain has no chain selector: its telemetry will be dropped", "network", network, "chainID", chainID, "err", err) + }) + return 0, false } - return details.ChainSelector, nil + return details.ChainSelector, true } -func routeAttrs(route telemetryRoute, telemType synchronization.TelemetryType) attribute.Set { - return attribute.NewSet( - attribute.String("domain", string(route.domain)), - attribute.String("telemetry_type", string(telemType)), - ) +func (m *DomainManager) reportOnce(key string, report func()) { + if _, loaded := m.reported.LoadOrStore(key, struct{}{}); !loaded { + report() + } } -// send builds the event and hands it to emitter. Fates are counted, never logged per message. -func (m *DomainManager) send(emitter beholder.DomainEmitter, route telemetryRoute, telemType synchronization.TelemetryType, - network, contractID string, selector uint64, attrs attribute.Set, telemetry []byte) { +// send builds the event and hands it to the route's emitter. Fates are counted; the emitter +// rate-limits its own failure logs. +func (m *DomainManager) send(route *telemetryRoute, network, contractID string, selector uint64, telemetry []byte) { // SendLog has no context (libocr's MonitoringEndpoint); the emitter only queues. ctx := context.Background() - ev, err := m.newEvent(route, telemType, network, contractID, selector, telemetry) + ev, err := m.newEvent(route, network, contractID, selector, telemetry) if err != nil { - m.metrics.dropped.Add(ctx, 1, otelmetric.WithAttributeSet(attrs)) + m.recordDropped(ctx, route, errorCodeEventBuild) return } - err = emitter.EmitEvent(ctx, ev, func(sendErr error) { - if sendErr == nil { - m.metrics.sent.Add(ctx, 1, otelmetric.WithAttributeSet(attrs)) - return - } - errAttrs := append(attrs.ToSlice(), attribute.String("error_code", batch.ErrorCodeFor(sendErr))) - m.metrics.sendErrors.Add(ctx, 1, otelmetric.WithAttributes(errAttrs...)) - }) - if err != nil { - m.metrics.dropped.Add(ctx, 1, otelmetric.WithAttributeSet(attrs)) + if err := route.emitter.EmitEvent(ctx, ev, func(sendErr error) { m.recordFate(ctx, route, sendErr) }); err != nil { + m.recordDropped(ctx, route, errorCodeEmitRejected) + } +} + +// recordFate counts an event's fate as reported by the emitter: published, failed to publish, +// or dropped before publishing (full buffer or shutdown). +func (m *DomainManager) recordFate(ctx context.Context, route *telemetryRoute, err error) { + switch { + case err == nil: + m.metrics.sent.Add(ctx, 1, route.okOpt) + route.promSent.Inc() + case errors.Is(err, batch.ErrMessageBufferFull), errors.Is(err, batch.ErrClientShutdown): + m.recordDropped(ctx, route, batch.ErrorCodeFor(err)) + default: + m.metrics.sendErrors.Add(ctx, 1, route.errorOpt(batch.ErrorCodeFor(err))) + route.promSendErrors.Inc() + } +} + +func (m *DomainManager) recordDropped(ctx context.Context, route *telemetryRoute, code string) { + m.metrics.dropped.Add(ctx, 1, route.errorOpt(code)) + route.promDropped.Inc() +} + +// errorOpt returns the route's attributes plus error_code, cached per code (a bounded set). +func (r *telemetryRoute) errorOpt(code string) otelmetric.AddOption { + if v, ok := r.errorOpts.Load(code); ok { + return v.(otelmetric.AddOption) } + opt := otelmetric.WithAttributes(append(r.attrs.ToSlice(), attribute.String("error_code", code))...) + v, _ := r.errorOpts.LoadOrStore(code, opt) + return v.(otelmetric.AddOption) +} + +func (m *DomainManager) countUnsupported(telemType synchronization.TelemetryType, reason string) { + m.metrics.unsupported.Add(context.Background(), 1, otelmetric.WithAttributes( + attribute.String("telemetry_type", string(telemType)), + attribute.String("reason", reason), + )) } // newEvent builds the CloudEvent legacy-mode chip ingress expects. Each extension becomes a // "ce_" Kafka header; the comments name the header OTI used for the same value. Chip // ingress derives nodeoperatorname/nodename (from the CSA key's NOP info) and receivedat itself. -func (m *DomainManager) newEvent(route telemetryRoute, telemType synchronization.TelemetryType, - network, contractID string, selector uint64, telemetry []byte) (chipingress.CloudEvent, error) { +func (m *DomainManager) newEvent(route *telemetryRoute, network, contractID string, selector uint64, telemetry []byte) (chipingress.CloudEvent, error) { now := m.now() - event, err := chipingress.NewEvent(route.source, route.entity, telemetry, map[string]any{"time": now.UTC()}) + event, err := chipingress.NewEvent(route.source, route.entity, telemetry, map[string]any{ + "time": now.UTC(), + "recordedtime": now, + }) if err != nil { return event, fmt.Errorf("failed creating CloudEvent: %w", err) } event.SetExtension("legacytelemetry", "true") // marks node-sent legacy telemetry - event.SetExtension("telemetrytype", string(telemType)) // "telemetry-type" + event.SetExtension("telemetrytype", string(route.telemType)) // "telemetry-type" event.SetExtension("chainselector", strconv.FormatUint(selector, 10)) // chain-selectors ID of the job's chain - event.SetExtension("networkname", network) // e.g. "EVM" + event.SetExtension("networkname", network) // lower case, e.g. "evm" event.SetExtension("contractid", contractID) // "contract-address" event.SetExtension("csapublickey", m.csaPubKeyHex) // "node-operator-csa-public-key" - event.SetExtension("partitionkey", contractID+"-"+m.csaPubKeyHex) // Kafka record key - event.SetExtension("sentat", strconv.FormatInt(now.UnixNano(), 10)) // "sent-at", unix ns + event.SetExtension("partitionkey", contractID+"-"+m.csaPubKeyHex) // Kafka record key, as OTI's + event.SetExtension("sentat", strconv.FormatInt(now.UnixNano(), 10)) // "sent-at", unix ns as over WSRPC return event, nil } -func (m *DomainManager) unsupportedAgent(telemType synchronization.TelemetryType, reason string) *unsupportedAgent { - return &unsupportedAgent{m: m, telemType: telemType, reason: reason} -} - // domainAgent sends one telemetry type through its domain's emitter. type domainAgent struct { m *DomainManager - route telemetryRoute - emitter beholder.DomainEmitter - telemType synchronization.TelemetryType + route *telemetryRoute network string contractID string selector uint64 - attrs attribute.Set } func (a *domainAgent) SendLog(telemetry []byte) { - a.m.send(a.emitter, a.route, a.telemType, a.network, a.contractID, a.selector, a.attrs, telemetry) + a.m.send(a.route, a.network, a.contractID, a.selector, telemetry) } // multitypeDomainAgent routes each message by the type it carries. @@ -275,15 +357,16 @@ type multitypeDomainAgent struct { } func (a *multitypeDomainAgent) SendTypedLog(telemType synchronization.TelemetryType, telemetry []byte) { - route, ok := telemetryRoutes[telemType] + route, ok := a.m.routes[telemType] if !ok { - a.m.unsupportedAgent(telemType, unsupportedReasonTelemetryType).SendLog(telemetry) + a.m.countUnsupported(telemType, unsupportedReasonTelemetryType) return } - a.m.send(a.m.emitters[route.domain], route, telemType, a.network, a.contractID, a.selector, routeAttrs(route, telemType), telemetry) + a.m.send(route, a.network, a.contractID, a.selector, telemetry) } -// unsupportedAgent drops everything it is given and counts it. +// unsupportedAgent drops everything it is given and counts it. telemType is empty for a +// multitype endpoint, whose messages carry their own type. type unsupportedAgent struct { m *DomainManager telemType synchronization.TelemetryType @@ -291,16 +374,12 @@ type unsupportedAgent struct { } func (a *unsupportedAgent) SendLog([]byte) { - a.m.metrics.unsupported.Add(context.Background(), 1, otelmetric.WithAttributes( - attribute.String("telemetry_type", string(a.telemType)), - attribute.String("reason", a.reason), - )) + a.m.countUnsupported(a.telemType, a.reason) } -func (a *unsupportedAgent) SendTypedLog(telemType synchronization.TelemetryType, telemetry []byte) { - if a.telemType == "" { - a.m.unsupportedAgent(telemType, a.reason).SendLog(telemetry) - return +func (a *unsupportedAgent) SendTypedLog(telemType synchronization.TelemetryType, _ []byte) { + if a.telemType != "" { + telemType = a.telemType } - a.SendLog(telemetry) + a.m.countUnsupported(telemType, a.reason) } diff --git a/core/services/telemetry/domain_manager_beholder_test.go b/core/services/telemetry/domain_manager_beholder_test.go new file mode 100644 index 00000000000..c7bc4f21b5b --- /dev/null +++ b/core/services/telemetry/domain_manager_beholder_test.go @@ -0,0 +1,143 @@ +package telemetry + +import ( + "context" + "net" + "sync" + "testing" + "time" + + cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" + sdklog "go.opentelemetry.io/otel/sdk/log" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" + + "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" +) + +type capturedBatch struct { + method string + md metadata.MD + events []*cepb.CloudEvent +} + +// capturingChipServer records PublishBatch / PublishBatchNoAck calls and answers each event +// with a positional success, as chip ingress does. +type capturingChipServer struct { + pb.UnimplementedChipIngressServer + + mu sync.Mutex + batches []capturedBatch +} + +func (s *capturingChipServer) record(ctx context.Context, method string, in *pb.CloudEventBatch) (*pb.PublishResponse, error) { + md, _ := metadata.FromIncomingContext(ctx) + s.mu.Lock() + s.batches = append(s.batches, capturedBatch{method: method, md: md, events: in.Events}) + s.mu.Unlock() + return &pb.PublishResponse{Results: make([]*pb.PublishResult, len(in.Events))}, nil +} + +func (s *capturingChipServer) PublishBatch(ctx context.Context, in *pb.CloudEventBatch) (*pb.PublishResponse, error) { + return s.record(ctx, "PublishBatch", in) +} + +func (s *capturingChipServer) PublishBatchNoAck(ctx context.Context, in *pb.CloudEventBatch) (*pb.PublishResponse, error) { + return s.record(ctx, "PublishBatchNoAck", in) +} + +// received returns one entry per event: the RPC method, its chainlink-domain header and the event. +func (s *capturingChipServer) received() (methods, domains []string, events []*cepb.CloudEvent) { + s.mu.Lock() + defer s.mu.Unlock() + for _, b := range s.batches { + for _, ev := range b.events { + methods = append(methods, b.method) + domains = append(domains, b.md.Get(beholder.DomainHeaderKey)...) + events = append(events, ev) + } + } + return methods, domains, events +} + +type nopLogExporter struct{} + +func (nopLogExporter) Export(context.Context, []sdklog.Record) error { return nil } +func (nopLogExporter) Shutdown(context.Context) error { return nil } +func (nopLogExporter) ForceFlush(context.Context) error { return nil } + +func TestDomainManager_ThroughRealBeholderClient(t *testing.T) { + t.Parallel() + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer() + chip := &capturingChipServer{} + pb.RegisterChipIngressServer(srv, chip) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + client, err := beholder.NewGRPCClient(beholder.Config{ + OtelExporterGRPCEndpoint: "localhost:4317", // never reached: logs go to nopLogExporter + InsecureConnection: true, + ChipIngressEmitterEnabled: true, + ChipIngressEmitterGRPCEndpoint: lis.Addr().String(), + ChipIngressInsecureConnection: true, + ChipIngressSendInterval: 10 * time.Millisecond, + ChipIngressDomainEmittersEnabled: true, + AuthHeaders: map[string]string{"X-Beholder-Node-Auth-Token": "1:abc:2:def"}, + }, func(...otlploggrpc.Option) (sdklog.Exporter, error) { return nopLogExporter{}, nil }) + require.NoError(t, err) + require.NoError(t, client.Start(t.Context())) + + m, reader := newTestDomainManager(t, client) + + // Concurrent senders across two domains, through single-type and multitype endpoints. + const workers, perWorker = 4, 25 + var wg sync.WaitGroup + for range workers { + wg.Go(func() { + feeds := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median) + streams := m.GenMultitypeMonitoringEndpoint("evm", ethMainnetChainID, "0xdef/1") + for range perWorker { + feeds.SendLog([]byte("p")) + streams.SendTypedLog(synchronization.LLOOutcome, []byte("p")) + } + }) + } + wg.Wait() + + const total = 2 * workers * perWorker + require.Eventually(t, func() bool { return counterValue(t, reader, metricMessagesSent) == total }, + 10*time.Second, 20*time.Millisecond, "every event is acknowledged by the per-event result") + + methods, domains, events := chip.received() + require.Len(t, events, total) + for i, ev := range events { + assert.Equal(t, "PublishBatchNoAck", methods[i]) + switch ev.Source { + case "data-feeds.telemetry.ocr2-median": + assert.Equal(t, string(beholder.DomainDataFeeds), domains[i]) + case "data-streams.telemetry.llo-outcome": + assert.Equal(t, string(beholder.DomainDataStreams), domains[i]) + default: + t.Fatalf("unexpected source %q", ev.Source) + } + assert.Equal(t, "true", ev.Attributes["legacytelemetry"].GetCeString()) + assert.Equal(t, testCSAPubKeyHex, ev.Attributes["csapublickey"].GetCeString()) + } + + // Closing the client drains and closes the domain emitters exactly once. OTel provider + // shutdown may fail to flush to the absent collector; nothing else may fail. + if err := client.Close(); err != nil { + assert.NotContains(t, err.Error(), "client connection is closing") + } + assert.Zero(t, counterValue(t, reader, metricMessagesSendErrors)) + assert.Zero(t, counterValue(t, reader, metricMessagesDropped)) + +} diff --git a/core/services/telemetry/domain_manager_test.go b/core/services/telemetry/domain_manager_test.go index 306811f804f..9834a6c4b91 100644 --- a/core/services/telemetry/domain_manager_test.go +++ b/core/services/telemetry/domain_manager_test.go @@ -7,17 +7,21 @@ import ( "strings" "sync" "testing" + "time" + "github.com/prometheus/client_golang/prometheus/testutil" chainselectors "github.com/smartcontractkit/chain-selectors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" @@ -25,12 +29,15 @@ import ( const testCSAPubKeyHex = "755294c8a1f0" -// fakeDomainEmitter records EmitEvent calls and reports callbackErr as each event's fate. +// fakeDomainEmitter records EmitEvent calls. Like the real emitter, it can report each event's +// fate from another goroutine (async). type fakeDomainEmitter struct { - mu sync.Mutex - events []chipingress.CloudEvent - emitErr error // returned by EmitEvent (callback not invoked) - callbackErr error // reported to the callback + async bool + emitErr error // returned by EmitEvent; the callback is then not invoked + callbackErr error // reported to the callback as the event's fate + + mu sync.Mutex + events []chipingress.CloudEvent } func (f *fakeDomainEmitter) Emit(context.Context, []byte, ...any) error { return nil } @@ -43,7 +50,12 @@ func (f *fakeDomainEmitter) EmitEvent(_ context.Context, event chipingress.Cloud f.mu.Lock() f.events = append(f.events, event) f.mu.Unlock() - if callback != nil { + if callback == nil { + return nil + } + if f.async { + go callback(f.callbackErr) + } else { callback(f.callbackErr) } return nil @@ -74,21 +86,22 @@ func newFakeEmitterSource() fakeEmitterSource { } } -// installTestMeter routes otel.Meter to a manual reader for the test's duration. -func installTestMeter(t *testing.T) *sdkmetric.ManualReader { +// newTestDomainManager returns a manager whose OTel metrics go to its own reader, so tests can +// run in parallel. +func newTestDomainManager(t *testing.T, src DomainEmitterSource) (*DomainManager, *sdkmetric.ManualReader) { t.Helper() reader := sdkmetric.NewManualReader() - prev := otel.GetMeterProvider() - otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))) - t.Cleanup(func() { otel.SetMeterProvider(prev) }) - return reader + m, err := NewDomainManager(src, testCSAPubKeyHex, logger.Test(t), + WithMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)))) + require.NoError(t, err) + return m, reader } // counterValue sums the int64 counter name over data points whose attributes include want. func counterValue(t *testing.T, reader *sdkmetric.ManualReader, name string, want ...attribute.KeyValue) int64 { t.Helper() var rm metricdata.ResourceMetrics - require.NoError(t, reader.Collect(context.Background(), &rm)) + require.NoError(t, reader.Collect(t.Context(), &rm)) var total int64 for _, sm := range rm.ScopeMetrics { for _, m := range sm.Metrics { @@ -114,71 +127,109 @@ func counterValue(t *testing.T, reader *sdkmetric.ManualReader, name string, wan return total } -func newTestDomainManager(t *testing.T, src fakeEmitterSource) *DomainManager { - t.Helper() - m, err := NewDomainManager(src, testCSAPubKeyHex, logger.Test(t)) - require.NoError(t, err) - return m -} - var ethMainnetChainID = strconv.FormatUint(chainselectors.ETHEREUM_MAINNET.EvmChainID, 10) func TestNewDomainManager_RequiresEveryDomain(t *testing.T) { + t.Parallel() src := newFakeEmitterSource() delete(src, beholder.DomainCCIP) + delete(src, beholder.DomainDataStreams) _, err := NewDomainManager(src, testCSAPubKeyHex, logger.Test(t)) - require.Error(t, err, "a missing domain emitter fails construction") + require.Error(t, err) + assert.Contains(t, err.Error(), `"ccip"`, "domains are resolved in a stable order, so the same one is reported every time") } -func TestNewDomainManager_RequiresCSAPublicKey(t *testing.T) { - _, err := NewDomainManager(newFakeEmitterSource(), "", logger.Test(t)) +func TestNewDomainManager_RequiresEmitterSourceAndCSAPublicKey(t *testing.T) { + t.Parallel() + _, err := NewDomainManager(nil, testCSAPubKeyHex, logger.Test(t)) + require.Error(t, err) + _, err = NewDomainManager(newFakeEmitterSource(), "", logger.Test(t)) require.Error(t, err) } func TestDomainManager_RoutesEachTypeToItsDomain(t *testing.T) { + t.Parallel() + const ocr2, ocr3 = "offchainreporting2.TelemetryWrapper", "offchainreporting3.TelemetryWrapper" for _, tc := range []struct { telemType synchronization.TelemetryType domain beholder.Domain source string + entity string }{ - {synchronization.OCR, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr"}, - {synchronization.OCR2Median, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr2-median"}, - {synchronization.OCR3DataFeeds, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr3-data-feeds"}, - {synchronization.EnhancedEA, beholder.DomainDataFeeds, "data-feeds.telemetry.enhanced-ea"}, - {synchronization.OCR3Mercury, beholder.DomainDataStreams, "data-streams.telemetry.ocr3-mercury"}, - {synchronization.EnhancedEAMercury, beholder.DomainDataStreams, "data-streams.telemetry.enhanced-ea-mercury"}, - {synchronization.LLOObservation, beholder.DomainDataStreams, "data-streams.telemetry.llo-observation"}, - {synchronization.LLOOutcome, beholder.DomainDataStreams, "data-streams.telemetry.llo-outcome"}, - {synchronization.LLOAttributedObservation, beholder.DomainDataStreams, "data-streams.telemetry.llo-attributed-observation"}, - {synchronization.PipelineBridge, beholder.DomainDataStreams, "data-streams.telemetry.pipeline-bridge"}, - {synchronization.OCR2CCIPCommit, beholder.DomainCCIP, "ccip.telemetry.ocr2"}, - {synchronization.OCR2CCIPExec, beholder.DomainCCIP, "ccip.telemetry.ocr2"}, - {synchronization.OCR3CCIPCommit, beholder.DomainCCIP, "ccip.telemetry.ocr3"}, - {synchronization.OCR3CCIPExec, beholder.DomainCCIP, "ccip.telemetry.ocr3"}, - {synchronization.HeadReport, beholder.DomainCCIP, "ccip.telemetry.head-report"}, + {synchronization.OCR, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr", "offchainreporting.TelemetryWrapper"}, + {synchronization.OCR2Median, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr2-median", ocr2}, + {synchronization.OCR3DataFeeds, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr3-data-feeds", ocr3}, + {synchronization.EnhancedEA, beholder.DomainDataFeeds, "data-feeds.telemetry.enhanced-ea", "telem.EnhancedEA"}, + {synchronization.OCR3Mercury, beholder.DomainDataStreams, "data-streams.telemetry.ocr3-mercury", ocr3}, + {synchronization.EnhancedEAMercury, beholder.DomainDataStreams, "data-streams.telemetry.enhanced-ea-mercury", "telem.EnhancedEAMercury"}, + {synchronization.LLOObservation, beholder.DomainDataStreams, "data-streams.telemetry.llo-observation", "telem.LLOObservationTelemetry"}, + {synchronization.LLOOutcome, beholder.DomainDataStreams, "data-streams.telemetry.llo-outcome", "telem.LLOOutcomeTelemetry"}, + {synchronization.PipelineBridge, beholder.DomainDataStreams, "data-streams.telemetry.pipeline-bridge", "telem.LLOBridgeTelemetry"}, + {synchronization.OCR2CCIPCommit, beholder.DomainCCIP, "ccip.telemetry.ocr2", ocr2}, + {synchronization.OCR2CCIPExec, beholder.DomainCCIP, "ccip.telemetry.ocr2", ocr2}, + {synchronization.OCR3CCIPCommit, beholder.DomainCCIP, "ccip.telemetry.ocr3", ocr3}, + {synchronization.OCR3CCIPExec, beholder.DomainCCIP, "ccip.telemetry.ocr3", ocr3}, + {synchronization.HeadReport, beholder.DomainCCIP, "ccip.telemetry.head-report", "telem.HeadReportRequest"}, } { t.Run(string(tc.telemType), func(t *testing.T) { + t.Parallel() src := newFakeEmitterSource() - m := newTestDomainManager(t, src) + m, _ := newTestDomainManager(t, src) m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", tc.telemType).SendLog([]byte("payload")) for d, em := range src { - if d == tc.domain { - require.Len(t, em.sent(), 1, "sent on %s", d) - assert.Equal(t, tc.source, em.sent()[0].Source()) - } else { + if d != tc.domain { assert.Empty(t, em.sent(), "nothing on %s", d) + continue } + require.Len(t, em.sent(), 1, "sent on %s", d) + assert.Equal(t, tc.source, em.sent()[0].Source()) + assert.Equal(t, tc.entity, em.sent()[0].Type()) } }) } } +func TestDomainManager_DropsTypesItDoesNotSupport(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, reader := newTestDomainManager(t, src) + + unsupported := []synchronization.TelemetryType{ + synchronization.OCR2Functions, + synchronization.FunctionsRequests, + synchronization.LLOAttributedObservation, + synchronization.LLOReport, + synchronization.OCR2Threshold, + synchronization.OCR2S4, + synchronization.OCR3Rebalancer, + synchronization.OCR3CCIPBootstrap, + "vault-plugin", + } + for _, tt := range unsupported { + ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", tt) + require.NotNil(t, ep) + ep.SendLog([]byte("p")) + ep.SendLog([]byte("p")) + } + + for d, em := range src { + assert.Empty(t, em.sent(), "nothing sent on %s", d) + } + assert.Equal(t, int64(2), counterValue(t, reader, metricMessagesUnsupported, + attribute.String("telemetry_type", string(synchronization.LLOAttributedObservation)), + attribute.String("reason", unsupportedReasonTelemetryType)), "counted per message") + assert.Equal(t, int64(2*len(unsupported)), counterValue(t, reader, metricMessagesUnsupported)) +} + func TestDomainManager_EventCarriesLegacyTelemetryExtensions(t *testing.T) { + t.Parallel() src := newFakeEmitterSource() - m := newTestDomainManager(t, src) + m, _ := newTestDomainManager(t, src) + now := time.Date(2026, 10, 28, 12, 0, 0, 0, time.UTC) + m.now = func() time.Time { return now } m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("payload")) @@ -187,124 +238,192 @@ func TestDomainManager_EventCarriesLegacyTelemetryExtensions(t *testing.T) { ev := events[0] require.NoError(t, ev.Validate()) - assert.Equal(t, "offchainreporting2.TelemetryWrapper", ev.Type()) assert.Equal(t, []byte("payload"), ev.Data()) - assert.False(t, ev.Time().IsZero()) + assert.True(t, ev.Time().Equal(now)) ext := ev.Extensions() assert.Equal(t, "true", ext["legacytelemetry"]) assert.Equal(t, "ocr2-median", ext["telemetrytype"]) assert.Equal(t, strconv.FormatUint(chainselectors.ETHEREUM_MAINNET.Selector, 10), ext["chainselector"]) - assert.Equal(t, "EVM", ext["networkname"]) + assert.Equal(t, "evm", ext["networkname"], "network is normalized to lower case") assert.Equal(t, "0xabc", ext["contractid"]) assert.Equal(t, testCSAPubKeyHex, ext["csapublickey"]) assert.Equal(t, "0xabc-"+testCSAPubKeyHex, ext["partitionkey"]) - sentAt, ok := ext["sentat"].(string) + assert.Equal(t, strconv.FormatInt(now.UnixNano(), 10), ext["sentat"], "sentat is unix nanoseconds, as the node sends OTI today") + recorded, ok := ext["recordedtime"] require.True(t, ok) - _, err := strconv.ParseInt(sentAt, 10, 64) - assert.NoError(t, err, "sentat is unix nanoseconds") + assert.Contains(t, recorded.(interface{ String() string }).String(), "2026-10-28T12:00:00", "recordedtime uses the manager's clock") } -func TestDomainManager_SendsForAnyNetworkWithoutPerNetworkConfig(t *testing.T) { - src := newFakeEmitterSource() - m := newTestDomainManager(t, src) - - for _, chain := range []chainselectors.Chain{chainselectors.ETHEREUM_MAINNET, chainselectors.POLYGON_MAINNET} { - m.GenMonitoringEndpoint("EVM", strconv.FormatUint(chain.EvmChainID, 10), "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) - } - assert.Len(t, src[beholder.DomainDataFeeds].sent(), 2) -} +func TestDomainManager_ResolvesChainsInEveryFamily(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + network, chainID string + selector uint64 + }{ + {"EVM", ethMainnetChainID, chainselectors.ETHEREUM_MAINNET.Selector}, + {"evm", ethMainnetChainID, chainselectors.ETHEREUM_MAINNET.Selector}, + {"Solana", "5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d", 124615329519749607}, // genesis hash: the relayer's chain ID + {"StarkNet", "SN_MAIN", 511843109281680063}, + {"Aptos", "1", 4741433654826277614}, + {"Tron", "728126428", 1546563616611573945}, + {"TON", "-239", 16448340667252469081}, + {"Sui", "1", 17529533435026248318}, + } { + t.Run(tc.network+"/"+tc.chainID, func(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, _ := newTestDomainManager(t, src) -func TestDomainManager_UnsupportedTypesAreDroppedAndCounted(t *testing.T) { - reader := installTestMeter(t) - src := newFakeEmitterSource() - m := newTestDomainManager(t, src) + m.GenMonitoringEndpoint(tc.network, tc.chainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) - unsupported := []synchronization.TelemetryType{ - synchronization.OCR2Functions, - synchronization.FunctionsRequests, - synchronization.OCR2Threshold, - synchronization.OCR2S4, - synchronization.OCR3Rebalancer, - synchronization.LLOReport, - "vault-plugin", - } - for _, tt := range unsupported { - ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", tt) - require.NotNil(t, ep) - ep.SendLog([]byte("p")) - } - - for d, em := range src { - assert.Empty(t, em.sent(), "nothing sent on %s", d) + events := src[beholder.DomainDataFeeds].sent() + require.Len(t, events, 1) + assert.Equal(t, strconv.FormatUint(tc.selector, 10), events[0].Extensions()["chainselector"]) + assert.Equal(t, strings.ToLower(tc.network), events[0].Extensions()["networkname"]) + }) } - assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, - attribute.String("telemetry_type", string(synchronization.OCR2Functions)), - attribute.String("reason", unsupportedReasonTelemetryType))) - assert.Equal(t, int64(len(unsupported)), counterValue(t, reader, metricMessagesUnsupported)) } func TestDomainManager_UnresolvableChainIsDroppedAndCounted(t *testing.T) { - reader := installTestMeter(t) + t.Parallel() src := newFakeEmitterSource() - m := newTestDomainManager(t, src) + m, reader := newTestDomainManager(t, src) m.GenMonitoringEndpoint("EVM", "999999999999", "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + mt := m.GenMultitypeMonitoringEndpoint("EVM", "999999999999", "0xabc/1") + mt.SendTypedLog(synchronization.LLOOutcome, []byte("p")) + mt.SendTypedLog(synchronization.LLOObservation, []byte("p")) - assert.Empty(t, src[beholder.DomainDataFeeds].sent()) - assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, - attribute.String("reason", unsupportedReasonChain))) + for d, em := range src { + assert.Empty(t, em.sent(), "nothing sent on %s", d) + } + reason := attribute.String("reason", unsupportedReasonChain) + assert.Equal(t, int64(3), counterValue(t, reader, metricMessagesUnsupported, reason)) + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, reason, + attribute.String("telemetry_type", string(synchronization.LLOOutcome))), "multitype counts carry the message's type") } func TestDomainManager_MultitypeEndpointRoutesPerMessage(t *testing.T) { + t.Parallel() src := newFakeEmitterSource() - m := newTestDomainManager(t, src) + m, reader := newTestDomainManager(t, src) ep := m.GenMultitypeMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc/1") ep.SendTypedLog(synchronization.LLOOutcome, []byte("p")) - ep.SendTypedLog(synchronization.LLOReport, []byte("p")) // unmapped: dropped + ep.SendTypedLog(synchronization.LLOReport, []byte("p")) // unsupported: dropped events := src[beholder.DomainDataStreams].sent() require.Len(t, events, 1) assert.Equal(t, "data-streams.telemetry.llo-outcome", events[0].Source()) assert.Equal(t, "llo-outcome", events[0].Extensions()["telemetrytype"]) + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, + attribute.String("telemetry_type", string(synchronization.LLOReport)))) } -func TestDomainManager_MetricsFollowEachEventsFate(t *testing.T) { - dataFeeds := attribute.String("domain", string(beholder.DomainDataFeeds)) - ocr2 := attribute.String("telemetry_type", string(synchronization.OCR2Median)) +func TestDomainManager_CountsEachEventsFate(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + emitErr error + callbackErr error + metric string + errorCode string + }{ + {name: "sent", metric: metricMessagesSent}, + {name: "server unavailable", callbackErr: status.Error(codes.Unavailable, "down"), metric: metricMessagesSendErrors, errorCode: codes.Unavailable.String()}, + {name: "server without no-ack", callbackErr: status.Error(codes.Unimplemented, "no"), metric: metricMessagesSendErrors, errorCode: codes.Unimplemented.String()}, + {name: "buffer full", callbackErr: batch.ErrMessageBufferFull, metric: metricMessagesDropped, errorCode: batch.ErrorCodeFor(batch.ErrMessageBufferFull)}, + {name: "emitter shutting down", callbackErr: batch.ErrClientShutdown, metric: metricMessagesDropped, errorCode: batch.ErrorCodeFor(batch.ErrClientShutdown)}, + {name: "emitter refuses the event", emitErr: errors.New("not started"), metric: metricMessagesDropped, errorCode: errorCodeEmitRejected}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + src[beholder.DomainDataFeeds].async = true + src[beholder.DomainDataFeeds].emitErr = tc.emitErr + src[beholder.DomainDataFeeds].callbackErr = tc.callbackErr + m, reader := newTestDomainManager(t, src) - t.Run("sent", func(t *testing.T) { - reader := installTestMeter(t) - m := newTestDomainManager(t, newFakeEmitterSource()) - m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) - assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesSent, dataFeeds, ocr2)) - }) - t.Run("send error from the per-message result", func(t *testing.T) { - reader := installTestMeter(t) - src := newFakeEmitterSource() - src[beholder.DomainDataFeeds].callbackErr = errors.New("rejected") - m := newTestDomainManager(t, src) - m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) - assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesSendErrors, dataFeeds, ocr2)) - assert.Zero(t, counterValue(t, reader, metricMessagesSent)) - }) - t.Run("dropped when the emitter refuses the event", func(t *testing.T) { - reader := installTestMeter(t) + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + + want := []attribute.KeyValue{ + attribute.String("domain", string(beholder.DomainDataFeeds)), + attribute.String("telemetry_type", string(synchronization.OCR2Median)), + } + if tc.errorCode != "" { + want = append(want, attribute.String("error_code", tc.errorCode)) + } + require.Eventually(t, func() bool { return counterValue(t, reader, tc.metric, want...) == 1 }, + 5*time.Second, 10*time.Millisecond) + for _, other := range []string{metricMessagesSent, metricMessagesSendErrors, metricMessagesDropped} { + if other != tc.metric { + assert.Zero(t, counterValue(t, reader, other), "%s", other) + } + } + }) + } +} + +func TestDomainManager_ConcurrentSends(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + for _, em := range src { + em.async = true + } + m, reader := newTestDomainManager(t, src) + + const workers, perWorker = 16, 50 + var wg sync.WaitGroup + for i := range workers { + wg.Go(func() { + ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0x"+strconv.Itoa(i), synchronization.OCR2Median) + mt := m.GenMultitypeMonitoringEndpoint("EVM", ethMainnetChainID, "0x"+strconv.Itoa(i)+"/1") + for range perWorker { + ep.SendLog([]byte("p")) + mt.SendTypedLog(synchronization.LLOOutcome, []byte("p")) + } + }) + } + wg.Wait() + + require.Eventually(t, func() bool { return counterValue(t, reader, metricMessagesSent) == 2*workers*perWorker }, + 5*time.Second, 10*time.Millisecond) + assert.Len(t, src[beholder.DomainDataFeeds].sent(), workers*perWorker) + assert.Len(t, src[beholder.DomainDataStreams].sent(), workers*perWorker) +} + +// The Prometheus counters are process-global, so this test does not run in parallel: sequential +// tests finish before parallel ones resume, which keeps the deltas below exact. +// +//nolint:paralleltest // asserts deltas on process-global Prometheus counters +func TestDomainManager_AlsoCountsInPrometheus(t *testing.T) { + const endpoint = "beholder/data-feeds" + tt := string(synchronization.OCR2Median) + sent := testutil.ToFloat64(synchronization.TelemetryClientMessagesSent.WithLabelValues(endpoint, tt)) + sendErrs := testutil.ToFloat64(synchronization.TelemetryClientMessagesSendErrors.WithLabelValues(endpoint, tt)) + dropped := testutil.ToFloat64(synchronization.TelemetryClientMessagesDropped.WithLabelValues(endpoint, tt)) + + for _, callbackErr := range []error{nil, status.Error(codes.Unavailable, "down"), batch.ErrMessageBufferFull} { src := newFakeEmitterSource() - src[beholder.DomainDataFeeds].emitErr = errors.New("stopped") - m := newTestDomainManager(t, src) + src[beholder.DomainDataFeeds].callbackErr = callbackErr + m, _ := newTestDomainManager(t, src) m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) - assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesDropped, dataFeeds, ocr2)) - }) + } + + assert.Equal(t, sent+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesSent.WithLabelValues(endpoint, tt))) + assert.Equal(t, sendErrs+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesSendErrors.WithLabelValues(endpoint, tt))) + assert.Equal(t, dropped+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesDropped.WithLabelValues(endpoint, tt))) } // Beholder domain emitters reject events whose source is not under their domain, so every -// route's source must be the domain or a subdomain of it. -func TestTelemetryRoutes_SourcesSitUnderTheirDomain(t *testing.T) { - for telemType, route := range telemetryRoutes { - d := string(route.domain) - assert.True(t, route.source == d || strings.HasPrefix(route.source, d+"."), - "%s: source %q is not under domain %q", telemType, route.source, d) +// supported type's source must be the domain or a subdomain of it. +func TestSupportedTelemetry_SourcesSitUnderTheirDomain(t *testing.T) { + t.Parallel() + for telemType, d := range supportedTelemetryDomains() { + source, _, err := synchronization.TelemetryTypeToDomainAndEntity(telemType) + require.NoError(t, err, "%s has a CloudEvent source", telemType) + assert.True(t, source == string(d) || strings.HasPrefix(source, string(d)+"."), + "%s: source %q is not under domain %q", telemType, source, d) } } From e0d8f2864dab90a7a7bb9aee618c95dc1a9b1646 Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:46:02 -0400 Subject: [PATCH 04/23] Extract telemetry manager selection; parallel tests - newTelemetryManager picks DomainManager or the WSRPC Manager, and rejects a nil Beholder client before it becomes an interface value - New tests run in parallel; the cmd test starts and closes its client Co-Authored-By: Claude Opus 5.5 --- core/cmd/beholder_client_test.go | 11 ++++ core/services/chainlink/application.go | 20 +----- .../config_telemetry_ingress_test.go | 3 + core/services/chainlink/telemetry_manager.go | 39 +++++++++++ .../chainlink/telemetry_manager_test.go | 66 +++++++++++++++++++ 5 files changed, 122 insertions(+), 17 deletions(-) create mode 100644 core/services/chainlink/telemetry_manager.go create mode 100644 core/services/chainlink/telemetry_manager_test.go diff --git a/core/cmd/beholder_client_test.go b/core/cmd/beholder_client_test.go index 4e97c3ce017..4c902686650 100644 --- a/core/cmd/beholder_client_test.go +++ b/core/cmd/beholder_client_test.go @@ -3,6 +3,7 @@ package cmd import ( "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/beholder" @@ -13,6 +14,7 @@ import ( ) func TestNewBeholderClient_DomainEmittersFollowTelemetryIngressBeholderEnabled(t *testing.T) { + t.Parallel() const beholderTOML = ` [Telemetry] Enabled = true @@ -30,6 +32,7 @@ ChipIngressInsecureConnection = true {name: "on", toml: beholderTOML + "\n[TelemetryIngress]\nBeholderEnabled = true\n", enabled: true}, } { t.Run(tt.name, func(t *testing.T) { + t.Parallel() cfg, err := chainlink.GeneralConfigOpts{ConfigStrings: []string{tt.toml}}.New() require.NoError(t, err) @@ -38,6 +41,14 @@ ChipIngressInsecureConnection = true client, err := newBeholderClient(logger.TestLogger(t), ks, cfg.Tracing(), cfg, cfg.Telemetry(), "abc123", nil) require.NoError(t, err) + require.NoError(t, client.Start(t.Context())) + t.Cleanup(func() { + // OTel shutdown may fail to flush to the absent collector; the domain emitters + // must close cleanly. + if err := client.Close(); err != nil { + assert.NotContains(t, err.Error(), "client connection is closing") + } + }) _, err = client.ForDomain(beholder.DomainDataFeeds) if tt.enabled { diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index d926281049d..f7365d1fbc5 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -86,7 +86,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/shardorchestrator" "github.com/smartcontractkit/chainlink/v2/core/services/standardcapabilities" "github.com/smartcontractkit/chainlink/v2/core/services/streams" - "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" "github.com/smartcontractkit/chainlink/v2/core/services/vrf" workflowstore "github.com/smartcontractkit/chainlink/v2/core/services/workflows/store" "github.com/smartcontractkit/chainlink/v2/core/sessions" @@ -553,22 +552,9 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err globalLogger.Info("Nurse service (automatic pprof profiling) is disabled") } - var telemetryManager interface { - commonsrv.Service - telemetry.MonitoringEndpointGenerator - } - if cfg.TelemetryIngress().BeholderEnabled() { - if opts.BeholderClient == nil { - return nil, errors.New("TelemetryIngress.BeholderEnabled requires a Beholder client") - } - domainManager, dmErr := telemetry.NewDomainManager(opts.BeholderClient, csaPubKeyHex, globalLogger) - if dmErr != nil { - return nil, fmt.Errorf("failed to create telemetry domain manager: %w", dmErr) - } - globalLogger.Info("Telemetry is sent through Beholder; TelemetryIngress.Endpoints are ignored") - telemetryManager = domainManager - } else { - telemetryManager = telemetry.NewManager(cfg.TelemetryIngress(), csaKeystore, globalLogger) + telemetryManager, err := newTelemetryManager(cfg.TelemetryIngress(), opts.BeholderClient, csaKeystore, csaPubKeyHex, globalLogger) + if err != nil { + return nil, err } srvcs = append(srvcs, telemetryManager) diff --git a/core/services/chainlink/config_telemetry_ingress_test.go b/core/services/chainlink/config_telemetry_ingress_test.go index 4a50d8c9bfa..a6500e7867d 100644 --- a/core/services/chainlink/config_telemetry_ingress_test.go +++ b/core/services/chainlink/config_telemetry_ingress_test.go @@ -61,12 +61,14 @@ func TestTelemetryIngressConfig_ChipIngressEnabled(t *testing.T) { } func TestTelemetryIngressConfig_BeholderEnabled(t *testing.T) { + t.Parallel() trueVal, falseVal := true, false assert.True(t, (&telemetryIngressConfig{c: toml.TelemetryIngress{BeholderEnabled: &trueVal}}).BeholderEnabled()) assert.False(t, (&telemetryIngressConfig{c: toml.TelemetryIngress{BeholderEnabled: &falseVal}}).BeholderEnabled()) } func TestCoreValidateConfig_TelemetryIngressBeholderEnabled(t *testing.T) { + t.Parallel() const enabled = ` [TelemetryIngress] BeholderEnabled = true @@ -91,6 +93,7 @@ ChipIngressEndpoint = 'chip.example.com:443' {name: "valid with Beholder chip ingress", toml: enabled + beholderChip}, } { t.Run(tt.name, func(t *testing.T) { + t.Parallel() var c Config require.NoError(t, commonconfig.DecodeTOML(strings.NewReader(tt.toml), &c)) c.setDefaults() diff --git a/core/services/chainlink/telemetry_manager.go b/core/services/chainlink/telemetry_manager.go new file mode 100644 index 00000000000..3a6f6912925 --- /dev/null +++ b/core/services/chainlink/telemetry_manager.go @@ -0,0 +1,39 @@ +package chainlink + +import ( + "errors" + "fmt" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + commonsrv "github.com/smartcontractkit/chainlink-common/pkg/services" + + "github.com/smartcontractkit/chainlink/v2/core/config" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore" + "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" +) + +// telemetryManager generates the node's telemetry endpoints. +type telemetryManager interface { + commonsrv.Service + telemetry.MonitoringEndpointGenerator +} + +// newTelemetryManager returns the DomainManager, which sends telemetry through Beholder's +// per-domain emitters, when TelemetryIngress.BeholderEnabled is set; otherwise the WSRPC Manager. +func newTelemetryManager(cfg config.TelemetryIngress, beholderClient *beholder.Client, csaKeystore keystore.CSA, + csaPubKeyHex string, lggr logger.SugaredLogger) (telemetryManager, error) { + if !cfg.BeholderEnabled() { + return telemetry.NewManager(cfg, csaKeystore, lggr), nil + } + // Checked before beholderClient becomes an interface value, where a nil pointer is not nil. + if beholderClient == nil { + return nil, errors.New("TelemetryIngress.BeholderEnabled requires a Beholder client") + } + dm, err := telemetry.NewDomainManager(beholderClient, csaPubKeyHex, lggr) + if err != nil { + return nil, fmt.Errorf("failed to create telemetry domain manager: %w", err) + } + lggr.Info("Telemetry is sent through Beholder; TelemetryIngress.Endpoints and its sending settings are ignored") + return dm, nil +} diff --git a/core/services/chainlink/telemetry_manager_test.go b/core/services/chainlink/telemetry_manager_test.go new file mode 100644 index 00000000000..b111b6cf238 --- /dev/null +++ b/core/services/chainlink/telemetry_manager_test.go @@ -0,0 +1,66 @@ +package chainlink + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" + sdklog "go.opentelemetry.io/otel/sdk/log" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + + "github.com/smartcontractkit/chainlink/v2/core/logger" + ksmocks "github.com/smartcontractkit/chainlink/v2/core/services/keystore/mocks" + "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" +) + +type nopLogExporter struct{} + +func (nopLogExporter) Export(context.Context, []sdklog.Record) error { return nil } +func (nopLogExporter) Shutdown(context.Context) error { return nil } +func (nopLogExporter) ForceFlush(context.Context) error { return nil } + +func TestNewTelemetryManager(t *testing.T) { + t.Parallel() + telemetryIngress := func(t *testing.T, beholderEnabled bool) GeneralConfig { + t.Helper() + toml := "" + if beholderEnabled { + toml = "[TelemetryIngress]\nBeholderEnabled = true\n" + } + cfg, err := GeneralConfigOpts{ConfigStrings: []string{toml}}.New() + require.NoError(t, err) + return cfg + } + lggr := logger.Sugared(logger.TestLogger(t)) + + t.Run("flag off: the WSRPC manager", func(t *testing.T) { + t.Parallel() + m, err := newTelemetryManager(telemetryIngress(t, false).TelemetryIngress(), nil, ksmocks.NewCSA(t), "abc123", lggr) + require.NoError(t, err) + assert.IsType(t, &telemetry.Manager{}, m) + }) + t.Run("flag on without a Beholder client: an error, not a panic", func(t *testing.T) { + t.Parallel() + var noClient *beholder.Client + _, err := newTelemetryManager(telemetryIngress(t, true).TelemetryIngress(), noClient, ksmocks.NewCSA(t), "abc123", lggr) + require.Error(t, err) + }) + t.Run("flag on: the domain manager on Beholder's domain emitters", func(t *testing.T) { + t.Parallel() + client, err := beholder.NewGRPCClient(beholder.Config{ + OtelExporterGRPCEndpoint: "localhost:4317", + InsecureConnection: true, + ChipIngressEmitterGRPCEndpoint: "localhost:9999", // never dialed: nothing is sent + ChipIngressInsecureConnection: true, + ChipIngressDomainEmittersEnabled: true, + }, func(...otlploggrpc.Option) (sdklog.Exporter, error) { return nopLogExporter{}, nil }) + require.NoError(t, err) + + m, err := newTelemetryManager(telemetryIngress(t, true).TelemetryIngress(), client, ksmocks.NewCSA(t), "abc123", lggr) + require.NoError(t, err) + assert.IsType(t, &telemetry.DomainManager{}, m) + }) +} From 35f173ff79b286a3434b31d7234c313379df982c Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:46:37 -0400 Subject: [PATCH 05/23] docs: BeholderEnabled ignores TelemetryIngress sending settings; needs no-ack Co-Authored-By: Claude Opus 5.5 --- core/config/docs/core.toml | 7 +++++-- docs/CONFIG.md | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 513bb15a658..f30d0b68c53 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -116,8 +116,11 @@ SendTimeout = '10s' # Default UseBatchSend = true # Default # ChipIngressEnabled enables sending telemetry to CHIP Ingress. ChipIngressEnabled = false # Default -# BeholderEnabled sends telemetry through Beholder instead of the Endpoints below, which are then ignored. -# Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint. Cannot be combined with ChipIngressEnabled. +# BeholderEnabled sends telemetry through Beholder, one emitter per product domain, instead of the Endpoints below. +# When set, Endpoints and the settings above (BufferSize, MaxBatchSize, SendInterval, SendTimeout, UseBatchSend, UniConn, Logging) +# are ignored: the Telemetry.ChipIngress* settings size one buffer per domain, shared by all of that domain's jobs. +# Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint, and chip ingress that serves PublishBatchNoAck for every domain +# (there is no fallback). Cannot be combined with ChipIngressEnabled. BeholderEnabled = false # Default [[TelemetryIngress.Endpoints]] # Example diff --git a/docs/CONFIG.md b/docs/CONFIG.md index cce65db70cb..6640af4bbfe 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -337,8 +337,11 @@ ChipIngressEnabled enables sending telemetry to CHIP Ingress. ```toml BeholderEnabled = false # Default ``` -BeholderEnabled sends telemetry through Beholder instead of the Endpoints below, which are then ignored. -Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint. Cannot be combined with ChipIngressEnabled. +BeholderEnabled sends telemetry through Beholder, one emitter per product domain, instead of the Endpoints below. +When set, Endpoints and the settings above (BufferSize, MaxBatchSize, SendInterval, SendTimeout, UseBatchSend, UniConn, Logging) +are ignored: the Telemetry.ChipIngress* settings size one buffer per domain, shared by all of that domain's jobs. +Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint, and chip ingress that serves PublishBatchNoAck for every domain +(there is no fallback). Cannot be combined with ChipIngressEnabled. ## TelemetryIngress.Endpoints ```toml From 59e32e52488a088d1544e1555d9b661ba848206b Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:51:05 -0400 Subject: [PATCH 06/23] Fix golangci-lint v2.15.0 findings (gci, shadow, testifylint, whitespace) Co-Authored-By: Claude Opus 5.5 --- core/cmd/beholder_client_test.go | 5 ++--- .../chainlink/config_telemetry_ingress_test.go | 3 +-- core/services/chainlink/telemetry_manager.go | 1 - core/services/chainlink/telemetry_manager_test.go | 1 - core/services/telemetry/domain_manager.go | 6 +++--- .../services/telemetry/domain_manager_beholder_test.go | 2 -- core/services/telemetry/domain_manager_test.go | 10 +++++----- 7 files changed, 11 insertions(+), 17 deletions(-) diff --git a/core/cmd/beholder_client_test.go b/core/cmd/beholder_client_test.go index 4c902686650..e657de66dda 100644 --- a/core/cmd/beholder_client_test.go +++ b/core/cmd/beholder_client_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/beholder" - "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" "github.com/smartcontractkit/chainlink/v2/core/services/keystore/mocks" @@ -45,8 +44,8 @@ ChipIngressInsecureConnection = true t.Cleanup(func() { // OTel shutdown may fail to flush to the absent collector; the domain emitters // must close cleanly. - if err := client.Close(); err != nil { - assert.NotContains(t, err.Error(), "client connection is closing") + if closeErr := client.Close(); closeErr != nil { + assert.NotContains(t, closeErr.Error(), "client connection is closing") } }) diff --git a/core/services/chainlink/config_telemetry_ingress_test.go b/core/services/chainlink/config_telemetry_ingress_test.go index a6500e7867d..228df4cd6b2 100644 --- a/core/services/chainlink/config_telemetry_ingress_test.go +++ b/core/services/chainlink/config_telemetry_ingress_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/require" commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" - "github.com/smartcontractkit/chainlink/v2/core/config/toml" ) @@ -97,7 +96,7 @@ ChipIngressEndpoint = 'chip.example.com:443' var c Config require.NoError(t, commonconfig.DecodeTOML(strings.NewReader(tt.toml), &c)) c.setDefaults() - err := c.Core.ValidateConfig() + err := c.ValidateConfig() if tt.wantErr == "" { if err != nil { assert.NotContains(t, err.Error(), "TelemetryIngress.BeholderEnabled") diff --git a/core/services/chainlink/telemetry_manager.go b/core/services/chainlink/telemetry_manager.go index 3a6f6912925..1c8fb3d1e72 100644 --- a/core/services/chainlink/telemetry_manager.go +++ b/core/services/chainlink/telemetry_manager.go @@ -6,7 +6,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/beholder" commonsrv "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink/v2/core/config" "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/keystore" diff --git a/core/services/chainlink/telemetry_manager_test.go b/core/services/chainlink/telemetry_manager_test.go index b111b6cf238..b528e47b279 100644 --- a/core/services/chainlink/telemetry_manager_test.go +++ b/core/services/chainlink/telemetry_manager_test.go @@ -10,7 +10,6 @@ import ( sdklog "go.opentelemetry.io/otel/sdk/log" "github.com/smartcontractkit/chainlink-common/pkg/beholder" - "github.com/smartcontractkit/chainlink/v2/core/logger" ksmocks "github.com/smartcontractkit/chainlink/v2/core/services/keystore/mocks" "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index c9bdfe6d2e2..909765795d3 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -11,18 +11,18 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - chainselector "github.com/smartcontractkit/chain-selectors" - "github.com/smartcontractkit/libocr/commontypes" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" otelmetric "go.opentelemetry.io/otel/metric" + chainselector "github.com/smartcontractkit/chain-selectors" + "github.com/smartcontractkit/libocr/commontypes" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/chipingress" "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) diff --git a/core/services/telemetry/domain_manager_beholder_test.go b/core/services/telemetry/domain_manager_beholder_test.go index c7bc4f21b5b..c20ce57acad 100644 --- a/core/services/telemetry/domain_manager_beholder_test.go +++ b/core/services/telemetry/domain_manager_beholder_test.go @@ -17,7 +17,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" - "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) @@ -139,5 +138,4 @@ func TestDomainManager_ThroughRealBeholderClient(t *testing.T) { } assert.Zero(t, counterValue(t, reader, metricMessagesSendErrors)) assert.Zero(t, counterValue(t, reader, metricMessagesDropped)) - } diff --git a/core/services/telemetry/domain_manager_test.go b/core/services/telemetry/domain_manager_test.go index 9834a6c4b91..bafea133e0b 100644 --- a/core/services/telemetry/domain_manager_test.go +++ b/core/services/telemetry/domain_manager_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus/testutil" - chainselectors "github.com/smartcontractkit/chain-selectors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel/attribute" @@ -19,11 +18,12 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + chainselectors "github.com/smartcontractkit/chain-selectors" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/chipingress" "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) @@ -411,9 +411,9 @@ func TestDomainManager_AlsoCountsInPrometheus(t *testing.T) { m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) } - assert.Equal(t, sent+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesSent.WithLabelValues(endpoint, tt))) - assert.Equal(t, sendErrs+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesSendErrors.WithLabelValues(endpoint, tt))) - assert.Equal(t, dropped+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesDropped.WithLabelValues(endpoint, tt))) + assert.InDelta(t, sent+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesSent.WithLabelValues(endpoint, tt)), 0) + assert.InDelta(t, sendErrs+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesSendErrors.WithLabelValues(endpoint, tt)), 0) + assert.InDelta(t, dropped+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesDropped.WithLabelValues(endpoint, tt)), 0) } // Beholder domain emitters reject events whose source is not under their domain, so every From 8eedadcdf0bc687c8d4679e0618b7a670c31fa8d Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:32:01 -0400 Subject: [PATCH 07/23] Fix CI: test fake implements PublishBatchNoAck; go mod tidy The chipingress bump added PublishBatchNoAck to chipingress.Client; the new tests import otel log packages directly. Co-Authored-By: Claude Opus 5.5 --- .../synchronization/chip_ingress_batch_worker_test.go | 4 ++++ go.mod | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/services/synchronization/chip_ingress_batch_worker_test.go b/core/services/synchronization/chip_ingress_batch_worker_test.go index 8dc68ad5fbd..742b1fdea19 100644 --- a/core/services/synchronization/chip_ingress_batch_worker_test.go +++ b/core/services/synchronization/chip_ingress_batch_worker_test.go @@ -25,6 +25,10 @@ func (noopChipIngressPublisher) PublishBatch(ctx context.Context, batch *pb.Clou return &pb.PublishResponse{}, nil } +func (noopChipIngressPublisher) PublishBatchNoAck(ctx context.Context, batch *pb.CloudEventBatch, opts ...grpc.CallOption) (*pb.PublishResponse, error) { + return &pb.PublishResponse{}, nil +} + func (noopChipIngressPublisher) Ping(ctx context.Context, req *pb.EmptyRequest, opts ...grpc.CallOption) (*pb.PingResponse, error) { return &pb.PingResponse{}, nil } diff --git a/go.mod b/go.mod index f445a0e6353..d2edfa09be8 100644 --- a/go.mod +++ b/go.mod @@ -124,8 +124,10 @@ require ( go.opentelemetry.io/contrib/bridges/prometheus v0.68.0 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0 go.opentelemetry.io/otel v1.46.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 go.opentelemetry.io/otel/log v0.22.0 go.opentelemetry.io/otel/metric v1.46.0 + go.opentelemetry.io/otel/sdk/log v0.22.0 go.opentelemetry.io/otel/sdk/metric v1.46.0 go.opentelemetry.io/otel/trace v1.46.0 go.uber.org/atomic v1.11.0 @@ -374,7 +376,6 @@ require ( go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 // indirect @@ -385,7 +386,6 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 // indirect go.opentelemetry.io/otel/sdk v1.46.0 // indirect - go.opentelemetry.io/otel/sdk/log v0.22.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect From 67963782d95857404cb2ec110cffa5d91803a88c Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:37:04 -0400 Subject: [PATCH 08/23] DomainManager: explicit Beholder client, loud refusals, core metric names - AppFactory.NewApplication takes the Shell's Beholder client explicitly instead of reading the global - Log the first refused/unbuildable event per route and error code at Error - Warn once for unsupported types on multitype endpoints; cache the option - Precompute per-endpoint partitionkey and chain selector strings - Rename metrics to core convention (telemetry_messages_*_total); note the Prometheus counter semantics next to the code Co-Authored-By: Claude Opus 5.5 --- core/cmd/shell.go | 9 +- core/cmd/shell_local.go | 6 +- core/internal/cltest/mocks.go | 5 +- core/services/telemetry/domain_manager.go | 136 ++++++++++++------ .../services/telemetry/domain_manager_test.go | 43 ++++++ 5 files changed, 142 insertions(+), 57 deletions(-) diff --git a/core/cmd/shell.go b/core/cmd/shell.go index f96a14e9953..1d7b8caa405 100644 --- a/core/cmd/shell.go +++ b/core/cmd/shell.go @@ -271,14 +271,14 @@ func (s *Shell) configExitErr(validateFn func() error) cli.ExitCoder { // AppFactory implements the NewApplication method. type AppFactory interface { - NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master) (chainlink.Application, error) + NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master, beholderClient *beholder.Client) (chainlink.Application, error) } // ChainlinkAppFactory is used to create a new Application. type ChainlinkAppFactory struct{} // NewApplication returns a new instance of the node with the given config. -func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master) (app chainlink.Application, err error) { +func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master, beholderClient *beholder.Client) (app chainlink.Application, err error) { err = migrate.SetMigrationENVVars(cfg.EVMConfigs()) if err != nil { return nil, err @@ -316,9 +316,8 @@ func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.G } return chainlink.NewApplication(ctx, chainlink.ApplicationOpts{ - Opts: creOpts, - // Set as the global client in BeforeNode; NewApplication takes it explicitly. - BeholderClient: beholder.GetClient(), + Opts: creOpts, + BeholderClient: beholderClient, Config: cfg, DS: ds, KeyStore: keyStore, diff --git a/core/cmd/shell_local.go b/core/cmd/shell_local.go index 2e32712b590..77ae6d195ab 100644 --- a/core/cmd/shell_local.go +++ b/core/cmd/shell_local.go @@ -428,7 +428,7 @@ func (s *Shell) runNode(c *cli.Context) error { os.Exit(-1) }) - app, err := s.AppFactory.NewApplication(rootCtx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore) + app, err := s.AppFactory.NewApplication(rootCtx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore, s.BeholderClient) if err != nil { return s.errorOut(fmt.Errorf("fatal error instantiating application: %w", err)) } @@ -780,7 +780,7 @@ func (s *Shell) RebroadcastTransactions(c *cli.Context) (err error) { lggr := logger.Sugared(s.Logger.Named("RebroadcastTransactions")) - app, err := s.AppFactory.NewApplication(ctx, s.Config, lggr, s.Registerer, s.DS, s.KeyStore) + app, err := s.AppFactory.NewApplication(ctx, s.Config, lggr, s.Registerer, s.DS, s.KeyStore, s.BeholderClient) if err != nil { return s.errorOut(fmt.Errorf("fatal error instantiating application: %w", err)) } @@ -1145,7 +1145,7 @@ func (s *Shell) RemoveBlocks(c *cli.Context) error { s.afterNode(lggr) }) - app, err := s.AppFactory.NewApplication(ctx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore) + app, err := s.AppFactory.NewApplication(ctx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore, s.BeholderClient) if err != nil { return s.errorOut(fmt.Errorf("fatal error instantiating application: %w", err)) } diff --git a/core/internal/cltest/mocks.go b/core/internal/cltest/mocks.go index 3a6af54bc19..81d68f27a7d 100644 --- a/core/internal/cltest/mocks.go +++ b/core/internal/cltest/mocks.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-evm/pkg/chains/legacyevm" @@ -90,7 +91,7 @@ type InstanceAppFactory struct { } // NewApplication creates a new application with specified config -func (f InstanceAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master) (chainlink.Application, error) { +func (f InstanceAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master, *beholder.Client) (chainlink.Application, error) { return f.App, nil } @@ -98,7 +99,7 @@ type seededAppFactory struct { Application chainlink.Application } -func (s seededAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master) (chainlink.Application, error) { +func (s seededAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master, *beholder.Client) (chainlink.Application, error) { return noopStopApplication{s.Application}, nil } diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index 909765795d3..0126922e2dc 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -56,13 +56,18 @@ func supportedTelemetryDomains() map[synchronization.TelemetryType]beholder.Doma } // Metric names and attribute values. The manager's metrics are OTel (Beholder) metrics, so they -// reach us from every node; the existing Prometheus telemetry counters are incremented too, so -// node-local dashboards keep working. +// reach us from every node. +// +// It also increments the existing Prometheus telemetry_client_messages_{sent,send_errors,dropped} +// counters for node-local dashboards, with two differences from the WSRPC path: they count +// messages, where the WSRPC batch worker counts batches, and their endpoint label is +// "beholder/" rather than the ingress URL. Unsupported types and chains are counted only +// in OTel (telemetry_messages_unsupported_total). const ( - metricMessagesSent = "telemetry.messages_sent" - metricMessagesSendErrors = "telemetry.messages_send_errors" - metricMessagesDropped = "telemetry.messages_dropped" - metricMessagesUnsupported = "telemetry.messages_unsupported" + metricMessagesSent = "telemetry_messages_sent_total" + metricMessagesSendErrors = "telemetry_messages_send_errors_total" + metricMessagesDropped = "telemetry_messages_dropped_total" + metricMessagesUnsupported = "telemetry_messages_unsupported_total" unsupportedReasonTelemetryType = "telemetry_type" unsupportedReasonChain = "chain" @@ -83,12 +88,13 @@ type DomainManager struct { services.Service eng *services.Engine - routes map[synchronization.TelemetryType]*telemetryRoute - csaPubKeyHex string - meterProvider otelmetric.MeterProvider - metrics domainManagerMetrics - now func() time.Time - reported sync.Map // keys of unsupported types and chains already logged + routes map[synchronization.TelemetryType]*telemetryRoute + csaPubKeyHex string + meterProvider otelmetric.MeterProvider + metrics domainManagerMetrics + now func() time.Time + reported sync.Map // keys of unsupported types, chains and refusals already logged + unsupportedOpts sync.Map // "/" -> otelmetric.AddOption } // telemetryRoute is where one telemetry type goes, with its metric handles precomputed. @@ -219,16 +225,14 @@ func newDomainManagerMetrics(mp otelmetric.MeterProvider) (m domainManagerMetric func (m *DomainManager) GenMonitoringEndpoint(network string, chainID string, contractID string, telemType synchronization.TelemetryType) commontypes.MonitoringEndpoint { route, ok := m.routes[telemType] if !ok { - m.reportOnce("type:"+string(telemType), func() { - m.eng.Warnw("Telemetry type is not supported: its telemetry will be dropped", "telemType", telemType) - }) + m.warnUnsupportedType(telemType) return &unsupportedAgent{m: m, telemType: telemType, reason: unsupportedReasonTelemetryType} } selector, ok := m.chainSelector(network, chainID) if !ok { return &unsupportedAgent{m: m, telemType: telemType, reason: unsupportedReasonChain} } - return &domainAgent{m: m, route: route, network: strings.ToLower(network), contractID: contractID, selector: selector} + return &domainAgent{m: m, route: route, target: m.newEventTarget(network, contractID, selector)} } // GenMultitypeMonitoringEndpoint returns an endpoint whose messages carry their own type. @@ -237,7 +241,24 @@ func (m *DomainManager) GenMultitypeMonitoringEndpoint(network string, chainID s if !ok { return &unsupportedAgent{m: m, reason: unsupportedReasonChain} } - return &multitypeDomainAgent{m: m, network: strings.ToLower(network), contractID: contractID, selector: selector} + return &multitypeDomainAgent{m: m, target: m.newEventTarget(network, contractID, selector)} +} + +// eventTarget holds an endpoint's per-event values, computed once when the endpoint is created. +type eventTarget struct { + network string // lower case, e.g. "evm" + contractID string + selector string // decimal chain selector + partitionKey string // -, the Kafka record key, as OTI's +} + +func (m *DomainManager) newEventTarget(network, contractID string, selector uint64) eventTarget { + return eventTarget{ + network: strings.ToLower(network), + contractID: contractID, + selector: strconv.FormatUint(selector, 10), + partitionKey: contractID + "-" + m.csaPubKeyHex, + } } // chainSelector resolves the job's chain to its chain-selectors selector, which legacy-mode chip @@ -260,21 +281,32 @@ func (m *DomainManager) reportOnce(key string, report func()) { } } -// send builds the event and hands it to the route's emitter. Fates are counted; the emitter -// rate-limits its own failure logs. -func (m *DomainManager) send(route *telemetryRoute, network, contractID string, selector uint64, telemetry []byte) { +// send builds the event and hands it to the route's emitter. Fates are counted; publish failures +// are logged (rate-limited) by the emitter, and events that never reach it are logged here, once +// per route and error code, since they mean telemetry is being lost. +func (m *DomainManager) send(route *telemetryRoute, target eventTarget, telemetry []byte) { // SendLog has no context (libocr's MonitoringEndpoint); the emitter only queues. ctx := context.Background() - ev, err := m.newEvent(route, network, contractID, selector, telemetry) + ev, err := m.newEvent(route, target, telemetry) if err != nil { - m.recordDropped(ctx, route, errorCodeEventBuild) + m.refused(ctx, route, errorCodeEventBuild, err) return } if err := route.emitter.EmitEvent(ctx, ev, func(sendErr error) { m.recordFate(ctx, route, sendErr) }); err != nil { - m.recordDropped(ctx, route, errorCodeEmitRejected) + m.refused(ctx, route, errorCodeEmitRejected, err) } } +// refused counts an event that never reached the emitter's buffer and logs the first one per +// route and error code at Error. +func (m *DomainManager) refused(ctx context.Context, route *telemetryRoute, code string, err error) { + m.recordDropped(ctx, route, code) + m.reportOnce("refused:"+string(route.telemType)+"/"+code, func() { + m.eng.Errorw("Telemetry refused: it is being dropped (further refusals are only counted)", + "telemType", route.telemType, "domain", route.domain, "errorCode", code, "err", err) + }) +} + // recordFate counts an event's fate as reported by the emitter: published, failed to publish, // or dropped before publishing (full buffer or shutdown). func (m *DomainManager) recordFate(ctx context.Context, route *telemetryRoute, err error) { @@ -305,17 +337,30 @@ func (r *telemetryRoute) errorOpt(code string) otelmetric.AddOption { return v.(otelmetric.AddOption) } +// countUnsupported counts a dropped message whose type or chain is not supported. The metric +// option is cached per (type, reason): unsupported LLO types arrive on the hot path. func (m *DomainManager) countUnsupported(telemType synchronization.TelemetryType, reason string) { - m.metrics.unsupported.Add(context.Background(), 1, otelmetric.WithAttributes( - attribute.String("telemetry_type", string(telemType)), - attribute.String("reason", reason), - )) + key := string(telemType) + "/" + reason + v, ok := m.unsupportedOpts.Load(key) + if !ok { + v, _ = m.unsupportedOpts.LoadOrStore(key, otelmetric.WithAttributes( + attribute.String("telemetry_type", string(telemType)), + attribute.String("reason", reason), + )) + } + m.metrics.unsupported.Add(context.Background(), 1, v.(otelmetric.AddOption)) +} + +func (m *DomainManager) warnUnsupportedType(telemType synchronization.TelemetryType) { + m.reportOnce("type:"+string(telemType), func() { + m.eng.Warnw("Telemetry type is not supported: its telemetry will be dropped", "telemType", telemType) + }) } // newEvent builds the CloudEvent legacy-mode chip ingress expects. Each extension becomes a // "ce_" Kafka header; the comments name the header OTI used for the same value. Chip // ingress derives nodeoperatorname/nodename (from the CSA key's NOP info) and receivedat itself. -func (m *DomainManager) newEvent(route *telemetryRoute, network, contractID string, selector uint64, telemetry []byte) (chipingress.CloudEvent, error) { +func (m *DomainManager) newEvent(route *telemetryRoute, target eventTarget, telemetry []byte) (chipingress.CloudEvent, error) { now := m.now() event, err := chipingress.NewEvent(route.source, route.entity, telemetry, map[string]any{ "time": now.UTC(), @@ -324,45 +369,42 @@ func (m *DomainManager) newEvent(route *telemetryRoute, network, contractID stri if err != nil { return event, fmt.Errorf("failed creating CloudEvent: %w", err) } - event.SetExtension("legacytelemetry", "true") // marks node-sent legacy telemetry - event.SetExtension("telemetrytype", string(route.telemType)) // "telemetry-type" - event.SetExtension("chainselector", strconv.FormatUint(selector, 10)) // chain-selectors ID of the job's chain - event.SetExtension("networkname", network) // lower case, e.g. "evm" - event.SetExtension("contractid", contractID) // "contract-address" - event.SetExtension("csapublickey", m.csaPubKeyHex) // "node-operator-csa-public-key" - event.SetExtension("partitionkey", contractID+"-"+m.csaPubKeyHex) // Kafka record key, as OTI's - event.SetExtension("sentat", strconv.FormatInt(now.UnixNano(), 10)) // "sent-at", unix ns as over WSRPC + event.SetExtension("legacytelemetry", "true") // marks node-sent legacy telemetry + event.SetExtension("telemetrytype", string(route.telemType)) // "telemetry-type" + event.SetExtension("chainselector", target.selector) // chain-selectors ID of the job's chain + event.SetExtension("networkname", target.network) // lower case, e.g. "evm" + event.SetExtension("contractid", target.contractID) // "contract-address" + event.SetExtension("csapublickey", m.csaPubKeyHex) // "node-operator-csa-public-key" + event.SetExtension("partitionkey", target.partitionKey) // Kafka record key, as OTI's + event.SetExtension("sentat", strconv.FormatInt(now.UnixNano(), 10)) // "sent-at", unix ns as over WSRPC return event, nil } // domainAgent sends one telemetry type through its domain's emitter. type domainAgent struct { - m *DomainManager - route *telemetryRoute - network string - contractID string - selector uint64 + m *DomainManager + route *telemetryRoute + target eventTarget } func (a *domainAgent) SendLog(telemetry []byte) { - a.m.send(a.route, a.network, a.contractID, a.selector, telemetry) + a.m.send(a.route, a.target, telemetry) } // multitypeDomainAgent routes each message by the type it carries. type multitypeDomainAgent struct { - m *DomainManager - network string - contractID string - selector uint64 + m *DomainManager + target eventTarget } func (a *multitypeDomainAgent) SendTypedLog(telemType synchronization.TelemetryType, telemetry []byte) { route, ok := a.m.routes[telemType] if !ok { + a.m.warnUnsupportedType(telemType) a.m.countUnsupported(telemType, unsupportedReasonTelemetryType) return } - a.m.send(route, a.network, a.contractID, a.selector, telemetry) + a.m.send(route, a.target, telemetry) } // unsupportedAgent drops everything it is given and counts it. telemType is empty for a diff --git a/core/services/telemetry/domain_manager_test.go b/core/services/telemetry/domain_manager_test.go index bafea133e0b..626359c7798 100644 --- a/core/services/telemetry/domain_manager_test.go +++ b/core/services/telemetry/domain_manager_test.go @@ -15,6 +15,8 @@ import ( "go.opentelemetry.io/otel/attribute" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -427,3 +429,44 @@ func TestSupportedTelemetry_SourcesSitUnderTheirDomain(t *testing.T) { "%s: source %q is not under domain %q", telemType, source, d) } } + +func newObservedDomainManager(t *testing.T, src DomainEmitterSource) (*DomainManager, *observer.ObservedLogs) { + t.Helper() + lggr, logs := logger.TestObserved(t, zapcore.WarnLevel) + m, err := NewDomainManager(src, testCSAPubKeyHex, lggr, + WithMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(sdkmetric.NewManualReader())))) + require.NoError(t, err) + return m, logs +} + +// A refused event means telemetry is being lost, so the first refusal per route and error code +// is logged at Error; repeats are only counted. +func TestDomainManager_LogsTheFirstRefusalPerRouteAtError(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + src[beholder.DomainDataFeeds].emitErr = errors.New("not started") + m, logs := newObservedDomainManager(t, src) + + ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median) + for range 3 { + ep.SendLog([]byte("p")) + } + + refusals := logs.FilterLevelExact(zapcore.ErrorLevel).FilterMessageSnippet("refused").All() + require.Len(t, refusals, 1, "logged once, however many events are refused") + assert.Equal(t, synchronization.OCR2Median, refusals[0].ContextMap()["telemType"]) +} + +func TestDomainManager_WarnsOnceForAnUnsupportedTypeOnAMultitypeEndpoint(t *testing.T) { + t.Parallel() + m, logs := newObservedDomainManager(t, newFakeEmitterSource()) + + ep := m.GenMultitypeMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc/1") + for range 3 { + ep.SendTypedLog(synchronization.LLOReport, []byte("p")) + } + + warnings := logs.FilterLevelExact(zapcore.WarnLevel).FilterMessageSnippet("not supported").All() + require.Len(t, warnings, 1) + assert.Equal(t, synchronization.LLOReport, warnings[0].ContextMap()["telemType"]) +} From 0b58cf0c05e4715a46c1cf7f36a38ba66e34fa17 Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:40:02 -0400 Subject: [PATCH 09/23] Tests: fast real-client tests, buffer-full end to end, exact counts after Close - Real-client tests point OTel at the local server (10s -> <1s) and close via t.Cleanup; per-event header lookup cannot panic - Buffer-full through a real Beholder client: dropped, never send errors; sent + dropped == total and sent == received after Close - Helper test covers the no-op client (Beholder off) and closes its client Co-Authored-By: Claude Opus 5.5 --- core/cmd/beholder_client_test.go | 17 +- .../chainlink/telemetry_manager_test.go | 20 ++- .../telemetry/domain_manager_beholder_test.go | 153 +++++++++++++----- 3 files changed, 144 insertions(+), 46 deletions(-) diff --git a/core/cmd/beholder_client_test.go b/core/cmd/beholder_client_test.go index e657de66dda..9c2330249bb 100644 --- a/core/cmd/beholder_client_test.go +++ b/core/cmd/beholder_client_test.go @@ -1,10 +1,13 @@ package cmd import ( + "fmt" + "net" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/grpc" "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink/v2/core/logger" @@ -14,14 +17,20 @@ import ( func TestNewBeholderClient_DomainEmittersFollowTelemetryIngressBeholderEnabled(t *testing.T) { t.Parallel() - const beholderTOML = ` + // A local gRPC server with no services answers OTel exports at once, so Close is quick. + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer() + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + beholderTOML := fmt.Sprintf(` [Telemetry] Enabled = true -Endpoint = 'localhost:4317' +Endpoint = '%s' InsecureConnection = true -ChipIngressEndpoint = 'localhost:9999' +ChipIngressEndpoint = '%s' ChipIngressInsecureConnection = true -` +`, lis.Addr(), lis.Addr()) for _, tt := range []struct { name string toml string diff --git a/core/services/chainlink/telemetry_manager_test.go b/core/services/chainlink/telemetry_manager_test.go index b528e47b279..08792d7345d 100644 --- a/core/services/chainlink/telemetry_manager_test.go +++ b/core/services/chainlink/telemetry_manager_test.go @@ -2,12 +2,14 @@ package chainlink import ( "context" + "net" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" sdklog "go.opentelemetry.io/otel/sdk/log" + "google.golang.org/grpc" "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink/v2/core/logger" @@ -47,16 +49,30 @@ func TestNewTelemetryManager(t *testing.T) { _, err := newTelemetryManager(telemetryIngress(t, true).TelemetryIngress(), noClient, ksmocks.NewCSA(t), "abc123", lggr) require.Error(t, err) }) + t.Run("flag on with Beholder telemetry off (the no-op client): an error", func(t *testing.T) { + t.Parallel() + _, err := newTelemetryManager(telemetryIngress(t, true).TelemetryIngress(), beholder.NewNoopClient(), ksmocks.NewCSA(t), "abc123", lggr) + require.Error(t, err, "the no-op client has no domain emitters") + }) t.Run("flag on: the domain manager on Beholder's domain emitters", func(t *testing.T) { t.Parallel() + // A local gRPC server with no services answers OTel exports at once, so Close is quick. + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer() + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + client, err := beholder.NewGRPCClient(beholder.Config{ - OtelExporterGRPCEndpoint: "localhost:4317", + OtelExporterGRPCEndpoint: lis.Addr().String(), InsecureConnection: true, - ChipIngressEmitterGRPCEndpoint: "localhost:9999", // never dialed: nothing is sent + ChipIngressEmitterGRPCEndpoint: lis.Addr().String(), ChipIngressInsecureConnection: true, ChipIngressDomainEmittersEnabled: true, }, func(...otlploggrpc.Option) (sdklog.Exporter, error) { return nopLogExporter{}, nil }) require.NoError(t, err) + require.NoError(t, client.Start(t.Context())) + t.Cleanup(func() { _ = client.Close() }) m, err := newTelemetryManager(telemetryIngress(t, true).TelemetryIngress(), client, ksmocks.NewCSA(t), "abc123", lggr) require.NoError(t, err) diff --git a/core/services/telemetry/domain_manager_beholder_test.go b/core/services/telemetry/domain_manager_beholder_test.go index c20ce57acad..0203cb7a06c 100644 --- a/core/services/telemetry/domain_manager_beholder_test.go +++ b/core/services/telemetry/domain_manager_beholder_test.go @@ -10,35 +10,53 @@ import ( cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" sdklog "go.opentelemetry.io/otel/sdk/log" "google.golang.org/grpc" "google.golang.org/grpc/metadata" "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" + "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) -type capturedBatch struct { +// receivedEvent is one event as chip ingress saw it. +type receivedEvent struct { method string - md metadata.MD - events []*cepb.CloudEvent + domain string // the chainlink-domain header, "" if absent + event *cepb.CloudEvent } -// capturingChipServer records PublishBatch / PublishBatchNoAck calls and answers each event -// with a positional success, as chip ingress does. +// capturingChipServer records PublishBatch / PublishBatchNoAck calls and answers each event with +// a positional success, as chip ingress does. With hold set, it answers nothing until released. type capturingChipServer struct { pb.UnimplementedChipIngressServer + hold chan struct{} - mu sync.Mutex - batches []capturedBatch + mu sync.Mutex + events []receivedEvent } func (s *capturingChipServer) record(ctx context.Context, method string, in *pb.CloudEventBatch) (*pb.PublishResponse, error) { + if s.hold != nil { + select { + case <-s.hold: + case <-ctx.Done(): + return nil, ctx.Err() + } + } md, _ := metadata.FromIncomingContext(ctx) + domain := "" + if v := md.Get(beholder.DomainHeaderKey); len(v) > 0 { + domain = v[0] + } s.mu.Lock() - s.batches = append(s.batches, capturedBatch{method: method, md: md, events: in.Events}) + for _, ev := range in.Events { + s.events = append(s.events, receivedEvent{method: method, domain: domain, event: ev}) + } s.mu.Unlock() return &pb.PublishResponse{Results: make([]*pb.PublishResult, len(in.Events))}, nil } @@ -51,18 +69,10 @@ func (s *capturingChipServer) PublishBatchNoAck(ctx context.Context, in *pb.Clou return s.record(ctx, "PublishBatchNoAck", in) } -// received returns one entry per event: the RPC method, its chainlink-domain header and the event. -func (s *capturingChipServer) received() (methods, domains []string, events []*cepb.CloudEvent) { +func (s *capturingChipServer) received() []receivedEvent { s.mu.Lock() defer s.mu.Unlock() - for _, b := range s.batches { - for _, ev := range b.events { - methods = append(methods, b.method) - domains = append(domains, b.md.Get(beholder.DomainHeaderKey)...) - events = append(events, ev) - } - } - return methods, domains, events + return append([]receivedEvent(nil), s.events...) } type nopLogExporter struct{} @@ -71,29 +81,64 @@ func (nopLogExporter) Export(context.Context, []sdklog.Record) error { return ni func (nopLogExporter) Shutdown(context.Context) error { return nil } func (nopLogExporter) ForceFlush(context.Context) error { return nil } -func TestDomainManager_ThroughRealBeholderClient(t *testing.T) { - t.Parallel() +// startChipServer serves chip on a local listener and returns its address. +func startChipServer(t *testing.T, chip *capturingChipServer) string { + t.Helper() lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") require.NoError(t, err) srv := grpc.NewServer() - chip := &capturingChipServer{} pb.RegisterChipIngressServer(srv, chip) go func() { _ = srv.Serve(lis) }() t.Cleanup(srv.Stop) + return lis.Addr().String() +} - client, err := beholder.NewGRPCClient(beholder.Config{ - OtelExporterGRPCEndpoint: "localhost:4317", // never reached: logs go to nopLogExporter +// startBeholderClient starts a real Beholder client with domain emitters against addr. Its OTel +// exporters also point at addr, so shutdown gets a prompt answer instead of dialing an absent +// collector. closeClient closes it once and returns Close's error; cleanup closes it otherwise. +func startBeholderClient(t *testing.T, addr string, tune func(*beholder.Config)) (client *beholder.Client, closeClient func() error) { + t.Helper() + cfg := beholder.Config{ + OtelExporterGRPCEndpoint: addr, InsecureConnection: true, ChipIngressEmitterEnabled: true, - ChipIngressEmitterGRPCEndpoint: lis.Addr().String(), + ChipIngressEmitterGRPCEndpoint: addr, ChipIngressInsecureConnection: true, ChipIngressSendInterval: 10 * time.Millisecond, ChipIngressDomainEmittersEnabled: true, AuthHeaders: map[string]string{"X-Beholder-Node-Auth-Token": "1:abc:2:def"}, - }, func(...otlploggrpc.Option) (sdklog.Exporter, error) { return nopLogExporter{}, nil }) + } + if tune != nil { + tune(&cfg) + } + client, err := beholder.NewGRPCClient(cfg, func(...otlploggrpc.Option) (sdklog.Exporter, error) { return nopLogExporter{}, nil }) require.NoError(t, err) require.NoError(t, client.Start(t.Context())) + var once sync.Once + var closeErr error + closeClient = func() error { + once.Do(func() { closeErr = client.Close() }) + return closeErr + } + t.Cleanup(func() { _ = closeClient() }) + return client, closeClient +} + +// requireCleanClose asserts that closing the client did not fail because of the domain emitters. +// OTel provider shutdown may still report export errors (the test server is not a collector). +func requireCleanClose(t *testing.T, err error) { + t.Helper() + if err != nil { + assert.NotContains(t, err.Error(), "client connection is closing", "domain emitter connections close once") + } +} + +func TestDomainManager_ThroughRealBeholderClient(t *testing.T) { + t.Parallel() + chip := &capturingChipServer{} + addr := startChipServer(t, chip) + client, closeClient := startBeholderClient(t, addr, nil) m, reader := newTestDomainManager(t, client) // Concurrent senders across two domains, through single-type and multitype endpoints. @@ -115,27 +160,55 @@ func TestDomainManager_ThroughRealBeholderClient(t *testing.T) { require.Eventually(t, func() bool { return counterValue(t, reader, metricMessagesSent) == total }, 10*time.Second, 20*time.Millisecond, "every event is acknowledged by the per-event result") - methods, domains, events := chip.received() - require.Len(t, events, total) - for i, ev := range events { - assert.Equal(t, "PublishBatchNoAck", methods[i]) - switch ev.Source { + received := chip.received() + require.Len(t, received, total) + for _, r := range received { + assert.Equal(t, "PublishBatchNoAck", r.method) + switch r.event.Source { case "data-feeds.telemetry.ocr2-median": - assert.Equal(t, string(beholder.DomainDataFeeds), domains[i]) + assert.Equal(t, string(beholder.DomainDataFeeds), r.domain) case "data-streams.telemetry.llo-outcome": - assert.Equal(t, string(beholder.DomainDataStreams), domains[i]) + assert.Equal(t, string(beholder.DomainDataStreams), r.domain) default: - t.Fatalf("unexpected source %q", ev.Source) + t.Errorf("unexpected source %q", r.event.Source) } - assert.Equal(t, "true", ev.Attributes["legacytelemetry"].GetCeString()) - assert.Equal(t, testCSAPubKeyHex, ev.Attributes["csapublickey"].GetCeString()) + assert.Equal(t, "true", r.event.Attributes["legacytelemetry"].GetCeString()) + assert.Equal(t, testCSAPubKeyHex, r.event.Attributes["csapublickey"].GetCeString()) } - // Closing the client drains and closes the domain emitters exactly once. OTel provider - // shutdown may fail to flush to the absent collector; nothing else may fail. - if err := client.Close(); err != nil { - assert.NotContains(t, err.Error(), "client connection is closing") - } + requireCleanClose(t, closeClient()) + // Nothing is counted twice, and nothing else is counted, once the emitters have drained. + assert.Equal(t, int64(total), counterValue(t, reader, metricMessagesSent)) assert.Zero(t, counterValue(t, reader, metricMessagesSendErrors)) assert.Zero(t, counterValue(t, reader, metricMessagesDropped)) } + +func TestDomainManager_ThroughRealBeholderClient_BufferFull(t *testing.T) { + t.Parallel() + chip := &capturingChipServer{hold: make(chan struct{})} + addr := startChipServer(t, chip) + client, closeClient := startBeholderClient(t, addr, func(c *beholder.Config) { + c.ChipIngressBufferSize = 3 + c.ChipIngressMaxBatchSize = 1 + c.ChipIngressMaxConcurrentSends = 1 + }) + m, reader := newTestDomainManager(t, client) + + // The server holds every request, so the tiny buffer fills and the rest are dropped. + const total = 50 + ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median) + for range total { + ep.SendLog([]byte("p")) + } + bufferFull := attribute.String("error_code", batch.ErrorCodeFor(batch.ErrMessageBufferFull)) + require.Positive(t, counterValue(t, reader, metricMessagesDropped, bufferFull), "drops are counted as dropped, not send errors") + + close(chip.hold) // let the held and queued events through, then drain on close + requireCleanClose(t, closeClient()) + + sent := counterValue(t, reader, metricMessagesSent) + dropped := counterValue(t, reader, metricMessagesDropped) + assert.Equal(t, int64(total), sent+dropped, "every event is counted exactly once") + assert.Equal(t, sent, int64(len(chip.received())), "sent matches what the server received") + assert.Zero(t, counterValue(t, reader, metricMessagesSendErrors)) +} From bd92375c8c95534f239e221fbe9b6bfdfb4f7215 Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:40:56 -0400 Subject: [PATCH 10/23] docs: BeholderEnabled sends for every chain; list the supported telemetry types Co-Authored-By: Claude Opus 5.5 --- core/config/docs/core.toml | 3 +++ core/services/telemetry/domain_manager.go | 1 + docs/CONFIG.md | 3 +++ 3 files changed, 7 insertions(+) diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index f30d0b68c53..b1f07e0c1e3 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -121,6 +121,9 @@ ChipIngressEnabled = false # Default # are ignored: the Telemetry.ChipIngress* settings size one buffer per domain, shared by all of that domain's jobs. # Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint, and chip ingress that serves PublishBatchNoAck for every domain # (there is no fallback). Cannot be combined with ChipIngressEnabled. +# Telemetry is sent for every chain the node's jobs run on: Endpoints no longer select chains. Supported types are ocr, +# ocr2-median, ocr3-data-feeds, enhanced-ea, ocr3-mercury, enhanced-ea-mercury, llo-observation, llo-outcome, pipeline-bridge, +# ocr2-ccip-commit, ocr2-ccip-exec, ocr3-ccip-commit, ocr3-ccip-exec and head-report; other types are dropped. BeholderEnabled = false # Default [[TelemetryIngress.Endpoints]] # Example diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index 0126922e2dc..615544eb232 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -34,6 +34,7 @@ type DomainEmitterSource interface { // supportedTelemetryDomains lists the telemetry types DomainManager sends and the Beholder domain // whose emitter carries each: the types ingested in production. Their CloudEvent source and type // come from synchronization.TelemetryTypeToDomainAndEntity. Every other type is dropped and counted. +// The TelemetryIngress.BeholderEnabled docs (core/config/docs/core.toml) list these types: keep them in sync. func supportedTelemetryDomains() map[synchronization.TelemetryType]beholder.Domain { return map[synchronization.TelemetryType]beholder.Domain{ synchronization.OCR: beholder.DomainDataFeeds, diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 6640af4bbfe..5d276e5c090 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -342,6 +342,9 @@ When set, Endpoints and the settings above (BufferSize, MaxBatchSize, SendInterv are ignored: the Telemetry.ChipIngress* settings size one buffer per domain, shared by all of that domain's jobs. Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint, and chip ingress that serves PublishBatchNoAck for every domain (there is no fallback). Cannot be combined with ChipIngressEnabled. +Telemetry is sent for every chain the node's jobs run on: Endpoints no longer select chains. Supported types are ocr, +ocr2-median, ocr3-data-feeds, enhanced-ea, ocr3-mercury, enhanced-ea-mercury, llo-observation, llo-outcome, pipeline-bridge, +ocr2-ccip-commit, ocr2-ccip-exec, ocr3-ccip-commit, ocr3-ccip-exec and head-report; other types are dropped. ## TelemetryIngress.Endpoints ```toml From 10cb619294dcff7cb1d080d48148327ea43fbaa2 Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:44:48 -0400 Subject: [PATCH 11/23] Fix gci import grouping Co-Authored-By: Claude Opus 5.5 --- core/services/telemetry/domain_manager_beholder_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/services/telemetry/domain_manager_beholder_test.go b/core/services/telemetry/domain_manager_beholder_test.go index 0203cb7a06c..bb49f0f4ec1 100644 --- a/core/services/telemetry/domain_manager_beholder_test.go +++ b/core/services/telemetry/domain_manager_beholder_test.go @@ -19,7 +19,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" - "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) From 730ab996ee340db9f761f5414f72757ea0255078 Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:19:25 -0400 Subject: [PATCH 12/23] Re-pin chainlink-common#2504 (2cd02358): ChipDomainEmittersEnabled Follows the upstream rename of beholder Config.ChipIngressDomainEmittersEnabled to ChipDomainEmittersEnabled (chip ingress is an implementation detail). Test Beholder configs set ChipIngressLogger, now required with domain emitters. Co-Authored-By: Claude Opus 5.5 --- core/cmd/shell.go | 32 +++++++++---------- core/scripts/go.mod | 2 +- core/scripts/go.sum | 4 +-- .../chainlink/telemetry_manager_test.go | 11 ++++--- .../telemetry/domain_manager_beholder_test.go | 18 ++++++----- deployment/go.mod | 2 +- deployment/go.sum | 4 +-- go.mod | 2 +- go.sum | 4 +-- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 +-- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +-- system-tests/lib/go.mod | 2 +- system-tests/lib/go.sum | 4 +-- system-tests/tests/go.mod | 2 +- system-tests/tests/go.sum | 4 +-- 17 files changed, 53 insertions(+), 50 deletions(-) diff --git a/core/cmd/shell.go b/core/cmd/shell.go index 1d7b8caa405..91f179636a9 100644 --- a/core/cmd/shell.go +++ b/core/cmd/shell.go @@ -161,22 +161,22 @@ func newBeholderClient( ChipIngressInsecureConnection: cfgTelemetry.ChipIngressInsecureConnection(), ChipIngressBatchEmitterEnabled: cfgTelemetry.ChipIngressBatchEmitterEnabled(), // TelemetryIngress.BeholderEnabled sends node telemetry through the per-domain emitters. - ChipIngressDomainEmittersEnabled: cfgApp.TelemetryIngress().BeholderEnabled(), - ChipIngressBufferSize: cfgTelemetry.ChipIngressBufferSize(), - ChipIngressMaxBatchSize: cfgTelemetry.ChipIngressMaxBatchSize(), - ChipIngressMaxConcurrentSends: cfgTelemetry.ChipIngressMaxConcurrentSends(), - ChipIngressSendInterval: cfgTelemetry.ChipIngressSendInterval(), - ChipIngressSendTimeout: cfgTelemetry.ChipIngressSendTimeout(), - ChipIngressDrainTimeout: cfgTelemetry.ChipIngressDrainTimeout(), - ChipIngressMaxGRPCRequestSize: cfgTelemetry.ChipIngressMaxGRPCRequestSize(), - ChipIngressLogger: lggr, - LogStreamingEnabled: cfgTelemetry.LogStreamingEnabled(), - LogLevel: cfgTelemetry.LogLevel(), - LogBatchProcessor: cfgTelemetry.LogBatchProcessor(), - LogExportTimeout: cfgTelemetry.LogExportTimeout(), - LogExportMaxBatchSize: cfgTelemetry.LogExportMaxBatchSize(), - LogExportInterval: cfgTelemetry.LogExportInterval(), - LogMaxQueueSize: cfgTelemetry.LogMaxQueueSize(), + ChipDomainEmittersEnabled: cfgApp.TelemetryIngress().BeholderEnabled(), + ChipIngressBufferSize: cfgTelemetry.ChipIngressBufferSize(), + ChipIngressMaxBatchSize: cfgTelemetry.ChipIngressMaxBatchSize(), + ChipIngressMaxConcurrentSends: cfgTelemetry.ChipIngressMaxConcurrentSends(), + ChipIngressSendInterval: cfgTelemetry.ChipIngressSendInterval(), + ChipIngressSendTimeout: cfgTelemetry.ChipIngressSendTimeout(), + ChipIngressDrainTimeout: cfgTelemetry.ChipIngressDrainTimeout(), + ChipIngressMaxGRPCRequestSize: cfgTelemetry.ChipIngressMaxGRPCRequestSize(), + ChipIngressLogger: lggr, + LogStreamingEnabled: cfgTelemetry.LogStreamingEnabled(), + LogLevel: cfgTelemetry.LogLevel(), + LogBatchProcessor: cfgTelemetry.LogBatchProcessor(), + LogExportTimeout: cfgTelemetry.LogExportTimeout(), + LogExportMaxBatchSize: cfgTelemetry.LogExportMaxBatchSize(), + LogExportInterval: cfgTelemetry.LogExportInterval(), + LogMaxQueueSize: cfgTelemetry.LogMaxQueueSize(), // Due to OpenTelemetry semantics, histogram bucket boundaries must be set // when the Beholder client is constructed. MetricViews: metricViews(), diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 18b8ebf8a68..17bb10fbe5f 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -29,7 +29,7 @@ require ( github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a diff --git a/core/scripts/go.sum b/core/scripts/go.sum index ab0349c6f4c..ba173f6f02c 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1686,8 +1686,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 h1:NRsBDJ7SHAU0u8iauVLuBHCCnfMfNYYtonN32U8ZNWU= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/core/services/chainlink/telemetry_manager_test.go b/core/services/chainlink/telemetry_manager_test.go index 08792d7345d..075a16f07da 100644 --- a/core/services/chainlink/telemetry_manager_test.go +++ b/core/services/chainlink/telemetry_manager_test.go @@ -64,11 +64,12 @@ func TestNewTelemetryManager(t *testing.T) { t.Cleanup(srv.Stop) client, err := beholder.NewGRPCClient(beholder.Config{ - OtelExporterGRPCEndpoint: lis.Addr().String(), - InsecureConnection: true, - ChipIngressEmitterGRPCEndpoint: lis.Addr().String(), - ChipIngressInsecureConnection: true, - ChipIngressDomainEmittersEnabled: true, + OtelExporterGRPCEndpoint: lis.Addr().String(), + InsecureConnection: true, + ChipIngressEmitterGRPCEndpoint: lis.Addr().String(), + ChipIngressInsecureConnection: true, + ChipDomainEmittersEnabled: true, + ChipIngressLogger: logger.TestLogger(t), }, func(...otlploggrpc.Option) (sdklog.Exporter, error) { return nopLogExporter{}, nil }) require.NoError(t, err) require.NoError(t, client.Start(t.Context())) diff --git a/core/services/telemetry/domain_manager_beholder_test.go b/core/services/telemetry/domain_manager_beholder_test.go index bb49f0f4ec1..283ad5e49d7 100644 --- a/core/services/telemetry/domain_manager_beholder_test.go +++ b/core/services/telemetry/domain_manager_beholder_test.go @@ -19,6 +19,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) @@ -98,14 +99,15 @@ func startChipServer(t *testing.T, chip *capturingChipServer) string { func startBeholderClient(t *testing.T, addr string, tune func(*beholder.Config)) (client *beholder.Client, closeClient func() error) { t.Helper() cfg := beholder.Config{ - OtelExporterGRPCEndpoint: addr, - InsecureConnection: true, - ChipIngressEmitterEnabled: true, - ChipIngressEmitterGRPCEndpoint: addr, - ChipIngressInsecureConnection: true, - ChipIngressSendInterval: 10 * time.Millisecond, - ChipIngressDomainEmittersEnabled: true, - AuthHeaders: map[string]string{"X-Beholder-Node-Auth-Token": "1:abc:2:def"}, + OtelExporterGRPCEndpoint: addr, + InsecureConnection: true, + ChipIngressEmitterEnabled: true, + ChipIngressEmitterGRPCEndpoint: addr, + ChipIngressInsecureConnection: true, + ChipIngressSendInterval: 10 * time.Millisecond, + ChipDomainEmittersEnabled: true, + ChipIngressLogger: logger.Test(t), + AuthHeaders: map[string]string{"X-Beholder-Node-Auth-Token": "1:abc:2:def"}, } if tune != nil { tune(&cfg) diff --git a/deployment/go.mod b/deployment/go.mod index 3ffc85f4a89..b21db25c30a 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -41,7 +41,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/deployment v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 diff --git a/deployment/go.sum b/deployment/go.sum index 37967adca33..4cbb8211b0e 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1411,8 +1411,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 h1:NRsBDJ7SHAU0u8iauVLuBHCCnfMfNYYtonN32U8ZNWU= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/go.mod b/go.mod index d2edfa09be8..3773eac1f63 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 diff --git a/go.sum b/go.sum index 44ffba99adb..9cd02b0ae87 100644 --- a/go.sum +++ b/go.sum @@ -1104,8 +1104,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62/go.mod h1:c++Ed0jeYn4w8WIfiAboRwN6ClRRVGglL0qwB/QjJvM= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 h1:NRsBDJ7SHAU0u8iauVLuBHCCnfMfNYYtonN32U8ZNWU= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index c4c0b8b1797..f92b1ca32e4 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -30,7 +30,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a diff --git a/integration-tests/go.sum b/integration-tests/go.sum index a1569b2f373..0b3de7cee7f 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1396,8 +1396,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 h1:NRsBDJ7SHAU0u8iauVLuBHCCnfMfNYYtonN32U8ZNWU= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 3bdcbbc36b6..1a862bec7c6 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -20,7 +20,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.9 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index cea9e662efb..cb758924cb9 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1636,8 +1636,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 h1:NRsBDJ7SHAU0u8iauVLuBHCCnfMfNYYtonN32U8ZNWU= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 852c6ff9913..6dec7afb2ab 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -32,7 +32,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index ee861bea0d4..ce6d358be67 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1657,8 +1657,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 h1:NRsBDJ7SHAU0u8iauVLuBHCCnfMfNYYtonN32U8ZNWU= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index 70a1c6bf715..22b30da69cf 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -73,7 +73,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index c1df9c39139..267f1482f08 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1836,8 +1836,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f h1:Omu++3bshSAaklDTCtXoi7RciOdoEnNzwf8RrnpGLXQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261028183552-b3de7aa7469f/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 h1:NRsBDJ7SHAU0u8iauVLuBHCCnfMfNYYtonN32U8ZNWU= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= From 9e036483c6919ea0067bfd15c174be6b70bb127e Mon Sep 17 00:00:00 2001 From: Thomas Kaliakos Date: Fri, 13 Nov 2026 13:09:40 +0200 Subject: [PATCH 13/23] Re-pin chainlink-common to main a3881fed (per-domain chip ingress emitters, orgresolver registry field removal, OCR3 config digests); adapt newOrgResolver to the removed orgresolver fields --- core/scripts/go.mod | 8 ++++---- core/scripts/go.sum | 16 ++++++++-------- core/services/cre/cre.go | 26 ++++++-------------------- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- integration-tests/go.mod | 8 ++++---- integration-tests/go.sum | 16 ++++++++-------- 7 files changed, 42 insertions(+), 56 deletions(-) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 17bb10fbe5f..cea1f98a25f 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -29,7 +29,7 @@ require ( github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261112155645-a3881feda2f8 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a @@ -500,7 +500,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca // indirect github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261112155645-a3881feda2f8 // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 // indirect @@ -515,7 +515,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-discovery v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735 // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261013212015-e622e8b83b6b // indirect + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261103151748-aa8c64acc53c // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect @@ -525,7 +525,7 @@ require ( github.com/smartcontractkit/chainlink-protos/rmn/v1.6/go v0.0.0-20250131130834-15e0d4cde2a6 // indirect github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 // indirect github.com/smartcontractkit/chainlink-protos/svr v1.3.0 // indirect - github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260821155228-fa1d775e2138 // indirect + github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20261028174148-0d48af26262a // indirect github.com/smartcontractkit/chainlink-solana/contracts v0.0.0-20260513123719-d347eaf314e1 // indirect github.com/smartcontractkit/chainlink-stellar/bindings v0.0.0-20261005072841-71e45e418098 // indirect github.com/smartcontractkit/chainlink-stellar/deployment v0.0.0-20261005072841-71e45e418098 // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index ba173f6f02c..f640c2fd280 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1686,12 +1686,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 h1:NRsBDJ7SHAU0u8iauVLuBHCCnfMfNYYtonN32U8ZNWU= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261112155645-a3881feda2f8 h1:JIUY7AIcqFcbq9s4s+BSp2NoEGtaSUvVUJ/VNuxc68s= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261112155645-a3881feda2f8/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261112155645-a3881feda2f8 h1:/azbvl3dOhNnQbU0gT+JNNcjafrTNMbi5OIxOVrM/rQ= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261112155645-a3881feda2f8/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= @@ -1726,8 +1726,8 @@ github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0- github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735/go.mod h1:zAJq6Tpkx5AdFUwW67dIYnW+Bdf50drCCpMR81Qxb4E= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d h1:AJy55QJ/pBhXkZjc7N+ATnWfxrcjq9BI9DmdtdjwDUQ= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d/go.mod h1:5JdppgngCOUS76p61zCinSCgOhPeYQ+OcDUuome5THQ= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261013212015-e622e8b83b6b h1:tlxPkVQ6ZpJIElqUWc4lU5VH4BB6OzhckG/gHwpaaa0= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261013212015-e622e8b83b6b/go.mod h1:/i8hjTPFdVWHiY+QjeSiVS2Z3GB3WAZznGgXHstC02E= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261103151748-aa8c64acc53c h1:vt2LhimyZm6zPN46kGKlaRUmIiVbv+8VrPT7Hh4raAI= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261103151748-aa8c64acc53c/go.mod h1:/i8hjTPFdVWHiY+QjeSiVS2Z3GB3WAZznGgXHstC02E= github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 h1:SG+wAsNyAcA6Kk19ljuxi3HK9Ll2lpHik8OKoY4x7A0= github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36/go.mod h1:vL1bDgPSJjV0EqHYs4dDlR+EEE0cJchgvGLYXhwIjXY= github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea h1:gdA7tZgcRYMq3ryi7uNopYgn6s26g04Haw9SdmLRHzE= @@ -1750,8 +1750,8 @@ github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 h1:B7itmjy+C github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0/go.mod h1:h6kqaGajbNRrezm56zhx03p0mVmmA2xxj7E/M4ytLUA= github.com/smartcontractkit/chainlink-protos/svr v1.3.0 h1:vOw+MbXtkPK8l/hA7uLASIgCwv05YzmuoIMuOXPa+g0= github.com/smartcontractkit/chainlink-protos/svr v1.3.0/go.mod h1:TcOliTQU6r59DwG4lo3U+mFM9WWyBHGuFkkxQpvSujo= -github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260821155228-fa1d775e2138 h1:8c/R1dHoAeGTV3/s3ELHjCFuPGuStx5VlJCqe9X0gcM= -github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260821155228-fa1d775e2138/go.mod h1:Oz1I5l+jsFc+YWPjLIrYdn5XUtjA3Iw/E6Z2G6pq5Q0= +github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20261028174148-0d48af26262a h1:CzHgJU/57hwO0Cm7Ev+S8UvuTZjIXDqSGRBM9z++Mck= +github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20261028174148-0d48af26262a/go.mod h1:9cjvCME22gVZthWieZDmjWbdWvwOzv0RPfPjAkqZ0oc= github.com/smartcontractkit/chainlink-solana v1.3.1-0.20260605202330-b5a89c32fdc1 h1:e4vdi3czYy+eK2j/eO5r3ceMxxkx4Qq5IsiAeSAQ9uc= github.com/smartcontractkit/chainlink-solana v1.3.1-0.20260605202330-b5a89c32fdc1/go.mod h1:wi1QdXqhSJnADt9YRaRtEWomqknLcrdkTS0JotupuOQ= github.com/smartcontractkit/chainlink-solana/contracts v0.0.0-20260513123719-d347eaf314e1 h1:/xvuNFI7DwOoTQnmAdYPDdY+sConn3RgZ2rMy/8AXlo= diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index 7f77d5d1ec8..ba2b1c63fb3 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -643,27 +643,13 @@ func newOrgResolver( ds sqlutil.DataSource, lggr logger.Logger, ) (orgresolver.OrgResolver, error) { - var wrChainDetails chainselectors.ChainDetails - if capCfg.WorkflowRegistry().Address() != "" { - var err error - wrChainDetails, err = chainselectors.GetChainDetailsByChainIDAndFamily( - capCfg.WorkflowRegistry().ChainID(), - capCfg.WorkflowRegistry().NetworkID(), - ) - if err != nil { - return nil, fmt.Errorf("failed to get workflow registry chain details by chain ID and network ID: %w", err) - } - } - orgResolverConfig := orgresolver.Config{ - URL: cfg.CRE().Linking().URL(), - TLSEnabled: cfg.CRE().Linking().TLSEnabled(), - RequestTimeout: cfg.CRE().Linking().RequestTimeout(), - WorkflowRegistryAddress: capCfg.WorkflowRegistry().Address(), - WorkflowRegistryChainSelector: wrChainDetails.ChainSelector, - JWTGenerator: opts.JWTGenerator, - Client: opts.LinkingClient, - Meter: opts.Meter, + URL: cfg.CRE().Linking().URL(), + TLSEnabled: cfg.CRE().Linking().TLSEnabled(), + RequestTimeout: cfg.CRE().Linking().RequestTimeout(), + JWTGenerator: opts.JWTGenerator, + Client: opts.LinkingClient, + Meter: opts.Meter, } resolver, err := orgResolverConfig.New(lggr) diff --git a/go.mod b/go.mod index 3773eac1f63..dbdbca28f60 100644 --- a/go.mod +++ b/go.mod @@ -82,9 +82,9 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261112155645-a3881feda2f8 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261112155645-a3881feda2f8 github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 @@ -93,7 +93,7 @@ require ( github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20261009212658-df7dcfcb7a77 github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260625152110-9afcf56e4053 github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4 - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261013212015-e622e8b83b6b + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261103151748-aa8c64acc53c github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 @@ -101,7 +101,7 @@ require ( github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260904122233-4d79005babd9 github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 - github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260821155228-fa1d775e2138 + github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20261028174148-0d48af26262a github.com/smartcontractkit/chainlink-sui/codec v0.0.0-20260720132736-e99278bfdc96 github.com/smartcontractkit/chainlink-ton/cciplib v0.1.1-0.20260716214810-db5ecc877490 github.com/smartcontractkit/cre-sdk-go v1.17.0-capdev.1 diff --git a/go.sum b/go.sum index 9cd02b0ae87..3e2b1449ca2 100644 --- a/go.sum +++ b/go.sum @@ -1104,12 +1104,12 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62/go.mod h1:c++Ed0jeYn4w8WIfiAboRwN6ClRRVGglL0qwB/QjJvM= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 h1:NRsBDJ7SHAU0u8iauVLuBHCCnfMfNYYtonN32U8ZNWU= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261112155645-a3881feda2f8 h1:JIUY7AIcqFcbq9s4s+BSp2NoEGtaSUvVUJ/VNuxc68s= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261112155645-a3881feda2f8/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261112155645-a3881feda2f8 h1:/azbvl3dOhNnQbU0gT+JNNcjafrTNMbi5OIxOVrM/rQ= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261112155645-a3881feda2f8/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 h1:HeQlHO9MrUXTJmxYCNX4xPX14glLAdwIN3igc9Am/g8= @@ -1140,8 +1140,8 @@ github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0- github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735/go.mod h1:zAJq6Tpkx5AdFUwW67dIYnW+Bdf50drCCpMR81Qxb4E= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d h1:AJy55QJ/pBhXkZjc7N+ATnWfxrcjq9BI9DmdtdjwDUQ= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d/go.mod h1:5JdppgngCOUS76p61zCinSCgOhPeYQ+OcDUuome5THQ= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261013212015-e622e8b83b6b h1:tlxPkVQ6ZpJIElqUWc4lU5VH4BB6OzhckG/gHwpaaa0= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261013212015-e622e8b83b6b/go.mod h1:/i8hjTPFdVWHiY+QjeSiVS2Z3GB3WAZznGgXHstC02E= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261103151748-aa8c64acc53c h1:vt2LhimyZm6zPN46kGKlaRUmIiVbv+8VrPT7Hh4raAI= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261103151748-aa8c64acc53c/go.mod h1:/i8hjTPFdVWHiY+QjeSiVS2Z3GB3WAZznGgXHstC02E= github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 h1:SG+wAsNyAcA6Kk19ljuxi3HK9Ll2lpHik8OKoY4x7A0= github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36/go.mod h1:vL1bDgPSJjV0EqHYs4dDlR+EEE0cJchgvGLYXhwIjXY= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= @@ -1160,8 +1160,8 @@ github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 h1:B7itmjy+C github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0/go.mod h1:h6kqaGajbNRrezm56zhx03p0mVmmA2xxj7E/M4ytLUA= github.com/smartcontractkit/chainlink-protos/svr v1.3.0 h1:vOw+MbXtkPK8l/hA7uLASIgCwv05YzmuoIMuOXPa+g0= github.com/smartcontractkit/chainlink-protos/svr v1.3.0/go.mod h1:TcOliTQU6r59DwG4lo3U+mFM9WWyBHGuFkkxQpvSujo= -github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260821155228-fa1d775e2138 h1:8c/R1dHoAeGTV3/s3ELHjCFuPGuStx5VlJCqe9X0gcM= -github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260821155228-fa1d775e2138/go.mod h1:Oz1I5l+jsFc+YWPjLIrYdn5XUtjA3Iw/E6Z2G6pq5Q0= +github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20261028174148-0d48af26262a h1:CzHgJU/57hwO0Cm7Ev+S8UvuTZjIXDqSGRBM9z++Mck= +github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20261028174148-0d48af26262a/go.mod h1:9cjvCME22gVZthWieZDmjWbdWvwOzv0RPfPjAkqZ0oc= github.com/smartcontractkit/chainlink-sui/codec v0.0.0-20260720132736-e99278bfdc96 h1:3R9f6pguDIjei+nKx5dUT9j6fX8urGmlZA7Ad3x9UBs= github.com/smartcontractkit/chainlink-sui/codec v0.0.0-20260720132736-e99278bfdc96/go.mod h1:pSW9q0/YqQE5R65Z/2zsyD9c50JZf+yRJJvN6Jo3LCU= github.com/smartcontractkit/chainlink-ton/cciplib v0.1.1-0.20260716214810-db5ecc877490 h1:ots2WddFOQJL/tDFGmvrhRrVu2iRXCd0XEAxhpkAE/Y= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index f92b1ca32e4..06a6096ff71 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -30,7 +30,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261112155645-a3881feda2f8 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261022195454-600e2b321c7a @@ -405,7 +405,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/deployment v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca // indirect github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261112155645-a3881feda2f8 // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect @@ -419,7 +419,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-discovery v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735 // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261013212015-e622e8b83b6b // indirect + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261103151748-aa8c64acc53c // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect @@ -430,7 +430,7 @@ require ( github.com/smartcontractkit/chainlink-protos/rmn/v1.6/go v0.0.0-20250131130834-15e0d4cde2a6 // indirect github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 // indirect github.com/smartcontractkit/chainlink-protos/svr v1.3.0 // indirect - github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260821155228-fa1d775e2138 // indirect + github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20261028174148-0d48af26262a // indirect github.com/smartcontractkit/chainlink-solana v1.3.1-0.20260605202330-b5a89c32fdc1 // indirect github.com/smartcontractkit/chainlink-stellar/bindings v0.0.0-20261005072841-71e45e418098 // indirect github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.9 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 0b3de7cee7f..996b38863e9 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1396,12 +1396,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37 h1:eZj3cJUTFmbpuRMsUwAM99QYPRiD4udbfcuRLWZ2WWg= github.com/smartcontractkit/chainlink-ccv v0.15.1-0.20261020204039-1a47492cdb37/go.mod h1:A107VkPT8sqCjZJ/2GULxEQOaky70t5tkcE7hK9NDpI= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571 h1:NRsBDJ7SHAU0u8iauVLuBHCCnfMfNYYtonN32U8ZNWU= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261029170441-2cd023586571/go.mod h1:ScyzQBJ5zn64fsAB+67dJtx0UCOyliTBLTLg8Sqz78E= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261112155645-a3881feda2f8 h1:JIUY7AIcqFcbq9s4s+BSp2NoEGtaSUvVUJ/VNuxc68s= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261112155645-a3881feda2f8/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd h1:5pkLw8ZJwMPc27ccq335YullY+kHYiPCIqCvk6zJvBA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260923164324-86cd57d9fdbd/go.mod h1:r8u2d+ZpMweYchIytSRmkRFR+aKQ5c7/HoNv2hzuoTY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c h1:lgshnYtVJKzYnh8ejHpE7MAoR2Jx3O/kezw13PmifZo= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261026195813-8ed7b172ab8c/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261112155645-a3881feda2f8 h1:/azbvl3dOhNnQbU0gT+JNNcjafrTNMbi5OIxOVrM/rQ= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261112155645-a3881feda2f8/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 h1:HeQlHO9MrUXTJmxYCNX4xPX14glLAdwIN3igc9Am/g8= @@ -1434,8 +1434,8 @@ github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0- github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735/go.mod h1:zAJq6Tpkx5AdFUwW67dIYnW+Bdf50drCCpMR81Qxb4E= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d h1:AJy55QJ/pBhXkZjc7N+ATnWfxrcjq9BI9DmdtdjwDUQ= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d/go.mod h1:5JdppgngCOUS76p61zCinSCgOhPeYQ+OcDUuome5THQ= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261013212015-e622e8b83b6b h1:tlxPkVQ6ZpJIElqUWc4lU5VH4BB6OzhckG/gHwpaaa0= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261013212015-e622e8b83b6b/go.mod h1:/i8hjTPFdVWHiY+QjeSiVS2Z3GB3WAZznGgXHstC02E= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261103151748-aa8c64acc53c h1:vt2LhimyZm6zPN46kGKlaRUmIiVbv+8VrPT7Hh4raAI= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20261103151748-aa8c64acc53c/go.mod h1:/i8hjTPFdVWHiY+QjeSiVS2Z3GB3WAZznGgXHstC02E= github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 h1:SG+wAsNyAcA6Kk19ljuxi3HK9Ll2lpHik8OKoY4x7A0= github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36/go.mod h1:vL1bDgPSJjV0EqHYs4dDlR+EEE0cJchgvGLYXhwIjXY= github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea h1:gdA7tZgcRYMq3ryi7uNopYgn6s26g04Haw9SdmLRHzE= @@ -1458,8 +1458,8 @@ github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 h1:B7itmjy+C github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0/go.mod h1:h6kqaGajbNRrezm56zhx03p0mVmmA2xxj7E/M4ytLUA= github.com/smartcontractkit/chainlink-protos/svr v1.3.0 h1:vOw+MbXtkPK8l/hA7uLASIgCwv05YzmuoIMuOXPa+g0= github.com/smartcontractkit/chainlink-protos/svr v1.3.0/go.mod h1:TcOliTQU6r59DwG4lo3U+mFM9WWyBHGuFkkxQpvSujo= -github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260821155228-fa1d775e2138 h1:8c/R1dHoAeGTV3/s3ELHjCFuPGuStx5VlJCqe9X0gcM= -github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260821155228-fa1d775e2138/go.mod h1:Oz1I5l+jsFc+YWPjLIrYdn5XUtjA3Iw/E6Z2G6pq5Q0= +github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20261028174148-0d48af26262a h1:CzHgJU/57hwO0Cm7Ev+S8UvuTZjIXDqSGRBM9z++Mck= +github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20261028174148-0d48af26262a/go.mod h1:9cjvCME22gVZthWieZDmjWbdWvwOzv0RPfPjAkqZ0oc= github.com/smartcontractkit/chainlink-solana v1.3.1-0.20260605202330-b5a89c32fdc1 h1:e4vdi3czYy+eK2j/eO5r3ceMxxkx4Qq5IsiAeSAQ9uc= github.com/smartcontractkit/chainlink-solana v1.3.1-0.20260605202330-b5a89c32fdc1/go.mod h1:wi1QdXqhSJnADt9YRaRtEWomqknLcrdkTS0JotupuOQ= github.com/smartcontractkit/chainlink-stellar/bindings v0.0.0-20261005072841-71e45e418098 h1:gFe6t6i9283xHx2nsBA604rpKxn932504sbxPERpe24= From f5c1858f44c22ed93bef47aa47a9ffc2eae2512d Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:32:41 -0500 Subject: [PATCH 14/23] Add chain_selector to the domain manager OTel counters telemetry_messages_{sent,send_errors,dropped}_total now carry the target chain's chain_selector, so volume and drops can be attributed to a chain on multi-chain DONs. Metric options are cached per (selector, telemetry type, error code); unsupported-type/chain counts keep their (type, reason) attributes since no selector resolves for them. --- core/services/telemetry/domain_manager.go | 46 ++++++++++--------- .../telemetry/domain_manager_beholder_test.go | 5 +- .../services/telemetry/domain_manager_test.go | 2 + 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index 615544eb232..ddbcb4c3f24 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -57,7 +57,8 @@ func supportedTelemetryDomains() map[synchronization.TelemetryType]beholder.Doma } // Metric names and attribute values. The manager's metrics are OTel (Beholder) metrics, so they -// reach us from every node. +// reach us from every node. The counters carry the route's domain and telemetry_type, the target +// chain's chain_selector, and error_code or reason where they apply. // // It also increments the existing Prometheus telemetry_client_messages_{sent,send_errors,dropped} // counters for node-local dashboards, with two differences from the WSRPC path: they count @@ -96,6 +97,7 @@ type DomainManager struct { now func() time.Time reported sync.Map // keys of unsupported types, chains and refusals already logged unsupportedOpts sync.Map // "/" -> otelmetric.AddOption + metricOpts sync.Map // "\x00\x00" -> otelmetric.AddOption } // telemetryRoute is where one telemetry type goes, with its metric handles precomputed. @@ -106,9 +108,7 @@ type telemetryRoute struct { entity string emitter beholder.DomainEmitter - attrs attribute.Set - okOpt otelmetric.AddOption - errorOpts sync.Map // error code -> otelmetric.AddOption + attrs attribute.Set promSent, promSendErrors, promDropped prometheus.Counter } @@ -189,7 +189,6 @@ func NewDomainManager(src DomainEmitterSource, csaPubKeyHex string, lggr logger. entity: entity, emitter: emitters[d], attrs: attrs, - okOpt: otelmetric.WithAttributeSet(attrs), promSent: synchronization.TelemetryClientMessagesSent.WithLabelValues(endpoint, string(telemType)), promSendErrors: synchronization.TelemetryClientMessagesSendErrors.WithLabelValues(endpoint, string(telemType)), promDropped: synchronization.TelemetryClientMessagesDropped.WithLabelValues(endpoint, string(telemType)), @@ -290,18 +289,18 @@ func (m *DomainManager) send(route *telemetryRoute, target eventTarget, telemetr ctx := context.Background() ev, err := m.newEvent(route, target, telemetry) if err != nil { - m.refused(ctx, route, errorCodeEventBuild, err) + m.refused(ctx, route, target, errorCodeEventBuild, err) return } - if err := route.emitter.EmitEvent(ctx, ev, func(sendErr error) { m.recordFate(ctx, route, sendErr) }); err != nil { - m.refused(ctx, route, errorCodeEmitRejected, err) + if err := route.emitter.EmitEvent(ctx, ev, func(sendErr error) { m.recordFate(ctx, route, target, sendErr) }); err != nil { + m.refused(ctx, route, target, errorCodeEmitRejected, err) } } // refused counts an event that never reached the emitter's buffer and logs the first one per // route and error code at Error. -func (m *DomainManager) refused(ctx context.Context, route *telemetryRoute, code string, err error) { - m.recordDropped(ctx, route, code) +func (m *DomainManager) refused(ctx context.Context, route *telemetryRoute, target eventTarget, code string, err error) { + m.recordDropped(ctx, route, target, code) m.reportOnce("refused:"+string(route.telemType)+"/"+code, func() { m.eng.Errorw("Telemetry refused: it is being dropped (further refusals are only counted)", "telemType", route.telemType, "domain", route.domain, "errorCode", code, "err", err) @@ -310,31 +309,36 @@ func (m *DomainManager) refused(ctx context.Context, route *telemetryRoute, code // recordFate counts an event's fate as reported by the emitter: published, failed to publish, // or dropped before publishing (full buffer or shutdown). -func (m *DomainManager) recordFate(ctx context.Context, route *telemetryRoute, err error) { +func (m *DomainManager) recordFate(ctx context.Context, route *telemetryRoute, target eventTarget, err error) { switch { case err == nil: - m.metrics.sent.Add(ctx, 1, route.okOpt) + m.metrics.sent.Add(ctx, 1, m.metricOpt(route, target, "")) route.promSent.Inc() case errors.Is(err, batch.ErrMessageBufferFull), errors.Is(err, batch.ErrClientShutdown): - m.recordDropped(ctx, route, batch.ErrorCodeFor(err)) + m.recordDropped(ctx, route, target, batch.ErrorCodeFor(err)) default: - m.metrics.sendErrors.Add(ctx, 1, route.errorOpt(batch.ErrorCodeFor(err))) + m.metrics.sendErrors.Add(ctx, 1, m.metricOpt(route, target, batch.ErrorCodeFor(err))) route.promSendErrors.Inc() } } -func (m *DomainManager) recordDropped(ctx context.Context, route *telemetryRoute, code string) { - m.metrics.dropped.Add(ctx, 1, route.errorOpt(code)) +func (m *DomainManager) recordDropped(ctx context.Context, route *telemetryRoute, target eventTarget, code string) { + m.metrics.dropped.Add(ctx, 1, m.metricOpt(route, target, code)) route.promDropped.Inc() } -// errorOpt returns the route's attributes plus error_code, cached per code (a bounded set). -func (r *telemetryRoute) errorOpt(code string) otelmetric.AddOption { - if v, ok := r.errorOpts.Load(code); ok { +// metricOpt returns the route's attributes plus the target chain's chain_selector and, when code +// is non-empty, the error code. Cached per (selector, telemetry type, error code): a bounded set. +func (m *DomainManager) metricOpt(route *telemetryRoute, target eventTarget, code string) otelmetric.AddOption { + key := target.selector + "\x00" + string(route.telemType) + "\x00" + code + if v, ok := m.metricOpts.Load(key); ok { return v.(otelmetric.AddOption) } - opt := otelmetric.WithAttributes(append(r.attrs.ToSlice(), attribute.String("error_code", code))...) - v, _ := r.errorOpts.LoadOrStore(code, opt) + kvs := append(route.attrs.ToSlice(), attribute.String("chain_selector", target.selector)) + if code != "" { + kvs = append(kvs, attribute.String("error_code", code)) + } + v, _ := m.metricOpts.LoadOrStore(key, otelmetric.WithAttributes(kvs...)) return v.(otelmetric.AddOption) } diff --git a/core/services/telemetry/domain_manager_beholder_test.go b/core/services/telemetry/domain_manager_beholder_test.go index 283ad5e49d7..d3a472eb1b9 100644 --- a/core/services/telemetry/domain_manager_beholder_test.go +++ b/core/services/telemetry/domain_manager_beholder_test.go @@ -180,6 +180,8 @@ func TestDomainManager_ThroughRealBeholderClient(t *testing.T) { requireCleanClose(t, closeClient()) // Nothing is counted twice, and nothing else is counted, once the emitters have drained. assert.Equal(t, int64(total), counterValue(t, reader, metricMessagesSent)) + assert.Equal(t, int64(total), counterValue(t, reader, metricMessagesSent, + attribute.String("chain_selector", ethMainnetSelector)), "sent counts carry the chain selector") assert.Zero(t, counterValue(t, reader, metricMessagesSendErrors)) assert.Zero(t, counterValue(t, reader, metricMessagesDropped)) } @@ -202,7 +204,8 @@ func TestDomainManager_ThroughRealBeholderClient_BufferFull(t *testing.T) { ep.SendLog([]byte("p")) } bufferFull := attribute.String("error_code", batch.ErrorCodeFor(batch.ErrMessageBufferFull)) - require.Positive(t, counterValue(t, reader, metricMessagesDropped, bufferFull), "drops are counted as dropped, not send errors") + require.Positive(t, counterValue(t, reader, metricMessagesDropped, bufferFull, + attribute.String("chain_selector", ethMainnetSelector)), "drops are counted as dropped, not send errors") close(chip.hold) // let the held and queued events through, then drain on close requireCleanClose(t, closeClient()) diff --git a/core/services/telemetry/domain_manager_test.go b/core/services/telemetry/domain_manager_test.go index 626359c7798..ac54d55e016 100644 --- a/core/services/telemetry/domain_manager_test.go +++ b/core/services/telemetry/domain_manager_test.go @@ -131,6 +131,8 @@ func counterValue(t *testing.T, reader *sdkmetric.ManualReader, name string, wan var ethMainnetChainID = strconv.FormatUint(chainselectors.ETHEREUM_MAINNET.EvmChainID, 10) +var ethMainnetSelector = strconv.FormatUint(chainselectors.ETHEREUM_MAINNET.Selector, 10) + func TestNewDomainManager_RequiresEveryDomain(t *testing.T) { t.Parallel() src := newFakeEmitterSource() From a2d24a13bc0f2497844b6c496d9b1abfcb6c9d0a Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:54:40 -0500 Subject: [PATCH 15/23] Use a struct key for the domain manager metric option cache The (selector, telemetry type, error code) cache key was a NUL-joined string; a small struct is self-documenting and collision-proof. No behaviour change. --- core/services/telemetry/domain_manager.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index ddbcb4c3f24..f9714333811 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -97,7 +97,7 @@ type DomainManager struct { now func() time.Time reported sync.Map // keys of unsupported types, chains and refusals already logged unsupportedOpts sync.Map // "/" -> otelmetric.AddOption - metricOpts sync.Map // "\x00\x00" -> otelmetric.AddOption + metricOpts sync.Map // metricKey -> otelmetric.AddOption } // telemetryRoute is where one telemetry type goes, with its metric handles precomputed. @@ -327,10 +327,18 @@ func (m *DomainManager) recordDropped(ctx context.Context, route *telemetryRoute route.promDropped.Inc() } +// metricKey identifies a cached metric option: the target chain's selector, the route's +// telemetry type and the error code ("" for the sent counter). +type metricKey struct { + selector string + telemType synchronization.TelemetryType + errorCode string +} + // metricOpt returns the route's attributes plus the target chain's chain_selector and, when code -// is non-empty, the error code. Cached per (selector, telemetry type, error code): a bounded set. +// is non-empty, the error code. Cached per metricKey: a bounded set. func (m *DomainManager) metricOpt(route *telemetryRoute, target eventTarget, code string) otelmetric.AddOption { - key := target.selector + "\x00" + string(route.telemType) + "\x00" + code + key := metricKey{selector: target.selector, telemType: route.telemType, errorCode: code} if v, ok := m.metricOpts.Load(key); ok { return v.(otelmetric.AddOption) } From 28a269ec03267ee0cc6d57142650ae9c7da268bc Mon Sep 17 00:00:00 2001 From: Thomas Kaliakos Date: Tue, 17 Nov 2026 15:29:22 +0200 Subject: [PATCH 16/23] telemetry: bound unsupported-type metric cardinality and record the domain on unsupported drops --- core/services/telemetry/domain_manager.go | 88 ++++++++++++------- .../services/telemetry/domain_manager_test.go | 35 +++++++- 2 files changed, 89 insertions(+), 34 deletions(-) diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index f9714333811..02d499e58a0 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -35,25 +35,24 @@ type DomainEmitterSource interface { // whose emitter carries each: the types ingested in production. Their CloudEvent source and type // come from synchronization.TelemetryTypeToDomainAndEntity. Every other type is dropped and counted. // The TelemetryIngress.BeholderEnabled docs (core/config/docs/core.toml) list these types: keep them in sync. -func supportedTelemetryDomains() map[synchronization.TelemetryType]beholder.Domain { - return map[synchronization.TelemetryType]beholder.Domain{ - synchronization.OCR: beholder.DomainDataFeeds, - synchronization.OCR2Median: beholder.DomainDataFeeds, - synchronization.OCR3DataFeeds: beholder.DomainDataFeeds, - synchronization.EnhancedEA: beholder.DomainDataFeeds, - synchronization.OCR3Mercury: beholder.DomainDataStreams, - synchronization.EnhancedEAMercury: beholder.DomainDataStreams, - synchronization.LLOObservation: beholder.DomainDataStreams, - synchronization.LLOOutcome: beholder.DomainDataStreams, - synchronization.PipelineBridge: beholder.DomainDataStreams, - synchronization.OCR2CCIPCommit: beholder.DomainCCIP, - synchronization.OCR2CCIPExec: beholder.DomainCCIP, - synchronization.OCR3CCIPCommit: beholder.DomainCCIP, - synchronization.OCR3CCIPExec: beholder.DomainCCIP, - // TODO(INFOPLAT-19709): head reports are emitted for every chain on every node; confirm - // ccip is their domain. - synchronization.HeadReport: beholder.DomainCCIP, - } +// A package-level map: countUnsupported reads it per message and must not allocate. +var supportedTelemetryDomains = map[synchronization.TelemetryType]beholder.Domain{ + synchronization.OCR: beholder.DomainDataFeeds, + synchronization.OCR2Median: beholder.DomainDataFeeds, + synchronization.OCR3DataFeeds: beholder.DomainDataFeeds, + synchronization.EnhancedEA: beholder.DomainDataFeeds, + synchronization.OCR3Mercury: beholder.DomainDataStreams, + synchronization.EnhancedEAMercury: beholder.DomainDataStreams, + synchronization.LLOObservation: beholder.DomainDataStreams, + synchronization.LLOOutcome: beholder.DomainDataStreams, + synchronization.PipelineBridge: beholder.DomainDataStreams, + synchronization.OCR2CCIPCommit: beholder.DomainCCIP, + synchronization.OCR2CCIPExec: beholder.DomainCCIP, + synchronization.OCR3CCIPCommit: beholder.DomainCCIP, + synchronization.OCR3CCIPExec: beholder.DomainCCIP, + // TODO(INFOPLAT-19709): head reports are emitted for every chain on every node; confirm + // ccip is their domain. + synchronization.HeadReport: beholder.DomainCCIP, } // Metric names and attribute values. The manager's metrics are OTel (Beholder) metrics, so they @@ -96,7 +95,7 @@ type DomainManager struct { metrics domainManagerMetrics now func() time.Time reported sync.Map // keys of unsupported types, chains and refusals already logged - unsupportedOpts sync.Map // "/" -> otelmetric.AddOption + unsupportedOpts sync.Map // unsupportedKey -> otelmetric.AddOption metricOpts sync.Map // metricKey -> otelmetric.AddOption } @@ -153,10 +152,9 @@ func NewDomainManager(src DomainEmitterSource, csaPubKeyHex string, lggr logger. } m.metrics = metrics - supported := supportedTelemetryDomains() // Resolve domains in a stable order so a missing one is reported the same way every time. var domains []beholder.Domain - for _, d := range supported { + for _, d := range supportedTelemetryDomains { if !slices.Contains(domains, d) { domains = append(domains, d) } @@ -171,8 +169,8 @@ func NewDomainManager(src DomainEmitterSource, csaPubKeyHex string, lggr logger. emitters[d] = em } - m.routes = make(map[synchronization.TelemetryType]*telemetryRoute, len(supported)) - for telemType, d := range supported { + m.routes = make(map[synchronization.TelemetryType]*telemetryRoute, len(supportedTelemetryDomains)) + for telemType, d := range supportedTelemetryDomains { source, entity, err := synchronization.TelemetryTypeToDomainAndEntity(telemType) if err != nil { return nil, fmt.Errorf("telemetry: no CloudEvent source for %q: %w", telemType, err) @@ -350,22 +348,50 @@ func (m *DomainManager) metricOpt(route *telemetryRoute, target eventTarget, cod return v.(otelmetric.AddOption) } -// countUnsupported counts a dropped message whose type or chain is not supported. The metric -// option is cached per (type, reason): unsupported LLO types arrive on the hot path. +// unsupportedTypeLabel clamps an unsupported telemetry type for metric labels and log +// dedup keys: the generic telemetry adapter accepts arbitrary strings, so a raw value +// would grow the metric series and the option cache without bound. Types with a +// CloudEvent mapping (synchronization.TelemetryTypeToDomainAndEntity is the registry) +// keep their label; anything else is "unknown" — its raw value is logged once by +// warnUnsupportedType. +func unsupportedTypeLabel(telemType synchronization.TelemetryType) string { + if _, _, err := synchronization.TelemetryTypeToDomainAndEntity(telemType); err == nil { + return string(telemType) + } + return "unknown" +} + +// countUnsupported counts a dropped message whose type or chain is not supported. The +// telemetry_type label is clamped (see unsupportedTypeLabel) and the domain is recorded +// whenever the type maps to one, so domain-filtered loss dashboards see chain-reason +// drops of mapped types. The option is cached per (label, reason): hot path. +// unsupportedKey identifies a cached unsupported-metric option: the clamped telemetry +// type label and the drop reason. Bounded by construction — labels come from the +// CloudEvent registry plus "unknown", reasons from a fixed set. +type unsupportedKey struct { + telemType string + reason string +} + func (m *DomainManager) countUnsupported(telemType synchronization.TelemetryType, reason string) { - key := string(telemType) + "/" + reason + label := unsupportedTypeLabel(telemType) + key := unsupportedKey{telemType: label, reason: reason} v, ok := m.unsupportedOpts.Load(key) if !ok { - v, _ = m.unsupportedOpts.LoadOrStore(key, otelmetric.WithAttributes( - attribute.String("telemetry_type", string(telemType)), + attrs := []attribute.KeyValue{ + attribute.String("telemetry_type", label), attribute.String("reason", reason), - )) + } + if d, mapped := supportedTelemetryDomains[telemType]; mapped { + attrs = append(attrs, attribute.String("domain", string(d))) + } + v, _ = m.unsupportedOpts.LoadOrStore(key, otelmetric.WithAttributes(attrs...)) } m.metrics.unsupported.Add(context.Background(), 1, v.(otelmetric.AddOption)) } func (m *DomainManager) warnUnsupportedType(telemType synchronization.TelemetryType) { - m.reportOnce("type:"+string(telemType), func() { + m.reportOnce("type:"+unsupportedTypeLabel(telemType), func() { m.eng.Warnw("Telemetry type is not supported: its telemetry will be dropped", "telemType", telemType) }) } diff --git a/core/services/telemetry/domain_manager_test.go b/core/services/telemetry/domain_manager_test.go index ac54d55e016..8433295ab52 100644 --- a/core/services/telemetry/domain_manager_test.go +++ b/core/services/telemetry/domain_manager_test.go @@ -306,6 +306,34 @@ func TestDomainManager_UnresolvableChainIsDroppedAndCounted(t *testing.T) { assert.Equal(t, int64(3), counterValue(t, reader, metricMessagesUnsupported, reason)) assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, reason, attribute.String("telemetry_type", string(synchronization.LLOOutcome))), "multitype counts carry the message's type") + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, reason, + attribute.String("telemetry_type", string(synchronization.OCR2Median)), + attribute.String("domain", string(beholder.DomainDataFeeds))), "chain drops of mapped types carry their domain") +} + +func TestDomainManager_UnknownTypesCollapseToOneSeries(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, reader := newTestDomainManager(t, src) + + // The generic telemetry adapter accepts arbitrary type strings; distinct values + // must not create a metric series or a cache entry each. + for i := 0; i < 100; i++ { + tt := synchronization.TelemetryType("plugin-garbage-" + strconv.Itoa(i)) + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", tt).SendLog([]byte("p")) + } + + assert.Equal(t, int64(100), counterValue(t, reader, metricMessagesUnsupported, + attribute.String("telemetry_type", "unknown"), + attribute.String("reason", unsupportedReasonTelemetryType)), "every message is counted") + assert.Equal(t, 1, syncMapCount(&m.unsupportedOpts), "one cache entry per clamped label, not per raw value") + assert.Equal(t, 1, syncMapCount(&m.reported), "log dedup keys are clamped too") +} + +func syncMapCount(m *sync.Map) int { + n := 0 + m.Range(func(_, _ any) bool { n++; return true }) + return n } func TestDomainManager_MultitypeEndpointRoutesPerMessage(t *testing.T) { @@ -315,14 +343,15 @@ func TestDomainManager_MultitypeEndpointRoutesPerMessage(t *testing.T) { ep := m.GenMultitypeMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc/1") ep.SendTypedLog(synchronization.LLOOutcome, []byte("p")) - ep.SendTypedLog(synchronization.LLOReport, []byte("p")) // unsupported: dropped + ep.SendTypedLog(synchronization.LLOReport, []byte("p")) // unsupported: dropped (no CloudEvent mapping yet) events := src[beholder.DomainDataStreams].sent() require.Len(t, events, 1) assert.Equal(t, "data-streams.telemetry.llo-outcome", events[0].Source()) assert.Equal(t, "llo-outcome", events[0].Extensions()["telemetrytype"]) assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, - attribute.String("telemetry_type", string(synchronization.LLOReport)))) + attribute.String("telemetry_type", "unknown")), + "types without a CloudEvent mapping collapse onto the unknown label") } func TestDomainManager_CountsEachEventsFate(t *testing.T) { @@ -424,7 +453,7 @@ func TestDomainManager_AlsoCountsInPrometheus(t *testing.T) { // supported type's source must be the domain or a subdomain of it. func TestSupportedTelemetry_SourcesSitUnderTheirDomain(t *testing.T) { t.Parallel() - for telemType, d := range supportedTelemetryDomains() { + for telemType, d := range supportedTelemetryDomains { source, _, err := synchronization.TelemetryTypeToDomainAndEntity(telemType) require.NoError(t, err, "%s has a CloudEvent source", telemType) assert.True(t, source == string(d) || strings.HasPrefix(source, string(d)+"."), From 64a94d4f8daab0efe5c8f275f644c95365791620 Mon Sep 17 00:00:00 2001 From: Thomas Kaliakos Date: Tue, 17 Nov 2026 16:38:01 +0200 Subject: [PATCH 17/23] Amend @pkcll's comments --- core/services/chainlink/telemetry_manager.go | 8 ++++- .../chainlink/telemetry_manager_test.go | 7 ++++ core/services/telemetry/domain_manager.go | 32 +++++++++++++------ .../services/telemetry/domain_manager_test.go | 28 ++++++++++++++-- 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/core/services/chainlink/telemetry_manager.go b/core/services/chainlink/telemetry_manager.go index 1c8fb3d1e72..19e539b6d3a 100644 --- a/core/services/chainlink/telemetry_manager.go +++ b/core/services/chainlink/telemetry_manager.go @@ -29,7 +29,13 @@ func newTelemetryManager(cfg config.TelemetryIngress, beholderClient *beholder.C if beholderClient == nil { return nil, errors.New("TelemetryIngress.BeholderEnabled requires a Beholder client") } - dm, err := telemetry.NewDomainManager(beholderClient, csaPubKeyHex, lggr) + // The manager's counters share the client's meter provider, as the emitters' chip + // metrics already do. Nil on a hand-built client, where the option would panic. + var opts []telemetry.DomainManagerOpt + if beholderClient.MeterProvider != nil { + opts = append(opts, telemetry.WithMeterProvider(beholderClient.MeterProvider)) + } + dm, err := telemetry.NewDomainManager(beholderClient, csaPubKeyHex, lggr, opts...) if err != nil { return nil, fmt.Errorf("failed to create telemetry domain manager: %w", err) } diff --git a/core/services/chainlink/telemetry_manager_test.go b/core/services/chainlink/telemetry_manager_test.go index 075a16f07da..73fffc3d6a0 100644 --- a/core/services/chainlink/telemetry_manager_test.go +++ b/core/services/chainlink/telemetry_manager_test.go @@ -54,6 +54,13 @@ func TestNewTelemetryManager(t *testing.T) { _, err := newTelemetryManager(telemetryIngress(t, true).TelemetryIngress(), beholder.NewNoopClient(), ksmocks.NewCSA(t), "abc123", lggr) require.Error(t, err, "the no-op client has no domain emitters") }) + t.Run("flag on with a hand-built client: an error, not a panic", func(t *testing.T) { + t.Parallel() + // A hand-built client has no meter provider and no domain emitters; the + // manager must fall back to the global provider instead of panicking. + _, err := newTelemetryManager(telemetryIngress(t, true).TelemetryIngress(), &beholder.Client{}, ksmocks.NewCSA(t), "abc123", lggr) + require.Error(t, err, "a hand-built client has no domain emitters") + }) t.Run("flag on: the domain manager on Beholder's domain emitters", func(t *testing.T) { t.Parallel() // A local gRPC server with no services answers OTel exports at once, so Close is quick. diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index 02d499e58a0..6477a1e1e94 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/prometheus/client_golang/prometheus" @@ -95,6 +96,7 @@ type DomainManager struct { metrics domainManagerMetrics now func() time.Time reported sync.Map // keys of unsupported types, chains and refusals already logged + reportedKeys atomic.Int64 unsupportedOpts sync.Map // unsupportedKey -> otelmetric.AddOption metricOpts sync.Map // metricKey -> otelmetric.AddOption } @@ -273,10 +275,20 @@ func (m *DomainManager) chainSelector(network, chainID string) (uint64, bool) { return details.ChainSelector, true } +// maxReportedKeys caps how many distinct one-time reports are remembered. The keys are +// sender-influenced (telemetry types, network/chainID pairs), so the cap bounds +// worst-case memory; past it, new keys are only counted — never logged or retained. +const maxReportedKeys = 25 + func (m *DomainManager) reportOnce(key string, report func()) { - if _, loaded := m.reported.LoadOrStore(key, struct{}{}); !loaded { - report() + if _, loaded := m.reported.LoadOrStore(key, struct{}{}); loaded { + return + } + if m.reportedKeys.Add(1) > maxReportedKeys { + m.reported.Delete(key) + return } + report() } // send builds the event and hands it to the route's emitter. Fates are counted; publish failures @@ -362,9 +374,9 @@ func unsupportedTypeLabel(telemType synchronization.TelemetryType) string { } // countUnsupported counts a dropped message whose type or chain is not supported. The -// telemetry_type label is clamped (see unsupportedTypeLabel) and the domain is recorded -// whenever the type maps to one, so domain-filtered loss dashboards see chain-reason -// drops of mapped types. The option is cached per (label, reason): hot path. +// telemetry_type label is clamped (see unsupportedTypeLabel) and the domain is always +// recorded ("unknown" when the type maps to none), so domain-filtered loss dashboards +// see every drop. The option is cached per (label, reason): hot path. // unsupportedKey identifies a cached unsupported-metric option: the clamped telemetry // type label and the drop reason. Bounded by construction — labels come from the // CloudEvent registry plus "unknown", reasons from a fixed set. @@ -378,12 +390,14 @@ func (m *DomainManager) countUnsupported(telemType synchronization.TelemetryType key := unsupportedKey{telemType: label, reason: reason} v, ok := m.unsupportedOpts.Load(key) if !ok { + d, mapped := supportedTelemetryDomains[telemType] + if !mapped { + d = "unknown" + } attrs := []attribute.KeyValue{ attribute.String("telemetry_type", label), attribute.String("reason", reason), - } - if d, mapped := supportedTelemetryDomains[telemType]; mapped { - attrs = append(attrs, attribute.String("domain", string(d))) + attribute.String("domain", string(d)), } v, _ = m.unsupportedOpts.LoadOrStore(key, otelmetric.WithAttributes(attrs...)) } @@ -391,7 +405,7 @@ func (m *DomainManager) countUnsupported(telemType synchronization.TelemetryType } func (m *DomainManager) warnUnsupportedType(telemType synchronization.TelemetryType) { - m.reportOnce("type:"+unsupportedTypeLabel(telemType), func() { + m.reportOnce("type:"+string(telemType), func() { m.eng.Warnw("Telemetry type is not supported: its telemetry will be dropped", "telemType", telemType) }) } diff --git a/core/services/telemetry/domain_manager_test.go b/core/services/telemetry/domain_manager_test.go index 8433295ab52..407d3f4b657 100644 --- a/core/services/telemetry/domain_manager_test.go +++ b/core/services/telemetry/domain_manager_test.go @@ -318,16 +318,37 @@ func TestDomainManager_UnknownTypesCollapseToOneSeries(t *testing.T) { // The generic telemetry adapter accepts arbitrary type strings; distinct values // must not create a metric series or a cache entry each. - for i := 0; i < 100; i++ { + for i := range 100 { tt := synchronization.TelemetryType("plugin-garbage-" + strconv.Itoa(i)) m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", tt).SendLog([]byte("p")) } assert.Equal(t, int64(100), counterValue(t, reader, metricMessagesUnsupported, attribute.String("telemetry_type", "unknown"), + attribute.String("domain", "unknown"), attribute.String("reason", unsupportedReasonTelemetryType)), "every message is counted") assert.Equal(t, 1, syncMapCount(&m.unsupportedOpts), "one cache entry per clamped label, not per raw value") - assert.Equal(t, 1, syncMapCount(&m.reported), "log dedup keys are clamped too") + assert.LessOrEqual(t, syncMapCount(&m.reported), maxReportedKeys, + "the log-dedup map is capped however many distinct types arrive") +} + +func TestDomainManager_UnresolvableChainsAreCapped(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, reader := newTestDomainManager(t, src) + + // Distinct network/chainID pairs are plugin-suppliable via the generic adapter; + // past the cap they must stop being logged and retained, but stay counted. + for i := range 100 { + m.GenMonitoringEndpoint("EVM", "bogus-"+strconv.Itoa(i), "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + } + + assert.Equal(t, int64(100), counterValue(t, reader, metricMessagesUnsupported, + attribute.String("telemetry_type", string(synchronization.OCR2Median)), + attribute.String("domain", string(beholder.DomainDataFeeds)), + attribute.String("reason", unsupportedReasonChain)), "every drop is counted") + assert.LessOrEqual(t, syncMapCount(&m.reported), maxReportedKeys, + "the log-dedup map is capped however many distinct chains arrive") } func syncMapCount(m *sync.Map) int { @@ -350,7 +371,8 @@ func TestDomainManager_MultitypeEndpointRoutesPerMessage(t *testing.T) { assert.Equal(t, "data-streams.telemetry.llo-outcome", events[0].Source()) assert.Equal(t, "llo-outcome", events[0].Extensions()["telemetrytype"]) assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, - attribute.String("telemetry_type", "unknown")), + attribute.String("telemetry_type", "unknown"), + attribute.String("domain", "unknown")), "types without a CloudEvent mapping collapse onto the unknown label") } From 445f7b7e9575fa6d2200556dcded65b50bd9109a Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:20:49 -0500 Subject: [PATCH 18/23] telemetry: route ocr3_1-mercury to data-streams LLO v31 (OCR3.1) oracles send their protocol telemetry as ocr3_1-mercury since #24197; without a route, BeholderEnabled dropped it. Co-Authored-By: Claude Opus 5.5 --- core/config/docs/core.toml | 4 ++-- core/services/telemetry/domain_manager.go | 1 + core/services/telemetry/domain_manager_test.go | 1 + docs/CONFIG.md | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index f94fdb34610..0ac7ff96f07 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -122,8 +122,8 @@ ChipIngressEnabled = false # Default # Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint, and chip ingress that serves PublishBatchNoAck for every domain # (there is no fallback). Cannot be combined with ChipIngressEnabled. # Telemetry is sent for every chain the node's jobs run on: Endpoints no longer select chains. Supported types are ocr, -# ocr2-median, ocr3-data-feeds, enhanced-ea, ocr3-mercury, enhanced-ea-mercury, llo-observation, llo-outcome, pipeline-bridge, -# ocr2-ccip-commit, ocr2-ccip-exec, ocr3-ccip-commit, ocr3-ccip-exec and head-report; other types are dropped. +# ocr2-median, ocr3-data-feeds, enhanced-ea, ocr3-mercury, ocr3_1-mercury, enhanced-ea-mercury, llo-observation, llo-outcome, +# pipeline-bridge, ocr2-ccip-commit, ocr2-ccip-exec, ocr3-ccip-commit, ocr3-ccip-exec and head-report; other types are dropped. BeholderEnabled = false # Default [[TelemetryIngress.Endpoints]] # Example diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index 6477a1e1e94..a3ca0c39044 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -43,6 +43,7 @@ var supportedTelemetryDomains = map[synchronization.TelemetryType]beholder.Domai synchronization.OCR3DataFeeds: beholder.DomainDataFeeds, synchronization.EnhancedEA: beholder.DomainDataFeeds, synchronization.OCR3Mercury: beholder.DomainDataStreams, + synchronization.OCR3_1Mercury: beholder.DomainDataStreams, synchronization.EnhancedEAMercury: beholder.DomainDataStreams, synchronization.LLOObservation: beholder.DomainDataStreams, synchronization.LLOOutcome: beholder.DomainDataStreams, diff --git a/core/services/telemetry/domain_manager_test.go b/core/services/telemetry/domain_manager_test.go index 407d3f4b657..42dbb7f5bfd 100644 --- a/core/services/telemetry/domain_manager_test.go +++ b/core/services/telemetry/domain_manager_test.go @@ -166,6 +166,7 @@ func TestDomainManager_RoutesEachTypeToItsDomain(t *testing.T) { {synchronization.OCR3DataFeeds, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr3-data-feeds", ocr3}, {synchronization.EnhancedEA, beholder.DomainDataFeeds, "data-feeds.telemetry.enhanced-ea", "telem.EnhancedEA"}, {synchronization.OCR3Mercury, beholder.DomainDataStreams, "data-streams.telemetry.ocr3-mercury", ocr3}, + {synchronization.OCR3_1Mercury, beholder.DomainDataStreams, "data-streams.telemetry.ocr3_1-mercury", "offchainreporting3_1.TelemetryWrapper"}, {synchronization.EnhancedEAMercury, beholder.DomainDataStreams, "data-streams.telemetry.enhanced-ea-mercury", "telem.EnhancedEAMercury"}, {synchronization.LLOObservation, beholder.DomainDataStreams, "data-streams.telemetry.llo-observation", "telem.LLOObservationTelemetry"}, {synchronization.LLOOutcome, beholder.DomainDataStreams, "data-streams.telemetry.llo-outcome", "telem.LLOOutcomeTelemetry"}, diff --git a/docs/CONFIG.md b/docs/CONFIG.md index f600ce5eaa5..de07dd2f332 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -343,8 +343,8 @@ are ignored: the Telemetry.ChipIngress* settings size one buffer per domain, sha Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint, and chip ingress that serves PublishBatchNoAck for every domain (there is no fallback). Cannot be combined with ChipIngressEnabled. Telemetry is sent for every chain the node's jobs run on: Endpoints no longer select chains. Supported types are ocr, -ocr2-median, ocr3-data-feeds, enhanced-ea, ocr3-mercury, enhanced-ea-mercury, llo-observation, llo-outcome, pipeline-bridge, -ocr2-ccip-commit, ocr2-ccip-exec, ocr3-ccip-commit, ocr3-ccip-exec and head-report; other types are dropped. +ocr2-median, ocr3-data-feeds, enhanced-ea, ocr3-mercury, ocr3_1-mercury, enhanced-ea-mercury, llo-observation, llo-outcome, +pipeline-bridge, ocr2-ccip-commit, ocr2-ccip-exec, ocr3-ccip-commit, ocr3-ccip-exec and head-report; other types are dropped. ## TelemetryIngress.Endpoints ```toml From 8f734c823032652501e4f94fd0da5be5b7001740 Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:20:57 -0500 Subject: [PATCH 19/23] telemetry: send recordedtime in UTC; fix countUnsupported doc comment placement Co-Authored-By: Claude Opus 5.5 --- core/services/telemetry/domain_manager.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index a3ca0c39044..ee13583b0e1 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -374,10 +374,6 @@ func unsupportedTypeLabel(telemType synchronization.TelemetryType) string { return "unknown" } -// countUnsupported counts a dropped message whose type or chain is not supported. The -// telemetry_type label is clamped (see unsupportedTypeLabel) and the domain is always -// recorded ("unknown" when the type maps to none), so domain-filtered loss dashboards -// see every drop. The option is cached per (label, reason): hot path. // unsupportedKey identifies a cached unsupported-metric option: the clamped telemetry // type label and the drop reason. Bounded by construction — labels come from the // CloudEvent registry plus "unknown", reasons from a fixed set. @@ -386,6 +382,10 @@ type unsupportedKey struct { reason string } +// countUnsupported counts a dropped message whose type or chain is not supported. The +// telemetry_type label is clamped (see unsupportedTypeLabel) and the domain is always +// recorded ("unknown" when the type maps to none), so domain-filtered loss dashboards +// see every drop. The option is cached per (label, reason): hot path. func (m *DomainManager) countUnsupported(telemType synchronization.TelemetryType, reason string) { label := unsupportedTypeLabel(telemType) key := unsupportedKey{telemType: label, reason: reason} @@ -418,7 +418,7 @@ func (m *DomainManager) newEvent(route *telemetryRoute, target eventTarget, tele now := m.now() event, err := chipingress.NewEvent(route.source, route.entity, telemetry, map[string]any{ "time": now.UTC(), - "recordedtime": now, + "recordedtime": now.UTC(), }) if err != nil { return event, fmt.Errorf("failed creating CloudEvent: %w", err) From 7035964c48fa8961306b32a3d0d06e05d0522dc3 Mon Sep 17 00:00:00 2001 From: Thomas Kaliakos Date: Wed, 18 Nov 2026 14:47:53 +0200 Subject: [PATCH 20/23] telemetry: derive send and unsupported-count contexts from the service engine --- core/services/telemetry/domain_manager.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index ee13583b0e1..8a99f538708 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -296,8 +296,11 @@ func (m *DomainManager) reportOnce(key string, report func()) { // are logged (rate-limited) by the emitter, and events that never reach it are logged here, once // per route and error code, since they mean telemetry is being lost. func (m *DomainManager) send(route *telemetryRoute, target eventTarget, telemetry []byte) { - // SendLog has no context (libocr's MonitoringEndpoint); the emitter only queues. - ctx := context.Background() + // SendLog has no context (libocr's MonitoringEndpoint), so tie it to the + // service lifecycle. Cancelling on return is safe: after the emitter's sync + // queue check, only otel adds (which tolerate cancellation) read it. + ctx, cancel := m.eng.NewCtx() + defer cancel() ev, err := m.newEvent(route, target, telemetry) if err != nil { m.refused(ctx, route, target, errorCodeEventBuild, err) @@ -402,7 +405,9 @@ func (m *DomainManager) countUnsupported(telemType synchronization.TelemetryType } v, _ = m.unsupportedOpts.LoadOrStore(key, otelmetric.WithAttributes(attrs...)) } - m.metrics.unsupported.Add(context.Background(), 1, v.(otelmetric.AddOption)) + ctx, cancel := m.eng.NewCtx() + defer cancel() + m.metrics.unsupported.Add(ctx, 1, v.(otelmetric.AddOption)) } func (m *DomainManager) warnUnsupportedType(telemType synchronization.TelemetryType) { From 70f4b296c3bb7c8e5ce7f8cb7e411f2c75ba537f Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Wed, 18 Nov 2026 12:46:51 -0500 Subject: [PATCH 21/23] telemetry: promote the telemetry manager interface to telemetry.Manager Move the Service + MonitoringEndpointGenerator interface from core/services/chainlink into the telemetry package as Manager, and rename the WSRPC struct it displaced to WSRPCManager (NewManager -> NewWSRPCManager). The service name stays "TelemetryManager", so logs and health checks are unchanged. Co-Authored-By: Claude Opus 5.5 --- core/services/chainlink/telemetry_manager.go | 13 ++------ .../chainlink/telemetry_manager_test.go | 2 +- core/services/telemetry/common.go | 9 ++++++ core/services/telemetry/domain_manager.go | 8 ++--- core/services/telemetry/manager.go | 21 ++++++++----- core/services/telemetry/manager_test.go | 30 +++++++++---------- 6 files changed, 45 insertions(+), 38 deletions(-) diff --git a/core/services/chainlink/telemetry_manager.go b/core/services/chainlink/telemetry_manager.go index 19e539b6d3a..1d3d6fce7a3 100644 --- a/core/services/chainlink/telemetry_manager.go +++ b/core/services/chainlink/telemetry_manager.go @@ -5,25 +5,18 @@ import ( "fmt" "github.com/smartcontractkit/chainlink-common/pkg/beholder" - commonsrv "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/config" "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/keystore" "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" ) -// telemetryManager generates the node's telemetry endpoints. -type telemetryManager interface { - commonsrv.Service - telemetry.MonitoringEndpointGenerator -} - // newTelemetryManager returns the DomainManager, which sends telemetry through Beholder's -// per-domain emitters, when TelemetryIngress.BeholderEnabled is set; otherwise the WSRPC Manager. +// per-domain emitters, when TelemetryIngress.BeholderEnabled is set; otherwise the WSRPCManager. func newTelemetryManager(cfg config.TelemetryIngress, beholderClient *beholder.Client, csaKeystore keystore.CSA, - csaPubKeyHex string, lggr logger.SugaredLogger) (telemetryManager, error) { + csaPubKeyHex string, lggr logger.SugaredLogger) (telemetry.Manager, error) { if !cfg.BeholderEnabled() { - return telemetry.NewManager(cfg, csaKeystore, lggr), nil + return telemetry.NewWSRPCManager(cfg, csaKeystore, lggr), nil } // Checked before beholderClient becomes an interface value, where a nil pointer is not nil. if beholderClient == nil { diff --git a/core/services/chainlink/telemetry_manager_test.go b/core/services/chainlink/telemetry_manager_test.go index 73fffc3d6a0..4c2b7361690 100644 --- a/core/services/chainlink/telemetry_manager_test.go +++ b/core/services/chainlink/telemetry_manager_test.go @@ -41,7 +41,7 @@ func TestNewTelemetryManager(t *testing.T) { t.Parallel() m, err := newTelemetryManager(telemetryIngress(t, false).TelemetryIngress(), nil, ksmocks.NewCSA(t), "abc123", lggr) require.NoError(t, err) - assert.IsType(t, &telemetry.Manager{}, m) + assert.IsType(t, &telemetry.WSRPCManager{}, m) }) t.Run("flag on without a Beholder client: an error, not a panic", func(t *testing.T) { t.Parallel() diff --git a/core/services/telemetry/common.go b/core/services/telemetry/common.go index 7eb049e35e9..311e9caabd4 100644 --- a/core/services/telemetry/common.go +++ b/core/services/telemetry/common.go @@ -3,9 +3,18 @@ package telemetry import ( ocrtypes "github.com/smartcontractkit/libocr/commontypes" + "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) +// Manager is the node's telemetry service: it runs for the node's lifetime and generates +// the monitoring endpoints that jobs send telemetry through. WSRPCManager and DomainManager +// implement it. +type Manager interface { + services.Service + MonitoringEndpointGenerator +} + type MonitoringEndpointGenerator interface { GenMonitoringEndpoint(network string, chainID string, contractID string, telemType synchronization.TelemetryType) ocrtypes.MonitoringEndpoint GenMultitypeMonitoringEndpoint(network string, chainID string, contractID string) MultitypeMonitoringEndpoint diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index 8a99f538708..2b0835d1ef3 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -81,11 +81,11 @@ const ( errorCodeEventBuild = "event_build" ) -var _ MonitoringEndpointGenerator = (*DomainManager)(nil) +var _ Manager = (*DomainManager)(nil) -// DomainManager is a MonitoringEndpointGenerator that sends telemetry through Beholder's -// per-domain emitters. Each supported telemetry type maps to a domain; endpoints bind their -// route when created, so sending does no lookups. Network and chain only describe the event: +// DomainManager is the Manager that sends telemetry through Beholder's per-domain +// emitters. Each supported telemetry type maps to a domain; endpoints bind their route +// when created, so sending does no lookups. Network and chain only describe the event: // nothing is configured or routed per network. type DomainManager struct { services.Service diff --git a/core/services/telemetry/manager.go b/core/services/telemetry/manager.go index 9b24d46c627..a9394db02c6 100644 --- a/core/services/telemetry/manager.go +++ b/core/services/telemetry/manager.go @@ -18,7 +18,12 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) -type Manager struct { +var _ Manager = (*WSRPCManager)(nil) + +// WSRPCManager is the Manager that sends telemetry to the TelemetryIngress.Endpoints +// configured per network and chain, over WSRPC, or through ChIP Ingress when +// TelemetryIngress.ChipIngressEnabled is set. +type WSRPCManager struct { services.Service eng *services.Engine @@ -46,15 +51,15 @@ type telemetryEndpoint struct { chipIngressClient synchronization.ChipIngressService } -// NewManager create a new telemetry manager that is responsible for configuring telemetry agents and generating the defined telemetry endpoints and monitoring endpoints -func NewManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, lggr common.Logger) *Manager { +// NewWSRPCManager create a new telemetry manager that is responsible for configuring telemetry agents and generating the defined telemetry endpoints and monitoring endpoints +func NewWSRPCManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, lggr common.Logger) *WSRPCManager { var chipIngressClient chipingress.Client if cfg.ChipIngressEnabled() { lggr.Info("ChIP Ingress is enabled for telemetry") chipIngressClient = beholder.GetClient().Chip } - m := &Manager{ + m := &WSRPCManager{ bufferSize: cfg.BufferSize(), ks: csaKeyStore, logging: cfg.Logging(), @@ -83,7 +88,7 @@ func NewManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, lggr comm } // GenMonitoringEndpoint creates a new monitoring endpoints based on the existing available endpoints defined in the core config TOML, if no endpoint for the network and chainID exists, a NOOP agent will be used and the telemetry will not be sent -func (m *Manager) GenMonitoringEndpoint(network string, chainID string, contractID string, telemType synchronization.TelemetryType) commontypes.MonitoringEndpoint { +func (m *WSRPCManager) GenMonitoringEndpoint(network string, chainID string, contractID string, telemType synchronization.TelemetryType) commontypes.MonitoringEndpoint { e, found := m.getEndpoint(network, chainID) if !found { @@ -108,7 +113,7 @@ func (m *Manager) GenMonitoringEndpoint(network string, chainID string, contract return NewTypedIngressAgent(e.client, network, chainID, contractID, telemType) } -func (m *Manager) GenMultitypeMonitoringEndpoint(network string, chainID string, contractID string) MultitypeMonitoringEndpoint { +func (m *WSRPCManager) GenMultitypeMonitoringEndpoint(network string, chainID string, contractID string) MultitypeMonitoringEndpoint { e, found := m.getEndpoint(network, chainID) if !found { @@ -133,7 +138,7 @@ func (m *Manager) GenMultitypeMonitoringEndpoint(network string, chainID string, return NewMultiIngressAgent(e.client, network, chainID, contractID) } -func (m *Manager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Logger, cfg config.TelemetryIngress) (services.Service, error) { +func (m *WSRPCManager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Logger, cfg config.TelemetryIngress) (services.Service, error) { if e.Network() == "" { return nil, errors.New("cannot add telemetry endpoint, network cannot be empty") } @@ -191,7 +196,7 @@ func (m *Manager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Log return te.client, nil } -func (m *Manager) getEndpoint(network string, chainID string) (*telemetryEndpoint, bool) { +func (m *WSRPCManager) getEndpoint(network string, chainID string) (*telemetryEndpoint, bool) { for _, e := range m.endpoints { if e.Network == strings.ToUpper(network) && e.ChainID == strings.ToUpper(chainID) { return e, true diff --git a/core/services/telemetry/manager_test.go b/core/services/telemetry/manager_test.go index 143066e0894..c6a012f06fc 100644 --- a/core/services/telemetry/manager_test.go +++ b/core/services/telemetry/manager_test.go @@ -51,14 +51,14 @@ func TestManagerAgents(t *testing.T) { ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) require.Equal(t, "*synchronization.telemetryIngressBatchClient", reflect.TypeOf(tm.endpoints[0].client).String()) me := tm.GenMonitoringEndpoint("network-1", "network-1-chainID-1", "", "") assert.Equal(t, "*telemetry.TypedIngressAgentBatch", reflect.TypeOf(me).String()) tic = setupMockConfig(t, false, false) tic.On("Endpoints").Return([]config.TelemetryIngressEndpoint{te}) - tm = NewManager(tic, ks, lggr) + tm = NewWSRPCManager(tic, ks, lggr) require.Equal(t, "*synchronization.telemetryIngressClient", reflect.TypeOf(tm.endpoints[0].client).String()) me = tm.GenMonitoringEndpoint("network-1", "network-1-chainID-1", "", "") assert.Equal(t, "*telemetry.TypedIngressAgent", reflect.TypeOf(me).String()) @@ -154,7 +154,7 @@ func TestNewManager(t *testing.T) { key := csakey.MustNewV2XXXTestingOnly(big.NewInt(0)) ks.On("GetAll").Return([]csakey.KeyV2{key}, nil) ks.On("Get", key.ID()).Return(key, nil) - m := NewManager(tic, ks, lggr) + m := NewWSRPCManager(tic, ks, lggr) require.Equal(t, uint(123), m.bufferSize) require.Equal(t, ks, m.ks) @@ -203,7 +203,7 @@ func TestCorrectEndpointRouting(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) type testEndpoint struct { network string @@ -290,7 +290,7 @@ func TestManager_ChipIngressClient(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) assert.Nil(t, tm.chipIngressClient) }) @@ -300,7 +300,7 @@ func TestManager_ChipIngressClient(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) assert.NotNil(t, tm.chipIngressClient) }) } @@ -319,7 +319,7 @@ func TestManager_ChipIngressEndpoint(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) require.Len(t, tm.endpoints, 1) assert.NotNil(t, tm.endpoints[0].chipIngressClient) assert.Nil(t, tm.endpoints[0].client) @@ -341,7 +341,7 @@ func TestManager_ChipIngressEndpoint(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) require.Len(t, tm.endpoints, 1) assert.Nil(t, tm.endpoints[0].chipIngressClient) assert.NotNil(t, tm.endpoints[0].client) @@ -362,7 +362,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) me := tm.GenMonitoringEndpoint("EVM", "1", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) }) @@ -380,7 +380,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.DebugLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) // Start and cleanup the manager to ensure background goroutines are properly stopped require.NoError(t, tm.Start(t.Context())) t.Cleanup(func() { @@ -410,7 +410,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) // Invalid network/chainID combination that doesn't exist in chain-selectors me := tm.GenMonitoringEndpoint("INVALID", "999", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.NoopAgent", reflect.TypeOf(me).String()) @@ -445,7 +445,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) me := tm.GenMonitoringEndpoint("EVM", "1", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.TypedIngressAgentBatch", reflect.TypeOf(me).String()) }) @@ -465,7 +465,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) me := tm.GenMultitypeMonitoringEndpoint("EVM", "137", "0x456") assert.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) }) @@ -483,7 +483,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) // Invalid network/chainID combination me := tm.GenMultitypeMonitoringEndpoint("INVALID", "888", "0x456") assert.Equal(t, "*telemetry.NoopAgent", reflect.TypeOf(me).String()) @@ -518,7 +518,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) me := tm.GenMultitypeMonitoringEndpoint("EVM", "137", "0x456") assert.Equal(t, "*telemetry.MultiIngressAgentBatch", reflect.TypeOf(me).String()) }) From 2a9bd9f1a21bfce37f62def32c50c9630812ab6a Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:05:44 -0500 Subject: [PATCH 22/23] telemetry: reuse one lifecycle context for DomainManager sends send and countUnsupported called m.eng.NewCtx() per message, which starts a goroutine each time on the telemetry hot path. Create one context in NewDomainManager and cancel it in the service's Close hook instead; sends are still tied to the manager's lifecycle and are refused after Close. Co-Authored-By: Claude Opus 5.5 --- core/services/telemetry/domain_manager.go | 21 ++++++++++-------- .../services/telemetry/domain_manager_test.go | 22 ++++++++++++++++++- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index 2b0835d1ef3..a5619934d08 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -91,6 +91,11 @@ type DomainManager struct { services.Service eng *services.Engine + // ctx is cancelled when the manager closes. SendLog has no context (libocr's + // MonitoringEndpoint), so sends and counters use this one. + ctx context.Context + cancel context.CancelFunc + routes map[synchronization.TelemetryType]*telemetryRoute csaPubKeyHex string meterProvider otelmetric.MeterProvider @@ -196,7 +201,11 @@ func NewDomainManager(src DomainEmitterSource, csaPubKeyHex string, lggr logger. } } - m.Service, m.eng = services.Config{Name: "DomainManager"}.NewServiceEngine(lggr) + m.ctx, m.cancel = context.WithCancel(context.Background()) + m.Service, m.eng = services.Config{ + Name: "DomainManager", + Close: func() error { m.cancel(); return nil }, + }.NewServiceEngine(lggr) return m, nil } @@ -296,11 +305,7 @@ func (m *DomainManager) reportOnce(key string, report func()) { // are logged (rate-limited) by the emitter, and events that never reach it are logged here, once // per route and error code, since they mean telemetry is being lost. func (m *DomainManager) send(route *telemetryRoute, target eventTarget, telemetry []byte) { - // SendLog has no context (libocr's MonitoringEndpoint), so tie it to the - // service lifecycle. Cancelling on return is safe: after the emitter's sync - // queue check, only otel adds (which tolerate cancellation) read it. - ctx, cancel := m.eng.NewCtx() - defer cancel() + ctx := m.ctx ev, err := m.newEvent(route, target, telemetry) if err != nil { m.refused(ctx, route, target, errorCodeEventBuild, err) @@ -405,9 +410,7 @@ func (m *DomainManager) countUnsupported(telemType synchronization.TelemetryType } v, _ = m.unsupportedOpts.LoadOrStore(key, otelmetric.WithAttributes(attrs...)) } - ctx, cancel := m.eng.NewCtx() - defer cancel() - m.metrics.unsupported.Add(ctx, 1, v.(otelmetric.AddOption)) + m.metrics.unsupported.Add(m.ctx, 1, v.(otelmetric.AddOption)) } func (m *DomainManager) warnUnsupportedType(telemType synchronization.TelemetryType) { diff --git a/core/services/telemetry/domain_manager_test.go b/core/services/telemetry/domain_manager_test.go index 42dbb7f5bfd..edb6e79998f 100644 --- a/core/services/telemetry/domain_manager_test.go +++ b/core/services/telemetry/domain_manager_test.go @@ -45,7 +45,11 @@ type fakeDomainEmitter struct { func (f *fakeDomainEmitter) Emit(context.Context, []byte, ...any) error { return nil } func (f *fakeDomainEmitter) Close() error { return nil } -func (f *fakeDomainEmitter) EmitEvent(_ context.Context, event chipingress.CloudEvent, callback func(error)) error { +func (f *fakeDomainEmitter) EmitEvent(ctx context.Context, event chipingress.CloudEvent, callback func(error)) error { + // Like the real emitter, refuse events whose context is already done. + if err := ctx.Err(); err != nil { + return err + } if f.emitErr != nil { return f.emitErr } @@ -421,6 +425,22 @@ func TestDomainManager_CountsEachEventsFate(t *testing.T) { } } +func TestDomainManager_RefusesSendsAfterClose(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, reader := newTestDomainManager(t, src) + require.NoError(t, m.Start(t.Context())) + ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median) + require.NoError(t, m.Close()) + + ep.SendLog([]byte("p")) + + assert.Empty(t, src[beholder.DomainDataFeeds].sent()) + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesDropped, + attribute.String("telemetry_type", string(synchronization.OCR2Median)), + attribute.String("error_code", errorCodeEmitRejected))) +} + func TestDomainManager_ConcurrentSends(t *testing.T) { t.Parallel() src := newFakeEmitterSource() From 5cac6d015aeda86eab97af7584b8e81992c36bfc Mon Sep 17 00:00:00 2001 From: 4of9 <177086174+4of9@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:16:52 -0500 Subject: [PATCH 23/23] telemetry: allow DomainManager's lifecycle context field (containedctx) libocr's MonitoringEndpoint.SendLog takes no context, so the manager keeps one cancelled on Close; same exception as ocr3_1/beholderwrapper. Co-Authored-By: Claude Opus 5.5 --- core/services/telemetry/domain_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go index a5619934d08..d7bb05904cd 100644 --- a/core/services/telemetry/domain_manager.go +++ b/core/services/telemetry/domain_manager.go @@ -93,7 +93,7 @@ type DomainManager struct { // ctx is cancelled when the manager closes. SendLog has no context (libocr's // MonitoringEndpoint), so sends and counters use this one. - ctx context.Context + ctx context.Context //nolint:containedctx // libocr's MonitoringEndpoint.SendLog takes no ctx cancel context.CancelFunc routes map[synchronization.TelemetryType]*telemetryRoute