diff --git a/.changeset/telemetry-domain-manager.md b/.changeset/telemetry-domain-manager.md new file mode 100644 index 00000000000..5eac5be4806 --- /dev/null +++ b/.changeset/telemetry-domain-manager.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +#added `TelemetryIngress.BeholderEnabled` (off by default): sends node telemetry through Beholder's per-domain emitters instead of the WSRPC `TelemetryIngress.Endpoints`. Requires `Telemetry.Enabled` and `Telemetry.ChipIngressEndpoint`; cannot be combined with `TelemetryIngress.ChipIngressEnabled`. diff --git a/core/cmd/beholder_client_test.go b/core/cmd/beholder_client_test.go new file mode 100644 index 00000000000..9c2330249bb --- /dev/null +++ b/core/cmd/beholder_client_test.go @@ -0,0 +1,69 @@ +package cmd + +import ( + "fmt" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/mocks" +) + +func TestNewBeholderClient_DomainEmittersFollowTelemetryIngressBeholderEnabled(t *testing.T) { + t.Parallel() + // A local gRPC server with no services answers OTel exports at once, so Close is quick. + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer() + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + beholderTOML := fmt.Sprintf(` +[Telemetry] +Enabled = true +Endpoint = '%s' +InsecureConnection = true +ChipIngressEndpoint = '%s' +ChipIngressInsecureConnection = true +`, lis.Addr(), lis.Addr()) + for _, tt := range []struct { + name string + toml string + enabled bool + }{ + {name: "off", toml: beholderTOML}, + {name: "on", toml: beholderTOML + "\n[TelemetryIngress]\nBeholderEnabled = true\n", enabled: true}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + cfg, err := chainlink.GeneralConfigOpts{ConfigStrings: []string{tt.toml}}.New() + require.NoError(t, err) + + ks := mocks.NewMaster(t) + ks.On("CSA").Return(mocks.NewCSA(t)).Maybe() + + client, err := newBeholderClient(logger.TestLogger(t), ks, cfg.Tracing(), cfg, cfg.Telemetry(), "abc123", nil) + require.NoError(t, err) + require.NoError(t, client.Start(t.Context())) + t.Cleanup(func() { + // OTel shutdown may fail to flush to the absent collector; the domain emitters + // must close cleanly. + if closeErr := client.Close(); closeErr != nil { + assert.NotContains(t, closeErr.Error(), "client connection is closing") + } + }) + + _, err = client.ForDomain(beholder.DomainDataFeeds) + if tt.enabled { + require.NoError(t, err, "BeholderEnabled builds the domain emitters") + } else { + require.Error(t, err, "no domain emitters unless BeholderEnabled") + } + }) + } +} diff --git a/core/cmd/shell.go b/core/cmd/shell.go index 9f2f36fed74..91f179636a9 100644 --- a/core/cmd/shell.go +++ b/core/cmd/shell.go @@ -160,21 +160,23 @@ func newBeholderClient( ChipIngressEmitterGRPCEndpoint: cfgTelemetry.ChipIngressEndpoint(), ChipIngressInsecureConnection: cfgTelemetry.ChipIngressInsecureConnection(), ChipIngressBatchEmitterEnabled: cfgTelemetry.ChipIngressBatchEmitterEnabled(), - ChipIngressBufferSize: cfgTelemetry.ChipIngressBufferSize(), - ChipIngressMaxBatchSize: cfgTelemetry.ChipIngressMaxBatchSize(), - ChipIngressMaxConcurrentSends: cfgTelemetry.ChipIngressMaxConcurrentSends(), - ChipIngressSendInterval: cfgTelemetry.ChipIngressSendInterval(), - ChipIngressSendTimeout: cfgTelemetry.ChipIngressSendTimeout(), - ChipIngressDrainTimeout: cfgTelemetry.ChipIngressDrainTimeout(), - ChipIngressMaxGRPCRequestSize: cfgTelemetry.ChipIngressMaxGRPCRequestSize(), - ChipIngressLogger: lggr, - LogStreamingEnabled: cfgTelemetry.LogStreamingEnabled(), - LogLevel: cfgTelemetry.LogLevel(), - LogBatchProcessor: cfgTelemetry.LogBatchProcessor(), - LogExportTimeout: cfgTelemetry.LogExportTimeout(), - LogExportMaxBatchSize: cfgTelemetry.LogExportMaxBatchSize(), - LogExportInterval: cfgTelemetry.LogExportInterval(), - LogMaxQueueSize: cfgTelemetry.LogMaxQueueSize(), + // TelemetryIngress.BeholderEnabled sends node telemetry through the per-domain emitters. + ChipDomainEmittersEnabled: cfgApp.TelemetryIngress().BeholderEnabled(), + ChipIngressBufferSize: cfgTelemetry.ChipIngressBufferSize(), + ChipIngressMaxBatchSize: cfgTelemetry.ChipIngressMaxBatchSize(), + ChipIngressMaxConcurrentSends: cfgTelemetry.ChipIngressMaxConcurrentSends(), + ChipIngressSendInterval: cfgTelemetry.ChipIngressSendInterval(), + ChipIngressSendTimeout: cfgTelemetry.ChipIngressSendTimeout(), + ChipIngressDrainTimeout: cfgTelemetry.ChipIngressDrainTimeout(), + ChipIngressMaxGRPCRequestSize: cfgTelemetry.ChipIngressMaxGRPCRequestSize(), + ChipIngressLogger: lggr, + LogStreamingEnabled: cfgTelemetry.LogStreamingEnabled(), + LogLevel: cfgTelemetry.LogLevel(), + LogBatchProcessor: cfgTelemetry.LogBatchProcessor(), + LogExportTimeout: cfgTelemetry.LogExportTimeout(), + LogExportMaxBatchSize: cfgTelemetry.LogExportMaxBatchSize(), + LogExportInterval: cfgTelemetry.LogExportInterval(), + LogMaxQueueSize: cfgTelemetry.LogMaxQueueSize(), // Due to OpenTelemetry semantics, histogram bucket boundaries must be set // when the Beholder client is constructed. MetricViews: metricViews(), @@ -269,14 +271,14 @@ func (s *Shell) configExitErr(validateFn func() error) cli.ExitCoder { // AppFactory implements the NewApplication method. type AppFactory interface { - NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master) (chainlink.Application, error) + NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master, beholderClient *beholder.Client) (chainlink.Application, error) } // ChainlinkAppFactory is used to create a new Application. type ChainlinkAppFactory struct{} // NewApplication returns a new instance of the node with the given config. -func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master) (app chainlink.Application, err error) { +func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, appRegisterer prometheus.Registerer, ds sqlutil.DataSource, keyStore keystore.Master, beholderClient *beholder.Client) (app chainlink.Application, err error) { err = migrate.SetMigrationENVVars(cfg.EVMConfigs()) if err != nil { return nil, err @@ -315,6 +317,7 @@ func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.G return chainlink.NewApplication(ctx, chainlink.ApplicationOpts{ Opts: creOpts, + BeholderClient: beholderClient, Config: cfg, DS: ds, KeyStore: keyStore, diff --git a/core/cmd/shell_local.go b/core/cmd/shell_local.go index 2e32712b590..77ae6d195ab 100644 --- a/core/cmd/shell_local.go +++ b/core/cmd/shell_local.go @@ -428,7 +428,7 @@ func (s *Shell) runNode(c *cli.Context) error { os.Exit(-1) }) - app, err := s.AppFactory.NewApplication(rootCtx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore) + app, err := s.AppFactory.NewApplication(rootCtx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore, s.BeholderClient) if err != nil { return s.errorOut(fmt.Errorf("fatal error instantiating application: %w", err)) } @@ -780,7 +780,7 @@ func (s *Shell) RebroadcastTransactions(c *cli.Context) (err error) { lggr := logger.Sugared(s.Logger.Named("RebroadcastTransactions")) - app, err := s.AppFactory.NewApplication(ctx, s.Config, lggr, s.Registerer, s.DS, s.KeyStore) + app, err := s.AppFactory.NewApplication(ctx, s.Config, lggr, s.Registerer, s.DS, s.KeyStore, s.BeholderClient) if err != nil { return s.errorOut(fmt.Errorf("fatal error instantiating application: %w", err)) } @@ -1145,7 +1145,7 @@ func (s *Shell) RemoveBlocks(c *cli.Context) error { s.afterNode(lggr) }) - app, err := s.AppFactory.NewApplication(ctx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore) + app, err := s.AppFactory.NewApplication(ctx, s.Config, s.Logger, s.Registerer, s.DS, s.KeyStore, s.BeholderClient) if err != nil { return s.errorOut(fmt.Errorf("fatal error instantiating application: %w", err)) } diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 80f81e5efb2..0ac7ff96f07 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -116,6 +116,15 @@ SendTimeout = '10s' # Default UseBatchSend = true # Default # ChipIngressEnabled enables sending telemetry to CHIP Ingress. ChipIngressEnabled = false # Default +# BeholderEnabled sends telemetry through Beholder, one emitter per product domain, instead of the Endpoints below. +# When set, Endpoints and the settings above (BufferSize, MaxBatchSize, SendInterval, SendTimeout, UseBatchSend, UniConn, Logging) +# are ignored: the Telemetry.ChipIngress* settings size one buffer per domain, shared by all of that domain's jobs. +# Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint, and chip ingress that serves PublishBatchNoAck for every domain +# (there is no fallback). Cannot be combined with ChipIngressEnabled. +# Telemetry is sent for every chain the node's jobs run on: Endpoints no longer select chains. Supported types are ocr, +# ocr2-median, ocr3-data-feeds, enhanced-ea, ocr3-mercury, ocr3_1-mercury, enhanced-ea-mercury, llo-observation, llo-outcome, +# pipeline-bridge, ocr2-ccip-commit, ocr2-ccip-exec, ocr3-ccip-commit, ocr3-ccip-exec and head-report; other types are dropped. +BeholderEnabled = false # Default [[TelemetryIngress.Endpoints]] # Example # Network aka EVM, Solana, Starknet diff --git a/core/config/mocks/telemetry_ingress.go b/core/config/mocks/telemetry_ingress.go index 94e83549e51..2cc9532131c 100644 --- a/core/config/mocks/telemetry_ingress.go +++ b/core/config/mocks/telemetry_ingress.go @@ -22,6 +22,51 @@ func (_m *TelemetryIngress) EXPECT() *TelemetryIngress_Expecter { return &TelemetryIngress_Expecter{mock: &_m.Mock} } +// BeholderEnabled provides a mock function with no fields +func (_m *TelemetryIngress) BeholderEnabled() bool { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for BeholderEnabled") + } + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// TelemetryIngress_BeholderEnabled_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BeholderEnabled' +type TelemetryIngress_BeholderEnabled_Call struct { + *mock.Call +} + +// BeholderEnabled is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) BeholderEnabled() *TelemetryIngress_BeholderEnabled_Call { + return &TelemetryIngress_BeholderEnabled_Call{Call: _e.mock.On("BeholderEnabled")} +} + +func (_c *TelemetryIngress_BeholderEnabled_Call) Run(run func()) *TelemetryIngress_BeholderEnabled_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_BeholderEnabled_Call) Return(_a0 bool) *TelemetryIngress_BeholderEnabled_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_BeholderEnabled_Call) RunAndReturn(run func() bool) *TelemetryIngress_BeholderEnabled_Call { + _c.Call.Return(run) + return _c +} + // BufferSize provides a mock function with no fields func (_m *TelemetryIngress) BufferSize() uint { ret := _m.Called() diff --git a/core/config/telemetry_ingress_config.go b/core/config/telemetry_ingress_config.go index 918af269601..867e7fe2497 100644 --- a/core/config/telemetry_ingress_config.go +++ b/core/config/telemetry_ingress_config.go @@ -15,6 +15,7 @@ type TelemetryIngress interface { UseBatchSend() bool Endpoints() []TelemetryIngressEndpoint ChipIngressEnabled() bool + BeholderEnabled() bool } type TelemetryIngressEndpoint interface { diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 460d9512c7e..8b293a3dbc2 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -141,6 +141,15 @@ func (c *Core) ValidateConfig() (err error) { } } + if *c.TelemetryIngress.BeholderEnabled { + if *c.TelemetryIngress.ChipIngressEnabled { + err = errors.Join(err, configutils.InvalidError{Name: "TelemetryIngress.BeholderEnabled", Value: true, Msg: "cannot be combined with TelemetryIngress.ChipIngressEnabled"}) + } + if !*c.Telemetry.Enabled || *c.Telemetry.ChipIngressEndpoint == "" { + err = errors.Join(err, configutils.InvalidError{Name: "TelemetryIngress.BeholderEnabled", Value: true, Msg: "requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint"}) + } + } + return err } @@ -982,6 +991,7 @@ type TelemetryIngress struct { UseBatchSend *bool Endpoints []TelemetryIngressEndpoint `toml:",omitempty"` ChipIngressEnabled *bool + BeholderEnabled *bool } type TelemetryIngressEndpoint struct { @@ -1019,6 +1029,9 @@ func (t *TelemetryIngress) setFrom(f *TelemetryIngress) { if v := f.ChipIngressEnabled; v != nil { t.ChipIngressEnabled = v } + if v := f.BeholderEnabled; v != nil { + t.BeholderEnabled = v + } } type AuditLogger struct { diff --git a/core/internal/cltest/mocks.go b/core/internal/cltest/mocks.go index 3a6af54bc19..81d68f27a7d 100644 --- a/core/internal/cltest/mocks.go +++ b/core/internal/cltest/mocks.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-evm/pkg/chains/legacyevm" @@ -90,7 +91,7 @@ type InstanceAppFactory struct { } // NewApplication creates a new application with specified config -func (f InstanceAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master) (chainlink.Application, error) { +func (f InstanceAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master, *beholder.Client) (chainlink.Application, error) { return f.App, nil } @@ -98,7 +99,7 @@ type seededAppFactory struct { Application chainlink.Application } -func (s seededAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master) (chainlink.Application, error) { +func (s seededAppFactory) NewApplication(context.Context, chainlink.GeneralConfig, logger.Logger, prometheus.Registerer, sqlutil.DataSource, keystore.Master, *beholder.Client) (chainlink.Application, error) { return noopStopApplication{s.Application}, nil } diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index 0f0f50fb310..e8f67d3d1ae 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -88,7 +88,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/shardorchestrator" "github.com/smartcontractkit/chainlink/v2/core/services/standardcapabilities" "github.com/smartcontractkit/chainlink/v2/core/services/streams" - "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" "github.com/smartcontractkit/chainlink/v2/core/services/vrf" workflowstore "github.com/smartcontractkit/chainlink/v2/core/services/workflows/store" "github.com/smartcontractkit/chainlink/v2/core/sessions" @@ -214,6 +213,8 @@ type ApplicationOpts struct { NewOracleFactoryFn standardcapabilities.NewOracleFactoryFn EVMFactoryConfigFn func(*EVMFactoryConfig) DonTimeStore *dontime.Store + // BeholderClient is the node's Beholder client. Required when TelemetryIngress.BeholderEnabled is set. + BeholderClient *beholder.Client // LLOKeyValueDatabaseFactory replaces the on-disk store LLO v31 jobs use. // Tests only. LLOKeyValueDatabaseFactory ocr3_1types.KeyValueDatabaseFactory @@ -567,7 +568,10 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err globalLogger.Info("Nurse service (automatic pprof profiling) is disabled") } - telemetryManager := telemetry.NewManager(cfg.TelemetryIngress(), csaKeystore, globalLogger) + telemetryManager, err := newTelemetryManager(cfg.TelemetryIngress(), opts.BeholderClient, csaKeystore, csaPubKeyHex, globalLogger) + if err != nil { + return nil, err + } srvcs = append(srvcs, telemetryManager) backupCfg := cfg.Database().Backup() diff --git a/core/services/chainlink/config_telemetry_ingress.go b/core/services/chainlink/config_telemetry_ingress.go index 21c50d967a0..c633fa10e66 100644 --- a/core/services/chainlink/config_telemetry_ingress.go +++ b/core/services/chainlink/config_telemetry_ingress.go @@ -78,3 +78,7 @@ func (t *telemetryIngressEndpointConfig) URL() *url.URL { func (t *telemetryIngressEndpointConfig) ServerPubKey() string { return *t.c.ServerPubKey } + +func (t *telemetryIngressConfig) BeholderEnabled() bool { + return *t.c.BeholderEnabled +} diff --git a/core/services/chainlink/config_telemetry_ingress_test.go b/core/services/chainlink/config_telemetry_ingress_test.go index 2bcdf3c80b1..228df4cd6b2 100644 --- a/core/services/chainlink/config_telemetry_ingress_test.go +++ b/core/services/chainlink/config_telemetry_ingress_test.go @@ -1,12 +1,14 @@ package chainlink import ( + "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" "github.com/smartcontractkit/chainlink/v2/core/config/toml" ) @@ -56,3 +58,53 @@ func TestTelemetryIngressConfig_ChipIngressEnabled(t *testing.T) { assert.True(t, config.ChipIngressEnabled()) }) } + +func TestTelemetryIngressConfig_BeholderEnabled(t *testing.T) { + t.Parallel() + trueVal, falseVal := true, false + assert.True(t, (&telemetryIngressConfig{c: toml.TelemetryIngress{BeholderEnabled: &trueVal}}).BeholderEnabled()) + assert.False(t, (&telemetryIngressConfig{c: toml.TelemetryIngress{BeholderEnabled: &falseVal}}).BeholderEnabled()) +} + +func TestCoreValidateConfig_TelemetryIngressBeholderEnabled(t *testing.T) { + t.Parallel() + const enabled = ` +[TelemetryIngress] +BeholderEnabled = true +` + const beholderChip = ` +[Telemetry] +Enabled = true +ChipIngressEndpoint = 'chip.example.com:443' +` + for _, tt := range []struct { + name string + toml string + wantErr string // empty: the BeholderEnabled rule must not fire + }{ + {name: "off by default", toml: ``}, + {name: "requires Beholder telemetry", toml: enabled, + wantErr: "TelemetryIngress.BeholderEnabled: invalid value (true): requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint"}, + {name: "requires a chip ingress endpoint", toml: enabled + "\n[Telemetry]\nEnabled = true\n", + wantErr: "TelemetryIngress.BeholderEnabled: invalid value (true): requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint"}, + {name: "cannot combine with ChipIngressEnabled", toml: "[TelemetryIngress]\nBeholderEnabled = true\nChipIngressEnabled = true\n" + beholderChip, + wantErr: "TelemetryIngress.BeholderEnabled: invalid value (true): cannot be combined with TelemetryIngress.ChipIngressEnabled"}, + {name: "valid with Beholder chip ingress", toml: enabled + beholderChip}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + var c Config + require.NoError(t, commonconfig.DecodeTOML(strings.NewReader(tt.toml), &c)) + c.setDefaults() + err := c.ValidateConfig() + if tt.wantErr == "" { + if err != nil { + assert.NotContains(t, err.Error(), "TelemetryIngress.BeholderEnabled") + } + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 0d9219aa82b..d380a3094b8 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -254,6 +254,7 @@ func TestConfig_Marshal(t *testing.T) { SendTimeout: commonconfig.MustNewDuration(5 * time.Second), UseBatchSend: new(true), ChipIngressEnabled: new(false), + BeholderEnabled: new(false), Endpoints: []toml.TelemetryIngressEndpoint{ { Network: new("EVM"), @@ -915,6 +916,7 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/services/chainlink/telemetry_manager.go b/core/services/chainlink/telemetry_manager.go new file mode 100644 index 00000000000..1d3d6fce7a3 --- /dev/null +++ b/core/services/chainlink/telemetry_manager.go @@ -0,0 +1,37 @@ +package chainlink + +import ( + "errors" + "fmt" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink/v2/core/config" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore" + "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" +) + +// newTelemetryManager returns the DomainManager, which sends telemetry through Beholder's +// per-domain emitters, when TelemetryIngress.BeholderEnabled is set; otherwise the WSRPCManager. +func newTelemetryManager(cfg config.TelemetryIngress, beholderClient *beholder.Client, csaKeystore keystore.CSA, + csaPubKeyHex string, lggr logger.SugaredLogger) (telemetry.Manager, error) { + if !cfg.BeholderEnabled() { + return telemetry.NewWSRPCManager(cfg, csaKeystore, lggr), nil + } + // Checked before beholderClient becomes an interface value, where a nil pointer is not nil. + if beholderClient == nil { + return nil, errors.New("TelemetryIngress.BeholderEnabled requires a Beholder client") + } + // The manager's counters share the client's meter provider, as the emitters' chip + // metrics already do. Nil on a hand-built client, where the option would panic. + var opts []telemetry.DomainManagerOpt + if beholderClient.MeterProvider != nil { + opts = append(opts, telemetry.WithMeterProvider(beholderClient.MeterProvider)) + } + dm, err := telemetry.NewDomainManager(beholderClient, csaPubKeyHex, lggr, opts...) + if err != nil { + return nil, fmt.Errorf("failed to create telemetry domain manager: %w", err) + } + lggr.Info("Telemetry is sent through Beholder; TelemetryIngress.Endpoints and its sending settings are ignored") + return dm, nil +} diff --git a/core/services/chainlink/telemetry_manager_test.go b/core/services/chainlink/telemetry_manager_test.go new file mode 100644 index 00000000000..4c2b7361690 --- /dev/null +++ b/core/services/chainlink/telemetry_manager_test.go @@ -0,0 +1,89 @@ +package chainlink + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" + sdklog "go.opentelemetry.io/otel/sdk/log" + "google.golang.org/grpc" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink/v2/core/logger" + ksmocks "github.com/smartcontractkit/chainlink/v2/core/services/keystore/mocks" + "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" +) + +type nopLogExporter struct{} + +func (nopLogExporter) Export(context.Context, []sdklog.Record) error { return nil } +func (nopLogExporter) Shutdown(context.Context) error { return nil } +func (nopLogExporter) ForceFlush(context.Context) error { return nil } + +func TestNewTelemetryManager(t *testing.T) { + t.Parallel() + telemetryIngress := func(t *testing.T, beholderEnabled bool) GeneralConfig { + t.Helper() + toml := "" + if beholderEnabled { + toml = "[TelemetryIngress]\nBeholderEnabled = true\n" + } + cfg, err := GeneralConfigOpts{ConfigStrings: []string{toml}}.New() + require.NoError(t, err) + return cfg + } + lggr := logger.Sugared(logger.TestLogger(t)) + + t.Run("flag off: the WSRPC manager", func(t *testing.T) { + t.Parallel() + m, err := newTelemetryManager(telemetryIngress(t, false).TelemetryIngress(), nil, ksmocks.NewCSA(t), "abc123", lggr) + require.NoError(t, err) + assert.IsType(t, &telemetry.WSRPCManager{}, m) + }) + t.Run("flag on without a Beholder client: an error, not a panic", func(t *testing.T) { + t.Parallel() + var noClient *beholder.Client + _, err := newTelemetryManager(telemetryIngress(t, true).TelemetryIngress(), noClient, ksmocks.NewCSA(t), "abc123", lggr) + require.Error(t, err) + }) + t.Run("flag on with Beholder telemetry off (the no-op client): an error", func(t *testing.T) { + t.Parallel() + _, err := newTelemetryManager(telemetryIngress(t, true).TelemetryIngress(), beholder.NewNoopClient(), ksmocks.NewCSA(t), "abc123", lggr) + require.Error(t, err, "the no-op client has no domain emitters") + }) + t.Run("flag on with a hand-built client: an error, not a panic", func(t *testing.T) { + t.Parallel() + // A hand-built client has no meter provider and no domain emitters; the + // manager must fall back to the global provider instead of panicking. + _, err := newTelemetryManager(telemetryIngress(t, true).TelemetryIngress(), &beholder.Client{}, ksmocks.NewCSA(t), "abc123", lggr) + require.Error(t, err, "a hand-built client has no domain emitters") + }) + t.Run("flag on: the domain manager on Beholder's domain emitters", func(t *testing.T) { + t.Parallel() + // A local gRPC server with no services answers OTel exports at once, so Close is quick. + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer() + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + client, err := beholder.NewGRPCClient(beholder.Config{ + OtelExporterGRPCEndpoint: lis.Addr().String(), + InsecureConnection: true, + ChipIngressEmitterGRPCEndpoint: lis.Addr().String(), + ChipIngressInsecureConnection: true, + ChipDomainEmittersEnabled: true, + ChipIngressLogger: logger.TestLogger(t), + }, func(...otlploggrpc.Option) (sdklog.Exporter, error) { return nopLogExporter{}, nil }) + require.NoError(t, err) + require.NoError(t, client.Start(t.Context())) + t.Cleanup(func() { _ = client.Close() }) + + m, err := newTelemetryManager(telemetryIngress(t, true).TelemetryIngress(), client, ksmocks.NewCSA(t), "abc123", lggr) + require.NoError(t, err) + assert.IsType(t, &telemetry.DomainManager{}, m) + }) +} diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index 3e6fb0dd366..0da4e4c6a87 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -44,6 +44,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index 06a9b4c014f..cf3a25547c5 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -44,6 +44,7 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 98600eeaa73..a025d3febf0 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -44,6 +44,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = true diff --git a/core/services/telemetry/common.go b/core/services/telemetry/common.go index 7eb049e35e9..311e9caabd4 100644 --- a/core/services/telemetry/common.go +++ b/core/services/telemetry/common.go @@ -3,9 +3,18 @@ package telemetry import ( ocrtypes "github.com/smartcontractkit/libocr/commontypes" + "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) +// Manager is the node's telemetry service: it runs for the node's lifetime and generates +// the monitoring endpoints that jobs send telemetry through. WSRPCManager and DomainManager +// implement it. +type Manager interface { + services.Service + MonitoringEndpointGenerator +} + type MonitoringEndpointGenerator interface { GenMonitoringEndpoint(network string, chainID string, contractID string, telemType synchronization.TelemetryType) ocrtypes.MonitoringEndpoint GenMultitypeMonitoringEndpoint(network string, chainID string, contractID string) MultitypeMonitoringEndpoint diff --git a/core/services/telemetry/domain_manager.go b/core/services/telemetry/domain_manager.go new file mode 100644 index 00000000000..d7bb05904cd --- /dev/null +++ b/core/services/telemetry/domain_manager.go @@ -0,0 +1,489 @@ +package telemetry + +import ( + "context" + "errors" + "fmt" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/metric" + + chainselector "github.com/smartcontractkit/chain-selectors" + "github.com/smartcontractkit/libocr/commontypes" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" +) + +// DomainEmitterSource provides Beholder's per-domain emitters. *beholder.Client implements it. +type DomainEmitterSource interface { + ForDomain(domain beholder.Domain) (beholder.DomainEmitter, error) +} + +// supportedTelemetryDomains lists the telemetry types DomainManager sends and the Beholder domain +// whose emitter carries each: the types ingested in production. Their CloudEvent source and type +// come from synchronization.TelemetryTypeToDomainAndEntity. Every other type is dropped and counted. +// The TelemetryIngress.BeholderEnabled docs (core/config/docs/core.toml) list these types: keep them in sync. +// A package-level map: countUnsupported reads it per message and must not allocate. +var supportedTelemetryDomains = map[synchronization.TelemetryType]beholder.Domain{ + synchronization.OCR: beholder.DomainDataFeeds, + synchronization.OCR2Median: beholder.DomainDataFeeds, + synchronization.OCR3DataFeeds: beholder.DomainDataFeeds, + synchronization.EnhancedEA: beholder.DomainDataFeeds, + synchronization.OCR3Mercury: beholder.DomainDataStreams, + synchronization.OCR3_1Mercury: beholder.DomainDataStreams, + synchronization.EnhancedEAMercury: beholder.DomainDataStreams, + synchronization.LLOObservation: beholder.DomainDataStreams, + synchronization.LLOOutcome: beholder.DomainDataStreams, + synchronization.PipelineBridge: beholder.DomainDataStreams, + synchronization.OCR2CCIPCommit: beholder.DomainCCIP, + synchronization.OCR2CCIPExec: beholder.DomainCCIP, + synchronization.OCR3CCIPCommit: beholder.DomainCCIP, + synchronization.OCR3CCIPExec: beholder.DomainCCIP, + // TODO(INFOPLAT-19709): head reports are emitted for every chain on every node; confirm + // ccip is their domain. + synchronization.HeadReport: beholder.DomainCCIP, +} + +// Metric names and attribute values. The manager's metrics are OTel (Beholder) metrics, so they +// reach us from every node. The counters carry the route's domain and telemetry_type, the target +// chain's chain_selector, and error_code or reason where they apply. +// +// It also increments the existing Prometheus telemetry_client_messages_{sent,send_errors,dropped} +// counters for node-local dashboards, with two differences from the WSRPC path: they count +// messages, where the WSRPC batch worker counts batches, and their endpoint label is +// "beholder/" rather than the ingress URL. Unsupported types and chains are counted only +// in OTel (telemetry_messages_unsupported_total). +const ( + metricMessagesSent = "telemetry_messages_sent_total" + metricMessagesSendErrors = "telemetry_messages_send_errors_total" + metricMessagesDropped = "telemetry_messages_dropped_total" + metricMessagesUnsupported = "telemetry_messages_unsupported_total" + + unsupportedReasonTelemetryType = "telemetry_type" + unsupportedReasonChain = "chain" + + // errorCodeEmitRejected is the error_code of events the emitter refused outright (not + // started, invalid, or outside its domain); other codes come from batch.ErrorCodeFor. + errorCodeEmitRejected = "emit_rejected" + errorCodeEventBuild = "event_build" +) + +var _ Manager = (*DomainManager)(nil) + +// DomainManager is the Manager that sends telemetry through Beholder's per-domain +// emitters. Each supported telemetry type maps to a domain; endpoints bind their route +// when created, so sending does no lookups. Network and chain only describe the event: +// nothing is configured or routed per network. +type DomainManager struct { + services.Service + eng *services.Engine + + // ctx is cancelled when the manager closes. SendLog has no context (libocr's + // MonitoringEndpoint), so sends and counters use this one. + ctx context.Context //nolint:containedctx // libocr's MonitoringEndpoint.SendLog takes no ctx + cancel context.CancelFunc + + routes map[synchronization.TelemetryType]*telemetryRoute + csaPubKeyHex string + meterProvider otelmetric.MeterProvider + metrics domainManagerMetrics + now func() time.Time + reported sync.Map // keys of unsupported types, chains and refusals already logged + reportedKeys atomic.Int64 + unsupportedOpts sync.Map // unsupportedKey -> otelmetric.AddOption + metricOpts sync.Map // metricKey -> otelmetric.AddOption +} + +// telemetryRoute is where one telemetry type goes, with its metric handles precomputed. +type telemetryRoute struct { + telemType synchronization.TelemetryType + domain beholder.Domain + source string + entity string + emitter beholder.DomainEmitter + + attrs attribute.Set + + promSent, promSendErrors, promDropped prometheus.Counter +} + +type domainManagerMetrics struct { + sent otelmetric.Int64Counter + sendErrors otelmetric.Int64Counter + dropped otelmetric.Int64Counter + unsupported otelmetric.Int64Counter +} + +// DomainManagerOpt configures NewDomainManager. +type DomainManagerOpt func(*DomainManager) + +// WithMeterProvider sets the OTel meter provider for the manager's metrics. The default is the +// global provider, which is Beholder's once the Beholder client is set globally. +func WithMeterProvider(mp otelmetric.MeterProvider) DomainManagerOpt { + return func(m *DomainManager) { m.meterProvider = mp } +} + +// NewDomainManager resolves an emitter for every domain the supported telemetry types use; a +// missing one is an error. csaPubKeyHex is the node's CSA public key, which legacy-mode chip +// ingress binds to the authenticated key. +func NewDomainManager(src DomainEmitterSource, csaPubKeyHex string, lggr logger.Logger, opts ...DomainManagerOpt) (*DomainManager, error) { + if src == nil { + return nil, errors.New("telemetry: a Beholder domain emitter source is required") + } + if csaPubKeyHex == "" { + return nil, errors.New("telemetry: CSA public key is required") + } + m := &DomainManager{ + csaPubKeyHex: csaPubKeyHex, + meterProvider: otel.GetMeterProvider(), + now: time.Now, + } + for _, opt := range opts { + opt(m) + } + metrics, err := newDomainManagerMetrics(m.meterProvider) + if err != nil { + return nil, err + } + m.metrics = metrics + + // Resolve domains in a stable order so a missing one is reported the same way every time. + var domains []beholder.Domain + for _, d := range supportedTelemetryDomains { + if !slices.Contains(domains, d) { + domains = append(domains, d) + } + } + slices.Sort(domains) + emitters := make(map[beholder.Domain]beholder.DomainEmitter, len(domains)) + for _, d := range domains { + em, err := src.ForDomain(d) + if err != nil { + return nil, fmt.Errorf("telemetry: no Beholder emitter for domain %q: %w", d, err) + } + emitters[d] = em + } + + m.routes = make(map[synchronization.TelemetryType]*telemetryRoute, len(supportedTelemetryDomains)) + for telemType, d := range supportedTelemetryDomains { + source, entity, err := synchronization.TelemetryTypeToDomainAndEntity(telemType) + if err != nil { + return nil, fmt.Errorf("telemetry: no CloudEvent source for %q: %w", telemType, err) + } + attrs := attribute.NewSet( + attribute.String("domain", string(d)), + attribute.String("telemetry_type", string(telemType)), + ) + endpoint := "beholder/" + string(d) + m.routes[telemType] = &telemetryRoute{ + telemType: telemType, + domain: d, + source: source, + entity: entity, + emitter: emitters[d], + attrs: attrs, + promSent: synchronization.TelemetryClientMessagesSent.WithLabelValues(endpoint, string(telemType)), + promSendErrors: synchronization.TelemetryClientMessagesSendErrors.WithLabelValues(endpoint, string(telemType)), + promDropped: synchronization.TelemetryClientMessagesDropped.WithLabelValues(endpoint, string(telemType)), + } + } + + m.ctx, m.cancel = context.WithCancel(context.Background()) + m.Service, m.eng = services.Config{ + Name: "DomainManager", + Close: func() error { m.cancel(); return nil }, + }.NewServiceEngine(lggr) + return m, nil +} + +func newDomainManagerMetrics(mp otelmetric.MeterProvider) (m domainManagerMetrics, err error) { + meter := mp.Meter("github.com/smartcontractkit/chainlink/v2/core/services/telemetry") + if m.sent, err = meter.Int64Counter(metricMessagesSent, + otelmetric.WithDescription("Telemetry messages accepted by chip ingress")); err != nil { + return m, err + } + if m.sendErrors, err = meter.Int64Counter(metricMessagesSendErrors, + otelmetric.WithDescription("Telemetry messages that failed to publish")); err != nil { + return m, err + } + if m.dropped, err = meter.Int64Counter(metricMessagesDropped, + otelmetric.WithDescription("Telemetry messages dropped before publishing (buffer full, shutdown, refused)")); err != nil { + return m, err + } + if m.unsupported, err = meter.Int64Counter(metricMessagesUnsupported, + otelmetric.WithDescription("Telemetry messages dropped because their type or chain is not supported")); err != nil { + return m, err + } + return m, nil +} + +// GenMonitoringEndpoint returns an endpoint for one telemetry type. Unsupported types and chains +// get an endpoint that drops and counts what it is given. +func (m *DomainManager) GenMonitoringEndpoint(network string, chainID string, contractID string, telemType synchronization.TelemetryType) commontypes.MonitoringEndpoint { + route, ok := m.routes[telemType] + if !ok { + m.warnUnsupportedType(telemType) + return &unsupportedAgent{m: m, telemType: telemType, reason: unsupportedReasonTelemetryType} + } + selector, ok := m.chainSelector(network, chainID) + if !ok { + return &unsupportedAgent{m: m, telemType: telemType, reason: unsupportedReasonChain} + } + return &domainAgent{m: m, route: route, target: m.newEventTarget(network, contractID, selector)} +} + +// GenMultitypeMonitoringEndpoint returns an endpoint whose messages carry their own type. +func (m *DomainManager) GenMultitypeMonitoringEndpoint(network string, chainID string, contractID string) MultitypeMonitoringEndpoint { + selector, ok := m.chainSelector(network, chainID) + if !ok { + return &unsupportedAgent{m: m, reason: unsupportedReasonChain} + } + return &multitypeDomainAgent{m: m, target: m.newEventTarget(network, contractID, selector)} +} + +// eventTarget holds an endpoint's per-event values, computed once when the endpoint is created. +type eventTarget struct { + network string // lower case, e.g. "evm" + contractID string + selector string // decimal chain selector + partitionKey string // -, the Kafka record key, as OTI's +} + +func (m *DomainManager) newEventTarget(network, contractID string, selector uint64) eventTarget { + return eventTarget{ + network: strings.ToLower(network), + contractID: contractID, + selector: strconv.FormatUint(selector, 10), + partitionKey: contractID + "-" + m.csaPubKeyHex, + } +} + +// chainSelector resolves the job's chain to its chain-selectors selector, which legacy-mode chip +// ingress maps to the network's topics. A chain that does not resolve (e.g. one newer than the +// pinned chain-selectors) loses its telemetry, so it is logged at Error, once per chain. +func (m *DomainManager) chainSelector(network, chainID string) (uint64, bool) { + details, err := chainselector.GetChainDetailsByChainIDAndFamily(chainID, strings.ToLower(network)) + if err != nil { + m.reportOnce("chain:"+strings.ToLower(network)+"/"+chainID, func() { + m.eng.Errorw("Chain has no chain selector: its telemetry will be dropped", "network", network, "chainID", chainID, "err", err) + }) + return 0, false + } + return details.ChainSelector, true +} + +// maxReportedKeys caps how many distinct one-time reports are remembered. The keys are +// sender-influenced (telemetry types, network/chainID pairs), so the cap bounds +// worst-case memory; past it, new keys are only counted — never logged or retained. +const maxReportedKeys = 25 + +func (m *DomainManager) reportOnce(key string, report func()) { + if _, loaded := m.reported.LoadOrStore(key, struct{}{}); loaded { + return + } + if m.reportedKeys.Add(1) > maxReportedKeys { + m.reported.Delete(key) + return + } + report() +} + +// send builds the event and hands it to the route's emitter. Fates are counted; publish failures +// are logged (rate-limited) by the emitter, and events that never reach it are logged here, once +// per route and error code, since they mean telemetry is being lost. +func (m *DomainManager) send(route *telemetryRoute, target eventTarget, telemetry []byte) { + ctx := m.ctx + ev, err := m.newEvent(route, target, telemetry) + if err != nil { + m.refused(ctx, route, target, errorCodeEventBuild, err) + return + } + if err := route.emitter.EmitEvent(ctx, ev, func(sendErr error) { m.recordFate(ctx, route, target, sendErr) }); err != nil { + m.refused(ctx, route, target, errorCodeEmitRejected, err) + } +} + +// refused counts an event that never reached the emitter's buffer and logs the first one per +// route and error code at Error. +func (m *DomainManager) refused(ctx context.Context, route *telemetryRoute, target eventTarget, code string, err error) { + m.recordDropped(ctx, route, target, code) + m.reportOnce("refused:"+string(route.telemType)+"/"+code, func() { + m.eng.Errorw("Telemetry refused: it is being dropped (further refusals are only counted)", + "telemType", route.telemType, "domain", route.domain, "errorCode", code, "err", err) + }) +} + +// recordFate counts an event's fate as reported by the emitter: published, failed to publish, +// or dropped before publishing (full buffer or shutdown). +func (m *DomainManager) recordFate(ctx context.Context, route *telemetryRoute, target eventTarget, err error) { + switch { + case err == nil: + m.metrics.sent.Add(ctx, 1, m.metricOpt(route, target, "")) + route.promSent.Inc() + case errors.Is(err, batch.ErrMessageBufferFull), errors.Is(err, batch.ErrClientShutdown): + m.recordDropped(ctx, route, target, batch.ErrorCodeFor(err)) + default: + m.metrics.sendErrors.Add(ctx, 1, m.metricOpt(route, target, batch.ErrorCodeFor(err))) + route.promSendErrors.Inc() + } +} + +func (m *DomainManager) recordDropped(ctx context.Context, route *telemetryRoute, target eventTarget, code string) { + m.metrics.dropped.Add(ctx, 1, m.metricOpt(route, target, code)) + route.promDropped.Inc() +} + +// metricKey identifies a cached metric option: the target chain's selector, the route's +// telemetry type and the error code ("" for the sent counter). +type metricKey struct { + selector string + telemType synchronization.TelemetryType + errorCode string +} + +// metricOpt returns the route's attributes plus the target chain's chain_selector and, when code +// is non-empty, the error code. Cached per metricKey: a bounded set. +func (m *DomainManager) metricOpt(route *telemetryRoute, target eventTarget, code string) otelmetric.AddOption { + key := metricKey{selector: target.selector, telemType: route.telemType, errorCode: code} + if v, ok := m.metricOpts.Load(key); ok { + return v.(otelmetric.AddOption) + } + kvs := append(route.attrs.ToSlice(), attribute.String("chain_selector", target.selector)) + if code != "" { + kvs = append(kvs, attribute.String("error_code", code)) + } + v, _ := m.metricOpts.LoadOrStore(key, otelmetric.WithAttributes(kvs...)) + return v.(otelmetric.AddOption) +} + +// unsupportedTypeLabel clamps an unsupported telemetry type for metric labels and log +// dedup keys: the generic telemetry adapter accepts arbitrary strings, so a raw value +// would grow the metric series and the option cache without bound. Types with a +// CloudEvent mapping (synchronization.TelemetryTypeToDomainAndEntity is the registry) +// keep their label; anything else is "unknown" — its raw value is logged once by +// warnUnsupportedType. +func unsupportedTypeLabel(telemType synchronization.TelemetryType) string { + if _, _, err := synchronization.TelemetryTypeToDomainAndEntity(telemType); err == nil { + return string(telemType) + } + return "unknown" +} + +// unsupportedKey identifies a cached unsupported-metric option: the clamped telemetry +// type label and the drop reason. Bounded by construction — labels come from the +// CloudEvent registry plus "unknown", reasons from a fixed set. +type unsupportedKey struct { + telemType string + reason string +} + +// countUnsupported counts a dropped message whose type or chain is not supported. The +// telemetry_type label is clamped (see unsupportedTypeLabel) and the domain is always +// recorded ("unknown" when the type maps to none), so domain-filtered loss dashboards +// see every drop. The option is cached per (label, reason): hot path. +func (m *DomainManager) countUnsupported(telemType synchronization.TelemetryType, reason string) { + label := unsupportedTypeLabel(telemType) + key := unsupportedKey{telemType: label, reason: reason} + v, ok := m.unsupportedOpts.Load(key) + if !ok { + d, mapped := supportedTelemetryDomains[telemType] + if !mapped { + d = "unknown" + } + attrs := []attribute.KeyValue{ + attribute.String("telemetry_type", label), + attribute.String("reason", reason), + attribute.String("domain", string(d)), + } + v, _ = m.unsupportedOpts.LoadOrStore(key, otelmetric.WithAttributes(attrs...)) + } + m.metrics.unsupported.Add(m.ctx, 1, v.(otelmetric.AddOption)) +} + +func (m *DomainManager) warnUnsupportedType(telemType synchronization.TelemetryType) { + m.reportOnce("type:"+string(telemType), func() { + m.eng.Warnw("Telemetry type is not supported: its telemetry will be dropped", "telemType", telemType) + }) +} + +// newEvent builds the CloudEvent legacy-mode chip ingress expects. Each extension becomes a +// "ce_" Kafka header; the comments name the header OTI used for the same value. Chip +// ingress derives nodeoperatorname/nodename (from the CSA key's NOP info) and receivedat itself. +func (m *DomainManager) newEvent(route *telemetryRoute, target eventTarget, telemetry []byte) (chipingress.CloudEvent, error) { + now := m.now() + event, err := chipingress.NewEvent(route.source, route.entity, telemetry, map[string]any{ + "time": now.UTC(), + "recordedtime": now.UTC(), + }) + if err != nil { + return event, fmt.Errorf("failed creating CloudEvent: %w", err) + } + event.SetExtension("legacytelemetry", "true") // marks node-sent legacy telemetry + event.SetExtension("telemetrytype", string(route.telemType)) // "telemetry-type" + event.SetExtension("chainselector", target.selector) // chain-selectors ID of the job's chain + event.SetExtension("networkname", target.network) // lower case, e.g. "evm" + event.SetExtension("contractid", target.contractID) // "contract-address" + event.SetExtension("csapublickey", m.csaPubKeyHex) // "node-operator-csa-public-key" + event.SetExtension("partitionkey", target.partitionKey) // Kafka record key, as OTI's + event.SetExtension("sentat", strconv.FormatInt(now.UnixNano(), 10)) // "sent-at", unix ns as over WSRPC + return event, nil +} + +// domainAgent sends one telemetry type through its domain's emitter. +type domainAgent struct { + m *DomainManager + route *telemetryRoute + target eventTarget +} + +func (a *domainAgent) SendLog(telemetry []byte) { + a.m.send(a.route, a.target, telemetry) +} + +// multitypeDomainAgent routes each message by the type it carries. +type multitypeDomainAgent struct { + m *DomainManager + target eventTarget +} + +func (a *multitypeDomainAgent) SendTypedLog(telemType synchronization.TelemetryType, telemetry []byte) { + route, ok := a.m.routes[telemType] + if !ok { + a.m.warnUnsupportedType(telemType) + a.m.countUnsupported(telemType, unsupportedReasonTelemetryType) + return + } + a.m.send(route, a.target, telemetry) +} + +// unsupportedAgent drops everything it is given and counts it. telemType is empty for a +// multitype endpoint, whose messages carry their own type. +type unsupportedAgent struct { + m *DomainManager + telemType synchronization.TelemetryType + reason string +} + +func (a *unsupportedAgent) SendLog([]byte) { + a.m.countUnsupported(a.telemType, a.reason) +} + +func (a *unsupportedAgent) SendTypedLog(telemType synchronization.TelemetryType, _ []byte) { + if a.telemType != "" { + telemType = a.telemType + } + a.m.countUnsupported(telemType, a.reason) +} diff --git a/core/services/telemetry/domain_manager_beholder_test.go b/core/services/telemetry/domain_manager_beholder_test.go new file mode 100644 index 00000000000..d3a472eb1b9 --- /dev/null +++ b/core/services/telemetry/domain_manager_beholder_test.go @@ -0,0 +1,218 @@ +package telemetry + +import ( + "context" + "net" + "sync" + "testing" + "time" + + cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" + sdklog "go.opentelemetry.io/otel/sdk/log" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" +) + +// receivedEvent is one event as chip ingress saw it. +type receivedEvent struct { + method string + domain string // the chainlink-domain header, "" if absent + event *cepb.CloudEvent +} + +// capturingChipServer records PublishBatch / PublishBatchNoAck calls and answers each event with +// a positional success, as chip ingress does. With hold set, it answers nothing until released. +type capturingChipServer struct { + pb.UnimplementedChipIngressServer + hold chan struct{} + + mu sync.Mutex + events []receivedEvent +} + +func (s *capturingChipServer) record(ctx context.Context, method string, in *pb.CloudEventBatch) (*pb.PublishResponse, error) { + if s.hold != nil { + select { + case <-s.hold: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + md, _ := metadata.FromIncomingContext(ctx) + domain := "" + if v := md.Get(beholder.DomainHeaderKey); len(v) > 0 { + domain = v[0] + } + s.mu.Lock() + for _, ev := range in.Events { + s.events = append(s.events, receivedEvent{method: method, domain: domain, event: ev}) + } + s.mu.Unlock() + return &pb.PublishResponse{Results: make([]*pb.PublishResult, len(in.Events))}, nil +} + +func (s *capturingChipServer) PublishBatch(ctx context.Context, in *pb.CloudEventBatch) (*pb.PublishResponse, error) { + return s.record(ctx, "PublishBatch", in) +} + +func (s *capturingChipServer) PublishBatchNoAck(ctx context.Context, in *pb.CloudEventBatch) (*pb.PublishResponse, error) { + return s.record(ctx, "PublishBatchNoAck", in) +} + +func (s *capturingChipServer) received() []receivedEvent { + s.mu.Lock() + defer s.mu.Unlock() + return append([]receivedEvent(nil), s.events...) +} + +type nopLogExporter struct{} + +func (nopLogExporter) Export(context.Context, []sdklog.Record) error { return nil } +func (nopLogExporter) Shutdown(context.Context) error { return nil } +func (nopLogExporter) ForceFlush(context.Context) error { return nil } + +// startChipServer serves chip on a local listener and returns its address. +func startChipServer(t *testing.T, chip *capturingChipServer) string { + t.Helper() + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer() + pb.RegisterChipIngressServer(srv, chip) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + return lis.Addr().String() +} + +// startBeholderClient starts a real Beholder client with domain emitters against addr. Its OTel +// exporters also point at addr, so shutdown gets a prompt answer instead of dialing an absent +// collector. closeClient closes it once and returns Close's error; cleanup closes it otherwise. +func startBeholderClient(t *testing.T, addr string, tune func(*beholder.Config)) (client *beholder.Client, closeClient func() error) { + t.Helper() + cfg := beholder.Config{ + OtelExporterGRPCEndpoint: addr, + InsecureConnection: true, + ChipIngressEmitterEnabled: true, + ChipIngressEmitterGRPCEndpoint: addr, + ChipIngressInsecureConnection: true, + ChipIngressSendInterval: 10 * time.Millisecond, + ChipDomainEmittersEnabled: true, + ChipIngressLogger: logger.Test(t), + AuthHeaders: map[string]string{"X-Beholder-Node-Auth-Token": "1:abc:2:def"}, + } + if tune != nil { + tune(&cfg) + } + client, err := beholder.NewGRPCClient(cfg, func(...otlploggrpc.Option) (sdklog.Exporter, error) { return nopLogExporter{}, nil }) + require.NoError(t, err) + require.NoError(t, client.Start(t.Context())) + + var once sync.Once + var closeErr error + closeClient = func() error { + once.Do(func() { closeErr = client.Close() }) + return closeErr + } + t.Cleanup(func() { _ = closeClient() }) + return client, closeClient +} + +// requireCleanClose asserts that closing the client did not fail because of the domain emitters. +// OTel provider shutdown may still report export errors (the test server is not a collector). +func requireCleanClose(t *testing.T, err error) { + t.Helper() + if err != nil { + assert.NotContains(t, err.Error(), "client connection is closing", "domain emitter connections close once") + } +} + +func TestDomainManager_ThroughRealBeholderClient(t *testing.T) { + t.Parallel() + chip := &capturingChipServer{} + addr := startChipServer(t, chip) + client, closeClient := startBeholderClient(t, addr, nil) + m, reader := newTestDomainManager(t, client) + + // Concurrent senders across two domains, through single-type and multitype endpoints. + const workers, perWorker = 4, 25 + var wg sync.WaitGroup + for range workers { + wg.Go(func() { + feeds := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median) + streams := m.GenMultitypeMonitoringEndpoint("evm", ethMainnetChainID, "0xdef/1") + for range perWorker { + feeds.SendLog([]byte("p")) + streams.SendTypedLog(synchronization.LLOOutcome, []byte("p")) + } + }) + } + wg.Wait() + + const total = 2 * workers * perWorker + require.Eventually(t, func() bool { return counterValue(t, reader, metricMessagesSent) == total }, + 10*time.Second, 20*time.Millisecond, "every event is acknowledged by the per-event result") + + received := chip.received() + require.Len(t, received, total) + for _, r := range received { + assert.Equal(t, "PublishBatchNoAck", r.method) + switch r.event.Source { + case "data-feeds.telemetry.ocr2-median": + assert.Equal(t, string(beholder.DomainDataFeeds), r.domain) + case "data-streams.telemetry.llo-outcome": + assert.Equal(t, string(beholder.DomainDataStreams), r.domain) + default: + t.Errorf("unexpected source %q", r.event.Source) + } + assert.Equal(t, "true", r.event.Attributes["legacytelemetry"].GetCeString()) + assert.Equal(t, testCSAPubKeyHex, r.event.Attributes["csapublickey"].GetCeString()) + } + + requireCleanClose(t, closeClient()) + // Nothing is counted twice, and nothing else is counted, once the emitters have drained. + assert.Equal(t, int64(total), counterValue(t, reader, metricMessagesSent)) + assert.Equal(t, int64(total), counterValue(t, reader, metricMessagesSent, + attribute.String("chain_selector", ethMainnetSelector)), "sent counts carry the chain selector") + assert.Zero(t, counterValue(t, reader, metricMessagesSendErrors)) + assert.Zero(t, counterValue(t, reader, metricMessagesDropped)) +} + +func TestDomainManager_ThroughRealBeholderClient_BufferFull(t *testing.T) { + t.Parallel() + chip := &capturingChipServer{hold: make(chan struct{})} + addr := startChipServer(t, chip) + client, closeClient := startBeholderClient(t, addr, func(c *beholder.Config) { + c.ChipIngressBufferSize = 3 + c.ChipIngressMaxBatchSize = 1 + c.ChipIngressMaxConcurrentSends = 1 + }) + m, reader := newTestDomainManager(t, client) + + // The server holds every request, so the tiny buffer fills and the rest are dropped. + const total = 50 + ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median) + for range total { + ep.SendLog([]byte("p")) + } + bufferFull := attribute.String("error_code", batch.ErrorCodeFor(batch.ErrMessageBufferFull)) + require.Positive(t, counterValue(t, reader, metricMessagesDropped, bufferFull, + attribute.String("chain_selector", ethMainnetSelector)), "drops are counted as dropped, not send errors") + + close(chip.hold) // let the held and queued events through, then drain on close + requireCleanClose(t, closeClient()) + + sent := counterValue(t, reader, metricMessagesSent) + dropped := counterValue(t, reader, metricMessagesDropped) + assert.Equal(t, int64(total), sent+dropped, "every event is counted exactly once") + assert.Equal(t, sent, int64(len(chip.received())), "sent matches what the server received") + assert.Zero(t, counterValue(t, reader, metricMessagesSendErrors)) +} diff --git a/core/services/telemetry/domain_manager_test.go b/core/services/telemetry/domain_manager_test.go new file mode 100644 index 00000000000..edb6e79998f --- /dev/null +++ b/core/services/telemetry/domain_manager_test.go @@ -0,0 +1,546 @@ +package telemetry + +import ( + "context" + "errors" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + chainselectors "github.com/smartcontractkit/chain-selectors" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" +) + +const testCSAPubKeyHex = "755294c8a1f0" + +// fakeDomainEmitter records EmitEvent calls. Like the real emitter, it can report each event's +// fate from another goroutine (async). +type fakeDomainEmitter struct { + async bool + emitErr error // returned by EmitEvent; the callback is then not invoked + callbackErr error // reported to the callback as the event's fate + + mu sync.Mutex + events []chipingress.CloudEvent +} + +func (f *fakeDomainEmitter) Emit(context.Context, []byte, ...any) error { return nil } +func (f *fakeDomainEmitter) Close() error { return nil } + +func (f *fakeDomainEmitter) EmitEvent(ctx context.Context, event chipingress.CloudEvent, callback func(error)) error { + // Like the real emitter, refuse events whose context is already done. + if err := ctx.Err(); err != nil { + return err + } + if f.emitErr != nil { + return f.emitErr + } + f.mu.Lock() + f.events = append(f.events, event) + f.mu.Unlock() + if callback == nil { + return nil + } + if f.async { + go callback(f.callbackErr) + } else { + callback(f.callbackErr) + } + return nil +} + +func (f *fakeDomainEmitter) sent() []chipingress.CloudEvent { + f.mu.Lock() + defer f.mu.Unlock() + return append([]chipingress.CloudEvent(nil), f.events...) +} + +// fakeEmitterSource stands in for *beholder.Client. +type fakeEmitterSource map[beholder.Domain]*fakeDomainEmitter + +func (s fakeEmitterSource) ForDomain(d beholder.Domain) (beholder.DomainEmitter, error) { + em, ok := s[d] + if !ok { + return nil, errors.New("no emitter for domain " + string(d)) + } + return em, nil +} + +func newFakeEmitterSource() fakeEmitterSource { + return fakeEmitterSource{ + beholder.DomainDataFeeds: {}, + beholder.DomainDataStreams: {}, + beholder.DomainCCIP: {}, + } +} + +// newTestDomainManager returns a manager whose OTel metrics go to its own reader, so tests can +// run in parallel. +func newTestDomainManager(t *testing.T, src DomainEmitterSource) (*DomainManager, *sdkmetric.ManualReader) { + t.Helper() + reader := sdkmetric.NewManualReader() + m, err := NewDomainManager(src, testCSAPubKeyHex, logger.Test(t), + WithMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)))) + require.NoError(t, err) + return m, reader +} + +// counterValue sums the int64 counter name over data points whose attributes include want. +func counterValue(t *testing.T, reader *sdkmetric.ManualReader, name string, want ...attribute.KeyValue) int64 { + t.Helper() + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + var total int64 + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != name { + continue + } + sum, ok := m.Data.(metricdata.Sum[int64]) + require.True(t, ok, "%s is an int64 counter", name) + for _, dp := range sum.DataPoints { + matches := true + for _, kv := range want { + if v, ok := dp.Attributes.Value(kv.Key); !ok || v != kv.Value { + matches = false + break + } + } + if matches { + total += dp.Value + } + } + } + } + return total +} + +var ethMainnetChainID = strconv.FormatUint(chainselectors.ETHEREUM_MAINNET.EvmChainID, 10) + +var ethMainnetSelector = strconv.FormatUint(chainselectors.ETHEREUM_MAINNET.Selector, 10) + +func TestNewDomainManager_RequiresEveryDomain(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + delete(src, beholder.DomainCCIP) + delete(src, beholder.DomainDataStreams) + + _, err := NewDomainManager(src, testCSAPubKeyHex, logger.Test(t)) + require.Error(t, err) + assert.Contains(t, err.Error(), `"ccip"`, "domains are resolved in a stable order, so the same one is reported every time") +} + +func TestNewDomainManager_RequiresEmitterSourceAndCSAPublicKey(t *testing.T) { + t.Parallel() + _, err := NewDomainManager(nil, testCSAPubKeyHex, logger.Test(t)) + require.Error(t, err) + _, err = NewDomainManager(newFakeEmitterSource(), "", logger.Test(t)) + require.Error(t, err) +} + +func TestDomainManager_RoutesEachTypeToItsDomain(t *testing.T) { + t.Parallel() + const ocr2, ocr3 = "offchainreporting2.TelemetryWrapper", "offchainreporting3.TelemetryWrapper" + for _, tc := range []struct { + telemType synchronization.TelemetryType + domain beholder.Domain + source string + entity string + }{ + {synchronization.OCR, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr", "offchainreporting.TelemetryWrapper"}, + {synchronization.OCR2Median, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr2-median", ocr2}, + {synchronization.OCR3DataFeeds, beholder.DomainDataFeeds, "data-feeds.telemetry.ocr3-data-feeds", ocr3}, + {synchronization.EnhancedEA, beholder.DomainDataFeeds, "data-feeds.telemetry.enhanced-ea", "telem.EnhancedEA"}, + {synchronization.OCR3Mercury, beholder.DomainDataStreams, "data-streams.telemetry.ocr3-mercury", ocr3}, + {synchronization.OCR3_1Mercury, beholder.DomainDataStreams, "data-streams.telemetry.ocr3_1-mercury", "offchainreporting3_1.TelemetryWrapper"}, + {synchronization.EnhancedEAMercury, beholder.DomainDataStreams, "data-streams.telemetry.enhanced-ea-mercury", "telem.EnhancedEAMercury"}, + {synchronization.LLOObservation, beholder.DomainDataStreams, "data-streams.telemetry.llo-observation", "telem.LLOObservationTelemetry"}, + {synchronization.LLOOutcome, beholder.DomainDataStreams, "data-streams.telemetry.llo-outcome", "telem.LLOOutcomeTelemetry"}, + {synchronization.PipelineBridge, beholder.DomainDataStreams, "data-streams.telemetry.pipeline-bridge", "telem.LLOBridgeTelemetry"}, + {synchronization.OCR2CCIPCommit, beholder.DomainCCIP, "ccip.telemetry.ocr2", ocr2}, + {synchronization.OCR2CCIPExec, beholder.DomainCCIP, "ccip.telemetry.ocr2", ocr2}, + {synchronization.OCR3CCIPCommit, beholder.DomainCCIP, "ccip.telemetry.ocr3", ocr3}, + {synchronization.OCR3CCIPExec, beholder.DomainCCIP, "ccip.telemetry.ocr3", ocr3}, + {synchronization.HeadReport, beholder.DomainCCIP, "ccip.telemetry.head-report", "telem.HeadReportRequest"}, + } { + t.Run(string(tc.telemType), func(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, _ := newTestDomainManager(t, src) + + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", tc.telemType).SendLog([]byte("payload")) + + for d, em := range src { + if d != tc.domain { + assert.Empty(t, em.sent(), "nothing on %s", d) + continue + } + require.Len(t, em.sent(), 1, "sent on %s", d) + assert.Equal(t, tc.source, em.sent()[0].Source()) + assert.Equal(t, tc.entity, em.sent()[0].Type()) + } + }) + } +} + +func TestDomainManager_DropsTypesItDoesNotSupport(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, reader := newTestDomainManager(t, src) + + unsupported := []synchronization.TelemetryType{ + synchronization.OCR2Functions, + synchronization.FunctionsRequests, + synchronization.LLOAttributedObservation, + synchronization.LLOReport, + synchronization.OCR2Threshold, + synchronization.OCR2S4, + synchronization.OCR3Rebalancer, + synchronization.OCR3CCIPBootstrap, + "vault-plugin", + } + for _, tt := range unsupported { + ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", tt) + require.NotNil(t, ep) + ep.SendLog([]byte("p")) + ep.SendLog([]byte("p")) + } + + for d, em := range src { + assert.Empty(t, em.sent(), "nothing sent on %s", d) + } + assert.Equal(t, int64(2), counterValue(t, reader, metricMessagesUnsupported, + attribute.String("telemetry_type", string(synchronization.LLOAttributedObservation)), + attribute.String("reason", unsupportedReasonTelemetryType)), "counted per message") + assert.Equal(t, int64(2*len(unsupported)), counterValue(t, reader, metricMessagesUnsupported)) +} + +func TestDomainManager_EventCarriesLegacyTelemetryExtensions(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, _ := newTestDomainManager(t, src) + now := time.Date(2026, 10, 28, 12, 0, 0, 0, time.UTC) + m.now = func() time.Time { return now } + + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("payload")) + + events := src[beholder.DomainDataFeeds].sent() + require.Len(t, events, 1) + ev := events[0] + + require.NoError(t, ev.Validate()) + assert.Equal(t, []byte("payload"), ev.Data()) + assert.True(t, ev.Time().Equal(now)) + + ext := ev.Extensions() + assert.Equal(t, "true", ext["legacytelemetry"]) + assert.Equal(t, "ocr2-median", ext["telemetrytype"]) + assert.Equal(t, strconv.FormatUint(chainselectors.ETHEREUM_MAINNET.Selector, 10), ext["chainselector"]) + assert.Equal(t, "evm", ext["networkname"], "network is normalized to lower case") + assert.Equal(t, "0xabc", ext["contractid"]) + assert.Equal(t, testCSAPubKeyHex, ext["csapublickey"]) + assert.Equal(t, "0xabc-"+testCSAPubKeyHex, ext["partitionkey"]) + assert.Equal(t, strconv.FormatInt(now.UnixNano(), 10), ext["sentat"], "sentat is unix nanoseconds, as the node sends OTI today") + recorded, ok := ext["recordedtime"] + require.True(t, ok) + assert.Contains(t, recorded.(interface{ String() string }).String(), "2026-10-28T12:00:00", "recordedtime uses the manager's clock") +} + +func TestDomainManager_ResolvesChainsInEveryFamily(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + network, chainID string + selector uint64 + }{ + {"EVM", ethMainnetChainID, chainselectors.ETHEREUM_MAINNET.Selector}, + {"evm", ethMainnetChainID, chainselectors.ETHEREUM_MAINNET.Selector}, + {"Solana", "5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d", 124615329519749607}, // genesis hash: the relayer's chain ID + {"StarkNet", "SN_MAIN", 511843109281680063}, + {"Aptos", "1", 4741433654826277614}, + {"Tron", "728126428", 1546563616611573945}, + {"TON", "-239", 16448340667252469081}, + {"Sui", "1", 17529533435026248318}, + } { + t.Run(tc.network+"/"+tc.chainID, func(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, _ := newTestDomainManager(t, src) + + m.GenMonitoringEndpoint(tc.network, tc.chainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + + events := src[beholder.DomainDataFeeds].sent() + require.Len(t, events, 1) + assert.Equal(t, strconv.FormatUint(tc.selector, 10), events[0].Extensions()["chainselector"]) + assert.Equal(t, strings.ToLower(tc.network), events[0].Extensions()["networkname"]) + }) + } +} + +func TestDomainManager_UnresolvableChainIsDroppedAndCounted(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, reader := newTestDomainManager(t, src) + + m.GenMonitoringEndpoint("EVM", "999999999999", "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + mt := m.GenMultitypeMonitoringEndpoint("EVM", "999999999999", "0xabc/1") + mt.SendTypedLog(synchronization.LLOOutcome, []byte("p")) + mt.SendTypedLog(synchronization.LLOObservation, []byte("p")) + + for d, em := range src { + assert.Empty(t, em.sent(), "nothing sent on %s", d) + } + reason := attribute.String("reason", unsupportedReasonChain) + assert.Equal(t, int64(3), counterValue(t, reader, metricMessagesUnsupported, reason)) + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, reason, + attribute.String("telemetry_type", string(synchronization.LLOOutcome))), "multitype counts carry the message's type") + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, reason, + attribute.String("telemetry_type", string(synchronization.OCR2Median)), + attribute.String("domain", string(beholder.DomainDataFeeds))), "chain drops of mapped types carry their domain") +} + +func TestDomainManager_UnknownTypesCollapseToOneSeries(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, reader := newTestDomainManager(t, src) + + // The generic telemetry adapter accepts arbitrary type strings; distinct values + // must not create a metric series or a cache entry each. + for i := range 100 { + tt := synchronization.TelemetryType("plugin-garbage-" + strconv.Itoa(i)) + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", tt).SendLog([]byte("p")) + } + + assert.Equal(t, int64(100), counterValue(t, reader, metricMessagesUnsupported, + attribute.String("telemetry_type", "unknown"), + attribute.String("domain", "unknown"), + attribute.String("reason", unsupportedReasonTelemetryType)), "every message is counted") + assert.Equal(t, 1, syncMapCount(&m.unsupportedOpts), "one cache entry per clamped label, not per raw value") + assert.LessOrEqual(t, syncMapCount(&m.reported), maxReportedKeys, + "the log-dedup map is capped however many distinct types arrive") +} + +func TestDomainManager_UnresolvableChainsAreCapped(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, reader := newTestDomainManager(t, src) + + // Distinct network/chainID pairs are plugin-suppliable via the generic adapter; + // past the cap they must stop being logged and retained, but stay counted. + for i := range 100 { + m.GenMonitoringEndpoint("EVM", "bogus-"+strconv.Itoa(i), "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + } + + assert.Equal(t, int64(100), counterValue(t, reader, metricMessagesUnsupported, + attribute.String("telemetry_type", string(synchronization.OCR2Median)), + attribute.String("domain", string(beholder.DomainDataFeeds)), + attribute.String("reason", unsupportedReasonChain)), "every drop is counted") + assert.LessOrEqual(t, syncMapCount(&m.reported), maxReportedKeys, + "the log-dedup map is capped however many distinct chains arrive") +} + +func syncMapCount(m *sync.Map) int { + n := 0 + m.Range(func(_, _ any) bool { n++; return true }) + return n +} + +func TestDomainManager_MultitypeEndpointRoutesPerMessage(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, reader := newTestDomainManager(t, src) + + ep := m.GenMultitypeMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc/1") + ep.SendTypedLog(synchronization.LLOOutcome, []byte("p")) + ep.SendTypedLog(synchronization.LLOReport, []byte("p")) // unsupported: dropped (no CloudEvent mapping yet) + + events := src[beholder.DomainDataStreams].sent() + require.Len(t, events, 1) + assert.Equal(t, "data-streams.telemetry.llo-outcome", events[0].Source()) + assert.Equal(t, "llo-outcome", events[0].Extensions()["telemetrytype"]) + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesUnsupported, + attribute.String("telemetry_type", "unknown"), + attribute.String("domain", "unknown")), + "types without a CloudEvent mapping collapse onto the unknown label") +} + +func TestDomainManager_CountsEachEventsFate(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + emitErr error + callbackErr error + metric string + errorCode string + }{ + {name: "sent", metric: metricMessagesSent}, + {name: "server unavailable", callbackErr: status.Error(codes.Unavailable, "down"), metric: metricMessagesSendErrors, errorCode: codes.Unavailable.String()}, + {name: "server without no-ack", callbackErr: status.Error(codes.Unimplemented, "no"), metric: metricMessagesSendErrors, errorCode: codes.Unimplemented.String()}, + {name: "buffer full", callbackErr: batch.ErrMessageBufferFull, metric: metricMessagesDropped, errorCode: batch.ErrorCodeFor(batch.ErrMessageBufferFull)}, + {name: "emitter shutting down", callbackErr: batch.ErrClientShutdown, metric: metricMessagesDropped, errorCode: batch.ErrorCodeFor(batch.ErrClientShutdown)}, + {name: "emitter refuses the event", emitErr: errors.New("not started"), metric: metricMessagesDropped, errorCode: errorCodeEmitRejected}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + src[beholder.DomainDataFeeds].async = true + src[beholder.DomainDataFeeds].emitErr = tc.emitErr + src[beholder.DomainDataFeeds].callbackErr = tc.callbackErr + m, reader := newTestDomainManager(t, src) + + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + + want := []attribute.KeyValue{ + attribute.String("domain", string(beholder.DomainDataFeeds)), + attribute.String("telemetry_type", string(synchronization.OCR2Median)), + } + if tc.errorCode != "" { + want = append(want, attribute.String("error_code", tc.errorCode)) + } + require.Eventually(t, func() bool { return counterValue(t, reader, tc.metric, want...) == 1 }, + 5*time.Second, 10*time.Millisecond) + for _, other := range []string{metricMessagesSent, metricMessagesSendErrors, metricMessagesDropped} { + if other != tc.metric { + assert.Zero(t, counterValue(t, reader, other), "%s", other) + } + } + }) + } +} + +func TestDomainManager_RefusesSendsAfterClose(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + m, reader := newTestDomainManager(t, src) + require.NoError(t, m.Start(t.Context())) + ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median) + require.NoError(t, m.Close()) + + ep.SendLog([]byte("p")) + + assert.Empty(t, src[beholder.DomainDataFeeds].sent()) + assert.Equal(t, int64(1), counterValue(t, reader, metricMessagesDropped, + attribute.String("telemetry_type", string(synchronization.OCR2Median)), + attribute.String("error_code", errorCodeEmitRejected))) +} + +func TestDomainManager_ConcurrentSends(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + for _, em := range src { + em.async = true + } + m, reader := newTestDomainManager(t, src) + + const workers, perWorker = 16, 50 + var wg sync.WaitGroup + for i := range workers { + wg.Go(func() { + ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0x"+strconv.Itoa(i), synchronization.OCR2Median) + mt := m.GenMultitypeMonitoringEndpoint("EVM", ethMainnetChainID, "0x"+strconv.Itoa(i)+"/1") + for range perWorker { + ep.SendLog([]byte("p")) + mt.SendTypedLog(synchronization.LLOOutcome, []byte("p")) + } + }) + } + wg.Wait() + + require.Eventually(t, func() bool { return counterValue(t, reader, metricMessagesSent) == 2*workers*perWorker }, + 5*time.Second, 10*time.Millisecond) + assert.Len(t, src[beholder.DomainDataFeeds].sent(), workers*perWorker) + assert.Len(t, src[beholder.DomainDataStreams].sent(), workers*perWorker) +} + +// The Prometheus counters are process-global, so this test does not run in parallel: sequential +// tests finish before parallel ones resume, which keeps the deltas below exact. +// +//nolint:paralleltest // asserts deltas on process-global Prometheus counters +func TestDomainManager_AlsoCountsInPrometheus(t *testing.T) { + const endpoint = "beholder/data-feeds" + tt := string(synchronization.OCR2Median) + sent := testutil.ToFloat64(synchronization.TelemetryClientMessagesSent.WithLabelValues(endpoint, tt)) + sendErrs := testutil.ToFloat64(synchronization.TelemetryClientMessagesSendErrors.WithLabelValues(endpoint, tt)) + dropped := testutil.ToFloat64(synchronization.TelemetryClientMessagesDropped.WithLabelValues(endpoint, tt)) + + for _, callbackErr := range []error{nil, status.Error(codes.Unavailable, "down"), batch.ErrMessageBufferFull} { + src := newFakeEmitterSource() + src[beholder.DomainDataFeeds].callbackErr = callbackErr + m, _ := newTestDomainManager(t, src) + m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median).SendLog([]byte("p")) + } + + assert.InDelta(t, sent+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesSent.WithLabelValues(endpoint, tt)), 0) + assert.InDelta(t, sendErrs+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesSendErrors.WithLabelValues(endpoint, tt)), 0) + assert.InDelta(t, dropped+1, testutil.ToFloat64(synchronization.TelemetryClientMessagesDropped.WithLabelValues(endpoint, tt)), 0) +} + +// Beholder domain emitters reject events whose source is not under their domain, so every +// supported type's source must be the domain or a subdomain of it. +func TestSupportedTelemetry_SourcesSitUnderTheirDomain(t *testing.T) { + t.Parallel() + for telemType, d := range supportedTelemetryDomains { + source, _, err := synchronization.TelemetryTypeToDomainAndEntity(telemType) + require.NoError(t, err, "%s has a CloudEvent source", telemType) + assert.True(t, source == string(d) || strings.HasPrefix(source, string(d)+"."), + "%s: source %q is not under domain %q", telemType, source, d) + } +} + +func newObservedDomainManager(t *testing.T, src DomainEmitterSource) (*DomainManager, *observer.ObservedLogs) { + t.Helper() + lggr, logs := logger.TestObserved(t, zapcore.WarnLevel) + m, err := NewDomainManager(src, testCSAPubKeyHex, lggr, + WithMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(sdkmetric.NewManualReader())))) + require.NoError(t, err) + return m, logs +} + +// A refused event means telemetry is being lost, so the first refusal per route and error code +// is logged at Error; repeats are only counted. +func TestDomainManager_LogsTheFirstRefusalPerRouteAtError(t *testing.T) { + t.Parallel() + src := newFakeEmitterSource() + src[beholder.DomainDataFeeds].emitErr = errors.New("not started") + m, logs := newObservedDomainManager(t, src) + + ep := m.GenMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc", synchronization.OCR2Median) + for range 3 { + ep.SendLog([]byte("p")) + } + + refusals := logs.FilterLevelExact(zapcore.ErrorLevel).FilterMessageSnippet("refused").All() + require.Len(t, refusals, 1, "logged once, however many events are refused") + assert.Equal(t, synchronization.OCR2Median, refusals[0].ContextMap()["telemType"]) +} + +func TestDomainManager_WarnsOnceForAnUnsupportedTypeOnAMultitypeEndpoint(t *testing.T) { + t.Parallel() + m, logs := newObservedDomainManager(t, newFakeEmitterSource()) + + ep := m.GenMultitypeMonitoringEndpoint("EVM", ethMainnetChainID, "0xabc/1") + for range 3 { + ep.SendTypedLog(synchronization.LLOReport, []byte("p")) + } + + warnings := logs.FilterLevelExact(zapcore.WarnLevel).FilterMessageSnippet("not supported").All() + require.Len(t, warnings, 1) + assert.Equal(t, synchronization.LLOReport, warnings[0].ContextMap()["telemType"]) +} diff --git a/core/services/telemetry/manager.go b/core/services/telemetry/manager.go index 9b24d46c627..a9394db02c6 100644 --- a/core/services/telemetry/manager.go +++ b/core/services/telemetry/manager.go @@ -18,7 +18,12 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) -type Manager struct { +var _ Manager = (*WSRPCManager)(nil) + +// WSRPCManager is the Manager that sends telemetry to the TelemetryIngress.Endpoints +// configured per network and chain, over WSRPC, or through ChIP Ingress when +// TelemetryIngress.ChipIngressEnabled is set. +type WSRPCManager struct { services.Service eng *services.Engine @@ -46,15 +51,15 @@ type telemetryEndpoint struct { chipIngressClient synchronization.ChipIngressService } -// NewManager create a new telemetry manager that is responsible for configuring telemetry agents and generating the defined telemetry endpoints and monitoring endpoints -func NewManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, lggr common.Logger) *Manager { +// NewWSRPCManager create a new telemetry manager that is responsible for configuring telemetry agents and generating the defined telemetry endpoints and monitoring endpoints +func NewWSRPCManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, lggr common.Logger) *WSRPCManager { var chipIngressClient chipingress.Client if cfg.ChipIngressEnabled() { lggr.Info("ChIP Ingress is enabled for telemetry") chipIngressClient = beholder.GetClient().Chip } - m := &Manager{ + m := &WSRPCManager{ bufferSize: cfg.BufferSize(), ks: csaKeyStore, logging: cfg.Logging(), @@ -83,7 +88,7 @@ func NewManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, lggr comm } // GenMonitoringEndpoint creates a new monitoring endpoints based on the existing available endpoints defined in the core config TOML, if no endpoint for the network and chainID exists, a NOOP agent will be used and the telemetry will not be sent -func (m *Manager) GenMonitoringEndpoint(network string, chainID string, contractID string, telemType synchronization.TelemetryType) commontypes.MonitoringEndpoint { +func (m *WSRPCManager) GenMonitoringEndpoint(network string, chainID string, contractID string, telemType synchronization.TelemetryType) commontypes.MonitoringEndpoint { e, found := m.getEndpoint(network, chainID) if !found { @@ -108,7 +113,7 @@ func (m *Manager) GenMonitoringEndpoint(network string, chainID string, contract return NewTypedIngressAgent(e.client, network, chainID, contractID, telemType) } -func (m *Manager) GenMultitypeMonitoringEndpoint(network string, chainID string, contractID string) MultitypeMonitoringEndpoint { +func (m *WSRPCManager) GenMultitypeMonitoringEndpoint(network string, chainID string, contractID string) MultitypeMonitoringEndpoint { e, found := m.getEndpoint(network, chainID) if !found { @@ -133,7 +138,7 @@ func (m *Manager) GenMultitypeMonitoringEndpoint(network string, chainID string, return NewMultiIngressAgent(e.client, network, chainID, contractID) } -func (m *Manager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Logger, cfg config.TelemetryIngress) (services.Service, error) { +func (m *WSRPCManager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Logger, cfg config.TelemetryIngress) (services.Service, error) { if e.Network() == "" { return nil, errors.New("cannot add telemetry endpoint, network cannot be empty") } @@ -191,7 +196,7 @@ func (m *Manager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Log return te.client, nil } -func (m *Manager) getEndpoint(network string, chainID string) (*telemetryEndpoint, bool) { +func (m *WSRPCManager) getEndpoint(network string, chainID string) (*telemetryEndpoint, bool) { for _, e := range m.endpoints { if e.Network == strings.ToUpper(network) && e.ChainID == strings.ToUpper(chainID) { return e, true diff --git a/core/services/telemetry/manager_test.go b/core/services/telemetry/manager_test.go index 143066e0894..c6a012f06fc 100644 --- a/core/services/telemetry/manager_test.go +++ b/core/services/telemetry/manager_test.go @@ -51,14 +51,14 @@ func TestManagerAgents(t *testing.T) { ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) require.Equal(t, "*synchronization.telemetryIngressBatchClient", reflect.TypeOf(tm.endpoints[0].client).String()) me := tm.GenMonitoringEndpoint("network-1", "network-1-chainID-1", "", "") assert.Equal(t, "*telemetry.TypedIngressAgentBatch", reflect.TypeOf(me).String()) tic = setupMockConfig(t, false, false) tic.On("Endpoints").Return([]config.TelemetryIngressEndpoint{te}) - tm = NewManager(tic, ks, lggr) + tm = NewWSRPCManager(tic, ks, lggr) require.Equal(t, "*synchronization.telemetryIngressClient", reflect.TypeOf(tm.endpoints[0].client).String()) me = tm.GenMonitoringEndpoint("network-1", "network-1-chainID-1", "", "") assert.Equal(t, "*telemetry.TypedIngressAgent", reflect.TypeOf(me).String()) @@ -154,7 +154,7 @@ func TestNewManager(t *testing.T) { key := csakey.MustNewV2XXXTestingOnly(big.NewInt(0)) ks.On("GetAll").Return([]csakey.KeyV2{key}, nil) ks.On("Get", key.ID()).Return(key, nil) - m := NewManager(tic, ks, lggr) + m := NewWSRPCManager(tic, ks, lggr) require.Equal(t, uint(123), m.bufferSize) require.Equal(t, ks, m.ks) @@ -203,7 +203,7 @@ func TestCorrectEndpointRouting(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) type testEndpoint struct { network string @@ -290,7 +290,7 @@ func TestManager_ChipIngressClient(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) assert.Nil(t, tm.chipIngressClient) }) @@ -300,7 +300,7 @@ func TestManager_ChipIngressClient(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) assert.NotNil(t, tm.chipIngressClient) }) } @@ -319,7 +319,7 @@ func TestManager_ChipIngressEndpoint(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) require.Len(t, tm.endpoints, 1) assert.NotNil(t, tm.endpoints[0].chipIngressClient) assert.Nil(t, tm.endpoints[0].client) @@ -341,7 +341,7 @@ func TestManager_ChipIngressEndpoint(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) require.Len(t, tm.endpoints, 1) assert.Nil(t, tm.endpoints[0].chipIngressClient) assert.NotNil(t, tm.endpoints[0].client) @@ -362,7 +362,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) me := tm.GenMonitoringEndpoint("EVM", "1", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) }) @@ -380,7 +380,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.DebugLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) // Start and cleanup the manager to ensure background goroutines are properly stopped require.NoError(t, tm.Start(t.Context())) t.Cleanup(func() { @@ -410,7 +410,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) // Invalid network/chainID combination that doesn't exist in chain-selectors me := tm.GenMonitoringEndpoint("INVALID", "999", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.NoopAgent", reflect.TypeOf(me).String()) @@ -445,7 +445,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) me := tm.GenMonitoringEndpoint("EVM", "1", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.TypedIngressAgentBatch", reflect.TypeOf(me).String()) }) @@ -465,7 +465,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) me := tm.GenMultitypeMonitoringEndpoint("EVM", "137", "0x456") assert.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) }) @@ -483,7 +483,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) // Invalid network/chainID combination me := tm.GenMultitypeMonitoringEndpoint("INVALID", "888", "0x456") assert.Equal(t, "*telemetry.NoopAgent", reflect.TypeOf(me).String()) @@ -518,7 +518,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewWSRPCManager(tic, ks, lggr) me := tm.GenMultitypeMonitoringEndpoint("EVM", "137", "0x456") assert.Equal(t, "*telemetry.MultiIngressAgentBatch", reflect.TypeOf(me).String()) }) diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index 3e6fb0dd366..0da4e4c6a87 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -44,6 +44,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 8735192c3ca..b74a0caa5cf 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -44,6 +44,7 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index d031ee10283..86abfb1d0d4 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -44,6 +44,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = true diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 8919ca2c7ad..de07dd2f332 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -281,6 +281,7 @@ SendInterval = '500ms' # Default SendTimeout = '10s' # Default UseBatchSend = true # Default ChipIngressEnabled = false # Default +BeholderEnabled = false # Default ``` @@ -332,6 +333,19 @@ ChipIngressEnabled = false # Default ``` ChipIngressEnabled enables sending telemetry to CHIP Ingress. +### BeholderEnabled +```toml +BeholderEnabled = false # Default +``` +BeholderEnabled sends telemetry through Beholder, one emitter per product domain, instead of the Endpoints below. +When set, Endpoints and the settings above (BufferSize, MaxBatchSize, SendInterval, SendTimeout, UseBatchSend, UniConn, Logging) +are ignored: the Telemetry.ChipIngress* settings size one buffer per domain, shared by all of that domain's jobs. +Requires Telemetry.Enabled and Telemetry.ChipIngressEndpoint, and chip ingress that serves PublishBatchNoAck for every domain +(there is no fallback). Cannot be combined with ChipIngressEnabled. +Telemetry is sent for every chain the node's jobs run on: Endpoints no longer select chains. Supported types are ocr, +ocr2-median, ocr3-data-feeds, enhanced-ea, ocr3-mercury, ocr3_1-mercury, enhanced-ea-mercury, llo-observation, llo-outcome, +pipeline-bridge, ocr2-ccip-commit, ocr2-ccip-exec, ocr3-ccip-commit, ocr3-ccip-exec and head-report; other types are dropped. + ## TelemetryIngress.Endpoints ```toml [[TelemetryIngress.Endpoints]] # Example diff --git a/go.mod b/go.mod index 23e3f7ff938..8f2b7894353 100644 --- a/go.mod +++ b/go.mod @@ -124,8 +124,10 @@ require ( go.opentelemetry.io/contrib/bridges/prometheus v0.68.0 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0 go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 go.opentelemetry.io/otel/log v1.47.0 go.opentelemetry.io/otel/metric v1.47.0 + go.opentelemetry.io/otel/sdk/log v1.47.0 go.opentelemetry.io/otel/sdk/metric v1.47.0 go.opentelemetry.io/otel/trace v1.47.0 go.uber.org/atomic v1.11.0 @@ -373,7 +375,6 @@ require ( go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 // indirect @@ -384,7 +385,6 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 // indirect go.opentelemetry.io/otel/sdk v1.47.0 // indirect - go.opentelemetry.io/otel/sdk/log v1.47.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/testdata/scripts/config/merge_raw_configs.txtar b/testdata/scripts/config/merge_raw_configs.txtar index 1463970ca35..a069ca9926e 100644 --- a/testdata/scripts/config/merge_raw_configs.txtar +++ b/testdata/scripts/config/merge_raw_configs.txtar @@ -191,6 +191,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index e423ec523f9..bd384f9d7d8 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -56,6 +56,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 20f3b565be0..1076fa70894 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -117,6 +117,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 721d1701c82..f1a32080b9c 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -100,6 +100,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index d5337a40c9a..6940039a950 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -100,6 +100,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index 7ff4cbf59b5..6abf9de76a0 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -100,6 +100,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index e34c251e4de..a966635764b 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -199,6 +199,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar index 3ed1117dd42..6af411c6ee5 100644 --- a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar +++ b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar @@ -85,6 +85,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 1c27e505de9..7f76893794a 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -96,6 +96,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index cfdb2cae79b..7179d2de3f5 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -97,6 +97,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index 9e15d037a5b..63febc37d50 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -79,6 +79,7 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +BeholderEnabled = false [AuditLogger] Enabled = false