diff --git a/core/cmd/shell.go b/core/cmd/shell.go index 64468775e00..d963e19ec3f 100644 --- a/core/cmd/shell.go +++ b/core/cmd/shell.go @@ -181,6 +181,21 @@ func newBeholderClient( MetricCardinalityLimit: cfgTelemetry.MetricCardinalityLimit(), } + // Opt-in gRPC-level retries for the chip ingress client (nil keeps them off). + // Unset knobs fall back to chipingress.DefaultRetryPolicy values. + retryPolicy, err := beholder.ChipIngressRetryConfig{ + Enabled: cfgTelemetry.ChipIngressRetryEnabled(), + MaxAttempts: cfgTelemetry.ChipIngressRetryMaxAttempts(), + InitialBackoff: cfgTelemetry.ChipIngressRetryInitialBackoff(), + MaxBackoff: cfgTelemetry.ChipIngressRetryMaxBackoff(), + BackoffMultiplier: cfgTelemetry.ChipIngressRetryBackoffMultiplier(), + RetryableStatusCodes: cfgTelemetry.ChipIngressRetryableStatusCodes(), + }.RetryPolicy() + if err != nil { + return nil, err + } + clientCfg.ChipIngressRetryPolicy = retryPolicy + if cfgTracing.Enabled() { tracingCfg := tracingConfig(cfgTracing, lggr) // add tracing attributes to resource attributes diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 30bc5c82a23..51b1fd0619d 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -904,6 +904,18 @@ ChipIngressSendTimeout = '10s' # Default ChipIngressDrainTimeout = '30s' # Default # ChipIngressMaxGRPCRequestSize is the max serialized PublishBatch request size in bytes. Batches exceeding this are split before send (min 1 MiB enforced by batch client). ChipIngressMaxGRPCRequestSize = 10485760 # Default +# ChipIngressRetryEnabled turns on gRPC-level retries for the node's chip ingress client. Retries replay failed RPCs (Publish/PublishBatch) on transient failures; without them every UNAVAILABLE/RESOURCE_EXHAUSTED publish is dropped. Off by default: replayed publishes can duplicate events unless an idempotency key is set. +ChipIngressRetryEnabled = false # Default +# ChipIngressRetryMaxAttempts is the max call attempts including the original (min 2; gRPC silently installs no retry policy below 2). Applies when ChipIngressRetryEnabled is true. +ChipIngressRetryMaxAttempts = 3 # Default +# ChipIngressRetryInitialBackoff is the base delay before the first retry attempt. Applies when ChipIngressRetryEnabled is true. +ChipIngressRetryInitialBackoff = '100ms' # Default +# ChipIngressRetryMaxBackoff caps the exponential backoff delay between retry attempts. Applies when ChipIngressRetryEnabled is true. +ChipIngressRetryMaxBackoff = '1s' # Default +# ChipIngressRetryBackoffMultiplier grows the backoff delay after each attempt. Applies when ChipIngressRetryEnabled is true. +ChipIngressRetryBackoffMultiplier = 2.0 # Default +# ChipIngressRetryableStatusCodes lists gRPC status code names eligible for retry (codes.Code.String() spelling, e.g. 'Unavailable', 'ResourceExhausted'). Applies when ChipIngressRetryEnabled is true. +ChipIngressRetryableStatusCodes = ['Unavailable', 'ResourceExhausted'] # Default # DurableEmitterEnabled enables persisting outbound CHIP events to Postgres for at-least-once delivery. DurableEmitterEnabled = true # Default # DurableEmitterRetransmitBatchSize is the number of pending events the durable emitter replays per retransmit tick. diff --git a/core/config/telemetry_config.go b/core/config/telemetry_config.go index 108786f0cb4..62c3d4d665e 100644 --- a/core/config/telemetry_config.go +++ b/core/config/telemetry_config.go @@ -26,6 +26,12 @@ type Telemetry interface { ChipIngressSendTimeout() time.Duration ChipIngressDrainTimeout() time.Duration ChipIngressMaxGRPCRequestSize() int + ChipIngressRetryEnabled() bool + ChipIngressRetryMaxAttempts() int + ChipIngressRetryInitialBackoff() time.Duration + ChipIngressRetryMaxBackoff() time.Duration + ChipIngressRetryBackoffMultiplier() float64 + ChipIngressRetryableStatusCodes() []string DurableEmitterEnabled() bool DurableEmitterRetransmitBatchSize() int DurableEmitterEventTTL() time.Duration diff --git a/core/config/toml/types.go b/core/config/toml/types.go index a69fcde9338..1c176ea89bd 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -3028,6 +3028,12 @@ type Telemetry struct { ChipIngressSendTimeout *commonconfig.Duration ChipIngressDrainTimeout *commonconfig.Duration ChipIngressMaxGRPCRequestSize *int + ChipIngressRetryEnabled *bool + ChipIngressRetryMaxAttempts *int + ChipIngressRetryInitialBackoff *commonconfig.Duration + ChipIngressRetryMaxBackoff *commonconfig.Duration + ChipIngressRetryBackoffMultiplier *float64 + ChipIngressRetryableStatusCodes []string DurableEmitterEnabled *bool DurableEmitterRetransmitBatchSize *int DurableEmitterEventTTL *commonconfig.Duration @@ -3105,6 +3111,24 @@ func (b *Telemetry) setFrom(f *Telemetry) { if v := f.ChipIngressMaxGRPCRequestSize; v != nil { b.ChipIngressMaxGRPCRequestSize = v } + if v := f.ChipIngressRetryEnabled; v != nil { + b.ChipIngressRetryEnabled = v + } + if v := f.ChipIngressRetryMaxAttempts; v != nil { + b.ChipIngressRetryMaxAttempts = v + } + if v := f.ChipIngressRetryInitialBackoff; v != nil { + b.ChipIngressRetryInitialBackoff = v + } + if v := f.ChipIngressRetryMaxBackoff; v != nil { + b.ChipIngressRetryMaxBackoff = v + } + if v := f.ChipIngressRetryBackoffMultiplier; v != nil { + b.ChipIngressRetryBackoffMultiplier = v + } + if v := f.ChipIngressRetryableStatusCodes; v != nil { + b.ChipIngressRetryableStatusCodes = v + } if v := f.DurableEmitterEnabled; v != nil { b.DurableEmitterEnabled = v } @@ -3190,6 +3214,18 @@ func (b *Telemetry) ValidateConfig() (err error) { if v := b.ChipIngressMaxGRPCRequestSize; v != nil && *v <= 0 { err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressMaxGRPCRequestSize", Value: *v, Msg: "must be greater than 0"}) } + if v := b.ChipIngressRetryMaxAttempts; v != nil && *v < 2 { + err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressRetryMaxAttempts", Value: *v, Msg: "must be at least 2 (gRPC silently installs no retry policy below 2 attempts)"}) + } + if v := b.ChipIngressRetryInitialBackoff; v != nil && v.Duration() <= 0 { + err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressRetryInitialBackoff", Value: v.Duration(), Msg: "must be greater than 0"}) + } + if v := b.ChipIngressRetryMaxBackoff; v != nil && v.Duration() <= 0 { + err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressRetryMaxBackoff", Value: v.Duration(), Msg: "must be greater than 0"}) + } + if v := b.ChipIngressRetryBackoffMultiplier; v != nil && *v <= 0 { + err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressRetryBackoffMultiplier", Value: *v, Msg: "must be greater than 0"}) + } return err } diff --git a/core/scripts/go.mod b/core/scripts/go.mod index e74ee12b257..37edb9e2458 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -33,7 +33,7 @@ require ( github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chain-selectors v1.0.111 github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260918135944-fa1a268dac47 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-deployments-framework v0.123.3 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261005112317-b723176adfe8 @@ -506,7 +506,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 // indirect github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 82d7ec0494d..78ca4280939 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1688,12 +1688,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 h1:ihs6hN8KnaszTFYZvtUxs03FKB/IxBg4bqJLzdDtWYw= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 h1:v3cQZmx/56xH6Jy+ui9HHsivMZ+QZ1Ehlwict/ikw8I= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227/go.mod h1:37Mu0pbAnGmHPgOZ8bQvgP4hzXw/Wu7smKX4cN46UOs= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 h1:Hh+OQI53w12AhLJ/zruSzWqpc0yA3kcCh4YOXwEumig= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 h1:C7Dv8SRUU97tsUJvDDOmntJI6RZxQflIi88NlvdUnCQ= diff --git a/core/services/chainlink/config_telemetry.go b/core/services/chainlink/config_telemetry.go index 399542883a4..2d878c9b909 100644 --- a/core/services/chainlink/config_telemetry.go +++ b/core/services/chainlink/config_telemetry.go @@ -23,6 +23,21 @@ const ( defaultDurableEmitterInsertBatchFlushInterval = 50 * time.Millisecond ) +// Defaults for the chip ingress client gRPC retry policy knobs. They mirror +// chainlink-common's chipingress.DefaultRetryPolicy so that enabling retries +// without overrides yields the recommended policy (mirrored in docs/core.toml). +const ( + defaultChipIngressRetryMaxAttempts = 3 + defaultChipIngressRetryInitialBackoff = 100 * time.Millisecond + defaultChipIngressRetryMaxBackoff = 1 * time.Second + defaultChipIngressRetryBackoffMultiplier = 2 +) + +// defaultChipIngressRetryableStatusCodes are the gRPC status code names +// (codes.Code.String() spelling, per chipingress.ParseStatusCodes) retried by +// default: transient transport/server pressure failures. +var defaultChipIngressRetryableStatusCodes = []string{"Unavailable", "ResourceExhausted"} + // defaultDurableEmitterMaxQueuePayloadBytes is the denominator used for the // durable_emitter.queue.capacity_usage_ratio gauge when the operator does not // override it via TOML. 1 GiB gives capacity tracking a sane default ceiling. @@ -168,6 +183,48 @@ func (b *telemetryConfig) ChipIngressMaxGRPCRequestSize() int { return *b.s.ChipIngressMaxGRPCRequestSize } +func (b *telemetryConfig) ChipIngressRetryEnabled() bool { + if b.s.ChipIngressRetryEnabled == nil { + return false + } + return *b.s.ChipIngressRetryEnabled +} + +func (b *telemetryConfig) ChipIngressRetryMaxAttempts() int { + if b.s.ChipIngressRetryMaxAttempts == nil { + return defaultChipIngressRetryMaxAttempts + } + return *b.s.ChipIngressRetryMaxAttempts +} + +func (b *telemetryConfig) ChipIngressRetryInitialBackoff() time.Duration { + if b.s.ChipIngressRetryInitialBackoff == nil { + return defaultChipIngressRetryInitialBackoff + } + return b.s.ChipIngressRetryInitialBackoff.Duration() +} + +func (b *telemetryConfig) ChipIngressRetryMaxBackoff() time.Duration { + if b.s.ChipIngressRetryMaxBackoff == nil { + return defaultChipIngressRetryMaxBackoff + } + return b.s.ChipIngressRetryMaxBackoff.Duration() +} + +func (b *telemetryConfig) ChipIngressRetryBackoffMultiplier() float64 { + if b.s.ChipIngressRetryBackoffMultiplier == nil { + return defaultChipIngressRetryBackoffMultiplier + } + return *b.s.ChipIngressRetryBackoffMultiplier +} + +func (b *telemetryConfig) ChipIngressRetryableStatusCodes() []string { + if b.s.ChipIngressRetryableStatusCodes == nil { + return defaultChipIngressRetryableStatusCodes + } + return b.s.ChipIngressRetryableStatusCodes +} + func (b *telemetryConfig) DurableEmitterEnabled() bool { if b.s.DurableEmitterEnabled == nil { return true diff --git a/core/services/chainlink/config_telemetry_test.go b/core/services/chainlink/config_telemetry_test.go index e4b20e259a8..9b8844e62da 100644 --- a/core/services/chainlink/config_telemetry_test.go +++ b/core/services/chainlink/config_telemetry_test.go @@ -5,8 +5,10 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" "github.com/smartcontractkit/chainlink/v2/core/config/docs" "github.com/smartcontractkit/chainlink/v2/core/config/toml" @@ -601,3 +603,141 @@ func TestTelemetryConfig_MetricCardinalityLimit(t *testing.T) { assert.Equal(t, 100000, tc.MetricCardinalityLimit()) }) } + +func TestTelemetryConfig_ChipIngressRetry(t *testing.T) { + t.Parallel() + + t.Run("Enabled", func(t *testing.T) { + t.Parallel() + tests := []struct { + name string + telemetry toml.Telemetry + expected bool + }{ + {"Set", toml.Telemetry{ChipIngressRetryEnabled: new(true)}, true}, + {"NilDisables", toml.Telemetry{}, false}, + {"ExplicitFalse", toml.Telemetry{ChipIngressRetryEnabled: new(false)}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tc := telemetryConfig{s: tt.telemetry} + assert.Equal(t, tt.expected, tc.ChipIngressRetryEnabled()) + }) + } + }) + + t.Run("MaxAttempts", func(t *testing.T) { + t.Parallel() + tests := []struct { + name string + telemetry toml.Telemetry + expected int + }{ + {"Set", toml.Telemetry{ChipIngressRetryMaxAttempts: new(5)}, 5}, + {"NilDefaults", toml.Telemetry{}, defaultChipIngressRetryMaxAttempts}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tc := telemetryConfig{s: tt.telemetry} + assert.Equal(t, tt.expected, tc.ChipIngressRetryMaxAttempts()) + }) + } + }) + + t.Run("InitialBackoff", func(t *testing.T) { + t.Parallel() + tests := []struct { + name string + telemetry toml.Telemetry + expected time.Duration + }{ + {"Set", toml.Telemetry{ChipIngressRetryInitialBackoff: commonconfig.MustNewDuration(250 * time.Millisecond)}, 250 * time.Millisecond}, + {"NilDefaults", toml.Telemetry{}, defaultChipIngressRetryInitialBackoff}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tc := telemetryConfig{s: tt.telemetry} + assert.Equal(t, tt.expected, tc.ChipIngressRetryInitialBackoff()) + }) + } + }) + + t.Run("MaxBackoff", func(t *testing.T) { + t.Parallel() + tests := []struct { + name string + telemetry toml.Telemetry + expected time.Duration + }{ + {"Set", toml.Telemetry{ChipIngressRetryMaxBackoff: commonconfig.MustNewDuration(5 * time.Second)}, 5 * time.Second}, + {"NilDefaults", toml.Telemetry{}, defaultChipIngressRetryMaxBackoff}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tc := telemetryConfig{s: tt.telemetry} + assert.Equal(t, tt.expected, tc.ChipIngressRetryMaxBackoff()) + }) + } + }) + + t.Run("BackoffMultiplier", func(t *testing.T) { + t.Parallel() + tests := []struct { + name string + telemetry toml.Telemetry + expected float64 + }{ + {"Set", toml.Telemetry{ChipIngressRetryBackoffMultiplier: new(1.5)}, 1.5}, + {"NilDefaults", toml.Telemetry{}, defaultChipIngressRetryBackoffMultiplier}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tc := telemetryConfig{s: tt.telemetry} + assert.InDelta(t, tt.expected, tc.ChipIngressRetryBackoffMultiplier(), 0.0001) + }) + } + }) + + t.Run("RetryableStatusCodes", func(t *testing.T) { + t.Parallel() + tests := []struct { + name string + telemetry toml.Telemetry + expected []string + }{ + {"Set", toml.Telemetry{ChipIngressRetryableStatusCodes: []string{"Unavailable"}}, []string{"Unavailable"}}, + {"NilDefaults", toml.Telemetry{}, defaultChipIngressRetryableStatusCodes}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tc := telemetryConfig{s: tt.telemetry} + assert.Equal(t, tt.expected, tc.ChipIngressRetryableStatusCodes()) + }) + } + }) + + t.Run("DefaultsMirrorCommonRecommendedPolicy", func(t *testing.T) { + t.Parallel() + // The getters' defaults must resolve to the same policy as + // chainlink-common's chipingress.DefaultRetryPolicy so that enabling + // retries without overrides yields the recommended policy. + tc := telemetryConfig{s: toml.Telemetry{}} + assert.False(t, tc.ChipIngressRetryEnabled(), "retries must stay off unless explicitly enabled") + + expected := chipingress.DefaultRetryPolicy() + assert.Equal(t, expected.MaxAttempts, tc.ChipIngressRetryMaxAttempts()) + assert.Equal(t, expected.InitialBackoff, tc.ChipIngressRetryInitialBackoff()) + assert.Equal(t, expected.MaxBackoff, tc.ChipIngressRetryMaxBackoff()) + assert.InDelta(t, expected.BackoffMultiplier, tc.ChipIngressRetryBackoffMultiplier(), 0.0001) + + codes, err := chipingress.ParseStatusCodes(tc.ChipIngressRetryableStatusCodes()) + require.NoError(t, err) + assert.Equal(t, expected.RetryableStatusCodes, codes) + }) +} diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index f1dc56f7ef2..f6b9b0fa45a 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -569,6 +569,12 @@ func TestConfig_Marshal(t *testing.T) { ChipIngressSendTimeout: commonconfig.MustNewDuration(10 * time.Second), ChipIngressDrainTimeout: commonconfig.MustNewDuration(30 * time.Second), ChipIngressMaxGRPCRequestSize: new(10485760), + ChipIngressRetryEnabled: new(false), + ChipIngressRetryMaxAttempts: new(3), + ChipIngressRetryInitialBackoff: commonconfig.MustNewDuration(100 * time.Millisecond), + ChipIngressRetryMaxBackoff: commonconfig.MustNewDuration(1 * time.Second), + ChipIngressRetryBackoffMultiplier: new(2.0), + ChipIngressRetryableStatusCodes: []string{"Unavailable", "ResourceExhausted"}, DurableEmitterEnabled: new(false), DurableEmitterRetransmitBatchSize: new(500), DurableEmitterEventTTL: commonconfig.MustNewDuration(6 * time.Hour), diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index c1459f2bd6e..ebba7beb5bc 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -353,6 +353,12 @@ ChipIngressSendInterval = '500ms' ChipIngressSendTimeout = '10s' ChipIngressDrainTimeout = '30s' ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressRetryEnabled = false +ChipIngressRetryMaxAttempts = 3 +ChipIngressRetryInitialBackoff = '100ms' +ChipIngressRetryMaxBackoff = '1s' +ChipIngressRetryBackoffMultiplier = 2.0 +ChipIngressRetryableStatusCodes = ['Unavailable', 'ResourceExhausted'] DurableEmitterEnabled = true DurableEmitterRetransmitBatchSize = 500 DurableEmitterEventTTL = '6h0m0s' diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index 1e3ea6346ee..75f0bea3385 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -388,6 +388,12 @@ ChipIngressSendInterval = '500ms' ChipIngressSendTimeout = '10s' ChipIngressDrainTimeout = '30s' ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressRetryEnabled = false +ChipIngressRetryMaxAttempts = 3 +ChipIngressRetryInitialBackoff = '100ms' +ChipIngressRetryMaxBackoff = '1s' +ChipIngressRetryBackoffMultiplier = 2.0 +ChipIngressRetryableStatusCodes = ['Unavailable', 'ResourceExhausted'] DurableEmitterEnabled = false DurableEmitterRetransmitBatchSize = 500 DurableEmitterEventTTL = '6h0m0s' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index dba938357aa..34240764077 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -353,6 +353,12 @@ ChipIngressSendInterval = '500ms' ChipIngressSendTimeout = '10s' ChipIngressDrainTimeout = '30s' ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressRetryEnabled = false +ChipIngressRetryMaxAttempts = 3 +ChipIngressRetryInitialBackoff = '100ms' +ChipIngressRetryMaxBackoff = '1s' +ChipIngressRetryBackoffMultiplier = 2.0 +ChipIngressRetryableStatusCodes = ['Unavailable', 'ResourceExhausted'] DurableEmitterEnabled = true DurableEmitterRetransmitBatchSize = 500 DurableEmitterEventTTL = '6h0m0s' diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go index 60eef3c0c75..763274e8da7 100644 --- a/core/services/workflows/v2/confidential_module_test.go +++ b/core/services/workflows/v2/confidential_module_test.go @@ -50,6 +50,10 @@ func (s *stubExecutionHelper) GetRawSecrets(ctx context.Context, request *sdkpb. return nil, nil } +func (s *stubExecutionHelper) GetRawSecretsResponse(ctx context.Context, request *sdkpb.GetSecretsRequest, fetcher host.EncryptionKeyFetcher) (*vaultcommon.GetSecretsResponse, error) { + return nil, nil +} + func (s *stubExecutionHelper) GetOwner() string { return "" } diff --git a/deployment/go.mod b/deployment/go.mod index 4083d90008b..a308c9e434c 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -46,7 +46,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/deployment v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 github.com/smartcontractkit/chainlink-deployments-framework v0.123.3 @@ -435,7 +435,7 @@ require ( github.com/smartcontractkit/chainlink-canton/authentication v1.0.0 // indirect github.com/smartcontractkit/chainlink-canton/contracts/v2 v2.1.0 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260723212800-b2f21d31b1d2 // indirect diff --git a/deployment/go.sum b/deployment/go.sum index d7cac93d354..6cc158c1012 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1409,12 +1409,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 h1:ihs6hN8KnaszTFYZvtUxs03FKB/IxBg4bqJLzdDtWYw= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 h1:v3cQZmx/56xH6Jy+ui9HHsivMZ+QZ1Ehlwict/ikw8I= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227/go.mod h1:37Mu0pbAnGmHPgOZ8bQvgP4hzXw/Wu7smKX4cN46UOs= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 h1:Hh+OQI53w12AhLJ/zruSzWqpc0yA3kcCh4YOXwEumig= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 h1:C7Dv8SRUU97tsUJvDDOmntJI6RZxQflIi88NlvdUnCQ= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.123.3 h1:CeN/QrCpLFCdWBe/7Tzh/HuXjgTtSq/98YYj40/hHlQ= diff --git a/docs/CHIP_INGRESS_RETRY.md b/docs/CHIP_INGRESS_RETRY.md new file mode 100644 index 00000000000..cc1dd620340 --- /dev/null +++ b/docs/CHIP_INGRESS_RETRY.md @@ -0,0 +1,48 @@ +--- +title: Chip Ingress Retry Policy +sidebar_position: 4 +--- + +# Chip Ingress retry policy + +The node's CHIP ingress publishes (custom events, workflow telemetry, heartbeats) go over gRPC. +Without retries, every transient publish failure — an `UNAVAILABLE` disconnect, a +`RESOURCE_EXHAUSTED` server under pressure — drops the batch permanently. gRPC-level retries +can be enabled per node in Core TOML, scoped to the `ChipIngress` service: + +```text +[Telemetry] +ChipIngressRetryEnabled = true +``` + +Off by default. When enabled, unset knobs fall back to the recommended policy +(3 attempts, 100ms initial backoff doubling to a 1s cap, retrying `Unavailable` and +`ResourceExhausted`), paired with gRPC retry throttling so a struggling server is not +amplified: + +```text +[Telemetry] +ChipIngressRetryEnabled = true +ChipIngressRetryMaxAttempts = 5 +ChipIngressRetryInitialBackoff = '100ms' +ChipIngressRetryMaxBackoff = '1s' +ChipIngressRetryBackoffMultiplier = 2.0 +ChipIngressRetryableStatusCodes = ['Unavailable', 'ResourceExhausted'] +``` + +- `ChipIngressRetryMaxAttempts` counts the original attempt and must be at least 2; gRPC + silently installs no retry policy below that. +- `ChipIngressRetryableStatusCodes` uses `codes.Code.String()` spelling + (e.g. `Unavailable`, `ResourceExhausted`, `DeadlineExceeded`); unknown names fail node + startup instead of being ignored. +- Retries share the caller's RPC deadline; they cannot rescue calls that already exhausted it. + +Every value can also be set via environment variable, e.g. +`CL_TELEMETRY_CHIP_INGRESS_RETRY_ENABLED=true`, and the same settings are passed to LOOP +plugin subprocesses (their CHIP publishes use the same policy). + +Retried publishes can duplicate events: an `UNAVAILABLE` may be returned after the server +already did the work. Event producers that set the `idempotencykey` extension are protected; +otherwise weigh enabling retries against duplicate tolerance for your event types. The +durable emitter path is unaffected — it already guarantees delivery via its own retransmit +loop and is intentionally left without gRPC retries. diff --git a/docs/CONFIG.md b/docs/CONFIG.md index e619ec40f0f..63539d4350d 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2381,6 +2381,12 @@ ChipIngressSendInterval = '500ms' # Default ChipIngressSendTimeout = '10s' # Default ChipIngressDrainTimeout = '30s' # Default ChipIngressMaxGRPCRequestSize = 10485760 # Default +ChipIngressRetryEnabled = false # Default +ChipIngressRetryMaxAttempts = 3 # Default +ChipIngressRetryInitialBackoff = '100ms' # Default +ChipIngressRetryMaxBackoff = '1s' # Default +ChipIngressRetryBackoffMultiplier = 2.0 # Default +ChipIngressRetryableStatusCodes = ['Unavailable', 'ResourceExhausted'] # Default DurableEmitterEnabled = true # Default DurableEmitterRetransmitBatchSize = 500 # Default DurableEmitterEventTTL = '6h0m0s' # Default @@ -2513,6 +2519,42 @@ ChipIngressMaxGRPCRequestSize = 10485760 # Default ``` ChipIngressMaxGRPCRequestSize is the max serialized PublishBatch request size in bytes. Batches exceeding this are split before send (min 1 MiB enforced by batch client). +### ChipIngressRetryEnabled +```toml +ChipIngressRetryEnabled = false # Default +``` +ChipIngressRetryEnabled turns on gRPC-level retries for the node's chip ingress client. Retries replay failed RPCs (Publish/PublishBatch) on transient failures; without them every UNAVAILABLE/RESOURCE_EXHAUSTED publish is dropped. Off by default: replayed publishes can duplicate events unless an idempotency key is set. + +### ChipIngressRetryMaxAttempts +```toml +ChipIngressRetryMaxAttempts = 3 # Default +``` +ChipIngressRetryMaxAttempts is the max call attempts including the original (min 2; gRPC silently installs no retry policy below 2). Applies when ChipIngressRetryEnabled is true. + +### ChipIngressRetryInitialBackoff +```toml +ChipIngressRetryInitialBackoff = '100ms' # Default +``` +ChipIngressRetryInitialBackoff is the base delay before the first retry attempt. Applies when ChipIngressRetryEnabled is true. + +### ChipIngressRetryMaxBackoff +```toml +ChipIngressRetryMaxBackoff = '1s' # Default +``` +ChipIngressRetryMaxBackoff caps the exponential backoff delay between retry attempts. Applies when ChipIngressRetryEnabled is true. + +### ChipIngressRetryBackoffMultiplier +```toml +ChipIngressRetryBackoffMultiplier = 2.0 # Default +``` +ChipIngressRetryBackoffMultiplier grows the backoff delay after each attempt. Applies when ChipIngressRetryEnabled is true. + +### ChipIngressRetryableStatusCodes +```toml +ChipIngressRetryableStatusCodes = ['Unavailable', 'ResourceExhausted'] # Default +``` +ChipIngressRetryableStatusCodes lists gRPC status code names eligible for retry (codes.Code.String() spelling, e.g. 'Unavailable', 'ResourceExhausted'). Applies when ChipIngressRetryEnabled is true. + ### DurableEmitterEnabled ```toml DurableEmitterEnabled = true # Default diff --git a/docs/index.md b/docs/index.md index bb5a8fc1138..a8c1951536e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,6 +11,7 @@ This local docs bundle exposes documentation from the `chainlink` repository thr - [Local CRE](local-cre/index.md) - [Configuration Reference](CONFIG.md) +- [Chip Ingress Retry Policy](CHIP_INGRESS_RETRY.md) - [Secrets Reference](SECRETS.md) - [Contributing](CONTRIBUTING.md) - [Community](COMMUNITY.md) diff --git a/go.mod b/go.mod index e049e8ce915..b35f006ff8d 100644 --- a/go.mod +++ b/go.mod @@ -82,9 +82,9 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261005112317-b723176adfe8 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 diff --git a/go.sum b/go.sum index 2642a62b24c..ea75edf2f77 100644 --- a/go.sum +++ b/go.sum @@ -1104,12 +1104,12 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62/go.mod h1:c++Ed0jeYn4w8WIfiAboRwN6ClRRVGglL0qwB/QjJvM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 h1:ihs6hN8KnaszTFYZvtUxs03FKB/IxBg4bqJLzdDtWYw= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 h1:v3cQZmx/56xH6Jy+ui9HHsivMZ+QZ1Ehlwict/ikw8I= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227/go.mod h1:37Mu0pbAnGmHPgOZ8bQvgP4hzXw/Wu7smKX4cN46UOs= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 h1:Hh+OQI53w12AhLJ/zruSzWqpc0yA3kcCh4YOXwEumig= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 h1:C7Dv8SRUU97tsUJvDDOmntJI6RZxQflIi88NlvdUnCQ= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261005112317-b723176adfe8 h1:49f3Vk1Igc5UTQc26lYrS1/+du+imLBKoYW2IX/SiCM= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 2708b26f03b..edf5efc1a42 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -34,7 +34,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260918135944-fa1a268dac47 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-deployments-framework v0.123.3 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261005112317-b723176adfe8 @@ -411,7 +411,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/deployment v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 7bf7053772e..d91a1988e47 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1394,12 +1394,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 h1:ihs6hN8KnaszTFYZvtUxs03FKB/IxBg4bqJLzdDtWYw= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 h1:v3cQZmx/56xH6Jy+ui9HHsivMZ+QZ1Ehlwict/ikw8I= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227/go.mod h1:37Mu0pbAnGmHPgOZ8bQvgP4hzXw/Wu7smKX4cN46UOs= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 h1:Hh+OQI53w12AhLJ/zruSzWqpc0yA3kcCh4YOXwEumig= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 h1:C7Dv8SRUU97tsUJvDDOmntJI6RZxQflIi88NlvdUnCQ= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.123.3 h1:CeN/QrCpLFCdWBe/7Tzh/HuXjgTtSq/98YYj40/hHlQ= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index d20a73a9f6a..81300c7c935 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -24,7 +24,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260918135944-fa1a268dac47 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 github.com/smartcontractkit/chainlink-deployments-framework v0.123.3 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261005112317-b723176adfe8 github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.8 @@ -482,7 +482,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index fabac03ab44..f542b4b9bdb 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1634,12 +1634,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 h1:ihs6hN8KnaszTFYZvtUxs03FKB/IxBg4bqJLzdDtWYw= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 h1:v3cQZmx/56xH6Jy+ui9HHsivMZ+QZ1Ehlwict/ikw8I= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227/go.mod h1:37Mu0pbAnGmHPgOZ8bQvgP4hzXw/Wu7smKX4cN46UOs= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 h1:Hh+OQI53w12AhLJ/zruSzWqpc0yA3kcCh4YOXwEumig= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 h1:C7Dv8SRUU97tsUJvDDOmntJI6RZxQflIi88NlvdUnCQ= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.123.3 h1:CeN/QrCpLFCdWBe/7Tzh/HuXjgTtSq/98YYj40/hHlQ= diff --git a/plugins/loop_registry.go b/plugins/loop_registry.go index ef22f6eff1d..fc5a50659bf 100644 --- a/plugins/loop_registry.go +++ b/plugins/loop_registry.go @@ -165,6 +165,12 @@ func (m *LoopRegistry) Register(id string) (*RegisteredLoop, error) { envCfg.ChipIngressSendTimeout = m.cfgTelemetry.ChipIngressSendTimeout() envCfg.ChipIngressDrainTimeout = m.cfgTelemetry.ChipIngressDrainTimeout() envCfg.ChipIngressMaxGRPCRequestSize = m.cfgTelemetry.ChipIngressMaxGRPCRequestSize() + envCfg.ChipIngressRetryEnabled = m.cfgTelemetry.ChipIngressRetryEnabled() + envCfg.ChipIngressRetryMaxAttempts = m.cfgTelemetry.ChipIngressRetryMaxAttempts() + envCfg.ChipIngressRetryInitialBackoff = m.cfgTelemetry.ChipIngressRetryInitialBackoff() + envCfg.ChipIngressRetryMaxBackoff = m.cfgTelemetry.ChipIngressRetryMaxBackoff() + envCfg.ChipIngressRetryBackoffMultiplier = m.cfgTelemetry.ChipIngressRetryBackoffMultiplier() + envCfg.ChipIngressRetryableStatusCodes = m.cfgTelemetry.ChipIngressRetryableStatusCodes() envCfg.TelemetryLogStreamingEnabled = m.cfgTelemetry.LogStreamingEnabled() envCfg.TelemetryLogLevel = m.cfgTelemetry.LogLevel() envCfg.TelemetryLogBatchProcessor = m.cfgTelemetry.LogBatchProcessor() diff --git a/plugins/loop_registry_test.go b/plugins/loop_registry_test.go index 91fb8c6c8b6..e24be7d2ba7 100644 --- a/plugins/loop_registry_test.go +++ b/plugins/loop_registry_test.go @@ -100,6 +100,22 @@ func (m mockCfgTelemetry) ChipIngressDrainTimeout() time.Duration { return 10 * func (m mockCfgTelemetry) ChipIngressMaxGRPCRequestSize() int { return 10485760 } +func (m mockCfgTelemetry) ChipIngressRetryEnabled() bool { return true } + +func (m mockCfgTelemetry) ChipIngressRetryMaxAttempts() int { return 5 } + +func (m mockCfgTelemetry) ChipIngressRetryInitialBackoff() time.Duration { + return 250 * time.Millisecond +} + +func (m mockCfgTelemetry) ChipIngressRetryMaxBackoff() time.Duration { return 5 * time.Second } + +func (m mockCfgTelemetry) ChipIngressRetryBackoffMultiplier() float64 { return 1.5 } + +func (m mockCfgTelemetry) ChipIngressRetryableStatusCodes() []string { + return []string{"Unavailable", "ResourceExhausted"} +} + func (m mockCfgTelemetry) HeartbeatInterval() time.Duration { return 5 * time.Second } @@ -313,4 +329,10 @@ func TestLoopRegistry_Register(t *testing.T) { require.Equal(t, 10*time.Second, envCfg.ChipIngressSendTimeout) require.Equal(t, 10*time.Second, envCfg.ChipIngressDrainTimeout) require.Equal(t, 10485760, envCfg.ChipIngressMaxGRPCRequestSize) + require.True(t, envCfg.ChipIngressRetryEnabled) + require.Equal(t, 5, envCfg.ChipIngressRetryMaxAttempts) + require.Equal(t, 250*time.Millisecond, envCfg.ChipIngressRetryInitialBackoff) + require.Equal(t, 5*time.Second, envCfg.ChipIngressRetryMaxBackoff) + require.InDelta(t, 1.5, envCfg.ChipIngressRetryBackoffMultiplier, 0.0001) + require.Equal(t, []string{"Unavailable", "ResourceExhausted"}, envCfg.ChipIngressRetryableStatusCodes) } diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index b7279b5b9d0..e25e8709682 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -37,7 +37,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.111 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-deployments-framework v0.123.3 @@ -491,7 +491,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index a27e90b69d7..b60fc9420bd 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1659,12 +1659,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 h1:ihs6hN8KnaszTFYZvtUxs03FKB/IxBg4bqJLzdDtWYw= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 h1:v3cQZmx/56xH6Jy+ui9HHsivMZ+QZ1Ehlwict/ikw8I= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227/go.mod h1:37Mu0pbAnGmHPgOZ8bQvgP4hzXw/Wu7smKX4cN46UOs= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 h1:Hh+OQI53w12AhLJ/zruSzWqpc0yA3kcCh4YOXwEumig= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 h1:C7Dv8SRUU97tsUJvDDOmntJI6RZxQflIi88NlvdUnCQ= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index 9c7f2169741..db0c0bfde1d 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -76,9 +76,9 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.111 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers v0.0.0-20260812145307-d77342c53d7d github.com/smartcontractkit/chainlink-deployments-framework v0.123.3 diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index da9527882fa..f91d12205f3 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1838,12 +1838,12 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3 h1:ihs6hN8KnaszTFYZvtUxs03FKB/IxBg4bqJLzdDtWYw= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227 h1:v3cQZmx/56xH6Jy+ui9HHsivMZ+QZ1Ehlwict/ikw8I= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227/go.mod h1:37Mu0pbAnGmHPgOZ8bQvgP4hzXw/Wu7smKX4cN46UOs= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 h1:Hh+OQI53w12AhLJ/zruSzWqpc0yA3kcCh4YOXwEumig= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers v0.0.0-20260812145307-d77342c53d7d h1:CZ0Om7lANyhpFJcmZE8RwQpEP4PyvoNLhOJckjm+zao=