Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions core/cmd/shell.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,21 @@ func newBeholderClient(
MetricCardinalityLimit: cfgTelemetry.MetricCardinalityLimit(),
}

// Opt-in gRPC-level retries for the chip ingress client (nil keeps them off).
// Unset knobs fall back to chipingress.DefaultRetryPolicy values.
retryPolicy, err := beholder.ChipIngressRetryConfig{
Enabled: cfgTelemetry.ChipIngressRetryEnabled(),
MaxAttempts: cfgTelemetry.ChipIngressRetryMaxAttempts(),
InitialBackoff: cfgTelemetry.ChipIngressRetryInitialBackoff(),
MaxBackoff: cfgTelemetry.ChipIngressRetryMaxBackoff(),
BackoffMultiplier: cfgTelemetry.ChipIngressRetryBackoffMultiplier(),
RetryableStatusCodes: cfgTelemetry.ChipIngressRetryableStatusCodes(),
}.RetryPolicy()
if err != nil {
return nil, err
}
clientCfg.ChipIngressRetryPolicy = retryPolicy

if cfgTracing.Enabled() {
tracingCfg := tracingConfig(cfgTracing, lggr)
// add tracing attributes to resource attributes
Expand Down
12 changes: 12 additions & 0 deletions core/config/docs/core.toml
Original file line number Diff line number Diff line change
Expand Up @@ -904,6 +904,18 @@ ChipIngressSendTimeout = '10s' # Default
ChipIngressDrainTimeout = '30s' # Default
# ChipIngressMaxGRPCRequestSize is the max serialized PublishBatch request size in bytes. Batches exceeding this are split before send (min 1 MiB enforced by batch client).
ChipIngressMaxGRPCRequestSize = 10485760 # Default
# ChipIngressRetryEnabled turns on gRPC-level retries for the node's chip ingress client. Retries replay failed RPCs (Publish/PublishBatch) on transient failures; without them every UNAVAILABLE/RESOURCE_EXHAUSTED publish is dropped. Off by default: replayed publishes can duplicate events unless an idempotency key is set.
ChipIngressRetryEnabled = false # Default
# ChipIngressRetryMaxAttempts is the max call attempts including the original (min 2; gRPC silently installs no retry policy below 2). Applies when ChipIngressRetryEnabled is true.
ChipIngressRetryMaxAttempts = 3 # Default
# ChipIngressRetryInitialBackoff is the base delay before the first retry attempt. Applies when ChipIngressRetryEnabled is true.
ChipIngressRetryInitialBackoff = '100ms' # Default
# ChipIngressRetryMaxBackoff caps the exponential backoff delay between retry attempts. Applies when ChipIngressRetryEnabled is true.
ChipIngressRetryMaxBackoff = '1s' # Default
# ChipIngressRetryBackoffMultiplier grows the backoff delay after each attempt. Applies when ChipIngressRetryEnabled is true.
ChipIngressRetryBackoffMultiplier = 2.0 # Default
# ChipIngressRetryableStatusCodes lists gRPC status code names eligible for retry (codes.Code.String() spelling, e.g. 'Unavailable', 'ResourceExhausted'). Applies when ChipIngressRetryEnabled is true.
ChipIngressRetryableStatusCodes = ['Unavailable', 'ResourceExhausted'] # Default
# DurableEmitterEnabled enables persisting outbound CHIP events to Postgres for at-least-once delivery.
DurableEmitterEnabled = true # Default
# DurableEmitterRetransmitBatchSize is the number of pending events the durable emitter replays per retransmit tick.
Expand Down
6 changes: 6 additions & 0 deletions core/config/telemetry_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ type Telemetry interface {
ChipIngressSendTimeout() time.Duration
ChipIngressDrainTimeout() time.Duration
ChipIngressMaxGRPCRequestSize() int
ChipIngressRetryEnabled() bool
ChipIngressRetryMaxAttempts() int
ChipIngressRetryInitialBackoff() time.Duration
ChipIngressRetryMaxBackoff() time.Duration
ChipIngressRetryBackoffMultiplier() float64
ChipIngressRetryableStatusCodes() []string
DurableEmitterEnabled() bool
DurableEmitterRetransmitBatchSize() int
DurableEmitterEventTTL() time.Duration
Expand Down
36 changes: 36 additions & 0 deletions core/config/toml/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -3028,6 +3028,12 @@ type Telemetry struct {
ChipIngressSendTimeout *commonconfig.Duration
ChipIngressDrainTimeout *commonconfig.Duration
ChipIngressMaxGRPCRequestSize *int
ChipIngressRetryEnabled *bool
ChipIngressRetryMaxAttempts *int
ChipIngressRetryInitialBackoff *commonconfig.Duration
ChipIngressRetryMaxBackoff *commonconfig.Duration
ChipIngressRetryBackoffMultiplier *float64
ChipIngressRetryableStatusCodes []string
DurableEmitterEnabled *bool
DurableEmitterRetransmitBatchSize *int
DurableEmitterEventTTL *commonconfig.Duration
Expand Down Expand Up @@ -3105,6 +3111,24 @@ func (b *Telemetry) setFrom(f *Telemetry) {
if v := f.ChipIngressMaxGRPCRequestSize; v != nil {
b.ChipIngressMaxGRPCRequestSize = v
}
if v := f.ChipIngressRetryEnabled; v != nil {
b.ChipIngressRetryEnabled = v
}
if v := f.ChipIngressRetryMaxAttempts; v != nil {
b.ChipIngressRetryMaxAttempts = v
}
if v := f.ChipIngressRetryInitialBackoff; v != nil {
b.ChipIngressRetryInitialBackoff = v
}
if v := f.ChipIngressRetryMaxBackoff; v != nil {
b.ChipIngressRetryMaxBackoff = v
}
if v := f.ChipIngressRetryBackoffMultiplier; v != nil {
b.ChipIngressRetryBackoffMultiplier = v
}
if v := f.ChipIngressRetryableStatusCodes; v != nil {
b.ChipIngressRetryableStatusCodes = v
}
if v := f.DurableEmitterEnabled; v != nil {
b.DurableEmitterEnabled = v
}
Expand Down Expand Up @@ -3190,6 +3214,18 @@ func (b *Telemetry) ValidateConfig() (err error) {
if v := b.ChipIngressMaxGRPCRequestSize; v != nil && *v <= 0 {
err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressMaxGRPCRequestSize", Value: *v, Msg: "must be greater than 0"})
}
if v := b.ChipIngressRetryMaxAttempts; v != nil && *v < 2 {
err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressRetryMaxAttempts", Value: *v, Msg: "must be at least 2 (gRPC silently installs no retry policy below 2 attempts)"})
}
if v := b.ChipIngressRetryInitialBackoff; v != nil && v.Duration() <= 0 {
err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressRetryInitialBackoff", Value: v.Duration(), Msg: "must be greater than 0"})
}
if v := b.ChipIngressRetryMaxBackoff; v != nil && v.Duration() <= 0 {
err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressRetryMaxBackoff", Value: v.Duration(), Msg: "must be greater than 0"})
}
if v := b.ChipIngressRetryBackoffMultiplier; v != nil && *v <= 0 {
err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressRetryBackoffMultiplier", Value: *v, Msg: "must be greater than 0"})
}
return err
}

Expand Down
4 changes: 2 additions & 2 deletions core/scripts/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ require (
github.com/shopspring/decimal v1.4.0
github.com/smartcontractkit/chain-selectors v1.0.111
github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260918135944-fa1a268dac47
github.com/smartcontractkit/chainlink-common v0.11.2-0.20260930142354-8fdc7816bef3
github.com/smartcontractkit/chainlink-common v0.11.2-0.20261006033306-84edf0848227
github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d
github.com/smartcontractkit/chainlink-deployments-framework v0.123.3
github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261005112317-b723176adfe8
Expand Down Expand Up @@ -506,7 +506,7 @@ require (
github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 // indirect
github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260918135944-fa1a268dac47 // indirect
github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect
github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect
github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261006033306-84edf0848227 // indirect
github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 // indirect
github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261002081259-6c2163b21db9 // indirect
github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect
Expand Down
8 changes: 4 additions & 4 deletions core/scripts/go.sum

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

57 changes: 57 additions & 0 deletions core/services/chainlink/config_telemetry.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,21 @@ const (
defaultDurableEmitterInsertBatchFlushInterval = 50 * time.Millisecond
)

// Defaults for the chip ingress client gRPC retry policy knobs. They mirror
// chainlink-common's chipingress.DefaultRetryPolicy so that enabling retries
// without overrides yields the recommended policy (mirrored in docs/core.toml).
const (
defaultChipIngressRetryMaxAttempts = 3
defaultChipIngressRetryInitialBackoff = 100 * time.Millisecond
defaultChipIngressRetryMaxBackoff = 1 * time.Second
defaultChipIngressRetryBackoffMultiplier = 2
)

// defaultChipIngressRetryableStatusCodes are the gRPC status code names
// (codes.Code.String() spelling, per chipingress.ParseStatusCodes) retried by
// default: transient transport/server pressure failures.
var defaultChipIngressRetryableStatusCodes = []string{"Unavailable", "ResourceExhausted"}

// defaultDurableEmitterMaxQueuePayloadBytes is the denominator used for the
// durable_emitter.queue.capacity_usage_ratio gauge when the operator does not
// override it via TOML. 1 GiB gives capacity tracking a sane default ceiling.
Expand Down Expand Up @@ -168,6 +183,48 @@ func (b *telemetryConfig) ChipIngressMaxGRPCRequestSize() int {
return *b.s.ChipIngressMaxGRPCRequestSize
}

func (b *telemetryConfig) ChipIngressRetryEnabled() bool {
if b.s.ChipIngressRetryEnabled == nil {
return false
}
return *b.s.ChipIngressRetryEnabled
}

func (b *telemetryConfig) ChipIngressRetryMaxAttempts() int {
if b.s.ChipIngressRetryMaxAttempts == nil {
return defaultChipIngressRetryMaxAttempts
}
return *b.s.ChipIngressRetryMaxAttempts
}

func (b *telemetryConfig) ChipIngressRetryInitialBackoff() time.Duration {
if b.s.ChipIngressRetryInitialBackoff == nil {
return defaultChipIngressRetryInitialBackoff
}
return b.s.ChipIngressRetryInitialBackoff.Duration()
}

func (b *telemetryConfig) ChipIngressRetryMaxBackoff() time.Duration {
if b.s.ChipIngressRetryMaxBackoff == nil {
return defaultChipIngressRetryMaxBackoff
}
return b.s.ChipIngressRetryMaxBackoff.Duration()
}

func (b *telemetryConfig) ChipIngressRetryBackoffMultiplier() float64 {
if b.s.ChipIngressRetryBackoffMultiplier == nil {
return defaultChipIngressRetryBackoffMultiplier
}
return *b.s.ChipIngressRetryBackoffMultiplier
}

func (b *telemetryConfig) ChipIngressRetryableStatusCodes() []string {
if b.s.ChipIngressRetryableStatusCodes == nil {
return defaultChipIngressRetryableStatusCodes
}
return b.s.ChipIngressRetryableStatusCodes
}

func (b *telemetryConfig) DurableEmitterEnabled() bool {
if b.s.DurableEmitterEnabled == nil {
return true
Expand Down
140 changes: 140 additions & 0 deletions core/services/chainlink/config_telemetry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ import (
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zapcore"

"github.com/smartcontractkit/chainlink-common/pkg/chipingress"
commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config"
"github.com/smartcontractkit/chainlink/v2/core/config/docs"
"github.com/smartcontractkit/chainlink/v2/core/config/toml"
Expand Down Expand Up @@ -601,3 +603,141 @@ func TestTelemetryConfig_MetricCardinalityLimit(t *testing.T) {
assert.Equal(t, 100000, tc.MetricCardinalityLimit())
})
}

func TestTelemetryConfig_ChipIngressRetry(t *testing.T) {
t.Parallel()

t.Run("Enabled", func(t *testing.T) {
t.Parallel()
tests := []struct {
name string
telemetry toml.Telemetry
expected bool
}{
{"Set", toml.Telemetry{ChipIngressRetryEnabled: new(true)}, true},
{"NilDisables", toml.Telemetry{}, false},
{"ExplicitFalse", toml.Telemetry{ChipIngressRetryEnabled: new(false)}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
tc := telemetryConfig{s: tt.telemetry}
assert.Equal(t, tt.expected, tc.ChipIngressRetryEnabled())
})
}
})

t.Run("MaxAttempts", func(t *testing.T) {
t.Parallel()
tests := []struct {
name string
telemetry toml.Telemetry
expected int
}{
{"Set", toml.Telemetry{ChipIngressRetryMaxAttempts: new(5)}, 5},
{"NilDefaults", toml.Telemetry{}, defaultChipIngressRetryMaxAttempts},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
tc := telemetryConfig{s: tt.telemetry}
assert.Equal(t, tt.expected, tc.ChipIngressRetryMaxAttempts())
})
}
})

t.Run("InitialBackoff", func(t *testing.T) {
t.Parallel()
tests := []struct {
name string
telemetry toml.Telemetry
expected time.Duration
}{
{"Set", toml.Telemetry{ChipIngressRetryInitialBackoff: commonconfig.MustNewDuration(250 * time.Millisecond)}, 250 * time.Millisecond},
{"NilDefaults", toml.Telemetry{}, defaultChipIngressRetryInitialBackoff},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
tc := telemetryConfig{s: tt.telemetry}
assert.Equal(t, tt.expected, tc.ChipIngressRetryInitialBackoff())
})
}
})

t.Run("MaxBackoff", func(t *testing.T) {
t.Parallel()
tests := []struct {
name string
telemetry toml.Telemetry
expected time.Duration
}{
{"Set", toml.Telemetry{ChipIngressRetryMaxBackoff: commonconfig.MustNewDuration(5 * time.Second)}, 5 * time.Second},
{"NilDefaults", toml.Telemetry{}, defaultChipIngressRetryMaxBackoff},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
tc := telemetryConfig{s: tt.telemetry}
assert.Equal(t, tt.expected, tc.ChipIngressRetryMaxBackoff())
})
}
})

t.Run("BackoffMultiplier", func(t *testing.T) {
t.Parallel()
tests := []struct {
name string
telemetry toml.Telemetry
expected float64
}{
{"Set", toml.Telemetry{ChipIngressRetryBackoffMultiplier: new(1.5)}, 1.5},
{"NilDefaults", toml.Telemetry{}, defaultChipIngressRetryBackoffMultiplier},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
tc := telemetryConfig{s: tt.telemetry}
assert.InDelta(t, tt.expected, tc.ChipIngressRetryBackoffMultiplier(), 0.0001)
})
}
})

t.Run("RetryableStatusCodes", func(t *testing.T) {
t.Parallel()
tests := []struct {
name string
telemetry toml.Telemetry
expected []string
}{
{"Set", toml.Telemetry{ChipIngressRetryableStatusCodes: []string{"Unavailable"}}, []string{"Unavailable"}},
{"NilDefaults", toml.Telemetry{}, defaultChipIngressRetryableStatusCodes},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
tc := telemetryConfig{s: tt.telemetry}
assert.Equal(t, tt.expected, tc.ChipIngressRetryableStatusCodes())
})
}
})

t.Run("DefaultsMirrorCommonRecommendedPolicy", func(t *testing.T) {
t.Parallel()
// The getters' defaults must resolve to the same policy as
// chainlink-common's chipingress.DefaultRetryPolicy so that enabling
// retries without overrides yields the recommended policy.
tc := telemetryConfig{s: toml.Telemetry{}}
assert.False(t, tc.ChipIngressRetryEnabled(), "retries must stay off unless explicitly enabled")

expected := chipingress.DefaultRetryPolicy()
assert.Equal(t, expected.MaxAttempts, tc.ChipIngressRetryMaxAttempts())
assert.Equal(t, expected.InitialBackoff, tc.ChipIngressRetryInitialBackoff())
assert.Equal(t, expected.MaxBackoff, tc.ChipIngressRetryMaxBackoff())
assert.InDelta(t, expected.BackoffMultiplier, tc.ChipIngressRetryBackoffMultiplier(), 0.0001)

codes, err := chipingress.ParseStatusCodes(tc.ChipIngressRetryableStatusCodes())
require.NoError(t, err)
assert.Equal(t, expected.RetryableStatusCodes, codes)
})
}
6 changes: 6 additions & 0 deletions core/services/chainlink/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -569,6 +569,12 @@ func TestConfig_Marshal(t *testing.T) {
ChipIngressSendTimeout: commonconfig.MustNewDuration(10 * time.Second),
ChipIngressDrainTimeout: commonconfig.MustNewDuration(30 * time.Second),
ChipIngressMaxGRPCRequestSize: new(10485760),
ChipIngressRetryEnabled: new(false),
ChipIngressRetryMaxAttempts: new(3),
ChipIngressRetryInitialBackoff: commonconfig.MustNewDuration(100 * time.Millisecond),
ChipIngressRetryMaxBackoff: commonconfig.MustNewDuration(1 * time.Second),
ChipIngressRetryBackoffMultiplier: new(2.0),
ChipIngressRetryableStatusCodes: []string{"Unavailable", "ResourceExhausted"},
DurableEmitterEnabled: new(false),
DurableEmitterRetransmitBatchSize: new(500),
DurableEmitterEventTTL: commonconfig.MustNewDuration(6 * time.Hour),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,12 @@ ChipIngressSendInterval = '500ms'
ChipIngressSendTimeout = '10s'
ChipIngressDrainTimeout = '30s'
ChipIngressMaxGRPCRequestSize = 10485760
ChipIngressRetryEnabled = false
ChipIngressRetryMaxAttempts = 3
ChipIngressRetryInitialBackoff = '100ms'
ChipIngressRetryMaxBackoff = '1s'
ChipIngressRetryBackoffMultiplier = 2.0
ChipIngressRetryableStatusCodes = ['Unavailable', 'ResourceExhausted']
DurableEmitterEnabled = true
DurableEmitterRetransmitBatchSize = 500
DurableEmitterEventTTL = '6h0m0s'
Expand Down
Loading
Loading