diff --git a/CONFIG.md b/CONFIG.md index 902b52d64e..68ae48b2bc 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -957,6 +957,7 @@ GasEstimator.PriceMax overrides the maximum gas price for this key. See EVM.GasE [NodePool] PollFailureThreshold = 5 # Default PollSuccessThreshold = 0 # Default +RedialBackoffMin = '1s' # Default PollInterval = '10s' # Default SelectionMode = 'HighestHead' # Default SyncThreshold = 5 # Default @@ -991,6 +992,15 @@ PollSuccessThreshold indicates how many consecutive polls must succeed in order Set to zero to require no successful polls (previous behavior). +### RedialBackoffMin +```toml +RedialBackoffMin = '1s' # Default +``` +RedialBackoffMin is how long to wait before the first redial of a node that has been marked as unreachable. +Later redials back off exponentially, with jitter, up to 15s. Values of 15s or more always wait 15s. + +Set to zero to redial immediately. Later redials then back off starting from 100ms. + ### PollInterval ```toml PollInterval = '10s' # Default diff --git a/pkg/client/config_builder.go b/pkg/client/config_builder.go index 05bc216830..300bd30399 100644 --- a/pkg/client/config_builder.go +++ b/pkg/client/config_builder.go @@ -33,6 +33,7 @@ func NewClientConfigs( nodeCfgs []NodeConfig, pollFailureThreshold *uint32, pollSuccessThreshold *uint32, + redialBackoffMin time.Duration, pollInterval time.Duration, syncThreshold *uint32, nodeIsSyncingEnabled *bool, @@ -58,6 +59,7 @@ func NewClientConfigs( LeaseDuration: commonconfig.MustNewDuration(leaseDuration), PollFailureThreshold: pollFailureThreshold, PollSuccessThreshold: pollSuccessThreshold, + RedialBackoffMin: commonconfig.MustNewDuration(redialBackoffMin), PollInterval: commonconfig.MustNewDuration(pollInterval), SyncThreshold: syncThreshold, NodeIsSyncingEnabled: nodeIsSyncingEnabled, diff --git a/pkg/client/config_builder_test.go b/pkg/client/config_builder_test.go index 9024effa99..e7d95cc78c 100644 --- a/pkg/client/config_builder_test.go +++ b/pkg/client/config_builder_test.go @@ -20,6 +20,7 @@ func TestClientConfigBuilder(t *testing.T) { leaseDuration := 0 * time.Second pollFailureThreshold := new(uint32(5)) pollSuccessThreshold := new(uint32(3)) + redialBackoffMin := 2 * time.Second pollInterval := 10 * time.Second syncThreshold := new(uint32(5)) nodeIsSyncingEnabled := new(false) @@ -43,7 +44,7 @@ func TestClientConfigBuilder(t *testing.T) { noNewHeadsThreshold := time.Second newHeadsPollInterval := 0 * time.Second chainCfg, nodePool, nodes, err := client.NewClientConfigs(selectionMode, leaseDuration, chainTypeStr, nodeConfigs, - pollFailureThreshold, pollSuccessThreshold, pollInterval, syncThreshold, nodeIsSyncingEnabled, noNewHeadsThreshold, finalityDepth, + pollFailureThreshold, pollSuccessThreshold, redialBackoffMin, pollInterval, syncThreshold, nodeIsSyncingEnabled, noNewHeadsThreshold, finalityDepth, finalityTagEnabled, SafeTagSupported, finalizedBlockOffset, enforceRepeatableRead, deathDeclarationDelay, noNewFinalizedBlocksThreshold, pollInterval, newHeadsPollInterval, confirmationTimeout, safeDepth) require.NoError(t, err) @@ -53,6 +54,7 @@ func TestClientConfigBuilder(t *testing.T) { require.Equal(t, leaseDuration, nodePool.LeaseDuration()) require.Equal(t, *pollFailureThreshold, nodePool.PollFailureThreshold()) require.Equal(t, *pollSuccessThreshold, nodePool.PollSuccessThreshold()) + require.Equal(t, redialBackoffMin, nodePool.RedialBackoffMin()) require.Equal(t, pollInterval, nodePool.PollInterval()) require.Equal(t, *syncThreshold, nodePool.SyncThreshold()) require.Equal(t, *nodeIsSyncingEnabled, nodePool.NodeIsSyncingEnabled()) diff --git a/pkg/client/evm_client_test.go b/pkg/client/evm_client_test.go index 12b88f60fc..47c4c5b878 100644 --- a/pkg/client/evm_client_test.go +++ b/pkg/client/evm_client_test.go @@ -30,6 +30,7 @@ func TestNewEvmClient(t *testing.T) { leaseDuration := 0 * time.Second pollFailureThreshold := new(uint32(5)) pollSuccessThreshold := new(uint32(0)) + redialBackoffMin := time.Second pollInterval := 10 * time.Second syncThreshold := new(uint32(5)) nodeIsSyncingEnabled := new(false) @@ -53,7 +54,7 @@ func TestNewEvmClient(t *testing.T) { finalityTagEnabled := new(true) SafeTagSupported := new(true) chainCfg, nodePool, nodes, err := client.NewClientConfigs(selectionMode, leaseDuration, chainTypeStr, nodeConfigs, - pollFailureThreshold, pollSuccessThreshold, pollInterval, syncThreshold, nodeIsSyncingEnabled, noNewHeadsThreshold, finalityDepth, + pollFailureThreshold, pollSuccessThreshold, redialBackoffMin, pollInterval, syncThreshold, nodeIsSyncingEnabled, noNewHeadsThreshold, finalityDepth, finalityTagEnabled, SafeTagSupported, finalizedBlockOffset, enforceRepeatableRead, deathDeclarationDelay, noNewFinalizedBlocksThreshold, finalizedBlockPollInterval, newHeadsPollInterval, confirmationTimeout, safeDepth) require.NoError(t, err) @@ -80,7 +81,7 @@ func TestChainClientMetrics(t *testing.T) { }, } chainCfg, nodePool, nodes, err := client.NewClientConfigs(new("HighestHead"), time.Duration(0), "", nodeConfigs, - new(uint32(5)), new(uint32(0)), 10*time.Second, new(uint32(5)), new(false), time.Minute, new(uint32(5)), new(false), new(false), + new(uint32(5)), new(uint32(0)), time.Second, 10*time.Second, new(uint32(5)), new(false), time.Minute, new(uint32(5)), new(false), new(false), new(uint32(5)), new(false), 10*time.Second, 10*time.Second, 10*time.Second, 10*time.Second, 60*time.Second, new(uint32(10))) require.NoError(t, err) diff --git a/pkg/client/helpers_test.go b/pkg/client/helpers_test.go index 8e90539c1c..d9867cb52c 100644 --- a/pkg/client/helpers_test.go +++ b/pkg/client/helpers_test.go @@ -92,6 +92,7 @@ func (c *TestClientErrors) FinalizedStateUnavailable() string { return c.finaliz type TestNodePoolConfig struct { NodePollFailureThreshold uint32 NodePollSuccessThreshold uint32 + NodeRedialBackoffMin time.Duration NodePollInterval time.Duration NodeSelectionMode string NodeSyncThreshold uint32 @@ -107,11 +108,12 @@ type TestNodePoolConfig struct { FinalizedStateCheckFailureThresholdVal uint32 } -func (tc TestNodePoolConfig) PollFailureThreshold() uint32 { return tc.NodePollFailureThreshold } -func (tc TestNodePoolConfig) PollSuccessThreshold() uint32 { return tc.NodePollSuccessThreshold } -func (tc TestNodePoolConfig) PollInterval() time.Duration { return tc.NodePollInterval } -func (tc TestNodePoolConfig) SelectionMode() string { return tc.NodeSelectionMode } -func (tc TestNodePoolConfig) SyncThreshold() uint32 { return tc.NodeSyncThreshold } +func (tc TestNodePoolConfig) PollFailureThreshold() uint32 { return tc.NodePollFailureThreshold } +func (tc TestNodePoolConfig) PollSuccessThreshold() uint32 { return tc.NodePollSuccessThreshold } +func (tc TestNodePoolConfig) RedialBackoffMin() time.Duration { return tc.NodeRedialBackoffMin } +func (tc TestNodePoolConfig) PollInterval() time.Duration { return tc.NodePollInterval } +func (tc TestNodePoolConfig) SelectionMode() string { return tc.NodeSelectionMode } +func (tc TestNodePoolConfig) SyncThreshold() uint32 { return tc.NodeSyncThreshold } func (tc TestNodePoolConfig) LeaseDuration() time.Duration { return tc.NodeLeaseDuration } diff --git a/pkg/config/chain_scoped_node_pool.go b/pkg/config/chain_scoped_node_pool.go index 88679e7de6..fd61de03c6 100644 --- a/pkg/config/chain_scoped_node_pool.go +++ b/pkg/config/chain_scoped_node_pool.go @@ -19,6 +19,10 @@ func (n *NodePoolConfig) PollSuccessThreshold() uint32 { return *n.C.PollSuccessThreshold } +func (n *NodePoolConfig) RedialBackoffMin() time.Duration { + return n.C.RedialBackoffMin.Duration() +} + func (n *NodePoolConfig) PollInterval() time.Duration { return n.C.PollInterval.Duration() } diff --git a/pkg/config/config.go b/pkg/config/config.go index 2612f41675..3921aa749e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -202,6 +202,7 @@ type Workflow interface { type NodePool interface { PollFailureThreshold() uint32 PollSuccessThreshold() uint32 + RedialBackoffMin() time.Duration PollInterval() time.Duration SelectionMode() string SyncThreshold() uint32 diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 548d0a68e5..f18911fdca 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -363,6 +363,7 @@ func TestNodePoolConfig(t *testing.T) { require.Equal(t, time.Duration(10000000000), cfg.EVM().NodePool().PollInterval()) require.Equal(t, uint32(5), cfg.EVM().NodePool().PollFailureThreshold()) require.Equal(t, uint32(0), cfg.EVM().NodePool().PollSuccessThreshold()) + require.Equal(t, time.Second, cfg.EVM().NodePool().RedialBackoffMin()) require.False(t, cfg.EVM().NodePool().NodeIsSyncingEnabled()) require.True(t, cfg.EVM().NodePool().EnforceRepeatableRead()) require.Equal(t, time.Minute, cfg.EVM().NodePool().DeathDeclarationDelay()) diff --git a/pkg/config/toml/config.go b/pkg/config/toml/config.go index d7c620cbea..2f0f7e2866 100644 --- a/pkg/config/toml/config.go +++ b/pkg/config/toml/config.go @@ -1105,6 +1105,7 @@ func (r *ClientErrors) setFrom(f *ClientErrors) bool { type NodePool struct { PollFailureThreshold *uint32 PollSuccessThreshold *uint32 + RedialBackoffMin *commonconfig.Duration PollInterval *commonconfig.Duration SelectionMode *string SyncThreshold *uint32 @@ -1128,6 +1129,9 @@ func (p *NodePool) setFrom(f *NodePool) { if v := f.PollSuccessThreshold; v != nil { p.PollSuccessThreshold = v } + if v := f.RedialBackoffMin; v != nil { + p.RedialBackoffMin = v + } if v := f.PollInterval; v != nil { p.PollInterval = v } diff --git a/pkg/config/toml/config_test.go b/pkg/config/toml/config_test.go index fef3f67211..c3cb4134bf 100644 --- a/pkg/config/toml/config_test.go +++ b/pkg/config/toml/config_test.go @@ -372,6 +372,7 @@ var fullConfig = EVMConfig{ NodePool: NodePool{ PollFailureThreshold: new(uint32(5)), PollSuccessThreshold: new(uint32(0)), + RedialBackoffMin: config.MustNewDuration(time.Second), PollInterval: config.MustNewDuration(time.Minute), SelectionMode: new(multinode.NodeSelectionModeHighestHead), SyncThreshold: new(uint32(13)), diff --git a/pkg/config/toml/defaults/fallback.toml b/pkg/config/toml/defaults/fallback.toml index fb41327a12..cd74768282 100644 --- a/pkg/config/toml/defaults/fallback.toml +++ b/pkg/config/toml/defaults/fallback.toml @@ -81,6 +81,7 @@ PersistenceBatchSize = 100 [NodePool] PollFailureThreshold = 5 PollSuccessThreshold = 0 +RedialBackoffMin = '1s' PollInterval = '10s' SelectionMode = 'HighestHead' SyncThreshold = 5 diff --git a/pkg/config/toml/docs.toml b/pkg/config/toml/docs.toml index 624ecbd603..c07c3e5d6a 100644 --- a/pkg/config/toml/docs.toml +++ b/pkg/config/toml/docs.toml @@ -435,6 +435,11 @@ PollFailureThreshold = 5 # Default # # Set to zero to require no successful polls (previous behavior). PollSuccessThreshold = 0 # Default +# RedialBackoffMin is how long to wait before the first redial of a node that has been marked as unreachable. +# Later redials back off exponentially, with jitter, up to 15s. Values of 15s or more always wait 15s. +# +# Set to zero to redial immediately. Later redials then back off starting from 100ms. +RedialBackoffMin = '1s' # Default # PollInterval controls how often to poll the node to check for liveness. # # Set to zero to disable poll checking. diff --git a/pkg/config/toml/testdata/config-full.toml b/pkg/config/toml/testdata/config-full.toml index f2599a6cd1..392f3895a3 100644 --- a/pkg/config/toml/testdata/config-full.toml +++ b/pkg/config/toml/testdata/config-full.toml @@ -120,6 +120,7 @@ PriceMax = '79.228162514264337593543950335 gether' [NodePool] PollFailureThreshold = 5 PollSuccessThreshold = 0 +RedialBackoffMin = '1s' PollInterval = '1m0s' SelectionMode = 'HighestHead' SyncThreshold = 13