Redis hardening: retry strategy (#18565)

* Add an error handler to ensure the Redis server connection is forcibly closed
* Explain unknown errors are usually server idle timeouts
* Add an automatic retry_strategy
* Explicitly set connect_timeout to the default value of 1 hour

Author: JamesMGreene

lib/redis/create-client.js

@@ -6,10 +6,13 @@ const { REDIS_MIN_DB, REDIS_MAX_DB } = process.env
 const redisMinDb = REDIS_MIN_DB || 0
 const redisMaxDb = REDIS_MAX_DB || 15
 
+// Maximum delay between reconnection attempts after backoff
+const maxReconnectDelay = 5000
+
 function formatRedisError (error) {
   const errorCode = error ? error.code : null
-  const errorName = error ? error.constructor.name : 'Error'
-  const errorMsg = error ? error.toString() : 'unknown error'
+  const errorName = error ? error.constructor.name : 'Server disconnection'
+  const errorMsg = error ? error.toString() : 'unknown (commonly a server idle timeout)'
   const preamble = errorName + (errorCode ? ` with code "${errorCode}"` : '')
   return preamble + ': ' + errorMsg
 }
@@ -51,6 +54,35 @@ module.exports = function createClient (options = {}) {
     // This is also critical to preventing a backend pile-up!
     enable_offline_queue: false,
 
+    // This timeout value will be applied to both the initial connection
+    // and any auto-reconnect attempts (if the `retry_strategy` option is
+    // provided). If not using the `retry_strategy` option, this value can be
+    // set to a very low number. If using the `retry_strategy` option to allow
+    // more than one reconnection attempt, this value must be set to a higher
+    // number. Defaults to 1 hour if not configured!
+    connect_timeout: 60 * 60 * 1000, // 60 minutes
+
+    // Be aware that this retry (NOT just reconnection) strategy appears to
+    // be a major point of confusion (and possibly legitimate issues) between
+    // reconnecting and retrying failed commands.
+    retry_strategy:
+      function ({
+        attempt,
+        error,
+        total_retry_time: totalRetryTime,
+        times_connected: timesConnected
+      }) {
+        let delayPerAttempt = 100
+
+        // If the server appears to be unavailable, slow down faster
+        if (error && error.code === 'ECONNREFUSED') {
+          delayPerAttempt *= 5
+        }
+
+        // Reconnect after delay
+        return Math.min(attempt * delayPerAttempt, maxReconnectDelay)
+      },
+
     // Expand whatever other options and overrides were provided
     ...options
   })
@@ -72,6 +104,7 @@ module.exports = function createClient (options = {}) {
   // Add event listeners for basic logging
   client.on('connect', () => { console.log(logPrefix, 'Connection opened') })
   client.on('ready', () => { console.log(logPrefix, 'Ready to receive commands') })
+  client.on('end', () => { console.log(logPrefix, 'Connection closed') })
   client.on(
     'reconnecting',
     ({
@@ -93,7 +126,6 @@ module.exports = function createClient (options = {}) {
       )
     }
   )
-  client.on('end', () => { console.log(logPrefix, 'Connection closed') })
   client.on('warning', (msg) => { console.warn(logPrefix, 'Warning:', msg) })
   client.on('error', (error) => { console.error(logPrefix, formatRedisError(error)) })