Failsafe on ActiveRecord::ConnectionFailed

lib/solid_cache/store/failsafe.rb

@@ -5,6 +5,7 @@ class Store
     module Failsafe
       TRANSIENT_ACTIVE_RECORD_ERRORS = [
         ActiveRecord::AdapterTimeout,
+        ActiveRecord::ConnectionFailed,
         ActiveRecord::ConnectionNotEstablished,
         ActiveRecord::Deadlocked,
         ActiveRecord::LockWaitTimeout,

test/test_helper.rb

@@ -95,12 +95,20 @@ def shard_keys(cache, shard)
     shard_keys.map { |key| key.delete_prefix("#{@namespace}:") }
   end
 
-  def emulating_timeouts
+  def emulating_errors(error_class)
     ar_methods = [ :select_all, :delete, :exec_insert_all ]
     stub_matcher = ActiveRecord::Base.connection.class.any_instance
-    ar_methods.each { |method| stub_matcher.stubs(method).raises(ActiveRecord::StatementTimeout) }
+    ar_methods.each { |method| stub_matcher.stubs(method).raises(error_class) }
     yield
   ensure
     ar_methods.each { |method| stub_matcher.unstub(method) }
   end
+
+  def emulating_timeouts(&block)
+    emulating_errors(ActiveRecord::StatementTimeout, &block)
+  end
+
+  def emulating_connection_failures(&block)
+    emulating_errors(ActiveRecord::ConnectionFailed, &block)
+  end
 end

test/unit/failsafe_test.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class SolidCacheFailsafeErrorsTest < ActiveSupport::TestCase
+  setup do
+    @namespace = "test-#{SecureRandom.hex}"
+  end
+
+  # ActiveRecord::ConnectionFailed is raised when the server drops the connection
+  # mid-query (e.g. Postgres idle_in_transaction_session_timeout, MySQL server
+  # gone away). It is a StatementInvalid, not a ConnectionNotEstablished, so it
+  # must be listed explicitly to degrade to a cache miss like other transient
+  # connection failures. SolidCacheConnectionFailedFailsafeTest covers the
+  # degrading of every cache operation; these cover the error handler contract.
+  def test_connection_failed_is_passed_to_the_error_handler
+    handled = []
+    handler = ->(method:, returning:, exception:) { handled << [ returning, exception ] }
+
+    emulating_errors(ActiveRecord::ConnectionFailed) do
+      lookup_store(namespace: @namespace, error_handler: handler).read("key")
+    end
+
+    returning, exception = handled.first
+    assert_nil returning
+    assert_instance_of ActiveRecord::ConnectionFailed, exception
+  end
+
+  def test_connection_failed_is_raised_when_the_error_handler_raises
+    cache = lookup_store(namespace: @namespace, error_handler: ->(method:, returning:, exception:) { raise exception })
+
+    assert_raise ActiveRecord::ConnectionFailed do
+      emulating_errors(ActiveRecord::ConnectionFailed) { cache.read("key") }
+    end
+  end
+
+  # The failsafe only rescues transient errors so that misconfiguration and
+  # bugs are not silently turned into cache misses (see #182).
+  def test_non_transient_errors_are_not_swallowed
+    [ ActiveRecord::StatementInvalid, ActiveRecord::NoDatabaseError ].each do |error|
+      assert_raise error, "#{error} should propagate" do
+        emulating_errors(error) { lookup_store(namespace: @namespace).read("key") }
+      end
+    end
+  end
+end

test/unit/solid_cache_test.rb

@@ -89,6 +89,32 @@ def emulating_unavailability
   end
 end
 
+class SolidCacheConnectionFailedFailsafeTest < ActiveSupport::TestCase
+  include FailureSafetyBehavior
+
+  setup do
+    @cache = nil
+    @namespace = "test-#{SecureRandom.hex}"
+
+    @cache = lookup_store(expires_in: 60)
+    # @cache.logger = Logger.new($stdout)  # For test debugging
+
+    # For LocalCacheBehavior tests
+    @peek = lookup_store(expires_in: 60)
+  end
+
+  # A terminated connection raises ActiveRecord::ConnectionFailed, which is a
+  # subclass of ActiveRecord::StatementInvalid rather than of
+  # ActiveRecord::ConnectionNotEstablished, so it needs its own failsafe entry.
+  # Regression test for https://github.com/rails/solid_cache/issues/307.
+  def emulating_unavailability
+    wait_for_background_tasks(@cache)
+    emulating_connection_failures do
+      yield lookup_store(namespace: @namespace)
+    end
+  end
+end
+
 class SolidCacheRaisingTest < ActiveSupport::TestCase
   include FailureRaisingBehavior