fix(db): PoolSuite no longer races the driver's refusal (#320)

bin/gate

@@ -50,10 +50,9 @@
 # step the gate names any process listening on 8080 and carries on; a red ClientSuite under that
 # line is the squatter, not the code.
 #
-# Two suites are timing sensitive and can go red under machine load with no code cause.
-# PoolSuite's "with the database down, the first read waits the acquire timeout" fires its 500 ms
-# acquire timeout before the driver's refusal is recorded. LiveServerSuite's abnormal drop rejoin
-# can time out on a slow runner. A lone red in one of those: rerun once before diagnosing.
+# One suite is timing sensitive and can go red under machine load with no code cause:
+# LiveServerSuite's abnormal drop rejoin can time out on a slow runner. A lone red there: rerun
+# once before diagnosing.
 #
 # `bin/gate --dry-run` prints the steps and the lock paths without running sbt, and changes
 # nothing, not even the git config.

examples/build.sbt

@@ -92,7 +92,7 @@ lazy val blog = (project in file("blog"))
       // Postgres: what it is asserting about is the SQL `JdbcStore.owned` narrows with, and an
       // in-memory substitute is the half `modules/http` already tests. Test scope, so nothing the
       // blog runs with carries it.
-      "org.testcontainers" % "postgresql" % "1.21.3" % Test
+      Dependencies.testcontainersPg
     ),
     Compile / run / mainClass := Some("Main"),
     runInProjectDir
@@ -121,7 +121,7 @@ lazy val shop = (project in file("shop"))
       layer("eezo-upickle"),
       "com.lihaoyi" %% "upickle" % "4.4.3",
       "org.scalameta" %% "munit" % "1.3.4" % Test,
-      "org.testcontainers" % "postgresql" % "1.21.3" % Test
+      Dependencies.testcontainersPg
     ),
     // `Stripe` reads its two secrets when the route table is built, and a suite builds the table,
     // so the suite runs in a JVM of its own that can be handed stand ins. Nothing in it calls

examples/project/build.sbt

@@ -3,3 +3,9 @@
 // come from one level further up. This line compiles that same file into the meta build as well,
 // so `build.sbt` reads the version from the one definition rather than from a second copy of it.
 Compile / unmanagedSources += baseDirectory.value / "project" / "EezoVersion.scala"
+
+// The framework's `Dependencies`, compiled into this meta build as well, so an example whose suite
+// needs what a module's suite needs, a real Postgres through testcontainers, takes the same version
+// instead of a second literal that falls behind: a Docker Engine upgrade once broke the examples
+// alone for that reason.
+Compile / unmanagedSources += baseDirectory.value.getParentFile.getParentFile / "project" / "Dependencies.scala"

modules/db/src/test/scala/io/eezo/db/PoolSuite.scala

@@ -9,6 +9,7 @@ import java.sql.{Connection, SQLException}
 import java.time.Duration
 import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit}
 import java.util.concurrent.atomic.AtomicReference
+import scala.annotation.tailrec
 
 /** The pool against a real Postgres: what it reuses, when it opens, and how it fails.
   *
@@ -168,19 +169,46 @@ class PoolSuite extends PgSuite {
   test(
     "with the database down, the first read waits the acquire timeout and fails in eezo's words"
   ) {
+    // Nothing here says whether the driver's refusal has been recorded when the timeout fires: the
+    // two are independent timers, and which comes first depends on how busy the machine is. The
+    // driver's word is asserted below, once it has had time to arrive.
     val timeout = Duration.ofMillis(500)
     val pool    = new Pool(Refusing.jdbcUrl(), "nobody", "hunter2", _ => (), size = 1, timeout)
     installing(new Database(pool)) {
       val start  = System.nanoTime()
       val e      = intercept[ConnectionUnavailable](read { backend() })
       val waited = millisSince(start)
       assert(waited >= timeout.toMillis - 50, s"the borrow gave up after $waited ms")
-      assert(e.getMessage.contains("refused"), s"the driver's failure is missing: ${e.getMessage}")
+      assert(
+        e.getMessage.contains("no database connection within 500 ms from the pool of size 1"),
+        e.getMessage
+      )
       assert(!e.getMessage.contains("hunter2"), e.getMessage)
       assert(!e.getMessage.contains("detached"), s"no detached block is involved: ${e.getMessage}")
     }
   }
 
+  /** Borrows until the pool has the driver's failure on record, or the deadline passes, and answers
+    * the last refusal either way. HikariCP records a failed connect on its own thread, so a borrow
+    * that gives up first reports that none is recorded yet; the next one finds it.
+    */
+  @tailrec
+  private def unavailableOnceDriverFails(deadline: Long): ConnectionUnavailable = {
+    val e = intercept[ConnectionUnavailable](read { backend() })
+    if (e.getMessage.contains("refused") || System.nanoTime() >= deadline) e
+    else unavailableOnceDriverFails(deadline)
+  }
+
+  test("with the database down, a read after the driver has failed says what the driver said") {
+    val pool =
+      new Pool(Refusing.jdbcUrl(), "nobody", "hunter2", _ => (), size = 1, Duration.ofMillis(500))
+    installing(new Database(pool)) {
+      val e = unavailableOnceDriverFails(System.nanoTime() + Duration.ofSeconds(10).toNanos)
+      assert(e.getMessage.contains("refused"), s"the driver's failure is missing: ${e.getMessage}")
+      assert(!e.getMessage.contains("hunter2"), e.getMessage)
+    }
+  }
+
   test("a detached block that needs a second connection from a full pool fails, and says why") {
     // A pool of one, and the read around the detached block holds it: the second borrow can only
     // wait on the thread that is waiting, so the timeout is all that stands between this and a hang.

project/Dependencies.scala

@@ -10,7 +10,7 @@ object Dependencies {
   object V {
     val munit          = "1.3.4"
     val postgresql     = "42.7.1"
-    val testcontainers = "1.21.3"
+    val testcontainers = "1.21.4"
 
     /** spring-security-crypto. Picked because it is the one bcrypt implementation on the JVM that
       * ships as a jar with no transitive dependency of its own, so `eezo-auth` costs an application