fix: slice 040 AC3 measures the watcher, not the machine

.gitignore

@@ -5,6 +5,10 @@
 
 # Build and dependency output
 /_build/
+# A second build tree for the Postgres adapter, which is chosen at compile time by TRINITY_DB and
+# so cannot share _build with the SQLite one. `MIX_BUILD_ROOT=_build_pg TRINITY_DB=postgres` is how
+# the postgres leg is reproduced locally.
+/_build_pg/
 /deps/
 /cover/
 /doc/

config/test.exs

@@ -174,6 +174,17 @@ else
   # a write while another connection holds the write lock, because waiting there could deadlock.
   # No timeout affects that path. The raise is kept because it does help ordinary lock waiting,
   # but it is not the fix for what is recorded as R26 in docs/06-risk-register.md.
+  #
+  # And the line below does not reach a sandboxed test at all, which is worth knowing before
+  # trusting it. `Ecto.Adapters.SQL.Sandbox.post_checkout/3` begins each test's transaction with
+  # `handle_begin([mode: :transaction] ++ opts, ...)`, and `Exqlite.Connection` resolves the mode
+  # as `Keyword.get(options, :mode, state.default_transaction_mode)`, so the explicit option wins
+  # over this setting and `:transaction` is a plain DEFERRED `BEGIN`. Every sandboxed test is
+  # therefore a DEFERRED transaction, and one that reads before it writes is on R26's unguarded
+  # upgrade path with no timeout able to help. `default_transaction_mode` below still governs the
+  # transactions the application itself opens, which is why it stays.
+  # `test/trinity/repo/sandbox_transaction_mode_test.exs` asserts both dependency lines so this
+  # paragraph fails rather than rots.
   config :trinity, Trinity.Repo,
     database: Path.expand("../trinity_test.db", __DIR__),
     pool: Ecto.Adapters.SQL.Sandbox,
@@ -260,6 +271,18 @@ config :trinity, :skills,
   # user_dir at a temporary directory first.
   pending_dir: Path.join(System.tmp_dir!(), "trinity-test-pending-skills"),
   watch: true,
+  # The shipped poll is every second (lib/trinity/skills/registry.ex) and it compares mtimes at
+  # one-second resolution, so its worst case is about two seconds, which is the whole of slice
+  # 040 AC3's bound. With no margin, the test asserting that bound failed intermittently on a
+  # loaded runner while the watcher was behaving exactly as designed. A shorter interval here
+  # leaves the mechanism under test, the fs_poll backend over mtimes, and takes the runner's
+  # scheduler out of the measurement. The shipped default is unchanged.
+  # One backend everywhere, so slice 040 AC3 measures the same mechanism on every machine. The
+  # gate's runners have no `inotifywait` and already take this path; a developer box takes the
+  # native one, and takes it badly when the machine is at its inotify instance ceiling, where the
+  # watch is never established and no edit ever arrives.
+  poll: true,
+  poll_interval_ms: 100,
   index_tokens: 338
 
 # Slice 060: no MCP client boots from rows in the suite (each test starts the clients it

lib/trinity/skills/registry.ex

@@ -304,13 +304,30 @@ defmodule Trinity.Skills.Registry do
   # has no native backend for, poll once a second (AC3's 2 s still holds; the poll compares
   # mtimes at one-second resolution). `TRINITY_SKILLS_POLL=1` forces the poll, so the
   # fallback can be exercised on a machine that has inotifywait.
+  #
+  # The interval is configurable because AC3's bound and this default leave no margin. A poll
+  # every second over mtimes at one-second resolution has a worst case of about two seconds all
+  # by itself, which is the whole of the criterion, so on a loaded shared runner the test that
+  # asserts it failed intermittently while the product was behaving exactly as designed. The
+  # default is unchanged and is what ships; the suite sets a shorter one so that what it measures
+  # is the watcher noticing rather than the runner's scheduler.
   defp backend do
-    poll = [backend: :fs_poll, interval: 1_000]
+    poll = [backend: :fs_poll, interval: poll_interval_ms()]
 
     cond do
       System.get_env("TRINITY_SKILLS_POLL") == "1" ->
         poll
 
+      # Same switch, from configuration rather than the environment, so the suite can pin one
+      # backend. Which backend the registry picks otherwise depends on the machine: a developer
+      # box has `inotifywait` and the gate's runners do not, so AC3's test measured a different
+      # mechanism in each place, and on a box already at its inotify **instance** ceiling
+      # (`/proc/sys/fs/inotify/max_user_instances`, 128 here with 119 in use) the native backend
+      # obtains no watch and fails silently, which the test could only report as an edit that
+      # never arrived.
+      Keyword.get(Application.get_env(:trinity, :skills, []), :poll, false) ->
+        poll
+
       match?({:unix, :linux}, :os.type()) ->
         if System.find_executable("inotifywait"), do: [], else: poll
 
@@ -327,6 +344,9 @@ defmodule Trinity.Skills.Registry do
 
   defp watch?, do: Application.get_env(:trinity, :skills, []) |> Keyword.get(:watch, true)
 
+  defp poll_interval_ms,
+    do: Application.get_env(:trinity, :skills, []) |> Keyword.get(:poll_interval_ms, 1_000)
+
   ## Helpers
 
   defp ensure_project(nil), do: :ok

test/support/boot_isolation.ex

@@ -0,0 +1,120 @@
+# SPDX-FileCopyrightText: Sudo Apt Holdings LLC
+# SPDX-License-Identifier: Apache-2.0
+defmodule Trinity.BootIsolation do
+  @moduledoc """
+  Gives a spawned boot its own databases, whichever adapter is compiled in.
+
+  `test/trinity/regulated_boot_node_test.exs` boots Trinity in a separate OS process, because
+  `start/2` cannot be re-run in a VM where the application is already up. Such a child needs
+  databases that are **not** the suite's own: it writes a boot receipt, and six children appending
+  to the chain the suite is using is not isolation.
+
+  Two things make that awkward, and this module exists to hold both in one place.
+
+  **The database is named differently per adapter.** Under SQLite it is a file path; under
+  `TRINITY_DB=postgres` it is a name inside a URL. `config/test.exs` fixes either one and derives
+  neither from `XDG_DATA_HOME`, so pointing a child at its own data directory does not move its
+  database. The config is rewritten here instead, in whichever shape the compiled adapter uses.
+
+  **Migrations do not run in the child.** `Trinity.Application`'s `skip_migrations?/0` is
+  `Code.ensure_loaded?(Mix)`, so under `mix run` the `Ecto.Migrator` child is skipped and a fresh
+  database would boot with no tables. `isolate!/1` therefore creates the storage and runs the
+  migrations itself, before the application starts.
+
+  The sandbox pool is removed as well. A child is a real boot, not a sandboxed test, and nothing
+  in it owns a sandbox connection. So is the data-directory lock's pinned path, for the same reason
+  the database is renamed: `config/test.exs` fixes it per test partition rather than per process.
+  """
+
+  @repos [Trinity.Repo, Trinity.Repo.Receipts]
+
+  @doc """
+  Called **in the child**, with `--no-start`, before `Application.ensure_all_started/1`.
+
+  Rewrites both repos' configuration to a database named after `tag`, creates it, and migrates it.
+  """
+  @spec isolate!(String.t()) :: :ok
+  def isolate!(tag) when is_binary(tag) and tag != "" do
+    # The data-directory lock is pinned by `config/test.exs` to one path per test partition, so the
+    # suite holds it and every child would refuse to boot with `{:data_dir_held, ...}`. Dropping the
+    # override returns `Trinity.Application`'s `lock_dir/0` to `Trinity.Paths.ensure_data_dir/0`,
+    # which reads `XDG_DATA_HOME` — the variable the caller already sets per case. Under
+    # `MIX_ENV=dev` there was no override and this was free; it is the same class of problem as the
+    # pinned database, and it belongs here beside it rather than in six call sites.
+    Application.delete_env(:trinity, Trinity.DataDir.Lock)
+
+    Enum.each(@repos, fn repo ->
+      config = rewrite(repo, tag)
+      Application.put_env(:trinity, repo, config)
+
+      # `:already_up` is not an error here: a tag is unique per case, but a retried run of the same
+      # case reuses it, and a database that exists is exactly what was wanted.
+      _ = repo.__adapter__().storage_up(config)
+
+      Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true), pool_size: 2)
+    end)
+  end
+
+  @doc """
+  Called **in the parent**, from `on_exit`, to drop what a child created.
+
+  Matters most under Postgres, where a leaked database outlives the run on a developer's machine.
+  Under SQLite the files sit in the case's temporary directory and go with it.
+  """
+  @spec drop!(String.t()) :: :ok
+  def drop!(tag) when is_binary(tag) do
+    Enum.each(@repos, fn repo ->
+      _ = repo.__adapter__().storage_down(rewrite(repo, tag))
+    end)
+  end
+
+  @doc "The tag a case uses, unique per case and short enough to be a Postgres database name."
+  @spec tag(String.t()) :: String.t()
+  def tag(name) do
+    "trinity_boot_#{name}_#{System.unique_integer([:positive])}"
+    |> String.replace(~r/[^a-z0-9_]/, "_")
+    |> String.slice(0, 50)
+  end
+
+  defp rewrite(repo, tag) do
+    config = Application.get_env(:trinity, repo, [])
+    name = "#{tag}_#{repo |> Module.split() |> List.last() |> String.downcase()}"
+
+    config
+    # A child is a real boot. Nothing in it owns a sandbox connection.
+    |> Keyword.delete(:pool)
+    |> name_database(name)
+  end
+
+  # Postgres: the database is a name inside a URL. The URL is replaced with explicit keys rather
+  # than patched, so there is no second source of truth for the same field.
+  defp name_database(config, name) do
+    case Keyword.get(config, :url) do
+      url when is_binary(url) and url != "" ->
+        uri = URI.parse(url)
+        {username, password} = userinfo(uri)
+
+        config
+        |> Keyword.delete(:url)
+        |> Keyword.merge(
+          username: username,
+          password: password,
+          hostname: uri.host || "localhost",
+          port: uri.port || 5432,
+          database: name
+        )
+
+      _ ->
+        Keyword.put(config, :database, Path.join(System.tmp_dir!(), name <> ".db"))
+    end
+  end
+
+  defp userinfo(%URI{userinfo: info}) when is_binary(info) do
+    case String.split(info, ":", parts: 2) do
+      [user, pass] -> {user, pass}
+      [user] -> {user, nil}
+    end
+  end
+
+  defp userinfo(_), do: {"postgres", nil}
+end

test/test_helper.exs

@@ -7,14 +7,6 @@
 # Slice 003: :fips (tests that need FIPS mode on) runs only where the leg declares itself with
 # TRINITY_FIPS_LEG=1; excluded by tag elsewhere, never skipped. test/fips/mode_test.exs runs
 # everywhere and is what makes a leg that failed to enter the mode red rather than quiet.
-# Slice regulated-boot: :needs_dev_compile (tests that spawn a child OS process running
-# `mix run` under MIX_ENV=dev) is excluded on the FIPS leg alone. The leg has no prebuilt dev
-# tree, so the child compiles the dev dependency tree from scratch, and that reaches the network
-# to fetch the `tokenizers` precompiled NIF. OTP's TLS client cannot complete that download
-# (HelloRetryRequest middlebox assertion, erlang/otp#8470, docs/fips-leg.md finding 2 and slice
-# 002), so the child dies at `mix compile` and every assertion about what it booted is really an
-# assertion about a failed download. Excluded here rather than skipped by hand, and only here:
-# gate and postgres run the file in full.
 # Slice 032: :local_model (the real embedder through the serving) runs where
 # TRINITY_LOCAL_MODEL_CACHE names a cache that already holds all-MiniLM-L6-v2 (no download,
 # no network: a missing model is a failure there, not a skip); excluded by tag elsewhere.
@@ -24,8 +16,7 @@ local_model? = System.get_env("TRINITY_LOCAL_MODEL_CACHE") not in [nil, ""]
 ExUnit.start(
   exclude:
     [:live, :desktop, :eval] ++
-      if(fips_leg?, do: [:needs_dev_compile], else: [:fips]) ++
-      if(local_model?, do: [], else: [:local_model])
+      if(fips_leg?, do: [], else: [:fips]) ++ if(local_model?, do: [], else: [:local_model])
 )
 
 # Slice 024's boot receipt is written by a transient Task after the tree starts; once the