From 02c168f39df1286036e7618416c8ea4f64cadf74 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Thu, 1 Oct 2026 12:37:56 +0200
Subject: [PATCH 01/26] Prefetch tasks in per-process batches

Add a count to ThreadmillTaskBackend.acquire so a worker reserves up to
`count` tasks in one broker round-trip, and fill a per-process buffer
from a dedicated fetcher thread. The buffer defaults to 4 x threads and
is tunable with --prefetch-count; 1 disables batching.

- Redis acquire pops a round-robin batch atomically and advances the
  rotation one position per call
- the fetcher is a daemon thread, stops on max_tasks, shutdown, or drain,
  abandons a full buffer once no consumer is left, and logs plus re-raises
  a fetch failure so the child exits non-zero
- a worker whose consumers died is recycled instead of parking forever
- document the option and its soft limits in the README
---
 README.md                                    |  26 +-
 benchmarks/test_backends.py                  |  21 +
 tests/backends/test_redis.py                 | 181 +++++---
 tests/test_command.py                        |  64 ++-
 tests/test_executor.py                       | 459 +++++++++++++++++--
 tests/test_inspector.py                      |  11 +-
 tests/testapp/backends.py                    |  14 +
 tests/testapp/settings.py                    |   4 +
 threadmill/backends/base.py                  |  20 +-
 threadmill/backends/lua/acquire.lua          |  66 +--
 threadmill/backends/redis.py                 |   8 +-
 threadmill/executor.py                       | 139 +++++-
 threadmill/management/commands/threadmill.py |  14 +
 13 files changed, 861 insertions(+), 166 deletions(-)
 create mode 100644 tests/testapp/backends.py

diff --git a/README.md b/README.md
index 13b8506..3424e15 100644
--- a/README.md
+++ b/README.md
@@ -82,9 +82,24 @@ Depending on your workload, you can tweak the number of processes and threads.
 Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO.
 
 ```console
-uv run manage.py threadmill worker --processes 4 --threads 2
+uv run manage.py threadmill worker --workers 4 --threads 2
 ```
 
+Each worker process runs one fetcher thread that reserves a batch of tasks in a single broker round-trip.
+Worker threads drain that buffer, which amortizes broker latency across fast tasks.
+A full buffer blocks the fetcher until a worker thread frees a slot.
+
+Set the batch size with `--prefetch-count`.
+It defaults to four times the thread count and applies per process, not per thread.
+A value of `1` disables batching.
+
+Prefetching has soft limits:
+
+- Tasks are marked `RUNNING` when they are fetched, so the time they spend in the buffer counts against `lease_ttl`.
+- The priority lookahead widens to the buffer size, so ordering is no longer strictly global.
+- `--max-tasks` may overshoot by up to the buffer size, because a prefetched task always runs.
+- `worker_ids` records the fetcher of the process, not the thread that runs the task.
+
 #### Health
 
 If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:
@@ -95,12 +110,15 @@ uv run manage.py threadmill worker --max-tasks 1000 --max-tasks-jitter 100
 
 This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time.
 
+The limit is soft: a worker still drains its prefetch buffer, so it may process up to `--prefetch-count` tasks beyond the configured maximum.
+
 Should a worker crash or be killed, the pool will automatically restart it.
 
 #### Shutdown
 
 A graceful shutdown is possible with the `SIGTERM` or a keyboard interrupt.
-All workers will finish the tasks they acquired and acknowledge them.
+All workers will finish the tasks they acquired and acknowledge them, including the tasks already in their prefetch buffer.
+A hard kill cannot be intercepted, so buffered tasks are left to the lease reaper.
 
 You can use `--exit-empty` to exit immediately after all tasks have been processed,
 which might be useful for draining a one-off queue.
@@ -124,7 +142,7 @@ The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
 
 | Option              | Default                   | Description                                                             |
 | ------------------- | ------------------------- | ----------------------------------------------------------------------- |
-| `lease_ttl`         | `timedelta(hours=1)`      | Max processing time before a started task is marked FAILED.             |
+| `lease_ttl`         | `timedelta(hours=1)`      | Max time from fetch to acknowledgement before a task is marked FAILED.  |
 | `result_ttl`        | `timedelta(days=1)`       | How long task results are retained before automatic removal.            |
 | `broker_interval`   | `timedelta(seconds=1)`    | Interval between background broker maintenance passes.                  |
 | `batch_size`        | `100`                     | Max tasks to move or requeue per broker pass.                           |
@@ -133,7 +151,7 @@ The `RedisTaskBackend` accepts the following options under `OPTIONS` in your
 
 A task that is started but never acknowledged (lease expired) is marked FAILED
 with an `AcknowledgementTimeout` error. Set `lease_ttl` comfortably above your
-worst-case task runtime.
+worst-case task runtime plus the time a task may wait in a prefetch buffer.
 
 All keys for one backend alias share a Redis Cluster hash tag (`{alias}`), so
 every multi-key operation — including the cross-queue acquire — runs on a single
diff --git a/benchmarks/test_backends.py b/benchmarks/test_backends.py
index 4313a71..d38b48f 100644
--- a/benchmarks/test_backends.py
+++ b/benchmarks/test_backends.py
@@ -12,6 +12,8 @@
 ``test_start_worker__benchmark`` from ``test_process_queue__benchmark`` and divide
 the queue depth by the difference to get the marginal throughput of a busy queue.
 
+Threadmill is measured twice, with its default prefetch buffer and with batching
+disabled, so the prefetch cost can be subtracted from both worker benchmarks.
 Threadmill, django-tasks-db and django-tasks-redis run one worker process that
 drains a queue and exits. Celery has no such mode, so the benchmark queues a
 sentinel task last and waits for it to be processed. That wait is what proves the
@@ -119,6 +121,20 @@ def drain_with_threadmill_worker() -> None:
     )
 
 
+def drain_with_threadmill_worker_no_prefetch() -> None:
+    """Process every queued task with one threadmill worker without batching."""
+    call_command(
+        "threadmill",
+        "worker",
+        backend=DEFAULT_TASK_BACKEND_ALIAS,
+        queues=[DEFAULT_TASK_QUEUE_NAME],
+        workers=1,
+        prefetch_count=1,
+        exit_empty=True,
+        verbosity=0,
+    )
+
+
 def drain_with_django_tasks_db_worker() -> None:
     """Process every queued task with the django-tasks-db worker."""
     call_command(
@@ -250,6 +266,11 @@ def django_task_backend(
     django_task_backend(
         "threadmill", DEFAULT_TASK_BACKEND_ALIAS, drain_with_threadmill_worker
     ),
+    django_task_backend(
+        "threadmill (no prefetch)",
+        DEFAULT_TASK_BACKEND_ALIAS,
+        drain_with_threadmill_worker_no_prefetch,
+    ),
     django_task_backend(
         "django-tasks-db", "django-tasks-db", drain_with_django_tasks_db_worker
     ),
diff --git a/tests/backends/test_redis.py b/tests/backends/test_redis.py
index 4cb58b9..28bc925 100644
--- a/tests/backends/test_redis.py
+++ b/tests/backends/test_redis.py
@@ -101,8 +101,9 @@ def test_mover__moves_deferred_task_to_ready(self):
         task_result = default_task_backend.enqueue(deferred_task, args=[])
         broker = RedisBroker(default_task_backend)
         broker.main()
-        acquired = default_task_backend.acquire(timeout=datetime.timedelta(seconds=1))
-        assert acquired is not None
+        (acquired,) = default_task_backend.acquire(
+            timeout=datetime.timedelta(seconds=1)
+        )
         assert acquired.id == task_result.id
 
     def test_error_path__maintain_continues_after_exception(self, caplog):
@@ -138,10 +139,9 @@ def test_acquire__moves_to_running_set(self):
         )
         try:
             task_result = backend.enqueue(echo, args=[42])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="worker-1"
             )
-            assert acquired is not None
             assert acquired.id == task_result.id
 
             # Verify task is in running set, not in any processing set
@@ -175,10 +175,9 @@ def test_acquire__sets_last_attempted_at(self):
         )
         try:
             task_result = backend.enqueue(echo, args=[42])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="test-worker"
             )
-            assert acquired is not None
             assert acquired.last_attempted_at is not None
             assert acquired.worker_ids == ["test-worker"]
 
@@ -208,10 +207,9 @@ async def test_running_reaper__fails_expired_tasks(self):
         )
         try:
             task_result = backend.enqueue(echo, args=[42])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="reaper-test"
             )
-            assert acquired is not None
 
             # Wait for lease to expire
             time.sleep(1.1)
@@ -250,10 +248,9 @@ def test_stale_acknowledge__is_noop(self):
         )
         try:
             task_result = backend.enqueue(echo, args=[42])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="stale-ack-test"
             )
-            assert acquired is not None
 
             # Wait for lease to expire
             time.sleep(1.1)
@@ -311,10 +308,9 @@ async def test_queue_stats__counts_tasks(self):
             backend.enqueue(echo, args=[42])
             backend.enqueue(boom, args=[])
 
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="telemetry-test"
             )
-            assert acquired is not None
             backend.acknowledge(
                 dataclasses.replace(
                     acquired,
@@ -323,10 +319,9 @@ async def test_queue_stats__counts_tasks(self):
                 )
             )
 
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="telemetry-test"
             )
-            assert acquired is not None
             backend.acknowledge(
                 dataclasses.replace(
                     acquired,
@@ -361,10 +356,9 @@ async def test_queue_stats__counts_successful_and_failed(self):
             backend.enqueue(echo, args=[3])
 
             for _ in range(2):
-                acquired = backend.acquire(
+                (acquired,) = backend.acquire(
                     timeout=datetime.timedelta(seconds=1), worker="egress-test"
                 )
-                assert acquired is not None
                 backend.acknowledge(
                     dataclasses.replace(
                         acquired,
@@ -372,10 +366,9 @@ async def test_queue_stats__counts_successful_and_failed(self):
                         finished_at=timezone.now(),
                     )
                 )
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="egress-test"
             )
-            assert acquired is not None
             backend.acknowledge(
                 dataclasses.replace(
                     acquired,
@@ -413,10 +406,9 @@ async def test_queue_stats__successful_failed_evicted_by_result_ttl(self):
 
             def _ack(status: TaskResultStatus) -> str:
                 enqueued = backend.enqueue(echo, args=[1])
-                acquired = backend.acquire(
+                (acquired,) = backend.acquire(
                     timeout=datetime.timedelta(seconds=1), worker="eviction-test"
                 )
-                assert acquired is not None
                 backend.acknowledge(
                     dataclasses.replace(
                         acquired, status=status, finished_at=timezone.now()
@@ -507,7 +499,7 @@ def test_acknowledge__publishes_egress_telemetry(self):
         pubsub = backend.client.pubsub()
         try:
             backend.enqueue(echo, args=[1])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="publish-test"
             )
             pubsub.subscribe(backend.telemetry_channel)
@@ -542,10 +534,9 @@ def test_requeue__publishes_ingress_telemetry(self):
         pubsub = backend.client.pubsub()
         try:
             backend.enqueue(echo, args=[1])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="requeue-publish-test"
             )
-            assert acquired is not None
             failed = dataclasses.replace(
                 acquired,
                 status=TaskResultStatus.FAILED,
@@ -564,7 +555,7 @@ def test_requeue__publishes_ingress_telemetry(self):
     def _acknowledge(self, status: TaskResultStatus) -> str:
         """Enqueue, acquire, and acknowledge a task with the given status."""
         task_result = default_task_backend.enqueue(echo, args=[1])
-        acquired = default_task_backend.acquire(
+        (acquired,) = default_task_backend.acquire(
             timeout=datetime.timedelta(seconds=1), worker="peek-test"
         )
         assert acquired.id == task_result.id
@@ -587,7 +578,7 @@ def test_peek__ready_tasks(self):
     def test_peek__running_tasks(self):
         """Peek RUNNING returns acquired tasks with worker info."""
         default_task_backend.enqueue(echo, args=[1])
-        acquired = default_task_backend.acquire(
+        (acquired,) = default_task_backend.acquire(
             timeout=datetime.timedelta(seconds=1), worker="peek-test"
         )
         results = list(
@@ -675,10 +666,9 @@ def test_requeue__moves_from_running_to_deferred(self) -> None:
         )
         try:
             task_result = backend.enqueue(boom_with_retry, args=[])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="requeue-test"
             )
-            assert acquired is not None
 
             # Simulate a failed execution
             from django.tasks.base import TaskError
@@ -722,10 +712,9 @@ def test_requeue__preserves_id_and_errors(self) -> None:
         )
         try:
             task_result = backend.enqueue(boom_with_retry, args=[])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="preserve-test"
             )
-            assert acquired is not None
 
             from django.tasks.base import TaskError
 
@@ -773,10 +762,9 @@ def test_requeue__task_is_re_acquirable_after_delay(self) -> None:
         )
         try:
             task_result = backend.enqueue(boom_with_retry, args=[])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="requeue-acq-test"
             )
-            assert acquired is not None
 
             from django.tasks.base import TaskError
 
@@ -801,10 +789,9 @@ def test_requeue__task_is_re_acquirable_after_delay(self) -> None:
             broker.main()
 
             # The task should be acquirable again
-            re_acquired = backend.acquire(
+            (re_acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="requeue-acq-test-2"
             )
-            assert re_acquired is not None
             assert re_acquired.id == task_result.id
         finally:
             backend.close()
@@ -824,10 +811,9 @@ def test_requeue__cleans_up_failed_and_result_keys(self) -> None:
         )
         try:
             backend.enqueue(echo, args=[1])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="cleanup-test"
             )
-            assert acquired is not None
             backend.acknowledge(
                 dataclasses.replace(
                     acquired,
@@ -893,10 +879,9 @@ def test_dequeue__removes_failed_task_from_results(self) -> None:
         )
         try:
             backend.enqueue(echo, args=[1])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="dequeue-failed-test"
             )
-            assert acquired is not None
             backend.acknowledge(
                 dataclasses.replace(
                     acquired,
@@ -938,10 +923,9 @@ def test_purge__removes_all_tasks_across_segments(self) -> None:
             backend.acquire(timeout=datetime.timedelta(seconds=1), worker="purge-test")
             # One failed task
             backend.enqueue(echo, args=[3])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="purge-test-2"
             )
-            assert acquired is not None
             backend.acknowledge(
                 dataclasses.replace(
                     acquired,
@@ -951,10 +935,9 @@ def test_purge__removes_all_tasks_across_segments(self) -> None:
             )
             # One successful task
             backend.enqueue(echo, args=[4])
-            acquired = backend.acquire(
+            (acquired,) = backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="purge-test-3"
             )
-            assert acquired is not None
             backend.acknowledge(
                 dataclasses.replace(
                     acquired,
@@ -1083,8 +1066,7 @@ def test_acquire__resets_wait_after_success(self):
                 assert buildup_deltas[2] >= 0.15
 
             backend.enqueue(echo, args=[1])
-            acquired = backend.acquire(timeout=datetime.timedelta(seconds=1))
-            assert acquired is not None
+            (acquired,) = backend.acquire(timeout=datetime.timedelta(seconds=1))
             assert len(script.calls) == buildup_end + 1
 
             with pytest.raises(TimeoutError):
@@ -1138,14 +1120,14 @@ def test_acquire__serves_backlogged_queue_round_robin(self):
                 for value in range(neighbour_tasks)
             }
 
-            acquired_ids = {
-                backend.acquire(
+            acquired_ids = set()
+            for _ in range(2 * neighbour_tasks):
+                (task_result,) = backend.acquire(
                     "compute",
                     "io",
                     timeout=datetime.timedelta(seconds=1),
-                ).id
-                for _ in range(2 * neighbour_tasks)
-            }
+                )
+                acquired_ids.add(task_result.id)
             assert neighbour_ids <= acquired_ids
         finally:
             backend.close()
@@ -1160,16 +1142,16 @@ def test_acquire__rebase_rotation_index_beyond_queue_count(self):
         try:
             backend.enqueue(replace(echo, queue_name="compute"), args=[1])
             backend.enqueue(replace(echo, queue_name="io"), args=[2])
-            acquired = [
-                backend.acquire(
+            acquired = []
+            for _ in range(2):
+                (task_result,) = backend.acquire(
                     "default",
                     "compute",
                     "io",
                     timeout=datetime.timedelta(seconds=1),
                     worker="worker-1",
                 )
-                for _ in range(2)
-            ]
+                acquired.append(task_result)
             assert [result.task.queue_name for result in acquired] == ["compute", "io"]
             running_by_queue = {
                 queue_name: {
@@ -1206,9 +1188,10 @@ def test_acquire__wraps_rotation_offset_through_every_queue(self):
 
             acquired = []
             for _ in range(2 * len(queue_names)):
-                acquired.append(
-                    backend.acquire(*queue_names, timeout=datetime.timedelta(seconds=1))
+                (task_result,) = backend.acquire(
+                    *queue_names, timeout=datetime.timedelta(seconds=1)
                 )
+                acquired.append(task_result)
                 assert 0 <= backend._rotation_offset < len(queue_names)
 
             assert [result.task.queue_name for result in acquired] == [
@@ -1216,11 +1199,12 @@ def test_acquire__wraps_rotation_offset_through_every_queue(self):
                 "default",
                 "compute",
             ] * 2
-            assert [sent_args[-1] for sent_args in recorder.sent_args] == [
+            assert [sent_args[-2] for sent_args in recorder.sent_args] == [
                 "2",
                 "0",
                 "1",
             ] * 2
+            assert {sent_args[-1] for sent_args in recorder.sent_args} == {"1"}
         finally:
             backend.close()
 
@@ -1241,11 +1225,96 @@ def test_acquire__keep_rotation_on_idle_polls(self):
                     timeout=datetime.timedelta(seconds=0.3),
                 )
             assert len(recorder.sent_args) > 1
-            assert {sent_args[-1] for sent_args in recorder.sent_args} == {"5"}
+            assert {sent_args[-2] for sent_args in recorder.sent_args} == {"5"}
+            assert {sent_args[-1] for sent_args in recorder.sent_args} == {"1"}
             assert backend._rotation_offset == 5
         finally:
             backend.close()
 
+    def test_acquire__passes_count_to_script(self):
+        """acquire() sends the requested count as the last script argument."""
+        backend = _make_backend("acquire_count_arg_test")
+        recorder = RecordingAcquireScript(backend._acquire_script)
+        backend._acquire_script = recorder
+        try:
+            backend.enqueue(echo, args=[1])
+            (acquired,) = backend.acquire(
+                count=3, timeout=datetime.timedelta(seconds=1), worker="count-test"
+            )
+            assert recorder.sent_args[-1][-1] == "3"
+            assert acquired.worker_ids == ["count-test"]
+        finally:
+            backend.close()
+
+    def test_acquire__returns_up_to_count_tasks(self):
+        """acquire(count=N) locks at most N tasks and leaves the surplus ready."""
+        backend = _make_backend("acquire_batch_test")
+        try:
+            enqueued = [backend.enqueue(echo, args=[value]) for value in range(3)]
+            acquired = backend.acquire(
+                count=2, timeout=datetime.timedelta(seconds=1), worker="batch-test"
+            )
+            assert len(acquired) == 2
+            assert {task_result.id for task_result in acquired} <= {
+                task_result.id for task_result in enqueued
+            }
+            assert all(
+                task_result.status is TaskResultStatus.RUNNING
+                for task_result in acquired
+            )
+            assert all(
+                task_result.worker_ids == ["batch-test"] for task_result in acquired
+            )
+            running_key = backend._segment_key(TaskResultStatus.RUNNING, "default")
+            ready_key = backend._segment_key(TaskResultStatus.READY, "default")
+            assert backend.client.zcard(running_key) == 2
+            assert backend.client.zcard(ready_key) == 1
+        finally:
+            backend.close()
+
+    def test_acquire__short_batch_does_not_wait_for_count(self):
+        """acquire(count=N) returns a short batch without waiting for the rest."""
+        backend = _make_backend("acquire_short_batch_test")
+        try:
+            enqueued = backend.enqueue(echo, args=[1])
+            started_at = time.monotonic()
+            acquired = backend.acquire(
+                count=5,
+                timeout=datetime.timedelta(seconds=5),
+                worker="short-batch-test",
+            )
+            assert [task_result.id for task_result in acquired] == [enqueued.id]
+            assert time.monotonic() - started_at < 2
+        finally:
+            backend.close()
+
+    def test_acquire__spreads_batch_across_queues(self):
+        """acquire(count=N) pops round-robin, spreading one batch over the queues."""
+        backend = _make_backend(
+            "acquire_batch_round_robin_test",
+            queues=["compute", "io"],
+        )
+        backend._rotation_offset = 0
+        try:
+            for value in range(2):
+                backend.enqueue(replace(echo, queue_name="compute"), args=[value])
+                backend.enqueue(replace(echo, queue_name="io"), args=[value])
+            acquired = backend.acquire(
+                "compute",
+                "io",
+                count=4,
+                timeout=datetime.timedelta(seconds=1),
+            )
+            assert [task_result.task.queue_name for task_result in acquired] == [
+                "compute",
+                "io",
+                "compute",
+                "io",
+            ]
+            assert backend._rotation_offset == 1
+        finally:
+            backend.close()
+
     def test_init__randomize_rotation_offset(self):
         """Seed each backend differently so recycled workers spread across queues."""
         queues = ["default", "compute", "io"]
diff --git a/tests/test_command.py b/tests/test_command.py
index aeb93c2..f229b76 100644
--- a/tests/test_command.py
+++ b/tests/test_command.py
@@ -1,5 +1,7 @@
 import argparse
 import datetime
+import io
+import json
 import logging
 import re
 import signal
@@ -7,9 +9,14 @@
 
 import pytest
 from django.core.management import CommandError, call_command
-from django.tasks import default_task_backend
-
-from tests.testapp.tasks import compute_workload, io_workload, memory_workload
+from django.tasks import TaskResultStatus, default_task_backend
+
+from tests.testapp.tasks import (
+    compute_workload,
+    echo,
+    io_workload,
+    memory_workload,
+)
 from threadmill.executor import JsonFormatter, handler
 from threadmill.management.commands import threadmill
 
@@ -43,6 +50,7 @@ def test_add_arguments__register_all_worker_options(self):
         assert parsed_arguments.max_tasks_jitter == 0
         assert parsed_arguments.poll_interval == 0.01
         assert parsed_arguments.poll_max_interval == 1
+        assert parsed_arguments.prefetch_count is None
         assert parsed_arguments.log_format is None
 
     def test_add_arguments__parse_poll_intervals_as_floats(self):
@@ -55,6 +63,13 @@ def test_add_arguments__parse_poll_intervals_as_floats(self):
         assert parsed_arguments.poll_interval == 0.05
         assert parsed_arguments.poll_max_interval == 0.2
 
+    def test_add_arguments__parse_prefetch_count(self):
+        """Parse the prefetch count option as an int."""
+        parser = argparse.ArgumentParser()
+        threadmill.WorkerCommand().add_arguments(parser)
+        parsed_arguments = parser.parse_args(["--prefetch-count", "8"])
+        assert parsed_arguments.prefetch_count == 8
+
     def test_call_command__log_format(self):
         """Run the worker with the given log format string."""
         call_command(
@@ -113,6 +128,49 @@ def test_call_command__log_format__empty_string(self):
         )
         assert handler.formatter.format(record) == "Hello world"
 
+    def test_call_command__prefetch_count(self):
+        """Pass the prefetch count through to the running task executor."""
+        enqueued = default_task_backend.enqueue(echo, args=[1])
+        original_stream = handler.stream
+        parent_log = io.StringIO()
+        handler.setStream(parent_log)
+        try:
+            call_command(
+                "threadmill",
+                "worker",
+                verbosity=0,
+                workers=1,
+                queues=["default"],
+                exit_empty=True,
+                prefetch_count=7,
+            )
+        finally:
+            handler.setStream(original_stream)
+        records = [
+            json.loads(line)
+            for line in parent_log.getvalue().splitlines()
+            if line.startswith("{")
+        ]
+        assert any("7 prefetched tasks each" in record["message"] for record in records)
+        assert (
+            default_task_backend.get_result(enqueued.id).status
+            is TaskResultStatus.SUCCESSFUL
+        )
+
+    @pytest.mark.parametrize("prefetch_count", [0, -1])
+    def test_call_command__prefetch_count__raise_command_error(self, prefetch_count):
+        """Reject a prefetch count below one with a CommandError."""
+        with pytest.raises(
+            CommandError,
+            match=re.escape(f"Invalid prefetch count: {prefetch_count!r}"),
+        ):
+            call_command(
+                "threadmill",
+                "worker",
+                verbosity=0,
+                prefetch_count=prefetch_count,
+            )
+
     def test_call_command__poll_intervals(self):
         """Convert poll options to timedeltas for the task executor."""
         with patch.object(threadmill.TaskExecutor, "run", autospec=True) as run:
diff --git a/tests/test_executor.py b/tests/test_executor.py
index 19ecbbf..812c0dd 100644
--- a/tests/test_executor.py
+++ b/tests/test_executor.py
@@ -3,6 +3,7 @@
 import json
 import logging
 import multiprocessing
+import queue
 import sys
 import threading
 import time
@@ -28,10 +29,11 @@
     echo,
     log_message,
 )
-from threadmill.backends.base import Broker
+from threadmill.backends.base import Broker, ThreadmillTaskBackend
 from threadmill.executor import (
     JsonFormatter,
     TaskExecutor,
+    TaskPrefetcher,
     WorkerProcess,
     WorkerThread,
     configure_logging,
@@ -75,18 +77,109 @@ def _task_result(task, *args, **kwargs) -> TaskResult:
 def _make_worker(
     *,
     max_tasks: int | None = None,
+    prefetch_count: int = 1,
     poll_interval: datetime.timedelta | None = None,
     poll_max_interval: datetime.timedelta | None = None,
 ) -> WorkerProcess:
     """Build an unstarted `WorkerProcess`."""
+    options = {}
+    if poll_interval is not None:
+        options["poll_interval"] = poll_interval
+    if poll_max_interval is not None:
+        options["poll_max_interval"] = poll_max_interval
     return WorkerProcess(
         thread_count=1,
         max_tasks=max_tasks,
+        prefetch_count=prefetch_count,
         backend_alias="default",
         queues=("default",),
-        poll_interval=poll_interval,
-        poll_max_interval=poll_max_interval,
         log_formatter=JsonFormatter(),
+        **options,
+    )
+
+
+def _prefetched_thread(
+    task_result: TaskResult,
+    *,
+    max_tasks: int = 1,
+    backend: ThreadmillTaskBackend = default_task_backend,
+) -> WorkerThread:
+    """Build a worker thread whose prefetch buffer already holds one task."""
+    worker = _make_worker(max_tasks=max_tasks)
+    worker.lock = threading.Lock()
+    worker.expired = threading.Event()
+    prefetcher = TaskPrefetcher(worker=worker, backend=backend, prefetch_count=1)
+    prefetcher.task_buffer.put(task_result)
+    prefetcher.finished.set()
+    worker.prefetcher = prefetcher
+    return WorkerThread(worker=worker, index=0, backend=backend)
+
+
+class StubPrefetchBackend:
+    """Scripted backend stub for prefetcher tests without broker round-trips."""
+
+    def __init__(self, *responses: list[TaskResult] | Exception) -> None:
+        self.responses = list(responses)
+        self.calls: list[dict] = []
+
+    def acquire(self, *queue_names, count=1, timeout=None, worker=""):
+        """Return the next scripted batch or raise the next scripted error."""
+        self.calls.append(
+            {
+                "queues": queue_names,
+                "count": count,
+                "timeout": timeout,
+                "worker": worker,
+            }
+        )
+        response = self.responses.pop(0) if self.responses else TimeoutError("drained")
+        if isinstance(response, Exception):
+            raise response
+        return response
+
+
+class StubPrefetchWorker:
+    """Minimal worker stub exposing the state the prefetcher reads."""
+
+    def __init__(
+        self, *, remaining: int | None = None, exit_empty: bool = False
+    ) -> None:
+        self.pid = 4242
+        self.queues = ("default",)
+        self.remaining = remaining
+        self.exit_empty = exit_empty
+        self.expired = threading.Event()
+        self.shutdown_requested = threading.Event()
+
+    def remaining_tasks(self) -> int | None:
+        """Return the scripted task budget."""
+        return self.remaining
+
+
+class CountingEvent(threading.Event):
+    """Event that counts is_set() checks to observe consumer poll cycles."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.checks = 0
+
+    def is_set(self) -> bool:
+        self.checks += 1
+        return super().is_set()
+
+
+def _make_prefetcher(
+    backend: StubPrefetchBackend,
+    *,
+    prefetch_count: int = 4,
+    remaining: int | None = None,
+    exit_empty: bool = False,
+) -> TaskPrefetcher:
+    """Build a prefetcher over a scripted stub backend and worker."""
+    return TaskPrefetcher(
+        worker=StubPrefetchWorker(remaining=remaining, exit_empty=exit_empty),
+        backend=backend,
+        prefetch_count=prefetch_count,
     )
 
 
@@ -222,6 +315,32 @@ def test_post_init__thread_count_at_least_one(self):
         )
         assert executor.thread_count == 1
 
+    def test_post_init__derives_prefetch_count_from_threads(self):
+        """__post_init__ defaults the prefetch count to four tasks per thread."""
+        executor = TaskExecutor(
+            backend=default_task_backend, threads=3, queues=("default",)
+        )
+        assert executor.prefetch_count == 12
+
+    def test_post_init__keeps_explicit_prefetch_count(self):
+        """__post_init__ keeps an explicitly configured prefetch count."""
+        executor = TaskExecutor(
+            backend=default_task_backend, prefetch_count=7, queues=("default",)
+        )
+        assert executor.prefetch_count == 7
+
+    @pytest.mark.parametrize("prefetch_count", [0, -1])
+    def test_post_init__raise_value_error_for_prefetch_count_below_one(
+        self, prefetch_count
+    ):
+        """__post_init__ rejects a prefetch count below one."""
+        with pytest.raises(ValueError, match="prefetch_count must be at least 1"):
+            TaskExecutor(
+                backend=default_task_backend,
+                prefetch_count=prefetch_count,
+                queues=("default",),
+            )
+
     def test_get_maximum_tasks_per_child__returns_none_when_max_tasks_is_zero(self):
         """get_maximum_tasks_per_child returns None when max_tasks is 0."""
         executor = TaskExecutor(
@@ -252,12 +371,25 @@ def test_get_maximum_tasks_per_child__applies_jitter(self):
         result = executor.get_maximum_tasks_per_child()
         assert 100 <= result <= 110  # (100 + randint(0, 10)) // 1
 
+    def test_get_maximum_tasks_per_child__floors_at_one(self):
+        """get_maximum_tasks_per_child never returns less than one task per child."""
+        executor = TaskExecutor(
+            backend=default_task_backend,
+            max_tasks=2,
+            max_tasks_jitter=0,
+            threads=8,
+            queues=("default",),
+        )
+        assert executor.get_maximum_tasks_per_child() == 1
+
     def test_create_worker_process__starts_worker(self):
         """create_worker_process creates and starts a WorkerProcess."""
         executor = TaskExecutor(backend=default_task_backend, queues=("default",))
         worker = executor.create_worker_process()
         assert worker.is_alive()
         assert worker.log_formatter is executor.log_formatter
+        assert executor.prefetch_count == 4
+        assert worker.prefetch_count == executor.prefetch_count
         worker.shutdown()
 
     def test_run__processes_enqueued_tasks_end_to_end(self):
@@ -349,19 +481,13 @@ def test_run__executes_model_task_in_spawned_worker(self):
 
     @pytest.mark.django_db(transaction=True)
     def test_worker_acquires_updates_and_acknowledges(self):
-        """Worker acquires, executes, and acknowledges via its own backend."""
+        """Worker thread executes and acknowledges a prefetched task via its backend."""
         enqueued = default_task_backend.enqueue(echo, args=[42])
-
-        worker = _make_worker(max_tasks=1)
-        worker.lock = threading.Lock()
-        worker.expired = threading.Event()
-
-        thread = WorkerThread(
-            worker=worker,
-            index=0,
-            backend=default_task_backend,
+        (acquired,) = default_task_backend.acquire(
+            timeout=datetime.timedelta(seconds=1), worker="test-worker"
         )
-        thread.run()
+
+        _prefetched_thread(acquired).run()
 
         persisted = default_task_backend.get_result(enqueued.id)
         assert persisted.status == TaskResultStatus.SUCCESSFUL
@@ -471,6 +597,22 @@ def test_record_task__noop_before_run_sets_lock_and_expired(self):
         worker.record_task()
         assert worker.task_count == 0
 
+    def test_remaining_tasks__none_when_unlimited(self):
+        """remaining_tasks reports no budget when max_tasks is disabled."""
+        assert _make_worker(max_tasks=None).remaining_tasks() is None
+
+    def test_remaining_tasks__subtracts_task_count(self):
+        """remaining_tasks reports the unreached part of the worker budget."""
+        worker = _make_worker(max_tasks=5)
+        worker.task_count = 2
+        assert worker.remaining_tasks() == 3
+
+    def test_remaining_tasks__floors_at_zero(self):
+        """remaining_tasks never reports a negative budget."""
+        worker = _make_worker(max_tasks=2)
+        worker.task_count = 5
+        assert worker.remaining_tasks() == 0
+
     def test_shutdown_requested__is_settable(self):
         """shutdown_requested event can be set on an unstarted worker."""
         worker = _make_worker()
@@ -487,12 +629,59 @@ def test_run__applies_log_formatter_and_stops(self):
         assert not run_thread.is_alive()
         assert handler.formatter is worker.log_formatter
 
+    def test_run__raises_when_prefetcher_fails(self, monkeypatch):
+        """Re-raise a prefetch failure so the worker process exits non-zero."""
+        thread_failures = []
+        monkeypatch.setattr(threading, "excepthook", thread_failures.append)
+        monkeypatch.setattr(
+            "threadmill.executor.TASK_WAIT_TIMEOUT", datetime.timedelta(seconds=0.01)
+        )
+        worker = WorkerProcess(
+            thread_count=1,
+            backend_alias="stub",
+            queues=("default",),
+            log_formatter=JsonFormatter(),
+        )
+
+        with pytest.raises(RuntimeError, match="backend unavailable"):
+            worker.run()
+
+        assert worker.prefetcher is not None
+        assert isinstance(worker.prefetcher.failure, RuntimeError)
+        assert worker.prefetcher.finished.is_set()
+        assert len(thread_failures) == 1
+        assert isinstance(thread_failures[0].exc_value, RuntimeError)
+
+    def test_run__child_exits_non_zero_on_prefetch_failure(self, capfd):
+        """A child whose prefetcher failed exits non-zero and logs the failure."""
+        worker = WorkerProcess(
+            thread_count=1,
+            backend_alias="stub",
+            queues=("default",),
+            log_formatter=JsonFormatter(),
+        )
+
+        worker.start()
+        worker.join(timeout=5)
+        if worker.is_alive():
+            worker.terminate()
+
+        assert worker.exitcode == 1
+        assert "exits after a fetch failure" in capfd.readouterr().out
+
 
 class TestWorkerThread:
     """Tests for the WorkerThread class."""
 
     pytestmark = pytest.mark.django_db(transaction=True)
 
+    @pytest.fixture(autouse=True)
+    def fast_task_wait(self, monkeypatch):
+        """Shorten the buffer wait so drained run() tests return quickly."""
+        monkeypatch.setattr(
+            "threadmill.executor.TASK_WAIT_TIMEOUT", datetime.timedelta(seconds=0.01)
+        )
+
     def test_execute_task_result__successful_execution(self):
         """execute_task_result runs a task and returns SUCCESSFUL result."""
         result = WorkerThread(
@@ -575,6 +764,7 @@ def test_retry_delay__none_when_callback_returns_none(self) -> None:
     def test_retry_delay__none_when_callback_raises(self, caplog) -> None:
         """Return None and log when the retry callback raises an exception."""
         mp_logger = multiprocessing.get_logger()
+        level = mp_logger.level
         mp_logger.addHandler(caplog.handler)
         mp_logger.setLevel(logging.ERROR)
         result = _task_result(boom_retry_raises)
@@ -587,6 +777,7 @@ def test_retry_delay__none_when_callback_raises(self, caplog) -> None:
             assert WorkerThread.retry_delay(result) is None
         finally:
             mp_logger.removeHandler(caplog.handler)
+            mp_logger.setLevel(level)
         assert "Retry callback failed" in caplog.text
 
     def test_retry_delay__passes_task_context(self) -> None:
@@ -619,16 +810,11 @@ def test_retry_delay__callback_receives_errors(self) -> None:
     def test_run__requeues_failed_task_with_retry(self) -> None:
         """run() requeues a FAILED task when retry_delay returns a timedelta."""
         enqueued = default_task_backend.enqueue(boom_with_retry, args=[])
-        worker = _make_worker(max_tasks=1)
-        worker.lock = threading.Lock()
-        worker.expired = threading.Event()
-
-        thread = WorkerThread(
-            worker=worker,
-            index=0,
-            backend=default_task_backend,
+        (acquired,) = default_task_backend.acquire(
+            timeout=datetime.timedelta(seconds=1), worker="test-worker"
         )
-        thread.run()
+
+        _prefetched_thread(acquired).run()
 
         # The task should have been requeued to the deferred set, not acknowledged
         from threadmill.backends.redis import RedisTaskBackend
@@ -641,16 +827,11 @@ def test_run__requeues_failed_task_with_retry(self) -> None:
     def test_run__acknowledges_failed_task_without_retry(self) -> None:
         """run() acknowledges a FAILED task when retry_delay returns None."""
         enqueued = default_task_backend.enqueue(boom_no_retry, args=[])
-        worker = _make_worker(max_tasks=1)
-        worker.lock = threading.Lock()
-        worker.expired = threading.Event()
-
-        thread = WorkerThread(
-            worker=worker,
-            index=0,
-            backend=default_task_backend,
+        (acquired,) = default_task_backend.acquire(
+            timeout=datetime.timedelta(seconds=1), worker="test-worker"
         )
-        thread.run()
+
+        _prefetched_thread(acquired).run()
 
         # The task should be acknowledged (FAILED result, not in deferred)
         from threadmill.backends.redis import RedisTaskBackend
@@ -666,16 +847,11 @@ def test_run__acknowledges_failed_task_without_retry(self) -> None:
     def test_run__acknowledges_failed_task_when_callback_raises(self) -> None:
         """run() acknowledges a FAILED task when the retry callback raises."""
         enqueued = default_task_backend.enqueue(boom_retry_raises, args=[])
-        worker = _make_worker(max_tasks=1)
-        worker.lock = threading.Lock()
-        worker.expired = threading.Event()
-
-        thread = WorkerThread(
-            worker=worker,
-            index=0,
-            backend=default_task_backend,
+        (acquired,) = default_task_backend.acquire(
+            timeout=datetime.timedelta(seconds=1), worker="test-worker"
         )
-        thread.run()
+
+        _prefetched_thread(acquired).run()
 
         from threadmill.backends.redis import RedisTaskBackend
 
@@ -686,3 +862,204 @@ def test_run__acknowledges_failed_task_when_callback_raises(self) -> None:
 
         result = default_task_backend.get_result(enqueued.id)
         assert result.status == TaskResultStatus.FAILED
+
+    def test_run__returns_when_buffer_drained_and_finished(self) -> None:
+        """run() returns once the buffer is drained and the prefetcher finished."""
+        worker = _make_worker(max_tasks=1)
+        worker.lock = threading.Lock()
+        worker.expired = threading.Event()
+        prefetcher = TaskPrefetcher(
+            worker=worker, backend=default_task_backend, prefetch_count=1
+        )
+        prefetcher.finished.set()
+        worker.prefetcher = prefetcher
+
+        WorkerThread(worker=worker, index=0, backend=default_task_backend).run()
+
+    def test_run__waits_for_buffer_until_finished(self) -> None:
+        """run() keeps polling an empty buffer while the prefetcher is alive."""
+        worker = _make_worker(max_tasks=1)
+        worker.lock = threading.Lock()
+        worker.expired = threading.Event()
+        prefetcher = TaskPrefetcher(
+            worker=worker, backend=default_task_backend, prefetch_count=1
+        )
+        finished = CountingEvent()
+        prefetcher.finished = finished
+        worker.prefetcher = prefetcher
+
+        reader = threading.Thread(
+            target=WorkerThread(
+                worker=worker, index=0, backend=default_task_backend
+            ).run,
+            daemon=True,
+        )
+        reader.start()
+        deadline = time.monotonic() + 2
+        while finished.checks < 1 and time.monotonic() < deadline:
+            time.sleep(0.005)
+        assert reader.is_alive()
+
+        finished.set()
+        reader.join(timeout=1)
+
+        assert not reader.is_alive()
+        assert finished.checks >= 2
+
+
+class TestTaskPrefetcher:
+    """Tests for the TaskPrefetcher thread."""
+
+    def test_run__buffers_until_stop_requested(self):
+        """Fill the buffer from batches and poll again after an empty acquire."""
+        first = _task_result(echo, 1)
+        second = _task_result(echo, 2)
+        backend = StubPrefetchBackend(
+            TimeoutError("drained"), [first, second], TimeoutError("drained")
+        )
+        prefetcher = _make_prefetcher(backend, prefetch_count=4)
+        thread = threading.Thread(target=prefetcher.run)
+        thread.start()
+        try:
+            deadline = time.monotonic() + 2
+            while prefetcher.task_buffer.qsize() < 2 and time.monotonic() < deadline:
+                time.sleep(0.01)
+            assert prefetcher.task_buffer.qsize() == 2
+        finally:
+            prefetcher.stop_requested.set()
+            thread.join(timeout=2)
+        assert not thread.is_alive()
+        assert prefetcher.finished.is_set()
+        assert prefetcher.failure is None
+        assert [prefetcher.task_buffer.get_nowait().id for _ in range(2)] == [
+            first.id,
+            second.id,
+        ]
+        assert all(call["count"] == 4 for call in backend.calls)
+
+    def test_run__stops_on_expired(self):
+        """Return immediately when the worker has already expired."""
+        backend = StubPrefetchBackend()
+        prefetcher = _make_prefetcher(backend)
+
+        prefetcher.worker.expired.set()
+        prefetcher.run()
+
+        assert prefetcher.finished.is_set()
+        assert prefetcher.failure is None
+        assert backend.calls == []
+
+    def test_run__stops_on_stop_requested(self):
+        """Return immediately when a stop was requested before the loop starts."""
+        backend = StubPrefetchBackend()
+        prefetcher = _make_prefetcher(backend)
+
+        prefetcher.stop_requested.set()
+        prefetcher.run()
+
+        assert prefetcher.finished.is_set()
+        assert backend.calls == []
+
+    def test_run__breaks_when_no_budget_remains(self):
+        """Stop without acquiring once the worker budget is exhausted."""
+        backend = StubPrefetchBackend()
+        prefetcher = _make_prefetcher(backend, remaining=0)
+
+        prefetcher.run()
+
+        assert backend.calls == []
+        assert prefetcher.finished.is_set()
+
+    def test_run__caps_count_at_remaining_budget(self):
+        """Acquire no more tasks than the remaining worker budget allows."""
+        task_result = _task_result(echo, 1)
+        backend = StubPrefetchBackend([task_result])
+        prefetcher = _make_prefetcher(backend, prefetch_count=5, remaining=2)
+
+        prefetcher.worker.shutdown_requested.set()
+        prefetcher.run()
+
+        assert backend.calls[0]["count"] == 2
+        assert prefetcher.task_buffer.get_nowait().id == task_result.id
+        assert len(backend.calls) == 1
+        assert prefetcher.finished.is_set()
+
+    def test_run__stops_after_batch_when_shutdown_requested(self):
+        """Buffer one final batch, then stop when a shutdown was requested."""
+        first = _task_result(echo, 1)
+        second = _task_result(echo, 2)
+        backend = StubPrefetchBackend([first, second])
+        prefetcher = _make_prefetcher(backend, prefetch_count=2)
+
+        prefetcher.worker.shutdown_requested.set()
+        prefetcher.run()
+
+        assert [prefetcher.task_buffer.get_nowait().id for _ in range(2)] == [
+            first.id,
+            second.id,
+        ]
+        assert len(backend.calls) == 1
+        assert prefetcher.finished.is_set()
+
+    def test_run__stops_on_empty_when_exit_empty(self):
+        """Break out of the fetch loop when the queue drained and exit_empty is set."""
+        backend = StubPrefetchBackend(TimeoutError("drained"))
+        prefetcher = _make_prefetcher(backend, exit_empty=True)
+
+        prefetcher.run()
+
+        assert len(backend.calls) == 1
+        assert prefetcher.finished.is_set()
+        assert prefetcher.failure is None
+
+    def test_run__stops_on_empty_when_shutdown_requested(self):
+        """Break out of the fetch loop when a shutdown was requested."""
+        backend = StubPrefetchBackend(queue.Empty("drained"))
+        prefetcher = _make_prefetcher(backend)
+
+        prefetcher.worker.shutdown_requested.set()
+        prefetcher.run()
+
+        assert len(backend.calls) == 1
+        assert prefetcher.finished.is_set()
+
+    def test_run__records_failure_and_reraises(self):
+        """Record a fetch failure, mark the fetcher finished, and re-raise."""
+        backend = StubPrefetchBackend(RuntimeError("backend unavailable"))
+        prefetcher = _make_prefetcher(backend)
+
+        with pytest.raises(RuntimeError, match="backend unavailable"):
+            prefetcher.run()
+
+        assert isinstance(prefetcher.failure, RuntimeError)
+        assert prefetcher.finished.is_set()
+
+    def test_buffer__returns_false_when_full_and_stop_requested(self):
+        """Abandon a full buffer as soon as the prefetcher must stop."""
+        buffered = _task_result(echo, 1)
+        overflow = _task_result(echo, 2)
+        backend = StubPrefetchBackend([overflow])
+        prefetcher = _make_prefetcher(backend, prefetch_count=1)
+        prefetcher.task_buffer.put(buffered)
+
+        thread = threading.Thread(target=prefetcher.run)
+        thread.start()
+        try:
+            time.sleep(0.2)
+        finally:
+            prefetcher.stop_requested.set()
+            thread.join(timeout=3)
+
+        assert not thread.is_alive()
+        assert prefetcher.finished.is_set()
+        assert prefetcher.failure is None
+        assert prefetcher.task_buffer.qsize() == 1
+        assert prefetcher.task_buffer.get_nowait().id == buffered.id
+
+    def test_buffer__returns_true_when_space_available(self):
+        """Buffer a task result when the queue has room."""
+        task_result = _task_result(echo, 1)
+        prefetcher = _make_prefetcher(StubPrefetchBackend())
+
+        assert prefetcher.buffer(task_result) is True
+        assert prefetcher.task_buffer.get_nowait().id == task_result.id
diff --git a/tests/test_inspector.py b/tests/test_inspector.py
index 0c3b7cc..415a041 100644
--- a/tests/test_inspector.py
+++ b/tests/test_inspector.py
@@ -325,7 +325,7 @@ async def test_refresh_falls_back_to_first_when_selected_gone(self):
             task_list.selected_task = next(
                 r for r in task_list._current_results if r.id == first.id
             )
-            acquired = default_task_backend.acquire(
+            (acquired,) = default_task_backend.acquire(
                 timeout=datetime.timedelta(seconds=1), worker="inspector-test"
             )
             assert acquired.id == first.id
@@ -433,10 +433,9 @@ async def test_digit_binding_switches_tab(self):
     async def test_successful_tab_lists_finished_task(self):
         """The successful tab renders acknowledged tasks with the finished column."""
         default_task_backend.enqueue(echo, args=[1])
-        acquired = default_task_backend.acquire(
+        (acquired,) = default_task_backend.acquire(
             timeout=datetime.timedelta(seconds=1), worker="succ-test"
         )
-        assert acquired is not None
         default_task_backend.acknowledge(
             dataclasses.replace(
                 acquired,
@@ -530,10 +529,9 @@ async def test_action_refresh_refreshes_task_list(self):
 def _acknowledge_failed() -> str:
     """Enqueue, acquire, and acknowledge a task as FAILED. Return its ID."""
     task_result = default_task_backend.enqueue(echo, args=[1])
-    acquired = default_task_backend.acquire(
+    (acquired,) = default_task_backend.acquire(
         timeout=datetime.timedelta(seconds=1), worker="inspector-test"
     )
-    assert acquired is not None
     default_task_backend.acknowledge(
         dataclasses.replace(
             acquired, status=TaskResultStatus.FAILED, finished_at=timezone.now()
@@ -545,10 +543,9 @@ def _acknowledge_failed() -> str:
 def _acknowledge_successful() -> str:
     """Enqueue, acquire, and acknowledge a task as SUCCESSFUL. Return its ID."""
     task_result = default_task_backend.enqueue(echo, args=[1])
-    acquired = default_task_backend.acquire(
+    (acquired,) = default_task_backend.acquire(
         timeout=datetime.timedelta(seconds=1), worker="inspector-test"
     )
-    assert acquired is not None
     default_task_backend.acknowledge(
         dataclasses.replace(
             acquired, status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now()
diff --git a/tests/testapp/backends.py b/tests/testapp/backends.py
new file mode 100644
index 0000000..2587bfe
--- /dev/null
+++ b/tests/testapp/backends.py
@@ -0,0 +1,14 @@
+"""Test support task backends without a broker."""
+
+from threadmill.backends.base import ThreadmillTaskBackend
+
+
+class StubTaskBackend(ThreadmillTaskBackend):
+    """Backend whose acquire always fails, to exercise prefetch error paths."""
+
+    def enqueue(self, task, args, kwargs):
+        raise NotImplementedError
+
+    def acquire(self, *queue_names, count=1, timeout=None, worker=""):
+        """Raise a fetch failure for every call."""
+        raise RuntimeError("backend unavailable")
diff --git a/tests/testapp/settings.py b/tests/testapp/settings.py
index 3a1fa38..a6737b7 100644
--- a/tests/testapp/settings.py
+++ b/tests/testapp/settings.py
@@ -115,6 +115,10 @@
     "dummy": {
         "BACKEND": "django.tasks.backends.dummy.DummyBackend",
     },
+    "stub": {
+        "BACKEND": "tests.testapp.backends.StubTaskBackend",
+        "QUEUES": [DEFAULT_TASK_QUEUE_NAME],
+    },
 }
 
 # Password validation
diff --git a/threadmill/backends/base.py b/threadmill/backends/base.py
index fbab995..843c9fd 100644
--- a/threadmill/backends/base.py
+++ b/threadmill/backends/base.py
@@ -211,20 +211,28 @@ def validate_task(self, task: RetryTask) -> None:
     def acquire(
         self,
         *queue_names: str,
+        count: int = 1,
         timeout: datetime.timedelta | None = None,
         worker: str = "",
-    ) -> TaskResult:
+    ) -> list[TaskResult]:
         """
-        Return and lock the next task to be processed without removing it from the queue.
+        Return and lock up to `count` tasks without removing them from the queue.
+
+        Waits up to `timeout` for the first task only and fills the remaining
+        `count - 1` tasks without waiting.
 
         Args:
             queue_names: The names of the queues to acquire tasks from.
-            timeout: The maximum time to wait for a task. If None, wait indefinitely.
-            worker: The name of the worker thread acquiring the task.
+            count: The maximum number of tasks to acquire, at least 1.
+            timeout: The maximum time to wait for the first task. If None, wait indefinitely.
+            worker: The name of the worker thread acquiring the tasks.
 
         Raises:
-            TimeoutError: If no task is available within the specified timeout.
-            queue.Empty: If no task is available and timeout is None.
+            TimeoutError: If the first task does not arrive within the specified timeout.
+            queue.Empty: If the first task is unavailable and timeout is None.
+
+        Returns:
+            Between one and `count` task results, never an empty list.
         """
         raise NotImplementedError
 
diff --git a/threadmill/backends/lua/acquire.lua b/threadmill/backends/lua/acquire.lua
index e7c9762..ee0cc09 100644
--- a/threadmill/backends/lua/acquire.lua
+++ b/threadmill/backends/lua/acquire.lua
@@ -1,7 +1,8 @@
--- Atomically pop the lowest-scored task from any of the given priority queues,
--- update its JSON data with worker info, and move it directly to the running
--- set. Scans the queues round-robin from ARGV[7] and returns the first available
--- task, so a backlogged queue cannot starve its neighbours.
+-- Atomically pop up to ARGV[8] lowest-scored tasks from any of the given
+-- priority queues, update their JSON data with worker info, and move them
+-- directly to the running set. Scans the queues round-robin from ARGV[7], one
+-- queue per pop, so a batch spreads across queues instead of draining one and a
+-- backlogged queue cannot starve its neighbours.
 --
 -- KEYS[1..N]  -- interleaved running keys and queue keys, one pair per queue:
 --                KEYS[1] = running set, KEYS[2] = queue set, KEYS[3] = running,
@@ -14,36 +15,45 @@
 -- ARGV[6]     -- lease TTL in milliseconds
 -- ARGV[7]     -- start_index; 0-based index of the queue pair to scan first, so
 --                start_index 0 is the pair at KEYS[1] and KEYS[2]
--- Returns: updated serialized data on success, nil if all queues are empty.
+-- ARGV[8]     -- maximum number of tasks to pop
+-- Returns: array of updated serialized task data, empty if all queues are empty.
 
 local num_queues = tonumber(ARGV[4])
 local lease_ttl_ms = tonumber(ARGV[6])
 local start_index = tonumber(ARGV[7])
-for offset = 0, num_queues - 1 do
-  local queue_index = (start_index + offset) % num_queues + 1
+local max_count = tonumber(ARGV[8])
+local task_key_prefix = ARGV[3]
+local now_ms = tonumber(ARGV[1])
+local tasks = {}
+local misses = 0
+local queue_index = start_index % num_queues + 1
+-- joe: one full round without a task ends the scan, so a queue whose head
+-- entry is a ghost yields a short batch; retry the queue if ghosts ever dominate
+while #tasks < max_count and misses < num_queues do
   local result = redis.call('ZPOPMIN', KEYS[queue_index * 2])
-  if #result > 0 then
-    local task_id = result[1]
-    local data = redis.call('HGET', ARGV[3] .. task_id, 'data')
-    if data then
-      local ok, parsed = pcall(cjson.decode, data)
-      if ok then
-        parsed.status = 'RUNNING'
-        parsed.last_attempted_at = ARGV[2]
-        if not parsed.started_at then
-          parsed.started_at = ARGV[2]
-        end
-        if not parsed.worker_ids then
-          parsed.worker_ids = {}
-        end
-        table.insert(parsed.worker_ids, ARGV[5])
-        local updated_data = cjson.encode(parsed)
-        local deadline = tonumber(ARGV[1]) + lease_ttl_ms
-        redis.call('ZADD', KEYS[queue_index * 2 - 1], deadline, task_id)
-        redis.call('HSET', ARGV[3] .. task_id, 'data', updated_data)
-        return updated_data
+  misses = misses + 1
+  local data = #result > 0
+    and redis.call('HGET', task_key_prefix .. result[1], 'data')
+  if data then
+    local ok, parsed = pcall(cjson.decode, data)
+    if ok then
+      parsed.status = 'RUNNING'
+      parsed.last_attempted_at = ARGV[2]
+      if not parsed.started_at then
+        parsed.started_at = ARGV[2]
       end
+      if not parsed.worker_ids then
+        parsed.worker_ids = {}
+      end
+      table.insert(parsed.worker_ids, ARGV[5])
+      local updated_data = cjson.encode(parsed)
+      local deadline = now_ms + lease_ttl_ms
+      redis.call('ZADD', KEYS[queue_index * 2 - 1], deadline, result[1])
+      redis.call('HSET', task_key_prefix .. result[1], 'data', updated_data)
+      table.insert(tasks, updated_data)
+      misses = 0
     end
   end
+  queue_index = queue_index % num_queues + 1
 end
-return nil
+return tasks
diff --git a/threadmill/backends/redis.py b/threadmill/backends/redis.py
index 9ab1a80..9cd8ead 100644
--- a/threadmill/backends/redis.py
+++ b/threadmill/backends/redis.py
@@ -128,7 +128,7 @@ class RedisTaskBackend(ThreadmillTaskBackend):
     TELEMETRY_CHANNEL = "{prefix}:telemetry"
 
     ACQUIRE_SCRIPT = _load_lua("acquire")
-    """Pop the next task from a priority queue and move it directly to the running set."""
+    """Pop up to a given number of tasks from the priority queues and move them directly to the running set."""
     ACKNOWLEDGE_SCRIPT = _load_lua("acknowledge")
     """Remove from running, persist the result, and clean up."""
 
@@ -249,9 +249,10 @@ def enqueue(
     def acquire(
         self,
         *queue_names: str,
+        count: int = 1,
         timeout: datetime.timedelta | None = None,
         worker: str = "",
-    ) -> TaskResult:
+    ) -> list[TaskResult]:
         queue_names = queue_names or tuple(self.queues)
         deadline = time.monotonic() + timeout.total_seconds() if timeout else None
         keys = [
@@ -278,11 +279,12 @@ def acquire(
                     worker,
                     str(int(self.lease_ttl.total_seconds() * 1000)),
                     str(self._rotation_offset),
+                    str(count),
                 ],
             ):
                 self._miss_count = 0
                 self._rotation_offset = (self._rotation_offset + 1) % len(queue_names)
-                return self.deserialize_task_result(data)
+                return [self.deserialize_task_result(item) for item in data]
 
             try:
                 remaining = deadline - time.monotonic()
diff --git a/threadmill/executor.py b/threadmill/executor.py
index 1f4a58c..283a5ba 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -6,6 +6,7 @@
 import json
 import logging
 import multiprocessing
+import queue
 import random
 import socket
 import sys
@@ -14,7 +15,7 @@
 import typing
 from concurrent.futures import ThreadPoolExecutor
 from inspect import iscoroutinefunction
-from queue import Empty
+from queue import Empty, Full
 from traceback import format_exception
 
 import django
@@ -99,6 +100,10 @@ def configure_logging(formatter: logging.Formatter) -> None:
     root_logger.setLevel(logging.INFO)
 
 
+# Maximum wait for a task from the backend or the prefetch buffer.
+TASK_WAIT_TIMEOUT = datetime.timedelta(seconds=1)
+
+
 @dataclasses.dataclass(kw_only=True, slots=True)
 class TaskExecutor:
     """Tasks consumed from shared joinable queues via process and thread pools."""
@@ -108,6 +113,7 @@ class TaskExecutor:
     threads: int = 1
     max_tasks: int = 0
     max_tasks_jitter: int = 0
+    prefetch_count: int | None = None
     poll_interval: datetime.timedelta = datetime.timedelta(seconds=0.01)
     poll_max_interval: datetime.timedelta = datetime.timedelta(seconds=1)
     is_publishing: bool = dataclasses.field(default=True, init=False)
@@ -125,19 +131,23 @@ def __post_init__(self) -> None:
         """Initialize derived orchestration fields and queues."""
         self.process_count = self.workers or max(multiprocessing.cpu_count() - 1, 1)
         self.thread_count = max(self.threads, 1)
+        if self.prefetch_count is None:
+            self.prefetch_count = self.thread_count * 4
+        if self.prefetch_count < 1:
+            raise ValueError("prefetch_count must be at least 1")
 
     def get_maximum_tasks_per_child(self) -> int | None:
         """Return worker recycling limit based on config and thread count."""
         if self.max_tasks:
-            return (
-                self.max_tasks + random.randint(0, self.max_tasks_jitter)  # noqa: S311
-            ) // self.thread_count
+            jitter = random.randint(0, self.max_tasks_jitter)  # noqa: S311
+            return max((self.max_tasks + jitter) // self.thread_count, 1)
 
     def create_worker_process(self) -> WorkerProcess:
         """Create and start a new worker process."""
         worker = WorkerProcess(
             thread_count=self.thread_count,
             max_tasks=self.get_maximum_tasks_per_child(),
+            prefetch_count=self.prefetch_count,
             backend_alias=self.backend.alias,
             queues=self.queues,
             exit_empty=self.exit_empty,
@@ -151,6 +161,12 @@ def create_worker_process(self) -> WorkerProcess:
     def run(self) -> None:
         """Start consuming tasks until shutdown is requested."""
         configure_logging(self.log_formatter)
+        logger.info(
+            "Starting %d worker processes with %d threads and %d prefetched tasks each",
+            self.process_count,
+            self.thread_count,
+            self.prefetch_count,
+        )
         self.worker_processes = [
             self.create_worker_process() for _ in range(self.process_count)
         ]
@@ -194,13 +210,14 @@ def maintain_worker_pool(self) -> None:
 
 
 class WorkerProcess(multiprocessing.Process):
-    """Single worker process running thread_count consumer threads."""
+    """Single worker process running a prefetcher and thread_count consumer threads."""
 
     def __init__(
         self,
         *,
         thread_count: int,
         max_tasks: int | None = None,
+        prefetch_count: int = 1,
         backend_alias: str = "",
         queues: tuple[str, ...] = (),
         exit_empty: bool = False,
@@ -213,6 +230,7 @@ def __init__(
         super().__init__(daemon=True)
         self.thread_count = thread_count
         self.max_tasks = max_tasks
+        self.prefetch_count = prefetch_count
         self.backend_alias = backend_alias
         self.queues = queues
         self.exit_empty = exit_empty
@@ -222,9 +240,9 @@ def __init__(
         self.task_count = 0
         self.lock: threading.Lock | None = None
         self.expired: threading.Event | None = None
+        self.prefetcher: TaskPrefetcher | None = None
 
     def run(self) -> None:
-        """Start consumer execution inside this process."""
         django.setup()
         configure_logging(self.log_formatter)
         logger.info("Starting worker process %s", self.name)
@@ -233,16 +251,32 @@ def run(self) -> None:
         backend = task_backends[self.backend_alias]
         backend.poll_interval = self.poll_interval
         backend.poll_max_interval = self.poll_max_interval
+        self.prefetcher = TaskPrefetcher(
+            worker=self, backend=backend, prefetch_count=self.prefetch_count
+        )
+        self.prefetcher.start()
         consumer_threads = [
             WorkerThread(worker=self, index=index, backend=backend)
             for index in range(self.thread_count)
         ]
         for consumer_thread in consumer_threads:
             consumer_thread.start()
+        join_timeout = (
+            backend.result_ttl.total_seconds() if backend.result_ttl else None
+        )
         for consumer_thread in consumer_threads:
-            consumer_thread.join(
-                backend.result_ttl.total_seconds() if backend.result_ttl else None
-            )
+            consumer_thread.join(join_timeout)
+        self.prefetcher.stop_requested.set()
+        self.prefetcher.join(join_timeout)
+        if (failure := self.prefetcher.failure) is not None:
+            logger.error("Worker process %s exits after a fetch failure", self.name)
+            raise failure
+
+    def remaining_tasks(self) -> int | None:
+        """Return how many more tasks this process may run, or None when unlimited."""
+        if self.max_tasks is None:
+            return None
+        return max(self.max_tasks - self.task_count, 0)
 
     def record_task(self) -> None:
         """Record one processed task and stop when max_tasks is reached."""
@@ -262,8 +296,79 @@ def shutdown(self) -> None:
         self.join()
 
 
+class TaskPrefetcher(threading.Thread):
+    """Single prefetcher thread filling the task buffer of one worker process."""
+
+    def __init__(
+        self,
+        *,
+        worker: WorkerProcess,
+        backend: ThreadmillTaskBackend,
+        prefetch_count: int,
+    ) -> None:
+        super().__init__(name=f"{socket.gethostname()}:{worker.pid}-fetch", daemon=True)
+        self.worker = worker
+        self.backend = backend
+        self.prefetch_count = prefetch_count
+        self.task_buffer: queue.Queue[TaskResult] = queue.Queue(maxsize=prefetch_count)
+        self.finished = threading.Event()
+        self.stop_requested = threading.Event()
+        self.failure: Exception | None = None
+
+    def run(self) -> None:
+        try:
+            while not self.stop_requested.is_set() and not self.worker.expired.is_set():
+                remaining = self.worker.remaining_tasks()
+                count = (
+                    self.prefetch_count
+                    if remaining is None
+                    else min(self.prefetch_count, remaining)
+                )
+                if count < 1:
+                    break
+                try:
+                    batch = self.backend.acquire(
+                        *self.worker.queues,
+                        count=count,
+                        timeout=TASK_WAIT_TIMEOUT,
+                        worker=self.name,
+                    )
+                except Empty, TimeoutError:
+                    if (
+                        self.worker.exit_empty
+                        or self.worker.shutdown_requested.is_set()
+                    ):
+                        break
+                else:
+                    # joe: buffered tasks are leased at fetch time, so buffer dwell
+                    # counts against lease_ttl; renew leases if dwell ever matters
+                    for task_result in batch:
+                        if not self.buffer(task_result):
+                            break
+                    if self.worker.shutdown_requested.is_set():
+                        break
+        except Exception as exception:
+            self.failure = exception
+            logger.exception("Task prefetcher '%s' failed", self.name)
+            raise
+        finally:
+            self.finished.set()
+
+    def buffer(self, task_result: TaskResult) -> bool:
+        """Buffer one task result; return False when the prefetcher must stop."""
+        while not self.stop_requested.is_set():
+            try:
+                self.task_buffer.put(
+                    task_result, timeout=TASK_WAIT_TIMEOUT.total_seconds()
+                )
+            except Full:
+                continue
+            return True
+        return False
+
+
 class WorkerThread(threading.Thread):
-    """Single worker thread consuming tasks from the process queue."""
+    """Single worker thread consuming tasks from the process prefetch buffer."""
 
     def __init__(
         self,
@@ -278,16 +383,14 @@ def __init__(
         self.backend = backend
 
     def run(self) -> None:
-        """Start consuming tasks for this thread."""
-        while self.worker.expired is None or not self.worker.expired.is_set():
+        prefetcher = self.worker.prefetcher
+        while True:
             try:
-                task_result = self.backend.acquire(
-                    *self.worker.queues,
-                    timeout=datetime.timedelta(seconds=1),
-                    worker=self.name,
+                task_result = prefetcher.task_buffer.get(
+                    timeout=TASK_WAIT_TIMEOUT.total_seconds()
                 )
-            except Empty, TimeoutError:
-                if self.worker.shutdown_requested.is_set() or self.worker.exit_empty:
+            except Empty:
+                if prefetcher.finished.is_set():
                     return
                 continue
 
diff --git a/threadmill/management/commands/threadmill.py b/threadmill/management/commands/threadmill.py
index 2c39524..4618344 100644
--- a/threadmill/management/commands/threadmill.py
+++ b/threadmill/management/commands/threadmill.py
@@ -68,6 +68,16 @@ def add_arguments(self, parser):
             default=0,
             help="Maximum random jitter to add to the max-tasks value by randint(0, max_tasks_jitter).",
         )
+        parser.add_argument(
+            "--prefetch-count",
+            type=int,
+            default=None,
+            help=(
+                "Number of tasks to prefetch per worker process."
+                " Defaults to 4 × the number of threads; use 1 to disable batching."
+                " Buffered tasks still run after a worker reaches its max-tasks limit."
+            ),
+        )
         parser.add_argument(
             "--poll-interval",
             type=float,
@@ -103,6 +113,7 @@ def handle(
         threads,
         max_tasks,
         max_tasks_jitter,
+        prefetch_count,
         poll_interval,
         poll_max_interval,
         exit_empty,
@@ -136,12 +147,15 @@ def handle(
             )
         except (TypeError, ValueError) as e:
             raise CommandError(f"Invalid log format: {log_format!r}") from e
+        if prefetch_count is not None and prefetch_count < 1:
+            raise CommandError(f"Invalid prefetch count: {prefetch_count!r}")
         exe = TaskExecutor(
             backend=backend,
             workers=workers,
             threads=threads,
             max_tasks=max_tasks,
             max_tasks_jitter=max_tasks_jitter,
+            prefetch_count=prefetch_count,
             poll_interval=datetime.timedelta(seconds=poll_interval),
             poll_max_interval=datetime.timedelta(seconds=poll_max_interval),
             exit_empty=exit_empty,

From e0654ba5eb0b9934a8a2283329a1afd5588493b5 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Thu, 1 Oct 2026 13:15:34 +0200
Subject: [PATCH 02/26] Add dramatiq to the benchmark and refresh the chart

Measure dramatiq beside celery and the task backends on the same trivial
echo task, pinned to one process, one worker thread and a prefetch of one
message so it matches the others.

The threadmill queues grew to 60,000 tasks and dramatiq keeps 5,000: the
fixed cost of a cold worker start and stop is quantized to about a second,
which swamped the marginal drain of a shallower queue and made the
prefetch comparison unmeasurable.

- per-queue depth on QueueUnderTest, recorded in the benchmark extra info
- fail loudly instead of writing a negative throughput when a drain is
  degenerate (process mean below start mean)
- chart height follows the row count, and its subtitle reports the depths
  actually measured
- the chart plots threadmill at its default configuration only; the
  no-prefetch run stays a benchmark diagnostic, since on a local broker
  the two land within a percent of each other
---
 README.md                                |   3 +-
 benchmarks/chart.py                      |  41 ++++++--
 benchmarks/dramatiq_app.py               |  34 +++++++
 benchmarks/test_backends.py              | 123 ++++++++++++++++++++---
 docs/images/backend-comparison-dark.svg  |  25 +++--
 docs/images/backend-comparison-light.svg |  25 +++--
 pyproject.toml                           |   1 +
 7 files changed, 206 insertions(+), 46 deletions(-)
 create mode 100644 benchmarks/dramatiq_app.py

diff --git a/README.md b/README.md
index 3424e15..971e6e2 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-dark.svg">
     <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
-    <img alt="Tasks per second with one worker: threadmill 4,989, celery 2,363, django-tasks-db 2,179, django-tasks-redis 1,712." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
+    <img alt="Tasks per second with one worker: threadmill 5,965, celery 2,243, django-tasks-db 2,055, django-tasks-redis 1,458, dramatiq 105." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
   </picture>
 </p>
 
@@ -88,6 +88,7 @@ uv run manage.py threadmill worker --workers 4 --threads 2
 Each worker process runs one fetcher thread that reserves a batch of tasks in a single broker round-trip.
 Worker threads drain that buffer, which amortizes broker latency across fast tasks.
 A full buffer blocks the fetcher until a worker thread frees a slot.
+The gain is largest when the broker is a network hop away; against a local broker a task spends most of its time executing and acknowledging rather than waiting to be fetched.
 
 Set the batch size with `--prefetch-count`.
 It defaults to four times the thread count and applies per process, not per thread.
diff --git a/benchmarks/chart.py b/benchmarks/chart.py
index 2c12c66..ecc8a65 100644
--- a/benchmarks/chart.py
+++ b/benchmarks/chart.py
@@ -16,12 +16,16 @@
 DARK_THEME_PATH = IMAGE_DIRECTORY / "backend-comparison-dark.svg"
 
 WIDTH = 900
-HEIGHT = 332
 
 LABEL_X = 180
 PLOT_X0 = 200
 PLOT_WIDTH = 560
 
+FIRST_ROW_CENTER = 110
+ROW_HEIGHT = 40
+FOOTNOTE_GAP = 38
+BOTTOM_PADDING = 64
+
 FONT = (
     'system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif'
 )
@@ -73,6 +77,14 @@ class Theme:
     accent_bar="#6366f1",
 )
 
+DIAGNOSTIC_QUEUES = frozenset({"threadmill (no prefetch)"})
+"""Queues the benchmark measures but the chart leaves out.
+
+The harness runs threadmill twice to bracket its prefetch buffer, and on a local
+broker the two land within a percent of each other, so plotting both would rank
+them on measurement noise.
+"""
+
 
 @dataclasses.dataclass(frozen=True, kw_only=True, slots=True)
 class QueueResult:
@@ -104,6 +116,8 @@ def read_results(json_path: pathlib.Path) -> list[QueueResult]:
     ]
     results = []
     for queue_name in queue_names:
+        if queue_name in DIAGNOSTIC_QUEUES:
+            continue
         process = means[("test_process_queue__benchmark", queue_name)]
         start = means[("test_start_worker__benchmark", queue_name)]
         enqueue = means[("test_enqueue__benchmark", queue_name)]
@@ -139,20 +153,30 @@ def build_chart(results: list[QueueResult], theme: Theme) -> str:
     """Return the chart as an SVG document drawn in the given theme."""
     fastest = results[0]
     scale = PLOT_WIDTH / fastest.throughput
-    row_centers = [110 + index * 40 for index in range(len(results))]
+    minimum_task_count = min(result.task_count for result in results)
+    maximum_task_count = max(result.task_count for result in results)
+    depth_label = (
+        f"{minimum_task_count:,}"
+        if minimum_task_count == maximum_task_count
+        else f"{minimum_task_count:,}–{maximum_task_count:,}"
+    )
+    row_centers = [
+        FIRST_ROW_CENTER + index * ROW_HEIGHT for index in range(len(results))
+    ]
+    height = row_centers[-1] + FOOTNOTE_GAP + BOTTOM_PADDING
 
     parts = [
-        f'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {WIDTH} {HEIGHT}" '
-        f'width="{WIDTH}" height="{HEIGHT}" role="img" '
+        f'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {WIDTH} {height}" '
+        f'width="{WIDTH}" height="{height}" role="img" '
         f'aria-label="{describe(results)}.">',
         "<style>svg{max-width:100%;height:auto}</style>",
-        f'<rect x="0.5" y="0.5" width="{WIDTH - 1}" height="{HEIGHT - 1}" rx="14" '
+        f'<rect x="0.5" y="0.5" width="{WIDTH - 1}" height="{height - 1}" rx="14" '
         f'fill="{theme.canvas}" stroke="{theme.border}"/>',
         text(28, 46, "Queue throughput", theme=theme, size=19, weight=700),
         text(
             28,
             68,
-            f"{fastest.task_count:,} trivial tasks per queue · one worker process, "
+            f"{depth_label} trivial tasks per queue · one worker process, "
             "one thread · higher is better",
             theme=theme,
             size=12.5,
@@ -195,8 +219,9 @@ def build_chart(results: list[QueueResult], theme: Theme) -> str:
     parts.append(
         text(
             28,
-            row_centers[-1] + 38,
-            "One message in flight per worker — no queue reads ahead.",
+            row_centers[-1] + FOOTNOTE_GAP,
+            "Threadmill reads a batch ahead; every other worker reads one message "
+            "at a time.",
             theme=theme,
             size=11.5,
             fill=theme.faint,
diff --git a/benchmarks/dramatiq_app.py b/benchmarks/dramatiq_app.py
new file mode 100644
index 0000000..8df5228
--- /dev/null
+++ b/benchmarks/dramatiq_app.py
@@ -0,0 +1,34 @@
+"""The dramatiq broker the comparison benchmarks hand tasks to.
+
+The dramatiq worker CLI imports this module without setting up Django, so it
+must not import Django or any Django application.
+"""
+
+import os
+
+import dramatiq
+import redis
+from dramatiq.brokers.redis import RedisBroker
+
+REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
+
+PROCESSED_KEY = "benchmark:processed"
+"""Key the sentinel task increments once every earlier task was processed."""
+
+client = redis.Redis.from_url(REDIS_URL)
+
+# joe: the echo value is discarded, one warning per task; celery stores its
+# results in Redis, so add the Results middleware to compare result storage too.
+redis_broker = RedisBroker(url=REDIS_URL)
+
+
+@dramatiq.actor(broker=redis_broker)
+def dramatiq_echo(value):
+    """Return the given value."""
+    return value
+
+
+@dramatiq.actor(broker=redis_broker)
+def dramatiq_mark_processed():
+    """Record that every earlier task in the queue has been processed."""
+    client.incr(PROCESSED_KEY)
diff --git a/benchmarks/test_backends.py b/benchmarks/test_backends.py
index d38b48f..e23acb7 100644
--- a/benchmarks/test_backends.py
+++ b/benchmarks/test_backends.py
@@ -7,23 +7,34 @@
 - ``test_start_worker__benchmark``: time for a worker to start and process one queued task.
 - ``test_process_queue__benchmark``: time for a worker to process a full queue.
 
+The processing benchmark queues 20,000 tasks, except for dramatiq, which queues
+5,000: dramatiq reads one message at a time and backs off before polling again,
+about 9.5 ms per task, so the default depth would need minutes to drain. The
+fixed cost of a cold worker start and stop is quantized to about a second, so a
+deep queue is what makes the marginal drain per task — the number the chart
+plots — measurable.
+
 Both worker benchmarks include the worker's fixed start cost, and the queues
 that exit on their own include their stop cost too. Subtract
 ``test_start_worker__benchmark`` from ``test_process_queue__benchmark`` and divide
-the queue depth by the difference to get the marginal throughput of a busy queue.
+that queue's task count by the difference to get the marginal throughput of a
+busy queue.
 
 Threadmill is measured twice, with its default prefetch buffer and with batching
-disabled, so the prefetch cost can be subtracted from both worker benchmarks.
+disabled, so the prefetch cost can be subtracted from both worker benchmarks. Its
+queues are deeper than the others because its marginal drain is only seconds long,
+which would otherwise sit inside the one-second quantization of the fixed cost.
 Threadmill, django-tasks-db and django-tasks-redis run one worker process that
-drains a queue and exits. Celery has no such mode, so the benchmark queues a
-sentinel task last and waits for it to be processed. That wait is what proves the
-queue was drained. Its worker is stopped after the measurement, because a graceful
-shutdown takes seconds and would dominate a short drain.
+drains a queue and exits. Celery and dramatiq have no such mode, so the benchmark
+queues a sentinel task last and waits for it to be processed. That wait is what
+proves the queue was drained. Their workers are stopped after the measurement,
+because a graceful shutdown takes seconds and would dominate a short drain.
 """
 
 import collections.abc
 import dataclasses
 import io
+import os
 import subprocess
 import sys
 import tempfile
@@ -48,13 +59,14 @@
     celery_echo,
     celery_mark_processed,
 )
+from benchmarks.dramatiq_app import dramatiq_echo, dramatiq_mark_processed
 from tests.testapp.tasks import echo
 
 ENQUEUE_ITERATIONS = 500
 """Tasks enqueued within one enqueue benchmark round."""
 
-QUEUE_DEPTH = 5000
-"""Tasks queued before one processing benchmark round."""
+QUEUE_DEPTH = 20_000
+"""Default tasks queued before one processing benchmark round."""
 
 CELERY_WORKER = (
     sys.executable,
@@ -76,6 +88,23 @@
 the task handler state it expects.
 """
 
+DRAMATIQ_WORKER = (
+    sys.executable,
+    "-m",
+    "dramatiq",
+    "benchmarks.dramatiq_app:redis_broker",
+    "--processes",
+    "1",
+    "--threads",
+    "1",
+)
+"""dramatiq worker running as one process with one thread, reading one message at a time.
+
+One thread alone still reads two messages ahead, and the CLI has no flag for it,
+so the worker environment carries ``dramatiq_queue_prefetch=1``; dramatiq reads
+that variable while it imports its worker module, before the CLI runs.
+"""
+
 WORKER_STOP_TIMEOUT_SECONDS = 20
 """Seconds to wait for a worker process to stop after SIGTERM."""
 
@@ -107,6 +136,9 @@ class QueueUnderTest:
     verify: collections.abc.Callable[[TaskResult | None], None] | None = None
     """Assert that the drained queue was processed."""
 
+    task_count: int = QUEUE_DEPTH
+    """Tasks enqueued before one processing round; slow queues queue fewer."""
+
 
 def drain_with_threadmill_worker() -> None:
     """Process every queued task with a single threadmill worker process."""
@@ -166,7 +198,19 @@ def drain_with_celery_worker() -> None:
     drain_with_subprocess_worker(CELERY_WORKER)
 
 
-def drain_with_subprocess_worker(argv: collections.abc.Sequence[str]) -> None:
+def drain_with_dramatiq_worker() -> None:
+    """Process every queued task with a single-process, single-thread dramatiq worker."""
+    dramatiq_mark_processed.send()
+    drain_with_subprocess_worker(
+        DRAMATIQ_WORKER,
+        env={**os.environ, "dramatiq_queue_prefetch": "1"},
+    )
+
+
+def drain_with_subprocess_worker(
+    argv: collections.abc.Sequence[str],
+    env: collections.abc.Mapping[str, str] | None = None,
+) -> None:
     """Run a worker CLI until the sentinel task queued last was processed."""
     client = redis.Redis.from_url(REDIS_URL)
     client.delete(PROCESSED_KEY)
@@ -176,6 +220,7 @@ def drain_with_subprocess_worker(argv: collections.abc.Sequence[str]) -> None:
         argv,
         stdout=log,
         stderr=subprocess.STDOUT,
+        env=env,
     )
     running_workers.append(WorkerProcess(process=process, log=log))
     wait_until_processed(client, process, log)
@@ -252,24 +297,45 @@ def enqueue_celery_tasks(count: int) -> None:
         celery_echo.delay(index)
 
 
+def enqueue_dramatiq_tasks(count: int) -> None:
+    """Accept `count` echo tasks on the dramatiq queue."""
+    for index in range(count):
+        dramatiq_echo.send(index)
+
+
 def django_task_backend(
     name: str,
     alias: str,
     drain: collections.abc.Callable[[], None] | None = None,
+    task_count: int = QUEUE_DEPTH,
 ) -> QueueUnderTest:
     """Build a comparison entry for a Django task backend."""
     enqueue, verify = django_task_enqueuer(alias)
-    return QueueUnderTest(name=name, enqueue=enqueue, drain=drain, verify=verify)
+    return QueueUnderTest(
+        name=name, enqueue=enqueue, drain=drain, verify=verify, task_count=task_count
+    )
 
 
+THREADMILL_TASK_COUNT = 60_000
+"""Tasks threadmill queues, so its marginal drain outruns the one-second fixed cost.
+
+Threadmill drains a queue in about 3 seconds per 20,000 tasks, and the fixed cost
+of a cold worker start and stop is quantized to about a second, so a shallower
+queue would leave the prefetch comparison within that step.
+"""
+
 WORKER_QUEUES = (
     django_task_backend(
-        "threadmill", DEFAULT_TASK_BACKEND_ALIAS, drain_with_threadmill_worker
+        "threadmill",
+        DEFAULT_TASK_BACKEND_ALIAS,
+        drain_with_threadmill_worker,
+        task_count=THREADMILL_TASK_COUNT,
     ),
     django_task_backend(
         "threadmill (no prefetch)",
         DEFAULT_TASK_BACKEND_ALIAS,
         drain_with_threadmill_worker_no_prefetch,
+        task_count=THREADMILL_TASK_COUNT,
     ),
     django_task_backend(
         "django-tasks-db", "django-tasks-db", drain_with_django_tasks_db_worker
@@ -282,6 +348,12 @@ def django_task_backend(
         enqueue=enqueue_celery_tasks,
         drain=drain_with_celery_worker,
     ),
+    QueueUnderTest(
+        name="dramatiq",
+        enqueue=enqueue_dramatiq_tasks,
+        drain=drain_with_dramatiq_worker,
+        task_count=5_000,  # about 9.5 ms per task; see the module docstring
+    ),
 )
 """Queues that ship a worker to process queued tasks."""
 
@@ -301,6 +373,13 @@ def identify_backend(queue: QueueUnderTest) -> str:
 running_workers: list[WorkerProcess] = []
 """Workers started by the running benchmark, stopped once it is measured."""
 
+start_seconds: dict[str, float] = {}
+"""Mean seconds each queue needs to start a worker and drain one task.
+
+``TestWorkerStart`` records it and ``TestQueueProcessing`` compares its own drain
+against it, because a drain shorter than the start cost is not measurable.
+"""
+
 
 @pytest.fixture(autouse=True)
 def stop_workers(empty_queues):
@@ -322,7 +401,13 @@ def empty_queues():
     client = task_backends[DEFAULT_TASK_BACKEND_ALIAS].client
 
     def delete_queued_tasks() -> None:
-        for key_pattern in ("threadmill:*", "django_tasks:*", "celery*", "_kombu*"):
+        for key_pattern in (
+            "threadmill:*",
+            "django_tasks:*",
+            "celery*",
+            "dramatiq:*",
+            "_kombu*",
+        ):
             if keys := client.keys(key_pattern):
                 client.delete(*keys)
         client.delete(PROCESSED_KEY)
@@ -370,6 +455,7 @@ def test_start_worker__benchmark(self, benchmark, queue_under_test, empty_queues
             iterations=1,
             warmup_rounds=0,
         )
+        start_seconds[queue_under_test.name] = benchmark.stats["mean"]
 
 
 class TestQueueProcessing:
@@ -379,9 +465,9 @@ class TestQueueProcessing:
     @pytest.mark.django_db(transaction=True)
     @pytest.mark.parametrize("queue_under_test", WORKER_QUEUES, ids=identify_backend)
     def test_process_queue__benchmark(self, benchmark, queue_under_test, empty_queues):
-        """Benchmark the time to process QUEUE_DEPTH queued tasks."""
-        enqueued_task_result = queue_under_test.enqueue(QUEUE_DEPTH)
-        benchmark.extra_info["tasks"] = QUEUE_DEPTH
+        """Benchmark the time to process a queue at its own depth."""
+        enqueued_task_result = queue_under_test.enqueue(queue_under_test.task_count)
+        benchmark.extra_info["tasks"] = queue_under_test.task_count
 
         benchmark.pedantic(
             queue_under_test.drain,
@@ -390,5 +476,12 @@ def test_process_queue__benchmark(self, benchmark, queue_under_test, empty_queue
             warmup_rounds=0,
         )
 
+        process_mean = benchmark.stats["mean"]
+        if process_mean <= (start_mean := start_seconds[queue_under_test.name]):
+            pytest.fail(
+                f"{queue_under_test.name}: drain mean {process_mean:.4f}s does not "
+                f"exceed start mean {start_mean:.4f}s; the run is not measurable"
+            )
+
         if queue_under_test.verify:
             queue_under_test.verify(enqueued_task_result)
diff --git a/docs/images/backend-comparison-dark.svg b/docs/images/backend-comparison-dark.svg
index 9ab93ac..534ca1e 100644
--- a/docs/images/backend-comparison-dark.svg
+++ b/docs/images/backend-comparison-dark.svg
@@ -1,19 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 332" width="900" height="332" role="img" aria-label="Tasks per second with one worker: threadmill 4,989, celery 2,363, django-tasks-db 2,179, django-tasks-redis 1,712.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,965, celery 2,243, django-tasks-db 2,055, django-tasks-redis 1,458, dramatiq 105.">
 <style>svg{max-width:100%;height:auto}</style>
-<rect x="0.5" y="0.5" width="899" height="331" rx="14" fill="#0d1117" stroke="#30363d"/>
+<rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#0d1117" stroke="#30363d"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#e6edf3" text-anchor="start">Queue throughput</text>
-<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#8b949e" text-anchor="start">5,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
+<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#8b949e" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
 <text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#e6edf3" text-anchor="end">threadmill</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#6366f1" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">4,989/s</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">5,965/s</text>
 <text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">celery</text>
-<rect x="200" y="140" width="265.25" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="475.254" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,363/s</text>
+<rect x="200" y="140" width="210.53" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="420.535" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,243/s</text>
 <text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="180" width="244.60" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="454.595" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,179/s</text>
+<rect x="200" y="180" width="192.91" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="402.908" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,055/s</text>
 <text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-redis</text>
-<rect x="200" y="220" width="192.10" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="402.103" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">1,712/s</text>
-<text x="28" y="268" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#6e7681" text-anchor="start">One message in flight per worker — no queue reads ahead.</text>
+<rect x="200" y="220" width="136.84" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="346.839" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">1,458/s</text>
+<text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">dramatiq</text>
+<rect x="200" y="260" width="9.90" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="219.903" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">105/s</text>
+<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#6e7681" text-anchor="start">Threadmill reads a batch ahead; every other worker reads one message at a time.</text>
 </svg>
diff --git a/docs/images/backend-comparison-light.svg b/docs/images/backend-comparison-light.svg
index 98d82d5..968febb 100644
--- a/docs/images/backend-comparison-light.svg
+++ b/docs/images/backend-comparison-light.svg
@@ -1,19 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 332" width="900" height="332" role="img" aria-label="Tasks per second with one worker: threadmill 4,989, celery 2,363, django-tasks-db 2,179, django-tasks-redis 1,712.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,965, celery 2,243, django-tasks-db 2,055, django-tasks-redis 1,458, dramatiq 105.">
 <style>svg{max-width:100%;height:auto}</style>
-<rect x="0.5" y="0.5" width="899" height="331" rx="14" fill="#ffffff" stroke="#e5e7eb"/>
+<rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#ffffff" stroke="#e5e7eb"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#111827" text-anchor="start">Queue throughput</text>
-<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#6b7280" text-anchor="start">5,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
+<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#6b7280" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
 <text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#111827" text-anchor="end">threadmill</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#4f46e5" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">4,989/s</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">5,965/s</text>
 <text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">celery</text>
-<rect x="200" y="140" width="265.25" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="475.254" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,363/s</text>
+<rect x="200" y="140" width="210.53" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="420.535" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,243/s</text>
 <text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="180" width="244.60" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="454.595" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,179/s</text>
+<rect x="200" y="180" width="192.91" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="402.908" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,055/s</text>
 <text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-redis</text>
-<rect x="200" y="220" width="192.10" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="402.103" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">1,712/s</text>
-<text x="28" y="268" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#9ca3af" text-anchor="start">One message in flight per worker — no queue reads ahead.</text>
+<rect x="200" y="220" width="136.84" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="346.839" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">1,458/s</text>
+<text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">dramatiq</text>
+<rect x="200" y="260" width="9.90" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="219.903" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">105/s</text>
+<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#9ca3af" text-anchor="start">Threadmill reads a batch ahead; every other worker reads one message at a time.</text>
 </svg>
diff --git a/pyproject.toml b/pyproject.toml
index 340e138..e5ad3e8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,6 +52,7 @@ test = [
   "celery",
   "django-tasks-db",
   "django-tasks-redis",
+  "dramatiq[redis]>=2.2.1",
   "pytest",
   "pytest-asyncio",
   "pytest-benchmark",

From dc8665f02666d89881b4c91c299ceb7a12a49e12 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Thu, 1 Oct 2026 13:51:28 +0200
Subject: [PATCH 03/26] Measure each queue at its own default read-ahead

dramatiq's Redis consumer polls rather than blocks: with its read-ahead
window full it sleeps compute_backoff(0), a jittered 5-10 ms, so pinning
it to one message in flight cost a sleep between every task and made the
chart read 105/s instead of its real figure. Unpinned it reads 211/s,
which is five times celery rather than twenty.

Celery's pinned prefetch multiplier goes with it: its consumer blocks on
Redis, so the pin measured nothing (2,145/s pinned against 2,080/s at its
default over 20,000 tasks).

The methodology is now one worker process and one thread, each queue at
its own default read-ahead, and the chart says so.
---
 README.md                                |  2 +-
 benchmarks/chart.py                      |  7 +++-
 benchmarks/test_backends.py              | 48 ++++++++++++------------
 docs/images/backend-comparison-dark.svg  | 22 +++++------
 docs/images/backend-comparison-light.svg | 22 +++++------
 5 files changed, 52 insertions(+), 49 deletions(-)

diff --git a/README.md b/README.md
index 971e6e2..a0925b5 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-dark.svg">
     <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
-    <img alt="Tasks per second with one worker: threadmill 5,965, celery 2,243, django-tasks-db 2,055, django-tasks-redis 1,458, dramatiq 105." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
+    <img alt="Tasks per second with one worker: threadmill 5,436, celery 2,251, django-tasks-db 2,035, django-tasks-redis 1,406, dramatiq 211." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
   </picture>
 </p>
 
diff --git a/benchmarks/chart.py b/benchmarks/chart.py
index ecc8a65..6712a92 100644
--- a/benchmarks/chart.py
+++ b/benchmarks/chart.py
@@ -220,8 +220,11 @@ def build_chart(results: list[QueueResult], theme: Theme) -> str:
         text(
             28,
             row_centers[-1] + FOOTNOTE_GAP,
-            "Threadmill reads a batch ahead; every other worker reads one message "
-            "at a time.",
+            # joe: width checked by hand (right edge 836.7 of 900 at 11.5px); add a
+            # width guard if the canvas width or the font stack changes.
+            "One worker process and one thread; each queue at its own default "
+            "read-ahead. dramatiq's Redis broker polls, so two in flight cost a "
+            "backoff per poll.",
             theme=theme,
             size=11.5,
             fill=theme.faint,
diff --git a/benchmarks/test_backends.py b/benchmarks/test_backends.py
index e23acb7..1ca3a49 100644
--- a/benchmarks/test_backends.py
+++ b/benchmarks/test_backends.py
@@ -8,11 +8,11 @@
 - ``test_process_queue__benchmark``: time for a worker to process a full queue.
 
 The processing benchmark queues 20,000 tasks, except for dramatiq, which queues
-5,000: dramatiq reads one message at a time and backs off before polling again,
-about 9.5 ms per task, so the default depth would need minutes to drain. The
-fixed cost of a cold worker start and stop is quantized to about a second, so a
-deep queue is what makes the marginal drain per task — the number the chart
-plots — measurable.
+5,000: dramatiq's Redis consumer polls rather than blocks, so its default
+read-ahead of two backs off about 4.5 ms per task, and the default depth would
+need minutes to drain. The fixed cost of a cold worker start and stop is
+quantized to about a second, so a deep queue is what makes the marginal drain per
+task — the number the chart plots — measurable.
 
 Both worker benchmarks include the worker's fixed start cost, and the queues
 that exit on their own include their stop cost too. Subtract
@@ -20,6 +20,10 @@
 that queue's task count by the difference to get the marginal throughput of a
 busy queue.
 
+Every queue runs one worker process and one thread at its own default read-ahead,
+because a read-ahead pinned below a polling consumer's default measures the pin
+instead of the queue.
+
 Threadmill is measured twice, with its default prefetch buffer and with batching
 disabled, so the prefetch cost can be subtracted from both worker benchmarks. Its
 queues are deeper than the others because its marginal drain is only seconds long,
@@ -34,7 +38,6 @@
 import collections.abc
 import dataclasses
 import io
-import os
 import subprocess
 import sys
 import tempfile
@@ -76,16 +79,17 @@
     "benchmarks.celery_app:celery_app",
     "worker",
     "--pool=solo",
-    "--prefetch-multiplier=1",
     "--loglevel=WARNING",
     "--without-gossip",
     "--without-mingle",
     "--without-heartbeat",
 )
-"""Celery worker running as one process with one thread, reading one message at a time.
+"""Celery worker running as one process with one thread, at its default read-ahead.
 
 The default prefork pool crashes on CPython 3.14, where the pool child loses
-the task handler state it expects.
+the task handler state it expects, so the worker runs on the solo pool. Celery's
+Redis consumer blocks while its queue is empty, so its default prefetch
+multiplier adds no sleep per message.
 """
 
 DRAMATIQ_WORKER = (
@@ -98,11 +102,14 @@
     "--threads",
     "1",
 )
-"""dramatiq worker running as one process with one thread, reading one message at a time.
-
-One thread alone still reads two messages ahead, and the CLI has no flag for it,
-so the worker environment carries ``dramatiq_queue_prefetch=1``; dramatiq reads
-that variable while it imports its worker module, before the CLI runs.
+"""dramatiq worker running as one process with one thread, at its default read-ahead.
+
+The Redis broker polls rather than blocks: its consumer fetches only while fewer
+than its read-ahead of messages are unacked and, with that window full, sleeps a
+jittered 5-10 ms backoff before polling again. An unset
+``dramatiq_queue_prefetch`` gives two messages per thread, so that sleep lands
+between poll cycles; pinning it to one would land a sleep between every message
+and measure the backoff instead of the queue.
 """
 
 WORKER_STOP_TIMEOUT_SECONDS = 20
@@ -201,16 +208,10 @@ def drain_with_celery_worker() -> None:
 def drain_with_dramatiq_worker() -> None:
     """Process every queued task with a single-process, single-thread dramatiq worker."""
     dramatiq_mark_processed.send()
-    drain_with_subprocess_worker(
-        DRAMATIQ_WORKER,
-        env={**os.environ, "dramatiq_queue_prefetch": "1"},
-    )
+    drain_with_subprocess_worker(DRAMATIQ_WORKER)
 
 
-def drain_with_subprocess_worker(
-    argv: collections.abc.Sequence[str],
-    env: collections.abc.Mapping[str, str] | None = None,
-) -> None:
+def drain_with_subprocess_worker(argv: collections.abc.Sequence[str]) -> None:
     """Run a worker CLI until the sentinel task queued last was processed."""
     client = redis.Redis.from_url(REDIS_URL)
     client.delete(PROCESSED_KEY)
@@ -220,7 +221,6 @@ def drain_with_subprocess_worker(
         argv,
         stdout=log,
         stderr=subprocess.STDOUT,
-        env=env,
     )
     running_workers.append(WorkerProcess(process=process, log=log))
     wait_until_processed(client, process, log)
@@ -352,7 +352,7 @@ def django_task_backend(
         name="dramatiq",
         enqueue=enqueue_dramatiq_tasks,
         drain=drain_with_dramatiq_worker,
-        task_count=5_000,  # about 9.5 ms per task; see the module docstring
+        task_count=5_000,  # about 4.5 ms per task; see the module docstring
     ),
 )
 """Queues that ship a worker to process queued tasks."""
diff --git a/docs/images/backend-comparison-dark.svg b/docs/images/backend-comparison-dark.svg
index 534ca1e..6471791 100644
--- a/docs/images/backend-comparison-dark.svg
+++ b/docs/images/backend-comparison-dark.svg
@@ -1,22 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,965, celery 2,243, django-tasks-db 2,055, django-tasks-redis 1,458, dramatiq 105.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,436, celery 2,251, django-tasks-db 2,035, django-tasks-redis 1,406, dramatiq 211.">
 <style>svg{max-width:100%;height:auto}</style>
 <rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#0d1117" stroke="#30363d"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#e6edf3" text-anchor="start">Queue throughput</text>
 <text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#8b949e" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
 <text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#e6edf3" text-anchor="end">threadmill</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#6366f1" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">5,965/s</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">5,436/s</text>
 <text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">celery</text>
-<rect x="200" y="140" width="210.53" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="420.535" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,243/s</text>
+<rect x="200" y="140" width="231.87" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="441.875" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,251/s</text>
 <text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="180" width="192.91" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="402.908" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,055/s</text>
+<rect x="200" y="180" width="209.61" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="419.613" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,035/s</text>
 <text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-redis</text>
-<rect x="200" y="220" width="136.84" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="346.839" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">1,458/s</text>
+<rect x="200" y="220" width="144.87" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="354.871" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">1,406/s</text>
 <text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">dramatiq</text>
-<rect x="200" y="260" width="9.90" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="219.903" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">105/s</text>
-<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#6e7681" text-anchor="start">Threadmill reads a batch ahead; every other worker reads one message at a time.</text>
+<rect x="200" y="260" width="21.74" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="231.745" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">211/s</text>
+<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#6e7681" text-anchor="start">One worker process and one thread; each queue at its own default read-ahead. dramatiq's Redis broker polls, so two in flight cost a backoff per poll.</text>
 </svg>
diff --git a/docs/images/backend-comparison-light.svg b/docs/images/backend-comparison-light.svg
index 968febb..63f7db7 100644
--- a/docs/images/backend-comparison-light.svg
+++ b/docs/images/backend-comparison-light.svg
@@ -1,22 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,965, celery 2,243, django-tasks-db 2,055, django-tasks-redis 1,458, dramatiq 105.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,436, celery 2,251, django-tasks-db 2,035, django-tasks-redis 1,406, dramatiq 211.">
 <style>svg{max-width:100%;height:auto}</style>
 <rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#ffffff" stroke="#e5e7eb"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#111827" text-anchor="start">Queue throughput</text>
 <text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#6b7280" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
 <text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#111827" text-anchor="end">threadmill</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#4f46e5" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">5,965/s</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">5,436/s</text>
 <text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">celery</text>
-<rect x="200" y="140" width="210.53" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="420.535" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,243/s</text>
+<rect x="200" y="140" width="231.87" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="441.875" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,251/s</text>
 <text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="180" width="192.91" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="402.908" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,055/s</text>
+<rect x="200" y="180" width="209.61" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="419.613" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,035/s</text>
 <text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-redis</text>
-<rect x="200" y="220" width="136.84" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="346.839" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">1,458/s</text>
+<rect x="200" y="220" width="144.87" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="354.871" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">1,406/s</text>
 <text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">dramatiq</text>
-<rect x="200" y="260" width="9.90" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="219.903" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">105/s</text>
-<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#9ca3af" text-anchor="start">Threadmill reads a batch ahead; every other worker reads one message at a time.</text>
+<rect x="200" y="260" width="21.74" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="231.745" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">211/s</text>
+<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#9ca3af" text-anchor="start">One worker process and one thread; each queue at its own default read-ahead. dramatiq's Redis broker polls, so two in flight cost a backoff per poll.</text>
 </svg>

From 71aa0895143fc3ca49bd223a01edf2a841f5f9c1 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Thu, 1 Oct 2026 14:10:39 +0200
Subject: [PATCH 04/26] Measure every queue at the same read-ahead

Threadmill reserves four tasks per worker by default, so every queue that
can be told now reads four ahead: celery through --prefetch-multiplier=4
and dramatiq through dramatiq_queue_prefetch=4. The prefetch buffer is no
longer a comparison advantage.

dramatiq reads 420 tasks/s at that rate, twice the 211 it scored at its
own default of two; its polling consumer still pays a jittered 5-10 ms
backoff roughly once per four messages, which the chart footnote states.

The two Django backends keep reading one message at a time because their
shipped workers expose no read-ahead setting: db_worker claims one task
per loop and run_redis_tasks hardcodes max_messages=1. Patching a third
party worker would measure the patch rather than the library, so they stay
as they ship and the footnote says so.
---
 README.md                                |  2 +-
 benchmarks/chart.py                      |  8 ++---
 benchmarks/test_backends.py              | 46 ++++++++++++++----------
 docs/images/backend-comparison-dark.svg  | 22 ++++++------
 docs/images/backend-comparison-light.svg | 22 ++++++------
 5 files changed, 55 insertions(+), 45 deletions(-)

diff --git a/README.md b/README.md
index a0925b5..430454f 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-dark.svg">
     <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
-    <img alt="Tasks per second with one worker: threadmill 5,436, celery 2,251, django-tasks-db 2,035, django-tasks-redis 1,406, dramatiq 211." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
+    <img alt="Tasks per second with one worker: threadmill 5,430, celery 2,240, django-tasks-db 2,033, django-tasks-redis 1,472, dramatiq 420." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
   </picture>
 </p>
 
diff --git a/benchmarks/chart.py b/benchmarks/chart.py
index 6712a92..f21d0f3 100644
--- a/benchmarks/chart.py
+++ b/benchmarks/chart.py
@@ -220,11 +220,11 @@ def build_chart(results: list[QueueResult], theme: Theme) -> str:
         text(
             28,
             row_centers[-1] + FOOTNOTE_GAP,
-            # joe: width checked by hand (right edge 836.7 of 900 at 11.5px); add a
+            # joe: width checked by hand (right edge 873.7 of 900 at 11.5px); add a
             # width guard if the canvas width or the font stack changes.
-            "One worker process and one thread; each queue at its own default "
-            "read-ahead. dramatiq's Redis broker polls, so two in flight cost a "
-            "backoff per poll.",
+            "One worker process and one thread; each reads four messages ahead. "
+            "The Django backends read one at a time - their workers have no "
+            "read-ahead setting.",
             theme=theme,
             size=11.5,
             fill=theme.faint,
diff --git a/benchmarks/test_backends.py b/benchmarks/test_backends.py
index 1ca3a49..5385c07 100644
--- a/benchmarks/test_backends.py
+++ b/benchmarks/test_backends.py
@@ -8,9 +8,9 @@
 - ``test_process_queue__benchmark``: time for a worker to process a full queue.
 
 The processing benchmark queues 20,000 tasks, except for dramatiq, which queues
-5,000: dramatiq's Redis consumer polls rather than blocks, so its default
-read-ahead of two backs off about 4.5 ms per task, and the default depth would
-need minutes to drain. The fixed cost of a cold worker start and stop is
+5,000: dramatiq's Redis consumer polls rather than blocks, so its four-message
+read-ahead costs about 2.3 ms per task in backoff, and the full depth would take
+about 46 seconds to drain. The fixed cost of a cold worker start and stop is
 quantized to about a second, so a deep queue is what makes the marginal drain per
 task — the number the chart plots — measurable.
 
@@ -20,9 +20,11 @@
 that queue's task count by the difference to get the marginal throughput of a
 busy queue.
 
-Every queue runs one worker process and one thread at its own default read-ahead,
-because a read-ahead pinned below a polling consumer's default measures the pin
-instead of the queue.
+Every queue runs one worker process and one thread and reads four messages ahead,
+threadmill's default prefetch count of four per thread, so no queue is measured
+with a read-ahead advantage. The two Django backends are the exception: their
+shipped workers read one task at a time and expose no read-ahead setting.
+dramatiq pays a jittered 5-10 ms poll backoff roughly once per four messages.
 
 Threadmill is measured twice, with its default prefetch buffer and with batching
 disabled, so the prefetch cost can be subtracted from both worker benchmarks. Its
@@ -38,6 +40,7 @@
 import collections.abc
 import dataclasses
 import io
+import os
 import subprocess
 import sys
 import tempfile
@@ -79,17 +82,19 @@
     "benchmarks.celery_app:celery_app",
     "worker",
     "--pool=solo",
+    "--prefetch-multiplier=4",
     "--loglevel=WARNING",
     "--without-gossip",
     "--without-mingle",
     "--without-heartbeat",
 )
-"""Celery worker running as one process with one thread, at its default read-ahead.
+"""Celery worker running as one process with one thread, four messages ahead.
 
 The default prefork pool crashes on CPython 3.14, where the pool child loses
-the task handler state it expects, so the worker runs on the solo pool. Celery's
-Redis consumer blocks while its queue is empty, so its default prefetch
-multiplier adds no sleep per message.
+the task handler state it expects, so the worker runs on the solo pool. With one
+concurrent task, ``--prefetch-multiplier=4`` sets the prefetch count to four, the
+benchmark rate. Celery's Redis consumer blocks while its queue is empty, so the
+prefetch adds no sleep per message.
 """
 
 DRAMATIQ_WORKER = (
@@ -102,14 +107,12 @@
     "--threads",
     "1",
 )
-"""dramatiq worker running as one process with one thread, at its default read-ahead.
+"""dramatiq worker running as one process with one thread, four messages ahead.
 
 The Redis broker polls rather than blocks: its consumer fetches only while fewer
 than its read-ahead of messages are unacked and, with that window full, sleeps a
-jittered 5-10 ms backoff before polling again. An unset
-``dramatiq_queue_prefetch`` gives two messages per thread, so that sleep lands
-between poll cycles; pinning it to one would land a sleep between every message
-and measure the backoff instead of the queue.
+jittered 5-10 ms backoff before polling again. The CLI has no read-ahead flag, so
+the worker environment carries ``dramatiq_queue_prefetch=4``, the benchmark rate.
 """
 
 WORKER_STOP_TIMEOUT_SECONDS = 20
@@ -208,10 +211,16 @@ def drain_with_celery_worker() -> None:
 def drain_with_dramatiq_worker() -> None:
     """Process every queued task with a single-process, single-thread dramatiq worker."""
     dramatiq_mark_processed.send()
-    drain_with_subprocess_worker(DRAMATIQ_WORKER)
+    drain_with_subprocess_worker(
+        DRAMATIQ_WORKER,
+        env={**os.environ, "dramatiq_queue_prefetch": "4"},
+    )
 
 
-def drain_with_subprocess_worker(argv: collections.abc.Sequence[str]) -> None:
+def drain_with_subprocess_worker(
+    argv: collections.abc.Sequence[str],
+    env: collections.abc.Mapping[str, str] | None = None,
+) -> None:
     """Run a worker CLI until the sentinel task queued last was processed."""
     client = redis.Redis.from_url(REDIS_URL)
     client.delete(PROCESSED_KEY)
@@ -219,6 +228,7 @@ def drain_with_subprocess_worker(argv: collections.abc.Sequence[str]) -> None:
     # The command is a fixed worker CLI, never caller input.
     process = subprocess.Popen(  # noqa: S603
         argv,
+        env=env,
         stdout=log,
         stderr=subprocess.STDOUT,
     )
@@ -352,7 +362,7 @@ def django_task_backend(
         name="dramatiq",
         enqueue=enqueue_dramatiq_tasks,
         drain=drain_with_dramatiq_worker,
-        task_count=5_000,  # about 4.5 ms per task; see the module docstring
+        task_count=5_000,  # about 2.3 ms per task, a drain near 12 s; see the module docstring
     ),
 )
 """Queues that ship a worker to process queued tasks."""
diff --git a/docs/images/backend-comparison-dark.svg b/docs/images/backend-comparison-dark.svg
index 6471791..c3e4437 100644
--- a/docs/images/backend-comparison-dark.svg
+++ b/docs/images/backend-comparison-dark.svg
@@ -1,22 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,436, celery 2,251, django-tasks-db 2,035, django-tasks-redis 1,406, dramatiq 211.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,430, celery 2,240, django-tasks-db 2,033, django-tasks-redis 1,472, dramatiq 420.">
 <style>svg{max-width:100%;height:auto}</style>
 <rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#0d1117" stroke="#30363d"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#e6edf3" text-anchor="start">Queue throughput</text>
 <text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#8b949e" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
 <text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#e6edf3" text-anchor="end">threadmill</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#6366f1" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">5,436/s</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">5,430/s</text>
 <text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">celery</text>
-<rect x="200" y="140" width="231.87" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="441.875" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,251/s</text>
+<rect x="200" y="140" width="231.03" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="441.033" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,240/s</text>
 <text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="180" width="209.61" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="419.613" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,035/s</text>
+<rect x="200" y="180" width="209.70" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="419.705" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,033/s</text>
 <text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-redis</text>
-<rect x="200" y="220" width="144.87" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="354.871" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">1,406/s</text>
+<rect x="200" y="220" width="151.79" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="361.795" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">1,472/s</text>
 <text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">dramatiq</text>
-<rect x="200" y="260" width="21.74" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="231.745" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">211/s</text>
-<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#6e7681" text-anchor="start">One worker process and one thread; each queue at its own default read-ahead. dramatiq's Redis broker polls, so two in flight cost a backoff per poll.</text>
+<rect x="200" y="260" width="43.35" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="253.347" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">420/s</text>
+<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#6e7681" text-anchor="start">One worker process and one thread; each reads four messages ahead. The Django backends read one at a time - their workers have no read-ahead setting.</text>
 </svg>
diff --git a/docs/images/backend-comparison-light.svg b/docs/images/backend-comparison-light.svg
index 63f7db7..949ab45 100644
--- a/docs/images/backend-comparison-light.svg
+++ b/docs/images/backend-comparison-light.svg
@@ -1,22 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,436, celery 2,251, django-tasks-db 2,035, django-tasks-redis 1,406, dramatiq 211.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,430, celery 2,240, django-tasks-db 2,033, django-tasks-redis 1,472, dramatiq 420.">
 <style>svg{max-width:100%;height:auto}</style>
 <rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#ffffff" stroke="#e5e7eb"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#111827" text-anchor="start">Queue throughput</text>
 <text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#6b7280" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
 <text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#111827" text-anchor="end">threadmill</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#4f46e5" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">5,436/s</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">5,430/s</text>
 <text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">celery</text>
-<rect x="200" y="140" width="231.87" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="441.875" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,251/s</text>
+<rect x="200" y="140" width="231.03" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="441.033" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,240/s</text>
 <text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="180" width="209.61" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="419.613" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,035/s</text>
+<rect x="200" y="180" width="209.70" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="419.705" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,033/s</text>
 <text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-redis</text>
-<rect x="200" y="220" width="144.87" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="354.871" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">1,406/s</text>
+<rect x="200" y="220" width="151.79" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="361.795" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">1,472/s</text>
 <text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">dramatiq</text>
-<rect x="200" y="260" width="21.74" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="231.745" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">211/s</text>
-<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#9ca3af" text-anchor="start">One worker process and one thread; each queue at its own default read-ahead. dramatiq's Redis broker polls, so two in flight cost a backoff per poll.</text>
+<rect x="200" y="260" width="43.35" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="253.347" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">420/s</text>
+<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#9ca3af" text-anchor="start">One worker process and one thread; each reads four messages ahead. The Django backends read one at a time - their workers have no read-ahead setting.</text>
 </svg>

From bab323faa850eda5173f65f10b16e378d7c3646d Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Thu, 1 Oct 2026 14:32:47 +0200
Subject: [PATCH 05/26] Read 128 messages ahead in the queue comparison

An earlier harness in this repository ran dramatiq with a prefetch window
of 128 and it led the field; a later commit dropped it because its
single-threaded consumer sleeps a poll backoff between messages and could
not be compared fairly against queues that read one message at a time.

That window is the fix, not the problem. A shared rate of four still left
dramatiq penalised: its jittered 5-10 ms backoff lands once per window, so
the cost per task is inverse in the depth and a shallow window measures the
poll, not the queue. Every queue that can be told now reads 128 ahead,
which amortises that backoff to about 0.06 ms per task and leaves celery
and threadmill blocking on Redis as they always did.

dramatiq also takes its Results middleware back, which stores results the
way celery does, so neither queue is measured discarding the value. Its
depth rises to 20,000 like the other third-party queues, because at this
rate a 5,000 task drain fits inside the one-second quantisation of the
fixed start cost.

dramatiq leads at 6,975 tasks/s against threadmill's 5,437, close to the
7,676 the earlier harness measured, and threadmill's own prefetch buffer
is worth about nine percent over reading one at a time.
---
 README.md                                |  2 +-
 benchmarks/chart.py                      |  6 +--
 benchmarks/dramatiq_app.py               | 11 +++--
 benchmarks/test_backends.py              | 53 ++++++++++++++----------
 docs/images/backend-comparison-dark.svg  | 34 +++++++--------
 docs/images/backend-comparison-light.svg | 34 +++++++--------
 6 files changed, 77 insertions(+), 63 deletions(-)

diff --git a/README.md b/README.md
index 430454f..ad7a43a 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-dark.svg">
     <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
-    <img alt="Tasks per second with one worker: threadmill 5,430, celery 2,240, django-tasks-db 2,033, django-tasks-redis 1,472, dramatiq 420." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
+    <img alt="Tasks per second with one worker: dramatiq 6,975, threadmill 5,437, celery 2,080, django-tasks-db 1,981, django-tasks-redis 1,391." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
   </picture>
 </p>
 
diff --git a/benchmarks/chart.py b/benchmarks/chart.py
index f21d0f3..9e54f62 100644
--- a/benchmarks/chart.py
+++ b/benchmarks/chart.py
@@ -220,10 +220,10 @@ def build_chart(results: list[QueueResult], theme: Theme) -> str:
         text(
             28,
             row_centers[-1] + FOOTNOTE_GAP,
-            # joe: width checked by hand (right edge 873.7 of 900 at 11.5px); add a
+            # joe: width checked by hand (right edge 856.1 of 900 at 11.5px); add a
             # width guard if the canvas width or the font stack changes.
-            "One worker process and one thread; each reads four messages ahead. "
-            "The Django backends read one at a time - their workers have no "
+            "One process and one thread each, reading 128 messages ahead. "
+            "The Django backends read one at a time - their workers expose no "
             "read-ahead setting.",
             theme=theme,
             size=11.5,
diff --git a/benchmarks/dramatiq_app.py b/benchmarks/dramatiq_app.py
index 8df5228..9a864c6 100644
--- a/benchmarks/dramatiq_app.py
+++ b/benchmarks/dramatiq_app.py
@@ -1,7 +1,9 @@
 """The dramatiq broker the comparison benchmarks hand tasks to.
 
 The dramatiq worker CLI imports this module without setting up Django, so it
-must not import Django or any Django application.
+must not import Django or any Django application. The broker carries the Results
+middleware so the echo actor stores its return value in Redis, the way the Celery
+app's result backend does.
 """
 
 import os
@@ -9,6 +11,8 @@
 import dramatiq
 import redis
 from dramatiq.brokers.redis import RedisBroker
+from dramatiq.results import Results
+from dramatiq.results.backends.redis import RedisBackend
 
 REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
 
@@ -17,12 +21,11 @@
 
 client = redis.Redis.from_url(REDIS_URL)
 
-# joe: the echo value is discarded, one warning per task; celery stores its
-# results in Redis, so add the Results middleware to compare result storage too.
 redis_broker = RedisBroker(url=REDIS_URL)
+redis_broker.add_middleware(Results(backend=RedisBackend(client=client)))
 
 
-@dramatiq.actor(broker=redis_broker)
+@dramatiq.actor(broker=redis_broker, store_results=True)
 def dramatiq_echo(value):
     """Return the given value."""
     return value
diff --git a/benchmarks/test_backends.py b/benchmarks/test_backends.py
index 5385c07..ee14625 100644
--- a/benchmarks/test_backends.py
+++ b/benchmarks/test_backends.py
@@ -7,12 +7,9 @@
 - ``test_start_worker__benchmark``: time for a worker to start and process one queued task.
 - ``test_process_queue__benchmark``: time for a worker to process a full queue.
 
-The processing benchmark queues 20,000 tasks, except for dramatiq, which queues
-5,000: dramatiq's Redis consumer polls rather than blocks, so its four-message
-read-ahead costs about 2.3 ms per task in backoff, and the full depth would take
-about 46 seconds to drain. The fixed cost of a cold worker start and stop is
-quantized to about a second, so a deep queue is what makes the marginal drain per
-task — the number the chart plots — measurable.
+The processing benchmark queues 20,000 tasks per queue. The fixed cost of a cold
+worker start and stop is quantized to about a second, so a deep queue is what
+makes the marginal drain per task — the number the chart plots — measurable.
 
 Both worker benchmarks include the worker's fixed start cost, and the queues
 that exit on their own include their stop cost too. Subtract
@@ -20,14 +17,14 @@
 that queue's task count by the difference to get the marginal throughput of a
 busy queue.
 
-Every queue runs one worker process and one thread and reads four messages ahead,
-threadmill's default prefetch count of four per thread, so no queue is measured
-with a read-ahead advantage. The two Django backends are the exception: their
-shipped workers read one task at a time and expose no read-ahead setting.
-dramatiq pays a jittered 5-10 ms poll backoff roughly once per four messages.
+Every queue runs one worker process and one thread and reads ``READ_AHEAD`` (128)
+messages ahead, so the numbers rank the queues rather than their polling
+strategies: dramatiq's poll backoff is amortized over the window, while celery and
+threadmill block on an empty queue. The two Django backends are the exception:
+their shipped workers read one task at a time and expose no read-ahead setting.
 
-Threadmill is measured twice, with its default prefetch buffer and with batching
-disabled, so the prefetch cost can be subtracted from both worker benchmarks. Its
+Threadmill is measured twice, reading 128 messages ahead and reading one at a
+time, so the read-ahead cost can be subtracted from both worker benchmarks. Its
 queues are deeper than the others because its marginal drain is only seconds long,
 which would otherwise sit inside the one-second quantization of the fixed cost.
 Threadmill, django-tasks-db and django-tasks-redis run one worker process that
@@ -74,6 +71,16 @@
 QUEUE_DEPTH = 20_000
 """Default tasks queued before one processing benchmark round."""
 
+READ_AHEAD = 128
+"""Messages each worker reads ahead, where its queue has such a setting.
+
+Deep enough that each consumer's wait mechanism stops deciding the ranking:
+dramatiq's Redis consumer polls, and its jittered 5-10 ms backoff costs about
+0.06 ms per task over 128 messages, while celery and threadmill block on an empty
+queue. The Django backends expose no read-ahead setting and read one task at a
+time.
+"""
+
 CELERY_WORKER = (
     sys.executable,
     "-m",
@@ -82,17 +89,17 @@
     "benchmarks.celery_app:celery_app",
     "worker",
     "--pool=solo",
-    "--prefetch-multiplier=4",
+    f"--prefetch-multiplier={READ_AHEAD}",
     "--loglevel=WARNING",
     "--without-gossip",
     "--without-mingle",
     "--without-heartbeat",
 )
-"""Celery worker running as one process with one thread, four messages ahead.
+"""Celery worker running as one process with one thread, ``READ_AHEAD`` messages ahead.
 
 The default prefork pool crashes on CPython 3.14, where the pool child loses
 the task handler state it expects, so the worker runs on the solo pool. With one
-concurrent task, ``--prefetch-multiplier=4`` sets the prefetch count to four, the
+concurrent task, ``--prefetch-multiplier=128`` sets the prefetch count to 128, the
 benchmark rate. Celery's Redis consumer blocks while its queue is empty, so the
 prefetch adds no sleep per message.
 """
@@ -107,12 +114,12 @@
     "--threads",
     "1",
 )
-"""dramatiq worker running as one process with one thread, four messages ahead.
+"""dramatiq worker running as one process with one thread, ``READ_AHEAD`` messages ahead.
 
 The Redis broker polls rather than blocks: its consumer fetches only while fewer
 than its read-ahead of messages are unacked and, with that window full, sleeps a
 jittered 5-10 ms backoff before polling again. The CLI has no read-ahead flag, so
-the worker environment carries ``dramatiq_queue_prefetch=4``, the benchmark rate.
+the worker environment carries ``dramatiq_queue_prefetch=128``, the benchmark rate.
 """
 
 WORKER_STOP_TIMEOUT_SECONDS = 20
@@ -158,13 +165,18 @@ def drain_with_threadmill_worker() -> None:
         backend=DEFAULT_TASK_BACKEND_ALIAS,
         queues=[DEFAULT_TASK_QUEUE_NAME],
         workers=1,
+        prefetch_count=READ_AHEAD,
         exit_empty=True,
         verbosity=0,
     )
 
 
 def drain_with_threadmill_worker_no_prefetch() -> None:
-    """Process every queued task with one threadmill worker without batching."""
+    """Process every queued task with one threadmill worker reading one at a time.
+
+    The no-prefetch ablation of the entry above, so the read-ahead cost can be
+    subtracted.
+    """
     call_command(
         "threadmill",
         "worker",
@@ -213,7 +225,7 @@ def drain_with_dramatiq_worker() -> None:
     dramatiq_mark_processed.send()
     drain_with_subprocess_worker(
         DRAMATIQ_WORKER,
-        env={**os.environ, "dramatiq_queue_prefetch": "4"},
+        env={**os.environ, "dramatiq_queue_prefetch": str(READ_AHEAD)},
     )
 
 
@@ -362,7 +374,6 @@ def django_task_backend(
         name="dramatiq",
         enqueue=enqueue_dramatiq_tasks,
         drain=drain_with_dramatiq_worker,
-        task_count=5_000,  # about 2.3 ms per task, a drain near 12 s; see the module docstring
     ),
 )
 """Queues that ship a worker to process queued tasks."""
diff --git a/docs/images/backend-comparison-dark.svg b/docs/images/backend-comparison-dark.svg
index c3e4437..e86f3e4 100644
--- a/docs/images/backend-comparison-dark.svg
+++ b/docs/images/backend-comparison-dark.svg
@@ -1,22 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,430, celery 2,240, django-tasks-db 2,033, django-tasks-redis 1,472, dramatiq 420.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: dramatiq 6,975, threadmill 5,437, celery 2,080, django-tasks-db 1,981, django-tasks-redis 1,391.">
 <style>svg{max-width:100%;height:auto}</style>
 <rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#0d1117" stroke="#30363d"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#e6edf3" text-anchor="start">Queue throughput</text>
-<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#8b949e" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
-<text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#e6edf3" text-anchor="end">threadmill</text>
+<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#8b949e" text-anchor="start">20,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
+<text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#e6edf3" text-anchor="end">dramatiq</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#6366f1" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">5,430/s</text>
-<text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">celery</text>
-<rect x="200" y="140" width="231.03" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="441.033" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,240/s</text>
-<text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="180" width="209.70" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="419.705" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,033/s</text>
-<text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-redis</text>
-<rect x="200" y="220" width="151.79" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="361.795" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">1,472/s</text>
-<text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">dramatiq</text>
-<rect x="200" y="260" width="43.35" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="253.347" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">420/s</text>
-<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#6e7681" text-anchor="start">One worker process and one thread; each reads four messages ahead. The Django backends read one at a time - their workers have no read-ahead setting.</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">6,975/s</text>
+<text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">threadmill</text>
+<rect x="200" y="140" width="436.55" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="646.55" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">5,437/s</text>
+<text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">celery</text>
+<rect x="200" y="180" width="166.99" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="376.994" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,080/s</text>
+<text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-db</text>
+<rect x="200" y="220" width="159.02" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="369.017" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">1,981/s</text>
+<text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-redis</text>
+<rect x="200" y="260" width="111.66" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="321.658" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">1,391/s</text>
+<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#6e7681" text-anchor="start">One process and one thread each, reading 128 messages ahead. The Django backends read one at a time - their workers expose no read-ahead setting.</text>
 </svg>
diff --git a/docs/images/backend-comparison-light.svg b/docs/images/backend-comparison-light.svg
index 949ab45..6f68748 100644
--- a/docs/images/backend-comparison-light.svg
+++ b/docs/images/backend-comparison-light.svg
@@ -1,22 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,430, celery 2,240, django-tasks-db 2,033, django-tasks-redis 1,472, dramatiq 420.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: dramatiq 6,975, threadmill 5,437, celery 2,080, django-tasks-db 1,981, django-tasks-redis 1,391.">
 <style>svg{max-width:100%;height:auto}</style>
 <rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#ffffff" stroke="#e5e7eb"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#111827" text-anchor="start">Queue throughput</text>
-<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#6b7280" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
-<text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#111827" text-anchor="end">threadmill</text>
+<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#6b7280" text-anchor="start">20,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
+<text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#111827" text-anchor="end">dramatiq</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#4f46e5" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">5,430/s</text>
-<text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">celery</text>
-<rect x="200" y="140" width="231.03" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="441.033" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,240/s</text>
-<text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="180" width="209.70" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="419.705" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,033/s</text>
-<text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-redis</text>
-<rect x="200" y="220" width="151.79" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="361.795" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">1,472/s</text>
-<text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">dramatiq</text>
-<rect x="200" y="260" width="43.35" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="253.347" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">420/s</text>
-<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#9ca3af" text-anchor="start">One worker process and one thread; each reads four messages ahead. The Django backends read one at a time - their workers have no read-ahead setting.</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">6,975/s</text>
+<text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">threadmill</text>
+<rect x="200" y="140" width="436.55" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="646.55" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">5,437/s</text>
+<text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">celery</text>
+<rect x="200" y="180" width="166.99" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="376.994" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,080/s</text>
+<text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-db</text>
+<rect x="200" y="220" width="159.02" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="369.017" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">1,981/s</text>
+<text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-redis</text>
+<rect x="200" y="260" width="111.66" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="321.658" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">1,391/s</text>
+<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#9ca3af" text-anchor="start">One process and one thread each, reading 128 messages ahead. The Django backends read one at a time - their workers expose no read-ahead setting.</text>
 </svg>

From 6a13e134ae97bb24730c1e9362957bad2beaf9a1 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Thu, 1 Oct 2026 15:40:45 +0200
Subject: [PATCH 06/26] Delete the dramatiq results the benchmark now writes

Storing results for parity with celery left keys the cleanup could not
match: with the default result backend the key is a bare md5 hex, so the
harness's dramatiq:* pattern missed it and the results stayed in Redis
until their TTL expired.

Naming the result namespace makes the key greppable and the existing
pattern deletes it, so no new cleanup code is needed.
---
 benchmarks/dramatiq_app.py  | 16 ++++++++++++++--
 benchmarks/test_backends.py |  4 ++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/benchmarks/dramatiq_app.py b/benchmarks/dramatiq_app.py
index 9a864c6..b3f474a 100644
--- a/benchmarks/dramatiq_app.py
+++ b/benchmarks/dramatiq_app.py
@@ -3,7 +3,9 @@
 The dramatiq worker CLI imports this module without setting up Django, so it
 must not import Django or any Django application. The broker carries the Results
 middleware so the echo actor stores its return value in Redis, the way the Celery
-app's result backend does.
+app's result backend does. The result backend names its keys
+``dramatiq:results:<queue>:<actor>:<message_id>`` rather than the default bare MD5
+hash, so the benchmark cleanup's ``dramatiq:*`` pattern deletes them.
 """
 
 import os
@@ -22,7 +24,17 @@
 client = redis.Redis.from_url(REDIS_URL)
 
 redis_broker = RedisBroker(url=REDIS_URL)
-redis_broker.add_middleware(Results(backend=RedisBackend(client=client)))
+# A greppable namespace the benchmark cleanup's "dramatiq:*" pattern matches;
+# the backend's default key is a bare MD5 hash no pattern can name.
+redis_broker.add_middleware(
+    Results(
+        backend=RedisBackend(
+            client=client,
+            namespace="dramatiq:results",
+            use_namespace_prefix_keys=True,
+        ),
+    ),
+)
 
 
 @dramatiq.actor(broker=redis_broker, store_results=True)
diff --git a/benchmarks/test_backends.py b/benchmarks/test_backends.py
index ee14625..5b305b9 100644
--- a/benchmarks/test_backends.py
+++ b/benchmarks/test_backends.py
@@ -418,7 +418,7 @@ def stop_workers(empty_queues):
 
 @pytest.fixture
 def empty_queues():
-    """Delete queued tasks from every compared queue before and after a benchmark."""
+    """Delete queued tasks and stored results from every compared queue before and after a benchmark."""
     client = task_backends[DEFAULT_TASK_BACKEND_ALIAS].client
 
     def delete_queued_tasks() -> None:
@@ -426,7 +426,7 @@ def delete_queued_tasks() -> None:
             "threadmill:*",
             "django_tasks:*",
             "celery*",
-            "dramatiq:*",
+            "dramatiq:*",  # broker keys and the dramatiq:results:* results
             "_kombu*",
         ):
             if keys := client.keys(key_pattern):

From f0192a8db247693428719aa03f7e7fbb75c92ea4 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 20:52:45 +0200
Subject: [PATCH 07/26] Refresh the chart after the merge

The merged worker leases tasks beside their payload instead of rewriting
it, which roughly doubled threadmill's drain: 11,973 tasks/s against
5,437 before the merge, and it now leads the comparison. The prefetch
buffer is worth about a fifth over reading one at a time, 11,973 against
9,972, where the same merge left the ablation at its pre-merge rate.

The chart now carries django-tasks-rq, which main added in place of
django-tasks-redis, and drops the dependency the replacement orphaned.
Its queue drains at 90 tasks/s, so it measures 5,000 tasks.
---
 README.md                                |  5 ++--
 benchmarks/chart.py                      |  2 +-
 benchmarks/test_backends.py              | 19 ++++++++-------
 docs/images/backend-comparison-dark.svg  | 30 ++++++++++++------------
 docs/images/backend-comparison-light.svg | 30 ++++++++++++------------
 pyproject.toml                           |  1 -
 tests/test_command.py                    | 23 ++++++++++--------
 7 files changed, 58 insertions(+), 52 deletions(-)

diff --git a/README.md b/README.md
index 3cda142..18db044 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-dark.svg">
     <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
-    <img alt="Tasks per second with one worker: dramatiq 6,975, threadmill 5,437, celery 2,080, django-tasks-db 1,981, django-tasks-redis 1,391." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
+    <img alt="Tasks per second with one worker: threadmill 11,973, dramatiq 7,331, celery 2,307, django-tasks-db 2,080, django-tasks-rq 90." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
   </picture>
 </p>
 
@@ -88,7 +88,8 @@ uv run manage.py threadmill worker --workers 4 --threads 2
 Each worker process runs one fetcher thread that reserves a batch of tasks in a single broker round-trip.
 Worker threads drain that buffer, which amortizes broker latency across fast tasks.
 A full buffer blocks the fetcher until a worker thread frees a slot.
-The gain is largest when the broker is a network hop away; against a local broker a task spends most of its time executing and acknowledging rather than waiting to be fetched.
+The gain is largest when the broker is a network hop away.
+Against a local broker the buffer is worth about a fifth of the worker's throughput in the queue comparison.
 
 Set the batch size with `--prefetch-count`.
 It defaults to four times the thread count and applies per process, not per thread.
diff --git a/benchmarks/chart.py b/benchmarks/chart.py
index 8a065ad..d42813c 100644
--- a/benchmarks/chart.py
+++ b/benchmarks/chart.py
@@ -224,7 +224,7 @@ def build_chart(results: list[QueueResult], theme: Theme) -> str:
             # joe: width checked by hand (right edge 856.1 of 900 at 11.5px); add a
             # width guard if the canvas width or the font stack changes.
             "One process and one thread each. Threadmill, celery and dramatiq read "
-            "128 ahead; django-tasks-db, -redis and -rq read one message at a time.",
+            "128 ahead; django-tasks-db and -rq read one message at a time.",
             theme=theme,
             size=11.5,
             fill=theme.faint,
diff --git a/benchmarks/test_backends.py b/benchmarks/test_backends.py
index 72d2945..62201d5 100644
--- a/benchmarks/test_backends.py
+++ b/benchmarks/test_backends.py
@@ -26,16 +26,16 @@
 1,983 at 16 and 5,658 at 64, where the sleep stops setting the rate. Threadmill
 reads the same rate into its prefetch buffer.
 
-django-tasks-db and django-tasks-redis read one task at a time because their
-shipped workers expose no read-ahead setting. django-tasks-rq forks a work horse
-per job, so its drain includes that fork.
+django-tasks-db reads one task at a time, because its shipped worker exposes no
+read-ahead setting. django-tasks-rq forks a work horse per job, so its drain
+includes that fork and it reads one task at a time too.
 
 Threadmill is measured twice, reading 128 messages ahead and reading one at a
 time, so the read-ahead cost can be subtracted from both worker benchmarks. Its
 queues are deeper than the others because its marginal drain is only seconds long,
 which would otherwise sit inside the one-second quantization of the fixed cost.
-Threadmill, django-tasks-db, django-tasks-redis and django-tasks-rq run one worker
-process that drains a queue and exits. Celery and dramatiq have no such mode, so
+Threadmill, django-tasks-db and django-tasks-rq run one worker process that
+drains a queue and exits. Celery and dramatiq have no such mode, so
 the benchmark queues a sentinel task last and waits for it to be processed. That
 wait is what proves the queue was drained. Their workers are stopped after the
 measurement, because a graceful shutdown takes seconds and would dominate a short
@@ -88,8 +88,8 @@
 fills, so a shallow window measures that backoff rather than the queue: the same
 worker drains about 451 tasks/s at two messages, 1,983 at 16 and 5,658 at 64,
 where the sleep stops setting the rate. At 128 the backoff costs about 0.06 ms
-per task. django-tasks-db and django-tasks-redis expose no read-ahead setting and
-read one task at a time; django-tasks-rq forks a work horse per job.
+per task. django-tasks-db reads one task at a time, and django-tasks-rq forks a
+work horse per job, so neither can be told to read ahead.
 """
 
 CELERY_WORKER = (
@@ -382,7 +382,10 @@ def django_task_backend(
         "django-tasks-db", "django-tasks-db", drain_with_django_tasks_db_worker
     ),
     django_task_backend(
-        "django-tasks-rq", "django-tasks-rq", drain_with_django_tasks_rq_worker
+        "django-tasks-rq",
+        "django-tasks-rq",
+        drain_with_django_tasks_rq_worker,
+        task_count=5_000,  # about 80 tasks/s; see the module docstring
     ),
     QueueUnderTest(
         name="celery",
diff --git a/docs/images/backend-comparison-dark.svg b/docs/images/backend-comparison-dark.svg
index 488f20a..c69f4c9 100644
--- a/docs/images/backend-comparison-dark.svg
+++ b/docs/images/backend-comparison-dark.svg
@@ -1,22 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,023, celery 1,951, django-tasks-db 1,942, dramatiq 669, django-tasks-rq 80.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 11,973, dramatiq 7,331, celery 2,307, django-tasks-db 2,080, django-tasks-rq 90.">
 <style>svg{max-width:100%;height:auto}</style>
 <rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#0d1117" stroke="#30363d"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#e6edf3" text-anchor="start">Queue throughput</text>
-<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#8b949e" text-anchor="start">5,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
+<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#8b949e" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
 <text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#e6edf3" text-anchor="end">threadmill</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#6366f1" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">5,023/s</text>
-<text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">celery</text>
-<rect x="200" y="140" width="217.46" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="427.465" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">1,951/s</text>
-<text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="180" width="216.50" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="426.499" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">1,942/s</text>
-<text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">dramatiq</text>
-<rect x="200" y="220" width="74.58" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="284.583" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">669/s</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">11,973/s</text>
+<text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">dramatiq</text>
+<rect x="200" y="140" width="342.90" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="552.899" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">7,331/s</text>
+<text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">celery</text>
+<rect x="200" y="180" width="107.91" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="317.91" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,307/s</text>
+<text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-db</text>
+<rect x="200" y="220" width="97.30" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="307.295" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,080/s</text>
 <text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-rq</text>
-<rect x="200" y="260" width="8.91" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="218.905" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">80/s</text>
-<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#6e7681" text-anchor="start">threadmill, django-tasks-db and django-tasks-rq read one message at a time; celery and dramatiq four.</text>
+<rect x="200" y="260" width="4.23" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="214.23" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">90/s</text>
+<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#6e7681" text-anchor="start">One process and one thread each. Threadmill, celery and dramatiq read 128 ahead; django-tasks-db and -rq read one message at a time.</text>
 </svg>
diff --git a/docs/images/backend-comparison-light.svg b/docs/images/backend-comparison-light.svg
index cca0b6e..2c1fd71 100644
--- a/docs/images/backend-comparison-light.svg
+++ b/docs/images/backend-comparison-light.svg
@@ -1,22 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 5,023, celery 1,951, django-tasks-db 1,942, dramatiq 669, django-tasks-rq 80.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 11,973, dramatiq 7,331, celery 2,307, django-tasks-db 2,080, django-tasks-rq 90.">
 <style>svg{max-width:100%;height:auto}</style>
 <rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#ffffff" stroke="#e5e7eb"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#111827" text-anchor="start">Queue throughput</text>
-<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#6b7280" text-anchor="start">5,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
+<text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#6b7280" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
 <text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#111827" text-anchor="end">threadmill</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#4f46e5" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">5,023/s</text>
-<text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">celery</text>
-<rect x="200" y="140" width="217.46" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="427.465" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">1,951/s</text>
-<text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="180" width="216.50" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="426.499" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">1,942/s</text>
-<text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">dramatiq</text>
-<rect x="200" y="220" width="74.58" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="284.583" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">669/s</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">11,973/s</text>
+<text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">dramatiq</text>
+<rect x="200" y="140" width="342.90" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="552.899" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">7,331/s</text>
+<text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">celery</text>
+<rect x="200" y="180" width="107.91" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="317.91" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,307/s</text>
+<text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-db</text>
+<rect x="200" y="220" width="97.30" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="307.295" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,080/s</text>
 <text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-rq</text>
-<rect x="200" y="260" width="8.91" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="218.905" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">80/s</text>
-<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#9ca3af" text-anchor="start">threadmill, django-tasks-db and django-tasks-rq read one message at a time; celery and dramatiq four.</text>
+<rect x="200" y="260" width="4.23" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="214.23" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">90/s</text>
+<text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#9ca3af" text-anchor="start">One process and one thread each. Threadmill, celery and dramatiq read 128 ahead; django-tasks-db and -rq read one message at a time.</text>
 </svg>
diff --git a/pyproject.toml b/pyproject.toml
index 4788410..32c7483 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -51,7 +51,6 @@ dev = [
 test = [
   "celery",
   "django-tasks-db",
-  "django-tasks-redis",
   "django-tasks-rq",
   "dramatiq[redis]>=2.2.1",
   "pytest",
diff --git a/tests/test_command.py b/tests/test_command.py
index f229b76..39248d3 100644
--- a/tests/test_command.py
+++ b/tests/test_command.py
@@ -128,22 +128,25 @@ def test_call_command__log_format__empty_string(self):
         )
         assert handler.formatter.format(record) == "Hello world"
 
-    def test_call_command__prefetch_count(self):
+    def test_call_command__prefetch_count(self, caplog):
         """Pass the prefetch count through to the running task executor."""
         enqueued = default_task_backend.enqueue(echo, args=[1])
         original_stream = handler.stream
         parent_log = io.StringIO()
         handler.setStream(parent_log)
         try:
-            call_command(
-                "threadmill",
-                "worker",
-                verbosity=0,
-                workers=1,
-                queues=["default"],
-                exit_empty=True,
-                prefetch_count=7,
-            )
+            # The worker no longer forces a level on the root logger, so the
+            # startup record is only emitted when the caller asks for INFO.
+            with caplog.at_level(logging.INFO):
+                call_command(
+                    "threadmill",
+                    "worker",
+                    verbosity=0,
+                    workers=1,
+                    queues=["default"],
+                    exit_empty=True,
+                    prefetch_count=7,
+                )
         finally:
             handler.setStream(original_stream)
         records = [

From 385a28a986268699e567b5b75b610746459c0dbb Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 21:00:13 +0200
Subject: [PATCH 08/26] Buffer prefetched tasks in a priority queue

The buffer hands out the highest priority task first and keeps fetch order
within a priority, so prefetching weakens the queue's ordering less: a
batch that interleaves queues still dispatches by priority.

It holds a small ordered wrapper rather than bare task results, keyed on
the negated priority, because the backend pops the highest priority first,
and a fetch sequence counter that keeps equal priorities in queue order.
---
 README.md              |  2 +-
 tests/test_executor.py | 39 ++++++++++++++++++++++++++-------------
 threadmill/executor.py | 29 ++++++++++++++++++++++++-----
 3 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index 18db044..2a6b192 100644
--- a/README.md
+++ b/README.md
@@ -98,7 +98,7 @@ A value of `1` disables batching.
 Prefetching has soft limits:
 
 - Tasks are marked `RUNNING` when they are fetched, so the time they spend in the buffer counts against `lease_ttl`.
-- The priority lookahead widens to the buffer size, so ordering is no longer strictly global.
+- A task enqueued after a fetch waits for the buffer to drain before a worker picks it up, though the buffer itself dispatches the highest priority task first and keeps queue order within a priority.
 - `--max-tasks` may overshoot by up to the buffer size, because a prefetched task always runs.
 - `worker_ids` records the fetcher of the process, not the thread that runs the task.
 
diff --git a/tests/test_executor.py b/tests/test_executor.py
index dc18f05..8b9b8ea 100644
--- a/tests/test_executor.py
+++ b/tests/test_executor.py
@@ -109,7 +109,7 @@ def _prefetched_thread(
     worker.lock = threading.Lock()
     worker.expired = threading.Event()
     prefetcher = TaskPrefetcher(worker=worker, backend=backend, prefetch_count=1)
-    prefetcher.task_buffer.put(task_result)
+    prefetcher.buffer(task_result)
     prefetcher.finished.set()
     worker.prefetcher = prefetcher
     return WorkerThread(worker=worker, index=0, backend=backend)
@@ -880,10 +880,9 @@ def test_run__buffers_until_stop_requested(self):
         assert not thread.is_alive()
         assert prefetcher.finished.is_set()
         assert prefetcher.failure is None
-        assert [prefetcher.task_buffer.get_nowait().id for _ in range(2)] == [
-            first.id,
-            second.id,
-        ]
+        assert [
+            prefetcher.task_buffer.get_nowait().task_result.id for _ in range(2)
+        ] == [first.id, second.id]
         assert all(call["count"] == 4 for call in backend.calls)
 
     def test_run__stops_on_expired(self):
@@ -929,7 +928,7 @@ def test_run__caps_count_at_remaining_budget(self):
         prefetcher.run()
 
         assert backend.calls[0]["count"] == 2
-        assert prefetcher.task_buffer.get_nowait().id == task_result.id
+        assert prefetcher.task_buffer.get_nowait().task_result.id == task_result.id
         assert len(backend.calls) == 1
         assert prefetcher.finished.is_set()
 
@@ -943,10 +942,9 @@ def test_run__stops_after_batch_when_shutdown_requested(self):
         prefetcher.worker.shutdown_requested.set()
         prefetcher.run()
 
-        assert [prefetcher.task_buffer.get_nowait().id for _ in range(2)] == [
-            first.id,
-            second.id,
-        ]
+        assert [
+            prefetcher.task_buffer.get_nowait().task_result.id for _ in range(2)
+        ] == [first.id, second.id]
         assert len(backend.calls) == 1
         assert prefetcher.finished.is_set()
 
@@ -989,7 +987,7 @@ def test_buffer__returns_false_when_full_and_stop_requested(self):
         overflow = _task_result(echo, 2)
         backend = StubPrefetchBackend([overflow])
         prefetcher = _make_prefetcher(backend, prefetch_count=1)
-        prefetcher.task_buffer.put(buffered)
+        prefetcher.buffer(buffered)
 
         thread = threading.Thread(target=prefetcher.run)
         thread.start()
@@ -1003,7 +1001,7 @@ def test_buffer__returns_false_when_full_and_stop_requested(self):
         assert prefetcher.finished.is_set()
         assert prefetcher.failure is None
         assert prefetcher.task_buffer.qsize() == 1
-        assert prefetcher.task_buffer.get_nowait().id == buffered.id
+        assert prefetcher.task_buffer.get_nowait().task_result.id == buffered.id
 
     def test_buffer__returns_true_when_space_available(self):
         """Buffer a task result when the queue has room."""
@@ -1011,4 +1009,19 @@ def test_buffer__returns_true_when_space_available(self):
         prefetcher = _make_prefetcher(StubPrefetchBackend())
 
         assert prefetcher.buffer(task_result) is True
-        assert prefetcher.task_buffer.get_nowait().id == task_result.id
+        assert prefetcher.task_buffer.get_nowait().task_result.id == task_result.id
+
+    def test_buffer__dispatches_highest_priority_first(self):
+        """Hand out the highest priority task first and keep fetch order on ties."""
+        low = _task_result(dataclasses.replace(echo, priority=1), 1)
+        high = _task_result(dataclasses.replace(echo, priority=5), 2)
+        middle = _task_result(dataclasses.replace(echo, priority=3), 3)
+        later_high = _task_result(dataclasses.replace(echo, priority=5), 4)
+        prefetcher = _make_prefetcher(StubPrefetchBackend())
+
+        for task_result in (low, high, middle, later_high):
+            assert prefetcher.buffer(task_result) is True
+
+        assert [
+            prefetcher.task_buffer.get_nowait().task_result.id for _ in range(4)
+        ] == [high.id, later_high.id, middle.id, low.id]
diff --git a/threadmill/executor.py b/threadmill/executor.py
index a8cb7f5..3f0db9a 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -3,6 +3,7 @@
 import asyncio
 import dataclasses
 import datetime
+import itertools
 import json
 import logging
 import multiprocessing
@@ -295,6 +296,20 @@ def shutdown(self) -> None:
         self.join()
 
 
+@dataclasses.dataclass(order=True, slots=True)
+class PrefetchedTask:
+    """A buffered task result, awaiting dispatch to a worker thread.
+
+    The buffer holds these rather than bare task results so it can hand out the
+    highest priority task first, and preserve fetch order within a priority.
+    """
+
+    sort_key: tuple[int, int]
+    """Negated task priority, then fetch order; the heap compares on this."""
+
+    task_result: TaskResult = dataclasses.field(compare=False)
+
+
 class TaskPrefetcher(threading.Thread):
     """Single prefetcher thread filling the task buffer of one worker process."""
 
@@ -309,7 +324,10 @@ def __init__(
         self.worker = worker
         self.backend = backend
         self.prefetch_count = prefetch_count
-        self.task_buffer: queue.Queue[TaskResult] = queue.Queue(maxsize=prefetch_count)
+        self.task_buffer: queue.PriorityQueue[PrefetchedTask] = queue.PriorityQueue(
+            maxsize=prefetch_count
+        )
+        self.fetch_sequence = itertools.count()
         self.finished = threading.Event()
         self.stop_requested = threading.Event()
         self.failure: Exception | None = None
@@ -355,11 +373,12 @@ def run(self) -> None:
 
     def buffer(self, task_result: TaskResult) -> bool:
         """Buffer one task result; return False when the prefetcher must stop."""
+        item = PrefetchedTask(
+            (-task_result.task.priority, next(self.fetch_sequence)), task_result
+        )
         while not self.stop_requested.is_set():
             try:
-                self.task_buffer.put(
-                    task_result, timeout=TASK_WAIT_TIMEOUT.total_seconds()
-                )
+                self.task_buffer.put(item, timeout=TASK_WAIT_TIMEOUT.total_seconds())
             except Full:
                 continue
             return True
@@ -387,7 +406,7 @@ def run(self) -> None:
             try:
                 task_result = prefetcher.task_buffer.get(
                     timeout=TASK_WAIT_TIMEOUT.total_seconds()
-                )
+                ).task_result
             except Empty:
                 if prefetcher.finished.is_set():
                     return

From 3c95fd7d4a7472ee0d88e5cf347e3d23d04307db Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 21:03:38 +0200
Subject: [PATCH 09/26] Document the buffer's lease dwell instead of planning a
 fix

A task waits in the buffer while it holds its lease, so a deep buffer needs
a matching lease_ttl. That is a downside the operator sizes with
--prefetch-count, not something the worker should compensate for, so the
note that suggested renewing leases is gone and the trade-off is stated
where the knob is set: the README's soft-limit list and the flag's help.
---
 README.md                                    | 2 +-
 threadmill/executor.py                       | 2 --
 threadmill/management/commands/threadmill.py | 2 ++
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 2a6b192..0f79594 100644
--- a/README.md
+++ b/README.md
@@ -97,7 +97,7 @@ A value of `1` disables batching.
 
 Prefetching has soft limits:
 
-- Tasks are marked `RUNNING` when they are fetched, so the time they spend in the buffer counts against `lease_ttl`.
+- Tasks are leased when they are fetched, so the time a task waits in the buffer counts against `lease_ttl`. Size `--prefetch-count` for your workload: the buffer holds at most that many tasks, and each of them is waiting.
 - A task enqueued after a fetch waits for the buffer to drain before a worker picks it up, though the buffer itself dispatches the highest priority task first and keeps queue order within a priority.
 - `--max-tasks` may overshoot by up to the buffer size, because a prefetched task always runs.
 - `worker_ids` records the fetcher of the process, not the thread that runs the task.
diff --git a/threadmill/executor.py b/threadmill/executor.py
index 3f0db9a..fe6d5a4 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -357,8 +357,6 @@ def run(self) -> None:
                     ):
                         break
                 else:
-                    # joe: buffered tasks are leased at fetch time, so buffer dwell
-                    # counts against lease_ttl; renew leases if dwell ever matters
                     for task_result in batch:
                         if not self.buffer(task_result):
                             break
diff --git a/threadmill/management/commands/threadmill.py b/threadmill/management/commands/threadmill.py
index 4618344..3f1a9aa 100644
--- a/threadmill/management/commands/threadmill.py
+++ b/threadmill/management/commands/threadmill.py
@@ -75,6 +75,8 @@ def add_arguments(self, parser):
             help=(
                 "Number of tasks to prefetch per worker process."
                 " Defaults to 4 × the number of threads; use 1 to disable batching."
+                " Buffered tasks hold their lease while they wait, so a deep buffer"
+                " needs a matching lease_ttl."
                 " Buffered tasks still run after a worker reaches its max-tasks limit."
             ),
         )

From 178b7f2dcd07b5dd5aa372fcf53ff59b7bb607d6 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 21:18:27 +0200
Subject: [PATCH 10/26] Scope the wait timeout and simplify the prefetch
 derivation

The wait that bounds a broker acquire and a buffer get moves from a module
global onto WorkerProcess, which already owns its threads' configuration
and is carried by both the fetcher and the consumers.

The prefetch count now derives in one line: a falsy value or zero takes
four tasks per thread, an explicit value is kept, and a negative value is
floored at one, because a negative count left a worker that fetched
nothing while the pool respawned it. The CLI still rejects a negative
count outright, so an operator gets an error rather than a silently
adjusted configuration.
---
 tests/test_command.py                        | 12 ++----
 tests/test_executor.py                       | 39 ++++++++++++--------
 threadmill/executor.py                       | 20 +++++-----
 threadmill/management/commands/threadmill.py |  2 +-
 4 files changed, 38 insertions(+), 35 deletions(-)

diff --git a/tests/test_command.py b/tests/test_command.py
index 39248d3..a9e4847 100644
--- a/tests/test_command.py
+++ b/tests/test_command.py
@@ -160,18 +160,14 @@ def test_call_command__prefetch_count(self, caplog):
             is TaskResultStatus.SUCCESSFUL
         )
 
-    @pytest.mark.parametrize("prefetch_count", [0, -1])
-    def test_call_command__prefetch_count__raise_command_error(self, prefetch_count):
-        """Reject a prefetch count below one with a CommandError."""
-        with pytest.raises(
-            CommandError,
-            match=re.escape(f"Invalid prefetch count: {prefetch_count!r}"),
-        ):
+    def test_call_command__prefetch_count__raise_command_error(self):
+        """Reject a negative prefetch count with a CommandError."""
+        with pytest.raises(CommandError, match=re.escape("Invalid prefetch count: -1")):
             call_command(
                 "threadmill",
                 "worker",
                 verbosity=0,
-                prefetch_count=prefetch_count,
+                prefetch_count=-1,
             )
 
     def test_call_command__poll_intervals(self):
diff --git a/tests/test_executor.py b/tests/test_executor.py
index 8b9b8ea..bca5303 100644
--- a/tests/test_executor.py
+++ b/tests/test_executor.py
@@ -141,6 +141,11 @@ def acquire(self, *queue_names, count=1, timeout=None, worker=""):
 class StubPrefetchWorker:
     """Minimal worker stub exposing the state the prefetcher reads."""
 
+    @property
+    def task_wait_timeout(self) -> datetime.timedelta:
+        """Follow the worker's timeout, so patching it reaches this stub too."""
+        return WorkerProcess.task_wait_timeout
+
     def __init__(
         self, *, remaining: int | None = None, exit_empty: bool = False
     ) -> None:
@@ -316,10 +321,14 @@ def test_post_init__thread_count_at_least_one(self):
         )
         assert executor.thread_count == 1
 
-    def test_post_init__derives_prefetch_count_from_threads(self):
+    @pytest.mark.parametrize("prefetch_count", [None, 0])
+    def test_post_init__derives_prefetch_count_from_threads(self, prefetch_count):
         """__post_init__ defaults the prefetch count to four tasks per thread."""
         executor = TaskExecutor(
-            backend=default_task_backend, threads=3, queues=("default",)
+            backend=default_task_backend,
+            prefetch_count=prefetch_count,
+            threads=3,
+            queues=("default",),
         )
         assert executor.prefetch_count == 12
 
@@ -330,17 +339,15 @@ def test_post_init__keeps_explicit_prefetch_count(self):
         )
         assert executor.prefetch_count == 7
 
-    @pytest.mark.parametrize("prefetch_count", [0, -1])
-    def test_post_init__raise_value_error_for_prefetch_count_below_one(
-        self, prefetch_count
-    ):
-        """__post_init__ rejects a prefetch count below one."""
-        with pytest.raises(ValueError, match="prefetch_count must be at least 1"):
-            TaskExecutor(
-                backend=default_task_backend,
-                prefetch_count=prefetch_count,
-                queues=("default",),
-            )
+    @pytest.mark.parametrize("prefetch_count", [-1, -5])
+    def test_post_init__floors_prefetch_count_at_one(self, prefetch_count):
+        """__post_init__ floors a negative count, which would fetch nothing at all."""
+        executor = TaskExecutor(
+            backend=default_task_backend,
+            prefetch_count=prefetch_count,
+            queues=("default",),
+        )
+        assert executor.prefetch_count == 1
 
     def test_get_maximum_tasks_per_child__returns_none_when_max_tasks_is_zero(self):
         """get_maximum_tasks_per_child returns None when max_tasks is 0."""
@@ -635,7 +642,8 @@ def test_run__raises_when_prefetcher_fails(self, monkeypatch):
         thread_failures = []
         monkeypatch.setattr(threading, "excepthook", thread_failures.append)
         monkeypatch.setattr(
-            "threadmill.executor.TASK_WAIT_TIMEOUT", datetime.timedelta(seconds=0.01)
+            "threadmill.executor.WorkerProcess.task_wait_timeout",
+            datetime.timedelta(seconds=0.01),
         )
         worker = WorkerProcess(
             thread_count=1,
@@ -680,7 +688,8 @@ class TestWorkerThread:
     def fast_task_wait(self, monkeypatch):
         """Shorten the buffer wait so drained run() tests return quickly."""
         monkeypatch.setattr(
-            "threadmill.executor.TASK_WAIT_TIMEOUT", datetime.timedelta(seconds=0.01)
+            "threadmill.executor.WorkerProcess.task_wait_timeout",
+            datetime.timedelta(seconds=0.01),
         )
 
     def test_execute_task_result__successful_execution(self):
diff --git a/threadmill/executor.py b/threadmill/executor.py
index fe6d5a4..096d413 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -100,10 +100,6 @@ def configure_logging(formatter: logging.Formatter) -> None:
     root_logger.addHandler(handler)
 
 
-# Maximum wait for a task from the backend or the prefetch buffer.
-TASK_WAIT_TIMEOUT = datetime.timedelta(seconds=1)
-
-
 @dataclasses.dataclass(kw_only=True, slots=True)
 class TaskExecutor:
     """Tasks consumed from shared joinable queues via process and thread pools."""
@@ -131,10 +127,7 @@ def __post_init__(self) -> None:
         """Initialize derived orchestration fields and queues."""
         self.process_count = self.workers or max(multiprocessing.cpu_count() - 1, 1)
         self.thread_count = max(self.threads, 1)
-        if self.prefetch_count is None:
-            self.prefetch_count = self.thread_count * 4
-        if self.prefetch_count < 1:
-            raise ValueError("prefetch_count must be at least 1")
+        self.prefetch_count = max(self.prefetch_count or self.thread_count * 4, 1)
 
     def get_maximum_tasks_per_child(self) -> int | None:
         """Return worker recycling limit based on config and thread count."""
@@ -212,6 +205,9 @@ def maintain_worker_pool(self) -> None:
 class WorkerProcess(multiprocessing.Process):
     """Single worker process running a prefetcher and thread_count consumer threads."""
 
+    task_wait_timeout: datetime.timedelta = datetime.timedelta(seconds=1)
+    """How long a thread waits on the broker or the buffer before it re-checks its stop condition."""
+
     def __init__(
         self,
         *,
@@ -347,7 +343,7 @@ def run(self) -> None:
                     batch = self.backend.acquire(
                         *self.worker.queues,
                         count=count,
-                        timeout=TASK_WAIT_TIMEOUT,
+                        timeout=self.worker.task_wait_timeout,
                         worker=self.name,
                     )
                 except Empty, TimeoutError:
@@ -376,7 +372,9 @@ def buffer(self, task_result: TaskResult) -> bool:
         )
         while not self.stop_requested.is_set():
             try:
-                self.task_buffer.put(item, timeout=TASK_WAIT_TIMEOUT.total_seconds())
+                self.task_buffer.put(
+                    item, timeout=self.worker.task_wait_timeout.total_seconds()
+                )
             except Full:
                 continue
             return True
@@ -403,7 +401,7 @@ def run(self) -> None:
         while True:
             try:
                 task_result = prefetcher.task_buffer.get(
-                    timeout=TASK_WAIT_TIMEOUT.total_seconds()
+                    timeout=self.worker.task_wait_timeout.total_seconds()
                 ).task_result
             except Empty:
                 if prefetcher.finished.is_set():
diff --git a/threadmill/management/commands/threadmill.py b/threadmill/management/commands/threadmill.py
index 3f1a9aa..c1ef6c6 100644
--- a/threadmill/management/commands/threadmill.py
+++ b/threadmill/management/commands/threadmill.py
@@ -149,7 +149,7 @@ def handle(
             )
         except (TypeError, ValueError) as e:
             raise CommandError(f"Invalid log format: {log_format!r}") from e
-        if prefetch_count is not None and prefetch_count < 1:
+        if prefetch_count is not None and prefetch_count < 0:
             raise CommandError(f"Invalid prefetch count: {prefetch_count!r}")
         exe = TaskExecutor(
             backend=backend,

From 6db87c9993c9ccada56e38911eaa89ae5f5501ca Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 21:20:37 +0200
Subject: [PATCH 11/26] Drop the worker startup log

It repeated counts the operator had already configured, and the command
prints that it is starting workers. The test that watched the line now
asserts on the executor the command builds, the way the poll-interval
tests do, so the prefetch plumbing stays pinned without a log to watch.
---
 tests/test_command.py  | 44 ++++++++++--------------------------------
 threadmill/executor.py |  6 ------
 2 files changed, 10 insertions(+), 40 deletions(-)

diff --git a/tests/test_command.py b/tests/test_command.py
index a9e4847..80b9ad6 100644
--- a/tests/test_command.py
+++ b/tests/test_command.py
@@ -1,7 +1,5 @@
 import argparse
 import datetime
-import io
-import json
 import logging
 import re
 import signal
@@ -9,11 +7,10 @@
 
 import pytest
 from django.core.management import CommandError, call_command
-from django.tasks import TaskResultStatus, default_task_backend
+from django.tasks import default_task_backend
 
 from tests.testapp.tasks import (
     compute_workload,
-    echo,
     io_workload,
     memory_workload,
 )
@@ -128,37 +125,16 @@ def test_call_command__log_format__empty_string(self):
         )
         assert handler.formatter.format(record) == "Hello world"
 
-    def test_call_command__prefetch_count(self, caplog):
+    def test_call_command__prefetch_count(self):
         """Pass the prefetch count through to the running task executor."""
-        enqueued = default_task_backend.enqueue(echo, args=[1])
-        original_stream = handler.stream
-        parent_log = io.StringIO()
-        handler.setStream(parent_log)
-        try:
-            # The worker no longer forces a level on the root logger, so the
-            # startup record is only emitted when the caller asks for INFO.
-            with caplog.at_level(logging.INFO):
-                call_command(
-                    "threadmill",
-                    "worker",
-                    verbosity=0,
-                    workers=1,
-                    queues=["default"],
-                    exit_empty=True,
-                    prefetch_count=7,
-                )
-        finally:
-            handler.setStream(original_stream)
-        records = [
-            json.loads(line)
-            for line in parent_log.getvalue().splitlines()
-            if line.startswith("{")
-        ]
-        assert any("7 prefetched tasks each" in record["message"] for record in records)
-        assert (
-            default_task_backend.get_result(enqueued.id).status
-            is TaskResultStatus.SUCCESSFUL
-        )
+        with patch.object(threadmill.TaskExecutor, "run", autospec=True) as run:
+            call_command(
+                "threadmill",
+                "worker",
+                verbosity=0,
+                prefetch_count=7,
+            )
+        assert run.call_args.args[0].prefetch_count == 7
 
     def test_call_command__prefetch_count__raise_command_error(self):
         """Reject a negative prefetch count with a CommandError."""
diff --git a/threadmill/executor.py b/threadmill/executor.py
index 096d413..076f87c 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -154,12 +154,6 @@ def create_worker_process(self) -> WorkerProcess:
     def run(self) -> None:
         """Start consuming tasks until shutdown is requested."""
         configure_logging(self.log_formatter)
-        logger.info(
-            "Starting %d worker processes with %d threads and %d prefetched tasks each",
-            self.process_count,
-            self.thread_count,
-            self.prefetch_count,
-        )
         self.worker_processes = [
             self.create_worker_process() for _ in range(self.process_count)
         ]

From e36bd01c98bbe95fa94209c9222e6a83e01f249a Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 21:29:02 +0200
Subject: [PATCH 12/26] Let the worker budget stop the fetcher on its own

The prefetch budget clamp only shrank the last request, so it bought a
margin of at most one task on a limit the README already called soft. The
worker's expiry, set when the budget is reached and read at the top of the
fetch loop, already stops fetching, so the clamp and its remaining_tasks
helper are gone and the fetcher always asks for a full buffer.

The documented overshoot had to grow with it: the buffer and the batch in
the fetcher's hand both still run, so a recycled worker can finish roughly
twice the buffer size plus the thread count past its budget, which is what
the README and the flag help now say.
---
 README.md                                    |  4 +-
 tests/test_executor.py                       | 46 +++-----------------
 threadmill/executor.py                       | 16 +------
 threadmill/management/commands/threadmill.py |  3 +-
 4 files changed, 11 insertions(+), 58 deletions(-)

diff --git a/README.md b/README.md
index 0f79594..684c94d 100644
--- a/README.md
+++ b/README.md
@@ -99,7 +99,7 @@ Prefetching has soft limits:
 
 - Tasks are leased when they are fetched, so the time a task waits in the buffer counts against `lease_ttl`. Size `--prefetch-count` for your workload: the buffer holds at most that many tasks, and each of them is waiting.
 - A task enqueued after a fetch waits for the buffer to drain before a worker picks it up, though the buffer itself dispatches the highest priority task first and keeps queue order within a priority.
-- `--max-tasks` may overshoot by up to the buffer size, because a prefetched task always runs.
+- `--max-tasks` recycles a worker, it does not cap how many tasks run: a full buffer and the batch in hand both still run, so expect roughly twice the buffer size plus the thread count beyond the budget.
 - `worker_ids` records the fetcher of the process, not the thread that runs the task.
 
 #### Health
@@ -112,7 +112,7 @@ uv run manage.py threadmill worker --max-tasks 1000 --max-tasks-jitter 100
 
 This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time.
 
-The limit is soft: a worker still drains its prefetch buffer, so it may process up to `--prefetch-count` tasks beyond the configured maximum.
+The limit is soft: a worker drains its buffer and the batch already fetched before it stops, so it may process roughly twice `--prefetch-count` tasks beyond the configured maximum, plus whatever its threads were already running.
 
 Should a worker crash or be killed, the pool will automatically restart it.
 
diff --git a/tests/test_executor.py b/tests/test_executor.py
index bca5303..0a6afa4 100644
--- a/tests/test_executor.py
+++ b/tests/test_executor.py
@@ -146,20 +146,13 @@ def task_wait_timeout(self) -> datetime.timedelta:
         """Follow the worker's timeout, so patching it reaches this stub too."""
         return WorkerProcess.task_wait_timeout
 
-    def __init__(
-        self, *, remaining: int | None = None, exit_empty: bool = False
-    ) -> None:
+    def __init__(self, *, exit_empty: bool = False) -> None:
         self.pid = 4242
         self.queues = ("default",)
-        self.remaining = remaining
         self.exit_empty = exit_empty
         self.expired = threading.Event()
         self.shutdown_requested = threading.Event()
 
-    def remaining_tasks(self) -> int | None:
-        """Return the scripted task budget."""
-        return self.remaining
-
 
 class CountingEvent(threading.Event):
     """Event that counts is_set() checks to observe consumer poll cycles."""
@@ -177,12 +170,11 @@ def _make_prefetcher(
     backend: StubPrefetchBackend,
     *,
     prefetch_count: int = 4,
-    remaining: int | None = None,
     exit_empty: bool = False,
 ) -> TaskPrefetcher:
     """Build a prefetcher over a scripted stub backend and worker."""
     return TaskPrefetcher(
-        worker=StubPrefetchWorker(remaining=remaining, exit_empty=exit_empty),
+        worker=StubPrefetchWorker(exit_empty=exit_empty),
         backend=backend,
         prefetch_count=prefetch_count,
     )
@@ -605,22 +597,6 @@ def test_record_task__noop_before_run_sets_lock_and_expired(self):
         worker.record_task()
         assert worker.task_count == 0
 
-    def test_remaining_tasks__none_when_unlimited(self):
-        """remaining_tasks reports no budget when max_tasks is disabled."""
-        assert _make_worker(max_tasks=None).remaining_tasks() is None
-
-    def test_remaining_tasks__subtracts_task_count(self):
-        """remaining_tasks reports the unreached part of the worker budget."""
-        worker = _make_worker(max_tasks=5)
-        worker.task_count = 2
-        assert worker.remaining_tasks() == 3
-
-    def test_remaining_tasks__floors_at_zero(self):
-        """remaining_tasks never reports a negative budget."""
-        worker = _make_worker(max_tasks=2)
-        worker.task_count = 5
-        assert worker.remaining_tasks() == 0
-
     def test_shutdown_requested__is_settable(self):
         """shutdown_requested event can be set on an unstarted worker."""
         worker = _make_worker()
@@ -917,26 +893,16 @@ def test_run__stops_on_stop_requested(self):
         assert prefetcher.finished.is_set()
         assert backend.calls == []
 
-    def test_run__breaks_when_no_budget_remains(self):
-        """Stop without acquiring once the worker budget is exhausted."""
-        backend = StubPrefetchBackend()
-        prefetcher = _make_prefetcher(backend, remaining=0)
-
-        prefetcher.run()
-
-        assert backend.calls == []
-        assert prefetcher.finished.is_set()
-
-    def test_run__caps_count_at_remaining_budget(self):
-        """Acquire no more tasks than the remaining worker budget allows."""
+    def test_run__acquires_a_full_buffer(self):
+        """Acquire a full buffer; the worker budget only stops the loop."""
         task_result = _task_result(echo, 1)
         backend = StubPrefetchBackend([task_result])
-        prefetcher = _make_prefetcher(backend, prefetch_count=5, remaining=2)
+        prefetcher = _make_prefetcher(backend, prefetch_count=5)
 
         prefetcher.worker.shutdown_requested.set()
         prefetcher.run()
 
-        assert backend.calls[0]["count"] == 2
+        assert backend.calls[0]["count"] == 5
         assert prefetcher.task_buffer.get_nowait().task_result.id == task_result.id
         assert len(backend.calls) == 1
         assert prefetcher.finished.is_set()
diff --git a/threadmill/executor.py b/threadmill/executor.py
index 076f87c..1f0e264 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -262,12 +262,6 @@ def run(self) -> None:
             logger.error("Worker process %s exits after a fetch failure", self.name)
             raise failure
 
-    def remaining_tasks(self) -> int | None:
-        """Return how many more tasks this process may run, or None when unlimited."""
-        if self.max_tasks is None:
-            return None
-        return max(self.max_tasks - self.task_count, 0)
-
     def record_task(self) -> None:
         """Record one processed task and stop when max_tasks is reached."""
         if self.max_tasks is None:
@@ -325,18 +319,10 @@ def __init__(
     def run(self) -> None:
         try:
             while not self.stop_requested.is_set() and not self.worker.expired.is_set():
-                remaining = self.worker.remaining_tasks()
-                count = (
-                    self.prefetch_count
-                    if remaining is None
-                    else min(self.prefetch_count, remaining)
-                )
-                if count < 1:
-                    break
                 try:
                     batch = self.backend.acquire(
                         *self.worker.queues,
-                        count=count,
+                        count=self.prefetch_count,
                         timeout=self.worker.task_wait_timeout,
                         worker=self.name,
                     )
diff --git a/threadmill/management/commands/threadmill.py b/threadmill/management/commands/threadmill.py
index c1ef6c6..47d1c81 100644
--- a/threadmill/management/commands/threadmill.py
+++ b/threadmill/management/commands/threadmill.py
@@ -77,7 +77,8 @@ def add_arguments(self, parser):
                 " Defaults to 4 × the number of threads; use 1 to disable batching."
                 " Buffered tasks hold their lease while they wait, so a deep buffer"
                 " needs a matching lease_ttl."
-                " Buffered tasks still run after a worker reaches its max-tasks limit."
+                " Buffered tasks and the batch in hand still run after a worker reaches"
+                " its max-tasks limit."
             ),
         )
         parser.add_argument(

From 8d3e9f2924ce44c33c398d2aecacfcaabc24178f Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 21:39:01 +0200
Subject: [PATCH 13/26] Order the buffered task results themselves

ThreadmillTaskResult gains a comparison ordered by descending priority and
then enqueue time, which is the order the queue pops in, so the prefetch
buffer holds the leased results directly. That removes the PrefetchedTask
wrapper, the fetch-sequence counter and the unwrapping at the consumer.

The acquire contract now says what the buffer needs: it returns
ThreadmillTaskResult, which carries the lease and is orderable.
---
 tests/test_executor.py       | 56 +++++++++++++++++++++++-------------
 threadmill/backends/base.py  | 13 +++++++--
 threadmill/backends/redis.py |  2 +-
 threadmill/executor.py       | 31 ++++----------------
 4 files changed, 54 insertions(+), 48 deletions(-)

diff --git a/tests/test_executor.py b/tests/test_executor.py
index 0a6afa4..cbcdeae 100644
--- a/tests/test_executor.py
+++ b/tests/test_executor.py
@@ -29,7 +29,11 @@
     echo,
     log_message,
 )
-from threadmill.backends.base import Broker, ThreadmillTaskBackend
+from threadmill.backends.base import (
+    Broker,
+    ThreadmillTaskBackend,
+    ThreadmillTaskResult,
+)
 from threadmill.executor import (
     JsonFormatter,
     TaskExecutor,
@@ -56,9 +60,9 @@ async def _async_task():
     return 99
 
 
-def _task_result(task, *args, **kwargs) -> TaskResult:
-    """Build a READY `TaskResult` without touching Redis."""
-    return TaskResult(
+def _task_result(task, *args, **kwargs) -> ThreadmillTaskResult:
+    """Build a READY `ThreadmillTaskResult` without touching Redis."""
+    return ThreadmillTaskResult(
         task=task,
         id=str(uuid.uuid7()),
         status=TaskResultStatus.READY,
@@ -865,9 +869,10 @@ def test_run__buffers_until_stop_requested(self):
         assert not thread.is_alive()
         assert prefetcher.finished.is_set()
         assert prefetcher.failure is None
-        assert [
-            prefetcher.task_buffer.get_nowait().task_result.id for _ in range(2)
-        ] == [first.id, second.id]
+        assert [prefetcher.task_buffer.get_nowait().id for _ in range(2)] == [
+            first.id,
+            second.id,
+        ]
         assert all(call["count"] == 4 for call in backend.calls)
 
     def test_run__stops_on_expired(self):
@@ -903,7 +908,7 @@ def test_run__acquires_a_full_buffer(self):
         prefetcher.run()
 
         assert backend.calls[0]["count"] == 5
-        assert prefetcher.task_buffer.get_nowait().task_result.id == task_result.id
+        assert prefetcher.task_buffer.get_nowait().id == task_result.id
         assert len(backend.calls) == 1
         assert prefetcher.finished.is_set()
 
@@ -917,9 +922,10 @@ def test_run__stops_after_batch_when_shutdown_requested(self):
         prefetcher.worker.shutdown_requested.set()
         prefetcher.run()
 
-        assert [
-            prefetcher.task_buffer.get_nowait().task_result.id for _ in range(2)
-        ] == [first.id, second.id]
+        assert [prefetcher.task_buffer.get_nowait().id for _ in range(2)] == [
+            first.id,
+            second.id,
+        ]
         assert len(backend.calls) == 1
         assert prefetcher.finished.is_set()
 
@@ -976,7 +982,7 @@ def test_buffer__returns_false_when_full_and_stop_requested(self):
         assert prefetcher.finished.is_set()
         assert prefetcher.failure is None
         assert prefetcher.task_buffer.qsize() == 1
-        assert prefetcher.task_buffer.get_nowait().task_result.id == buffered.id
+        assert prefetcher.task_buffer.get_nowait().id == buffered.id
 
     def test_buffer__returns_true_when_space_available(self):
         """Buffer a task result when the queue has room."""
@@ -984,19 +990,29 @@ def test_buffer__returns_true_when_space_available(self):
         prefetcher = _make_prefetcher(StubPrefetchBackend())
 
         assert prefetcher.buffer(task_result) is True
-        assert prefetcher.task_buffer.get_nowait().task_result.id == task_result.id
+        assert prefetcher.task_buffer.get_nowait().id == task_result.id
 
     def test_buffer__dispatches_highest_priority_first(self):
-        """Hand out the highest priority task first and keep fetch order on ties."""
+        """Hand out the highest priority task first, earlier enqueued on ties."""
+        enqueued_at = timezone.now()
         low = _task_result(dataclasses.replace(echo, priority=1), 1)
-        high = _task_result(dataclasses.replace(echo, priority=5), 2)
+        first_high = dataclasses.replace(
+            _task_result(dataclasses.replace(echo, priority=5), 2),
+            enqueued_at=enqueued_at,
+        )
         middle = _task_result(dataclasses.replace(echo, priority=3), 3)
-        later_high = _task_result(dataclasses.replace(echo, priority=5), 4)
+        later_high = dataclasses.replace(
+            _task_result(dataclasses.replace(echo, priority=5), 4),
+            enqueued_at=enqueued_at + datetime.timedelta(seconds=1),
+        )
         prefetcher = _make_prefetcher(StubPrefetchBackend())
 
-        for task_result in (low, high, middle, later_high):
+        for task_result in (low, first_high, middle, later_high):
             assert prefetcher.buffer(task_result) is True
 
-        assert [
-            prefetcher.task_buffer.get_nowait().task_result.id for _ in range(4)
-        ] == [high.id, later_high.id, middle.id, low.id]
+        assert [prefetcher.task_buffer.get_nowait().id for _ in range(4)] == [
+            first_high.id,
+            later_high.id,
+            middle.id,
+            low.id,
+        ]
diff --git a/threadmill/backends/base.py b/threadmill/backends/base.py
index 10e7496..a2a07d2 100644
--- a/threadmill/backends/base.py
+++ b/threadmill/backends/base.py
@@ -69,6 +69,13 @@ def __reduce__(self):
 class ThreadmillTaskResult(TaskResult):
     lease_token: str | None = None
 
+    def __lt__(self, other: ThreadmillTaskResult) -> bool:
+        """Order by descending priority, then by enqueue time, as the queue does."""
+        return (-self.task.priority, self.enqueued_at) < (
+            -other.task.priority,
+            other.enqueued_at,
+        )
+
     @classmethod
     def from_result(
         cls, task_result: TaskResult, *, lease_token: str | None
@@ -261,7 +268,7 @@ def acquire(
         count: int = 1,
         timeout: datetime.timedelta | None = None,
         worker: str = "",
-    ) -> list[TaskResult]:
+    ) -> list[ThreadmillTaskResult]:
         """
         Return and lock up to `count` tasks without removing them from the queue.
 
@@ -279,7 +286,9 @@ def acquire(
             queue.Empty: If the first task is unavailable and timeout is None.
 
         Returns:
-            Between one and `count` task results, never an empty list.
+            Between one and `count` leased task results, never an empty list. They
+            are `ThreadmillTaskResult`s, which order by priority and enqueue time
+            so the prefetch buffer can hand out the most important task first.
         """
         raise NotImplementedError
 
diff --git a/threadmill/backends/redis.py b/threadmill/backends/redis.py
index 0cf7e85..38d2703 100644
--- a/threadmill/backends/redis.py
+++ b/threadmill/backends/redis.py
@@ -318,7 +318,7 @@ def acquire(
         count: int = 1,
         timeout: datetime.timedelta | None = None,
         worker: str = "",
-    ) -> list[TaskResult]:
+    ) -> list[ThreadmillTaskResult]:
         queue_names = queue_names or tuple(self.queues)
         deadline = time.monotonic() + timeout.total_seconds() if timeout else None
         keys = [
diff --git a/threadmill/executor.py b/threadmill/executor.py
index 1f0e264..96f213b 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -3,7 +3,6 @@
 import asyncio
 import dataclasses
 import datetime
-import itertools
 import json
 import logging
 import multiprocessing
@@ -28,7 +27,7 @@
 from django.utils.json import normalize_json
 
 if typing.TYPE_CHECKING:
-    from .backends.base import Broker, ThreadmillTaskBackend
+    from .backends.base import Broker, ThreadmillTaskBackend, ThreadmillTaskResult
 
 
 class JsonFormatter(logging.Formatter):
@@ -280,20 +279,6 @@ def shutdown(self) -> None:
         self.join()
 
 
-@dataclasses.dataclass(order=True, slots=True)
-class PrefetchedTask:
-    """A buffered task result, awaiting dispatch to a worker thread.
-
-    The buffer holds these rather than bare task results so it can hand out the
-    highest priority task first, and preserve fetch order within a priority.
-    """
-
-    sort_key: tuple[int, int]
-    """Negated task priority, then fetch order; the heap compares on this."""
-
-    task_result: TaskResult = dataclasses.field(compare=False)
-
-
 class TaskPrefetcher(threading.Thread):
     """Single prefetcher thread filling the task buffer of one worker process."""
 
@@ -308,10 +293,9 @@ def __init__(
         self.worker = worker
         self.backend = backend
         self.prefetch_count = prefetch_count
-        self.task_buffer: queue.PriorityQueue[PrefetchedTask] = queue.PriorityQueue(
-            maxsize=prefetch_count
+        self.task_buffer: queue.PriorityQueue[ThreadmillTaskResult] = (
+            queue.PriorityQueue(maxsize=prefetch_count)
         )
-        self.fetch_sequence = itertools.count()
         self.finished = threading.Event()
         self.stop_requested = threading.Event()
         self.failure: Exception | None = None
@@ -345,15 +329,12 @@ def run(self) -> None:
         finally:
             self.finished.set()
 
-    def buffer(self, task_result: TaskResult) -> bool:
+    def buffer(self, task_result: ThreadmillTaskResult) -> bool:
         """Buffer one task result; return False when the prefetcher must stop."""
-        item = PrefetchedTask(
-            (-task_result.task.priority, next(self.fetch_sequence)), task_result
-        )
         while not self.stop_requested.is_set():
             try:
                 self.task_buffer.put(
-                    item, timeout=self.worker.task_wait_timeout.total_seconds()
+                    task_result, timeout=self.worker.task_wait_timeout.total_seconds()
                 )
             except Full:
                 continue
@@ -382,7 +363,7 @@ def run(self) -> None:
             try:
                 task_result = prefetcher.task_buffer.get(
                     timeout=self.worker.task_wait_timeout.total_seconds()
-                ).task_result
+                )
             except Empty:
                 if prefetcher.finished.is_set():
                     return

From 880209771405a84bb6ff9483617db846d7105b58 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 21:41:36 +0200
Subject: [PATCH 14/26] Rewrite the new docstrings in simplified technical
 English

Short sentences, one fact for each sentence, and no em dashes, semicolons
or colons in the prose. The acquire docstring says what the method locks
and what it returns. The prefetcher and worker docstrings say what each
thread does instead of naming it again.

The range notation in the benchmark docs becomes words, "5 to 10 ms", and
the definition list of benchmarks becomes three sentences, because a colon
after each name reads as structure and not as prose.
---
 benchmarks/chart.py                          |   8 +-
 benchmarks/test_backends.py                  | 120 ++++++++++---------
 threadmill/backends/base.py                  |  26 ++--
 threadmill/backends/redis.py                 |   2 +-
 threadmill/executor.py                       |  10 +-
 threadmill/management/commands/threadmill.py |   2 +-
 6 files changed, 90 insertions(+), 78 deletions(-)

diff --git a/benchmarks/chart.py b/benchmarks/chart.py
index d42813c..f89ec8f 100644
--- a/benchmarks/chart.py
+++ b/benchmarks/chart.py
@@ -78,11 +78,11 @@ class Theme:
 )
 
 DIAGNOSTIC_QUEUES = frozenset({"threadmill (no prefetch)"})
-"""Queues the benchmark measures but the chart leaves out.
+"""Queues that the benchmark measures and the chart leaves out.
 
-The harness runs threadmill twice to bracket its prefetch buffer, and on a local
-broker the two land within a percent of each other, so plotting both would rank
-them on measurement noise.
+The harness runs threadmill twice to compare its prefetch buffer with a single
+task. Against a local broker the two results differ by less than one percent. A
+chart with both rows ranks them on measurement noise.
 """
 
 
diff --git a/benchmarks/test_backends.py b/benchmarks/test_backends.py
index 62201d5..3fbbb57 100644
--- a/benchmarks/test_backends.py
+++ b/benchmarks/test_backends.py
@@ -1,45 +1,48 @@
 """Benchmarks comparing threadmill against other Django task queues.
 
-Every backend is measured on the same trivial echo task and the same queue, so
-the numbers reflect queue and worker overhead instead of task work:
+Every backend is measured on the same trivial echo task and the same queue. The
+numbers therefore show queue and worker overhead and not task work.
 
-- ``test_enqueue__benchmark``: time for a queue to accept a single task.
-- ``test_start_worker__benchmark``: time for a worker to start and process one queued task.
-- ``test_process_queue__benchmark``: time for a worker to process a full queue.
+- ``test_enqueue__benchmark`` measures the time for a queue to accept one task.
+- ``test_start_worker__benchmark`` measures the time for a worker to start and process one queued task.
+- ``test_process_queue__benchmark`` measures the time for a worker to process a full queue.
 
 The processing benchmark queues 20,000 tasks per queue. The fixed cost of a cold
-worker start and stop is quantized to about a second, so a deep queue is what
-makes the marginal drain per task — the number the chart plots — measurable.
+worker start and stop is quantized to about a second. A deep queue is therefore
+necessary to measure the marginal drain per task, which is the number that the
+chart plots.
 
-Both worker benchmarks include the worker's fixed start cost, and the queues
-that exit on their own include their stop cost too. Subtract
+Both worker benchmarks include the fixed start cost of the worker. Queues that
+exit on their own include their stop cost too. Subtract
 ``test_start_worker__benchmark`` from ``test_process_queue__benchmark`` and divide
-that queue's task count by the difference to get the marginal throughput of a
-busy queue.
-
-Every queue runs one worker process and one thread, and each reads ``READ_AHEAD``
-(128) messages ahead where its queue has such a setting, so the numbers rank the
-queues rather than their polling strategies. Celery's consumer blocks on an empty
-queue, so its window costs no sleep per message. Dramatiq polls instead and sleeps
-a jittered backoff once its window fills, so a shallow window measures that backoff
-rather than the queue: the same worker drains roughly 450 tasks/s at two messages,
-1,983 at 16 and 5,658 at 64, where the sleep stops setting the rate. Threadmill
-reads the same rate into its prefetch buffer.
-
-django-tasks-db reads one task at a time, because its shipped worker exposes no
-read-ahead setting. django-tasks-rq forks a work horse per job, so its drain
-includes that fork and it reads one task at a time too.
-
-Threadmill is measured twice, reading 128 messages ahead and reading one at a
-time, so the read-ahead cost can be subtracted from both worker benchmarks. Its
-queues are deeper than the others because its marginal drain is only seconds long,
-which would otherwise sit inside the one-second quantization of the fixed cost.
-Threadmill, django-tasks-db and django-tasks-rq run one worker process that
-drains a queue and exits. Celery and dramatiq have no such mode, so
-the benchmark queues a sentinel task last and waits for it to be processed. That
-wait is what proves the queue was drained. Their workers are stopped after the
-measurement, because a graceful shutdown takes seconds and would dominate a short
-drain.
+the task count of the queue by the difference. The result is the marginal
+throughput of a busy queue.
+
+Every queue runs one worker process and one thread. Each queue reads ``READ_AHEAD``
+messages ahead where the queue has such a setting. The numbers therefore rank the
+queues and not their polling strategies. The consumer of Celery blocks on an empty
+queue, so its window costs no sleep for each message. Dramatiq polls instead and
+sleeps a jittered backoff of 5 to 10 ms when its window is full. A shallow window
+therefore measures that backoff and not the queue.
+
+The same worker drains about 451 tasks per second at two messages, 1,983 at 16 and
+5,658 at 64. At 64 the sleep stops to set the rate. Threadmill reads the same rate
+into its prefetch buffer.
+
+django-tasks-db reads one task at a time, because its shipped worker does not
+expose a read-ahead setting. django-tasks-rq forks a work horse for each job, so
+its drain includes that fork and it reads one task at a time too.
+
+Threadmill is measured twice, with 128 messages ahead and with one message at a
+time. The read-ahead cost can therefore be subtracted from both worker benchmarks.
+Its queues are deeper than the others, because its marginal drain is only seconds
+long. A shallow queue sits inside the one-second quantization of the fixed cost.
+
+Threadmill, django-tasks-db and django-tasks-rq run one worker process that drains
+a queue and exits. Celery and dramatiq have no such mode, so the benchmark queues a
+sentinel task last and waits for it. That wait proves that the queue was drained.
+Their workers are stopped after the measurement. A graceful shutdown takes seconds.
+It dominates a short drain.
 """
 
 import collections.abc
@@ -82,14 +85,18 @@
 READ_AHEAD = 128
 """Messages each worker reads ahead, where its queue has such a setting.
 
-Deep enough that each consumer's wait mechanism stops deciding the ranking.
-Celery's consumer blocks on an empty queue, so its window costs no sleep.
-Dramatiq polls instead and sleeps a jittered 5-10 ms backoff once its window
-fills, so a shallow window measures that backoff rather than the queue: the same
-worker drains about 451 tasks/s at two messages, 1,983 at 16 and 5,658 at 64,
-where the sleep stops setting the rate. At 128 the backoff costs about 0.06 ms
-per task. django-tasks-db reads one task at a time, and django-tasks-rq forks a
-work horse per job, so neither can be told to read ahead.
+This window is deep enough that the wait mechanism of each consumer stops to
+decide the ranking. The consumer of Celery blocks on an empty queue, so its
+window costs no sleep. Dramatiq polls instead and sleeps a jittered backoff of
+5 to 10 ms when its window is full. A shallow window therefore measures that
+backoff and not the queue.
+
+The same worker drains about 451 tasks per second at two messages, 1,983 at 16
+and 5,658 at 64. At 64 the sleep stops to set the rate. At 128 the backoff costs
+about 0.06 ms for each task.
+
+django-tasks-db reads one task at a time. django-tasks-rq forks a work horse for
+each job. Neither queue can be told to read ahead.
 """
 
 CELERY_WORKER = (
@@ -108,11 +115,12 @@
 )
 """Celery worker running as one process with one thread, ``READ_AHEAD`` messages ahead.
 
-The default prefork pool crashes on CPython 3.14, where the pool child loses
-the task handler state it expects, so the worker runs on the solo pool. With one
-concurrent task, ``--prefetch-multiplier={READ_AHEAD}`` sets the prefetch count to
-``READ_AHEAD``, the benchmark rate. Celery's Redis consumer blocks while its queue
-is empty, so the prefetch adds no sleep per message.
+The default prefork pool crashes on CPython 3.14. The pool child loses the task
+handler state that it expects. The worker therefore runs on the solo pool. With
+one concurrent task, ``--prefetch-multiplier={READ_AHEAD}`` sets the prefetch
+count to ``READ_AHEAD``, which is the benchmark rate. The Redis consumer of
+Celery blocks while its queue is empty, so the prefetch adds no sleep for each
+message.
 """
 
 DRAMATIQ_WORKER = (
@@ -127,11 +135,11 @@
 )
 """dramatiq worker running as one process with one thread, ``READ_AHEAD`` messages ahead.
 
-The Redis broker polls rather than blocks: its consumer fetches only while fewer
-than its read-ahead of messages are unacked and, with that window full, sleeps a
-jittered 5-10 ms backoff before polling again. The CLI has no read-ahead flag, so
-the worker environment carries ``dramatiq_queue_prefetch`` set to ``READ_AHEAD``,
-the benchmark rate.
+The Redis broker polls and does not block. Its consumer fetches only while fewer
+messages than its read-ahead are unacked. When that window is full, the consumer
+sleeps a jittered backoff of 5 to 10 ms and then polls again. The command line
+has no read-ahead flag, so the worker environment carries
+``dramatiq_queue_prefetch``. The benchmark sets this variable to ``READ_AHEAD``.
 """
 
 WORKER_STOP_TIMEOUT_SECONDS = 20
@@ -360,9 +368,9 @@ def django_task_backend(
 THREADMILL_TASK_COUNT = 60_000
 """Tasks threadmill queues, so its marginal drain outruns the one-second fixed cost.
 
-Threadmill drains a queue in about 3 seconds per 20,000 tasks, and the fixed cost
-of a cold worker start and stop is quantized to about a second, so a shallower
-queue would leave the prefetch comparison within that step.
+Threadmill drains a queue in about 3 seconds for each 20,000 tasks. The fixed cost
+of a cold worker start and stop is quantized to about a second. A shallower queue
+therefore keeps the prefetch comparison inside that step.
 """
 
 WORKER_QUEUES = (
diff --git a/threadmill/backends/base.py b/threadmill/backends/base.py
index a2a07d2..5e82af0 100644
--- a/threadmill/backends/base.py
+++ b/threadmill/backends/base.py
@@ -70,7 +70,10 @@ class ThreadmillTaskResult(TaskResult):
     lease_token: str | None = None
 
     def __lt__(self, other: ThreadmillTaskResult) -> bool:
-        """Order by descending priority, then by enqueue time, as the queue does."""
+        """Order the results by priority from highest to lowest.
+
+        Two results with the same priority keep their enqueue order.
+        """
         return (-self.task.priority, self.enqueued_at) < (
             -other.task.priority,
             other.enqueued_at,
@@ -270,25 +273,26 @@ def acquire(
         worker: str = "",
     ) -> list[ThreadmillTaskResult]:
         """
-        Return and lock up to `count` tasks without removing them from the queue.
+        Return and lock up to `count` tasks. The tasks stay in the queue.
 
-        Waits up to `timeout` for the first task only and fills the remaining
-        `count - 1` tasks without waiting.
+        The method waits up to `timeout` for the first task. It fills the
+        remaining tasks without a wait.
 
         Args:
             queue_names: The names of the queues to acquire tasks from.
             count: The maximum number of tasks to acquire, at least 1.
-            timeout: The maximum time to wait for the first task. If None, wait indefinitely.
-            worker: The name of the worker thread acquiring the tasks.
+            timeout: The maximum time to wait for the first task. If None, the method waits without a limit.
+            worker: The name of the worker thread that acquires the tasks.
 
         Raises:
-            TimeoutError: If the first task does not arrive within the specified timeout.
-            queue.Empty: If the first task is unavailable and timeout is None.
+            TimeoutError: The first task does not arrive within the specified timeout.
+            queue.Empty: The first task is unavailable and the timeout is None.
 
         Returns:
-            Between one and `count` leased task results, never an empty list. They
-            are `ThreadmillTaskResult`s, which order by priority and enqueue time
-            so the prefetch buffer can hand out the most important task first.
+            Between one and `count` leased task results. The list is never empty.
+            Each result is a `ThreadmillTaskResult` that orders by priority and
+            enqueue time. The prefetch buffer uses this order to select the next
+            task.
         """
         raise NotImplementedError
 
diff --git a/threadmill/backends/redis.py b/threadmill/backends/redis.py
index 38d2703..5737e5c 100644
--- a/threadmill/backends/redis.py
+++ b/threadmill/backends/redis.py
@@ -178,7 +178,7 @@ class RedisTaskBackend(ThreadmillTaskBackend):
     TELEMETRY_CHANNEL = "{prefix}:telemetry"
 
     ACQUIRE_SCRIPT = _load_lua("acquire")
-    """Lease up to a given number of tasks from the priority queues in one round-trip."""
+    """Lease up to a given number of tasks from the priority queues in one broker call."""
     ACKNOWLEDGE_SCRIPT = _load_lua("acknowledge")
     """Remove from running, persist the result, and clean up."""
 
diff --git a/threadmill/executor.py b/threadmill/executor.py
index 96f213b..1d65cd7 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -196,10 +196,10 @@ def maintain_worker_pool(self) -> None:
 
 
 class WorkerProcess(multiprocessing.Process):
-    """Single worker process running a prefetcher and thread_count consumer threads."""
+    """One worker process with one prefetcher and `thread_count` consumer threads."""
 
     task_wait_timeout: datetime.timedelta = datetime.timedelta(seconds=1)
-    """How long a thread waits on the broker or the buffer before it re-checks its stop condition."""
+    """How long a thread waits before it examines its stop condition again."""
 
     def __init__(
         self,
@@ -280,7 +280,7 @@ def shutdown(self) -> None:
 
 
 class TaskPrefetcher(threading.Thread):
-    """Single prefetcher thread filling the task buffer of one worker process."""
+    """The prefetcher thread of one worker process. It fills the task buffer."""
 
     def __init__(
         self,
@@ -330,7 +330,7 @@ def run(self) -> None:
             self.finished.set()
 
     def buffer(self, task_result: ThreadmillTaskResult) -> bool:
-        """Buffer one task result; return False when the prefetcher must stop."""
+        """Buffer one task result. Return False when the prefetcher must stop."""
         while not self.stop_requested.is_set():
             try:
                 self.task_buffer.put(
@@ -343,7 +343,7 @@ def buffer(self, task_result: ThreadmillTaskResult) -> bool:
 
 
 class WorkerThread(threading.Thread):
-    """Single worker thread consuming tasks from the process prefetch buffer."""
+    """A worker thread that runs the tasks from the prefetch buffer."""
 
     def __init__(
         self,
diff --git a/threadmill/management/commands/threadmill.py b/threadmill/management/commands/threadmill.py
index 47d1c81..fe7af47 100644
--- a/threadmill/management/commands/threadmill.py
+++ b/threadmill/management/commands/threadmill.py
@@ -74,7 +74,7 @@ def add_arguments(self, parser):
             default=None,
             help=(
                 "Number of tasks to prefetch per worker process."
-                " Defaults to 4 × the number of threads; use 1 to disable batching."
+                " Defaults to 4 × the number of threads. Use 1 to disable batching."
                 " Buffered tasks hold their lease while they wait, so a deep buffer"
                 " needs a matching lease_ttl."
                 " Buffered tasks and the batch in hand still run after a worker reaches"

From 981762bb3beb362c6a9b0e31cef175f864cc25e3 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 21:44:58 +0200
Subject: [PATCH 15/26] Trim the prefetch section of the README

The section said the same facts in more words. It now uses short sentences
and one idea for each sentence, and it drops the em dashes, the semicolons
and the two colons that joined clauses. The prefetch buffer paragraph, the
health note, the shutdown note and the lease note all lost a third of
their length and kept every number.
---
 README.md | 35 ++++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 684c94d..76f137a 100644
--- a/README.md
+++ b/README.md
@@ -85,22 +85,23 @@ Processes allow for parallel compute (no GIL) while threads are great for low-me
 uv run manage.py threadmill worker --workers 4 --threads 2
 ```
 
-Each worker process runs one fetcher thread that reserves a batch of tasks in a single broker round-trip.
-Worker threads drain that buffer, which amortizes broker latency across fast tasks.
-A full buffer blocks the fetcher until a worker thread frees a slot.
-The gain is largest when the broker is a network hop away.
-Against a local broker the buffer is worth about a fifth of the worker's throughput in the queue comparison.
+Each worker process runs one fetcher thread.
+The fetcher reserves a batch of tasks in one broker round trip, and the worker threads drain that batch.
+This keeps the worker threads busy while the broker answers.
+A full buffer blocks the fetcher until a thread frees a slot.
+The gain is largest when the broker is far from the worker.
+Against a local broker the buffer is worth about a fifth of the throughput.
 
 Set the batch size with `--prefetch-count`.
-It defaults to four times the thread count and applies per process, not per thread.
+It defaults to four times the thread count and applies to each process.
 A value of `1` disables batching.
 
 Prefetching has soft limits:
 
-- Tasks are leased when they are fetched, so the time a task waits in the buffer counts against `lease_ttl`. Size `--prefetch-count` for your workload: the buffer holds at most that many tasks, and each of them is waiting.
-- A task enqueued after a fetch waits for the buffer to drain before a worker picks it up, though the buffer itself dispatches the highest priority task first and keeps queue order within a priority.
-- `--max-tasks` recycles a worker, it does not cap how many tasks run: a full buffer and the batch in hand both still run, so expect roughly twice the buffer size plus the thread count beyond the budget.
-- `worker_ids` records the fetcher of the process, not the thread that runs the task.
+- A fetched task holds its lease while it waits. The buffer holds at most `--prefetch-count` tasks, so size that value for your workload.
+- A task enqueued after a fetch waits for the buffer to drain. Inside the buffer the highest priority task runs first, and tasks of one priority keep queue order.
+- `--max-tasks` recycles a worker. It is not a hard limit. The buffer and the batch in hand still run, so a worker can run about twice `--prefetch-count` tasks more than its budget.
+- `worker_ids` records the fetcher of the process and not the thread that runs the task.
 
 #### Health
 
@@ -112,15 +113,15 @@ uv run manage.py threadmill worker --max-tasks 1000 --max-tasks-jitter 100
 
 This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time.
 
-The limit is soft: a worker drains its buffer and the batch already fetched before it stops, so it may process roughly twice `--prefetch-count` tasks beyond the configured maximum, plus whatever its threads were already running.
+The limit is soft. A worker drains its buffer and the batch in hand before it stops. It can then run about twice `--prefetch-count` tasks more than the configured maximum.
 
 Should a worker crash or be killed, the pool will automatically restart it.
 
 #### Shutdown
 
 A graceful shutdown is possible with `SIGTERM` or a keyboard interrupt.
-All workers will finish the tasks they acquired and acknowledge them, including the tasks already in their prefetch buffer.
-A hard kill cannot be intercepted, so buffered tasks are left to the lease reaper.
+All workers finish the tasks they acquired and acknowledge them. This includes the tasks in their prefetch buffer.
+A hard kill cannot be intercepted, so the lease reaper collects the buffered tasks after the lease expires.
 
 You can use `--exit-empty` to exit immediately after all tasks have been processed,
 which might be useful for draining a one-off queue.
@@ -159,10 +160,10 @@ A task whose lease expired reaches the `retry` callback as an
 A claimed task whose stored payload cannot be read any more is dropped with the
 read error logged. A dropped task records no result, so it leaves the inspector
 and cannot be requeued.
-Keep `lease_ttl` above your worst-case runtime plus the time a task may wait in a
-prefetch buffer: a task that outlives its lease can still be running, so a retry
-may execute concurrently with it. The acknowledgement of the lease holder wins:
-the late result of an expired attempt is discarded.
+Keep `lease_ttl` above your worst-case runtime and above the time a task waits in a
+prefetch buffer. A task that outlives its lease can still run, so a retry can run
+at the same time. The acknowledgement of the lease holder wins. The late result
+of an expired attempt is discarded.
 
 All keys for one backend alias share a Redis Cluster hash tag (`{alias}`), so
 every multi-key operation — including the cross-queue acquire — runs on a single

From 804b2b297116d0179dd1e42751a5d831062a1e9c Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 21:45:16 +0200
Subject: [PATCH 16/26] Drop the comparison docstring

The method is a dunder, and the class does not document its other dunders
either. The expression states the order on its own.
---
 threadmill/backends/base.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/threadmill/backends/base.py b/threadmill/backends/base.py
index 5e82af0..dabdbd8 100644
--- a/threadmill/backends/base.py
+++ b/threadmill/backends/base.py
@@ -70,10 +70,6 @@ class ThreadmillTaskResult(TaskResult):
     lease_token: str | None = None
 
     def __lt__(self, other: ThreadmillTaskResult) -> bool:
-        """Order the results by priority from highest to lowest.
-
-        Two results with the same priority keep their enqueue order.
-        """
         return (-self.task.priority, self.enqueued_at) < (
             -other.task.priority,
             other.enqueued_at,

From 8ee65d529ac79212112275c0e3c1e5e8961edac4 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 21:50:50 +0200
Subject: [PATCH 17/26] Move the fetch loop into its own method

The try block in run now holds one call, so the handler shows what it does.
It stores the failure for the worker process, logs it and raises it again.
The loop that fills the buffer lives in fill_buffer.
---
 threadmill/executor.py | 43 ++++++++++++++++++++++--------------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/threadmill/executor.py b/threadmill/executor.py
index 1d65cd7..63c904d 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -302,33 +302,36 @@ def __init__(
 
     def run(self) -> None:
         try:
-            while not self.stop_requested.is_set() and not self.worker.expired.is_set():
-                try:
-                    batch = self.backend.acquire(
-                        *self.worker.queues,
-                        count=self.prefetch_count,
-                        timeout=self.worker.task_wait_timeout,
-                        worker=self.name,
-                    )
-                except Empty, TimeoutError:
-                    if (
-                        self.worker.exit_empty
-                        or self.worker.shutdown_requested.is_set()
-                    ):
-                        break
-                else:
-                    for task_result in batch:
-                        if not self.buffer(task_result):
-                            break
-                    if self.worker.shutdown_requested.is_set():
-                        break
+            self.fill_buffer()
         except Exception as exception:
+            # The worker process reads this after the join, so a failed fetch
+            # cannot look like a drained queue.
             self.failure = exception
             logger.exception("Task prefetcher '%s' failed", self.name)
             raise
         finally:
             self.finished.set()
 
+    def fill_buffer(self) -> None:
+        """Fill the task buffer until the worker stops or the queue is drained."""
+        while not self.stop_requested.is_set() and not self.worker.expired.is_set():
+            try:
+                batch = self.backend.acquire(
+                    *self.worker.queues,
+                    count=self.prefetch_count,
+                    timeout=self.worker.task_wait_timeout,
+                    worker=self.name,
+                )
+            except Empty, TimeoutError:
+                if self.worker.exit_empty or self.worker.shutdown_requested.is_set():
+                    break
+            else:
+                for task_result in batch:
+                    if not self.buffer(task_result):
+                        break
+                if self.worker.shutdown_requested.is_set():
+                    break
+
     def buffer(self, task_result: ThreadmillTaskResult) -> bool:
         """Buffer one task result. Return False when the prefetcher must stop."""
         while not self.stop_requested.is_set():

From c8ac2f426e376504536601d6eb95bf5fdc87ed46 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 21:58:28 +0200
Subject: [PATCH 18/26] Hand the prefetch outcome over with one future

The fetcher had two signals for one event, an event for completion and a
field for the exception, and a failure was reported three times. It reached
the structured log, came out again through the default thread hook and came
out a third time when the worker raised it.

One Future now carries the outcome. The fetch loop sets an exception on it
or a result, the consumers wait for done, and the worker reads it once after
the join. A failure is logged once with its traceback, and the child exits
with code 1 through SystemExit, which prints nothing of its own.
---
 tests/test_executor.py | 106 ++++++++++++++++++++++++-----------------
 threadmill/executor.py |  28 +++++------
 2 files changed, 77 insertions(+), 57 deletions(-)

diff --git a/tests/test_executor.py b/tests/test_executor.py
index cbcdeae..3fefa10 100644
--- a/tests/test_executor.py
+++ b/tests/test_executor.py
@@ -9,6 +9,7 @@
 import threading
 import time
 import uuid
+from concurrent.futures import Future
 
 import pytest
 from django.tasks import (
@@ -114,7 +115,7 @@ def _prefetched_thread(
     worker.expired = threading.Event()
     prefetcher = TaskPrefetcher(worker=worker, backend=backend, prefetch_count=1)
     prefetcher.buffer(task_result)
-    prefetcher.finished.set()
+    prefetcher.completion.set_result(None)
     worker.prefetcher = prefetcher
     return WorkerThread(worker=worker, index=0, backend=backend)
 
@@ -158,16 +159,17 @@ def __init__(self, *, exit_empty: bool = False) -> None:
         self.shutdown_requested = threading.Event()
 
 
-class CountingEvent(threading.Event):
-    """Event that counts is_set() checks to observe consumer poll cycles."""
+class CountingFuture(Future):
+    """Future that counts done() checks to observe consumer poll cycles."""
 
     def __init__(self) -> None:
         super().__init__()
         self.checks = 0
 
-    def is_set(self) -> bool:
+    def done(self) -> bool:
+        """Count the check before reporting the future state."""
         self.checks += 1
-        return super().is_set()
+        return super().done()
 
 
 def _make_prefetcher(
@@ -617,10 +619,8 @@ def test_run__applies_log_formatter_and_stops(self):
         assert not run_thread.is_alive()
         assert handler.formatter is worker.log_formatter
 
-    def test_run__raises_when_prefetcher_fails(self, monkeypatch):
-        """Re-raise a prefetch failure so the worker process exits non-zero."""
-        thread_failures = []
-        monkeypatch.setattr(threading, "excepthook", thread_failures.append)
+    def test_run__exits_when_prefetcher_fails(self, monkeypatch):
+        """Exit with SystemExit(1) after the prefetcher recorded a fetch failure."""
         monkeypatch.setattr(
             "threadmill.executor.WorkerProcess.task_wait_timeout",
             datetime.timedelta(seconds=0.01),
@@ -632,14 +632,12 @@ def test_run__raises_when_prefetcher_fails(self, monkeypatch):
             log_formatter=JsonFormatter(),
         )
 
-        with pytest.raises(RuntimeError, match="backend unavailable"):
+        with pytest.raises(SystemExit) as exit_info:
             worker.run()
 
+        assert exit_info.value.code == 1
         assert worker.prefetcher is not None
-        assert isinstance(worker.prefetcher.failure, RuntimeError)
-        assert worker.prefetcher.finished.is_set()
-        assert len(thread_failures) == 1
-        assert isinstance(thread_failures[0].exc_value, RuntimeError)
+        assert isinstance(worker.prefetcher.completion.exception(), RuntimeError)
 
     def test_run__child_exits_non_zero_on_prefetch_failure(self, capfd):
         """A child whose prefetcher failed exits non-zero and logs the failure."""
@@ -658,6 +656,25 @@ def test_run__child_exits_non_zero_on_prefetch_failure(self, capfd):
         assert worker.exitcode == 1
         assert "exits after a fetch failure" in capfd.readouterr().out
 
+    def test_run__reports_fetch_failure_once(self, capfd):
+        """Report a child's fetch failure in exactly one traceback."""
+        worker = WorkerProcess(
+            thread_count=1,
+            backend_alias="stub",
+            queues=("default",),
+            log_formatter=JsonFormatter(),
+        )
+
+        worker.start()
+        worker.join(timeout=5)
+        if worker.is_alive():
+            worker.terminate()
+
+        captured = capfd.readouterr()
+        assert worker.exitcode == 1
+        output = captured.out + captured.err
+        assert output.count("Traceback (most recent call last):") == 1, output
+
 
 class TestWorkerThread:
     """Tests for the WorkerThread class."""
@@ -801,29 +818,33 @@ def test_run__acknowledges_failed_task_when_callback_raises(self) -> None:
         result = default_task_backend.get_result(enqueued.id)
         assert result.status == TaskResultStatus.FAILED
 
-    def test_run__returns_when_buffer_drained_and_finished(self) -> None:
-        """run() returns once the buffer is drained and the prefetcher finished."""
+    def test_run__returns_when_buffer_drained_and_completion_done(self) -> None:
+        """run() returns once the buffer is drained and the prefetcher completed."""
         worker = _make_worker(max_tasks=1)
         worker.lock = threading.Lock()
         worker.expired = threading.Event()
         prefetcher = TaskPrefetcher(
             worker=worker, backend=default_task_backend, prefetch_count=1
         )
-        prefetcher.finished.set()
+        prefetcher.completion.set_result(None)
         worker.prefetcher = prefetcher
 
         WorkerThread(worker=worker, index=0, backend=default_task_backend).run()
 
-    def test_run__waits_for_buffer_until_finished(self) -> None:
+    def test_run__waits_for_buffer_until_completion_done(self, monkeypatch) -> None:
         """run() keeps polling an empty buffer while the prefetcher is alive."""
+        monkeypatch.setattr(
+            "threadmill.executor.WorkerProcess.task_wait_timeout",
+            datetime.timedelta(seconds=0.01),
+        )
         worker = _make_worker(max_tasks=1)
         worker.lock = threading.Lock()
         worker.expired = threading.Event()
         prefetcher = TaskPrefetcher(
             worker=worker, backend=default_task_backend, prefetch_count=1
         )
-        finished = CountingEvent()
-        prefetcher.finished = finished
+        completion = CountingFuture()
+        prefetcher.completion = completion
         worker.prefetcher = prefetcher
 
         reader = threading.Thread(
@@ -834,15 +855,15 @@ def test_run__waits_for_buffer_until_finished(self) -> None:
         )
         reader.start()
         deadline = time.monotonic() + 2
-        while finished.checks < 1 and time.monotonic() < deadline:
+        while completion.checks < 1 and time.monotonic() < deadline:
             time.sleep(0.005)
         assert reader.is_alive()
 
-        finished.set()
+        completion.set_result(None)
         reader.join(timeout=1)
 
         assert not reader.is_alive()
-        assert finished.checks >= 2
+        assert completion.checks >= 2
 
 
 class TestTaskPrefetcher:
@@ -867,8 +888,8 @@ def test_run__buffers_until_stop_requested(self):
             prefetcher.stop_requested.set()
             thread.join(timeout=2)
         assert not thread.is_alive()
-        assert prefetcher.finished.is_set()
-        assert prefetcher.failure is None
+        assert prefetcher.completion.done()
+        assert prefetcher.completion.exception() is None
         assert [prefetcher.task_buffer.get_nowait().id for _ in range(2)] == [
             first.id,
             second.id,
@@ -883,8 +904,8 @@ def test_run__stops_on_expired(self):
         prefetcher.worker.expired.set()
         prefetcher.run()
 
-        assert prefetcher.finished.is_set()
-        assert prefetcher.failure is None
+        assert prefetcher.completion.done()
+        assert prefetcher.completion.exception() is None
         assert backend.calls == []
 
     def test_run__stops_on_stop_requested(self):
@@ -895,7 +916,7 @@ def test_run__stops_on_stop_requested(self):
         prefetcher.stop_requested.set()
         prefetcher.run()
 
-        assert prefetcher.finished.is_set()
+        assert prefetcher.completion.done()
         assert backend.calls == []
 
     def test_run__acquires_a_full_buffer(self):
@@ -910,7 +931,7 @@ def test_run__acquires_a_full_buffer(self):
         assert backend.calls[0]["count"] == 5
         assert prefetcher.task_buffer.get_nowait().id == task_result.id
         assert len(backend.calls) == 1
-        assert prefetcher.finished.is_set()
+        assert prefetcher.completion.done()
 
     def test_run__stops_after_batch_when_shutdown_requested(self):
         """Buffer one final batch, then stop when a shutdown was requested."""
@@ -927,7 +948,7 @@ def test_run__stops_after_batch_when_shutdown_requested(self):
             second.id,
         ]
         assert len(backend.calls) == 1
-        assert prefetcher.finished.is_set()
+        assert prefetcher.completion.done()
 
     def test_run__stops_on_empty_when_exit_empty(self):
         """Break out of the fetch loop when the queue drained and exit_empty is set."""
@@ -937,8 +958,8 @@ def test_run__stops_on_empty_when_exit_empty(self):
         prefetcher.run()
 
         assert len(backend.calls) == 1
-        assert prefetcher.finished.is_set()
-        assert prefetcher.failure is None
+        assert prefetcher.completion.done()
+        assert prefetcher.completion.exception() is None
 
     def test_run__stops_on_empty_when_shutdown_requested(self):
         """Break out of the fetch loop when a shutdown was requested."""
@@ -949,18 +970,17 @@ def test_run__stops_on_empty_when_shutdown_requested(self):
         prefetcher.run()
 
         assert len(backend.calls) == 1
-        assert prefetcher.finished.is_set()
+        assert prefetcher.completion.done()
 
-    def test_run__records_failure_and_reraises(self):
-        """Record a fetch failure, mark the fetcher finished, and re-raise."""
-        backend = StubPrefetchBackend(RuntimeError("backend unavailable"))
-        prefetcher = _make_prefetcher(backend)
+    def test_run__records_failure_on_completion(self):
+        """Record a fetch failure on the completion future without raising."""
+        failure = RuntimeError("backend unavailable")
+        prefetcher = _make_prefetcher(StubPrefetchBackend(failure))
 
-        with pytest.raises(RuntimeError, match="backend unavailable"):
-            prefetcher.run()
+        prefetcher.run()
 
-        assert isinstance(prefetcher.failure, RuntimeError)
-        assert prefetcher.finished.is_set()
+        assert prefetcher.completion.done()
+        assert prefetcher.completion.exception() is failure
 
     def test_buffer__returns_false_when_full_and_stop_requested(self):
         """Abandon a full buffer as soon as the prefetcher must stop."""
@@ -979,8 +999,8 @@ def test_buffer__returns_false_when_full_and_stop_requested(self):
             thread.join(timeout=3)
 
         assert not thread.is_alive()
-        assert prefetcher.finished.is_set()
-        assert prefetcher.failure is None
+        assert prefetcher.completion.done()
+        assert prefetcher.completion.exception() is None
         assert prefetcher.task_buffer.qsize() == 1
         assert prefetcher.task_buffer.get_nowait().id == buffered.id
 
diff --git a/threadmill/executor.py b/threadmill/executor.py
index 63c904d..2b40939 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -13,7 +13,7 @@
 import threading
 import time
 import typing
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import Future, ThreadPoolExecutor
 from inspect import iscoroutinefunction
 from queue import Empty, Full
 from traceback import format_exception
@@ -257,9 +257,14 @@ def run(self) -> None:
             consumer_thread.join(join_timeout)
         self.prefetcher.stop_requested.set()
         self.prefetcher.join(join_timeout)
-        if (failure := self.prefetcher.failure) is not None:
-            logger.error("Worker process %s exits after a fetch failure", self.name)
-            raise failure
+        completion = self.prefetcher.completion
+        if completion.done() and (failure := completion.exception()) is not None:
+            logger.error(
+                "Worker process %s exits after a fetch failure",
+                self.name,
+                exc_info=failure,
+            )
+            raise SystemExit(1)
 
     def record_task(self) -> None:
         """Record one processed task and stop when max_tasks is reached."""
@@ -296,21 +301,16 @@ def __init__(
         self.task_buffer: queue.PriorityQueue[ThreadmillTaskResult] = (
             queue.PriorityQueue(maxsize=prefetch_count)
         )
-        self.finished = threading.Event()
+        self.completion: Future[None] = Future()
         self.stop_requested = threading.Event()
-        self.failure: Exception | None = None
 
     def run(self) -> None:
         try:
             self.fill_buffer()
         except Exception as exception:
-            # The worker process reads this after the join, so a failed fetch
-            # cannot look like a drained queue.
-            self.failure = exception
-            logger.exception("Task prefetcher '%s' failed", self.name)
-            raise
-        finally:
-            self.finished.set()
+            self.completion.set_exception(exception)
+        else:
+            self.completion.set_result(None)
 
     def fill_buffer(self) -> None:
         """Fill the task buffer until the worker stops or the queue is drained."""
@@ -368,7 +368,7 @@ def run(self) -> None:
                     timeout=self.worker.task_wait_timeout.total_seconds()
                 )
             except Empty:
-                if prefetcher.finished.is_set():
+                if prefetcher.completion.done():
                     return
                 continue
 

From 1bd3f8a5b67186b04004f122fa8c5aad5254c1fa Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 22:06:02 +0200
Subject: [PATCH 19/26] Reduce the prefetch note to one paragraph

The section explained the same feature three times over, once as prose,
once as the flag and once as four soft limits. The flag help carries the
size and the default, the Redis options carry the lease effect, and the
health note carries the max-tasks overshoot, so the README keeps one
paragraph that says what the extra thread does and which flag sizes it.
---
 README.md | 19 ++-----------------
 1 file changed, 2 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 76f137a..bc0f06f 100644
--- a/README.md
+++ b/README.md
@@ -85,23 +85,8 @@ Processes allow for parallel compute (no GIL) while threads are great for low-me
 uv run manage.py threadmill worker --workers 4 --threads 2
 ```
 
-Each worker process runs one fetcher thread.
-The fetcher reserves a batch of tasks in one broker round trip, and the worker threads drain that batch.
-This keeps the worker threads busy while the broker answers.
-A full buffer blocks the fetcher until a thread frees a slot.
-The gain is largest when the broker is far from the worker.
-Against a local broker the buffer is worth about a fifth of the throughput.
-
-Set the batch size with `--prefetch-count`.
-It defaults to four times the thread count and applies to each process.
-A value of `1` disables batching.
-
-Prefetching has soft limits:
-
-- A fetched task holds its lease while it waits. The buffer holds at most `--prefetch-count` tasks, so size that value for your workload.
-- A task enqueued after a fetch waits for the buffer to drain. Inside the buffer the highest priority task runs first, and tasks of one priority keep queue order.
-- `--max-tasks` recycles a worker. It is not a hard limit. The buffer and the batch in hand still run, so a worker can run about twice `--prefetch-count` tasks more than its budget.
-- `worker_ids` records the fetcher of the process and not the thread that runs the task.
+Each worker process runs a fetcher thread that leases a batch of tasks ahead of the worker threads.
+Set the batch size with `--prefetch-count`, four times the thread count by default.
 
 #### Health
 

From 5d46ecbbe5959ab1721efdbe19ade978003e8c01 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 22:10:23 +0200
Subject: [PATCH 20/26] Work buffered tasks off in lease order

The buffer is a plain queue again, and the comparison on
ThreadmillTaskResult is gone with it. A worker takes the task that the
fetcher leased first, which is the order the queue handed them out.

The ordering rule is unchanged from before prefetching. The backend still
picks each queue's head by priority and enqueue time, and the buffer now
preserves that lease order instead of reordering it.
---
 tests/test_executor.py      | 26 ++++++++------------------
 threadmill/backends/base.py | 13 +++----------
 threadmill/executor.py      |  4 ++--
 3 files changed, 13 insertions(+), 30 deletions(-)

diff --git a/tests/test_executor.py b/tests/test_executor.py
index 3fefa10..69cf519 100644
--- a/tests/test_executor.py
+++ b/tests/test_executor.py
@@ -1012,27 +1012,17 @@ def test_buffer__returns_true_when_space_available(self):
         assert prefetcher.buffer(task_result) is True
         assert prefetcher.task_buffer.get_nowait().id == task_result.id
 
-    def test_buffer__dispatches_highest_priority_first(self):
-        """Hand out the highest priority task first, earlier enqueued on ties."""
-        enqueued_at = timezone.now()
-        low = _task_result(dataclasses.replace(echo, priority=1), 1)
-        first_high = dataclasses.replace(
-            _task_result(dataclasses.replace(echo, priority=5), 2),
-            enqueued_at=enqueued_at,
-        )
-        middle = _task_result(dataclasses.replace(echo, priority=3), 3)
-        later_high = dataclasses.replace(
-            _task_result(dataclasses.replace(echo, priority=5), 4),
-            enqueued_at=enqueued_at + datetime.timedelta(seconds=1),
-        )
+    def test_buffer__dispatches_in_lease_order(self):
+        """Hand out the tasks in the order the fetcher leased them."""
+        leased = [
+            _task_result(dataclasses.replace(echo, priority=priority), value)
+            for value, priority in enumerate((1, 5, 3, 5))
+        ]
         prefetcher = _make_prefetcher(StubPrefetchBackend())
 
-        for task_result in (low, first_high, middle, later_high):
+        for task_result in leased:
             assert prefetcher.buffer(task_result) is True
 
         assert [prefetcher.task_buffer.get_nowait().id for _ in range(4)] == [
-            first_high.id,
-            later_high.id,
-            middle.id,
-            low.id,
+            task_result.id for task_result in leased
         ]
diff --git a/threadmill/backends/base.py b/threadmill/backends/base.py
index dabdbd8..7bb325c 100644
--- a/threadmill/backends/base.py
+++ b/threadmill/backends/base.py
@@ -69,12 +69,6 @@ def __reduce__(self):
 class ThreadmillTaskResult(TaskResult):
     lease_token: str | None = None
 
-    def __lt__(self, other: ThreadmillTaskResult) -> bool:
-        return (-self.task.priority, self.enqueued_at) < (
-            -other.task.priority,
-            other.enqueued_at,
-        )
-
     @classmethod
     def from_result(
         cls, task_result: TaskResult, *, lease_token: str | None
@@ -285,10 +279,9 @@ def acquire(
             queue.Empty: The first task is unavailable and the timeout is None.
 
         Returns:
-            Between one and `count` leased task results. The list is never empty.
-            Each result is a `ThreadmillTaskResult` that orders by priority and
-            enqueue time. The prefetch buffer uses this order to select the next
-            task.
+            Between one and `count` leased task results in lease order. The list is
+            never empty. Each result is a `ThreadmillTaskResult` that carries the
+            lease token for its acknowledgement.
         """
         raise NotImplementedError
 
diff --git a/threadmill/executor.py b/threadmill/executor.py
index 2b40939..a239d27 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -298,8 +298,8 @@ def __init__(
         self.worker = worker
         self.backend = backend
         self.prefetch_count = prefetch_count
-        self.task_buffer: queue.PriorityQueue[ThreadmillTaskResult] = (
-            queue.PriorityQueue(maxsize=prefetch_count)
+        self.task_buffer: queue.Queue[ThreadmillTaskResult] = queue.Queue(
+            maxsize=prefetch_count
         )
         self.completion: Future[None] = Future()
         self.stop_requested = threading.Event()

From 97f4619184bd8c05adf79f68928d937b9531bd39 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 22:10:49 +0200
Subject: [PATCH 21/26] Trim the prefetch flag help

The flag name and its default are enough for the help. The lease effect
lives with the Redis options and the max-tasks overshoot lives in the
health note, so the help no longer repeats them.
---
 threadmill/management/commands/threadmill.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/threadmill/management/commands/threadmill.py b/threadmill/management/commands/threadmill.py
index fe7af47..ed2eae7 100644
--- a/threadmill/management/commands/threadmill.py
+++ b/threadmill/management/commands/threadmill.py
@@ -75,10 +75,6 @@ def add_arguments(self, parser):
             help=(
                 "Number of tasks to prefetch per worker process."
                 " Defaults to 4 × the number of threads. Use 1 to disable batching."
-                " Buffered tasks hold their lease while they wait, so a deep buffer"
-                " needs a matching lease_ttl."
-                " Buffered tasks and the batch in hand still run after a worker reaches"
-                " its max-tasks limit."
             ),
         )
         parser.add_argument(

From ea9b8d2d1a5d35d89b3136e884f2a1e9c6c968b8 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 22:15:51 +0200
Subject: [PATCH 22/26] Resolve the completion future on every exit path

The future replaced a finished event that a finally block always set, so
the new code lost that guarantee. A BaseException escaping the fetch loop
left the future unresolved, and a consumer waits for a result that never
comes. The finally block now resolves the future when the handler has not,
and the BaseException still escapes the thread.
---
 tests/test_executor.py | 14 ++++++++++++--
 threadmill/executor.py |  7 +++++--
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/tests/test_executor.py b/tests/test_executor.py
index 69cf519..8dc79bc 100644
--- a/tests/test_executor.py
+++ b/tests/test_executor.py
@@ -123,7 +123,7 @@ def _prefetched_thread(
 class StubPrefetchBackend:
     """Scripted backend stub for prefetcher tests without broker round-trips."""
 
-    def __init__(self, *responses: list[TaskResult] | Exception) -> None:
+    def __init__(self, *responses: list[TaskResult] | BaseException) -> None:
         self.responses = list(responses)
         self.calls: list[dict] = []
 
@@ -138,7 +138,7 @@ def acquire(self, *queue_names, count=1, timeout=None, worker=""):
             }
         )
         response = self.responses.pop(0) if self.responses else TimeoutError("drained")
-        if isinstance(response, Exception):
+        if isinstance(response, BaseException):
             raise response
         return response
 
@@ -982,6 +982,16 @@ def test_run__records_failure_on_completion(self):
         assert prefetcher.completion.done()
         assert prefetcher.completion.exception() is failure
 
+    def test_run__resolves_completion_on_base_exception(self):
+        """Resolve the completion future when a BaseException escapes the fetch."""
+        prefetcher = _make_prefetcher(StubPrefetchBackend(KeyboardInterrupt("stop")))
+
+        with pytest.raises(KeyboardInterrupt):
+            prefetcher.run()
+
+        assert prefetcher.completion.done()
+        assert prefetcher.completion.exception() is None
+
     def test_buffer__returns_false_when_full_and_stop_requested(self):
         """Abandon a full buffer as soon as the prefetcher must stop."""
         buffered = _task_result(echo, 1)
diff --git a/threadmill/executor.py b/threadmill/executor.py
index a239d27..5d18b84 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -309,8 +309,11 @@ def run(self) -> None:
             self.fill_buffer()
         except Exception as exception:
             self.completion.set_exception(exception)
-        else:
-            self.completion.set_result(None)
+        finally:
+            # A BaseException would leave the consumers waiting for a result
+            # that never comes.
+            if not self.completion.done():
+                self.completion.set_result(None)
 
     def fill_buffer(self) -> None:
         """Fill the task buffer until the worker stops or the queue is drained."""

From a96fb49905d1a106dfbbb4a6cb8ac9aa2f401f66 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 22:16:04 +0200
Subject: [PATCH 23/26] Drop the comment on the completion guard

The guard reads on its own.
---
 threadmill/executor.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/threadmill/executor.py b/threadmill/executor.py
index 5d18b84..71615f0 100644
--- a/threadmill/executor.py
+++ b/threadmill/executor.py
@@ -310,8 +310,6 @@ def run(self) -> None:
         except Exception as exception:
             self.completion.set_exception(exception)
         finally:
-            # A BaseException would leave the consumers waiting for a result
-            # that never comes.
             if not self.completion.done():
                 self.completion.set_result(None)
 

From b6b6a5f399da16c5c0015004ef200364e7dee81f Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 22:16:24 +0200
Subject: [PATCH 24/26] Drop the prefetch paragraph

The flag help documents the setting, so the README does not repeat it.
---
 README.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/README.md b/README.md
index bc0f06f..fbe285e 100644
--- a/README.md
+++ b/README.md
@@ -85,9 +85,6 @@ Processes allow for parallel compute (no GIL) while threads are great for low-me
 uv run manage.py threadmill worker --workers 4 --threads 2
 ```
 
-Each worker process runs a fetcher thread that leases a batch of tasks ahead of the worker threads.
-Set the batch size with `--prefetch-count`, four times the thread count by default.
-
 #### Health
 
 If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:

From 38c558d7e2fe5abf5d911bd43bb7f751322047b6 Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 22:25:27 +0200
Subject: [PATCH 25/26] Spawn the child in the prefetch failure tests

A forkserver child inherits the stdout of the long-lived forkserver
instead of the file descriptor capfd replaces, so the fetch failure
report never reached the fixture and both tests failed on Linux, where
Python 3.14 defaults to forkserver. Force spawn for the duration of the
child, like test_run__routes_task_logs_to_stdout does.
---
 tests/test_executor.py | 56 +++++++++++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 20 deletions(-)

diff --git a/tests/test_executor.py b/tests/test_executor.py
index 8dc79bc..45bccfb 100644
--- a/tests/test_executor.py
+++ b/tests/test_executor.py
@@ -641,34 +641,50 @@ def test_run__exits_when_prefetcher_fails(self, monkeypatch):
 
     def test_run__child_exits_non_zero_on_prefetch_failure(self, capfd):
         """A child whose prefetcher failed exits non-zero and logs the failure."""
-        worker = WorkerProcess(
-            thread_count=1,
-            backend_alias="stub",
-            queues=("default",),
-            log_formatter=JsonFormatter(),
-        )
+        original_start_method = multiprocessing.get_start_method()
+        # A forkserver worker inherits the stdout of the long-lived forkserver
+        # instead of the file descriptor this fixture replaces, so its records
+        # would never reach capfd.
+        multiprocessing.set_start_method("spawn", force=True)
+        try:
+            worker = WorkerProcess(
+                thread_count=1,
+                backend_alias="stub",
+                queues=("default",),
+                log_formatter=JsonFormatter(),
+            )
 
-        worker.start()
-        worker.join(timeout=5)
-        if worker.is_alive():
-            worker.terminate()
+            worker.start()
+            worker.join(timeout=5)
+            if worker.is_alive():
+                worker.terminate()
+        finally:
+            multiprocessing.set_start_method(original_start_method, force=True)
 
         assert worker.exitcode == 1
         assert "exits after a fetch failure" in capfd.readouterr().out
 
     def test_run__reports_fetch_failure_once(self, capfd):
         """Report a child's fetch failure in exactly one traceback."""
-        worker = WorkerProcess(
-            thread_count=1,
-            backend_alias="stub",
-            queues=("default",),
-            log_formatter=JsonFormatter(),
-        )
+        original_start_method = multiprocessing.get_start_method()
+        # A forkserver worker inherits the stdout of the long-lived forkserver
+        # instead of the file descriptor this fixture replaces, so its records
+        # would never reach capfd.
+        multiprocessing.set_start_method("spawn", force=True)
+        try:
+            worker = WorkerProcess(
+                thread_count=1,
+                backend_alias="stub",
+                queues=("default",),
+                log_formatter=JsonFormatter(),
+            )
 
-        worker.start()
-        worker.join(timeout=5)
-        if worker.is_alive():
-            worker.terminate()
+            worker.start()
+            worker.join(timeout=5)
+            if worker.is_alive():
+                worker.terminate()
+        finally:
+            multiprocessing.set_start_method(original_start_method, force=True)
 
         captured = capfd.readouterr()
         assert worker.exitcode == 1

From dc339844384b3cf466d70bbc78bcd08a839e6f9f Mon Sep 17 00:00:00 2001
From: Johannes Maron <johannes@maron.family>
Date: Wed, 7 Oct 2026 22:35:05 +0200
Subject: [PATCH 26/26] Refresh the comparison charts

Regenerated from a run on the current branch. Threadmill holds 11,977
tasks per second, dramatiq 7,168, celery 2,183, django-tasks-db 2,154 and
django-tasks-rq 87, which is within a few percent of the previous chart
for every row.

The no-prefetch ablation moved by a quarter, from the one-second
quantization of the worker start and stop rather than a change in the
drain. Its start mean fell by a second while its process mean rose by the
same second, so the row is noisy at this depth and the chart does not
plot it.
---
 README.md                                |  2 +-
 docs/images/backend-comparison-dark.svg  | 20 ++++++++++----------
 docs/images/backend-comparison-light.svg | 20 ++++++++++----------
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/README.md b/README.md
index fbe285e..8ef46a7 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-dark.svg">
     <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
-    <img alt="Tasks per second with one worker: threadmill 11,973, dramatiq 7,331, celery 2,307, django-tasks-db 2,080, django-tasks-rq 90." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
+    <img alt="Tasks per second with one worker: threadmill 11,977, dramatiq 7,168, celery 2,183, django-tasks-db 2,154, django-tasks-rq 87." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/backend-comparison-light.svg">
   </picture>
 </p>
 
diff --git a/docs/images/backend-comparison-dark.svg b/docs/images/backend-comparison-dark.svg
index c69f4c9..c0ce3a5 100644
--- a/docs/images/backend-comparison-dark.svg
+++ b/docs/images/backend-comparison-dark.svg
@@ -1,22 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 11,973, dramatiq 7,331, celery 2,307, django-tasks-db 2,080, django-tasks-rq 90.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 11,977, dramatiq 7,168, celery 2,183, django-tasks-db 2,154, django-tasks-rq 87.">
 <style>svg{max-width:100%;height:auto}</style>
 <rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#0d1117" stroke="#30363d"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#e6edf3" text-anchor="start">Queue throughput</text>
 <text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#8b949e" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
 <text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#e6edf3" text-anchor="end">threadmill</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#6366f1" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">11,973/s</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#818cf8" text-anchor="start">11,977/s</text>
 <text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">dramatiq</text>
-<rect x="200" y="140" width="342.90" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="552.899" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">7,331/s</text>
+<rect x="200" y="140" width="335.14" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="545.141" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">7,168/s</text>
 <text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">celery</text>
-<rect x="200" y="180" width="107.91" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="317.91" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,307/s</text>
+<rect x="200" y="180" width="102.07" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="312.07" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,183/s</text>
 <text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="220" width="97.30" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="307.295" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,080/s</text>
+<rect x="200" y="220" width="100.72" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="310.723" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">2,154/s</text>
 <text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#8b949e" text-anchor="end">django-tasks-rq</text>
-<rect x="200" y="260" width="4.23" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
-<text x="214.23" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">90/s</text>
+<rect x="200" y="260" width="4.08" height="20" rx="5" fill="#8b949e" opacity="0.25"/>
+<text x="214.079" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#8b949e" text-anchor="start">87/s</text>
 <text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#6e7681" text-anchor="start">One process and one thread each. Threadmill, celery and dramatiq read 128 ahead; django-tasks-db and -rq read one message at a time.</text>
 </svg>
diff --git a/docs/images/backend-comparison-light.svg b/docs/images/backend-comparison-light.svg
index 2c1fd71..a69b065 100644
--- a/docs/images/backend-comparison-light.svg
+++ b/docs/images/backend-comparison-light.svg
@@ -1,22 +1,22 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 11,973, dramatiq 7,331, celery 2,307, django-tasks-db 2,080, django-tasks-rq 90.">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 372" width="900" height="372" role="img" aria-label="Tasks per second with one worker: threadmill 11,977, dramatiq 7,168, celery 2,183, django-tasks-db 2,154, django-tasks-rq 87.">
 <style>svg{max-width:100%;height:auto}</style>
 <rect x="0.5" y="0.5" width="899" height="371" rx="14" fill="#ffffff" stroke="#e5e7eb"/>
 <text x="28" y="46" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="19" font-weight="700" fill="#111827" text-anchor="start">Queue throughput</text>
 <text x="28" y="68" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="12.5" font-weight="400" fill="#6b7280" text-anchor="start">5,000–60,000 trivial tasks per queue · one worker process, one thread · higher is better</text>
 <text x="180" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="700" fill="#111827" text-anchor="end">threadmill</text>
 <rect x="200" y="100" width="560.00" height="20" rx="5" fill="#4f46e5" opacity="1"/>
-<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">11,973/s</text>
+<text x="770" y="115" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="700" fill="#4f46e5" text-anchor="start">11,977/s</text>
 <text x="180" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">dramatiq</text>
-<rect x="200" y="140" width="342.90" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="552.899" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">7,331/s</text>
+<rect x="200" y="140" width="335.14" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="545.141" y="155" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">7,168/s</text>
 <text x="180" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">celery</text>
-<rect x="200" y="180" width="107.91" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="317.91" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,307/s</text>
+<rect x="200" y="180" width="102.07" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="312.07" y="195" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,183/s</text>
 <text x="180" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-db</text>
-<rect x="200" y="220" width="97.30" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="307.295" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,080/s</text>
+<rect x="200" y="220" width="100.72" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="310.723" y="235" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">2,154/s</text>
 <text x="180" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="14" font-weight="400" fill="#6b7280" text-anchor="end">django-tasks-rq</text>
-<rect x="200" y="260" width="4.23" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
-<text x="214.23" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">90/s</text>
+<rect x="200" y="260" width="4.08" height="20" rx="5" fill="#6b7280" opacity="0.25"/>
+<text x="214.079" y="275" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="13" font-weight="400" fill="#6b7280" text-anchor="start">87/s</text>
 <text x="28" y="308" font-family='system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' font-size="11.5" font-weight="400" fill="#9ca3af" text-anchor="start">One process and one thread each. Threadmill, celery and dramatiq read 128 ahead; django-tasks-db and -rq read one message at a time.</text>
 </svg>