From 4599c6a9e7d0f21c8ff0ead2413fa64dab4856d2 Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Thu, 5 Nov 2026 18:04:30 +0700 Subject: [PATCH 1/2] fix(p2p): trim the simple priority queue on every push The simple priority queue trimmed its heap on push only when the signal channel was already pending. A push that refilled an empty signal skipped the trim, so with nobody draining, the heap ran past maxSize until the next 10ms tick. Instrumented under -race, a 2560-envelope flood at capacity 8 took the heap 7 to 18 envelopes past maxSize every time. TestLoadRouterSharedQueueOccupancyAndDrops checks that the queue retains no more than maxSize + cap(input) + cap(output) and failed when the drain caught that overshoot (85 and 87 retained against a ceiling of 84). The ceiling is right for the intended queue, so the queue now trims after every push and the ticker only re-signals. The test comment spells out why the envelope the run loop holds between input and heap cannot add to the count. Co-Authored-By: Claude Opus 5.5 --- internal/p2p/load_router_queue_test.go | 11 ++++++++--- internal/p2p/rqueue.go | 25 ++++++++++++------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/internal/p2p/load_router_queue_test.go b/internal/p2p/load_router_queue_test.go index 4421bc07c..f9fd53544 100644 --- a/internal/p2p/load_router_queue_test.go +++ b/internal/p2p/load_router_queue_test.go @@ -33,9 +33,14 @@ func TestLoadRouterSharedQueueOccupancyAndDrops(t *testing.T) { // queue the router builds, at a size small enough to reach its ceiling. const capacity = 8 q := newSimplePriorityQueue(ctx, capacity) - // Account for the queue's heap and both channel buffers. Depending on - // scheduling, draining can make every one of these retained envelopes - // observable before the queue gets another chance to trim its heap. + // The queue holds envelopes in three places: waiting in the input buffer, + // in the heap, and popped into the output buffer. The heap is trimmed to + // maxSize on every push. The run loop holds one envelope in none of these + // between taking it from input and pushing it, which frees an input slot + // for the producer, but that push trims the heap back to maxSize, so the + // envelope replaces one already held instead of adding to the count. A + // flood that nobody drains fills all three, and draining can then hand + // over every one of them before a later push trims the heap again. ceiling := q.maxSize + cap(q.input) + cap(q.output) // Nothing drains while this runs, which is the case that matters: a diff --git a/internal/p2p/rqueue.go b/internal/p2p/rqueue.go index 669acfede..23b6a1b26 100644 --- a/internal/p2p/rqueue.go +++ b/internal/p2p/rqueue.go @@ -73,19 +73,11 @@ func (q *simpleQueue) run(ctx context.Context) { timestamp: time.Now().UTC(), }) - select { - case signal <- struct{}{}: - default: - if len(pq) > q.maxSize { - sort.Sort(pq) - for _, dropped := range pq[q.maxSize:] { - dropped.envelope.NotifyDelivery() - } - pq = pq[:q.maxSize] - } - } - - case <-ticker.C: + // Trim on every push, so the heap never holds more than + // maxSize. Trimming only when the signal was already pending + // let every push that refilled it skip the trim, so with + // nobody draining, the heap ran past maxSize until the next + // tick. if len(pq) > q.maxSize { sort.Sort(pq) for _, dropped := range pq[q.maxSize:] { @@ -93,6 +85,13 @@ func (q *simpleQueue) run(ctx context.Context) { } pq = pq[:q.maxSize] } + + select { + case signal <- struct{}{}: + default: + } + + case <-ticker.C: if len(pq) > 0 { select { case signal <- struct{}{}: From fb3e02bfc772facdefade06c2eeca6743e23e329 Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Sat, 7 Nov 2026 14:22:00 +0700 Subject: [PATCH 2/2] fix(p2p): trim the simple priority queue below its limit An overflowing heap was sorted whole and trimmed back to exactly maxSize, so the next overflowing push had to sort again, and the run loop accepts and delivers nothing while it sorts. The vote channel's capacity of 4096 gives a limit of 16,777,216 envelopes, where a full queue took 2 to 3 seconds per push with nobody draining it. Trim to 15/16 of the limit so one sort covers the pushes that follow, and clear the dropped slots so those envelopes can be collected. A full queue at that capacity now accepts about 187,000 pushes per second. This changes behaviour under a flood. The cost of sorting used to slow senders to about the consumer's rate, so little was dropped. Senders now rarely wait, and the queue drops what the consumer cannot keep up with. Co-Authored-By: Claude Opus 5.5 --- internal/p2p/load_router_queue_test.go | 15 ++++--- internal/p2p/rqueue.go | 32 +++++++++++--- internal/p2p/rqueue_test.go | 61 ++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 14 deletions(-) diff --git a/internal/p2p/load_router_queue_test.go b/internal/p2p/load_router_queue_test.go index f9fd53544..a1022e67a 100644 --- a/internal/p2p/load_router_queue_test.go +++ b/internal/p2p/load_router_queue_test.go @@ -34,13 +34,14 @@ func TestLoadRouterSharedQueueOccupancyAndDrops(t *testing.T) { const capacity = 8 q := newSimplePriorityQueue(ctx, capacity) // The queue holds envelopes in three places: waiting in the input buffer, - // in the heap, and popped into the output buffer. The heap is trimmed to - // maxSize on every push. The run loop holds one envelope in none of these - // between taking it from input and pushing it, which frees an input slot - // for the producer, but that push trims the heap back to maxSize, so the - // envelope replaces one already held instead of adding to the count. A - // flood that nobody drains fills all three, and draining can then hand - // over every one of them before a later push trims the heap again. + // in the heap, and popped into the output buffer. A push that takes the + // heap past maxSize trims it, so it holds at most maxSize once a push + // finishes. The run loop holds one envelope in none of these between taking + // it from input and pushing it, which frees an input slot for the producer, + // but if that push overflows the heap it also trims it, so the envelope + // cannot add to the count. A flood that nobody drains can fill all three, + // and draining can then hand over every one of them before a later push + // trims the heap again. ceiling := q.maxSize + cap(q.input) + cap(q.output) // Nothing drains while this runs, which is the case that matters: a diff --git a/internal/p2p/rqueue.go b/internal/p2p/rqueue.go index 23b6a1b26..0673cb15a 100644 --- a/internal/p2p/rqueue.go +++ b/internal/p2p/rqueue.go @@ -9,6 +9,10 @@ import ( "github.com/cosmos/gogoproto/proto" ) +// simpleQueueTrimFraction sets how far below maxSize an overflowing heap is +// trimmed: by 1/simpleQueueTrimFraction of maxSize. +const simpleQueueTrimFraction = 16 + type simpleQueue struct { input chan Envelope output chan Envelope @@ -73,17 +77,13 @@ func (q *simpleQueue) run(ctx context.Context) { timestamp: time.Now().UTC(), }) - // Trim on every push, so the heap never holds more than - // maxSize. Trimming only when the signal was already pending + // Check on every push, so the heap never holds more than + // maxSize. Checking only when the signal was already pending // let every push that refilled it skip the trim, so with // nobody draining, the heap ran past maxSize until the next // tick. if len(pq) > q.maxSize { - sort.Sort(pq) - for _, dropped := range pq[q.maxSize:] { - dropped.envelope.NotifyDelivery() - } - pq = pq[:q.maxSize] + pq = q.trim(pq) } select { @@ -117,3 +117,21 @@ func (q *simpleQueue) run(ctx context.Context) { } } } + +// trim drops the lowest-ranked envelopes from a heap that has outgrown maxSize. +// Sorting takes time in proportion to the whole heap, and the run loop accepts +// nothing while it sorts, so trimming back to exactly maxSize would sort again +// on the very next push. Trimming a fraction below it leaves room for the pushes +// that follow, which spreads the cost of one sort across all of them. The sorted +// slice it returns is still a valid heap. +func (q *simpleQueue) trim(pq priorityQueue) priorityQueue { + sort.Sort(pq) + keep := q.maxSize - q.maxSize/simpleQueueTrimFraction + for _, dropped := range pq[keep:] { + dropped.envelope.NotifyDelivery() + } + // Later pushes reclaim these slots only one at a time, so release the + // dropped envelopes now. + clear(pq[keep:]) + return pq[:keep] +} diff --git a/internal/p2p/rqueue_test.go b/internal/p2p/rqueue_test.go index 0ea137ee8..359ac69bf 100644 --- a/internal/p2p/rqueue_test.go +++ b/internal/p2p/rqueue_test.go @@ -1,9 +1,12 @@ package p2p import ( + "container/heap" "context" "testing" "time" + + "github.com/stretchr/testify/require" ) func TestSimpleQueue(t *testing.T) { @@ -45,3 +48,61 @@ RETRY: } } + +// An overflowing heap is trimmed far enough below maxSize that the next pushes +// do not each need another sort. It keeps the highest-ranked envelopes, stays a +// valid heap, and releases both the senders waiting on what it drops and the +// memory those envelopes hold. +func TestSimpleQueueTrimDropsLowestRankedBelowMaxSize(t *testing.T) { + testCases := []struct { + name string + maxSize int + keep int + }{ + {name: "large enough for a margin", maxSize: 64, keep: 60}, + {name: "too small for a margin", maxSize: 4, keep: 4}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + q := &simpleQueue{maxSize: tc.maxSize} + overflow := tc.maxSize + 1 + + // Timestamps this far apart decide the rank on their own, and the + // newest ranks highest, so the oldest are the ones to lose. + start := time.Now() + pq := make(priorityQueue, 0, overflow) + delivered := make([]<-chan struct{}, overflow) + timestamps := make([]time.Time, overflow) + for i := range overflow { + pqEnv := &pqEnvelope{timestamp: start.Add(time.Duration(i) * time.Second)} + delivered[i] = pqEnv.envelope.EnableDeliveryNotification() + timestamps[i] = pqEnv.timestamp + heap.Push(&pq, pqEnv) + } + + trimmed := q.trim(pq) + require.Len(t, trimmed, tc.keep) + + firstKept := overflow - tc.keep + for i, done := range delivered { + select { + case <-done: + require.Less(t, i, firstKept, "released the sender of a kept envelope") + default: + require.GreaterOrEqual(t, i, firstKept, "did not release the sender of a dropped envelope") + } + } + for _, slot := range trimmed[tc.keep:overflow] { + require.Nil(t, slot, "a dropped envelope is still reachable") + } + + for i := overflow - 1; i >= firstKept; i-- { + next, ok := heap.Pop(&trimmed).(*pqEnvelope) + require.True(t, ok) + require.True(t, timestamps[i].Equal(next.timestamp), "the trimmed heap is out of order") + } + require.Empty(t, trimmed) + }) + } +}