From 944bccf5b9175ec12c80beb7b122e5c8c064e991 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Wed, 2 Sep 2026 14:23:34 -0400 Subject: [PATCH 1/2] Count requests still setting up in the autoscaler metric The evictor counted only requests in its manager map, which they enter after session acquisition and store setup. Admission and substreams_active_requests count them from arrival, so the autoscaler metric could read below the plain count it is meant to replace. --- docs/release-notes/change-log.md | 5 +++-- service/active_requests/evictor.go | 25 +++++++++++++++++++++---- service/active_requests/evictor_test.go | 11 +++++++++++ service/tier1.go | 1 + 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/docs/release-notes/change-log.md b/docs/release-notes/change-log.md index cace098f6..bf4eeeff5 100644 --- a/docs/release-notes/change-log.md +++ b/docs/release-notes/change-log.md @@ -37,8 +37,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), horizontal autoscaler input on tier1: the higher of the plain active-request count and the number of requests the CPU budget is being spent at (`nominal_capacity * cpu_usage_ratio / cpu_eviction_target_ratio`). A pod full of expensive requests, or one holding its CPU down by eviction, reports itself at capacity, so the autoscaler will - add more pods. `CPUEviction.NominalCapacity` should be set to the autoscaler's per-pod request target; it defaults - to the active-requests soft limit. + add more pods. Its active-request count is the one admission uses, so it includes requests still setting up and + never reads below `substreams_active_requests`. `CPUEviction.NominalCapacity` should be set to the autoscaler's + per-pod request target; it defaults to the active-requests soft limit. - Trimmed tier2's per-segment logging: a large backfill fans out into tens of thousands of `ProcessRange` calls, and each one was logging ~10 `Info` lines with no steady-state diagnostic value, which could spike a pod's log diff --git a/service/active_requests/evictor.go b/service/active_requests/evictor.go index 00e281a4f..8e0be2eaf 100644 --- a/service/active_requests/evictor.go +++ b/service/active_requests/evictor.go @@ -145,8 +145,9 @@ type Evictor struct { reader *CPUReader logger *zap.Logger - overloaded atomic.Bool - onEvaluate func() + overloaded atomic.Bool + onEvaluate func() + countActiveRequests func() int // zero time = condition not currently holding overloadSince time.Time @@ -193,6 +194,22 @@ func (ev *Evictor) OnEvaluate(fn func()) { ev.onEvaluate = fn } +// CountActiveRequestsWith overrides how the evictor counts the pod's active +// requests when it publishes substreams_tier1_effective_active_requests. The +// manager's map only holds requests that finished setting up, while admission +// and substreams_active_requests count them from the moment they arrive, and +// the autoscaler metric has to agree with those. Must be called before Run. +func (ev *Evictor) CountActiveRequestsWith(fn func() int) { + ev.countActiveRequests = fn +} + +func (ev *Evictor) activeRequests(inManager int) int { + if ev.countActiveRequests == nil { + return inManager + } + return ev.countActiveRequests() +} + func (ev *Evictor) notify() { if ev.onEvaluate != nil { ev.onEvaluate() @@ -227,9 +244,9 @@ func (ev *Evictor) tick(now time.Time) { ev.clearOverload() return } - candidates, totalActive := ev.sampleBurnRates(now) + candidates, inManager := ev.sampleBurnRates(now) firing := ev.classify(signals, now) - ev.publishMetrics(signals, totalActive) + ev.publishMetrics(signals, ev.activeRequests(inManager)) ev.notify() if !ev.overloaded.Load() { diff --git a/service/active_requests/evictor_test.go b/service/active_requests/evictor_test.go index ea4be0d3a..7782ea6ca 100644 --- a/service/active_requests/evictor_test.go +++ b/service/active_requests/evictor_test.go @@ -205,3 +205,14 @@ func TestTick_UnreadableCgroupClearsOverload(t *testing.T) { assert.False(t, ev.IsOverloaded()) assert.Equal(t, 2, evaluations) } + +func TestActiveRequests_PrefersTheHostCount(t *testing.T) { + ev := newTestEvictor(DefaultEvictorConfig()) + + // without an override the manager's own count is used + assert.Equal(t, 3, ev.activeRequests(3)) + + // the host counts requests still setting up, which the manager does not hold yet + ev.CountActiveRequestsWith(func() int { return 7 }) + assert.Equal(t, 7, ev.activeRequests(3)) +} diff --git a/service/tier1.go b/service/tier1.go index e2258a5a7..c835cd63b 100644 --- a/service/tier1.go +++ b/service/tier1.go @@ -294,6 +294,7 @@ func NewTier1( } evictor := active_requests.NewEvictor(evictorConfig, s.activeRequestsManager, reader, logger) evictor.OnEvaluate(s.refreshReadiness) + evictor.CountActiveRequestsWith(s.getActiveRequestCount) s.evictor = evictor go evictor.Run(s.Terminating()) } From 54f05be6879752ef3603c6f59a3d605940b42bd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 3 Sep 2026 15:54:27 -0400 Subject: [PATCH 2/2] Bump golang.org/x/crypto to v0.56.0 Fixes the two high-severity advisories the Docker Scout check flags on v0.55.0. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4e6583999..82b0784a3 100644 --- a/go.mod +++ b/go.mod @@ -316,7 +316,7 @@ require ( go.opentelemetry.io/otel/metric v1.45.0 // indirect go.opentelemetry.io/otel/sdk v1.45.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.55.0 // indirect + golang.org/x/crypto v0.56.0 // indirect golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 // indirect golang.org/x/term v0.45.0 // indirect diff --git a/go.sum b/go.sum index ba6ccf290..b6d024af1 100644 --- a/go.sum +++ b/go.sum @@ -842,8 +842,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 h1:SbTAbRFnd5kjQXbczszQ0hdk3ctwYf3qBNH9jIsGclE=