Count requests still setting up in the eviction autoscaler metric

docs/release-notes/change-log.md

@@ -37,8 +37,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
   horizontal autoscaler input on tier1: the higher of the plain active-request count and the number of requests the
   CPU budget is being spent at (`nominal_capacity * cpu_usage_ratio / cpu_eviction_target_ratio`). A pod full of
   expensive requests, or one holding its CPU down by eviction, reports itself at capacity, so the autoscaler will
-  add more pods. `CPUEviction.NominalCapacity` should be set to the autoscaler's per-pod request target; it defaults
-  to the active-requests soft limit.
+  add more pods. Its active-request count is the one admission uses, so it includes requests still setting up and
+  never reads below `substreams_active_requests`. `CPUEviction.NominalCapacity` should be set to the autoscaler's
+  per-pod request target; it defaults to the active-requests soft limit.
 
 - Trimmed tier2's per-segment logging: a large backfill fans out into tens of thousands of `ProcessRange` calls,
   and each one was logging ~10 `Info` lines with no steady-state diagnostic value, which could spike a pod's log

service/active_requests/evictor.go

@@ -145,8 +145,9 @@ type Evictor struct {
 	reader  *CPUReader
 	logger  *zap.Logger
 
-	overloaded atomic.Bool
-	onEvaluate func()
+	overloaded          atomic.Bool
+	onEvaluate          func()
+	countActiveRequests func() int
 
 	// zero time = condition not currently holding
 	overloadSince time.Time
@@ -193,6 +194,22 @@ func (ev *Evictor) OnEvaluate(fn func()) {
 	ev.onEvaluate = fn
 }
 
+// CountActiveRequestsWith overrides how the evictor counts the pod's active
+// requests when it publishes substreams_tier1_effective_active_requests. The
+// manager's map only holds requests that finished setting up, while admission
+// and substreams_active_requests count them from the moment they arrive, and
+// the autoscaler metric has to agree with those. Must be called before Run.
+func (ev *Evictor) CountActiveRequestsWith(fn func() int) {
+	ev.countActiveRequests = fn
+}
+
+func (ev *Evictor) activeRequests(inManager int) int {
+	if ev.countActiveRequests == nil {
+		return inManager
+	}
+	return ev.countActiveRequests()
+}
+
 func (ev *Evictor) notify() {
 	if ev.onEvaluate != nil {
 		ev.onEvaluate()
@@ -227,9 +244,9 @@ func (ev *Evictor) tick(now time.Time) {
 		ev.clearOverload()
 		return
 	}
-	candidates, totalActive := ev.sampleBurnRates(now)
+	candidates, inManager := ev.sampleBurnRates(now)
 	firing := ev.classify(signals, now)
-	ev.publishMetrics(signals, totalActive)
+	ev.publishMetrics(signals, ev.activeRequests(inManager))
 	ev.notify()
 
 	if !ev.overloaded.Load() {

service/active_requests/evictor_test.go

@@ -205,3 +205,14 @@ func TestTick_UnreadableCgroupClearsOverload(t *testing.T) {
 	assert.False(t, ev.IsOverloaded())
 	assert.Equal(t, 2, evaluations)
 }
+
+func TestActiveRequests_PrefersTheHostCount(t *testing.T) {
+	ev := newTestEvictor(DefaultEvictorConfig())
+
+	// without an override the manager's own count is used
+	assert.Equal(t, 3, ev.activeRequests(3))
+
+	// the host counts requests still setting up, which the manager does not hold yet
+	ev.CountActiveRequestsWith(func() int { return 7 })
+	assert.Equal(t, 7, ev.activeRequests(3))
+}

service/tier1.go

@@ -294,6 +294,7 @@ func NewTier1(
 			}
 			evictor := active_requests.NewEvictor(evictorConfig, s.activeRequestsManager, reader, logger)
 			evictor.OnEvaluate(s.refreshReadiness)
+			evictor.CountActiveRequestsWith(s.getActiveRequestCount)
 			s.evictor = evictor
 			go evictor.Run(s.Terminating())
 		}