From 944ef7a44ac0a7e6e6fcb1c2114e874eb84b16b3 Mon Sep 17 00:00:00 2001 From: Beinan Date: Thu, 8 Oct 2026 21:05:29 +0000 Subject: [PATCH] fix(master): pay back the running-pool gauge on task panic #348 incremented the running gauge before spawning the task and decremented it after run_task returned. A panic inside run_task unwinds past the decrement; the permits are released by Drop but the gauge stays elevated, so the pool reads as consumed forever. Hold the decrement in a Drop guard so unwinding pays it back. Refs #333. Review finding 5 on #348. Co-Authored-By: Claude Fable 5 --- crates/lance-context-master/src/scheduler.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/lance-context-master/src/scheduler.rs b/crates/lance-context-master/src/scheduler.rs index b5e405e7..d30637ef 100644 --- a/crates/lance-context-master/src/scheduler.rs +++ b/crates/lance-context-master/src/scheduler.rs @@ -1172,6 +1172,14 @@ fn spawn_pool_poller( spawn_filtered_pool_poller(state, pool, kinds, report_depth, false) } +/// Decrements a gauge when dropped, so unwinding pays it back. +struct GaugeHold(metrics::Gauge); +impl Drop for GaugeHold { + fn drop(&mut self) { + self.0.decrement(1.0); + } +} + fn spawn_filtered_pool_poller( state: Arc, pool: Arc, @@ -1245,11 +1253,15 @@ fn spawn_filtered_pool_poller( let running = running.clone(); running.increment(1.0); tokio::spawn(async move { + // Decrement on every exit, including a panic in + // run_task: the permits are dropped by unwinding, + // so the gauge must be too, or capacity reads as + // consumed forever. + let _held = GaugeHold(running); run_task(&st, claim, timing).await; drop(permit); drop(compact_permit); drop(operation); - running.decrement(1.0); }); } Ok(None) => break,