From 54c861c28b1861dcbbd7dd9bd71198899440066b Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Tue, 18 Aug 2026 06:22:10 +0000 Subject: [PATCH] fix: resolve result_future on inference error without unloading the model _commit_completed_packet (added in the upstream stream-error-surfacing rework) still fires registry.register_unload() on every inference error and exits the worker loop. That unload runs the pipeline destructor, which can touch a corrupted device context after a real GPU error and SIGABRT the whole process, taking every other loaded model down with it (the second half of #153, the first half was fixed by the stream-error rework already on main). Resolve the caller's future and keep the worker/model alive instead. Fixes #153 --- src/server/worker_registry.py | 10 +++++----- tests/unit/test_worker_registry_unit.py | 9 ++++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/server/worker_registry.py b/src/server/worker_registry.py index 82f6bc7..d658646 100644 --- a/src/server/worker_registry.py +++ b/src/server/worker_registry.py @@ -97,13 +97,13 @@ def _commit_completed_packet( ) -> bool: """Complete the request future. Return True if the worker should exit.""" if completed.error is not None: - logger.error( - f"[{model_name}] Inference failed, triggering model unload..." - ) + # Resolve the caller and keep the worker/model alive; unloading here runs the + # pipeline destructor, which can touch a corrupted device context after certain + # errors and SIGABRT the whole process, taking every other loaded model with it. + logger.error(f"[{model_name}] Inference failed: {completed.error}") if packet.result_future is not None and not packet.result_future.done(): packet.result_future.set_exception(completed.error) - asyncio.create_task(registry.register_unload(model_name)) - return True + return False if packet.result_future is not None and not packet.result_future.done(): packet.result_future.set_result(completed) return False diff --git a/tests/unit/test_worker_registry_unit.py b/tests/unit/test_worker_registry_unit.py index 415f14a..055745b 100644 --- a/tests/unit/test_worker_registry_unit.py +++ b/tests/unit/test_worker_registry_unit.py @@ -310,7 +310,7 @@ async def transcribe(self, gen_config): assert packet.segments is None -def test_commit_treats_error_field_not_error_prefix() -> None: +def test_commit_resolves_error_without_unloading_the_model() -> None: class DummyRegistry: def __init__(self): self.unloaded = [] @@ -342,11 +342,14 @@ async def _run(): result_future=err_future, error=RuntimeError("gpu oom"), ) - assert worker_module._commit_completed_packet(err, err, "m", registry) is True + # On error the worker resolves the caller's future and keeps the model loaded; + # unloading here runs the pipeline destructor, which can SIGABRT the process if + # the device context is corrupted (see queue_worker_llm docstring). + assert worker_module._commit_completed_packet(err, err, "m", registry) is False with pytest.raises(RuntimeError, match="gpu oom"): err_future.result() await asyncio.sleep(0) - assert registry.unloaded == ["m"] + assert registry.unloaded == [] asyncio.run(_run())