Repository navigation
Fix MTPHead unpack crash in prefetch queue cleanup (CORE-644) #17180
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
3d210ab
Fix MTPHead unpack crash in prefetch queue cleanup (CORE-644)
Glary-Bot a6edefd
Merge remote-tracking branch 'origin/master' into fix-mtp
christian-byrne d033958
Test that prefetch cleanup skips an unreached grouped head slot
christian-byrne 520869e
Keep unpinning other prefetch queues when one queue's cleanup fails
christian-byrne 2522c20
Always clear _prefetch markers when prefetched module cleanup fails
Glary-Bot 43e9a9a
Merge remote-tracking branch 'origin/master' into fix
christian-byrne acc3b5a
Clean prefetch queues when malloc graph cleanup fails
christian-byrne e4a32d4
Merge branch 'master' into glary/fix-mtp-prefetch-cleanup
christian-byrne File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| import pytest | ||
| import torch | ||
|
|
||
| from comfy.cli_args import args as cli_args | ||
|
|
||
| if not torch.cuda.is_available(): | ||
| cli_args.cpu = True | ||
|
|
||
| import comfy.model_prefetch as model_prefetch # noqa: E402 | ||
|
|
||
|
|
||
| class MTPHead(torch.nn.Module): | ||
| pass | ||
|
|
||
|
|
||
| @pytest.fixture(autouse=True) | ||
| def _reset_queues(): | ||
| model_prefetch.PREFETCH_QUEUES = [] | ||
| yield | ||
| model_prefetch.PREFETCH_QUEUES = [] | ||
|
|
||
|
|
||
| def _prefetched(module): | ||
| module._prefetch = {"signature": None} | ||
| return (None, (module, [module])) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize("pending_unit", [tuple, list]) | ||
| def test_cleanup_skips_unconsumed_grouped_unit(pending_unit): | ||
| # Qwen3.5 MTP draft queue interrupted after the embed pop: the embed slot has | ||
| # been pinned, the grouped (head, mtp) slot has not been reached yet. | ||
| embed = torch.nn.Embedding(4, 4) | ||
| head = torch.nn.Linear(4, 4) | ||
| queue = [_prefetched(embed), pending_unit((head, MTPHead())), None] | ||
| model_prefetch.PREFETCH_QUEUES = [queue] | ||
|
|
||
| model_prefetch.cleanup_prefetch_queues() | ||
|
|
||
| assert model_prefetch.PREFETCH_QUEUES == [] | ||
| assert not hasattr(embed, "_prefetch") | ||
|
|
||
|
|
||
| def test_cleanup_resets_queues_when_entry_cleanup_fails(monkeypatch): | ||
| def boom(module, comfy_modules): | ||
| raise RuntimeError("unpin failed") | ||
|
|
||
| monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", boom) | ||
| model_prefetch.PREFETCH_QUEUES = [[_prefetched(torch.nn.Linear(4, 4)), None]] | ||
|
|
||
| with pytest.raises(RuntimeError): | ||
| model_prefetch.cleanup_prefetch_queues() | ||
| # A failed cleanup must not poison every later node on the worker. | ||
| assert model_prefetch.PREFETCH_QUEUES == [] | ||
| model_prefetch.cleanup_prefetch_queues() | ||
|
|
||
|
|
||
| @pytest.mark.parametrize("pending_unit", [tuple, list]) | ||
| def test_cleanup_skips_unreached_grouped_head(pending_unit): | ||
| # Interrupted between popping the consumed slot and pinning the next one: the | ||
| # head of the queue is still the raw (head, mtp) group, not a pinned slot. | ||
| queue = [pending_unit((torch.nn.Linear(4, 4), MTPHead())), None] | ||
| model_prefetch.PREFETCH_QUEUES = [queue] | ||
|
|
||
| model_prefetch.cleanup_prefetch_queues() | ||
|
|
||
| assert model_prefetch.PREFETCH_QUEUES == [] | ||
|
|
||
|
|
||
| def test_cleanup_unpins_later_queues_when_an_earlier_one_fails(monkeypatch): | ||
| real_cleanup = model_prefetch.cleanup_prefetched_modules | ||
| failing = torch.nn.Linear(4, 4) | ||
| later = torch.nn.Linear(4, 4) | ||
|
|
||
| def cleanup(module, comfy_modules): | ||
| if module is failing: | ||
| raise RuntimeError("unpin failed") | ||
| real_cleanup(module, comfy_modules) | ||
|
|
||
| monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", cleanup) | ||
| model_prefetch.PREFETCH_QUEUES = [[_prefetched(failing), None], [_prefetched(later), None]] | ||
|
|
||
| with pytest.raises(RuntimeError, match="unpin failed"): | ||
| model_prefetch.cleanup_prefetch_queues() | ||
| # The first failure is re-raised, but only after every other queue was cleaned up. | ||
| assert not hasattr(later, "_prefetch") | ||
| assert model_prefetch.PREFETCH_QUEUES == [] | ||
|
|
||
|
|
||
| def test_failed_unpin_still_clears_prefetch_markers(monkeypatch): | ||
| # A module whose unpin raised must not keep _prefetch, or a later cast would skip | ||
| # the fresh cast and reuse the stale transfer state. The rest of the group is still cleaned. | ||
| calls = [] | ||
|
|
||
| def vbar_unpin(v): | ||
| calls.append(v) | ||
| if v == "first": | ||
| raise RuntimeError("unpin failed") | ||
|
|
||
| monkeypatch.setattr(model_prefetch.comfy_aimdo.model_vbar, "vbar_unpin", vbar_unpin) | ||
| first = torch.nn.Linear(4, 4) | ||
| second = torch.nn.Linear(4, 4) | ||
| for module, v in ((first, "first"), (second, "second")): | ||
| module._v = v | ||
| module._prefetch = {"signature": "sig"} | ||
| model_prefetch.PREFETCH_QUEUES = [[(None, (first, [first, second])), None]] | ||
|
|
||
| with pytest.raises(RuntimeError, match="unpin failed"): | ||
| model_prefetch.cleanup_prefetch_queues() | ||
| assert calls == ["first", "second"] | ||
| assert not hasattr(first, "_prefetch") | ||
| assert not hasattr(second, "_prefetch") | ||
|
|
||
|
|
||
| def test_malloc_graph_cleanup_failure_still_cleans_queues(monkeypatch): | ||
| # After a GPU failure the graph abort can raise too. Queues and _prefetch markers must not | ||
| # survive into the next node, and the graph error still reaches the caller. | ||
| def boom(): | ||
| raise RuntimeError("graph abort failed") | ||
|
|
||
| monkeypatch.setattr(model_prefetch, "cleanup_malloc_graph", boom) | ||
| module = torch.nn.Linear(4, 4) | ||
| model_prefetch.PREFETCH_QUEUES = [[_prefetched(module), None]] | ||
|
|
||
| with pytest.raises(RuntimeError, match="graph abort failed"): | ||
| model_prefetch.cleanup_prefetch_queues() | ||
|
|
||
| assert model_prefetch.PREFETCH_QUEUES == [] | ||
| assert not hasattr(module, "_prefetch") |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.