diff --git a/CHANGELOG.md b/CHANGELOG.md index c18ae84..ad8ae1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## v0.8.70 + +llama.cpp stays at `9daa0aa22`. The `generate_mtp_tokens` NIF changed, so +this release ships new precompiled NIFs. + +### Fixed + +- Reusing an `LlamaCppEx.MTP` session right after abandoning one of its + streams (`Enum.take/2`, an exception) no longer races the cancelled loop. + Cancellation is unacknowledged, so the next `generate/3` or `stream/3` + could decode into the session's two contexts while the cancelled loop was + still inside a decode; at `9daa0aa22` that aborted the VM in 4 of 7 runs + (`GGML_ASSERT(logits != nullptr)`). `generate_mtp_tokens` now holds the + session for its whole run, so a second call waits, at most one + draft-and-verify iteration after a cancel. The test moves from the + known-broken `:mtp_cancel` tag into `:mtp` and checks the greedy output + after three cancelled streams; the `:mtp_cancel` tag is gone. A call made + while another is still running on the same session also waits for it now, + where it used to decode beside it. + ## v0.8.69 llama.cpp bumped to `9daa0aa22` (b12361+7), 163 commits from `a691b2345` diff --git a/c_src/llama_cpp_ex/llama_nif.cpp b/c_src/llama_cpp_ex/llama_nif.cpp index 68bfa66..d166ce0 100644 --- a/c_src/llama_cpp_ex/llama_nif.cpp +++ b/c_src/llama_cpp_ex/llama_nif.cpp @@ -3013,6 +3013,10 @@ fine::Ok<> generate_mtp_tokens( fine::ResourcePtr cancel) { auto& sp = *spec_res; + // One run per session at a time; see LlamaSpeculative::run_mutex. Taken + // before anything below touches the contexts, and released after the + // abort scope has restored the target's callback. + std::lock_guard run_lock(sp.run_mutex); auto* ctx_tgt = sp.ctx_tgt->ctx; auto* ctx_dft = sp.ctx_dft->ctx; auto* sampler = sampler_res->sampler; diff --git a/c_src/llama_cpp_ex/llama_nif.h b/c_src/llama_cpp_ex/llama_nif.h index e2f9f71..a19fee3 100644 --- a/c_src/llama_cpp_ex/llama_nif.h +++ b/c_src/llama_cpp_ex/llama_nif.h @@ -334,6 +334,13 @@ class LlamaSpeculative { std::atomic us_other{0}; std::atomic us_total{0}; + // Held by generate_mtp_tokens for its whole run. A session is reusable + // across calls, and a cancelled stream is fire-and-forget: the consumer + // returns while the cancelled loop can still be inside a target or draft + // decode, and only notices the flag at its next check. A second call + // waits here instead of decoding into the same two contexts at once. + std::mutex run_mutex; + LlamaSpeculative(common_speculative* s, fine::ResourcePtr tgt, fine::ResourcePtr dft, diff --git a/docs/release-guide.md b/docs/release-guide.md index 234cc72..e2acb98 100644 --- a/docs/release-guide.md +++ b/docs/release-guide.md @@ -482,11 +482,6 @@ LLAMA_RPC=1 LLAMA_BACKEND=metal MIX_ENV=test mix run --no-halt -e \ LLAMA_RPC=1 LLAMA_RPC_ENDPOINT=127.0.0.1:50052 \ mix test --include rpc_live -# :mtp_cancel is the one tag that is not expected to pass; run it to confirm -# how it fails, and update test/mtp_model_test.exs if the failure mode moved. -LLAMA_SMOKE_MTP_MODEL=~/Downloads/Qwen3.6-35B-A3B-MTP-UD-Q4_K_XL.gguf \ - mix test --only mtp_cancel - # Verify formatting and types mix format --check-formatted mix dialyzer diff --git a/lib/llama_cpp_ex/mtp.ex b/lib/llama_cpp_ex/mtp.ex index 59f87d7..bc9ec1e 100644 --- a/lib/llama_cpp_ex/mtp.ex +++ b/lib/llama_cpp_ex/mtp.ex @@ -69,18 +69,11 @@ defmodule LlamaCppEx.MTP do Upstream currently requires `n_parallel = 1` for MTP. This module reflects that — a single MTP session decodes one sequence at a time. Reuse the same `%MTP{}` value across calls to `stream/3` / `generate/3` to avoid rebuilding - the contexts; KV caches are cleared on each call. - - > #### Do not reuse a session straight after abandoning a stream {: .warning} - > - > Cancellation is asynchronous and unacknowledged: abandoning a `stream/3` - > early (`Enum.take/2`, `break`, an exception) sets a flag that the draft loop - > notices and then exits without reporting that it has done so. A session's two - > contexts are long-lived and shared by every call on it, so starting the next - > `generate/3` or `stream/3` immediately can put a second writer on a KV cache - > the cancelled loop has not finished with — which aborts the VM rather than - > returning an error. Let an abandoned stream run to a terminal event, or build - > a fresh session with `init/2`, before using the session again. + the contexts; KV caches are cleared on each call. Calls on one session run + one after another: a call made while another is still decoding, including a + stream abandoned early (`Enum.take/2`, an exception) whose loop has not yet + noticed its cancel flag, waits for it to finish. The wait after an abandoned + stream is at most one draft-and-verify iteration. MTP is the only speculative type this binding exposes. Upstream llama.cpp also implements EAGLE-3, DFlash (block-diffusion drafting), n-gram diff --git a/mix.exs b/mix.exs index df99b7b..368785e 100644 --- a/mix.exs +++ b/mix.exs @@ -133,7 +133,7 @@ end defmodule LlamaCppEx.MixProject do use Mix.Project - @version "0.8.69" + @version "0.8.70" @source_url "https://github.com/nyo16/llama_cpp_ex" def project do diff --git a/test/mtp_model_test.exs b/test/mtp_model_test.exs index 76ab342..e978ddc 100644 --- a/test/mtp_model_test.exs +++ b/test/mtp_model_test.exs @@ -138,6 +138,26 @@ defmodule LlamaCppEx.MTPModelTest do assert MTP.print_stats(mtp) == :ok end + # Regression: cancelling a stream is fire-and-forget, so the consumer gets + # control back while the cancelled loop can still be inside a decode on the + # session's two contexts. A call made right then decoded into the same KV + # caches: from b10435 to 9daa0aa22 this aborted the VM in roughly half the + # runs (last as `GGML_ASSERT(logits != nullptr)` in the MTP drafter) or + # failed with "verify decode failed". `generate_mtp_tokens` now holds the + # session for its whole run, so the next call waits for the cancelled one. + # Three rounds, and the greedy text compared, because a lost race need not + # crash: it can only corrupt the cache. + test "a session is reusable right after a halted stream", %{session: mtp} do + opts = [max_tokens: 8, temp: 0.0, seed: 7] + assert {:ok, expected} = MTP.generate(mtp, "2 + 2 =", opts) + + for _ <- 1..3 do + taken = mtp |> MTP.stream("Write an endless story:", max_tokens: 400) |> Enum.take(3) + assert length(taken) == 3 + assert {:ok, ^expected} = MTP.generate(mtp, "2 + 2 =", opts) + end + end + # The event vocabulary stream_events/3 documents. defp event_kind({:token, id, text}) when is_integer(id) and is_binary(text), do: :token defp event_kind({:stats, snapshot}) when is_map(snapshot), do: :stats @@ -147,89 +167,6 @@ defmodule LlamaCppEx.MTPModelTest do defp event_kind(_), do: :undocumented end -defmodule LlamaCppEx.MTPCancelTest do - # KNOWN BUG, kept as an executable record rather than deleted. - # - # At b10435 this test did not fail — it took the VM down, with - # `GGML_ASSERT(offset + size <= ggml_nbytes(tensor))` or a plain SIGSEGV. - # At b10582 it no longer aborts: it fails, racily, with - # `{:error, "prompt decode failed: code=-1"}` or `"verify decode failed: - # code=-1"` from the `generate/3` below — and sometimes passes. Measured on - # M1 Max / Metal with Qwen3.6-35B-A3B-MTP, four runs each: b10435 aborted 4/4 - # (one exit 134, three exit 139), b10582 aborted 0/4 (3 failed, 1 passed). - # The race is unchanged and unfixed; only its consequence moved from "kills - # the BEAM" to "returns an error", so the two decode paths now refuse the - # half-released context instead of writing through it. - # - # It aborts again, somewhere between b10582 and b11479: six runs each at - # 88dcc460d (b11479+4) and at 23b0202a1 (b11552) aborted 4/6 (exit 134 or - # 139); the rest failed with `"verify decode failed: code=-1"`, `"prompt - # decode failed: code=-2"`, or `unknown exception thrown within NIF` raised - # from the cancelled stream's own `generate_mtp_tokens` call. Same model and - # machine as above, so the abort is not new in 23b0202a1; nothing between - # v0.8.49 and v0.8.56 ran this tag. At a0e6dc738 (b11630+9) four runs aborted - # 2/4 (exit 134); the other two failed with the same `unknown exception` and - # `"prompt decode failed: code=-2"` as above. At 785322d17 (b12032+1) four - # runs aborted 3/4 (SIGSEGV, exit 139) and the fourth failed with the same - # `unknown exception`. At a691b2345 (b12205) four runs aborted 2/4 (exit 134: - # `GGML_ASSERT(idx == i - off)` in `common_batch`, `GGML_ASSERT(backend)` in - # `ggml-backend.cpp`) and the other two passed. At 9daa0aa22 (b12361+7), with - # Qwen3.6-27B (the 35B-A3B was not on the machine), seven runs aborted 4/7 - # (exit 134, every time `GGML_ASSERT(logits != nullptr)` in - # `common/sampling.cpp`, reached from the MTP drafter's - # `common_sampler_sample`) and the other three passed. - # - # It therefore still lives in its own module carrying only `:mtp_cancel`, and - # deliberately *not* `:mtp`: `--include` beats `--exclude` in ExUnit, so a - # second gate tag would drag a flaky test back into `--include mtp` runs. - # Run it on purpose: - # - # LLAMA_BACKEND=auto \ - # LLAMA_SMOKE_MTP_MODEL=... mix test --include mtp_cancel - # - # Cancellation is fire-and-forget: `request_cancel` sets an atomic flag and the - # MTP loop then "stops quietly" (`llama_nif.cpp:2009`) without emitting any - # terminal event. `Generator.stop/1` cancels, unlinks and drains the mailbox, - # but it has no completion signal to wait on, so it returns while the dirty - # scheduler may still be inside `llama_decode`. For `LlamaCppEx.stream/3` that - # is harmless — each call owns a context that dies with it. An `%MTP{}` session - # is not: it holds two long-lived contexts and is documented as reusable across - # calls, so the `generate/3` below can start decoding on the very contexts the - # cancelled loop has not released. Two writers, one KV cache. - # - # The fix is to acknowledge cancellation — have the loop emit a terminal event - # when it observes the flag and have `Generator.stop/1` await it — which changes - # the NIF and the cancellation protocol that non-MTP streaming also uses. That - # is why it is not bundled into a submodule bump. - use ExUnit.Case, async: false - - alias LlamaCppEx.MTP - - @moduletag :mtp_cancel - @moduletag timeout: 300_000 - - setup_all do - :ok = LlamaCppEx.init() - - {:ok, model} = - LlamaCppEx.load_model(LlamaCppEx.TestModels.path!(:mtp), - n_gpu_layers: -1, - load_mtp: true - ) - - {:ok, mtp} = MTP.init(model, n_ctx: 2048, n_draft: 3) - %{session: mtp} - end - - test "a halted stream stops generation instead of running to max_tokens", %{session: mtp} do - taken = mtp |> MTP.stream("Write an endless story:", max_tokens: 400) |> Enum.take(3) - assert length(taken) == 3 - - # The MTP value is still usable afterwards. - assert {:ok, _} = MTP.generate(mtp, "2 + 2 =", max_tokens: 4, temp: 0.0) - end -end - defmodule LlamaCppEx.MTPSidecarTest do # The target/draft split, which is how Qwen 3.8 ships MTP: the target GGUF # carries zero nextn layers and a separate `mtp-*.gguf` carries the head. This diff --git a/test/test_helper.exs b/test/test_helper.exs index 57b98ed..8987819 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -18,14 +18,6 @@ # unset, so `--include mtp_sidecar` with only the Qwen pair # stays green. Its own tag rather than `:mtp` because that # tag's single-file model cannot satisfy it. -# :mtp_cancel — one known-broken MTP test, excluded on its own tag so that -# `--include mtp` is green. Cancelling an MTP stream is -# fire-and-forget, so reusing the session immediately afterwards -# races the still-running draft loop over shared contexts. At -# b10435 that aborted the VM; at b10582 it failed racily instead -# (3 of 4 runs) and did not abort; by b11479 it aborts again -# (4 of 6 runs at 23b0202a1). Either way it does not belong in -# a green run. See test/mtp_model_test.exs. # :dflash — DFlash speculative decoding (MTP.init type: :dflash); needs # LLAMA_SMOKE_DFLASH_MODEL and LLAMA_SMOKE_DFLASH_DRAFT_MODEL, # a target and the `dflash` drafter trained for it (e.g. @@ -120,7 +112,6 @@ ExUnit.start( :embeddings, :slow, :mtp, - :mtp_cancel, :mtp_sidecar, :decision, :decision_clef,