Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
# Changelog

## Unreleased
## v0.8.64

Quieter llama.cpp logging with a configurable level, `:tools` coverage in
CI, and dependency updates. llama.cpp stays at `418f905a4`.

### Added

Expand Down Expand Up @@ -32,6 +35,21 @@
EEF-CVE-2026-32686 and EEF-CVE-2026-97853 from `mix deps.get`. Both are
optional dependencies, so this changes nothing a user resolves.

Verified on macOS, M1 Max, Metal (`GGML_METAL_NO_RESIDENCY=1`): default
**436 passed, 194 excluded**; `--include smoke --include embeddings --include
slow --include mtp` (Qwen3.5-0.8B-UD-Q4_K_XL, Qwen3-Embedding-0.6B-f16,
Qwen3.6-35B-A3B-MTP-UD-Q4_K_XL) **585 passed**; CI's set, `--include smoke
--include embeddings --include slow --include decision --include tools`
(SmolLM2-135M, bge-small, tinylaya, tinyopenjev, Qwen3.5-0.8B) **601
passed**; `--include embeddings` on embeddinggemma-2-Q8_0 **447 passed**;
`--include decision_clef` on Clef Flash Q8_0 **440 passed** (4 of 4 clef);
`--include tools` on Qwen3.5-0.8B **447 passed** on Metal and on CPU (11 of
11 tool tests; Llama-3.2-1B-Instruct passes 10 of 11, calling the tool again
in the round trip); `--include mtp_sidecar` on Qwen3.8-27B-Q4_K_M +
mtp-Qwen3.8-27B-Q4_0 **442 passed, 6 skipped** (the Gemma 4 E4B pair). The
Hex package builds from source with `LLAMA_BACKEND=cpu`. Format, credo
`--strict` and dialyzer (0 errors) clean.

## v0.8.63

Security fixes for special-token text in caller input (chat and decisions),
Expand Down
2 changes: 1 addition & 1 deletion mix.exs
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ end
defmodule LlamaCppEx.MixProject do
use Mix.Project

@version "0.8.63"
@version "0.8.64"
@source_url "https://github.com/nyo16/llama_cpp_ex"

def project do
Expand Down
48 changes: 34 additions & 14 deletions test/llama_cpp_ex_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -819,29 +819,53 @@ defmodule LlamaCppExTest do
ref = make_ref()
test_pid = self()

start_id = "test-request-start-#{inspect(ref)}"
done_id = "test-request-done-#{inspect(ref)}"
tick_id = "test-tick-#{inspect(ref)}"

on_exit(fn ->
:telemetry.detach(start_id)
:telemetry.detach(done_id)
:telemetry.detach(tick_id)
end)

# Handlers are global: pin the server so a lingering server from an
# earlier test cannot satisfy these assertions.
:telemetry.attach(
"test-request-start-#{inspect(ref)}",
start_id,
[:llama_cpp_ex, :server, :request, :start],
fn _event, measurements, metadata, _config ->
send(test_pid, {:start_telemetry, measurements, metadata})
fn
_event, measurements, %{server: ^server} = metadata, _config ->
send(test_pid, {:start_telemetry, measurements, metadata})

_event, _measurements, _metadata, _config ->
:ok
end,
nil
)

:telemetry.attach(
"test-request-done-#{inspect(ref)}",
done_id,
[:llama_cpp_ex, :server, :request, :done],
fn _event, measurements, metadata, _config ->
send(test_pid, {:telemetry, measurements, metadata})
fn
_event, measurements, %{server: ^server} = metadata, _config ->
send(test_pid, {:telemetry, measurements, metadata})

_event, _measurements, _metadata, _config ->
:ok
end,
nil
)

:telemetry.attach(
"test-tick-#{inspect(ref)}",
tick_id,
[:llama_cpp_ex, :server, :tick],
fn _event, measurements, _metadata, _config ->
send(test_pid, {:tick_telemetry, measurements})
fn
_event, measurements, %{server: ^server}, _config ->
send(test_pid, {:tick_telemetry, measurements})

_event, _measurements, _metadata, _config ->
:ok
end,
nil
)
Expand Down Expand Up @@ -873,7 +897,7 @@ defmodule LlamaCppExTest do
# be strictly less than duration.
assert measurements.ttft_ms < measurements.duration_ms

assert is_pid(metadata.server)
assert metadata.server == server
assert is_integer(metadata.seq_id)
assert metadata.mode == :generate
assert metadata.stop_reason in [:eog, :max_tokens]
Expand All @@ -883,10 +907,6 @@ defmodule LlamaCppExTest do
assert is_number(tick_measurements.batch_size)
assert tick_measurements.batch_size > 0
assert is_number(tick_measurements.eval_ms)

:telemetry.detach("test-request-start-#{inspect(ref)}")
:telemetry.detach("test-request-done-#{inspect(ref)}")
:telemetry.detach("test-tick-#{inspect(ref)}")
end

test "cache_prompt sequential requests don't crash on hybrid models", %{
Expand Down
22 changes: 15 additions & 7 deletions test/server_smoke_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,23 @@ defmodule LlamaCppEx.ServerSmokeTest do
server
end

defp attach_collector(event) do
# Telemetry handlers are global, so filter on the server under test: the
# previous test's server can still be emitting `:done` after its caller got
# the reply (finish_slot replies first), and ExUnit does not wait for that
# linked server to stop before starting the next test.
defp attach_collector(server, event) do
parent = self()
handler_id = {__MODULE__, self(), event}

:telemetry.attach(
handler_id,
event,
fn _event, measurements, metadata, _ ->
send(parent, {:telemetry, measurements, metadata})
fn
_event, measurements, %{server: ^server} = metadata, _ ->
send(parent, {:telemetry, measurements, metadata})

_event, _measurements, _metadata, _ ->
:ok
end,
nil
)
Expand Down Expand Up @@ -83,7 +91,7 @@ defmodule LlamaCppEx.ServerSmokeTest do

test "per-request cache_prompt overrides: hit, no-reuse, no-retention" do
server = start_server(n_parallel: 1)
attach_collector([:llama_cpp_ex, :server, :request, :start])
attach_collector(server, [:llama_cpp_ex, :server, :request, :start])

prompt = "User: Name three colors.\nAssistant:"
{:ok, reply} = Server.generate(server, prompt, max_tokens: 12)
Expand All @@ -106,7 +114,7 @@ defmodule LlamaCppEx.ServerSmokeTest do

test "session affinity keeps interleaved conversations on their slots" do
server = start_server([])
attach_collector([:llama_cpp_ex, :server, :request, :start])
attach_collector(server, [:llama_cpp_ex, :server, :request, :start])

# The two prompts must share no leading tokens. pick_cached_slot/2 reuses a
# slot when longest-common-prefix / prompt_len > 0.1, so a shared opener
Expand Down Expand Up @@ -195,7 +203,7 @@ defmodule LlamaCppEx.ServerSmokeTest do

test "halting a stream early cancels generation and frees the slot" do
server = start_server(n_parallel: 1)
attach_collector([:llama_cpp_ex, :server, :request, :done])
attach_collector(server, [:llama_cpp_ex, :server, :request, :done])

_ = Server.stream(server, "Write an endless story:", max_tokens: 400) |> Enum.take(3)

Expand Down Expand Up @@ -295,7 +303,7 @@ defmodule LlamaCppEx.ServerSmokeTest do
# hand one request's KV to another when their prompts share a prefix.
test "a cached prefix is never reused across cache scopes" do
server = start_server(n_parallel: 1, prompt_cache_ram_mb: 64)
attach_collector([:llama_cpp_ex, :server, :request, :start])
attach_collector(server, [:llama_cpp_ex, :server, :request, :start])

prompt =
"Internal memo, confidential. " <>
Expand Down