From 49160197033064551fb27fefb59399cd26f5064d Mon Sep 17 00:00:00 2001 From: Niko Maroulis Date: Sat, 31 Oct 2026 14:20:59 -0400 Subject: [PATCH] Run :tools in CI, filter llama.cpp debug logs, require elixir_make 0.10.1 - CI pins Qwen3.5-0.8B-UD-Q4_K_XL and runs --include tools on CPU. - The NIF installs a level-filtering llama/ggml log callback, :info by default, with LlamaCppEx.set_log_level/1 and config :llama_cpp_ex, log_level. - elixir_make ~> 0.10.1: newer OTP patches cap httpc headers at 10 KB. - Lock: decimal 3.2.0 / ecto 3.14.2 for two decimal advisories. --- .github/workflows/ci.yml | 24 +++++++++++++-- CHANGELOG.md | 25 +++++++++++++++- README.md | 11 +++++++ c_src/llama_cpp_ex/llama_nif.cpp | 51 ++++++++++++++++++++++++++++++++ lib/llama_cpp_ex.ex | 40 +++++++++++++++++++++++++ lib/llama_cpp_ex/application.ex | 9 ++++++ lib/llama_cpp_ex/nif.ex | 3 ++ mix.exs | 8 ++++- mix.lock | 4 +-- test/log_level_test.exs | 23 ++++++++++++++ test/test_helper.exs | 8 ++--- 11 files changed, 195 insertions(+), 11 deletions(-) create mode 100644 test/log_level_test.exs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ec7ddd..0243dbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,13 @@ env: DECISION_OPENJEV_REPO: ggml-org/tinyopenjev-for-testing-gguf DECISION_OPENJEV_REV: 953e44649ca8a71e5a8eee4676edebff2a45b2a2 DECISION_OPENJEV_FILE: tinyopenjev-for-testing-Q8_0.gguf + # A model whose chat template supports tool calling, for the :tools tag (559 + # MB). SmolLM2's template has no tools. Qwen3.5-0.8B is what the tool tests + # and the README were written against, and a second chat template for the + # special-token tests. + TOOLS_MODEL_REPO: unsloth/Qwen3.5-0.8B-GGUF + TOOLS_MODEL_REV: 6ab461498e2023f6e3c1baea90a8f0fe38ab64d0 + TOOLS_MODEL_FILE: Qwen3.5-0.8B-UD-Q4_K_XL.gguf jobs: setup: @@ -391,13 +398,13 @@ jobs: fail-on-cache-miss: true # Every URL is revision-pinned so a repo re-upload cannot silently change - # what CI runs. Total ~318 MB, cached on the pins. + # what CI runs. Total ~877 MB, cached on the pins. - name: Cache GGUF models id: models uses: actions/cache@v5 with: path: models - key: gguf-${{ env.GEN_MODEL_REV }}-${{ env.EMB_MODEL_REV }}-${{ env.DECISION_LAYA_REV }}-${{ env.DECISION_OPENJEV_REV }} + key: gguf-${{ env.GEN_MODEL_REV }}-${{ env.EMB_MODEL_REV }}-${{ env.DECISION_LAYA_REV }}-${{ env.DECISION_OPENJEV_REV }}-${{ env.TOOLS_MODEL_REV }} - name: Download GGUF models if: steps.models.outputs.cache-hit != 'true' @@ -411,6 +418,8 @@ jobs: "https://huggingface.co/${DECISION_LAYA_REPO}/resolve/${DECISION_LAYA_REV}/${DECISION_LAYA_FILE}" curl -fsSL --retry 3 -o "models/${DECISION_OPENJEV_FILE}" \ "https://huggingface.co/${DECISION_OPENJEV_REPO}/resolve/${DECISION_OPENJEV_REV}/${DECISION_OPENJEV_FILE}" + curl -fsSL --retry 3 -o "models/${TOOLS_MODEL_FILE}" \ + "https://huggingface.co/${TOOLS_MODEL_REPO}/resolve/${TOOLS_MODEL_REV}/${TOOLS_MODEL_FILE}" # This is the job the audit says the repo was missing: CI compiled the # 2100-line NIF but never ran a single forward pass, while 61 of 223 @@ -421,14 +430,17 @@ jobs: # tag was excluded by default and included by nothing, even though the # generation model it needs is already cached above. :decision covers # LlamaCppEx.Decision, the port of upstream's server-decision code. + # :tools covers tool calling and the special-token tests on a second + # chat template. - name: Run inference tests - run: mix test --include smoke --include embeddings --include slow --include decision + run: mix test --include smoke --include embeddings --include slow --include decision --include tools env: LLAMA_BACKEND: cpu LLAMA_SMOKE_GEN_MODEL: models/SmolLM2-135M-Instruct-Q8_0.gguf LLAMA_SMOKE_EMB_MODEL: models/bge-small-en-v1.5-q8_0.gguf LLAMA_SMOKE_DECISION_LAYA_MODEL: models/tinylaya-for-testing-Q8_0.gguf LLAMA_SMOKE_DECISION_OPENJEV_MODEL: models/tinyopenjev-for-testing-Q8_0.gguf + LLAMA_SMOKE_TOOLS_MODEL: models/Qwen3.5-0.8B-UD-Q4_K_XL.gguf # The taxonomy's invariant, checked rather than trusted: each gate tag must # stand alone, because `--include` beats `--exclude` and a test carrying two @@ -455,3 +467,9 @@ jobs: LLAMA_BACKEND: cpu LLAMA_SMOKE_DECISION_LAYA_MODEL: models/tinylaya-for-testing-Q8_0.gguf LLAMA_SMOKE_DECISION_OPENJEV_MODEL: models/tinyopenjev-for-testing-Q8_0.gguf + + - name: Tools tag runs alone + run: mix test --include tools + env: + LLAMA_BACKEND: cpu + LLAMA_SMOKE_TOOLS_MODEL: models/Qwen3.5-0.8B-UD-Q4_K_XL.gguf diff --git a/CHANGELOG.md b/CHANGELOG.md index a425660..bda92d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,35 @@ ## Unreleased -### Changed +### Added +- `LlamaCppEx.set_log_level/1` and `config :llama_cpp_ex, log_level: level` + (`:debug | :info | :warning | :error | :none`). The NIF now installs its own + llama.cpp/ggml log callback instead of llama.cpp's default, which wrote every + level to stderr, and also sets the chat-template code's log threshold. + +### Changed + +- llama.cpp's DEBUG log lines are no longer printed by default (`:info`). + With `:tools`, a lazy grammar logged "Grammar still awaiting trigger" once + per generated token: a 100-token Qwen3.5 tool call printed 1,663 stderr + lines, now 199. Everything at INFO and above is printed as before. +- CI runs the `:tools` tag on CPU against a pinned Qwen3.5-0.8B-UD-Q4_K_XL, so + tool calling and the chat special-token tests on a second template run on + every push. +- `elixir_make` requirement raised to `~> 0.10.1`. OTP 29.0.6, 28.5.0.6 and + 27.3.4.17 limit `httpc` response headers to 10,240 bytes, and GitHub's + release redirects can exceed it; 0.10.1 raises the limit, so an older + elixir_make could fail to download the precompiled NIF there. On earlier OTP + patch releases `httpc` logs `Invalid option {max_header_size,65536} + ignored`, which is harmless (README, "Logging"). - `:req` (optional, for `LlamaCppEx.Hub`) now also accepts `~> 0.7`, and the lock moves to elixir_make 0.10.1, req 0.7.5, finch 0.24.0 and mint 1.11.0. Their older releases set `xref: [exclude: ...]`, which Elixir 1.20 reports as deprecated on every build (three CI annotations per run). +- Lock: decimal 2.4.1 → 3.2.0 and ecto 3.13.5 → 3.14.2, clearing + EEF-CVE-2026-32686 and EEF-CVE-2026-97853 from `mix deps.get`. Both are + optional dependencies, so this changes nothing a user resolves. ## v0.8.63 diff --git a/README.md b/README.md index b8741da..5ec8dc2 100644 --- a/README.md +++ b/README.md @@ -728,6 +728,17 @@ CPU build at Q8_0 (one 324-token prompt). The opt-in `test/decision_clef_test.exs` runs against this model (`LLAMA_SMOKE_DECISION_CLEF_MODEL`, tag `:decision_clef`). +## Logging + +llama.cpp, ggml and the chat-template code write their own log lines to stderr. `LlamaCppEx.set_log_level/1` sets the lowest level printed: `:debug`, `:info` (the default: everything except llama.cpp's DEBUG lines), `:warning`, `:error` or `:none`. Set it in config to have it applied when the application starts, before any backend or model is initialized: + +```elixir +# config/config.exs +config :llama_cpp_ex, log_level: :warning +``` + +Installing the precompiled NIF on an OTP patch release older than 29.0.6, 28.5.0.6 or 27.3.4.17 may log `Invalid option {max_header_size,65536} ignored`. That comes from `httpc` and is harmless: elixir_make sets the option because those OTP releases and newer limit response headers to 10 KB, which GitHub's download redirects can exceed. + ## Lower-level API For fine-grained control over the inference pipeline: diff --git a/c_src/llama_cpp_ex/llama_nif.cpp b/c_src/llama_cpp_ex/llama_nif.cpp index 9c6746b..50f193f 100644 --- a/c_src/llama_cpp_ex/llama_nif.cpp +++ b/c_src/llama_cpp_ex/llama_nif.cpp @@ -3,6 +3,7 @@ #include #include #include "json-schema-to-grammar.h" +#include "log.h" #include "sampling.h" #include "speculative.h" #include @@ -55,6 +56,56 @@ inline auto no_devices = fine::Atom("no_devices"); inline auto bind_timeout = fine::Atom("bind_timeout"); } // namespace atoms +// --- Logging --- +// +// llama.cpp's default log callback writes every message to stderr, DEBUG +// included: a lazy tool-call grammar alone logged "Grammar still awaiting +// trigger" once per generated token. This callback drops what is below the +// level LlamaCppEx.set_log_level/1 sets, INFO by default, so the default +// output is what it was minus DEBUG. Installed for llama and ggml (one call +// sets both) when the library is loaded, before any model is. +namespace { + +// Lowest ggml_log_level that is printed. GGML_LOG_LEVEL_CONT (above ERROR) +// prints nothing. +std::atomic g_log_min{GGML_LOG_LEVEL_INFO}; + +// A CONT message continues the previous one on its thread, so it takes that +// message's level. +thread_local int t_log_last = GGML_LOG_LEVEL_INFO; + +void log_callback(ggml_log_level level, const char* text, void* /*user_data*/) { + int effective = level; + if (level == GGML_LOG_LEVEL_CONT) { + effective = t_log_last; + } else { + // LLAMA_LOG(...) (model summaries) logs at NONE; treat it as INFO. + if (level == GGML_LOG_LEVEL_NONE) { + effective = GGML_LOG_LEVEL_INFO; + } + t_log_last = effective; + } + if (effective >= g_log_min.load(std::memory_order_relaxed)) { + fputs(text, stderr); + } +} + +[[maybe_unused]] const bool log_callback_installed = (llama_log_set(log_callback, nullptr), true); + +} // namespace + +// min_level: the lowest ggml_log_level printed (GGML_LOG_LEVEL_CONT for none). +// common_threshold: common_log's verbosity threshold, for the chat-template +// code's own logger, which compares each message's verbosity against it. +fine::Ok<> log_set_level(ErlNifEnv* env, int64_t min_level, int64_t common_threshold) { + g_log_min.store(static_cast(min_level), std::memory_order_relaxed); + // An int store; common documents the setter as not thread-safe because a + // concurrent reader may see the old value for one message. + common_log_set_verbosity_thold(static_cast(common_threshold)); + return fine::Ok(); +} +FINE_NIF(log_set_level, 0); + // --- Input validation at the NIF boundary --- // // llama.cpp's GGML_ASSERT is never NDEBUG-gated, so it calls ggml_abort() -> diff --git a/lib/llama_cpp_ex.ex b/lib/llama_cpp_ex.ex index ca0c47e..7441b16 100644 --- a/lib/llama_cpp_ex.ex +++ b/lib/llama_cpp_ex.ex @@ -106,6 +106,46 @@ defmodule LlamaCppEx do LlamaCppEx.NIF.backend_init() end + # ggml_log_level values (ggml.h): DEBUG 1, INFO 2, WARN 3, ERROR 4; CONT (5), + # above every level, prints nothing. And common_log's verbosity threshold + # (common/log.h): DEBUG 5, INFO 3, WARN 2, ERROR 1; 0 keeps only tool output, + # which the library never writes, so -1 silences it. + @log_levels %{ + debug: {1, 5}, + info: {2, 3}, + warning: {3, 2}, + error: {4, 1}, + none: {5, -1} + } + + @doc """ + Sets the lowest level of llama.cpp's own log output that is printed to stderr. + + `level` is `:debug`, `:info` (the default), `:warning`, `:error` or `:none`. + It covers llama.cpp, ggml (backends, Metal/CUDA setup) and the chat-template + code, for the whole VM. `:info` prints what llama.cpp prints by default + except its DEBUG lines, which include one per generated token while a lazy + tool-call grammar waits for its trigger. + + To set it at startup, configure it and the application applies it when it + starts: + + config :llama_cpp_ex, log_level: :warning + + """ + @spec set_log_level(:debug | :info | :warning | :error | :none) :: :ok + def set_log_level(level) do + case Map.fetch(@log_levels, level) do + {:ok, {min_level, common_threshold}} -> + LlamaCppEx.NIF.log_set_level(min_level, common_threshold) + + :error -> + raise ArgumentError, + "invalid log level #{inspect(level)}, expected one of " <> + inspect([:debug, :info, :warning, :error, :none]) + end + end + @doc """ Lists the available ggml backend devices (GPUs, integrated GPUs, accelerators, and the CPU). diff --git a/lib/llama_cpp_ex/application.ex b/lib/llama_cpp_ex/application.ex index cf45e89..c67dc98 100644 --- a/lib/llama_cpp_ex/application.ex +++ b/lib/llama_cpp_ex/application.ex @@ -22,12 +22,21 @@ defmodule LlamaCppEx.Application do This is separate from `LlamaCppEx.ModelSupervisor`, which is opt-in and starts the `Registry`/`DynamicSupervisor` pair that the multi-model manager needs. + + When `config :llama_cpp_ex, log_level: level` is set, it is applied here with + `LlamaCppEx.set_log_level/1`. Unset, the NIF's own default (`:info`) stands + and starting the application does not touch the NIF. """ use Application @impl true def start(_type, _args) do + case Application.fetch_env(:llama_cpp_ex, :log_level) do + {:ok, level} -> :ok = LlamaCppEx.set_log_level(level) + :error -> :ok + end + children = [ {Registry, keys: :unique, name: LlamaCppEx.Registry} ] diff --git a/lib/llama_cpp_ex/nif.ex b/lib/llama_cpp_ex/nif.ex index 17aa49e..ef0c2ec 100644 --- a/lib/llama_cpp_ex/nif.ex +++ b/lib/llama_cpp_ex/nif.ex @@ -21,6 +21,9 @@ defmodule LlamaCppEx.NIF do def backend_init, do: :erlang.nif_error(:not_loaded) def backend_free, do: :erlang.nif_error(:not_loaded) + # Logging + def log_set_level(_min_level, _common_threshold), do: :erlang.nif_error(:not_loaded) + # Devices def device_list, do: :erlang.nif_error(:not_loaded) diff --git a/mix.exs b/mix.exs index 89d8450..d746b4e 100644 --- a/mix.exs +++ b/mix.exs @@ -187,7 +187,13 @@ defmodule LlamaCppEx.MixProject do defp deps do [ - {:elixir_make, "~> 0.8", runtime: false}, + # 0.10.1 raises httpc's response-header limit for the NIF download. OTP + # 29.0.6, 28.5.0.6 and 27.3.4.17 cap headers at 10,240 bytes, and a + # GitHub release redirect can exceed that, which fails the download on + # older elixir_make. On OTP patch releases before those, httpc does not + # know the option and logs "Invalid option {max_header_size,65536} + # ignored", which is harmless. + {:elixir_make, "~> 0.10.1", runtime: false}, {:fine, "~> 0.1", runtime: false}, {:telemetry, "~> 1.0"}, {:ecto, "~> 3.0", optional: true}, diff --git a/mix.lock b/mix.lock index 32a2597..e4398cf 100644 --- a/mix.lock +++ b/mix.lock @@ -4,11 +4,11 @@ "benchee_json": {:hex, :benchee_json, "1.0.0", "cc661f4454d5995c08fe10dd1f2f72f229c8f0fb1c96f6b327a8c8fc96a91fe5", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "da05d813f9123505f870344d68fb7c86a4f0f9074df7d7b7e2bb011a63ec231c"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "credo": {:hex, :credo, "1.7.19", "cc52129665fc7c15143d47838fda0f9cd6dac9ceced7bf4da6f85fcbfe64b12a", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "2d8bc95d5a7bb99dd2613621d4f08c6a3575c3fd4b62e6a2b48a100352a557b8"}, - "decimal": {:hex, :decimal, "2.4.1", "6c0fbede12fb122ba685e9ab41c6a40c129e322b3aa192f9e072e61f3a6ffaf2", [:mix], [], "hexpm", "7e618897933a8455f19a727d7c5e50a2c071a544b700e5e724298ecb4340187f"}, + "decimal": {:hex, :decimal, "3.2.0", "f1235f05d841a211fda78a08fdb9777f2ff51a71f18455233fae186ff76187c6", [:mix], [], "hexpm", "190bca80580ce61aa69bb0eacadd7677710205c770225c2c2c07e32a5c31b234"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, - "ecto": {:hex, :ecto, "3.13.5", "9d4a69700183f33bf97208294768e561f5c7f1ecf417e0fa1006e4a91713a834", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "df9efebf70cf94142739ba357499661ef5dbb559ef902b68ea1f3c1fabce36de"}, + "ecto": {:hex, :ecto, "3.14.2", "99db28a864293a789c970651de711e3cae184291e0e7ea1166c54055ac41c1f3", [:mix], [{:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "25d60b8c816a07d19d85b80bdf60978bd8b102209dda198d768cd7c6745339a6"}, "elixir_make": {:hex, :elixir_make, "0.10.1", "f5a44ab6b48421de1c4fd33c147e22cd2f82fd86e8a9a4d24809d466cad63c5e", [:mix], [], "hexpm", "ce1eaa234bc87f0279718e793fd6f8ad35b4e8ee8d412329e3879eee8d40adba"}, "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, "ex_doc": {:hex, :ex_doc, "0.40.1", "67542e4b6dde74811cfd580e2c0149b78010fd13001fda7cfeb2b2c2ffb1344d", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "bcef0e2d360d93ac19f01a85d58f91752d930c0a30e2681145feea6bd3516e00"}, diff --git a/test/log_level_test.exs b/test/log_level_test.exs new file mode 100644 index 0000000..7a6a5de --- /dev/null +++ b/test/log_level_test.exs @@ -0,0 +1,23 @@ +defmodule LlamaCppEx.LogLevelTest do + # The level is VM-global, so nothing else may run while it changes. + use ExUnit.Case, async: false + + setup do + on_exit(fn -> LlamaCppEx.set_log_level(:info) end) + end + + test "accepts the documented levels" do + for level <- [:debug, :info, :warning, :error, :none] do + assert LlamaCppEx.set_log_level(level) == :ok + end + end + + # Logger's spelling is :warning; :warn and strings are a likely slip. + test "rejects anything else, naming the accepted levels" do + for level <- [:warn, "info", nil] do + assert_raise ArgumentError, ~r/invalid log level .* expected one of \[:debug/, fn -> + LlamaCppEx.set_log_level(level) + end + end + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 670239d..703fe2e 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -4,10 +4,10 @@ # :smoke — generation/chat/grammar/server paths; needs LLAMA_SMOKE_GEN_MODEL # :tools — tool calling through chat_completion/3; needs # LLAMA_SMOKE_TOOLS_MODEL, a model whose chat template supports -# tools (Qwen3.5-0.8B, Llama-3.2-1B-Instruct). Its own tag -# because CI's smoke model (SmolLM2-135M) has a template without -# tool support. Pick a different template from the :smoke model -# and the chat special-token tests cover two templates. +# tools. Its own tag because the :smoke model CI uses +# (SmolLM2-135M) has a template without tool support; CI pins +# Qwen3.5-0.8B-UD-Q4_K_XL for this one, which also gives the +# chat special-token tests a second template. # :embeddings — embedding paths; needs LLAMA_SMOKE_EMB_MODEL # :mtp — MTP speculative decoding; needs LLAMA_SMOKE_MTP_MODEL # :mtp_sidecar — MTP with the head in a *separate* sidecar GGUF. Qwen 3.8