From a0a7e3f83da1af37f563150861c15b89252cfb2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?George=20Guimar=C3=A3es?= Date: Mon, 26 Oct 2026 13:21:48 -0300 Subject: [PATCH 1/2] fix: Free the MLX array when a tensor is deallocated Nx.backend_deallocate/1 marked an EMLX tensor as deallocated, but the MLX array stayed in memory until the BEAM collected the term. TensorP::deallocate() only took 1 from the refcount, and ~TensorP() destroyed the array only for a count that was already 0, which never happened. A decode loop that frees its dead KV cache each step kept one more copy of the cache, and more when lazy outputs chained the old buffers. The TensorP that brings the count to 0 now replaces the array with a scalar of 4 bytes, so the buffer is freed at once. This TensorP is the deallocate call itself, or the last NIF call that still held the tensor. The resource destructor still destroys the array object at GC. A binary from to_blob aliased the buffer through the tensor resource, so it could read a freed buffer after a deallocate. The binary now keeps a minimal resource with its own reference to the buffer, for contiguous and non-contiguous tensors. EMLX.deallocate/1 also gave a FunctionClauseError for a tensor that was already deallocated. It now returns :already_deallocated, as Nx.backend_deallocate/1 expects. --- emlx/c_src/emlx_nif.cpp | 47 ++++++++++++++++------------------ emlx/c_src/emlx_nif_shared.hpp | 12 +++++---- emlx/lib/emlx.ex | 15 ++++++++++- emlx/lib/emlx/backend.ex | 6 ++--- emlx/test/emlx/memory_test.exs | 35 +++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 34 deletions(-) diff --git a/emlx/c_src/emlx_nif.cpp b/emlx/c_src/emlx_nif.cpp index b7d1128..898e6e4 100644 --- a/emlx/c_src/emlx_nif.cpp +++ b/emlx/c_src/emlx_nif.cpp @@ -121,36 +121,33 @@ NIF(copy) { // legacy path; a process-independent msg_env for the worker path). static ERL_NIF_TERM to_blob_term(ErlNifEnv *out_env, mlx::core::array *t, size_t byte_size) { - if (t->flags().row_contiguous) { - // Zero-copy: alias the MLX buffer via the existing tensor resource. - // Invariant: lib/emlx.ex calls eval(tensor) before this NIF, so - // data() is guaranteed non-null and stable (MLX arrays are immutable - // once materialised). enif_make_resource_binary keeps the resource alive - // until the binary is GC'd, decoupling the binary lifetime from Elixir GC - // of the tensor term. - return enif_make_resource_binary(out_env, static_cast(t), - t->data(), byte_size); + // Zero-copy: the binary aliases the MLX buffer. Invariant: lib/emlx.ex calls + // eval(tensor) before this NIF, so data() is non-null and stable (MLX + // arrays are immutable once materialised). A non-contiguous tensor first gets + // a fresh row-major copy. + // + // The binary keeps a minimal ERTS resource alive until it is GC'd. That + // resource holds its own mlx::core::array, which shares the buffer, so the + // binary stays valid after a deallocate frees the array of the tensor. The + // resource holds only sizeof(mlx::core::array), without the TensorP refcount + // and deleted-flag tail, because TensorP and the Elixir side never see it; + // default_dtor (~array()) is its only destructor path. + mlx::core::array source = *t; + if (!t->flags().row_contiguous) { + source = mlx::core::contiguous(*t); + mlx::core::eval(source); } - // Non-contiguous: materialise a fresh row-major copy, wrap it in a minimal - // ERTS resource, and alias that buffer zero-copy. - // The resource holds only sizeof(mlx::core::array) — no TensorP refcount/ - // deleted-flag tail — because it is never exposed to TensorP or the Elixir - // side; only the binary holds a reference and default_dtor - // (~array()) is the sole destructor path. - auto ct = mlx::core::contiguous(*t); - mlx::core::eval(ct); - - auto *ct_ptr = static_cast(enif_alloc_resource( + auto *ptr = static_cast(enif_alloc_resource( resource_object::type, sizeof(mlx::core::array))); - if (!ct_ptr) { - throw std::runtime_error("Unable to allocate contiguous-copy resource"); + if (!ptr) { + throw std::runtime_error("Unable to allocate the resource of a binary"); } - new (ct_ptr) mlx::core::array(std::move(ct)); - ERL_NIF_TERM resource_bin = enif_make_resource_binary( - out_env, ct_ptr, ct_ptr->data(), byte_size); - enif_release_resource(ct_ptr); + new (ptr) mlx::core::array(std::move(source)); + ERL_NIF_TERM resource_bin = + enif_make_resource_binary(out_env, ptr, ptr->data(), byte_size); + enif_release_resource(ptr); return resource_bin; } diff --git a/emlx/c_src/emlx_nif_shared.hpp b/emlx/c_src/emlx_nif_shared.hpp index a69756a..ddb8acb 100644 --- a/emlx/c_src/emlx_nif_shared.hpp +++ b/emlx/c_src/emlx_nif_shared.hpp @@ -91,12 +91,14 @@ class TensorP { TensorP(const TensorP &) = delete; TensorP &operator=(const TensorP &) = delete; + // The resource starts with a count of 1, which deallocate() takes away, and each + // TensorP adds 1 while it lives. So the TensorP that brings the count to 0 is the last + // user of a deallocated tensor. It replaces the array with a scalar of 4 bytes, which + // frees the MLX buffer now. The array object stays valid, because the resource + // destructor still destroys it when the BEAM collects the term. ~TensorP() { - if (is_valid()) { - // decrease reference count - if (refcount->fetch_sub(1) == 0) { - ptr->~array(); // Call MLX tensor destructor - } + if (is_valid() && refcount->fetch_sub(1) == 1) { + *ptr = mlx::core::array(0.0f); } } diff --git a/emlx/lib/emlx.ex b/emlx/lib/emlx.ex index 9a6dd44..10034de 100644 --- a/emlx/lib/emlx.ex +++ b/emlx/lib/emlx.ex @@ -1590,7 +1590,20 @@ defmodule EMLX do defp merge_device(_, :gpu), do: :gpu defp merge_device(_, _), do: :cpu - defvalue deallocate(tensor_ref) + @doc """ + Frees the MLX array of `tensor_ref` now, without waiting for the BEAM to collect the + term. A later use of the tensor raises. Returns `:ok`, or `:already_deallocated` for a + tensor that was already deallocated, as `Nx.backend_deallocate/1` expects. + """ + @mlx_function {:deallocate, 1} + def deallocate(tensor_ref) do + {[tensor_ref], _device} = prepare_tensors!([tensor_ref]) + + case EMLX.NIF.deallocate(tensor_ref) do + :already_deallocated -> :already_deallocated + result -> unwrap!(result) + end + end @doc """ Evaluates a (possibly lazy) MLX tensor by routing the work through an diff --git a/emlx/lib/emlx/backend.ex b/emlx/lib/emlx/backend.ex index b9f1414..f647092 100644 --- a/emlx/lib/emlx/backend.ex +++ b/emlx/lib/emlx/backend.ex @@ -267,9 +267,9 @@ defmodule EMLX.Backend do case opts[:mode] do :local -> # Eval happens inside tensor_data_ptr (same pattern as to_blob). - # The pointer is valid until another mx::eval on the same array or - # until the Elixir tensor term is GC'd. On Apple Silicon (unified - # memory) the address is accessible from both CPU and GPU. + # The pointer is valid until another mx::eval on the same array, or + # until the tensor is deallocated or its Elixir term is GC'd. On Apple + # Silicon (unified memory) the address is accessible from both CPU and GPU. {addr, byte_size} = EMLX.tensor_data_ptr(ref) %Nx.Pointer{kind: :local, address: addr, data_size: byte_size} diff --git a/emlx/test/emlx/memory_test.exs b/emlx/test/emlx/memory_test.exs index 42a40e9..225d292 100644 --- a/emlx/test/emlx/memory_test.exs +++ b/emlx/test/emlx/memory_test.exs @@ -32,6 +32,41 @@ defmodule EMLX.MemoryTest do assert info.cache_memory == 0 end + # Nx.backend_deallocate/1 frees the MLX array at once. Before, it only marked the tensor, + # and the memory stayed in use until the BEAM collected the term. The deallocated tensor + # keeps a scalar of 4 bytes until then. + test "deallocate frees the memory of a tensor without a garbage collection" do + EMLX.clear_cache() + t = Nx.iota({1024, 1024}, type: :f32, backend: EMLX.Backend) + EMLX.eval(EMLX.Backend.from_nx(t)) + before = EMLX.memory_info().active_memory + + assert Nx.backend_deallocate(t) == :ok + assert EMLX.memory_info().active_memory <= before - 1024 * 1024 * 4 + 4 + end + + test "a deallocated tensor raises on use, and a second deallocate does nothing" do + t = Nx.iota({4}, type: :f32, backend: EMLX.Backend) + assert Nx.backend_deallocate(t) == :ok + assert Nx.backend_deallocate(t) == :already_deallocated + assert_raise EMLX.NIFError, ~r/deallocated/, fn -> Nx.to_flat_list(t) end + end + + # A binary from Nx.to_binary/1 shares the buffer of a contiguous tensor without a copy. + # The binary keeps its own reference to the buffer, so a deallocate of the tensor does not + # free the memory that the binary still reads. + test "a binary keeps the buffer of a deallocated tensor" do + EMLX.clear_cache() + t = Nx.iota({1024, 1024}, type: :f32, backend: EMLX.Backend) + expected = t |> Nx.backend_copy(Nx.BinaryBackend) |> Nx.to_binary() + binary = Nx.to_binary(t) + before = EMLX.memory_info().active_memory + + Nx.backend_deallocate(t) + assert EMLX.memory_info().active_memory >= before - 4 + assert binary == expected + end + test "reset_peak_memory resets the counter" do _t = Nx.iota({1024, 1024}, type: :f32, backend: EMLX.Backend) EMLX.reset_peak_memory() From 6e416a9fdb99e6b383b473987358cb80dad19389 Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:13:23 -0300 Subject: [PATCH 2/2] fix memory assertions --- emlx/c_src/emlx_nif.cpp | 2 +- emlx/lib/emlx.ex | 7 +++---- emlx/test/emlx/memory_test.exs | 18 +++++++++++++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/emlx/c_src/emlx_nif.cpp b/emlx/c_src/emlx_nif.cpp index 898e6e4..6d55b9b 100644 --- a/emlx/c_src/emlx_nif.cpp +++ b/emlx/c_src/emlx_nif.cpp @@ -55,7 +55,7 @@ NIF(deallocate) { if (t.deallocate()) { return nx::nif::ok(env); } else { - return nx::nif::atom(env, "already_deallocated"); + return nx::nif::ok(env, nx::nif::atom(env, "already_deallocated")); } } diff --git a/emlx/lib/emlx.ex b/emlx/lib/emlx.ex index 10034de..4fb67b8 100644 --- a/emlx/lib/emlx.ex +++ b/emlx/lib/emlx.ex @@ -1599,10 +1599,9 @@ defmodule EMLX do def deallocate(tensor_ref) do {[tensor_ref], _device} = prepare_tensors!([tensor_ref]) - case EMLX.NIF.deallocate(tensor_ref) do - :already_deallocated -> :already_deallocated - result -> unwrap!(result) - end + tensor_ref + |> EMLX.NIF.deallocate() + |> unwrap!() end @doc """ diff --git a/emlx/test/emlx/memory_test.exs b/emlx/test/emlx/memory_test.exs index 225d292..7d24668 100644 --- a/emlx/test/emlx/memory_test.exs +++ b/emlx/test/emlx/memory_test.exs @@ -42,7 +42,23 @@ defmodule EMLX.MemoryTest do before = EMLX.memory_info().active_memory assert Nx.backend_deallocate(t) == :ok - assert EMLX.memory_info().active_memory <= before - 1024 * 1024 * 4 + 4 + assert wait_for_memory(before - 1024 * 1024 * 4 + 4) + end + + # On the CPU stream, the thread that ran the eval can still hold a copy of the array for a + # short time after the eval returns, so the buffer frees a moment after the deallocate. + defp wait_for_memory(limit, attempts \\ 100) do + cond do + EMLX.memory_info().active_memory <= limit -> + true + + attempts == 0 -> + false + + true -> + Process.sleep(10) + wait_for_memory(limit, attempts - 1) + end end test "a deallocated tensor raises on use, and a second deallocate does nothing" do