diff --git a/emlx/c_src/emlx_nif.cpp b/emlx/c_src/emlx_nif.cpp index 0c1bee6..fdc0837 100644 --- a/emlx/c_src/emlx_nif.cpp +++ b/emlx/c_src/emlx_nif.cpp @@ -55,7 +55,7 @@ NIF(deallocate) { if (t.deallocate()) { return nx::nif::ok(env); } else { - return nx::nif::atom(env, "already_deallocated"); + return nx::nif::ok(env, nx::nif::atom(env, "already_deallocated")); } } @@ -121,36 +121,33 @@ NIF(copy) { // legacy path; a process-independent msg_env for the worker path). static ERL_NIF_TERM to_blob_term(ErlNifEnv *out_env, mlx::core::array *t, size_t byte_size) { - if (t->flags().row_contiguous) { - // Zero-copy: alias the MLX buffer via the existing tensor resource. - // Invariant: lib/emlx.ex calls eval(tensor) before this NIF, so - // data() is guaranteed non-null and stable (MLX arrays are immutable - // once materialised). enif_make_resource_binary keeps the resource alive - // until the binary is GC'd, decoupling the binary lifetime from Elixir GC - // of the tensor term. - return enif_make_resource_binary(out_env, static_cast(t), - t->data(), byte_size); + // Zero-copy: the binary aliases the MLX buffer. Invariant: lib/emlx.ex calls + // eval(tensor) before this NIF, so data() is non-null and stable (MLX + // arrays are immutable once materialised). A non-contiguous tensor first gets + // a fresh row-major copy. + // + // The binary keeps a minimal ERTS resource alive until it is GC'd. That + // resource holds its own mlx::core::array, which shares the buffer, so the + // binary stays valid after a deallocate frees the array of the tensor. The + // resource holds only sizeof(mlx::core::array), without the TensorP refcount + // and deleted-flag tail, because TensorP and the Elixir side never see it; + // default_dtor (~array()) is its only destructor path. + mlx::core::array source = *t; + if (!t->flags().row_contiguous) { + source = mlx::core::contiguous(*t); + mlx::core::eval(source); } - // Non-contiguous: materialise a fresh row-major copy, wrap it in a minimal - // ERTS resource, and alias that buffer zero-copy. - // The resource holds only sizeof(mlx::core::array) — no TensorP refcount/ - // deleted-flag tail — because it is never exposed to TensorP or the Elixir - // side; only the binary holds a reference and default_dtor - // (~array()) is the sole destructor path. - auto ct = mlx::core::contiguous(*t); - mlx::core::eval(ct); - - auto *ct_ptr = static_cast(enif_alloc_resource( + auto *ptr = static_cast(enif_alloc_resource( resource_object::type, sizeof(mlx::core::array))); - if (!ct_ptr) { - throw std::runtime_error("Unable to allocate contiguous-copy resource"); + if (!ptr) { + throw std::runtime_error("Unable to allocate the resource of a binary"); } - new (ct_ptr) mlx::core::array(std::move(ct)); - ERL_NIF_TERM resource_bin = enif_make_resource_binary( - out_env, ct_ptr, ct_ptr->data(), byte_size); - enif_release_resource(ct_ptr); + new (ptr) mlx::core::array(std::move(source)); + ERL_NIF_TERM resource_bin = + enif_make_resource_binary(out_env, ptr, ptr->data(), byte_size); + enif_release_resource(ptr); return resource_bin; } diff --git a/emlx/c_src/emlx_nif_shared.hpp b/emlx/c_src/emlx_nif_shared.hpp index a69756a..ddb8acb 100644 --- a/emlx/c_src/emlx_nif_shared.hpp +++ b/emlx/c_src/emlx_nif_shared.hpp @@ -91,12 +91,14 @@ class TensorP { TensorP(const TensorP &) = delete; TensorP &operator=(const TensorP &) = delete; + // The resource starts with a count of 1, which deallocate() takes away, and each + // TensorP adds 1 while it lives. So the TensorP that brings the count to 0 is the last + // user of a deallocated tensor. It replaces the array with a scalar of 4 bytes, which + // frees the MLX buffer now. The array object stays valid, because the resource + // destructor still destroys it when the BEAM collects the term. ~TensorP() { - if (is_valid()) { - // decrease reference count - if (refcount->fetch_sub(1) == 0) { - ptr->~array(); // Call MLX tensor destructor - } + if (is_valid() && refcount->fetch_sub(1) == 1) { + *ptr = mlx::core::array(0.0f); } } diff --git a/emlx/lib/emlx.ex b/emlx/lib/emlx.ex index 5767fe1..a3a1fd1 100644 --- a/emlx/lib/emlx.ex +++ b/emlx/lib/emlx.ex @@ -1590,7 +1590,19 @@ defmodule EMLX do defp merge_device(_, :gpu), do: :gpu defp merge_device(_, _), do: :cpu - defvalue deallocate(tensor_ref) + @doc """ + Frees the MLX array of `tensor_ref` now, without waiting for the BEAM to collect the + term. A later use of the tensor raises. Returns `:ok`, or `:already_deallocated` for a + tensor that was already deallocated, as `Nx.backend_deallocate/1` expects. + """ + @mlx_function {:deallocate, 1} + def deallocate(tensor_ref) do + {[tensor_ref], _device} = prepare_tensors!([tensor_ref]) + + tensor_ref + |> EMLX.NIF.deallocate() + |> unwrap!() + end @doc """ Evaluates a (possibly lazy) MLX tensor by routing the work through an diff --git a/emlx/lib/emlx/backend.ex b/emlx/lib/emlx/backend.ex index b9f1414..f647092 100644 --- a/emlx/lib/emlx/backend.ex +++ b/emlx/lib/emlx/backend.ex @@ -267,9 +267,9 @@ defmodule EMLX.Backend do case opts[:mode] do :local -> # Eval happens inside tensor_data_ptr (same pattern as to_blob). - # The pointer is valid until another mx::eval on the same array or - # until the Elixir tensor term is GC'd. On Apple Silicon (unified - # memory) the address is accessible from both CPU and GPU. + # The pointer is valid until another mx::eval on the same array, or + # until the tensor is deallocated or its Elixir term is GC'd. On Apple + # Silicon (unified memory) the address is accessible from both CPU and GPU. {addr, byte_size} = EMLX.tensor_data_ptr(ref) %Nx.Pointer{kind: :local, address: addr, data_size: byte_size} diff --git a/emlx/test/emlx/memory_test.exs b/emlx/test/emlx/memory_test.exs index 42a40e9..7d24668 100644 --- a/emlx/test/emlx/memory_test.exs +++ b/emlx/test/emlx/memory_test.exs @@ -32,6 +32,57 @@ defmodule EMLX.MemoryTest do assert info.cache_memory == 0 end + # Nx.backend_deallocate/1 frees the MLX array at once. Before, it only marked the tensor, + # and the memory stayed in use until the BEAM collected the term. The deallocated tensor + # keeps a scalar of 4 bytes until then. + test "deallocate frees the memory of a tensor without a garbage collection" do + EMLX.clear_cache() + t = Nx.iota({1024, 1024}, type: :f32, backend: EMLX.Backend) + EMLX.eval(EMLX.Backend.from_nx(t)) + before = EMLX.memory_info().active_memory + + assert Nx.backend_deallocate(t) == :ok + assert wait_for_memory(before - 1024 * 1024 * 4 + 4) + end + + # On the CPU stream, the thread that ran the eval can still hold a copy of the array for a + # short time after the eval returns, so the buffer frees a moment after the deallocate. + defp wait_for_memory(limit, attempts \\ 100) do + cond do + EMLX.memory_info().active_memory <= limit -> + true + + attempts == 0 -> + false + + true -> + Process.sleep(10) + wait_for_memory(limit, attempts - 1) + end + end + + test "a deallocated tensor raises on use, and a second deallocate does nothing" do + t = Nx.iota({4}, type: :f32, backend: EMLX.Backend) + assert Nx.backend_deallocate(t) == :ok + assert Nx.backend_deallocate(t) == :already_deallocated + assert_raise EMLX.NIFError, ~r/deallocated/, fn -> Nx.to_flat_list(t) end + end + + # A binary from Nx.to_binary/1 shares the buffer of a contiguous tensor without a copy. + # The binary keeps its own reference to the buffer, so a deallocate of the tensor does not + # free the memory that the binary still reads. + test "a binary keeps the buffer of a deallocated tensor" do + EMLX.clear_cache() + t = Nx.iota({1024, 1024}, type: :f32, backend: EMLX.Backend) + expected = t |> Nx.backend_copy(Nx.BinaryBackend) |> Nx.to_binary() + binary = Nx.to_binary(t) + before = EMLX.memory_info().active_memory + + Nx.backend_deallocate(t) + assert EMLX.memory_info().active_memory >= before - 4 + assert binary == expected + end + test "reset_peak_memory resets the counter" do _t = Nx.iota({1024, 1024}, type: :f32, backend: EMLX.Backend) EMLX.reset_peak_memory()