From 4a3210c75e5b535fe73df49f9bfea958ff09b951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?George=20Guimar=C3=A3es?= Date: Sun, 25 Oct 2026 18:21:13 -0300 Subject: [PATCH] feat: Add EMLX.async_eval/1 and do not wait for compiled call inputs A decode loop waits for the GPU twice per token, and the CPU work of each step runs while the GPU is idle. EMLX.async_eval/1 schedules tensors and returns at once. eval_program now schedules its inputs with mx::async_eval instead of mx::eval. The eval NIF no longer calls a stream-wide synchronize, because it would also wait for work that async_eval scheduled later. A loop can then build step t + 1 while the device runs step t. --- emlx/c_src/emlx/compiler.cpp | 10 ++-- emlx/c_src/emlx_nif.cpp | 11 +++- emlx/lib/emlx.ex | 37 +++++++++++++ emlx/lib/emlx/nif.ex | 4 ++ emlx/test/emlx/async_eval_test.exs | 86 ++++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 emlx/test/emlx/async_eval_test.exs diff --git a/emlx/c_src/emlx/compiler.cpp b/emlx/c_src/emlx/compiler.cpp index 396ba66..2bd261e 100644 --- a/emlx/c_src/emlx/compiler.cpp +++ b/emlx/c_src/emlx/compiler.cpp @@ -2497,15 +2497,15 @@ ERL_NIF_TERM eval_program(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { LIST_PARAM(1, std::vector, inputs); - // Force-evaluate all inputs on the worker thread before passing them to the + // Schedule all inputs on the worker thread before passing them to the // compiled function. MLX tensors created via Elixir (e.g. Nx.tensor/2) may // arrive as unevaluated lazy nodes associated with a different computation. // If they are consumed inside compiled_fn while still lazy, MLX may // propagate stale or garbage data from a previously-evaluated graph that - // happened to share the same underlying buffer. Forcing eval here ensures - // all inputs are materialized scalars/arrays on the worker's stream before - // the compiled graph is built or replayed. - mlx::core::eval(inputs); + // happened to share the same underlying buffer. Scheduled inputs are not + // lazy nodes anymore, and async_eval does not wait for them, so the caller + // can build the next call while the device runs this one. + mlx::core::async_eval(inputs); std::vector outputs; { diff --git a/emlx/c_src/emlx_nif.cpp b/emlx/c_src/emlx_nif.cpp index b7d1128..0c1bee6 100644 --- a/emlx/c_src/emlx_nif.cpp +++ b/emlx/c_src/emlx_nif.cpp @@ -444,10 +444,11 @@ NIF(argsort) { TENSOR(mlx::core::argsort(*t, axis, device)); } +// No stream-wide synchronize: it would also wait for the work that +// async_eval scheduled after this array, such as the next decode step. NIF(eval) { TENSOR_PARAM(0, t); mlx::core::eval(*t); - mlx::core::synchronize(); return nx::nif::ok(env); } @@ -465,6 +466,12 @@ NIF(eval_many) { return nx::nif::ok(env); } +NIF(async_eval) { + LIST_PARAM(0, std::vector, inputs); + mlx::core::async_eval(inputs); + return nx::nif::ok(env); +} + NIF(to_device) { TENSOR_PARAM(0, t); DEVICE_PARAM(1, device); @@ -1747,6 +1754,7 @@ NIF(window_scatter_min) { ASYNC_NIF(eval) ASYNC_NIF(eval_many) +ASYNC_NIF(async_eval) ASYNC_NIF(to_blob) ASYNC_NIF(tensor_to_shm) ASYNC_NIF(item) @@ -1895,6 +1903,7 @@ static ErlNifFunc nif_funcs[] = { // as the ~150 sibling ASYNC_NIF ops below (item 3.8 fold-in). {"eval", 2, eval_async}, {"eval_many", 2, eval_many_async}, + {"async_eval", 2, async_eval_async}, {"to_device", 3, to_device_async}, {"to_blob", 2, to_blob_async}, {"to_blob", 3, to_blob_async}, diff --git a/emlx/lib/emlx.ex b/emlx/lib/emlx.ex index 9a6dd44..b48be05 100644 --- a/emlx/lib/emlx.ex +++ b/emlx/lib/emlx.ex @@ -1597,6 +1597,11 @@ defmodule EMLX do `EMLX.CommandQueue`. Blocks the caller until the worker thread has finished `mlx::core::eval/1` for this tensor. + It waits for this tensor only. Other work on the same stream, such as + tensors that `async_eval/1` scheduled after it, can still run when it + returns. Use `EMLX.CommandQueue.synchronize/1` to wait for all the work of + a queue. + Resolves the queue via `resolve_worker/1`: 1. If the calling process has bound a queue with @@ -1614,6 +1619,38 @@ defmodule EMLX do end) end + @doc """ + Schedules one or more (possibly lazy) MLX tensors on their device and + returns without waiting for the result. + + The worker thread builds the command buffers of the tensors and commits + them, so the device starts to compute them. A later `eval/1`, `item/1` or + `to_blob/1` of one of the tensors waits only for that tensor. A compiled + call that takes a scheduled tensor as an input does not wait for it either. + + This lets a loop build its next step while the device runs the current + one. For example, a decode loop calls the compiled step `t + 1` with the + lazy token of step `t`, schedules step `t + 1`, and then reads the token + of step `t`: + + next = step.(token) + EMLX.async_eval(EMLX.Backend.from_nx(next)) + Nx.to_number(token) + + Takes a tensor ref (`{device, ref}`) or a list of them. Tensors of each + device go to the worker of that device, resolved as in `eval/1`. + """ + def async_eval({device, ref} = tensor) when is_tensor(device, ref), do: async_eval([tensor]) + + def async_eval(tensors) when is_list(tensors) do + tensors + |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) + |> Enum.each(fn {device, refs} -> + {worker, _effective_device} = resolve_worker(device) + EMLX.NIF.async_eval(worker, refs) |> unwrap!() |> await_worker() + end) + end + @doc """ Moves a tensor to `target_device` (`:cpu` or `:gpu`) without deallocating the source. This is a no-op if the tensor is already on the target device. diff --git a/emlx/lib/emlx/nif.ex b/emlx/lib/emlx/nif.ex index d35fd5e..4d9ef44 100644 --- a/emlx/lib/emlx/nif.ex +++ b/emlx/lib/emlx/nif.ex @@ -46,6 +46,10 @@ defmodule EMLX.NIF do :erlang.nif_error(:nif_not_loaded) end + def async_eval(_worker, _tensor_refs) do + :erlang.nif_error(:nif_not_loaded) + end + def to_device(_worker, _tensor, _device) do :erlang.nif_error(:nif_not_loaded) end diff --git a/emlx/test/emlx/async_eval_test.exs b/emlx/test/emlx/async_eval_test.exs new file mode 100644 index 0000000..c0a9178 --- /dev/null +++ b/emlx/test/emlx/async_eval_test.exs @@ -0,0 +1,86 @@ +defmodule EMLX.AsyncEvalTest do + use EMLX.Case, async: true + + defp step(x, w), do: x |> Nx.dot(w) |> Nx.divide(16) |> Nx.add(1) + + defp ref(tensor), do: EMLX.Backend.from_nx(tensor) + + for device <- [:cpu, :gpu] do + describe "#{device}" do + if device == :gpu, do: @describetag(:metal) + + setup do + backend = {EMLX.Backend, device: unquote(device)} + x = Nx.iota({64, 64}, type: :f32, backend: backend) |> Nx.divide(4096) + w = Nx.iota({64, 64}, type: :f32, backend: backend) |> Nx.remainder(7) |> Nx.divide(7) + + expected = + x |> Nx.backend_copy(Nx.BinaryBackend) |> step(Nx.backend_copy(w, Nx.BinaryBackend)) + + %{backend: backend, x: x, w: w, expected: expected} + end + + test "a compiled call takes lazy inputs and the lazy outputs of another call", ctx do + compiled = + Nx.Defn.compile(&step/2, [ctx.x, ctx.w], compiler: EMLX, device: unquote(device)) + + lazy_x = ctx.x |> Nx.multiply(2) |> Nx.divide(2) + first = compiled.(lazy_x, ctx.w) + second = compiled.(first, ctx.w) + + expected_second = step(ctx.expected, Nx.backend_copy(ctx.w, Nx.BinaryBackend)) + assert_all_close(Nx.backend_copy(first, Nx.BinaryBackend), ctx.expected) + assert_all_close(Nx.backend_copy(second, Nx.BinaryBackend), expected_second) + end + + test "async_eval and then a read give the values of the tensors", ctx do + a = step(ctx.x, ctx.w) + b = Nx.multiply(a, 3) + + assert EMLX.async_eval([ref(a), ref(b)]) == :ok + assert_all_close(Nx.backend_copy(a, Nx.BinaryBackend), ctx.expected) + assert_all_close(Nx.backend_copy(b, Nx.BinaryBackend), Nx.multiply(ctx.expected, 3)) + end + + test "a read right after eval has the values while later work runs", ctx do + a = step(ctx.x, ctx.w) + EMLX.async_eval(ref(a)) + + later = Enum.reduce(1..20, a, fn _, acc -> step(acc, ctx.w) end) + EMLX.async_eval(ref(later)) + + EMLX.eval(ref(a)) + assert_all_close(Nx.backend_copy(a, Nx.BinaryBackend), ctx.expected) + assert later |> Nx.is_nan() |> Nx.any() |> Nx.to_number() == 0 + end + + test "a pipelined loop reads each step after it schedules the next one", ctx do + compiled = + Nx.Defn.compile(&step/2, [ctx.x, ctx.w], compiler: EMLX, device: unquote(device)) + + host_w = Nx.backend_copy(ctx.w, Nx.BinaryBackend) + + expected = + Enum.map_reduce(1..8, Nx.backend_copy(ctx.x, Nx.BinaryBackend), fn _, acc -> + next = step(acc, host_w) + {Nx.sum(next) |> Nx.to_number(), next} + end) + |> elem(0) + + {sums, last} = + Enum.map_reduce(1..8, ctx.x, fn i, previous -> + next = compiled.(previous, ctx.w) + EMLX.async_eval(ref(next)) + sum = if i > 1, do: [Nx.sum(previous) |> Nx.to_number()], else: [] + {sum, next} + end) + + sums = List.flatten(sums) ++ [Nx.sum(last) |> Nx.to_number()] + + for {got, want} <- Enum.zip(sums, expected) do + assert_in_delta got, want, abs(want) * 1.0e-3 + end + end + end + end +end