Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions emlx/c_src/emlx/compiler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2497,15 +2497,15 @@ ERL_NIF_TERM eval_program(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {

LIST_PARAM(1, std::vector<mlx::core::array>, inputs);

// Force-evaluate all inputs on the worker thread before passing them to the
// Schedule all inputs on the worker thread before passing them to the
// compiled function. MLX tensors created via Elixir (e.g. Nx.tensor/2) may
// arrive as unevaluated lazy nodes associated with a different computation.
// If they are consumed inside compiled_fn while still lazy, MLX may
// propagate stale or garbage data from a previously-evaluated graph that
// happened to share the same underlying buffer. Forcing eval here ensures
// all inputs are materialized scalars/arrays on the worker's stream before
// the compiled graph is built or replayed.
mlx::core::eval(inputs);
// happened to share the same underlying buffer. Scheduled inputs are not
// lazy nodes anymore, and async_eval does not wait for them, so the caller
// can build the next call while the device runs this one.
mlx::core::async_eval(inputs);

std::vector<mlx::core::array> outputs;
{
Expand Down
11 changes: 10 additions & 1 deletion emlx/c_src/emlx_nif.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -444,10 +444,11 @@ NIF(argsort) {
TENSOR(mlx::core::argsort(*t, axis, device));
}

// No stream-wide synchronize: it would also wait for the work that
// async_eval scheduled after this array, such as the next decode step.
NIF(eval) {
TENSOR_PARAM(0, t);
mlx::core::eval(*t);
mlx::core::synchronize();
return nx::nif::ok(env);
}

Expand All @@ -465,6 +466,12 @@ NIF(eval_many) {
return nx::nif::ok(env);
}

NIF(async_eval) {
LIST_PARAM(0, std::vector<mlx::core::array>, inputs);
mlx::core::async_eval(inputs);
return nx::nif::ok(env);
}

NIF(to_device) {
TENSOR_PARAM(0, t);
DEVICE_PARAM(1, device);
Expand Down Expand Up @@ -1747,6 +1754,7 @@ NIF(window_scatter_min) {

ASYNC_NIF(eval)
ASYNC_NIF(eval_many)
ASYNC_NIF(async_eval)
ASYNC_NIF(to_blob)
ASYNC_NIF(tensor_to_shm)
ASYNC_NIF(item)
Expand Down Expand Up @@ -1895,6 +1903,7 @@ static ErlNifFunc nif_funcs[] = {
// as the ~150 sibling ASYNC_NIF ops below (item 3.8 fold-in).
{"eval", 2, eval_async},
{"eval_many", 2, eval_many_async},
{"async_eval", 2, async_eval_async},
{"to_device", 3, to_device_async},
{"to_blob", 2, to_blob_async},
{"to_blob", 3, to_blob_async},
Expand Down
37 changes: 37 additions & 0 deletions emlx/lib/emlx.ex
Original file line number Diff line number Diff line change
Expand Up @@ -1597,6 +1597,11 @@ defmodule EMLX do
`EMLX.CommandQueue`. Blocks the caller until the worker thread has
finished `mlx::core::eval/1` for this tensor.

It waits for this tensor only. Other work on the same stream, such as
tensors that `async_eval/1` scheduled after it, can still run when it
returns. Use `EMLX.CommandQueue.synchronize/1` to wait for all the work of
a queue.

Resolves the queue via `resolve_worker/1`:

1. If the calling process has bound a queue with
Expand All @@ -1614,6 +1619,38 @@ defmodule EMLX do
end)
end

@doc """
Schedules one or more (possibly lazy) MLX tensors on their device and
returns without waiting for the result.

The worker thread builds the command buffers of the tensors and commits
them, so the device starts to compute them. A later `eval/1`, `item/1` or
`to_blob/1` of one of the tensors waits only for that tensor. A compiled
call that takes a scheduled tensor as an input does not wait for it either.

This lets a loop build its next step while the device runs the current
one. For example, a decode loop calls the compiled step `t + 1` with the
lazy token of step `t`, schedules step `t + 1`, and then reads the token
of step `t`:

next = step.(token)
EMLX.async_eval(EMLX.Backend.from_nx(next))
Nx.to_number(token)

Takes a tensor ref (`{device, ref}`) or a list of them. Tensors of each
device go to the worker of that device, resolved as in `eval/1`.
"""
def async_eval({device, ref} = tensor) when is_tensor(device, ref), do: async_eval([tensor])

def async_eval(tensors) when is_list(tensors) do
tensors
|> Enum.group_by(&elem(&1, 0), &elem(&1, 1))
|> Enum.each(fn {device, refs} ->
{worker, _effective_device} = resolve_worker(device)
EMLX.NIF.async_eval(worker, refs) |> unwrap!() |> await_worker()
end)
end

@doc """
Moves a tensor to `target_device` (`:cpu` or `:gpu`) without deallocating
the source. This is a no-op if the tensor is already on the target device.
Expand Down
4 changes: 4 additions & 0 deletions emlx/lib/emlx/nif.ex
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ defmodule EMLX.NIF do
:erlang.nif_error(:nif_not_loaded)
end

def async_eval(_worker, _tensor_refs) do
:erlang.nif_error(:nif_not_loaded)
end

def to_device(_worker, _tensor, _device) do
:erlang.nif_error(:nif_not_loaded)
end
Expand Down
86 changes: 86 additions & 0 deletions emlx/test/emlx/async_eval_test.exs
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
defmodule EMLX.AsyncEvalTest do
use EMLX.Case, async: true

defp step(x, w), do: x |> Nx.dot(w) |> Nx.divide(16) |> Nx.add(1)

defp ref(tensor), do: EMLX.Backend.from_nx(tensor)

for device <- [:cpu, :gpu] do
describe "#{device}" do
if device == :gpu, do: @describetag(:metal)

setup do
backend = {EMLX.Backend, device: unquote(device)}
x = Nx.iota({64, 64}, type: :f32, backend: backend) |> Nx.divide(4096)
w = Nx.iota({64, 64}, type: :f32, backend: backend) |> Nx.remainder(7) |> Nx.divide(7)

expected =
x |> Nx.backend_copy(Nx.BinaryBackend) |> step(Nx.backend_copy(w, Nx.BinaryBackend))

%{backend: backend, x: x, w: w, expected: expected}
end

test "a compiled call takes lazy inputs and the lazy outputs of another call", ctx do
compiled =
Nx.Defn.compile(&step/2, [ctx.x, ctx.w], compiler: EMLX, device: unquote(device))

lazy_x = ctx.x |> Nx.multiply(2) |> Nx.divide(2)
first = compiled.(lazy_x, ctx.w)
second = compiled.(first, ctx.w)

expected_second = step(ctx.expected, Nx.backend_copy(ctx.w, Nx.BinaryBackend))
assert_all_close(Nx.backend_copy(first, Nx.BinaryBackend), ctx.expected)
assert_all_close(Nx.backend_copy(second, Nx.BinaryBackend), expected_second)
end

test "async_eval and then a read give the values of the tensors", ctx do
a = step(ctx.x, ctx.w)
b = Nx.multiply(a, 3)

assert EMLX.async_eval([ref(a), ref(b)]) == :ok
assert_all_close(Nx.backend_copy(a, Nx.BinaryBackend), ctx.expected)
assert_all_close(Nx.backend_copy(b, Nx.BinaryBackend), Nx.multiply(ctx.expected, 3))
end

test "a read right after eval has the values while later work runs", ctx do
a = step(ctx.x, ctx.w)
EMLX.async_eval(ref(a))

later = Enum.reduce(1..20, a, fn _, acc -> step(acc, ctx.w) end)
EMLX.async_eval(ref(later))

EMLX.eval(ref(a))
assert_all_close(Nx.backend_copy(a, Nx.BinaryBackend), ctx.expected)
assert later |> Nx.is_nan() |> Nx.any() |> Nx.to_number() == 0
end

test "a pipelined loop reads each step after it schedules the next one", ctx do
compiled =
Nx.Defn.compile(&step/2, [ctx.x, ctx.w], compiler: EMLX, device: unquote(device))

host_w = Nx.backend_copy(ctx.w, Nx.BinaryBackend)

expected =
Enum.map_reduce(1..8, Nx.backend_copy(ctx.x, Nx.BinaryBackend), fn _, acc ->
next = step(acc, host_w)
{Nx.sum(next) |> Nx.to_number(), next}
end)
|> elem(0)

{sums, last} =
Enum.map_reduce(1..8, ctx.x, fn i, previous ->
next = compiled.(previous, ctx.w)
EMLX.async_eval(ref(next))
sum = if i > 1, do: [Nx.sum(previous) |> Nx.to_number()], else: []
{sum, next}
end)

sums = List.flatten(sums) ++ [Nx.sum(last) |> Nx.to_number()]

for {got, want} <- Enum.zip(sums, expected) do
assert_in_delta got, want, abs(want) * 1.0e-3
end
end
end
end
end
Loading