Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 43 additions & 12 deletions emlx/c_src/emlx_runtime_call_bridge.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
#include <condition_variable>
#include <cstdint>
#include <cstring>
#include <memory>
#include <mutex>
#include <new>
#include <stdexcept>
Expand Down Expand Up @@ -141,6 +142,28 @@ inline void invoke_runtime_call(int64_t callback_index,
}
new (pending) PendingRuntimeCall();

// The worker must own a reference while waiting: the recipient may die or
// discard the message at any point. Also invalidate destinations on every
// exit path, so a delayed/duplicate reply can never write freed output memory.
auto release_pending = [](PendingRuntimeCall *p) {
{
std::lock_guard<std::mutex> lock(p->mu);
if (!p->done) {
p->done = true;
p->ok = false;
p->error = "runtime_call abandoned";
}
p->outputs.clear();
}
p->cv.notify_one();
enif_release_resource(p);
};
std::unique_ptr<PendingRuntimeCall, decltype(release_pending)> owner(pending, release_pending);
std::unique_ptr<ErlNifEnv, decltype(&enif_free_env)> alive_env(enif_alloc_env(), enif_free_env);
if (!alive_env) {
throw std::runtime_error("emlx::native: failed to allocate liveness env");
}

if (!alias_inputs) {
// Pre-allocate every output buffer now so the destination pointers are
// stable before we hand the request to the Elixir side.
Expand All @@ -154,7 +177,6 @@ inline void invoke_runtime_call(int64_t callback_index,

ErlNifEnv *msg_env = enif_alloc_env();
if (msg_env == nullptr) {
enif_release_resource(pending);
throw std::runtime_error("emlx::native: failed to allocate msg env for runtime_call");
}

Expand All @@ -174,7 +196,6 @@ inline void invoke_runtime_call(int64_t callback_index,
size_t nbytes = in.nbytes();
ErlNifBinary bin;
if (!enif_alloc_binary(nbytes, &bin)) {
enif_release_resource(pending);
enif_free_env(msg_env);
throw std::runtime_error(
"emlx::native: failed to allocate binary for runtime_call arg");
Expand All @@ -195,11 +216,7 @@ inline void invoke_runtime_call(int64_t callback_index,
}

ERL_NIF_TERM pending_term = enif_make_resource(msg_env, pending);
// enif_make_resource/enif_make_copy below keep their own references; drop
// ours now — the resource stays alive via the message term (and later,
// whatever term resolve_runtime_call's caller decodes it into) until its
// own destructor runs.
enif_release_resource(pending);
// Keep the worker-owned reference until this invocation has unwound.

ERL_NIF_TERM args_list = enif_make_list_from_array(
msg_env, arg_terms.data(), static_cast<unsigned>(arg_terms.size()));
Expand All @@ -218,8 +235,14 @@ inline void invoke_runtime_call(int64_t callback_index,
"emlx::native: failed to send runtime_call request (target process not alive?)");
}

auto is_done = [pending] {
auto is_done = [pending, &alive_env, &caller_pid] {
std::lock_guard<std::mutex> lock(pending->mu);
if (!pending->done && !enif_is_process_alive(alive_env.get(), &caller_pid)) {
pending->done = true;
pending->ok = false;
pending->error = "runtime_call caller exited";
pending->outputs.clear();
}
return pending->done;
};

Expand All @@ -233,8 +256,12 @@ inline void invoke_runtime_call(int64_t callback_index,
if (Worker *worker = g_current_worker) {
worker->pump_until(is_done);
} else {
std::unique_lock<std::mutex> lock(pending->mu);
pending->cv.wait(lock, [pending] { return pending->done; });
// A dead caller cannot reply or notify this condition variable. Wake
// periodically to check its liveness, just as pump_until does above.
while (!is_done()) {
std::unique_lock<std::mutex> lock(pending->mu);
pending->cv.wait_for(lock, std::chrono::milliseconds(5));
}
}

std::lock_guard<std::mutex> lock(pending->mu);
Expand All @@ -244,8 +271,8 @@ inline void invoke_runtime_call(int64_t callback_index,

if (alias_inputs) {
size_t passthrough_start = inputs.size() - outputs.size();
for (size_t i = passthrough_start; i < outputs.size(); i++) {
outputs[i].copy_shared_buffer(inputs[i]);
for (size_t i = 0; i < outputs.size(); i++) {
outputs[i].copy_shared_buffer(inputs[passthrough_start + i]);
}
}
}
Expand Down Expand Up @@ -280,6 +307,10 @@ inline ERL_NIF_TERM resolve_runtime_call(ErlNifEnv *env, int argc,
{
std::lock_guard<std::mutex> lock(pending->mu);

if (pending->done) {
return nx::nif::error(env, "runtime_call already resolved or abandoned");
}

if (is_ok) {
unsigned n;
if (!enif_get_list_length(env, argv[2], &n)) {
Expand Down
82 changes: 53 additions & 29 deletions emlx/lib/emlx.ex
Original file line number Diff line number Diff line change
Expand Up @@ -146,11 +146,11 @@ defmodule EMLX do

## Compile once, replay many times

`Nx.Defn.jit/2` and `Nx.Defn.jit_apply/3` still retrace through Nx on every
For callback-free functions, `Nx.Defn.jit/2` and `Nx.Defn.jit_apply/3` retrace through Nx on every
call, but `EMLX.__compile__/4` caches the resulting eval closure in
`EMLX.CompileCache`, keyed by function identity, input templates, and
device. Repeated jit of the same `&Mod.f/a` with the same shapes hits that
cache and skips `native_compile/3`.
cache and skips native compilation.

For a hot loop (e.g. a decode step), prefer `Nx.Defn.compile/3`, which traces
and lowers **once** and returns a plain closure the caller holds — no ETS
Expand All @@ -172,6 +172,14 @@ defmodule EMLX do
Cache bounds are documented on `EMLX.CompileCache`. A caller that keeps the
`Nx.Defn.compile/3` closure can pass `cache: false` to bypass the cache mechanism.

## Named hooks

Use the compiler's `:hooks` option to override named `io_call` callbacks.
Otherwise, the inline callback runs, if provided.

Executables with callbacks bypass global caches; keep the function returned
by `Nx.Defn.compile/3` to reuse them.

## Compile-time debug flags

Several development-only checks are gated by `Application.compile_env/3`
Expand Down Expand Up @@ -1854,6 +1862,8 @@ defmodule EMLX do
end)
rescue
e -> {:error, Exception.format(:error, e, __STACKTRACE__)}
catch
kind, reason -> {:error, Exception.format(kind, reason, __STACKTRACE__)}
end

case reply do
Expand Down Expand Up @@ -1898,7 +1908,7 @@ defmodule EMLX do

@valid_compiler_keys [:device, :max_concurrency, :command_queue, :hooks, :cache]

# Structural program cache for `get_or_compile_program/6`.
# Structural program cache for callback-free programs.
@native_dispatch_cache_table :emlx_native_dispatch_cache

# Expr-id cache in front of `dispatch_key/3`.
Expand All @@ -1922,26 +1932,17 @@ defmodule EMLX do

hooks = Keyword.get(opts, :hooks, %{})

case hooks do
empty when empty == %{} ->
:ok

_ ->
raise ArgumentError,
"EMLX does not support the :hooks named-override map (got callbacks for " <>
"#{inspect(Map.keys(hooks))}) — :hook/:io_call expr nodes lower natively " <>
"with their own inline default callback, but swapping in a different " <>
"callback per hook name via compiler opts isn't wired through"
end
unless is_map(hooks), do: raise(ArgumentError, ":hooks option must be a map")

queue = Keyword.get(opts, :command_queue)
device = Keyword.get(opts, :device, default_device())

eval_fn =
if Keyword.get(opts, :cache, true) do
if Keyword.get(opts, :cache, true) and hooks == %{} do
cached_compile(compile_cache_key(key, vars, hooks, device), vars, fun, device)
else
native_compile(vars, fun, device)
{built, _cacheable?} = native_compile(vars, fun, device, hooks)
built
end

wrap_with_queue(queue, eval_fn)
Expand All @@ -1953,24 +1954,28 @@ defmodule EMLX do
cached

{:error, :cache_miss} ->
built = native_compile(vars, fun, device)
EMLX.CompileCache.put(cache_key, built)
{built, cacheable?} = native_compile(vars, fun, device, %{})
if cacheable?, do: EMLX.CompileCache.put(cache_key, built), else: built
end
end

# Attempts to lower `fun.(vars)` to an `EMLX.Native.Expr` program and build a
# compiled program resource, returning the resulting eval closure. Single-mode:
# compiled program resource, returning the eval closure and cache eligibility. Single-mode:
# any op not yet implemented raises `ArgumentError` ("does not yet lower op
# ...") straight through to the caller — there is no whole-`defn` fallback lane.
defp native_compile(vars, fun, device) do
defp native_compile(vars, fun, device, hooks) do
output_expr = fun.(vars)
{worker, effective_device} = resolve_worker(device)
# `vars`' flattened leaf count is the true call arity — passed through so
# `lower/2` can densify its input list even when `output_expr` doesn't
# reference every parameter position (see EMLX.Native.Expr.lower/2 doc).
num_inputs = vars |> Nx.Defn.Composite.flatten_list() |> length()

base_key = dispatch_key(output_expr, num_inputs, effective_device)
program = EMLX.Native.Expr.lower(output_expr, num_inputs, %{}, hooks)

base_key =
if program.runtime_calls == [] and hooks == %{},
do: dispatch_key(output_expr, num_inputs, effective_device)

{resource, runtime_calls} =
get_or_compile_program(
Expand All @@ -1979,7 +1984,9 @@ defmodule EMLX do
fn -> output_expr end,
num_inputs,
worker,
effective_device
effective_device,
hooks,
program
)

output_template = Nx.Defn.Composite.traverse(output_expr, &Nx.to_template/1)
Expand Down Expand Up @@ -2030,7 +2037,7 @@ defmodule EMLX do
nil
end)

fn [params] ->
eval_fn = fn [params] ->
{worker, dev} = resolve_worker(effective_device)
tensors = materialise_input_tensors(params, dev)
quant_signature = quant_signature(tensors, quantizable_positions)
Expand All @@ -2039,7 +2046,7 @@ defmodule EMLX do
# (and cached under `{base_key, %{}}`) by our caller, so the common
# case skips a redundant cache round-trip; a quantized signature is
# looked up (or compiled once and cached) here — see
# get_or_compile_program/6. `fn -> fun.(vars) end` is only forced on
# get_or_compile_program. For callback-free programs, `fn -> fun.(vars) end` is only forced on
# that cache's own miss (once per distinct `{base_key, quant_signature}`
# while it stays in the cache) — never eagerly, so a cache *hit* here
# costs nothing beyond the lookup itself.
Expand All @@ -2053,7 +2060,8 @@ defmodule EMLX do
fn -> fun.(vars) end,
num_inputs,
worker,
dev
dev,
hooks
)
end

Expand Down Expand Up @@ -2134,6 +2142,10 @@ defmodule EMLX do

[output_container]
end

# Callbacks may capture request aliases or other short-lived state. Their
# lifetime belongs to this returned executable, never to a global cache.
{eval_fn, runtime_calls == []}
end

# Reassembles the 2-3 physical refs `EMLX.Native.Expr.lower/2` emitted for
Expand Down Expand Up @@ -2325,19 +2337,31 @@ defmodule EMLX do
output_expr_thunk,
num_inputs,
worker,
dev
dev,
hooks,
program \\ nil
) do
cache_key = {base_key, quant_signature}
table = dispatch_cache_table()
cached = if base_key, do: :ets.lookup(table, cache_key), else: []

case :ets.lookup(table, cache_key) do
case cached do
[{_key, compiled, _inserted_ms, _index}] ->
compiled

[] ->
program = EMLX.Native.Expr.lower(output_expr_thunk.(), num_inputs, quant_signature)
program =
program ||
EMLX.Native.Expr.lower(output_expr_thunk.(), num_inputs, quant_signature, hooks)

resource = compile_native_program(worker, dev, program)
EMLX.CompileCache.put_new(table, cache_key, {resource, program.runtime_calls})
compiled = {resource, program.runtime_calls}

if base_key != nil and program.runtime_calls == [] do
EMLX.CompileCache.put_new(table, cache_key, compiled)
else
compiled
end
end
end

Expand Down
9 changes: 7 additions & 2 deletions emlx/lib/emlx/compile_cache.ex
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
defmodule EMLX.CompileCache do
@moduledoc """
Bounded cache of `native_compile/3` eval closures and compiled programs.
Configurable cache of callback-free eval closures and compiled programs.

`EMLX.__compile__/4` stores one closure per `{fun, templates, hooks, device}`
so repeated `Nx.Defn.jit/2` of the same call site skips retrace. The
so repeated `Nx.Defn.jit/2` of the same call site skips retrace. Executables
with runtime callbacks (including inline/default hooks) and compilations
with nonempty `:hooks` options bypass these caches: callback closures may
capture request-specific aliases and must live only as long as the returned
executable. Keep a `Nx.Defn.compile/3` result to reuse such an executable.
The
structural program table `:emlx_native_dispatch_cache` and its key table
`:emlx_dispatch_key_by_id` are shared across call sites. Each compiled
program holds device memory, for example its captured constants. All three
Expand Down
Loading
Loading