From e6e44b0e76c06913fbacbad928a99f8b30e6d0e5 Mon Sep 17 00:00:00 2001 From: Leandro Pereira Date: Sat, 26 Sep 2026 13:40:51 -0400 Subject: [PATCH 1/3] feat: add load_data_fun to load dynamic data Closes #54 --- lib/rustler_precompiled.ex | 27 ++++++++++-- lib/rustler_precompiled/config.ex | 31 +++++++++++++- test/rustler_precompiled/config_test.exs | 52 ++++++++++++++++++++++++ test/rustler_precompiled_test.exs | 51 +++++++++++++++++++++++ 4 files changed, 156 insertions(+), 5 deletions(-) diff --git a/lib/rustler_precompiled.ex b/lib/rustler_precompiled.ex index 5e72093..464288a 100644 --- a/lib/rustler_precompiled.ex +++ b/lib/rustler_precompiled.ex @@ -95,6 +95,12 @@ defmodule RustlerPrecompiled do %{"x86_64-unknown-linux-gnu" => [old_glibc: fn _config -> has_old_glibc?() end]} + * `:load_data` - Any valid term. This value is passed into the NIF when it is loaded. + Defaults to `0`. + + * `:load_data_fun` - A `{module, function}` tuple naming a zero-arity function that + returns the load data. Cannot be combined with `:load_data`. + In case "force build" is used, all options except the ones use by RustlerPrecompiled are going to be passed down to `Rustler`. So if you need to configure the build, check the `Rustler` options. @@ -198,7 +204,6 @@ defmodule RustlerPrecompiled do {:ok, config} -> @on_load :load_rustler_precompiled @rustler_precompiled_load_from config.load_from - @rustler_precompiled_load_data config.load_data @doc false def load_rustler_precompiled do @@ -212,7 +217,22 @@ defmodule RustlerPrecompiled do |> Application.app_dir(path) |> to_charlist() - :erlang.load_nif(load_path, @rustler_precompiled_load_data) + :erlang.load_nif(load_path, __rustler_precompiled_load_data__()) + end + + if config.load_data_fun do + @rustler_precompiled_load_data_fun config.load_data_fun + + # Called on load, not at compile time, like `Rustler` does: + # https://github.com/rusterlium/rustler/blob/rustler-0.38.0/rustler_mix/lib/rustler.ex#L152-L167 + defp __rustler_precompiled_load_data__ do + {module, function} = @rustler_precompiled_load_data_fun + apply(module, function, []) + end + else + @rustler_precompiled_load_data config.load_data + + defp __rustler_precompiled_load_data__, do: @rustler_precompiled_load_data end {:error, precomp_error} -> @@ -741,7 +761,8 @@ defmodule RustlerPrecompiled do result = %{ load?: true, load_from: {name, Path.join("priv/native", lib_name)}, - load_data: config.load_data + load_data: config.load_data, + load_data_fun: config.load_data_fun } if File.exists?(cached_tar_gz) do diff --git a/lib/rustler_precompiled/config.ex b/lib/rustler_precompiled/config.ex index 57b33b7..e01d76b 100644 --- a/lib/rustler_precompiled/config.ex +++ b/lib/rustler_precompiled/config.ex @@ -10,6 +10,7 @@ defmodule RustlerPrecompiled.Config do :crate, :base_cache_dir, :load_data, + :load_data_fun, :force_build?, :targets, :nif_versions, @@ -54,6 +55,9 @@ defmodule RustlerPrecompiled.Config do |> Keyword.get(:nif_versions, @default_nif_versions) |> validate_list!(:nif_versions, @available_nif_versions) + # Defaults to 0 like Rustler + load_data = opts[:load_data] || 0 + %__MODULE__{ otp_app: otp_app, base_url: base_url, @@ -61,8 +65,8 @@ defmodule RustlerPrecompiled.Config do version: version, force_build?: pre_release?(version) or Keyword.fetch!(opts, :force_build), crate: opts[:crate], - # Default to `0` like `Rustler`. - load_data: opts[:load_data] || 0, + load_data: load_data, + load_data_fun: validate_load_data_fun!(load_data, opts[:load_data_fun]), base_cache_dir: opts[:base_cache_dir], targets: targets, nif_versions: nif_versions, @@ -111,6 +115,29 @@ defmodule RustlerPrecompiled.Config do end end + # Same validations as done by Rustler + # https://github.com/rusterlium/rustler/blob/rustler-0.38.0/rustler_mix/lib/rustler.ex#L114-L146 + defp validate_load_data_fun!(_load_data, nil), do: nil + + defp validate_load_data_fun!(0, {module, function} = load_data_fun) when is_atom(module) and is_atom(function) do + load_data_fun + else + + defp validate_load_data_fun!(0, provided_value) do + raise """ + `load_data_fun` has to be `{Module, :function}`. + Instead received: #{inspect(provided_value)} + """ + end + + defp validate_load_data_fun!(load_data, load_data_fun) do + raise """ + Only `load_data` or `load_data_fun` can be provided. Instead received: + >>> load_data: #{inspect(load_data)} + >>> load_data_fun: #{inspect(load_data_fun)} + """ + end + defp validate_list!(nil, option, _valid_values), do: raise_for_nil_field_value(option) defp validate_list!([_ | _] = values, option, valid_values) do diff --git a/test/rustler_precompiled/config_test.exs b/test/rustler_precompiled/config_test.exs index 9b557cd..bbc42aa 100644 --- a/test/rustler_precompiled/config_test.exs +++ b/test/rustler_precompiled/config_test.exs @@ -42,6 +42,58 @@ defmodule RustlerPrecompiled.ConfigTest do end end + test "new/1 accepts `load_data_fun`" do + config = + Config.new( + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_url: + "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", + version: "0.2.0-dev", + load_data_fun: {RustlerPrecompilationExample.Native, :load_data} + ) + + assert config.load_data_fun == {RustlerPrecompilationExample.Native, :load_data} + end + + test "new/1 validates `load_data_fun`" do + opts = [ + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_url: + "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", + version: "0.2.0-dev" + ] + + assert_raise RuntimeError, + """ + `load_data_fun` has to be `{Module, :function}`. + Instead received: {RustlerPrecompilationExample.Native, :load_data, []} + """, + fn -> + Config.new( + opts ++ + [load_data_fun: {RustlerPrecompilationExample.Native, :load_data, []}] + ) + end + + assert_raise RuntimeError, + """ + Only `load_data` or `load_data_fun` can be provided. Instead received: + >>> load_data: :data + >>> load_data_fun: {RustlerPrecompilationExample.Native, :load_data} + """, + fn -> + Config.new( + opts ++ + [ + load_data: :data, + load_data_fun: {RustlerPrecompilationExample.Native, :load_data} + ] + ) + end + end + test "new/1 validates the given targets" do opts = [ otp_app: :rustler_precompiled, diff --git a/test/rustler_precompiled_test.exs b/test/rustler_precompiled_test.exs index 02c7e8e..d39a936 100644 --- a/test/rustler_precompiled_test.exs +++ b/test/rustler_precompiled_test.exs @@ -775,6 +775,50 @@ defmodule RustlerPrecompiledTest do end end + @tag :tmp_dir + test "use RustlerPrecompiled calls `load_data_fun` every time the module is loaded", %{ + tmp_dir: tmp_dir + } do + nif_fixtures_dir = Path.join(File.cwd!(), "test/fixtures") + checksum_sample = File.read!(Path.join(nif_fixtures_dir, "checksum-sample-file.exs")) + Process.register(self(), :load_data_fun_test) + + in_tmp(tmp_dir, fn -> + File.write!("checksum-Elixir.RustlerPrecompilationExample.Native.exs", checksum_sample) + + {[{module, binary}], _log} = + with_log(fn -> + Code.compile_quoted( + quote do + defmodule RustlerPrecompilationExample.Native do + use RustlerPrecompiled, + otp_app: :rustler_precompiled, + crate: "example", + base_url: + "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", + base_cache_dir: unquote(nif_fixtures_dir), + version: "0.2.0", + force_build: false, + load_data_fun: {RustlerPrecompiledTest.LoadData, :load_data} + + def add(_a, _b), do: :erlang.nif_error(:nif_not_loaded) + end + end + ) + end) + + assert_receive :load_data_called + assert module.add(1, 2) == 3 + + # Loading the same binary again calls it again, + # so the data is not fixed at compile time. + :code.delete(module) + :code.purge(module) + assert {:module, ^module} = :code.load_binary(module, ~c"nofile", binary) + assert_receive :load_data_called + end) + end + describe "build_metadata/1" do test "builds a valid metadata" do config = @@ -1155,6 +1199,13 @@ defmodule RustlerPrecompiledTest do end end + defmodule LoadData do + def load_data do + send(:load_data_fun_test, :load_data_called) + :load_data + end + end + def in_tmp(tmp_path, function) do path = Path.join([tmp_path, random_string(10)]) From cc687b8740417952f442e722510f1a489e3ea286 Mon Sep 17 00:00:00 2001 From: Leandro Pereira Date: Sat, 26 Sep 2026 18:25:19 -0400 Subject: [PATCH 2/3] fixes --- lib/rustler_precompiled/config.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/rustler_precompiled/config.ex b/lib/rustler_precompiled/config.ex index e01d76b..0701d8f 100644 --- a/lib/rustler_precompiled/config.ex +++ b/lib/rustler_precompiled/config.ex @@ -119,9 +119,10 @@ defmodule RustlerPrecompiled.Config do # https://github.com/rusterlium/rustler/blob/rustler-0.38.0/rustler_mix/lib/rustler.ex#L114-L146 defp validate_load_data_fun!(_load_data, nil), do: nil - defp validate_load_data_fun!(0, {module, function} = load_data_fun) when is_atom(module) and is_atom(function) do + defp validate_load_data_fun!(0, {module, function} = load_data_fun) + when is_atom(module) and is_atom(function) do load_data_fun - else + end defp validate_load_data_fun!(0, provided_value) do raise """ From 6e7d7bc63a9e7fce2988635f38f6b597e6863e64 Mon Sep 17 00:00:00 2001 From: Leandro Pereira Date: Sun, 27 Sep 2026 15:33:25 -0400 Subject: [PATCH 3/3] rm lib_file --- lib/rustler_precompiled.ex | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/rustler_precompiled.ex b/lib/rustler_precompiled.ex index 464288a..52dedd5 100644 --- a/lib/rustler_precompiled.ex +++ b/lib/rustler_precompiled.ex @@ -752,8 +752,11 @@ defmodule RustlerPrecompiled do cached_tar_gz = Map.fetch!(metadata, :cached_tar_gz) cache_dir = Path.dirname(cached_tar_gz) - file_name = Map.fetch!(metadata, :file_name) - lib_file = Path.join(native_dir, file_name) + lib_file = + metadata + |> Map.fetch!(:file_name) + |> String.replace_suffix(".tar.gz", "") + |> then(&Path.join(native_dir, &1)) base_url = config.base_url nif_module = config.module @@ -766,11 +769,8 @@ defmodule RustlerPrecompiled do } if File.exists?(cached_tar_gz) do - # Remove existing NIF file so we don't have processes using it. - # See: https://github.com/rusterlium/rustler/blob/46494d261cbedd3c798f584459e42ab7ee6ea1f4/rustler_mix/lib/rustler/compiler.ex#L134 - File.rm(lib_file) - with :ok <- check_file_integrity(cached_tar_gz, nif_module), + :ok <- remove_existing_nif(lib_file), :ok <- :erl_tar.extract(cached_tar_gz, [:compressed, cwd: Path.dirname(lib_file)]) do Logger.debug("Copying NIF from cache and extracting to #{lib_file}") {:ok, result} @@ -785,6 +785,7 @@ defmodule RustlerPrecompiled do with_retry(fn -> download_nif_artifact(tar_gz_url) end, config.max_retries), :ok <- File.write(cached_tar_gz, tar_gz), :ok <- check_file_integrity(cached_tar_gz, nif_module), + :ok <- remove_existing_nif(lib_file), :ok <- :erl_tar.extract({:binary, tar_gz}, [:compressed, cwd: Path.dirname(lib_file)]) do Logger.debug("NIF cached at #{cached_tar_gz} and extracted to #{lib_file}") @@ -794,6 +795,15 @@ defmodule RustlerPrecompiled do end end + # Remove existing NIF file so we don't have processes using it. + # See: https://github.com/rusterlium/rustler/blob/46494d261cbedd3c798f584459e42ab7ee6ea1f4/rustler_mix/lib/rustler/compiler.ex#L134 + defp remove_existing_nif(path) do + case File.rm(path) do + {:error, :enoent} -> :ok + result -> result + end + end + defp checksum_map(nif_module) when is_atom(nif_module) do nif_module |> checksum_file()