From 211488675628e03848f83927c8933e955e33bbdb Mon Sep 17 00:00:00 2001 From: Alexy Khrabrov Date: Sun, 6 Sep 2026 04:58:09 +0000 Subject: [PATCH] build: keep the prebuilt cache out of the crate source tree; opt-in localization of bundled C symbols MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The downloaded prebuilt archive now lives under OUT_DIR (or LBUG_PREBUILT_CACHE_DIR for a persistent, shared cache) instead of .cache/lbug-prebuilt inside CARGO_MANIFEST_DIR. For a registry dependency that directory is ~/.cargo/registry/src/…, which Cargo treats as immutable; writing there broke `cargo vendor`, read-only registries and offline builds. An archive already present in the legacy location is still used. - LBUG_LOCALIZE_BUNDLED_SYMBOLS=1 (Linux, static) partially links liblbug.a into one relocatable object, makes every strong unmangled symbol outside the lbug_* C API local, and archives it back. liblbug.a exports zstd, lz4, brotli, simsimd, yyjson and CRoaring with their ordinary names (192 ZSTD_*, 194 roaring/bitset/… including bare helpers such as `get` and `union`); a binary that also links zstd-sys, lz4-sys, simsimd or croaring-sys fails with rust-lld on Linux (duplicate symbol), while ld64 on macOS silently keeps whichever copy comes first. Opt-in because an extension that resolves one of those symbols from the host binary would stop finding it. Weak (COMDAT) symbols stay global so the linker can still deduplicate them. Verified: 0 ZSTD_/roaring_/simsimd_/LZ4 globals remain, 179 lbug_* and the mangled lbug:: API do, and the examples crate links and runs. - Crate docs: the two new variables, a bulk-loading section (COPY FROM and the Arrow table registration; one MERGE per row measured at tens of milliseconds each), and a note on the buffer pool's host-sized default. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013H2j2wjVBdmiK6v5sWM4u1 --- build.rs | 166 ++++++++++++++++++++++++++++++++++++++++++-- examples/Cargo.lock | 44 +++++++++++- src/lib.rs | 30 ++++++++ 3 files changed, 232 insertions(+), 8 deletions(-) diff --git a/build.rs b/build.rs index d9f8292..658bf89 100644 --- a/build.rs +++ b/build.rs @@ -49,7 +49,10 @@ fn link_openssl() { #[cfg(target_os = "macos")] { - for prefix in ["/opt/homebrew/opt/openssl/lib", "/usr/local/opt/openssl/lib"] { + for prefix in [ + "/opt/homebrew/opt/openssl/lib", + "/usr/local/opt/openssl/lib", + ] { let path = PathBuf::from(prefix); if path.is_dir() { println!("cargo:rustc-link-search=native={}", path.display()); @@ -173,11 +176,28 @@ fn prebuilt_cache_key() -> String { .collect() } +/// Where the downloaded prebuilt archive lives. +/// +/// `LBUG_PREBUILT_CACHE_DIR` names a persistent cache shared across builds and +/// projects. Otherwise the archive goes under this build's `OUT_DIR`, which +/// Cargo owns: a registry checkout under `~/.cargo/registry/src` is meant to be +/// immutable, and writing the archive there broke `cargo vendor`, read-only +/// registries and offline builds. An archive already present in the legacy +/// in-tree cache is still used, so existing checkouts do not download again. fn prebuilt_lib_dir(manifest_dir: &Path) -> PathBuf { - manifest_dir - .join(PREBUILT_CACHE_DIR) - .join(prebuilt_cache_key()) - .join("lib") + let key = prebuilt_cache_key(); + let legacy = manifest_dir.join(PREBUILT_CACHE_DIR).join(&key).join("lib"); + if legacy.join(static_lbug_file_name()).exists() { + return legacy; + } + let root = env::var_os("LBUG_PREBUILT_CACHE_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| out_dir().join("lbug-prebuilt")); + root.join(key).join("lib") +} + +fn out_dir() -> PathBuf { + PathBuf::from(env::var_os("OUT_DIR").expect("cargo sets OUT_DIR for build scripts")) } fn prebuilt_source_desc() -> String { @@ -210,6 +230,8 @@ fn try_download_prebuilt_lbug(manifest_dir: &Path) -> bool { "LBUG_LIB_KIND", "LBUG_BUILD_FROM_SOURCE", "LBUG_RUST_BUILD_FROM_SOURCE", + "LBUG_PREBUILT_CACHE_DIR", + "LBUG_LOCALIZE_BUNDLED_SYMBOLS", ] { println!("cargo:rerun-if-env-changed={var}"); } @@ -277,13 +299,143 @@ fn use_prebuilt_lbug(manifest_dir: &Path) -> Option> { return None; } + // The downloaded directory holds the headers as well as the archive; a + // localized archive lives elsewhere but the headers stay where they are. let lib_dir = prebuilt_lib_dir(manifest_dir); - println!("cargo:rustc-link-search=native={}", lib_dir.display()); + let link_dir = localize_bundled_symbols(&lib_dir).unwrap_or_else(|| lib_dir.clone()); + println!("cargo:rustc-link-search=native={}", link_dir.display()); println!("cargo:rerun-if-changed={}", lib_dir.display()); - emit_lbug_metadata(&prebuilt_source_desc(), &lib_dir); + emit_lbug_metadata(&prebuilt_source_desc(), &link_dir); Some(vec![lib_dir]) } +/// Hide the third-party C symbols the prebuilt archive carries, on request. +/// +/// `liblbug.a` bundles zstd, lz4, brotli, simsimd, yyjson and CRoaring with +/// their ordinary exported names (`ZSTD_compressBound`, `roaring_bitmap_add`, +/// even bare helpers such as `get` and `union`). A binary that also links the +/// Rust crates for those libraries (`zstd-sys`, `lz4-sys`, `simsimd`, +/// `croaring-sys`, …) fails to link on Linux with `rust-lld`: duplicate +/// symbol, twenty times over, whereas macOS `ld64` silently keeps whichever +/// copy comes first. +/// +/// With `LBUG_LOCALIZE_BUNDLED_SYMBOLS=1` on an ELF target the archive is +/// partially linked into one relocatable object, every unmangled global symbol +/// except the `lbug_*` C API is made local, and the result is archived back +/// under `OUT_DIR`. Internal references resolve inside that object, the C++ API +/// (mangled `lbug::` symbols) and the C API stay global, and the bundled +/// libraries no longer collide with anyone else's copy. It is opt-in because a +/// dynamically loaded extension that expects to resolve one of those bundled +/// symbols from the host binary would stop finding it; the maintainers can +/// flip the default once extensions are checked. Needs `ld`, `objcopy`, `nm` +/// and `ar` (GNU binutils); without them the archive is used as downloaded. +fn localize_bundled_symbols(lib_dir: &Path) -> Option { + let requested = env::var("LBUG_LOCALIZE_BUNDLED_SYMBOLS") + .map(|value| value != "0" && !value.is_empty()) + .unwrap_or(false); + if !requested || !cfg!(target_os = "linux") || link_mode() != "static" { + return None; + } + let archive = lib_dir.join(static_lbug_file_name()); + let work = out_dir().join("lbug-localized"); + let out_lib_dir = work.join("lib"); + let localized = out_lib_dir.join(static_lbug_file_name()); + if localized.exists() { + return Some(out_lib_dir); + } + if let Err(error) = std::fs::create_dir_all(&out_lib_dir) { + println!( + "cargo:warning=Cannot create {}: {error}; linking liblbug as downloaded", + work.display() + ); + return None; + } + let merged = work.join("lbug-merged.o"); + let keep = work.join("keep-global.txt"); + let steps: [(&str, Vec); 3] = [ + ( + "ld", + vec![ + "-r".into(), + "--whole-archive".into(), + archive.clone().into(), + "--no-whole-archive".into(), + "-o".into(), + merged.clone().into(), + ], + ), + ( + "nm", + vec![ + "--defined-only".into(), + "-g".into(), + "--format=posix".into(), + merged.clone().into(), + ], + ), + ( + "ar", + vec![ + "rcs".into(), + localized.clone().into(), + merged.clone().into(), + ], + ), + ]; + for (tool, args) in &steps { + let output = match Command::new(tool).args(args).output() { + Ok(output) => output, + Err(error) => { + println!( + "cargo:warning=Cannot run {tool} ({error}); linking liblbug as downloaded" + ); + return None; + } + }; + if !output.status.success() { + println!("cargo:warning={tool} failed while localizing liblbug symbols; linking it as downloaded"); + return None; + } + if *tool == "nm" { + // Localize only strong, unmangled symbols outside the C API: the + // bundled libraries' functions and data. Mangled C++ symbols are + // the API this crate binds, and weak symbols (COMDAT groups such + // as `DW.ref.__gxx_personality_v0`, template instantiations) must + // stay global so the linker can still deduplicate them. + let listing = String::from_utf8_lossy(&output.stdout); + let keep_names: Vec<&str> = listing + .lines() + .filter_map(|line| { + let mut fields = line.split(' '); + Some((fields.next()?, fields.next()?)) + }) + .filter(|(name, kind)| { + let strong = matches!(*kind, "T" | "D" | "B" | "R"); + !strong || name.starts_with("_Z") || name.starts_with("lbug_") + }) + .map(|(name, _)| name) + .collect(); + if std::fs::write(&keep, keep_names.join("\n")).is_err() { + return None; + } + // `--keep-global-symbols` makes every defined symbol local except + // the listed ones; undefined references are left alone. + let status = Command::new("objcopy") + .arg(format!("--keep-global-symbols={}", keep.display())) + .arg(&merged) + .status(); + match status { + Ok(status) if status.success() => {} + _ => { + println!("cargo:warning=objcopy failed while localizing liblbug symbols; linking it as downloaded"); + return None; + } + } + } + } + Some(out_lib_dir) +} + fn get_lbug_root() -> PathBuf { let manifest_dir = manifest_dir(); if let Ok(lbug_source_dir) = std::env::var("LBUG_SOURCE_DIR") { diff --git a/examples/Cargo.lock b/examples/Cargo.lock index f27121c..2e092cf 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -490,6 +490,12 @@ dependencies = [ "cc", ] +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "js-sys" version = "0.3.78" @@ -502,7 +508,7 @@ dependencies = [ [[package]] name = "lbug" -version = "0.16.0" +version = "0.18.0" dependencies = [ "arrow", "cmake", @@ -510,8 +516,10 @@ dependencies = [ "cxx-build", "rust_decimal", "rustversion", + "serde_json", "time", "uuid", + "vcpkg", ] [[package]] @@ -792,6 +800,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -812,6 +829,19 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "shlex" version = "1.3.0" @@ -900,6 +930,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -1088,3 +1124,9 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/src/lib.rs b/src/lib.rs index 1fdb1d0..00f3b38 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -53,6 +53,36 @@ //! export LBUG_SHARED=1 //! ``` //! +//! - `LBUG_PREBUILT_CACHE_DIR`: Directory to keep downloaded prebuilt archives in, shared across +//! builds. By default they live under the build's `OUT_DIR`. +//! - `LBUG_LOCALIZE_BUNDLED_SYMBOLS`: On Linux static builds, set to `1` to hide the C symbols of +//! the libraries bundled in the prebuilt archive (zstd, lz4, brotli, simsimd, yyjson, CRoaring) +//! so they cannot collide with the Rust crates for the same libraries (`zstd-sys`, `lz4-sys`, +//! `simsimd`, `croaring-sys`, …) at link time. Needs GNU binutils. Leave it unset when the binary +//! loads extensions that resolve those symbols from the host. +//! +//! ## Bulk loading +//! +//! One `CREATE` or `MERGE` per row costs a full statement each: planning, execution and, outside +//! an explicit transaction, a commit. Measured through this crate on a 4-vCPU host, a single-row +//! relationship `MERGE … SET` costs tens of milliseconds of CPU, so loading a few hundred thousand +//! rows that way takes hours. Load in bulk instead: +//! +//! - From files: `COPY Person FROM 'people.csv'` / `COPY Knows FROM 'knows.parquet'`. +//! - From memory, with the `arrow` feature: register Arrow record batches as tables with +//! [`Connection::create_arrow_table`] and [`Connection::create_arrow_rel_table`], then +//! `COPY … FROM` them, or query them directly. +//! - Point lookups by primary key cost about a millisecond of planning each through +//! [`Connection::prepare`] + [`Connection::execute`]; fetch a neighbourhood with one +//! `MATCH (a)-[r]->(b) WHERE a.id = $id RETURN …` rather than one lookup per neighbour. +//! +//! ## Memory +//! +//! [`SystemConfig::default`] leaves `buffer_pool_size` at 0, which the engine resolves from the +//! host's physical memory, so an embedding process reserves a large share of RAM regardless of +//! the database's size (about 6 GB resident on a 15 GiB host for a 200,000-edge graph). Set +//! [`SystemConfig::buffer_pool_size`] explicitly when the process shares the machine. +//! //! ## Using Extensions //! By default, binaries created using this library will not work with Lbug's //! [extensions](https://docs.ladybugdb.com/extensions/) (except on Windows/MSVC, where the linker works differently).