From a7075eaaf2084a771c4abe4bb5845a5afae1ce44 Mon Sep 17 00:00:00 2001 From: Beinan Date: Mon, 19 Oct 2026 02:13:13 +0000 Subject: [PATCH] chore: upgrade Lance dependencies to 13.0.0 --- Cargo.lock | 821 +++++++++++------- crates/lance-context-core/Cargo.toml | 13 +- .../examples/generic_batch_get.rs | 1 + .../lance-context-core/src/datagen_store.rs | 2 +- .../lance-context-core/src/generic_store.rs | 2 +- .../src/merge_write_scope.rs | 50 +- .../lance-context-core/src/preparation_io.rs | 16 + .../lance-context-core/src/rollout_append.rs | 64 +- .../lance-context-core/src/rollout_store.rs | 121 ++- crates/lance-context-core/src/store_base.rs | 66 +- .../tests/storage_reliability.rs | 2 + crates/lance-context-master/Cargo.toml | 2 +- crates/lance-context-master/src/lib.rs | 1 + crates/lance-context-master/src/main.rs | 2 + crates/lance-context-server/Cargo.toml | 2 +- crates/lance-context-server/src/main.rs | 2 + docs/design/lance-13-upgrade.md | 33 + python/pyproject.toml | 8 +- python/uv.lock | 36 +- 19 files changed, 808 insertions(+), 436 deletions(-) create mode 100644 docs/design/lance-13-upgrade.md diff --git a/Cargo.lock b/Cargo.lock index 58efe35b..4686434a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -222,7 +222,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -435,6 +435,15 @@ dependencies = [ "loom", ] +[[package]] +name = "asyncband" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5161fb43cb8f231f06983cb2b8f9ee9cee14be0b1356d8383398cc0f312800d1" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "atoi" version = "2.0.0" @@ -911,6 +920,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1405,12 +1420,13 @@ dependencies = [ ] [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc-fast" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "rustc_version", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] @@ -1521,16 +1537,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ctor" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" -dependencies = [ - "ctor-proc-macro", - "dtor", -] - [[package]] name = "ctor" version = "1.0.12" @@ -1541,12 +1547,6 @@ dependencies = [ "linktime-proc-macro", ] -[[package]] -name = "ctor-proc-macro" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" - [[package]] name = "ctutils" version = "0.4.2" @@ -1642,7 +1642,7 @@ dependencies = [ "indexmap 2.14.0", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "sqlparser", "tempfile", @@ -1671,7 +1671,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "tokio", ] @@ -1696,7 +1696,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", ] [[package]] @@ -1716,7 +1716,7 @@ dependencies = [ "itertools 0.14.0", "libc", "log", - "object_store", + "object_store 0.13.2", "sqlparser", "tokio", "uuid", @@ -1757,7 +1757,7 @@ dependencies = [ "glob", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "rand 0.9.4", "tokio", @@ -1784,7 +1784,7 @@ dependencies = [ "datafusion-session", "futures", "itertools 0.14.0", - "object_store", + "object_store 0.13.2", "tokio", ] @@ -1806,7 +1806,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store", + "object_store 0.13.2", "regex", "tokio", ] @@ -1829,7 +1829,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store", + "object_store 0.13.2", "tokio", "tokio-stream", ] @@ -1855,7 +1855,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "rand 0.9.4", "tempfile", @@ -1904,7 +1904,7 @@ checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.22.1", "blake2", "blake3", "chrono", @@ -1919,7 +1919,7 @@ dependencies = [ "hex", "itertools 0.14.0", "log", - "md-5 0.11.0", + "md-5", "memchr", "num-traits", "rand 0.9.4", @@ -2211,6 +2211,12 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "datasketches" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + [[package]] name = "der" version = "0.7.10" @@ -2277,6 +2283,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -2297,21 +2313,6 @@ dependencies = [ "const-random", ] -[[package]] -name = "dtor" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" -dependencies = [ - "dtor-proc-macro", -] - -[[package]] -name = "dtor-proc-macro" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" - [[package]] name = "dunce" version = "1.0.5" @@ -2502,6 +2503,71 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer-common" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" +dependencies = [ + "anyhow", + "bytes", + "cfg-if 1.0.4", + "foyer-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "twox-hash", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" +dependencies = [ + "anyhow", + "asyncband", + "bitflags", + "datasketches", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "foyer-tokio", + "futures-util", + "hashbrown 0.17.1", + "itertools 0.15.0", + "mixtrics", + "parking_lot", + "paste", + "pin-project", + "serde", + "tracing", +] + +[[package]] +name = "foyer-tokio" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" +dependencies = [ + "tokio", +] + +[[package]] +name = "frostem" +version = "1.20260821.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2488a10218a17366ce6bd38b6d081b4387a9548d350029db9feddc9bb44d6d" + [[package]] name = "fs_extra" version = "1.3.0" @@ -2510,9 +2576,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f1155128aba964cf6925c22a667ed763b52c8c653693c4b46f8a79d509a8c1" +checksum = "de52c5e8a41983a44223740e4322122b5a9111909215ebdb4abd9b3b4cf04a67" dependencies = [ "arrow-array", "rand 0.9.4", @@ -2861,18 +2927,25 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.5" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +checksum = "1e471f7bce40936cf2023855c7350e31e11b3f703eb7cb2ce73c9ffc451f87d2" dependencies = [ + "arc-swap", "async-trait", "bytes", "dashmap", + "fastrand", + "foyer-common", + "foyer-memory", + "futures", "hostname", + "io-uring", + "itoa", + "libc", + "lru 0.16.4", "prost", - "prost-types", "rand 0.9.4", - "reqwest 0.12.28", "serde", "thiserror 2.0.18", "tokio", @@ -2881,6 +2954,7 @@ dependencies = [ "tonic-prost", "tracing", "uuid", + "xxhash-rust", ] [[package]] @@ -3005,18 +3079,21 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd" +checksum = "f24754aef4f6c3d57bc4a0dc2648ca74487050eda01c361b3eefa11feeafd662" dependencies = [ + "anyhow", "async-trait", "bytes", "http 1.4.1", "more-asserts", "serde", + "serde_json", "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "uuid", "xet-client", @@ -3200,7 +3277,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -3547,6 +3624,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -3706,9 +3792,9 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d04bed056e254bc6e31264b031c8492507ca57939586f016924081dcf221a9" +checksum = "0e8dc0a0759fa0fcdbb970eec48ade5d02dbbd255147ff1f7b06d402d2f86b97" dependencies = [ "arc-swap", "arrow", @@ -3724,7 +3810,7 @@ dependencies = [ "async-recursion", "async-trait", "async_cell", - "aws-credential-types", + "blake3", "byteorder", "bytes", "chrono", @@ -3739,7 +3825,6 @@ dependencies = [ "either", "fst", "futures", - "half", "humantime", "itertools 0.14.0", "lance-arrow", @@ -3757,7 +3842,7 @@ dependencies = [ "lance-tokenizer", "log", "moka", - "object_store", + "object_store 0.14.2", "permutation", "pin-project", "prost", @@ -3781,9 +3866,9 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6e7b87c8183988c40a6bd30a6d8ec588b84e53f702b82f19859dc71ba6c02bc" +checksum = "bab6e64549f9ad9a6380a35b56d7dca612555a6272c07b3abca84e59ded6f463" dependencies = [ "arrow-array", "arrow-buffer", @@ -3831,9 +3916,9 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a4d84a3f36133c70bf89d306d813a29f8eb8555bba2b84995260867cfafdd5e" +checksum = "38984429670b348e62eda38315a6d01a930c5c5e8b85bbfd8b998e7d1fe6a257" dependencies = [ "arrayref", "crunchy", @@ -3856,7 +3941,7 @@ name = "lance-context-api" version = "0.6.7" dependencies = [ "arrow-schema", - "base64", + "base64 0.22.1", "chrono", "serde", "serde_json", @@ -3884,7 +3969,7 @@ dependencies = [ "arrow-schema", "arrow-select", "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "clap", @@ -3893,13 +3978,14 @@ dependencies = [ "futures", "lance", "lance-context-api", + "lance-file", "lance-index", "lance-io", "lance-namespace", "lance-table", "metrics", "metrics-util", - "object_store", + "object_store 0.14.2", "serde", "serde_json", "tempfile", @@ -3925,7 +4011,7 @@ dependencies = [ "lance-context-core", "lance-context-merge", "lance-context-metrics", - "lru", + "lru 0.12.5", "metrics", "reqwest 0.12.28", "serde", @@ -3997,7 +4083,7 @@ dependencies = [ "lance-context-core", "lance-context-merge", "lance-context-metrics", - "lru", + "lru 0.12.5", "metrics", "metrics-util", "serde", @@ -4013,21 +4099,20 @@ dependencies = [ [[package]] name = "lance-core" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238c8a58308e7718d6bd96b53494eb7953fa299778bc4911cc571c3576e9446d" +checksum = "d3e377ee5bc5118efe5c321ecff0a07ac8f56db6bed16e31304e7d876ddcd96c" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "async-trait", - "byteorder", + "blake3", "bytes", "datafusion-common", "datafusion-sql", "futures", - "itertools 0.14.0", "lance-arrow", "lance-derive", "libc", @@ -4035,16 +4120,16 @@ dependencies = [ "log", "moka", "num_cpus", - "object_store", + "object_store 0.14.2", "pin-project", "prost", + "quick_cache", "rand 0.9.4", "roaring", "serde_json", "snafu", "tempfile", "tokio", - "tokio-stream", "tokio-util", "tracing", "twox-hash", @@ -4053,9 +4138,9 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f0ac4e1cf2f9b1b2fb5ee11bcd04af617bdd6f6cca13726a41c4212220193e" +checksum = "7be43992965983ed8238a969b36239093102fdfe6c3dfcaa0007eeb17195ca34" dependencies = [ "arrow", "arrow-array", @@ -4071,10 +4156,10 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "futures", + "half", "jsonb", "lance-arrow", "lance-core", - "lance-datagen", "lance-geo", "log", "pin-project", @@ -4084,30 +4169,11 @@ dependencies = [ "tracing", ] -[[package]] -name = "lance-datagen" -version = "9.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd65e7ea88ab28e5d3e91b58a56c02bfd44c47474caae5f8aed1322df1611476" -dependencies = [ - "arrow", - "arrow-array", - "arrow-cast", - "arrow-schema", - "chrono", - "futures", - "half", - "hex", - "rand 0.9.4", - "rand_distr", - "rand_xoshiro", -] - [[package]] name = "lance-derive" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf46656b359786f0b73f13936193583972b7af6e53ccb542da87830be18e94b2" +checksum = "e847c0e4bf23d21967fe79473d8001309c7351895afb347e1a997d5fadca59fd" dependencies = [ "proc-macro2", "quote", @@ -4116,9 +4182,9 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4725f14fe6cc1b2a5644786b4afa828d0c243064e208fa17378f839243d049c0" +checksum = "3c04a91e545b505b5e6ec98854c7eadc56689cbc072f138dd26805ef0095e81a" dependencies = [ "arrow-arith", "arrow-array", @@ -4143,8 +4209,6 @@ dependencies = [ "num-traits", "prost", "prost-build", - "rand 0.9.4", - "strum", "tokio", "tracing", "xxhash-rust", @@ -4153,13 +4217,14 @@ dependencies = [ [[package]] name = "lance-file" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c161b00eb5813f98f1d6d52e06f1b712f9eebb0a7f535a8dc1e1122ceca7307" +checksum = "f0393605ef4f2bdd9894950de6c68fd2ca99a1de76d743e52e001177ccd9994f" dependencies = [ "arrow-arith", "arrow-array", "arrow-buffer", + "arrow-cast", "arrow-data", "arrow-schema", "arrow-select", @@ -4175,19 +4240,20 @@ dependencies = [ "lance-io", "log", "num-traits", - "object_store", + "object_store 0.14.2", "prost", "prost-build", "prost-types", + "serde", "tokio", "tracing", ] [[package]] name = "lance-geo" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97739c10f2c818ac4aa8a798a115e11c790c174d016c0836792aa3b8d0f981ab" +checksum = "d18de60ec696890176fd63f696b8c453d0ee3ff820f63fc21f8fc018bd283492" dependencies = [ "datafusion", "geo-traits", @@ -4201,14 +4267,15 @@ dependencies = [ [[package]] name = "lance-index" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7980b0a6287fd46b9308a31e2cf284065d5693bfb43336297f0400172670ad" +checksum = "21b5e0ccd1b4bffdccbb3b5c989e62eb545587d84b0785da50eda5a4d87d0a5d" dependencies = [ "arc-swap", "arrow", "arrow-arith", "arrow-array", + "arrow-ipc", "arrow-ord", "arrow-schema", "arrow-select", @@ -4217,7 +4284,6 @@ dependencies = [ "async-trait", "bitvec", "bytes", - "chrono", "crossbeam-queue", "datafusion", "datafusion-common", @@ -4230,6 +4296,8 @@ dependencies = [ "geoarrow-array", "geoarrow-schema", "half", + "hashbrown 0.16.1", + "hex", "itertools 0.14.0", "jsonb", "lance-arrow", @@ -4237,7 +4305,6 @@ dependencies = [ "lance-bitpacking", "lance-core", "lance-datafusion", - "lance-datagen", "lance-encoding", "lance-file", "lance-geo", @@ -4251,7 +4318,7 @@ dependencies = [ "log", "ndarray", "num-traits", - "object_store", + "object_store 0.14.2", "prost", "prost-build", "prost-types", @@ -4267,14 +4334,14 @@ dependencies = [ "tempfile", "tokio", "tracing", - "uuid", + "twox-hash", ] [[package]] name = "lance-index-core" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a95f46ac3e4cdd710ca6929226cfb0dd343d5d2370ecb4b40e1c452043f7d27a" +checksum = "f90986278ab82f1f1b1fe43eed060f547e0da0ad98a1038785f055e407fca3db" dependencies = [ "arrow-array", "arrow-schema", @@ -4292,23 +4359,18 @@ dependencies = [ "roaring", "serde", "serde_json", + "tokio", ] [[package]] name = "lance-io" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6deecd351ed6849184cc83000eabb87c8a5bfc519996f92451284498af7b42c" +checksum = "224730479e781d24b81a0afecfe230b5b21a86a4f767ba7655b974402d0c67c5" dependencies = [ "arrow", - "arrow-arith", "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", "arrow-schema", - "arrow-select", - "async-recursion", "async-trait", "aws-config", "aws-credential-types", @@ -4319,62 +4381,67 @@ dependencies = [ "goosefs-sdk", "http 1.4.1", "io-uring", - "lance-arrow", "lance-core", "lance-namespace", "log", "moka", - "object_store", + "object_store 0.14.2", "object_store_opendal", "opendal", + "opendal-service-goosefs", "path_abs", "pin-project", "prost", "rand 0.9.4", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", "serde", + "serde_json", "tempfile", "tokio", "tracing", "url", + "uuid", ] [[package]] name = "lance-linalg" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e5a9b99bd1f49bc2fe5afb81323141abc506c818f589de95dbab4f6143d9a88" +checksum = "84e97d99cab96a320119f2be2c1ccbc4b69e5190c152b9bac96ad902b4409efe" dependencies = [ "arrow-array", - "arrow-buffer", "arrow-schema", "cc", "half", "lance-arrow", "lance-core", "num-traits", - "rand 0.9.4", "rayon", ] [[package]] name = "lance-namespace" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8624fd33a894b5f2eb411cb56588d29138137ce100dee8e335843828e249b860" +checksum = "eedb0930505992009d9a1b9aefc2e9fde6026fcc1b345211e2a609dace5be022" dependencies = [ "arrow", "async-trait", "bytes", "lance-core", "lance-namespace-reqwest-client", + "serde", + "serde_json", "snafu", ] [[package]] name = "lance-namespace-reqwest-client" -version = "0.8.6" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d" +checksum = "828f291764feda98898452bea4ce3756b73e5d5c7c67b62edda27456b7cc72c1" dependencies = [ "reqwest 0.12.28", "serde", @@ -4386,15 +4453,14 @@ dependencies = [ [[package]] name = "lance-select" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb5c718c141ea4f067203b11a54ccf57f8effbf963f345bc811f4837a660ad57" +checksum = "6cd00ec21363c9449f10409bf5b56e914b807dd9110745f0d616f65c16188ddb" dependencies = [ "arrow-array", "arrow-buffer", "arrow-schema", "byteorder", - "bytes", "itertools 0.14.0", "lance-core", "roaring", @@ -4403,9 +4469,9 @@ dependencies = [ [[package]] name = "lance-table" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc125611b4fe46f99f00b5262e71e17bd947a3bc3b81f3a4a604cc9ca290b3f" +checksum = "0b770bac5033a6b3f11b3e3ffb05f8bce2cfc004ecb7992354af7627fd278747" dependencies = [ "arrow", "arrow-array", @@ -4413,6 +4479,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "async-trait", + "blake3", "byteorder", "bytes", "chrono", @@ -4423,7 +4490,7 @@ dependencies = [ "lance-io", "lance-select", "log", - "object_store", + "object_store 0.14.2", "prost", "prost-build", "prost-types", @@ -4442,12 +4509,12 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "9.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73060a844ceda2405759b8f0f16a1c586b5b285dbdf7a3784350d048f991cfd3" +checksum = "3fec4342e22fa493efec9cc96584efe4cecbdc1ae218d6a40f47b16bd4f1ea13" dependencies = [ + "frostem", "icu_segmenter", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -4459,7 +4526,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.8", ] [[package]] @@ -4611,9 +4678,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.30" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loom" @@ -4637,6 +4704,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -4705,16 +4781,6 @@ dependencies = [ "thread-tree", ] -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if 1.0.4", - "digest 0.10.7", -] - [[package]] name = "md-5" version = "0.11.0" @@ -4725,15 +4791,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "mea" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2640d335e7273dacdcf51044026139b2e269c3bb0dfc3f8cb3496b85e3f6a42c" -dependencies = [ - "slab", -] - [[package]] name = "memchr" version = "2.8.1" @@ -4765,7 +4822,7 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" dependencies = [ - "base64", + "base64 0.22.1", "indexmap 2.14.0", "metrics", "metrics-util", @@ -4850,6 +4907,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mixtrics" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd94ebec15e61a36cd63b1e8816c83434e42926962cd21ee6e4db898d4d4f05" +dependencies = [ + "itertools 0.15.0", + "parking_lot", +] + [[package]] name = "moka" version = "0.12.15" @@ -4889,7 +4956,7 @@ dependencies = [ "httparse", "memchr", "mime", - "spin", + "spin 0.9.8", "version_check", ] @@ -4914,6 +4981,18 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if 1.0.4", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -5054,6 +5133,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "objc2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08849bbd4767dfae9457696856ae1c84fe4e0281bbe4a7abff2d0e06fb7981f8" +dependencies = [ + "objc2-encode", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -5061,6 +5149,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags", + "objc2", ] [[package]] @@ -5073,6 +5179,17 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-open-directory" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb82bed227edf5201dfedf072bba4015a33d3d4a98519837295a90f0a23f676d" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + [[package]] name = "objc2-system-configuration" version = "0.3.2" @@ -5089,9 +5206,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", "bytes", "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.1", + "humantime", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "object_store" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +dependencies = [ + "async-trait", + "aws-lc-rs", + "base64 0.23.1", + "bytes", + "chrono", + "crc-fast", "form_urlencoded", "futures-channel", "futures-core", @@ -5101,14 +5246,14 @@ dependencies = [ "httparse", "humantime", "hyper", - "itertools 0.14.0", - "md-5 0.10.6", + "itertools 0.15.0", + "md-5", + "nix", "parking_lot", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "rand 0.10.1", - "reqwest 0.12.28", - "ring", + "reqwest 0.13.4", "rustls-pki-types", "serde", "serde_json", @@ -5120,20 +5265,21 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", ] [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.60.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "0bd9da6291f93eb19b7987dac17617de0131760a6ba97a4cd8d6b1a07fa89783" dependencies = [ "async-trait", + "asyncband", "bytes", "chrono", "futures", - "mea", - "object_store", + "object_store 0.14.2", "opendal", "pin-project", "tokio", @@ -5159,12 +5305,13 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] name = "opendal" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "46081824c74bac9f68f0bc4408077f77b0e315c006514719c97af1c2227ac2d2" dependencies = [ - "ctor 1.0.12", + "ctor", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -5182,24 +5329,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "33d5a635dfea36a7a0c3bbb2ca34c97af17f98dd69b0cdf31a1d436a346e0a12" dependencies = [ "anyhow", - "base64", + "asyncband", + "base64 0.23.1", "bytes", "futures", "http 1.4.1", - "http-body 1.0.1", "jiff", "log", - "md-5 0.11.0", - "mea", + "md-5", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.42.0", "reqsign-core", - "reqwest 0.13.4", "serde", "serde_json", "tokio", @@ -5208,23 +5353,37 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc025e163c43993e30cf1829a30b791934e5b0f7cd4d7e03acc902a9efb7e8f5" +dependencies = [ + "bytes", + "futures", + "http 1.4.1", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + [[package]] name = "opendal-layer-concurrent-limit" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "06b44bdf694cccb7d90515b3eac6623975a2813abf4f5c438b4949a329d6a45b" dependencies = [ + "asyncband", "futures", "http 1.4.1", - "mea", "opendal-core", ] [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "520d3de5d246c82a246c431ef0f52b8d2f091c7c49a3fd7a6f0ff519b1e7725f" dependencies = [ "log", "opendal-core", @@ -5232,9 +5391,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "2a02c59479b047878f4a0f269f23c847f24d67d1ff19f2b20fdd5a3bceefd451" dependencies = [ "backon", "log", @@ -5243,9 +5402,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "afffa0cf8a6690b3b44cc510816ba675638eb236699a7d449e44ac3344bafd36" dependencies = [ "opendal-core", "tokio", @@ -5253,17 +5412,17 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "b674507cd2f764f6b40d5f88d284bb293c519d76e8aa0103987f970da412eeac" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "http 1.4.1", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.42.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -5274,17 +5433,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "afeb287cf2b03ab8d99d77a8e9687e908eb7898e2da1af6ccd00590c7095cae6" dependencies = [ - "base64", + "asyncband", + "base64 0.23.1", "bytes", "http 1.4.1", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.42.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -5294,9 +5454,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "bdf9c9d0bfd1fa3638eec485c7f3448252cbb5a81774a2aa7abadc3e4f1ea43e" dependencies = [ "http 1.4.1", "opendal-core", @@ -5304,15 +5464,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "423127c9e81b1ef4253883743ea6696c06dfad0fa6dcb1247887039919e53772" dependencies = [ "bytes", "http 1.4.1", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.42.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -5321,9 +5481,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "f9e5197fce0cd5929db9129a7e71e503841aa20bea4f1dcc54ab097a5f3652e9" dependencies = [ "async-trait", "bytes", @@ -5331,57 +5491,58 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.42.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", "serde", "serde_json", "tokio", + "uuid", ] [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "6ca356f23424fa0129367c3eca426eb3ec0c3cfaf628bb2972fa597f8bd10805" dependencies = [ + "asyncband", "bytes", "goosefs-sdk", "log", "opendal-core", "serde", - "tokio", ] [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "32c1a4076d933180b66b57a0f305e48f2de1a72b50648114cb9943a8bc9e28b8" dependencies = [ + "asyncband", "bytes", "hf-xet", "http 1.4.1", "log", "opendal-core", "percent-encoding", - "reqwest 0.13.4", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "30e973fcebfa413432c024c41d65fad8034b22c065250c587e0c3a70f1a27eb6" dependencies = [ "bytes", "http 1.4.1", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.42.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -5390,18 +5551,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "0932321e27c712a30ef72db56b12f31e41e9f0d2d3841d63a91bc8075cd8f0ef" dependencies = [ - "base64", + "base64 0.23.1", "bytes", - "crc32c", + "crc-fast", "http 1.4.1", "log", - "md-5 0.11.0", + "md-5", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.42.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -5411,14 +5572,14 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.57.0" +version = "0.59.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +checksum = "96652f317be16f5031487efbf027eb2bf11884d27184f4779085d985820c41d7" dependencies = [ "bytes", "http 1.4.1", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.42.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", @@ -5587,11 +5748,11 @@ dependencies = [ [[package]] name = "pem" -version = "3.0.6" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" dependencies = [ - "base64", + "base64 0.23.1", "serde_core", ] @@ -5964,9 +6125,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.4" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -5974,14 +6135,26 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.40.1" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quick_cache" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6295,9 +6468,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "reqsign-aliyun-oss" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d" +checksum = "28cf3d65d37ac910073d921c1cc58c3e4e6c963795c6062eee196e6d19054dde" dependencies = [ "anyhow", "form_urlencoded", @@ -6311,19 +6484,18 @@ dependencies = [ ] [[package]] -name = "reqsign-aws-v4" -version = "3.0.1" +name = "reqsign-aws-core" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75624bd8a466e37ddc0a7b6c33ac859a85347c153a916e1dd9d0b68338f74a" +checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" dependencies = [ - "anyhow", "bytes", "form_urlencoded", "hex", "http 1.4.1", "log", "percent-encoding", - "quick-xml 0.40.1", + "quick-xml 0.42.0", "reqsign-core", "rust-ini", "serde", @@ -6332,14 +6504,30 @@ dependencies = [ "sha1 0.11.0", ] +[[package]] +name = "reqsign-aws-v4" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" +dependencies = [ + "bytes", + "http 1.4.1", + "log", + "quick-xml 0.42.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + [[package]] name = "reqsign-azure-storage" -version = "3.0.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b96928e73ad984de1d99e382749d09e5dab7dd707b767974f7e40aa926b82f" +checksum = "7260e6c1d89422ba62d624a32c491b561386fb364ac4bf0959c740e34b251a2e" dependencies = [ "anyhow", - "base64", + "asyncband", + "base64 0.23.1", "bytes", "form_urlencoded", "http 1.4.1", @@ -6355,15 +6543,14 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.0.1" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fa5cb48808693614d1701fcd3db0b30fa292e0f18e122ae068b6d32eaeed3f" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" dependencies = [ "anyhow", - "base64", + "asyncband", + "base64 0.23.1", "bytes", - "form_urlencoded", - "futures", "hex", "hmac 0.13.0", "http 1.4.1", @@ -6380,9 +6567,9 @@ dependencies = [ [[package]] name = "reqsign-file-read-tokio" -version = "3.0.1" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a4b6f3a3fd29ffcc99a90aec585a65217783badfd73acddf847b63ae683bda9" +checksum = "586a8e198057773f9616901fd723a4bc186812b9765bc0f20368661d6169d906" dependencies = [ "anyhow", "reqsign-core", @@ -6391,10 +6578,11 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2" +checksum = "719e55931cc81b8dcd69f95f0a2def55617cb76cb3aa2f8c439e10682e46fa10" dependencies = [ + "bytes", "form_urlencoded", "http 1.4.1", "log", @@ -6409,9 +6597,9 @@ dependencies = [ [[package]] name = "reqsign-tencent-cos" -version = "3.0.1" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84110aabba799fbcd48b3abb51fbbff4749f879252e5806b6f5d0cbe0fef6abb" +checksum = "9a98625e0f3eab846348b7524f1f5b5c97ffc348002845463520be371b7ab7cd" dependencies = [ "anyhow", "http 1.4.1", @@ -6424,9 +6612,9 @@ dependencies = [ [[package]] name = "reqsign-volcengine-tos" -version = "3.0.1" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d083a363b3577f519ce8425bb50f902622a28a83f7c4a26a5c990b66ec75b3" +checksum = "551a3bbd9f93765dff5a4a44af0375a821f97cf633c9f299c501d3236a96e6ec" dependencies = [ "anyhow", "http 1.4.1", @@ -6441,7 +6629,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -6487,10 +6675,11 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", + "h2", "http 1.4.1", "http-body 1.0.1", "http-body-util", @@ -6616,16 +6805,6 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-hash" version = "2.1.2" @@ -6941,7 +7120,7 @@ version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" dependencies = [ - "base64", + "base64 0.22.1", "bs58", "chrono", "hex", @@ -7004,7 +7183,6 @@ dependencies = [ "cfg-if 1.0.4", "cpufeatures 0.2.17", "digest 0.10.7", - "sha2-asm", ] [[package]] @@ -7018,15 +7196,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "sha2-asm" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" -dependencies = [ - "cc", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -7174,6 +7343,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" @@ -7259,9 +7434,6 @@ name = "strum" version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros", -] [[package]] name = "strum_macros" @@ -7332,15 +7504,16 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.38.4" +version = "0.39.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +checksum = "d2071df9448915b71c4fe6d25deaf1c22f12bd234f01540b77312bb8e41361e6" dependencies = [ "libc", "memchr", "ntapi", "objc2-core-foundation", "objc2-io-kit", + "objc2-open-directory", "windows", ] @@ -7593,6 +7766,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio_with_wasm" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34e40fbbbd95441133fe9483f522db15dbfd26dc636164ebd8f2dd28759a6aa6" +dependencies = [ + "js-sys", + "tokio", + "tokio_with_wasm_proc", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "tokio_with_wasm_proc" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01145a2c788d6aae4cd653afec1e8332534d7d783d01897cefcafe4428de992" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" @@ -7631,7 +7828,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2", "http 1.4.1", @@ -8665,20 +8862,19 @@ dependencies = [ [[package]] name = "xet-client" -version = "1.5.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" +checksum = "15500e5fc9e17be15224f9e34cb07d41fb2805d9dbbf7bd3507c982fd1eb3efd" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", - "clap", + "chrono", "crc32fast", "futures", "http 1.4.1", "hyper", - "lazy_static", "more-asserts", "rand 0.10.1", "redb", @@ -8692,10 +8888,11 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-retry", + "tokio_with_wasm", "tracing", - "tracing-subscriber", "url", "urlencoding", + "uuid", "web-time", "xet-core-structures", "xet-runtime", @@ -8703,24 +8900,21 @@ dependencies = [ [[package]] name = "xet-core-structures" -version = "1.5.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" +checksum = "5080d5803c8bf15fa56ef934050a46b225801bad3ea4cb8bb3c46c5c37976516" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "blake3", "bytemuck", "bytes", - "clap", "countio", - "csv", "futures", "futures-util", "getrandom 0.4.2", "heapify", "itertools 0.14.0", - "lazy_static", "lz4_flex", "more-asserts", "rand 0.10.1", @@ -8728,7 +8922,6 @@ dependencies = [ "safe-transmute", "serde", "static_assertions", - "tempfile", "thiserror 2.0.18", "tokio", "tokio-util", @@ -8740,32 +8933,31 @@ dependencies = [ [[package]] name = "xet-data" -version = "1.5.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fd409bef621411a9d9013798540bb8036cb2678f03ab39af89a5e88034ed8c" +checksum = "1287a1657fb67de846030f6605a9960dcdad401c467b490657e6e909d2e0d1c0" dependencies = [ "anyhow", "async-trait", "bytes", "chrono", - "clap", "gearhash", "http 1.4.1", "itertools 0.14.0", - "lazy_static", "more-asserts", "rand 0.10.1", "serde", "serde_json", - "sha2 0.10.9", + "sha2 0.11.0", "tempfile", "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "url", "uuid", - "walkdir", + "web-time", "xet-client", "xet-core-structures", "xet-runtime", @@ -8773,9 +8965,9 @@ dependencies = [ [[package]] name = "xet-runtime" -version = "1.5.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d8f121c33866f7648b737abe70d0e2dd9c0af4ffdd7219207531d0283aa63d" +checksum = "921953c885ce83e02b21e8d26444b75c78c9e83076699ced08d5c6fc1cdb48ac" dependencies = [ "anyhow", "async-trait", @@ -8783,13 +8975,12 @@ dependencies = [ "chrono", "colored", "const-str", - "ctor 0.6.3", + "ctor", "dirs", "futures", "git-version", "humantime", "konst", - "lazy_static", "libc", "more-asserts", "oneshot", @@ -8803,9 +8994,11 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "tracing-appender", "tracing-subscriber", + "web-time", "whoami", "winapi", ] diff --git a/crates/lance-context-core/Cargo.toml b/crates/lance-context-core/Cargo.toml index ac86edcd..2780f4ef 100644 --- a/crates/lance-context-core/Cargo.toml +++ b/crates/lance-context-core/Cargo.toml @@ -19,9 +19,10 @@ metrics = ["dep:metrics"] [dependencies] async-trait = "0.1.92" -lance-table = "9.0.0" -lance-io = "9.0.0" -object_store = "0.13.2" +lance-table = "13.0.0" +lance-io = "13.0.0" +lance-file = "13.0.0" +object_store = "0.14.2" base64 = "0.22" bytes = "1" arrow-array = "58" @@ -31,10 +32,10 @@ arrow-schema = "58" arrow-select = "58" chrono = { version = "0.4", default-features = false, features = ["clock"] } datafusion = { version = "54", default-features = false, features = ["nested_expressions"] } -lance = "9.0.0" +lance = "13.0.0" lance-context-api = { version = "0.6.7", path = "../lance-context-api" } -lance-index = "9.0.0" -lance-namespace = "9.0.0" +lance-index = "13.0.0" +lance-namespace = "13.0.0" # Version-matched with lance-context-server/-master so one process-wide recorder # serves every crate. metrics = { version = "0.24", optional = true } diff --git a/crates/lance-context-core/examples/generic_batch_get.rs b/crates/lance-context-core/examples/generic_batch_get.rs index 0b7af70f..99867e9e 100644 --- a/crates/lance-context-core/examples/generic_batch_get.rs +++ b/crates/lance-context-core/examples/generic_batch_get.rs @@ -1,5 +1,6 @@ //! Run with `cargo run -p lance-context-core --example generic_batch_get`. //! Supply `--uri` to benchmark an existing store without changing its data. +#![recursion_limit = "256"] use std::time::Instant; diff --git a/crates/lance-context-core/src/datagen_store.rs b/crates/lance-context-core/src/datagen_store.rs index cea5156d..f3598441 100644 --- a/crates/lance-context-core/src/datagen_store.rs +++ b/crates/lance-context-core/src/datagen_store.rs @@ -335,7 +335,7 @@ impl DatagenStore { let mut generations: Vec<(u64, String)> = snapshots .iter() .flat_map(|snapshot| { - snapshot.flushed_generations.iter().map(|generation| { + snapshot.sstables.iter().map(|generation| { ( generation.generation, self.base diff --git a/crates/lance-context-core/src/generic_store.rs b/crates/lance-context-core/src/generic_store.rs index 8798d075..aaa7a356 100644 --- a/crates/lance-context-core/src/generic_store.rs +++ b/crates/lance-context-core/src/generic_store.rs @@ -528,7 +528,7 @@ impl GenericStore { // pending WAL generations before the key/work budgets can reject it. collect_scan_keys(dataset, &mut ids, &mut id_bytes, &mut scanned).await?; for snapshot in snapshots { - for generation in &snapshot.flushed_generations { + for generation in &snapshot.sstables { let path = self .base .flushed_generation_uri(snapshot.shard_id, &generation.path); diff --git a/crates/lance-context-core/src/merge_write_scope.rs b/crates/lance-context-core/src/merge_write_scope.rs index 7b87bd10..7a8c833a 100644 --- a/crates/lance-context-core/src/merge_write_scope.rs +++ b/crates/lance-context-core/src/merge_write_scope.rs @@ -318,6 +318,15 @@ struct Layered { } impl lance_io::object_store::WrappingObjectStore for Layered { + fn wrap_paginated( + &self, + name: &str, + original: Arc, + ) -> Option> { + self.inner + .wrap_paginated(name, original) + .and_then(|inner| self.outer.wrap_paginated(name, inner)) + } fn wrap( &self, name: &str, @@ -439,7 +448,7 @@ pub(crate) async fn drain_generations( let resource = format!("shard:{}", store.shard_id()); for _ in 0..10 { let mut next = store - .read_latest() + .refresh_latest() .await? .ok_or_else(|| Error::io("Shard manifest not found"))?; if next.writer_epoch != epoch { @@ -449,17 +458,17 @@ pub(crate) async fn drain_generations( .version .checked_add(1) .ok_or_else(|| Error::io("manifest version overflow"))?; - next.flushed_generations + next.sstables .retain(|g| !generations.contains(&g.generation)); authorize(&resource, next.version).await?; let writer = store.clone(); let scope = CURRENT.try_with(Arc::clone).ok(); let (next, written) = shield_observed( async move { - let written = writer.write(&next).await; + let written = commit_exact_shard(&writer, &next).await; if written .as_ref() - .is_err_and(|e| !e.to_string().contains("already exists")) + .is_err_and(|e| !matches!(e, Error::RetryableCommitConflict { .. })) { if let Some(scope) = scope { scope.mark_uncertain(); @@ -472,13 +481,38 @@ pub(crate) async fn drain_generations( .await?; match written { Ok(_) => return Ok(next), - Err(error) if error.to_string().contains("already exists") => continue, + Err(Error::RetryableCommitConflict { .. }) => continue, Err(error) => return Err(error), } } Err(Error::io("shard drain exceeded conditional-write retries")) } +/// Adapt Lance's retrying update API to one previously authorized version. +/// Returning the SAME manifest on a retry makes Lance's successor check reject +/// a moved predecessor before any new PUT; the caller must authorize again. +pub(crate) async fn commit_exact_shard( + store: &lance::dataset::mem_wal::ShardManifestStore, + next: &lance_index::mem_wal::ShardManifest, +) -> Result { + let moved = std::sync::atomic::AtomicBool::new(false); + let result = store + .commit_update(next.writer_epoch, |current| { + if current.version.checked_add(1) != Some(next.version) { + moved.store(true, Ordering::Relaxed); + } + next.clone() + }) + .await; + if result.is_err() && moved.load(Ordering::Relaxed) { + return Err(Error::retryable_commit_conflict_source( + next.version, + "shard manifest predecessor changed; reauthorize".into(), + )); + } + result +} + /// The owned protocol is supported only with immutable conditional manifest /// creation, not Lance's fallback UnsafeCommitHandler or external catalogues. pub fn supports_version_fencing(uri: &str) -> bool { @@ -569,7 +603,7 @@ pub async fn fence_manifest_versions( let mut fenced = false; for _ in 0..64 { let mut next = store - .read_latest() + .refresh_latest() .await? .ok_or_else(|| Error::io("Shard manifest missing during recovery"))?; if next.version > *high { @@ -580,9 +614,9 @@ pub async fn fence_manifest_versions( .version .checked_add(1) .ok_or_else(|| Error::io("manifest version overflow"))?; - match store.write(&next).await { + match commit_exact_shard(&store, &next).await { Ok(_) => {} - Err(error) if error.to_string().contains("already exists") => {} + Err(Error::RetryableCommitConflict { .. }) => {} Err(error) => return Err(error), } } diff --git a/crates/lance-context-core/src/preparation_io.rs b/crates/lance-context-core/src/preparation_io.rs index 93c208a1..2976c933 100644 --- a/crates/lance-context-core/src/preparation_io.rs +++ b/crates/lance-context-core/src/preparation_io.rs @@ -13,6 +13,13 @@ use std::sync::{ pub(crate) struct ProgressWrapper(pub Arc); impl WrappingObjectStore for ProgressWrapper { + fn wrap_paginated( + &self, + _: &str, + original: Arc, + ) -> Option> { + Some(original) + } fn wrap(&self, _: &str, original: Arc) -> Arc { Arc::new(ProgressStore { inner: original, @@ -81,6 +88,7 @@ impl ObjectStore for ProgressStore { let meta = result.meta.clone(); let range = result.range.clone(); let attributes = result.attributes.clone(); + let extensions = result.extensions.clone(); let steps = self.steps.clone(); // Count body chunks after they arrive, not successful response headers. // into_stream also observes actual local file reads without buffering. @@ -99,6 +107,7 @@ impl ObjectStore for ProgressStore { meta, range, attributes, + extensions, }) } @@ -257,6 +266,13 @@ pub mod test_support { pub struct ThrottleWrapper(pub Throttle); impl WrappingObjectStore for ThrottleWrapper { + fn wrap_paginated( + &self, + _: &str, + original: Arc, + ) -> Option> { + Some(original) + } fn wrap(&self, _: &str, original: Arc) -> Arc { Arc::new(ThrottledStore { inner: original, diff --git a/crates/lance-context-core/src/rollout_append.rs b/crates/lance-context-core/src/rollout_append.rs index 42a42fac..cf8aacea 100644 --- a/crates/lance-context-core/src/rollout_append.rs +++ b/crates/lance-context-core/src/rollout_append.rs @@ -16,7 +16,7 @@ use lance::dataset::{ CommitBuilder, Dataset, InsertBuilder, WriteMode, WriteParams, }; use lance::index::DatasetIndexExt; -use lance_index::mem_wal::{MemWalIndexDetails, MergedGeneration, MEM_WAL_INDEX_NAME}; +use lance_index::mem_wal::{CompactedSsTable, MemWalIndexDetails, MEM_WAL_INDEX_NAME}; use lance_table::format::{pb, Fragment}; use lance_table::system_index::mem_wal::ShardManifest; use serde::{Deserialize, Serialize}; @@ -293,7 +293,7 @@ pub(crate) async fn watermarks(dataset: &Dataset) -> Result> .ok_or_else(|| Error::io("MemWAL index has no details"))?; let details = MemWalIndexDetails::try_from(details.to_msg::()?)?; Ok(details - .merged_generations + .compacted_sstables .into_iter() .map(|g| (g.shard_id, g.generation)) .collect()) @@ -315,11 +315,11 @@ pub(crate) async fn reconcile_shard(dataset: &Dataset, shard: Uuid) -> Result = manifest - .flushed_generations + .sstables .iter() .filter(|g| g.generation <= high) .map(|g| g.generation) @@ -327,7 +327,7 @@ pub(crate) async fn reconcile_shard(dataset: &Dataset, shard: Uuid) -> Result 0 { let paths: Vec<_> = manifest - .flushed_generations + .sstables .iter() .filter(|g| generations.contains(&g.generation)) .map(|g| g.path.clone()) @@ -372,7 +372,9 @@ impl AppendCoordinator { let dataset = StorageBase::load_with_options(uri, None, session).await?; let actual: Schema = dataset.schema().into(); let expected = crate::rollout_schema(); - if dataset.manifest().should_use_legacy_format() { + if dataset.manifest().data_storage_format.version + == lance_file::version::ConcreteFileVersion::V1 + { return Err(Error::invalid_input( "rollout staging requires Lance V2 files", )); @@ -402,8 +404,12 @@ impl AppendCoordinator { for name in shards { let id = derive_shard_id(Some(name)); if seen.insert(id) { - if let Some(manifest) = shard_store(&self.dataset, id).await?.read_latest().await? { - if !manifest.flushed_generations.is_empty() { + if let Some(manifest) = shard_store(&self.dataset, id) + .await? + .refresh_latest() + .await? + { + if !manifest.sstables.is_empty() { return Ok(true); } } @@ -452,7 +458,7 @@ impl AppendCoordinator { reclaimed += reconcile_shard(&self.dataset, shard).await?; let Some(manifest) = shard_store(&self.dataset, shard) .await? - .read_latest() + .refresh_latest() .await? else { continue; @@ -490,7 +496,7 @@ impl AppendCoordinator { let cutoff = if previous_epoch != Some(manifest.writer_epoch) || prior_ranges.is_none() { let high = manifest - .flushed_generations + .sstables .iter() .map(|g| g.generation) .max() @@ -555,7 +561,7 @@ impl AppendCoordinator { previous_cutoff.unwrap_or(0) }; let mut generations: Vec<_> = manifest - .flushed_generations + .sstables .into_iter() .map(|g| Generation { number: g.generation, @@ -648,10 +654,10 @@ impl AppendCoordinator { } let manifest = shard_store(&self.dataset, part.plan.shard) .await? - .read_latest() + .refresh_latest() .await? .ok_or_else(|| Error::io("staged shard disappeared"))?; - let mut pending: Vec<_> = manifest.flushed_generations.iter().collect(); + let mut pending: Vec<_> = manifest.sstables.iter().collect(); pending.sort_by_key(|g| g.generation); if pending.len() < selected.len() || pending @@ -671,7 +677,7 @@ impl AppendCoordinator { return Err(Error::invalid_input("staged row count mismatch")); } fragments.extend(part.fragments); - merged.push(MergedGeneration::new(part.plan.shard, high)); + merged.push(CompactedSsTable::new(part.plan.shard, high)); } if !merged.is_empty() { commit_files(&mut self.dataset, fragments, merged).await?; @@ -694,7 +700,7 @@ pub(crate) fn has_cutover(dataset: &Dataset, shard: Uuid) -> bool { pub(crate) async fn commit_files( dataset: &mut Dataset, fragments: Vec, - merged: Vec, + merged: Vec, ) -> Result<()> { let _phase = crate::merge_write_scope::work_stage("base_commit"); let operation = Operation::Update { @@ -702,7 +708,7 @@ pub(crate) async fn commit_files( updated_fragments: Vec::new(), new_fragments: fragments, fields_modified: Vec::new(), - merged_generations: merged, + compacted_sstables: merged, fields_for_preserving_frag_bitmap: Vec::new(), update_mode: None, inserted_rows_filter: None, @@ -737,7 +743,9 @@ pub async fn stage( .await?; let dataset_uri = dataset.uri().to_owned(); let schema: Arc = Arc::new(dataset.schema().into()); - if dataset.manifest().should_use_legacy_format() { + if dataset.manifest().data_storage_format.version + == lance_file::version::ConcreteFileVersion::V1 + { return Err(Error::invalid_input( "rollout staging requires Lance V2 files", )); @@ -1487,7 +1495,7 @@ mod tests { assert_eq!(rows(uri).await, 2); let shard = derive_shard_id(Some("a")); let store = shard_store(&coordinator.dataset, shard).await.unwrap(); - let before = store.read_latest().await.unwrap().unwrap(); + let before = store.refresh_latest().await.unwrap().unwrap(); // Reproduce a persisted replay cursor lagging the last durable flush. // The WAL bytes survive a merge; reopening can replay the old tail into // a NEW generation even though its original generation was committed. @@ -1505,9 +1513,9 @@ mod tests { .add(&[artifact_record("new", &[42; 4096])]) .await .unwrap(); - let opened = store.read_latest().await.unwrap().unwrap(); + let opened = store.refresh_latest().await.unwrap().unwrap(); assert!(opened.writer_epoch > before.writer_epoch); - assert!(opened.flushed_generations.is_empty()); + assert!(opened.sstables.is_empty()); // Planning may observe the new epoch before its replay memtable flushes. // It must protect that active generation, not only the empty disk prefix. let (empty, _) = coordinator @@ -1564,7 +1572,7 @@ mod tests { a.close().await.unwrap(); let shard = derive_shard_id(Some("a")); let store = shard_store(&coordinator.dataset, shard).await.unwrap(); - let before = store.read_latest().await.unwrap().unwrap(); + let before = store.refresh_latest().await.unwrap().unwrap(); // Real replay: lag the durable cursor, so reopening replays "tail" and // the eight still-pending rows into the first generation of the epoch. store @@ -1658,13 +1666,13 @@ mod tests { let mut clean = Vec::new(); for epoch in 0..3 { put(&a, &format!("first-{epoch}"), 4096).await; - let current = store.read_latest().await.unwrap().unwrap(); + let current = store.refresh_latest().await.unwrap().unwrap(); clean.push(current.current_generation); put(&a, &format!("steady-{epoch}"), 4096).await; a.close().await.unwrap(); expected.push( store - .read_latest() + .refresh_latest() .await .unwrap() .unwrap() @@ -1710,7 +1718,7 @@ mod tests { let current = shard_store(&coordinator.dataset, shard) .await .unwrap() - .read_latest() + .refresh_latest() .await .unwrap() .unwrap(); @@ -1740,7 +1748,7 @@ mod tests { .unwrap() .unwrap(); assert!(state.contains(1)); - assert!(state.contains(current.flushed_generations[0].generation)); + assert!(state.contains(current.sstables[0].generation)); assert!(!state.contains(current.current_generation)); a.close().await.unwrap(); } @@ -1769,7 +1777,7 @@ mod tests { .unwrap(); let shard = derive_shard_id(Some("a")); let store = shard_store(&coordinator.dataset, shard).await.unwrap(); - let current = store.read_latest().await.unwrap().unwrap(); + let current = store.refresh_latest().await.unwrap().unwrap(); // Remove only old metadata, retaining the exact latest version so that // normal WAL discovery and publication are still possible. let manifest_dir = dir @@ -1837,7 +1845,7 @@ mod tests { let current = shard_store(&coordinator.dataset, shard) .await .unwrap() - .read_latest() + .refresh_latest() .await .unwrap() .unwrap(); @@ -1863,7 +1871,7 @@ mod tests { base_version: coordinator.version(), shard, generations: current - .flushed_generations + .sstables .iter() .map(|g| Generation { number: g.generation, diff --git a/crates/lance-context-core/src/rollout_store.rs b/crates/lance-context-core/src/rollout_store.rs index 5564b3c2..4f573ff2 100644 --- a/crates/lance-context-core/src/rollout_store.rs +++ b/crates/lance-context-core/src/rollout_store.rs @@ -813,7 +813,7 @@ impl RolloutStore { let last_updated = self.base.dataset.manifest.timestamp().timestamp_millis(); let pending_wal_generations = shard_snapshots .iter() - .map(|snapshot| snapshot.flushed_generations.len() as i64) + .map(|snapshot| snapshot.sstables.len() as i64) .sum(); Ok(RolloutObservation { row_count, @@ -1067,7 +1067,7 @@ impl RolloutStore { .await?; for snapshot in snapshots { - for generation in snapshot.flushed_generations { + for generation in snapshot.sstables { let uri = self.flushed_generation_uri(snapshot.shard_id, &generation.path); let dataset = self.open_flushed_dataset(&uri).await?; Self::append_non_blob_records_from_dataset( @@ -1144,7 +1144,7 @@ impl RolloutStore { let mut generation_uris = Vec::new(); for snapshot in shard_snapshots { - for generation in snapshot.flushed_generations { + for generation in snapshot.sstables { generation_uris .push(self.flushed_generation_uri(snapshot.shard_id, &generation.path)); } @@ -1471,7 +1471,7 @@ impl RolloutStore { let uris: Vec = snapshots .iter() .flat_map(|snapshot| { - snapshot.flushed_generations.iter().map(|generation| { + snapshot.sstables.iter().map(|generation| { self.flushed_generation_uri(snapshot.shard_id, &generation.path) }) }) @@ -2338,16 +2338,16 @@ mod tests { snapshot.shard_id, DEFAULT_MANIFEST_SCAN_BATCH_SIZE, ); - let current = manifests.read_latest().await.unwrap().unwrap(); + let current = manifests.refresh_latest().await.unwrap().unwrap(); manifests .commit_update(current.writer_epoch, |m| ShardManifest { version: m.version + 1, - flushed_generations: vec![], + sstables: vec![], ..m.clone() }) .await .unwrap(); - for generation in &snapshot.flushed_generations { + for generation in &snapshot.sstables { store .remove_dir_all( path.clone() @@ -2749,10 +2749,10 @@ mod tests { store.base.write_shard, 16, ); - let before = manifests.read_latest().await.unwrap().unwrap(); + let before = manifests.refresh_latest().await.unwrap().unwrap(); let mut late_drain = before.clone(); late_drain.version += 1; - late_drain.flushed_generations.clear(); + late_drain.sstables.clear(); let plan = std::collections::BTreeMap::from([( format!("shard:{}", store.base.write_shard), late_drain.version, @@ -2760,25 +2760,106 @@ mod tests { crate::merge_write_scope::fence_manifest_versions(uri, None, &plan, "test-recovery") .await .unwrap(); - assert!(manifests - .write(&late_drain) - .await - .unwrap_err() - .to_string() - .contains("already exists")); - let after = manifests.read_latest().await.unwrap().unwrap(); + assert!(matches!( + crate::merge_write_scope::commit_exact_shard(&manifests, &late_drain).await, + Err(lance::Error::RetryableCommitConflict { .. }) + )); + let after = manifests.refresh_latest().await.unwrap().unwrap(); assert!(after.version > late_drain.version); assert_eq!( after.writer_epoch, before.writer_epoch, "recovery must not fence live ingest" ); - assert_eq!(after.flushed_generations, before.flushed_generations); + assert_eq!(after.sstables, before.sstables); store.cleanup_own_shard().await.unwrap(); assert_eq!(flushed_generation_count(&store).await, 0); assert_eq!(store.base.dataset.count_rows(None).await.unwrap(), 2); store.close().await.unwrap(); } + #[tokio::test] + async fn shard_drain_reauthorizes_after_a_concurrent_manifest_commit() { + use crate::merge_write_scope::{drain_generations, CommitAuthorizer, MergeWriteScope}; + use lance::dataset::mem_wal::ShardManifestStore; + use std::sync::atomic::{AtomicU64, Ordering}; + + #[derive(Debug)] + struct RaceThenRevoke { + peer: ShardManifestStore, + epoch: u64, + admitted: AtomicU64, + refused: AtomicU64, + } + impl CommitAuthorizer for RaceThenRevoke { + fn authorize<'a>( + &'a self, + _resource: &'a str, + version: u64, + ) -> std::pin::Pin> + Send + 'a>> + { + Box::pin(async move { + if self.admitted.swap(version, Ordering::SeqCst) != 0 { + self.refused.store(version, Ordering::SeqCst); + return Err(lance::Error::io("execution revoked")); + } + // A different handle wins the admitted immutable version + // before this drain writes. Lance may internally retry, + // but it must not drain at the next, unadmitted version. + let won = self + .peer + .commit_update(self.epoch, |current| { + let mut next = current.clone(); + next.version += 1; + next + }) + .await?; + assert_eq!(won.version, version); + Ok(()) + }) + } + } + + let dir = tempfile::tempdir().unwrap(); + let mut store = RolloutStore::open(dir.path().to_str().unwrap()) + .await + .unwrap(); + store.add(&[assistant_record("pending")]).await.unwrap(); + store.flush().await.unwrap(); + let object_store = store.base.dataset.object_store(None).await.unwrap(); + let path = &store.base.dataset.branch_location().path; + let shard = store.base.write_shard; + let manifests = ShardManifestStore::new(object_store.clone(), path, shard, 16); + let before = manifests.refresh_latest().await.unwrap().unwrap(); + let gate = Arc::new(RaceThenRevoke { + peer: ShardManifestStore::new(object_store, path, shard, 16), + epoch: before.writer_epoch, + admitted: AtomicU64::new(0), + refused: AtomicU64::new(0), + }); + let scope = MergeWriteScope::with_authorizer(gate.clone()); + let result = scope + .run(drain_generations( + manifests, + before.writer_epoch, + before.sstables.iter().map(|s| s.generation).collect(), + )) + .await; + assert!(result + .unwrap_err() + .to_string() + .contains("execution revoked")); + scope.drain().await; + assert_eq!(gate.refused.load(Ordering::SeqCst), before.version + 2); + let after = gate.peer.refresh_latest().await.unwrap().unwrap(); + assert_eq!(after.version, before.version + 1); + assert_eq!(after.sstables, before.sstables); + assert_eq!(scope.completed_steps(), 0); + assert!(!scope.has_uncertain_commit()); + store.cleanup_own_shard().await.unwrap(); + assert_eq!(store.base.dataset.count_rows(None).await.unwrap(), 1); + store.close().await.unwrap(); + } + fn assistant_record(id: &str) -> RolloutRecord { RolloutRecord { id: id.to_string(), @@ -3729,10 +3810,10 @@ mod tests { DEFAULT_MANIFEST_SCAN_BATCH_SIZE, ); manifest_store - .read_latest() + .refresh_latest() .await .unwrap() - .map(|m| m.flushed_generations.len()) + .map(|m| m.sstables.len()) .unwrap_or(0) } @@ -3749,7 +3830,7 @@ mod tests { DEFAULT_MANIFEST_SCAN_BATCH_SIZE, ); manifest_store - .read_latest() + .refresh_latest() .await .unwrap() .map(|m| m.writer_epoch) diff --git a/crates/lance-context-core/src/store_base.rs b/crates/lance-context-core/src/store_base.rs index 06bfaf52..c8604b52 100644 --- a/crates/lance-context-core/src/store_base.rs +++ b/crates/lance-context-core/src/store_base.rs @@ -772,8 +772,8 @@ impl StorageBase { let Some(writer) = writer else { return Ok(FlushOutcome::Noop); }; - match writer.force_seal_active().await { - Ok(()) => {} + let fence = match writer.force_seal_active().await { + Ok(fence) => fence, // A fence means a merge superseded our epoch; the next `put` reopens // and replays. Nothing to flush against the dead epoch. Err(err) if is_fenced_error(&err) => { @@ -781,8 +781,8 @@ impl StorageBase { return Ok(FlushOutcome::Fenced); } Err(err) => return Err(err), - } - writer.wait_for_flush_drain().await?; + }; + fence.wait().await?; Ok(FlushOutcome::Sealed) } @@ -940,18 +940,18 @@ impl StorageBase { DEFAULT_MANIFEST_SCAN_BATCH_SIZE, ); Ok(manifest_store - .read_latest() + .refresh_latest() .await? .map(|manifest| ShardWatermark { shard_id: manifest.shard_id, writer_epoch: manifest.writer_epoch, sealed_through: manifest - .flushed_generations + .sstables .iter() .map(|g| g.generation) .max() .unwrap_or(0), - pending_generations: manifest.flushed_generations.len(), + pending_generations: manifest.sstables.len(), merged_through: None, })) } @@ -968,9 +968,10 @@ impl StorageBase { self.write_shard, DEFAULT_MANIFEST_SCAN_BATCH_SIZE, ); - Ok(manifest_store.read_latest().await?.is_some_and(|manifest| { - manifest.flushed_generations.len() >= self.merge_after_generations - })) + Ok(manifest_store + .refresh_latest() + .await? + .is_some_and(|manifest| manifest.sstables.len() >= self.merge_after_generations)) } /// [`Self::prepare_merge_if_ready`], but seals the active memtable *before* @@ -1004,10 +1005,10 @@ impl StorageBase { self.write_shard, DEFAULT_MANIFEST_SCAN_BATCH_SIZE, ); - let Some(manifest) = manifest_store.read_latest().await? else { + let Some(manifest) = manifest_store.refresh_latest().await? else { return Ok(None); }; - let pending = manifest.flushed_generations.len(); + let pending = manifest.sstables.len(); if pending == 0 || pending < threshold.max(1) { return Ok(None); } @@ -1058,13 +1059,11 @@ impl StorageBase { let marks = crate::rollout_append::watermarks(&latest).await?; let mut manifest = manifest.clone(); if let Some(high) = marks.get(&self.write_shard) { - manifest - .flushed_generations - .retain(|g| g.generation > *high); + manifest.sstables.retain(|g| g.generation > *high); } - manifest.flushed_generations.sort_by_key(|g| g.generation); + manifest.sstables.sort_by_key(|g| g.generation); let manifest = &manifest; - if manifest.flushed_generations.is_empty() { + if manifest.sstables.is_empty() { return Ok(None); } @@ -1131,11 +1130,11 @@ impl StorageBase { // Several sweepers can prepare the same immutable generations under a // shared lock. Once the first commit drains them, later prepared copies // are stale and must not write the same rows again. - let Some(current_manifest) = manifest_store.read_latest().await? else { + let Some(current_manifest) = manifest_store.refresh_latest().await? else { return Ok(false); }; if !current_manifest - .flushed_generations + .sstables .iter() .any(|generation| merged_generations.contains(&generation.generation)) { @@ -1161,7 +1160,7 @@ impl StorageBase { merged_generations .iter() .max() - .map(|high| lance_index::mem_wal::MergedGeneration::new(self.write_shard, *high)) + .map(|high| lance_index::mem_wal::CompactedSsTable::new(self.write_shard, *high)) } else { None }; @@ -1317,12 +1316,12 @@ impl StorageBase { // first generation alone exceeds the budget. Leftovers stay pending. let mut buffered_bytes = 0usize; let budget = if self.merge_max_generations == 0 { - manifest.flushed_generations.len() + manifest.sstables.len() } else { self.merge_max_generations }; - 'generations: for flushed in manifest.flushed_generations.iter().take(budget) { + 'generations: for flushed in manifest.sstables.iter().take(budget) { let gen_uri = format!( "{}/_mem_wal/{}/{}", base_uri, self.write_shard, flushed.path @@ -1401,7 +1400,7 @@ impl StorageBase { &mut self, batches: Vec, merge_schema: Arc, - watermark: Option, + watermark: Option, ) -> LanceResult<()> { // Predicate deletion projects only the key and row IDs, including for // unindexed fragments. It can use existing BTree/ZoneMap coverage but @@ -2120,17 +2119,17 @@ impl StorageBase { shard_id, DEFAULT_MANIFEST_SCAN_BATCH_SIZE, ); - Ok::<_, LanceError>(manifest_store.read_latest().await?.map(|manifest| { + Ok::<_, LanceError>(manifest_store.refresh_latest().await?.map(|manifest| { ShardWatermark { shard_id: manifest.shard_id, writer_epoch: manifest.writer_epoch, sealed_through: manifest - .flushed_generations + .sstables .iter() .map(|g| g.generation) .max() .unwrap_or(0), - pending_generations: manifest.flushed_generations.len(), + pending_generations: manifest.sstables.len(), merged_through: merged.get(&manifest.shard_id).copied(), } })) @@ -2169,10 +2168,10 @@ impl StorageBase { shard_id, DEFAULT_MANIFEST_SCAN_BATCH_SIZE, ); - let Some(manifest) = manifest_store.read_latest().await? else { + let Some(manifest) = manifest_store.refresh_latest().await? else { return Ok(None); }; - let pending = manifest.flushed_generations.len(); + let pending = manifest.sstables.len(); observe_value!(crate::metrics::ROLLOUT_WAL_PENDING_GENERATIONS, pending); if max_at != 0 && pending > max_at { // A read over this shard would open `pending` datasets @@ -2205,9 +2204,8 @@ impl StorageBase { let mut snapshot = ShardSnapshot::new(shard_id) .with_spec_id(manifest.shard_spec_id) .with_current_generation(manifest.current_generation); - for flushed in manifest.flushed_generations { - snapshot = - snapshot.with_flushed_generation(flushed.generation, flushed.path); + for sstable in manifest.sstables { + snapshot = snapshot.with_sstable(sstable.generation, sstable.path); } Ok::<_, LanceError>(Some(snapshot)) } @@ -2223,7 +2221,7 @@ impl StorageBase { // and still opened 25k datasets on one lookup (OOMKilled the worker). let total: usize = snapshots .iter() - .map(|snapshot| snapshot.flushed_generations.len()) + .map(|snapshot| snapshot.sstables.len()) .sum(); if max_at != 0 && total > max_at { note_read_refused(self.dataset.uri()); @@ -2244,7 +2242,7 @@ impl StorageBase { .wal_shard_snapshots() .await? .iter() - .map(|snapshot| snapshot.flushed_generations.len()) + .map(|snapshot| snapshot.sstables.len()) .sum()) } @@ -2254,7 +2252,7 @@ impl StorageBase { let generation_paths: Vec = snapshots .iter() .flat_map(|snapshot| { - snapshot.flushed_generations.iter().map(|generation| { + snapshot.sstables.iter().map(|generation| { self.flushed_generation_uri(snapshot.shard_id, &generation.path) }) }) diff --git a/crates/lance-context-core/tests/storage_reliability.rs b/crates/lance-context-core/tests/storage_reliability.rs index a81e9e97..d212e045 100644 --- a/crates/lance-context-core/tests/storage_reliability.rs +++ b/crates/lance-context-core/tests/storage_reliability.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "256"] + use lance_context_api::AddRecordRequest; use lance_context_core::{ record_from_add_request, ContextStore, RolloutRecord, RolloutStore, RolloutStoreOptions, diff --git a/crates/lance-context-master/Cargo.toml b/crates/lance-context-master/Cargo.toml index dc1a0ca3..cd1e0d2c 100644 --- a/crates/lance-context-master/Cargo.toml +++ b/crates/lance-context-master/Cargo.toml @@ -25,7 +25,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] } clap = { version = "4", features = ["derive", "env"] } etcd-client = { version = "0.19", features = ["tls"] } futures = "0.3" -lance = "9.0.0" +lance = "13.0.0" lru = "0.12" metrics = "0.24" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } diff --git a/crates/lance-context-master/src/lib.rs b/crates/lance-context-master/src/lib.rs index a0330986..47fa122a 100644 --- a/crates/lance-context-master/src/lib.rs +++ b/crates/lance-context-master/src/lib.rs @@ -1,4 +1,5 @@ //! Control-plane (master) library surface. +#![recursion_limit = "256"] pub mod admission; pub mod catchup; diff --git a/crates/lance-context-master/src/main.rs b/crates/lance-context-master/src/main.rs index b0cb777e..804b9e1c 100644 --- a/crates/lance-context-master/src/main.rs +++ b/crates/lance-context-master/src/main.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "256"] + use axum::Router; use clap::Parser; use tokio::net::TcpListener; diff --git a/crates/lance-context-server/Cargo.toml b/crates/lance-context-server/Cargo.toml index 2e478bc9..169ec7c9 100644 --- a/crates/lance-context-server/Cargo.toml +++ b/crates/lance-context-server/Cargo.toml @@ -35,7 +35,7 @@ uuid = { version = "1", features = ["v4"] } [dev-dependencies] tokio = { version = "1", features = ["test-util", "io-util"] } -lance = "9.0.0" +lance = "13.0.0" tower = { version = "0.5", features = ["util"] } lance-context-client = { version = "0.6.7", path = "../lance-context-client" } etcd-client = { version = "0.19", features = ["tls"] } diff --git a/crates/lance-context-server/src/main.rs b/crates/lance-context-server/src/main.rs index b00c4f71..5e8a5af6 100644 --- a/crates/lance-context-server/src/main.rs +++ b/crates/lance-context-server/src/main.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "256"] + mod body_timeout; mod config; mod drain; diff --git a/docs/design/lance-13-upgrade.md b/docs/design/lance-13-upgrade.md new file mode 100644 index 00000000..bc9c4161 --- /dev/null +++ b/docs/design/lance-13-upgrade.md @@ -0,0 +1,33 @@ +# Upgrading lance-context to Lance 13 + +Rust Lance dependencies and Python's optional `pylance` dependency now use 13.0.0. +The Rust object_store dependency moves to 0.14.2; Arrow remains at 58 and +DataFusion at 54. Python's matching lance-namespace range is 0.11.x. + +## Prepare existing MemWAL data before deployment + +Lance 13 reads flushed SSTables by field ID. Older flushed generations can use +positional column numbers; when schema field IDs have gaps, the old files cannot +be identified or interpreted safely by the new reader. See the upstream +[MemWAL upgrading notes](https://docs.rs/lance/13.0.0/src/lance/dataset/mem_wal/api.rs.html). + +Before deploying the new binaries, stop writes and use the **old Lance version** +to flush and merge all shards into the base dataset. Confirm no old pending +generations remain, then upgrade readers and writers together. Validate rows and +payloads before resuming writes. Avoid a mixed-version rolling upgrade while old +flushed generations remain. The upstream field-ID change does not affect durable +WAL entries themselves. This code upgrade does not migrate existing datasets. + +## Compatibility changes + +- Use SSTable names for upstream generation metadata; context's public API names + remain unchanged. +- Refresh authoritative shard manifests explicitly, bypassing the new cache when + observing concurrent writer or merge progress. +- Wait on the SealFence returned by flush, propagating its errors and bounding + the wait to the captured set of flushes. +- Preserve exact-version merge authorization across Lance's internal commit + retries. If the predecessor changed, refresh and authorize again before any + write at a newer version. A concurrent-commit regression test covers revocation. +- Adapt object-store pagination wrappers and retain GetResult extensions. +- Increase affected crate recursion limits to accommodate the new async types. diff --git a/python/pyproject.toml b/python/pyproject.toml index ceb44385..03f320a8 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -41,8 +41,8 @@ openai = ["openai>=1.0"] sentence-transformers = ["sentence-transformers>=2.0"] graph = ["lance-graph==0.5.4"] lance-python = [ - "pylance>=9.0.0,<10", - "lance-namespace>=0.8.5,<0.9", + "pylance>=13.0.0,<14", + "lance-namespace>=0.11.1,<0.12", "lancedb==0.27.1", ] # `moto[server]` pulls in flask + flask-cors so moto.server can be launched @@ -54,8 +54,8 @@ tests = [ "moto[s3,server]", "boto3", "botocore", - "pylance>=9.0.0,<10", - "lance-namespace>=0.8.5,<0.9", + "pylance>=13.0.0,<14", + "lance-namespace>=0.11.1,<0.12", "lancedb==0.27.1", ] dev = ["ruff", "pyright"] diff --git a/python/uv.lock b/python/uv.lock index c58b464a..51cc473d 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1185,7 +1185,7 @@ wheels = [ [[package]] name = "lance-context" -version = "0.6.5" +version = "0.6.7" source = { editable = "." } dependencies = [ { name = "pyarrow" }, @@ -1227,15 +1227,15 @@ requires-dist = [ { name = "boto3", marker = "extra == 'tests'" }, { name = "botocore", marker = "extra == 'tests'" }, { name = "lance-graph", marker = "extra == 'graph'", specifier = "==0.5.4" }, - { name = "lance-namespace", marker = "extra == 'lance-python'", specifier = ">=0.8.5,<0.9" }, - { name = "lance-namespace", marker = "extra == 'tests'", specifier = ">=0.8.5,<0.9" }, + { name = "lance-namespace", marker = "extra == 'lance-python'", specifier = ">=0.11.1,<0.12" }, + { name = "lance-namespace", marker = "extra == 'tests'", specifier = ">=0.11.1,<0.12" }, { name = "lancedb", marker = "extra == 'lance-python'", specifier = "==0.27.1" }, { name = "lancedb", marker = "extra == 'tests'", specifier = "==0.27.1" }, { name = "moto", extras = ["s3", "server"], marker = "extra == 'tests'" }, { name = "openai", marker = "extra == 'openai'", specifier = ">=1.0" }, { name = "pyarrow", specifier = ">=14" }, - { name = "pylance", marker = "extra == 'lance-python'", specifier = ">=9.0.0,<10" }, - { name = "pylance", marker = "extra == 'tests'", specifier = ">=9.0.0,<10" }, + { name = "pylance", marker = "extra == 'lance-python'", specifier = ">=13.0.0,<14" }, + { name = "pylance", marker = "extra == 'tests'", specifier = ">=13.0.0,<14" }, { name = "pyright", marker = "extra == 'dev'" }, { name = "pytest", marker = "extra == 'tests'" }, { name = "pytest-asyncio", marker = "extra == 'tests'" }, @@ -1267,19 +1267,19 @@ wheels = [ [[package]] name = "lance-namespace" -version = "0.8.6" +version = "0.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace-urllib3-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/af/12/f7ab93b29be3edbf5fc3610714bf2d06088e7f4524bfb38dfd6852458b08/lance_namespace-0.8.6.tar.gz", hash = "sha256:18232e721c8188145f4ec9389cc2dfbeeabf54a619d94885ea1b3375bee9f4af", size = 11529, upload-time = "2026-06-12T17:36:41.651Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/93/da5f7fcac690db9b282a3439ed9e34960c147619a0d6e1f4eb8cd240e7a5/lance_namespace-0.11.1.tar.gz", hash = "sha256:f67cfbbe0647b7cb42f23b673e7edf8a75b7d8a047265a916492f8d247ee1bc2", size = 11631, upload-time = "2026-08-18T17:40:06.294Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/1b/5b1668ee2dc8910965f390640359112a31157092fcf8e000b89c79b58708/lance_namespace-0.8.6-py3-none-any.whl", hash = "sha256:571eae34f9aad70e5b05020416c2860889b9ec82993ccd0eb015e7b39c3ea309", size = 13383, upload-time = "2026-06-12T17:36:43.456Z" }, + { url = "https://files.pythonhosted.org/packages/fa/bc/601f2b3cc4cfa0070d858a33223bc823fffdd7981a25c45984a5216ca952/lance_namespace-0.11.1-py3-none-any.whl", hash = "sha256:07643fce9a42ad4d58cc8bf91e3f592bc7f4cbd8d0ad5233223506debf67551c", size = 13507, upload-time = "2026-08-18T17:40:03.561Z" }, ] [[package]] name = "lance-namespace-urllib3-client" -version = "0.8.6" +version = "0.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -1287,9 +1287,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c7/80/fb224b4a89c1c1638cde949cb6cce6c3aca7759effbfea46a3d9c3960b21/lance_namespace_urllib3_client-0.8.6.tar.gz", hash = "sha256:b6fb1d306e74a7576e5309919020be744527de484a63dbf5eed10f8b368548df", size = 228772, upload-time = "2026-06-12T17:36:42.609Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/c5/2bdd0ff98b469894c8a73be809d26ffdad5402517b0e5f9e758026cba29e/lance_namespace_urllib3_client-0.11.1.tar.gz", hash = "sha256:145a9e9424d7597487249b5b95ee274423bf2910e1a9160b6a07b676b61ea46a", size = 237345, upload-time = "2026-08-18T17:40:07.308Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/90/1e27de15cd1b16785a1c7312beb0a59e75c8344a815f600f58173a565bd1/lance_namespace_urllib3_client-0.8.6-py3-none-any.whl", hash = "sha256:9d78249c3fb15aa3d15d668f78f04a275af3d08d800a7027492f37996ac4968b", size = 369950, upload-time = "2026-06-12T17:36:40.438Z" }, + { url = "https://files.pythonhosted.org/packages/2a/2a/eaaefd55d1190291207049fedc6b3eb22b506e57d6de91bae46bbaaa9c60/lance_namespace_urllib3_client-0.11.1-py3-none-any.whl", hash = "sha256:36537f529294da6d884ba0fe783704483f0a75463497c7705fd083a4d0257990", size = 406311, upload-time = "2026-08-18T17:40:04.842Z" }, ] [[package]] @@ -2170,7 +2170,7 @@ wheels = [ [[package]] name = "pylance" -version = "9.0.0" +version = "13.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace" }, @@ -2178,12 +2178,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/be/45733acd64801991852aac8e658601fd8fc12f76ceb81e57fca690896b90/pylance-9.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:8257213501d3298c5b6a344d60938e4bbe4de9f00cd3265371a56d1dc3dd15ca", size = 68377982, upload-time = "2026-07-24T16:53:45.247Z" }, - { url = "https://files.pythonhosted.org/packages/d0/3e/1ef707cb215cc7268c63ad84a91344ad6313d3984b343eeb19a9b708698e/pylance-9.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:804eedfa1fda2e703cca8580c76f0b44a1b849b725a8908c06f8acfca811732f", size = 71844362, upload-time = "2026-07-24T16:56:07.6Z" }, - { url = "https://files.pythonhosted.org/packages/c8/fb/a499e5c53ddb75c7de44100fd2bb1f7cc735966200d20292eed7af9ef552/pylance-9.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a0b75595e3766c1d5f4c90abdc52337b4de60f1a52d123a4f8c4e5bcbbbfa8f", size = 75663088, upload-time = "2026-07-24T17:10:31.283Z" }, - { url = "https://files.pythonhosted.org/packages/e9/80/0714e09f64a68dbdf62558955e737a7436df763b9834a6aba506861b5352/pylance-9.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d2f69c5c390ae3710c35a429905fe15f769951777fafb1b72a359e035ec121f7", size = 71866858, upload-time = "2026-07-24T16:56:35.937Z" }, - { url = "https://files.pythonhosted.org/packages/4b/3c/78d3a6d6ca0d843b7c3ac0c30d9cd2cf4635b0c2cabe6ec66583d5bfc1a1/pylance-9.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:836268a7832d62f3d5ccbe1c3fca239621971d90297bcfff14a70b3cb6842aa8", size = 75642656, upload-time = "2026-07-24T17:13:11.879Z" }, - { url = "https://files.pythonhosted.org/packages/cf/c1/dc9c9a31e171530ec0add024d922488c046437d32a7087c29e254a7eacc7/pylance-9.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:96441d27a5ed3805300388ccf8f31835cd280c98751f80b6a1a11dcd6808fc43", size = 81668288, upload-time = "2026-07-24T17:05:38.707Z" }, + { url = "https://files.pythonhosted.org/packages/90/12/fa8b39d84bfac672fd44d1369b31069021829e585ca565d49d33cedc90a1/pylance-13.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:38cbe8d204785e697909e8faca91adb3b00a2f6a865c425987cb232d0865c010", size = 74507904, upload-time = "2026-10-07T07:00:00.369Z" }, + { url = "https://files.pythonhosted.org/packages/27/e1/0399a1dc66664ed6d66fc4cd7fdaeb457c4dd5b2ea536241643388a888fb/pylance-13.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22a37a0af446964e79cfd6046f9ea2735fcb42f8020eab63947b2bdca04989c2", size = 78945779, upload-time = "2026-10-07T07:03:56.389Z" }, + { url = "https://files.pythonhosted.org/packages/7d/72/7ba2a773a9fc3be815f39fae39f1f26b87e1a82aa7adf7cc748cb294ea36/pylance-13.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c94649a35161c6100ed822ac022756bfefb840b7e01cf491505c5245186f7b4", size = 83169863, upload-time = "2026-10-07T07:19:35.436Z" }, + { url = "https://files.pythonhosted.org/packages/03/48/81ffda7fb308a87e81416f5f5ca67507abd8f0470c7820f68e9b32881260/pylance-13.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:22e47adcc2c7300ff876fb398ee9c625973932163ec055adae14871f8a09d7b0", size = 78969946, upload-time = "2026-10-07T07:05:09.987Z" }, + { url = "https://files.pythonhosted.org/packages/77/4c/8734e6c12500521cc92e3594715cb5d58cd770938127458e4b7252a17a92/pylance-13.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a54496c4e6c01a8c3d49479fce98474c95a265515091d173aa9e16df271d837d", size = 83164662, upload-time = "2026-10-07T07:17:16.085Z" }, + { url = "https://files.pythonhosted.org/packages/ee/39/7ff19ec586f460f7851f96e07ee20e67815806c19eccc08155aa110a7d0d/pylance-13.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:8a340dcf750171dd6386db0b6ed303bb22594e1c1b266b7fe149ea1684bf4a9a", size = 89711200, upload-time = "2026-10-07T07:07:25.895Z" }, ] [[package]]