diff --git a/python/Cargo.lock b/python/Cargo.lock index dfe5562..25d42c3 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -21,6 +21,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "asn1-rs" version = "0.7.2" @@ -45,7 +54,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -57,7 +66,18 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", ] [[package]] @@ -72,12 +92,41 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-lc-rs" +version = "1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bit-vec" version = "0.9.1" @@ -158,6 +207,37 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "const-oid" version = "0.10.2" @@ -198,6 +278,16 @@ dependencies = [ "libc", ] +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -272,9 +362,21 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + [[package]] name = "equivalent" version = "1.0.2" @@ -333,12 +435,34 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + [[package]] name = "futures-core" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + [[package]] name = "futures-macro" version = "0.3.32" @@ -347,7 +471,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -369,9 +493,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", + "futures-io", "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -493,12 +619,41 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + [[package]] name = "hybrid-array" version = "0.4.13" @@ -508,6 +663,90 @@ dependencies = [ "typenum", ] +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -621,12 +860,76 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" version = "0.1.35" @@ -676,6 +979,7 @@ dependencies = [ "log", "memchr", "minarrow", + "object_store", "quinn", "rustls-pki-types", "ryu", @@ -685,6 +989,7 @@ dependencies = [ "tokio-rustls", "tokio-tungstenite", "tracing", + "url", "vec64", "wtransport", "zstd", @@ -714,6 +1019,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" @@ -726,6 +1040,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.0" @@ -816,6 +1140,46 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object_store" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +dependencies = [ + "async-trait", + "aws-lc-rs", + "base64 0.23.1", + "bytes", + "chrono", + "crc-fast", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "httparse", + "humantime", + "hyper", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.10.2", + "reqwest", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "octets" version = "0.3.6" @@ -843,13 +1207,36 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "pem" version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -952,7 +1339,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -964,7 +1351,17 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", ] [[package]] @@ -993,6 +1390,7 @@ version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.4.3", "lru-slab", @@ -1112,6 +1510,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "ref-cast" version = "1.0.25" @@ -1129,7 +1536,46 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", +] + +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", ] [[package]] @@ -1176,6 +1622,7 @@ version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -1206,12 +1653,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + [[package]] name = "rustls-webpki" version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -1229,6 +1704,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -1238,6 +1722,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.7.0" @@ -1294,7 +1784,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1310,6 +1800,18 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1" version = "0.10.7" @@ -1353,6 +1855,16 @@ dependencies = [ "value-trait", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -1387,6 +1899,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1410,6 +1928,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1418,7 +1956,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1444,7 +1982,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1525,7 +2063,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1567,6 +2105,51 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" @@ -1586,7 +2169,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1598,6 +2181,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "tungstenite" version = "0.28.0" @@ -1683,6 +2272,25 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1711,6 +2319,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.126" @@ -1730,7 +2348,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -1743,6 +2361,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" @@ -1753,6 +2394,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -1771,12 +2421,74 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -1954,7 +2666,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -1975,7 +2687,7 @@ checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1995,7 +2707,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -2035,7 +2747,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] diff --git a/python/Cargo.toml b/python/Cargo.toml index b46f69c..46dac81 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -45,6 +45,16 @@ lightstream = { version = "0.7", path = "../rust", features = ["mmap", "uds"] } [features] default = [] extension-module = ["pyo3/extension-module"] +# Object storage URLs in place of file paths for `read` and `write`, with +# the module-level `register_options` function. Enables `memory://` URLs, +# and the backend features below enable their cloud schemes. +object_storage = ["lightstream/object_storage"] +# Amazon S3 and S3-compatible stores (`s3://`, `s3a://`) +aws = ["object_storage", "lightstream/aws"] +# Google Cloud Storage (`gs://`) +gcp = ["object_storage", "lightstream/gcp"] +# Azure Blob Storage (`az://`, `abfs://`, `abfss://`, `adl://`, `azure://`) +azure = ["object_storage", "lightstream/azure"] [[example]] name = "tcp_server" diff --git a/python/README.md b/python/README.md index e5b2f05..8502d0f 100644 --- a/python/README.md +++ b/python/README.md @@ -44,6 +44,31 @@ Arrow IPC readers use memory mapping and out-of-core routing where appropriate. Readers yield `minarrow.Table` batches. `read_all()` returns a `Table` when the result is contiguous or a `ChunkedTable` when multiple chunks must be preserved. +### Object storage + +Builds with the `object_storage` feature accept object URLs wherever a file path is accepted. An object prefix with chunk objects beneath it reads in the same way as a chunk directory. + +```python +ls.register_options("s3://telemetry", {"aws_region": "eu-west-2"}) + +table = ls.read("s3://telemetry/trades/2026-10-14.parquet").read_all() + +with ls.write("s3://telemetry/trades/summary.arrow") as writer: + writer.write(table) +``` + +| Store | URL | Feature | +| ------------------------ | --------------------------------- | ---------------- | +| In-process memory | `memory:///path` | `object_storage` | +| Amazon S3 and compatible | `s3://bucket/path` | `aws` | +| Google Cloud Storage | `gs://bucket/path` | `gcp` | +| Azure Blob Storage | `az://container/path`, `abfss://` | `azure` | + +- Credentials come from each backend's standard environment variables. +- `register_options` sets backend options, and the upload settings `part_size` and `max_concurrent_parts`, for every URL beneath a prefix. +- Written objects become visible in their store when the writer closes. +- Memory-mapped reads apply to local files only. + ### Network transports The same interface streams raw Arrow IPC over TCP, Unix domain sockets, WebSocket, HTTP, QUIC, WebTransport and standard I/O. @@ -225,6 +250,8 @@ maturin develop pytest tests/ ``` +Optional features are enabled at build time, for example `maturin develop --features object_storage` or `maturin develop --features aws,gcp,azure`. + ## Licence Copyright © 2025–2026 Peter Garfield Bower. diff --git a/python/src/input.rs b/python/src/input.rs index cfb4971..cef92c7 100644 --- a/python/src/input.rs +++ b/python/src/input.rs @@ -11,7 +11,6 @@ //! `Protocol` picks the wire framing, with both arms running over every //! transport, and each leaf variant wraps one lightstream end reader. -use std::fs::File; use std::future::poll_fn; use std::io::{BufReader, Stdin}; use std::path::PathBuf; @@ -224,6 +223,9 @@ impl FileIO { /// Reads the next batch from the file, advancing the batch cursor for /// the indexed IPC readers and the chunk iterator for directories. fn read(&mut self) -> Result, IoError> { + // Object URL reads wait on the embedded runtime from this thread. + #[cfg(feature = "object_storage")] + let _guard = runtime().enter(); match self { FileIO::Ipc { reader, cursor } => { if *cursor >= reader.num_batches() { @@ -248,8 +250,8 @@ impl FileIO { return Ok(None); } *done = true; - let file = File::open(&path).map_err(IoError::Io)?; - Ok(Some(load_parquet_table(file)?)) + let reader = StorageReader::open(&path).map_err(IoError::Io)?; + Ok(Some(load_parquet_table(reader)?)) } FileIO::ParquetChunked { reader } => reader.next().transpose(), FileIO::Csv { reader } => Ok(reader.next_batch()?), diff --git a/python/src/lib.rs b/python/src/lib.rs index d8a5abe..7374369 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -19,6 +19,13 @@ mod runtime; mod tls; mod uri; +#[cfg(feature = "object_storage")] +use std::collections::HashMap; + +#[cfg(feature = "object_storage")] +use lightstream::error::IoError; +#[cfg(feature = "object_storage")] +use lightstream::object_storage::register_options; use pyo3::prelude::*; pub use errors::{FormatError, LightstreamError, ProtocolError, TransportError, to_py_err}; @@ -27,16 +34,27 @@ pub use message::PyMessage; pub use output::PyDataStreamWriter; pub use runtime::{block_on_py, runtime}; -/// Opens a source for reading. The scheme picks the medium, `protocol` -/// picks the wire protocol, and the format comes from `format` or the -/// path extension. Returns a `Reader` that yields `minarrow.Table` -/// batches. `accept=True` makes the reader the accepting peer on wire -/// sources, holding the endpoint bound for the life of the process and -/// blocking until a writer connects. `parallel` takes a stream count -/// on TCP sources, where the reader always accepts that many -/// connections. The quic, wt, wss, and https schemes carry TLS, -/// taking `tls_cert` and `tls_key` PEM paths on the accepting peer -/// and a `tls_ca` PEM path on the connecting peer. +/// Opens a source for reading and returns a `Reader` that yields +/// `minarrow.Table` batches. +/// +/// The URI scheme selects the medium and `protocol` selects the wire +/// protocol. The format comes from `format` or the path extension. +/// +/// ## Wire sources +/// - `accept=True` makes the reader the accepting peer. The endpoint stays +/// bound for the life of the process, and the call blocks until a writer +/// connects. +/// - `parallel` sets the stream count on TCP sources, where the reader +/// accepts that many connections. +/// - The quic, wt, wss and https schemes use TLS. The accepting peer takes +/// `tls_cert` and `tls_key` PEM paths, and the connecting peer takes a +/// `tls_ca` PEM path. +/// +/// ## Object storage +/// Builds with the `object_storage` feature accept object URLs, such as +/// `s3://bucket/key.parquet` or `memory:///key.parquet`, in place of file +/// paths. An object prefix with chunk objects immediately beneath it reads +/// as a chunk directory. #[pyfunction] #[pyo3(signature = (uri, *, format=None, protocol=None, parallel=0, accept=false, tls_cert=None, tls_key=None, tls_ca=None, mmap=None, out_of_core=false, base=None, delimiter=None, header=None, batch_size=None))] fn read( @@ -77,14 +95,24 @@ fn read( Ok(PyDataStreamReader::new(resolved)) } -/// Opens a target for writing. The scheme picks the medium, `protocol` -/// picks the wire protocol, and the format comes from `format` or the -/// path extension. Returns a `Writer` accepting any Arrow-compatible -/// object. `accept=True` makes the writer the accepting peer on wire -/// targets, holding the endpoint bound for the life of the process and -/// blocking until a reader connects. The quic, wt, wss, and https -/// schemes carry TLS, taking `tls_cert` and `tls_key` PEM paths on -/// the accepting peer and a `tls_ca` PEM path on the connecting peer. +/// Opens a target for writing and returns a `Writer` that accepts any +/// Arrow-compatible object. +/// +/// The URI scheme selects the medium and `protocol` selects the wire +/// protocol. The format comes from `format` or the path extension. +/// +/// ## Wire targets +/// - `accept=True` makes the writer the accepting peer. The endpoint stays +/// bound for the life of the process, and the call blocks until a reader +/// connects. +/// - The quic, wt, wss and https schemes use TLS. The accepting peer takes +/// `tls_cert` and `tls_key` PEM paths, and the connecting peer takes a +/// `tls_ca` PEM path. +/// +/// ## Object storage +/// Builds with the `object_storage` feature accept object URLs in place of +/// file paths. Each object becomes visible in its store when the writer +/// closes. #[pyfunction] #[pyo3(signature = (uri, *, format=None, protocol=None, parallel=0, accept=false, tls_cert=None, tls_key=None, tls_ca=None, compression=None, delimiter=None, header=None))] fn write( @@ -110,6 +138,39 @@ fn write( Ok(PyDataStreamWriter::new(resolved)) } +/// Registers object storage options for every object URL beneath `prefix`, +/// for example `s3://telemetry` or `s3://telemetry/trades`. +/// +/// Settings not registered here come from the backend's standard +/// environment variables, such as `AWS_REGION`. +/// +/// ## Options +/// Keys and values are strings. The accepted keys are: +/// - backend configuration keys, for example `aws_region`, `aws_endpoint` +/// or `google_service_account`; +/// - HTTP client keys, for example `timeout`; +/// - the upload settings `part_size` and `max_concurrent_parts`. +/// +/// ## Behaviour +/// - Registering a prefix again replaces its earlier options. +/// - Each URL uses the options of the longest registered prefix that +/// contains it. +/// +/// ## Errors +/// - Raises `LightstreamError` when `prefix` is not an object URL or an +/// upload setting is invalid. +/// - Unknown backend keys raise `LightstreamError` when a URL beneath the +/// prefix is first opened. +#[cfg(feature = "object_storage")] +#[pyfunction(name = "register_options")] +fn register_object_options(prefix: &str, options: HashMap) -> PyResult<()> { + let options: Vec<(&str, &str)> = options + .iter() + .map(|(key, value)| (key.as_str(), value.as_str())) + .collect(); + register_options(prefix, &options).map_err(|e| to_py_err(IoError::Io(e))) +} + /// Streaming Arrow I/O for Python. #[pymodule] #[pyo3(name = "lightstream")] @@ -122,6 +183,8 @@ fn lightstream_py(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(read, m)?)?; m.add_function(wrap_pyfunction!(write, m)?)?; + #[cfg(feature = "object_storage")] + m.add_function(wrap_pyfunction!(register_object_options, m)?)?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/output.rs b/python/src/output.rs index a8c0a0b..57d4439 100644 --- a/python/src/output.rs +++ b/python/src/output.rs @@ -12,8 +12,7 @@ //! over every transport, and each leaf variant wraps one lightstream end //! writer. -use std::fs::File; -use std::io; +use std::io::{self, Write}; use std::net::SocketAddr; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -27,6 +26,7 @@ use lightstream::models::writers::csv::CsvWriter; use lightstream::models::writers::ipc::sync_table::SyncTableWriter; use lightstream::models::writers::json::JsonWriter; use lightstream::models::writers::lightstream::LightstreamWriter; +use lightstream::models::writers::object_storage::StorageWriter; use lightstream::models::writers::parallel::tcp::TcpParallelTableWriter; use lightstream::models::writers::parquet::write_parquet_table; use lightstream::models::writers::http::HttpTableWriter; @@ -86,7 +86,7 @@ pub enum FileIO { Ipc { path: PathBuf, compression: Option, - writer: Option>, + writer: Option>, }, Parquet { path: PathBuf, @@ -661,6 +661,9 @@ impl FileIO { /// from the first batch's schema, and the document formats buffer it /// for `finish`. fn write(&mut self, table: &Table) -> Result<(), IoError> { + // Object URL uploads wait on the embedded runtime from this thread. + #[cfg(feature = "object_storage")] + let _guard = runtime().enter(); match self { FileIO::Ipc { path, @@ -670,7 +673,7 @@ impl FileIO { if writer.is_none() { let fields: Vec = table.cols.iter().map(|col| (*col.field).clone()).collect(); - let file = File::create(&path).map_err(IoError::Io)?; + let file = StorageWriter::create(&path).map_err(IoError::Io)?; *writer = Some(SyncTableWriter::new( file, fields, @@ -709,6 +712,9 @@ impl FileIO { /// document formats write their buffered batches as one file. A target /// that received no batches writes nothing. fn finish(&mut self) -> Result<(), IoError> { + // Object URL uploads wait on the embedded runtime from this thread. + #[cfg(feature = "object_storage")] + let _guard = runtime().enter(); match self { FileIO::Ipc { writer, .. } => match writer.as_mut() { Some(writer) => Ok(writer.finish()?), @@ -724,8 +730,10 @@ impl FileIO { } let tables: Vec = batches.drain(..).map(|t| (*t).clone()).collect(); let table = tables.consolidate(); - let file = File::create(&path).map_err(IoError::Io)?; - write_parquet_table(&table, file, *compression) + let mut file = StorageWriter::create(&path).map_err(IoError::Io)?; + write_parquet_table(&table, &mut file, *compression)?; + // Flushing completes the object when `path` is an object URL. + Ok(file.flush()?) } FileIO::CsvStdio { .. } | FileIO::JsonStdio { .. } => Ok(()), FileIO::Csv { @@ -738,7 +746,7 @@ impl FileIO { } let name = Some(batches[0].name.clone()); let super_table = SuperTable::from_batches(batches.drain(..).collect(), name); - let file = File::create(&path).map_err(IoError::Io)?; + let file = StorageWriter::create(&path).map_err(IoError::Io)?; let mut writer = CsvWriter::with_options(file, options.clone()); writer.write_supertable(&super_table)?; Ok(writer.flush()?) @@ -753,7 +761,7 @@ impl FileIO { } let name = Some(batches[0].name.clone()); let super_table = SuperTable::from_batches(batches.drain(..).collect(), name); - let file = File::create(&path).map_err(IoError::Io)?; + let file = StorageWriter::create(&path).map_err(IoError::Io)?; let mut writer = JsonWriter::new(file, options.clone()); writer.write_supertable(&super_table)?; Ok(writer.flush()?) @@ -912,7 +920,8 @@ impl PyDataStreamWriter { } /// File targets write through on each batch, so flush delegates to the - /// operating system. + /// operating system. Objects at object URLs are completed on `close`, + /// which makes them visible in their store. fn flush(&self) -> PyResult<()> { Ok(()) } diff --git a/python/src/runtime.rs b/python/src/runtime.rs index c418270..7cf7c7f 100644 --- a/python/src/runtime.rs +++ b/python/src/runtime.rs @@ -7,8 +7,10 @@ //! # Runtime //! //! The embedded Tokio runtime that drives the async readers and writers. -//! File-format paths use the synchronous readers and writers and never -//! touch it. +//! File-format reads and writes use the synchronous readers and writers. +//! With the `object_storage` feature, they enter the runtime on the +//! calling Python thread, which is outside the runtime's executor, and +//! object URL requests wait on the runtime through `Handle::block_on`. use std::sync::OnceLock; diff --git a/python/src/uri.rs b/python/src/uri.rs index 5c61192..7fa3b7d 100644 --- a/python/src/uri.rs +++ b/python/src/uri.rs @@ -11,12 +11,15 @@ //! protocol arm, and the format comes from the `format` argument or the //! path extension. +#[cfg(unix)] use std::fs; use std::io; use std::io::BufReader; use std::path::{Path, PathBuf}; use lightstream::compression::Compression; +#[cfg(feature = "object_storage")] +use lightstream::enums::BufferChunkSize; use lightstream::error::IoError; use lightstream::models::decoders::csv::CsvDecodeOptions; use lightstream::models::decoders::json::JsonDecodeOptions; @@ -33,6 +36,7 @@ use lightstream::models::readers::http::HttpTableReader; use lightstream::models::readers::parallel::tcp::TcpParallelTableReader; use lightstream::models::readers::json::JsonReader; use lightstream::models::readers::lightstream::LightstreamReader; +use lightstream::models::readers::object_storage::list_dir; use lightstream::models::readers::stdio::StdinTableReader; use lightstream::models::readers::tcp::TcpTableReader; #[cfg(unix)] @@ -41,6 +45,8 @@ use lightstream::models::readers::quic::QuicTableReader; use lightstream::enums::IPCMessageProtocol; use lightstream::models::readers::websocket::WebSocketTableReader; use lightstream::models::readers::webtransport::WebTransportTableReader; +#[cfg(feature = "object_storage")] +use lightstream::models::streams::object_store::ObjectByteStream; use lightstream::models::streams::websocket::{WsRead, WsWrite}; use lightstream::models::transports::http::HttpTransport; use lightstream::models::transports::quic::QuicTransport; @@ -50,6 +56,7 @@ use lightstream::models::transports::uds::UdsTransport; use lightstream::models::transports::webtransport::WebTransport; use lightstream::models::transports::websocket::WebSocketTransport; use lightstream::models::writers::lightstream::LightstreamWriter; +use lightstream::models::writers::object_storage::AsyncStorageWriter; use lightstream::traits::chunked_table_reader::ChunkedTableReader; use pyo3::PyResult; use pyo3::exceptions::PyValueError; @@ -132,6 +139,7 @@ fn resolve_protocol(protocol: Option<&str>) -> PyResult { /// A URI resolved to its addressing form. Wire endpoints carry the /// address their transport connects with. enum Endpoint { + /// A local file path or an object URL. Path(PathBuf), Tcp(String), Ws(String), @@ -145,8 +153,24 @@ enum Endpoint { Stdio, } +/// URL schemes that address object storage, matching the schemes of +/// lightstream's object storage backends. +const OBJECT_SCHEMES: [&str; 9] = [ + "s3", "s3a", "gs", "az", "adl", "azure", "abfs", "abfss", "memory", +]; + +/// True when `path` holds an object URL rather than a local path. +fn is_object_url(path: &Path) -> bool { + path.to_str() + .and_then(|text| text.split_once("://")) + .is_some_and(|(scheme, _)| { + OBJECT_SCHEMES.contains(&scheme.to_ascii_lowercase().as_str()) + }) +} + /// Splits a URI into its endpoint, raising for schemes whose transport -/// is not in this build. +/// is not in this build. Object URLs resolve to path endpoints, keeping +/// the whole URL. fn resolve_endpoint(uri: &str) -> PyResult { if let Some((scheme, rest)) = uri.split_once("://") { return match scheme { @@ -164,6 +188,12 @@ fn resolve_endpoint(uri: &str) -> PyResult { "wt" => Ok(Endpoint::Wt(rest.to_string())), "wss" => Ok(Endpoint::Wss(uri.to_string())), "https" => Ok(Endpoint::Https(uri.to_string())), + #[cfg(feature = "object_storage")] + _ if is_object_url(Path::new(uri)) => Ok(Endpoint::Path(PathBuf::from(uri))), + #[cfg(not(feature = "object_storage"))] + _ if is_object_url(Path::new(uri)) => Err(TransportError::new_err( + "object storage URLs require a build with the object_storage feature", + )), other => Err(TransportError::new_err(format!( "unknown scheme '{}'", other @@ -291,10 +321,10 @@ fn infer_chunk_base( base: Option<&str>, ) -> PyResult<(String, Format)> { let mut bases: Vec<(String, Format)> = Vec::new(); - for entry in fs::read_dir(dir).map_err(|e| FormatError::new_err(e.to_string()))? { - let entry = entry.map_err(|e| FormatError::new_err(e.to_string()))?; - let name = entry.file_name(); - let Some(name) = name.to_str() else { continue }; + for entry in list_dir(dir).map_err(|e| FormatError::new_err(e.to_string()))? { + let Some(name) = entry.file_name().and_then(|name| name.to_str()) else { + continue; + }; let Some((stem, ext_format)) = CHUNK_EXTENSIONS .iter() .find_map(|(ext, f)| name.strip_suffix(ext).map(|stem| (stem, *f))) @@ -430,6 +460,16 @@ pub fn resolve_source( "format arguments apply to file-format reads, not protocol streams", )); } + #[cfg(feature = "object_storage")] + if is_object_url(&path) { + let object = runtime() + .block_on(ObjectByteStream::open(uri, BufferChunkSize::FileIO)) + .map_err(|e| to_py_err(IoError::Io(e)))?; + let reader = LightstreamReader::new(object, None); + return Ok(input::Source::Stream(input::Protocol::Lightstream( + input::LightstreamIO::Stream(reader), + ))); + } let file = runtime() .block_on(TokioFile::open(&path)) .map_err(|e| to_py_err(IoError::Io(e)))?; @@ -988,7 +1028,26 @@ pub fn resolve_source( } let delimiter_byte = resolve_delimiter(delimiter)?; - if path.is_dir() { + // Object URL requests wait on the embedded runtime from this thread. + #[cfg(feature = "object_storage")] + let _guard = runtime().enter(); + let object_url = is_object_url(&path); + if object_url && mmap == Some(true) { + return Err(FormatError::new_err( + "mmap applies to local files only", + )); + } + // Object storage has no directories. An object URL addresses a chunk + // set when objects exist immediately beneath it, and a single object + // otherwise. + let chunk_set = if object_url { + !list_dir(&path) + .map_err(|e| FormatError::new_err(e.to_string()))? + .is_empty() + } else { + path.is_dir() + }; + if chunk_set { let named_format = format.map(Format::from_name).transpose()?; let (base, chunk_format) = match (base, named_format) { (Some(base), Some(format)) => (base.to_string(), format), @@ -1034,7 +1093,7 @@ pub fn resolve_source( let file_io = match resolved_format { Format::Ipc => { #[cfg(unix)] - { + if !object_url { let size = fs::metadata(&path) .map_err(|e| FormatError::new_err(e.to_string()))? .len(); @@ -1147,7 +1206,7 @@ pub fn resolve_target( )); } let file = runtime() - .block_on(TokioFile::create(&path)) + .block_on(AsyncStorageWriter::create(&path)) .map_err(|e| to_py_err(IoError::Io(e)))?; let writer = LightstreamWriter::new(Box::new(file) as _); return Ok(output::Target::Stream(output::Protocol::Lightstream( @@ -1715,7 +1774,7 @@ pub fn resolve_target( "file targets have no parallel form", )); } - if path.is_dir() { + if !is_object_url(&path) && path.is_dir() { return Err(FormatError::new_err( "chunked directory targets are not available in this build", )); diff --git a/python/tests/test_files.py b/python/tests/test_files.py index 229977d..029156b 100644 --- a/python/tests/test_files.py +++ b/python/tests/test_files.py @@ -154,6 +154,16 @@ def test_unknown_transport_scheme_raises(tmp_path): ls.write("ftp://sink:9000") +@pytest.mark.skipif( + hasattr(ls, "register_options"), reason="built with the object_storage feature" +) +def test_object_url_without_object_storage_raises(): + with pytest.raises(ls.TransportError, match="object_storage feature"): + ls.read("memory:///quotes.arrow") + with pytest.raises(ls.TransportError, match="object_storage feature"): + ls.write("s3://bucket/quotes.arrow") + + def test_unknown_protocol_raises(): with pytest.raises(ls.ProtocolError, match="unknown protocol"): ls.read("feed.arrow", protocol="flight") diff --git a/python/tests/test_object_storage.py b/python/tests/test_object_storage.py new file mode 100644 index 0000000..1584fcc --- /dev/null +++ b/python/tests/test_object_storage.py @@ -0,0 +1,204 @@ +# Copyright Peter G. Bower 2025-2026. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Object storage URLs through ls.read and ls.write. + +Runs the file-format round-trips against `memory://` URLs, which address +an in-memory store shared across the process. Each test writes beneath +its own prefix to keep its objects separate. The module requires a build +with the `object_storage` feature, which also adds `ls.register_options`. +""" + +import gc +import uuid + +import lightstream as ls +import minarrow +import pyarrow as pa +import pytest + +pytestmark = pytest.mark.skipif( + not hasattr(ls, "register_options"), + reason="built without the object_storage feature", +) + + +def sample_table(offset=0): + return pa.table( + { + "sym": ["A", "B", "C"], + "px": [1.5 + offset, 2.5 + offset, 3.5 + offset], + "qty": [10 + offset, 20 + offset, 30 + offset], + } + ) + + +@pytest.fixture +def base(): + """A `memory://` prefix unique to the test.""" + return f"memory:///tests/{uuid.uuid4().hex}" + + +def test_ipc_round_trip(base): + url = f"{base}/quotes.arrow" + with ls.write(url) as w: + w.write(sample_table()) + + result = ls.read(url).read_all() + assert isinstance(result, minarrow.Table) + assert pa.table(result).equals(sample_table()) + + +def test_ipc_multi_batch_reads_as_chunked_table(base): + url = f"{base}/quotes.arrow" + with ls.write(url, compression="zstd") as w: + w.write(sample_table(0)) + w.write(sample_table(100)) + + batches = list(ls.read(url)) + assert len(batches) == 2 + assert pa.table(batches[1]).equals(sample_table(100)) + assert isinstance(ls.read(url).read_all(), minarrow.ChunkedTable) + + +def test_out_of_core_ipc_read(base): + url = f"{base}/quotes.arrow" + with ls.write(url) as w: + w.write(sample_table()) + + result = ls.read(url, out_of_core=True).read_all() + assert pa.table(result).equals(sample_table()) + + +def test_parquet_round_trip(base): + url = f"{base}/quotes.parquet" + with ls.write(url) as w: + w.write(sample_table(0)) + w.write(sample_table(100)) + + result = pa.table(ls.read(url).read_all()) + assert result.num_rows == 6 + assert result.column("qty").to_pylist() == [10, 20, 30, 110, 120, 130] + + +def test_csv_round_trip(base): + url = f"{base}/quotes.csv" + with ls.write(url, delimiter=";") as w: + w.write(sample_table()) + + result = pa.table(ls.read(url, delimiter=";").read_all()) + assert result.to_pydict() == sample_table().to_pydict() + + +def test_csv_batch_size_splits_batches(base): + url = f"{base}/quotes.csv" + with ls.write(url) as w: + w.write(sample_table()) + + assert len(list(ls.read(url, batch_size=2))) == 2 + + +@pytest.mark.parametrize("name", ["quotes.json", "quotes.ndjson"]) +def test_json_write_creates_the_object(base, name): + url = f"{base}/{name}" + with ls.write(url) as w: + w.write(sample_table()) + + # The JSON reader needs a schema, so the read fails at decoding after + # the object opens. + with pytest.raises(ls.LightstreamError, match="schema"): + ls.read(url).read_all() + + +def test_lightstream_protocol_round_trip(base): + url = f"{base}/feed.tlv" + w = ls.write(url, protocol="lightstream") + w.register_table("quotes", sample_table()) + w.register_message("health") + w.write(sample_table(0), name="quotes") + w.write_message("health", b"\x01\x02\x03") + w.write(sample_table(100), name="quotes") + w.close() + + r = ls.read(url, protocol="lightstream") + r.register_table("quotes", sample_table()) + r.register_message("health") + frames = list(r) + assert [f.is_table() for f in frames] == [True, False, True] + assert frames[1].payload == b"\x01\x02\x03" + assert pa.table(frames[2].table).to_pydict() == sample_table(100).to_pydict() + + +@pytest.mark.parametrize("ext", ["arrow", "parquet", "csv"]) +def test_chunk_prefix_reads_in_order(base, ext): + for idx in range(2): + with ls.write(f"{base}/part-{idx:010}.{ext}") as w: + w.write(sample_table(idx * 100)) + + batches = list(ls.read(base)) + assert len(batches) == 2 + assert pa.table(batches[0]).to_pydict() == sample_table(0).to_pydict() + assert pa.table(batches[1]).to_pydict() == sample_table(100).to_pydict() + + +def test_chunk_prefix_base_disambiguation(base): + for name in ("left", "right"): + with ls.write(f"{base}/{name}-0000000000.arrow") as w: + w.write(sample_table()) + + with pytest.raises(ls.FormatError, match="multiple chunk sets"): + ls.read(base) + + assert len(list(ls.read(base, base="left"))) == 1 + + +def test_reader_exports_through_the_capsule_stream(base): + url = f"{base}/quotes.arrow" + with ls.write(url) as w: + w.write(sample_table(0)) + w.write(sample_table(100)) + + reader = pa.RecordBatchReader.from_stream(ls.read(url)) + assert reader.read_all().num_rows == 6 + + +def test_writer_drop_completes_the_object(base): + url = f"{base}/quotes.arrow" + w = ls.write(url) + w.write(sample_table()) + del w + gc.collect() + + assert pa.table(ls.read(url).read_all()).equals(sample_table()) + + +def test_missing_object_raises(base): + with pytest.raises(ls.LightstreamError): + ls.read(f"{base}/missing.arrow") + + +def test_mmap_on_an_object_url_raises(base): + with pytest.raises(ls.FormatError, match="local files only"): + ls.read(f"{base}/quotes.arrow", mmap=True) + + +def test_registered_options_apply_beneath_the_prefix(base): + ls.register_options(base, {"max_concurrent_parts": "1", "part_size": "6291456"}) + url = f"{base}/quotes.parquet" + with ls.write(url) as w: + w.write(sample_table()) + + assert pa.table(ls.read(url).read_all()).to_pydict() == sample_table().to_pydict() + + +def test_register_options_rejects_a_local_path(tmp_path): + with pytest.raises(ls.LightstreamError, match="not an object storage URL"): + ls.register_options(str(tmp_path), {}) + + +def test_register_options_rejects_an_invalid_upload_setting(base): + with pytest.raises(ls.LightstreamError, match="max_concurrent_parts"): + ls.register_options(base, {"max_concurrent_parts": "0"})