From 89382f43b7490e3d6f51982a9b6ae185c1adf4b3 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 7 Sep 2026 16:49:45 +0700 Subject: [PATCH 01/14] Require arctic 0.1.11 and ps-reclaim 0.1.4 ps-reclaim 0.1.4 closes a use-after-free. A retirement published between the participant scan and the extraction of garbage was judged against a decision taken before it existed, so a live reader's object could be reclaimed under it. The fix is a sequence cutoff captured under the garbage mutex before the scan. It also stops bounded drains being quadratic: `extract_if(..).take(k)` compacts the unchecked tail on drop, so a 128,000 backlog at `advance_up_to(256)` moved 23.29 ms of records under that mutex and now moves 1.28 ms. Both were already being picked up by resolution, because `^0.1` allows them. Raising the floors says they are required rather than merely permitted, which is what a correctness fix means. arctic 0.1.11 is the release where `smr-ps-reclaim` stopped implying `std`. It changes nothing here - WorkTable links `std` and takes ps-reclaim directly with default features, so feature unification gives it `std` either way - but the floor is what lets a no_std consumer downstream rely on it. cargo test 927 passed, 0 failed cargo clippy --all-targets clean --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e32eb50c..e18d14ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"] # pointer-only fast path. Publication is append-only and asserted at each swap. arc-swap = "1" async-trait = "0.1" -arctic = { package = "arctic-wt", version = "^0.1, >=0.1.9", default-features = false, features = ["smr-ps-reclaim"] } +arctic = { package = "arctic-wt", version = "^0.1, >=0.1.11", default-features = false, features = ["smr-ps-reclaim"] } congee = { package = "congee-wt", version = "^0.4, >=0.4.4" } convert_case = "0.6" crc32fast = "1" @@ -59,7 +59,7 @@ prettytable-rs = "0.10" psc-nanoid = { version = "3", features = ["rkyv", "packed"] } rkyv = { version = "0.8", features = ["uuid-1"] } reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] } -ps-reclaim = { version = "^0.1, >=0.1.3" } +ps-reclaim = { version = "^0.1, >=0.1.4" } rustc-hash = "2" rusty-s3 = { version = "0.10", optional = true } smart-default = "0.7" From 93b938e1ade51e6d2f107ac65c396571dace2b53 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 7 Sep 2026 16:50:25 +0700 Subject: [PATCH 02/14] Write down what a data page has to say about itself beta.19 changed the on-disk format and every `.wt.data` on this machine had to be thrown away and rebuilt on 6 September 2026, because nothing could read the old shape. That is a regeneration event, and the reason it happened is that a data page cannot be read without the index that points into it. This test states the requirement for beta.20 as an executable assertion rather than a paragraph in a release note: a page that describes itself can be read by a reader that has never seen the writer's index. --- tests/slotted_page_requirement.rs | 203 ++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 tests/slotted_page_requirement.rs diff --git a/tests/slotted_page_requirement.rs b/tests/slotted_page_requirement.rs new file mode 100644 index 00000000..32af729f --- /dev/null +++ b/tests/slotted_page_requirement.rs @@ -0,0 +1,203 @@ +//! What a data page has to say about itself, for beta.20. +//! +//! # Where this comes from +//! +//! beta.19 changed the on-disk format and every existing `.wt.data` had to be +//! thrown away and rebuilt, because nothing could read the old shape. That is a +//! regeneration event, and it happened on 6 September 2026 across every store on +//! this machine. +//! +//! It does not have to happen again. The reason it did is that a data page +//! cannot be read without the index that points into it: +//! +//! ```ignore +//! pub struct DataPage { +//! pub length: u32, +//! pub data: [u8; DATA_LENGTH], +//! } +//! ``` +//! +//! Rows are bump allocated into `data` and `length` is a high water mark. There +//! are no delimiters, so nothing can tell where one row ends and the next +//! begins. `empty_links_list` in the `SpaceInfoPage` records freed ranges and is +//! explicitly lossy: `bound_empty_links_list` truncates it when it outgrows the +//! info page and logs "space leak, not corruption". +//! +//! **The schema is already there and this is not asking for it again.** +//! `SpaceInfoPage` carries `row_schema`, `primary_key_fields` and +//! `secondary_index_types`, and `ensure_schema` refuses a mismatch by name. A +//! reader already knows how to decode a row. What it cannot do is find one. +//! +//! # The two things, in order of how much they matter +//! +//! 1. **A row directory in the data page**, the usual slotted layout: an +//! `(offset, length)` per row growing down from the end of the page, with a +//! count. Then a page describes itself, a reader needs no index, and the CRC +//! on that page validates the directory together with the rows it points at. +//! +//! 2. **A reader for the format beta.19 writes**, so beta.20 is an upgrade +//! rather than another regeneration. One already exists and is switched off: +//! `src/page/iterators.rs` in DataBucket, where `LinksIterator` walks index +//! pages for links and `DataIterator` follows them, decoding through +//! `row_schema`. It is 226 lines, commented out at `src/page/mod.rs:4`, and +//! enabling it produces nine errors that are bit rot rather than design: +//! `crate::IndexData` and `super::SpaceInfo` were renamed, and one call site +//! predates the API going async. +//! +//! # How the two fit together +//! +//! `DATA_VERSION` is 2 today and lives in every page's `GeneralHeader`, so it is +//! per page rather than per file. +//! +//! - **beta.20 ships both.** It writes 3 and reads 2 and 3. +//! - **beta.21 ships neither of the old ones.** The v2 path is deleted. +//! +//! So v2 is a one way ramp rather than dual support: a store is loaded through +//! it once, written back as v3, and never read that way again. It does not need +//! to be fast and it never needs append, which is most of why it is cheap. +//! +//! Two things to settle rather than discover: +//! +//! - Once a page has a directory and an index, both know where a row is and they +//! can disagree. One has to be authoritative. The directory is the better +//! candidate: it is local to the page and validated by the same CRC, where the +//! index is a separate structure with a different topology per backend. Under +//! `validate-reads` a load can compare the two and name a disagreement instead +//! of silently preferring one. +//! - Whether one file may hold both v2 and v3 pages. Per page versioning allows +//! it, which makes migration an append rather than a rewrite, but then no +//! reader may assume uniformity. +//! +//! # What is missing here, and is the next piece of work +//! +//! A committed `.wt.data` written by beta.19, so the ramp can be tested against +//! a real old file rather than against one this build just wrote. Until that +//! fixture exists, `a_store_reopens_without_being_rebuilt` below only proves the +//! current version reopens, which is the weaker half. + +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: SlottedRow, + version: 1, + persist: true, + columns: { + id: u64 primary_key autoincrement, + blob: String, + }, +); + +/// Enough rows to fill more than one page, so a directory would be doing real +/// work rather than describing a single row. +const ROWS: u64 = 4_000; + +async fn filled(dir: &str) -> SlottedRowWorkTable { + let _ = std::fs::remove_dir_all(dir); + std::fs::create_dir_all(dir).expect("a directory"); + + let engine = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await + .expect("an engine"); + let table = SlottedRowWorkTable::load(engine).await.expect("a table"); + + for n in 0..ROWS { + table + .insert(SlottedRowRow { + id: table.get_next_pk().into(), + blob: format!("row {n}, long enough to make the page boundaries interesting"), + }) + .await + .expect("a row"); + } + table.wait_for_ops().await.expect("the queue drains"); + table +} + +fn data_file(dir: &str) -> std::path::PathBuf { + std::path::Path::new(dir) + .join(SlottedRowWorkTable::name_snake_case()) + .join(".wt.data") +} + +/// A data page should say where its rows are, without an index. +/// +/// **This is the beta.20 requirement.** The check goes straight at the bytes on +/// purpose. Reading the page through the engine would prove only that the index +/// still works, and the index is exactly what a self describing page is supposed +/// to make unnecessary. +/// +/// Written against bytes rather than against an API that does not exist yet, so +/// this file compiles today and fails on the missing behaviour rather than on a +/// missing symbol. +#[tokio::test] +#[ignore = "beta.20: a data page carries no row directory"] +async fn a_data_page_says_where_its_rows_are() { + let dir = "tests/data/slotted_page/self_describing"; + let table = filled(dir).await; + table.close().await.expect("the table closes"); + + let bytes = std::fs::read(data_file(dir)).expect("the file"); + assert!( + bytes.len() > PAGE_SIZE, + "the fixture has to span pages: {} bytes", + bytes.len() + ); + + // A slotted page keeps its directory at the end: a row count in the last + // four bytes, then that many (offset, length) pairs growing back up. Any + // layout would do; what matters is that something in the page delimits the + // rows. Today the tail is write padding, so this reads zero. + let mut described = 0usize; + for page in bytes.chunks_exact(PAGE_SIZE).skip(1) { + let mut tail = [0u8; 4]; + tail.copy_from_slice(&page[PAGE_SIZE - 4..]); + described += u32::from_le_bytes(tail) as usize; + } + + assert_eq!( + described, ROWS as usize, + "no page says how many rows it holds, so the {ROWS} rows in this file \ + cannot be found without the index. A row directory in the data page is \ + what makes a page readable on its own, and what makes the next format \ + change an upgrade instead of a regeneration." + ); + let _ = std::fs::remove_dir_all(dir); +} + +/// A store reopens without being deleted first. +/// +/// **Not ignored, and passing.** It guards the property at the current version, +/// so a format change that breaks reopening trips here rather than in somebody's +/// deploy. It is the weaker half of the requirement: proving beta.20 can read +/// beta.19 needs a beta.19 file committed as a fixture, which does not exist +/// yet. +#[tokio::test] +async fn a_store_reopens_without_being_rebuilt() { + let dir = "tests/data/slotted_page/reopen"; + let table = filled(dir).await; + table.close().await.expect("the table closes"); + + let engine = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await + .expect("an engine"); + let reopened = SlottedRowWorkTable::load(engine) + .await + .expect("a store reopens rather than needing to be rebuilt"); + + assert_eq!( + reopened.select_all().execute().expect("a read").len(), + ROWS as usize, + "no rows are lost reopening a store" + ); + reopened.close().await.expect("the table closes"); + let _ = std::fs::remove_dir_all(dir); +} From 5dcbf21b195dfcda7a9cee05e58b18ea00e1e2ac Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 7 Sep 2026 16:50:25 +0700 Subject: [PATCH 03/14] Map paper two onto evidence that already exists The CIDR submission deferred the lock-discipline scaling comparison, crash consistency, protocol checking, the cost of monomorphization, and baselines beyond redb and LMDB. This plans the paper those deferrals point at, and pins each candidate contribution to the code and the measurements that back it, so the writing starts from what has landed rather than from an outline. --- docs/paper-2-plan.md | 115 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docs/paper-2-plan.md diff --git a/docs/paper-2-plan.md b/docs/paper-2-plan.md new file mode 100644 index 00000000..474414be --- /dev/null +++ b/docs/paper-2-plan.md @@ -0,0 +1,115 @@ +# Paper two: plan and evidence map + +Written 2026-09-06 against master `d5656e6` (1.0.0-beta.19). Companion to the CIDR 2027 +submission (beta.6, submitted for the 2026-08-04 deadline; notification 2026-10-06). + +## Thesis candidates + +The CIDR paper argued *compile-time engine specialization*. It deferred (§5, §7): the lock +discipline scaling comparison, crash consistency, formal checking of the protocols, the cost +of monomorphization, and external baselines beyond redb/LMDB. Paper two should be the paper +those deferrals point at, not a re-statement of the thesis. + +**A. Lifecycle paper (recommended).** "Row lifecycle in a specialized engine: exact-cell +locking, epoch reclamation, and reactive vacuum without a transaction manager." Everything +in it has landed since beta.6 and has numbers. Contributions: + +1. Exact-cell synchronization replacing hashed stripes and the table-global barrier + (`src/in_memory/data.rs` `CellLocks`; `docs/versioned-row-publication.md`). +2. Quiescent-state reclamation via a one-word `!Send` guard (`ps-reclaim`), replacing a + global reader counter; the `Send`-guard use-after-free found on the way is a good + cautionary section (`docs/TODO.md` "ps-reclaim 0.1.1"). +3. Reactive vacuum planned from the free-range registry, gated by a mutation lease and a + quiet epoch; root-cause note that all four reclaim bugs came from indexes storing + physical addresses (`docs/vacuum-design-directions.md`, `src/table/vacuum/`). +4. The 1-32 thread grid across three index backends: the lock-discipline scaling result the + CIDR paper promised (`docs/beta18-validation.md`). +5. `partition_by` with the loom model of the slot protocol (`src/partition/loom_tests.rs`), + the first model-checked component; HFT review that produced `partition_ref` + (`wt-review.md`). + +**B. Persistence paper.** "Index persistence by replaying the tree's own CDC stream, and what +it costs." Needs the durability work first: structural CDC is ~28% of a 126 ns insert +(`docs/wti-dirty-generation-persistence-plan.md`); ART logical WAL exists +(`src/persistence/space/art_index.rs`) but data pages and WTI still end at `flush()`; the +watermark/fsync design is proposal only (`docs/durability-visibility-proposal.md`). Not +ready before a Q1 2027 deadline unless the journal lands. + +**C. Schema-as-IR paper (PL venue).** `worktable_dsl`: schema read as data, diff with a +per-change cost model, declarations baked into generated code, `wt-dsl` CLI, migration +engine with on-disk version detection (`dsl/`, `docs/migration.md`). Fits PEPM / OOPSLA +better than a DB venue; overlaps with the PEPM plan in the other session. + +## Evidence already in hand (all M4 Max, local trees; needs a pinned Linux rerun) + +| Claim | Number | Source | +|---|---|---| +| Hot-page writer, exact-cell vs hashed | 322 ns vs 1,536 ns | beta18-validation | +| Per-page vs table-global barrier | +25% @4, +65% @8 disjoint writers | versioned-row-publication | +| Generation retire, beta.18 vs 15 | 33.6x (WTI), 16.1x (Arctic) | beta18-validation | +| Read scaling best/1T at 16 threads | 3.6x / 3.3x / 2.6x per backend | beta18-validation | +| 10%-write mix ceiling | peaks at 4 threads | beta18-validation (also beta.13/15) | +| Reactive vacuum foreground penalty | -1.1..-5.1% vs 16-41% unpaced | beta17/18-validation | +| Vacuum reclamation | 18/18 cells, 196 pages, 100% | beta17-validation | +| Point lookup vs stripped index | 15.65 ns vs 9.25 ns bare Arctic vs 33.5 Vec+BTreeMap | beta18-validation | +| Memory overhead | 0.14 B/row over control | beta18-validation | +| Partition route | 0.73 ns Vec vs 9.5 ns string hash; `partition_ref` 3.35 ns | TODO.md, partition docs | +| Persisted insert, beta.15 to 18 | -33..-42% (Arctic 6,130 to 3,753 ns/row) | beta18-validation | +| CDC share of insert | ~35 ns of 126 ns | wti-dirty-generation plan | + +Not in hand: `paper-bench/results/` (never committed), `compile_cost.sh` never run, no +sled/SQLite/DashMap baselines, no beta.19 rerun of Table 2. + +## Consumer workloads (surveyed 2026-09-06, all under ~/code) + +| Repo | Shape | What it gives the paper | Open? | +|---|---|---|---| +| `agentcode` | 11 tables, 8 persisted, Arctic on nearly every index, u128 keys. Stress = 8 concurrent 800-file `update_latency` processes. | The concurrency bug that motivates the paper: `docs/known-defects.md:70-137`, torn/corrupt page header in secondary-index batch apply at beta.11, 6/8 runs failing, 28/28 after moving to Arctic. Map it to the beta.18 fix ("torn reads and premature physical-link reuse") and show the same harness clean on beta.19. Also: Arctic vs WTI at 20k rows, insert 2.18M/s vs 0.89M/s, lookup 17.0M/s vs 5.0M/s (`docs/benchmarks/index-backends.md`); state 42.4 to 22.2 MB after u128 keys (`state-growth.md`); the request for a non-unique fixed-width index that became `ArcticMultiIndex`. | Proprietary | +| `agencyzero` | 19 persisted tables, String PKs, migration engine, `LoadMode::Recovery`, single-writer flock, QA fixture of 248 projects (~30 MB store). | The reclamation case study: `docs/store-recovery.md` records four production corruptions, including a variable-width index page that forgot fragmentation across restart (174 live entries, 2,664 B dead, 64 B tail overlap) and beta.5 whole-row rebuilds churning `pr_project_idx` (38 disagreeing rows) fixed by in-place updates. Production migrations via schema fingerprint. This is the "why indexes must not store physical addresses" story with real data. | Private (GitHub) | +| `karen` | 2 persisted tables, tiny (~700 rows). | Row-level, queryable, durable learned session state instead of one blob: the per-turn Confirm write-through gives 15-20 points top-1. One paragraph of motivation, not evaluation. Uses `unload_gracefully`. | Closed | +| `ekopathrs` | No `worktable!` at all. Uses `worktable-vec::AtomicKeyTable` for two in-memory profiling tables. | The honest negative: `docs/STORAGE-REVIEW.md` rejects full WorkTable (318-package resolve, no `no_std`) for 399 entries. Cite as the boundary of the design space; `worktable-vec` is the lock-free, `no_std` sibling. | Private | + +Use agentcode as the headline stress workload in §4 alongside the shadow-state harness; use +agencyzero as the recovery/fragmentation case study; mention karen and ekopathrs in one +paragraph each in the experience section. Get written OK before naming private repos. + +## Gaps to close for option A + +- Rerun the beta.18 grid and `paper-bench` on a quiet pinned x86 box; commit `results/`. +- Lock-discipline ablation as a proper figure: field vs row vs table lock, 1-32 threads, + skewed keys (`paper-bench/src/bin/contention`). +- Semi-formal statement of the cell-lock + reclamation invariants; ideally extend loom + beyond partitions to the cell/retire path (the CIDR reviewers will ask). +- Wart sweep: 9 `todo!()` sites remain (`codegen/.../queries/in_place.rs`, `update.rs`, + `src/features/s3_support.rs`), `Avaiable` typo in 2 files. +- Merge or explicitly exclude `feat/columnar-fields-indexes` (branch, Aug 6, unmerged). + +## Target: EDBT 2027, 3rd cycle (verified 2026-09-06) + +- Submission **2026-10-07, 5pm PST** (31 days out). Author feedback 11-19, notification + Acc/Rej/Revise 12-05, revised paper 2027-01-04, final 01-27, camera-ready 02-10. + Conference Lille, April 6-9, 2027. +- Paper types: Research long (12p) or short (6p, title prefixed "[Short Paper]"), + Experiments & Analysis, Vision (6p). Topics list includes "Concurrency control, recovery, + and transaction management", "Storage, indexing, and physical database design", + "Data management on modern hardware", "Benchmarking and performance evaluation". +- The revise cycle matters: a paper that gets "revise" on Dec 5 has until Jan 4 to add + the pinned-Linux rerun, so the Oct 7 draft can ship on the M4 grid with the caveat stated. +- CIDR notification is Oct 6, one day before: paper two cannot depend on the outcome and + must not overlap the CIDR text (still under review until then). Option A is disjoint by + construction; cite the CIDR paper as "under submission". + +Alternatives if A slips: ICDE 2027 R2 (2026-11-11), PVLDB rolling (monthly to 2027-03-01), +SIGMOD R4 (2026-10-17). DaMoN 2027 CFP not posted. + +## 31-day schedule for option A (long paper) + +| Week | Dates | Deliverable | +|---|---|---| +| 1 | Sep 7-13 | Freeze the claim list. Run `paper-bench` contention (field/row/table/inplace, 1-32 tasks) and beta.18 grid on the pinned Linux box if available, else M4 with three rotated passes; commit `results/`. Decide long vs short by Sep 13 based on whether the scaling figure holds. | +| 2 | Sep 14-20 | Draft §2 protocols (cell lock, retire, vacuum lease) with invariants stated; §3 partition + loom. Wart sweep PR (`todo!()`, `Avaiable`). | +| 3 | Sep 21-27 | Draft §4 evaluation from `results/`; figures; related work (Hekaton, epoch/QSBR: Fraser, Hart et al., Bw-tree, OLC, DaMoN vacuum/compaction lineage). | +| 4 | Sep 28-Oct 4 | Full read-through, internal review, page trim to 12. | +| 5 | Oct 5-7 | Buffer. Submit by Oct 6 evening local time (Oct 7 5pm PST is 07:00 Oct 8 in Bangkok, but do not use it). | + +Short-paper fallback (6p): contributions 1-3 only, one scaling figure, one vacuum figure. From 1cd8301be257ab38124e37939fabff62f4128579 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 7 Sep 2026 17:12:45 +0700 Subject: [PATCH 04/14] Cut the page run with as_chunks, which is what CI's clippy asks for `chunks_exact` with a constant chunk size is a lint on a newer clippy than the one installed here, so the local run was clean and CI was not. `as_chunks` also gives fixed-size arrays rather than slices, which is what the loop wanted. --- tests/slotted_page_requirement.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/slotted_page_requirement.rs b/tests/slotted_page_requirement.rs index 32af729f..8b21460d 100644 --- a/tests/slotted_page_requirement.rs +++ b/tests/slotted_page_requirement.rs @@ -153,7 +153,8 @@ async fn a_data_page_says_where_its_rows_are() { // layout would do; what matters is that something in the page delimits the // rows. Today the tail is write padding, so this reads zero. let mut described = 0usize; - for page in bytes.chunks_exact(PAGE_SIZE).skip(1) { + let (pages, _) = bytes.as_chunks::(); + for page in pages.iter().skip(1) { let mut tail = [0u8; 4]; tail.copy_from_slice(&page[PAGE_SIZE - 4..]); described += u32::from_le_bytes(tail) as usize; From 440c452b6ebeb9a45d1fd2299d84b1a37d18c475 Mon Sep 17 00:00:00 2001 From: meh Date: Tue, 8 Sep 2026 02:22:22 +0700 Subject: [PATCH 05/14] Run the formatting check CI was missing `scripts/ci-local.sh` runs `cargo fmt --all --check` as its first step and CI did not, which made the local script stricter than CI rather than equal to it, so formatting drift could reach master unnoticed. Same command, same arguments. Measured before adding it: the tree already passes, exit 0 with no diffs, so this job is green as written rather than a red first run to clean up. --- .github/workflows/rust.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ecb2281e..695ace57 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -13,6 +13,18 @@ permissions: contents: read jobs: + # `scripts/ci-local.sh` runs `cargo fmt --all --check` as its first step. CI + # did not, which made the script stricter than CI instead of equal to it, and + # formatting drift reached master unnoticed. Same command, same arguments. + fmt: + name: Formatting + runs-on: ubicloud-standard-2 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - name: Formatting (cargo fmt --all --check) + run: cargo fmt --all --check + build: name: Build and test (${{ matrix.name }}) runs-on: ubicloud-standard-2 From 89b58609a9ed04e48a37aa733b13f17431186bc3 Mon Sep 17 00:00:00 2001 From: meh Date: Tue, 8 Sep 2026 02:22:22 +0700 Subject: [PATCH 06/14] Backfill the change log to 0.3.10 The log carried beta.18 and beta.19 and nothing before them, so the history a consumer needs in order to judge an upgrade was only in the commits. Entries are derived from this repository's own history. --- CHANGELOG.md | 691 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 691 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b37f3eed..98d36fda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,697 @@ Change Log - Persisted primary/secondary index reconstruction and validation failures that could otherwise expose missing, duplicate, or mismatched rows. +## [1.0.0-beta.17] + +### Added + +- `worktable_dsl`, a standalone crate holding the schema language. A schema can + now be read as data and written back, two schemas can be compared and the + cost of the difference reported, and declarations can be found across a + source tree. +- Every generated table embeds its own declaration, so the schema is + recoverable from the code the macro produced. + +### Changed + +- Dependency requirements on the index and reclamation crates are carets rather + than exact pins, and `ps-reclaim` moved to 0.1.1 taken from the registry. +- Retirement runs through the reclamation domain rather than through the guard. + +## [1.0.0-beta.16] + +### Changed + +- A batch pins its reclamation domain once instead of once per row, and + reclamation goes through `ps-reclaim`. +- Requires data_bucket 0.5.5. + +## [1.0.0-beta.15] + +### BC Breaks + +- Non-unique index entries are identified and ordered by their `(key, value)` + pair, and the discriminator is gone. This is a persisted format change. An + index file written by beta.14 or earlier orders entries within a key by + discriminator, so it must be reindexed rather than loaded. + +### Fixed + +- Inserting into a non-unique index no longer scans every entry sharing the + key. On a table that puts a whole generation under one key, a one-file update + measured 698 ms on beta.13 and 15.1 s on beta.14; the per-row cost is back + from 330 us to roughly 9 us. +- Index pages reconstruct in order of their minimum rather than their node id, + so a page that merely ends late no longer sorts ahead of one that starts + earlier. + +## [1.0.0-beta.14] + +### Added + +- `insert_many` with all-or-nothing semantics and CDC batch operations, and + `reserve_pks` for atomic primary key range reservation, both generated on + in-memory and persisted tables. +- Per-table epoch pin domains. The global reader counter is replaced by + epoch-based retirement reclamation, and removed partitions are reclaimed + through the shared router under the same grace period. +- Non-unique Arctic indexes for fixed-width integer keys, generated for + in-memory and persisted tables, with a pair-list checkpoint and WAL. + +### Changed + +- The persistence queue takes batches on a single wakeup, deduplicates page + queries when collecting a multi-row batch, and caps rows per group id so the + analyzer drain stays linear. +- The table-global page barrier is narrowed to one barrier per page. +- Requires WorkTablesIndex 0.0.8 and data_bucket 0.5.4. + +### Fixed + +- Mutation stripes are acquired as a batch without deadlocking. +- A unique-collision unwind on a persisted table survives reload. + +## [1.0.0-beta.13] + +The audit-fix release. Most of it is durability and concurrency correctness +rather than new surface. + +### BC Breaks + +- Persisted tables reject any `page_size` other than 16384 instead of writing a + file that cannot be read back. +- A torn table-of-contents page 1 fails loudly instead of silently starting + from an empty table. +- In-place update is rejected on indexed columns rather than leaving the index + stale. +- An exhausted autoincrement generator panics instead of wrapping around and + handing out keys that are already in use. + +### Fixed + +- A failed data write no longer leaves published index keys behind. Insert, + update and delete each roll their index changes back. +- Index pages are written before the table of contents that references them, + and table-of-contents key updates are guarded against segment overflow. +- Data-file accounting: the u32 page-offset wrap when writing the last page's + data length, files whose length is an exact page multiple failing to reopen, + and non-extending writes being counted into the last page's length. +- Vacuum no longer panics on a failed row move, no longer counts its scratch + pages in `pages_freed`, and never reports a source page fully moved when a + row was skipped. +- A cancelled lock wait releases its registered op-lock, and a row that + vanishes mid-update returns `NotFound` instead of panicking. +- The persistence worker refuses new operations once `Drop` has aborted it, + propagates `insert_cdc` serialization failure instead of panicking, and keeps + surviving data-only writes when event removal empties a batch. +- ART checkpoints are atomic and clean up stale temporaries. +- Arctic returns an empty range for `Excluded` bounds with no neighbour. +- Row counts include inserts and deletes that reused a slot, and the + `PageIsFull` page switch is serialized against racing inserters. +- A misplaced `persist` or `partition_by` in a declaration now names the + position it belongs in. + +## [1.0.0-beta.12] + +### Added + +- `partition_by`: one declared table type, many routed instances, with + `partition_ref` for borrowing a partition rather than cloning it. + +### Changed + +- `system_info` no longer copies every data page, and partition metrics scan + and allocate once instead of three times. + +### Fixed + +- A use-after-free in partition removal. +- `close` reported success having persisted nothing. +- One panic inside the router no longer disables the router. + +## [1.0.0-beta.11] + +### Changed + +- Requires the reviewed ART backend releases. + +## [1.0.0-beta.10] + +### Fixed + +- Table-of-contents inserts carry across persisted segments, reload insertion + stays on the fast path, and the insert API keeps its previous shape. + +## [1.0.0-beta.9] + +### Fixed + +- Persistence health is preserved across page splits. + +## [1.0.0-beta.8] + +### Fixed + +- Multi-row persistence order is preserved, and overlapping durable row writes + are ordered against each other. + +## [1.0.0-beta.7] + +### Fixed + +- The sized indexed update path is preserved. + +## [1.0.0-beta.6] + +### Changed + +- Fixed-width updates stay in place. + +### Fixed + +- Vacuum revalidates links after row locking. + +## [1.0.0-beta.5] + +### Added + +- Checked offline recovery load. + +### Changed + +- WorkTablesIndex structural persistence moved off the mutation path. +- Logical WTI mutation stripes hash with FxHash, reusable data ranges are + subtracted in one pass, and full-row updates generate distinct paths. + +### Fixed + +- Same-size unsized updates apply in place instead of going through reinsert. +- Full-table scans re-resolve stale links. +- Vacuumed pages are reusable after reload. +- Cancelled lock acquirers are cleaned up. +- A panic while loading a persisted table is contained instead of unwinding + into the caller. + +## [1.0.0-beta.4] + +### Fixed + +- Release hardening and torn-store refusal are consolidated, so a torn store + refuses cleanly. + +## [1.0.0-beta.3] + +### Changed + +- Depends on the published index dependency chain rather than git revisions. +- Row publication is concurrency-safe by default. +- Persistence failures are terminal instead of leaving the table in a state + that looks usable. + +### Fixed + +- Synchronous insert is serialized against row mutations. +- Page and link reclamation no longer overlap, and vacuum page reuse is + deferred through the read grace period. +- Upsert retry backoff is bounded and its shift is capped, so same-key churn + cannot livelock. +- Fragmented unsized index pages are compacted. +- A stale multimap removal lookup is avoided. + +## [1.0.0-beta.2] + +### Added + +- Native ART index backends persist. + +### Changed + +- The temporary rusty-s3 fork is retired in favour of the published crate. +- Stable index reads use the specialized path by default. + +### Fixed + +- Same-key upserts linearize. +- Bounded retry for transient index misses is gated rather than always on. +- Reused persistence slots coalesce. + +## [1.0.0-beta.1] + +### Added + +- Per-index backend selection in the `worktable!` declaration, with unique-index + adapters for Arctic, Congee and a parallel upstream indexset. Persistence is + preserved across indexset providers. + +## [0.9.4] + +### Changed + +- Requires data_bucket 0.4.1, and the temporary git patch is retired. + +### Fixed + +- A torn store refuses cleanly instead of terminating the process by signal. + +## [0.9.3] + +### Fixed + +- `worktable_version!` stays read-only when the primary key is unsized. + +## [0.9.2] + +### Fixed + +- Duplicate-key secondary indexes reconstruct correctly on reload. +- Nodes sharing a maximum key order correctly, and pages are no longer re-sorted + on reload. +- Space files flush before an operation reports done. + +## [0.9.1] + +### Changed + +- The proc-macro crate is published as `worktable_codegen` again, after a brief + release under the name `worktable_macros`. + +## [0.9.0] + +### Changed + +- Moves to WorkTablesIndex 0.0.1 and data_bucket 0.4.0. +- The unsound lock-free persistence queue is replaced with a mutexed + `VecDeque`. + +### Fixed + +- Row lock acquisition and vacuum no longer race between check and act. +- `wait_for_ops` no longer returns while a popped operation is still in flight. +- Upsert retries an existence flip instead of surfacing it to the caller. +- A multi-row update locks one validated snapshot, predicate included, and + delete by non-unique index snapshots validated primary keys. +- Gapped event streams are never force-applied to the on-disk index, and the + whole batch is scanned for event-id gaps rather than the last thirty events. +- A failed batch sub-operation is reported without cancelling the rest of the + work. +- `save_batch_data` tracks the real maximum created page id. +- Vacuum persists row moves through CDC, so persisted tables survive + defragmentation. + +## [0.9.0-beta0.2.3] + +### Fixed + +- Primary key generator state is preserved across migration reinserts. + +## [0.9.0-beta0.2.2] + +### Changed + +- Range ordering query logic reworked. + +## [0.9.0-beta0.2.1] + +### Changed + +- Update locks spin before returning a `Pending` state. + +## [0.9.0-beta0.2.0] + +### Added + +- Migrations. + +## [0.9.0-beta0.1.4] + +### Fixed + +- Page-not-found bug in the table of contents. + +## [0.9.0-beta0.1.1] + +### Fixed + +- Persistence bug affecting operations that fail. + +## [0.9.0-alpha8] + +### Changed + +- S3 integration moves to a different client crate. + +## [0.9.0-alpha7] + +### Fixed + +- S3 integration bug. + +## [0.9.0-alpha6] + +### Changed + +- Moves to rustls. + +## [0.9.0-alpha5] + +### Added + +- nanoid support for primary keys. + +## [0.9.0-alpha4] + +### Fixed + +- Vacuum logic. + +## [0.9.0-alpha3] + +### Fixed + +- The S3 macro. + +## [0.9.0-alpha2] + +### Added + +- S3 sync feature. + +## [0.9.0-alpha1] + +### Changed + +- Persistence is moved behind separate traits. + +## [0.8.23] + +### Changed + +- `Lock`s are reworked around RAII guards. + +## [0.8.22] + +### Added + +- `MemStat` derive on the generated primary key type. + +### Changed + +- `DataPages` select is generic over the input link type. + +## [0.8.21] + +### Changed + +- `delete` is generic, matching `insert` and `update`. + +## [0.8.20] + +### Added + +- Vacuum. + +## [0.8.19] + +### Fixed + +- Optional fields in persisted tables. + +## [0.8.18] + +### Fixed + +- Persisted table code failed to compile when the declaration used `optional` + fields. + +## [0.8.17] + +### Changed + +- Updated `indexset`. + +## [0.8.16] + +### Changed + +- Dependencies are pinned to exact versions. + +## [0.8.15] + +### Fixed + +- Empty link registry. + +## [0.8.13] + +### Changed + +- Bumped `indexset`. + +## [0.8.12] + +### Changed + +- Bumped `data_bucket` to 0.3.5 and `wt-indexset` to 0.12.11, and the crate now + declares its repository. + +## [0.8.11] + +### Changed + +- Bumped `indexset`. + +## [0.8.10] + +### Changed + +- Bumped `data_bucket` to 0.3.3 and `wt-indexset` to 0.12.9. + +## [0.8.9] + +### Fixed + +- Empty node bug. + +## [0.8.8] + +### Added + +- Every `AtomicU*` and `AtomicI*` type is usable as a primary key. + +## [0.8.7] + +### Changed + +- Dependency bumps. + +## [0.8.6] + +### Fixed + +- An `update`-related bug. + +## [0.8.5] + +### Fixed + +- Another `update`-related bug. + +## [0.8.4] + +### Changed + +- Codegen version bump. + +## [0.8.3] + +### Fixed + +- `delete` queries on a table whose primary key is not named `id`. +- An update bug, by way of an `indexset` update. + +## [0.8.1] + +### Added + +- The macro reports an error when an index names a column that does not exist, + and declaration errors are raised as `syn::Error`s with usable messages. + +### Fixed + +- `UnsizedNode` split. + +## [0.8.0] + +### Fixed + +- Unsized node bug. + +## [0.7.2] + +### Fixed + +- A further `update` bug. + +## [0.7.1] + +### Fixed + +- An update violation. + +## [0.7.0] + +### Fixed + +- Reinsert bug. + +## [0.6.14] + +### Added + +- Ghost inserts. A row is staged invisible and becomes visible only once its + index entries are in place, so a concurrent reader never observes a + half-inserted row. + +## [0.6.13] + +### Fixed + +- Concurrency bugs in `select`. + +## [0.6.12] + +### Fixed + +- A further locking bug. + +## [0.6.11] + +### Fixed + +- Locking bugs for unsized types, and an `UnsizedNode` bug on `update`. + +### Changed + +- Dependency bumps. + +## [0.6.10] + +### Changed + +- Republished against `worktable_codegen` 0.6.9. No library change. + +## [0.6.9] + +### Fixed + +- Concurrent persistence issues. + +## [0.6.8] + +### Fixed + +- `wait_for_ops` logic. + +## [0.6.7] + +### Fixed + +- `delete` on persisted tables. + +## [0.6.5] + +### Added + +- Custom derives can be attached to the generated row type. + +## [0.6.4] + +### Fixed + +- `uuid` usage. + +## [0.6.3] + +### Fixed + +- A debug `println!` on the persistence batch path no longer writes to stdout. + +## [0.6.2] + +### Fixed + +- Table-of-contents corrections. + +## [0.6.1] + +### Added + +- `update_in_place`. + +### Changed + +- The persistence queue is optimized. + +### Fixed + +- `insert` with an already-existing key. +- A `use rkyv::Archive` import was required for some declarations. +- `wait_for_ops`. + +## [0.5.6] + +### Changed + +- Updated `indexset`. + +## [0.5.5] + +### Fixed + +- Array-typed fields. + +### Changed + +- Moves to the newer Rust edition. + +## [0.5.4] + +### Added + +- Unsized index space, so index keys are no longer limited to fixed-width + types. +- `SystemInfo` for the table and its indexes. +- `where_by` on `SelectBuilder` for any column, indexed or not. +- Float columns are usable in indexes, including ranges. + +### Fixed + +- Re-reading a table from file. +- Index difference logic for `update` queries. + +## [0.5.1] + +### Changed + +- Persistence I/O is asynchronous. + +## [0.5.0] + +### Added + +- `select_where_{field}` queries for selecting data ranges. +- `count` on the table. +- Persist sync logic. + +### Changed + +- Non-unique indexes are backed by `IndexMultiMap`. + +### Fixed + +- Secondary index left inconsistent after an update. +- Diff logic for a full-row update. + ## [0.4.1] ### Added From cce51f653e0758c3a76366e90b1fb848fb96bbec Mon Sep 17 00:00:00 2001 From: meh Date: Tue, 8 Sep 2026 02:22:22 +0700 Subject: [PATCH 07/14] Name the cause of a persistence event gap, not only its symptom `BatchOperation::validate` refuses an event stream with a hole in it. A hole is normally transient, and one that survives the whole deferral budget is not, but the stall message could not tell the two apart: it reported the range and nothing else. Two very different bugs produce that line. Either an id was assigned by the index and its event was dropped instead of queued, in which case nothing will ever deliver it, or the operation was queued and batch collection keeps assembling batches that exclude it. Distinguishing them needs a record of what was queued. Assignment happens inside the index and cannot be hooked from here, but every event reaching persistence passes through `Queue`, so an id present in the stream and absent from the ledger was assigned and never queued. That is the leak signature. Producers are named by `std::panic::Location`, which is why `push_at` takes the caller location and why the old `push` and `push_many` wrappers are deleted rather than kept: calling them loses exactly the location the ledger exists to record. This observes and does not fix. `docs/TODO.md` records the three leak sites the instrument points at, all in generated code: the `NotFound` arm of the update query returns without building an acknowledge while its `AlreadyExists` sibling builds one, and two `res?;` sites in update and delete propagate before acknowledging. The TODO's congee section is corrected while here. The crossbeam-epoch port was already done in congee-wt 0.4.4, and it was a design change rather than a rename: per-tree `Domain`, batched retires, guard provenance checked, and a new `Guard` that is `!Send` and tree-scoped, which constrains future callers. --- docs/TODO.md | 123 +++-- src/persistence/event_ledger.rs | 735 +++++++++++++++++++++++++++++ src/persistence/mod.rs | 2 + src/persistence/operation/batch.rs | 69 ++- src/persistence/task.rs | 194 +++++++- 5 files changed, 1061 insertions(+), 62 deletions(-) create mode 100644 src/persistence/event_ledger.rs diff --git a/docs/TODO.md b/docs/TODO.md index 15824525..8b6f79fb 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -3,7 +3,9 @@ What is known to be unfinished, and enough context to act on it without the conversation it came from. Ordered by whether it blocks a release. -Last reviewed 2026-09-04, against `master` after beta.17 publication. +Last reviewed 2026-09-07. Sections below still describe the repository as of +beta.17; `master` is now at 1.0.0-beta.19 and this file has not been swept for +what those two releases closed. ## Closed, and how @@ -111,43 +113,88 @@ yank beta.16 once beta.17 supersedes it. ## Not blocking, but wrong today -### `congee-wt` still pulls `crossbeam-epoch` - -beta.16 removes crossbeam from WorkTable's own reclamation, not from the build. -`congee-wt` depends on it directly and re-exports its `Guard`, which -`src/index/congee.rs:101` names in a signature; `crossbeam-skiplist` also -arrives under `WorkTablesIndex` and `indexset`. - -congee's use is shallow: 34 references, none of them `Atomic<>`, `Owned::` or -`Shared<>`, and most in tests. It only ever calls `pin()` and passes `Guard` -around as an opaque token, so porting it to `ps-reclaim` is mechanical. The -catch is that `Guard` is in congee-wt's public API, so it is a breaking change -there plus the call sites here. - -`arctic-wt` should **not** be ported. It reclaims through `seize`, and is right -to: a trie with short reads reaches quiescence constantly, which is the exact -property that makes `seize` wrong for this crate, where `select` holds a read -guard. - -### Persistence stalls on a primary index event gap, rarely - -One run of `cargo test --workspace --all-targets --all-features` failed with - - persistence stalled on primary index event gap: last applied Id(1439), - next available Id(1455) (attempt 9) - -in `tests/persistence/loaded_index_growth.rs`. Not a flaky timeout: the guard -at `src/persistence/operation/batch.rs:346` is deliberate, added in `c0c06ba`, -and its comment says a gap that persists past eight deferrals means an event id -was consumed without its event being queued, which only non-CDC index mutations -do. The gap is 16 ids wide. - -1 failure in 6 full runs on this branch, 0 in 3 on master, 0 in 15 -persistence-only runs, so it needs whole-suite load and is not a beta.16 -regression. Do not start with a repro hunt: instrument `IndexChangeEventId` -assignment against event queueing so the next occurrence names its own cause. -Evidence at `~/code/wt-event-gap-2026-09-01.txt`. - +### `congee-wt` no longer pulls `crossbeam-epoch` + +Corrected 2026-09-07. This section said the port was outstanding and mechanical. +Both halves were wrong. + +`congee-wt` dropped `crossbeam-epoch` in 0.4.4. `Cargo.toml:22` now reads +`ps-reclaim = { version = "0.1.4", default-features = false, features = +["libc", "spin"] }`, no `crossbeam_epoch` reference survives in its sources or +tests, and this repository's `Cargo.lock` already resolves `congee-wt 0.4.4`. +The two remaining `crossbeam` strings in that crate are attribution comments on +a seqlock and a backoff loop. + +The call site this section worried about needed no change. It has moved to +`src/index/congee.rs:120` and still reads +`fn retire_old(pointer: usize, guard: &congee::epoch::Guard) -> Arc`. The +`congee::epoch::Guard` path was deliberately preserved across the port, and the +lifetime the new guard carries elides in reference position. + +The port was not the mechanical rename described here. A literal swap would have +kept one global epoch; what shipped gives each tree its own `Domain`, adds a +bounded pending-retire batch, checks guard provenance so a guard from another +tree panics rather than corrupting, and drains the tree's own domain on `Drop`. +Worth knowing because the new `Guard` is `!Send` and tree-scoped, so a guard may +not be created outside the thread and tree that uses it. Nothing in either +repository does; every threaded test builds its guard inside the spawned +closure. + +### Persistence event gap: three leak sites found, instrumented not yet fixed + +Updated 2026-09-07. This section previously said the cause was unknown and that +the next step was instrumentation rather than a repro hunt. The instrumentation +was built, and reading for it found the leaks. + +`IndexChangeEventId` is `indexset::cdc::change::Id`, allocated by +`event_id.fetch_add` in the same statement that stamps the event, so indexset +never consumes an id without emitting its event. Every leak is on our side: an +event handed back and then dropped. Three sites, all in generated persisted +query code, all on secondary streams, all confirmed by reading: + +- `codegen/src/generators/persist/queries/update.rs:569`. The + `IndexError::NotFound => Err(WorkTableError::NotFound)` arm returns with no + acknowledge, while its sibling `AlreadyExists` arm immediately above builds an + `Acknowledge` carrying `merged_events` and applies it. The events from + `process_difference_insert_cdc` are dropped on the `NotFound` path. This + asymmetry between two adjacent arms is the clearest of the three. +- `codegen/src/generators/persist/queries/update.rs:593`, + `gen_process_diffs_remove_on_index`: `let (secondary_keys_events_remove, res) + = ...; res?;`. On `Err` the `?` returns before + `op.extend_secondary_key_events`, dropping the events bound on the line above. +- `codegen/src/generators/persist/queries/delete.rs:101`: the same `res?;` shape + after `delete_row_cdc`. + +Checked and NOT leaks: the rollback arms of `insert_cdc`, `insert_many_cdc` and +`reinsert_cdc` in `src/table/mod.rs` all merge forward and rollback events into +an `Acknowledge`, as does the data-delete-failure restore path. Vacuum's +`update_index_after_move` takes the non-CDC branch only when `persistence` is +`None`, so the one case commit `c0c06ba` named is closed for persisted tables. + +The only structurally possible primary-stream leak is a refused +`apply_operation` after the index mutation already consumed ids, and more +generally any `?` between a CDC index mutation and its `apply_operation`. + +**The failure text quoted in earlier versions of this section is stale.** It +said "attempt 9"; `GIVE_UP_AFTER_ATTEMPTS` is now 120, and +`COLLECT_WHOLE_QUEUE_AFTER_ATTEMPTS` plus its regression test +`collection_recovers_when_event_order_and_operation_order_disagree` were added +since, for a symptom that reads identically but is a collection failure rather +than a leak. Telling those two apart is exactly what the new ledger does. + +`src/persistence/event_ledger.rs` records queued, collected, requeued, trimmed +and applied per stream in a bounded 8192-id window, and the guard's message now +ends in a verdict: either ASSIGNED BUT NEVER QUEUED with the id range and the +producer sites either side of the gap, or QUEUED BUT NOT APPLIED with the per-id +stage history. It says so plainly when part of the gap fell outside the retained +window, so it never claims "never queued" about an id it cannot answer for. +Always compiled, gated at run time on `debug_assertions` or `WT_EVENT_LEDGER`, +which puts it on exactly where the bug appears, since the stall needs a full +debug `--all-features` run. + +Not fixed, and not compiled. The three sites need the same treatment the +rollback arms already use: build an `Acknowledge` carrying the orphaned events +before propagating the error. ## Housekeeping - `CHANGELOG.md` stops at 0.4.1, long before the 1.0.0-beta line. diff --git a/src/persistence/event_ledger.rs b/src/persistence/event_ledger.rs new file mode 100644 index 00000000..560cb7ee --- /dev/null +++ b/src/persistence/event_ledger.rs @@ -0,0 +1,735 @@ +//! Pairs index change event id *assignment* with event *queueing*, so that a +//! persistence stall on an event gap names its own cause instead of only its +//! symptom. +//! +//! # The defect this exists for +//! +//! `BatchOperation::validate` refuses to apply an event stream with a hole in +//! it (see the guard there and commit `c0c06ba`). A hole is normally +//! transient: the operation carrying the missing id has been produced but not +//! yet batched. A hole that survives the whole deferral budget means something +//! else, and the old message could not tell the two apart. It reported the +//! range and nothing more: +//! +//! ```text +//! persistence stalled on primary index event gap: last applied Id(1439), +//! next available Id(1455) (attempt 9) +//! ``` +//! +//! Two very different bugs produce that line: +//! +//! 1. **A leak.** An id was assigned by the index and its event was then +//! dropped instead of being pushed onto the persistence queue. Nothing will +//! ever deliver it and the stream is permanently gapped. +//! 2. **A collection failure.** The operation carrying the id *was* queued and +//! is still sitting in the analyzer, but batch collection keeps assembling +//! batches that exclude it. +//! +//! Distinguishing those needs a record of what was queued, which is what this +//! ledger keeps. It is deliberately not a fix for either: it only observes. +//! +//! # Where the two sides are observed +//! +//! Assignment happens inside the index (`indexset` bumps an `AtomicU64` and +//! stamps the event in the same commit), which this crate cannot hook. What it +//! *can* hook is the other end: every event that reaches persistence passes +//! through [`crate::persistence::task::Queue`], so an id present in the stream +//! but absent from this ledger was assigned and never queued. That is the leak +//! signature, and it is what [`EventLedger::gap_report`] reports. +//! +//! The producer is named by [`std::panic::Location`], captured with +//! `#[track_caller]` at the queue push, so a leak points at the call site that +//! produced its neighbours: the generated query, `insert_many`, an +//! acknowledge path, or vacuum's `apply_move`. A full backtrace is captured +//! too, but only when `RUST_BACKTRACE` is set; `Backtrace::capture` is +//! essentially free otherwise. +//! +//! # Gating: `debug_assertions`, not a cargo feature +//! +//! Recording is compiled in unconditionally and gated at run time by +//! [`enabled`], which is true when `debug_assertions` is on or when +//! `WT_EVENT_LEDGER` is set in the environment. +//! +//! `debug_assertions` was chosen over a new cargo feature for one reason: the +//! stall is only ever observed under a full `cargo test --workspace +//! --all-targets --all-features` run, and that run is a debug build, so the +//! instrumentation is on exactly where the bug appears. A feature would also +//! have been enabled by `--all-features`, but it would have to be declared in +//! `Cargo.toml`, and a diagnostic that lives behind a flag nobody sets in the +//! failing configuration is worthless. Release builds fold `enabled()` down +//! to the environment check and every recording call returns immediately, so +//! they pay a predictable-branch and nothing else. +//! +//! `WT_EVENT_LEDGER=1` exists so a release build can be told to record without +//! being rebuilt, for the day the stall shows up outside a test run. +//! +//! Memory is bounded by [`WINDOW`] ids per stream. The gap sits at the head of +//! the stream by construction, so a recent window always covers it; the report +//! states the window it holds so a reader can see that for themselves. + +use std::backtrace::Backtrace; +use std::collections::{BTreeMap, HashMap}; +use std::fmt::Write as _; +use std::panic::Location; +use std::sync::LazyLock; + +use data_bucket::Link; +use indexset::cdc::change::ChangeEvent; +use indexset::core::pair::Pair; +use parking_lot::Mutex; + +use crate::persistence::{OperationId, OperationType}; + +/// Ids retained per stream. The gap the guard reports is at the head of the +/// stream, so a window this size covers it many times over: the one observed +/// stall was 16 ids wide. +const WINDOW: usize = 8192; + +/// Gap ids listed individually in a report before it summarises the rest. +const MAX_LISTED_GAP_IDS: usize = 64; + +static ENABLED: LazyLock = LazyLock::new(|| { + if cfg!(debug_assertions) { + return true; + } + match std::env::var("WT_EVENT_LEDGER") { + Ok(value) => !value.is_empty() && value != "0" && !value.eq_ignore_ascii_case("false"), + Err(_) => false, + } +}); + +/// Whether event bookkeeping is recording in this process. +/// +/// See the module comment for why this is `debug_assertions` plus an +/// environment override rather than a cargo feature. +#[inline] +pub fn enabled() -> bool { + *ENABLED +} + +/// Which index's event id sequence a record belongs to. +/// +/// Every index keeps its own counter, so ids only mean anything relative to a +/// stream. `Primary` allocates nothing, which keeps the hot path free of +/// allocation; secondary streams are labelled by the `Debug` rendering of the +/// table's `AvailableIndexes` value, which is the only name available to +/// non-generic code here. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum EventStream { + Primary, + Secondary(String), +} + +impl std::fmt::Display for EventStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EventStream::Primary => f.write_str("primary"), + EventStream::Secondary(index) => write!(f, "secondary {index}"), + } + } +} + +/// What has been observed happening to one event id. +/// +/// Flags, not a state machine: an id is queued, then collected into a batch, +/// then possibly trimmed back out and requeued, possibly several times over. +/// The report reads the whole set. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Stages(u8); + +impl Stages { + /// Pushed onto the persistence queue by a producer. + pub const QUEUED: Stages = Stages(1 << 0); + /// Pulled into a `BatchOperation` by the analyzer. + pub const COLLECTED: Stages = Stages(1 << 1); + /// Handed back to the analyzer's queue after a deferral or a trim. + pub const REQUEUED: Stages = Stages(1 << 2); + /// Removed from a batch by `remove_operations_from_events`. + pub const TRIMMED: Stages = Stages(1 << 3); + /// Covered by the applied watermark reported after a batch. + pub const APPLIED: Stages = Stages(1 << 4); + + fn insert(&mut self, other: Stages) { + self.0 |= other.0; + } + + /// Both flag sets at once. + pub const fn union(self, other: Stages) -> Stages { + Stages(self.0 | other.0) + } + + fn contains(self, other: Stages) -> bool { + self.0 & other.0 == other.0 + } +} + +impl std::fmt::Display for Stages { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut first = true; + for (flag, name) in [ + (Stages::QUEUED, "queued"), + (Stages::COLLECTED, "collected"), + (Stages::REQUEUED, "requeued"), + (Stages::TRIMMED, "trimmed"), + (Stages::APPLIED, "applied"), + ] { + if self.contains(flag) { + if !first { + f.write_str("+")?; + } + f.write_str(name)?; + first = false; + } + } + if first { + f.write_str("none")?; + } + Ok(()) + } +} + +#[derive(Debug)] +struct IdRecord { + stages: Stages, + /// The operation that carried this id when it was first queued. + op_id: Option, + op_type: Option, + /// Producer call site, from `#[track_caller]` at the queue push. + site: Option<&'static Location<'static>>, + /// Captured only when `RUST_BACKTRACE` is set; otherwise `Disabled` and + /// free. Boxed so that the common empty case costs a pointer instead of an + /// inline `Backtrace` in each of the thousands of records held per stream. + backtrace: Option>, + collected: u32, + requeued: u32, +} + +impl IdRecord { + fn new() -> Self { + Self { + stages: Stages::default(), + op_id: None, + op_type: None, + site: None, + backtrace: None, + collected: 0, + requeued: 0, + } + } +} + +#[derive(Debug, Default)] +struct StreamLedger { + ids: BTreeMap, + /// Ids below this were evicted by the window and cannot be answered for. + evicted_below: u64, + /// Highest id ever seen at any stage on this stream. + highest_seen: u64, + /// Highest id the analyzer reported as applied. + applied_upto: u64, + queued_total: u64, +} + +impl StreamLedger { + fn entry(&mut self, id: u64) -> &mut IdRecord { + if id > self.highest_seen { + self.highest_seen = id; + } + self.ids.entry(id).or_insert_with(IdRecord::new) + } + + fn trim(&mut self) { + while self.ids.len() > WINDOW { + // Ids are close to monotonic, so the lowest key is the oldest. + if let Some((id, _)) = self.ids.pop_first() { + self.evicted_below = self.evicted_below.max(id + 1); + } else { + break; + } + } + } + + /// Lowest id this ledger can still answer for. + fn window_start(&self) -> u64 { + self.ids.keys().next().copied().unwrap_or(self.evicted_below) + } +} + +/// Per-table record of which index change event ids reached the persistence +/// queue, and what happened to them afterwards. +/// +/// Shared by `Arc` between the queue (the producer side), the analyzer, and +/// the `BatchOperation` whose guard reads it. +#[derive(Debug)] +pub struct EventLedger { + label: String, + /// True when no producer writes to this ledger, so "never queued" here + /// means "never recorded", not "leaked". Reports say so rather than + /// accusing a producer that was never watched. + detached: bool, + streams: Mutex>, +} + +impl EventLedger { + pub fn new(label: impl Into) -> Self { + Self { + label: label.into(), + detached: false, + streams: Mutex::new(HashMap::new()), + } + } + + /// A ledger attached to nothing, for analyzers and `BatchOperation`s built + /// outside `run_engine` (unit tests, defensive callers). It records + /// normally; it is simply never shared with a producer, so its reports say + /// so instead of reading a missing record as a leak. + pub fn detached() -> Self { + Self { + label: "".to_owned(), + detached: true, + streams: Mutex::new(HashMap::new()), + } + } + + pub fn label(&self) -> &str { + &self.label + } + + /// Collects the event ids of `evs`, for a later [`EventLedger::record_queued`]. + /// + /// Split from the recording itself because the queue only knows a push was + /// accepted after the operation has been moved into it, and a refused push + /// must not be recorded as queued. + pub fn event_ids(evs: &[ChangeEvent>]) -> Vec { + if !enabled() { + return Vec::new(); + } + evs.iter().map(|ev| ev.id().inner()).collect() + } + + /// Records every id in `ids` as queued by `site`. + /// + /// Called from the persistence queue push, which is the single point every + /// operation passes through on its way to the engine. An id in the applied + /// stream that never appears here was assigned by the index and dropped + /// before it reached persistence. + pub fn record_queued( + &self, + stream: EventStream, + ids: &[u64], + op_id: OperationId, + op_type: OperationType, + site: &'static Location<'static>, + ) { + if !enabled() || ids.is_empty() { + return; + } + // Costs nothing unless RUST_BACKTRACE is set: `capture` returns + // `Disabled` without walking any frames. One capture per push, moved + // onto the first newly recorded id, because every id in this vector + // came from the same producer. + let backtrace = Backtrace::capture(); + let mut backtrace = + matches!(backtrace.status(), std::backtrace::BacktraceStatus::Captured).then_some(backtrace); + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + for id in ids.iter().copied() { + let record = ledger.entry(id); + let first_time = !record.stages.contains(Stages::QUEUED); + record.stages.insert(Stages::QUEUED); + if first_time { + record.op_id = Some(op_id); + record.op_type = Some(op_type); + record.site = Some(site); + // One backtrace per push, not per id: every id in this vector + // has the same producer, and keeping one each would multiply + // the cost of a `RUST_BACKTRACE` run for no extra signal. + if record.backtrace.is_none() { + record.backtrace = backtrace.take().map(Box::new); + } + } else { + record.stages.insert(Stages::REQUEUED); + record.requeued = record.requeued.saturating_add(1); + } + } + ledger.queued_total = ledger.queued_total.saturating_add(ids.len() as u64); + ledger.trim(); + } + + /// Records a stage transition for a single id already known to a stream. + pub fn record_stage(&self, stream: EventStream, id: u64, stage: Stages) { + if !enabled() { + return; + } + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + let record = ledger.entry(id); + record.stages.insert(stage); + if stage.contains(Stages::COLLECTED) { + record.collected = record.collected.saturating_add(1); + } + if stage.contains(Stages::REQUEUED) { + record.requeued = record.requeued.saturating_add(1); + } + ledger.trim(); + } + + /// Records a stage transition for every id in `evs`. + pub fn record_stage_for_events(&self, stream: EventStream, evs: &[ChangeEvent>], stage: Stages) { + if !enabled() || evs.is_empty() { + return; + } + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + for ev in evs { + let record = ledger.entry(ev.id().inner()); + record.stages.insert(stage); + if stage.contains(Stages::COLLECTED) { + record.collected = record.collected.saturating_add(1); + } + if stage.contains(Stages::REQUEUED) { + record.requeued = record.requeued.saturating_add(1); + } + } + ledger.trim(); + } + + /// Records the applied watermark reported after a batch was accepted. + pub fn record_applied_upto(&self, stream: EventStream, id: u64) { + if !enabled() || id == 0 { + return; + } + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + if id > ledger.applied_upto { + ledger.applied_upto = id; + } + if id > ledger.highest_seen { + ledger.highest_seen = id; + } + for (_, record) in ledger.ids.range_mut(..=id) { + record.stages.insert(Stages::APPLIED); + } + } + + /// Explains the gap between `last_applied` and `next_available`. + /// + /// This is the whole point of the ledger. For every id in the hole it says + /// whether the id ever reached the persistence queue, and if it did, what + /// happened to it afterwards, so the reader can tell a leaked event from a + /// batch collection that keeps missing one. + pub fn gap_report(&self, stream: &EventStream, last_applied: u64, next_available: u64) -> String { + let mut out = String::new(); + if !enabled() { + let _ = write!( + out, + " Event bookkeeping is off in this build, so the gap cannot be attributed. \ + Re-run with WT_EVENT_LEDGER=1 (and RUST_BACKTRACE=1 for producer backtraces) \ + to have the next occurrence name its own cause." + ); + return out; + } + + if self.detached { + let _ = write!( + out, + " This analyzer holds detached event bookkeeping: no producer ever wrote to it, \ + so the gap cannot be attributed. Only analyzers built outside `run_engine` are detached." + ); + return out; + } + + let streams = self.streams.lock(); + let Some(ledger) = streams.get(stream) else { + let _ = write!( + out, + " Event bookkeeping for table {label} holds no records at all for the {stream} stream, \ + which should be impossible while it is applying that stream's events.", + label = self.label, + ); + return out; + }; + + let window_start = ledger.window_start(); + let first_missing = last_applied.saturating_add(1); + if next_available <= first_missing { + return out; + } + + // Bounded on purpose: this runs while building a panic message, and a + // corrupt watermark could otherwise make it walk billions of ids. + let scan_end = next_available.min(first_missing.saturating_add(WINDOW as u64)); + let mut never_queued = Vec::new(); + let mut queued_not_applied = Vec::new(); + for id in first_missing..scan_end { + match ledger.ids.get(&id) { + Some(record) if record.stages.contains(Stages::QUEUED) => queued_not_applied.push((id, record)), + _ => never_queued.push(id), + } + } + + let _ = write!( + out, + " Bookkeeping for table {label}, {stream} stream: window covers ids {window_start}..={highest}, \ + applied watermark {applied}, {total} id(s) queued in all, \ + {gap_len} id(s) in the gap ({scanned} scanned), {never} never queued, {queued} queued.", + label = self.label, + window_start = window_start, + highest = ledger.highest_seen, + applied = ledger.applied_upto, + total = ledger.queued_total, + gap_len = next_available - first_missing, + scanned = scan_end - first_missing, + never = never_queued.len(), + queued = queued_not_applied.len(), + ); + + if first_missing < window_start { + let _ = write!( + out, + " CAUTION: part of the gap ({first_missing}..{window_start}) fell out of the retained window, \ + so those ids are unattributable rather than proven missing." + ); + } + + if !never_queued.is_empty() { + let _ = write!(out, " ASSIGNED BUT NEVER QUEUED: {}.", format_ids(&never_queued)); + let _ = write!( + out, + " Those ids were consumed by the index and their events never reached the persistence queue, \ + so nothing will ever deliver them: this is an event leak upstream of the analyzer, \ + not a batch collection problem." + ); + let _ = write!(out, "{}", bracketing_producers(ledger, last_applied, next_available)); + } + + if !queued_not_applied.is_empty() { + let _ = write!(out, " QUEUED BUT NOT APPLIED:"); + for (id, record) in queued_not_applied.iter().take(MAX_LISTED_GAP_IDS) { + let _ = write!( + out, + " [{id}: {stages}, collected {collected}x, requeued {requeued}x, {op_type} op {op_id} from {site}]", + stages = record.stages, + collected = record.collected, + requeued = record.requeued, + op_type = OptionDisplay(record.op_type.as_ref().map(|t| format!("{t:?}"))), + op_id = OptionDisplay(record.op_id.as_ref().map(|id| format!("{id:?}"))), + site = OptionDisplay(record.site.map(|site| site.to_string())), + ); + } + if queued_not_applied.len() > MAX_LISTED_GAP_IDS { + let _ = write!(out, " and {} more", queued_not_applied.len() - MAX_LISTED_GAP_IDS); + } + let _ = write!( + out, + ". Those events did reach the queue, so their operations are still somewhere in the analyzer \ + and batch collection is failing to assemble them: the bug is in collection, not in event production." + ); + } + + out + } +} + +/// Names the producers on either side of the hole. +/// +/// A leaked id has no record of its own, so the closest evidence about who +/// should have produced it is who produced its neighbours. One index allocates +/// its ids from one counter, so the neighbours are almost always the same call +/// path. +fn bracketing_producers(ledger: &StreamLedger, last_applied: u64, next_available: u64) -> String { + let mut out = String::new(); + let before = ledger.ids.range(..=last_applied).next_back(); + let after = ledger.ids.range(next_available..).next(); + for (side, entry) in [("before the gap", before), ("after the gap", after)] { + let Some((id, record)) = entry else { + let _ = write!(out, " No record {side}."); + continue; + }; + let _ = write!( + out, + " Producer {side} (id {id}): {op_type}, op {op_id}, pushed from {site}.", + op_type = OptionDisplay(record.op_type.as_ref().map(|t| format!("{t:?}"))), + op_id = OptionDisplay(record.op_id.as_ref().map(|id| format!("{id:?}"))), + site = OptionDisplay(record.site.map(|site| site.to_string())), + ); + if let Some(backtrace) = &record.backtrace { + let _ = write!(out, " Backtrace:\n{backtrace}\n"); + } + } + if !ledger.ids.values().any(|record| record.backtrace.is_some()) { + let _ = write!( + out, + " Re-run with RUST_BACKTRACE=1 for the full producer backtraces, which this run did not capture." + ); + } + out +} + +impl Default for EventLedger { + fn default() -> Self { + Self::detached() + } +} + +struct OptionDisplay(Option); + +impl std::fmt::Display for OptionDisplay { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.0 { + Some(value) => f.write_str(value), + None => f.write_str(""), + } + } +} + +/// Renders a sorted id list compactly, collapsing runs. +fn format_ids(ids: &[u64]) -> String { + let mut out = String::new(); + let mut listed = 0usize; + let mut i = 0usize; + while i < ids.len() && listed < MAX_LISTED_GAP_IDS { + let start = ids[i]; + let mut end = start; + while i + 1 < ids.len() && ids[i + 1] == end + 1 { + i += 1; + end = ids[i]; + } + if !out.is_empty() { + out.push_str(", "); + } + if start == end { + let _ = write!(out, "{start}"); + } else { + let _ = write!(out, "{start}..={end}"); + } + listed += 1; + i += 1; + } + if i < ids.len() { + let _ = write!(out, ", and {} more", ids.len() - i); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn insert_at(id: u64) -> ChangeEvent> { + ChangeEvent::InsertAt { + event_id: id.into(), + max_value: Pair { + key: id, + value: Link::default(), + }, + value: Pair { + key: id, + value: Link::default(), + }, + index: 0, + } + } + + #[track_caller] + fn queue(ledger: &EventLedger, ids: &[u64], op_type: OperationType) { + let evs = ids.iter().copied().map(insert_at).collect::>(); + let ids = EventLedger::event_ids(&evs); + ledger.record_queued( + EventStream::Primary, + &ids, + OperationId::Single(uuid::Uuid::from_u128(1)), + op_type, + Location::caller(), + ); + } + + /// These tests assert on what the ledger recorded, so they are only + /// meaningful where it records. That is every normal test run + /// (`debug_assertions`); a `--release` test run skips them rather than + /// failing on a report that correctly says bookkeeping was off. + fn recording() -> bool { + enabled() + } + + #[test] + fn names_ids_that_were_never_queued() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3], OperationType::Insert); + // 4 and 5 are assigned by the index and leaked: nothing queues them. + queue(&ledger, &[6, 7], OperationType::Update); + + let report = ledger.gap_report(&EventStream::Primary, 3, 6); + + assert!(report.contains("ASSIGNED BUT NEVER QUEUED"), "{report}"); + assert!(report.contains("4..=5"), "{report}"); + assert!(report.contains("event leak upstream of the analyzer"), "{report}"); + } + + #[test] + fn distinguishes_a_queued_id_from_a_leaked_one() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3], OperationType::Insert); + queue(&ledger, &[4], OperationType::Update); + queue(&ledger, &[6], OperationType::Insert); + + let report = ledger.gap_report(&EventStream::Primary, 3, 6); + + assert!(report.contains("QUEUED BUT NOT APPLIED"), "{report}"); + assert!(report.contains("the bug is in collection"), "{report}"); + // Only id 5 is missing; 4 was queued. + assert!(report.contains("ASSIGNED BUT NEVER QUEUED: 5."), "{report}"); + } + + #[test] + fn reports_a_gapless_stream_as_nothing() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3], OperationType::Insert); + assert!(ledger.gap_report(&EventStream::Primary, 3, 4).is_empty()); + } + + #[test] + fn window_eviction_is_reported_rather_than_guessed() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + let ids = (1..=(WINDOW as u64 + 64)).collect::>(); + queue(&ledger, &ids, OperationType::Insert); + + // Ask about a gap far below the retained window. + let report = ledger.gap_report(&EventStream::Primary, 1, 20); + assert!(report.contains("fell out of the retained window"), "{report}"); + } + + #[test] + fn applied_watermark_marks_everything_behind_it() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3, 5], OperationType::Insert); + ledger.record_applied_upto(EventStream::Primary, 3); + + let report = ledger.gap_report(&EventStream::Primary, 3, 5); + assert!(report.contains("ASSIGNED BUT NEVER QUEUED: 4."), "{report}"); + } + + #[test] + fn stages_render_every_flag_set() { + let mut stages = Stages::default(); + assert_eq!(stages.to_string(), "none"); + stages.insert(Stages::QUEUED); + stages.insert(Stages::TRIMMED); + assert_eq!(stages.to_string(), "queued+trimmed"); + } +} diff --git a/src/persistence/mod.rs b/src/persistence/mod.rs index b5e03ce3..b5ddb33a 100644 --- a/src/persistence/mod.rs +++ b/src/persistence/mod.rs @@ -10,6 +10,7 @@ pub use error::{ PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, PersistenceResult, PersistenceState, load_persisted_state, }; +pub use event_ledger::{EventLedger, EventStream, Stages}; pub use operation::{ AcknowledgeOperation, DeleteOperation, InsertOperation, Operation, OperationId, OperationType, UpdateOperation, validate_events, @@ -91,6 +92,7 @@ impl std::error::Error for UnloadFailure {} mod engine; mod error; +pub mod event_ledger; pub mod operation; mod readonly_engine; mod space; diff --git a/src/persistence/operation/batch.rs b/src/persistence/operation/batch.rs index 958bcfc0..ea49a1d8 100644 --- a/src/persistence/operation/batch.rs +++ b/src/persistence/operation/batch.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::fmt::Debug; use std::hash::Hash; use std::marker::PhantomData; +use std::sync::Arc; use data_bucket::page::PageId; use data_bucket::{Link, SizeMeasurable}; @@ -10,6 +11,7 @@ use indexset::core::pair::Pair; use worktable_codegen::{MemStat, worktable}; use crate::persistence::OperationType; +use crate::persistence::event_ledger::{self, EventLedger, EventStream, Stages}; use crate::persistence::space::{BatchChangeEvent, BatchData}; use crate::persistence::task::{LastEventIds, QueueInnerRow}; use crate::prelude::*; @@ -149,6 +151,10 @@ fn latest_data_writes( pub struct BatchOperation { ops: Vec>, info_wt: BatchInnerWorkTable, + /// Event bookkeeping shared with the queue that produced `ops`, read by + /// the event-gap guard in `validate` so a stall names its own cause. + /// Diagnostics only, and `None` for batches built outside the analyzer. + event_ledger: Option>, prepared_index_evs: Option>, phantom_data: PhantomData, } @@ -173,11 +179,23 @@ where Self { ops, info_wt, + event_ledger: None, prepared_index_evs: None, phantom_data: PhantomData, } } + /// Attaches the analyzer's event bookkeeping, so the gap guard below can + /// say which ids in a gap ever reached the persistence queue. + /// + /// A builder method rather than a `new` parameter, so every existing + /// caller of `new` keeps working unchanged and the batch stays usable + /// without any bookkeeping at all. + pub fn with_event_ledger(mut self, ledger: Arc) -> Self { + self.event_ledger = Some(ledger); + self + } + /// Remove metadata immediately after `self.ops.remove(removed_pos)`. /// /// At entry, `self.ops.len()` is already one shorter while `info_wt` still @@ -272,9 +290,54 @@ where prepared_evs.secondary_evs.remove(op_secondary); } + self.record_stage(&removed_ops, Stages::TRIMMED); + Ok(removed_ops) } + /// Records `stage` against every event id carried by `ops`. + /// + /// Diagnostics only. Skipped entirely when bookkeeping is off, which keeps + /// the `Debug` formatting of secondary index labels out of release builds. + fn record_stage(&self, ops: &[Operation], stage: Stages) { + let Some(ledger) = &self.event_ledger else { + return; + }; + if !event_ledger::enabled() { + return; + } + for op in ops { + if let Some(evs) = op.primary_key_events() { + ledger.record_stage_for_events(EventStream::Primary, evs, stage); + } + // See the matching note in `QueueAnalyzer::record_ops_stage`: the + // producer side records primary ids only, so an id observed on a + // secondary stream here is known to have been queued. + for (index, id) in op.secondary_key_events().iter_event_ids() { + ledger.record_stage( + EventStream::Secondary(format!("{index:?}")), + id.inner(), + stage.union(Stages::QUEUED), + ); + } + } + } + + /// The bookkeeping's account of a gap, or a note saying there is none. + fn gap_report( + &self, + stream: &EventStream, + last_applied: IndexChangeEventId, + next_available: IndexChangeEventId, + ) -> String { + match &self.event_ledger { + Some(ledger) => ledger.gap_report(stream, last_applied.inner(), next_available.inner()), + None => { + " This batch was built without event bookkeeping attached, so the gap cannot be attributed.".to_owned() + } + } + } + pub fn get_last_event_ids(&self) -> LastEventIds { let prepared_evs = self .prepared_index_evs @@ -358,8 +421,9 @@ where // that persists is a bug upstream of the analyzer; report it // loudly instead of force-applying and corrupting the file. if attempts > GIVE_UP_AFTER_ATTEMPTS { + let report = self.gap_report(&EventStream::Primary, last_ids.primary_id, id); return Err(eyre::eyre!( - "persistence stalled on primary index event gap: last applied {:?}, next available {:?} after {attempts} attempts, with {} operations queued. Every one of them was collected and the stream is still gapped, so the operation carrying the missing id never reached the queue: an event id was consumed without its event being pushed. The producer is upstream of the analyzer, not here.", + "persistence stalled on primary index event gap: last applied {:?}, next available {:?} after {attempts} attempts, with {} operations queued.{report}", last_ids.primary_id, id, self.ops.len() @@ -381,8 +445,9 @@ where // stream, defer until the missing event arrives, and report // a persistent gap as the bug it is. if attempts > GIVE_UP_AFTER_ATTEMPTS { + let report = self.gap_report(&EventStream::Secondary(format!("{index:?}")), *last, id); return Err(eyre::eyre!( - "persistence stalled on secondary index {index:?} event gap: last applied {last:?}, next available {id:?} after {attempts} attempts, with {} operations queued. All of them were collected and the stream is still gapped, so the operation carrying the missing id never reached the queue.", + "persistence stalled on secondary index {index:?} event gap: last applied {last:?}, next available {id:?} after {attempts} attempts, with {} operations queued.{report}", self.ops.len() )); } diff --git a/src/persistence/task.rs b/src/persistence/task.rs index fc67f09f..edba8f5c 100644 --- a/src/persistence/task.rs +++ b/src/persistence/task.rs @@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt::Debug; use std::hash::Hash; use std::marker::PhantomData; +use std::panic::Location; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; @@ -12,7 +13,8 @@ use tokio::sync::Notify; use tokio::task::JoinHandle; use worktable_codegen::worktable; -use crate::persistence::operation::{BatchInnerRow, BatchInnerWorkTable, BatchOperation, OperationId}; +use crate::persistence::event_ledger::{self, EventLedger, EventStream, Stages}; +use crate::persistence::operation::{BatchInnerRow, BatchInnerWorkTable, BatchOperation, OperationId, OperationType}; use crate::persistence::{ PersistenceEngine, PersistenceError, PersistenceIndexCorruption, PersistenceResult, PersistenceState, }; @@ -207,6 +209,12 @@ pub struct QueueAnalyzer, } #[derive(Debug)] @@ -260,9 +268,19 @@ where page_limit: MAX_PAGE_AMOUNT, attempts: 0, no_progress: 0, + event_ledger: Arc::new(EventLedger::detached()), } } + /// Shares the feeding queue's event bookkeeping with this analyzer. + /// + /// Only the producer side records who pushed an event, so the analyzer has + /// to read the *same* ledger the queue writes for its gap reports to mean + /// anything. + pub fn attach_event_ledger(&mut self, ledger: Arc) { + self.event_ledger = ledger; + } + pub fn push(&mut self, value: Operation) -> eyre::Result<()> { let link = value.link(); let mut row = QueueInnerRow { @@ -301,6 +319,39 @@ where .map(|(id, _)| id) } + /// Records `stage` against every event id carried by `ops`. + /// + /// Diagnostics only. The whole body is skipped when bookkeeping is off, + /// which keeps the `Debug` formatting of secondary index labels off the + /// path of a release build entirely. + fn record_ops_stage(&self, ops: &[Operation], stage: Stages) + where + SecondaryKeys: TableSecondaryIndexEventsOps, + { + if !event_ledger::enabled() { + return; + } + for op in ops { + if let Some(evs) = op.primary_key_events() { + self.event_ledger + .record_stage_for_events(EventStream::Primary, evs, stage); + } + // `Stages::QUEUED` is unioned in for secondary streams because the + // producer side records primary ids only: an operation reaching + // the analyzer at all proves it was queued, and without this the + // secondary gap report would call every id it knows about leaked. + // The cost is that a secondary record carries no producer call + // site, which the report prints as ``. + for (index, id) in op.secondary_key_events().iter_event_ids() { + self.event_ledger.record_stage( + EventStream::Secondary(format!("{index:?}")), + id.inner(), + stage.union(Stages::QUEUED), + ); + } + } + } + pub async fn collect_batch_from_op_id( &mut self, op_id: OperationId, @@ -440,13 +491,23 @@ where ops.push(op); } - let mut op = BatchOperation::new(ops, info_wt); + self.record_ops_stage(&ops, Stages::COLLECTED); + let mut op = BatchOperation::new(ops, info_wt).with_event_ledger(self.event_ledger.clone()); let invalid_for_this_batch_ops = op.validate(&self.last_events_ids, self.attempts).await?; if let Some(invalid_for_this_batch_ops) = invalid_for_this_batch_ops { + self.record_ops_stage(&invalid_for_this_batch_ops, Stages::REQUEUED); self.extend_from_iter(invalid_for_this_batch_ops.into_iter())?; let previous_primary = self.last_events_ids.primary_id; let last_ids = op.get_last_event_ids(); let advanced = last_ids.primary_id > previous_primary; + self.event_ledger + .record_applied_upto(EventStream::Primary, last_ids.primary_id.inner()); + if event_ledger::enabled() { + for (index, id) in &last_ids.secondary_ids { + self.event_ledger + .record_applied_upto(EventStream::Secondary(format!("{index:?}")), id.inner()); + } + } self.last_events_ids.merge(last_ids); self.last_invalid_batch_size = 0; self.page_limit = MAX_PAGE_AMOUNT; @@ -461,6 +522,7 @@ where } else { // can't collect batch for now let ops = op.ops(); + self.record_ops_stage(&ops, Stages::REQUEUED); self.attempts += 1; self.no_progress += 1; if self.last_invalid_batch_size == ops.len() { @@ -918,7 +980,7 @@ mod lifecycle_tests { #[tokio::test] async fn wake_landing_inside_the_pop_race_window_is_not_lost() { let lifecycle = Arc::new(PersistenceLifecycle::new()); - let mut queue = Queue::<(), u64, TestEvents>::new(lifecycle.clone()); + let mut queue = Queue::<(), u64, TestEvents>::new(lifecycle.clone(), "tests/queue"); let gate = Arc::new(PopRaceWindowGate::new()); queue.pop_race_window_gate = Some(gate.clone()); let queue = Arc::new(queue); @@ -1148,6 +1210,41 @@ impl PopRaceWindowGate { } } +/// Primary index event ids lifted off an operation before it is moved into the +/// queue, so they can be recorded once the push is known to have been accepted. +/// +/// Primary only: `Queue` is generic over the secondary event type with no bound +/// that could iterate it, and the primary stream is the one whose gap guard +/// stalls the engine. The analyzer records secondary ids at collection time, +/// where that bound does exist. +/// +/// Cheap when bookkeeping is off: `EventLedger::event_ids` returns an empty +/// `Vec`, which allocates nothing, and the rest is two `Copy` field reads. +struct QueuedEventIds { + ids: Vec, + op_id: OperationId, + op_type: OperationType, +} + +impl QueuedEventIds { + fn of( + value: &Operation, + ) -> Self { + Self { + ids: value + .primary_key_events() + .map(|evs| EventLedger::event_ids(evs.as_slice())) + .unwrap_or_default(), + op_id: value.operation_id(), + op_type: value.operation_type(), + } + } + + fn record(&self, ledger: &EventLedger, site: &'static Location<'static>) { + ledger.record_queued(EventStream::Primary, &self.ids, self.op_id, self.op_type, site); + } +} + #[derive(Debug)] pub struct Queue { // Not `lockfree::queue::Queue`: its `Removable::empty` materializes the @@ -1161,38 +1258,71 @@ pub struct Queue { // queue that still holds work. len: Arc, lifecycle: Arc, + /// Producer-side half of the event-gap bookkeeping: every operation that + /// reaches persistence passes through this queue, so an event id the + /// engine is waiting for that never appears here was assigned by the index + /// and dropped before it was queued. Shared with the analyzer, which reads + /// it when the gap guard fires. Diagnostics only, and inert unless + /// [`event_ledger::enabled`]. + event_ledger: Arc, #[cfg(test)] pop_race_window_gate: Option>, } impl Queue { - fn new(lifecycle: Arc) -> Self { + fn new(lifecycle: Arc, table_path: &str) -> Self { Self { queue: ParkingMutex::new(VecDeque::new()), notify: Notify::new(), len: Arc::new(AtomicUsize::new(0)), lifecycle, + event_ledger: Arc::new(EventLedger::new(table_path)), #[cfg(test)] pop_race_window_gate: None, } } - pub fn push(&self, value: Operation) -> PersistenceResult { - self.push_message(PersistenceMessage::Operation(value)) + /// The event bookkeeping this queue writes, for sharing with the analyzer. + pub fn event_ledger(&self) -> Arc { + self.event_ledger.clone() + } + + /// Enqueues one operation, naming the producer's call site. + /// + /// The site is passed explicitly rather than taken with `#[track_caller]`, + /// because that only reaches one frame up: a wrapper that wants its own + /// caller named in a gap report has to forward a location through here. + /// through here rather than call `push` and lose it. + pub fn push_at( + &self, + value: Operation, + site: &'static Location<'static>, + ) -> PersistenceResult { + // The ids have to be lifted out before the operation is moved into the + // queue, but they are only recorded once the push is accepted: a + // refused push (the engine is closing or already failed) genuinely + // does not queue its events, and recording it as queued would hide + // exactly that leak mode. + let queued = QueuedEventIds::of(&value); + self.push_message(PersistenceMessage::Operation(value))?; + queued.record(&self.event_ledger, site); + Ok(()) } - /// Enqueues a whole batch of operations under one lifecycle check, one - /// queue lock acquisition and one worker wake-up, so callers producing - /// many operations at once (`insert_many`) pay the intake overhead once - /// instead of per row. All-or-nothing: either every operation is accepted - /// or none is. - pub fn push_many( + /// Enqueues a whole batch under one lifecycle check, one queue lock and one + /// worker wake-up, so a caller producing many operations at once + /// (`insert_many`) pays the intake overhead once instead of per row. + /// All-or-nothing: either every operation is accepted or none is. Takes the + /// producer's call site for the same reason as [`Queue::push_at`]. + pub fn push_many_at( &self, values: Vec>, + site: &'static Location<'static>, ) -> PersistenceResult { if values.is_empty() { return Ok(()); } + let queued = values.iter().map(QueuedEventIds::of).collect::>(); let state = self.lifecycle.state.lock(); match &*state { PersistenceState::Running => {} @@ -1205,6 +1335,11 @@ impl Queue>>, secondary_keys_events: SecondaryKeys, ) -> PersistenceResult { - self.push(Operation::Update(UpdateOperation { - id: OperationId::Single(uuid::Uuid::now_v7()), - primary_key_events, - secondary_keys_events, - bytes, - link: new_link, - })) + // `Location::caller()` without `#[track_caller]` resolves to this line + // rather than to vacuum's call site. That is deliberate: the trait + // declaration lives outside this module and cannot be annotated, and + // this line is already a unique producer label, because `apply_move` + // is only ever reached from a vacuum row move. + self.push_at( + Operation::Update(UpdateOperation { + id: OperationId::Single(uuid::Uuid::now_v7()), + primary_key_events, + secondary_keys_events, + bytes, + link: new_link, + }), + Location::caller(), + ) } fn reclaim_pages(&self, page_ids: Vec) -> PersistenceResult { @@ -1391,17 +1534,21 @@ impl Drop impl PersistenceTask { + /// `#[track_caller]` so an event-gap report names the producer that + /// pushed the operation rather than this forwarding line. + #[track_caller] pub fn apply_operation(&self, op: Operation) -> PersistenceResult { - self.queue.push(op) + self.queue.push_at(op, Location::caller()) } /// Enqueues a batch of operations atomically with a single worker /// wake-up. See [`Queue::push_many`]. + #[track_caller] pub fn apply_operations( &self, ops: Vec>, ) -> PersistenceResult { - self.queue.push_many(ops) + self.queue.push_many_at(ops, Location::caller()) } pub fn ensure_running(&self) -> PersistenceResult { @@ -1448,12 +1595,15 @@ impl { let table_path = engine.config().table_path().to_owned(); let lifecycle = Arc::new(PersistenceLifecycle::new()); - let queue = Arc::new(Queue::new(lifecycle.clone())); + let queue = Arc::new(Queue::new(lifecycle.clone(), &table_path)); let engine_queue = queue.clone(); let engine_lifecycle = lifecycle.clone(); let analyzer_inner_wt: Arc = Default::default(); let mut analyzer = QueueAnalyzer::new(analyzer_inner_wt.clone()); + // Producer and consumer must share one ledger: the queue records who + // pushed an event, the analyzer's gap guard reads it back. + analyzer.attach_event_ledger(queue.event_ledger()); let analyzer_in_progress = Arc::new(AtomicBool::new(true)); let task_analyzer_in_progress = analyzer_in_progress.clone(); From f6447e9ac081d9ff64c6dbfdea46d7a93b460f2f Mon Sep 17 00:00:00 2001 From: meh Date: Tue, 8 Sep 2026 02:41:43 +0700 Subject: [PATCH 08/14] Acknowledge orphaned index events before propagating the error The ledger from the previous commit points at three sites in generated persisted query code where an event the index had already assigned an id to was dropped instead of queued. A dropped event is a hole `BatchOperation::validate` refuses forever, so the table stalls and the message names a range rather than a cause. All three now do what the rollback arms already did: build an `Acknowledge` carrying the orphaned events and apply it before propagating. The `IndexError::NotFound` arm of the update insert path acknowledges the events its sibling `AlreadyExists` arm was already acknowledging, and the two bare `res?` sites, in `gen_process_diffs_remove_on_index` and after `delete_row_cdc`, became `if let Err(e) = res` blocks that acknowledge and then return. The events are moved into the acknowledge rather than cloned, and that is load-bearing rather than a preference. Cloning does not compile: the events type is still an inference variable at that point, pinned only by the `op.extend_secondary_key_events` call further down, and method resolution for `.clone()` needs the type known where the call is written. It fails as an `E0282` reported against the `worktable!` invocation with no inner span, which is expensive to diagnose. The comment in `docs/TODO.md` records that so nobody pays for it twice. The write failures are not forcible through the public API, so the wiring is pinned on the emitted tokens the way the surrounding tests already do it. Both assertions check the acknowledge is emitted *before* the return or the extend, not merely that one appears somewhere in the output. The in-memory generator has the same two `NotFound` arms and is deliberately untouched: there is no persistence stream behind them to gap. --- .../src/generators/persist/queries/delete.rs | 34 +++++++- .../src/generators/persist/queries/update.rs | 84 ++++++++++++++++++- docs/TODO.md | 34 ++++++-- 3 files changed, 144 insertions(+), 8 deletions(-) diff --git a/codegen/src/generators/persist/queries/delete.rs b/codegen/src/generators/persist/queries/delete.rs index c2efa53c..ebe4ea45 100644 --- a/codegen/src/generators/persist/queries/delete.rs +++ b/codegen/src/generators/persist/queries/delete.rs @@ -98,7 +98,23 @@ impl PersistGenerator { row, link, ); - res?; + // `delete_row_cdc` produces events whether or not it succeeds, and + // the index has already assigned their ids. Propagating the error + // without queueing them leaves a hole the persistence stream can + // never fill, exactly as the restore path below is careful not to. + if let core::result::Result::Err(e) = res { + let ack_op: Operation< + <<#pk_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #pk_ident, + #secondary_events_ident + > = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events, + }); + self.1.apply_operation(ack_op)?; + return core::result::Result::Err(e.into()); + } let (_, primary_key_events) = self.0.primary_index.remove_cdc(pk.clone(), link); if let core::result::Result::Err(e) = self.0.data.delete(link) { let mut secondary_keys_events = secondary_keys_events; @@ -387,5 +403,21 @@ mod tests { emitted.contains("Operation :: Acknowledge"), "acknowledge op missing:\n{emitted}" ); + + // A failed secondary removal used to propagate through a bare `res?`, + // dropping the events `delete_row_cdc` had already produced. Their ids + // are assigned when the index produces them, so the persistence stream + // gapped permanently and the stall named a range rather than a cause. + let secondary = emitted.find("delete_row_cdc").expect("secondary removal emitted"); + let tail = &emitted[secondary..]; + let ack = tail + .find("Operation :: Acknowledge") + .expect("a failed secondary removal must acknowledge its events"); + assert!( + ack < tail + .find("remove_cdc (pk . clone () , link)") + .expect("primary removal emitted"), + "the secondary removal propagates before acknowledging, which gaps the stream:\n{emitted}" + ); } } diff --git a/codegen/src/generators/persist/queries/update.rs b/codegen/src/generators/persist/queries/update.rs index ffd41658..380df89d 100644 --- a/codegen/src/generators/persist/queries/update.rs +++ b/codegen/src/generators/persist/queries/update.rs @@ -501,6 +501,7 @@ impl PersistGenerator { fn gen_process_diffs_insert_on_index(&self, idents: &[Ident], idx_idents: Option<&Vec>) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let avt_type_ident = name_generator.get_available_type_ident(); + let pk_ident = name_generator.get_primary_key_type_ident(); // `updated_bytes` is bound by gen_data_write_and_fetch, which captures // the real row bytes right after the data write. let diff_container = if idx_idents.is_some() { @@ -567,7 +568,27 @@ impl PersistGenerator { Err(WorkTableError::AlreadyExists(at.to_string_value())) } - IndexError::NotFound => Err(WorkTableError::NotFound), + IndexError::NotFound => { + // The insert side produced events before it + // failed, and the index has already assigned + // their ids. Returning without queueing them + // leaves a hole the persistence stream can + // never fill, which is what the sibling arm + // above avoids and what this arm used to + // cause. + let ack_op: Operation< + <<#pk_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #pk_ident, + #secondary_events_ident + > = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events: secondary_events.clone(), + }); + self.1.apply_operation(ack_op)?; + + Err(WorkTableError::NotFound) + } }; } let mut secondary_keys_events = secondary_events; @@ -587,10 +608,29 @@ impl PersistGenerator { } fn gen_process_diffs_remove_on_index(&self, idx_idents: Option<&Vec>) -> TokenStream { + let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); + let pk_ident = name_generator.get_primary_key_type_ident(); + let secondary_events_ident = name_generator.get_space_secondary_index_events_ident(); if idx_idents.is_some() { quote! { let (secondary_keys_events_remove, res) = self.0.indexes.process_difference_remove_cdc(link, diffs); - res?; + // The removal produced events whether or not it succeeded, and + // their ids are already assigned. Propagating the error without + // queueing them gaps the stream permanently, so acknowledge + // them first and then propagate unchanged. + if let core::result::Result::Err(e) = res { + let ack_op: Operation< + <<#pk_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #pk_ident, + #secondary_events_ident + > = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events: secondary_keys_events_remove, + }); + self.1.apply_operation(ack_op)?; + return core::result::Result::Err(e.into()); + } op.extend_secondary_key_events(secondary_keys_events_remove); } } else { @@ -1143,5 +1183,45 @@ mod tests { .find("Operation :: Update (UpdateOperation") .expect("update op emitted"); assert!(insert < write && write < op_build, "emission order broken:\n{emitted}"); + + // Every event the index produced must reach the persistence stream, + // including on the paths that fail. The index assigns an event id at + // the moment it produces the event, so a path that returns without + // queueing one leaves a hole `BatchOperation::validate` will refuse + // forever, and the stall it causes names a range rather than a cause. + // + // The `NotFound` arm used to be exactly that: its sibling + // `AlreadyExists` arm built an Acknowledge and it did not. + let not_found = emitted + .find("IndexError :: NotFound =>") + .expect("not-found arm emitted"); + let tail = &emitted[not_found..]; + let ack = tail + .find("Operation :: Acknowledge") + .expect("not-found arm must acknowledge its events"); + let returns = tail + .find("Err (WorkTableError :: NotFound)") + .expect("not-found arm returns"); + assert!( + ack < returns, + "the not-found arm returns before acknowledging, which gaps the stream:\n{emitted}" + ); + + // Same for the removal side, where the events were dropped by a bare + // `res?` before the extend that would have carried them. + let removal = emitted + .find("process_difference_remove_cdc (link , diffs)") + .expect("old-key removal emitted"); + let tail = &emitted[removal..]; + let ack = tail + .find("Operation :: Acknowledge") + .expect("a failed removal must acknowledge its events"); + let extend = tail + .find("op . extend_secondary_key_events") + .expect("successful removal extends the operation"); + assert!( + ack < extend, + "a failed removal propagates before acknowledging, which gaps the stream:\n{emitted}" + ); } } diff --git a/docs/TODO.md b/docs/TODO.md index 8b6f79fb..6e8d0a85 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -140,7 +140,7 @@ not be created outside the thread and tree that uses it. Nothing in either repository does; every threaded test builds its guard inside the spawned closure. -### Persistence event gap: three leak sites found, instrumented not yet fixed +### Persistence event gap: three leak sites found, instrumented, and fixed Updated 2026-09-07. This section previously said the cause was unknown and that the next step was instrumentation rather than a repro hunt. The instrumentation @@ -192,12 +192,36 @@ Always compiled, gated at run time on `debug_assertions` or `WT_EVENT_LEDGER`, which puts it on exactly where the bug appears, since the stall needs a full debug `--all-features` run. -Not fixed, and not compiled. The three sites need the same treatment the -rollback arms already use: build an `Acknowledge` carrying the orphaned events -before propagating the error. +**Fixed 2026-09-08.** All three sites now do what the rollback arms already did: +build an `Acknowledge` carrying the orphaned events and apply it before +propagating the error. The two `res?` sites became `if let Err(e) = res` so the +events are moved into the acknowledge and the error is returned explicitly, and +the `NotFound` arm acknowledges the events its sibling arm was already +acknowledging. + +The events are **moved** into the acknowledge rather than cloned, and that is +load-bearing rather than tidy. Cloning them fails to compile: the events type is +still an inference variable at that point in the generated code, pinned only by +the `op.extend_secondary_key_events` call further down, and method resolution for +`.clone()` needs the type resolved where the call is written. The result is an +`E0282` reported against the `worktable!` invocation with no inner span, which is +an expensive thing to diagnose twice. + +Covered by emitted-token assertions in both generators +(`indexed_update_write_failure_unwinds_and_acknowledges` and +`delete_data_failure_restores_indexes_and_acknowledges`), which assert the +acknowledge is emitted **before** the return or the extend rather than merely +present somewhere in the output. The write failure itself is not forcible through +the public API, so the wiring is pinned on the tokens, which is the same approach +those tests already took. + +The in-memory generator has the same two `NotFound` arms +(`codegen/src/generators/in_memory/queries/update.rs:528` and `552`) and they are +correctly untouched: there is no persistence stream behind them to gap. ## Housekeeping -- `CHANGELOG.md` stops at 0.4.1, long before the 1.0.0-beta line. +- `CHANGELOG.md` was backfilled to 0.3.10 on 2026-09-08. It previously stopped at + beta.18. - `.github/workflows/rust.yml` has no `cargo fmt --check` job, so formatting drift accumulates unnoticed; `scripts/ci-local.sh` does check it, which makes the script stricter than CI rather than equal to it. From 670fca952fd759c5d2bcefd5ab74f6e10d001188 Mon Sep 17 00:00:00 2001 From: meh Date: Tue, 8 Sep 2026 19:03:21 +0700 Subject: [PATCH 09/14] Stop copying the page list every time a page is appended `DataPages::pages` was a `Vec` behind an `ArcSwap`, and both append paths cloned the whole thing to add one page. Every page in it is an `Arc`, so the clone was one atomic increment per existing page, each touching a separately allocated page header: a cache miss apiece. Appending page N cost O(N), and filling a table cost O(N^2). It hid behind row size, because row size is what decides how many pages a table has. At 256 bytes a page holds sixty-odd rows and twenty thousand rows is three hundred pages, where the copy is invisible and per-row cost is flat. At 4 KiB a page holds three, the same rows are six thousand pages, and per-row cost climbed from 2.95 to 30.08 microseconds across the load. The pages now live in fixed-size chunks, so an append copies one chunk and the spine of chunk pointers rather than every page. Readers still take an `ArcSwap` snapshot and never block, and page access still goes through the directory first, so the read path is unchanged. Measured over twenty thousand 4 KiB rows, per-row cost goes flat - 1.53 to 1.95 microseconds against 2.95 to 30.08 - and inserting into an unpersisted table goes from 254 MB/s to 3062. A persisted table gains far less, 208 to 279 MB/s, because with the copy gone persistence is what the load now waits on. Co-Authored-By: Claude Opus 5 --- src/in_memory/pages.rs | 151 +++++++++++++++++++++++++++++------------ 1 file changed, 106 insertions(+), 45 deletions(-) diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index f118dff5..e7c1ad4a 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -67,6 +67,88 @@ const RECLAIM_BATCH_LIMIT: usize = 256; /// gain. const RECLAIM_BACKLOG_TRIGGER: usize = RECLAIM_BATCH_LIMIT; +/// Pages per chunk of [`PageList`]. +/// +/// The list is copy-on-write, so an append copies whatever the writer has to +/// replace. Chunking bounds that at one chunk plus the (much shorter) spine, +/// instead of the whole list. +const PAGE_LIST_CHUNK: usize = 256; + +/// Owns every page, and appends one without copying the ones already there. +/// +/// **This was a `Vec` behind an `ArcSwap`, and appending cloned all of it.** +/// Every existing page is an `Arc`, so the clone was one atomic increment per +/// page, each touching a separately allocated page header: a cache miss apiece. +/// Appending page N cost O(N) and filling a table cost O(N^2). It only showed +/// up with large rows, because those are what make pages plentiful - at 4 KiB a +/// row, three rows to a page, per-row insert cost grew tenfold over twenty +/// thousand rows while a 256-byte row stayed flat. +/// +/// Readers still take an `ArcSwap` snapshot and never block. +#[derive(Debug)] +struct PageList { + chunks: ArcSwap>>>>, +} + +impl PageList { + fn from_pages(pages: Vec>) -> Self { + let chunks = pages + .chunks(PAGE_LIST_CHUNK) + .map(|chunk| Arc::new(chunk.to_vec())) + .collect::>(); + Self { + chunks: ArcSwap::from_pointee(chunks), + } + } + + /// Append a page. Copies the last chunk, or starts a new one, plus the + /// spine of chunk pointers. + fn push(&self, page: Arc) { + let chunks = self.chunks.load_full(); + let mut next = (*chunks).clone(); + match next.last() { + // Every chunk but the last is full, so only the last can take one. + Some(last) if last.len() < PAGE_LIST_CHUNK => { + let mut grown = (**last).clone(); + grown.push(page); + *next.last_mut().expect("the branch matched on it") = Arc::new(grown); + } + _ => { + let mut chunk = Vec::with_capacity(PAGE_LIST_CHUNK); + chunk.push(page); + next.push(Arc::new(chunk)); + } + } + self.chunks.store(Arc::new(next)); + } + + fn len(&self) -> usize { + let chunks = self.chunks.load(); + match chunks.last() { + None => 0, + // Full but for the last, so its length is the only remainder. + Some(last) => (chunks.len() - 1) * PAGE_LIST_CHUNK + last.len(), + } + } + + fn get(&self, index: usize) -> Option> { + let chunks = self.chunks.load(); + chunks + .get(index / PAGE_LIST_CHUNK)? + .get(index % PAGE_LIST_CHUNK) + .cloned() + } + + fn for_each(&self, mut visit: impl FnMut(&Arc)) { + let chunks = self.chunks.load(); + for chunk in chunks.iter() { + for page in chunk.iter() { + visit(page); + } + } + } +} + #[derive(Debug)] struct PageDirectoryChunk { pages: [AtomicPtr; PAGE_DIRECTORY_CHUNK_SIZE], @@ -261,7 +343,7 @@ where /// Immutable page-directory snapshots. Reads load one snapshot without a /// shared read-modify-write; rare growth copies and swaps the short vector. - pages: ArcSwap::WrappedRow, DATA_LENGTH>>>>, + pages: PageList::WrappedRow, DATA_LENGTH>>, /// Stable pointers for point access without ArcSwap's shared snapshot /// accounting. The corresponding `Arc`s remain owned by `pages`. page_directory: PageDirectory::WrappedRow, DATA_LENGTH>>, @@ -303,11 +385,11 @@ where return Ok(page); } - let page = { - let pages = self.pages.load(); - pages.get(index).map(Arc::as_ptr) - } - .ok_or(ExecutionError::PageNotFound(page_id))?; + let page = self + .pages + .get(index) + .map(|page| Arc::as_ptr(&page)) + .ok_or(ExecutionError::PageNotFound(page_id))?; // SAFETY: as above, the current directory retains this allocation and // all future directory snapshots clone its Arc. @@ -578,7 +660,7 @@ where queued_page_retirements: AtomicUsize::new(0), // We are starting ID's from `1` because `0`'s page in file is info page. page_directory: PageDirectory::new(&pages), - pages: ArcSwap::from_pointee(pages), + pages: PageList::from_pages(pages), pages_write: Mutex::new(()), empty_links: EmptyLinkRegistry::::default(), empty_pages: Default::default(), @@ -602,7 +684,7 @@ where pending_retirements: AtomicUsize::new(0), queued_page_retirements: AtomicUsize::new(0), page_directory, - pages: ArcSwap::from_pointee(vec), + pages: PageList::from_pages(vec), pages_write: Mutex::new(()), empty_links: EmptyLinkRegistry::default(), empty_pages: Default::default(), @@ -743,18 +825,8 @@ where let _write = self.pages_write.lock(); if tried_page == page_id_mapper(self.current_page_id.load(Ordering::Acquire) as usize) { let index = self.last_page_id.fetch_add(1, Ordering::AcqRel) + 1; - let pages = self.pages.load_full(); - let mut next = (*pages).clone(); let page = Arc::new(Data::new(index.into())); - next.push(page.clone()); - debug_assert_eq!(next.len(), pages.len() + 1); - debug_assert!( - next[..pages.len()] - .iter() - .zip(pages.iter()) - .all(|(new, old)| Arc::ptr_eq(new, old)) - ); - self.pages.store(Arc::new(next)); + self.pages.push(page.clone()); self.publish_page(&page); self.current_page_id.store(index, Ordering::Release); } @@ -771,9 +843,11 @@ where }; if let Some(page_id) = page_id { - let pages = self.pages.load(); let index = page_id_mapper(page_id.into()); - let page = pages[index].clone(); + let page = self + .pages + .get(index) + .expect("an empty page id names a page that was allocated"); { let _page_guard = page.access.write(); page.reset(); @@ -785,17 +859,7 @@ where let _write = self.pages_write.lock(); let index = self.last_page_id.fetch_add(1, Ordering::AcqRel) + 1; let page = Arc::new(Data::new(index.into())); - let pages = self.pages.load_full(); - let mut next = (*pages).clone(); - next.push(page.clone()); - debug_assert_eq!(next.len(), pages.len() + 1); - debug_assert!( - next[..pages.len()] - .iter() - .zip(pages.iter()) - .all(|(new, old)| Arc::ptr_eq(new, old)) - ); - self.pages.store(Arc::new(next)); + self.pages.push(page.clone()); self.publish_page(&page); page @@ -842,12 +906,12 @@ where + Deserialize<::WrappedRow, HighDeserializer> + for<'a> rkyv::bytecheck::CheckBytes>, { - let pages = self.pages.load(); let page_id: usize = link.page_id.into(); let page_index = page_id .checked_sub(1) .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let page = pages + let page = self + .pages .get(page_index) .ok_or(ExecutionError::PageNotFound(link.page_id))?; let wrapped = page.get_row_checked(link).map_err(ExecutionError::DataPageError)?; @@ -1119,9 +1183,7 @@ where } pub fn get_page(&self, page_id: PageId) -> Option::WrappedRow, DATA_LENGTH>>> { - let pages = self.pages.load(); - let page = pages.get(page_id_mapper(page_id.into()))?; - Some(page.clone()) + self.pages.get(page_id_mapper(page_id.into())) } /// Registers an already-indexed cell while rebuilding runtime metadata @@ -1170,11 +1232,10 @@ where /// Approximate under concurrency: a failing `save_row`'s transient /// reservation may be counted before its rollback. Metrics only. pub fn used_bytes(&self) -> u64 { - let pages = self.pages.load(); - pages - .iter() - .map(|p| u64::from(p.free_offset.load(Ordering::Relaxed))) - .sum() + let mut total = 0u64; + self.pages + .for_each(|page| total += u64::from(page.free_offset.load(Ordering::Relaxed))); + total } /// Copies a row to another page without exposing either mutable byte @@ -1230,7 +1291,7 @@ where } pub fn get_page_count(&self) -> usize { - self.pages.load().len() + self.pages.len() } pub fn get_empty_links(&self) -> Vec { @@ -1253,12 +1314,12 @@ where /// figure without it cannot be checked, because a sweep that never runs /// looks exactly like a sweep that is free. pub fn allocated_pages(&self) -> usize { - self.pages.load().len() + self.pages.len() } /// Heap bytes reserved by the fixed-size data-page allocations. pub fn allocated_bytes(&self) -> usize { - self.pages.load().len() * std::mem::size_of::::WrappedRow, DATA_LENGTH>>() + self.pages.len() * std::mem::size_of::::WrappedRow, DATA_LENGTH>>() } /// Pages allocated but currently on the empty list, so reusable without From 08cc8651654b622e21603ffd72f39bf3365a353f Mon Sep 17 00:00:00 2001 From: meh Date: Tue, 8 Sep 2026 19:03:42 +0700 Subject: [PATCH 10/14] Let the page directory reach past 64 MiB, and measure the insert path The directory holds stable page pointers so a point access does not take an `ArcSwap` snapshot, and it reached 64 * 64 = 4,096 pages, which is 64 MiB at the default page size. Past that `publish` returned early and every access fell back to the snapshot. A table of 4 KiB rows crosses that after twelve thousand rows, which is not a large table. Raising it to 1,024 roots reaches 1 GiB for an 8 KiB array of pointers. On its own it made no measurable difference to insert cost, because the page list copy was the term that mattered; this moves a ceiling rather than removing a cost, and it is committed separately so the two are not confused. The tests are the harness the page list work was measured with: whether insert cost scales with row size, whether it grows as the table fills, what the floor is with few pages, and what the persistence path sustains end to end. They are all `#[ignore]`d, since they are measurements and not assertions. Co-Authored-By: Claude Opus 5 --- src/in_memory/pages.rs | 13 +- tests/persistence/insert_cost_shape.rs | 216 +++++++++++++++++++++ tests/persistence/local_write_bandwidth.rs | 201 +++++++++++++++++++ tests/persistence/mod.rs | 2 + 4 files changed, 431 insertions(+), 1 deletion(-) create mode 100644 tests/persistence/insert_cost_shape.rs create mode 100644 tests/persistence/local_write_bandwidth.rs diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index e7c1ad4a..a28d210d 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -37,7 +37,18 @@ fn page_id_mapper(page_id: usize) -> usize { } const PAGE_DIRECTORY_CHUNK_SIZE: usize = 64; -const PAGE_DIRECTORY_ROOTS: usize = 64; +/// Roots in the page directory, so its reach is `ROOTS * CHUNK_SIZE` pages. +/// +/// **This was 64, which reached 4,096 pages: 64 MiB at the default page size.** +/// Past that, `publish` returns early and every page access falls back to an +/// `ArcSwap` snapshot of the owning list. A table of 4 KiB rows, which fit +/// three to a page, crosses it after twelve thousand rows. +/// +/// Raising it did not measurably change insert cost on that fixture - the +/// copy-on-write page list dominated, and still does at these sizes - so this +/// is a ceiling being moved rather than a cost being removed. 1,024 roots +/// reach 65,536 pages, or 1 GiB, for an 8 KiB array of pointers. +const PAGE_DIRECTORY_ROOTS: usize = 1024; const GHOSTED: u8 = 1 << 0; const DELETED: u8 = 1 << 1; const VACUUMED: u8 = 1 << 2; diff --git a/tests/persistence/insert_cost_shape.rs b/tests/persistence/insert_cost_shape.rs new file mode 100644 index 00000000..72cc3ba9 --- /dev/null +++ b/tests/persistence/insert_cost_shape.rs @@ -0,0 +1,216 @@ +//! Where the time in an insert goes, since it is not the disk. +//! +//! WorkTable's bulk load runs at about 200 MB/s while the write path under it +//! does gigabytes, and removing persistence entirely changes nothing. So the +//! cost is in the insert. This asks the first question that splits the +//! candidates: does it scale with the size of the row, or is it a fixed price +//! per row? + +use worktable::prelude::*; +use worktable_codegen::worktable; + +worktable!( + name: InsertShape, + columns: { + id: u64 primary_key, + payload: String, + } +); + +#[test] +#[ignore = "a measurement, not an assertion"] +fn does_insert_cost_scale_with_row_size() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + const ROWS: u64 = 25_000; + println!(" payload rows/s us/row MB/s"); + for size in [8usize, 64, 256, 1024, 2048, 4096, 8192] { + let payload = "x".repeat(size); + // Warm: allocator and the table's first growth are not the subject. + { + let warm = InsertShapeWorkTable::default(); + for id in 0..1_000u64 { + warm.insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + } + let table = InsertShapeWorkTable::default(); + let at = std::time::Instant::now(); + for id in 0..ROWS { + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + let elapsed = at.elapsed().as_secs_f64(); + println!( + " {size:>7} {:>9.0} {:>8.2} {:>7.0}", + ROWS as f64 / elapsed, + elapsed * 1e6 / ROWS as f64, + (ROWS as usize * size) as f64 / 1e6 / elapsed, + ); + } + }); +} + +/// The same rows, with the table taken out of it: what the loop costs when all +/// it does is build the row and drop it. Anything the insert arm spends beyond +/// this is the table. +#[test] +#[ignore = "a measurement, not an assertion"] +fn what_the_loop_costs_without_the_table() { + const ROWS: u64 = 25_000; + println!(" payload rows/s us/row"); + for size in [8usize, 4096] { + let payload = "x".repeat(size); + let at = std::time::Instant::now(); + let mut sink = 0usize; + for id in 0..ROWS { + let row = InsertShapeRow { + id, + payload: payload.clone(), + }; + sink = sink.wrapping_add(row.payload.len()); + std::hint::black_box(&row); + } + let elapsed = at.elapsed().as_secs_f64(); + std::hint::black_box(sink); + println!( + " {size:>7} {:>9.0} {:>8.3}", + ROWS as f64 / elapsed, + elapsed * 1e6 / ROWS as f64, + ); + } +} + +/// A long run of the expensive case, so a sampling profiler has something to +/// look at. Not a measurement in itself. +#[test] +#[ignore = "for profiling only"] +fn keep_inserting_four_kilobyte_rows() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let payload = "x".repeat(4096); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while std::time::Instant::now() < deadline { + let table = InsertShapeWorkTable::default(); + for id in 0..20_000u64 { + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + } + }); +} + +/// Does an insert get slower as the table fills? +/// +/// A cost that scales with rows already present is O(n) per insert and O(n^2) +/// overall, which is what a super-linear response to row size would look like +/// if bigger rows simply reach any given page count sooner. +#[test] +#[ignore = "a measurement, not an assertion"] +fn does_insert_slow_down_as_the_table_fills() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + for size in [256usize, 4096] { + let payload = "x".repeat(size); + let table = InsertShapeWorkTable::default(); + const BLOCK: u64 = 2_000; + const BLOCKS: u64 = 10; + println!(" payload {size}: us/row by block of {BLOCK}"); + let mut line = String::new(); + for block in 0..BLOCKS { + let at = std::time::Instant::now(); + for n in 0..BLOCK { + let id = block * BLOCK + n; + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + let per_row = at.elapsed().as_secs_f64() * 1e6 / BLOCK as f64; + line.push_str(&format!("{per_row:>8.2}")); + } + println!(" {line}"); + } + }); +} + +/// The per-row cost with the page-list clone nearly absent, which is the floor +/// a fix for it would approach. +/// +/// The clone is O(pages), so a table that has barely any pages barely pays it. +/// Timing small tables gives the cost of everything else: the row clone, the +/// rkyv serialize, and the copy into the page. +#[test] +#[ignore = "a measurement, not an assertion"] +fn the_floor_with_almost_no_pages() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + println!(" payload rows pages us/row MB/s"); + for size in [256usize, 1024, 4096] { + let payload = "x".repeat(size); + let per_page = (16356 / size).max(1); + for rows in [30u64, 120, 480] { + // Median of several fresh tables: a single small run is noise. + let mut samples = Vec::new(); + for _ in 0..25 { + let table = InsertShapeWorkTable::default(); + let at = std::time::Instant::now(); + for id in 0..rows { + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + samples.push(at.elapsed().as_secs_f64() / rows as f64); + } + samples.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let per_row = samples[samples.len() / 2]; + println!( + " {size:>7} {rows:>5} {:>5} {:>7.3} {:>7.0}", + rows as usize / per_page, + per_row * 1e6, + size as f64 / 1e6 / per_row, + ); + } + } + }); +} diff --git a/tests/persistence/local_write_bandwidth.rs b/tests/persistence/local_write_bandwidth.rs new file mode 100644 index 00000000..ab36e386 --- /dev/null +++ b/tests/persistence/local_write_bandwidth.rs @@ -0,0 +1,201 @@ +//! What WorkTable's local persistence path sustains, in bytes per second. +//! +//! Measured through `insert` and `wait_for_ops` rather than against +//! `persist_page` directly, so it counts everything the engine does to make a +//! write durable and not just the call at the bottom of it. + +use worktable::prelude::PersistedWorkTable; +use worktable::prelude::*; +use worktable_codegen::worktable; + +use crate::remove_dir_if_exists; + +worktable!( + name: WriteBandwidth, + persist: true, + columns: { + id: u64 primary_key, + payload: String, + } +); + +// The same table without persistence, so the cost of being a WorkTable can be +// told apart from the cost of writing to disk. Anything this arm spends is +// spent by the persisted one too, before any page is written. +worktable!( + name: WriteBandwidthMemory, + columns: { + id: u64 primary_key, + payload: String, + } +); + +/// A page of the on-disk format, which is the unit a write actually lands in. +const PAGE: usize = 4096 * 4; + +/// Per-page checksums of every file, so two snapshots say how many pages a +/// stretch of work really wrote. +fn page_checksums(dir: &str) -> Vec<(String, Vec)> { + fn walk(dir: &std::path::Path, into: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + if path.is_dir() { + walk(&path, into); + } else if path.is_file() { + into.push(path); + } + } + } + let mut paths = Vec::new(); + walk(std::path::Path::new(dir), &mut paths); + paths.sort(); + paths + .into_iter() + .map(|path| { + let bytes = std::fs::read(&path).expect("a table file"); + ( + path.to_string_lossy().into_owned(), + bytes.chunks(PAGE).map(crc32fast::hash).collect(), + ) + }) + .collect() +} + +/// Bytes that differ between two snapshots, counted a page at a time. +fn written_bytes(before: &[(String, Vec)], after: &[(String, Vec)]) -> u64 { + let mut pages = 0u64; + for (name, now) in after { + let then = before + .iter() + .find(|(n, _)| n == name) + .map(|(_, c)| c.as_slice()) + .unwrap_or(&[]); + pages += now + .iter() + .enumerate() + .filter(|(index, checksum)| then.get(*index) != Some(*checksum)) + .count() as u64; + } + pages * PAGE as u64 +} + +fn table_bytes(dir: &str) -> u64 { + fn walk(dir: &std::path::Path, total: &mut u64) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + if path.is_dir() { + walk(&path, total); + } else if let Ok(meta) = entry.metadata() { + *total += meta.len(); + } + } + } + let mut total = 0; + walk(std::path::Path::new(dir), &mut total); + total +} + +#[test] +#[ignore = "a measurement, not an assertion"] +fn local_write_bandwidth() { + let dir = "tests/data/local_write_bandwidth"; + let config = DiskConfig::new_with_table_name( + dir, + WriteBandwidthWorkTable::name_snake_case(), + WriteBandwidthWorkTable::version(), + ); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_io() + .enable_time() + .build() + .unwrap(); + + runtime.block_on(async { + remove_dir_if_exists(dir.to_string()).await; + let engine = WriteBandwidthPersistenceEngine::new(config.clone()).await.unwrap(); + let table = WriteBandwidthWorkTable::load(engine).await.unwrap(); + let payload = "x".repeat(4096); + + // ---- bulk load: consecutive pages, which is the batch path's shape + const ROWS: u64 = 25_000; + let at = std::time::Instant::now(); + for id in 0..ROWS { + table + .insert(WriteBandwidthRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + table.wait_for_ops().await.expect("the queue drains"); + let bulk = at.elapsed().as_secs_f64(); + let bytes = table_bytes(dir); + + // ---- scattered updates into what already exists + const UPDATES: u64 = 2_000; + let replacement = "y".repeat(4096); + let pages_before = page_checksums(dir); + let at = std::time::Instant::now(); + for n in 0..UPDATES { + // Spread across the whole table rather than a contiguous run. + let id = (n * (ROWS / UPDATES)) % ROWS; + table + .update(WriteBandwidthRow { + id, + payload: replacement.clone(), + }) + .await + .unwrap(); + } + table.wait_for_ops().await.expect("the queue drains"); + let scattered = at.elapsed().as_secs_f64(); + let scattered_bytes = written_bytes(&pages_before, &page_checksums(dir)); + + println!("table {:.1} MB on disk", bytes as f64 / 1e6); + println!( + " bulk load, {ROWS} rows : {:>8.1} ms {:>7.0} MB/s {:>8.0} rows/s", + bulk * 1e3, + bytes as f64 / 1e6 / bulk, + ROWS as f64 / bulk, + ); + println!( + " scattered update, {UPDATES} rows : {:>8.1} ms {:>7.0} MB/s {:>8.0} rows/s {:.1} MB written", + scattered * 1e3, + scattered_bytes as f64 / 1e6 / scattered, + UPDATES as f64 / scattered, + scattered_bytes as f64 / 1e6, + ); + + // ---- the same inserts with nothing underneath them + let memory = WriteBandwidthMemoryWorkTable::default(); + let at = std::time::Instant::now(); + for id in 0..ROWS { + memory + .insert(WriteBandwidthMemoryRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + let in_memory = at.elapsed().as_secs_f64(); + println!( + " in memory, no persistence : {:>8.1} ms {:>7.0} MB/s {:>8.0} rows/s", + in_memory * 1e3, + bytes as f64 / 1e6 / in_memory, + ROWS as f64 / in_memory, + ); + println!( + " ^ persistence adds {:.1} ms on top of {:.1} ms of table work", + (bulk - in_memory) * 1e3, + in_memory * 1e3, + ); + + remove_dir_if_exists(dir.to_string()).await; + }); +} diff --git a/tests/persistence/mod.rs b/tests/persistence/mod.rs index e282cc9f..e07610a5 100644 --- a/tests/persistence/mod.rs +++ b/tests/persistence/mod.rs @@ -10,9 +10,11 @@ mod exact_boundary_load; mod failure; mod in_place_durability; mod index_page; +mod insert_cost_shape; mod insert_many; mod insert_many_bench; mod loaded_index_growth; +mod local_write_bandwidth; mod multi_row_backend_order; mod read; mod recovery_load; From c852f9590466008be9890a198f345c3984ae0a43 Mon Sep 17 00:00:00 2001 From: meh Date: Tue, 8 Sep 2026 19:15:56 +0700 Subject: [PATCH 11/14] Measure insert latency as two populations, not one distribution An insert that fits in the page already open and one that has to add a page are different operations with different costs, and averaging them hides the second behind the first. How much it hides depends on row size: at 256 bytes one insert in sixty allocates, so allocation is a tail event, while at 4 KiB one in three does and it is a third of all traffic. Split by whether the page count moved, the page list change reads as what it is - a tail-latency fix that leaves the common path alone. On 4 KiB rows the allocating population goes from 59.67 to 7.08 microseconds at p50, 133.38 to 14.46 at p99, and 451.54 to 42.33 at its worst, while the existing-page population does not move. On 256-byte rows the same change is worth about a third, on the one insert in sixty that pays it. Co-Authored-By: Claude Opus 5 --- tests/persistence/insert_latency.rs | 124 ++++++++++++++++++++++++++++ tests/persistence/mod.rs | 1 + 2 files changed, 125 insertions(+) create mode 100644 tests/persistence/insert_latency.rs diff --git a/tests/persistence/insert_latency.rs b/tests/persistence/insert_latency.rs new file mode 100644 index 00000000..00afe6a0 --- /dev/null +++ b/tests/persistence/insert_latency.rs @@ -0,0 +1,124 @@ +//! Per-insert latency, split by whether the insert had to allocate a page. +//! +//! These are two populations, not one distribution. An insert that fits in the +//! page already open is cheap; one that has to add a page pays for the page +//! list as well. Blending them hides the second behind the first, and how much +//! it hides depends on row size: at 4 KiB a page holds three rows, so a third +//! of all inserts allocate and the expensive population is not a tail at all. + +use worktable::prelude::PersistedWorkTable; +use worktable::prelude::*; +use worktable_codegen::worktable; + +use crate::remove_dir_if_exists; + +worktable!( + name: InsertLatency, + persist: true, + columns: { + id: u64 primary_key, + payload: String, + } +); + +worktable!( + name: InsertLatencyMemory, + columns: { + id: u64 primary_key, + payload: String, + } +); + +fn report(label: &str, mut us: Vec, of: usize) { + if us.is_empty() { + println!(" {label:<34} (none)"); + return; + } + us.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let at = |q: f64| us[((us.len() as f64 * q) as usize).min(us.len() - 1)]; + println!( + " {label:<34} {:>5.1}% of inserts p50 {:>7.2} p99 {:>8.2} max {:>9.2} (us)", + 100.0 * us.len() as f64 / of as f64, + at(0.50), + at(0.99), + us[us.len() - 1], + ); +} + +#[test] +#[ignore = "a measurement, not an assertion"] +fn insert_latency_split_by_page_allocation() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + const ROWS: u64 = 20_000; + for size in [256usize, 4096] { + let payload = "x".repeat(size); + println!("payload {size} B, {ROWS} rows"); + + // ---- no persistence + let table = InsertLatencyMemoryWorkTable::default(); + let (mut same, mut fresh) = (Vec::new(), Vec::new()); + let mut pages = table.0.data.get_page_count(); + for id in 0..ROWS { + let at = std::time::Instant::now(); + table + .insert(InsertLatencyMemoryRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + let took = at.elapsed().as_secs_f64() * 1e6; + let now = table.0.data.get_page_count(); + if now == pages { + same.push(took) + } else { + fresh.push(took) + } + pages = now; + } + report("in memory, existing page", same, ROWS as usize); + report("in memory, allocated a page", fresh, ROWS as usize); + + // ---- persisted + let dir = "tests/data/insert_latency"; + remove_dir_if_exists(dir.to_string()).await; + let config = DiskConfig::new_with_table_name( + dir, + InsertLatencyWorkTable::name_snake_case(), + InsertLatencyWorkTable::version(), + ); + let engine = InsertLatencyPersistenceEngine::new(config).await.unwrap(); + let persisted = InsertLatencyWorkTable::load(engine).await.unwrap(); + let (mut same, mut fresh) = (Vec::new(), Vec::new()); + let mut pages = persisted.0.data.get_page_count(); + for id in 0..ROWS { + let at = std::time::Instant::now(); + persisted + .insert(InsertLatencyRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + let took = at.elapsed().as_secs_f64() * 1e6; + let now = persisted.0.data.get_page_count(); + if now == pages { + same.push(took) + } else { + fresh.push(took) + } + pages = now; + } + report("persisted, existing page", same, ROWS as usize); + report("persisted, allocated a page", fresh, ROWS as usize); + persisted.wait_for_ops().await.expect("the queue drains"); + remove_dir_if_exists(dir.to_string()).await; + } + }); +} diff --git a/tests/persistence/mod.rs b/tests/persistence/mod.rs index e07610a5..06864a9d 100644 --- a/tests/persistence/mod.rs +++ b/tests/persistence/mod.rs @@ -11,6 +11,7 @@ mod failure; mod in_place_durability; mod index_page; mod insert_cost_shape; +mod insert_latency; mod insert_many; mod insert_many_bench; mod loaded_index_growth; From db5e2836f5e1af81ae8c884b010fbd05c6036ab1 Mon Sep 17 00:00:00 2001 From: meh Date: Tue, 8 Sep 2026 20:52:57 +0700 Subject: [PATCH 12/14] Borrow the page on the link read instead of cloning its Arc The chunked page list handed back an owned `Arc` from `get`, which costs an atomic increment and the matching decrement on drop. The link-based read is a read path and needs the page only for the length of one call, so it paid both for nothing. It measured. A delete went from 665 to 751 ns, while a select by primary key - which goes through the page directory and never touches this - did not move. Borrowing through `with_page` puts delete back at 654 ns, and the criterion cases either side of it are unchanged or better: insert 442 -> 86 ns, select by primary key 21.0 -> 20.1 ns. Found by running the benchmark suite that already existed rather than the ad-hoc probes the rest of this work was measured with. Co-Authored-By: Claude Opus 5 --- src/in_memory/pages.rs | 41 +++++++++++----- tests/persistence/mod.rs | 1 + tests/persistence/persistence_is_what.rs | 60 ++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 tests/persistence/persistence_is_what.rs diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index a28d210d..08cfeaa1 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -150,6 +150,21 @@ impl PageList { .cloned() } + /// Run `visit` against the page at `index`, borrowing it rather than + /// handing back an owned `Arc`. + /// + /// **`get` costs an atomic increment and the matching decrement on drop.** + /// A read that only needs the page for the length of one call pays both for + /// nothing, and it is measurable: routing the link-based read through `get` + /// moved a delete from 665 to 751 ns, while a select by primary key - which + /// goes through the page directory and never touches this - did not move at + /// all. + fn with_page(&self, index: usize, visit: impl FnOnce(&T) -> R) -> Option { + let chunks = self.chunks.load(); + let page = chunks.get(index / PAGE_LIST_CHUNK)?.get(index % PAGE_LIST_CHUNK)?; + Some(visit(page)) + } + fn for_each(&self, mut visit: impl FnMut(&Arc)) { let chunks = self.chunks.load(); for chunk in chunks.iter() { @@ -921,18 +936,20 @@ where let page_index = page_id .checked_sub(1) .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let page = self - .pages - .get(page_index) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let wrapped = page.get_row_checked(link).map_err(ExecutionError::DataPageError)?; - if wrapped.is_ghosted() { - return Err(ExecutionError::Ghosted); - } - if wrapped.is_deleted() { - return Err(ExecutionError::Deleted); - } - Ok(wrapped.get_inner()) + // Borrowed rather than cloned: this is a read path and an `Arc` bump + // here showed up as a 13% slower delete. + self.pages + .with_page(page_index, |page| { + let wrapped = page.get_row_checked(link).map_err(ExecutionError::DataPageError)?; + if wrapped.is_ghosted() { + return Err(ExecutionError::Ghosted); + } + if wrapped.is_deleted() { + return Err(ExecutionError::Deleted); + } + Ok(wrapped.get_inner()) + }) + .ok_or(ExecutionError::PageNotFound(link.page_id))? } pub fn select_non_vacuumed(&self, link: Link) -> Result diff --git a/tests/persistence/mod.rs b/tests/persistence/mod.rs index 06864a9d..9b2535a9 100644 --- a/tests/persistence/mod.rs +++ b/tests/persistence/mod.rs @@ -17,6 +17,7 @@ mod insert_many_bench; mod loaded_index_growth; mod local_write_bandwidth; mod multi_row_backend_order; +mod persistence_is_what; mod read; mod recovery_load; mod same_size_in_place; diff --git a/tests/persistence/persistence_is_what.rs b/tests/persistence/persistence_is_what.rs new file mode 100644 index 00000000..ca17484a --- /dev/null +++ b/tests/persistence/persistence_is_what.rs @@ -0,0 +1,60 @@ +//! Is the persistence path waiting, or working? +//! +//! Bulk load persists at about 310 MB/s while the disk under it does gigabytes +//! and DataBucket's own write path does 500+ MB/s single threaded. So something +//! between them is the limit. CPU time against wall time says which kind of +//! limit it is: near or above wall means it is computing, well under means it +//! is waiting. + +use worktable::prelude::PersistedWorkTable; +use worktable::prelude::*; +use worktable_codegen::worktable; + +use crate::remove_dir_if_exists; + +worktable!( + name: PersistShape, + persist: true, + columns: { id: u64 primary_key, payload: String } +); + +#[test] +#[ignore = "a measurement, not an assertion"] +fn is_persistence_waiting_or_working() { + let dir = "tests/data/persistence_is_what"; + let config = DiskConfig::new_with_table_name( + dir, + PersistShapeWorkTable::name_snake_case(), + PersistShapeWorkTable::version(), + ); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + remove_dir_if_exists(dir.to_string()).await; + let engine = PersistShapePersistenceEngine::new(config).await.unwrap(); + let table = PersistShapeWorkTable::load(engine).await.unwrap(); + let payload = "x".repeat(4096); + + // Marked so the times either side can be attributed to this and not to + // building the table or tearing it down. + println!("MARK begin"); + let at = std::time::Instant::now(); + for id in 0..25_000u64 { + table + .insert(PersistShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + table.wait_for_ops().await.expect("the queue drains"); + println!("MARK end {:.1} ms", at.elapsed().as_secs_f64() * 1e3); + + remove_dir_if_exists(dir.to_string()).await; + }); +} From f30eff4d74db846fe887427dc0766b7aa275d586 Mon Sep 17 00:00:00 2001 From: meh Date: Wed, 9 Sep 2026 01:33:06 +0700 Subject: [PATCH 13/14] Call the insert benchmark what it measures It was named "bulk load", which reads as loading a table from disk. It inserts 25,000 rows. Co-Authored-By: Claude Opus 5 --- tests/persistence/local_write_bandwidth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/persistence/local_write_bandwidth.rs b/tests/persistence/local_write_bandwidth.rs index ab36e386..5d74f201 100644 --- a/tests/persistence/local_write_bandwidth.rs +++ b/tests/persistence/local_write_bandwidth.rs @@ -158,7 +158,7 @@ fn local_write_bandwidth() { println!("table {:.1} MB on disk", bytes as f64 / 1e6); println!( - " bulk load, {ROWS} rows : {:>8.1} ms {:>7.0} MB/s {:>8.0} rows/s", + " bulk insert, {ROWS} rows : {:>8.1} ms {:>7.0} MB/s {:>8.0} rows/s", bulk * 1e3, bytes as f64 / 1e6 / bulk, ROWS as f64 / bulk, From ac66ae0ec8e75322ea5a0327796dcd4ca929be24 Mon Sep 17 00:00:00 2001 From: meh Date: Wed, 9 Sep 2026 16:26:17 +0700 Subject: [PATCH 14/14] Take WorkTable off tokio::fs, let a persisted table choose its page size Collapses what was PR #102 onto the arctic, event-ledger and page-list work already on this branch, so WorkTable carries one pull request. #97, #99 and #100 are folded in; #58 is not, being 224 commits behind master and already conflicted, which is its own job. `fsx` and every persistence signature go through `nagoya::io`, so the production path names no runtime. A persisted table can set `page_size`, which was refused before because the seeks computed offsets from a crate constant while the generated table threaded the configured one. Four places where the folded branches disagreed, each resolved by keeping both rather than either: * `arctic` and `ps-reclaim` keep #97's raised version floors and gain the `default-features = false` the no_std work needs. * `allocated_bytes` keeps the page list as #100 left it and takes `core::mem` from the no_std work; the `.load()` in the older branch belonged to a shape #100 replaced. * `batch.rs` and `task.rs` take the `core`/`alloc` imports, plus the `Arc` and `Location` the no_std lists had dropped and the code still uses. * The s3 test goes back to tokio's extension traits. It talks to a tokio `TcpStream` it starts itself, so the sweep that took the storage path off tokio should never have touched it. `event_ledger` is new here and was written against `std`. Its bookkeeping moves to `core` and `alloc`; only the two parts that genuinely need an operating system are gated, the backtrace capture and reading `WT_EVENT_LEDGER`, so without `std` the ledger is simply never enabled. `futures/std` goes in this crate's own `std` feature rather than on the dependency line, where a `--no-default-features` build would still have turned it on. cargo test --workspace --all-targets 933 passed ... --all-features 935 passed ... --features versioned-row-publication 1029 passed cargo clippy --workspace --all-targets [--all-features] clean cargo check --no-default-features clean --- Cargo.toml | 71 ++++- benches/cases/full_featured.rs | 18 +- benches/cases/non_unique_index.rs | 10 +- benches/cases/nonunique_arctic_vs_wti.rs | 12 +- benches/cases/partition_routing.rs | 6 +- benches/cases/simple.rs | 10 +- benches/cases/unique_index.rs | 16 +- codegen/Cargo.toml | 10 +- codegen/src/generators/in_memory/index/cdc.rs | 4 +- .../src/generators/in_memory/index/usual.rs | 4 +- codegen/src/generators/in_memory/locks.rs | 16 +- .../src/generators/in_memory/primary_key.rs | 16 +- .../generators/in_memory/queries/delete.rs | 2 +- .../src/generators/in_memory/queries/locks.rs | 6 +- .../generators/in_memory/queries/select.rs | 4 +- .../generators/in_memory/queries/update.rs | 16 +- .../src/generators/in_memory/table/impls.rs | 22 +- .../generators/in_memory/table/index_fns.rs | 12 +- codegen/src/generators/in_memory/table/mod.rs | 2 +- .../in_memory/table/select_executor.rs | 12 +- codegen/src/generators/partitions.rs | 10 +- codegen/src/generators/persist/index/cdc.rs | 4 +- codegen/src/generators/persist/index/usual.rs | 4 +- codegen/src/generators/persist/locks.rs | 16 +- codegen/src/generators/persist/primary_key.rs | 16 +- .../src/generators/persist/queries/delete.rs | 2 +- .../src/generators/persist/queries/locks.rs | 6 +- .../src/generators/persist/queries/select.rs | 4 +- .../src/generators/persist/queries/update.rs | 16 +- codegen/src/generators/persist/table/impls.rs | 32 +- .../src/generators/persist/table/index_fns.rs | 12 +- .../persist/table/select_executor.rs | 12 +- .../src/generators/read_only/index/usual.rs | 4 +- codegen/src/generators/read_only/locks.rs | 16 +- .../src/generators/read_only/primary_key.rs | 16 +- .../generators/read_only/queries/select.rs | 4 +- .../src/generators/read_only/table/impls.rs | 14 +- .../generators/read_only/table/index_fns.rs | 12 +- .../read_only/table/select_executor.rs | 12 +- codegen/src/mem_stat/mod.rs | 4 +- codegen/src/persist_index/generator.rs | 38 +-- codegen/src/persist_index/mod.rs | 2 +- codegen/src/persist_index/parser.rs | 2 +- codegen/src/persist_index/space/events.rs | 10 +- codegen/src/persist_index/space/index.rs | 17 +- codegen/src/persist_table/generator/space.rs | 46 +-- .../persist_table/generator/space_file/mod.rs | 46 +-- .../generator/space_file/worktable_impls.rs | 21 +- codegen/src/worktable/mod.rs | 28 +- docs/crate.md | 4 +- docs/page-size.md | 105 +++++++ docs/wt-user-guide.pdf | Bin 0 -> 111263 bytes docs/wt-user-guide.typ | 276 ++++++++++++++++++ dsl/src/validate.rs | 43 ++- dsl/tests/check.rs | 2 +- dsl/tests/cli.rs | 13 +- examples/guide_check.rs | 31 ++ examples/system_info_render.rs | 24 ++ src/features/s3_support.rs | 15 +- src/fsx.rs | 57 ++++ src/in_memory/data.rs | 26 +- src/in_memory/empty_link_registry.rs | 19 +- src/in_memory/pages.rs | 42 +-- src/in_memory/row.rs | 2 +- src/index/arctic.rs | 18 +- src/index/arctic_multi.rs | 18 +- src/index/available_index.rs | 1 + src/index/congee.rs | 30 +- src/index/mod.rs | 10 +- src/index/persistent_art.rs | 20 +- src/index/persistent_wti.rs | 13 +- src/index/primary_index.rs | 7 +- src/index/table_index/cdc.rs | 17 +- src/index/table_index/mod.rs | 17 +- src/index/table_index/util.rs | 5 + src/index/table_secondary_index/cdc.rs | 3 +- .../table_secondary_index/index_events.rs | 2 +- src/index/table_secondary_index/info.rs | 1 + src/index/table_secondary_index/mod.rs | 3 +- src/index/unique.rs | 24 +- src/index/unsized_node.rs | 13 +- src/lib.rs | 52 +++- src/lock/map.rs | 23 +- src/lock/mod.rs | 21 +- src/lock/row_lock.rs | 8 +- src/mem_stat/mod.rs | 56 ++-- src/mem_stat/primitives.rs | 31 +- src/migration/mod.rs | 6 +- src/partition/mod.rs | 35 +-- src/partition/tests.rs | 17 +- src/persistence/engine.rs | 9 +- src/persistence/error.rs | 15 +- src/persistence/event_ledger.rs | 59 +++- src/persistence/mod.rs | 36 ++- src/persistence/operation/batch.rs | 27 +- src/persistence/operation/mod.rs | 8 +- src/persistence/operation/operation.rs | 5 +- src/persistence/operation/util.rs | 5 +- src/persistence/readonly_engine.rs | 4 +- src/persistence/space/art_index.rs | 77 ++--- src/persistence/space/data.rs | 31 +- src/persistence/space/index/mod.rs | 91 +++--- src/persistence/space/index/page_aliases.rs | 3 +- src/persistence/space/index/reconstruct.rs | 7 +- .../space/index/table_of_contents.rs | 61 ++-- src/persistence/space/index/unsized_.rs | 103 ++++--- src/persistence/space/index/util.rs | 15 +- src/persistence/space/logical_index.rs | 70 +++-- src/persistence/space/mod.rs | 18 +- src/persistence/task.rs | 40 +-- src/primary_key.rs | 8 +- src/table/mod.rs | 44 +-- src/table/select/mod.rs | 2 +- src/table/select/query.rs | 3 +- src/table/system_info.rs | 77 ++++- src/table/vacuum/fragmentation_info.rs | 3 +- src/table/vacuum/manager.rs | 9 +- src/table/vacuum/mod.rs | 2 + src/table/vacuum/pacing.rs | 10 +- src/table/vacuum/vacuum.rs | 30 +- src/util/mod.rs | 14 + src/util/offset_eq_link.rs | 12 +- src/util/optimized_vec.rs | 3 +- src/util/ordered_float.rs | 4 +- tests/mod.rs | 4 +- tests/persistence/custom_page_size.rs | 70 +++++ .../persistence/duplicate_key_index_reload.rs | 15 +- tests/persistence/index_page/read.rs | 100 +++---- tests/persistence/index_page/unsized_read.rs | 156 +++++----- tests/persistence/mod.rs | 1 + tests/persistence/read.rs | 26 +- tests/persistence/recovery_load.rs | 4 +- tests/persistence/s3/mod.rs | 2 + tests/persistence/schema.rs | 15 +- tests/persistence/space_data.rs | 11 +- .../space_index/indexset_compatibility.rs | 14 +- .../persistence/space_index/unsized_write.rs | 23 +- tests/persistence/space_index/write.rs | 36 ++- .../sync/string_secondary_index.rs | 12 +- tests/persistence/toc/read.rs | 134 ++++----- tests/persistence/toc/unsized_read.rs | 67 ++--- tests/persistence/toc/write.rs | 9 +- tests/worktable/concurrency.rs | 16 +- tests/worktable/index/insert.rs | 4 +- tests/worktable/key_widths.rs | 2 +- tests/worktable/nonunique_arctic.rs | 2 +- tests/worktable/partitioned.rs | 8 +- 147 files changed, 2190 insertions(+), 1251 deletions(-) create mode 100644 docs/page-size.md create mode 100644 docs/wt-user-guide.pdf create mode 100644 docs/wt-user-guide.typ create mode 100644 examples/guide_check.rs create mode 100644 examples/system_info_render.rs create mode 100644 src/fsx.rs create mode 100644 tests/persistence/custom_page_size.rs diff --git a/Cargo.toml b/Cargo.toml index e18d14ee..c08a4e56 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,17 @@ keywords = ["database", "embedded", "in-memory", "index", "storage"] categories = ["database-implementations", "data-structures", "caching"] [features] -default = ["wti-predictable-search"] +default = ["std", "wti-predictable-search", "vanilla-index"] +# `futures/std` belongs here rather than on the dependency +# line: written as `features = ["std"]` beside the version, a +# `--no-default-features` build of this crate would still turn them on. The +# s3 tests reach `futures::io`, whose traits live behind that feature. +std = ["nagoya/std", "ps-reclaim/std", "congee/std", "indexset/std", "futures/std"] +# The upstream IndexSet backend, selectable per index with `using indexset`. +# Optional because it reaches `crossbeam-utils`, `parking_lot` and `serde`, +# none of which build without `std`, and because it is one backend of four +# rather than something the table needs. +vanilla-index = ["dep:vanilla_indexset"] perf_measurements = ["dep:performance_measurement", "dep:performance_measurement_codegen"] s3-support = ["dep:rusty-s3", "dep:url", "dep:reqwest", "dep:walkdir", "worktable_codegen/s3-support"] # Moves unique WorkTablesIndex structural CDC work out of the table mutation @@ -35,36 +45,52 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"] [dependencies] # Read-mostly snapshots own page Arcs while the fixed directory supplies a # pointer-only fast path. Publication is append-only and asserted at each swap. -arc-swap = "1" +arc-swap = { version = "1", default-features = false } async-trait = "0.1" arctic = { package = "arctic-wt", version = "^0.1, >=0.1.11", default-features = false, features = ["smr-ps-reclaim"] } -congee = { package = "congee-wt", version = "^0.4, >=0.4.4" } -convert_case = "0.6" -crc32fast = "1" -data_bucket = { version = "^0.5, >=0.5.7" } -derive_more = { version = "2", features = ["from", "error", "display", "debug", "into"] } +# `default-features = false`, or its `std` turns `ps-reclaim/std` on and this +# crate links the standard library through a dependency. That matters for +# EKOPathRS bootstrapping, which is what `no_std` here is for, not for any +# bare-metal target: we ship on Linux, macOS and Windows. +congee = { package = "congee-wt", version = "^0.4, >=0.4.4", default-features = false } +convert_case = { version = "0.6", default-features = false } +crc32fast = { version = "1", default-features = false } +# 0.6 because the page stride is a parameter now and the error type is +# concrete: `^0.5` cannot resolve the crate this depends on. +data_bucket = { version = "^0.6" } +derive_more = { version = "2", default-features = false, features = ["from", "error", "display", "debug", "into"] } eyre = "0.6" fastrand = "2" -futures = "0.3" +hashbrown = "0.15" +futures = { version = "0.3", default-features = false, features = ["alloc"] } +# The file traits and the host implementation. `std` here; the crate takes it +# without default features wherever the filesystem is not involved. +nagoya = { version = "0.1", default-features = false } indexset = { package = "WorkTablesIndex", version = "^0.0, >=0.0.12", default-features = false, features = ["concurrent", "cdc", "multimap"] } -vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"] } +# Pulls `serde` and `serde_core` into every build, and nothing here uses them. +# They are not reachable from this line: `indexset` 0.15 declares `ftree` with +# `features = ["serde"]` unconditionally, so no `default-features = false` here +# removes them and forking `ftree` would not either. This is one of four +# selectable index backends and making it optional is the fix. +vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"], optional = true } # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", version = "^0.12", features = ["concurrent", "cdc", "multimap"] } -log = "0.4" -ordered-float = "5" +log = { version = "0.4", default-features = false } +ordered-float = { version = "5", default-features = false } parking_lot = "0.12" performance_measurement = { path = "performance_measurement", version = "^0.1", optional = true } performance_measurement_codegen = { path = "performance_measurement/codegen", version = "^0.1", optional = true } -prettytable-rs = "0.10" psc-nanoid = { version = "3", features = ["rkyv", "packed"] } -rkyv = { version = "0.8", features = ["uuid-1"] } +rkyv = { version = "0.8", default-features = false, features = ["alloc", "bytecheck", "uuid-1"] } reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] } -ps-reclaim = { version = "^0.1, >=0.1.4" } -rustc-hash = "2" +# `spin` rather than `std`: without it this crate reaches for `thread_local!`, +# which needs a platform. See its `slot` module for what it falls back to. +ps-reclaim = { version = "^0.1, >=0.1.4", default-features = false, features = ["spin"] } +rustc-hash = { version = "2", default-features = false } rusty-s3 = { version = "0.10", optional = true } smart-default = "0.7" tokio = { version = "1", features = ["full"] } -tracing = "0.1" +tracing = { version = "0.1", default-features = false } url = { version = "2", optional = true } uuid = { version = "1", features = ["v4", "v7"] } walkdir = { version = "2", optional = true } @@ -95,3 +121,16 @@ unexpected_cfgs = { level = "warn", check-cfg = ['cfg(wt_loom)'] } [[bench]] name = "worktable_benchmarks" harness = false + + +# None of these three are published yet, and `[patch]` only takes effect from +# the workspace root, so the whole chain has to be named here: a dependency's +# own patches do not reach its consumer. Without this the branch resolves +# nowhere, for a reviewer and for CI alike. +# +# Delete as each publishes: ps-st3 0.5.1, then nagoya 0.1.0, then data_bucket +# 0.6.0. +[patch.crates-io] +data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "feat/tunable-page-stride" } +nagoya = { git = "https://github.com/pathscale/nagoya", branch = "feat/timers" } +ps-st3 = { git = "https://github.com/pathscale/ps-st3", branch = "perf/skip-the-wake-when-nobody-sleeps" } diff --git a/benches/cases/full_featured.rs b/benches/cases/full_featured.rs index 0e5b222d..69c95bfa 100644 --- a/benches/cases/full_featured.rs +++ b/benches/cases/full_featured.rs @@ -19,7 +19,7 @@ fn insert(c: &mut Criterion) { another: format!("another_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(black_box(row))) + nagoya::block_on(table.insert(black_box(row))) }, BatchSize::SmallInput, ) @@ -37,7 +37,7 @@ fn select_by_pk(c: &mut Criterion) { another: format!("another_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -60,7 +60,7 @@ fn select_by_unique_index(c: &mut Criterion) { another: format!("another_{}", i), something: i, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("full_featured_select_by_val1", |b| { @@ -82,7 +82,7 @@ fn select_by_non_unique_index(c: &mut Criterion) { another: format!("cat_{}", i % 10), something: i, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("full_featured_select_by_another", |b| { @@ -196,7 +196,7 @@ fn in_place_update(c: &mut Criterion) { another: "test".to_string(), something: 0, }; - futures::executor::block_on(table.insert(row)).unwrap().into() + nagoya::block_on(table.insert(row)).unwrap().into() }; c.bench_function("full_featured_in_place_update_val", |b| { @@ -219,7 +219,7 @@ fn delete(c: &mut Criterion) { another: format!("temp_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: FullFeaturedPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -242,7 +242,7 @@ fn delete_by_index_query(c: &mut Criterion) { another: another.clone(), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); another }, |another: String| rt.block_on(async { table.delete_by_another(another).await.unwrap() }), @@ -319,7 +319,7 @@ fn batch_insert(c: &mut Criterion) { another: format!("another_{}", i), something: i as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -347,7 +347,7 @@ fn batch_select_pk(c: &mut Criterion) { another: format!("another_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/benches/cases/non_unique_index.rs b/benches/cases/non_unique_index.rs index 7e5f2038..c2589030 100644 --- a/benches/cases/non_unique_index.rs +++ b/benches/cases/non_unique_index.rs @@ -33,7 +33,7 @@ fn select_by_pk(c: &mut Criterion) { value: fastrand::u64(..), category: fastrand::u64(0..10), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -54,7 +54,7 @@ fn select_by_non_unique_index(c: &mut Criterion) { value: fastrand::u64(..), category: i % 10, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("non_unique_index_select_by_category", |b| { @@ -108,7 +108,7 @@ fn delete(c: &mut Criterion) { value: fastrand::u64(..), category: fastrand::u64(0..10), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: NonUniqueIndexPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -176,7 +176,7 @@ fn batch_insert(c: &mut Criterion) { value: i as u64, category: (i % 10) as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -202,7 +202,7 @@ fn batch_select_pk(c: &mut Criterion) { value: fastrand::u64(..), category: fastrand::u64(0..10), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/benches/cases/nonunique_arctic_vs_wti.rs b/benches/cases/nonunique_arctic_vs_wti.rs index 6ea1d8a7..9d4e19ed 100644 --- a/benches/cases/nonunique_arctic_vs_wti.rs +++ b/benches/cases/nonunique_arctic_vs_wti.rs @@ -84,7 +84,7 @@ fn select_by_key(c: &mut Criterion) { for (fan_out, keys) in SHAPES { group.throughput(Throughput::Elements(fan_out)); - let table = futures::executor::block_on(populated_wti(fan_out, keys)); + let table = nagoya::block_on(populated_wti(fan_out, keys)); group.bench_with_input(BenchmarkId::new("wti_string", fan_out), &fan_out, |b, _| { b.iter(|| { let key = string_key(fastrand::u64(0..keys)); @@ -92,7 +92,7 @@ fn select_by_key(c: &mut Criterion) { }) }); - let table = futures::executor::block_on(populated_arctic(fan_out, keys)); + let table = nagoya::block_on(populated_arctic(fan_out, keys)); group.bench_with_input(BenchmarkId::new("arctic_u128", fan_out), &fan_out, |b, _| { b.iter(|| { let key = hash_of(fastrand::u64(0..keys)); @@ -108,7 +108,7 @@ fn insert(c: &mut Criterion) { for (fan_out, keys) in SHAPES { // Steady state: the table already holds `fan_out` rows per key and // each measured insert lands on an existing key. - let table = futures::executor::block_on(populated_wti(fan_out, keys)); + let table = nagoya::block_on(populated_wti(fan_out, keys)); group.bench_with_input(BenchmarkId::new("wti_string", fan_out), &fan_out, |b, _| { b.iter_batched( || WtiStringAdjacencyRow { @@ -116,12 +116,12 @@ fn insert(c: &mut Criterion) { source: string_key(fastrand::u64(0..keys)), payload: u64::MAX, }, - |row| futures::executor::block_on(table.insert(black_box(row))).unwrap(), + |row| nagoya::block_on(table.insert(black_box(row))).unwrap(), BatchSize::SmallInput, ) }); - let table = futures::executor::block_on(populated_arctic(fan_out, keys)); + let table = nagoya::block_on(populated_arctic(fan_out, keys)); group.bench_with_input(BenchmarkId::new("arctic_u128", fan_out), &fan_out, |b, _| { b.iter_batched( || ArcticHashAdjacencyRow { @@ -129,7 +129,7 @@ fn insert(c: &mut Criterion) { source: hash_of(fastrand::u64(0..keys)), payload: u64::MAX, }, - |row| futures::executor::block_on(table.insert(black_box(row))).unwrap(), + |row| nagoya::block_on(table.insert(black_box(row))).unwrap(), BatchSize::SmallInput, ) }); diff --git a/benches/cases/partition_routing.rs b/benches/cases/partition_routing.rs index 7d59b03a..a207634d 100644 --- a/benches/cases/partition_routing.rs +++ b/benches/cases/partition_routing.rs @@ -46,7 +46,7 @@ async fn populated() -> RoutePartitions { /// The four ways to reach a partition, single threaded, one hot key. fn lookup(c: &mut Criterion) { - let routes = futures::executor::block_on(populated()); + let routes = nagoya::block_on(populated()); let cached = routes.partition(7).unwrap(); let mut group = c.benchmark_group("partition_lookup"); @@ -87,7 +87,7 @@ fn contended(c: &mut Criterion, name: &str, same_key: bool) { for api in ["partition_ref", "pinned_get", "partition_arc"] { group.bench_with_input(BenchmarkId::new(api, threads), &threads, |b, &threads| { b.iter_custom(|iters| { - let routes = Arc::new(futures::executor::block_on(populated())); + let routes = Arc::new(nagoya::block_on(populated())); let go = Arc::new(AtomicBool::new(false)); let workers: Vec<_> = (0..threads) @@ -151,7 +151,7 @@ fn distinct_key_readers(c: &mut Criterion) { /// Accounting over 500 partitions. These were routed through `system_info`, /// which copied every data page, and through `keys` then `partition` per key. fn metrics(c: &mut Criterion) { - let routes = futures::executor::block_on(populated()); + let routes = nagoya::block_on(populated()); let mut group = c.benchmark_group("partition_metrics"); group.bench_function("memory_total", |b| b.iter(|| black_box(routes.memory_total()))); group.bench_function("rows_by_key", |b| b.iter(|| black_box(routes.rows_by_key()))); diff --git a/benches/cases/simple.rs b/benches/cases/simple.rs index a035b53a..afbcc353 100644 --- a/benches/cases/simple.rs +++ b/benches/cases/simple.rs @@ -15,7 +15,7 @@ fn insert(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(black_box(row))) + nagoya::block_on(table.insert(black_box(row))) }, BatchSize::SmallInput, ) @@ -30,7 +30,7 @@ fn select_by_pk(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -82,7 +82,7 @@ fn delete(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: SimplePrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -146,7 +146,7 @@ fn batch_insert(c: &mut Criterion) { id: table.get_next_pk().into(), value: i as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -171,7 +171,7 @@ fn batch_select_pk(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/benches/cases/unique_index.rs b/benches/cases/unique_index.rs index 7509ae63..73144ff2 100644 --- a/benches/cases/unique_index.rs +++ b/benches/cases/unique_index.rs @@ -52,7 +52,7 @@ fn select_by_pk(c: &mut Criterion) { test: fastrand::i64(..), another: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -73,7 +73,7 @@ fn select_by_unique_index(c: &mut Criterion) { test: i, another: i as u64, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("unique_index_select_by_test", |b| { @@ -93,7 +93,7 @@ fn select_by_unique_index_range(c: &mut Criterion) { test: i, another: i as u64, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("unique_index_select_by_test_range", |b| { @@ -109,8 +109,8 @@ fn art_primary_key_ranges(c: &mut Criterion) { let congee = CongeeRangeBenchmarkWorkTable::default(); let arctic = ArcticRangeBenchmarkWorkTable::default(); for id in 0..ROWS { - futures::executor::block_on(congee.insert(CongeeRangeBenchmarkRow { id, value: id })).unwrap(); - futures::executor::block_on(arctic.insert(ArcticRangeBenchmarkRow { id, value: id })).unwrap(); + nagoya::block_on(congee.insert(CongeeRangeBenchmarkRow { id, value: id })).unwrap(); + nagoya::block_on(arctic.insert(ArcticRangeBenchmarkRow { id, value: id })).unwrap(); } let mut group = c.benchmark_group("art_primary_key_single_row_range"); @@ -173,7 +173,7 @@ fn delete(c: &mut Criterion) { test: fastrand::i64(..), another: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: UniqueIndexPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -241,7 +241,7 @@ fn batch_insert(c: &mut Criterion) { test: i as i64, another: i as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -267,7 +267,7 @@ fn batch_select_pk(c: &mut Criterion) { test: fastrand::i64(..), another: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index cab40835..5057c7f4 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -24,9 +24,17 @@ proc-macro = true # policy. beta.18.1 is the first DSL artifact that exports every validator this # code generator calls; beta.18 was published before that API landed. worktable_dsl = { path = "../dsl", version = "^1.0.0-beta.18.1" } -rkyv = { version = "0.8" } +# Test-only. As a normal dependency this proc-macro crate put `rkyv` with its +# default features into the graph, which turned on `rkyv/std` for the target +# build too and dragged `ptr_meta` with it. The generated code names `rkyv` +# paths through `quote!`, which needs no dependency here at all. +# +# See the `[dev-dependencies]` section below. syn = { version = "2", features = ["full"] } quote = "1" proc-macro2 = "1" convert_case = "0.6" indexmap = "2" + +[dev-dependencies] +rkyv = { version = "0.8" } diff --git a/codegen/src/generators/in_memory/index/cdc.rs b/codegen/src/generators/in_memory/index/cdc.rs index ed85e087..3394f3f5 100644 --- a/codegen/src/generators/in_memory/index/cdc.rs +++ b/codegen/src/generators/in_memory/index/cdc.rs @@ -311,7 +311,7 @@ impl InMemoryGenerator { fn process_difference_remove_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { #(#process_difference_rows)* (#events_ident { @@ -381,7 +381,7 @@ impl InMemoryGenerator { fn process_difference_insert_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; let mut partial_events = #events_ident::default(); diff --git a/codegen/src/generators/in_memory/index/usual.rs b/codegen/src/generators/in_memory/index/usual.rs index 2c90abc7..5329a29e 100644 --- a/codegen/src/generators/in_memory/index/usual.rs +++ b/codegen/src/generators/in_memory/index/usual.rs @@ -245,7 +245,7 @@ impl InMemoryGenerator { fn process_difference_remove( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { #(#process_difference_remove_rows)* core::result::Result::Ok(()) @@ -304,7 +304,7 @@ impl InMemoryGenerator { fn process_difference_insert( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { let mut inserted_indexes: Vec<#avt_index_ident> = vec![]; #(#process_difference_insert_rows)* diff --git a/codegen/src/generators/in_memory/locks.rs b/codegen/src/generators/in_memory/locks.rs index 78b40aec..1153c65c 100644 --- a/codegen/src/generators/in_memory/locks.rs +++ b/codegen/src/generators/in_memory/locks.rs @@ -24,7 +24,7 @@ impl InMemoryGenerator { .keys() .map(|i| { let name = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); - quote! { #name: Option>, } + quote! { #name: Option>, } }) .collect(); @@ -124,8 +124,8 @@ impl InMemoryGenerator { .collect(); quote! { - fn with_lock(id: u16) -> (Self, std::sync::Arc) { - let lock = std::sync::Arc::new(Lock::new(id)); + fn with_lock(id: u16) -> (Self, worktable::prelude::Arc) { + let lock = worktable::prelude::Arc::new(Lock::new(id)); ( Self { #(#rows),* @@ -154,9 +154,9 @@ impl InMemoryGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let lock = std::sync::Arc::new(Lock::new(id)); + fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* (set, lock) @@ -186,8 +186,8 @@ impl InMemoryGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> std::collections::HashSet> { - let mut set = std::collections::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { + let mut set = worktable::prelude::HashSet::new(); #(#rows)* set } diff --git a/codegen/src/generators/in_memory/primary_key.rs b/codegen/src/generators/in_memory/primary_key.rs index 8d58b84f..ac4db3e9 100644 --- a/codegen/src/generators/in_memory/primary_key.rs +++ b/codegen/src/generators/in_memory/primary_key.rs @@ -140,14 +140,14 @@ impl InMemoryGenerator { /// atomic of primitive. fn get_generator_from_type(type_: &TokenStream, i: &Ident) -> syn::Result { Ok(match type_.to_string().as_str() { - "u8" => quote! { std::sync::atomic::AtomicU8 }, - "u16" => quote! { std::sync::atomic::AtomicU16 }, - "u32" => quote! { std::sync::atomic::AtomicU32 }, - "u64" => quote! { std::sync::atomic::AtomicU64 }, - "i8" => quote! { std::sync::atomic::AtomicI8 }, - "i16" => quote! { std::sync::atomic::AtomicI16 }, - "i32" => quote! { std::sync::atomic::AtomicI32 }, - "i64" => quote! { std::sync::atomic::AtomicI64 }, + "u8" => quote! { core::sync::atomic::AtomicU8 }, + "u16" => quote! { core::sync::atomic::AtomicU16 }, + "u32" => quote! { core::sync::atomic::AtomicU32 }, + "u64" => quote! { core::sync::atomic::AtomicU64 }, + "i8" => quote! { core::sync::atomic::AtomicI8 }, + "i16" => quote! { core::sync::atomic::AtomicI16 }, + "i32" => quote! { core::sync::atomic::AtomicI32 }, + "i64" => quote! { core::sync::atomic::AtomicI64 }, // The accepted set is `worktable_dsl::AUTOINCREMENT_TYPES`, and the // arms above must stay equal to it. `check` uses that list to // answer "would the macro accept this", so a second copy drifting diff --git a/codegen/src/generators/in_memory/queries/delete.rs b/codegen/src/generators/in_memory/queries/delete.rs index 911f7e9a..efcde879 100644 --- a/codegen/src/generators/in_memory/queries/delete.rs +++ b/codegen/src/generators/in_memory/queries/delete.rs @@ -186,7 +186,7 @@ impl InMemoryGenerator { quote! { pub async fn #name(&self, by: #type_) -> core::result::Result<(), WorkTableError> { let _bulk_mutation = self.0.lock_manager.bulk_mutation_guard(); - let pks = std::cell::RefCell::new(Vec::new()); + let pks = core::cell::RefCell::new(Vec::new()); self.iter_with(|row| { if row.#field == by { pks.borrow_mut().push(row.get_primary_key()); diff --git a/codegen/src/generators/in_memory/queries/locks.rs b/codegen/src/generators/in_memory/queries/locks.rs index cf80019f..11e15c77 100644 --- a/codegen/src/generators/in_memory/queries/locks.rs +++ b/codegen/src/generators/in_memory/queries/locks.rs @@ -95,9 +95,9 @@ impl InMemoryGenerator { quote! { #[allow(clippy::mutable_key_type)] - pub fn #ident(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let new_lock = std::sync::Arc::new(Lock::new(id)); + pub fn #ident(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let new_lock = worktable::prelude::Arc::new(Lock::new(id)); #(#inner)* (set, new_lock) } diff --git a/codegen/src/generators/in_memory/queries/select.rs b/codegen/src/generators/in_memory/queries/select.rs index 42139128..744cd0ea 100644 --- a/codegen/src/generators/in_memory/queries/select.rs +++ b/codegen/src/generators/in_memory/queries/select.rs @@ -32,7 +32,7 @@ impl InMemoryGenerator { // Acquire the grace-period guard only when iteration starts. // Merely constructing and retaining a query builder must not // stall retired-link reclamation. - let iter = std::iter::once_with(move || { + let iter = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .iter_values() @@ -55,7 +55,7 @@ impl InMemoryGenerator { return None; } current_link = replacement; - std::hint::spin_loop(); + core::hint::spin_loop(); } None }) diff --git a/codegen/src/generators/in_memory/queries/update.rs b/codegen/src/generators/in_memory/queries/update.rs index 04af1554..a10c90bf 100644 --- a/codegen/src/generators/in_memory/queries/update.rs +++ b/codegen/src/generators/in_memory/queries/update.rs @@ -42,7 +42,7 @@ impl InMemoryGenerator { .keys() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -273,8 +273,8 @@ impl InMemoryGenerator { let avt_type_ident = name_generator.get_available_type_ident(); quote! { if let core::result::Result::Err(e) = #write { - let mut reversed_diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = - std::collections::HashMap::new(); + let mut reversed_diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = + worktable::prelude::HashMap::new(); for (key, diff) in diffs { reversed_diffs.insert(key, Difference { old: diff.new, new: diff.old }); } @@ -460,7 +460,7 @@ impl InMemoryGenerator { let row_old = self.0.data.select_non_ghosted(link)?; let row_new = row.clone(); let updated_bytes: Vec = vec![]; - let mut diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = std::collections::HashMap::new(); + let mut diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = worktable::prelude::HashMap::new(); } } else { quote! { @@ -605,7 +605,7 @@ impl InMemoryGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -680,7 +680,7 @@ impl InMemoryGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -830,7 +830,7 @@ impl InMemoryGenerator { pks.sort_unstable(); pks.dedup(); - let mut guards: std::collections::HashMap<_, _> = std::collections::HashMap::new(); + let mut guards: worktable::prelude::HashMap<_, _> = worktable::prelude::HashMap::new(); // Full-row locks, not per-column custom locks: each row's // unsized reinsert path mutates the whole row under these // guards, and one uniform lock kind keeps every concurrent @@ -902,7 +902,7 @@ impl InMemoryGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); diff --git a/codegen/src/generators/in_memory/table/impls.rs b/codegen/src/generators/in_memory/table/impls.rs index 198e390a..9d4554de 100644 --- a/codegen/src/generators/in_memory/table/impls.rs +++ b/codegen/src/generators/in_memory/table/impls.rs @@ -109,7 +109,7 @@ impl InMemoryGenerator { #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds + 'a, + R: core::ops::RangeBounds + 'a, Pk: Clone + 'a, { let converted_range = ( @@ -118,7 +118,7 @@ impl InMemoryGenerator { ); // Delay the grace-period guard until the returned iterator is // consumed so an idle query builder cannot pin reclamation. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .range_links(converted_range) @@ -302,7 +302,7 @@ impl InMemoryGenerator { let exponent = core::cmp::min(backoff_spins - 8, 8); let micros = core::cmp::min(1u64 << exponent, 256); backoff_spins = backoff_spins.saturating_add(1); - tokio::time::sleep(std::time::Duration::from_micros(micros)).await; + tokio::time::sleep(core::time::Duration::from_micros(micros)).await; } } } @@ -343,7 +343,7 @@ impl InMemoryGenerator { /// assigned contiguous keys before `insert_many`. Interleaved /// `get_next_pk` calls keep working and never overlap a /// reservation. - pub fn reserve_pks(&self, count: usize) -> std::ops::Range<#pk_inner_type> { + pub fn reserve_pks(&self, count: usize) -> core::ops::Range<#pk_inner_type> { self.0.reserve_pks(count) } } @@ -375,7 +375,7 @@ impl InMemoryGenerator { quote! { pub async fn iter_with_async< F: Fn(#row_type) -> Fut, - Fut: std::future::Future> + Fut: core::future::Future> >(&self, f: F) -> core::result::Result<(), WorkTableError> { #inner } @@ -436,8 +436,8 @@ impl InMemoryGenerator { let lock_type = name_generator.get_lock_type_ident(); quote! { - pub fn vacuum(&self) -> std::sync::Arc { - std::sync::Arc::new(EmptyDataVacuum::< + pub fn vacuum(&self) -> worktable::prelude::Arc { + worktable::prelude::Arc::new(EmptyDataVacuum::< _, _, _, @@ -448,10 +448,10 @@ impl InMemoryGenerator { _ >::new( #table_name, - std::sync::Arc::clone(&self.0.data), - std::sync::Arc::clone(&self.0.lock_manager), - std::sync::Arc::clone(&self.0.primary_index), - std::sync::Arc::clone(&self.0.indexes), + worktable::prelude::Arc::clone(&self.0.data), + worktable::prelude::Arc::clone(&self.0.lock_manager), + worktable::prelude::Arc::clone(&self.0.primary_index), + worktable::prelude::Arc::clone(&self.0.indexes), )) } } diff --git a/codegen/src/generators/in_memory/table/index_fns.rs b/codegen/src/generators/in_memory/table/index_fns.rs index 35d51744..1317e50c 100644 --- a/codegen/src/generators/in_memory/table/index_fns.rs +++ b/codegen/src/generators/in_memory/table/index_fns.rs @@ -99,7 +99,7 @@ impl InMemoryGenerator { if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None }; @@ -140,7 +140,7 @@ impl InMemoryGenerator { #column_range_type, #row_fields_ident> { - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.indexes.#field_ident .get(#by) @@ -173,7 +173,7 @@ impl InMemoryGenerator { let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { ( predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), @@ -183,7 +183,7 @@ impl InMemoryGenerator { ) } else { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { predicate_range.clone() }, ) }; @@ -216,7 +216,7 @@ impl InMemoryGenerator { }; let predicate_filter = quote! { .filter(move |row| { - std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + core::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) }) }; @@ -231,7 +231,7 @@ impl InMemoryGenerator { #predicate_setup // Query construction is not an active read. Pin the grace // period on the first row lookup instead. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); #index_range .filter_map(#select_row) diff --git a/codegen/src/generators/in_memory/table/mod.rs b/codegen/src/generators/in_memory/table/mod.rs index d5bd9e39..10d3768c 100644 --- a/codegen/src/generators/in_memory/table/mod.rs +++ b/codegen/src/generators/in_memory/table/mod.rs @@ -126,7 +126,7 @@ impl InMemoryGenerator { #index_type, #lock_ident, <#primary_key_type as TablePrimaryKey>::Generator, - { INNER_PAGE_SIZE }, + { #inner_const_name }, #node_type > ); diff --git a/codegen/src/generators/in_memory/table/select_executor.rs b/codegen/src/generators/in_memory/table/select_executor.rs index 0dd2560b..4538de0e 100644 --- a/codegen/src/generators/in_memory/table/select_executor.rs +++ b/codegen/src/generators/in_memory/table/select_executor.rs @@ -44,7 +44,7 @@ impl InMemoryGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - #variant_ident(std::ops::#range_ident<#ty_ident>), + #variant_ident(core::ops::#range_ident<#ty_ident>), } }) .collect(); @@ -65,8 +65,8 @@ impl InMemoryGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - impl From> for #column_range_type { - fn from(range: std::ops::#range_ident<#ty_ident>) -> Self { + impl From> for #column_range_type { + fn from(range: core::ops::#range_ident<#ty_ident>) -> Self { Self::#variant_ident(range) } } @@ -100,8 +100,8 @@ impl InMemoryGenerator { let col_ident = Ident::new(&column.to_string(), Span::call_site()); quote! { #row_fields_ident::#column_variant => { - let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(std::cmp::Ordering::Equal); - if cmp != std::cmp::Ordering::Equal { + let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(core::cmp::Ordering::Equal); + if cmp != core::cmp::Ordering::Equal { return match order { Order::Asc => cmp, Order::Desc => cmp.reverse(), @@ -165,7 +165,7 @@ impl InMemoryGenerator { _ => continue, } } - std::cmp::Ordering::Equal + core::cmp::Ordering::Equal }); iter = Box::new(items.into_iter()); }; diff --git a/codegen/src/generators/partitions.rs b/codegen/src/generators/partitions.rs index 5cf07732..5fed340e 100644 --- a/codegen/src/generators/partitions.rs +++ b/codegen/src/generators/partitions.rs @@ -38,7 +38,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok pub fn partition_or_create( &self, #key_name: #key_ty, - ) -> Result, worktable::partition::PartitionError> { + ) -> Result, worktable::partition::PartitionError> { self.inner.get_or_create(#key_name as u64, <#table as Default>::default) } } @@ -82,7 +82,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok /// The partition routed to by `#key_name`, if it exists. #[inline] - pub fn partition(&self, #key_name: #key_ty) -> Option> { + pub fn partition(&self, #key_name: #key_ty) -> Option> { self.inner.partition(#key_name as u64) } @@ -94,7 +94,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok &self, #key_name: #key_ty, make: F, - ) -> Result, worktable::partition::PartitionError> + ) -> Result, worktable::partition::PartitionError> where F: FnOnce() -> #table, { @@ -146,7 +146,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok /// grace period). Removal and creation reclaim opportunistically, /// so a router shared behind an `Arc` does not accumulate removed /// partitions; `collect` is available for removal-only phases. - pub fn remove(&self, #key_name: #key_ty) -> Option> { + pub fn remove(&self, #key_name: #key_ty) -> Option> { self.inner.remove(#key_name as u64) } @@ -156,7 +156,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok } /// Every live partition with its key. - pub fn iter(&self) -> Vec<(#key_ty, std::sync::Arc<#table>)> { + pub fn iter(&self) -> Vec<(#key_ty, worktable::prelude::Arc<#table>)> { self.inner .iter() .into_iter() diff --git a/codegen/src/generators/persist/index/cdc.rs b/codegen/src/generators/persist/index/cdc.rs index 8aee8d6a..69bde2a4 100644 --- a/codegen/src/generators/persist/index/cdc.rs +++ b/codegen/src/generators/persist/index/cdc.rs @@ -338,7 +338,7 @@ impl PersistGenerator { fn process_difference_remove_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { #(#process_difference_rows)* (#events_ident { @@ -408,7 +408,7 @@ impl PersistGenerator { fn process_difference_insert_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; let mut partial_events = #events_ident::default(); diff --git a/codegen/src/generators/persist/index/usual.rs b/codegen/src/generators/persist/index/usual.rs index e8629cc8..19663c79 100644 --- a/codegen/src/generators/persist/index/usual.rs +++ b/codegen/src/generators/persist/index/usual.rs @@ -236,7 +236,7 @@ impl PersistGenerator { fn process_difference_remove( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { #(#process_difference_remove_rows)* core::result::Result::Ok(()) @@ -293,7 +293,7 @@ impl PersistGenerator { fn process_difference_insert( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { let mut inserted_indexes: Vec<#avt_index_ident> = vec![]; #(#process_difference_insert_rows)* diff --git a/codegen/src/generators/persist/locks.rs b/codegen/src/generators/persist/locks.rs index 92a88a0c..86b55b60 100644 --- a/codegen/src/generators/persist/locks.rs +++ b/codegen/src/generators/persist/locks.rs @@ -24,7 +24,7 @@ impl PersistGenerator { .keys() .map(|i| { let name = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); - quote! { #name: Option>, } + quote! { #name: Option>, } }) .collect(); @@ -124,8 +124,8 @@ impl PersistGenerator { .collect(); quote! { - fn with_lock(id: u16) -> (Self, std::sync::Arc) { - let lock = std::sync::Arc::new(Lock::new(id)); + fn with_lock(id: u16) -> (Self, worktable::prelude::Arc) { + let lock = worktable::prelude::Arc::new(Lock::new(id)); ( Self { #(#rows),* @@ -154,9 +154,9 @@ impl PersistGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let lock = std::sync::Arc::new(Lock::new(id)); + fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* (set, lock) @@ -186,8 +186,8 @@ impl PersistGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> std::collections::HashSet> { - let mut set = std::collections::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { + let mut set = worktable::prelude::HashSet::new(); #(#rows)* set } diff --git a/codegen/src/generators/persist/primary_key.rs b/codegen/src/generators/persist/primary_key.rs index 59b002b9..47f3192b 100644 --- a/codegen/src/generators/persist/primary_key.rs +++ b/codegen/src/generators/persist/primary_key.rs @@ -133,14 +133,14 @@ impl PersistGenerator { fn get_generator_from_type(type_: &TokenStream, i: &Ident) -> syn::Result { Ok(match type_.to_string().as_str() { - "u8" => quote! { std::sync::atomic::AtomicU8 }, - "u16" => quote! { std::sync::atomic::AtomicU16 }, - "u32" => quote! { std::sync::atomic::AtomicU32 }, - "u64" => quote! { std::sync::atomic::AtomicU64 }, - "i8" => quote! { std::sync::atomic::AtomicI8 }, - "i16" => quote! { std::sync::atomic::AtomicI16 }, - "i32" => quote! { std::sync::atomic::AtomicI32 }, - "i64" => quote! { std::sync::atomic::AtomicI64 }, + "u8" => quote! { core::sync::atomic::AtomicU8 }, + "u16" => quote! { core::sync::atomic::AtomicU16 }, + "u32" => quote! { core::sync::atomic::AtomicU32 }, + "u64" => quote! { core::sync::atomic::AtomicU64 }, + "i8" => quote! { core::sync::atomic::AtomicI8 }, + "i16" => quote! { core::sync::atomic::AtomicI16 }, + "i32" => quote! { core::sync::atomic::AtomicI32 }, + "i64" => quote! { core::sync::atomic::AtomicI64 }, // The accepted set is `worktable_dsl::AUTOINCREMENT_TYPES`, and the // arms above must stay equal to it. `check` uses that list to // answer "would the macro accept this", so a second copy drifting diff --git a/codegen/src/generators/persist/queries/delete.rs b/codegen/src/generators/persist/queries/delete.rs index ebe4ea45..8aca38f3 100644 --- a/codegen/src/generators/persist/queries/delete.rs +++ b/codegen/src/generators/persist/queries/delete.rs @@ -235,7 +235,7 @@ impl PersistGenerator { quote! { pub async fn #name(&self, by: #type_) -> core::result::Result<(), WorkTableError> { let _bulk_mutation = self.0.lock_manager.bulk_mutation_guard(); - let pks = std::cell::RefCell::new(Vec::new()); + let pks = core::cell::RefCell::new(Vec::new()); self.iter_with(|row| { if row.#field == by { pks.borrow_mut().push(row.get_primary_key()); diff --git a/codegen/src/generators/persist/queries/locks.rs b/codegen/src/generators/persist/queries/locks.rs index c6f0c3ee..3b685c39 100644 --- a/codegen/src/generators/persist/queries/locks.rs +++ b/codegen/src/generators/persist/queries/locks.rs @@ -95,9 +95,9 @@ impl PersistGenerator { quote! { #[allow(clippy::mutable_key_type)] - pub fn #ident(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let new_lock = std::sync::Arc::new(Lock::new(id)); + pub fn #ident(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let new_lock = worktable::prelude::Arc::new(Lock::new(id)); #(#inner)* (set, new_lock) } diff --git a/codegen/src/generators/persist/queries/select.rs b/codegen/src/generators/persist/queries/select.rs index 7627a6fb..581c4fd6 100644 --- a/codegen/src/generators/persist/queries/select.rs +++ b/codegen/src/generators/persist/queries/select.rs @@ -32,7 +32,7 @@ impl PersistGenerator { // Acquire the grace-period guard only when iteration starts. // Merely constructing and retaining a query builder must not // stall retired-link reclamation. - let iter = std::iter::once_with(move || { + let iter = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .iter_values() @@ -55,7 +55,7 @@ impl PersistGenerator { return None; } current_link = replacement; - std::hint::spin_loop(); + core::hint::spin_loop(); } None }) diff --git a/codegen/src/generators/persist/queries/update.rs b/codegen/src/generators/persist/queries/update.rs index 380df89d..f91b65d8 100644 --- a/codegen/src/generators/persist/queries/update.rs +++ b/codegen/src/generators/persist/queries/update.rs @@ -42,7 +42,7 @@ impl PersistGenerator { .keys() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -311,8 +311,8 @@ impl PersistGenerator { // compensation above). let mut merged_events = secondary_keys_events; if row_holds_old_values { - let mut reversed_diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = - std::collections::HashMap::new(); + let mut reversed_diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = + worktable::prelude::HashMap::new(); for (key, diff) in diffs { reversed_diffs.insert(key, Difference { old: diff.new, new: diff.old }); } @@ -508,7 +508,7 @@ impl PersistGenerator { quote! { let row_old = self.0.data.select_non_ghosted(link)?; let row_new = row.clone(); - let mut diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = std::collections::HashMap::new(); + let mut diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = worktable::prelude::HashMap::new(); } } else { quote! {} @@ -655,7 +655,7 @@ impl PersistGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -733,7 +733,7 @@ impl PersistGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -900,7 +900,7 @@ impl PersistGenerator { pks.sort_unstable(); pks.dedup(); - let mut guards: std::collections::HashMap<_, _> = std::collections::HashMap::new(); + let mut guards: worktable::prelude::HashMap<_, _> = worktable::prelude::HashMap::new(); // Full-row locks, not per-column custom locks: each row's // unsized reinsert path mutates the whole row under these // guards, and one uniform lock kind keeps every concurrent @@ -973,7 +973,7 @@ impl PersistGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); diff --git a/codegen/src/generators/persist/table/impls.rs b/codegen/src/generators/persist/table/impls.rs index 39e453b2..80a1ef93 100644 --- a/codegen/src/generators/persist/table/impls.rs +++ b/codegen/src/generators/persist/table/impls.rs @@ -254,13 +254,13 @@ impl PersistGenerator { }; let index_setup = if self.columns.primary_index_backend == crate::common::model::IndexBackend::Arctic { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( PersistentArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); } } else if pk_types_unsized { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( #wti_map::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name) )); } @@ -268,19 +268,19 @@ impl PersistGenerator { match self.columns.primary_index_backend { crate::common::model::IndexBackend::WorktablesIndex => quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( #wti_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) )); }, crate::common::model::IndexBackend::Indexset => quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( UpstreamIndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) )); }, crate::common::model::IndexBackend::Arctic => unreachable!("handled before variable-size dispatch"), crate::common::model::IndexBackend::Congee => quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( PersistentCongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); }, @@ -409,7 +409,7 @@ impl PersistGenerator { #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds + 'a, + R: core::ops::RangeBounds + 'a, Pk: Clone + 'a, { let converted_range = ( @@ -418,7 +418,7 @@ impl PersistGenerator { ); // Delay the grace-period guard until the returned iterator is // consumed so an idle query builder cannot pin reclamation. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .range_links(converted_range) @@ -648,7 +648,7 @@ impl PersistGenerator { let exponent = core::cmp::min(backoff_spins - 8, 8); let micros = core::cmp::min(1u64 << exponent, 256); backoff_spins = backoff_spins.saturating_add(1); - tokio::time::sleep(std::time::Duration::from_micros(micros)).await; + tokio::time::sleep(core::time::Duration::from_micros(micros)).await; } } } @@ -689,7 +689,7 @@ impl PersistGenerator { /// assigned contiguous keys before `insert_many`. Interleaved /// `get_next_pk` calls keep working and never overlap a /// reservation. - pub fn reserve_pks(&self, count: usize) -> std::ops::Range<#pk_inner_type> { + pub fn reserve_pks(&self, count: usize) -> core::ops::Range<#pk_inner_type> { self.0.reserve_pks(count) } } @@ -732,7 +732,7 @@ impl PersistGenerator { quote! { pub async fn iter_with_async< F: Fn(#row_type) -> Fut, - Fut: std::future::Future> + Fut: core::future::Future> >(&self, f: F) -> core::result::Result<(), WorkTableError> { #inner } @@ -794,8 +794,8 @@ impl PersistGenerator { let lock_type = name_generator.get_lock_type_ident(); quote! { - pub fn vacuum(&self) -> std::sync::Arc { - std::sync::Arc::new(EmptyDataVacuum::< + pub fn vacuum(&self) -> worktable::prelude::Arc { + worktable::prelude::Arc::new(EmptyDataVacuum::< _, _, _, @@ -807,10 +807,10 @@ impl PersistGenerator { #secondary_index_events >::new( #table_name, - std::sync::Arc::clone(&self.0.data), - std::sync::Arc::clone(&self.0.lock_manager), - std::sync::Arc::clone(&self.0.primary_index), - std::sync::Arc::clone(&self.0.indexes), + worktable::prelude::Arc::clone(&self.0.data), + worktable::prelude::Arc::clone(&self.0.lock_manager), + worktable::prelude::Arc::clone(&self.0.primary_index), + worktable::prelude::Arc::clone(&self.0.indexes), ).with_persistence(self.1.vacuum_sink())) } } diff --git a/codegen/src/generators/persist/table/index_fns.rs b/codegen/src/generators/persist/table/index_fns.rs index e6712995..20f6f5e2 100644 --- a/codegen/src/generators/persist/table/index_fns.rs +++ b/codegen/src/generators/persist/table/index_fns.rs @@ -99,7 +99,7 @@ impl PersistGenerator { if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None }; @@ -140,7 +140,7 @@ impl PersistGenerator { #column_range_type, #row_fields_ident> { - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.indexes.#field_ident .get(#by) @@ -173,7 +173,7 @@ impl PersistGenerator { let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { ( predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), @@ -183,7 +183,7 @@ impl PersistGenerator { ) } else { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { predicate_range.clone() }, ) }; @@ -216,7 +216,7 @@ impl PersistGenerator { }; let predicate_filter = quote! { .filter(move |row| { - std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + core::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) }) }; @@ -231,7 +231,7 @@ impl PersistGenerator { #predicate_setup // Query construction is not an active read. Pin the grace // period on the first row lookup instead. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); #index_range .filter_map(#select_row) diff --git a/codegen/src/generators/persist/table/select_executor.rs b/codegen/src/generators/persist/table/select_executor.rs index 1499d250..bec09452 100644 --- a/codegen/src/generators/persist/table/select_executor.rs +++ b/codegen/src/generators/persist/table/select_executor.rs @@ -44,7 +44,7 @@ impl PersistGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - #variant_ident(std::ops::#range_ident<#ty_ident>), + #variant_ident(core::ops::#range_ident<#ty_ident>), } }) .collect(); @@ -65,8 +65,8 @@ impl PersistGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - impl From> for #column_range_type { - fn from(range: std::ops::#range_ident<#ty_ident>) -> Self { + impl From> for #column_range_type { + fn from(range: core::ops::#range_ident<#ty_ident>) -> Self { Self::#variant_ident(range) } } @@ -100,8 +100,8 @@ impl PersistGenerator { let col_ident = Ident::new(&column.to_string(), Span::call_site()); quote! { #row_fields_ident::#column_variant => { - let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(std::cmp::Ordering::Equal); - if cmp != std::cmp::Ordering::Equal { + let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(core::cmp::Ordering::Equal); + if cmp != core::cmp::Ordering::Equal { return match order { Order::Asc => cmp, Order::Desc => cmp.reverse(), @@ -165,7 +165,7 @@ impl PersistGenerator { _ => continue, } } - std::cmp::Ordering::Equal + core::cmp::Ordering::Equal }); iter = Box::new(items.into_iter()); }; diff --git a/codegen/src/generators/read_only/index/usual.rs b/codegen/src/generators/read_only/index/usual.rs index 0af30608..9739b3bc 100644 --- a/codegen/src/generators/read_only/index/usual.rs +++ b/codegen/src/generators/read_only/index/usual.rs @@ -236,7 +236,7 @@ impl ReadOnlyGenerator { fn process_difference_remove( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { #(#process_difference_remove_rows)* core::result::Result::Ok(()) @@ -293,7 +293,7 @@ impl ReadOnlyGenerator { fn process_difference_insert( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { let mut inserted_indexes: Vec<#avt_index_ident> = vec![]; #(#process_difference_insert_rows)* diff --git a/codegen/src/generators/read_only/locks.rs b/codegen/src/generators/read_only/locks.rs index 280afd28..ffa50040 100644 --- a/codegen/src/generators/read_only/locks.rs +++ b/codegen/src/generators/read_only/locks.rs @@ -24,7 +24,7 @@ impl ReadOnlyGenerator { .keys() .map(|i| { let name = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); - quote! { #name: Option>, } + quote! { #name: Option>, } }) .collect(); @@ -124,8 +124,8 @@ impl ReadOnlyGenerator { .collect(); quote! { - fn with_lock(id: u16) -> (Self, std::sync::Arc) { - let lock = std::sync::Arc::new(Lock::new(id)); + fn with_lock(id: u16) -> (Self, worktable::prelude::Arc) { + let lock = worktable::prelude::Arc::new(Lock::new(id)); ( Self { #(#rows),* @@ -154,9 +154,9 @@ impl ReadOnlyGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let lock = std::sync::Arc::new(Lock::new(id)); + fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* (set, lock) @@ -186,8 +186,8 @@ impl ReadOnlyGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> std::collections::HashSet> { - let mut set = std::collections::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { + let mut set = worktable::prelude::HashSet::new(); #(#rows)* set } diff --git a/codegen/src/generators/read_only/primary_key.rs b/codegen/src/generators/read_only/primary_key.rs index 192a2e44..8c8274c7 100644 --- a/codegen/src/generators/read_only/primary_key.rs +++ b/codegen/src/generators/read_only/primary_key.rs @@ -133,14 +133,14 @@ impl ReadOnlyGenerator { fn get_generator_from_type(type_: &TokenStream, i: &Ident) -> syn::Result { Ok(match type_.to_string().as_str() { - "u8" => quote! { std::sync::atomic::AtomicU8 }, - "u16" => quote! { std::sync::atomic::AtomicU16 }, - "u32" => quote! { std::sync::atomic::AtomicU32 }, - "u64" => quote! { std::sync::atomic::AtomicU64 }, - "i8" => quote! { std::sync::atomic::AtomicI8 }, - "i16" => quote! { std::sync::atomic::AtomicI16 }, - "i32" => quote! { std::sync::atomic::AtomicI32 }, - "i64" => quote! { std::sync::atomic::AtomicI64 }, + "u8" => quote! { core::sync::atomic::AtomicU8 }, + "u16" => quote! { core::sync::atomic::AtomicU16 }, + "u32" => quote! { core::sync::atomic::AtomicU32 }, + "u64" => quote! { core::sync::atomic::AtomicU64 }, + "i8" => quote! { core::sync::atomic::AtomicI8 }, + "i16" => quote! { core::sync::atomic::AtomicI16 }, + "i32" => quote! { core::sync::atomic::AtomicI32 }, + "i64" => quote! { core::sync::atomic::AtomicI64 }, // The accepted set is `worktable_dsl::AUTOINCREMENT_TYPES`, and the // arms above must stay equal to it. `check` uses that list to // answer "would the macro accept this", so a second copy drifting diff --git a/codegen/src/generators/read_only/queries/select.rs b/codegen/src/generators/read_only/queries/select.rs index 0adcbe70..8c46e611 100644 --- a/codegen/src/generators/read_only/queries/select.rs +++ b/codegen/src/generators/read_only/queries/select.rs @@ -32,7 +32,7 @@ impl ReadOnlyGenerator { // Acquire the grace-period guard only when iteration starts. // Merely constructing and retaining a query builder must not // stall retired-link reclamation. - let iter = std::iter::once_with(move || { + let iter = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .iter_values() @@ -55,7 +55,7 @@ impl ReadOnlyGenerator { return None; } current_link = replacement; - std::hint::spin_loop(); + core::hint::spin_loop(); } None }) diff --git a/codegen/src/generators/read_only/table/impls.rs b/codegen/src/generators/read_only/table/impls.rs index 6e4577b6..db0fd756 100644 --- a/codegen/src/generators/read_only/table/impls.rs +++ b/codegen/src/generators/read_only/table/impls.rs @@ -237,19 +237,19 @@ impl ReadOnlyGenerator { let pk_types_unsized = is_unsized_vec(pk_types); let index_setup = if self.columns.primary_index_backend == crate::common::model::IndexBackend::Arctic { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( ArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); } } else if self.columns.primary_index_backend == crate::common::model::IndexBackend::Congee { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( CongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); } } else if pk_types_unsized { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( IndexMap::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name) )); } @@ -260,7 +260,7 @@ impl ReadOnlyGenerator { }; quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( #pk_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) )); } @@ -369,7 +369,7 @@ impl ReadOnlyGenerator { #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds + 'a, + R: core::ops::RangeBounds + 'a, Pk: Clone + 'a, { let converted_range = ( @@ -378,7 +378,7 @@ impl ReadOnlyGenerator { ); // Delay the grace-period guard until the returned iterator is // consumed so an idle query builder cannot pin reclamation. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .range_links(converted_range) @@ -460,7 +460,7 @@ impl ReadOnlyGenerator { quote! { pub async fn iter_with_async< F: Fn(#row_type) -> Fut, - Fut: std::future::Future> + Fut: core::future::Future> >(&self, f: F) -> core::result::Result<(), WorkTableError> { #inner } diff --git a/codegen/src/generators/read_only/table/index_fns.rs b/codegen/src/generators/read_only/table/index_fns.rs index 98a1d0f6..13ebb9c8 100644 --- a/codegen/src/generators/read_only/table/index_fns.rs +++ b/codegen/src/generators/read_only/table/index_fns.rs @@ -99,7 +99,7 @@ impl ReadOnlyGenerator { if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None }; @@ -140,7 +140,7 @@ impl ReadOnlyGenerator { #column_range_type, #row_fields_ident> { - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.indexes.#field_ident .get(#by) @@ -173,7 +173,7 @@ impl ReadOnlyGenerator { let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { ( predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), @@ -183,7 +183,7 @@ impl ReadOnlyGenerator { ) } else { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { predicate_range.clone() }, ) }; @@ -216,7 +216,7 @@ impl ReadOnlyGenerator { }; let predicate_filter = quote! { .filter(move |row| { - std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + core::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) }) }; @@ -231,7 +231,7 @@ impl ReadOnlyGenerator { #predicate_setup // Query construction is not an active read. Pin the grace // period on the first row lookup instead. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); #index_range .filter_map(#select_row) diff --git a/codegen/src/generators/read_only/table/select_executor.rs b/codegen/src/generators/read_only/table/select_executor.rs index bd8b3f7c..7ebe9806 100644 --- a/codegen/src/generators/read_only/table/select_executor.rs +++ b/codegen/src/generators/read_only/table/select_executor.rs @@ -44,7 +44,7 @@ impl ReadOnlyGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - #variant_ident(std::ops::#range_ident<#ty_ident>), + #variant_ident(core::ops::#range_ident<#ty_ident>), } }) .collect(); @@ -65,8 +65,8 @@ impl ReadOnlyGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - impl From> for #column_range_type { - fn from(range: std::ops::#range_ident<#ty_ident>) -> Self { + impl From> for #column_range_type { + fn from(range: core::ops::#range_ident<#ty_ident>) -> Self { Self::#variant_ident(range) } } @@ -100,8 +100,8 @@ impl ReadOnlyGenerator { let col_ident = Ident::new(&column.to_string(), Span::call_site()); quote! { #row_fields_ident::#column_variant => { - let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(std::cmp::Ordering::Equal); - if cmp != std::cmp::Ordering::Equal { + let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(core::cmp::Ordering::Equal); + if cmp != core::cmp::Ordering::Equal { return match order { Order::Asc => cmp, Order::Desc => cmp.reverse(), @@ -165,7 +165,7 @@ impl ReadOnlyGenerator { _ => continue, } } - std::cmp::Ordering::Equal + core::cmp::Ordering::Equal }); iter = Box::new(items.into_iter()); }; diff --git a/codegen/src/mem_stat/mod.rs b/codegen/src/mem_stat/mod.rs index ea8d6b49..b674be14 100644 --- a/codegen/src/mem_stat/mod.rs +++ b/codegen/src/mem_stat/mod.rs @@ -3,11 +3,11 @@ use quote::quote; use syn::{Data, DeriveInput, Fields, Result, Type}; fn gen_heap_size_body(data: &Data) -> Result { - gen_mem_fn_body(data, quote! { heap_size() }, quote! { std::mem::size_of::() }) + gen_mem_fn_body(data, quote! { heap_size() }, quote! { core::mem::size_of::() }) } fn gen_used_size_body(data: &Data) -> Result { - gen_mem_fn_body(data, quote! { used_size() }, quote! { std::mem::size_of::() }) + gen_mem_fn_body(data, quote! { used_size() }, quote! { core::mem::size_of::() }) } fn gen_mem_fn_body(data: &Data, method: TokenStream, default_for_copy: TokenStream) -> Result { diff --git a/codegen/src/persist_index/generator.rs b/codegen/src/persist_index/generator.rs index 49b78938..754780a6 100644 --- a/codegen/src/persist_index/generator.rs +++ b/codegen/src/persist_index/generator.rs @@ -193,6 +193,7 @@ impl Generator { fn gen_persist_fn(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); let ident = name_generator.get_work_table_ident(); + let page_const_name = name_generator.get_page_size_const_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); let version_const_name = name_generator.get_version_const_ident(); let index_extension = Literal::string(WT_INDEX_EXTENSION); @@ -216,15 +217,15 @@ impl Generator { }, _ => quote! { { - let mut file = tokio::fs::File::create(format!("{}/{}{}", path, #index_name_literal, #index_extension)).await?; + let mut file = worktable::prelude::fsx::create(format!("{}/{}{}", path, #index_name_literal, #index_extension)).await?; let mut info = #ident::space_info_default(); info.inner.page_count = self.#i.1.len() as u32 + self.#i.0.len() as u32; - persist_page(&mut info, &mut file).await?; + persist_page::<_, { #page_const_name as u32 }>(&mut info, &mut file).await?; for mut page in &mut self.#i.0 { - persist_page(&mut page, &mut file).await?; + persist_page::<_, { #page_const_name as u32 }>(&mut page, &mut file).await?; } for mut page in &mut self.#i.1 { - persist_page(&mut page, &mut file).await?; + persist_page::<_, { #page_const_name as u32 }>(&mut page, &mut file).await?; } } }, @@ -295,18 +296,18 @@ impl Generator { _ => quote! { let #i: #parsed_type = { let mut #i = vec![]; - let mut file = tokio::fs::File::open(format!("{}/{}{}", path, #literal, #index_extension)).await?; - let info = parse_page::, { #page_const_name as u32 }>(&mut file, 0).await?; - let file_length = file.metadata().await?.len(); + let mut file = worktable::prelude::fsx::open(format!("{}/{}{}", path, #literal, #index_extension)).await?; + let info = parse_page::, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut file, 0).await?; + let file_length = worktable::prelude::fsx::file_metadata(&mut file).await?; // Pages sit at a fixed #page_const_name stride // (header inside the slot): the next free page id // is ceil(len / stride). The previous divisor used // stride + header and an unconditional +1. let page_id = file_length.div_ceil(#page_const_name as u64); - let next_page_id = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(page_id as u32)); - let toc = IndexTableOfContents::<_, { #page_const_name as u32 }>::parse_from_file(&mut file, 0.into(), next_page_id.clone()).await?; + let next_page_id = worktable::prelude::Arc::new(core::sync::atomic::AtomicU32::new(page_id as u32)); + let toc = IndexTableOfContents::<_, { #inner_const_name as u32 }, { #page_const_name as u32 }>::parse_from_file(&mut file, 0.into(), next_page_id.clone()).await?; for page_id in toc.iter().map(|(_, page_id)| page_id) { - let index = parse_page::<_, { #page_const_name as u32 }>(&mut file, (*page_id).into()).await?; + let index = parse_page::<_, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut file, (*page_id).into()).await?; #i.push(index); } (toc.pages, #i) @@ -370,6 +371,7 @@ impl Generator { fn gen_get_persisted_index_fn(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); let const_name = name_generator.get_page_inner_size_const_ident(); + let page_const_name = name_generator.get_page_size_const_ident(); let idents = self .struct_def @@ -401,7 +403,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(UnsizedIndexPage::from_node(node.as_ref())); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend == Some(ArtBackend::ArcticMulti) { @@ -415,7 +417,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend == Some(ArtBackend::Arctic) && is_unsized(&ty.to_string()) { @@ -428,7 +430,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(UnsizedIndexPage::from_node(node.as_ref())); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend == Some(ArtBackend::Arctic) { @@ -442,7 +444,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend.is_some() { @@ -462,7 +464,7 @@ impl Generator { let page = UnsizedIndexPage::from_node(node.as_ref()); pages.push(page); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else { @@ -472,7 +474,7 @@ impl Generator { let page = UnsizedIndexPage::from_node(node.as_ref()); pages.push(page); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } @@ -491,7 +493,7 @@ impl Generator { .collect(); pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else { @@ -502,7 +504,7 @@ impl Generator { let page = IndexPage::from_node(&node, size); pages.push(page); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } diff --git a/codegen/src/persist_index/mod.rs b/codegen/src/persist_index/mod.rs index ef17c461..c284b9d4 100644 --- a/codegen/src/persist_index/mod.rs +++ b/codegen/src/persist_index/mod.rs @@ -46,7 +46,7 @@ mod tests { #[derive(Debug, Default, Clone)] pub struct TestIndex { test_idx: TreeIndex, - exchnage_idx: TreeIndex>> + exchnage_idx: TreeIndex>> } }; diff --git a/codegen/src/persist_index/parser.rs b/codegen/src/persist_index/parser.rs index f7023697..4cb43d36 100644 --- a/codegen/src/persist_index/parser.rs +++ b/codegen/src/persist_index/parser.rs @@ -46,7 +46,7 @@ mod tests { #[derive(Debug, Default, Clone)] pub struct TestIndex { test_idx: TreeIndex, - exchnage_idx: TreeIndex>> + exchnage_idx: TreeIndex>> } }; assert!(Parser::parse_struct(input).is_ok()) diff --git a/codegen/src/persist_index/space/events.rs b/codegen/src/persist_index/space/events.rs index 28b66819..720096ad 100644 --- a/codegen/src/persist_index/space/events.rs +++ b/codegen/src/persist_index/space/events.rs @@ -100,8 +100,8 @@ impl Generator { .collect(); quote! { - fn first_evs(&self) -> std::collections::HashMap<#avt_index_ident, Option> { - let mut map = std::collections::HashMap::new(); + fn first_evs(&self) -> worktable::prelude::HashMap<#avt_index_ident, Option> { + let mut map = worktable::prelude::HashMap::new(); #(#fields_first)* map } @@ -126,8 +126,8 @@ impl Generator { .collect(); quote! { - fn last_evs(&self) -> std::collections::HashMap<#avt_index_ident, Option> { - let mut map = std::collections::HashMap::new(); + fn last_evs(&self) -> worktable::prelude::HashMap<#avt_index_ident, Option> { + let mut map = worktable::prelude::HashMap::new(); #(#fields_last)* map } @@ -200,7 +200,7 @@ impl Generator { quote! { fn iter_event_ids(&self) -> impl Iterator { - > as Iterator>::flatten( + > as Iterator>::flatten( vec![ #(#fields_iter),* ] diff --git a/codegen/src/persist_index/space/index.rs b/codegen/src/persist_index/space/index.rs index 166b514e..de1cfab1 100644 --- a/codegen/src/persist_index/space/index.rs +++ b/codegen/src/persist_index/space/index.rs @@ -9,6 +9,7 @@ impl Generator { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); let ident = name_generator.get_space_secondary_index_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); + let page_const_name = name_generator.get_page_size_const_ident(); let fields: Vec<_> = self .struct_def @@ -20,31 +21,31 @@ impl Generator { let t = self.field_types.get(i).expect("field type was collected"); Ok(match layout.art_backend { Some(ArtBackend::Arctic) if is_unsized(&t.to_string()) => quote! { - #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::Arctic) => quote! { - #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::ArcticMulti) if is_unsized(&t.to_string()) => quote! { - #i: SpaceLogicalMultiIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalMultiIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::ArcticMulti) => quote! { - #i: SpaceLogicalMultiIndex<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalMultiIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::Congee) => quote! { #i: SpaceCongeeIndex<#t, { #inner_const_name as u32}>, }, None if layout.logical_wti && is_unsized(&t.to_string()) => quote! { - #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, None if layout.logical_wti => quote! { - #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, None if is_unsized(&t.to_string()) => quote! { - #i: SpaceIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, None => quote! { - #i: SpaceIndex<#t, { #inner_const_name as u32}>, + #i: SpaceIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, }) }) diff --git a/codegen/src/persist_table/generator/space.rs b/codegen/src/persist_table/generator/space.rs index 35a9c929..a93e59a3 100644 --- a/codegen/src/persist_table/generator/space.rs +++ b/codegen/src/persist_table/generator/space.rs @@ -32,33 +32,35 @@ impl Generator { let ident = name_generator.get_persistence_engine_ident(); let primary_key_type = name_generator.get_primary_key_type_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); + let page_const_name = name_generator.get_page_size_const_ident(); let const_name = name_generator.get_page_size_const_ident(); let space_primary_index = name_generator.get_space_primary_index_ident(); let space_secondary_indexes = name_generator.get_space_secondary_index_ident(); let space_secondary_indexes_events = name_generator.get_space_secondary_index_events_ident(); let avt_index_ident = name_generator.get_available_indexes_ident(); - let space_index_type = - if self.attributes.pk_arctic_string || (self.attributes.pk_unsized && self.attributes.pk_wti_logical) { - quote! { - SpaceLogicalIndexUnsized<#primary_key_type, { #inner_const_name as u32 }> - } - } else if self.attributes.pk_unsized { - quote! { - SpaceIndexUnsized<#primary_key_type, { #inner_const_name as u32 }> - } - } else if self.attributes.pk_wti_logical || self.attributes.pk_arctic { - quote! { - SpaceLogicalIndex<#primary_key_type, { #inner_const_name as u32 }> - } - } else if self.attributes.pk_congee { - quote! { - SpaceCongeeIndex<#primary_key_type, { #inner_const_name as u32 }> - } - } else { - quote! { - SpaceIndex<#primary_key_type, { #inner_const_name as u32 }> - } - }; + let space_index_type = if self.attributes.pk_arctic_string + || (self.attributes.pk_unsized && self.attributes.pk_wti_logical) + { + quote! { + SpaceLogicalIndexUnsized<#primary_key_type, { #inner_const_name as u32 }, { #page_const_name as u32 }> + } + } else if self.attributes.pk_unsized { + quote! { + SpaceIndexUnsized<#primary_key_type, { #inner_const_name as u32 }, { #page_const_name as u32 }> + } + } else if self.attributes.pk_wti_logical || self.attributes.pk_arctic { + quote! { + SpaceLogicalIndex<#primary_key_type, { #inner_const_name as u32 }, { #page_const_name as u32 }> + } + } else if self.attributes.pk_congee { + quote! { + SpaceCongeeIndex<#primary_key_type, { #inner_const_name as u32 }> + } + } else { + quote! { + SpaceIndex<#primary_key_type, { #inner_const_name as u32 }, { #page_const_name as u32 }> + } + }; quote! { pub type #space_primary_index = #space_index_type; diff --git a/codegen/src/persist_table/generator/space_file/mod.rs b/codegen/src/persist_table/generator/space_file/mod.rs index 93626f25..c883e42e 100644 --- a/codegen/src/persist_table/generator/space_file/mod.rs +++ b/codegen/src/persist_table/generator/space_file/mod.rs @@ -241,7 +241,7 @@ impl Generator { data.set_page_id(page_id.into()); page_id += 1; - std::sync::Arc::new(data) + worktable::prelude::Arc::new(data) }) .collect(); let data = DataPages::from_data(data) @@ -251,13 +251,13 @@ impl Generator { #primary_index_init let table = WorkTable { - data: std::sync::Arc::new(data), - primary_index: std::sync::Arc::new(primary_index), - indexes: std::sync::Arc::new(indexes), + data: worktable::prelude::Arc::new(data), + primary_index: worktable::prelude::Arc::new(primary_index), + indexes: worktable::prelude::Arc::new(indexes), pk_gen: PrimaryKeyGeneratorState::from_state(self.data_info.inner.pk_gen_state), - lock_manager: std::sync::Arc::new(LockMap::<#lock_type, #pk_type>::default()), + lock_manager: worktable::prelude::Arc::new(LockMap::<#lock_type, #pk_type>::default()), table_name: #table_name, - pk_phantom: std::marker::PhantomData, + pk_phantom: core::marker::PhantomData, }; table.validate_persisted_state(path)?; @@ -310,7 +310,7 @@ impl Generator { data.set_page_id(page_id.into()); page_id += 1; - std::sync::Arc::new(data) + worktable::prelude::Arc::new(data) }) .collect(); let data = DataPages::from_data(data) @@ -320,13 +320,13 @@ impl Generator { #primary_index_init let table = WorkTable { - data: std::sync::Arc::new(data), - primary_index: std::sync::Arc::new(primary_index), - indexes: std::sync::Arc::new(indexes), + data: worktable::prelude::Arc::new(data), + primary_index: worktable::prelude::Arc::new(primary_index), + indexes: worktable::prelude::Arc::new(indexes), pk_gen: PrimaryKeyGeneratorState::from_state(self.data_info.inner.pk_gen_state), - lock_manager: std::sync::Arc::new(LockMap::<#lock_type, #pk_type>::default()), + lock_manager: worktable::prelude::Arc::new(LockMap::<#lock_type, #pk_type>::default()), table_name: #table_name, - pk_phantom: std::marker::PhantomData, + pk_phantom: core::marker::PhantomData, }; table.validate_persisted_state(path)?; @@ -353,11 +353,11 @@ impl Generator { let parse_pk_page = if self.attributes.pk_unsized { quote! { - let index = parse_page::, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; + let index = parse_page::, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; } } else { quote! { - let index = parse_page::, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; + let index = parse_page::, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; } }; @@ -372,17 +372,17 @@ impl Generator { quote! { { let mut primary_index = vec![]; - let mut primary_file = tokio::fs::File::open(format!("{}/primary{}", path, #index_extension)).await?; - let info = parse_page::, { #page_const_name as u32 }>(&mut primary_file, 0).await?; - let file_length = primary_file.metadata().await?.len(); + let mut primary_file = worktable::prelude::fsx::open(format!("{}/primary{}", path, #index_extension)).await?; + let info = parse_page::, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut primary_file, 0).await?; + let file_length = worktable::prelude::fsx::file_metadata(&mut primary_file).await?; // Pages sit at a fixed #page_const_name stride with the // general header inside the slot, so the next free page id // is ceil(len / stride). The previous divisor added the // header on top of the full stride and lagged one page // behind roughly every 512 pages. let count = file_length.div_ceil(#page_const_name as u64); - let next_page_id = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(count as u32)); - let toc = IndexTableOfContents::<_, { #page_const_name as u32 }>::parse_from_file(&mut primary_file, 0.into(), next_page_id.clone()).await?; + let next_page_id = worktable::prelude::Arc::new(core::sync::atomic::AtomicU32::new(count as u32)); + let toc = IndexTableOfContents::<_, { #inner_const_name as u32 }, { #page_const_name as u32 }>::parse_from_file(&mut primary_file, 0.into(), next_page_id.clone()).await?; for page_id in toc.iter().map(|(_, page_id)| page_id) { #parse_pk_page primary_index.push(index); @@ -399,9 +399,9 @@ impl Generator { let indexes = #persisted_index_name::parse_from_file(path).await?; let (data, data_info) = { let mut data = vec![]; - let mut data_file = tokio::fs::File::open(format!("{}/{}", path, #data_extension)).await?; - let info = parse_page::::Generator as PrimaryKeyGeneratorState>::State>, { #page_const_name as u32 }>(&mut data_file, 0).await?; - let file_length = data_file.metadata().await?.len(); + let mut data_file = worktable::prelude::fsx::open(format!("{}/{}", path, #data_extension)).await?; + let info = parse_page::::Generator as PrimaryKeyGeneratorState>::State>, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut data_file, 0).await?; + let file_length = worktable::prelude::fsx::file_metadata(&mut data_file).await?; // ceil(len / stride) counts every occupied page slot, // including the info page at id 0, whether or not the last // page fills its slot. The previous floor + inclusive @@ -410,7 +410,7 @@ impl Generator { // exactly fills its slot), failing the whole load. let count = file_length.div_ceil(#page_const_name as u64); for page_id in 1..count { - let index = parse_data_page::<{ #page_const_name as u32}, { #inner_const_name as usize }>(&mut data_file, page_id as u32).await?; + let index = parse_data_page::<{ #page_const_name as u32}, { #inner_const_name as usize }, { #page_const_name as u32 }>(&mut data_file, page_id as u32).await?; data.push(index); } (data, info) diff --git a/codegen/src/persist_table/generator/space_file/worktable_impls.rs b/codegen/src/persist_table/generator/space_file/worktable_impls.rs index c3d19f7d..e67694f0 100644 --- a/codegen/src/persist_table/generator/space_file/worktable_impls.rs +++ b/codegen/src/persist_table/generator/space_file/worktable_impls.rs @@ -33,13 +33,13 @@ impl Generator { /// Retires an Arc-owned table generation after the caller's /// quiesce barrier has stopped new leases and drained old ones. pub async fn unload_gracefully( - self: std::sync::Arc, - timeout: std::time::Duration, + self: worktable::prelude::Arc, + timeout: core::time::Duration, quiesce: F, ) -> Result> where F: FnOnce() -> Fut, - Fut: std::future::Future, + Fut: core::future::Future, { // Attribute the generation at the retirement request. The // quiesce callback can give background maintenance time to @@ -54,10 +54,10 @@ impl Generator { )); } - let owned = match std::sync::Arc::try_unwrap(self) { + let owned = match worktable::prelude::Arc::try_unwrap(self) { Ok(owned) => owned, Err(arc) => { - let outstanding = std::sync::Arc::strong_count(&arc).saturating_sub(1); + let outstanding = worktable::prelude::Arc::strong_count(&arc).saturating_sub(1); return Err(UnloadFailure::retained( arc, eyre::eyre!("cannot unload generation: {outstanding} Arc lease(s) remain"), @@ -80,7 +80,7 @@ impl Generator { /// Returns the physical size of this table's `.wt.data` file. /// Persisted vacuum makes freed pages reusable across reloads, /// but does not truncate this file. - pub async fn persisted_data_file_size_bytes(&self) -> std::io::Result { + pub async fn persisted_data_file_size_bytes(&self) -> Result { self.1.persisted_data_file_size_bytes().await } } @@ -184,6 +184,7 @@ impl Generator { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let pk_type = name_generator.get_primary_key_type_ident(); let const_name = name_generator.get_page_inner_size_const_ident(); + let page_const_name = name_generator.get_page_size_const_ident(); if self.attributes.pk_congee { // Congee durability is maintained by its native checkpoint/WAL. quote! {} @@ -198,7 +199,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(UnsizedIndexPage::from_node(node.as_ref())); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } @@ -214,7 +215,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } @@ -226,7 +227,7 @@ impl Generator { let page = UnsizedIndexPage::from_node(node.as_ref()); pages.push(page); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } @@ -257,7 +258,7 @@ impl Generator { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); let mut pages = vec![]; #collect_pages - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } diff --git a/codegen/src/worktable/mod.rs b/codegen/src/worktable/mod.rs index b7820ef8..fae07538 100644 --- a/codegen/src/worktable/mod.rs +++ b/codegen/src/worktable/mod.rs @@ -481,9 +481,14 @@ mod tests { ); } + /// This used to assert the opposite. A persisted table was refused any page + /// size but 16384, because the seeks computed every offset from a hardcoded + /// constant while the generated table threaded the configured one, so the + /// two disagreed and the file was silently corrupt. Both take the stride as + /// a parameter now. #[test] - fn persisted_tables_reject_non_default_page_size() { - let error = expand(quote! { + fn persisted_tables_accept_a_non_default_page_size() { + expand(quote! { name: PersistedSmallPages, persist: true, columns: { @@ -493,10 +498,27 @@ mod tests { page_size: 8192, } }) + .expect("a persisted table may choose its page size"); + } + + /// What is left of the rule: a page on disk carries a 28-byte header, so + /// one this small is mostly header. + #[test] + fn persisted_tables_reject_a_page_smaller_than_the_floor() { + let error = expand(quote! { + name: PersistedTinyPages, + persist: true, + columns: { + id: u64 primary_key, + }, + config: { + page_size: 64, + } + }) .unwrap_err(); assert!( - error.to_string().contains("cannot be combined with `persist: true`"), + error.to_string().contains("below the 512-byte"), "unexpected error: {error}" ); } diff --git a/docs/crate.md b/docs/crate.md index 993dc840..12155086 100644 --- a/docs/crate.md +++ b/docs/crate.md @@ -8,7 +8,7 @@ does not provide multi-table transactions or multi-process access. ## In-memory quick start ```rust -# fn main() { futures::executor::block_on(async { +# fn main() { nagoya::block_on(async { use worktable::prelude::*; use worktable::worktable; @@ -38,7 +38,7 @@ String and tuple primary keys accept borrowed forms, so callers do not need to write an explicit clone merely to perform a lookup or delete. ```rust -# fn main() { futures::executor::block_on(async { +# fn main() { nagoya::block_on(async { use worktable::prelude::*; use worktable::worktable; diff --git a/docs/page-size.md b/docs/page-size.md new file mode 100644 index 00000000..f33eee9b --- /dev/null +++ b/docs/page-size.md @@ -0,0 +1,105 @@ +# Page size: every place it is decided + +A WorkTable page has two sizes and they are not interchangeable. + +* **stride** is what one page occupies on disk, header included. It is what + every file offset is computed from, and getting it wrong puts a read in the + middle of a neighbouring page. In `data_bucket` it is the const generic + spelled `STRIDE`, a `u32`. +* **inner size** is the stride less the 28-byte `GeneralHeader`: how many bytes + of row or index content a page can hold. It is spelled `INNER_PAGE_SIZE`, + `DATA_LENGTH`, or `DATA_INNER_LENGTH` depending on where you are. + +A generated table emits both as constants named after itself, so `SomeTable` +gets `SOME_TABLE_PAGE_SIZE` and `SOME_TABLE_INNER_PAGE_SIZE`. Every location +below either defines one of those, threads one through, or consumes it. + +## What it costs to get this wrong + +A persisted table used to be refused any page size but 16384. The seeks computed +every offset from a hardcoded constant while the generated table threaded the +configured one, so the two disagreed and the file was silently corrupt. Both now +take the stride as a parameter and `page_size` works for a persisted table; +`tests/persistence/custom_page_size.rs` writes one at 8192, checks the file spans +several pages of that size, and reloads every row. + +**Three separate places had to be found before that passed, and each was silent.** +They are worth knowing because the next such parameter will hide in the same +kind of place: + +* `check_value_write_bounds` measured a slot write against the crate default + rather than the page. At a smaller page it let the write run past the page end + into its neighbour and returned success. +* one of two `WorkTable<...>` emissions in the in-memory generator hardcoded + `INNER_PAGE_SIZE` where the other used the table constant, so an in-memory page + and a persisted one disagreed whenever the two differed. +* `SpaceLogicalIndex` and its three siblings wrapped `SpaceIndex` without passing + a stride, and a `= DEFAULT_PAGE_STRIDE` default on that parameter made it + compile. The index file was then written at 16384 and read at 8192. + +That last one is the lesson: **the defaults were removed.** Every instantiation +now names its stride, so a wrapper that forgets to thread it fails to compile +rather than quietly picking 16384. + +## data_bucket + +| Location | What it decides | +|---|---| +| `src/page/mod.rs` | `PAGE_SIZE` (the 16384 default), `INNER_PAGE_SIZE`, and `DEFAULT_PAGE_STRIDE`, which is `PAGE_SIZE` as a `u32` so callers wanting the default need no cast in generic position | +| `src/page/util.rs` `page_start_offset` | The only multiplication of a page index by a stride. Everything else goes through it | +| `src/page/util.rs` `seek_to_page_start`, `seek_to_page_start_relatively`, `seek_by_link` | The three seeks | +| `src/page/util.rs` `persist_page`, `persist_page_in_place`, `persist_pages_batch` | The writes. `persist_page_in_place` also checks the payload against `STRIDE - GENERAL_HEADER_SIZE` rather than the crate default | +| `src/page/util.rs` `update_at` | In-place row rewrite. Two parameters because the bound it checks and the offset it seeks to are different quantities | +| `src/page/util.rs` `parse_page`, `parse_pages_batch`, `parse_general_header_by_index`, `parse_data_page`, `parse_data_pages_batch` | The reads. `parse_data_page` and `parse_data_pages_batch` already took a `const PAGE_SIZE: u32` that nothing used; the stride is a separate parameter and that one is still the payload length | +| `src/page/index/mod.rs` `IndexPageUtility` | `parse_index_page_utility` and `persist_index_page_utility`. The default body's overflow check uses the page's own capacity | +| `src/page/index/page.rs` | `read_value_with_index`, `persist_value`, `remove_value` | +| `src/page/index/page_for_unsized.rs` | `persist_value`, `read_value_with_offset` | +| `src/page/iterators.rs` | Reads at `DEFAULT_PAGE_STRIDE`. A standalone file reader with no table to ask | + +## worktable + +| Location | What it decides | +|---|---| +| `src/table/mod.rs` | `WorkTable<..., const DATA_LENGTH: usize = INNER_PAGE_SIZE, ...>`: the in-memory data page size | +| `src/in_memory/data.rs` | `DATA_INNER_LENGTH`, the row area of an in-memory page, and the one place still fixed to the crate default | +| `src/persistence/space/data.rs` | `SpaceData`. Its `PAGE_SIZE` is the stride and is passed to every data-file call | +| `src/persistence/space/index/mod.rs` | `SpaceIndex` | +| `src/persistence/space/index/unsized_.rs` | `SpaceIndexUnsized` | +| `src/persistence/space/index/util.rs` | `map_index_pages_to_toc_and_general` and its unsized form build a table of contents, so they carry the stride it will be read at | +| `src/persistence/space/logical_index.rs` | `SpaceLogicalIndex`, `SpaceLogicalIndexUnsized`, `SpaceLogicalMultiIndex` and `SpaceLogicalMultiIndexUnsized` each wrap one of the above and pass the stride through | +| `src/persistence/space/index/table_of_contents.rs` | `IndexTableOfContents`. It lives in the index file, so it takes that file's stride | +| `src/migration/mod.rs` | Reads at `DEFAULT_PAGE_STRIDE`. It opens files whose table it has not loaded | + +There are no defaults on these parameters. A default is what let +`SpaceLogicalIndex` wrap `SpaceIndex` without a stride and take 16384 in +silence, so every instantiation names it and the compiler finds the ones that +do not. + +## worktable_codegen + +| Location | What it decides | +|---|---| +| `generators/in_memory/table/mod.rs` `gen_page_size_consts` | Emits `_PAGE_SIZE` and `
_INNER_PAGE_SIZE` from `config.page_size`, or from the crate default when it is absent. **This is the hook.** The same function exists in `generators/persist/table/mod.rs` and `generators/read_only/table/mod.rs` | +| `generators/in_memory/table/mod.rs` | The emitted `WorkTable<...>` takes the inner constant as its `DATA_LENGTH`. There are two such emissions in this file and only one of them used to be parameterised | +| `persist_index/space/index.rs`, `persist_table/generator/space.rs` | Instantiate `SpaceIndex` and `SpaceIndexUnsized` with the inner constant and the page constant | +| `persist_index/generator.rs`, `persist_table/generator/space_file/mod.rs` | Every emitted `parse_page`, `persist_page` and `parse_data_page` call passes the table's page constant as the stride | + +## worktable_dsl + +| Location | What it decides | +|---|---| +| `dsl/src/parser/config.rs` | Parses `page_size` out of the `config` block | +| `dsl/src/model/config.rs` | `Config::page_size` and the span kept for diagnostics | +| `dsl/src/validate.rs` `validate_page_size` | Refuses a non-default size on a persisted table, and explains which half of the problem is still open | +| `dsl/src/validate.rs` `validate_arctic_page_size` | Refuses a size above 65535 for an Arctic-backed table: Arctic packs a link into one `u64` with 16-bit offset and length fields | + +## If you add another location + +1. Do not give the stride a default. Every one of the three bugs above was a + place that compiled because something else supplied 16384. +2. Assert file lengths as well as a round trip. A round trip alone proves + nothing: a table that silently fell back to the default still reads its own + writes. +3. `DataPage` is an inline `[u8; N]`, so a 32768-byte page overflows the + stack of a debug-build test before it reaches any of this. Measure large + pages in release. diff --git a/docs/wt-user-guide.pdf b/docs/wt-user-guide.pdf new file mode 100644 index 0000000000000000000000000000000000000000..38928a849a7f3e0570ba918743ff2395a2df493b GIT binary patch literal 111263 zcmd@730zF?_W+LDwGgRL!bq#M%uFkZ_FelfnfA0%lNKscN=n+%f}|9pMT?~pMfQYL zNTrR6EETE#&s}Dk${qQ8f4=|!@AdPV_xpBd?mf@h&w0*s?sNI{lvPFWqS8!!`@ugZ zCOl3Y=V7yxNls48$Zwaum>$W|-b+l$!`q#VlN8f(w)3(i5(wZT100b+M2`~aQ4&2$ zK}Rcjc_w>zI|vwbJqlEhPy0PM7)=zgnmw89?Cyy3B9lDH_I4E5vf>kALzGN} zjlhEug~>zVDS5bhySY$pRGmG65D+}IN#IWedqg8dh$!yL_O`Ah zPl&-J9GPU}YENY_eCPP+5Pa;>VM!yLLHLen1x@U41*RlqQEf9Fct>rFxnm@JCxTRp6)=WJv_(Jhl!&Z<2S}f!-@&$ zn~-e1z1^tTPO>|ggOeD6a1vnzq9ZW2h#V2sK{1DQ2aixRNqO|_J-wX0$oB5G6e@_~ zlk5@?0jD!IqW=h|5b6-=LIQDccD48N1Mn%75ydB2D}JiE(%J-`#$aJYC~{EpaJTjL z1P;yC4`mure3I3|_tHTeQF8=wS|e7qCy~890rIXSM=zX@hbP$y=i=_+1H?`p3MQ0B z(wLTwfJdmH_*r`Zoa}*f_3&`PxdS&$g*wS#VMd??z!5>$o9ycB4uS$AUPPjZbP-|@ z>Egv{FrzWet=?{4mg2yx!#Cpa;4hdC;H?QV@YoczBS-3_M)zbx6BF08=KB?a9t=_F{&+NbX`r_MUEDAf9=W5i1hY69ad8 zxH}?0tGK~Od3iA{OM(onAXd=TAX-770XPcg@NW>}2n7IdgaCjxfIYV%voFPb1&Ja#e&JZLh zX9yCMGjt#X3G_=mk}%=$NNNL)$jEqcWX!2YWOTeZG7?@K8Grno@;x#(UK|+}k7RH- zJdy{|9MS8Mya|UVATq(Du@R33%kgt`JT%7P(a49VoTI@Dj|MnA^&F*(dXCaZMCm~T z8=iVZ@gk!5qrnYNJ*Rw+uyfM);|2o_0~-t;jc9l@B95Pmw==iT~<|avlxEcr=RQ(Wr=*Ksir2pK?U`h(<#^^@v#5_&Lg7G&15PQ9e@7Q9LC7 zpMFFwQS$%lh+2f?|IrZ*rFb+q!Bfvsc~j3(y`Y|>@|HsN77b;1G03s zK56tmGzj6TN0e@9R8M74k48PBdPkYappzFoGGPG~E>upk=(uQ{z*Emryr}1t>nKMQ zKUvBV9S=Dl_e{{wh-Z>TDUqd&OgW;OPyHUHj1oI09SLYGA)v8i{D@#YevV)|evV*B z5Jxa3pfQ7h#+LDOgrfvB_Kbgz;5&Yf;5mMd-iMTOa0H~X0FLOm)FX-~^_=oMitm)~ zQGDSt0GMx0IU@Wc;3?yycv8<^vd5h&9@$~pCj z(t*Z20ujZJh>lM=q7jOKMx?1n^m^3e6HpILKt1;OIZ7|``?%&o>IQ4x_>~lhC`VAA zPC$J(0rklQ)Q629(d*Hv1VIYr5cM3D5Ng{5>Jc3e%|;1Rjwq+3(fd)`CLl@WI3*AR zAxbD!Z=|H4;%X0y0-#Qy2d3gMMxZtz4ypoNiZIR|FBJB0z?Z=OZroIqRvg+x#q1c3oqDgnb1;0ej}-~xLF6Dpv^K<|O35;4pk4j9ve zNW&v|Vl%zFK*RzpCSvFWJQJEGAXyw>E8u_x;rIFl$F}Lg0{DZG!3`1^o(qnP(}X3B z1JM#paXMP%DLn-5+BK z6+4iOgF2TKhCab*!1V9}kzGm}2Xc2Q3~h%Cr_+N4vH_6%gM42a!&$>=$uwa};H0sr zgVM`sLXpBrW5^C>B_@==cVTGqN2$LffeWoYprr;h@1xYpkt`2Q0@2wyn!r)2)d&=% zB28oYa9Z~#+>t^waBk9AlBFABP!3R`$Y6L`IJKNME;3k}F`Zm&92YFj_@e=h=anEO zMYCft!9;5nU}}kG%3yMd&cngGBP+&;RI7s(6|65pt3|9 zm{x-MqpTE877O|x^EN7USq#Sxr}NW85X?!TnkH1wgld^k{Spu6l~A1$s!`&_rNMtP z;6HF3NX+nH9UH1m;=$Yws!ig-91ahqGk9}p<5f9=dRCmM^F)}c?0x>;upn4=8L_ItRdw7tofI8#v;}yJG zFin`i#G&dV9)xSCs)z?>jR!W32L_D?%UV#y4XUz16*oMvV5nk-2eu1U$nZed@IcG( zK)>)nv+zKdLFEMlC%T#D-@7Kt>Ju`_qbwgGlXPUYgvPhPQ$0+Epo1f|9w| z8+9Ur%!nbcjH4LsWI%N79_1Ys44$Gxlc7v!x;74ll95b?0-a+LH3b)n-A;xARRg9E zg^~?UhB6(i7^et}yL5xzM5O}B15$&Oh4LCmsesp`lrxDS%R#=A#G&O5;QJ`WO`56G zWHRWQhB{%w6rbAl2Bs(&rUsW*{={5l9S&$60mIhdqU-dbVAud$s+~R*3=zT8#%V(# zVi*8iCZ0YN48FjO!-O(XT|k&a7f(T;L#x=xWD+f5gII@5iP1?RTFIt{yGfb?7q~IU zo}?+%W=2q{!jjH(3Y_(6xfeP$V#PA7oX3b}Up} zBrx0)d;un(C(0P46zK9bNGi~k2apz^YY!kX0C@q76)5syhyh-gz#MLpSgF0GT9(vV z@+4Nmt+1HzP*x&K4nV9D7)A%LicA}dB!g8nxYVR3gl>Q1GInio#y%XHLz914a@gD=43%S3etSuN6O4V6~Vg)@-Z!eTbS zf!r2dECbmsx)=uXTbO0wS8xZCioqohoGgUB0VrGiJ~2Z&DlqY)CQy_a+%&3+)bhwA zJ|ayU*h4|xi+@$hmBLUSq$>n-Ak;Shh9V57jI^6z4)piMX}BN+(jP2Y(Y4S?S@h&? za*+FC_;aKy29pewUue$^ltW{vIC2FhoQXOP5;{sZ4#?xEMW-n`L)V0<>IqGffto|e zVJZb(Gd5BNx(7|KAt*FvwzMuzUb*6F*H~ zIJ$^yG)WX_4*=T7Hi44qL|CY<1Thg*(y)97d_FSW`3x9g;c9FPu-f06^z+(9b` zxJBU41i+3RQY4K09BG-vg!5k!0A_cTCQTWvNCC468@X(~KsWeOb(0c$WiV!)NVn_%oB%KZr1aOyVtG%PP1u-B z7A-6q0CkGh((f2S5(lhY7Rekyjmg8)Qd|^(M*g!BLw+!Fa<+8=4<31eA0*JE0?A82 zM1@5Jj<8KB)n{ckr@B??I_gcA=s+NOYBI;H6}y;KII4VGg&OVfjizX$C{My(>&n`=km!1L? z`T~=oRKdV10!XHobf>`iqg9QQ46mKT06>Wm!%E?f4NSaXE26fG(DZRZT^-KplPB+#~9BX2m2~>dv#a}vY9#}(8$2ckCn?_w0YJ`A23)*Z0HAH}O zTF*kiwT7a0$RjCGfE33P4cys|sV7qbQP6+VrN5d4Oqu;`URP?B;sd%){ z1}_CuM9T}fSpw7MPy*oT%s&Z%+TuQ$fpCk)|H1gcmr@(#r{F+07*FOUrM(1cE}6hf z2rwos6t{!4oj_VeZ4;S-5M5J6g@UB_|FY^pX@wzRxj(o=2a}-x3=gE=5X0esSWbtj zQ-lL2H{eMPIBHE04qcWMUk{F6Har87Lz5rl;YT~~k# z2iDbp!h?uaZGdUSwp_SmIUXJm#sZRHM?`BiaN7(v9E6{6aQZhousjgltn+^`K2};r zziVl-f}>4F=;O^3_((rbfY*%w!_qI>3yG(+J;F{Dlzh>aHayxFg-6<-Xc48~@kDWx zNP7ts!>A2xlVjMlTAk1bzF|OEDHi>1CukBylY@hrE*>TU5DT@Gqa875TRBu#!!X!s zbwlBi78(d$2`r6)FTvE;m4;3X|PXR;M^d_N2X?%i8ov0`#!9#Wc zz!dN#QBZab)Uu>hf~J6@8$wZoL$?ut6p!L5r@*0Gr1*;Hg@6GRKQV;?&=;79QyD-_@Gu2JcZNF!4mEk6f)8DD zmdq8!HJ-8`1tep5<^ex`i!@Ln&){N+n|}Kd zg$+o{3FKTT4tg^B^t*|s!ol!TglTmXq1$Fa_+Y6qj4`IVO|%n`E>bq+KprtBplh@6 zbH?B|q?%8=f-r@Ax>iH2Sx`dZG%9=4`u>!mpf4~f`}@U8u#^Sj9Q7gODZ|mVw@LnB z!mgsAYL6-9X&E1~A`YZX?f;&F+_a2u91d2{fUm&h=|pV-YdHj@g$g`v4Xq{+(2c{u z!UwhXKN(T_9YGTZ05yAR?SBd!y5VS&H0W-`3Kkf!q(i^^XIz<)?ie5+Ni63GUxCS2 zc%DT-du9HiQbK?crY(9C=(qdOkN`6nA^{^mA;9~g{vVJqd2WaB1$$s6FK7AA;h%7ARmg9(S?nUQuFh!I#G6}|!!4kAmqL*<|PGtr%dCJ;b3GExYjwyH{E z8TlXjd!WfO;ok`W3o(@4kKolYjJQj`mj|A?(n=Se!vpjtmu3jl>g6F&dxE4W6^Y4? zg?=v&DqX}_{>4&}z8_lY(r@$muLh>>6ETHu=nHJEo@8YRAAuaESS$Uu9;%YUxf!rB zDJ^Y5^(njwoby1>fZ;ZtkGjQD0l#DBqkXK!c7p6;8b}6 zF?2KI|GKLZ%_b<092{Y(f+K7*06}=MYyyIGJpjdWkp>rFxzq=4rllT#FP^hBnjx2Dq!7=vWp&AfFdAmYM^U_(t@s|WFV%aq@j!xM>dm# z9SQsx4gBN+^=Gib8no9O`ZJc2(KY!=>8z>K)Q$f^=7WMp2cm8hO}~91>NeDa8ex!f0DA?uG;y>sePDRMPq1Q&A&o!uO-+^t%uP@ypEA9s-~U9B z2C|L|NrSR#G1+6lSNusDhe53UAa`S0QL1RUohxEqo^~pncqFx{I znP8hx_)DZ04r+>rEYA93XI}ZsLzzPu)x(S&aniF6j5@6O;txd2;YQ3I`D>D955&fazU4c5 z-IRtm;qbIZ0r7+G)dbqBA*|_MO{BegvICuh33#5A zI_aI#4>sj$+E=(yySFUGG?bJSNnZALU~6TVspI?0Q(LSiTm(y&<|2rb5KWXftcsbL zTY$}ZMTt`2)dApn5J^c-p={m+ulClz_(h^XmgXSKJOtJ1E%81HJLd#4v_fRB@agQ`n7H|RbFEsa1 zqK)C!c9N2^06WO5 z(%eI%48I)+qDThP6PkNybm4dRNQmP_fj&TuGBh_qOs2yHf;8X)c=yJ{k!Z95l9$0` z)=kw0BHYvg4FsxfdY&G(hW2DjG4PsW@CHa=Kyw5NBcP(z;5b@(-d<6yQ(98Qy%=I zI20lrLjGq6A#8yWf?cr~j1Y|GL@X$(`#)jJw6UPr!v6#d8sVS_LBfIAGLaV)>-wKy zLBk6aq2FUMrELh(GciMN8#46fZKx6~E(Qe@_-kcVVaFLwj>cn4_7-e z6?a%O3cW^*xeaiZx2e!sA^Mn zbV}>d1YAI=8ajtiBK!#@aIGQywp|T(2M@qs1A9mCa$iqBK?U$GaPWd(U1)nKfU+P3 zWg$TFu3g|=E6|I$5dsqh>7^{VQXE(<*zX@~ zyDcRO9+CwK9oSr-=1LH8X?_EbThjano{R*^tu%lMH4}n|Bx$aMp8SUP)u)UISPMQx zB2iRa97Gf3k?`@Gp@+k1k$|2Qr$qyrq(iTOpbQ4uX23^!{2YFue0(^Nuz}AwGoa_p zK{E`MNa&$+@Te!11;DjKA1XW$j=_iKdhiMgT6{q01!+Ca?|>6BwBv!28+f9XNTIjv?&xX{tV99w0A%o_usG@s$T$)3V4SEdXtR+vg*cnw|o`$#>0ClaY?5*zsaP&AN0WTv4VxNi`cX?{imGbMnbahb{- zBrcDShQ{Ub-;uaXz5O>~8RScV45-B$3{6capfC(|^#Wf7cS%zn8WM5G5kw;H_>Caa zfzNnkM?x-5L~Q9}JXOpbwxDEct?B-uN$I#Sna;U%*Nv_|W_U-~*UcPQi^` zG-Du}fKZfqg0rQWMp143cg}W-fm5CAg!DoLFCz*Zs}z{>ihxod$S#1VmnDjW;t?nt zOB1C;Wk5x8qKo~TG=rXlg;IzKSq4%IruA$z(={r!SPoWIM@>UtUs&0GC&|RykmT;A z=FPN)(d3bx;+IxX13zDnp*xNai zkOUmqFc<|alYl+m+8})78oU85i{%*sQaQPCC>fHBtf3-iIM=Ei?pXadg24RI%vD4

-@w0Y9I=byR|*p(~MAGPUN3mcC|YD;duh zcH2v;=T!+zxYS>6C@|)K{zN9vK5Gsw18c2Q^@;5>>zc))N)IR=v9~SpSj6Wue{aVQ zRu%oi3>AI$s%OT(1Xc;W+9Q_`#FDCOs%4?9A1kuSS=V{yNx%CNf~u|yHLr-xI~Ojb zH!n6o_cZsn1)5ugpRzec`G0x%I_~I}6?XCmi|(ZyV7gP37QbQ(yYrf%s?Ub40&~jQ zzq9|cSu=F*{OLO$vx83^iV3z(41Sk%#$|5#oUcZ$^XJbkH(OUBc6%GRtR}?zkdX5b z-|?%5toi4ZFORElO?umzS*4%fa{SVkUD{7e6UUzRhGrddTtDZ@oR?f_oE(PVB`bVwq?)h9_YdzbiL-@hq-ZxU0n%XyyhVI&1EaNXcI;%uc zDeZTEbz& ztPBh$?@yXls4iUAo~c){E>Txx=8YF$ug*(&X(+bs;#KvL9Y@k7t_E|sxw>4Hx@r-x zTbO3jlp|8pA?MPoL0ISMpVqlnm+%6y#<8Fbx-yGGO%+E!v8sB!j>B&A2 z!GkL!S2ZpyH|bGGU03G+lG!6S_`}@;$wGG>t_v;;RScA`Ow&kwtDIM> z5?a-&Ung>dXt|25b-+X`Wi+uWBUs>b>ou}|XjQDfib`CPPNQ&{2`P^pUlr?FP-)X} zK3!<9)$Oc@JB8du8chn_8abLGGTBAkuG?Ie-*)pc_`3P!Nx{YkJ&E=zdr#Ea^trh{ zT7he_w7hFWbX_nj$QRn=QFVxs^WqsoKVp&Zv?Xo^m~`Lwuo@9Pvo5Mo{X=qnC08 z-UgvnIr?>u#oB4{jwdg1oZh`?Ia{RYHeWNlJYU|W+U|L{n87*6Wb-6zp7JW)7UA@d zo*T$~R?RGfZ~YC%BNF9v-+W14y1hK~C2{EV^{ku?PV*kN%p~rL9<9g@%^lopEpcsB zvi6x{sz1BJ0=Wy3qWM3n%}rYp7I-kN)gt7V*`-Fb#4L!myp} zsNA=An>V*yD;ew`91Bsc>#qO4(tqQL_u;~ZK5=VIV-BfWg zhd-}kb-b{h=lv4W99FJy@hFB%p7K1~v_dW^mn1Xn4gWlcHBBJBDCdXBp#dY~BkD3C z=E`mh9b)hgSF)&7N9JC*Bve~a+z?bW)->ud{DgJoZFg`1*NY#TiogABhv)N$s#b;Gk> z_QI2!*`L^Mc6W2wcQTU(OA6onf9I%PLs;%JZ;X>X&x$~-!dDOOvHufanxtv=txID~XH+*l8R*cul)8#wJEoZtf%72Vq zVCu;l$XshnB3qVRuTt37wSI-UvB>i~4qJk5xfcbIbfvEK7-l`LU&J_@GwZSJ$RT+X z>n*ZvzOvp$8P^xx-?*N$C0<0pJCl7eE^_HT_gxqH`${ZR;_eK;5!uSJ`+hhvQO59a zonp$lI}KY>MD8?fO?h)CFWf6$rYv-R_SU1qEUjlW8DF#&f8Mk4Ox}GD)~+H3Ugv_Q zN^O}Zkx6Ir9&@jZ3YmSzNn820!#P3ky|1foYwmRU@bNK+_xEQJN5cGGuPWH8em3{= zqa{iqO>Hb;t8QA%3G#d(a#DD6^Ju8p8}G-V%r65wvj!grobOF1u`S*HZ2Ky$;?18U zZshM~?(=ScVJf-Zpz-~Nf!>aeQ%xr%mqm(vO-OD#ByIA%uXON-R?h7c-AadEp4#m2 z?X%aXE=xPn^0xb1UL|)&Cq6!TsMcAKf{w8_9RaOgECnhP=df8J=Zv8$;gcQ}(tw_`=X`YAWiSA;CAYW~DMIDtIyyf^iGyoKQVx61;xxy{mH?G>=_p z%n!rqE`IOFA(|SJRPPp(bUsJwP(ot-p>ud)wZ|-B5~e!Y2B+EL3QadS7)!qrPzikF zw=6g4yGPI_ftR}mZ?@Ug2YIziGku6~aF=8m90^cua2j4VcBMDKqP>x`xyXI zg5}re-f>STP#a6;wB%Ns8 zy9!?z7R-qbKl_+LpC^9FnKXt+%Exs>_>|3bLs)poOGwLDSr)g1M>K1uvTi-Uen(8- z@-Xs)mZ?ZWxWwoM#^hqd*Ez9_f1Cb%VMHV^Em3$OJvZWT5S8u=-luyw;0Yddb9`?`jF z$3^BQrt&DP;pC|z#vb2-&p%hXA$f4u65X+mP4C~GS+K~qUz}r+h_6{h*h1e|!C}3= zucGENFH-A2e_SHORqp8K_q^*p*|xq-;DJn_Cn+PXLeR&L&v(FC3~dA8AK15c67%BvP1<{MeM<^=z&bygN7 zBTKspu{kS^j2A81`}!Q;)dQ`K_?bj;{%zd5NabKCxN<~}Y@UW<5?4u(eIB#?_7oGA`HP#dm_m59@p1mTrRyC8; zr+BU5_u(reYNW=X=gC_)U6MY0X<&1x+WnmFZ=b%m_^&uSU~+&K#e^ser$FU zT}mofInZ`@uCmbS_*ox>PM?_7VOX(*$xL_VO7^E3Gp`H2n=zOB%<1N6W*LQ+9}ou7_-`I(;x_<5Lh1{=+lky%tIy1<%ZrS3NYXuHDPk8xj7}QnYdF^_{b8^=W!JKc-EnC$? zEi=3y%yL>>EnCI`MBDd+CnJum<9xX2$Kz}JhTT{%UUu((^)=@Wn~2Y)YkfBc$^!h` z0=y1MZyK;^7Qf`Ry`9@pEP2Z>v-max<1d{t(FNbq4lMrkDlKMxiPYhck$l18pd&l_ zx|=ySOTD!}etxr3{AR7W#)6!oN40IdZdv*93_^_oiDUDInR!m;A(Nt zL8!qMmn>pekm|Z8E=om8t!7olXW66u?Th)s3_Q;ZggJN~*JaVVN7^JyKBgof+rtw! z`~5Cu6E$$ju3+v^Mr{dCZi%HPyOlWBlXyae)@ZLJWiEDDMOw<{u#zOj)*)Ki!}~C> zQckU;dYmgY6tHaSDohf9&_@DF_;S|0HnK z3$O7$qO^42L6xP$`qzVxvzp5XOV5uex%KB&z8w0t?C9YKcUtSW7etpolYLt_<`85z z_FZ>#>DRWmaW4s+Nhf>w17gZ%#D)0s8Cz-$a~HllZ8yhoIcrULhPZ)Yq|JLRRe9&w zGqMKt;pzGjs++hTIXzUfW#sx;+qLB^Ta=X(_m(yWi}L4^TTf>{7h!%?LbON`+^u=o zP&dV)^`XGyC9M|)w$9(Z;c&qbnfc*f$_JbzEmGK+BD9vLDr}RDU+}&C=(z;1*e`)v z!UAEQ_XQo-w`-qES$f}GQ+9=*BXhfJR?3@rTm8rS_l->!u$_M0-<=bpa`jvHs>3_x zjacw}yXf9vaV306MDx0cCfic;z$2Qa3epM6=QBbyKb*H?N_coSw#q2~sHT)*jg?V; zoaTUGjhE3tlL6l+y=>PSJsG>;+4_ZzCwuT`&&D`i;$-l9-EqY?n{zkcN7MN-5d!N> zUmFr#m2LVGty6z?MVYfDN=5AYD&3PR)^y6@@U@4ZORnd>+EaJ`I+x9pcv0s@yuD_> zUwYx7hmC*VJ;~;c-}^6L&R?}}N1@4;OJC3I{Z#JuQ`5Zi%2G`5!~<zmJtShIjPw|8MLo1@#WYrBW6qf*^O(g)VEazuR< zd;ZK{toz3M3(?KH$=e^T$*y0yW1Ay6yRUYw<2|kuju&6pEl+%Q?254EzyPlk_qtnW z*Ed~1_gLhRCd13Nj1xV)4o5bG8pRLVaqniz7A#ffREVFKuDw=MSU-`q-6GwU@BSO* z2Bmm`@Go-ua~ssc_lGdls_%LAsdgJ*l|u!$2=flp!cUfs-j?P5pXx=qulEIsxr`<# zpSk=*Yq^z$>%+u^EOVFgz=xw-%6BMzx?@_u>2qS-#$?^{-1;tN!Nq5cgx=TabOtcy z-Z%VV$eDUeqk%|{e2klakg%ZIMx5i|(gle&S7sg*S?+pwhxELq^VIl!)mzt{YU7eG zRnzR(kB)ZO_werBCHpSkRhuurp+Ap#_59Dfm9Kt@cNvm7!xgMJD=K9)gnxy@cFoTQ zVM#*1C&H4de9gm>9{H9si*4-BUoG~erB_-_?9)5%9Vg>%+wIfOH1)W)$-O|`aB22O z$vwI!&0mc6HvamBFSf$p6^^LubS)Zl9{Xw-GtzRT@@ZO%-W&4W$Q4rBP5JxYM!B3~ zw;dDmtTJ<%QC#BFm85seHamYckIE)ql>z;l@W8yVJ^V9b?c%jmOAo3OK8q`R&j{Ag z^d_YovX@=^!exQiY2mdW!@h<5vN^f#*!yK_IT<@xzgqJ0dU|CL!r!FFIOU57LrVgc;+up zTUTz!{MwmI>viYt3z~ax_tq?_y2_>!u)AVG$en?n4~-kYbXfI_#+*0br|_o4%cSF? z`?B^Tkwrd>7t9SPdmECh;dp6Z?8tRuv4mu#-JqOPN6oySdbPg4tniK7J;p3NKJ@Oc z(%lqQt-7};FglP?L7S9zRdH0YRU@=3^ypfSCFhPAFlA>Qb6`55yGBdUZc`{(PhB9i zC;ixLcF&N?K(%%6=Y+1itL|!jKK@jwLQx!t-IMb&Ezj@etHsxf)W)&(*_nUPx_6hT zzS?Qcp_Gbct(z1_4u~m>u2Ef_eqeDOL#<}GeMm{-VoS!|(rX45w=ycq>8e*9{ua*? z8$Rb3`va+Gr)OTv{~|yJS+ompNYMH-}gu4nbNi1d#@hH+2jsoyALmG=EyCU6xp=# zikxBjyj#I_m$~XWJ@keeWP)Bcs%mcYo)M&WEN*0Fg+%cC#Tk(}TSh_xPUH*se%uO% z{7_sIV}3GjslrkIjr*T*d9M5#?$3a z2Y!9nKBshRO7w!IqO9Ky6+<@gXs@Z1<5|yNxq-)(ztT$il}5;jvTttrr2!_^xB709 z<|W5xhiVm;OVk@l=Wm}EFcK5iQ@5vnbfkRWHsw2wYNFp?-L*L^yQsI%zkhh}i4J?j zhx~-pyW81&TdR`>#{xbU34bhe2pa0nI<>7~*Q2oeo>FOnrl&4Q$2_PlNb?OiBU2L- znG*jrErQ6=_hm6obHBl?=WM4>%=pT9C1l1&#-|GVx6bO}shBy7yZf$HIbQ{1=ymQR z%raX=`?b^7ckbnNSSeS)z2O*_U1e_0sx&@3xt07)S`J%EtpZYcbxhB0XO|6XONs8_ zRhBdGe>9f!U|HtK=%(tfcSHKofxQ~NkIRgi*$V0Fes|2ZQ@rmqXs28cFNl*FKKylC5Z?&X?zj30g&}hf z>Kt)WPE1NWpY=R-@%JUQ&tE#_gtZ(mP7C-UHT)>+ZQv))7%>|w50iw)zC-heMc)p+ z87{L-;qYs5oWH%fHqg?|d;7^B_r$7FTQ`=!{u*c9SUmd6vN^@8x$n)3)y-vJL0Mv% z*qG6RyuH2h!|ll(Kjn^&L>4=KIPy(pj^L69kNvBUNxUf}a&I_K7B*hVqZw|X_?#{M z#Nw}vc_E8GGS(`DZ&mE!xieFdTc6V;d&3>ZgsgKO-68W9d&|poPn|LJx&A$NrW0ywjtJSU40X*sc3sd8IJm51oj(;G z3$uSYI!iXclKY}qYDch~e7vS^k%LuSMBkfz_eRa^`c+&d#j2C_@8Sf?)sh-c8*WM z(K5E>Jtmn-8lq@ZhnsQ#auBh>2n2DP?k)XC9N^F~rYOg@M{8v5`sqH5G`gc}8=9(F& z)1jcMO|Y>x@Di@pt69FmS52j_DJgQ}mv?5{I~fF`+0ttUOW$k_5YqqX>w%ThN&MeBbY3y0{scyC+ z#y5Q`XXh>ObMIa=QXoCAY**ozP?m2vK^b+6FT94wzvLVhCrZ?3mlZE<(sePnIgsf7 zX;|!|bHn#U4TB3$`M+;!e{H<$W`=ajxi#J$c5b^n;#`F6O!HU7zFgaGApFpRxL&(n z>t?}K^@)n(lw@=ZRn4_4Wabo2WAMZQnUS@7Vc zZKd<7l56%q_e3mdII1026&~lZ??!!){=Fl*tzc?+cIN83&$SWJ_CqC0S6lY)mX;rQ zr)1IjwdRQ8v-DYC_G?;|FTWHva!pHeWx{Iv2DP4~!WGSnod>dd+b&y?%K4Z~of-nh z?soYPiw$Jg-#OkW=^|QCo<5NFVeOc=Nqn60frF9TYVPTfcrR2C_l75t`Eo}e@P0Wh zvWZi#UExN#r5U?=y}B7YZ!~)V3B5qFVRWI_>S&&;x9gd$m`H}(>Kk8a1HotKs&nb1WA_))9XxA|?e&KLvn_b$$ZB}7HLu-d6hxzb7 zTRt&*b>6lK8M)}<^oj9i_TXAmlTX&$X3o7Cx54+JOk@(XL46v3r;Zk@&dy`IbNTqh zBsBv{;sjnSNj^L?C}#6d+km+iYRy*%wEWFB+l_Te1o&`a6b1p4TK~%bzFw+VbnMO_PT2Gt*RmTqd!-yAcT)w6cz z80o1yemj3_-j8r%pHD`y{{*RYwLB|Bz zACY1}O9cWA$G>f5)(%&`Yw+5DK+i0#e^n)P~Z59=`@`R?wKfa--iGoA+}RI~Q= zzgE@WyfyB^{uSRPU!Pxn_j&!Uqk$m>W-13G>4 zI_`h^cvrSwrjjLK;iI%!QqBu6*42v}ZGNm8u&~R8VTGC*uGwYb>-C$xq@U)lcD%7W zWOdiw8tJ2YXAe3*Y@7LG`{u_iwyPL@W}9pLTvN)j17xq3RZSM!$!!rO+v_Dq6wW?- zW#>vr^A__NyFZfGd3@QYQ+1|}Mh-XPNsce=27dl36V7ink9NE%%lrN^Bm3O4IIE^Z!4t@9ENo+kyNea62ws5qXEd8DlJ;A5%tL!(07 zsJis7^;dILA_gvWny$0h-l}6*+!~s8DXlrZ>vXz@DOtU?S)VLp|2Dkw!KR4E`X8Gw zb#1%)I3Qxx$uIl1Gg%!YIB3`E3SZNU`g$?FeK&c7n2`T*uCJ0RNf$bsO~?oRJ2t84 za_^IW6w}+xk;-$SFkaH}{zd`keK}p^nq{9k8qF&u3kwp9K5qQd!^^5kN(x(_r5RJxo9J6Dt^#@{kbUbi#D7tKQ=ttrnJH(ax5y12XA&e&Mj*yF{p@srIjhJByp-93>t^f+=vIIbS=C8-9$^3d%;v7qO@1FZ`C;oCx$hiW=VBXF# z+%M}ZJ2&W#vuIlnUpO*A2mWQLIYQ8*bn<0%rhJvWcrw%ZWnSx|n-+SFqqtFT(S{3>7iFxC zJ}MU|7ZcO{(yzHh+26Tl+hK z({x!;TT+OUf6$qr zu53%^N3LOhu5)JpnnK2zZ0|D7xun9xm(Q=}U*E!i;B|&j;iKwDjl6cHHF0kfUnkEy z_A@0<$4=2O@O}0--;UJ5lpl|M-hSHG%;~aahH^TGoM&f|MAK-|&bqi6X|soPz9xTI z7NFWR#_*JBi(|I6@8G^RC7;~6HeCC=Iu2c{xFL|}6t(64#u-d2xIWC%PqJM)yJOA_ zC9b8Cm)El#Hr`*#P;oC*t7D$F<0IyoUc?zq=@s|+du`qv+5dyI=cde+^L<0>Zu{&n zWqx<|Wa4@k{tGpH75+^7vz(o+>I%Xd>t{&K_*P~ZJo9d-b|F*Ey)z~=Ji_)fuh~;3 z>5C_;ito_>9Hg|?_wY6uKR>yj zV<40zOz!wm%e>VqWui=RF-qIy0{Fk}RzKqGRL#|M*H3%F$^})o4lmKJlDK$_F(Za~ zpVGY$)>lWZh8C{6`*mo^V4Uc)#XA{OnFE%$?g-{IK3}iN4DXU%Bk^NDrH4|U~R)fbq1#8NA&^2O=!_3>x9In08;cPUD5udwd- z*{7qOa=@d*`h8gF=+X#fKK>kHwi2_&6_$@zI9JNdi+LZr+isn?$|`=#n$;RoKEj=C zCcA`7kG!@ElnRm_Qw!wm%}5UERm>RD`E1su-Szz#>(bXp&zVJ8$wYaTgnX)Ci#e{r z#i^iqkDCxy7@L}ryf=j$m;GAE^Jq`!TY=Rvmuu{KcIj{9>3nyxRAk$HpObr|T;3Od ziT?cQYxC!`LIQH4`*<=Ak6p@M=+02Kl$-UZ_&2xku%&z0&(4Y4J%5&2F#qe|QzRnG zyBRF4GTSST_?nLd&LRF_YHs~Bxy9$+E zc_}eOFXL?@#o;dP+dC~v+2xZ@RXk~3^kx2!f?q3s*^d&$p8B3tf6(oFjqykq%e8}j znR}fDMmC8oRb`F~Z3;0V9lWA({K2VZZx@>!Nv_)3@6s=3lj)G|aM&R(lUb~dJxy&W z%!WZWeBb)&ZSk=d`3u|bwefCgd>ZLzrd!6Z!^`&K%!_@prylg)Wqf_aqNie=YlZ9R z?i(D9Ps$IAm@%B@*sG@~;$I+~t81$Bx+H*qrD~J$!1=4D z)DNF?u{vuCGJo(U|JsuMzPQFZ+c!2M=u&H!L5)Z8^%SY(`saE(<20fk2`62ZDD-pD zwA%Wtcjjp?S#$e-Tr6iIUi3=NEnZvhk1Yvt=L{~(sjo9`8j|qXktXMT@`>e1o8n=s zt;r9wbOtp7uRmzG_v=pedB0~jq9W#Bm0U2_Vc1UoT~aBRgYe5WE6y1-_&yQH{3JFU z<6~%c_TjmNxgQ$u+vz^4j(GckYY)dOqZUsSlLl~)`=w@qxT^tM#aAzjAGl^I;KylP zP{FUJ&S#+16nt{OPBq!uF<+oTc&_FMA@k>k`F``Vmx($SJAY@`oAIEbm+K?fgU4%o zGGE;6)D5bAFxJApG1;X2vqHuLbJ>xTGeV+xqmnjB=DX!?2

3R_W0*4Jj&B5!y4* z&z4lz?O&pxpD10z5?0j@k8z5yCvNYwTKGD zgi0K=a~YmT$c`!d>4{_Ovb`g5TYAT_xj!2%Vl+;#E5AdHLp>W8s=DHapCmyXHg8FeID)>+LeFq zrP;bTuB2H>h6jjEeFoysorU9m6cUO|T-}O_`oA{1+7}(FkUNo?-PtXlo%`sNZJd6) zOJU9NmVyVXg6>P2bq+4_5xPg(bAD&mBM;#Mk*s{RqCg;=~sjHd;?8aW#?FO@6jcC_d zrX(1T2y5QDvf8vnd>g6QP~l9@u3FuUUHy*)<$h(dui9-JsjLj!;i|US&ZF(=Y9orprP-8ZoT7yFPjcw+bZf+jIPV4?aPOamy3NySJ1Ms?IfL zncK)cS3>Q;%hNMD+4ozmxOOYfHi^CB>B6W8--E&7KBtd4-jtuqD{yW_$PMwvX0`pS zI04n+>k5ZGHD=<;O1S&yUf&2d+v}=&aZ^!kZro||JFMEw^?HkH&`@7! z`h~l>H|{@bmD*-?>&b{aw|+3kyQfS1qenGlqRcW{M?U8bXMQ=Dr)#}-L8N$yEL%;u zdcyu4U8+3#qEQ_0)OfgJ7;^nxc?;D;7!Ti5=7~5K&cv&DzHjFInTPXsS8aN_c1d`Y zu*qWQu0todOC_Y{avdjyaB_;@UP!!8s@6WPzldm35$P>{cBMnl0ZuhiHRo|wt4$^q zE5zG)9=^-*`l>#2^GWvicl&uVZgF;Vavf&R)H9XR(y-8x1S>Sr}+oSkOhKL1vB?|g15hqTc zpI^nKcX77LM!|vot%8D&_YXXYZ{-mz**UlWko$7u2mK>lOuj1|_T_5grRG{3e-YF~ zB&o7KJinh){&?*SqDf`scX8W|4_`48*^low;gFDy{b}oUJGfsjhVP4jC4U(Pa+qZrz`+rv6I`Z<3wUJ{HYuWJSI8UA7s~st79nCAx zt`^cM(Gp6t*IKwe@Y6>%gL}`-ONowe`&P!#m zlx5|>mawx=TNaQ<^6FZ6$Tu$(TGwO*|B{iOKfL4d%K1gEVy_G4B?_!_zcuQBkZiNcNyPUm? z56WB(yh1oHYT|QdW7+nXsuXrh7$6@vGOmhsrw2*d1<#nBVkrw7fH;q3l#{3~RMG zPa(SeB*ZkF9VS`wH4O_>CR#Eb@JBy>Z97WUGwU;y}X@Bcn9_bS!o#(PlsuytizbmWf4SZ7=%9iH)rRqcF7uH2A zsXL_>9}W*$%jYw1^|tMM##*F`mKp@Nbqh<(&kTy0zgbYpLe;4z`=gJx>K2!-LuJ;x zH$^^OJKMkHjGFAX=s1&{FX`A3Tnb`s5t^V6Wwi~~m;}*1vJzOrU zMcH|m|DlZuSFTE)Ecp@atNA2SuKdF2vp%ct6Am>e?C*(bKQ|KC)c;}MNyF9q#E066 zw|-cq4;0s?zsU&jWk}s;zcG1?Yrp)~2Y3dTAoA#DUz;xB1d&Dc|%qEbwur4^ie`fHt%Pz0V+_j&5Hn_pHE9(5=HC)?yzOO7g zX}fUsBAr#gp4M9PaT%EGO)g>uKz zI(GlCzSC`I?lSCKCa)yyRnH}L=1k;r@PAU|@|WdT2j{Yh&$>6`-8?;(9sK+MKa_n_ ztSG^@?Xqp#wr$(CZQHhOd+lZ0wr%faz5SB=lJjwroA=S_*_BQu)m1gSYRoZ5c>z!Z z)Bw-J8Nde=1Iz+^4}j1$fv;ZTL12JYQ^l$7Y<@0|Ma&sq%i|NvFc2Y&p$tSoVH^Y_ z!ZH|<#%RD3{~3ztVGs&LG&fXfqsKro$x%+2`El|}*kf0+)6?qha)5~>^df4?ks=Dc z6=L@`Sd74`wBI@#xsz5E>+EVXiY2pidWh?ghn1Ao)|pp1tCNfB3C4NRFeXMy+kFx+ zb(`G9G4gtR`+EPQAMw#bVVyH_Ayv7`iFMJ)J50NZlJS2iBq9OnSUxYlbE)D z#H1??7b4Gdkn89;X^0;6G_mOQR`=m_s{ZS&q-aT(f1Y9uo-kf zFn}i*df)Je+`I4t;(%^m;w}&0>t_(O!QC4mO1u}}G2HK%Jz}M5&0Xdr)ywZLxX|8j z%Z-%yvM>V0A>#aQZq`VR0Xl+c?~g7f+H(Th@qIu#oEIS+>7cDd>D0VX;D`%mS`zcR zKzd;?^{8YgOf^NqzqOJJ=IXquMbT742({Uh0H~j2&atr_X8O-Krj{HtT~so-Ureqd>)9C0YWeZMo{o53{+?;{Bk2ioR)@44PZIn zz_A9k6MIER5shC72Z=!mRPoSp=D)V3LOir2CkaWV>xaStY7Wd&%^ z3lUunIDwtI+&?SOJoI&8+EK7Ie6B;`0kGD7d23mW*UZoLPwdM$^K&G2~=KW*XD` z7_i;LQDWPXk`}{Ct|?yYV6&R7r=_AUAh=@U3q)El_*f61XyU`-9K>Xz$KoDDNJF&C zH(5%gUL#Bkq^A*{d!HjLpx_~NfQ4Mku;cUAnUlN2L(y9dxXSxOGAaga9x}p|lZaDP zkQadhosQMPIV8Lkf_zK{fq?xtammLRV)|n|;RkDsV~Q0_lD;rtMmbPmoGBbQMziLn+>X{v*zt(F z28t7HhIdSe*%D$sLTo^K!d?<%&&ja4D_56(TIaQx)x=*4Bd>+2H^B9Hk(%n6 zS@*nxI18(GPP7m(yjR!lQ3W@bNgT_Z$q%%oK%vdCAh{@66awaK)UX?0@`(eB0= zoA7T$+B7vY+Xl6UUz;rP^6*t#WDDK$>Lg~SmnC@fgra6{HBm7wHj!y9HBB-x{j=@3 zt2K)|beYwX>m4naSTC@;$V8J1VwbR57&j?u<*HY#P##^*FseObTnu1pN{$-TmZ6|L z%ba6U@rc1y9{Uw(SDdsgkW=Qtb#1Vv6ePsc0|`T2a1d2VsE+#&^ezL&q%e!Df&-1k z>)d}2BhrRI#*YC-hc`>o7|R$Pi;bhFq<%LF&H5J}=J?5s65CYO-jBc0`f}z3#t-{e z&)?MZPfy^&i|d9DDsuc@57Np&pH^N|liKRU9#_Y)wXtNJtZQf7ECiMJrq=&vZFaz3 z<5VPPmvkz52C7(bFBk2)>puCcZWE>A4f@={>=nM}buP}L$wz-wy*SuTEM5W{+qSc< zE+%1PP#cb>H8onTeAZdlna#A?)L5~pv0_)RDp$vioL<=0P-(8#B;c~y8<)z?)+CL| zXP+e70{T}9xi#n4rvCze7vY5bHw5-S@=X5+fn{X=U+~w@Uwj}w005d9h>QPL<9}q1 z{!fi;EbRaPApXBJ!z~7wu)81B?EwMr^CEg~TSO3xWUTQ*>jji8&en`+0b=YQj^jP} z#i^2^24S3TBGJb}lG8oCHg@>l?|tk~-34s;o8>j1X0KgW&Aj?8FD@qxe+KUt^D75= z-Mv-d_?FEY_Cm~Np1qI{=SQEH_xveh{H*&E1m}4xtkIW@lXq`szeiqO!$2k3mIs_s zGoHlG*~c&PSK*~F??uf#e$YOv_{Mg^Y)QDRo)_Dmlf)O<`yP{`_zd=?FdyakqQy48 z+s+nGZ!9rByTM*kXnKCvLrD=9?;~O6nD59sCBwC2PxvY6yAJ!oz`EIyAJc6)-Pegm z2_?(aVENx>h+kuOlPt)4Q&kE{f7+gWt~7vV{kd9x=vbty)u;Izt6TSj{tY3@(zKR} zsw%C~nBVm&&Og4_S9@K29ap5ZDNbIQDUy>E6xqw&0Zs~NRT5Pw+K}0SS@f2MDZRm+ zp=UUka8fq#3UHT*iXycJ)di~xZ3^-V))i1tFkEZ`l}N5u_nD|DZ>NWly^oMmZwZ3I z_Hl9+_m~ecK2oRzi~G$;$UbT>fpZ0Nu)RpS|DMl7!19IpTkj=T2Gz+-`_&1e0QCbY z#NHG#6Q7W=-zLQ=?l-_GIQb_4dOOzp)dR)@>IK|k-)boCIVZsR!t*fiQ8~1UeUbou z&^Bn_aJ0ZYwgU_#mNTAV3wJ1krv_bO8lsffT@) zQV6kSDvfQ2%36y?TdOVfS|H~qzuVk^fsd;z{O_;bz)bJk4)a;wSMR(hn!Pg_06+oI zS^>eIrhrWlfHynaJYH7wqYJWtO?bfZ#13aPn&SX{cm@|{fa^SN|I2OTFi&g1 zFje6BHYRa_F&Ci>OHlA!aJ_)McmS_i3Gs)Iy!s5f`N(GPAHh`Me!&ab;YlbCpw|I4 z3ve{I0Yr!)kaZYx2Q-{v>_XsOark2)>iL9#gL|}>zd1pc_be?#LL5-o2J{t#FwFxn zLx#9uUTHyCMhrM)15tV8gus&qs4hY%|A@-xx1N8@`G*Zybl|%T;c-IQ7vx_SNAu9V zW`W>~4G3!jb{wkAJ>8;Q9gkk90K@n~TXy z&rRg<0{~*T8^FW_nyv@>7i?ar@@{$kY!e5e4<>*X8xqxomm9FR2y^2I_G5Kpq91x~ z3m6^KEeh6~)EFUYjE87dgmPs7R~G`q4Kluwe+BFzV3!u7DUEqcdHujw7+}Dp=bKQq z2tc<8V%G%oI$&r{j|p-hr5_Bn1Nw_MjX#$J(vYj}d$1WpXy8i|8!gr=`&Qy)u zd%?NAxb%dt*$=}u;G+q*bwH6D^lT46X*wSSNHa#2>xW#A(w-39jsu^N;0Yq1u>0cm z3*6fm9YkPdOoq^uSWV_Vrtr+plC&+rEA*51m4-KgXwH34f_1=;P9A1G+~Pgq^*?{d zqu3V=pimirXot))gqsdEdPA)VK0ENV2$$mk8#@U3x`TERWcPvRCnx~K)rff3h$p_+ z$_ZnBVB`r2zen|?*+V1{KoOCz85SK)kThYlB)v;io0K$F6s~+w;a-vJk?WDyEumY4 zu`QW#q2&d+Ej3@#KBxT)=Y`%A?-w4)1e}RDtx=Y$IkYV;YbM8d*F3f1dd>HO{c@+1 zfjv8Oa2pE80VuxDKL09e4Avq{xDA!uh($gih7+>Yh`Bl_lo3?@fSm(y_m2=vnL z2oMpmHgzRupckk>!%WSL0|mzPbrCdo<=B-2*EM6sm>mPwBUh|OiOMs^#3G8&Wc?LP z{&6@E>Bx0nUpOwV^iPHSA5)<{jO zcvc+{m1CnHyNCX3+q#$D|4@we)%)hV@*}3-txdN!bpn3eRJOTC0l~S!rypx8)=D3T z?92KvK+C+kDpNB%F&fMWpm1s;9YqR8Re8h#W2MJ7gcdHS6rI&Rq_2zI_E~*mtx{0t zDdlrq5te0DPQY?%@_DFY;1b-YI6jd_8H;?YDE}beL5>(KiCn7m6btKE)xju7CgpoU z>{#8$WvBa5dNZ~(Rhs4fsS}gjzkgb34L)X!d{-%*E0;2xbyqtzfyXxLZY-QrV`CF> zG`TA%VVt9yi*>8wAFIOZm_}cBul{aclgrw;fgZj9`-#K&k8Ir0$p}Y`{jy=~_D(|n zSzzngQ7}2dm*Ffs%evU8(?Bba`|ILf^|M2~xbWv|{j4hI+OW7DAYd`~OZc!c(s9q^ zo$QALGDQ+!dCK7Cn5mqg2aA!s%dCvP4g-b!Unfu@m;(uC9n~7s!5fRYS~(z_tKKQE z<1RAG)L6w(gURYtU*#vqI+7JUeXjaJJLa6<{amfE&h)U_2)xoth*}oak4xcX)2C~j z7?f;nd@+~2+a{hZ@9_-TS6?VE@C?xl6324dMWMrCZx9Uo#?Gv8LPnUtO9vR44CxKK z#zJ0eW(HWj3Mv4&E`{WPm`E3~HFL!wJ0m3|=LTHU5RwCSZ1fH|QZZ;Gq3&tzZ4xT4 zet>Y?EV6gCL~|4UD(^DB*j_YjTH`*Mbemzf)2O&TKp zeMiXiK@~Fg@8$Zd==Z23Gx$i{U(r)mB09e z`K#bg1PY)6`6gU}KyYBB1~MdoK6*gs-5~d!ApH{{0WNjY0MC*@0_gy^$sh`yAQXIo z!u^2#>*It3{r&3_eu9a5^WlGkdZYBx+y9D}Jq;Fs==wYI10dW3FaT%B5fp@6g0P4Z z1k$gtkOWLR31j5KSC>njuQP<20AHU{qFTX z!H;;K4m&;=+ao%f$T~K%dJGlJK)5RLZhP?&^c8~#q+T}=s_`HE+Zjm_(>-C=l0A5B zbO{`H6u3YYxI%&1CTMzO!V=35j$}`8ofr|i;CDth}!xr8%Cu~kF-0spMsbiEaT!Vsn31|14G=CCY z4LHU_2jhTh*-Nk?3U$2w2KWrcmFW@Ak{AB9`7&8?=e-0(kR_{}UF3P80z}%~)V*!j zAa<5|9KVHx4aZqx9>)O$v{oHJGyyqb5)3edMoyq@fI48w1bIF^jV>UeZsne^G+-b= zD3Y@Uz=n)AK>QAA-p&ycB2j{*sCr(tgpa}L-Ibh$)i8O1NLJ8N(vcz~d_bY0q`jqL z-s<3MKw&1-)0y zm6QLNU#a`-5M4*aH_;0{K+l zjFS}iIwb|$DZR%4Z_|0#>9eGFyBV|~U5EMI`DWhojo4FlKYw<&K2nfh*Yp0N*aqYK zSa~E{+Ij+i8s~B`A5Vi+gA^PMw%6)U>qpc`0AUR2}g7yuuqpqN-sqabSDIUKJE^&1NS3M}GenbQlxgAm;5K%k~~nI#0j$YLKGwR&Za%e^BX z_9g4B7xEa?#ehO$cf)6DSj)NmTtW$G(2q+TYqk;< z`UEa=l$4vwG^#w$ulfz3tVRFM-Efe~#i=QM1bQNO$cp^$W$HlSvpY@?n>LTAI_zI~ zt1gu{wW-Y7x#dtQJgm=)^Tb~vb}I(;Z9kJeS#7;*oBZye+91i>afLvFHpE76K zPJ5=j<`s6Q6czoj(Z-?QEv=i5o4J3R<;eQ8C9<@^ySHXAQ-^I;89T%z*6+ z3OWal8b888(c=H^2TyxE^gK^wF%?5iaDn=txj6aoN?4b4pAl;nhzDTV7=(ixiDtjY zc8b%?&3C=wcAC>d{@xoD?~GfDIghY%F#5Cp~*hs*qzjTN5EyGJ4AcvRuqmg1L(;Vri+H7Q;yccRiIB}JXV#>K2Fyd-4$0tl=> zDofBWxDBDs(!o+{o;>Eae6+I@RskZOyo)e%GkcR}u%NAsMXIwomjh=3Z(x}Pz!t9z z5IsjykO21`0RY#gD$r~o>uJ>lKVNOyKq;)Y0;~{iIqp=0Dg9Zx3^(TVGewc@r?XU8 zP;)~{UYC#9`Xs)++^(3R$l+!IhQuN2C!8;KNZ(zvWeoO z$9F7jRycdLmzmFdJPv$@POaMUM0QO3%|?4R);P>O*3;Hncjxq&Or1-#R;5|>M`Nc3 zUESVeGs2uSS7-ML{E~zh2xmVex3N3dZQEvR19(IQ@cUqJLz}98qsuZGG9)K>gb!k=BD#~ zDjz$m@1wB8eQS%?pa1I?lqJV)f%p(#rq_Gzn*5J{li7p!SmQahi-EtS{xzDs5p@or z?WFhuRr$SC?#ktURH|}7Te)oBhdrOv>&0!5kF}LtJqIOg9m|At=DL%VudiI%acOsB z`w&g7ClKo4vkJQ|wATH=qK0B-?d+<-Rw#F-E*;x6c1_CU%~1@^iCYDY7TsEg^#n^c z&c$ZJ7G242%|mi+e(xHfBWI5lECLKtIU9skJsZS+QBc7{g;%Y9{rgqyrWd+v{ocXa z6q-UkGm0;M)yAunEd>)vqXqe{zp&wL9=bE0ez*9S`6CEjegpGv_eEaW*dC887~CXA z_kmCX$W@&i5#&tPdO~HW?l>bEE8~WX-2`B3WeVKv z0f1}S;K87db{X!q+?u?K!(^`fgtF+Rr}qkjo}1X`xnbN%i^b}v<4E2?mw66urf+|4 zA5R)*3(eR@AooT046+zM->sx1zI;_iQKL%gbatjsI3Cf32`a!dI2XLiet3^mFVigA z4d@>xIaEWPT=cbWcyGZ<_G?lfmMnb(8mA)FKtIrrg{NnCejggUK~nw5y~smN9ig*G z+x8GiJsH0k_{g@TfDq|u4_@4Y@Bp+$LqnQRO~3Ng5YK$+dea>m%SMwIgT*_qXInR2|%iOnJhEY>Tpd*MyL5Cy)CNw0G7VBUEO;c{_dP2)n zW-oGGEfzIt#FtB-V7>IH9bqqiPg`??`@$jC!2W$#!|-BzT(bCKDGw&#|Pv zo-MDV-67>a+l+SLt?oMs4xw+Y87TSN9?FsC5|7G|Lv8?`xQYldn2ga%OlsA@Xi0Q;wmxdo$*jj-K7_D5+HcFL^>R9TfHh9y~5 z#aHiCCs@6*?_3u}o$YI?mtZR{c41bqnxeIbs}*>J*p;x`Xe-So;Lk8^+Nq;YIZ5gp zp3zP?V^tnnl3-!hgE9f8oDdZ&RG_gbQ3^2CW&!t7s3Q+rmM6^DRT%OMXQ?8Ny{e$? zQnqU~*#^<;;^(tlm&pqi#F()!c-N1}8zxs)wFxxV+P-$~+IFuE3({ODQlRxR`)L|k zf=bRXqRlx$>J*-pa+U>_P>BIrD}Q9h@a?Bs84>+5Z#}TP14$gFq@hZK z$}pvF6$$PYd@4`_som(nM8SW+^Xi zi-D!ILg`L1ZZ*CpAz?fHwEZKFB?Yd_FH(-_?I7v6hkV%8Had=lYFFNwd?Q@FN4PO& z6T<2+(3{yBJ22SONgC^r=8P(c%`~P~^0eAjPA%RMB|j*C1>i}ptuIbo*OZZNrt+sgCz@zvXbo<602jk1r-|I+%}Ity?7^5KT-y+3*7 z_~1#4zKhKF>wcU``O=HePc$?^R@jfGB#ajVC!X*Fs(yhEza)9!ul|I3JjRe@fL=Rt zFG{jWdO1R{xOk6&V-OdF$IHN;+8MZU@!c<4d}o0-1gNnKTcYN*OPxeH#N`vOqeEvF zw155ZQxn#~Mc;ryQ?oRb6=b3KcR8;TV4>y_{1p>zlBTv)3f>hlQ8jpx-aKTAR9Rls z`}sS(_Kzqzy?6yx%st)$j^BOv5chbLgoUlm26Q)!HyfjN!!X*FgRZp1JLS*Q@HWQi>00;79)-DJt3 zG~fv(1KG`2wkS~cG#kStYi7$qigecHx<^(#6&BlTYncaY@-l-k(=~X3;vMW3J{gfh zC=hgEiD`I$!E0D@jSyBqMGWItiy&672tmws&H%DpSY2{F%YE04oK3P=Q_--T@kkW~ zE6P_qLK7+L)c{flU`#25>+jr`#jOvecAfh&%m3pBR~%hk*Q0&y^$J6|{rPr7{ikj> z;D}v>7k@t?`!ccnohodk{*NE+&wbh3AUQXKhyGV%tv7z}D<)*M8$f6F1qf<|p$JXz z(|k+RBLM>Fonn;-t70`|1G8kuU}5PSz&h(DNYx88_Uhj%RZa~~|IFH22{jcpiFp#` zrm{OHXO-hc?`2%{cqICz4MMdWwALG$Gt^rM(x0rL=?u&Q_3&hwf@_u~@@siD z9k>r=z~8JmxJZcfPX|B*>L)slLO!&E?V3ed!z@axg$A(F9V=H0)(@HLK;{z59<8Dt zlcuff^pJ}we>OC^(81KEKGOu4Rn%L_sHkrx0-eXAz9!*(L)Q(}V&6}$7u)rI71TY6 z@gW_B?`ial%?~;N(u3@8Ai_EL3v0deWSxpcqyDZ z)f@TIOjB3rfBmXncFRxUGsw6)$Ln;P_cD1cuQ|esRl>HPq*!ZD%S6(nPIObOZ-wK< zc>hfx`iumf#_DnYzUMdB%wXA|Mx*OCcodFg>p%fC0OctcAb6;D{k*dS?Nql|lOXgR5$e8v*qVi*aiuri_Zr{sTcm7)hc^tl{uA#$hm)%l0tGCi%m75Xe4N9V5#|w|Yxv>%%r|_DwcTw?VpSy0|dR zYg+Od7gndNVJ(c@2H_}I8ksQ+#?0EZ5H*^$QqE6WkrquHq+%EiJ63;MtMA&@ac7q( zLk;j9dw*(fs0fsy>i95eTh2D9sHsRC2mkfZFpL?q22C@pT6L^ojx9h#>xmeNMG0`< zbp~wqDeCdu5ShgwQ5q3WA~G>4lA!*kJ;BQmQ3*vMwa05om2K@-P-b2?jjmx{9A*V7 zdT*#+$x~T0e3ZYhV9x5mqrA1oz$kx?V0?Q^W~X%0kQ%_Hw`TZVjI1TedcHWfAY`@U zw2K*}H}={nPKlg?IR)uZ1J4{KMN<%uOdc6ZBo+!(mLX-AUqNXAxG7p%wN)p_^pE!F936$(K z(+(o8{(!m!5a3hRDR%VVB=K``xjTDbyuBZ(oG|Y`q*q(7yo>G@@UCIb3CZDBwQkkb ze0=}|aN6;02gB(B6Gs)ahMZ|wW{%Lbj04I+TlW$a&yjetqF!wyz?h^_I;PJ)4%f?N zN-d6LoxQ79>BhDE3m${SVW;YQ4x|h?=5ZJ+t}U^8Gq*vy!8IY=%EEA_>T^~Z^lz>q zcu0B(=WZG4$s{J5zeyr<+i)fp___UE@CwT|z=DG$_qQ40rlgJnVX51;VG1;$G{V%L zlF6;f!W50W)Gx-|)@|_#X?JJ&xf?8$M z@?7LBD|8djCw}#65S4kDa_9#xNk92(AhJ&+__*J1kX{T$t>bhr&Z=yXcXyL=UvOdmSC2YkY^t7dgRJQY_pDc zC_D$TmRkl*s0wh?SVRG?&8Fr!*d!$jl#iB5VJO&8kJ&Lpg*baC&%X(>#R?l*N^)&D zUS41Mk)gXPR=29qBA_33Rk<@~<@NpR$UE}**7vBDp70ZnJ&oqP|GH17CxE{bjFOp@ zg?pIz||Bo{=QhHL;7PnZmYI6j^PCyS0R*gB65Sh9&O*a_ow0YM>D1p%g- zE+y_aT|~Uc&_Sg9w@;fPyQ)J-?RhWP8*E}fStifSQB;x0{^d=V`SdL`;1_{c=XoF8 z#(ssx{xMmrF1{5V8sR@E7_tOq&xy3h)c1S5fi1p8#s7|eYpd0!REjrGi}e_1zmMFC zf&%8;AZ5q%kp22@qfLdz%X5(~r01va@%d1txbHu<`+$17{eHIR^6giSE;5S=LKldwLK;e*P!l)||Fy6wbS|xX| zpppHj-x2P??CY*XsHCF(W(>2i43jsxT4p#!M zSuiM|?9L)XU+6Bj@m!{TJ9aO`M!4>8NI89d9N2T5926lAHCGWVkHpJO zEo6iUT;Sj2^lgZeKYa;2k!y>e@;MxY-j5)E02wr!`f5VbGAi-Y8bgGs!y3@AA%+ME zA-A!brG6j}JfI-w9;Kufxp@jptm&CrglR1=Y8gv~lsM%fXq`X3Mi&W7Elq;tn@;|k z<|ZG$DNN26?#6HZ2=!im`7+(0F6Li!bL8zh8X|m8k6<77g9-u^$TQ^j3p!2*)_`*e zJtY|iMTbB!{1siD-Qy-4Im)Y-P|LZ))LQ*^}sVY1}LNNsjm{N}#a!#CT(lmu!4yZoDu8 z*YXI+XJX$_AH=3pyKm)Lihv~5qJ_LNlFE^(5Gg5an#9d(`oXD2TB%@$qCp=%GjWbv z8KL=lW7^TjWdjGI+sgvoATytS3TZr_g9w(sfGj5Yl33QwMHmApO#x#kOaYU0txtCL zIXp8DlBO3cNRg>qGJ{kM;FUYw`h~%%+?xC8@?))LvssSAVR&5wrxNYYN$BS^!-V5^ z<~!w1W8?qDciM=rrjyUF_odbCA}se--CQ5lHw2UuStZPIGFaZ%d$n3etF11*4LL{# zusZ1ygt!~Ss&>Q>)lgADV^Pj0Fd9KOS>jYRW(e$x#a|{@&8J3fNm^S;SPj7&WkW=w zjJf&`TV-iA!LcONF^CeAqyP>TDP>cjvHzom+ahvg6Kk2GCpAL0B9VkuPZ+4J&lI#n z68RS*ce;OlPQ3W6SaC~Jva|kpo21@<{hZXB^%CUvmOvt<3gmWI9Aag3`HW6>aVoBr zv9g%t<>~C;tIShOFfd60Ej_IOe}*jbbT`CqJ5au`I2kQFF8U}_J!3Xn@MAhuaQQeV zIhx*V5&t;FPs`_GA*GKjZy1DD01JZ<%Q=`|mNS&y>_S72k{4OsMnP@?(4JkAE<0zx zLPE{u)phfSc50w8=Xwy<%?i=H?~Jzhs^?YCQp^mMDd%Gvn|-ZSBU915L}!l-AS~@0 zfH~x#PGiGk#^alYNt#)@FNFrJoiK}w*YRnWWz+gEK!`_tY6TBr9-YIi4D~&C6UwQ; zy3yRU)V}OzXj=7D5m&HKJKy6O%oU(_5vhN8poz3E|*`h*clv+Kt|UcDvFSO-CNq z#hv>va3~#P77oAlysZ4+S2B90QCfDI-f~8I6w2JU>rT+QQVvJ2wi@AL#_Zus3NKCE z7c-X%6Ebn6FvrhO*!;h)9wz43+5xGMtq;;9TQ^qf^Jg?PpQ_{KNesw}U4UDfO4 zcJY;o1PU!80?jhmeKPJcKM6z8Wu8x`)Dg#1MR4x>FMH&3-*oS+OP&j|{*HzUQT9X( zy=O4Y@$X36`++@;PUOvy>`W=>J%4rAM{5|Fj6JMUwd>iNzjsnP`!o*-@#1r-2u?bI zdWOm=8n$|RZw6~Decj1&emh`H{2erbM+?~YVXA1Ngyu6`LMwSnq!lSn2%%4SN*2rL zLRf-35j?=P|8I#u89b$=Dl*$c#-zw(C9`KT`cxxRSIw0=)teQ;TQpm!$QnFCkF{2m~~CMK;su^TI3I!_<91-%!%$C1GGP`iO%U_jJfDP z8+lRq=R9WE2=o21jV_>yk60wfs2gF;;}T{$Cn+Y0QSV_&&Mc_{SJCZ?L;U43219;i zwQAJaDfV;fB&3+BHkBzU$xWwW7A>^wLi9_x^3>zvJwX3lY1yCp1oL4eA8t>2PlKmB zzi{-@+-jm=483P$tcP#>qxq=Gs3{X8Q}&-ZAESbj@oChH@83~sBx7-L@v*+<=-sy6 z%g(o69(`5b{CT=ER38Tpv`Xfhm!t}>2FD!7&cxffYF3)mw&(d`OQgMYZDMn&oTmbr^gU`4o^^X}_{p61k-|LQKTTs)iV?OQ ziKd-L^V*6D_Q`Ja>)jOuf|J080{0&n=;&ghvCza`7Yq$@Trl0bnNHddO%~F8SMl1? zrv9uXlm+;(oe6n}Qdnstb>2o(fvaVyo?B+g&J$zcD5o(wo;a}RBe!l3zvm7wX-;$6 z?lwN1bY%q<=)>=V#RR<{WpAU7mh>1xFaa+N@j_hJ5wR4d;7dmt**k4sXp_?(26BJr?E!xJwr4t8@-@+_C*0ON)!KlSsrzPjpC&D}(jzOj9{7wiGvV_L^Q| zwSDpF{%9s-Izm);lWFM{DcqrDDlaL2s)0WWCJSJL2Zuu!&tRxAou8VXU*WVp+fPc~c9iF6Yje4}TMV{qb{?@RF5d&(7`6mJ)hI&Pja!W! zbBuhniXEZkRJU{3mfATAFp9c%b`SIMH4r+B_u~dvUfA@tEnIIegJ#@hY4yj!TSyct zs1~l8Gm+qpX(v__m!vvE}VS&$#1|{Az_!1TH_Zs4V*VfQOc5|nE zrx_l~%_Xo6$Cs+L?s}~cMlGxJH+mjt%h_y@^`|>uw!Q6tJGeNS8qdidwkaQ8-DygV zsO@n&tXAGCpOQ~(t8YCGzIC1RiR4AZ{_s7lh{M-mC8sBAGMz-qJ%rQqeXR@tO`J-2 znGrtK)i`1qfr9+C%N?l|M8|HW)-XuHqqG|r3ExSrGdFKRm<}B8A zBvn<61bq+E*AJoeRr)H{TVODYK#gvvO0_QNv&4QbCgcU`hP<#V# z*&-y%!t+ltSHH|duDX6@0@=4$`?a(L&&v}}SmjhdocA>6sk8(zdAorvO;_wi5S>qI z_iVmGQ?Q5gFU#aczH)oSR4EgR0$hBpfT_DT-&bU1uNSf*Ax8;~y?tDp!sG*5QE(fo z6z8rG<=8*h>jAw`=YkfL=Z2=2d-N?Cr4Gg2uTTU$A(RaRBx=w5H9u>72=?$^Jjee9 zMJIBmKkWZ?QuFrSH(z7g+kIc>dhU6e3n+;(-O-05cpc6%ciO#Y&dmyPzwIF9AI-Mr`sZ@eQb_;MJaAkS>=aF-0Tr z*v)AvRIr4XQ|g5?AJBko%6rk4f%nvQ$(|)aNbf|Fc)#({Cvf3l9;C&s>&Un8gb8{h zTT@<*%rqs~VgY3cqL z$}oI)I8la#yj-u@!J>osrr19szE5t{V0RW4R(nKs;$`P_K4;W>bh!xc->%3k1|J7i z{<}kYo80)1T#i6i=Ig&?R%57SJ;N4_TDTFzZ7`k;2W=VbU^TDyS-U@9((R@#v;(Mh zGLXqf%HppV)Pj0~y(uP(S4z%!NfG?IeM-cc_ZE6L6@$;xcvv}_W{Rw-_-4Dvd!GBo z5AcWP{{9+YPa7v~Jl)pYtr$N`UNGZFm1yMukY7BELYA`AXM>dD(*sK}O9=p|Z8YV3 zx8!$yp_7{)Bj3)?CB7pDtdG=7*Fpl-W5H_h8LqfeEBrZJnpmSL)`72S$dmA4NR7^0 zU=)S7&moIpRd~G)r+3x}Mgdf2G#(Nt0r~(ntb!;#09O&K>}@hOU#&1r`C$$jKY35-%(PQThNhq)@k8K8ShfeT*Ie zBNKKV`jsVU0|;ic%|__e1d8?yx^sh3A+#k>FsyzV;Fcr`AHr8)1)5gV(LoU}K$HZE z9D_~+>~Bzr^%dtZJ%HzHyLFOQ^U*=F59khC#wF0B3($sGJ%ACGfM`7eeq|ZHvf1!P zMT`kCL#8FrtxP(PuFYRCx+LWi)DWYJIg>HCq+ZA-ge?5sjfJ4T4p zexeNc83$?_lUc?)CVa;Hehdw$83xE7fVO_53}9%lJ9ctc{*0{OwJ-F?zUg1CU!^r% z`mNo|bLPBE2GrBnjyc+vaJquQa}T&)$X*3q9n##GbgwJwCZVrz@$VCVtJPm`04S|?B_t*{s zgeHO@LXW+q34V7`;Q<4|4$2_@vEhaUD8gjzrLe#lrj-Sa1i_0z)YhK>AYOLY&V_dxVJ~Dq=_qNQp@V zYwHE5?)|{+z-wA&yf&)7st0^@@kh z4CwixYCdQb@5m1uzIj4-)ETgVSOd|5_w#Q_ukxUryY$dVAg`YBVMr{uPd}r{1Q^m! zc#i!4&g=8;d_{eAxyxHF-N}T!;GXNWf2e!?7ntU|6JE3zdGW-T!A0vp2GM_Z4dWuhVr|bahRI%$pZm1@~WTe z+1=XX-M(^M3;F1>77BM{w+p4&?xbf}_J(MaitqB5Fxwaf+3oG4(?O8fU(~SF^)NQm zrAXl1#vaGPfihhU9qpa1-M8DXJAmqdKtB!2>F4A`w%>Q{PE&u9%1eJi=WVNBoWdLB zRe;pnZq7INBnza!uKv3i{@~g1_zui7gN)~)Ty*vXH^GpB)Lt%3o{H3dMyf{F2GOk)9^i;v*y-LOc?h{s-aV=fAF1vu_H&jFyYmn` zG|DpU-i3V>s#G|(<8fki{QIocPJt9DQ=9b4eQlq~ZJqU{?)Sy2O#Wd_HhO27X&Bum z_xS!Yo3-^x9utI2{(mSt#~9Iq086)Ry|!)Jwr$(CZQHi{wQaq&ZQI_S%}z4OPBOc{ z?vIO9)vcSVx^>R^WO4~`vdTtI-pf=+&EBefytEE)z7Fr1&m5zAJ?W&Y+G`DNnpYRk zeP|lc-r%vX@|0P6&nmrY+8yiORQMJ&QWn>|&vcXi<0i)>XOBeZI?^*;nfY$gVbbxJ z#M~y+ue7)9mw?3YAYG1uX>lmAr^d^_i zEmlY@I&3e_$PGRT_J{Lo25DS;wBQ-%Et|H3O{M3Zk}1{v*`5J`qj(j@q|vh;L^Mtk%lJw1G266}Bo&_--Bvxbv#}9F*9b(V_0+U7 z&i6p}tVllcuE<1_>?{^lzUtt)>A-B~hAPBn)!Rkv9hJ%QM1-T&Ihuzy@Xd)>UnX{4 z8hBiHKgzJr4!5uMMsHMH@EY4DHo8;xkM0b^{5n@rG|6vIDijmnI6b*s3L(*Yb2a_Y zz-X3h(h4?~=5GcB5<96U8%efQR8(kAB7at+fDCRv-d}3tu1zJ}G*nY3%dM3~K!@u! z$iX*&mEp32DD|1-Ns>o&glh6_^Bv1S%Bx3sgpZg{C@Zy>Fdbl?AWC%9#u7zLjscA| z0@#bZ%gy@K2%`Y;c7y`z7O8ieEP?TYw)|&wg(!B2lJ9E@`cCNOfO(@N&~H;RmELUg z`$j=gUeSfoyh*reULBR|cPIPJp1$dIs=QC9&c6cs17|E{Z;#SzLG@xcns+A|NUFT` z>vxT60D6F6(7e%<8h4vAk-g#EhIXA!qx(nk4R0CDF}zh%zWG?-yrVUmw(%;NUL~DJ zcBvc|V0gjsg5ntdEPwnKYR1BB*XbORHw?1zzTi)p_t<{l$!rvG9@E|3IaN;HT~`h7 zizv@icQ6BORU7ef%q+;;rl-!vw89{}&GUe|{tkjO_mt zNp{N%@;|_vw_ev?|8z`An@Ey;;FA!9#Kj>HAdDeEAUgfy{Vxfs36kr>DL@aSs3pSD zq+tvT7UKw8BRenX&`unuku` zqB87h74un#iUN$4G0eg&)r<8uVgA5X-MPF1{OcpD$NZ6HykjvZ#p8!*hk&iGfre%s zU}X=9kq?bYmn#%!;1<&b@nH;gXfU&Q~WEY3t42GR2u&7rCLAA%MG+?_V%_;zuq(DHnZyVAq zCU%F@4d4Nwhkvd|dv1;gw=$r)ccYpOy&xa}99egGF z3{GBP??9^)Yyn{hPwmrKCIshiK>}ZLMAQ&FIjv*t&udQoM-aAKiJd<+XqPV?8L&_< zc7nc(YJr3`GqT_$58|aC_pM>$J>ly9$FAShU2)$S1t(xl9@fN}nj7{55FbzyARmND zA2OLAql!`29s*8)v1$ObVk#a?A`NU1w<27bzr`7OI|K-vrYsr3^M#jO_Xh$lbInetPyCK z#;bvIRagbr_56!YSMsLS)X?WC7*aCVCOE6xzuQAyBhnMR)Xn{k+raenL*4elCo7-} zgJa&QO%EqHu^By-?yVjk+g;a>z<_c5enMTLKMf}U0s-YK0f_(r@e=$z3Tvhjf|3hw z%e?x0JDxo(OFM`M_M_~EXbQKfhUqVLU7`)X4@;693huGD4*4gi*!2iuWGmDK#`DtQ ziR0FV(U#E)9O4S7Xu=tll!^$FX!Y^<^@r){!ITsB^VOPDq%thT=BoHdIr&q(@rg^R zONS5>(l|^UdIoyru1hayR<5EARXuE6N(f#& z$t1Y@5``!;NJqjXvR5YK$?px)97+;uP1Nm{xvT@+>x?SXE2SCG)FShS)P+%NNHzf; zF^}Y4!e0%4=d-PuLc|r&MTFFoDJGJMNg4>|5mHSsO(^MU6407NN*OJHYm2Z>i!Mh? zVM}*2j7y1lqQm`&|4Hwk%UyZ7{#1DSUffQK!7XrXw(xFBmXs=LqE&ZV4=q|?Azs~HK1Z!=pI^4@O_ek>QRCP5)a;G!xr4Nj z4bzE?O2Kw5Yc9W!Afv3xbuTlPTx96hd>?|{tymxB0ZDYH#A8j`j>ExDLB6X6m)tl5O>@y z@AqhabC4BQcNTWx$#*;Iw$7LD#H-Os@HfmrsRrON92>xt{R*QT%z`p01BrN28d^{S zR#7=wJ${Nf*&LxTxrh<+j+omhQAJc?Ih-0kOgpN(riEqN0A1+L#XoHw?$ggl#qOrI#K#?9=m8rO=I*(J3GEr5^DqTcK1-2g_b4u_~o<#jB0FZZQ#gb^Z@cN<5K9;EBuS9ohY z@iUWeV~5X9+{qR9EcJ8+sStnQJy)7Vwyj%4{pN$eM8K@MRJ}~VyMD<$h7zI=cVS~# zy$~Ogv`{ta8Y6e+L@fdQ>V{@kSB(Xb2cEb^$NVv>t3@xV^-#u{y=kc@p<%-S`2QSx&lwGj;F&{{tv?Pe z?e(>8LP5#*d!u>H**}j)X9*H-pD;g(e*fg;vb0GQAp(2=7Kz0I@P@oTS@@N@kC>O4 zk}1ZH8#0%CdzL<83(V}QWCwNLQ&^|SUr*2dzEgi9w24{xzQ=PP$nG}C_yZaH$!LK8 z1Pj%Y;Q}S3Q#Mm_)aQ<+_#Vc@Oa6%Vc0eQ#TxsURcp8w-2}qunP1$Qjb)2L*q8Y&+Dnp;}xw zuD5_H)p#dQcvddkvKpH^;jF2+KJ|t3@|MmX)H9<~EhKe=LZ)0ZX4kPLlTc_Z+> z?z3TxjBfYwqLJF`d=>ph_meF41)>0O=Cc9>9pF*bs0^t|h8Yq^A5$*iG~lzTv8ebS zU^>8NplD7eV|yT8c05&umSWuBs^Chi$VwG}D0Ua5LG>lOlt*+VtVg;^(Ii?9dFP_R zkArAwWhrq1@!NKt_C3(7>=Z&~um}*KC0fPgEWDnZk=@t*hIBxF-@U%`;4=GuZ!w~q z^6)e_RYbzz2NDOsxaF^73x)ff^fIG^#U%7DijFIk)x$c$9OI=9tsRY}W>M`!bK%}i#}QW3LwstM1> z`S~)B_xMlOCKj&hPUp(^;Q6}-d{Tfg0VM{z%X?C29Q20U1fV{fceVh(c?nCtVoeik zq8WX7Aa#s0`!uiDpJcZe2eA%1xHoYk912!i)sZt;*zOCctUBN4=fJJazb#!|g@s+- zRJlo^J{y`W9+()y+Cu{GatFuoeBY1WlVo^qr)<&xfc#F6AeXBl#1AFKaw4*uUo9Fv znu!O)0fNvB&Ma(dV)Lh_l>Njy2K%-o6hA9MOQ>)+d>_s8PW&DR1RUin?eKIu50>;K zE^Kf3*jsNC4diItXq7Tj1%Rrb-9M=lm#|S#D=ZTYp(3Lrry{~(STl%<%AqO8ATj0l zI1#na@>8xw#dKyMk~k=(10W3=S>zIc4V}C&&$|g1*AUSzKSS$z&Ej$QpQT|5#rVX%xBIzP?E@2(BDb`ah~= z=C3pw;PP4Ub@9JUMhnZ!Kb`J5IVI9YB8XwPpFeqaGY8}yWPDZ|(8YLD<_JzYdr7<*IuVwa74DnG@waJXoSpgVNpG@VVjmIY$Mmb` z*QyVbPteOWJ$ix=h(h5p(hrXtkDE8Q;PfuflFaJ?c0ZK+-ZP$Vub#XWDaC*HGRD9? z!!r?>j|e|dYLl|J$Qu0Hz%%M!DEwtG{iF%N3&K;^z^)0P#^b2FFp*JdjPINu2j}zYi+B*H){;MLpWYQq?^#0Osh_tpl%^)Fe{EaeB?yylXGf%9)P+ z2cG?KCv?fnN*#;UITF5g<+<6}6aBNg+&{HsOVB4)JAxQ5*jIG_t+`vIYbZ9(s|`n$}-9d%gR(sNftA3Q*aY-voPZ@(=d}T zGcjW^+o2d|q78C8W!6fpmYC9*WzQyLO*CQ)f@XL=`R-D5gG3?@FWOQ0@R$`|b3)gIOLM-huPy$v?;tlByRF-|jC z)n|x3wW=HKQuw9|oUdqAt3ge*m>OF2dFro2W;4 zCAKfA@IAJ6y>Bh8Ush+7<3t6YcAqNSeLtdt;<$dvcEo%HTH|Pq33ezQ^q0xS073e3 z8f*&1#6ZS#Yfu}DQcwozp$hNR^S8awpAdh?0cU|DxiJV>(==SI3S8BKGek!5(=S-c z$?PnDGity5F8-R6E)&s%5K2$!LHVD?_mg5dxuKrHb)zlsaH{5LnZ=VQaM6`ADlEW~ z`Wz&QhmOSxqx9FdsdEH^j7btlAb3D*aPRQ;Ham{RSCY*5U0^7@%|;)QEZ|jIdmFBV z&{?Ue)K$0p8kjvbc~dEksvM2@}IaXRaP;G zVUiBiZ;M5ntPr4k_UX%IBcUoQ z>8-a|sg;Sd+nUhzn9k{=yYKx&d!+aCK;@fMO)8ftCZVGI(YTW9b=|V_`93_@;T;(~ z(})(|2k*$OFa-YvFh>VITfN8_sKsK)AW$r6l2sD6bo~_Re?D8K*l<1;u*eoOD-Kf{ zMX7?~jWI8hE6yAFc3IQj3*SGyO#Di6IN zon)GjQ%fu!Ir|wu3uma`;HBw(EwEa(7rV0;PrXI*D0$Fxe#fBoEdxb9yOL=p^MWpu zvHPOsB1PEL6x?Sgqwtht!`u&?UE~8x^2+Km@mxNKAQhq4-KxvRpJ*oGg!-0gIyggE zm3UYZ=Hrr%bKN<6+w1Uib$bJETjg_ikmxN~$(S0$Ba&LvC)|Tl38Lo)2{^<;AobY4zQ_!~0d-2=2Pdhw}yAQRa(rO^(Brb8- zx%uG|cx3QKiYc^u!e!Y)-d-zWT|Xa?4att4>mga+Y z;4G%<-c4s>>*H}|e3er4x*rCfGE^j2jw5tCfsH$Rn*CGr;D1O3$1tC2}N8Efupx_nG%RoYCkAU}?Gn z@N5WRyAE*@#gR_4pZ%KI;@RB)GD|`oON}dW)*Tvrqmnu=sz#RApctwWq;pQIet6XJ za-WNb1rTmhI(%hg_>kP!>#AK*?R06l{=Kh`N^@7e;2X+x2+$%&1mO#2mOTy{zv=C$^^?TzEK$cfv5BxCPJjdkSBSnrHz zet&+~8Yv2Vw2b2J5W920@~d4iq56s6gCuEkEm~^jxF%65=v(*#V#S&}L4BZC)$^kjk2JZr^E0(xd<2P!| zoA!P&ewEBD-(|{}Yvful(w1dj6!rj5g zWddS&x0(k>xO}SDBShrC+CP(zoKuW)&3v50vWSo#5NZo(IrSEGua34Zs#gJ}98Zq4 zSS>4@MXrh7){XdM>wdQxRbA38B8g;&-_aej;*xg?nQT>&US84i$VNKQ%nKzd*imb5 zr<9a0FmssWhL;l^wBc4 z__2sCx@JP`Vnj*%iM5pD^?JgF=G#_(I_&`-i|yh0avjMd$E!QrX1;ZE&r52JB}&ru zw1~$WcjG`>WGAy^E})0!si4$){lWN9(X-(0sPnMQV>~NWF-)a9+#vL0WkRINRLW5Fqc^X5v_8@ zSu==HiDg`b5klw02aznoKfvx5`E#+#_y@4@hpXN=6p8+w@U4YViiZM1U85$a{*x&j zpfn6OJJ@dVK?D4?T5Vh(Umk)#>I^&HAmBHNkGx0`At9YTpEyW55a19Sa*r5U9iqr1 zzUtKeiQIXIvmF0fJdpStw~D-3MVZ1RQ7h3jG2^?i1>YfLRDu@6A-nD~@ZX(m{~y~M zV4&#%)idr{{2k=j{k29+;s`VUCwruej7WY7+_832hSo=(bfkw~St#czVC4iZYZ9&L3Gs~(sZ*yYeub1n@pQ+Xs>Zy z3%cu)-4+$R$ukpvkH+F2Y(UJrAY+SR^ zqk{>Dr#9D3=S}YYkMO&Kfi-8=ESecr)2b$xfY%Rzr*sKZB%i-lFg{;gr0j5=i(h~& z?cB9H{t?KG;L#LymF4B@3JMJelUQs$pN%y#9m{_EL*Kw7aVgKYFRvKitbTFDiN3{r z5?iBqds(kWpNNeqpTXAnT2HGjd;d&01y_XN(GMe@(h68_)m(d_CgHqnsPth7+SvA=3ZTO2U>6E z$rF^I$3>`9S%CwKaGxcT!?V{9FM^G%FP0d?yz{AN3aqK;1kA1ayN5eVUJ6 zYmP&@RZHPwLKbMJL}j6oZL=(*?XC~LaiK6ej~8+u(Hne0di!PKtP$xAew(L4N^)#0 zIsg24>4+E-7UNNkIVz;2xw%#ccMfju`EFmhgPvYJ!2BmXzn#54WjswwTNsJH7!(7( zGexE>VjkBrm)~Y5*?2jFg5I=<7rY{k?_1R;3pTLea0;`cKQ)|B7<0yIg+Pow3!6%!!_un zy>DkP7s}##IjjZt^Tw_l84kp0C=jE~9=W(^*idQqUcp-QderJ^(Vcr;u_<8#fjUp3MPALvtPCJzkV2tRRx*BaXp~VL zZU#4c6vPKRC>EhPMEIG8bv)j@f5w{p*U;t~Nw+OYc+@p@8c?DE%Tl1`%dnB^0$iwU zFn8NOUTnu1+uGWSyj&isIM;TX6+$u0E%+GG+PVX&A0x`Pbrm=3R1X30H-TgpDysl{ z+u!GR*jWXmu!nlo8}FwuCr~Rugp1EPM92;_RF`ob0A&X{k}uxT1KijJiY9>v=4Z(c zwTXu>8&hGF(6b9-B@a}JhkuiYn3jjb5+_jZ!ypfg)2Gx%eGhbZ4=Xn&d+xF56F+OSo=}( zQCqQYrdf_55B{`S`f(tMdKBaMJ~$V_LTgM10N~1>b!RSogi65E-cIIakivqw)>Qrw zzK`YAEAFemrU)>84>cwb;MZDJ2vUMRR!lI&8a*v}e%u-dT3QAQeYoggB=tbc{gUc1 z&<2Rrf##=>4+MyX1h9$(sPhDv$Nik)q1fs$Rrx5Yzt;db6`*?xFdz0iro(p=V1^Db zs)NnZS0JlaeA@sf4*;n{TImC#2-?y2+p7ae>C;MwEFr)?7$8oC9M#M3V8A-Lyw265 zrwTDw`!CVMr}i7B!_Ub>kjTS0Jz2dBWK;_wB8&(?Qimap1t5_Jl+de;>On<4Jp?vw z|0XWobo3rv$9mm_>J;a(uxlP2=-RhIsrK=U2cVEYUg`bDE3+jFLRN>S(Bq+pbt`W+;j@asHK);cs4)(UAwO&~&c&++RUm6wZ+KDm!u{b-@yo$m3Dafj>- zF1^Z`^7`*xJH6hH8Vr^9EEj(G7T8rN3C(QxNd4}rttwKLmF%WgIUeW@$!~=QwekEy zVgH3K$^SM|bM1Mu93pk72`|_6nuO6Gu)UkWZ=bhq6iXxxAZUHwIJpgFoj7z4BI9td zkpDozqqv?FpnzR&viCc!Xs3dZco?0}^1Z0f6Z_nV0>=IT)z&x8cRstmPDFl;_+P(3 z)&|>QWg*d>MXkpz{6X>QD#0aQ2LPB*)i(>5)?8bFPaY z?~r{5btQ1xo5VqsnWepSHaWovQY2V_qYdHh)Y1|AMpB3Is0lTBZ>5bL>K1H-_E9yz z?LQq{^H38&`TkS94Dbft0xWlzWc$rpb>zc@rJ!A5 zfRydwtHjbo$_wn-@7#6GrnqXJg6nN(i=w_A<`QuDQ`*9*7V`ep@G>6NkmJ4M6}GyQ z;9}INk+sKVbTiH*NGoc4p0A=l*l3%n4op)2%8R505O?axTJ1(#?T^|+P1>broE6K} zaj}#C7RDobs-TTLs)!c@Gy}gD)xY&+*w)sex#Q9rY~)w38__Kl_S4BrWq8^6FMI3A z1qYXe5@PU0SeNmBvKwKT=fQwtYUu@HOjH*|^=a$<;74@m-M2Pcax%VfEJ`FN5fikX zGZ&H`xAfGVv=1U3XhEM`dcN+c-#9!pf*z8zPkAh8FCKc(r5k3cF8rdQgH1Mw8)qNH zT|CUBFC9e4Wfc^mW)Lmq-2H0BT)teAq`hPTd1N5b2xy z_!0#@vw=J_3~8S*`K()XumZhEnV^2={su*l+}s;tCPtYNPEj5UdDiYiqq>^_qK|){ z#Vo-gUkmbsan4Tzy=q#`u3KX)!Jp89HrY0uzf6i@_@}JJck%!7^6BJGA=a zM_sS87iFjf%Yju3WS@bbo?N);uR`MlI&AMvW!1oAt;DEQgwb5}2PjB`IX8YuKPU>wLFXnsv z$<43x#nhYX8*d~haoTQ{^W)3nlLQJ7B}RU<>|<6_pyN&*>_X`3Qhdpr-0E;UE9bAD%<60#{(6z< z3-2YoZ|{3OyX3I!HzhQ6c;;TSY<@ASiKT5`h?A|f;$>S^`mWL_V;wU?JO9ma&j z`qHhDH*=n8Pmt!oy-HzF7}S@d?WZ@sic{}TyiYP`{2g>R;Su(7;(`3(pk&bj3B*nD3@!@NhY5zu_K{d`|@!{0-9aVnkO%Nd)H0o^wPV|uwpFgd=jpE54pMsMA)&Mrnb zFP}1AJHl|C4y>@_mi2|6ch6iQ6BMzcA7h-#l zUx7yTIPMTePvAAz9xU8vBYU}b^?NTPz_DNRW=NWTwBu$-4(QmgPhOp3Kb}MDeL~t^ z%v}Gnf%ut@n#r_<9WZr%tukS|eHd&lbZg*@mU$W3JjttS1+mEC7_ngIt+OQYxWMGC zi@WjcnRc4))533!-<4tKc@IqIa)-*bHFOM2SBCA!QZj8I`m2~$SKBvD2v)CN>!aFk z{^}2~TOBtC?(Q7Hy|Rtkfq#MyOk?9X99NB!9FRw2;&wyxs?D(2zm0invis>0XU#a^uC8^7XHPo7jjmyv$*?tPsAg+~!~fu# zNwYO!0J3jIgJM4b+hdMaVB`H-YT!%uP-2rfWcOcpJ+sFg1OZ}C*kg&C9I!`m2N`%A zoOS83w(uT8IW%B-=%;<<5S zj$v9Fu)&X+#dqgU<>OLdTlQwb90!UNPVuvFg zx4xyB+Gj5(gF8$=8#BlVVb7N%$D#-OaJo#x?t6EbLJ#s$64KaVNYBNbnCB?va8$<} zHI#G&kb)f9*2KT?fl2auYdQpwwv7i_*L4NIIEw2AX=hY4dU^X@maR!Iu ztZfrRC-7D1rL`(_)7q@**uQf#Lse)^*tIte=3_I#Ua)df(X>1uyd?trQ zAcdBq0R_@=>EkgVgpvRly8_|F3}yt#RSXHydLj$_kRui1v!6V^S3jA)pIbVh??17k z97X?~wwME-9^cN;5{jFf?*C|unOPYAClJoGj+Pyc1fuUu9eN_Dp?Yg)0I85{zgy!9bGpL@01NX1)te$b(Dn z6SWnHfqZ6TT{^_u^UP|1GYj9HW$WU^N(pze8kIVxOeL9zRq?N7W<0kjIq%BziPO&u zIe5O!?rxvg-}Yep`FMF@^Yi}bjr;K_hk(Nb^>}1fmk4YgN!X^J*K!B_jiRqJ|5P$y z1moDfC0HcMUWqK0tpodQH7;|iJ_;LfjxM}r9}FAlfn(aq|O(KB&UjOsuZWJR=V%E{o4M{Mf?&ZTjC?eoB7?i#CxW) zrxMvE08sqq7EkyBD~8MVEc^3P;S86bsdI=eOk7U+C39%0#G_(Qa>=Dwo4h<>%LvX? zrO{}Dd5SsrvS<^dT8frn*+xw zDax!Xqoqmwj;cVRALVMfAui0g$81JL z<0QFr(pzU~8p^&3hQ-{9KPJIAeKePXqMS*T%u1Le!J{hV;;-kNJ(rYed}?cFT?#D= zbkccckQ0+mqHb@>tiX^ya;wrvmAcOt??pI2)sF5N7-%~B4%zh+71;&Vqohiv-AV6t zqO?2@x=D#((3^&5C1k3*I7ha}@u=0=D6a;JGb*W~gKS%@Y(y6-?9(AX8(F6!|58{S zn4yl<2iqk!0#r(i&`w`)|9pMR0xc*Lzd$Q}J&^j$uELg9g$nYWU4{yRy88YmL0Coy;N(b`XQzsRh9iyC zo5C$sXYEMe*Eu(e3~5TTsAP6v99E5cWE8lP0h0m(5=d)Heqm4%RYy+7yZWrv=?IsM zUje{6svI7}*i?y6;LFK4Mb4ZqS5?J1YPHX%?nS(coAMS%^|7>g+9EUuEhKhoB(CF< z-RVNcpjM)pS}LCkiOq#mSPS{XrqPn}_x4k<`)XMH>lG;N($x5GMv<=Ei%0}JYDvm1 zKUFuX3=XrFO_z%AnlbK$i}j2#XQkzKJq>`$j8}cS!jXxm65ZPrrA)?D?SO|XnaUXo zV8}F*U2@8L7}Z%0v0p=}@(5%NV4$NnANfJZ+&Y*@admc2gx$1>fXY9&3GlBYn{q-l zO9VxYnnX>O?7Xb37!d)bkt}6}5kU?$iW5>Lo_1)BtB7jL_<>+(<{*@CRg1GNadm`r z=L;csu@j$}o4OMGn&vx+vnS2e0!a?CqCoFqenX z)|Vgmudgo^5jf}sZ1Z5y!48`v%hoH@_xzP=O6v8HI+>4)p|GS$DoHQ_D>klpL`G{k z+Bn}ebOlia%&2JJ+7v^E-(-&zB0i$E%eN|VC4N=FO{Ok1+F|Q2%>mQ3X!DBbis7>w z&zjYo(TSCZQv)$#wTgE-715{ddH%9-%0D_G7scl?0)ZKPNaR zwgxUTLgHgOtV>ZQ=hrE6a8gRvYU)uMEd6{M%WZ4I zP4K2HpMZMtL91vtpQQ2&C~T>Cm01aNJxJQ7et0(O{8?*yyl`tC%aE>9cW86pwt5^h zP8f!p)DOGYb*F04>YmJ+q31FDLt+#dxZ+vasg1bgg#bg)rbB|d9On%8tl41V2NvYY zE*g=(Hb8w{P~GgaV=hY(%Zm1+|l- zs`OV~roh=zyLQNA?sHjNbj=bf2xqt0dCd_@0)X6%zT(f))gPNY&+ z1l&KI5*#xtz<=#|FN}oF(=)0+P&u0QQ^n~{XkO;LDIPcX{vln@+}0FMD7yO))#fBk zG~2Q4X;wkgaLiB}6+CW>X* zSXX;Igh4OHL2Q%pvKF5Axj8we#lYWTQf#^xqXrkvee6ukVSz1Ho1k{F^CEP3z;}py zb=#j+3)VOsSA#vc?(4JHyaGEg4Vt?2RM}2&uBWa;iiREXbteKGfZ`+9bD&H7eQuyi z<070((BSsO-U#Jngi1CIL25wRi{O!ZFqdZ!J~m*1FF@T-zMXvqXROkRgG5B2fgjkn z;rVf58gwtiM1o*(s;yqkxQGt!&Fvd$5SSEjalCpH7I1if88OMiviVZU+bZt7O;4%A z7}M1x!S|G!#LpG*?^E*%o#DG3a+TTVC%)GqDkvo*`lQfHw$y+p(P?& z>EgZe77CCv@=^`jbeALn06-<^vEH%f3Z=-FAi9C5P~6X6iWX3wG>7`+s@TDXP#p39 zCeRMehOmD_W--mvcQRpNkbC-TiRyBDvUr%Fvn42If7alRfO;=Ajw+}_AO2;V!jEOl z-rriMukiiFujd-Gd!5G$Ws3+$eo(_SvCB$Td$g;x+cve3x-Vw%qoq=n}(S^PzJ@Hg-Am7aOSIT zeu?p|&b4UbK@Zzql7oNMyuL|G1Xo%R3Il1<((2JUQ%xshG^d|qe`*g5Py5hgaJPdl z@EtEgCImT>(c2+pxt43%(l%{3r8a`B z!y5#R-$z$+cN)YlbvJGQ+iviCf}@Tr6y4Kv^!*RJXZq$qc8pL+hgbViE$?fZv#rmvi{PG!_`M+lsLB0iQ~Z; zB`q=BjK4J9XTjLxZBsm-6=x)FKAA^_-Fhwe`Yl$)VxUCH^O1iig!v?m?T(wG8mzfH(QlA^bD)Xbrd$3^v9)>;FLa15t1}D6 z3j08n##G3ang#3)hr&BV10}qER-5}EJlSX7)2lj5=CjDgiP3|XfpW;BBP?dTPQzq1 zScCj?PZKnwj^<`s;cQymye_B~zd8&iQpgKNt;4;t`4QPSF?;>{mvr1*K(xQsTWkII zS7Ft55>>mN1kwbR8}WmeL09SQ;YYueLAM(L+VY?>u3m&lz}N1URNqZ+9j{obonWeeq&g@Yz6xkvZ^8Z94rjG_ou%A58+_uojyjr~52p*! z6ek1E3@abd4<6k2*<;55e)OCM{xkG_I}(VAPUJZvuUrVKRC$Z%=AO2;ZnV&d-w!&N z`+R8=>KAS1S=*O@IIWPNnouS>vP(Fvl!~`TZWTEs|z=j)ZrM-L*^N#Bt zY?Rf*BJnJ`L>7SLB-jyxEVxC9K;x7y%rHk5a!e;+DhYURFAV=w8HjC)ZAI;ZMdL;V zm5Kh?g{QMjq$iehWn#{XS4c5p!z<3?jtRy*7NK-m!96NbHEzpgAAQ-dyA9reTSB0n z7I-TFGbEr+up7l5MqxTj6wLm!7DZzv1;)fk71uBj%FMxGa4XtD#(aw#Q+Q1vsfksQ zjA`aL%%(SLoSCL!va&Zn>>Zd1oCDWTpM&cqmDD77WP?BE)Fy@SdiYZE*p_Rx+$d-@qpwqLQI5 z2M_5q+!k`%;#__%(Zw9XkK6#`m}(UBn0QX%GaQo27#L&*gMos?4ojX*@f-S&a`@yr zQQxpNiS7!Wk!71=op9Ro!F6kPuYQP|oU#M=`n_`z6oAmF0%`S}xv#}3je)Zj(Z8Vc z1fB~xlku=Ob^nX8a|-S(YS#WF6Wg5Fwr$(CZ6_1k#y_@g+qP}nIrE;|_f&n~U0?0J zYSpg2RzLmpuL%g6T;2aTG5_fqhO7;ene>LL=#EG7r-ppoJE3lo<+oiRQ^@M^b$9pr zn7R4+zM5LW!^7qEechPS`}w#g2y*?tJ{SqT;REz@|9n2<@oo0_yzU+N7)i-NmQ)oA z1fv89)NwN|x&5R;&sP-!Fg3iJTaiFRx97edte+lF3paY70uTiKe0~UmF@6r;Rt*;;3q3DJ z*fKsr8Q=3$G@PKz=6kw_Gtf+5OLaGRdS4$6JKdhodNaNsTN536e(s-l-%lGOcHM1W zPcJ>6_AebPetthKBl|Y-+~k5*;8w5@3HOi9?w(q=P;*#-pM;VhE#3=9gT87cQS z9n#$_y#8I%pJ@LDs1vn@>CaaUe0&`_`k90XDfr0{z945acNCOps{F(?IfQo5sUa_PnzJf*{&=SrEsZM3nSHNFjY5ihnRsu71kA#%)F1 zd(c_ozWGJ;_V5HQI_@Wr)5oN-cl6#;7yddj4s$Pjv^UC=mYg5mOQncQrRmb@1v}J} z#frFfn1xo>w(p|Dv+V~V7}hOWh$u8ky{rL34f;aow8yQ#hrJ2twDyem4}jo>q2^wL z)QX@1+UR&x+5*}v$MlzsKdLG8D!Zg(F;U+)W)R={-*89(21G|340n)P%8DVvAck}Rv;9e?e5Od+^G4Rhh;Fid=T}A z3hw3Bb8fg3CMKL^jnBtL?v$}s;pxx=7^jcLOoxo7m$o^ms0L3{Y$AJEH9k49S4JI4 z)>R*!Uz;GK;ZOAP(=ZJ6EpVf_QpQm{B{Ras(H89^qqD#oQZm8?KsNAmii&9(-Fzg2{@ zItO4bG5@ZGo{lkSW0k^84dz**A1=LG*Kg%jCpH`dvkI7L>Pw|B#joS6_v&MV z=K3?@Bx&M2v5RzO+MRNhdSCP_5Vz53xw>b<^zBGg)1LEsONO+sU8PMaAJ6(s*LkDY zU30KeVE~`Fc3p}KqvqFdO>NCSd0_6h+mEeWh@hlpkSD>pMckY;tpH1gA6b;hHj43| z3G+x#`WaW11{sT2qD6k9KP&Wg70QRk_aPLJ4|;X9S4I|EgjId)C3KQB9dXkT_QM!Y zLC$9QIu;p3L;^AP+(FI?A5mc>tsx(+KnIdKT2PS7RwcYF!xDSFT4y;3*Dyg;uwY1t zi;z)bw^ts|66uFbdHZPIR7=ujh@Ek-V{`$a=&(CBF#k9PT0~14(V;*)LQWYLqLA;1aeW^U*i;$U4^gu9K6dRgM?K*kmKnCZzKJAHs!K}ov4*y)m>JoQ zsj)>iW^X(=X+a(E3glo6T7B1P19uF`HZ+wxjx`M%%K|Dpd#q@uC@^~=yPQi?^dJ5> zCI#M7QgYnwH0Ug$_jgm3>`G4feKR!89x5DDi+Nda>fm$%qb?kkStUNlo-x)|A>Rz9 zURPloeGJ(TxF3fT<%~O|#hWUi$yHj#HQJb;9$6lkE)j%WyhWDB`{28k=bs^@?w>4KI>0R7cOTBd zNL}+rCy57v9X{6f=_`d>oj3(ENSQeez<;txLzv{5oLo=}11pbfbt}sn2c}S7A=*ap zCoIg3_lyRis(khrjN;si9}}vhh_mYyA7yJ6k<=2^hHyxKC^qe6kN1(78l5J(-#QTV}eL;PR)r<^g5=( zxu+qF*~FZymT49rZyVq?x?C$r-~IR;maJ7I*TNEYZQ}*ji}z3DTS0`@>okSOF-PV&L0b`u_;*^02Sa!^CR^LZOu=yI88N9DG2e>4+3;gkIt*vN z0+WMUogipO=C=d3d7>^Ih^7{-DhIbd1d9sRsL?v!%2aMWdQW8&xR@tlnANJl*x$P0 zR)(GO%@e1<@rPw6bVffUAj*5w+2P+<;i)rC3~(TMjs^C(#RN|6HX@yM;wN=i#MvVi z%Dl1Bk*FrP%nbR18a3mPk~SC;{+I&o5mM;Pfwob37jYtG*12T-1xcK;XyN742MU44cpLC$uUX5Tmg^xpu(T16iz02%7DWf$mA}XyVsvZBB@rDIaGubmT3FMGA zWQyC@c=%eoBkib*TZd>O@US>lVfi$Sw_G^}1q)%F-3KT_of>4>KY4ok4HwwM5q~QV zI4>S6{d|s3#qG;%tw=&JUKcS{UGR59zFHwuwj|oWgS_y8d<~g788RfOOE;qUsGIqa z9yUQE2dgHX3y}$BvNI(Zk51&Oe2yE>`M_;V^+ld8DXyna*MVOOAq)(>ibb7{_C`^d zfM1G`<2V}{gedwc?>AJ29gR12GazB+KqbQ)7d!Rv!DN;AgL~Rr1+2VW0>d-t9BaL% zb{55IUie{hoRJIJ=3000gf#P!-GT;=D=1;KS0Z-KRUo-eSUjEjs}?u6u>GC(T|dgj zYw11@tv9=f&~A2AWu~Mw7|NmOLq8o9gtx|idJxB0#Z`_syfUq$F4H9x=_Iq))(Yd ztgl^|_P{{@pa5V`rz24>3=SL{0Y2iiE$%Oj@*IwHEi@_FPJ(GqBpz$i>!DK4b6HDx zfDIyc%FD8j2`-Lette;S0)9o=mZJI>Q@~TB(MUE+;`&Ct%B2jJBCl_>Hr-pZ6I!5I zSMsaS?sBk0vYe1o{@=s)0OqIITJ+V6Y)SVB`DO4I*ep}+Dmtdrr0HwM zk?nt#*|Rbz`qDX?aIm*_{4H;Wh}yM`WN(HNi#&kNPU-(qqBS&9y>HVS0SV*-|F9DE zf*-e1?qr30L}Bax%Xz1x;my(D1x>KTg_UVH)QdP9vX3}5N$^Q0R}^qwwu=yg{|X-p zLRT3|Jtpg`DCXR{g~YhnLyU#$#nKd?jP^}L+CsS*kbLBW@yHrQ!ER)ug!1*(IsHtW zdfvW1RQ#4j(Q=x0Cj!H4Sahi^8Cc}pKJQJxh%srrwXwfDaaM>!+-O2dHlC1ZMBOof zO-9oJpvag!li#0XV1rO?rbClAcD5}3&BL$gWl!dT6&A!mq$fgGhC7V%7m;!#S;*VOYnX%43CkL8jsvDC2GcP z2_L~FSP%U=AOcSN=Y#XNDD$6fo(^&26liNkPc}W;${FXQc1zH;w&SCQLJo)|%K|4Z z3ho*Rb)3%Wmnt{%4ooflh;7&>-X5(}LLX8&Iv#~C^ zvIO_VA@R(LuvE8y zCK@-6ljbUq+fM6%%lh(%03X@~wO+}@ZF$+C;;bzYYwU*P@KVN2r+ktc)gjyKHJYn9 z4_W$CY%1jSfxr>>kaguM;0B&g&BpYB?9Lr>>#Iq|tUahZ@eg)C>Y*ld2-DBjQS*ct zd!OH*?v>Y<1W>!u$0L2?qDLz60c(fh*Kda=0!EqkGFR^y&LzlvK!Il6R;51y~0 z4ek7^UkmWaxvQkNKWeYqKh8l&EAtCF$JA(*DGkMH+pOu8n`su`zoKKy0-f*n+~n~{ z+KYmhT#3N<5rYcDy^v&^kWe=GW3>%xxJ+I8#~e;^HB-%);CM`m=qma`rWlw3tD07- zEmS&F7k0H-`5p&qGUXf(*fQlcR^abg+RI{D6c}KG7kjlM=V-12KWJ1V9XBwGverdqB!Nae=q7(8;es6zmm-!{fOOZ%?^3_5 z!j(`6`8DD5S6J{Bl&OvI_~|Wxkk(o7+E{1GPdu}-k_bT6(0RC8qIBV|^t%HtY^(_0 zDAwco-kO_{QeKPElNUFT9=c3cvBjG@_yZqw=ffL`=au^JHIh`gY6k-T5VS{h^Nf$r z@KDV}LB&z8GXAt0TUk;K(Ulc{HqA`^T!B(PVWue%lN_k&TJ*~U z-ni-McRXm)wsv-rPxc7Kn_6sWhgGp{1^*vK&nb@Y2FfBlpu{BmOP7dO+`j zw%X8+q=Aaut|F(Lig`*XwR7rVZn3y#j*oAcnqY%hOAr>Whr+pJyiX0$B)(XEz<*;hGp%8}U_rOV zmz8zexIwq8CI_l9Ms4AN#5RukgrsH{Ga*aH{BoD$7t@n4)`FrZP6*}H*`qJ{2*%Oo zMQrI1V30FRn-J==oL9-j?r16mzOSvA&rjglFDLIMQ9h>U;7ae!35z2 zo&qDwd#3}ocmKCk%T53~Qei+hZz@v@1x}u*%mSs_u+(+sjLCOw&2)}#DfOo;*sAF+ zQ%lTJZc(#doGT*6VALOQ8v9M4xG^B_0L)=O?h8baftrlzLM2jb=sF|Av)qh^L z6gI6RArHyp7$#dgDLYNb4v@V6EWqNV+XjuG`Z3@->%zj748O)5QyJoKw&0)|#JHB? zNOSDK#d3e7+TJKc)`Ai#lE-<;_{Ub<*6oQ_6ts<}G$k$S^j8(r(x2db`xB7mP)j6g zRsKQ}KO9v|#=4d=XxO=JO|u-F(?JzeMYTs^)|J-Prj+dM zAbJdfm-|`SaPcyRe;7M0v6BW$;o~Ks;{*^?XD&JKGUd+~xW>kJa*k`^Ojk=~0otbk zVhs%b8;`bfm@B<4x&gi)#;FDgp|gFh8+`W2-6lj$y&RiwY87Zz)&Zgj;||5&RyiL> zg2U@~ePjI({ZJ3b%ZcXwZMew`oRc6PXdThDqsJ$|2i1Lgx? zGb?`YcY99*%F|#S_DFOwnf}czbCv1l6#~ND1PuhwPsfSFWH^pzh!ske?VeU z3u|W+M>OZ!J!jrQ zNw`)aZ*hE|$OnP-2U6Zj(Y7ftfp4GQY+K%l*z2=w4GsYibJzBgOcAt;Aw!kB;Lf@B)?v)v>+jfhih1+aK=Ry_uF! zvz<f#QoRiOvin~N7mrrQJ_Tjr5qZ`^|x04dwdo&K+ zukR~AzK^91vmT%43Ol`>&;7l-&fK!jd5Dv1zukFLTNsiS;mmnxG*f^6N1sTk*lPU4#iCF=d$7|H zt~*lQ{fL)imp>MCmK22puu!z9m?OVe;e?Zw6Q&T&2O%x|RVIBu^>ZAqXgQDsTuY=N zd4=IPanuWL@%kv`O|^~uy=7lSgB;>?mI=WIYxbwfMWO<%_=IrP{=yaEZl4zj%l3Yi zI2K=vm;%cCJ%rR7Y6L+hMDALF92F)@f`T7oF%S=m6H**oyn<36c0rQ837fjP?|hp%-(bFN zp>G-)wD{+R+-6@m{9ur`nQTNEgn+S6;O6u?iezpup=w+YGN!}H;LIZ=0Fy*4!)He{ zjLM6Y<~PdvrD&Kr5r})NEkA?NuAy_{MI})LyI@RjdUG5G^%{BZh_jYhowTg{Tm~=? zvw_>d)hGseFe}7BWfPj~0BSvK3!zexewJ0zNkPh*w!n7!T|}Bn3gl7K4~bc4QbR6i zq9}*c9A*uKv`xBW0F2xGJZrhBFaxF8&r;0JlEniOyJO0AF%umsq!p->*zY50kbnjs ze8&I+Mz?pLTf7{Tlp%927=S|3JVueBphSmX3fDY_ltokolZAX1v=^XlKrtf!c+#F} zJqP`Og1U&@a)??oI`!~BVTJ?4$Wm-XSuIM3Ig1b(#Y+yRKvA3s_$1xcOMTx2>#Smb z{(^MgKxBRP8K$5%<8}NOamZ}QP{&o3OS7vl=t+C_Rui&qC_U)Snhw{V>m?o#PsfF} zFw)`%F&4jA5z86FU=sV~yrq=#rSb*F3h*Gwnb5zG4D(NyBzj?N1mOISkXhEmo@~OF zds}sOOvf; zYjhrh3LHit$ut}J9&;g@qJlYTtdb+$+#QES%*{ldlyxF}XZ>dlTpHLP8L5F}GSLyh zQGb+^L+1VJk%DYClLZ9pcqE-ykTk(@3iflTZio^L{? z&K#@{%QX=O>SlEnV%oXt#yMQ4ILk0&HJe0~=!a3G@> z>^g+YucV&GVe}9_o%eQaL7>xjAODxkE7Aw%UYVXpZbSr$WS4A2j0kiRb4flX_j<2s zErAm?p!4NfTC0;>7q5onzbKQAQzC#mL>+&W0Bv4MojQ9WQAXB|lLOuFc3Nb*9EtE0 z;}-;N^A@D^Mk%-YgmOH*CS&Zo7yZ>Bkw2rea#G}tn|Aw7lyaH2-Z#_Qq$@|(0Zph? z@3pM{2&@14e1sDN2H<-1EYdMf9Q_VO3a8tc0+W5X^s&I-=C$$%1kLoWSo zVr_}V-_WqYevKv;m>k2aJi7mdgbFH|j__T(*kWEcg(S-MDt5D5K)oIa)%OG}{0su| zP7j`>1v)xBCRdyAMGwwe_^IThKWw!dx9NDy5+MsgCOVz8e#nfP5E?hMR+$J+a>bcB zBN%Efwmz_inh{*J1tP%Z`^oGrRd@aKO+E4$Ro;u(=auoqPS%dBkW6$!;Byg4sY--X zD*1}5(zbRB{HjeB_ly6MJ+1Kk4%Q7OYhh#c*v$F3y1K}87)$)Bj=ag_pv`#pK{oqL z6a+^TcIttuoN97JIEt70kq5EZ!zeb0G6kIJL8KK;1<~CEgehP`Us({jUpfOtu-BO? zuH3rZ=4DP{u334YoAgG;r(@!caTJ78>4H=}jf!3x6Pi78k)N?JC-2A=eV`&e4uY^S zfO+AaA`vWaNC*@))a5tnD|A9ose<82iSt$pOjjDiQI|^T6zkY+C$pg?DFcI7p-IL4_OT}yQ`71Mu_ltaiEYkGHY)`$Ws6D%llTaTJJ+eFpp)jo znSy4R!X!s|xHJd-#WkJ!>5liHJD{f*Un7t{D7>=kNpHR_6wdAY?^XNwR-=KgT{c)) z(HhHN?h0m=RiX%IM2zU^`hD0FRu@qPzg4NZto^nfR^MkX&UhVEs)yF?7IGjIb*cn7mR;t0KPIC3#=cAfFG7$8&j8&5 zFNzl*rUf!#u-=K9ChBVpBqt%q^*h zDaYXfR(^SkrX}go*xPH~f(PVo0~>Q9(_?>?8aE^S|L zLt3b_sidbbqS00Dva4LOK3Ig%rvIn4dcvFPIK}4o%2UJ*piZv2-dV@U_ojB4++B^L z4FRKUr8a4+HMP02EKFu^*95Qrmd|~+SbZ*x7`Tz`Y5eAAKaGH>kA21 zDIl||u1t*p?+l|FUXd{t-pp@co+i>q=uBJJ~AuF?1d73W1JvUMHR1~ zJFWN)9(Ty$HvI0&{h!!5ZKz{C!$!5%6Qap|6(mD6yAb=@Rm&)}NP9X>7r=(6_%dQe zByb+O8LMcD&?~F5H!U;)7$4&f-0gZX3L{F!#@2j;aq@}FFnmB21?9qIjL)7P5@_PI z7*h_QMqm2)UfGi`C8HV6C)|qn*XmhnP-lr2k=hhGh0@duq1+HDa&d;gZc+(lazT@^ zYd^U-*f@I~gX;rcq><%ONtXsWK*vA40mK^@@gW~^R|ozGZfw9q3CQn*kpBh1v~RVp zPIVj5SqAgl3udVkPYAc_cg;^y=FJF}z(ZA?@T_;dDn!Q35U_w#poD)e=B~+GeLb#I zL^xtP*V_H4qPzrMI?>xnR8m~J;1*Jsh;&h9n;Wd7%_Jr!h5De!vz|0tX^Ek0;$@y9 zCeJy#!51bSe)|~Zs@|Vs4%nrj2@C|$_MG8jRi_;hBau2d9)j@tgGqnMwr3C?urx$~ za`yG0Zfcp2i-#IY)9G+e;^wS*P8|D_H-{*XWOS#Y;!L12PcARld_%sfV8XzLlrnWn zii=6@LJHD216gHbVyExdFm%)H&(y`w9ry^0AI{&iyX+31$ zk#DNf5INXVNd08$=uf)J)A)?u&y@LzlY|-3+mX-1mZLat<;P90QI#-e=p)`fm7gm_ z$C8KTJFn_2#mTzS4d;pLW8(fF&rhvIvOzoEs-;8CKEm&wJ?GjAP5C6N77a3ZWrfa2(tVWwyWGA{I-Ej7F)>S-dACwkm+Z2U!%S9SXv9CKmqm=iR@A9j#R!FXg$d z6}Ul!bI?L~Taf|bP z>RcU=%U6|-oqBdjUj&*Kk<=x8GtaZcyFA3oKed$sJ*d3&-PSJ2@crJEefP+>0CPofv+gPLKK?b%x$osd2Z{988skQPF+$S6Q#Caj}lYDU|W{<-@8< zgztS?gl!*mY3&-+XacrScuBmx3OX6JFFCn?Ou0XerHMc*G|eqS{tcC*b-w#)p!pLN z(d?R3&?t?^&?YVvnWyn{&DRabka=+A7lnt@U!C=kW`8W(A5-%CR`|K--jr9cTNRp} z&B_8ryq|goyCZw)b#XYvxGPJ2&qNb7Z+^qJsU*o*XYJ&DA0Og0`(G1XYWgV4?%434 zYFmno0^-v8fagko`JdKwgzN6?#3MuA6OQJwGFIpj^`I`ZZc%o>h@yi;i^cK2A_ps+ zO85hX1%C^&8N9WS@>PDV#cYw{K1>!gR(kTop2lsgDz!4v6KJ&9Fzu}4i5gm}k^x}K z;zAcI&Q?M&3{;`2V8RoQ1sh~#I3!UD%s+RnvHrYUyaGqLhB=b|yxm@i)l~2|lxg`Y32oktJSa!fl`2-0 z@MxvzUo$RFnMYJysV*=}uI5%YCSO0^E)_H$?C|xI!KVoE7wHIKm@gz7&9+&rWk9V) zisr;*GLQ31qu7Ig+qD3{L^m`X8RpPaVt?}NDj(OIoQBNd&o?-!LN<8snO&Y)NKdNY zgoF8^|14f@2$9&7u*^M0an)O@H!Pv5qNOj%O}8ULixZ2EymOGB<~ih=T>PokR2B^^ z-C$A`T~8{DwaHW9AmK#B8fTZA(qAoJKXVj7S&5FR2UiwLB|AgTEy7nq_*v z!z$aP>0G`R$y!EVIYua-uG>zW**s9Q;{ciPR2u$(&Xafr*TMnpX6yc~nM9@f{s85m zRPqab!)w#qsjns{68SP5d9eW6rh}csV1atqYA%W)!r@=KiH=lQ#4Mi3n10#0yeVYO z-4>(r;8ylY@kAJG!dg%X@lbfv(is&Xj(e}ZNTMWHoyH4gulJrrS2&a`l#M5HPse7t1WPPJ29EP9T>#-0@~bpiXu`jO zL(;4*RBFOuwf47|tNn5fPmfgcK=WZJLc*$Xq8>MIcgNSk%g;wgrX4RYukYvWM<$z} z$Iq$92_K)Y_viQCs92B3ODG=P4X)p-dnqxOlxgx_e`0PDTx|M5Nw)OMLSyIb4v<{2n+DY(=duS8Bw;t|* zU)t7dh@f?pn27>KVZ@nbPN|`540%30fAIBqc;D?mcc}f`J@x8!f1Y2c$$c}1`NMm+ zA_gW)Yf!Y|WGziag!VPdWJ(&1R`-~Rjs@%_9PU381<=LUB8KVsV zte*KS@3|3x{vPAecCD2NAbU#DC27S>QII&t_gWOdG_Bl{!bTEj3ZV)*JDU*6E>4-W zAo*M`7J8yjI}sTzM7bFQhQ14v(M)cUoqL6BB2aOw{aABd1(QKFso7lE>CtSW_(qgh7I-{Z&|!3e!felT3Z zCQ>(895vM;wZ20DO%7gvQ+Yhq_Kop!E0K?<%?>}nlAoELOVAk<{t&^OxfN`YR3Z88 zQ56z&-okc#y`0m-H?bLFJ;!KBtMQCV*1s6I{oS2}V?oy3hY~fn4s{QTRMQnyj3H8F z&z$h5SYnrAJYrb73i(Qq8!UeTrAWzQZ4QaVlA#CODPU6cTUuCuS==ZRJl+*Ok763! zB28)kGbqH#G%3lFkXf6)q7u3;V3)B=lb$y`CT5dhuNHFh0hsKy(vCa zm?YcEN+(k1o^kE3zjL1PV)BJfBabHL#{#3zq^RcV;u)3*(A3u98p_(|bfF#7@C*4h zmxj5Y-siEXpJV@lR-rp84bERl?`Pg;reMTv)eOvoti5VvcH%6;V)=z+`RgqyRctI& z7O$EKz}!qJ=DrMO=Hng@apt+0MKw&4>LvGHrMMog5`}H%?KGhe*I5n!D|BaSqlME0 zTU&7J=I$Qq{D9(kCGI(w$NC&Mo~({nWsvcSmzIGZ)7?VgQI;RCVzJD#v>W-9Y+mp= zw>BTW`~r`vA`v0I`@6hM_mp`=D3_vUUmRFeV$Wa-EZEL%Nz1LZdCR9YMirt6%poB&#q zv=(X7O7qaKBiN{5NvXyWY2EU$*sur?t9l{Dn=ne%!Cx{gQ@!|8Z|0sMs?t(6U;8YFgu^^z zoi4l@O(eOj^6w7~W@9XfilT0sX6zg*im1!IQc4838c2>I3Y*0iNzw9P zp^@d)TD0VLoS5jU$W!0JaG=wwu5f~ykhDSivYr$HQJBBn7|>d+OwyABz@twck3Faa zpk548pwwh8FiV1wS68$W;FccY8o7Hk#3aUAma-(T93c9E6ydfS!myE109j(%p)`UL#qYr;>TW0o^+7 z?;=*pdR;k5EkaI|61VnmS=|B;63nC&XN*LP1Y}M58{}gF@}7Jw8jC$OKd~6g&(kmF zD-nm*H3}VMy9P1QY)RSt>z?as2aPO-VMKzORD{_y=zmp~+H9_roiP(~AkwuSJ5-72B5IrB;?7*=VHo-HOq?N)nf4!WZn z?Vtg(lxugIPv>Q^R=$)~m2LKQ^~bufULLZ^NywLF@rdr6UaJu4w zaeXfF*HT!IX#e`H#-1PU@qZc*Zd5MX8|$SpZPw15@iE~zM_8bWdYl;8XJGQo37aW;*d|00!cM}2LQa~b zD<8q33#Nc~5`QJaHU%yqa{e=}Yq^M=``=F*!sbdK`M#M*rJdpH4)Dlv9Rm{fod+vk zI2lm|6KP2eIyPCl3<9`J=vr!`jry&aB%X$u&d7FTkpK4;``y#14an4TYb$!FS_~hGNo3y(%hf+mAxZ@)fyv9#V2F!jnJtf^zcSqunSj@y z)kRWbiS;w>%9=9GhBAoWMYZQ58uSahb}?_6hLSar^9#Bi1CA(K6U@10xzT6j6L9NJ zq51DVAe5}2oOh+k$(~3`c?|B0jY6Jn+-}2lU<2+a$C2~va}lBe&XNv6_Xxk1mFHi6 zF)rrlvp@MzLK3PtJUdu6oM~j>XI1s-o_a`PAf~Yn zxB6rPK>#J&&|e?+LLrvz;eb&6DxrUcvS9Z&GClGV0RiIo0FHr?t@$9WJsWOJ?}7Y< zBq#^T);=5-N~1|3WnT7Com%S$tqr3D0`jIrDOMF~cvdvBS@DZ>J70 zit)NXiR_gs8cAG_<1d2juq{zuZGKm=VaGTTo9TvvAP;EIM%+ZV9V80TOrRD6kh4LFLCpgo# zZ;cTGK=ZCbBybDF%ncpDC&+W=R4#uTC`UuJ$dB=e4bCU(ILHn*h|xA6PrO5_#*U|G91xkdXXqNDCn4C*F{vLmf$S*tYbs<%RN@jkiXrQv{W30Ip?pE@r{miY3lh zHGTZ?Y2!?nIjvpd$3t<_Z58)A2;gM%K>d=MUUxrUUY9_nP9zdvp_t;6nzcM7Z88$& zvXd=|g{=o&1Xt5?b3c%okO_ehkpe*{E-14gf-TH;RqE(XsyDEcmf08zg=!0UK{F+k58VF;}i~pw{_l>3lKy>KFaSZS{3qwR=h7^IY8L)oXl6TY zWu6`Nbkv~nfS(dqzpBn--DF+1$(zy_HIuY(jtD8 z{kLLdQ{KHn6Co<*oVbJn&0M)d5z2BoCa0%t5~a1+d-L^30jUgKm+nJI4$D1+Pa#&S zW}o0Q7qH~WpYy6Qyv&F)K3^dmX`HJP(quJRS|N5SMT!H7_2_?yG;F@TjyLe<5?Bz@( z1c+AFS1j$~G7Q!0@a{KmoRuikBX2Z3qXfOd>3af9C#G@>&H^vzz{}F1I*KZ#05r93 z0j1*$Qv+qGhl;=d&}Op)H$A^AVXCuPOv-C}Ca0?_an3CLLT)^AM`kMrVO=9fId%>A z`Zoo`A!g~^*8X(1oXEYD{2R+|Yg?SGb<^cgP1S+_>e;_^OA5NoYsY{iar+gJ=bUpj z?SkMU9bmRnF})U3?&`jnl^%Z{eeObo4V(qE`5LAp2&eIo1QBe_8%|KII+598lbQ8P z3JPdV)D*@L@>~t}!40hG>d#|Rf<=YpvQzOt@{hd2Z-Qhlzt_c?8HjRh-*H_=zTb|p zTPAPTj=&e4`8n;*4s@QCZ?fGcMJP-eH?W|zLXLc5;OV7p?Q|)g?EbqU5D|?7d{lV5rDH4l zXf~Xof>%zo0Y(|vd&~ayd&HdG7Uabf%tVqz3@#{$Y#~CC8bi2Y;&Y$6rSZ?-QiVVFT|d z@OrgcVSkC;HTqGHVi&hx27Rb)QgRE%7`p!Y4RJsV+~`y%AT%ZF&!D9uQ0sZ4Ns$cu zlxc;;XpfqMzn^X(2&kBQ1$6z%>}`8TX%y!t3QTi8OiO68XUDB%k5wpNw*Q1%`$L5x4YvE&=g(FFYE>D6iWxm_QP z?%hu%TzQDrL%tBXgfXyMMBfpeom_E*D3IXud+y$bg>*5(OIo%3-!#g6*afhAcUx1r z3V@v2Q>T5Y2nGmFp0w51$8$spYOVD+Yz$wxau5-<9oUp$Er}<@6wnZIw`Hjt?KAbC z`7n$n;!j@a5i^=Ma=myE7T_WOZz*B~{?=_b@4tUv5cYLaAMDO@+czJC>vYQzk}I7Q z1sOWuc_q8vc3x_}t#+m1fA~)KI_g}KPJ-*DoyeAMZx26+h=PJlbt$R%-bXpU4n}ZS zB}oZur`YSG`+uI}3yh5D1*6X6kwtNpAy9BV#%1pC;RmSo>qy>+7W@ zc``Nl&PTNwF1+c<=$La`GoJSwc@vQ=B6(aJu}>5VTbwvCJgX6*P4}JFwfSb?q}gaz zZ2d3SwNf2`ifXA*PCZ3ktej&Q7M9ZC1EYAm2u@##MSPVGwPa0{x{n?~?rBQ;Np>8s zo8()L|9#ROIyQw)++|t^G)2o{9HdWw>XHCDr=FRyT*X0KW&7S5TC_>}{o%RP1WkKT zC6P9zBdHZwQ}fUMB~l;Fdl~FXE_8unpeUt;@>7)z(F8!?-5X@RgD?Y@!q!&Gt-Y{q ze0GD?SLBw))P&M2vu9D}I$E^O(#aFSO=S5j{#b>0le)x|RmZCeHK87Qrgy9URC|>H=KAu$7SAq zZQ87y28Si{kVNQcvKUG^MbM&H5BX`;o@ur)(301}&V?h>Zd_@i>O&i%xZ_!hL3ABQ z3kjB4_5%}}FO@jg^2l2Sq{H;cS^v}6mB&N1zyEep>QJoUb##QO7gw z?*1mFpm4>5<^wUOvmW>5Kb;d7ws5@r+S@xe({UqDZFG=K2%DQx7b$J9SpKRiK;Bc- zH-BpAT#bh##T)KzI92=jx}&vK+K$GNeX%mxPt1$qpizzMqMsU;;y{a=DzUf874c$Hs!xPn2|Eit9o^EtVy#?`EuV@ z-4~6IOJZWoK5W@7HP%R&x~$aTZCKrvo3U@&waWeq@VMl^C30C`ks~Vm&L+KlEqZT# zu%z2Mt4y_rAIK23q;rX-qX!#*z}Yl7vwF4l@RO*0 z+=Wwn*-g}(ACax~{M`+mdgk2CC*4f#=<^fFFDf7%?520z~CJkDWYn$`A8Ll-I< zkFB3}Gj&)^M*5Jg4gTm$7bbx&vFL5jjVd3DZJHVd${GWf@$P_lMJoh{@PnN zg;||x2%YHButP1T^6aG*-tJk`yD4hACnVXQ({|IR_mMAGXlKl*)}PJW6EIyP+uXM3 zjJfW;>Q9aF9iM8Rdo&M>f7jeD{qV_AGH}E0xDV~09>xzG65i2F$9;VD>dM2?cj&QK zUtUYeLhT{akWoPe4+EMHMb!)6bac}`#~IO8&zuB3bDSamSI@l0%iB?x)>?AkS3reo zWW9W^dtX~?^?FX;jI24&yp!XF{p=@tx)2#x&g2F~F4}ceu*FA@a?vT-EkC;WYLrP( z?0ey9bMu|2E!yj~1>E^37e$!2?dT?0v7skN!_XsP)}U-HcRP(^zi&6unY&0oagooY z3Ebcn8!E4~E*s<*7&$1Us`S~@ntQ*L-1(*C&C~L_r+q(}yMWk;(b&7~%pEnqThg!#^Go&|l${*CD|s@4pY%O`p=##xl*Z%3ZX~YGdNusdNgH$w1;g+oZruy@%Hwew%}>iO%bG6-S+ay^0!4INu}w@mbA^) z;}%^tFBo{-cS3IP#>wMz?~is2Sh{qC)%4kHs@o1d8-Xk4Y*JO${vdXQoACD0bkmSE zUIET>PO8y%GU3nj+up0UM$0dR+>-qAR;zE?x|rfey>{okdy?L^2>0d1$KR|h;5OW( zTO<_)4&e`Hxik(PQ1?#?w%}vO%}4ju_b>Z6wO35lAm2fDR{gT{y=I$CIM~$J?qViy z&gIgflh8La=9%ZuTd8$-*L>rxx%!?VcG%QSk| zzUob;4n43$an@z-r8CPt%MCm~jf(V=rS=~q_FlN+0g|D<^i)fIyVH?{37M1ZUbrP_ z=NSxh$(ZCUv$svgR<78qTWK>_;+C>=4pfJjIS&9ohew`l9>GH9v%{&r;9vvdGl# z=2hNG-}^M=xt{viJJ!_xk{M^WCB}`Nt~2*1r{o#s5k{@aR^C^qFHi3somP{0P$S^Z zUq$px&5DvKrROxK9MaLyUf5eYrca?jWR1lZC)U4r9(v~(ClbA#u22W>!QOGblF|6i zO@68OG1>D~7yh2LS$|q{a`uZWX@A&HX`F^158qN%Hg1PtP?h*u%*2XeQ({&>(HJp& zg3IhXGt@)lbX!S_F@eHh7i!FnokY6ec&~SU=EXIWtxmWZoiH^~^jUi)i!O*V^is$j z*Bq`{7xX5}ab`>1UH6+O%kXTDo$zs9yph?oOV!=O=QK_-@*TP+{^6FKx2=b+O{`eb z9yDoT!_}qJW(IfoxFk)utMz<##k1M@w|Μ~N}4_Vn#Oud1PUv48yhB=VK(HN}G} zlF}2r)@bL9oXVCCuuO^=(tZ9XPqn6Y<5dxpb^L1ZB&V90HTqdQMr8G~%X2sD6%l{H z!pbc9YC~M2^zK8cguJS@asB$&W!){aUe%<`q;Dk0Y_&`q9g{!4x@vn>-=UWhvd?}R zKYvAuOK7&uz7=U^e0?-={Oz7$ekA(3>WUJII_27P{uL8(IHD>#x|Bmm%Tg7l^=3> zkgZlVDZH#lQ=j39RcBMIH|O%+oLg|m$Kh7lv0VrE8{ezXU&aZ(wd(4Hx78IhH2#S2 zs2+UCVA;cPs}pyVei=OZR#8seYV(TUVg|PKU)~Y(t67Bi;frVfc3ywE#)JA9U)TRd z+t!L5*crQ@ddy!tMO`bY^7lhmFMG^(-kZ8qR-$9N@2^>}b{8yQkDuH*{iai#_P9&^ z*AyiZtgwbER$Trl~u<-RB$*8BFhv{!o`Ev z?NhuwC>l6gS|-0zwq~cx+@xM zFuB>l-&UG+)Y?Xr@Nb!Y9AI8lD|DOk8+z!zxYoWP zVxmFtg2>#ek!4BThT|6g3w$-Q5?^$56ZYfejH9aJUv`8BiNdZMYi|lKUJ$kG=jmZS z&D&SL*xOt_Vav2Ij_I-O!kOAWQ**+rCoTQ+_}x7h>$X+IX5U|uJ=`igV@a9i)#ZKG zr5A)c6aDr5z1JC{cc&UR5A`nSmSWUzansQC8mEd$I5ztY+c5C4<(9&Tvq>J0t|r~AjnT68tS<7pQYfs`Z1ybuP+Gk|eH+(u zS4W%p{p+I8oLtLUj*@HJ|Gw{-le?p3$EM~E%Z`6ySc#=CciQ{?%SL1lpAbIA#dA_e z#608Rqdw?Vd}Y8Map300bWkmUZa2UHvirh zwUmaF&Ogx-_|0V5~G9tC!l(~I&<;VP!M*3&o1+F=&eqAk5PkqR_j%n!y+xq_G z*C){T%r8Gx7FPbfIe1X)Zrz6P!Smv;GqRo6TuVcD{`y`$FZ*qKQ18c0siRMvx`oes ze!gsKRg1e{M*rI*T?<;%(|Y(+YG&*?Z`EU!G2$9GPoCJyx*8qVCQKq72;}nSG79*7_IG71enP7%o z2T7m}dW6ov#+1MaKE`4!j*|?JHNxl#7`om$DpVr=Z%pXPJE1sG`4$9{M}HIV%li;; zoX_K;MTJ}kp+XQSPzboL5&jA#!P8;Zx2N{ZCr3*K0bxP~$6pv6Dz?yTEDZ7J4BHh18VGjf#pgi6TuTvS2RG zvMd)PxCCL0IE>{nVuc{uSS%m=Wda+aJU|vIRUid%loJa4B@qe>J+x-0Fo6MICzVFX zM4eLx25^NUVYpDNkaKYp9Le{uffR9KKX5uY{CD2)@E_RS zoPz&5zk(|b{x(AToi=_D1zkX*kciXLJTSmip>)8{s)sWk;v+% zUwq$9A`w}B@Bijp)vL;FDP@lk1b^At6hZLURfS@e3Jnk_J(i2CX|9z0%O&M34i2XFV!1*f4iMVgSc)AyZKeK!MxMboQP#E+ zPxLMD=MhHRZ^HWeY6ND|EL)Olwfboq*KoX){>(EHlZ@aqqOWS!Ugd-Mkqv~Vj)^u zDLbP43)9hPmM|Wg!j6_$Yb(I2OlQun(MIz`8g| zgMHvU2CR!ye4s0wPr`L+9PAf~0{BRb5B5tk&>j>@!8NEO1fW|SP z0~%)_9|8TICSh#RBnxaw^Y}nlG(`a&(0r&rn#SN*nt^*`pl{GD34E8qur6pcq!E0u z21AmN#)DYG@E~p=6Tz{33h0VK%2Krv!{9(Z7E?m)kG5oS4EO;{z%^Ks1~zATa9!j= z03%Qs!FBo2Usz;JARj|Q8rlfpV`)_yj$u4V!*NI>fSz%TBvf^eXaF~m_JGZCi~=@C z5DRRsgk87>&8q4T#~2#YSRSCE6x9U{U^b596yziLsy>8{L%_Z`&I9p_XaI+lOThW~ zAm^bY62L1Qr@N4aA>n*1251Dtdz?V$Fcp8n2?FFJoFD;a;{*@tkDx&8;{+etgP;jj z96~7+_%2ScgleoJ8mk&_IEgFEnIGa8(OA{k!bt+e5l)gIC*vdqFapuwen~#SNtE_! zC<}`Lzam+H8#oW+spbmgRy5cLkAU&ZBS9X-c|3quI3fX?^SZDRf?c3zWOA?%9t-k5 zPGJnt0fmE_0jCInVK{}1q-slw2XYSzBZy5DDkQ*I8el9=(GWK%2HJySd0-!?&H?hF zItTLM5O?_mz;;yXLBB%g0J`T>P^Wx8^k+Uz0{ikA7;k(Q>V?ljyrMCP#i$eoaY+*} zR%sIA3(bRip^@@c?S^UpnIByISRoGScAc#zc4t=GYkP^km(}lF+7OR zOc#4&_!Nk9hK4#|SWsi4Y7*vJ1oyz^ECKC{N;sfX7J-hc?pYp;1(pvnk)xh6C#& z1**mtfrJM7BM?<}9|WmsDYCudOSpeh#5$0t#zP9=``JaFD8 z&@mb4j}Oma$}j>k$VW{8um;Sv=mY`zXqcl=M-av!jSdm2YoI;|un{`i1O3qy34EM} zxfa0zsKW@F0q0esi=GEMrof!S;2?Js3;}XHLIZIAB@pZaJ)`yuuo0>mKwL6i}sK`fRXhui__CN>$FaptlpOP4b!}%z_YRsS(E$Bxe8tChh7!B-!Xdo_0 z3^^Ov2eb!?v7jy?QOgpH#ZjfBIv?tJfPLVwPDEWykUvPA2EI$8@KB9uWx53VLkIf%>rzMwsyd)Z&}$(n3hIxdp)3>wVxOc~fDt4gHMLaN zMR*Tvgxb5npZP2}`=QDa=nr)#fsdm!4eB)tL-&7FWT7w`tsgR&|g zTK7ADQOE?LBB4wJ-GXxsjTI{It8qK;>5)nlLO1o4H}nt{ZAtlujkm#nx;JWQ@SkqE b8v1m8Q$iq9bl$u}2NBfV(a_hob+r3G&E(@V literal 0 HcmV?d00001 diff --git a/docs/wt-user-guide.typ b/docs/wt-user-guide.typ new file mode 100644 index 00000000..64ad738a --- /dev/null +++ b/docs/wt-user-guide.typ @@ -0,0 +1,276 @@ +#set document(title: "WorkTable User Guide", author: "PathScale") +#set page(paper: "a4", margin: (x: 2.2cm, y: 2.4cm), numbering: "1") +#set text(font: ("Helvetica", "Arial"), size: 10pt) +#set par(justify: true, leading: 0.62em) +#show heading: set block(above: 1.4em, below: 0.7em) +#show heading.where(level: 1): set text(size: 17pt, weight: "bold") +#show heading.where(level: 2): set text(size: 12.5pt, weight: "bold") +#show heading.where(level: 3): set text(size: 10.5pt, weight: "bold") +#show raw.where(block: true): it => block( + fill: rgb("#f4f4f2"), inset: 9pt, radius: 3pt, width: 100%, breakable: false, text(size: 8.5pt, it), +) +#show raw.where(block: false): it => box(fill: rgb("#f0f0ee"), inset: (x: 2.5pt, y: 0pt), outset: (y: 2.5pt), radius: 2pt, text(size: 9pt, it)) +#show link: set text(fill: rgb("#1a4f8a")) +#let note(title, body) = block( + fill: rgb("#fbf6e8"), stroke: (left: 2.5pt + rgb("#c8a13a")), inset: 9pt, radius: 2pt, width: 100%, + [*#title.* #body], +) + +#align(center)[ + #text(size: 26pt, weight: "bold")[WorkTable] + #v(-0.4em) + #text(size: 11pt, style: "italic")[Absolutely not a database.] + #v(0.6em) + #text(size: 9.5pt)[A user's guide to the `worktable!` macro, its queries, its indexes and its + persistence tier. Written against 1.0.0-beta.19.] +] +#v(1.2em) + += What this is + +Embedded table storage for Rust. You declare a table with a macro and get a typed +struct back: a primary key, secondary indexes, and generated queries. Rows live in +memory as paged, zero-copy records. Persisting them to local disk or to S3 is opt-in. + +If you have used .NET's `DataTable` this will feel familiar. The differences are that +the type is generated for you, and that persistence is one feature away. + +#note("What it is not")[There is no transaction journal and no fsync on every batch. +A mutation returning means the change was accepted and queued, not that it is on +stable storage. Section 6 says exactly what each boundary guarantees.] + += Getting started + +```sh +cargo add worktable +``` + +A table is one macro invocation. The name is the only required key beyond the columns. + +```rust +use worktable::prelude::*; +use worktable::worktable; + +worktable! ( + name: Order, + columns: { + id: u64 primary_key autoincrement, + symbol: String, + quantity: u64, + }, + indexes: { + symbol_idx: symbol, + } +); +``` + +That generates `OrderWorkTable`, `OrderRow`, `OrderPrimaryKey`, and a `select_by_symbol` +method from the index. Nothing is written by hand per table. + +```rust +let table = OrderWorkTable::default(); +table + .insert(OrderRow { id: table.get_next_pk().into(), symbol: "ETH".into(), quantity: 3 }) + .await?; +let found = table.select_by_symbol("ETH".into()).execute()?; +``` + +Mutations are `async`; reads are not. This declaration and these three calls are +compiled and run by `examples/guide_check.rs`, so the guide cannot drift from the API +without the build noticing. + += Declaring a table + +The grammar is positional at the top and block-structured below it. The order is +*name, version, persist, partition_by*, and then the blocks `columns`, `indexes`, +`queries`, `config`, in any order. Putting `persist` after a block is an error that +names the required order rather than failing as an unexpected token. + +== Columns + +Each column is `name: Type` followed by any inline attributes. `primary_key` is +required on exactly one column, or on several to form a tuple key. `autoincrement` +asks the table to generate the key. `optional` makes the column an `Option`. + +== Indexes + +Each entry is `name: column`, optionally `unique`, optionally `using `. +A non-unique index maps one key to many rows. Every index adds a `select_by_` +method. + +== Queries + +Beyond the generated `select`, `insert`, `insert_many`, `upsert`, `update`, `delete` +and `select_all`, the `queries` block declares your own update and delete shapes. + +#note("in_place queries")[An `in_place` query hands you a mutable reference to the +archived column bytes and skips index maintenance entirely, so a column covered by any +index cannot be mutated that way. The macro refuses it rather than letting an index go +stale.] + += Index backends + +An index can name its physical structure with `using`. Four are available and they +differ in what they can express, not only in speed. + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Backend*], [*When it fits*], + [`worktables_index`], [The general one, and the default for persisted indexes. Takes an ordered key of any type.], + [`indexset`], [Vanilla IndexSet, selectable explicitly while keeping the same disk representation.], + [`arctic`], [Fixed-width keys only, and the fast one. Packs a row link into a single `u64`.], + [`congee`], [Fixed-width integer keys. Refuses `String` and other variable-width types.], +) + +Arctic and Congee must state `persist: true` or `persist: false` explicitly, because +their persistence uses native checkpoint and WAL adapters rather than the shared page +format. + +#note("Arctic and page size")[Arctic packs a link into 64 bits with 16-bit offset and +length fields, so it cannot address a page larger than 65535 bytes. The macro checks +this and refuses the combination.] + += Page size + +A page has two sizes and they are not interchangeable. The *stride* is what one page +occupies on disk, header included, and every file offset is computed from it. The +*inner size* is the stride less the 28-byte header: what a page can actually hold. + +Set it in the `config` block: + +```rust +worktable! ( + name: Small, + columns: { id: u64 primary_key, v: u64 }, + config: { page_size: 4096 } +); +``` + +#note("Persisted tables have a floor, not a fixed size")[`page_size` works for a +persisted table, and the only rule is a 512-byte minimum: a page on disk carries a +28-byte header, so anything much smaller is mostly header. An Arctic-backed table is +also capped at 65535. In-memory tables have neither limit. + +It was refused outright until recently, because the seeks computed offsets from a +hardcoded constant while the table threaded the configured one. Every location that +decides a page size, and the three silent bugs found while making them agree, are in +`docs/page-size.md`.] + += Persistence + +Persistence is implemented, not planned. Add `persist: true` and load the table through +an engine. + +```rust +let config = DiskConfig::new_with_table_name(dir, OrderWorkTable::name_snake_case(), OrderWorkTable::version()); +let engine = OrderPersistenceEngine::new(config).await?; +let table = OrderWorkTable::load(engine).await?; +``` + +S3 layers on top of the disk engine rather than replacing it: +`S3SyncDiskPersistenceEngine` wraps a `DiskPersistenceEngine` and syncs it. Enable the +`s3-support` feature. + +== The durability contract + +This is the part to read before relying on it. + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Boundary*], [*What you actually get*], + [A mutation returns], [The in-memory change was accepted and its persistence operation was queued.], + [`wait_for_ops()` returns], [The engine completed the queued operations. No fsync, no stable-storage guarantee.], + [`close()` returns], [Intake stopped, the queue drained, the engine task joined. Still no fsync guarantee.], + [Process crash or `SIGKILL`], [Acknowledged rows may be lost and the file may be torn.], + [Power loss], [No atomic-batch or stable-storage guarantee.], +) + +Call `close()` during orderly shutdown. `wait_for_ops()` is not a shutdown boundary on +its own: it does not stop another task from queueing later work, so it needs +application-level writer quiescence to mean anything. + +A persistence failure is terminal. An unrecoverable event gap, queue-analysis error, +batch-apply error or engine-task failure moves the table into a failed state, and the +original error is returned to waiters, to `close()`, and to later mutations. + +== Loading a torn store + +A normal load audits archived rows and both primary and secondary index consistency +before exposing the table, and refuses torn state with `PersistenceLoadError` rather +than opening plausible-but-invented rows. + +`LoadMode::Recovery` exists for offline tools only. It copies individually validated +rows through a surviving index into a clean table, which must then pass a normal strict +load before anyone reads it. It is not an in-place repair and must never serve live +traffic. + +== Vacuum + +Persisted vacuum compacts the in-memory layout and keeps disk indexes consistent with +moved rows. It does not truncate `.wt.data`. Watch physical growth with +`persisted_data_file_size_bytes().await` and decide when to snapshot and rebuild. + += The filesystem + +WorkTable reaches the filesystem through one module, `worktable::prelude::fsx`, and +names no async runtime. The file type is `std::fs::File` behind `AllowStdIo`, which +carries the `futures-io` traits the storage layer asks for while keeping blocking +semantics. + +That is a deliberate choice and it was measured: `tokio::fs` ran scattered updates at +12,316 rows per second against 74,728 for the same code on `std::fs`, a factor of 6.1, +with bulk insert within noise and the in-memory control matching. A scattered update is +many small IOs and `tokio::fs` pays a thread-pool round trip for each one. + +#note("Where to put the work")[Because the calls block, a persistence engine should own +a thread rather than share a runtime's worker pool. The calls were never waiting on the +disk through a runtime anyway: the persistence path measured 89 voluntary context +switches across 25,000 inserts.] + += Concurrency + +Indexes are lock-free with change-data-capture, and a row-level `LockMap` gives ordered +access when you need it. Generated reads always use immutable row-version publication, +including in `default-features = false` builds: turning off a Cargo feature must never +expose a safe API that races deserialization against page-byte mutation. + +Point lookups use a strict backend-specific visibility contract by default. +WorkTablesIndex pins the structural mapping until its selected node is locked, so both +hits and misses are definitive. + += Feature flags worth knowing + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Feature*], [*Effect*], + [`std`], [On by default. Off, the crate links no `std` and the whole persistence half is gone with it.], + [`s3-support`], [The S3 sync engine, and the HTTP stack under it.], + [`logical-index-persistence`], [Moves unique structural CDC work off the mutation path into the background worker. The page format is unchanged either way.], + [`wti-predictable-search`], [On by default. The branch-based node search, which avoids a measured regression on sequential numeric keys.], +) + +The three alternative search policies (`wti-hybrid-search`, `wti-std-search`, +`wti-superslice-search`) are compile-time gates. Enable one, and only one, for an +unambiguous build. If feature unification turns on several, WorkTablesIndex applies a +documented precedence rather than refusing the graph. + += Where to look next + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Document*], [*Covers*], + [`docs/persistence-durability.md`], [The durability contract in full, and the snapshot-restore procedure.], + [`docs/index-backend-dsl-proposal.md`], [The `using` syntax and the capability matrix per backend.], + [`docs/page-size.md`], [Every location across the four crates that decides a page size.], + [`docs/queries.md`], [The generated query surface and the custom query grammar.], + [`docs/migration.md`], [Moving a store between formats.], + [`docs/known-issues.md`], [What is known to be wrong right now.], +) diff --git a/dsl/src/validate.rs b/dsl/src/validate.rs index 36415291..174c32ce 100644 --- a/dsl/src/validate.rs +++ b/dsl/src/validate.rs @@ -19,31 +19,42 @@ use crate::model::{Columns, IndexBackend, Persistence}; -/// data_bucket's on-disk layer seeks with its own hardcoded `PAGE_SIZE` of -/// 16384 bytes (`seek_to_page_start`, `seek_by_link`, `persist_page`), while -/// the generated table threads the user's `page_size` through its page-id and -/// length arithmetic. Any other value therefore reads and writes the wrong -/// file offsets as soon as the table persists, silently corrupting it. -/// In-memory tables never seek a file: for them `page_size` only sizes index -/// nodes and stays configurable. -const DATA_BUCKET_PAGE_SIZE: u32 = 16384; +/// Bytes of `GeneralHeader` at the front of every persisted page. A page has +/// to be larger than this or there is no room left for a row. +const GENERAL_HEADER_SIZE: u32 = 28; +/// The smallest persisted page worth allowing. Below this the header is most +/// of the page and the table spends its time on page transitions; the number +/// is a floor against obvious mistakes, not a tuned value. +/// +/// It applies to persisted tables only. An in-memory table writes no header, +/// so its `page_size` only sizes index nodes and a small one is a legitimate +/// choice rather than a mistake. +const MINIMUM_PERSISTED_PAGE_SIZE: u32 = 512; + +/// A persisted table used to be refused any page size but 16384, because +/// `data_bucket` computed every offset from a hardcoded `PAGE_SIZE` in +/// `seek_to_page_start`, `seek_by_link` and `persist_page` while the generated +/// table threaded the configured size through its page-id arithmetic. The two +/// disagreed and the file was silently corrupt. +/// +/// Those seeks take the stride as a parameter now, and the generated table +/// passes its own constant to every one of them, in the data file and the index +/// file alike. The restriction is gone. What is left is the arithmetic that +/// still has to hold. pub fn validate_page_size(config: Option<&crate::model::Config>, persistence: Persistence) -> syn::Result<()> { let Some(config) = config else { return Ok(()) }; let Some(page_size) = config.page_size else { return Ok(()); }; - if persistence.is_persisted() && page_size != DATA_BUCKET_PAGE_SIZE { - let span = config.page_size_span.unwrap_or_else(proc_macro2::Span::call_site); + let span = config.page_size_span.unwrap_or_else(proc_macro2::Span::call_site); + if persistence.is_persisted() && page_size < MINIMUM_PERSISTED_PAGE_SIZE { return Err(syn::Error::new( span, format!( - "`page_size: {page_size}` cannot be combined with `persist: true`: the on-disk \ - layer (data_bucket) hardcodes {DATA_BUCKET_PAGE_SIZE}-byte pages in every file \ - seek, so a persisted table with any other page size reads and writes the wrong \ - pages and corrupts its files. Remove `page_size` (or set it to \ - {DATA_BUCKET_PAGE_SIZE}); custom page sizes remain available for in-memory \ - tables, where they only size index nodes" + "`page_size: {page_size}` is below the {MINIMUM_PERSISTED_PAGE_SIZE}-byte \ + minimum for a persisted table. Each page on disk carries a \ + {GENERAL_HEADER_SIZE}-byte header, so a page this small is mostly header" ), )); } diff --git a/dsl/tests/check.rs b/dsl/tests/check.rs index f10dc2a0..2e882e94 100644 --- a/dsl/tests/check.rs +++ b/dsl/tests/check.rs @@ -59,7 +59,7 @@ fn every_broken_rule_is_reported() { persist: true, columns: { id: u64 primary_key, label: String }, indexes: { label_idx: label unique using congee }, - config: { page_size: 4096 }", + config: { page_size: 64 }", ); assert!(checked.schema.is_some()); diff --git a/dsl/tests/cli.rs b/dsl/tests/cli.rs index bfcfe6b6..2358caa0 100644 --- a/dsl/tests/cli.rs +++ b/dsl/tests/cli.rs @@ -71,10 +71,13 @@ fn scan_finds_declarations_in_a_function_body() { /// declaration". /// /// The difference is not academic and this test exists because the first -/// version of the binary got it wrong. `page_size: 4096` beside -/// `persist: true` parses perfectly: it is a well-formed declaration. The -/// macro refuses it, because the on-disk layer hardcodes 16384-byte pages and -/// any other value reads and writes the wrong file offsets. +/// version of the binary got it wrong. `page_size: 64` parses perfectly: it is +/// a well-formed declaration. The macro refuses it, because a persisted page +/// that small is mostly its own 28-byte header. +/// +/// The fixture used to be `page_size: 4096` beside `persist: true`, refused +/// while the on-disk layer hardcoded its stride. It takes the stride as a +/// parameter now, so that combination is accepted and tests nothing. /// /// A round trip built on `Schema::parse` therefore reports success for output /// that does not compile, which is precisely the mistake a second @@ -82,7 +85,7 @@ fn scan_finds_declarations_in_a_function_body() { /// to stop. #[test] fn a_declaration_the_macro_refuses_is_not_a_successful_round_trip() { - let refused = "name: T, persist: true, columns: { id: u64 primary_key }, config: { page_size: 4096 },"; + let refused = "name: T, persist: true, columns: { id: u64 primary_key }, config: { page_size: 64 },"; // It parses. That is the trap. assert!( diff --git a/examples/guide_check.rs b/examples/guide_check.rs new file mode 100644 index 00000000..e61653bd --- /dev/null +++ b/examples/guide_check.rs @@ -0,0 +1,31 @@ +//! The declaration and the three calls printed in `docs/wt-user-guide.typ`. +//! If the guide drifts from the API, this stops compiling. +use worktable::prelude::*; +use worktable::worktable; + +worktable! ( + name: Order, + columns: { + id: u64 primary_key autoincrement, + symbol: String, + quantity: u64, + }, + indexes: { + symbol_idx: symbol, + } +); + +#[tokio::main] +async fn main() -> eyre::Result<()> { + let table = OrderWorkTable::default(); + table + .insert(OrderRow { + id: table.get_next_pk().into(), + symbol: "ETH".into(), + quantity: 3, + }) + .await?; + let found = table.select_by_symbol("ETH".into()).execute()?; + assert_eq!(found.len(), 1); + Ok(()) +} diff --git a/examples/system_info_render.rs b/examples/system_info_render.rs new file mode 100644 index 00000000..a6e23aa2 --- /dev/null +++ b/examples/system_info_render.rs @@ -0,0 +1,24 @@ +use worktable::prelude::*; +use worktable::worktable; + +worktable! ( + name: Shown, + columns: { id: u64 primary_key autoincrement, symbol: String, qty: u64 }, + indexes: { symbol_idx: symbol, qty_idx: qty } +); + +#[tokio::main] +async fn main() -> eyre::Result<()> { + let table = ShownWorkTable::default(); + for i in 0..5_000u64 { + table + .insert(ShownRow { + id: table.get_next_pk().into(), + symbol: format!("SYM{}", i % 97), + qty: i, + }) + .await?; + } + print!("{}", table.system_info()); + Ok(()) +} diff --git a/src/features/s3_support.rs b/src/features/s3_support.rs index 6a93ced7..4ec19810 100644 --- a/src/features/s3_support.rs +++ b/src/features/s3_support.rs @@ -1,8 +1,9 @@ -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; +use alloc::{string::String, string::ToString}; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; +use core::time::Duration; use std::path::Path; -use std::time::Duration; use reqwest::Client; use rusty_s3::{Bucket, Credentials, S3Action, UrlStyle}; @@ -135,7 +136,7 @@ where tracing::debug!(local_path = %local_path.display(), s3_key = %s3_key, "Uploading file to S3"); - let content = tokio::fs::read(local_path).await?; + let content = crate::fsx::read(local_path).await?; let action = self.bucket.put_object(Some(&self.credentials), &s3_key); let url = action.sign(Duration::from_secs(3600)); @@ -191,7 +192,7 @@ where return Ok(()); } - tokio::fs::create_dir_all(table_path).await?; + crate::fsx::create_dir_all(table_path).await?; for obj in parsed.contents { let s3_key = &obj.key; @@ -213,7 +214,7 @@ where let response = client.get(url).send().await?.error_for_status()?; let content = response.bytes().await?; - tokio::fs::write(&local_path, content).await?; + crate::fsx::write(&local_path, content).await?; } tracing::info!(table_name = %table_name, "S3 download sync complete"); diff --git a/src/fsx.rs b/src/fsx.rs new file mode 100644 index 00000000..cccab9f0 --- /dev/null +++ b/src/fsx.rs @@ -0,0 +1,57 @@ +//! The filesystem, without a runtime attached to it. +//! +//! A thin shim over [`nagoya::io`], which is where these operations live now. +//! They were here first, privately, and `data_bucket` could not name them at +//! all, so the two halves of one storage engine disagreed about what a file +//! was. The whole crate still goes through this module, so swapping backends +//! is still swapping one file. +//! +//! # Why the calls block +//! +//! Neither `tokio::fs` nor `async-fs` performs asynchronous file I/O: both hand +//! a blocking `std::fs` call to a thread pool, and what that buys is not +//! occupying a runtime worker rather than any actual overlap. It is not free. +//! Measured on this crate's scattered-update path, `tokio::fs` ran 12,316 rows +//! per second against 74,728 for the same code on blocking `std::fs`, and cold +//! reopen is 2.8x faster on an Arctic index without it. +//! +//! Because the calls block, the persistence engine should own a thread rather +//! than share a runtime's worker pool. It was never waiting on the disk through +//! a runtime anyway: the path measures 89 voluntary context switches across +//! 25,000 inserts. + +/// A file this crate reads and writes. +pub type File = nagoya::io::HostFile; + +pub use nagoya::io::{ + Error, SeekFrom, append, create, create_dir_all, open, open_or_create, read, remove_dir_all, remove_file, rename, + write, +}; + +/// How long the file at `path` is, without opening it. +/// +/// Named for what it does. `nagoya::io::metadata` returns the length rather +/// than a metadata handle, because a length is all anything here ever wanted. +pub use nagoya::io::metadata; + +/// The operations no I/O trait carries, as free functions. +/// +/// They are methods on [`nagoya::io::File`]; these wrappers exist so call sites +/// read the same as they did when this module owned the implementation, and so +/// that a backend swap stays a change to one file. +pub async fn sync_all(file: &mut File) -> Result<(), Error> { + nagoya::io::File::sync_all(file).await +} + +pub async fn sync_data(file: &mut File) -> Result<(), Error> { + nagoya::io::File::sync_data(file).await +} + +pub async fn set_len(file: &mut File, length: u64) -> Result<(), Error> { + nagoya::io::File::set_length(file, length).await +} + +/// How long an already-open file is. +pub async fn file_metadata(file: &mut File) -> Result { + nagoya::io::File::length(file).await +} diff --git a/src/in_memory/data.rs b/src/in_memory/data.rs index 7b15d55e..94c4c590 100644 --- a/src/in_memory/data.rs +++ b/src/in_memory/data.rs @@ -1,8 +1,9 @@ -use std::cell::UnsafeCell; -use std::fmt::Debug; -use std::marker::PhantomData; -use std::ops::{Deref, DerefMut}; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use alloc::vec::Vec; +use core::cell::UnsafeCell; +use core::fmt::Debug; +use core::marker::PhantomData; +use core::ops::{Deref, DerefMut}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use data_bucket::page::INNER_PAGE_SIZE; use data_bucket::page::PageId; @@ -37,7 +38,7 @@ struct CellLocks { impl Default for CellLocks { fn default() -> Self { Self { - slots: std::array::from_fn(|_| AtomicU64::new(0)), + slots: core::array::from_fn(|_| AtomicU64::new(0)), } } } @@ -59,10 +60,10 @@ impl CellLocks { #[inline] fn wait(spins: &mut u32) { if *spins < 64 { - std::hint::spin_loop(); + core::hint::spin_loop(); *spins += 1; } else { - std::thread::yield_now(); + crate::util::yield_now(); } } @@ -497,7 +498,7 @@ impl Data { // Use ptr::copy for overlapping memory regions (safe for shifting left) // When moving left (dst_offset < src_offset), this works correctly unsafe { - std::ptr::copy( + core::ptr::copy( inner_data.as_ptr().add(src_offset), inner_data.as_mut_ptr().add(dst_offset), length, @@ -567,10 +568,12 @@ impl Data { .map_err(|_| ExecutionError::LiveCellCountUnderflow) } + #[cfg(feature = "std")] pub(crate) fn has_live_cells(&self) -> bool { self.live_cells.load(Ordering::Acquire) != 0 } + #[cfg(feature = "std")] pub(crate) fn live_cell_count(&self) -> u32 { self.live_cells.load(Ordering::Acquire) } @@ -605,8 +608,9 @@ pub enum ExecutionError { #[cfg(test)] mod tests { - use std::sync::atomic::Ordering; - use std::sync::{Arc, mpsc}; + use alloc::sync::Arc; + use core::sync::atomic::Ordering; + use std::sync::mpsc; use std::thread; use rkyv::{Archive, Deserialize, Serialize}; diff --git a/src/in_memory/empty_link_registry.rs b/src/in_memory/empty_link_registry.rs index 825c7ccc..1bbab632 100644 --- a/src/in_memory/empty_link_registry.rs +++ b/src/in_memory/empty_link_registry.rs @@ -1,5 +1,6 @@ -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use data_bucket::Link; use data_bucket::page::PageId; @@ -65,13 +66,13 @@ impl IndexOrdLink { } impl PartialOrd for IndexOrdLink { - fn partial_cmp(&self, other: &Self) -> Option { + fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl Ord for IndexOrdLink { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { self.absolute_index().cmp(&other.absolute_index()) } } @@ -138,7 +139,7 @@ pub struct EmptyLinkRegistry { /// reclamation, and the most valuable. A scattered single-row delete says /// nothing about where to look, so [`Self::push`] does not record /// anything and only [`Self::push_many`] does. - targeted_pages: FairMutex>, + targeted_pages: FairMutex>, } /// A [`Link`] popped from the registry, together with the read guard that @@ -418,11 +419,11 @@ impl EmptyLinkRegistry { /// ranged delete emptied part of, and it is where a sweep should look /// first. fn note_coalesced_pages(&self, links: &[Link], runs: &[IndexOrdLink]) { - let mut links_per_page: std::collections::BTreeMap = Default::default(); + let mut links_per_page: alloc::collections::BTreeMap = Default::default(); for link in links { *links_per_page.entry(link.page_id).or_default() += 1; } - let mut runs_per_page: std::collections::BTreeMap = Default::default(); + let mut runs_per_page: alloc::collections::BTreeMap = Default::default(); for run in runs { *runs_per_page.entry(run.0.page_id).or_default() += 1; } @@ -443,8 +444,8 @@ impl EmptyLinkRegistry { /// Draining rather than reading: a sweep that has taken them is /// responsible for them, and leaving them would make every later sweep /// re-prioritise pages that are already compact. - pub fn take_targeted_pages(&self) -> std::collections::BTreeSet { - std::mem::take(&mut *self.targeted_pages.lock()) + pub fn take_targeted_pages(&self) -> alloc::collections::BTreeSet { + core::mem::take(&mut *self.targeted_pages.lock()) } /// Wakes a parked vacuum when freeing crossed the configured threshold. diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index 08cfeaa1..851f6770 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -1,6 +1,14 @@ +use alloc::collections::VecDeque; +use alloc::sync::Arc; +use alloc::{boxed::Box, vec::Vec}; use arc_swap::ArcSwap; +use core::fmt::Debug; +use core::marker::PhantomData; +use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize}; +use core::sync::atomic::{AtomicU64, Ordering}; use data_bucket::page::PageId; use derive_more::{Display, Error, From}; +use hashbrown::HashSet; use parking_lot::Mutex; use parking_lot::RwLock; #[cfg(feature = "perf_measurements")] @@ -12,14 +20,6 @@ use rkyv::{ ser::{Serializer, allocator::ArenaHandle, sharing::Share}, util::AlignedVec, }; -use std::collections::{HashSet, VecDeque}; -use std::marker::PhantomData; -use std::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize}; -use std::{ - fmt::Debug, - sync::Arc, - sync::atomic::{AtomicU64, Ordering}, -}; use crate::in_memory::empty_link_registry::EmptyLinkRegistry; use crate::prelude::ArchivedRowWrapper; @@ -183,7 +183,7 @@ struct PageDirectoryChunk { impl PageDirectoryChunk { fn new() -> Self { Self { - pages: std::array::from_fn(|_| AtomicPtr::new(std::ptr::null_mut())), + pages: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())), } } } @@ -203,7 +203,7 @@ struct PageDirectory { impl PageDirectory { fn new(pages: &[Arc]) -> Self { let directory = Self { - roots: std::array::from_fn(|_| AtomicPtr::new(std::ptr::null_mut())), + roots: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())), chunks: Mutex::new(Vec::new()), }; for (index, page) in pages.iter().enumerate() { @@ -223,7 +223,7 @@ impl PageDirectory { chunk = root.load(Ordering::Acquire); if chunk.is_null() { chunks.push(Box::new(PageDirectoryChunk::new())); - chunk = std::ptr::from_ref::>( + chunk = core::ptr::from_ref::>( chunks.last().expect("the chunk was just appended").as_ref(), ) .cast_mut(); @@ -466,7 +466,7 @@ where /// thread can later collect it; it executes only after every reader /// pinned right now has unpinned. fn retire(&self, item: Retired) { - self.retire_many(std::iter::once(item)); + self.retire_many(core::iter::once(item)); } /// Queue several retired items behind one grace marker. @@ -1228,16 +1228,19 @@ where Ok(()) } + #[cfg(feature = "std")] pub(crate) fn page_has_cells(&self, page_id: PageId) -> Result { let page = self.page_ref(page_id)?; Ok(page.has_live_cells()) } + #[cfg(feature = "std")] pub(crate) fn page_live_cell_count(&self, page_id: PageId) -> Result { let page = self.page_ref(page_id)?; Ok(page.live_cell_count()) } + #[cfg(feature = "std")] pub(crate) fn set_loaded_row_count(&self, count: usize) -> Result<(), ExecutionError> { let count = u64::try_from(count).map_err(|_| ExecutionError::RowCountOverflow)?; self.row_count.store(count, Ordering::Release); @@ -1246,6 +1249,7 @@ where /// Completes the vacuum's source-side accounting after every index has /// been swung to the destination link. + #[cfg(feature = "std")] pub(crate) fn remove_moved_cell(&self, link: Link) -> Result<(), ExecutionError> { self.remove_cell(link) } @@ -1277,6 +1281,7 @@ where /// concurrent low-level mutation may access either physical row while the /// move is in progress. After success, the caller must swing every index /// reference to the returned link before retiring `from_link`. + #[cfg(feature = "std")] pub(crate) unsafe fn move_row_for_vacuum( &self, from_link: Link, @@ -1347,7 +1352,7 @@ where /// Heap bytes reserved by the fixed-size data-page allocations. pub fn allocated_bytes(&self) -> usize { - self.pages.len() * std::mem::size_of::::WrappedRow, DATA_LENGTH>>() + self.pages.len() * core::mem::size_of::::WrappedRow, DATA_LENGTH>>() } /// Pages allocated but currently on the empty list, so reusable without @@ -1391,6 +1396,7 @@ where /// current page serves as the sweep's first destination. Concurrent /// inserts are safe: the insert path rechecks `current_page_id` under the /// page barrier before writing and retries if the target changed. + #[cfg(feature = "std")] pub(crate) fn rotate_current_for_vacuum(&self, page_id: PageId) { debug_assert!( self.get_page(page_id).is_some(), @@ -1437,12 +1443,12 @@ impl ExecutionError { #[cfg(test)] mod tests { - use std::collections::HashSet; - use std::sync::Arc; - use std::sync::atomic::Ordering; + use alloc::sync::Arc; + use core::sync::atomic::Ordering; + use core::time::Duration; + use hashbrown::HashSet; use std::sync::mpsc; use std::thread; - use std::time::Duration; use std::time::Instant; use parking_lot::RwLock; @@ -1716,7 +1722,7 @@ mod tests { impl Drop for RemoteReader { fn drop(&mut self) { let (disconnected, _rx) = mpsc::channel(); - let _ = std::mem::replace(&mut self.commands, disconnected); + let _ = core::mem::replace(&mut self.commands, disconnected); if let Some(thread) = self.thread.take() { thread.join().unwrap(); } diff --git a/src/in_memory/row.rs b/src/in_memory/row.rs index 5b38803b..38066fff 100644 --- a/src/in_memory/row.rs +++ b/src/in_memory/row.rs @@ -1,5 +1,5 @@ +use core::fmt::Debug; use rkyv::Archive; -use std::fmt::Debug; pub trait PublicationSafe: Send + Sync + 'static {} diff --git a/src/index/arctic.rs b/src/index/arctic.rs index 42336285..9ea8dcb3 100644 --- a/src/index/arctic.rs +++ b/src/index/arctic.rs @@ -1,10 +1,11 @@ //! Arctic adapter for memory-only unique WorkTable indexes. -use std::borrow::Borrow; -use std::fmt::{self, Debug}; -use std::marker::PhantomData; -use std::ops::{Bound, RangeBounds}; -use std::sync::atomic::{AtomicUsize, Ordering}; +use alloc::{string::String, vec::Vec}; +use core::borrow::Borrow; +use core::fmt::{self, Debug}; +use core::marker::PhantomData; +use core::ops::{Bound, RangeBounds}; +use core::sync::atomic::{AtomicUsize, Ordering}; use arctic::{ConcurrentMap, Key, Order}; @@ -327,6 +328,7 @@ where self.inner.allocated_node_bytes() } + #[cfg(feature = "std")] pub(crate) fn export_topology( &mut self, mut encode: impl FnMut(&V) -> T, @@ -337,6 +339,7 @@ where self.inner.export_topology(|value| encode(&V::from_arctic(*value))) } + #[cfg(feature = "std")] pub(crate) fn from_topology( topology: arctic::topology::Topology, mut decode: impl FnMut(T) -> V, @@ -481,8 +484,9 @@ where #[cfg(test)] mod tests { - use std::ops::Bound; - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use core::ops::Bound; + use std::sync::Barrier; use super::{ArcticIndex, UniqueIndex}; diff --git a/src/index/arctic_multi.rs b/src/index/arctic_multi.rs index 744eba81..f9fc346c 100644 --- a/src/index/arctic_multi.rs +++ b/src/index/arctic_multi.rs @@ -45,10 +45,11 @@ //! the dead entry and retries with a fresh slot, and the SMR guard it holds //! keeps the memory valid throughout. -use std::borrow::Borrow; -use std::fmt::{self, Debug}; -use std::ops::{Bound, ControlFlow, RangeBounds}; -use std::sync::atomic::{AtomicUsize, Ordering}; +use alloc::{boxed::Box, vec::Vec}; +use core::borrow::Borrow; +use core::fmt::{self, Debug}; +use core::ops::{Bound, ControlFlow, RangeBounds}; +use core::sync::atomic::{AtomicUsize, Ordering}; use arctic::{ConcurrentMap, Key as ArcticNativeKey, Order}; use parking_lot::RwLock; @@ -190,7 +191,7 @@ where /// Returns every `(key, value)` pair stored under `key`, in insertion /// order, as a stable snapshot. An unknown key yields an empty iterator. - pub fn get(&self, key: &K) -> std::vec::IntoIter<(K, V)> { + pub fn get(&self, key: &K) -> alloc::vec::IntoIter<(K, V)> { let raw = key.to_arctic(); let Some(slot) = self.inner.get(raw.borrow()) else { return Vec::new().into_iter(); @@ -222,7 +223,7 @@ where let mut slots = 0; while let Some((_, slot)) = entries.lend() { let links = slot.read(); - slots += std::mem::size_of::>>() + links.links.capacity() * std::mem::size_of::(); + slots += core::mem::size_of::>>() + links.links.capacity() * core::mem::size_of::(); } self.inner.allocated_node_bytes() + slots } @@ -287,8 +288,9 @@ where #[cfg(test)] mod tests { - use std::ops::Bound; - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use core::ops::Bound; + use std::sync::Barrier; use super::ArcticMultiIndex; diff --git a/src/index/available_index.rs b/src/index/available_index.rs index 32b18991..a9b3e4ac 100644 --- a/src/index/available_index.rs +++ b/src/index/available_index.rs @@ -1,3 +1,4 @@ +use alloc::{string::String, string::ToString}; pub trait AvailableIndex { fn to_string_value(&self) -> String; } diff --git a/src/index/congee.rs b/src/index/congee.rs index e564c754..42ba8a6d 100644 --- a/src/index/congee.rs +++ b/src/index/congee.rs @@ -1,9 +1,10 @@ //! Congee adapter for memory-only unique WorkTable indexes. -use std::fmt::{self, Debug}; -use std::ops::{Bound, RangeBounds}; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::{self, Debug}; +use core::ops::{Bound, RangeBounds}; +use core::sync::atomic::{AtomicUsize, Ordering}; use congee::{CongeeRaw, DefaultAllocator}; use parking_lot::Mutex; @@ -53,7 +54,7 @@ pub struct CongeeIndex { // serialize mutations until the backend offers the required visibility. mutation: Mutex<()>, len: AtomicUsize, - marker: std::marker::PhantomData<(K, V)>, + marker: core::marker::PhantomData<(K, V)>, } impl Debug for CongeeIndex { @@ -79,7 +80,7 @@ where inner: CongeeRaw::new_with_drainer(DefaultAllocator {}, drainer), mutation: Mutex::new(()), len: AtomicUsize::new(0), - marker: std::marker::PhantomData, + marker: core::marker::PhantomData, } } } @@ -113,7 +114,7 @@ where // SAFETY: callers guarantee that `pointer` was produced by // `Arc::into_raw(...).expose_provenance()` for the same `V` and still // owns one strong reference. - unsafe { Arc::from_raw(std::ptr::with_exposed_provenance(pointer)) } + unsafe { Arc::from_raw(core::ptr::with_exposed_provenance(pointer)) } } #[inline] @@ -180,12 +181,13 @@ where .map(|(key, pointer)| { // SAFETY: the pinned epoch keeps every returned tree-owned // pointer alive until its value has been cloned. - let value = unsafe { &*std::ptr::with_exposed_provenance::(pointer) }; + let value = unsafe { &*core::ptr::with_exposed_provenance::(pointer) }; (K::from_congee(key), value.clone()) }) .collect() } + #[cfg(feature = "std")] pub(crate) fn export_topology( &mut self, mut encode: impl FnMut(&V) -> T, @@ -193,10 +195,11 @@ where self.inner.export_topology(|pointer| { // SAFETY: every raw payload is a live tree-owned `Arc` pointer, // and the exclusive borrow prevents removal while it is cloned. - unsafe { encode(&*std::ptr::with_exposed_provenance::(pointer)) } + unsafe { encode(&*core::ptr::with_exposed_provenance::(pointer)) } }) } + #[cfg(feature = "std")] pub(crate) fn from_topology( topology: congee::topology::Topology, mut decode: impl FnMut(T) -> V, @@ -217,7 +220,7 @@ where inner, mutation: Mutex::new(()), len: AtomicUsize::new(len), - marker: std::marker::PhantomData, + marker: core::marker::PhantomData, }) } } @@ -238,7 +241,7 @@ where let pointer = self.inner.get(&key.into_congee(), &guard)?; // SAFETY: the epoch guard keeps the tree-owned `Arc` alive for the // duration of `read`, and the pointer originated from `Arc::into_raw`. - let value = unsafe { &*std::ptr::with_exposed_provenance::(pointer) }; + let value = unsafe { &*core::ptr::with_exposed_provenance::(pointer) }; Some(read(value)) } @@ -334,8 +337,9 @@ where #[cfg(test)] mod tests { - use std::ops::Bound; - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use core::ops::Bound; + use std::sync::Barrier; use super::{CongeeIndex, UniqueIndex}; diff --git a/src/index/mod.rs b/src/index/mod.rs index c36970cd..22456318 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -23,13 +23,15 @@ pub use persistent_art::{ }; pub use persistent_wti::PersistentWtiIndex; pub use primary_index::PrimaryIndex; -pub use table_index::{ - TableIndex, TableIndexCdc, convert_change_events, convert_multi_change_events, convert_upstream_change_events, -}; +#[cfg(feature = "vanilla-index")] +pub use table_index::convert_upstream_change_events; +pub use table_index::{TableIndex, TableIndexCdc, convert_change_events, convert_multi_change_events}; pub use table_secondary_index::{ IndexError, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, }; -pub use unique::{UniqueIndex, UpstreamIndexMap, UpstreamIndexPair}; +pub use unique::UniqueIndex; +#[cfg(feature = "vanilla-index")] +pub use unique::{UpstreamIndexMap, UpstreamIndexPair}; pub use unsized_node::UnsizedNode; #[derive(Clone, Debug)] diff --git a/src/index/persistent_art.rs b/src/index/persistent_art.rs index eda5a690..2f73d88b 100644 --- a/src/index/persistent_art.rs +++ b/src/index/persistent_art.rs @@ -6,12 +6,13 @@ //! different stripes remain concurrent because their Set/Remove records //! commute during recovery. -use std::array; -use std::collections::hash_map::DefaultHasher; -use std::fmt::{self, Debug}; -use std::hash::{Hash, Hasher}; -use std::ops::RangeBounds; -use std::sync::atomic::{AtomicU64, Ordering}; +use alloc::vec::Vec; +use core::array; +use core::fmt::{self, Debug}; +use core::hash::{Hash, Hasher}; +use core::ops::RangeBounds; +use core::sync::atomic::{AtomicU64, Ordering}; +use rustc_hash::FxHasher as DefaultHasher; use data_bucket::Link; use indexset::cdc::change::{ChangeEvent, Id}; @@ -49,7 +50,7 @@ where K: ArcticKey, V: Clone + Debug + PartialEq + Send + Sync + 'static, { - pub fn get(&self, key: &K) -> std::vec::IntoIter<(K, V)> { + pub fn get(&self, key: &K) -> alloc::vec::IntoIter<(K, V)> { self.inner.get(key) } @@ -128,7 +129,7 @@ impl PersistentArtIndex { } fn mutation_stripe(&self, key: &K) -> &Mutex<()> { - let mut hasher = DefaultHasher::new(); + let mut hasher = DefaultHasher::default(); key.hash(&mut hasher); &self.mutation_stripes[hasher.finish() as usize % MUTATION_STRIPES] } @@ -318,7 +319,8 @@ where #[cfg(test)] mod tests { - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use std::sync::Barrier; use super::*; diff --git a/src/index/persistent_wti.rs b/src/index/persistent_wti.rs index 8c05edde..b8e629d8 100644 --- a/src/index/persistent_wti.rs +++ b/src/index/persistent_wti.rs @@ -9,11 +9,12 @@ //! not claims about the live WTI node position or maximum; the shadow validates //! that marker and derives the real structural metadata itself. -use std::array; -use std::fmt::{self, Debug}; -use std::hash::{Hash, Hasher}; -use std::ops::RangeBounds; -use std::sync::atomic::{AtomicU64, Ordering}; +use alloc::vec::Vec; +use core::array; +use core::fmt::{self, Debug}; +use core::hash::{Hash, Hasher}; +use core::ops::RangeBounds; +use core::sync::atomic::{AtomicU64, Ordering}; use data_bucket::Link; use indexset::cdc::change::{ChangeEvent, Id}; @@ -285,7 +286,7 @@ where #[cfg(test)] mod tests { - use std::collections::HashSet; + use hashbrown::HashSet; use data_bucket::page::PageId; diff --git a/src/index/primary_index.rs b/src/index/primary_index.rs index d8883498..49969579 100644 --- a/src/index/primary_index.rs +++ b/src/index/primary_index.rs @@ -1,8 +1,9 @@ //! Primary-key to row-location index. -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; use data_bucket::Link; use indexset::cdc::change::ChangeEvent; diff --git a/src/index/table_index/cdc.rs b/src/index/table_index/cdc.rs index 0cb4a588..e8658a06 100644 --- a/src/index/table_index/cdc.rs +++ b/src/index/table_index/cdc.rs @@ -1,18 +1,26 @@ -use std::fmt::Debug; -use std::hash::Hash; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::Hash; use data_bucket::Link; use indexset::cdc::change::ChangeEvent; use indexset::core::multipair::MultiPair; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::concurrent::map::BTreeMap as VanillaIndexMap; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; -use crate::index::table_index::util::{convert_change_events, convert_upstream_change_events}; +#[cfg(feature = "vanilla-index")] +use crate::UpstreamIndexMap; +use crate::index::table_index::util::convert_change_events; +#[cfg(feature = "vanilla-index")] +use crate::index::table_index::util::convert_upstream_change_events; use crate::util::OffsetEqLink; -use crate::{ArcticIndex, ArcticKey, CongeeIndex, CongeeKey, IndexMap, IndexMultiMap, UniqueIndex, UpstreamIndexMap}; +use crate::{ArcticIndex, ArcticKey, CongeeIndex, CongeeKey, IndexMap, IndexMultiMap, UniqueIndex}; pub trait TableIndexCdc { fn insert_cdc(&self, value: T, link: Link) -> (Option, Vec>>); @@ -74,6 +82,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl TableIndexCdc for UpstreamIndexMap, Node> where T: Debug + Eq + Hash + Clone + Send + Ord, diff --git a/src/index/table_index/mod.rs b/src/index/table_index/mod.rs index a405b5d3..8a40d6ed 100644 --- a/src/index/table_index/mod.rs +++ b/src/index/table_index/mod.rs @@ -1,24 +1,32 @@ -use std::fmt::Debug; -use std::hash::Hash; +use core::fmt::Debug; +use core::hash::Hash; use data_bucket::Link; use indexset::core::multipair::MultiPair; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; +#[cfg(feature = "vanilla-index")] +use crate::UpstreamIndexMap; use crate::util::OffsetEqLink; +#[allow(unused_imports)] +use crate::{}; use crate::{ ArcticIndex, ArcticKey, ArcticMultiIndex, CongeeIndex, CongeeKey, IndexMap, IndexMultiMap, PersistentArcticIndex, - PersistentArcticMultiIndex, PersistentCongeeIndex, PersistentWtiIndex, UniqueIndex, UpstreamIndexMap, + PersistentArcticMultiIndex, PersistentCongeeIndex, PersistentWtiIndex, UniqueIndex, }; mod cdc; pub mod util; pub use cdc::TableIndexCdc; -pub use util::{convert_change_events, convert_multi_change_events, convert_upstream_change_events}; +#[cfg(feature = "vanilla-index")] +pub use util::convert_upstream_change_events; +pub use util::{convert_change_events, convert_multi_change_events}; pub trait TableIndex { fn insert(&self, value: T, link: Link) -> Option; @@ -114,6 +122,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl TableIndex for UpstreamIndexMap where T: Debug + Eq + Hash + Clone + Send + Ord, diff --git a/src/index/table_index/util.rs b/src/index/table_index/util.rs index 4ffc57cb..e056db39 100644 --- a/src/index/table_index/util.rs +++ b/src/index/table_index/util.rs @@ -1,7 +1,10 @@ +use alloc::vec::Vec; use indexset::cdc::change::ChangeEvent; use indexset::core::multipair::MultiPair; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::cdc::change::ChangeEvent as VanillaChangeEvent; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; pub fn convert_change_event(ev: ChangeEvent>) -> ChangeEvent> @@ -142,6 +145,7 @@ where /// Normalizes upstream IndexSet CDC events into WorkTablesIndex's event type, /// which remains the stable persistence boundary used by DataBucket. +#[cfg(feature = "vanilla-index")] pub fn convert_upstream_change_events( evs: Vec>>, ) -> Vec>> @@ -193,6 +197,7 @@ where .collect() } +#[cfg(feature = "vanilla-index")] fn upstream_pair(pair: VanillaPair) -> Pair where L1: Into, diff --git a/src/index/table_secondary_index/cdc.rs b/src/index/table_secondary_index/cdc.rs index 804014d8..0b7914da 100644 --- a/src/index/table_secondary_index/cdc.rs +++ b/src/index/table_secondary_index/cdc.rs @@ -1,4 +1,5 @@ -use std::collections::HashMap; +use alloc::vec::Vec; +use hashbrown::HashMap; use data_bucket::Link; diff --git a/src/index/table_secondary_index/index_events.rs b/src/index/table_secondary_index/index_events.rs index 9183da9b..acdf0a9f 100644 --- a/src/index/table_secondary_index/index_events.rs +++ b/src/index/table_secondary_index/index_events.rs @@ -1,6 +1,6 @@ use crate::prelude::IndexChangeEventId; +use hashbrown::HashMap; use indexset::cdc::change; -use std::collections::HashMap; pub trait TableSecondaryIndexEventsOps { fn extend(&mut self, another: Self) diff --git a/src/index/table_secondary_index/info.rs b/src/index/table_secondary_index/info.rs index 569a4953..6da1dfdc 100644 --- a/src/index/table_secondary_index/info.rs +++ b/src/index/table_secondary_index/info.rs @@ -1,4 +1,5 @@ use crate::prelude::IndexInfo; +use alloc::vec::Vec; pub trait TableSecondaryIndexInfo { fn index_info(&self) -> Vec; diff --git a/src/index/table_secondary_index/mod.rs b/src/index/table_secondary_index/mod.rs index 69434a82..415e4c61 100644 --- a/src/index/table_secondary_index/mod.rs +++ b/src/index/table_secondary_index/mod.rs @@ -1,9 +1,10 @@ +use alloc::vec::Vec; mod cdc; mod index_events; mod info; use data_bucket::Link; -use std::collections::HashMap; +use hashbrown::HashMap; use crate::WorkTableError; use crate::{AvailableIndex, Difference}; diff --git a/src/index/unique.rs b/src/index/unique.rs index 55e559f3..a6f63724 100644 --- a/src/index/unique.rs +++ b/src/index/unique.rs @@ -4,15 +4,23 @@ //! backend's guard type. That keeps generated code independent from the //! concurrency and reclamation strategy used by each index implementation. -use std::fmt::Debug; -use std::hash::Hash; -use std::ops::RangeBounds; +// Only `UpstreamIndexMap`'s default node type and the tests name `Vec`, and the +// first of those is behind `vanilla-index`. Ungated, this warns on every +// `--no-default-features` build, which is the build that has to stay quiet. +#[cfg(any(feature = "vanilla-index", test))] +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::Hash; +use core::ops::RangeBounds; use crate::IndexMap; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::concurrent::map::BTreeMap as VanillaIndexMap; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; /// Point, mutation, and ordered-scan operations used by generated unique @@ -131,6 +139,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl UniqueIndex for VanillaIndexMap where K: Debug + Eq + Hash + Clone + Send + Ord + 'static, @@ -198,15 +207,16 @@ where self.range(range).map(|(_, value)| value.clone()) } } - /// Vanilla upstream IndexSet map, kept distinct from WorkTable's default /// WorkTablesIndex alias so both implementations may coexist in one binary. +#[cfg(feature = "vanilla-index")] pub type UpstreamIndexMap>> = VanillaIndexMap; +#[cfg(feature = "vanilla-index")] pub type UpstreamIndexPair = VanillaPair; #[cfg(test)] mod tests { - use std::sync::Arc; + use alloc::sync::Arc; use super::{UniqueIndex, UpstreamIndexMap}; use crate::{ArcticIndex, CongeeIndex, IndexMap}; @@ -260,7 +270,7 @@ mod tests { let iterated = index.iter_values().find(|(candidate, _)| *candidate == key); panic!( "backend={}, key={key}, point={value:?}, iterated={iterated:?}", - std::any::type_name::(), + core::any::type_name::(), ); } } @@ -292,7 +302,7 @@ mod tests { threads.push(std::thread::spawn(move || { for sequence in 0..1_000_u64 { let key = worker * 1_000 + sequence; - let backend = std::any::type_name::(); + let backend = core::any::type_name::(); assert_eq!( index.insert_value_checked(key, key + 1), Some(()), diff --git a/src/index/unsized_node.rs b/src/index/unsized_node.rs index eefd0c8c..3214cada 100644 --- a/src/index/unsized_node.rs +++ b/src/index/unsized_node.rs @@ -1,11 +1,12 @@ +use alloc::vec::Vec; use data_bucket::{SizeMeasurable, UnsizedIndexPageUtility, VariableSizeMeasurable}; use indexset::core::node::NodeLike; -use std::borrow::Borrow; -use std::collections::Bound; -use std::fmt::Debug; -use std::ops::Deref; -use std::slice::Iter; +use core::borrow::Borrow; +use core::fmt::Debug; +use core::ops::Bound; +use core::ops::Deref; +use core::slice::Iter; pub const UNSIZED_HEADER_LENGTH: u32 = 64; @@ -238,7 +239,7 @@ where fn replace(&mut self, idx: usize, value: T) -> Option { let value_size = value.aligned_size(); if let Some(old) = self.inner.get_mut(idx) { - let old = std::mem::replace(old, value); + let old = core::mem::replace(old, value); self.length += value_size; self.removed_length += old.aligned_size(); if idx + 1 == self.inner.len() { diff --git a/src/lib.rs b/src/lib.rs index 173b172f..972e1d62 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,20 @@ +#![cfg_attr(not(feature = "std"), no_std)] #![doc = include_str!("../docs/crate.md")] +#[macro_use] +extern crate alloc; + +/// Generated code names `worktable::` paths, which must also resolve inside +/// this crate, where `worktable!` is invoked for the persistence queue. +extern crate self as worktable; + +#[cfg(feature = "std")] +pub mod fsx; pub mod in_memory; mod index; pub mod lock; mod mem_stat; +#[cfg(feature = "std")] pub mod migration; pub mod partition; pub mod persistence; @@ -16,6 +27,7 @@ mod util; pub mod features; pub use index::*; +#[cfg(feature = "std")] pub use persistence::{ LoadMode, PersistedWorkTable, PersistenceConfig, PersistenceLoadError, UnloadFailure, UnloadReport, }; @@ -34,38 +46,56 @@ pub use worktable_dsl; pub use worktable_codegen::s3_sync_persistence; pub mod prelude { + /// The filesystem this crate goes through. Generated code opens files by + /// this path, so a consumer of `worktable!` gets the same backend the + /// crate itself uses without naming it. + #[cfg(feature = "std")] + pub use crate::fsx; + pub use alloc::collections::BTreeMap; + pub use alloc::sync::Arc; + pub use alloc::vec::IntoIter; + pub use hashbrown::{HashMap, HashSet}; + pub use crate::in_memory::{ArchivedRowWrapper, Data, DataPages, Query, RowWrapper, StorableRow}; pub use crate::lock::FullRowLock; pub use crate::lock::{Lock, RowLock}; pub use crate::lock::{LockAcquirer, LockGuard, LockMap, PendingLock}; pub use crate::mem_stat::MemStat; pub use crate::partition::{MAX_PARTITIONS, PartRef, PartitionError, PartitionSet}; + pub use crate::persistence::{AcknowledgeOperation, DeleteOperation, InsertOperation, Operation, OperationId}; + #[cfg(feature = "std")] pub use crate::persistence::{ - AcknowledgeOperation, ArtPersistenceKey, DeleteOperation, DiskConfig, DiskPersistenceEngine, - IndexTableOfContents, InsertOperation, LoadMode, Operation, OperationId, PersistedWorkTable, PersistenceConfig, - PersistenceEngine, PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, PersistenceMonitor, - PersistenceResult, PersistenceState, PersistenceTask, ReadOnlyPersistenceEngine, SpaceArcticIndex, - SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, SpaceIndex, - SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, SpaceLogicalIndexUnsized, SpaceLogicalMultiIndex, - SpaceLogicalMultiIndexUnsized, SpaceSecondaryIndexOps, TocEntryOversizedError, UnloadFailure, UnloadReport, - UpdateOperation, load_persisted_state, map_index_pages_to_toc_and_general, - map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, validate_events, + ArtPersistenceKey, DiskConfig, DiskPersistenceEngine, IndexTableOfContents, LoadMode, PersistedWorkTable, + PersistenceConfig, PersistenceEngine, PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, + PersistenceMonitor, PersistenceResult, PersistenceState, PersistenceTask, ReadOnlyPersistenceEngine, + SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, + SpaceIndex, SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, SpaceLogicalIndexUnsized, + SpaceLogicalMultiIndex, SpaceLogicalMultiIndexUnsized, SpaceSecondaryIndexOps, TocEntryOversizedError, + UnloadFailure, UnloadReport, load_persisted_state, map_index_pages_to_toc_and_general, + map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, }; + pub use crate::persistence::{OperationType, UpdateOperation, validate_events}; pub use crate::primary_key::{ PrimaryKeyGenerator, PrimaryKeyGeneratorRange, PrimaryKeyGeneratorState, TablePrimaryKey, }; pub use crate::table::select::{Order, QueryParams, SelectQueryBuilder, SelectQueryExecutor}; pub use crate::table::system_info::{IndexInfo, IndexKind, SystemInfo}; pub use crate::util::{OffsetEqLink, OrderedF32Def, OrderedF64Def}; + #[allow(unused_imports)] + pub use crate::{}; pub use crate::{ ArcticEntry, ArcticIndex, ArcticKey, ArcticMultiIndex, ArcticStringKey, AvailableIndex, BatchDeleteError, BatchInsertError, CongeeIndex, CongeeKey, Difference, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, PersistentArcticIndex, PersistentArcticMultiIndex, PersistentArtIndex, PersistentCongeeIndex, PersistentWtiIndex, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, - UpstreamIndexMap, UpstreamIndexPair, WorkTable, WorkTableError, vacuum::EmptyDataVacuum, - vacuum::VacuumPersistence, vacuum::WorkTableVacuum, validate_arctic_link, + WorkTable, WorkTableError, validate_arctic_link, }; + /// The upstream IndexSet backend, when the `vanilla-index` feature selects it. + #[cfg(feature = "vanilla-index")] + pub use crate::{UpstreamIndexMap, UpstreamIndexPair}; + #[cfg(feature = "std")] + pub use crate::{vacuum::EmptyDataVacuum, vacuum::VacuumPersistence, vacuum::WorkTableVacuum}; pub use data_bucket::{ DATA_VERSION, DataPage, GENERAL_HEADER_SIZE, GeneralHeader, GeneralPage, INNER_PAGE_SIZE, IndexPage, Interval, Link, PAGE_SIZE, PageType, Persistable, PersistableIndex, SizeMeasurable, SizeMeasure, SpaceInfoPage, diff --git a/src/lock/map.rs b/src/lock/map.rs index ea79742e..22961737 100644 --- a/src/lock/map.rs +++ b/src/lock/map.rs @@ -1,10 +1,11 @@ -use std::collections::HashMap; -use std::collections::hash_map::DefaultHasher; -use std::fmt::Debug; -use std::hash::{Hash, Hasher}; -use std::ops::Deref; -use std::sync::Arc; -use std::sync::atomic::{AtomicU16, AtomicU64, AtomicUsize, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::{Hash, Hasher}; +use core::ops::Deref; +use core::sync::atomic::{AtomicU16, AtomicU64, AtomicUsize, Ordering}; +use hashbrown::HashMap; +use rustc_hash::FxHasher as DefaultHasher; use parking_lot::RwLock; @@ -138,7 +139,7 @@ impl Default for LockMap { Self { map: RwLock::new(HashMap::new()), next_id: AtomicU16::default(), - mutation_stripes: Arc::new(std::array::from_fn(|_| MutationStripe::default())), + mutation_stripes: Arc::new(core::array::from_fn(|_| MutationStripe::default())), bulk_mutations: Arc::default(), } } @@ -283,7 +284,7 @@ where } fn stripe_of(key: &PrimaryKey) -> usize { - let mut hasher = DefaultHasher::new(); + let mut hasher = DefaultHasher::default(); key.hash(&mut hasher); (hasher.finish() as usize) % MUTATION_STRIPE_COUNT } @@ -343,9 +344,9 @@ where while gate.serving.load(Ordering::Acquire) != ticket { if spins < 16 { spins += 1; - std::hint::spin_loop(); + core::hint::spin_loop(); } else { - std::thread::yield_now(); + crate::util::yield_now(); } } diff --git a/src/lock/mod.rs b/src/lock/mod.rs index db352d32..6c465e18 100644 --- a/src/lock/mod.rs +++ b/src/lock/mod.rs @@ -1,15 +1,16 @@ +use alloc::vec::Vec; mod map; mod row_lock; -use std::cell::Cell; -use std::fmt::Debug; -use std::future::Future; -use std::hash::{Hash, Hasher}; -use std::marker::PhantomData; -use std::pin::Pin; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::task::{Context, Poll}; +use alloc::sync::Arc; +use core::cell::Cell; +use core::fmt::Debug; +use core::future::Future; +use core::hash::{Hash, Hasher}; +use core::marker::PhantomData; +use core::pin::Pin; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::task::{Context, Poll}; use futures::task::AtomicWaker; use parking_lot::Mutex; @@ -261,7 +262,7 @@ impl Future for LockWait { // Spin phase: try up to MAX_SPINS before going async for _ in 0..MAX_SPINS { - std::hint::spin_loop(); + core::hint::spin_loop(); if !self.locked.load(Ordering::Acquire) { return Poll::Ready(()); } diff --git a/src/lock/row_lock.rs b/src/lock/row_lock.rs index 9e8aff49..e9022790 100644 --- a/src/lock/row_lock.rs +++ b/src/lock/row_lock.rs @@ -1,7 +1,7 @@ -use std::collections::HashSet; -use std::fmt::Debug; -use std::hash::Hash; -use std::sync::Arc; +use alloc::sync::Arc; +use core::fmt::Debug; +use core::hash::Hash; +use hashbrown::HashSet; use crate::lock::{Lock, LockGuard, LockMap, LockWait}; diff --git a/src/mem_stat/mod.rs b/src/mem_stat/mod.rs index 8b45b3db..345d9239 100644 --- a/src/mem_stat/mod.rs +++ b/src/mem_stat/mod.rs @@ -1,9 +1,10 @@ +use alloc::{boxed::Box, string::String, vec::Vec}; mod primitives; -use std::collections::HashMap; -use std::fmt::Debug; -use std::rc::Rc; -use std::sync::Arc; +use alloc::rc::Rc; +use alloc::sync::Arc; +use core::fmt::Debug; +use hashbrown::HashMap; use data_bucket::Link; use data_bucket::page::PageId; @@ -14,16 +15,22 @@ use ordered_float::OrderedFloat; use psc_nanoid::PackedNanoid; use psc_nanoid::packed::AlphabetPackExt; use uuid::Uuid; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; +#[cfg(feature = "vanilla-index")] +use crate::UpstreamIndexMap; use crate::in_memory::{RowWrapper, StorableRow}; use crate::persistence::OperationType; use crate::prelude::OperationId; use crate::util::OffsetEqLink; +#[allow(unused_imports)] +use crate::{}; use crate::{ ArcticIndex, ArcticKey, ArcticMultiIndex, ArcticValue, CongeeIndex, CongeeKey, IndexMultiMap, PersistentArtIndex, - PersistentWtiIndex, UniqueIndex, UpstreamIndexMap, WorkTable, + PersistentWtiIndex, UniqueIndex, WorkTable, }; use crate::{IndexMap, impl_memstat_zero}; @@ -55,7 +62,7 @@ impl< PkMap, > where - PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, PkMap: UniqueIndex> + MemStat, @@ -81,10 +88,10 @@ impl MemStat for Option { impl MemStat for Vec { fn heap_size(&self) -> usize { - self.capacity() * std::mem::size_of::() + self.iter().map(|v| v.heap_size()).sum::() + self.capacity() * core::mem::size_of::() + self.iter().map(|v| v.heap_size()).sum::() } fn used_size(&self) -> usize { - self.len() * std::mem::size_of::() + self.iter().map(|v| v.used_size()).sum::() + self.len() * core::mem::size_of::() + self.iter().map(|v| v.used_size()).sum::() } } @@ -104,7 +111,7 @@ where Node: NodeLike> + Send + 'static, { fn heap_size(&self) -> usize { - let slot_size = std::mem::size_of::>(); + let slot_size = core::mem::size_of::>(); let base_heap = self.capacity() * slot_size; let kv_heap: usize = self.iter().map(|(k, v)| k.heap_size() + v.heap_size()).sum(); @@ -113,7 +120,7 @@ where } fn used_size(&self) -> usize { - let pair_size = std::mem::size_of::>(); + let pair_size = core::mem::size_of::>(); let base = self.len() * pair_size; let used: usize = self.iter().map(|(k, v)| k.used_size() + v.used_size()).sum(); @@ -122,6 +129,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl MemStat for UpstreamIndexMap where K: Debug + Ord + Clone + 'static + MemStat + Send, @@ -129,14 +137,14 @@ where Node: VanillaNodeLike> + Send + 'static, { fn heap_size(&self) -> usize { - let slot_size = std::mem::size_of::>(); + let slot_size = core::mem::size_of::>(); let base_heap = self.capacity() * slot_size; let kv_heap: usize = self.iter().map(|(k, v)| k.heap_size() + v.heap_size()).sum(); base_heap + kv_heap } fn used_size(&self) -> usize { - let pair_size = std::mem::size_of::>(); + let pair_size = core::mem::size_of::>(); let base = self.len() * pair_size; let used: usize = self.iter().map(|(k, v)| k.used_size() + v.used_size()).sum(); base + used @@ -151,7 +159,7 @@ where fn heap_size(&self) -> usize { let values = self.iter_values().map(|(_, value)| value.heap_size()).sum::(); self.allocated_node_bytes() - + self.len() * (std::mem::size_of::() + 2 * std::mem::size_of::()) + + self.len() * (core::mem::size_of::() + 2 * core::mem::size_of::()) + values } @@ -170,7 +178,7 @@ where } fn used_size(&self) -> usize { - self.len() * std::mem::size_of::<(K, V)>() + self.len() * core::mem::size_of::<(K, V)>() } } @@ -184,7 +192,7 @@ where } fn used_size(&self) -> usize { - self.len() * std::mem::size_of::<(K, V)>() + self.len() * core::mem::size_of::<(K, V)>() } } @@ -220,7 +228,7 @@ where Node: NodeLike> + Send + 'static, { fn heap_size(&self) -> usize { - let slot_size = std::mem::size_of::>(); + let slot_size = core::mem::size_of::>(); let base_heap = self.capacity() * slot_size; let kv_heap: usize = self.iter().map(|(k, v)| k.heap_size() + v.heap_size()).sum(); @@ -229,7 +237,7 @@ where } fn used_size(&self) -> usize { - let pair_size = std::mem::size_of::>(); + let pair_size = core::mem::size_of::>(); let base = self.len() * pair_size; let used: usize = self.iter().map(|(k, v)| k.used_size() + v.used_size()).sum(); @@ -240,32 +248,32 @@ where impl MemStat for Box { fn heap_size(&self) -> usize { - std::mem::size_of::() + (**self).heap_size() + core::mem::size_of::() + (**self).heap_size() } fn used_size(&self) -> usize { - std::mem::size_of::() + (**self).used_size() + core::mem::size_of::() + (**self).used_size() } } impl MemStat for Arc { fn heap_size(&self) -> usize { - std::mem::size_of::() + (**self).heap_size() + core::mem::size_of::() + (**self).heap_size() } fn used_size(&self) -> usize { - std::mem::size_of::() + (**self).used_size() + core::mem::size_of::() + (**self).used_size() } } impl MemStat for Rc { fn heap_size(&self) -> usize { - std::mem::size_of::() + (**self).heap_size() + core::mem::size_of::() + (**self).heap_size() } fn used_size(&self) -> usize { - std::mem::size_of::() + (**self).used_size() + core::mem::size_of::() + (**self).used_size() } } -impl MemStat for HashMap { +impl MemStat for HashMap { fn heap_size(&self) -> usize { let bucket_size = size_of::<(K, V)>(); let base_heap = self.capacity() * bucket_size; diff --git a/src/mem_stat/primitives.rs b/src/mem_stat/primitives.rs index ee159f96..348b2484 100644 --- a/src/mem_stat/primitives.rs +++ b/src/mem_stat/primitives.rs @@ -29,23 +29,24 @@ impl_memstat_zero!( char, u128, i128, - std::num::NonZeroU8, - std::num::NonZeroU16, - std::num::NonZeroU32, - std::num::NonZeroU64, - std::num::NonZeroU128, - std::num::NonZeroUsize, - std::num::NonZeroI8, - std::num::NonZeroI16, - std::num::NonZeroI32, - std::num::NonZeroI64, - std::num::NonZeroI128, - std::num::NonZeroIsize, - std::time::Duration, - std::time::SystemTime, - std::time::Instant + core::num::NonZeroU8, + core::num::NonZeroU16, + core::num::NonZeroU32, + core::num::NonZeroU64, + core::num::NonZeroU128, + core::num::NonZeroUsize, + core::num::NonZeroI8, + core::num::NonZeroI16, + core::num::NonZeroI32, + core::num::NonZeroI64, + core::num::NonZeroI128, + core::num::NonZeroIsize, + core::time::Duration ); +#[cfg(feature = "std")] +impl_memstat_zero!(std::time::SystemTime, std::time::Instant); + impl_memstat_zero!( [u8], [i8], diff --git a/src/migration/mod.rs b/src/migration/mod.rs index 5074ed81..ca4e34dc 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use rkyv::api::high::HighDeserializer; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -5,7 +6,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize}; -use tokio::fs::File; use crate::prelude::{GeneralPage, Persistable, SpaceInfoPage, WT_DATA_EXTENSION, parse_page}; @@ -24,7 +24,7 @@ where SpaceInfoPage: Persistable, { let data_file_path = format!("{}/{}", table_path, WT_DATA_EXTENSION); - let mut file = File::open(&data_file_path).await?; - let info: GeneralPage> = parse_page::<_, 4096>(&mut file, 0).await?; + let mut file = crate::fsx::open(&data_file_path).await?; + let info: GeneralPage> = parse_page::<_, 4096, DEFAULT_PAGE_STRIDE>(&mut file, 0).await?; Ok(info.inner.version) } diff --git a/src/partition/mod.rs b/src/partition/mod.rs index fdc2779b..bef57a9a 100644 --- a/src/partition/mod.rs +++ b/src/partition/mod.rs @@ -55,22 +55,23 @@ //! instance measures 110 KB and 6.1 ms to construct, of which 95 percent is //! inside `PersistenceEngine::new`. +use alloc::{boxed::Box, vec::Vec}; // Under `--cfg wt_loom` the atomics and the mutex come from loom, which explores // every interleaving of them rather than whichever one this machine happened // to produce. `Arc` stays `std`: loom's has no `into_raw` or // `increment_strong_count`, and std's own atomics are already model-checked // upstream. What loom is being asked about here is the slot protocol and the // double-checked lock in `get_or_create`, not reference counting. +use alloc::collections::VecDeque; +use alloc::sync::Arc; +#[cfg(not(wt_loom))] +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; #[cfg(wt_loom)] use loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; #[cfg(wt_loom)] use loom::sync::{Mutex, MutexGuard}; #[cfg(not(wt_loom))] use parking_lot::{Mutex, MutexGuard}; -use std::collections::VecDeque; -use std::sync::Arc; -#[cfg(not(wt_loom))] -use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use crate::mem_stat::MemStat; #[cfg(not(wt_loom))] @@ -109,7 +110,7 @@ struct Chunk { impl Chunk { fn empty() -> Box { Box::new(Chunk { - slots: std::array::from_fn(|_| AtomicPtr::new(std::ptr::null_mut())), + slots: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())), }) } } @@ -144,7 +145,7 @@ pub struct PartRef<'a, T> { value: &'a T, } -impl std::ops::Deref for PartRef<'_, T> { +impl core::ops::Deref for PartRef<'_, T> { type Target = T; fn deref(&self) -> &T { @@ -180,7 +181,7 @@ impl Default for PartitionSet { impl PartitionSet { pub fn new() -> Self { Self { - spine: (0..MAX_CHUNKS).map(|_| AtomicPtr::new(std::ptr::null_mut())).collect(), + spine: (0..MAX_CHUNKS).map(|_| AtomicPtr::new(core::ptr::null_mut())).collect(), live: AtomicUsize::new(0), grow: Mutex::new(VecDeque::new()), #[cfg(not(wt_loom))] @@ -293,7 +294,7 @@ impl PartitionSet { /// So a tick loop should not pin per lookup. Pin once, read many: /// /// ``` - /// # fn main() { futures::executor::block_on(async { + /// # fn main() { nagoya::block_on(async { /// use worktable::prelude::*; /// use worktable::worktable; /// @@ -488,7 +489,7 @@ impl PartitionSet { let table = { let mut retired = self.lock(); let chunk = self.chunk(idx)?; - let p = chunk.slots[idx % CHUNK].swap(std::ptr::null_mut(), Ordering::AcqRel); + let p = chunk.slots[idx % CHUNK].swap(core::ptr::null_mut(), Ordering::AcqRel); if p.is_null() { return None; } @@ -594,13 +595,13 @@ impl PartitionSet { } } -impl std::fmt::Debug for PartitionSet { +impl core::fmt::Debug for PartitionSet { /// Deliberately shallow: a partition set can hold thousands of tables and /// printing them would be useless as well as slow. - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PartitionSet") .field("live", &self.len()) - .field("table", &std::any::type_name::()) + .field("table", &core::any::type_name::()) .finish() } } @@ -608,7 +609,7 @@ impl std::fmt::Debug for PartitionSet { impl Drop for PartitionSet { fn drop(&mut self) { for cell in &self.spine { - let p = cell.swap(std::ptr::null_mut(), Ordering::AcqRel); + let p = cell.swap(core::ptr::null_mut(), Ordering::AcqRel); if p.is_null() { continue; } @@ -616,7 +617,7 @@ impl Drop for PartitionSet { // published exactly once, and nothing else frees it. let chunk = unsafe { Box::from_raw(p) }; for slot in chunk.slots.iter() { - let sp = slot.swap(std::ptr::null_mut(), Ordering::AcqRel); + let sp = slot.swap(core::ptr::null_mut(), Ordering::AcqRel); if !sp.is_null() { // Safety: a live slot owns one strong reference. drop(unsafe { Arc::from_raw(sp as *const T) }); @@ -652,8 +653,8 @@ pub enum PartitionError { OutOfRange { key: u64 }, } -impl std::fmt::Display for PartitionError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for PartitionError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { PartitionError::OutOfRange { key } => { write!(f, "partition key {key} exceeds the maximum of {MAX_PARTITIONS}") @@ -662,7 +663,7 @@ impl std::fmt::Display for PartitionError { } } -impl std::error::Error for PartitionError {} +impl core::error::Error for PartitionError {} #[cfg(all(test, not(wt_loom)))] mod tests; diff --git a/src/partition/tests.rs b/src/partition/tests.rs index a18f9e68..1a920c2d 100644 --- a/src/partition/tests.rs +++ b/src/partition/tests.rs @@ -1,5 +1,6 @@ use super::*; -use std::sync::atomic::AtomicU32; +use alloc::{string::ToString, vec::Vec}; +use core::sync::atomic::AtomicU32; #[derive(Debug, PartialEq)] struct Counted(u64); @@ -119,7 +120,7 @@ fn concurrent_creation_of_one_key_makes_one_table() { #[test] fn concurrent_readers_see_a_partition_created_under_them() { let set: Arc> = Arc::new(PartitionSet::new()); - let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop = Arc::new(core::sync::atomic::AtomicBool::new(false)); let reader = { let set = set.clone(); let stop = stop.clone(); @@ -389,7 +390,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { let drops = Arc::new(AtomicU32::new(0)); let set: Arc> = Arc::new(PartitionSet::new()); - let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop = Arc::new(core::sync::atomic::AtomicBool::new(false)); let seen = Arc::new(AtomicU32::new(0)); let readers: Vec<_> = (0..if cfg!(miri) { 2 } else { 3 }) @@ -434,7 +435,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { for k in 0..KEYS { set.get_or_create(k, || make(k, &drops)).unwrap(); } - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let deadline = std::time::Instant::now() + core::time::Duration::from_secs(30); while seen.load(Ordering::Relaxed) == 0 && std::time::Instant::now() < deadline { std::thread::yield_now(); } @@ -447,7 +448,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { // read to finish, never a zero-reader instant. The pre-epoch retire list // could only assert the opposite here (everything retired, nothing // freed, unbounded growth through the shared router). - let reclaim_deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let reclaim_deadline = std::time::Instant::now() + core::time::Duration::from_secs(30); while drops.load(Ordering::SeqCst) == 0 && std::time::Instant::now() < reclaim_deadline { set.collect(); std::thread::yield_now(); @@ -473,7 +474,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { // Drain the remainder through the shared handle: no `&mut`, no // `Arc::try_unwrap` gymnastics needed for reclamation any more. - let drain_deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let drain_deadline = std::time::Instant::now() + core::time::Duration::from_secs(30); while set.retired_len() > 0 && std::time::Instant::now() < drain_deadline { set.collect(); } @@ -669,7 +670,7 @@ fn mem_stat_reports_each_partition_and_their_sum() { // `size_of::()` on top of the payload, because that is what the // allocation actually holds. Derived rather than hard coded so the // expectation is the rule, not one machine's numbers. - let overhead = std::mem::size_of::(); + let overhead = core::mem::size_of::(); for (k, size) in [(3u64, 17usize), (1, 5), (2048, 300)] { set.get_or_create(k, || Sized_(size)).unwrap(); } @@ -698,6 +699,6 @@ fn partition_error_says_which_key_and_which_bound() { ); // The `Error` impl is what a caller using `?` and `eyre` will format. - let as_error: &dyn std::error::Error = &out_of_range; + let as_error: &dyn core::error::Error = &out_of_range; assert_eq!(as_error.to_string(), text); } diff --git a/src/persistence/engine.rs b/src/persistence/engine.rs index 37254f6f..bcf9487d 100644 --- a/src/persistence/engine.rs +++ b/src/persistence/engine.rs @@ -1,8 +1,9 @@ -use std::fmt::Debug; +use alloc::{string::String, vec::Vec}; +use core::fmt::Debug; +use core::future::Future; +use core::hash::Hash; +use core::marker::PhantomData; use std::fs; -use std::future::Future; -use std::hash::Hash; -use std::marker::PhantomData; use std::panic::{AssertUnwindSafe, resume_unwind}; use std::path::Path; diff --git a/src/persistence/error.rs b/src/persistence/error.rs index fbfac49e..1acb39a2 100644 --- a/src/persistence/error.rs +++ b/src/persistence/error.rs @@ -1,9 +1,10 @@ -use std::error::Error; -use std::fmt::{Display, Formatter}; -use std::future::Future; +use alloc::sync::Arc; +use alloc::{string::String, string::ToString}; +use core::error::Error; +use core::fmt::{Display, Formatter}; +use core::future::Future; use std::panic::AssertUnwindSafe; use std::path::{Path, PathBuf}; -use std::sync::Arc; use futures::FutureExt; @@ -39,7 +40,7 @@ impl PersistenceLoadError { } impl Display for PersistenceLoadError { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut Formatter<'_>) -> core::fmt::Result { write!( formatter, "torn or corrupt persisted table at {}: {}", @@ -111,7 +112,7 @@ impl PersistenceIndexCorruption { } impl Display for PersistenceIndexCorruption { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut Formatter<'_>) -> core::fmt::Result { write!( formatter, "persisted index at {} was quarantined: {}", @@ -137,7 +138,7 @@ pub enum PersistenceError { } impl Display for PersistenceError { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut Formatter<'_>) -> core::fmt::Result { match self { Self::Closing => formatter.write_str("persistence task is closing"), Self::Closed => formatter.write_str("persistence task is closed"), diff --git a/src/persistence/event_ledger.rs b/src/persistence/event_ledger.rs index 560cb7ee..00ef9f4d 100644 --- a/src/persistence/event_ledger.rs +++ b/src/persistence/event_ledger.rs @@ -67,10 +67,24 @@ //! the stream by construction, so a recent window always covers it; the report //! states the window it holds so a reader can see that for themselves. +// The ledger itself is arithmetic and bookkeeping, so it takes `core` and +// `alloc`. Two of its capabilities genuinely need an operating system, and only +// those are gated: `Backtrace`, and reading `WT_EVENT_LEDGER` from the +// environment. Without `std` the ledger is simply never enabled, which is the +// right answer for a diagnostic that an environment variable turns on. +use alloc::borrow::ToOwned as _; +// Only the gated backtrace field boxes anything. +#[cfg(feature = "std")] +use alloc::boxed::Box; +use alloc::collections::BTreeMap; +use alloc::string::{String, ToString as _}; +use alloc::vec::Vec; +use core::fmt::Write as _; +use core::panic::Location; +use hashbrown::HashMap; +#[cfg(feature = "std")] use std::backtrace::Backtrace; -use std::collections::{BTreeMap, HashMap}; -use std::fmt::Write as _; -use std::panic::Location; +#[cfg(feature = "std")] use std::sync::LazyLock; use data_bucket::Link; @@ -88,6 +102,7 @@ const WINDOW: usize = 8192; /// Gap ids listed individually in a report before it summarises the rest. const MAX_LISTED_GAP_IDS: usize = 64; +#[cfg(feature = "std")] static ENABLED: LazyLock = LazyLock::new(|| { if cfg!(debug_assertions) { return true; @@ -98,13 +113,27 @@ static ENABLED: LazyLock = LazyLock::new(|| { } }); +/// Without `std` there is no environment to read the switch from, so the +/// ledger stays off. Everything below still compiles and can be driven +/// directly by a caller that wants it; what is missing is the ambient way to +/// turn it on. +#[cfg(not(feature = "std"))] +static ENABLED: bool = false; + /// Whether event bookkeeping is recording in this process. /// /// See the module comment for why this is `debug_assertions` plus an /// environment override rather than a cargo feature. #[inline] pub fn enabled() -> bool { - *ENABLED + #[cfg(feature = "std")] + { + *ENABLED + } + #[cfg(not(feature = "std"))] + { + ENABLED + } } /// Which index's event id sequence a record belongs to. @@ -120,8 +149,8 @@ pub enum EventStream { Secondary(String), } -impl std::fmt::Display for EventStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for EventStream { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { EventStream::Primary => f.write_str("primary"), EventStream::Secondary(index) => write!(f, "secondary {index}"), @@ -163,8 +192,8 @@ impl Stages { } } -impl std::fmt::Display for Stages { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for Stages { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { let mut first = true; for (flag, name) in [ (Stages::QUEUED, "queued"), @@ -199,6 +228,10 @@ struct IdRecord { /// Captured only when `RUST_BACKTRACE` is set; otherwise `Disabled` and /// free. Boxed so that the common empty case costs a pointer instead of an /// inline `Backtrace` in each of the thousands of records held per stream. + // Gated with the capture below: without `std` there is no `Backtrace`, + // and the ledger keeps the call site from `#[track_caller]` regardless, + // which is the half that names the producer. + #[cfg(feature = "std")] backtrace: Option>, collected: u32, requeued: u32, @@ -211,6 +244,7 @@ impl IdRecord { op_id: None, op_type: None, site: None, + #[cfg(feature = "std")] backtrace: None, collected: 0, requeued: 0, @@ -328,7 +362,9 @@ impl EventLedger { // `Disabled` without walking any frames. One capture per push, moved // onto the first newly recorded id, because every id in this vector // came from the same producer. + #[cfg(feature = "std")] let backtrace = Backtrace::capture(); + #[cfg(feature = "std")] let mut backtrace = matches!(backtrace.status(), std::backtrace::BacktraceStatus::Captured).then_some(backtrace); let mut streams = self.streams.lock(); @@ -344,6 +380,7 @@ impl EventLedger { // One backtrace per push, not per id: every id in this vector // has the same producer, and keeping one each would multiply // the cost of a `RUST_BACKTRACE` run for no extra signal. + #[cfg(feature = "std")] if record.backtrace.is_none() { record.backtrace = backtrace.take().map(Box::new); } @@ -553,10 +590,12 @@ fn bracketing_producers(ledger: &StreamLedger, last_applied: u64, next_available op_id = OptionDisplay(record.op_id.as_ref().map(|id| format!("{id:?}"))), site = OptionDisplay(record.site.map(|site| site.to_string())), ); + #[cfg(feature = "std")] if let Some(backtrace) = &record.backtrace { let _ = write!(out, " Backtrace:\n{backtrace}\n"); } } + #[cfg(feature = "std")] if !ledger.ids.values().any(|record| record.backtrace.is_some()) { let _ = write!( out, @@ -574,8 +613,8 @@ impl Default for EventLedger { struct OptionDisplay(Option); -impl std::fmt::Display for OptionDisplay { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for OptionDisplay { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match &self.0 { Some(value) => f.write_str(value), None => f.write_str(""), diff --git a/src/persistence/mod.rs b/src/persistence/mod.rs index b5ddb33a..037823ef 100644 --- a/src/persistence/mod.rs +++ b/src/persistence/mod.rs @@ -1,11 +1,13 @@ -use std::future::Future; - -use data_bucket::page::PageId; +use core::future::Future; +#[cfg(feature = "std")] use crate::persistence::operation::BatchOperation; +#[cfg(feature = "std")] pub use engine::DiskConfig; +#[cfg(feature = "std")] pub use engine::DiskPersistenceEngine; +#[cfg(feature = "std")] pub use error::{ PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, PersistenceResult, PersistenceState, load_persisted_state, @@ -15,7 +17,9 @@ pub use operation::{ AcknowledgeOperation, DeleteOperation, InsertOperation, Operation, OperationId, OperationType, UpdateOperation, validate_events, }; +#[cfg(feature = "std")] pub use readonly_engine::ReadOnlyPersistenceEngine; +#[cfg(feature = "std")] pub use space::{ ArtPersistenceKey, IndexTableOfContents, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, SpaceIndex, SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, @@ -23,6 +27,7 @@ pub use space::{ TocEntryOversizedError, map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, }; +#[cfg(feature = "std")] pub use task::{PersistenceMonitor, PersistenceTask}; /// Result of retiring one Arc-owned persisted table generation. @@ -40,13 +45,13 @@ pub struct UnloadReport { /// can keep serving it or retry. A failure returned by `close` has no retained /// generation because shutdown was already attempted and consumed it. pub struct UnloadFailure { - generation: Option>, + generation: Option>, error: eyre::Report, } impl UnloadFailure { #[doc(hidden)] - pub fn retained(generation: std::sync::Arc, error: eyre::Report) -> Self { + pub fn retained(generation: alloc::sync::Arc, error: eyre::Report) -> Self { Self { generation: Some(generation), error, @@ -62,7 +67,7 @@ impl UnloadFailure { } /// Returns the still-live generation when shutdown never began. - pub fn into_generation(self) -> Option> { + pub fn into_generation(self) -> Option> { self.generation } @@ -72,8 +77,8 @@ impl UnloadFailure { } } -impl std::fmt::Debug for UnloadFailure { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for UnloadFailure { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter .debug_struct("UnloadFailure") .field("generation_retained", &self.generation.is_some()) @@ -82,20 +87,28 @@ impl std::fmt::Debug for UnloadFailure { } } -impl std::fmt::Display for UnloadFailure { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for UnloadFailure { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.error.fmt(formatter) } } -impl std::error::Error for UnloadFailure {} +impl core::error::Error for UnloadFailure {} + +#[cfg(feature = "std")] +use data_bucket::page::PageId; +#[cfg(feature = "std")] mod engine; +#[cfg(feature = "std")] mod error; pub mod event_ledger; pub mod operation; +#[cfg(feature = "std")] mod readonly_engine; +#[cfg(feature = "std")] mod space; +#[cfg(feature = "std")] mod task; // TODO: remove this @@ -144,6 +157,7 @@ where } } +#[cfg(feature = "std")] pub trait PersistenceEngine { type Config: PersistenceConfig; diff --git a/src/persistence/operation/batch.rs b/src/persistence/operation/batch.rs index ea49a1d8..57164192 100644 --- a/src/persistence/operation/batch.rs +++ b/src/persistence/operation/batch.rs @@ -1,8 +1,10 @@ -use std::collections::HashMap; -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; -use std::sync::Arc; +use alloc::boxed::Box; +use alloc::sync::Arc; +use alloc::{string::ToString, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; +use hashbrown::HashMap; use data_bucket::page::PageId; use data_bucket::{Link, SizeMeasurable}; @@ -549,9 +551,10 @@ where #[cfg(test)] mod tests { - use std::collections::HashMap; + use hashbrown::HashMap; use data_bucket::Link; + use data_bucket::page::PageId; use indexset::core::pair::Pair; use uuid::Uuid; @@ -599,7 +602,7 @@ mod tests { // Deliberately reverse vector order: operation ids, not incidental // collection order, define which bytes are newest. let batch = latest_data_writes(&[insert(2, new_link, vec![2; 6]), insert(1, old_link, vec![1; 4])]); - let writes = batch.get(&1.into()).unwrap(); + let writes = batch.get(&PageId::from(1u32)).unwrap(); assert_eq!(writes, &vec![(new_link, vec![2; 6])]); } @@ -619,7 +622,7 @@ mod tests { for _ in 0..128 { let batch = latest_data_writes(&[insert(2, newer_link, vec![2; 8]), insert(1, older_link, vec![1; 8])]); - let writes = batch.get(&1.into()).unwrap(); + let writes = batch.get(&PageId::from(1u32)).unwrap(); assert_eq!(writes, &vec![(older_link, vec![1; 8]), (newer_link, vec![2; 8])]); } @@ -644,7 +647,7 @@ mod tests { ]); assert_eq!( - batch.get(&1.into()).unwrap(), + batch.get(&PageId::from(1u32)).unwrap(), &vec![(older_link, vec![1; 8]), (newer_link, vec![2; 8])] ); } @@ -667,7 +670,7 @@ mod tests { multi_insert(1, new_link, vec![2; 6]), ]); - assert_eq!(batch.get(&1.into()).unwrap(), &vec![(new_link, vec![2; 6])]); + assert_eq!(batch.get(&PageId::from(1u32)).unwrap(), &vec![(new_link, vec![2; 6])]); } #[tokio::test] @@ -699,7 +702,7 @@ mod tests { } fn iter_event_ids(&self) -> impl Iterator { - std::iter::empty() + core::iter::empty() } fn sort(&mut self) {} @@ -799,7 +802,7 @@ mod tests { let data = batch.get_batch_data_op().unwrap(); assert_eq!( - data.get(&1.into()).unwrap(), + data.get(&PageId::from(1u32)).unwrap(), &vec![(survivor_link, vec![7; 4])], "the surviving data-only write must stay in the applied batch" ); diff --git a/src/persistence/operation/mod.rs b/src/persistence/operation/mod.rs index 5f7954fc..3097abbe 100644 --- a/src/persistence/operation/mod.rs +++ b/src/persistence/operation/mod.rs @@ -1,11 +1,12 @@ +#[cfg(feature = "std")] mod batch; #[allow(clippy::module_inception)] mod operation; mod util; -use std::cmp::Ordering; -use std::fmt::Debug; -use std::hash::{Hash, Hasher}; +use core::cmp::Ordering; +use core::fmt::Debug; +use core::hash::{Hash, Hasher}; use data_bucket::SizeMeasurable; use derive_more::Display; @@ -14,6 +15,7 @@ use uuid::Uuid; use crate::prelude::From; +#[cfg(feature = "std")] pub use batch::{BatchInnerRow, BatchInnerWorkTable, BatchOperation}; pub use operation::{AcknowledgeOperation, DeleteOperation, InsertOperation, Operation, UpdateOperation}; pub use util::validate_events; diff --git a/src/persistence/operation/operation.rs b/src/persistence/operation/operation.rs index 36af8298..51a25c3c 100644 --- a/src/persistence/operation/operation.rs +++ b/src/persistence/operation/operation.rs @@ -1,5 +1,6 @@ -use std::fmt::Debug; -use std::hash::{Hash, Hasher}; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::{Hash, Hasher}; use data_bucket::Link; use indexset::cdc::change::ChangeEvent; diff --git a/src/persistence/operation/util.rs b/src/persistence/operation/util.rs index 405a9ce3..dd58bb20 100644 --- a/src/persistence/operation/util.rs +++ b/src/persistence/operation/util.rs @@ -1,7 +1,8 @@ +use alloc::vec::Vec; +use core::fmt::Debug; use data_bucket::Link; use indexset::cdc::change::{self, ChangeEvent}; use indexset::core::pair::Pair; -use std::fmt::Debug; pub fn validate_events(evs: &mut Vec>>) -> Vec>> where @@ -20,7 +21,7 @@ where } } - removed_events.sort_by_key(|ev2| std::cmp::Reverse(ev2.id())); + removed_events.sort_by_key(|ev2| core::cmp::Reverse(ev2.id())); removed_events } diff --git a/src/persistence/readonly_engine.rs b/src/persistence/readonly_engine.rs index 27a19d25..b35389bc 100644 --- a/src/persistence/readonly_engine.rs +++ b/src/persistence/readonly_engine.rs @@ -1,5 +1,5 @@ -use std::fmt::Debug; -use std::hash::Hash; +use core::fmt::Debug; +use core::hash::Hash; use crate::TableSecondaryIndexEventsOps; use crate::persistence::operation::{BatchOperation, Operation}; diff --git a/src/persistence/space/art_index.rs b/src/persistence/space/art_index.rs index d2db86c6..97f49a1a 100644 --- a/src/persistence/space/art_index.rs +++ b/src/persistence/space/art_index.rs @@ -6,17 +6,18 @@ //! applies the WAL, writes a new native checkpoint atomically, and drops the //! temporary tree; no duplicate ART is retained during normal operation. -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; +use alloc::{borrow::ToOwned, string::String, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; use std::path::{Path, PathBuf}; +use crate::fsx::File; use data_bucket::{Link, page::PageId}; use eyre::{Context, bail, eyre}; use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; -use tokio::fs::{File, OpenOptions}; -use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; +use nagoya::io::{Read as _, Seek as _, Write as _}; use crate::index::{ ArcticIndex, ArcticKey, ArcticMultiIndex, CongeeIndex, CongeeKey, PersistentArcticIndex, @@ -76,14 +77,14 @@ macro_rules! impl_art_persistence_key { ($($type:ty),+ $(,)?) => { $( impl ArtPersistenceKey for $type { - const WIDTH: u8 = std::mem::size_of::() as u8; + const WIDTH: u8 = core::mem::size_of::() as u8; fn encode_art_key(&self, output: &mut Vec) { output.extend_from_slice(&self.to_be_bytes()); } fn decode_art_key(bytes: &[u8]) -> eyre::Result { - let bytes: [u8; std::mem::size_of::()] = bytes + let bytes: [u8; core::mem::size_of::()] = bytes .try_into() .map_err(|_| eyre!("invalid {}-byte ART key", Self::WIDTH))?; Ok(Self::from_be_bytes(bytes)) @@ -112,7 +113,7 @@ macro_rules! impl_art_persistence_key_signed { ($($type:ty => $raw:ty),+ $(,)?) => { $( impl ArtPersistenceKey for $type { - const WIDTH: u8 = std::mem::size_of::() as u8; + const WIDTH: u8 = core::mem::size_of::() as u8; fn encode_art_key(&self, output: &mut Vec) { let raw = (*self as $raw) ^ ((1 as $raw) << (<$raw>::BITS - 1)); @@ -120,7 +121,7 @@ macro_rules! impl_art_persistence_key_signed { } fn decode_art_key(bytes: &[u8]) -> eyre::Result { - let bytes: [u8; std::mem::size_of::()] = bytes + let bytes: [u8; core::mem::size_of::()] = bytes .try_into() .map_err(|_| eyre!("invalid {}-byte ART key", Self::WIDTH))?; let raw = <$raw>::from_be_bytes(bytes) ^ ((1 as $raw) << (<$raw>::BITS - 1)); @@ -200,17 +201,17 @@ impl ArtFile { // with live state or block a future rename. let stale_temporary = temporary_path(&path); if stale_temporary.exists() { - tokio::fs::remove_file(&stale_temporary).await?; + crate::fsx::remove_file(&stale_temporary).await?; } if !path.exists() { Self::write_new_file(&path, backend, table_version, &empty_snapshot).await?; } let image = Self::read_image(&path, backend, table_version).await?; - let mut file = OpenOptions::new().read(true).write(true).open(&path).await?; + let mut file = crate::fsx::open(&path).await?; // Remove an incomplete final frame before appending. Leaving it in // place would make every later valid frame unreachable on recovery. - file.set_len(image.durable_len).await?; - file.seek(std::io::SeekFrom::End(0)).await?; + crate::fsx::set_len(&mut file, image.durable_len).await?; + file.seek(nagoya::io::SeekFrom::End(0)).await?; Ok(Self { path, file, @@ -222,7 +223,7 @@ impl ArtFile { } async fn read_image(path: &Path, backend: Backend, table_version: u32) -> eyre::Result> { - let mut file = File::open(path) + let mut file = crate::fsx::open(path) .await .wrap_err_with(|| format!("open ART index {}", path.display()))?; let mut bytes = Vec::new(); @@ -339,8 +340,8 @@ impl ArtFile { async fn rewrite(&mut self, snapshot: &[u8]) -> eyre::Result<()> { Self::write_file_atomically(&self.path, self.backend, self.table_version, snapshot).await?; - self.file = OpenOptions::new().read(true).write(true).open(&self.path).await?; - self.file.seek(std::io::SeekFrom::End(0)).await?; + self.file = crate::fsx::open(&self.path).await?; + self.file.seek(nagoya::io::SeekFrom::End(0)).await?; self.wal_bytes = 0; Ok(()) } @@ -356,7 +357,7 @@ impl ArtFile { ) -> eyre::Result<()> { let temporary = temporary_path(path); Self::write_new_file(&temporary, backend, table_version, snapshot).await?; - tokio::fs::rename(&temporary, path).await?; + crate::fsx::rename(&temporary, path).await?; Ok(()) } @@ -372,11 +373,11 @@ impl ArtFile { header.extend_from_slice(&0u32.to_le_bytes()); debug_assert_eq!(header.len(), HEADER_LEN); - let mut file = File::create(path).await?; + let mut file = crate::fsx::create(path).await?; file.write_all(&header).await?; file.write_all(snapshot).await?; file.flush().await?; - file.sync_data().await?; + crate::fsx::sync_data(&mut file).await?; Ok(()) } } @@ -384,7 +385,7 @@ impl ArtFile { fn encode_wal_record(record: &WalRecord) -> Vec { let mut key = Vec::new(); record.key.encode_art_key(&mut key); - let variable_prefix = usize::from(K::WIDTH == 0) * std::mem::size_of::(); + let variable_prefix = usize::from(K::WIDTH == 0) * core::mem::size_of::(); let mut bytes = Vec::with_capacity(9 + variable_prefix + key.len() + 12); bytes.extend_from_slice(&record.event_id.to_le_bytes()); match record.op { @@ -717,7 +718,7 @@ where path, Backend::ArcticVariable, table_version, - encode_multi_pairs(std::iter::empty::<(K, Link)>()), + encode_multi_pairs(core::iter::empty::<(K, Link)>()), ) .await?, }) @@ -922,7 +923,7 @@ where K: ArtPersistenceKey + ArcticKey, { async fn new(path: PathBuf, table_version: u32) -> eyre::Result { - let snapshot = encode_multi_pairs(std::iter::empty::<(K, Link)>()); + let snapshot = encode_multi_pairs(core::iter::empty::<(K, Link)>()); Ok(Self { file: ArtFile::open(path, Backend::ArcticMulti, table_version, snapshot).await?, }) @@ -1356,15 +1357,15 @@ mod tests { space.process_change_event(set_event(0, 7, link(7))).await.unwrap(); drop(space); - let durable_len = tokio::fs::metadata(&path).await.unwrap().len(); - let mut file = OpenOptions::new().append(true).open(&path).await.unwrap(); + let durable_len = crate::fsx::metadata(&path).await.unwrap(); + let mut file = crate::fsx::append(&path).await.unwrap(); file.write_all(&WAL_MAGIC[..2]).await.unwrap(); file.flush().await.unwrap(); drop(file); - assert_eq!(tokio::fs::metadata(&path).await.unwrap().len(), durable_len + 2); + assert_eq!(crate::fsx::metadata(&path).await.unwrap(), durable_len + 2); let mut space = SpaceArcticIndex::::new(path.clone(), 1).await.unwrap(); - assert_eq!(tokio::fs::metadata(&path).await.unwrap().len(), durable_len); + assert_eq!(crate::fsx::metadata(&path).await.unwrap(), durable_len); space.process_change_event(set_event(1, 8, link(8))).await.unwrap(); drop(space); @@ -1373,7 +1374,7 @@ mod tests { .unwrap(); assert_eq!(index.get_value(&7).unwrap().0, link(7)); assert_eq!(index.get_value(&8).unwrap().0, link(8)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1401,7 +1402,7 @@ mod tests { .unwrap(); assert_eq!(index.len(), 128); assert_eq!(index.get_value(&91).unwrap().0, link(92)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } fn remove_event(id: u64, key: u64, value: Link) -> ChangeEvent> { @@ -1439,7 +1440,7 @@ mod tests { assert_eq!(decode_multi_pairs::(&bytes).unwrap(), pairs); assert!(decode_multi_pairs::(&bytes[..bytes.len() - 1]).is_err()); assert_eq!( - decode_multi_pairs::(&encode_multi_pairs(std::iter::empty::<(u64, Link)>())).unwrap(), + decode_multi_pairs::(&encode_multi_pairs(core::iter::empty::<(u64, Link)>())).unwrap(), vec![] ); } @@ -1500,7 +1501,7 @@ mod tests { // A unique reader must refuse the multi file outright. assert!(ArtFile::::read_image(&path, Backend::Arctic, 5).await.is_err()); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1522,7 +1523,7 @@ mod tests { .unwrap(); assert_eq!(reloaded.len(), 50); assert_eq!(reloaded.get(&(u128::MAX - 3)).len(), 10); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1551,7 +1552,7 @@ mod tests { .unwrap(); assert_eq!(reloaded.len(), 4); assert_eq!(reloaded.get_value(&"🦀".to_owned()).unwrap().0, link(4)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[test] @@ -1569,7 +1570,7 @@ mod tests { // A leftover temporary from a crashed checkpoint must be cleaned up // when the index opens. - tokio::fs::write(&temporary, b"crashed checkpoint leftovers") + crate::fsx::write(&temporary, b"crashed checkpoint leftovers") .await .unwrap(); let mut space = SpaceArcticIndex::::new(path.clone(), 1).await.unwrap(); @@ -1597,7 +1598,7 @@ mod tests { .unwrap(); assert_eq!(reloaded.get_value(&7).unwrap().0, link(7)); assert_eq!(reloaded.get_value(&9).unwrap().0, link(9)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1611,16 +1612,16 @@ mod tests { assert!(ArtFile::::read_image(&path, Backend::Congee, 7).await.is_err()); assert!(ArtFile::::read_image(&path, Backend::Arctic, 7).await.is_err()); - let mut file = OpenOptions::new().read(true).write(true).open(&path).await.unwrap(); - file.seek(std::io::SeekFrom::End(-1)).await.unwrap(); + let mut file = crate::fsx::open(&path).await.unwrap(); + file.seek(nagoya::io::SeekFrom::End(-1)).await.unwrap(); let mut last = [0u8; 1]; file.read_exact(&mut last).await.unwrap(); - file.seek(std::io::SeekFrom::End(-1)).await.unwrap(); + file.seek(nagoya::io::SeekFrom::End(-1)).await.unwrap(); file.write_all(&[last[0] ^ 0x80]).await.unwrap(); file.flush().await.unwrap(); drop(file); assert!(ArtFile::::read_image(&path, Backend::Arctic, 7).await.is_err()); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } } diff --git a/src/persistence/space/data.rs b/src/persistence/space/data.rs index dd200019..93338d70 100644 --- a/src/persistence/space/data.rs +++ b/src/persistence/space/data.rs @@ -1,7 +1,9 @@ -use std::collections::HashSet; -use std::io::SeekFrom; +use alloc::{string::String, string::ToString, vec::Vec}; +use hashbrown::HashSet; +use nagoya::io::SeekFrom; use std::path::Path; +use crate::fsx::File; use crate::persistence::SpaceDataOps; use crate::persistence::space::{BatchData, open_or_create_file}; use crate::prelude::WT_DATA_EXTENSION; @@ -10,6 +12,7 @@ use data_bucket::{ DataPage, GeneralHeader, GeneralPage, Link, PageType, Persistable, SizeMeasurable, SpaceInfoPage, parse_data_pages_batch, parse_general_header_by_index, parse_page, persist_page, persist_pages_batch, update_at, }; +use nagoya::io::{Seek as _, Write as _}; use rkyv::api::high::HighDeserializer; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -17,8 +20,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize}; -use tokio::fs::File; -use tokio::io::{AsyncSeekExt, AsyncWriteExt}; fn link_sort_key(link: &Link) -> (u32, u32) { (link.page_id.into(), link.offset) @@ -206,7 +207,7 @@ impl SpaceData

) -> bool { - let free_ranges = std::mem::take(&mut self.info.inner.empty_links_list); + let free_ranges = core::mem::take(&mut self.info.inner.empty_links_list); let (remaining, changed) = subtract_used_ranges(free_ranges, used_links); self.info.inner.empty_links_list = remaining; changed @@ -241,8 +242,8 @@ where } else { open_or_create_file(path).await? }; - let info = parse_page::<_, PAGE_SIZE>(&mut data_file, 0).await?; - let file_length = data_file.metadata().await?.len(); + let info = parse_page::<_, PAGE_SIZE, PAGE_SIZE>(&mut data_file, 0).await?; + let file_length = crate::fsx::file_metadata(&mut data_file).await?; // Mirror the index file's ceil logic: a file whose length is an exact // page multiple ends with a full last page, so the plain floor // division names a page id one past EOF and reopening the table fails @@ -253,7 +254,7 @@ where } else { file_length / PAGE_SIZE as u64 }; - let last_page_header = parse_general_header_by_index(&mut data_file, page_id as u32).await?; + let last_page_header = parse_general_header_by_index::(&mut data_file, page_id as u32).await?; Ok(Self { data_file, @@ -279,7 +280,7 @@ where header: GeneralHeader::new(0.into(), PageType::SpaceInfo, 0.into()), inner: info, }; - persist_page(&mut page, file).await + Ok(persist_page::<_, PAGE_SIZE>(&mut page, file).await?) } async fn save_data(&mut self, link: Link, bytes: &[u8]) -> eyre::Result<()> { @@ -294,7 +295,7 @@ where data: [0; 1], }, }; - persist_page(&mut page, &mut self.data_file).await?; + persist_page::<_, PAGE_SIZE>(&mut page, &mut self.data_file).await?; self.current_data_length = 0; // High-water mark, as in the batch path below: the new page is the // one the link names, which can be more than one past the current @@ -320,8 +321,8 @@ where self.update_data_length().await?; } } - update_at::<{ PAGE_SIZE }>(&mut self.data_file, link, bytes).await?; - // `update_at` ends with a buffered `write_all` that `tokio::fs::File` + update_at::<{ PAGE_SIZE }, PAGE_SIZE>(&mut self.data_file, link, bytes).await?; + // `update_at` ends with a `write_all` that the file behind `fsx` // completes on a background blocking task. Flush before reporting the // save done so the bytes are visible to any other handle. self.data_file.flush().await?; @@ -368,7 +369,7 @@ where }) .collect::>(); let parsed_pages = - parse_data_pages_batch::(&mut self.data_file, ids_to_parse).await?; + parse_data_pages_batch::(&mut self.data_file, ids_to_parse).await?; let updated_pages = vec![parsed_pages, created_pages] .into_iter() @@ -397,7 +398,7 @@ where self.current_data_length = page.inner.length; } - persist_pages_batch(updated_pages, &mut self.data_file).await?; + persist_pages_batch::<_, PAGE_SIZE>(updated_pages, &mut self.data_file).await?; // The batch's last page write is a buffered `write_all`; flush so the // batch is visible to other handles once it reports done. self.data_file.flush().await?; @@ -447,7 +448,7 @@ where // Single choke point for the info page reaching disk: enforce the // page-0 slot budget however the free-range list was mutated. self.bound_empty_links_list(); - persist_page(&mut self.info, &mut self.data_file).await?; + persist_page::<_, PAGE_SIZE>(&mut self.info, &mut self.data_file).await?; // A generated table may immediately reopen this file through a // separate handle. Make the updated metadata visible before reporting // success, just as `save_data` does for row bytes. diff --git a/src/persistence/space/index/mod.rs b/src/persistence/space/index/mod.rs index d4cbc625..284ecb4d 100644 --- a/src/persistence/space/index/mod.rs +++ b/src/persistence/space/index/mod.rs @@ -1,16 +1,18 @@ +use alloc::{string::String, string::ToString, vec::Vec}; mod page_aliases; mod reconstruct; mod table_of_contents; mod unsized_; mod util; -use std::collections::HashMap; -use std::fmt::Debug; -use std::hash::Hash; +use alloc::sync::Arc; +use core::fmt::Debug; +use core::hash::Hash; +use core::sync::atomic::{AtomicU32, Ordering}; +use hashbrown::HashMap; use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; +use crate::fsx::File; use convert_case::{Case, Casing}; use data_bucket::page::{IndexValue, PageId}; use data_bucket::{ @@ -22,6 +24,7 @@ use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; use indexset::concurrent::multimap::BTreeMultiMap; use indexset::core::pair::Pair; +use nagoya::io::Write as _; use rkyv::de::Pool; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -29,8 +32,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; use crate::persistence::SpaceIndexOps; use crate::persistence::space::{BatchChangeEvent, open_or_create_file}; @@ -42,16 +43,16 @@ pub use unsized_::SpaceIndexUnsized; pub use util::{map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general}; #[derive(Debug)] -pub struct SpaceIndex { +pub struct SpaceIndex { space_id: SpaceId, - table_of_contents: IndexTableOfContents<(T, Link), INNER_PAGE_SIZE>, + table_of_contents: IndexTableOfContents<(T, Link), INNER_PAGE_SIZE, STRIDE>, next_page_id: Arc, index_file: File, #[allow(dead_code)] info: GeneralPage>, } -impl SpaceIndex +impl SpaceIndex where T: Archive + Ord @@ -110,9 +111,9 @@ where } else { open_or_create_file(index_file_path).await? }; - let info = parse_page::<_, INNER_PAGE_SIZE>(&mut index_file, 0).await?; + let info = parse_page::<_, INNER_PAGE_SIZE, STRIDE>(&mut index_file, 0).await?; - let file_length = index_file.metadata().await?.len(); + let file_length = crate::fsx::file_metadata(&mut index_file).await?; let page_id = if file_length % (INNER_PAGE_SIZE as u64 + GENERAL_HEADER_SIZE as u64) == 0 { file_length / (INNER_PAGE_SIZE as u64 + GENERAL_HEADER_SIZE as u64) } else { @@ -146,7 +147,7 @@ where async fn add_index_page(&mut self, node: IndexPage, page_id: PageId) -> eyre::Result<()> { let header = GeneralHeader::new(page_id, PageType::Index, self.space_id); let mut general_page = GeneralPage { inner: node, header }; - persist_page(&mut general_page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut general_page, &mut self.index_file).await?; Ok(()) } @@ -160,7 +161,7 @@ where let mut new_node_id = None; let size = get_index_page_size_from_data_length::(INNER_PAGE_SIZE as usize); - let mut utility = IndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + let mut utility = IndexPage::::parse_index_page_utility::(&mut self.index_file, page_id).await?; utility.slots.insert(index, utility.current_index); utility.slots.remove(size); utility.current_length += 1; @@ -168,16 +169,21 @@ where key: value.key.clone(), link: value.value, }; - utility.current_index = - IndexPage::::persist_value(&mut self.index_file, page_id, size, index_value, utility.current_index) - .await?; + utility.current_index = IndexPage::::persist_value::( + &mut self.index_file, + page_id, + size, + index_value, + utility.current_index, + ) + .await?; if node_id.key < value.key { utility.node_id = value.clone().into(); new_node_id = Some(value); } - IndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + IndexPage::::persist_index_page_utility::(&mut self.index_file, page_id, utility).await?; Ok(new_node_id) } @@ -192,7 +198,7 @@ where let mut new_node_id = None; let size = get_index_page_size_from_data_length::(INNER_PAGE_SIZE as usize); - let mut utility = IndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + let mut utility = IndexPage::::parse_index_page_utility::(&mut self.index_file, page_id).await?; let value_position = *utility .slots .get(index) @@ -203,7 +209,7 @@ where utility.slots.remove(index); utility.slots.push(0); utility.current_length -= 1; - IndexPage::::remove_value(&mut self.index_file, page_id, size, utility.current_index).await?; + IndexPage::::remove_value::(&mut self.index_file, page_id, size, utility.current_index).await?; if node_id.key == value.key { let index = *utility @@ -211,11 +217,12 @@ where .get(index - 1) .expect("slots always should exist in `size` bounds"); utility.node_id = - IndexPage::::read_value_with_index(&mut self.index_file, page_id, size, index as usize).await?; + IndexPage::::read_value_with_index::(&mut self.index_file, page_id, size, index as usize) + .await?; new_node_id = Some(utility.node_id.clone().into()) } - IndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + IndexPage::::persist_index_page_utility::(&mut self.index_file, page_id, utility).await?; Ok(new_node_id) } @@ -292,7 +299,8 @@ where .table_of_contents .get(&(node_id.key.clone(), node_id.value)) .ok_or(eyre!("Node with {:?} id is not found", node_id))?; - let mut page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, page_id.into()).await?; + let mut page = + parse_page::, INNER_PAGE_SIZE, STRIDE>(&mut self.index_file, page_id.into()).await?; let splitted_page = page.inner.split(split_index); let new_page_id = if let Some(id) = self.table_of_contents.pop_empty_page_id() { id @@ -316,7 +324,7 @@ where // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. self.add_index_page(splitted_page, new_page_id).await?; - persist_page(&mut page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut page, &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; Ok(()) @@ -327,7 +335,8 @@ where let indexset = BTreeMap::::with_maximum_node_size(size); let mut nodes = Vec::with_capacity(self.table_of_contents.iter().count()); for (_, page_id) in self.table_of_contents.iter() { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, (*page_id).into()).await?; + let page = + parse_page::, INNER_PAGE_SIZE, STRIDE>(&mut self.index_file, (*page_id).into()).await?; nodes.push(page.inner.get_node()); } indexset.attach_nodes(nodes); @@ -343,7 +352,8 @@ where let indexset = BTreeMultiMap::::with_maximum_node_size(size); let mut pages = Vec::with_capacity(self.table_of_contents.iter().count()); for ((key, link), page_id) in self.table_of_contents.iter() { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, (*page_id).into()).await?; + let page = + parse_page::, INNER_PAGE_SIZE, STRIDE>(&mut self.index_file, (*page_id).into()).await?; pages.push(( Pair { key: key.clone(), @@ -357,7 +367,7 @@ where } } -impl SpaceIndexOps for SpaceIndex +impl SpaceIndexOps for SpaceIndex where T: Archive + Ord @@ -410,7 +420,7 @@ where header: GeneralHeader::new(0.into(), PageType::SpaceInfo, 0.into()), inner: info, }; - persist_page(&mut page, file).await + Ok(persist_page::<_, STRIDE>(&mut page, file).await?) } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -442,7 +452,7 @@ where } => self.process_split_node(node_id, split_index).await, }?; // The partial page writes above can end with a buffered `write_all` - // that `tokio::fs::File` completes on a background blocking task. + // that the file behind `fsx` completes on the calling thread. // Flush before reporting the event processed so the bytes are visible // to any other handle that opens this file afterwards. self.index_file.flush().await?; @@ -482,8 +492,11 @@ where let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, page_index.into()) - .await?; + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( + &mut self.index_file, + page_index.into(), + ) + .await?; pages.insert(page_index, page); pages .get_mut(&page_index) @@ -564,8 +577,11 @@ where let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, page_index.into()) - .await?; + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( + &mut self.index_file, + page_index.into(), + ) + .await?; pages.insert(page_index, page); pages .get_mut(&page_index) @@ -628,7 +644,7 @@ where // authority (`parse_indexset` and the strict load audit iterate TOC // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. - persist_pages_batch(pages.values().cloned().collect(), &mut self.index_file).await?; + persist_pages_batch::<_, STRIDE>(pages.values().cloned().collect(), &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; // The batch's last write is buffered; flush so the batch is visible // to other handles once it reports done. @@ -639,6 +655,7 @@ where #[cfg(test)] mod test { + use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::{INNER_PAGE_SIZE, IndexPage, IndexValue, Persistable, get_index_page_size_from_data_length}; use super::*; @@ -651,7 +668,7 @@ mod test { #[tokio::test] async fn orphan_page_from_crash_between_page_and_toc_write_is_ignored_on_reload() { use indexset::cdc::change::ChangeEvent; - use tokio::io::AsyncWriteExt; + use nagoya::io::Write as _; let dir = std::env::temp_dir().join(format!("wt_orphan_crash_{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); @@ -669,7 +686,7 @@ mod test { }; { - let mut index = SpaceIndex::::new(&path, 0.into(), 1) + let mut index = SpaceIndex::::new(&path, 0.into(), 1) .await .unwrap(); index @@ -687,7 +704,7 @@ mod test { index.index_file.flush().await.unwrap(); } - let mut reloaded = SpaceIndex::::new(&path, 0.into(), 1) + let mut reloaded = SpaceIndex::::new(&path, 0.into(), 1) .await .unwrap(); let restored = reloaded.parse_indexset().await.unwrap(); diff --git a/src/persistence/space/index/page_aliases.rs b/src/persistence/space/index/page_aliases.rs index 131dd05d..627588e4 100644 --- a/src/persistence/space/index/page_aliases.rs +++ b/src/persistence/space/index/page_aliases.rs @@ -5,6 +5,7 @@ //! when a split or a max-remove re-keys a page mid-batch while later events //! still name a historical maximum. +use alloc::vec::Vec; use data_bucket::Link; use data_bucket::page::PageId; use eyre::eyre; @@ -35,7 +36,7 @@ pub(super) struct PageAliasEntry { impl Default for PageAliases { fn default() -> Self { Self { - inline: std::array::from_fn(|_| None), + inline: core::array::from_fn(|_| None), overflow: Vec::new(), } } diff --git a/src/persistence/space/index/reconstruct.rs b/src/persistence/space/index/reconstruct.rs index dab116ca..64ca1c13 100644 --- a/src/persistence/space/index/reconstruct.rs +++ b/src/persistence/space/index/reconstruct.rs @@ -4,7 +4,8 @@ //! generic function that can be unit-tested with synthetic pages; the proc //! macro only generates type plumbing and node attachment. -use std::fmt::Debug; +use alloc::vec::Vec; +use core::fmt::Debug; use indexset::core::multipair::MultiPair; use indexset::core::pair::Pair; @@ -173,7 +174,7 @@ mod tests { for b in nodes.iter().skip(i + 1) { assert_ne!( a.last().unwrap().cmp(b.last().unwrap()), - std::cmp::Ordering::Equal, + core::cmp::Ordering::Equal, "two node maxima compare Equal: {:?} vs {:?}", a.last().unwrap(), b.last().unwrap() @@ -247,7 +248,7 @@ mod tests { assert_eq!(flatten(&nodes[..1]), vec![(1, 1), (1, 2), (2, 30)]); // Every stored entry is distinct. That is what the discriminator counter was // for; identity is now the `(key, value)` pair itself. - let mut seen = std::collections::BTreeSet::new(); + let mut seen = alloc::collections::BTreeSet::new(); for p in nodes.iter().flatten() { assert!(seen.insert((p.key, p.value)), "duplicate entry {:?}", (p.key, p.value)); } diff --git a/src/persistence/space/index/table_of_contents.rs b/src/persistence/space/index/table_of_contents.rs index 633c1cf5..f6a594a9 100644 --- a/src/persistence/space/index/table_of_contents.rs +++ b/src/persistence/space/index/table_of_contents.rs @@ -1,7 +1,9 @@ -use std::fmt::Debug; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::sync::atomic::{AtomicU32, Ordering}; +use crate::fsx::File; use data_bucket::page::PageId; use data_bucket::{ GeneralHeader, GeneralPage, PageType, SizeMeasurable, SpaceId, TableOfContentsPage, parse_page, persist_page, @@ -13,7 +15,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; /// A table-of-contents entry whose serialized size can never fit a segment. /// @@ -26,8 +27,8 @@ pub struct TocEntryOversizedError { pub segment_capacity: usize, } -impl std::fmt::Display for TocEntryOversizedError { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for TocEntryOversizedError { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!( formatter, "table-of-contents entry needs {} bytes but a whole empty segment holds only {}", @@ -36,16 +37,16 @@ impl std::fmt::Display for TocEntryOversizedError { } } -impl std::error::Error for TocEntryOversizedError {} +impl core::error::Error for TocEntryOversizedError {} #[derive(Debug)] -pub struct IndexTableOfContents { +pub struct IndexTableOfContents { current_page: usize, next_page_id: Arc, pub pages: Vec>>, } -impl IndexTableOfContents +impl IndexTableOfContents where T: Debug + SizeMeasurable + Ord + Eq, { @@ -248,7 +249,7 @@ where + for<'a> rkyv::bytecheck::CheckBytes>, { for page in &mut self.pages { - persist_page(page, file).await?; + persist_page::<_, STRIDE>(page, file).await?; } Ok(()) @@ -265,7 +266,7 @@ where + Eq + for<'a> rkyv::bytecheck::CheckBytes>, { - let first_page = parse_page::, DATA_LENGTH>(file, 1).await; + let first_page = parse_page::, DATA_LENGTH, STRIDE>(file, 1).await; let page = match first_page { Ok(page) => page, Err(error) => { @@ -275,11 +276,14 @@ where // file extends into page 1's slot, the parse failure means a // torn or truncated table of contents, and silently starting // empty would discard the whole index. - let file_length = file.metadata().await?.len(); + let file_length = crate::fsx::file_metadata(file).await?; if file_length <= data_bucket::PAGE_SIZE as u64 { return Ok(Self::new(space_id, next_page_id)); } - return Err(error.wrap_err(format!( + // `wrap_err` belonged to `eyre::Report`. The parse error is a + // concrete `data_bucket::error::Error` now, so it becomes a report + // first and keeps the same context message. + return Err(eyre::Report::new(error).wrap_err(format!( "table of contents page 1 failed to parse in a {file_length}-byte index file that should contain it" ))); } @@ -297,7 +301,7 @@ where let mut ind = false; while !ind { - let page = parse_page::, DATA_LENGTH>(file, index).await?; + let page = parse_page::, DATA_LENGTH, STRIDE>(file, index).await?; ind = page.header.next_id.is_empty(); index = page.header.next_id.into(); table_of_contents_pages.push(page); @@ -316,13 +320,14 @@ where #[cfg(test)] mod tests { use crate::persistence::space::index::table_of_contents::IndexTableOfContents; + use alloc::sync::Arc; + use core::sync::atomic::AtomicU32; + use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::page::PageId; - use std::sync::Arc; - use std::sync::atomic::AtomicU32; #[test] fn empty() { - let toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); assert_eq!( toc.current_page, 0, "`current_page` is not set to 0, it is {}", @@ -333,7 +338,7 @@ mod tests { #[test] fn insert_to_empty() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); let key = 1; toc.insert(key, 1.into()); @@ -352,7 +357,7 @@ mod tests { #[test] fn checked_update_reports_a_missing_identity_without_mutating_the_toc() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); toc.insert(7, 2.into()); assert!(!toc.try_update_key(&8, 9).unwrap()); @@ -363,7 +368,10 @@ mod tests { #[test] fn growing_key_update_moves_the_entry_instead_of_overflowing_the_segment() { const DATA_LENGTH: u32 = 128; - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new( + 0.into(), + Arc::new(AtomicU32::new(1)), + ); // Fill the first segment close to capacity with short keys. let mut key = 0; @@ -395,7 +403,10 @@ mod tests { use crate::persistence::TocEntryOversizedError; const DATA_LENGTH: u32 = 128; - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new( + 0.into(), + Arc::new(AtomicU32::new(1)), + ); toc.insert("small".to_string(), PageId::from(9)); let oversized = "x".repeat(4 * DATA_LENGTH as usize); @@ -411,7 +422,7 @@ mod tests { #[test] fn insert_more_than_one_page() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); let mut keys = vec![]; for key in 0..10 { toc.insert(key, 1.into()); @@ -437,7 +448,7 @@ mod tests { #[test] fn insert_reaches_existing_tail_after_reload_resets_cursor() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); for key in 0..10 { toc.insert(key, u32::from(key).into()); } @@ -465,7 +476,7 @@ mod tests { #[test] fn insert_reports_a_truncated_segment_chain() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); for key in 0..10 { toc.insert(key, u32::from(key).into()); } @@ -480,7 +491,7 @@ mod tests { #[test] fn reinsert_on_empty_space() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); let mut keys = vec![]; for key in 0..10 { toc.insert(key, 1.into()); diff --git a/src/persistence/space/index/unsized_.rs b/src/persistence/space/index/unsized_.rs index dd7cdad0..babafe6b 100644 --- a/src/persistence/space/index/unsized_.rs +++ b/src/persistence/space/index/unsized_.rs @@ -1,9 +1,11 @@ -use std::collections::HashMap; -use std::fmt::Debug; -use std::hash::Hash; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; - +use alloc::sync::Arc; +use alloc::{string::String, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::sync::atomic::{AtomicU32, Ordering}; +use hashbrown::HashMap; + +use crate::fsx::File; use data_bucket::page::PageId; use data_bucket::{ GeneralHeader, GeneralPage, IndexPageUtility, IndexValue, Link, PageType, SizeMeasurable, SpaceId, SpaceInfoPage, @@ -14,6 +16,7 @@ use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; use indexset::concurrent::multimap::BTreeMultiMap; use indexset::core::pair::Pair; +use nagoya::io::Write as _; use rkyv::de::Pool; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -21,8 +24,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; use super::page_aliases::PageAliases; use crate::UnsizedNode; @@ -31,16 +32,16 @@ use crate::persistence::{IndexTableOfContents, SpaceIndex, SpaceIndexOps, recons use crate::prelude::WT_INDEX_EXTENSION; #[derive(Debug)] -pub struct SpaceIndexUnsized { +pub struct SpaceIndexUnsized { space_id: SpaceId, - table_of_contents: IndexTableOfContents<(T, Link), DATA_LENGTH>, + table_of_contents: IndexTableOfContents<(T, Link), DATA_LENGTH, STRIDE>, next_page_id: Arc, index_file: File, #[allow(dead_code)] info: GeneralPage>, } -impl SpaceIndexUnsized +impl SpaceIndexUnsized where T: Archive + Ord @@ -103,7 +104,7 @@ where } pub async fn new>(index_file_path: S, space_id: SpaceId, version: u32) -> eyre::Result { - let space_index = SpaceIndex::::new(index_file_path, space_id, version).await?; + let space_index = SpaceIndex::::new(index_file_path, space_id, version).await?; Ok(Self { space_id, table_of_contents: space_index.table_of_contents, @@ -126,7 +127,7 @@ where // happened to create the page. let header = GeneralHeader::new(page_id, PageType::IndexUnsized, self.space_id); let mut general_page = GeneralPage { inner: node, header }; - persist_page(&mut general_page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut general_page, &mut self.index_file).await?; Ok(()) } @@ -187,7 +188,8 @@ where let mut new_node_id = None; let mut utility = - UnsizedIndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + UnsizedIndexPage::::parse_index_page_utility::(&mut self.index_file, page_id) + .await?; let index_value = IndexValue { key: value.key.clone(), link: value.value, @@ -201,9 +203,11 @@ where let future_utility_size = UnsizedIndexPageUtility::::persisted_size(utility.slots_size as usize + 1, future_node_id_size); if future_utility_size + utility.last_value_offset as usize + value_size > DATA_LENGTH as usize { - let mut page = - parse_page::, DATA_LENGTH>(&mut self.index_file, page_id.into()) - .await?; + let mut page = parse_page::, DATA_LENGTH, STRIDE>( + &mut self.index_file, + page_id.into(), + ) + .await?; page.inner.apply_change_event(ChangeEvent::InsertAt { // The page mutation ignores event ids; this synthetic event // exists only to reuse the same insertion accounting. @@ -215,11 +219,11 @@ where Self::compact_page_if_needed(&mut page.inner)?; let changed_node_id = (page.inner.node_id.key != utility.node_id.key).then(|| Pair::from(page.inner.node_id.clone())); - persist_page(&mut page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut page, &mut self.index_file).await?; return Ok(changed_node_id); } let previous_offset = utility.last_value_offset; - let value_offset = UnsizedIndexPage::::persist_value( + let value_offset = UnsizedIndexPage::::persist_value::( &mut self.index_file, page_id, previous_offset, @@ -237,7 +241,12 @@ where new_node_id = Some(value); } - UnsizedIndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + UnsizedIndexPage::::persist_index_page_utility::( + &mut self.index_file, + page_id, + utility, + ) + .await?; Ok(new_node_id) } @@ -273,7 +282,8 @@ where let mut new_node_id = None; let mut utility = - UnsizedIndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + UnsizedIndexPage::::parse_index_page_utility::(&mut self.index_file, page_id) + .await?; utility.slots.remove(index); utility.slots_size -= 1; @@ -282,14 +292,23 @@ where .slots .get(index - 1) .expect("slots always should exist in `size` bounds"); - let node_id = - UnsizedIndexPage::::read_value_with_offset(&mut self.index_file, page_id, offset, len) - .await?; + let node_id = UnsizedIndexPage::::read_value_with_offset::( + &mut self.index_file, + page_id, + offset, + len, + ) + .await?; utility.update_node_id(node_id)?; new_node_id = Some(utility.node_id.clone().into()) } - UnsizedIndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + UnsizedIndexPage::::persist_index_page_utility::( + &mut self.index_file, + page_id, + utility, + ) + .await?; Ok(new_node_id) } @@ -300,7 +319,8 @@ where .get(&(node_id.key.clone(), node_id.value)) .ok_or(eyre!("Node with {:?} id is not found", node_id))?; let mut page = - parse_page::, DATA_LENGTH>(&mut self.index_file, page_id.into()).await?; + parse_page::, DATA_LENGTH, STRIDE>(&mut self.index_file, page_id.into()) + .await?; let splitted_page = page.inner.split(split_index); let new_page_id = if let Some(id) = self.table_of_contents.pop_empty_page_id() { id @@ -324,7 +344,7 @@ where // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. self.add_index_page(splitted_page, new_page_id).await?; - persist_page(&mut page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut page, &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; Ok(()) @@ -334,9 +354,11 @@ where let indexset = BTreeMap::>>::with_maximum_node_size(DATA_LENGTH as usize); let mut nodes = Vec::with_capacity(self.table_of_contents.iter().count()); for (_, page_id) in self.table_of_contents.iter() { - let page = - parse_page::, DATA_LENGTH>(&mut self.index_file, (*page_id).into()) - .await?; + let page = parse_page::, DATA_LENGTH, STRIDE>( + &mut self.index_file, + (*page_id).into(), + ) + .await?; let node = page.inner.get_node(); nodes.push(UnsizedNode::from_inner(node, DATA_LENGTH as usize)); } @@ -353,9 +375,11 @@ where let indexset = BTreeMultiMap::>::with_maximum_node_size(DATA_LENGTH as usize); let mut pages = Vec::with_capacity(self.table_of_contents.iter().count()); for ((key, link), page_id) in self.table_of_contents.iter() { - let page = - parse_page::, DATA_LENGTH>(&mut self.index_file, (*page_id).into()) - .await?; + let page = parse_page::, DATA_LENGTH, STRIDE>( + &mut self.index_file, + (*page_id).into(), + ) + .await?; pages.push(( Pair { key: key.clone(), @@ -373,7 +397,8 @@ where } } -impl SpaceIndexOps for SpaceIndexUnsized +impl SpaceIndexOps + for SpaceIndexUnsized where T: Archive + Ord @@ -412,7 +437,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndex::::bootstrap(file, table_name, version).await + SpaceIndex::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -444,7 +469,7 @@ where } => self.process_split_node(node_id, split_index).await, }?; // The partial page writes above can end with a buffered `write_all` - // that `tokio::fs::File` completes on a background blocking task. + // that the file behind `fsx` completes on the calling thread. // Flush before reporting the event processed so the bytes are visible // to any other handle that opens this file afterwards. self.index_file.flush().await?; @@ -484,7 +509,7 @@ where let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>( + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( &mut self.index_file, page_index.into(), ) @@ -562,7 +587,7 @@ where let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>( + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( &mut self.index_file, page_index.into(), ) @@ -633,7 +658,7 @@ where // authority (`parse_indexset` and the strict load audit iterate TOC // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. - persist_pages_batch(pages.values().cloned().collect(), &mut self.index_file).await?; + persist_pages_batch::<_, STRIDE>(pages.values().cloned().collect(), &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; // The batch's last write is buffered; flush so the batch is visible // to other handles once it reports done. diff --git a/src/persistence/space/index/util.rs b/src/persistence/space/index/util.rs index cde98019..b45fb700 100644 --- a/src/persistence/space/index/util.rs +++ b/src/persistence/space/index/util.rs @@ -1,16 +1,17 @@ use crate::prelude::IndexTableOfContents; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::sync::atomic::{AtomicU32, Ordering}; use data_bucket::{ GeneralHeader, GeneralPage, IndexPage, Link, PageType, SizeMeasurable, UnsizedIndexPage, VariableSizeMeasurable, }; -use std::fmt::Debug; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; #[allow(clippy::type_complexity)] -pub fn map_index_pages_to_toc_and_general( +pub fn map_index_pages_to_toc_and_general( pages: Vec>, ) -> ( - IndexTableOfContents<(T, Link), DATA_LENGTH>, + IndexTableOfContents<(T, Link), DATA_LENGTH, STRIDE>, Vec>>, ) where @@ -31,10 +32,10 @@ where } #[allow(clippy::type_complexity)] -pub fn map_unsized_index_pages_to_toc_and_general( +pub fn map_unsized_index_pages_to_toc_and_general( pages: Vec>, ) -> ( - IndexTableOfContents<(T, Link), DATA_LENGTH>, + IndexTableOfContents<(T, Link), DATA_LENGTH, STRIDE>, Vec>>, ) where diff --git a/src/persistence/space/logical_index.rs b/src/persistence/space/logical_index.rs index e2e948fc..7c3faa75 100644 --- a/src/persistence/space/logical_index.rs +++ b/src/persistence/space/logical_index.rs @@ -5,10 +5,12 @@ //! this persistence-worker-owned index derives the structural events required //! by the unchanged WTI disk format. -use std::fmt::Debug; -use std::hash::Hash; +use alloc::{string::String, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; use std::path::{Path, PathBuf}; +use crate::fsx::File; use data_bucket::{Link, SizeMeasurable, SpaceId, VariableSizeMeasurable}; use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; @@ -23,7 +25,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; use crate::UnsizedNode; use crate::convert_multi_change_events; @@ -158,25 +159,25 @@ where /// Sized-key WTI persistence with foreground logical CDC and a background /// structural shadow. The wrapped `SpaceIndex` retains the existing file /// layout byte-for-byte. -pub struct SpaceLogicalIndex +pub struct SpaceLogicalIndex where T: Send + Ord + Eq + Clone + 'static, { index_path: PathBuf, shadow: BTreeMap, - disk: SpaceIndex, + disk: SpaceIndex, } -impl Debug for SpaceLogicalIndex +impl Debug for SpaceLogicalIndex where T: Send + Ord + Eq + Clone + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter.debug_struct("SpaceLogicalIndex").finish_non_exhaustive() } } -impl SpaceLogicalIndex +impl SpaceLogicalIndex where T: Archive + Ord @@ -208,7 +209,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalIndex +impl SpaceIndexOps + for SpaceLogicalIndex where T: Archive + Ord @@ -245,7 +247,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndex::::bootstrap(file, table_name, version).await + SpaceIndex::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -260,27 +262,27 @@ where } /// Variable-sized-key counterpart to [`SpaceLogicalIndex`]. -pub struct SpaceLogicalIndexUnsized +pub struct SpaceLogicalIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { index_path: PathBuf, shadow: BTreeMap>>, - disk: SpaceIndexUnsized, + disk: SpaceIndexUnsized, } -impl Debug for SpaceLogicalIndexUnsized +impl Debug for SpaceLogicalIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter .debug_struct("SpaceLogicalIndexUnsized") .finish_non_exhaustive() } } -impl SpaceLogicalIndexUnsized +impl SpaceLogicalIndexUnsized where T: Archive + Ord @@ -313,7 +315,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalIndexUnsized +impl SpaceIndexOps + for SpaceLogicalIndexUnsized where T: Archive + Ord @@ -351,7 +354,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndexUnsized::::bootstrap(file, table_name, version).await + SpaceIndexUnsized::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -368,25 +371,25 @@ where /// Sized-key WTI persistence for a non-unique runtime backend that emits /// logical `(key, link)` mutations. The WTI file layout and its node topology /// remain compatible with earlier WorkTable releases. -pub struct SpaceLogicalMultiIndex +pub struct SpaceLogicalMultiIndex where T: Debug + Send + Ord + Eq + Clone + 'static, { index_path: PathBuf, shadow: BTreeMultiMap, - disk: SpaceIndex, + disk: SpaceIndex, } -impl Debug for SpaceLogicalMultiIndex +impl Debug for SpaceLogicalMultiIndex where T: Debug + Send + Ord + Eq + Clone + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter.debug_struct("SpaceLogicalMultiIndex").finish_non_exhaustive() } } -impl SpaceLogicalMultiIndex +impl SpaceLogicalMultiIndex where T: Archive + Ord @@ -418,7 +421,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalMultiIndex +impl SpaceIndexOps + for SpaceLogicalMultiIndex where T: Archive + Ord @@ -455,7 +459,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndex::::bootstrap(file, table_name, version).await + SpaceIndex::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -470,27 +474,28 @@ where } /// Variable-sized-key counterpart to [`SpaceLogicalMultiIndex`]. -pub struct SpaceLogicalMultiIndexUnsized +pub struct SpaceLogicalMultiIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { index_path: PathBuf, shadow: BTreeMultiMap>>, - disk: SpaceIndexUnsized, + disk: SpaceIndexUnsized, } -impl Debug for SpaceLogicalMultiIndexUnsized +impl Debug + for SpaceLogicalMultiIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter .debug_struct("SpaceLogicalMultiIndexUnsized") .finish_non_exhaustive() } } -impl SpaceLogicalMultiIndexUnsized +impl SpaceLogicalMultiIndexUnsized where T: Archive + Ord @@ -523,7 +528,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalMultiIndexUnsized +impl SpaceIndexOps + for SpaceLogicalMultiIndexUnsized where T: Archive + Ord @@ -561,7 +567,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndexUnsized::::bootstrap(file, table_name, version).await + SpaceIndexUnsized::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -577,7 +583,7 @@ where #[cfg(test)] mod tests { - use std::collections::BTreeMap as StdBTreeMap; + use alloc::collections::BTreeMap as StdBTreeMap; use data_bucket::page::PageId; diff --git a/src/persistence/space/mod.rs b/src/persistence/space/mod.rs index 4f9d280b..35bb6f62 100644 --- a/src/persistence/space/mod.rs +++ b/src/persistence/space/mod.rs @@ -1,17 +1,18 @@ +use alloc::{string::String, vec::Vec}; mod art_index; mod data; mod index; mod logical_index; -use std::collections::HashMap; -use std::future::Future; +use core::future::Future; +use hashbrown::HashMap; use std::path::Path; +use crate::fsx::File; use data_bucket::page::PageId; use data_bucket::{GeneralPage, Link, SpaceInfoPage}; use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; -use tokio::fs::{File, OpenOptions}; pub use art_index::{ ArtPersistenceKey, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, @@ -88,10 +89,9 @@ pub trait SpaceSecondaryIndexOps { pub async fn open_or_create_file>(path: S) -> eyre::Result { let path = Path::new(path.as_ref()); - Ok(OpenOptions::new() - .write(true) - .read(true) - .create(!path.exists()) - .open(path) - .await?) + Ok(if path.exists() { + crate::fsx::open(path).await? + } else { + crate::fsx::open_or_create(path).await? + }) } diff --git a/src/persistence/task.rs b/src/persistence/task.rs index edba8f5c..4aa521d7 100644 --- a/src/persistence/task.rs +++ b/src/persistence/task.rs @@ -1,11 +1,14 @@ -use std::collections::{HashMap, HashSet, VecDeque}; -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; -use std::panic::Location; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::time::Duration; +use alloc::boxed::Box; +use alloc::collections::VecDeque; +use alloc::sync::Arc; +use alloc::{borrow::ToOwned, string::String, string::ToString, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; +use core::panic::Location; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use core::time::Duration; +use hashbrown::{HashMap, HashSet}; use data_bucket::page::PageId; use parking_lot::Mutex as ParkingMutex; @@ -542,8 +545,8 @@ where #[cfg(test)] mod lifecycle_tests { - use std::collections::HashMap; - use std::sync::atomic::{AtomicUsize, Ordering}; + use core::sync::atomic::{AtomicUsize, Ordering}; + use hashbrown::HashMap; use super::*; @@ -570,7 +573,7 @@ mod lifecycle_tests { } fn iter_event_ids(&self) -> impl Iterator { - std::iter::empty() + core::iter::empty() } fn sort(&mut self) {} @@ -789,7 +792,7 @@ mod lifecycle_tests { .unwrap(); assert_eq!( - batch.get(&1.into()).unwrap(), + batch.get(&PageId::from(1u32)).unwrap(), &vec![ ( Link { @@ -853,7 +856,7 @@ mod lifecycle_tests { .get_batch_data_op() .unwrap(); - let page_one_writes = batch.get(&1.into()).unwrap(); + let page_one_writes = batch.get(&PageId::from(1u32)).unwrap(); assert_eq!( page_one_writes, &vec![ @@ -877,7 +880,7 @@ mod lifecycle_tests { "the complete earlier group must be applied" ); assert!( - !batch.contains_key(&2.into()), + !batch.contains_key(&PageId::from(2u32)), "the blocking group must stay queued, not be applied without its earlier events" ); assert_eq!(analyzer.len(), 2, "both rows of the blocked group remain queued"); @@ -1266,7 +1269,7 @@ pub struct Queue { /// [`event_ledger::enabled`]. event_ledger: Arc, #[cfg(test)] - pop_race_window_gate: Option>, + pop_race_window_gate: Option>, } impl Queue { @@ -1564,14 +1567,15 @@ impl /// This is intentionally separate from `VacuumStats`: online vacuum makes /// freed pages durably reusable, but does not truncate `.wt.data`. /// Operators can sample this value to observe physical growth and reuse. - pub async fn persisted_data_file_size_bytes(&self) -> std::io::Result { - tokio::fs::metadata(format!( + pub async fn persisted_data_file_size_bytes(&self) -> Result { + // `metadata` answers with the length, which is the only thing anything + // here ever asked a metadata handle for. + crate::fsx::metadata(format!( "{}/{}", self.table_path.trim_end_matches('/'), WT_DATA_EXTENSION )) .await - .map(|metadata| metadata.len()) } /// Returns a sink that lets vacuum queue persistence operations for row diff --git a/src/primary_key.rs b/src/primary_key.rs index ff73bbec..ab4469bd 100644 --- a/src/primary_key.rs +++ b/src/primary_key.rs @@ -1,4 +1,4 @@ -use std::sync::atomic::{ +use core::sync::atomic::{ AtomicI8, AtomicI16, AtomicI32, AtomicI64, AtomicU8, AtomicU16, AtomicU32, AtomicU64, Ordering, }; @@ -21,7 +21,7 @@ pub trait PrimaryKeyGeneratorRange { /// /// Concurrent `reserve` and [`PrimaryKeyGenerator::next`] calls never /// observe overlapping keys. - fn reserve(&self, count: usize) -> std::ops::Range; + fn reserve(&self, count: usize) -> core::ops::Range; } pub trait PrimaryKeyGeneratorState { @@ -52,7 +52,7 @@ macro_rules! atomic_primary_key { } impl PrimaryKeyGeneratorRange<$ty> for $atomic_ty { - fn reserve(&self, count: usize) -> std::ops::Range<$ty> { + fn reserve(&self, count: usize) -> core::ops::Range<$ty> { let count = <$ty>::try_from(count).unwrap_or_else(|_| { panic!( "autoincrement primary key space exhausted: cannot reserve {count} {} keys", @@ -142,7 +142,7 @@ mod tests { #[test] fn concurrent_reservations_never_overlap() { - use std::sync::Arc; + use alloc::sync::Arc; let generator = Arc::new(AtomicU64::from_state(0)); let mut handles = vec![]; diff --git a/src/table/mod.rs b/src/table/mod.rs index eeba4d01..9c1d77f8 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -1,9 +1,13 @@ +use alloc::{string::String, vec::Vec}; pub mod select; pub mod system_info; +#[cfg(feature = "std")] pub mod vacuum; use crate::in_memory::{ArchivedRowWrapper, DataPages, RowWrapper, StorableRow}; -use crate::persistence::{AcknowledgeOperation, InsertOperation, Operation, PersistenceLoadError}; +#[cfg(feature = "std")] +use crate::persistence::PersistenceLoadError; +use crate::persistence::{AcknowledgeOperation, InsertOperation, Operation}; use crate::prelude::{Link, LockMap, OperationId, PrimaryKeyGeneratorState}; use crate::primary_key::{PrimaryKeyGenerator, TablePrimaryKey}; use crate::util::OffsetEqLink; @@ -11,8 +15,13 @@ use crate::{ AvailableIndex, IndexError, IndexMap, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, convert_change_events, in_memory, }; +use alloc::sync::Arc; +use core::fmt::Debug; +use core::marker::PhantomData; use data_bucket::INNER_PAGE_SIZE; use derive_more::{Display, Error, From}; +#[cfg(feature = "std")] +use hashbrown::HashSet; use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; #[cfg(feature = "perf_measurements")] @@ -24,11 +33,8 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Portable, Serialize}; -use std::collections::HashSet; -use std::fmt::Debug; -use std::marker::PhantomData; +#[cfg(feature = "std")] use std::path::Path; -use std::sync::Arc; use uuid::Uuid; /// Keys per chunk when a bulk delete takes its mutation guards. /// @@ -51,7 +57,7 @@ pub struct WorkTable< const DATA_LENGTH: usize = INNER_PAGE_SIZE, PkMap = IndexMap>, > where - PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static, PkMap: crate::UniqueIndex>, { @@ -94,7 +100,7 @@ impl< PkMap, > where - PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + core::hash::Hash, SecondaryIndexes: Default, PkGen: Default, PkMap: crate::UniqueIndex>, @@ -127,7 +133,7 @@ impl< > WorkTable where Row: TableRow, - PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + core::hash::Hash, PkMap: crate::UniqueIndex>, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, @@ -138,6 +144,7 @@ where /// This load-only scan prevents a torn index link from turning zeroed or /// unrelated bytes into a plausible row. It deliberately does not run on /// steady-state operations. + #[cfg(feature = "std")] pub fn validate_persisted_state(&self, path: impl AsRef) -> Result<(), PersistenceLoadError> where <::WrappedRow as Archive>::Archived: Portable @@ -146,7 +153,7 @@ where { let path = path.as_ref(); let mut links = HashSet::with_capacity(self.primary_index.pk_map.len()); - let mut cells_by_page = std::collections::HashMap::::new(); + let mut cells_by_page = hashbrown::HashMap::::new(); for (primary_key, offset_link) in self.primary_index.pk_map.iter_values() { if !links.insert(offset_link) { @@ -211,7 +218,7 @@ where /// caller can iterate it directly while pre-assigning contiguous keys to a /// batch of rows for `insert_many`. Interleaved [`Self::get_next_pk`] /// calls keep working and never overlap a reservation. - pub fn reserve_pks(&self, count: usize) -> std::ops::Range + pub fn reserve_pks(&self, count: usize) -> core::ops::Range where PkGen: crate::primary_key::PrimaryKeyGeneratorRange, { @@ -238,7 +245,7 @@ where if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None } @@ -475,7 +482,7 @@ where /// Returns the keys actually deleted, in key order. pub fn delete_range(&self, range: R) -> Result, BatchDeleteError> where - R: std::ops::RangeBounds, + R: core::ops::RangeBounds, Row: Archive + Clone + for<'a> Serialize, Share>, rkyv::rancor::Error>>, @@ -531,8 +538,8 @@ where .primary_index .pk_map .range_values(( - std::ops::Bound::Included(chunk[0].clone()), - std::ops::Bound::Included(chunk[chunk.len() - 1].clone()), + core::ops::Bound::Included(chunk[0].clone()), + core::ops::Bound::Included(chunk[chunk.len() - 1].clone()), )) .map(|(key, link)| (key, link.into())) .collect(); @@ -1089,8 +1096,8 @@ where ops.push(Operation::Insert(InsertOperation { id: OperationId::Multi(batch_id), pk_gen_state: self.pk_gen.get_state(), - primary_key_events: std::mem::take(&mut forward_primary[row_index]), - secondary_keys_events: std::mem::take(&mut forward_secondary[row_index]), + primary_key_events: core::mem::take(&mut forward_primary[row_index]), + secondary_keys_events: core::mem::take(&mut forward_secondary[row_index]), bytes, link: *link, })); @@ -1349,7 +1356,7 @@ pub enum BatchInsertError { /// batch needs to know the prefix already succeeded rather than assume nothing /// happened. #[derive(Debug, Display, Error)] -pub enum BatchDeleteError { +pub enum BatchDeleteError { /// One key could not be deleted. Everything before it was. #[display("batch delete stopped at {key:?} after {deleted} deleted: {source}")] Key { @@ -1376,5 +1383,6 @@ pub enum WorkTableError { PrimaryUpdateTry, PagesError(in_memory::PagesExecutionError), #[display("{}", _0)] - PersistenceError(#[error(not(source))] std::sync::Arc), + #[cfg(feature = "std")] + PersistenceError(#[error(not(source))] alloc::sync::Arc), } diff --git a/src/table/select/mod.rs b/src/table/select/mod.rs index 5b9fc6e6..fe7de3ed 100644 --- a/src/table/select/mod.rs +++ b/src/table/select/mod.rs @@ -1,4 +1,4 @@ -use std::collections::VecDeque; +use alloc::collections::VecDeque; mod query; diff --git a/src/table/select/query.rs b/src/table/select/query.rs index 2b2f3f66..41bd6a01 100644 --- a/src/table/select/query.rs +++ b/src/table/select/query.rs @@ -1,7 +1,8 @@ use crate::WorkTableError; use crate::select::{Order, QueryParams}; +use alloc::vec::Vec; -use std::collections::VecDeque; +use alloc::collections::VecDeque; pub struct SelectQueryBuilder where diff --git a/src/table/system_info.rs b/src/table/system_info.rs index 49e95761..109f9e7c 100644 --- a/src/table/system_info.rs +++ b/src/table/system_info.rs @@ -1,5 +1,5 @@ -use prettytable::{Table, format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR, row}; -use std::fmt::{self, Debug, Display, Formatter}; +use alloc::{string::String, string::ToString, vec::Vec}; +use core::fmt::{self, Debug, Display, Formatter}; use crate::in_memory::{RowWrapper, StorableRow}; use crate::mem_stat::MemStat; @@ -55,7 +55,7 @@ impl< PkMap, > WorkTable where - PrimaryKey: Debug + Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, PkMap: UniqueIndex>, @@ -118,31 +118,76 @@ impl Display for SystemInfo { "Allocated Memory: {mem_fmt} (data) + {idx_fmt} (indexes) = {total_fmt} total\n" )?; - let mut table = Table::new(); - table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR); - table.add_row(row!["Index", "Type", "Keys", "Capacity", "Node Count", "Heap", "Used"]); - + // **Padded by hand rather than by a table crate.** + // + // This used to be `prettytable-rs`, which reaches `csv` and then + // `memchr` and fails to build without `std` in 505 places. Every + // alternative measured puts its usable API behind `std` too: `tabled` + // compiles without `std` but exposes no `Table` at all in that mode, + // and `comfy-table` and `ascii_table` do not compile. Seven columns of + // short strings are not worth a dependency that decides whether this + // crate can be embedded. + let mut rows: Vec<[String; COLUMNS]> = Vec::with_capacity(self.indexes_info.len() + 1); + rows.push([ + "Index".to_string(), + "Type".to_string(), + "Keys".to_string(), + "Capacity".to_string(), + "Node Count".to_string(), + "Heap".to_string(), + "Used".to_string(), + ]); for idx in &self.indexes_info { - table.add_row(row![ - idx.name, + rows.push([ + idx.name.to_string(), idx.index_type.to_string(), - idx.key_count, - idx.capacity, - idx.node_count, + idx.key_count.to_string(), + idx.capacity.to_string(), + idx.node_count.to_string(), fmt_bytes(idx.heap_size), fmt_bytes(idx.used_size), ]); } - let mut buffer = Vec::new(); - table.print(&mut buffer).unwrap(); - let table_str = String::from_utf8(buffer).unwrap(); - writeln!(f, "{}", table_str.trim_end())?; + // Width by character count, not byte length: an index named with any + // multi-byte character would otherwise pad short and skew every column + // after it. + let mut widths = [0usize; COLUMNS]; + for row in &rows { + for (width, cell) in widths.iter_mut().zip(row) { + *width = (*width).max(cell.chars().count()); + } + } + + for (index, row) in rows.iter().enumerate() { + for (column, (cell, width)) in row.iter().zip(&widths).enumerate() { + if column + 1 == COLUMNS { + write!(f, "{cell}")?; + } else { + let padding = width - cell.chars().count(); + write!(f, "{cell}{:padding$}{COLUMN_GAP}", "")?; + } + } + writeln!(f)?; + // A rule under the header, and nothing between the rows: the same + // shape `FORMAT_NO_BORDER_LINE_SEPARATOR` produced. + if index == 0 { + let rule: usize = widths.iter().sum::() + COLUMN_GAP.len() * (COLUMNS - 1); + writeln!(f, "{:- String { const KB: f64 = 1024.0; const MB: f64 = 1024.0 * KB; diff --git a/src/table/vacuum/fragmentation_info.rs b/src/table/vacuum/fragmentation_info.rs index 6541e014..68d0fb92 100644 --- a/src/table/vacuum/fragmentation_info.rs +++ b/src/table/vacuum/fragmentation_info.rs @@ -12,7 +12,8 @@ //! //! [`WorkTable`]: crate::table::WorkTable -use std::collections::HashMap; +use alloc::vec::Vec; +use hashbrown::HashMap; use data_bucket::page::PageId; use data_bucket::{INNER_PAGE_SIZE, Link}; diff --git a/src/table/vacuum/manager.rs b/src/table/vacuum/manager.rs index 566ed452..c7520644 100644 --- a/src/table/vacuum/manager.rs +++ b/src/table/vacuum/manager.rs @@ -1,7 +1,8 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; +use alloc::sync::Arc; +use alloc::{string::ToString, vec::Vec}; +use core::sync::atomic::{AtomicU64, Ordering}; +use core::time::Duration; +use hashbrown::HashMap; use tokio::task::AbortHandle; use parking_lot::RwLock; diff --git a/src/table/vacuum/mod.rs b/src/table/vacuum/mod.rs index 74fc55db..1c94e20f 100644 --- a/src/table/vacuum/mod.rs +++ b/src/table/vacuum/mod.rs @@ -1,3 +1,5 @@ +use alloc::boxed::Box; +use alloc::vec::Vec; use async_trait::async_trait; use data_bucket::Link; diff --git a/src/table/vacuum/pacing.rs b/src/table/vacuum/pacing.rs index 1ff64c9d..ea4bf81f 100644 --- a/src/table/vacuum/pacing.rs +++ b/src/table/vacuum/pacing.rs @@ -15,8 +15,8 @@ //! the preceding check. Every insert, delete and upsert passes through those //! stripes, including mutations that never ask for reclaimable space. -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::time::Duration; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use core::time::Duration; use smart_default::SmartDefault; @@ -114,7 +114,7 @@ pub trait ForegroundActivity { impl ForegroundActivity for crate::lock::LockMap where - PrimaryKey: Clone + std::fmt::Debug + Eq + std::hash::Hash, + PrimaryKey: Clone + core::fmt::Debug + Eq + core::hash::Hash, { fn mutations_in_flight(&self) -> usize { crate::lock::LockMap::mutations_in_flight(self) @@ -163,8 +163,8 @@ impl VacuumPacing { #[cfg(test)] mod tests { - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + use alloc::sync::Arc; + use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use super::*; diff --git a/src/table/vacuum/vacuum.rs b/src/table/vacuum/vacuum.rs index 47030bca..b95b787c 100644 --- a/src/table/vacuum/vacuum.rs +++ b/src/table/vacuum/vacuum.rs @@ -1,9 +1,12 @@ -use std::collections::VecDeque; -use std::fmt::Debug; -use std::marker::PhantomData; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; +use alloc::boxed::Box; +use alloc::collections::VecDeque; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::marker::PhantomData; +use core::sync::atomic::{AtomicU64, Ordering}; +use core::time::Duration; +use std::time::Instant; /// How long retirements have to stop arriving for a delete burst to count as /// over. Short enough that a sweep still follows a delete promptly, long @@ -85,7 +88,7 @@ pub struct EmptyDataVacuum< const DATA_LENGTH: usize, SecondaryEvents = (), > where - PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static + Debug, PkMap: UniqueIndex>, { @@ -139,7 +142,7 @@ impl< > where Row: TableRow + StorableRow + Send + Clone + 'static, - PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + core::hash::Hash, PkMap: UniqueIndex>, ::WrappedRow: RowWrapper, Row: Archive @@ -684,7 +687,7 @@ impl< > where Row: TableRow + StorableRow + Send + Sync + Clone + 'static, - PrimaryKey: Debug + Clone + Ord + Send + Sync + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + Sync + TablePrimaryKey + core::hash::Hash, PkMap: UniqueIndex> + Send + Sync + 'static, ::WrappedRow: RowWrapper, Row: Archive @@ -737,9 +740,10 @@ where #[cfg(test)] mod tests { - use std::collections::{HashMap, VecDeque}; - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use alloc::collections::VecDeque; + use alloc::sync::Arc; + use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use hashbrown::HashMap; use data_bucket::Link; use data_bucket::page::PageId; @@ -747,7 +751,7 @@ mod tests { use crate::in_memory::{ArchivedRowWrapper, RowWrapper, StorableRow}; use crate::prelude::*; - use std::time::Duration; + use core::time::Duration; use crate::vacuum::vacuum::{CandidateMove, EmptyDataVacuum}; use crate::vacuum::{VacuumGate, VacuumPacing, WorkTableVacuum}; diff --git a/src/util/mod.rs b/src/util/mod.rs index 7b1adf10..ac5cc7a4 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,8 +1,22 @@ pub(crate) mod epoch; mod offset_eq_link; +#[cfg(feature = "std")] mod optimized_vec; mod ordered_float; pub use offset_eq_link::OffsetEqLink; +#[cfg(feature = "std")] pub use optimized_vec::OptimizedVec; pub use ordered_float::{OrderedF32Def, OrderedF64Def}; + +/// Give up the rest of the timeslice after a spin has stopped paying. +/// +/// Under `std` that is the operating system's yield. Without one there is no +/// scheduler to yield to, so the spin hint is the whole of what can be done. +#[inline] +pub(crate) fn yield_now() { + #[cfg(feature = "std")] + std::thread::yield_now(); + #[cfg(not(feature = "std"))] + core::hint::spin_loop(); +} diff --git a/src/util/offset_eq_link.rs b/src/util/offset_eq_link.rs index 0b39d0db..a1c3c3c5 100644 --- a/src/util/offset_eq_link.rs +++ b/src/util/offset_eq_link.rs @@ -22,20 +22,20 @@ impl OffsetEqLink { } } -impl std::hash::Hash for OffsetEqLink { - fn hash(&self, state: &mut H) { +impl core::hash::Hash for OffsetEqLink { + fn hash(&self, state: &mut H) { self.absolute_index().hash(state); } } impl PartialOrd for OffsetEqLink { - fn partial_cmp(&self, other: &Self) -> Option { + fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl Ord for OffsetEqLink { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { self.absolute_index().cmp(&other.absolute_index()) } } @@ -48,7 +48,7 @@ impl PartialEq for OffsetEqLink { impl Eq for OffsetEqLink {} -impl std::ops::Deref for OffsetEqLink { +impl core::ops::Deref for OffsetEqLink { type Target = Link; fn deref(&self) -> &Self::Target { @@ -85,7 +85,7 @@ impl SizeMeasurable for OffsetEqLink { mod tests { use super::*; use data_bucket::page::PageId; - use std::collections::HashSet; + use hashbrown::HashSet; const TEST_DATA_LENGTH: usize = 4096; diff --git a/src/util/optimized_vec.rs b/src/util/optimized_vec.rs index c26114d0..9570ec9b 100644 --- a/src/util/optimized_vec.rs +++ b/src/util/optimized_vec.rs @@ -1,3 +1,4 @@ +use alloc::vec::Vec; /// Struct for storing data in a vector with stable indexes and slot reuse. /// Slots are `Option`: `remove` is `Option::take`, so the value moves out /// without a `Clone` bound and the slot is freed immediately. The previous @@ -201,7 +202,7 @@ mod tests { /// count proves the removed value is the only remaining owner. #[test] fn test_optimized_vec_remove_moves_without_clone() { - use std::rc::Rc; + use alloc::rc::Rc; struct NotClone(#[allow(dead_code)] Rc<()>); diff --git a/src/util/ordered_float.rs b/src/util/ordered_float.rs index 598be510..29ff13e3 100644 --- a/src/util/ordered_float.rs +++ b/src/util/ordered_float.rs @@ -4,7 +4,7 @@ use rkyv::{Archive, Deserialize, Serialize}; #[rkyv(remote = ordered_float::OrderedFloat, archived = ArchivedF64)] #[rkyv(derive(Debug))] pub struct OrderedF64Def { - #[rkyv(getter = std::ops::Deref::deref)] + #[rkyv(getter = core::ops::Deref::deref)] value: f64, } @@ -18,7 +18,7 @@ impl From for ordered_float::OrderedFloat { #[rkyv(remote = ordered_float::OrderedFloat, archived = ArchivedF32)] #[rkyv(derive(Debug))] pub struct OrderedF32Def { - #[rkyv(getter = std::ops::Deref::deref)] + #[rkyv(getter = core::ops::Deref::deref)] value: f32, } diff --git a/tests/mod.rs b/tests/mod.rs index 86ec0ed7..2e1e3256 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -46,12 +46,12 @@ pub fn check_if_dirs_are_same(got: String, expected: String) -> bool { pub async fn remove_file_if_exists(path: String) { if Path::new(path.as_str()).exists() { - tokio::fs::remove_file(path.as_str()).await.unwrap(); + ::worktable::prelude::fsx::remove_file(path.as_str()).await.unwrap(); } } pub async fn remove_dir_if_exists(path: String) { if Path::new(path.as_str()).exists() { - tokio::fs::remove_dir_all(path).await.unwrap() + ::worktable::prelude::fsx::remove_dir_all(path).await.unwrap() } } diff --git a/tests/persistence/custom_page_size.rs b/tests/persistence/custom_page_size.rs new file mode 100644 index 00000000..b7563cae --- /dev/null +++ b/tests/persistence/custom_page_size.rs @@ -0,0 +1,70 @@ +use worktable::prelude::*; +use worktable::worktable; + +use crate::remove_dir_if_exists; + +/// Half the default stride. Chosen so a page written at this size lands where +/// the default would put the middle of a page: a table that quietly fell back +/// to the default could not produce the file lengths asserted below. +const HALF: u32 = 8192; + +worktable! ( + name: HalfPage, + persist: true, + columns: { + id: u64 primary_key, + payload: u64, + }, + config: { + page_size: 8192 + } +); + +/// `page_size` beside `persist: true` used to be refused outright: the seeks +/// computed every offset from a hardcoded stride while the generated table +/// threaded the configured one, and the two disagreeing corrupted the file. +/// +/// A round trip alone would not prove the setting took effect, because a table +/// that fell back to the default would still reload its own writes. So the +/// file length is checked too, and it can only be a multiple of the configured +/// stride. +#[tokio::test] +async fn a_persisted_table_with_a_custom_page_size_reloads_what_it_wrote() { + let dir = "tests/data/custom_page_size/persisted"; + remove_dir_if_exists(dir.to_string()).await; + let config = + DiskConfig::new_with_table_name(dir, HalfPageWorkTable::name_snake_case(), HalfPageWorkTable::version()); + + let mut expected = Vec::new(); + { + let engine = HalfPagePersistenceEngine::new(config.clone()).await.unwrap(); + let table = HalfPageWorkTable::load(engine).await.unwrap(); + // Enough rows to need several pages at this stride, so the page + // transitions are exercised and not just the first page. + for i in 0..2_000u64 { + let row = HalfPageRow { id: i, payload: i }; + table.insert(row.clone()).await.unwrap(); + expected.push(row); + } + table.wait_for_ops().await.unwrap(); + } + + let data_file_path = format!("{dir}/{}/.wt.data", HalfPageWorkTable::name_snake_case()); + let length = std::fs::metadata(&data_file_path).unwrap().len(); + let pages = length.div_ceil(u64::from(HALF)); + assert!( + pages >= 2, + "expected several {HALF}-byte pages, got {pages} from {length} bytes" + ); + + let engine = HalfPagePersistenceEngine::new(config).await.unwrap(); + let table = HalfPageWorkTable::load(engine) + .await + .expect("a table with a custom page size must reload"); + assert_eq!(table.select_all().execute().unwrap().len(), expected.len()); + for row in &expected { + assert_eq!(table.select(row.id).as_ref(), Some(row)); + } + + remove_dir_if_exists(dir.to_string()).await; +} diff --git a/tests/persistence/duplicate_key_index_reload.rs b/tests/persistence/duplicate_key_index_reload.rs index d3b41405..c241c9b9 100644 --- a/tests/persistence/duplicate_key_index_reload.rs +++ b/tests/persistence/duplicate_key_index_reload.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::collections::{BTreeMap, BTreeSet}; use std::time::Duration; @@ -111,12 +112,11 @@ async fn assert_straddling_topology(dir: &str) { use data_bucket::INNER_PAGE_SIZE; use std::sync::Arc; use std::sync::atomic::AtomicU32; - use tokio::fs::OpenOptions; let path = format!("{dir}/duplicate_key_reload/score_idx.wt.idx"); - let mut file = OpenOptions::new().read(true).write(true).open(&path).await.unwrap(); + let mut file = worktable::prelude::fsx::open(&path).await.unwrap(); let next_id_gen = Arc::new(AtomicU32::new(1)); - let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -133,9 +133,12 @@ async fn assert_straddling_topology(dir: &str) { let mut pages_per_key: BTreeMap = BTreeMap::new(); for page_id in mappings { - let page = parse_page::, { DUPLICATE_KEY_RELOAD_PAGE_SIZE as u32 }>(&mut file, page_id.into()) - .await - .unwrap(); + let page = parse_page::, { DUPLICATE_KEY_RELOAD_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, + page_id.into(), + ) + .await + .unwrap(); let keys: BTreeSet = page.inner.index_values[..page.inner.current_length as usize] .iter() .map(|v| v.key) diff --git a/tests/persistence/index_page/read.rs b/tests/persistence/index_page/read.rs index 5cc90005..ebc19d28 100644 --- a/tests/persistence/index_page/read.rs +++ b/tests/persistence/index_page/read.rs @@ -1,16 +1,13 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::{INNER_PAGE_SIZE, IndexPage, parse_page}; -use tokio::fs::OpenOptions; #[tokio::test] async fn test_index_page_read_in_space() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/test_persist/primary.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/primary.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 99); @@ -20,14 +17,11 @@ async fn test_index_page_read_in_space() { #[tokio::test] async fn test_index_page_read_after_create_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -40,14 +34,11 @@ async fn test_index_page_read_after_create_node_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -64,14 +55,12 @@ async fn test_index_page_read_after_insert_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") + .await + .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 7); @@ -88,14 +77,11 @@ async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index #[tokio::test] async fn test_index_page_read_after_remove_at_node_id_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 3); @@ -109,14 +95,11 @@ async fn test_index_page_read_after_remove_at_node_id_in_space_index() { #[tokio::test] async fn test_index_page_read_after_remove_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_at.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -130,14 +113,12 @@ async fn test_index_page_read_after_remove_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_removed_place.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_removed_place.wt.idx") + .await + .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 7); @@ -159,14 +140,11 @@ async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_second_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_second_node.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -176,7 +154,7 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { assert_eq!(page.inner.index_values.first().unwrap().key, 5); assert_eq!(page.inner.index_values.first().unwrap().link.length, 24); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 3) .await .unwrap(); assert_eq!(page.inner.node_id.key, 15); @@ -189,14 +167,12 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_creation_of_node_after_remove_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") + .await + .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 10); @@ -206,7 +182,7 @@ async fn test_index_pages_read_after_creation_of_node_after_remove_node_in_space assert_eq!(page.inner.index_values.first().unwrap().key, 10); assert_eq!(page.inner.index_values.first().unwrap().link.length, 24); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 3) .await .unwrap(); assert_eq!(page.inner.node_id.key, 15); @@ -219,14 +195,11 @@ async fn test_index_pages_read_after_creation_of_node_after_remove_node_in_space #[tokio::test] async fn test_index_pages_read_full_page() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_big_amount.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_big_amount.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 1000); @@ -236,21 +209,18 @@ async fn test_index_pages_read_full_page() { #[tokio::test] async fn test_index_pages_read_after_node_split() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_split_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_split_node.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 457); assert_eq!(page.inner.current_index, 0); assert_eq!(page.inner.current_length, 453); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 3) .await .unwrap(); assert_eq!(page.inner.node_id.key, 1000); diff --git a/tests/persistence/index_page/unsized_read.rs b/tests/persistence/index_page/unsized_read.rs index e4673d55..c2e2fec4 100644 --- a/tests/persistence/index_page/unsized_read.rs +++ b/tests/persistence/index_page/unsized_read.rs @@ -1,19 +1,18 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::{INNER_PAGE_SIZE, Link, UnsizedIndexPage, parse_page}; -use tokio::fs::OpenOptions; #[tokio::test] async fn test_index_page_read_after_create_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -31,18 +30,18 @@ async fn test_index_page_read_after_create_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") - .await - .unwrap(); - - let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") .await .unwrap(); + let page = + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); + assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 1); let value = page.inner.index_values.first().unwrap(); @@ -57,9 +56,11 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { ); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 3, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Someone from somewhere".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -77,17 +78,16 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_remove_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 3, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Someone from somewhere".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -105,17 +105,16 @@ async fn test_index_pages_read_after_remove_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_insert_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 2); @@ -143,17 +142,16 @@ async fn test_index_pages_read_after_insert_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_remove_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -171,18 +169,18 @@ async fn test_index_page_read_after_remove_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_remove_at_node_id_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") - .await - .unwrap(); - - let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") .await .unwrap(); + let page = + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); + assert_eq!(page.inner.node_id.key, "Something else".to_string()); assert_eq!(page.inner.index_values.len(), 1); let value = page.inner.index_values.first().unwrap(); @@ -199,17 +197,18 @@ async fn test_index_page_read_after_remove_at_node_id_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at_with_node_id_update.wt.idx") - .await - .unwrap(); + let mut file = worktable::prelude::fsx::open( + "tests/data/expected/space_index_unsized/process_insert_at_with_node_id_update.wt.idx", + ) + .await + .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone 1".to_string()); assert_eq!(page.inner.index_values.len(), 2); @@ -237,18 +236,18 @@ async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index #[tokio::test] async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at_removed_place.wt.idx") - .await - .unwrap(); - - let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_insert_at_removed_place.wt.idx") .await .unwrap(); + let page = + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); + assert_eq!(page.inner.node_id.key, "Something from someone 1".to_string()); assert_eq!(page.inner.index_values.len(), 3); let first_value = &page.inner.index_values[0]; @@ -285,24 +284,25 @@ async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_node_split() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_split_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_split_node.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone 52"); assert_eq!(page.inner.slots_size, 53); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 3, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone _100"); assert_eq!(page.inner.slots_size, 48); } diff --git a/tests/persistence/mod.rs b/tests/persistence/mod.rs index 9b2535a9..1bdb95ea 100644 --- a/tests/persistence/mod.rs +++ b/tests/persistence/mod.rs @@ -5,6 +5,7 @@ use worktable::worktable; mod bulk_delete_durability; mod bulk_load_stall; mod concurrent; +mod custom_page_size; mod duplicate_key_index_reload; mod exact_boundary_load; mod failure; diff --git a/tests/persistence/read.rs b/tests/persistence/read.rs index d56ec83c..49faf282 100644 --- a/tests/persistence/read.rs +++ b/tests/persistence/read.rs @@ -1,4 +1,4 @@ -use tokio::fs::File; +use data_bucket::DEFAULT_PAGE_STRIDE; use worktable::prelude::PersistedWorkTable; use worktable::prelude::*; @@ -10,8 +10,10 @@ use crate::remove_dir_if_exists; #[tokio::test] async fn test_info_parse() { - let mut file = File::open("tests/data/expected/test_persist/.wt.data").await.unwrap(); - let info = parse_page::, { TEST_PERSIST_INNER_SIZE as u32 }>(&mut file, 0) + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/.wt.data") + .await + .unwrap(); + let info = parse_page::, { TEST_PERSIST_INNER_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 0) .await .unwrap(); @@ -31,10 +33,10 @@ async fn test_info_parse() { #[tokio::test] async fn test_primary_index_parse() { - let mut file = File::open("tests/data/expected/test_persist/primary.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/primary.wt.idx") .await .unwrap(); - let index = parse_page::, { TEST_PERSIST_PAGE_SIZE as u32 }>(&mut file, 2) + let index = parse_page::, { TEST_PERSIST_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); @@ -66,10 +68,10 @@ async fn test_primary_index_parse() { #[tokio::test] async fn test_another_idx_index_parse() { - let mut file = File::open("tests/data/expected/test_persist/another_idx.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/another_idx.wt.idx") .await .unwrap(); - let index = parse_page::, { TEST_PERSIST_PAGE_SIZE as u32 }>(&mut file, 2) + let index = parse_page::, { TEST_PERSIST_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); @@ -101,10 +103,16 @@ async fn test_another_idx_index_parse() { #[tokio::test] async fn test_data_parse() { - let mut file = File::open("tests/data/expected/test_persist/.wt.data").await.unwrap(); - let data = parse_data_page::<{ TEST_PERSIST_PAGE_SIZE as u32 }, { TEST_PERSIST_INNER_SIZE }>(&mut file, 1) + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/.wt.data") .await .unwrap(); + let data = parse_data_page::< + { TEST_PERSIST_PAGE_SIZE as u32 }, + { TEST_PERSIST_INNER_SIZE }, + { TEST_PERSIST_PAGE_SIZE as u32 }, + >(&mut file, 1) + .await + .unwrap(); assert_eq!(data.header.space_id, 0.into()); assert_eq!(data.header.page_id, 1.into()); diff --git a/tests/persistence/recovery_load.rs b/tests/persistence/recovery_load.rs index 3cf8feaf..63726804 100644 --- a/tests/persistence/recovery_load.rs +++ b/tests/persistence/recovery_load.rs @@ -52,7 +52,7 @@ async fn recovery_mode_reads_valid_rows_through_a_surviving_secondary_index() { let table_dir = format!("{DIR}/{}", RecoveryLoadWorkTable::name_snake_case()); let primary_path = format!("{table_dir}/primary{WT_INDEX_EXTENSION}"); - tokio::fs::rename(&primary_path, format!("{primary_path}.damaged")) + worktable::prelude::fsx::rename(&primary_path, format!("{primary_path}.damaged")) .await .unwrap(); @@ -104,7 +104,7 @@ async fn recovery_mode_rejects_corrupt_rows_reached_through_a_secondary_index() let table_dir = format!("{CORRUPT_DIR}/{}", RecoveryLoadWorkTable::name_snake_case()); let primary_path = format!("{table_dir}/primary{WT_INDEX_EXTENSION}"); - tokio::fs::rename(&primary_path, format!("{primary_path}.damaged")) + worktable::prelude::fsx::rename(&primary_path, format!("{primary_path}.damaged")) .await .unwrap(); diff --git a/tests/persistence/s3/mod.rs b/tests/persistence/s3/mod.rs index 04957c96..c2f9040d 100644 --- a/tests/persistence/s3/mod.rs +++ b/tests/persistence/s3/mod.rs @@ -1,4 +1,6 @@ use crate::remove_dir_if_exists; +// A tokio `TcpStream`, so tokio's extension traits: this is the mock S3 +// server the test talks to, not the storage path the crate took off tokio. use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use tokio::task::JoinHandle; diff --git a/tests/persistence/schema.rs b/tests/persistence/schema.rs index c815a18a..33d886c3 100644 --- a/tests/persistence/schema.rs +++ b/tests/persistence/schema.rs @@ -1,7 +1,6 @@ -use tokio::fs::File; - use super::*; use crate::remove_dir_if_exists; +use data_bucket::DEFAULT_PAGE_STRIDE; worktable!( name: SchemaMetadata, @@ -37,8 +36,10 @@ async fn generated_schema_is_persisted_and_mismatches_are_rejected() { let table = SchemaMetadataWorkTable::load(engine).await.unwrap(); table.close().await.unwrap(); - let mut file = File::open(format!("{table_path}/{}", WT_DATA_EXTENSION)).await.unwrap(); - let info = parse_page::, { PAGE_SIZE as u32 }>(&mut file, 0) + let mut file = worktable::prelude::fsx::open(format!("{table_path}/{}", WT_DATA_EXTENSION)) + .await + .unwrap(); + let info = parse_page::, { PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 0) .await .unwrap(); assert_eq!( @@ -80,8 +81,10 @@ async fn loading_a_legacy_empty_schema_does_not_rewrite_the_file() { let table = SchemaMetadataWorkTable::load(engine).await.unwrap(); table.close().await.unwrap(); - let mut file = File::open(format!("{table_path}/{}", WT_DATA_EXTENSION)).await.unwrap(); - let info = parse_page::, { PAGE_SIZE as u32 }>(&mut file, 0) + let mut file = worktable::prelude::fsx::open(format!("{table_path}/{}", WT_DATA_EXTENSION)) + .await + .unwrap(); + let info = parse_page::, { PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 0) .await .unwrap(); assert!(info.inner.row_schema.is_empty()); diff --git a/tests/persistence/space_data.rs b/tests/persistence/space_data.rs index 191c0322..3abe53af 100644 --- a/tests/persistence/space_data.rs +++ b/tests/persistence/space_data.rs @@ -1,4 +1,5 @@ -use std::collections::HashMap; +use data_bucket::DEFAULT_PAGE_STRIDE; +use worktable::prelude::HashMap; use data_bucket::{INNER_PAGE_SIZE, Link, PAGE_SIZE, parse_general_header_by_index}; use worktable::prelude::{SpaceData, SpaceDataOps}; @@ -46,7 +47,9 @@ async fn rewriting_one_link_does_not_inflate_the_persisted_data_length() { space.save_batch_data(batch).await.unwrap(); assert_eq!(space.current_data_length, 48); - let header = parse_general_header_by_index(&mut space.data_file, 1).await.unwrap(); + let header = parse_general_header_by_index::(&mut space.data_file, 1) + .await + .unwrap(); assert_eq!(header.data_length, 48); drop(space); @@ -86,7 +89,9 @@ async fn reclaiming_thousands_of_pages_bounds_the_info_page_instead_of_corruptin assert!(kept < 2_000, "the overflow condition was not constructed"); // Page 1 must be untouched by the info persist. - let header = parse_general_header_by_index(&mut space.data_file, 1).await.unwrap(); + let header = parse_general_header_by_index::(&mut space.data_file, 1) + .await + .unwrap(); assert_eq!(header.page_type, PageType::Data); drop(space); diff --git a/tests/persistence/space_index/indexset_compatibility.rs b/tests/persistence/space_index/indexset_compatibility.rs index c9c9ba9a..6f702a5a 100644 --- a/tests/persistence/space_index/indexset_compatibility.rs +++ b/tests/persistence/space_index/indexset_compatibility.rs @@ -1,4 +1,5 @@ mod sized { + use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use data_bucket::{INNER_PAGE_SIZE, Link}; @@ -11,7 +12,7 @@ mod sized { async fn test_indexset_node_creation() { remove_file_if_exists("tests/data/space_index/indexset/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/indexset/process_create_node.wt.idx", 0.into(), 1, @@ -46,7 +47,7 @@ mod sized { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/indexset/process_insert_at.wt.idx", 0.into(), 1, @@ -81,7 +82,7 @@ mod sized { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/indexset/process_insert_at_big_amount.wt.idx", 0.into(), 1, @@ -124,6 +125,7 @@ mod sized { } mod unsized_ { + use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use crate::{check_if_files_are_same, remove_file_if_exists}; @@ -137,7 +139,7 @@ mod unsized_ { async fn test_indexset_node_creation() { remove_file_if_exists("tests/data/space_index_unsized/indexset/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/indexset/process_create_node.wt.idx", 0.into(), 1, @@ -172,7 +174,7 @@ mod unsized_ { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/indexset/process_insert_at.wt.idx", 0.into(), 1, @@ -210,7 +212,7 @@ mod unsized_ { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/indexset/process_insert_at_big_amount.wt.idx", 0.into(), 1, diff --git a/tests/persistence/space_index/unsized_write.rs b/tests/persistence/space_index/unsized_write.rs index 61175c97..da84b980 100644 --- a/tests/persistence/space_index/unsized_write.rs +++ b/tests/persistence/space_index/unsized_write.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use data_bucket::{INNER_PAGE_SIZE, Link}; @@ -14,7 +15,7 @@ mod run_first { async fn test_space_index_process_create_node() { remove_file_if_exists("tests/data/space_index_unsized/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_create_node.wt.idx", 0.into(), 1, @@ -52,7 +53,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_create_second_node.wt.idx", 0.into(), 1, @@ -90,7 +91,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_remove_node.wt.idx", 0.into(), 1, @@ -128,7 +129,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at.wt.idx", 0.into(), 1, @@ -175,7 +176,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at_big_amount.wt.idx", 0.into(), 1, @@ -249,7 +250,7 @@ async fn test_space_index_process_remove_at() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_remove_at.wt.idx", 0.into(), 1, @@ -296,7 +297,7 @@ async fn test_space_index_process_remove_at_node_id() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_remove_at_node_id.wt.idx", 0.into(), 1, @@ -344,7 +345,7 @@ async fn test_space_index_process_insert_at_with_node_id_update() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at_with_node_id_update.wt.idx", 0.into(), 1, @@ -391,7 +392,7 @@ async fn test_space_index_process_insert_at_removed_place() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at_removed_place.wt.idx", 0.into(), 1, @@ -484,7 +485,7 @@ async fn test_space_index_process_create_node_after_remove() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_create_node_after_remove.wt.idx", 0.into(), 1, @@ -522,7 +523,7 @@ async fn test_space_index_process_split_node() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_split_node.wt.idx", 0.into(), 1, diff --git a/tests/persistence/space_index/write.rs b/tests/persistence/space_index/write.rs index d6f8d2c6..81a0befa 100644 --- a/tests/persistence/space_index/write.rs +++ b/tests/persistence/space_index/write.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use data_bucket::{INNER_PAGE_SIZE, Link}; @@ -14,7 +15,7 @@ mod run_first { async fn test_space_index_process_create_node() { remove_file_if_exists("tests/data/space_index/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_create_node.wt.idx", 0.into(), 1, @@ -52,7 +53,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_create_second_node.wt.idx", 0.into(), 1, @@ -90,7 +91,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at.wt.idx", 0.into(), 1, @@ -137,7 +138,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at_big_amount.wt.idx", 0.into(), 1, @@ -210,7 +211,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_remove_node.wt.idx", 0.into(), 1, @@ -249,7 +250,7 @@ async fn test_space_index_process_insert_at_with_node_id_update() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at_with_node_id_update.wt.idx", 0.into(), 1, @@ -296,7 +297,7 @@ async fn test_space_index_process_remove_at() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_remove_at.wt.idx", 0.into(), 1, @@ -343,7 +344,7 @@ async fn test_space_index_process_remove_at_node_id() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_remove_at_node_id.wt.idx", 0.into(), 1, @@ -390,7 +391,7 @@ async fn test_space_index_process_insert_at_removed_place() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at_removed_place.wt.idx", 0.into(), 1, @@ -483,7 +484,7 @@ async fn test_space_index_process_create_node_after_remove() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_create_node_after_remove.wt.idx", 0.into(), 1, @@ -521,7 +522,7 @@ async fn test_space_index_process_split_node() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_split_node.wt.idx", 0.into(), 1, @@ -560,10 +561,13 @@ async fn test_space_index_process_split_node() { async fn batch_split_then_max_remove_then_historical_identity_insert_applies() { remove_file_if_exists("tests/data/space_index/batch_alias.wt.idx".to_string()).await; - let mut space_index = - SpaceIndex::::new("tests/data/space_index/batch_alias.wt.idx", 0.into(), 1) - .await - .unwrap(); + let mut space_index = SpaceIndex::::new( + "tests/data/space_index/batch_alias.wt.idx", + 0.into(), + 1, + ) + .await + .unwrap(); fn link(offset: u32) -> Link { Link { @@ -636,7 +640,7 @@ async fn batch_split_then_max_remove_then_historical_identity_insert_applies() { async fn batch_replay_of_real_cdc_stream_with_splits_matches_the_source() { remove_file_if_exists("tests/data/space_index/batch_cdc_replay.wt.idx".to_string()).await; - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/batch_cdc_replay.wt.idx", 0.into(), 1, diff --git a/tests/persistence/sync/string_secondary_index.rs b/tests/persistence/sync/string_secondary_index.rs index 93a6fafc..e4b65ffc 100644 --- a/tests/persistence/sync/string_secondary_index.rs +++ b/tests/persistence/sync/string_secondary_index.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use worktable::prelude::PersistedWorkTable; use worktable::prelude::*; use worktable_codegen::worktable; @@ -104,11 +105,12 @@ fn fragmented_string_index_compacts_after_restart_before_appending() { } let index_path = format!("{path}/fragmented_string_secondary/project_idx.wt.idx"); - let mut index_file = tokio::fs::File::open(index_path).await.unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>( - &mut index_file, - 2, - ) + let mut index_file = worktable::prelude::fsx::open(index_path).await.unwrap(); + let page = parse_page::< + UnsizedIndexPage, + { INNER_PAGE_SIZE as u32 }, + DEFAULT_PAGE_STRIDE, + >(&mut index_file, 2) .await .unwrap(); let utility_size = worktable::data_bucket::UnsizedIndexPageUtility::::persisted_size( diff --git a/tests/persistence/toc/read.rs b/tests/persistence/toc/read.rs index d36d59b0..6460f931 100644 --- a/tests/persistence/toc/read.rs +++ b/tests/persistence/toc/read.rs @@ -1,37 +1,34 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::sync::Arc; use std::sync::atomic::AtomicU32; use data_bucket::{INNER_PAGE_SIZE, Link}; -use tokio::fs::OpenOptions; use worktable::prelude::IndexTableOfContents; #[tokio::test] async fn test_index_table_of_contents_read() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/persist_index_table_of_contents.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/persist_index_table_of_contents.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(1)); - let toc = - IndexTableOfContents::::parse_from_file(&mut file, 0.into(), next_id_gen) - .await - .unwrap(); + let toc = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + next_id_gen, + ) + .await + .unwrap(); assert_eq!(toc.get(&13), Some(1.into())) } #[tokio::test] async fn test_index_table_of_contents_read_from_space() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/test_persist/primary.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/primary.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(1)); - let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -53,14 +50,11 @@ async fn test_index_table_of_contents_read_from_space() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -83,14 +77,11 @@ async fn test_index_table_of_contents_read_from_space_index() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_after_insert() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -113,14 +104,12 @@ async fn test_index_table_of_contents_read_from_space_index_after_insert() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_updated_node_id() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -143,14 +132,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_updated_node_id #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_remove_at_node_id() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -173,14 +159,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_remove_at_node_ #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -214,14 +197,12 @@ async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_create_node_after_remove_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -255,14 +236,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_create_node_aft #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_after_split_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_split_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_split_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -297,36 +275,39 @@ async fn test_index_table_of_contents_read_from_space_index_after_split_node() { #[tokio::test] async fn test_truncated_table_of_contents_is_an_error_not_an_empty_index() { let path = std::env::temp_dir().join(format!("worktable-toc-truncated-{}.wt.idx", uuid::Uuid::new_v4())); - let mut file = OpenOptions::new() - .write(true) - .read(true) - .create(true) - .truncate(true) - .open(&path) - .await - .unwrap(); + let mut file = worktable::prelude::fsx::create(&path).await.unwrap(); // A DATA_LENGTH of 20 forces the table of contents to span several pages. - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); for key in 0..10 { toc.insert(key, u32::from(key).into()); } assert!(toc.pages.len() > 1, "fixture must span TOC pages"); toc.persist(&mut file).await.unwrap(); - file.sync_all().await.unwrap(); + worktable::prelude::fsx::sync_all(&mut file).await.unwrap(); // The intact file must round-trip. - let reloaded = IndexTableOfContents::::parse_from_file(&mut file, 0.into(), Arc::new(AtomicU32::new(1))) - .await - .unwrap(); + let reloaded = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + Arc::new(AtomicU32::new(1)), + ) + .await + .unwrap(); assert_eq!(reloaded.pages.len(), toc.pages.len()); // Tear the first table-of-contents page: the file still extends into its // slot, so the load must fail loudly instead of yielding an empty index. - file.set_len(data_bucket::PAGE_SIZE as u64 + 10).await.unwrap(); - let error = IndexTableOfContents::::parse_from_file(&mut file, 0.into(), Arc::new(AtomicU32::new(1))) + worktable::prelude::fsx::set_len(&mut file, data_bucket::PAGE_SIZE as u64 + 10) .await - .unwrap_err(); + .unwrap(); + let error = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + Arc::new(AtomicU32::new(1)), + ) + .await + .unwrap_err(); assert!( error.to_string().contains("table of contents page 1 failed to parse"), "unexpected error: {error:#}" @@ -334,13 +315,16 @@ async fn test_truncated_table_of_contents_is_an_error_not_an_empty_index() { // A file that never grew past page 0 is a real bootstrap and still loads // as a fresh empty table of contents. - file.set_len(0).await.unwrap(); - let bootstrapped = - IndexTableOfContents::::parse_from_file(&mut file, 0.into(), Arc::new(AtomicU32::new(1))) - .await - .unwrap(); + worktable::prelude::fsx::set_len(&mut file, 0).await.unwrap(); + let bootstrapped = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + Arc::new(AtomicU32::new(1)), + ) + .await + .unwrap(); assert_eq!(bootstrapped.pages.len(), 1); drop(file); - tokio::fs::remove_file(&path).await.unwrap(); + worktable::prelude::fsx::remove_file(&path).await.unwrap(); } diff --git a/tests/persistence/toc/unsized_read.rs b/tests/persistence/toc/unsized_read.rs index 1dffc0ad..ed460df7 100644 --- a/tests/persistence/toc/unsized_read.rs +++ b/tests/persistence/toc/unsized_read.rs @@ -1,20 +1,17 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::sync::Arc; use std::sync::atomic::AtomicU32; use data_bucket::{INNER_PAGE_SIZE, Link}; -use tokio::fs::OpenOptions; use worktable::prelude::IndexTableOfContents; #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -37,14 +34,12 @@ async fn test_index_table_of_contents_read_from_space_index_unsized() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_with_two_nodes() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(3)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -78,14 +73,11 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_with_two_nod #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -119,14 +111,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_insert_at() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -149,14 +138,11 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_after_insert #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove_at() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -179,14 +165,12 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove_at_node_id() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -209,14 +193,13 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_create_node_after_remove() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_node_after_remove.wt.idx") - .await - .unwrap(); + let mut file = worktable::prelude::fsx::open( + "tests/data/expected/space_index_unsized/process_create_node_after_remove.wt.idx", + ) + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, diff --git a/tests/persistence/toc/write.rs b/tests/persistence/toc/write.rs index 0bb04709..910ec0e9 100644 --- a/tests/persistence/toc/write.rs +++ b/tests/persistence/toc/write.rs @@ -1,7 +1,7 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::INNER_PAGE_SIZE; use std::sync::Arc; use std::sync::atomic::AtomicU32; -use tokio::fs::File; use worktable::prelude::IndexTableOfContents; use crate::{check_if_files_are_same, remove_file_if_exists}; @@ -10,11 +10,14 @@ use crate::{check_if_files_are_same, remove_file_if_exists}; async fn test_persist_index_table_of_contents() { remove_file_if_exists("tests/data/persist_index_table_of_contents.wt.idx".to_string()).await; - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new( + 0.into(), + Arc::new(AtomicU32::new(1)), + ); // Compile-time compatibility regression: the public API before PR #63 // returned unit, including for callers that bind the expression's type. let _: () = toc.insert(13, 1.into()); - let mut file = File::create("tests/data/persist_index_table_of_contents.wt.idx") + let mut file = worktable::prelude::fsx::create("tests/data/persist_index_table_of_contents.wt.idx") .await .unwrap(); toc.persist(&mut file).await.unwrap(); diff --git a/tests/worktable/concurrency.rs b/tests/worktable/concurrency.rs index 51eceb4f..75382664 100644 --- a/tests/worktable/concurrency.rs +++ b/tests/worktable/concurrency.rs @@ -115,7 +115,7 @@ macro_rules! backend_suite { scope.spawn(move || { for n in 0..per_writer { let id = w * per_writer + n; - futures::executor::block_on(table.insert(row(id))).expect("insert"); + nagoya::block_on(table.insert(row(id))).expect("insert"); // Read while the others write, so the scan and // the mutations actually overlap. let _ = table.select(id); @@ -166,7 +166,7 @@ macro_rules! backend_suite { scope.spawn(move || { for n in 0..per_writer { let id = seed + w * per_writer + n; - futures::executor::block_on(table.insert(row(id))).expect("insert"); + nagoya::block_on(table.insert(row(id))).expect("insert"); } }); } @@ -218,11 +218,11 @@ macro_rules! backend_suite { let rows: Vec<_> = (chunk..(chunk + 16).min(per_writer)) .map(|n| row(base + n)) .collect(); - futures::executor::block_on(table.insert_many(rows)).expect("insert_many"); + nagoya::block_on(table.insert_many(rows)).expect("insert_many"); } } else { for n in 0..per_writer { - futures::executor::block_on(table.insert(row(base + n))).expect("insert"); + nagoya::block_on(table.insert(row(base + n))).expect("insert"); } } }); @@ -263,7 +263,7 @@ macro_rules! backend_suite { scope.spawn(move || { // Distinct primary keys, one shared payload. let contended = ConcRow { id: w, payload: 42, bucket: 0 }; - if futures::executor::block_on(table.insert(contended)).is_ok() { + if nagoya::block_on(table.insert(contended)).is_ok() { winners.fetch_add(1, Ordering::Release); } }); @@ -363,8 +363,8 @@ macro_rules! backend_suite { (base + WINDOW, base) }; for i in 0..WINDOW { - futures::executor::block_on(table.delete(from + i)).expect("delete"); - futures::executor::block_on(table.insert(row(to + i))).expect("insert"); + nagoya::block_on(table.delete(from + i)).expect("delete"); + nagoya::block_on(table.insert(row(to + i))).expect("insert"); } } finished.fetch_add(1, Ordering::Release); @@ -452,7 +452,7 @@ macro_rules! backend_suite { let table = Arc::clone(&table); scope.spawn(move || { for n in 0..per_writer { - futures::executor::block_on(table.insert(row(w * per_writer + n))).expect("insert"); + nagoya::block_on(table.insert(row(w * per_writer + n))).expect("insert"); } }); } diff --git a/tests/worktable/index/insert.rs b/tests/worktable/index/insert.rs index f0ae7c85..b0615d77 100644 --- a/tests/worktable/index/insert.rs +++ b/tests/worktable/index/insert.rs @@ -228,7 +228,7 @@ async fn insert_when_unique_violated() { attr3: 123456789, attr4: row_new_attr_4.clone(), }; - assert!(futures::executor::block_on(shared.insert(row)).is_err()); + assert!(nagoya::block_on(shared.insert(row)).is_err()); } }); @@ -315,7 +315,7 @@ async fn insert_when_pk_violated() { attr3: 123456789, attr4: "Attribute__4".to_string(), }; - assert!(futures::executor::block_on(shared.insert(row)).is_err()); + assert!(nagoya::block_on(shared.insert(row)).is_err()); } }); diff --git a/tests/worktable/key_widths.rs b/tests/worktable/key_widths.rs index f0429fcb..349d0a35 100644 --- a/tests/worktable/key_widths.rs +++ b/tests/worktable/key_widths.rs @@ -66,7 +66,7 @@ macro_rules! width_case { ); // And the entry comes out again. - futures::executor::block_on(table.delete(2u64)).expect("delete"); + nagoya::block_on(table.delete(2u64)).expect("delete"); assert!( table.select_by_key(42 as $key).is_none(), "{}: {} index still resolves a deleted row", diff --git a/tests/worktable/nonunique_arctic.rs b/tests/worktable/nonunique_arctic.rs index bb9505ec..fcefb796 100644 --- a/tests/worktable/nonunique_arctic.rs +++ b/tests/worktable/nonunique_arctic.rs @@ -165,7 +165,7 @@ async fn concurrent_inserts_and_deletes_keep_the_index_consistent() { for n in 0..per_writer { let edge = (writer as u128) << 64 | n as u128; let source = (n as u128) % keys; - futures::executor::block_on(table.insert(row(&table, source, edge, n))).unwrap(); + nagoya::block_on(table.insert(row(&table, source, edge, n))).unwrap(); // Interleave point reads to race the writers. let _ = table.select_by_source_hash(source).execute().unwrap(); } diff --git a/tests/worktable/partitioned.rs b/tests/worktable/partitioned.rs index 56df81c2..672c1e32 100644 --- a/tests/worktable/partitioned.rs +++ b/tests/worktable/partitioned.rs @@ -101,7 +101,7 @@ async fn insert_with_a_custom_initialiser_runs_once_per_key() { .partition_or_insert_with(11, || { let t = PriceWorkTable::default(); for e in 0..3u8 { - futures::executor::block_on(t.insert(row(e, e as f64))).unwrap(); + nagoya::block_on(t.insert(row(e, e as f64))).unwrap(); } t }) @@ -177,7 +177,7 @@ fn concurrent_creation_and_reading_is_sound() { let table = prices.partition_or_create(k).unwrap(); // Every thread writes the same row for a key, so whichever // wins the insert the value must match the key. - let _ = futures::executor::block_on(table.insert(row(0, k as f64))); + let _ = nagoya::block_on(table.insert(row(0, k as f64))); let got = prices.partition(k).unwrap().select(0).unwrap(); assert_eq!(got.bid, k as f64, "thread {t} saw a torn partition at {k}"); } @@ -426,7 +426,7 @@ async fn concurrent_writers_on_disjoint_partitions_do_not_interfere() { std::thread::spawn(move || { let table = prices.partition_or_create(t).unwrap(); for e in 0..ROWS { - futures::executor::block_on(table.insert(row(e, t as f64 * 1000.0 + e as f64))).unwrap(); + nagoya::block_on(table.insert(row(e, t as f64 * 1000.0 + e as f64))).unwrap(); } }) }) @@ -490,7 +490,7 @@ async fn readers_survive_partitions_being_removed_under_them() { let t = prices .partition_or_insert_with(k, || { let t = PriceWorkTable::default(); - futures::executor::block_on(t.insert(row(0, k as f64))).unwrap(); + nagoya::block_on(t.insert(row(0, k as f64))).unwrap(); t }) .unwrap();