diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 60a66f05..3cffe506 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -5,6 +5,7 @@ on: branches: [ "master" ] pull_request: branches: [ "master" ] + workflow_dispatch: env: CARGO_TERM_COLOR: always @@ -32,16 +33,29 @@ jobs: # exceeded 30 minutes when GitHub's cache service was unavailable. Keep # enough room for a real from-scratch release gate. timeout-minutes: 45 + # Not on a pull request. `cargo test --workspace --all-targets` is a + # ten-minute compile on a two-core runner before it runs anything, three + # times over the matrix, and it is the same command `scripts/ci-local.sh` + # runs on a developer's machine in a fraction of that. Run it there. + # + # It still runs on master, which is what `publish` gates on, and on a manual + # `workflow_dispatch` when a change wants it earlier. + if: github.event_name != 'pull_request' strategy: fail-fast: false matrix: include: - name: default args: "" + test_debug: 2 - name: versioned-publication args: "--features versioned-row-publication" + test_debug: 2 - name: all-features args: "--all-features" + # Full DWARF makes Rust 1.98's bundled rust-lld crash while linking + # the large integration-test binary on a two-core runner. + test_debug: 0 steps: - uses: actions/checkout@v4 @@ -49,12 +63,45 @@ jobs: with: cache-on-failure: "true" add-job-id-key: "false" - - name: Build - run: cargo build --workspace --all-targets ${{ matrix.args }} --verbose - - name: Run tests + # There is no separate build step. `cargo test --all-targets` builds every + # target it is about to run, and the build step ran without + # `CARGO_PROFILE_TEST_DEBUG`: the two commands disagreed about the test + # profile, so the second one missed the first one's cache and compiled the + # workspace a second time. On the all-features leg it also compiled with the + # full DWARF that `test_debug: 0` is set to avoid, which is what makes + # rust-lld crash linking the integration-test binary on a two-core runner. + - name: Build and run tests run: cargo test --workspace --all-targets ${{ matrix.args }} --verbose + env: + CARGO_PROFILE_TEST_DEBUG: ${{ matrix.test_debug }} + # What a pull request gets instead of the full matrix: the library tests of + # every crate, without `--all-targets`. + # + # That leaves out the integration-test binary and the compile-fail harness, + # which is where the ten minutes live -- the trybuild tests shell out to + # nested cargo builds, and one of them alone took 56 seconds. The lib tests + # are the part that answers "did this change break something" quickly, and + # they are not a token gesture: the assertion that turned this whole workflow + # red was `worktable_dsl`'s `legacy_in_place_section_is_rejected`, a lib test + # this job runs and the fmt/clippy/no-std jobs all passed straight over. + # + # The full matrix still runs on master and on demand. + smoke: + name: Smoke test (lib tests) + runs-on: ubicloud-standard-2 + timeout-minutes: 15 + if: github.event_name == 'pull_request' + steps: + - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: "true" + add-job-id-key: "false" + - name: Library tests (cargo test --workspace --lib) + run: cargo test --workspace --lib --verbose + clippy_check: name: Clippy (${{ matrix.name }}) runs-on: ubicloud-standard-2 @@ -81,6 +128,12 @@ jobs: name: Archived-row lock concurrency models runs-on: ubicloud-standard-2 timeout-minutes: 15 + # Not on a pull request. The models explore an interleaving space rather + # than run a suite, they build into their own CARGO_TARGET_DIR so they share + # no cache with any other job and pay a cold compile every time, and the + # answer does not change with a review comment. They still run on master, + # where `publish` gates on them, and on demand. + if: github.event_name != 'pull_request' steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable diff --git a/CHANGELOG.md b/CHANGELOG.md index ec079e55..7178bfe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,46 @@ Change Log ========== +## [1.10.0-beta1] + +### Changed + +- Declared mutations now use `update:` and + `update_in_place:`. They generate typed selector calls such as + `update_by_id(id, InvoiceColumns::AMOUNT, value)` and + `update_in_place_by_id(id, InvoiceColumns::STATE, edit)`. Multi-column + declarations expose one selector for their exact atomic field set and take + the generated query struct. Multi-column `update_in_place` declarations pass + a tuple of mutable archived fields to one closure, preserving the declared + atomic field set. Full-row replacement is now `replace(row)`. + +### Added + +- The frozen `LinearTable` and `VecTable` API is now part of WorkTable itself, + replacing the retired `worktable-vec` compatibility crate. PathDB's indexed + lookup compiles to instruction-for-instruction identical AArch64 code before + and after the move. + +### Fixed + +- Generated fixed-size primary-key wrappers now report their actual archived + alignment. A `u128` primary key previously under-budgeted each persisted WTI + entry, producing an 18,518-byte index archive for a 16,356-byte default inner + page; the corrected 16-byte alignment selects a capacity whose archive fits. + +- Dropping a persisted table with queued writes now joins its private writer + before the final handle disappears. An immediate same-path reopen can no + longer race detached writes and observe a partial store or torn index header. + +- Generated persisted-table startup now constructs its page storage directly + in the final `Arc` allocation and pins nested load futures before awaiting + them. This removes roughly 16 KiB page and directory temporaries from normal + thread stacks. In AgentCode's eight-table empty-store startup, the generated + load future shrank from 17,832 bytes to 2,392 bytes and the restart path no + longer overflows Tokio's default 2 MiB worker stack. Existing-page reads and + mutations are unchanged; fresh construction and page-growing inserts use the + new final-allocation initializer. + ## [1.9.0-alpha1] @@ -39,9 +79,8 @@ Change Log - **`queries:` on a `vec: true` table.** It was refused wholesale; it now - generates `update_`, `delete_` and `update__in_place` under - the same names the paged table uses, so a declaration reads the same either - way. + generates typed `update_by_` and `update_in_place_by_` dispatch, + plus declared delete methods, under the same names the paged table uses. These are named wrappers rather than a new execution path: a declared update is `update(&pk, |row| ..)` with the columns filled in from a generated struct, @@ -198,15 +237,15 @@ Change Log declares 65,536 rows into a partition that holds 256), and `persist: true`, which a dense partition has no engine to honour. - It carries `queries:`. An `update` or `delete` query keyed by the primary key - generates the same method name against the same `Query` struct the - paged table generates, so a call reads identically; the signature does not, + It carries `queries:`. An `update` query keyed by the primary key uses the + same typed field-set selector as the paged table, and a multi-column selector + takes the same `Query` struct. The signature differs, deliberately, because there is no `.await` and no `WorkTableError`, and a call that moved between the shapes should fail to compile rather than quietly change what it guarantees. A query keyed by any other column is refused: a dense partition has no secondary index, and scanning it instead - would be a keyed operation silently becoming a linear one. `in_place` is - refused as a synonym, because every update here is already in place. + would be a keyed operation silently becoming a linear one. + `update_in_place` is refused, because every update here is already in place. Note that `memory_by_key` and `memory_total` **cannot see this saving**. They report `used_bytes`, which is rows plus indexes and excludes the fixed floor diff --git a/Cargo.toml b/Cargo.toml index 2ecc1abf..4d8c2aae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["codegen", "dsl", "examples", "performance_measurement", "performance [package] name = "worktable" -version = "1.9.0-alpha1" +version = "1.10.0-beta1" edition = "2024" authors = ["Handy-caT"] license = "MIT" @@ -147,12 +147,12 @@ walkdir = { version = "2", optional = true } # These pre-release workspace crates move as one train. The explicit caret # keeps the dependency policy consistent while the local path selects this # checkout during validation. -worktable_codegen = { path = "codegen", version = "^1.9.0-alpha1" } +worktable_codegen = { path = "codegen", version = "^1.10.0-beta1" } # Re-exported below. Each generated table carries its declaration as a const # whose documentation says to read it with `worktable_dsl::Schema::parse`; that # instruction is only true if a plain `worktable` dependency can reach the # crate. -worktable_dsl = { path = "dsl", version = "^1.0.0-beta.19", optional = true } +worktable_dsl = { path = "dsl", version = "^1.10.0-beta1", optional = true } [target.'cfg(unix)'.dependencies] libc = { version = "^0.2", default-features = false } diff --git a/README.md b/README.md index f7d15787..e6fae590 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ for table in database.catalog().system_tables() { ```toml [dependencies] -worktable = { version = "^1.9.0-alpha1", features = ["s3-support"] } # S3 sync, optional +worktable = { version = "^1.10.0-beta1", features = ["s3-support"] } # S3 sync, optional ``` Persisted indexes default to WorkTablesIndex. Vanilla IndexSet can be selected explicitly with `using indexset` while retaining the existing disk/S3 representation. Congee and Arctic persistence is experimental and uses their native checkpoint/WAL adapters; declarations using either backend must state `persist: true` or `persist: false` explicitly. The full syntax and capability matrix are documented in [Per-index backends with `using`](docs/index-backend-dsl-proposal.md). @@ -426,7 +426,7 @@ There are some default query implementations that are available for all `WorkTab into grouped persistence operations, and durability follows the usual `wait_for_ops` contract. Autoincrement tables additionally get `reserve_pks(&self, count: usize) -> Range` to pre-assign contiguous keys to a batch; - `upsert(&self, row: Row) -> Result<(), WorkTableError>`; -- `update(&self, row: Row) -> Result<(), WorkTableError>`; +- `replace(&self, row: Row) -> Result<(), WorkTableError>`; - `delete(&self, pk: PrimaryKey) -> Result<(), WorkTableError>`; - `select_all<'a>(&'a self) -> SelectQueryBuilder<'a, Row, Self>`; @@ -456,8 +456,13 @@ query. #### `update` query declaration -`update` queries are used to update row's data partially. Default generated `update` allows only full update of the row. -But if user's logic needs some simultaneous update of row parts from different code parts. `update` logic supports +`update` queries update only the declared fields through a typed, table-scoped +column selector. For example, +`table.update_by_id(id, TestColumns::ANOTHER, value).await?` changes only +`another`. A multi-column declaration exposes one atomic selector such as +`TestColumns::NAME_AND_AMOUNT` and takes its generated query struct. The +generated `replace(row)` method replaces the full row. +When application logic updates disjoint parts of a row concurrently, `update` supports smart lock logic that allows simultaneous update of not overlapping row fields. #### `select_all` query declaration diff --git a/benches/cases/full_featured.rs b/benches/cases/full_featured.rs index 69c95bfa..8ca9df13 100644 --- a/benches/cases/full_featured.rs +++ b/benches/cases/full_featured.rs @@ -123,7 +123,7 @@ fn update(c: &mut Criterion) { another: format!("updated_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - black_box(table.update(row).await) + black_box(table.replace(row).await) }) }); } @@ -151,7 +151,11 @@ fn update_by_pk_query(c: &mut Criterion) { let query = AnotherByIdQuery { another: format!("upd_{}", fastrand::u64(..)), }; - black_box(table.update_another_by_id(query, id).await) + black_box( + table + .update_by_id(id, FullFeaturedColumns::ANOTHER, (query).another) + .await, + ) }) }); } @@ -179,7 +183,11 @@ fn update_by_unique_index_query(c: &mut Criterion) { let query = AnotherByVal1Query { another: format!("upd_{}", fastrand::u64(..)), }; - black_box(table.update_another_by_val_1(query, val1).await) + black_box( + table + .update_by_val1(val1, FullFeaturedColumns::ANOTHER, (query).another) + .await, + ) }) }); } @@ -200,8 +208,11 @@ fn in_place_update(c: &mut Criterion) { }; c.bench_function("full_featured_in_place_update_val", |b| { - b.to_async(&rt) - .iter(|| async { table.update_val_by_id_in_place(|val| *val += 1, black_box(pk)).await }) + b.to_async(&rt).iter(|| async { + table + .update_in_place_by_id(black_box(pk), FullFeaturedColumns::VAL, |val| *val += 1) + .await + }) }); } diff --git a/benches/cases/non_unique_index.rs b/benches/cases/non_unique_index.rs index c2589030..a9ecd365 100644 --- a/benches/cases/non_unique_index.rs +++ b/benches/cases/non_unique_index.rs @@ -91,7 +91,7 @@ fn update(c: &mut Criterion) { value: fastrand::u64(..), category: fastrand::u64(0..10), }; - black_box(table.update(row).await) + black_box(table.replace(row).await) }) }); } diff --git a/benches/cases/simple.rs b/benches/cases/simple.rs index afbcc353..c9a437ce 100644 --- a/benches/cases/simple.rs +++ b/benches/cases/simple.rs @@ -66,7 +66,7 @@ fn update(c: &mut Criterion) { id: pk.into(), value: fastrand::u64(..), }; - black_box(table.update(row).await) + black_box(table.replace(row).await) }) }); } diff --git a/benches/cases/unique_index.rs b/benches/cases/unique_index.rs index 73144ff2..9b03baf9 100644 --- a/benches/cases/unique_index.rs +++ b/benches/cases/unique_index.rs @@ -156,7 +156,7 @@ fn update(c: &mut Criterion) { test: fastrand::i64(..), another: fastrand::u64(..), }; - black_box(table.update(row).await) + black_box(table.replace(row).await) }) }); } diff --git a/benches/cases/update_contention.rs b/benches/cases/update_contention.rs index eb3ed8c0..a7714365 100644 --- a/benches/cases/update_contention.rs +++ b/benches/cases/update_contention.rs @@ -41,7 +41,7 @@ fn single_row_update_contention(c: &mut Criterion) { another: format!("upd_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - black_box(table_clone.update(row).await) + black_box(table_clone.replace(row).await) }); } while join_set.join_next().await.is_some() {} @@ -81,7 +81,11 @@ fn single_row_in_place_contention(c: &mut Criterion) { for _ in 0..level { let table_clone = table.clone(); join_set.spawn(async move { - black_box(table_clone.update_val_by_id_in_place(|val| *val += 1, pk).await) + black_box( + table_clone + .update_in_place_by_id(pk, FullFeaturedColumns::VAL, |val| *val += 1) + .await, + ) }); } while join_set.join_next().await.is_some() {} diff --git a/benches/common/mod.rs b/benches/common/mod.rs index 13920717..ce50e30b 100644 --- a/benches/common/mod.rs +++ b/benches/common/mod.rs @@ -53,7 +53,7 @@ worktable!( another_idx: another, }, queries: { - in_place: { + update_in_place: { ValById(val) by id, } update: { diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 2db7b380..c70533c1 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "worktable_codegen" -version = "1.9.0-alpha1" +version = "1.10.0-beta1" edition = "2024" license = "MIT" description = "Proc-macro companion crate for worktable: the worktable! macro and its derives." @@ -26,7 +26,7 @@ proc-macro = true # a declaration. See its crate docs for why that needed a separate crate. # Name the reviewed prerelease while accepting compatible schema-model and # validator updates. Release checks verify the resolved generated API. -worktable_dsl = { path = "../dsl", version = "^1.0.0-beta.19" } +worktable_dsl = { path = "../dsl", version = "^1.10.0-beta1" } # Test-only. As a normal dependency this proc-macro crate put `rkyv` with its # default features into the graph, which turned on `rkyv/std` for the target # build too and dragged `ptr_meta` with it. The generated code names `rkyv` diff --git a/codegen/src/common/name_generator.rs b/codegen/src/common/name_generator.rs index b1865a12..613dd45d 100644 --- a/codegen/src/common/name_generator.rs +++ b/codegen/src/common/name_generator.rs @@ -1,6 +1,7 @@ use convert_case::{Case, Casing}; -use proc_macro2::{Ident, Literal}; +use proc_macro2::{Ident, Literal, TokenStream}; use quote::__private::Span; +use syn::{GenericArgument, PathArguments, Type}; pub fn is_unsized(ty_: &str) -> bool { matches!(ty_, "String") @@ -14,6 +15,119 @@ pub fn is_float(ty_: &str) -> bool { matches!(ty_, "f64" | "f32") } +/// Whether moving only this field's archived bytes can leave a relative +/// pointer referring to the temporary query buffer. +/// +/// The macro cannot inspect a user type's `Archive::Archived` layout. Keep the +/// field-swap fast path for Rust's known scalar shapes and make every opaque +/// user type take the full-row serialize/overwrite path. `String` is handled +/// by the existing variable-size path; wrapping it in `Option` still requires +/// the conservative full-row path because the option's archived payload can +/// contain the same relative pointer. +pub fn archived_field_requires_rebuild(ty: &TokenStream) -> bool { + fn type_requires_rebuild(ty: &Type) -> bool { + let Type::Path(type_path) = ty else { + return true; + }; + let Some(segment) = type_path.path.segments.last() else { + return true; + }; + + match segment.ident.to_string().as_str() { + "String" => false, + "bool" | "char" | "u8" | "u16" | "u32" | "u64" | "u128" | "usize" | "i8" | "i16" | "i32" | "i64" + | "i128" | "isize" | "f32" | "f64" | "Uuid" => false, + "Option" => match &segment.arguments { + PathArguments::AngleBracketed(arguments) => arguments + .args + .iter() + .find_map(|argument| match argument { + GenericArgument::Type(inner) => { + let is_string = matches!(inner, Type::Path(path) if path.path.segments.last().is_some_and(|segment| segment.ident == "String")); + Some(is_string || type_requires_rebuild(inner)) + } + _ => None, + }) + .unwrap_or(true), + _ => true, + }, + _ => true, + } + } + + syn::parse2::(ty.clone()) + .map(|ty| type_requires_rebuild(&ty)) + .unwrap_or(true) +} + +/// Whether this column's archived form is a fixed-size scalar sitting inline +/// in the cell, with no relative pointer. +/// +/// This is the safety condition for the zero-copy `select_with` path: a +/// concurrent writer can tear an inline scalar the reader's closure observes, +/// and the seqlock retry throws that away, but a torn *pointer* dereferenced +/// inside the closure is undefined behaviour. +/// +/// Distinct from [`archived_field_requires_rebuild`], which answers a +/// different question and treats `String` as fine because the variable-size +/// path handles it. Here `String` is precisely what must be excluded. +/// +/// Opaque user types are refused, because the macro cannot inspect a user +/// type's `Archive::Archived` layout. That is conservative in the safe +/// direction: an unrecognised column costs the table its zero-copy path, it +/// does not grant one unsoundly. +/// +/// "No relative pointer" is necessary but not sufficient: the type must also +/// have no validity invariant, because a torn read must yield a wrong *value* +/// and not an invalid one. `char` is the type that fails that second test and +/// is why this list is an allowlist rather than "anything `Copy`". It archives +/// to `rend::char_le`, whose `to_native` transmutes its `u32` on the promise +/// that it holds a valid scalar value. Two valid chars can tear into one that +/// is not: `U+1D800` is `[00 D8 01 00]` and `U+0041` is `[41 00 00 00]`, so a +/// copy taking the low half of the first and the high half of the second reads +/// `0x0000D800`, a surrogate. The closure transmutes that before `still_stable` +/// ever runs, which is undefined behaviour and not a discarded wrong number. +/// `bool` stays: it is one byte, so it cannot tear into a third value. +/// +/// A `char` column therefore costs its table `select_with` and nothing else. +/// This gate is only an optimisation. +pub fn archived_field_is_inline_scalar(ty: &TokenStream) -> bool { + fn type_is_inline(ty: &Type) -> bool { + let Type::Path(type_path) = ty else { + return false; + }; + let Some(segment) = type_path.path.segments.last() else { + return false; + }; + + match segment.ident.to_string().as_str() { + // `char` is deliberately absent: see the note above. Every type + // here has every bit pattern valid, so a torn read is a wrong + // number rather than an invalid value. + "bool" | "u8" | "u16" | "u32" | "u64" | "u128" | "usize" | "i8" | "i16" | "i32" | "i64" | "i128" + | "isize" | "f32" | "f64" => true, + // An archived `Option` of an inline `T` stays inline: rkyv + // encodes the niche or a discriminant beside the payload. + "Option" => match &segment.arguments { + PathArguments::AngleBracketed(arguments) => arguments + .args + .iter() + .find_map(|argument| match argument { + GenericArgument::Type(inner) => Some(type_is_inline(inner)), + _ => None, + }) + .unwrap_or(false), + _ => false, + }, + _ => false, + } + } + + syn::parse2::(ty.clone()) + .map(|ty| type_is_inline(&ty)) + .unwrap_or(false) +} + pub struct WorktableNameGenerator { pub(crate) name: String, } @@ -120,6 +234,31 @@ impl WorktableNameGenerator { quote::quote! { (#page_size - worktable::prelude::GENERAL_HEADER_SIZE) } } + /// Payload budget whose end is aligned for a tail-stored archived key. + /// Variable-width index entries are written backwards from this boundary. + /// + /// # On-disk format + /// + /// This is a **format-affecting** value for unsized primary keys: it fixes + /// how many entries an index node holds, so a change to it makes existing + /// index files unreadable at the new capacity. The house convention is + /// regenerate rather than migrate, but the change has to be stated rather + /// than discovered. + /// + /// Rounding down to the archived key's alignment is what corrects the + /// earlier under-budgeting (see the `u128_primary_index_capacity` test). A + /// plain `String` key is unaffected, because `align_of::()` + /// is 4 and the capacity was already a multiple of it; a composite key + /// containing a `u128` aligns to 16 and does change. + pub fn get_aligned_disk_page_capacity(&self, key_type: &proc_macro2::TokenStream) -> proc_macro2::TokenStream { + let capacity = self.get_disk_page_capacity(); + quote::quote! { + (#capacity - (#capacity % core::mem::align_of::< + <#key_type as worktable::prelude::rkyv::Archive>::Archived + >())) + } + } + pub fn get_page_inner_size_const_ident(&self) -> Ident { let upper_snake_case_name = self.name.from_case(Case::Pascal).to_case(Case::UpperSnake); Ident::new( @@ -171,3 +310,20 @@ impl WorktableNameGenerator { ) } } + +#[cfg(test)] +mod tests { + use super::archived_field_requires_rebuild; + use quote::quote; + + #[test] + fn archived_field_classification_is_conservative_for_opaque_types() { + assert!(!archived_field_requires_rebuild("e!(u64))); + assert!(!archived_field_requires_rebuild("e!(Option))); + assert!(!archived_field_requires_rebuild("e!(String))); + assert!(!archived_field_requires_rebuild("e!(Uuid))); + assert!(!archived_field_requires_rebuild("e!(Option))); + assert!(archived_field_requires_rebuild("e!(Option))); + assert!(archived_field_requires_rebuild("e!(EncryptedSecret))); + } +} diff --git a/codegen/src/generators/dense_table.rs b/codegen/src/generators/dense_table.rs index 985da4c5..42709489 100644 --- a/codegen/src/generators/dense_table.rs +++ b/codegen/src/generators/dense_table.rs @@ -26,8 +26,8 @@ pub struct DenseQueries { pub updates: Vec<(Ident, Operation)>, /// `delete () by `. pub deletes: Vec<(Ident, Operation)>, - /// `in_place (columns) by `. Refused: see [`expand`]. - pub in_place: Vec<(Ident, Operation)>, + /// `update_in_place (columns) by `. Refused: see [`expand`]. + pub updates_in_place: Vec<(Ident, Operation)>, } impl DenseQueries { @@ -42,12 +42,12 @@ impl DenseQueries { Self { updates: lift(&queries.updates), deletes: lift(&queries.deletes), - in_place: lift(&queries.in_place), + updates_in_place: lift(&queries.updates_in_place), } } fn is_empty(&self) -> bool { - self.updates.is_empty() && self.deletes.is_empty() && self.in_place.is_empty() + self.updates.is_empty() && self.deletes.is_empty() && self.updates_in_place.is_empty() } } @@ -199,7 +199,7 @@ pub fn expand( }) .collect::>(); - let query_methods = gen_queries(name, columns, &pk, &pk_type, queries)?; + let (query_dispatch, query_methods) = gen_queries(name, columns, &pk, &pk_type, queries)?; let table_doc = format!( "One partition of [`{name}Partitions`], addressed by position.\n\n\ @@ -213,6 +213,8 @@ pub fn expand( ); Ok(quote! { + #query_dispatch + #[doc = #table_doc] #[derive(Debug)] pub struct #table { @@ -285,7 +287,7 @@ pub fn expand( /// /// `None` means the key held nothing, and nothing was written: this /// updates, it does not insert. `upsert` is the one that does both. - pub fn update( + pub fn replace( &self, row: #row_ident, ) -> Result, worktable::partition::DenseError> { @@ -352,12 +354,12 @@ pub fn expand( }) } -/// Generate one method per `queries:` entry. +/// Generate storage methods and typed-selector dispatch for `queries:` entries. /// -/// The method names and the `Query` argument structs are the paged -/// table's: a partitioned declaration still generates the full table beside the -/// dense payload, so the query structs already exist and a caller keeps the -/// same call. What differs is the signature, deliberately, the same way every +/// A partitioned declaration still generates the full table beside the dense +/// payload, so the selector markers and multi-field query structs already +/// exist and a caller keeps the same call. What differs is the result, +/// deliberately, the same way every /// other pair of shapes in this crate differs: there is no `.await` and no /// `WorkTableError`, so moving a call between them fails to compile rather than /// quietly changing what it guarantees. @@ -367,20 +369,20 @@ fn gen_queries( pk: &Ident, pk_type: &TokenStream, queries: &DenseQueries, -) -> syn::Result> { +) -> syn::Result<(TokenStream, Vec)> { if queries.is_empty() { - return Ok(Vec::new()); + return Ok((quote! {}, Vec::new())); } - // `in_place` exists on the paged table because a write there is async and + // `update_in_place` exists on the paged table because a write there is async and // has to hold a column across a suspension point. Nothing here is async and // `update` is already in place, so generating both would be two names for // one method. - if let Some((query, _)) = queries.in_place.first() { + if let Some((query, _)) = queries.updates_in_place.first() { return Err(Error::new( query.span(), format!( - "`in_place {query}` has no meaning on a dense partition: every update here is \ + "`update_in_place {query}` has no meaning on a dense partition: every update here is \ already in place, because there is no page to rewrite and no await to hold a \ column across. Declare it as `update {query}`, or use `partition_max_size: u64` \ for the full table." @@ -389,10 +391,12 @@ fn gen_queries( } let mut out = Vec::new(); + let mut update_impls = Vec::new(); + let table = type_ident(name); for (query, op) in &queries.updates { by_must_be_the_key(pk, query, op, "update")?; - let method = format_ident!("update_{}", snake(query)); + let method = format_ident!("__wt_update_{}", snake(query)); let query_ty = format_ident!("{}Query", query); let fields = &op.columns; for column in fields { @@ -421,15 +425,72 @@ fn gen_queries( ); out.push(quote! { #[doc = #doc] - pub fn #method(&self, row: #query_ty, #pk: &#pk_type) -> Option<()> { + fn #method(&self, row: #query_ty, #pk: &#pk_type) -> Option<()> { let at = Self::at(#pk)?; self.inner.update(at, |target| { #(target.#fields = row.#fields;)* }) } }); + + let selector = fields.iter().map(ToString::to_string).collect::>().join("_and_"); + let selector_pascal = { + use convert_case::{Case, Casing as _}; + selector.from_case(Case::Snake).to_case(Case::Pascal) + }; + let selector_type = format_ident!("{name}{selector_pascal}Selector"); + let trait_ident = format_ident!("{name}DenseUpdateBy{}", { + use convert_case::{Case, Casing as _}; + pk.to_string().from_case(Case::Snake).to_case(Case::Pascal) + }); + let (value_type, value) = if fields.len() == 1 { + let field = &fields[0]; + let ty = columns + .columns_map + .get(field) + .ok_or_else(|| Error::new(field.span(), format!("no column `{field}`")))?; + (quote! { #ty }, quote! { #query_ty { #field: value } }) + } else { + (quote! { #query_ty }, quote! { value }) + }; + update_impls.push(quote! { + impl #trait_ident<#value_type> for #selector_type { + fn apply(self, table: &#table, key: &#pk_type, value: #value_type) -> Option<()> { + table.#method(#value, key) + } + } + }); } + let update_dispatch = if update_impls.is_empty() { + quote! {} + } else { + let sealed = format_ident!("__{}_mutation", { + use convert_case::{Case, Casing as _}; + name.to_string().from_case(Case::Pascal).to_case(Case::Snake) + }); + let trait_ident = format_ident!("{name}DenseUpdateBy{}", { + use convert_case::{Case, Casing as _}; + pk.to_string().from_case(Case::Snake).to_case(Case::Pascal) + }); + let method = format_ident!("update_by_{pk}"); + out.push(quote! { + pub fn #method(&self, key: #pk_type, selector: S, value: V) -> Option<()> + where S: #trait_ident + { + selector.apply(self, &key, value) + } + }); + quote! { + #[doc(hidden)] + #[allow(private_bounds)] + pub trait #trait_ident: #sealed::Sealed { + fn apply(self, table: &#table, key: &#pk_type, value: V) -> Option<()>; + } + #(#update_impls)* + } + }; + for (query, op) in &queries.deletes { by_must_be_the_key(pk, query, op, "delete")?; let method = format_ident!("delete_{}", snake(query)); @@ -447,7 +508,7 @@ fn gen_queries( }); } - Ok(out) + Ok((update_dispatch, out)) } /// A dense partition has no secondary index, so a query can only be keyed by diff --git a/codegen/src/generators/in_memory/locks.rs b/codegen/src/generators/in_memory/locks.rs index 1153c65c..224a4e10 100644 --- a/codegen/src/generators/in_memory/locks.rs +++ b/codegen/src/generators/in_memory/locks.rs @@ -144,8 +144,10 @@ impl InMemoryGenerator { .map(|i| { let col = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); quote! { - if let Some(lock) = &self.#col { - set.insert(lock.clone()); + if let Some(existing_lock) = &self.#col { + if !set.iter().any(|entry| worktable::prelude::Arc::ptr_eq(entry, existing_lock)) { + set.push(existing_lock.clone()); + } } self.#col = Some(lock.clone()); } @@ -153,9 +155,8 @@ impl InMemoryGenerator { .collect(); quote! { - #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { - let mut set = worktable::prelude::HashSet::new(); + fn lock(&mut self, id: u16) -> (Vec>, worktable::prelude::Arc) { + let mut set: Vec> = Vec::new(); let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* @@ -175,8 +176,8 @@ impl InMemoryGenerator { if let Some(#col) = &other.#col { if self.#col.is_none() { self.#col = Some(#col.clone()); - } else { - set.insert(#col.clone()); + } else if !set.iter().any(|existing| worktable::prelude::Arc::ptr_eq(existing, #col)) { + set.push(#col.clone()); } } other.#col = self.#col.clone(); @@ -185,9 +186,8 @@ impl InMemoryGenerator { .collect(); quote! { - #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { - let mut set = worktable::prelude::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> Vec> { + let mut set: Vec> = Vec::new(); #(#rows)* set } diff --git a/codegen/src/generators/in_memory/primary_key.rs b/codegen/src/generators/in_memory/primary_key.rs index fcdd0d17..389eb637 100644 --- a/codegen/src/generators/in_memory/primary_key.rs +++ b/codegen/src/generators/in_memory/primary_key.rs @@ -59,13 +59,7 @@ impl InMemoryGenerator { }) .collect::>(); - let unsized_derive = if is_unsized_vec(&types.iter().map(|v| v.to_string()).collect::>()) { - quote! { - VariableSizeMeasure, - } - } else { - quote! {} - }; + let is_unsized_key = is_unsized_vec(&types.iter().map(|v| v.to_string()).collect::>()); let (backend_derive, backend_impl) = primary_key_backend_impl(self.columns.primary_index_backend, &ident, types)?; let borrowed_impl = gen_borrowed_primary_key_impl(&ident, types); @@ -121,6 +115,79 @@ impl InMemoryGenerator { } }; + // `data_bucket_derive::SizeMeasure` reports only 8-byte field + // alignment, so a generated `u128` newtype under-budgets fixed index + // pages. It also cannot derive `VariableSizeMeasure` for a composite + // `(String, u128)` key because the fixed member does not implement the + // variable-size trait. Model the archived wrapper itself: fixed keys + // are exactly its archived root, while String keys add only their + // out-of-line payload to that root. + let string_fields = types + .iter() + .enumerate() + .filter(|(_, ty)| ty.to_string() == "String") + .map(|(index, ty)| (syn::Index::from(index), *ty)) + .collect::>(); + let string_field_indexes = string_fields.iter().map(|(index, _)| index); + let string_field_types = string_fields.iter().map(|(_, ty)| ty); + let aligned_size_body = if is_unsized_key { + quote! { + let len = core::mem::size_of::< + ::Archived + >() #(+ worktable::prelude::SizeMeasurable::aligned_size( + &self.#string_field_indexes + ).saturating_sub(core::mem::size_of::< + <#string_field_types as worktable::prelude::rkyv::Archive>::Archived + >()))*; + let alignment = core::mem::align_of::< + ::Archived + >(); + let remainder = len % alignment; + if remainder == 0 { len } else { len + alignment - remainder } + } + } else { + quote! { + core::mem::size_of::< + ::Archived + >() + } + }; + let size_measure_impl = quote! { + impl worktable::prelude::SizeMeasurable for #ident { + fn aligned_size(&self) -> usize { + #aligned_size_body + } + + fn align() -> Option { + Some(core::mem::align_of::< + ::Archived + >()) + } + } + }; + + let variable_size_measure_impl = if is_unsized_key { + let string_field_types = string_fields.iter().map(|(_, ty)| ty); + quote! { + impl worktable::prelude::VariableSizeMeasurable for #ident { + fn aligned_size(length: usize) -> usize { + let len = core::mem::size_of::< + ::Archived + >() #(+ <#string_field_types as worktable::prelude::VariableSizeMeasurable> + ::aligned_size(length).saturating_sub(core::mem::size_of::< + <#string_field_types as worktable::prelude::rkyv::Archive>::Archived + >()))*; + let alignment = core::mem::align_of::< + ::Archived + >(); + let remainder = len % alignment; + if remainder == 0 { len } else { len + alignment - remainder } + } + } + } + } else { + quote! {} + }; Ok(quote! { #[derive( Clone, @@ -135,9 +202,7 @@ impl InMemoryGenerator { PartialEq, PartialOrd, Ord, - SizeMeasure, MemStat, - #unsized_derive )] #[rkyv(crate = worktable::prelude::rkyv)] #[rkyv(derive(PartialEq, Eq, PartialOrd, Ord, Debug))] @@ -145,6 +210,8 @@ impl InMemoryGenerator { #from_impl #into_impl + #size_measure_impl + #variable_size_measure_impl #borrowed_impl #backend_impl diff --git a/codegen/src/generators/in_memory/queries/delete.rs b/codegen/src/generators/in_memory/queries/delete.rs index df23a6c9..626b647e 100644 --- a/codegen/src/generators/in_memory/queries/delete.rs +++ b/codegen/src/generators/in_memory/queries/delete.rs @@ -73,7 +73,9 @@ impl InMemoryGenerator { where #pk_ident: From { let pk: #pk_ident = pk.into(); - let _mutation_guard = self.0.lock_manager.mutation_guard(&pk); + // SAFETY: the table owns `lock_manager` and is borrowed for + // this whole operation, so the map outlives the guard. + let _mutation_guard = unsafe { self.0.lock_manager.mutation_guard(&pk) }; #publication #delete_logic core::result::Result::Ok(()) diff --git a/codegen/src/generators/in_memory/queries/in_place.rs b/codegen/src/generators/in_memory/queries/in_place.rs index 06450463..c0a1e82d 100644 --- a/codegen/src/generators/in_memory/queries/in_place.rs +++ b/codegen/src/generators/in_memory/queries/in_place.rs @@ -11,8 +11,8 @@ impl InMemoryGenerator { let table_ident = name_generator.get_work_table_ident(); let custom_in_place = if let Some(q) = &self.queries { - let profile = q.in_place_runtime.clone(); - let custom_in_place = self.gen_in_place_queries(q.in_place.clone()); + let profile = q.update_in_place_runtime.clone(); + let custom_in_place = self.gen_in_place_queries(q.updates_in_place.clone()); let custom_in_place = crate::generators::profile_dispatch::wrap( custom_in_place, profile.as_ref(), @@ -61,7 +61,7 @@ impl InMemoryGenerator { let lock_ident = WorktableNameGenerator::get_update_in_place_query_lock_ident(&snake_case_name); let method_ident = Ident::new( - format!("update_{snake_case_name}_in_place").as_str(), + format!("__wt_update_in_place_{snake_case_name}").as_str(), Span::mixed_site(), ); @@ -103,7 +103,7 @@ impl InMemoryGenerator { let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); quote! { - pub async fn #method_ident( + async fn #method_ident( &self, mut f: F, by: Pk, diff --git a/codegen/src/generators/in_memory/queries/locks.rs b/codegen/src/generators/in_memory/queries/locks.rs index 96522cd8..db3ef3a8 100644 --- a/codegen/src/generators/in_memory/queries/locks.rs +++ b/codegen/src/generators/in_memory/queries/locks.rs @@ -14,7 +14,7 @@ impl InMemoryGenerator { let lock_type_ident = name_generator.get_lock_type_ident(); let update_fns = Self::gen_update_query_locks(&q.updates); - let update_in_place_fns = Self::gen_in_place_update_query_locks(&q.in_place); + let update_in_place_fns = Self::gen_in_place_update_query_locks(&q.updates_in_place); Ok(quote! { impl #lock_type_ident { @@ -86,7 +86,13 @@ impl InMemoryGenerator { let col = Ident::new(format!("{col}_lock").as_str(), Span::mixed_site()); quote! { if let Some(lock) = &self.#col { - set.insert(lock.clone()); + // Dedup by pointer: columns can share one lock, and the + // caller must not wait on the same lock twice. Linear + // over a handful of entries, which is cheaper than + // seeding a hasher per operation. + if !set.iter().any(|existing| worktable::prelude::Arc::ptr_eq(existing, lock)) { + set.push(lock.clone()); + } } self.#col = Some(new_lock.clone()); } @@ -94,9 +100,8 @@ impl InMemoryGenerator { .collect::>(); quote! { - #[allow(clippy::mutable_key_type)] - pub fn #ident(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { - let mut set = worktable::prelude::HashSet::new(); + pub fn #ident(&mut self, id: u16) -> (Vec>, worktable::prelude::Arc) { + let mut set = Vec::new(); let new_lock = worktable::prelude::Arc::new(Lock::new(id)); #(#inner)* (set, new_lock) @@ -109,22 +114,37 @@ impl InMemoryGenerator { let lock_ident = name_generator.get_lock_type_ident(); quote! { - let lock_id = self.0.lock_manager.next_id(); + // Striped by the key: a table-wide label counter is one shared + // cache line per locked operation. See LockMap::next_ids. + let lock_id = self.0.lock_manager.next_id_for(&pk); // Same atomic acquire as the per-column path: see LockMap::get_or_insert_with. - let lock = self - .0 - .lock_manager - .get_or_insert_with(pk.clone(), #lock_ident::new); + // SAFETY: `self.0.lock_manager` is the table's own `Arc`, + // borrowed for this whole operation, so the map outlives the + // acquirer and the pending lock taken from it. See the "Borrowed + // guards" note on `LockMap`. + let lock = unsafe { + self.0 + .lock_manager + .get_or_insert_with(pk.clone(), #lock_ident::new) + }; let mut lock_guard = lock.write().await; - #[allow(clippy::mutable_key_type)] let (locks, op_lock) = lock_guard.lock(lock_id); drop(lock_guard); // The registered lock must be cancellation-covered BEFORE the // predecessor wait below: a future dropped at that await (tokio // timeout, task abort) would otherwise leave the registered lock // held forever and hang every later operation on this key. - let pending_lock = PendingLock::new(op_lock, self.0.lock_manager.clone(), pk.clone()); - worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + // SAFETY: as above; the map outlives this pending lock and the + // `LockGuard` it is converted into, both of which are locals of + // this operation. + let pending_lock = unsafe { PendingLock::new(op_lock, &self.0.lock_manager, pk.clone()) }; + // No predecessor is the common case on disjoint keys: the row had no + // entry, so every column slot was empty. Registering a waker on each + // of nothing, boxing the joined slice and suspending the operation to + // poll it once is all overhead there. + if !locks.is_empty() { + worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + } pending_lock } } @@ -134,26 +154,41 @@ impl InMemoryGenerator { let lock_ident = name_generator.get_lock_type_ident(); quote! { - let lock_id = self.0.lock_manager.next_id(); + // Striped by the key: a table-wide label counter is one shared + // cache line per locked operation. See LockMap::next_ids. + let lock_id = self.0.lock_manager.next_id_for(&pk); // One atomic acquire, no check-then-act. Splitting this into `get` // then `insert` let two tasks both miss, both build a lock and both // enter the row: the loser merged into the winner's lock, but the // winner had already registered its operation on a lock that was no // longer the map's, so it never waited for the loser. - let lock = self - .0 - .lock_manager - .get_or_insert_with(pk.clone(), #lock_ident::new); + // SAFETY: `self.0.lock_manager` is the table's own `Arc`, + // borrowed for this whole operation, so the map outlives the + // acquirer and the pending lock taken from it. See the "Borrowed + // guards" note on `LockMap`. + let lock = unsafe { + self.0 + .lock_manager + .get_or_insert_with(pk.clone(), #lock_ident::new) + }; let mut lock_guard = lock.write().await; - #[allow(clippy::mutable_key_type)] let (locks, op_lock) = lock_guard.#ident(lock_id); drop(lock_guard); // The registered lock must be cancellation-covered BEFORE the // predecessor wait below: a future dropped at that await (tokio // timeout, task abort) would otherwise leave the registered lock // held forever and hang every later operation on this key. - let pending_lock = PendingLock::new(op_lock, self.0.lock_manager.clone(), pk.clone()); - worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + // SAFETY: as above; the map outlives this pending lock and the + // `LockGuard` it is converted into, both of which are locals of + // this operation. + let pending_lock = unsafe { PendingLock::new(op_lock, &self.0.lock_manager, pk.clone()) }; + // No predecessor is the common case on disjoint keys: the row had no + // entry, so every column slot was empty. Registering a waker on each + // of nothing, boxing the joined slice and suspending the operation to + // poll it once is all overhead there. + if !locks.is_empty() { + worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + } pending_lock } } diff --git a/codegen/src/generators/in_memory/queries/update.rs b/codegen/src/generators/in_memory/queries/update.rs index 3001fdd3..e31e859d 100644 --- a/codegen/src/generators/in_memory/queries/update.rs +++ b/codegen/src/generators/in_memory/queries/update.rs @@ -1,5 +1,5 @@ use crate::common::model::{Index, Operation}; -use crate::common::name_generator::{WorktableNameGenerator, is_float}; +use crate::common::name_generator::{WorktableNameGenerator, archived_field_requires_rebuild, is_float}; use crate::generators::in_memory::InMemoryGenerator; use convert_case::{Case, Casing}; use indexmap::IndexMap; @@ -7,10 +7,22 @@ use proc_macro2::Literal; use proc_macro2::{Ident, Span, TokenStream}; use quote::quote; +struct UpdateStorage<'a> { + string_fields: Option>, + requires_rebuild: bool, +} + impl InMemoryGenerator { pub fn gen_query_update_impl(&mut self) -> syn::Result { - let custom_updates = if let Some(q) = &self.queries { + let (custom_updates, mutation_api) = if let Some(q) = &self.queries { let profile = q.update_runtime.clone(); + let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); + let mutation_api = crate::generators::mutation_builder::paged_mutation_api( + &self.name, + &name_generator.get_work_table_ident(), + &self.columns, + q, + )?; let custom_updates = self.gen_custom_updates(q.updates.clone()); let custom_updates = crate::generators::profile_dispatch::wrap( custom_updates, @@ -18,17 +30,16 @@ impl InMemoryGenerator { &WorktableNameGenerator::from_table_name(self.name.to_string()).get_row_type_ident(), )?; - quote! { - #custom_updates - } + (quote! { #custom_updates }, mutation_api) } else { - quote! {} + (quote! {}, quote! {}) }; let full_row_update = self.gen_full_row_update(); let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let table_ident = name_generator.get_work_table_ident(); Ok(quote! { + #mutation_api impl #table_ident { #full_row_update #custom_updates @@ -66,7 +77,8 @@ impl InMemoryGenerator { let persist_op = self.gen_persist_op(); let full_row_lock = self.gen_full_lock_for_update(); let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); - // A full-row `update(row)` replaces EVERY column, so it inherently + let requires_rebuild = self.columns.columns_map.values().any(archived_field_requires_rebuild); + // A full-row `replace(row)` replaces EVERY column, so it inherently // rewrites every secondary index. The in-place fast path only applies // when no updated field is indexed (it emits no index diff), so a // full-row update on a table with any secondary index must reinsert. @@ -76,8 +88,9 @@ impl InMemoryGenerator { // gen_size_check; gen_non_unique_update updates a non-unique-indexed // column and therefore always reinserts, correctly.) let const_name = name_generator.get_page_inner_size_const_ident(); - let full_row_in_place_eligible = !self.columns.is_sized && self.columns.indexes.is_empty(); - let update_body = if self.columns.is_sized { + let full_row_in_place_eligible = + (!self.columns.is_sized || requires_rebuild) && self.columns.indexes.is_empty(); + let update_body = if self.columns.is_sized && !requires_rebuild { quote! { let mut bytes = worktable::prelude::rkyv::to_bytes::(&row) .map_err(|_| WorkTableError::SerializeError)?; @@ -148,7 +161,7 @@ impl InMemoryGenerator { }; quote! { - pub async fn update(&self, row: #row_ident) -> core::result::Result<(), WorkTableError> { + pub async fn replace(&self, row: #row_ident) -> core::result::Result<(), WorkTableError> { let pk = row.get_primary_key(); let pending_lock = { #full_row_lock }; let guard = pending_lock.into_guard_with_mutation(); @@ -208,18 +221,19 @@ impl InMemoryGenerator { .collect::>(); if fields.is_empty() { None } else { Some(fields) } }; + let requires_rebuild = op + .columns + .iter() + .any(|column| archived_field_requires_rebuild(self.columns.columns_map.get(column).unwrap())); + let storage = UpdateStorage { + string_fields: unsized_columns, + requires_rebuild, + }; let idents = &op.columns; if let Some(index) = index { if index.is_unique { - self.gen_unique_update( - snake_case_name, - name, - index, - idents, - indexes_columns.as_ref(), - unsized_columns, - ) + self.gen_unique_update(snake_case_name, name, index, idents, indexes_columns.as_ref(), storage) } else { self.gen_non_unique_update( snake_case_name, @@ -227,12 +241,12 @@ impl InMemoryGenerator { index, idents, indexes_columns.as_ref(), - unsized_columns, + storage, ) } } else if self.columns.primary_keys.len() == 1 { if *self.columns.primary_keys.first().unwrap() == op.by { - self.gen_pk_update(snake_case_name, name, idents, indexes_columns.as_ref(), unsized_columns) + self.gen_pk_update(snake_case_name, name, idents, indexes_columns.as_ref(), storage) } else { todo!() } @@ -303,6 +317,7 @@ impl InMemoryGenerator { unsized_fields: Option>, idents: &[Ident], idx_idents: Option<&Vec>, + requires_rebuild: bool, ) -> TokenStream { // The in-place fast path re-serializes the row directly into its slot and // republishes it, bypassing the generated secondary-index diff. That is @@ -311,7 +326,26 @@ impl InMemoryGenerator { // path (and its unique-constraint check). Fall back to always-reinsert in // that case. let touches_index = idx_idents.map(|v| !v.is_empty()).unwrap_or(false); - if let (Some(f), false) = (unsized_fields, touches_index) { + if requires_rebuild { + let row_updates = idents + .iter() + .map(|i| quote! { row_new.#i = row.#i.clone(); }) + .collect::>(); + let full_row_lock = self.gen_full_lock_for_update(); + let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); + let const_name = name_generator.get_page_inner_size_const_ident(); + + // Nothing extra on a successful in-place write: an in-memory table + // has no CDC stream to publish it to. The persisted generator passes + // the operation there instead. See `opaque_rebuild`. + crate::generators::opaque_rebuild::gen_rebuild_arm( + &row_updates, + &full_row_lock, + &const_name, + touches_index, + "e! {}, + ) + } else if let (Some(f), false) = (unsized_fields, touches_index) { let fields_check: Vec<_> = f .iter() .map(|f| { @@ -635,10 +669,14 @@ impl InMemoryGenerator { name: &Ident, idents: &[Ident], idx_idents: Option<&Vec>, - unsized_fields: Option>, + storage: UpdateStorage<'_>, ) -> TokenStream { + let UpdateStorage { + string_fields: unsized_fields, + requires_rebuild, + } = storage; let pk_ident = &self.pk.as_ref().unwrap().ident; - let method_ident = Ident::new(format!("update_{snake_case_name}").as_str(), Span::mixed_site()); + let method_ident = Ident::new(format!("__wt_update_{snake_case_name}").as_str(), Span::mixed_site()); let query_ident = Ident::new(format!("{name}Query").as_str(), Span::mixed_site()); let lock_ident = WorktableNameGenerator::get_update_query_lock_ident(&snake_case_name); @@ -651,8 +689,9 @@ impl InMemoryGenerator { }) .collect::>(); - let archived_swap_is_safe = self.columns.is_sized || (unsized_fields.is_none() && idx_idents.is_none()); - let size_check = self.gen_size_check(unsized_fields, idents, idx_idents); + let archived_swap_is_safe = + !requires_rebuild && (self.columns.is_sized || (unsized_fields.is_none() && idx_idents.is_none())); + let size_check = self.gen_size_check(unsized_fields, idents, idx_idents, requires_rebuild); let diff_process_insert = self.gen_process_diffs_insert_on_index(idents, idx_idents); let diff_process_remove = self.gen_process_diffs_remove_on_index(idx_idents); let persist_call = self.gen_persist_call(); @@ -681,7 +720,7 @@ impl InMemoryGenerator { }; quote! { - pub async fn #method_ident(&self, row: #query_ident, pk: Pk) -> core::result::Result<(), WorkTableError> + async fn #method_ident(&self, row: #query_ident, pk: Pk) -> core::result::Result<(), WorkTableError> where #pk_ident: From { let pk: #pk_ident = pk.into(); @@ -711,11 +750,15 @@ impl InMemoryGenerator { index: &Index, idents: &[Ident], idx_idents: Option<&Vec>, - unsized_fields: Option>, + storage: UpdateStorage<'_>, ) -> TokenStream { + let UpdateStorage { + string_fields: unsized_fields, + requires_rebuild, + } = storage; let by_field = &index.field; let index = &index.name; - let method_ident = Ident::new(format!("update_{snake_case_name}").as_str(), Span::mixed_site()); + let method_ident = Ident::new(format!("__wt_update_{snake_case_name}").as_str(), Span::mixed_site()); let query_ident = Ident::new(format!("{name}Query").as_str(), Span::mixed_site()); let by_ident = Ident::new(format!("{name}By").as_str(), Span::mixed_site()); @@ -733,8 +776,44 @@ impl InMemoryGenerator { // every loop iteration itself (in-place or reinsert, then continue), // so the archived-swap tail is only emitted otherwise; emitting both // would leave unreachable code after the size_check block. - let has_unsized = unsized_fields.is_some(); - let size_check = if let Some(f) = unsized_fields { + let has_unsized = unsized_fields.is_some() || requires_rebuild; + let size_check = if requires_rebuild { + let row_updates = idents + .iter() + .map(|i| quote! { row_new.#i = row.#i.clone(); }) + .collect::>(); + let touches_index = idx_idents.map(|v| !v.is_empty()).unwrap_or(false); + let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); + let const_name = name_generator.get_page_inner_size_const_ident(); + if touches_index { + quote! { + { + let row_old = self.0.select(pk.clone()).ok_or(WorkTableError::NotFound)?; + let mut row_new = row_old.clone(); + #(#row_updates)* + self.reinsert(row_old, row_new).await?; + guards.remove(&pk); + continue; + } + } + } else { + quote! { + { + let row_old = self.0.select(pk.clone()).ok_or(WorkTableError::NotFound)?; + let mut row_new = row_old.clone(); + #(#row_updates)* + let in_place_ok = unsafe { + self.0.data.update_in_place::<{ #const_name }>(row_new.clone(), link).is_ok() + }; + if !in_place_ok { + self.reinsert(row_old, row_new).await?; + } + guards.remove(&pk); + continue; + } + } + } + } else if let Some(f) = unsized_fields { let fields_check: Vec<_> = f .iter() .map(|f| { @@ -844,7 +923,7 @@ impl InMemoryGenerator { }; quote! { - pub async fn #method_ident(&self, row: #query_ident, by: #by_ident) -> core::result::Result<(), WorkTableError> { + async fn #method_ident(&self, row: #query_ident, by: #by_ident) -> core::result::Result<(), WorkTableError> { // This query may update many rows. Keep vacuum out across the // snapshot, lock acquisition, and per-row mutation gaps // without adding work to each row. @@ -901,7 +980,9 @@ impl InMemoryGenerator { if self.0.data.select_non_ghosted(link)?.#by_field != by { continue; } - let _mutation_guard = self.0.lock_manager.mutation_guard(&pk); + // SAFETY: the table owns `lock_manager` and is borrowed + // for this whole operation, so the map outlives the guard. + let _mutation_guard = unsafe { self.0.lock_manager.mutation_guard(&pk) }; let mut bytes = worktable::prelude::rkyv::to_bytes::(&row) .map_err(|_| WorkTableError::SerializeError)?; @@ -926,8 +1007,12 @@ impl InMemoryGenerator { index: &Index, idents: &[Ident], idx_idents: Option<&Vec>, - unsized_fields: Option>, + storage: UpdateStorage<'_>, ) -> TokenStream { + let UpdateStorage { + string_fields: unsized_fields, + requires_rebuild, + } = storage; let by_field = &index.field; let by_is_float = is_float( self.columns @@ -938,7 +1023,7 @@ impl InMemoryGenerator { .as_str(), ); let index = &index.name; - let method_ident = Ident::new(format!("update_{snake_case_name}").as_str(), Span::mixed_site()); + let method_ident = Ident::new(format!("__wt_update_{snake_case_name}").as_str(), Span::mixed_site()); let query_ident = Ident::new(format!("{name}Query").as_str(), Span::mixed_site()); let by_ident = Ident::new(format!("{name}By").as_str(), Span::mixed_site()); @@ -957,8 +1042,9 @@ impl InMemoryGenerator { // nothing and the archived in-place swap below is both safe and the // only body this fn gets - gating on is_sized alone emitted a fn with // no tail expression for exactly that schema. - let archived_swap_is_safe = self.columns.is_sized || (unsized_fields.is_none() && idx_idents.is_none()); - let size_check = self.gen_size_check(unsized_fields, idents, idx_idents); + let archived_swap_is_safe = + !requires_rebuild && (self.columns.is_sized || (unsized_fields.is_none() && idx_idents.is_none())); + let size_check = self.gen_size_check(unsized_fields, idents, idx_idents, requires_rebuild); let diff_process_insert = self.gen_process_diffs_insert_on_index(idents, idx_idents); let diff_process_remove = self.gen_process_diffs_remove_on_index(idx_idents); let persist_call = self.gen_persist_call(); @@ -1012,7 +1098,7 @@ impl InMemoryGenerator { }; quote! { - pub async fn #method_ident(&self, row: #query_ident, by: #by_ident) -> core::result::Result<(), WorkTableError> { + async fn #method_ident(&self, row: #query_ident, by: #by_ident) -> core::result::Result<(), WorkTableError> { let mut bytes = worktable::prelude::rkyv::to_bytes::(&row) .map_err(|_| WorkTableError::SerializeError)?; @@ -1144,7 +1230,7 @@ mod tests { generator.queries = Some(Queries { updates, deletes: IndexMap::new(), - in_place: IndexMap::new(), + updates_in_place: IndexMap::new(), ..Default::default() }); generator.gen_primary_key_def().unwrap(); diff --git a/codegen/src/generators/in_memory/table/impls.rs b/codegen/src/generators/in_memory/table/impls.rs index 54778189..d01801b9 100644 --- a/codegen/src/generators/in_memory/table/impls.rs +++ b/codegen/src/generators/in_memory/table/impls.rs @@ -3,7 +3,7 @@ use proc_macro2::{Ident, Span, TokenStream}; use quote::quote; use crate::common::model::GeneratorType; -use crate::common::name_generator::WorktableNameGenerator; +use crate::common::name_generator::{WorktableNameGenerator, archived_field_is_inline_scalar}; use crate::generators::in_memory::InMemoryGenerator; impl InMemoryGenerator { @@ -75,11 +75,44 @@ impl InMemoryGenerator { let row_type = name_generator.get_row_type_ident(); let primary_key_type = name_generator.get_primary_key_type_ident(); + // `select_with` reads the cell in place with no copy, which is only + // sound when the archived row holds no relative pointers. Emit it only + // for those tables; one with a `String` column simply has no + // `select_with`, so a caller gets "no method named `select_with`" + // instead of a silent copy or a torn pointer. + let select_with_fn = if self + .columns + .columns_map + .values() + .all(|ty| archived_field_is_inline_scalar("e! { #ty })) + { + quote! { + /// Apply `f` to the archived inner row. No cell memcpy; `f` must copy out. + pub fn select_with(&self, pk: Pk, f: F) -> Option + where + #primary_key_type: From, + F: FnMut(&<#row_type as worktable::prelude::rkyv::Archive>::Archived) -> T, + { + self.0.select_with(pk.into(), f) + } + } + } else { + quote! {} + }; + quote! { pub fn select(&self, pk: Pk) -> Option<#row_type> where #primary_key_type: From { self.0.select(pk.into()) } + + /// Pin-guard plus archived inner row. Does not deserialize. + pub fn select_ref(&self, pk: Pk) -> Option> + where #primary_key_type: From { + self.0.select_ref(pk.into()) + } + + #select_with_fn } } @@ -226,7 +259,7 @@ impl InMemoryGenerator { let primary_key_type = name_generator.get_primary_key_type_ident(); quote! { - pub async fn reinsert(&self, row_old: #row_type, row_new: #row_type) -> core::result::Result<#primary_key_type, WorkTableError> { + async fn reinsert(&self, row_old: #row_type, row_new: #row_type) -> core::result::Result<#primary_key_type, WorkTableError> { self.0.reinsert(row_old, row_new).await } } diff --git a/codegen/src/generators/in_memory/wrapper.rs b/codegen/src/generators/in_memory/wrapper.rs index 84563e42..8b05d5d6 100644 --- a/codegen/src/generators/in_memory/wrapper.rs +++ b/codegen/src/generators/in_memory/wrapper.rs @@ -1,4 +1,4 @@ -use crate::common::name_generator::WorktableNameGenerator; +use crate::common::name_generator::{WorktableNameGenerator, archived_field_is_inline_scalar}; use crate::generators::in_memory::InMemoryGenerator; use proc_macro2::TokenStream; use quote::quote; @@ -9,12 +9,14 @@ impl InMemoryGenerator { let impl_ = self.gen_wrapper_impl(); let storable_impl = self.get_wrapper_storable_impl(); let archived_wrapper_impl = self.get_archived_wrapper_impl(); + let inline_archived_impl = self.get_inline_archived_impl(); quote! { #type_ #impl_ #storable_impl #archived_wrapper_impl + #inline_archived_impl } } @@ -72,6 +74,33 @@ impl InMemoryGenerator { } } + /// Emit `InlineArchived` only when every column is an inline scalar. + /// + /// This is what gates the zero-copy `select_with` path. A table with a + /// `String` column simply does not get the impl, so calling `select_with` + /// on it is a compile error naming the missing bound rather than a silent + /// copy or, worse, a torn relative pointer. + fn get_inline_archived_impl(&self) -> TokenStream { + let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); + let wrapper_ident = name_generator.get_wrapper_type_ident(); + + let all_inline = self + .columns + .columns_map + .values() + .all(|ty| archived_field_is_inline_scalar("e! { #ty })); + if !all_inline { + return quote! {}; + } + + quote! { + // SAFETY: every column is a fixed-size scalar, so the archived + // wrapper holds no relative pointers. The bookkeeping flags + // beside `inner` are `bool`. + unsafe impl worktable::prelude::InlineArchived for #wrapper_ident {} + } + } + fn get_wrapper_storable_impl(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let row_ident = name_generator.get_row_type_ident(); @@ -86,10 +115,21 @@ impl InMemoryGenerator { fn get_archived_wrapper_impl(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); - let row_ident = name_generator.get_archived_wrapper_type_ident(); + let archived_ident = name_generator.get_archived_wrapper_type_ident(); + let row_ident = name_generator.get_row_type_ident(); quote! { - impl ArchivedRowWrapper for #row_ident { + impl ArchivedRowWrapper for #archived_ident { + type Inner = <#row_ident as worktable::prelude::rkyv::Archive>::Archived; + + fn inner(&self) -> &Self::Inner { + &self.inner + } + + fn is_ghosted(&self) -> bool { + self.is_ghosted + } + fn unghost(&mut self) { self.is_ghosted = false; } diff --git a/codegen/src/generators/mod.rs b/codegen/src/generators/mod.rs index 1b0fe791..18a4397f 100644 --- a/codegen/src/generators/mod.rs +++ b/codegen/src/generators/mod.rs @@ -2,6 +2,8 @@ pub(crate) mod columnar; pub(crate) mod dense_table; pub mod in_memory; pub(crate) mod index_backend; +pub(crate) mod mutation_builder; +pub(crate) mod opaque_rebuild; pub mod partitions; pub mod persist; pub(crate) mod primary_key; diff --git a/codegen/src/generators/mutation_builder.rs b/codegen/src/generators/mutation_builder.rs new file mode 100644 index 00000000..580a0ab5 --- /dev/null +++ b/codegen/src/generators/mutation_builder.rs @@ -0,0 +1,544 @@ +use convert_case::{Case, Casing}; +use indexmap::{IndexMap, IndexSet}; +use proc_macro2::{Ident, Span, TokenStream}; +use quote::{format_ident, quote}; +use worktable_dsl::model::{Columns, Operation, Queries}; + +fn selector_name(operation: &Operation) -> String { + operation + .columns + .iter() + .map(ToString::to_string) + .collect::>() + .join("_and_") +} + +fn validate_collisions(operations: &IndexMap, family: &str) -> syn::Result<()> { + let mut generated: IndexMap<(String, String), &Ident> = IndexMap::new(); + for (name, operation) in operations { + let key = (operation.by.to_string(), selector_name(operation)); + if let Some(previous) = generated.insert(key.clone(), name) { + return Err(syn::Error::new( + name.span(), + format!( + "{family} queries `{previous}` and `{name}` both generate `{family}_by_{}(..., Columns::{}, ...)`", + key.0, + key.1.to_case(Case::UpperSnake) + ), + )); + } + } + Ok(()) +} + +fn selector_definitions(name: &Ident, queries: &Queries) -> syn::Result { + let columns = format_ident!("{name}Columns"); + let sealed = format_ident!("__{}_mutation", name.to_string().to_case(Case::Snake)); + let mut seen: IndexMap> = IndexMap::new(); + let mut definitions = Vec::new(); + let mut constants = Vec::new(); + + for operation in queries.updates.values().chain(queries.updates_in_place.values()) { + let field_set = operation.columns.iter().map(ToString::to_string).collect::>(); + let selector = selector_name(operation); + if let Some(previous) = seen.get(&selector) { + if previous != &field_set { + return Err(syn::Error::new( + operation.name.span(), + format!( + "declared field sets `{}` and `{}` generate the same selector name `{selector}`", + previous.join(", "), + field_set.join(", ") + ), + )); + } + continue; + } + seen.insert(selector.clone(), field_set); + let selector_pascal = selector.from_case(Case::Snake).to_case(Case::Pascal); + let selector_type = format_ident!("{name}{selector_pascal}Selector"); + // Preserve identifier spelling: Rust's conventional constant form is + // ASCII uppercase with the source underscores left in place. Case + // conversion would unexpectedly turn `attr1` into `ATTR_1`. + let constant = Ident::new(&selector.to_ascii_uppercase(), Span::mixed_site()); + definitions.push(quote! { + #[doc(hidden)] + #[derive(Clone, Copy, Debug, Default)] + pub struct #selector_type; + impl #sealed::Sealed for #selector_type {} + }); + constants.push(quote! { pub const #constant: #selector_type = #selector_type; }); + } + + if definitions.is_empty() { + return Ok(quote! {}); + } + Ok(quote! { + mod #sealed { pub trait Sealed {} } + pub struct #columns; + impl #columns { #(#constants)* } + #(#definitions)* + }) +} + +pub(crate) fn paged_mutation_api( + name: &Ident, + table: &Ident, + columns: &Columns, + queries: &Queries, +) -> syn::Result { + validate_collisions(&queries.updates, "update")?; + validate_collisions(&queries.updates_in_place, "update_in_place")?; + + let selectors = selector_definitions(name, queries)?; + let update = paged_updates(name, table, columns, &queries.updates, queries.update_runtime.is_some())?; + let update_in_place = paged_updates_in_place( + name, + table, + columns, + &queries.updates_in_place, + queries.update_in_place_runtime.is_some(), + )?; + Ok(quote! { #selectors #update #update_in_place }) +} + +/// The same typed selector surface for a single-writer `Vec` table. +/// +/// The dispatch is monomorphized and sealed exactly like the paged surface; +/// only the receiver and result reflect the synchronous storage shape. +pub(crate) fn vec_mutation_api( + name: &Ident, + table: &Ident, + columns: &Columns, + queries: &Queries, +) -> syn::Result { + validate_collisions(&queries.updates, "update")?; + validate_collisions(&queries.updates_in_place, "update_in_place")?; + let selectors = selector_definitions(name, queries)?; + let sealed = format_ident!("__{}_mutation", name.to_string().to_case(Case::Snake)); + let update = vec_updates(name, table, columns, &queries.updates, &sealed)?; + let update_in_place = vec_updates_in_place(name, table, columns, &queries.updates_in_place, &sealed)?; + Ok(quote! { #selectors #update #update_in_place }) +} + +fn vec_updates( + name: &Ident, + table: &Ident, + columns: &Columns, + operations: &IndexMap, + sealed: &Ident, +) -> syn::Result { + let mut by_fields = IndexSet::new(); + let mut implementations = Vec::new(); + for (query_name, operation) in operations { + by_fields.insert(operation.by.clone()); + let by = &operation.by; + let by_type = columns + .columns_map + .get(by) + .ok_or_else(|| syn::Error::new(by.span(), format!("no column `{by}`")))?; + let by_pascal = by.to_string().from_case(Case::Snake).to_case(Case::Pascal); + let trait_ident = format_ident!("{name}VecUpdateBy{by_pascal}"); + let selector_pascal = selector_name(operation).from_case(Case::Snake).to_case(Case::Pascal); + let selector = format_ident!("{name}{selector_pascal}Selector"); + let query = format_ident!("{query_name}Query"); + let hidden = format_ident!( + "__wt_update_{}", + query_name.to_string().from_case(Case::Pascal).to_case(Case::Snake) + ); + let (value_type, value) = if operation.columns.len() == 1 { + let field = &operation.columns[0]; + let ty = columns + .columns_map + .get(field) + .ok_or_else(|| syn::Error::new(field.span(), format!("no column `{field}`")))?; + (quote! { #ty }, quote! { #query { #field: value } }) + } else { + (quote! { #query }, quote! { value }) + }; + implementations.push(quote! { + impl #trait_ident<#value_type> for #selector { + type Key = #by_type; + fn apply(self, table: &mut #table, key: &Self::Key, value: #value_type) -> usize { + table.#hidden(#value, key) + } + } + }); + } + let mut traits_and_methods = Vec::new(); + for by in by_fields { + let by_type = columns + .columns_map + .get(&by) + .ok_or_else(|| syn::Error::new(by.span(), format!("no column `{by}`")))?; + let by_pascal = by.to_string().from_case(Case::Snake).to_case(Case::Pascal); + let trait_ident = format_ident!("{name}VecUpdateBy{by_pascal}"); + let method = format_ident!("update_by_{by}"); + traits_and_methods.push(quote! { + #[doc(hidden)] + #[allow(private_bounds)] + pub trait #trait_ident: #sealed::Sealed { + type Key; + fn apply(self, table: &mut #table, key: &Self::Key, value: V) -> usize; + } + impl #table { + pub fn #method(&mut self, key: #by_type, selector: S, value: V) -> usize + where S: #trait_ident + { + selector.apply(self, &key, value) + } + } + }); + } + Ok(quote! { #(#traits_and_methods)* #(#implementations)* }) +} + +fn vec_updates_in_place( + name: &Ident, + table: &Ident, + columns: &Columns, + operations: &IndexMap, + sealed: &Ident, +) -> syn::Result { + let mut by_fields = IndexSet::new(); + let mut implementations = Vec::new(); + for (query_name, operation) in operations { + by_fields.insert(operation.by.clone()); + let by = &operation.by; + let by_type = columns + .columns_map + .get(by) + .ok_or_else(|| syn::Error::new(by.span(), format!("no column `{by}`")))?; + let by_pascal = by.to_string().from_case(Case::Snake).to_case(Case::Pascal); + let trait_ident = format_ident!("{name}VecUpdateInPlaceBy{by_pascal}"); + let selector_pascal = selector_name(operation).from_case(Case::Snake).to_case(Case::Pascal); + let selector = format_ident!("{name}{selector_pascal}Selector"); + let fields = &operation.columns; + let field_types = fields + .iter() + .map(|field| { + columns + .columns_map + .get(field) + .ok_or_else(|| syn::Error::new(field.span(), format!("no column `{field}`"))) + }) + .collect::>>()?; + let closure_arg = if field_types.len() == 1 { + let ty = field_types[0]; + quote! { &mut #ty } + } else { + quote! { ( #(&mut #field_types),* ) } + }; + let hidden = format_ident!( + "__wt_update_in_place_{}", + query_name.to_string().from_case(Case::Pascal).to_case(Case::Snake) + ); + implementations.push(quote! { + impl #trait_ident for #selector where F: FnMut(#closure_arg) { + type Key = #by_type; + fn apply(self, table: &mut #table, key: &Self::Key, edit: F) -> usize { + table.#hidden(edit, key) + } + } + }); + } + let mut traits_and_methods = Vec::new(); + for by in by_fields { + let by_type = columns + .columns_map + .get(&by) + .ok_or_else(|| syn::Error::new(by.span(), format!("no column `{by}`")))?; + let by_pascal = by.to_string().from_case(Case::Snake).to_case(Case::Pascal); + let trait_ident = format_ident!("{name}VecUpdateInPlaceBy{by_pascal}"); + let method = format_ident!("update_in_place_by_{by}"); + traits_and_methods.push(quote! { + #[doc(hidden)] + #[allow(private_bounds)] + pub trait #trait_ident: #sealed::Sealed { + type Key; + fn apply(self, table: &mut #table, key: &Self::Key, edit: F) -> usize; + } + impl #table { + pub fn #method(&mut self, key: #by_type, selector: S, edit: F) -> usize + where S: #trait_ident + { + selector.apply(self, &key, edit) + } + } + }); + } + Ok(quote! { #(#traits_and_methods)* #(#implementations)* }) +} + +fn paged_updates( + name: &Ident, + table: &Ident, + columns: &Columns, + operations: &IndexMap, + scheduled: bool, +) -> syn::Result { + let sealed = format_ident!("__{}_mutation", name.to_string().to_case(Case::Snake)); + let mut by_fields = IndexSet::new(); + let mut implementations = Vec::new(); + + for (query_name, operation) in operations { + by_fields.insert(operation.by.clone()); + let by = &operation.by; + let by_type = columns + .columns_map + .get(by) + .ok_or_else(|| syn::Error::new(by.span(), format!("no column `{by}`")))?; + let key_type = if columns.primary_keys.contains(by) && !columns.indexes.values().any(|index| index.field == *by) + { + let primary_key = format_ident!("{name}PrimaryKey"); + quote! { #primary_key } + } else { + quote! { #by_type } + }; + let by_pascal = by.to_string().from_case(Case::Snake).to_case(Case::Pascal); + let trait_ident = format_ident!("{name}UpdateBy{by_pascal}"); + let selector_pascal = selector_name(operation).from_case(Case::Snake).to_case(Case::Pascal); + let selector = format_ident!("{name}{selector_pascal}Selector"); + let query = format_ident!("{query_name}Query"); + let hidden = format_ident!( + "__wt_update_{}", + query_name.to_string().from_case(Case::Pascal).to_case(Case::Snake) + ); + let (value_type, value) = if operation.columns.len() == 1 { + let field = &operation.columns[0]; + let ty = columns + .columns_map + .get(field) + .ok_or_else(|| syn::Error::new(field.span(), format!("no column `{field}`")))?; + (quote! { #ty }, quote! { #query { #field: value } }) + } else { + (quote! { #query }, quote! { value }) + }; + let table_ref = if scheduled { + quote! { &worktable::prelude::Arc<#table> } + } else { + quote! { &#table } + }; + implementations.push(quote! { + impl #trait_ident<#value_type> for #selector { + type Key = #key_type; + async fn apply(self, table: #table_ref, key: Self::Key, value: #value_type) + -> core::result::Result<(), WorkTableError> + { + table.#hidden(#value, key).await + } + } + }); + } + + let mut traits_and_methods = Vec::new(); + for by in by_fields { + let by_type = columns + .columns_map + .get(&by) + .ok_or_else(|| syn::Error::new(by.span(), format!("no column `{by}`")))?; + let key_type = if columns.primary_keys.contains(&by) && !columns.indexes.values().any(|index| index.field == by) + { + let primary_key = format_ident!("{name}PrimaryKey"); + quote! { #primary_key } + } else { + quote! { #by_type } + }; + let by_pascal = by.to_string().from_case(Case::Snake).to_case(Case::Pascal); + let trait_ident = format_ident!("{name}UpdateBy{by_pascal}"); + let method = format_ident!("update_by_{by}"); + let table_ref = if scheduled { + quote! { &worktable::prelude::Arc<#table> } + } else { + quote! { &#table } + }; + let receiver = if scheduled { + quote! { self: &worktable::prelude::Arc } + } else { + quote! { &self } + }; + let method_impl = + if columns.primary_keys.contains(&by) && !columns.indexes.values().any(|index| index.field == by) { + quote! { + impl #table { + pub async fn #method(#receiver, key: K, selector: S, value: V) + -> core::result::Result<(), WorkTableError> + where + S: #trait_ident, + #key_type: From, + { + selector.apply(self, key.into(), value).await + } + } + } + } else { + quote! { + impl #table { + pub async fn #method(#receiver, key: #by_type, selector: S, value: V) + -> core::result::Result<(), WorkTableError> + where S: #trait_ident + { + selector.apply(self, key, value).await + } + } + } + }; + traits_and_methods.push(quote! { + #[doc(hidden)] + #[allow(private_bounds)] + #[allow(async_fn_in_trait)] + pub trait #trait_ident: #sealed::Sealed { + type Key; + async fn apply(self, table: #table_ref, key: Self::Key, value: V) + -> core::result::Result<(), WorkTableError>; + } + + #method_impl + }); + } + Ok(quote! { #(#traits_and_methods)* #(#implementations)* }) +} + +fn paged_updates_in_place( + name: &Ident, + table: &Ident, + columns: &Columns, + operations: &IndexMap, + scheduled: bool, +) -> syn::Result { + let sealed = format_ident!("__{}_mutation", name.to_string().to_case(Case::Snake)); + let mut by_fields = IndexSet::new(); + let mut implementations = Vec::new(); + + for (query_name, operation) in operations { + by_fields.insert(operation.by.clone()); + let by = &operation.by; + let by_type = columns + .columns_map + .get(by) + .ok_or_else(|| syn::Error::new(by.span(), format!("no column `{by}`")))?; + let key_type = if columns.primary_keys.contains(by) && !columns.indexes.values().any(|index| index.field == *by) + { + let primary_key = format_ident!("{name}PrimaryKey"); + quote! { #primary_key } + } else { + quote! { #by_type } + }; + let by_pascal = by.to_string().from_case(Case::Snake).to_case(Case::Pascal); + let trait_ident = format_ident!("{name}UpdateInPlaceBy{by_pascal}"); + let selector_pascal = selector_name(operation).from_case(Case::Snake).to_case(Case::Pascal); + let selector = format_ident!("{name}{selector_pascal}Selector"); + let field_types = operation + .columns + .iter() + .map(|field| { + columns + .columns_map + .get(field) + .ok_or_else(|| syn::Error::new(field.span(), format!("no column `{field}`"))) + }) + .collect::>>()?; + let closure_arg = if field_types.len() == 1 { + let ty = field_types[0]; + quote! { &mut <#ty as worktable::prelude::rkyv::Archive>::Archived } + } else { + let archived = field_types.iter().map(|ty| { + quote! { &mut <#ty as worktable::prelude::rkyv::Archive>::Archived } + }); + quote! { ( #(#archived),* ) } + }; + let hidden = format_ident!( + "__wt_update_in_place_{}", + query_name.to_string().from_case(Case::Pascal).to_case(Case::Snake) + ); + let table_ref = if scheduled { + quote! { &worktable::prelude::Arc<#table> } + } else { + quote! { &#table } + }; + let send = if scheduled { + quote! { + Send + 'static } + } else { + quote! {} + }; + implementations.push(quote! { + impl #trait_ident for #selector + where F: FnMut(#closure_arg) #send + { + type Key = #key_type; + async fn apply(self, table: #table_ref, key: Self::Key, edit: F) + -> worktable::prelude::eyre::Result<()> + { + table.#hidden(edit, key).await + } + } + }); + } + + let mut traits_and_methods = Vec::new(); + for by in by_fields { + let by_type = columns + .columns_map + .get(&by) + .ok_or_else(|| syn::Error::new(by.span(), format!("no column `{by}`")))?; + let key_type = if columns.primary_keys.contains(&by) && !columns.indexes.values().any(|index| index.field == by) + { + let primary_key = format_ident!("{name}PrimaryKey"); + quote! { #primary_key } + } else { + quote! { #by_type } + }; + let by_pascal = by.to_string().from_case(Case::Snake).to_case(Case::Pascal); + let trait_ident = format_ident!("{name}UpdateInPlaceBy{by_pascal}"); + let method = format_ident!("update_in_place_by_{by}"); + let table_ref = if scheduled { + quote! { &worktable::prelude::Arc<#table> } + } else { + quote! { &#table } + }; + let receiver = if scheduled { + quote! { self: &worktable::prelude::Arc } + } else { + quote! { &self } + }; + let method_impl = + if columns.primary_keys.contains(&by) && !columns.indexes.values().any(|index| index.field == by) { + quote! { + impl #table { + pub async fn #method(#receiver, key: K, selector: S, edit: F) + -> worktable::prelude::eyre::Result<()> + where + S: #trait_ident, + #key_type: From, + { + selector.apply(self, key.into(), edit).await + } + } + } + } else { + quote! { + impl #table { + pub async fn #method(#receiver, key: #by_type, selector: S, edit: F) + -> worktable::prelude::eyre::Result<()> + where S: #trait_ident + { + selector.apply(self, key, edit).await + } + } + } + }; + traits_and_methods.push(quote! { + #[doc(hidden)] + #[allow(private_bounds)] + #[allow(async_fn_in_trait)] + pub trait #trait_ident: #sealed::Sealed { + type Key; + async fn apply(self, table: #table_ref, key: Self::Key, edit: F) + -> worktable::prelude::eyre::Result<()>; + } + + #method_impl + }); + } + Ok(quote! { #(#traits_and_methods)* #(#implementations)* }) +} diff --git a/codegen/src/generators/opaque_rebuild.rs b/codegen/src/generators/opaque_rebuild.rs new file mode 100644 index 00000000..43858f0e --- /dev/null +++ b/codegen/src/generators/opaque_rebuild.rs @@ -0,0 +1,84 @@ +//! The opaque-field rebuild arm of `gen_size_check`, shared by the in-memory +//! and persisted update generators. +//! +//! A column whose archived form the macro cannot inspect may contain relative +//! pointers, so an in-place update of one cannot be done field-by-field: the +//! whole row has to be rebuilt under its full row lock, so that every pointer +//! is based in the destination slot. The two generators emit exactly the same +//! code for that, differing only in whether the successful in-place write is +//! also published as a CDC operation. +//! +//! It lived as two verbatim copies, and this is a correctness-critical path: +//! a fix applied to one copy and not the other is a table whose opaque-field +//! update is right in memory and wrong on disk, or the reverse. Taking the +//! difference as a parameter is what keeps them in step. + +use proc_macro2::{Ident, TokenStream}; +use quote::quote; + +/// Emits the `requires_rebuild` arm. +/// +/// - `row_updates`: per-column assignments onto the rebuilt row. +/// - `full_row_lock`: the generator's full-row lock acquisition. +/// - `page_inner_size_const`: the table's page-inner-size constant. +/// - `touches_index`: whether any updated column is indexed. An indexed column +/// must keep the index-maintaining reinsert path and its unique-constraint +/// check, so the in-place attempt is skipped entirely. +/// - `on_in_place_success`: emitted inside the `in_place_ok` branch before it +/// returns. Empty for the in-memory generator; the persisted one publishes +/// the same-slot write there as an event-less data operation. +pub(crate) fn gen_rebuild_arm( + row_updates: &[TokenStream], + full_row_lock: &TokenStream, + page_inner_size_const: &Ident, + touches_index: bool, + on_in_place_success: &TokenStream, +) -> TokenStream { + if touches_index { + return quote! { + { + drop(_guard); + let pending_lock = { #full_row_lock }; + let _guard = pending_lock.into_guard_with_mutation(); + + let row_old = self.0.select(pk.clone()).ok_or(WorkTableError::NotFound)?; + let mut row_new = row_old.clone(); + #(#row_updates)* + self.reinsert(row_old, row_new).await?; + return core::result::Result::Ok(()); + } + }; + } + + quote! { + { + // An opaque archived field may contain relative pointers. + // Rebuild the complete row under its full lock so every + // pointer is based in the destination slot, then retain the + // current link when the serialized length still fits. + drop(_guard); + let pending_lock = { #full_row_lock }; + let _guard = pending_lock.into_guard_with_mutation(); + + let row_old = self.0.select(pk.clone()).ok_or(WorkTableError::NotFound)?; + let mut row_new = row_old.clone(); + #(#row_updates)* + let current_link: Link = self.0 + .primary_index + .pk_map + .get_value(&pk) + .map(Into::into) + .ok_or(WorkTableError::NotFound)?; + let in_place_ok = unsafe { + self.0.data.update_in_place::<{ #page_inner_size_const }>(row_new.clone(), current_link).is_ok() + }; + if in_place_ok { + #on_in_place_success + return core::result::Result::Ok(()); + } + + self.reinsert(row_old, row_new).await?; + return core::result::Result::Ok(()); + } + } +} diff --git a/codegen/src/generators/persist/locks.rs b/codegen/src/generators/persist/locks.rs index 86b55b60..0f6c577d 100644 --- a/codegen/src/generators/persist/locks.rs +++ b/codegen/src/generators/persist/locks.rs @@ -144,8 +144,10 @@ impl PersistGenerator { .map(|i| { let col = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); quote! { - if let Some(lock) = &self.#col { - set.insert(lock.clone()); + if let Some(existing_lock) = &self.#col { + if !set.iter().any(|entry| worktable::prelude::Arc::ptr_eq(entry, existing_lock)) { + set.push(existing_lock.clone()); + } } self.#col = Some(lock.clone()); } @@ -153,9 +155,8 @@ impl PersistGenerator { .collect(); quote! { - #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { - let mut set = worktable::prelude::HashSet::new(); + fn lock(&mut self, id: u16) -> (Vec>, worktable::prelude::Arc) { + let mut set: Vec> = Vec::new(); let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* @@ -175,8 +176,8 @@ impl PersistGenerator { if let Some(#col) = &other.#col { if self.#col.is_none() { self.#col = Some(#col.clone()); - } else { - set.insert(#col.clone()); + } else if !set.iter().any(|existing| worktable::prelude::Arc::ptr_eq(existing, #col)) { + set.push(#col.clone()); } } other.#col = self.#col.clone(); @@ -185,9 +186,8 @@ impl PersistGenerator { .collect(); quote! { - #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { - let mut set = worktable::prelude::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> Vec> { + let mut set: Vec> = Vec::new(); #(#rows)* set } diff --git a/codegen/src/generators/persist/primary_key.rs b/codegen/src/generators/persist/primary_key.rs index 5d1b4098..f3a2c4a1 100644 --- a/codegen/src/generators/persist/primary_key.rs +++ b/codegen/src/generators/persist/primary_key.rs @@ -55,13 +55,7 @@ impl PersistGenerator { .expect("should exist as got from definition") }) .collect::>(); - let unsized_derive = if is_unsized_vec(&types.iter().map(|v| v.to_string()).collect::>()) { - quote! { - VariableSizeMeasure, - } - } else { - quote! {} - }; + let is_unsized_key = is_unsized_vec(&types.iter().map(|v| v.to_string()).collect::>()); let (backend_derive, backend_impl) = primary_key_backend_impl(self.columns.primary_index_backend, &ident, types)?; let borrowed_impl = gen_borrowed_primary_key_impl(&ident, types); @@ -117,6 +111,79 @@ impl PersistGenerator { } }; + // `data_bucket_derive::SizeMeasure` reports only 8-byte field + // alignment, so a generated `u128` newtype under-budgets fixed index + // pages. It also cannot derive `VariableSizeMeasure` for a composite + // `(String, u128)` key because the fixed member does not implement the + // variable-size trait. Model the archived wrapper itself: fixed keys + // are exactly its archived root, while String keys add only their + // out-of-line payload to that root. + let string_fields = types + .iter() + .enumerate() + .filter(|(_, ty)| ty.to_string() == "String") + .map(|(index, ty)| (syn::Index::from(index), *ty)) + .collect::>(); + let string_field_indexes = string_fields.iter().map(|(index, _)| index); + let string_field_types = string_fields.iter().map(|(_, ty)| ty); + let aligned_size_body = if is_unsized_key { + quote! { + let len = core::mem::size_of::< + ::Archived + >() #(+ worktable::prelude::SizeMeasurable::aligned_size( + &self.#string_field_indexes + ).saturating_sub(core::mem::size_of::< + <#string_field_types as worktable::prelude::rkyv::Archive>::Archived + >()))*; + let alignment = core::mem::align_of::< + ::Archived + >(); + let remainder = len % alignment; + if remainder == 0 { len } else { len + alignment - remainder } + } + } else { + quote! { + core::mem::size_of::< + ::Archived + >() + } + }; + let size_measure_impl = quote! { + impl worktable::prelude::SizeMeasurable for #ident { + fn aligned_size(&self) -> usize { + #aligned_size_body + } + + fn align() -> Option { + Some(core::mem::align_of::< + ::Archived + >()) + } + } + }; + + let variable_size_measure_impl = if is_unsized_key { + let string_field_types = string_fields.iter().map(|(_, ty)| ty); + quote! { + impl worktable::prelude::VariableSizeMeasurable for #ident { + fn aligned_size(length: usize) -> usize { + let len = core::mem::size_of::< + ::Archived + >() #(+ <#string_field_types as worktable::prelude::VariableSizeMeasurable> + ::aligned_size(length).saturating_sub(core::mem::size_of::< + <#string_field_types as worktable::prelude::rkyv::Archive>::Archived + >()))*; + let alignment = core::mem::align_of::< + ::Archived + >(); + let remainder = len % alignment; + if remainder == 0 { len } else { len + alignment - remainder } + } + } + } + } else { + quote! {} + }; Ok(quote! { #[derive( Clone, @@ -131,9 +198,7 @@ impl PersistGenerator { PartialEq, PartialOrd, Ord, - SizeMeasure, MemStat, - #unsized_derive )] #[rkyv(crate = worktable::prelude::rkyv)] #[rkyv(derive(PartialEq, Eq, PartialOrd, Ord, Debug))] @@ -141,6 +206,8 @@ impl PersistGenerator { #from_impl #into_impl + #size_measure_impl + #variable_size_measure_impl #borrowed_impl #backend_impl diff --git a/codegen/src/generators/persist/queries/delete.rs b/codegen/src/generators/persist/queries/delete.rs index e57342f9..783fca7b 100644 --- a/codegen/src/generators/persist/queries/delete.rs +++ b/codegen/src/generators/persist/queries/delete.rs @@ -73,7 +73,9 @@ impl PersistGenerator { where #pk_ident: From { let pk: #pk_ident = pk.into(); - let _mutation_guard = self.0.lock_manager.mutation_guard(&pk); + // SAFETY: the table owns `lock_manager` and is borrowed for + // this whole operation, so the map outlives the guard. + let _mutation_guard = unsafe { self.0.lock_manager.mutation_guard(&pk) }; #publication #delete_logic core::result::Result::Ok(()) diff --git a/codegen/src/generators/persist/queries/in_place.rs b/codegen/src/generators/persist/queries/in_place.rs index 4b6d6289..f87e7ee0 100644 --- a/codegen/src/generators/persist/queries/in_place.rs +++ b/codegen/src/generators/persist/queries/in_place.rs @@ -13,8 +13,8 @@ impl PersistGenerator { let table_ident = name_generator.get_work_table_ident(); let custom_in_place = if let Some(q) = &self.queries { - let profile = q.in_place_runtime.clone(); - let custom_in_place = self.gen_in_place_queries(q.in_place.clone()); + let profile = q.update_in_place_runtime.clone(); + let custom_in_place = self.gen_in_place_queries(q.updates_in_place.clone()); let custom_in_place = crate::generators::profile_dispatch::wrap( custom_in_place, profile.as_ref(), @@ -64,7 +64,7 @@ impl PersistGenerator { let lock_ident = WorktableNameGenerator::get_update_in_place_query_lock_ident(&snake_case_name); let method_ident = Ident::new( - format!("update_{snake_case_name}_in_place").as_str(), + format!("__wt_update_in_place_{snake_case_name}").as_str(), Span::mixed_site(), ); @@ -106,7 +106,7 @@ impl PersistGenerator { let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); quote! { - pub async fn #method_ident( + async fn #method_ident( &self, mut f: F, by: Pk, diff --git a/codegen/src/generators/persist/queries/locks.rs b/codegen/src/generators/persist/queries/locks.rs index 6739cd86..b4aa4a9b 100644 --- a/codegen/src/generators/persist/queries/locks.rs +++ b/codegen/src/generators/persist/queries/locks.rs @@ -14,7 +14,7 @@ impl PersistGenerator { let lock_type_ident = name_generator.get_lock_type_ident(); let update_fns = Self::gen_update_query_locks(&q.updates); - let update_in_place_fns = Self::gen_in_place_update_query_locks(&q.in_place); + let update_in_place_fns = Self::gen_in_place_update_query_locks(&q.updates_in_place); Ok(quote! { impl #lock_type_ident { @@ -86,7 +86,13 @@ impl PersistGenerator { let col = Ident::new(format!("{col}_lock").as_str(), Span::mixed_site()); quote! { if let Some(lock) = &self.#col { - set.insert(lock.clone()); + // Dedup by pointer: columns can share one lock, and the + // caller must not wait on the same lock twice. Linear + // over a handful of entries, which is cheaper than + // seeding a hasher per operation. + if !set.iter().any(|existing| worktable::prelude::Arc::ptr_eq(existing, lock)) { + set.push(lock.clone()); + } } self.#col = Some(new_lock.clone()); } @@ -94,9 +100,8 @@ impl PersistGenerator { .collect::>(); quote! { - #[allow(clippy::mutable_key_type)] - pub fn #ident(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { - let mut set = worktable::prelude::HashSet::new(); + pub fn #ident(&mut self, id: u16) -> (Vec>, worktable::prelude::Arc) { + let mut set = Vec::new(); let new_lock = worktable::prelude::Arc::new(Lock::new(id)); #(#inner)* (set, new_lock) @@ -109,22 +114,37 @@ impl PersistGenerator { let lock_ident = name_generator.get_lock_type_ident(); quote! { - let lock_id = self.0.lock_manager.next_id(); + // Striped by the key: a table-wide label counter is one shared + // cache line per locked operation. See LockMap::next_ids. + let lock_id = self.0.lock_manager.next_id_for(&pk); // Same atomic acquire as the per-column path: see LockMap::get_or_insert_with. - let lock = self - .0 - .lock_manager - .get_or_insert_with(pk.clone(), #lock_ident::new); + // SAFETY: `self.0.lock_manager` is the table's own `Arc`, + // borrowed for this whole operation, so the map outlives the + // acquirer and the pending lock taken from it. See the "Borrowed + // guards" note on `LockMap`. + let lock = unsafe { + self.0 + .lock_manager + .get_or_insert_with(pk.clone(), #lock_ident::new) + }; let mut lock_guard = lock.write().await; - #[allow(clippy::mutable_key_type)] let (locks, op_lock) = lock_guard.lock(lock_id); drop(lock_guard); // The registered lock must be cancellation-covered BEFORE the // predecessor wait below: a future dropped at that await (tokio // timeout, task abort) would otherwise leave the registered lock // held forever and hang every later operation on this key. - let pending_lock = PendingLock::new(op_lock, self.0.lock_manager.clone(), pk.clone()); - worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + // SAFETY: as above; the map outlives this pending lock and the + // `LockGuard` it is converted into, both of which are locals of + // this operation. + let pending_lock = unsafe { PendingLock::new(op_lock, &self.0.lock_manager, pk.clone()) }; + // No predecessor is the common case on disjoint keys: the row had no + // entry, so every column slot was empty. Registering a waker on each + // of nothing, boxing the joined slice and suspending the operation to + // poll it once is all overhead there. + if !locks.is_empty() { + worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + } pending_lock } } @@ -134,26 +154,41 @@ impl PersistGenerator { let lock_ident = name_generator.get_lock_type_ident(); quote! { - let lock_id = self.0.lock_manager.next_id(); + // Striped by the key: a table-wide label counter is one shared + // cache line per locked operation. See LockMap::next_ids. + let lock_id = self.0.lock_manager.next_id_for(&pk); // One atomic acquire, no check-then-act. Splitting this into `get` // then `insert` let two tasks both miss, both build a lock and both // enter the row: the loser merged into the winner's lock, but the // winner had already registered its operation on a lock that was no // longer the map's, so it never waited for the loser. - let lock = self - .0 - .lock_manager - .get_or_insert_with(pk.clone(), #lock_ident::new); + // SAFETY: `self.0.lock_manager` is the table's own `Arc`, + // borrowed for this whole operation, so the map outlives the + // acquirer and the pending lock taken from it. See the "Borrowed + // guards" note on `LockMap`. + let lock = unsafe { + self.0 + .lock_manager + .get_or_insert_with(pk.clone(), #lock_ident::new) + }; let mut lock_guard = lock.write().await; - #[allow(clippy::mutable_key_type)] let (locks, op_lock) = lock_guard.#ident(lock_id); drop(lock_guard); // The registered lock must be cancellation-covered BEFORE the // predecessor wait below: a future dropped at that await (tokio // timeout, task abort) would otherwise leave the registered lock // held forever and hang every later operation on this key. - let pending_lock = PendingLock::new(op_lock, self.0.lock_manager.clone(), pk.clone()); - worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + // SAFETY: as above; the map outlives this pending lock and the + // `LockGuard` it is converted into, both of which are locals of + // this operation. + let pending_lock = unsafe { PendingLock::new(op_lock, &self.0.lock_manager, pk.clone()) }; + // No predecessor is the common case on disjoint keys: the row had no + // entry, so every column slot was empty. Registering a waker on each + // of nothing, boxing the joined slice and suspending the operation to + // poll it once is all overhead there. + if !locks.is_empty() { + worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + } pending_lock } } diff --git a/codegen/src/generators/persist/queries/update.rs b/codegen/src/generators/persist/queries/update.rs index ff379424..9a0a880d 100644 --- a/codegen/src/generators/persist/queries/update.rs +++ b/codegen/src/generators/persist/queries/update.rs @@ -1,5 +1,5 @@ use crate::common::model::{Index, Operation}; -use crate::common::name_generator::{WorktableNameGenerator, is_float}; +use crate::common::name_generator::{WorktableNameGenerator, archived_field_requires_rebuild, is_float}; use crate::generators::persist::PersistGenerator; use convert_case::{Case, Casing}; use indexmap::IndexMap; @@ -7,10 +7,22 @@ use proc_macro2::Literal; use proc_macro2::{Ident, Span, TokenStream}; use quote::quote; +struct UpdateStorage<'a> { + string_fields: Option>, + requires_rebuild: bool, +} + impl PersistGenerator { pub fn gen_query_update_impl(&mut self) -> syn::Result { - let custom_updates = if let Some(q) = &self.queries { + let (custom_updates, mutation_api) = if let Some(q) = &self.queries { let profile = q.update_runtime.clone(); + let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); + let mutation_api = crate::generators::mutation_builder::paged_mutation_api( + &self.name, + &name_generator.get_work_table_ident(), + &self.columns, + q, + )?; let custom_updates = self.gen_custom_updates(q.updates.clone()); let custom_updates = crate::generators::profile_dispatch::wrap( custom_updates, @@ -18,17 +30,16 @@ impl PersistGenerator { &WorktableNameGenerator::from_table_name(self.name.to_string()).get_row_type_ident(), )?; - quote! { - #custom_updates - } + (quote! { #custom_updates }, mutation_api) } else { - quote! {} + (quote! {}, quote! {}) }; let full_row_update = self.gen_full_row_update(); let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let table_ident = name_generator.get_work_table_ident(); Ok(quote! { + #mutation_api impl #table_ident { #full_row_update #custom_updates @@ -66,13 +77,15 @@ impl PersistGenerator { let persist_op = self.gen_persist_op(); let full_row_lock = self.gen_full_lock_for_update(); let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); + let requires_rebuild = self.columns.columns_map.values().any(archived_field_requires_rebuild); let const_name = name_generator.get_page_inner_size_const_ident(); let secondary_events_ident = name_generator.get_space_secondary_index_events_ident(); // A full-row update rewrites every column, hence every secondary // index; only a table with NO secondary indexes may take the // same-size in-place path (mirrors the in-memory generator). - let full_row_in_place_eligible = !self.columns.is_sized && self.columns.indexes.is_empty(); - let size_check = if self.columns.is_sized { + let full_row_in_place_eligible = + (!self.columns.is_sized || requires_rebuild) && self.columns.indexes.is_empty(); + let size_check = if self.columns.is_sized && !requires_rebuild { quote! {} } else { let in_place_attempt = if full_row_in_place_eligible { @@ -133,7 +146,7 @@ impl PersistGenerator { // from) every path, so the archived-swap tail is emitted only for // sized rows; emitting both would leave unreachable code behind the // diverging size_check block. - let update_body = if self.columns.is_sized { + let update_body = if self.columns.is_sized && !requires_rebuild { quote! { let mut bytes = worktable::prelude::rkyv::to_bytes::(&row).map_err(|_| WorkTableError::SerializeError)?; let mut archived_row = unsafe { worktable::prelude::rkyv::access_unchecked_mut::<<#row_ident as worktable::prelude::rkyv::Archive>::Archived>(&mut bytes[..]).unseal_unchecked() }; @@ -157,7 +170,7 @@ impl PersistGenerator { }; quote! { - pub async fn update(&self, row: #row_ident) -> core::result::Result<(), WorkTableError> { + pub async fn replace(&self, row: #row_ident) -> core::result::Result<(), WorkTableError> { let pk = row.get_primary_key(); let pending_lock = { #full_row_lock }; let guard = pending_lock.into_guard_with_mutation(); @@ -217,18 +230,19 @@ impl PersistGenerator { .collect::>(); if fields.is_empty() { None } else { Some(fields) } }; + let requires_rebuild = op + .columns + .iter() + .any(|column| archived_field_requires_rebuild(self.columns.columns_map.get(column).unwrap())); + let storage = UpdateStorage { + string_fields: unsized_columns, + requires_rebuild, + }; let idents = &op.columns; if let Some(index) = index { if index.is_unique { - self.gen_unique_update( - snake_case_name, - name, - index, - idents, - indexes_columns.as_ref(), - unsized_columns, - ) + self.gen_unique_update(snake_case_name, name, index, idents, indexes_columns.as_ref(), storage) } else { self.gen_non_unique_update( snake_case_name, @@ -236,12 +250,12 @@ impl PersistGenerator { index, idents, indexes_columns.as_ref(), - unsized_columns, + storage, ) } } else if self.columns.primary_keys.len() == 1 { if *self.columns.primary_keys.first().unwrap() == op.by { - self.gen_pk_update(snake_case_name, name, idents, indexes_columns.as_ref(), unsized_columns) + self.gen_pk_update(snake_case_name, name, idents, indexes_columns.as_ref(), storage) } else { todo!() } @@ -366,6 +380,7 @@ impl PersistGenerator { unsized_fields: Option>, idents: &[Ident], idx_idents: Option<&Vec>, + requires_rebuild: bool, ) -> TokenStream { // Port of the in-memory generator's gen_size_check: the in-place fast // path bypasses the index diff machinery, so it only applies when no @@ -375,7 +390,41 @@ impl PersistGenerator { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let secondary_events_ident = name_generator.get_space_secondary_index_events_ident(); let primary_key_ident = name_generator.get_primary_key_type_ident(); - if let (Some(f), false) = (unsized_fields, touches_index) { + if requires_rebuild { + let row_updates = idents + .iter() + .map(|i| quote! { row_new.#i = row.#i.clone(); }) + .collect::>(); + let full_row_lock = self.gen_full_lock_for_update(); + let const_name = name_generator.get_page_inner_size_const_ident(); + + // The one thing this generator adds over the in-memory twin: the + // same-slot write has to reach the CDC stream, as an event-less data + // operation. Everything around it is shared. See `opaque_rebuild`. + let publish_in_place = quote! { + let secondary_keys_events: #secondary_events_ident = core::default::Default::default(); + let op: Operation< + <<#primary_key_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #primary_key_ident, + #secondary_events_ident + > = Operation::Update(UpdateOperation { + retired_link: None, + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events, + bytes: self.0.data.select_raw(current_link)?, + link: current_link, + }); + self.1.apply_operation(op)?; + }; + crate::generators::opaque_rebuild::gen_rebuild_arm( + &row_updates, + &full_row_lock, + &const_name, + touches_index, + &publish_in_place, + ) + } else if let (Some(f), false) = (unsized_fields, touches_index) { let fields_check: Vec<_> = f .iter() .map(|f| { @@ -682,10 +731,14 @@ impl PersistGenerator { name: &Ident, idents: &[Ident], idx_idents: Option<&Vec>, - unsized_fields: Option>, + storage: UpdateStorage<'_>, ) -> TokenStream { + let UpdateStorage { + string_fields: unsized_fields, + requires_rebuild, + } = storage; let pk_ident = &self.pk.as_ref().unwrap().ident; - let method_ident = Ident::new(format!("update_{snake_case_name}").as_str(), Span::mixed_site()); + let method_ident = Ident::new(format!("__wt_update_{snake_case_name}").as_str(), Span::mixed_site()); let query_ident = Ident::new(format!("{name}Query").as_str(), Span::mixed_site()); let lock_ident = WorktableNameGenerator::get_update_query_lock_ident(&snake_case_name); @@ -701,8 +754,9 @@ impl PersistGenerator { // Same gate as the in-memory generator: when the size_check body // handles (and returns from) every path, emitting the archived-swap // tail too would leave unreachable code. - let archived_swap_is_safe = self.columns.is_sized || (unsized_fields.is_none() && idx_idents.is_none()); - let size_check = self.gen_size_check(unsized_fields, idents, idx_idents); + let archived_swap_is_safe = + !requires_rebuild && (self.columns.is_sized || (unsized_fields.is_none() && idx_idents.is_none())); + let size_check = self.gen_size_check(unsized_fields, idents, idx_idents, requires_rebuild); let diff_process_insert = self.gen_process_diffs_insert_on_index(idents, idx_idents); let diff_process_remove = self.gen_process_diffs_remove_on_index(idx_idents); let persist_call = self.gen_persist_call(); @@ -730,7 +784,7 @@ impl PersistGenerator { }; quote! { - pub async fn #method_ident(&self, row: #query_ident, pk: Pk) -> core::result::Result<(), WorkTableError> + async fn #method_ident(&self, row: #query_ident, pk: Pk) -> core::result::Result<(), WorkTableError> where #pk_ident: From { let pk: #pk_ident = pk.into(); @@ -760,11 +814,15 @@ impl PersistGenerator { index: &Index, idents: &[Ident], idx_idents: Option<&Vec>, - unsized_fields: Option>, + storage: UpdateStorage<'_>, ) -> TokenStream { + let UpdateStorage { + string_fields: unsized_fields, + requires_rebuild, + } = storage; let by_field = &index.field; let index = &index.name; - let method_ident = Ident::new(format!("update_{snake_case_name}").as_str(), Span::mixed_site()); + let method_ident = Ident::new(format!("__wt_update_{snake_case_name}").as_str(), Span::mixed_site()); let query_ident = Ident::new(format!("{name}Query").as_str(), Span::mixed_site()); let by_ident = Ident::new(format!("{name}By").as_str(), Span::mixed_site()); @@ -782,8 +840,61 @@ impl PersistGenerator { // every loop iteration itself (in-place or reinsert, then continue), // so the archived-swap tail is only emitted otherwise; emitting both // would leave unreachable code after the size_check block. - let has_unsized = unsized_fields.is_some(); - let size_check = if let Some(f) = unsized_fields { + let has_unsized = unsized_fields.is_some() || requires_rebuild; + let size_check = if requires_rebuild { + let row_updates = idents + .iter() + .map(|i| quote! { row_new.#i = row.#i.clone(); }) + .collect::>(); + let touches_index = idx_idents.map(|v| !v.is_empty()).unwrap_or(false); + let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); + let const_name = name_generator.get_page_inner_size_const_ident(); + let secondary_events_ident = name_generator.get_space_secondary_index_events_ident(); + let primary_key_ident = name_generator.get_primary_key_type_ident(); + if touches_index { + quote! { + { + let row_old = self.0.select(pk.clone()).ok_or(WorkTableError::NotFound)?; + let mut row_new = row_old.clone(); + #(#row_updates)* + self.reinsert(row_old, row_new).await?; + guards.remove(&pk); + continue; + } + } + } else { + quote! { + { + let row_old = self.0.select(pk.clone()).ok_or(WorkTableError::NotFound)?; + let mut row_new = row_old.clone(); + #(#row_updates)* + let in_place_ok = unsafe { + self.0.data.update_in_place::<{ #const_name }>(row_new.clone(), link).is_ok() + }; + if in_place_ok { + let secondary_keys_events: #secondary_events_ident = core::default::Default::default(); + let op: Operation< + <<#primary_key_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #primary_key_ident, + #secondary_events_ident + > = Operation::Update(UpdateOperation { + retired_link: None, + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events, + bytes: self.0.data.select_raw(link)?, + link, + }); + self.1.apply_operation(op)?; + } else { + self.reinsert(row_old, row_new).await?; + } + guards.remove(&pk); + continue; + } + } + } + } else if let Some(f) = unsized_fields { let fields_check: Vec<_> = f .iter() .map(|f| { @@ -911,7 +1022,7 @@ impl PersistGenerator { }; quote! { - pub async fn #method_ident(&self, row: #query_ident, by: #by_ident) -> core::result::Result<(), WorkTableError> { + async fn #method_ident(&self, row: #query_ident, by: #by_ident) -> core::result::Result<(), WorkTableError> { // This query may update many rows. Keep vacuum out across the // snapshot, lock acquisition, and per-row mutation gaps // without adding work to each row. @@ -969,7 +1080,9 @@ impl PersistGenerator { if self.0.data.select_non_ghosted(link)?.#by_field != by { continue; } - let _mutation_guard = self.0.lock_manager.mutation_guard(&pk); + // SAFETY: the table owns `lock_manager` and is borrowed + // for this whole operation, so the map outlives the guard. + let _mutation_guard = unsafe { self.0.lock_manager.mutation_guard(&pk) }; let mut bytes = worktable::prelude::rkyv::to_bytes::(&row) .map_err(|_| WorkTableError::SerializeError)?; @@ -994,8 +1107,12 @@ impl PersistGenerator { index: &Index, idents: &[Ident], idx_idents: Option<&Vec>, - unsized_fields: Option>, + storage: UpdateStorage<'_>, ) -> TokenStream { + let UpdateStorage { + string_fields: unsized_fields, + requires_rebuild, + } = storage; let by_field = &index.field; let by_is_float = is_float( self.columns @@ -1006,7 +1123,7 @@ impl PersistGenerator { .as_str(), ); let index = &index.name; - let method_ident = Ident::new(format!("update_{snake_case_name}").as_str(), Span::mixed_site()); + let method_ident = Ident::new(format!("__wt_update_{snake_case_name}").as_str(), Span::mixed_site()); let query_ident = Ident::new(format!("{name}Query").as_str(), Span::mixed_site()); let by_ident = Ident::new(format!("{name}By").as_str(), Span::mixed_site()); @@ -1023,8 +1140,9 @@ impl PersistGenerator { // Same gate as the in-memory generator: when the size_check body // handles (and returns from) every path, emitting the archived-swap // tail too would leave unreachable code. - let archived_swap_is_safe = self.columns.is_sized || (unsized_fields.is_none() && idx_idents.is_none()); - let size_check = self.gen_size_check(unsized_fields, idents, idx_idents); + let archived_swap_is_safe = + !requires_rebuild && (self.columns.is_sized || (unsized_fields.is_none() && idx_idents.is_none())); + let size_check = self.gen_size_check(unsized_fields, idents, idx_idents, requires_rebuild); let diff_process_insert = self.gen_process_diffs_insert_on_index(idents, idx_idents); let diff_process_remove = self.gen_process_diffs_remove_on_index(idx_idents); let persist_call = self.gen_persist_call(); @@ -1077,7 +1195,7 @@ impl PersistGenerator { }; quote! { - pub async fn #method_ident(&self, row: #query_ident, by: #by_ident) -> core::result::Result<(), WorkTableError> { + async fn #method_ident(&self, row: #query_ident, by: #by_ident) -> core::result::Result<(), WorkTableError> { let mut bytes = worktable::prelude::rkyv::to_bytes::(&row) .map_err(|_| WorkTableError::SerializeError)?; @@ -1203,7 +1321,7 @@ mod tests { generator.set_queries(Queries { updates, deletes: IndexMap::new(), - in_place: IndexMap::new(), + updates_in_place: IndexMap::new(), ..Default::default() }); generator.gen_primary_key_def().unwrap(); diff --git a/codegen/src/generators/persist/table/impls.rs b/codegen/src/generators/persist/table/impls.rs index 0e84ee3c..a0ce201a 100644 --- a/codegen/src/generators/persist/table/impls.rs +++ b/codegen/src/generators/persist/table/impls.rs @@ -3,7 +3,9 @@ use proc_macro2::{Ident, Literal, Span, TokenStream}; use quote::quote; use crate::common::model::GeneratorType; -use crate::common::name_generator::{WorktableNameGenerator, is_float, is_unsized_vec}; +use crate::common::name_generator::{ + WorktableNameGenerator, archived_field_is_inline_scalar, is_float, is_unsized_vec, +}; use crate::generators::persist::PersistGenerator; impl PersistGenerator { @@ -248,6 +250,8 @@ impl PersistGenerator { }) .collect::>(); let pk_types_unsized = is_unsized_vec(pk_types); + let pk_tokens = quote! { #pk_type }; + let unsized_node_capacity = name_generator.get_aligned_disk_page_capacity(&pk_tokens); let wti_map = if cfg!(feature = "logical-index-persistence") { quote! { PersistentWtiIndex } } else { @@ -262,7 +266,7 @@ impl PersistGenerator { } else if pk_types_unsized { quote! { inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( - #wti_map::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#node_capacity) + #wti_map::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#unsized_node_capacity) )); } } else { @@ -323,7 +327,7 @@ impl PersistGenerator { } async fn load(engine: E) -> worktable::prelude::eyre::Result { - Self::load_with(engine, LoadMode::Strict).await + worktable::prelude::Box::pin(Self::load_with(engine, LoadMode::Strict)).await } async fn load_with(mut engine: E, mode: LoadMode) -> worktable::prelude::eyre::Result { @@ -337,10 +341,10 @@ impl PersistGenerator { .await?; let table_path = engine.config().table_path().to_owned(); if !std::path::Path::new(&table_path).exists() { - return Self::new(engine).await; + return worktable::prelude::Box::pin(Self::new(engine)).await; }; let table = load_persisted_state(&table_path, async { - let space = #space_ident::parse_file(&table_path).await?; + let space = worktable::prelude::Box::pin(#space_ident::parse_file(&table_path)).await?; Ok::<_, worktable::prelude::eyre::Report>(space.into_worktable_with_mode(engine, &table_path, mode).await?) }).await?; Ok(table) @@ -379,11 +383,44 @@ impl PersistGenerator { let row_type = name_generator.get_row_type_ident(); let primary_key_type = name_generator.get_primary_key_type_ident(); + // `select_with` reads the cell in place with no copy, which is only + // sound when the archived row holds no relative pointers. Emit it only + // for those tables; one with a `String` column simply has no + // `select_with`, so a caller gets "no method named `select_with`" + // instead of a silent copy or a torn pointer. + let select_with_fn = if self + .columns + .columns_map + .values() + .all(|ty| archived_field_is_inline_scalar("e! { #ty })) + { + quote! { + /// Apply `f` to the archived inner row. No cell memcpy; `f` must copy out. + pub fn select_with(&self, pk: Pk, f: F) -> Option + where + #primary_key_type: From, + F: FnMut(&<#row_type as worktable::prelude::rkyv::Archive>::Archived) -> T, + { + self.0.select_with(pk.into(), f) + } + } + } else { + quote! {} + }; + quote! { pub fn select(&self, pk: Pk) -> Option<#row_type> where #primary_key_type: From { self.0.select(pk.into()) } + + /// Pin-guard plus archived inner row. Does not deserialize. + pub fn select_ref(&self, pk: Pk) -> Option> + where #primary_key_type: From { + self.0.select_ref(pk.into()) + } + + #select_with_fn } } @@ -577,7 +614,7 @@ impl PersistGenerator { let secondary_events_ident = name_generator.get_space_secondary_index_events_ident(); quote! { - pub async fn reinsert(&self, row_old: #row_type, row_new: #row_type) -> core::result::Result<#primary_key_type, WorkTableError> { + async fn reinsert(&self, row_old: #row_type, row_new: #row_type) -> core::result::Result<#primary_key_type, WorkTableError> { self.1.ensure_running()?; let (op, res) = self.0.reinsert_cdc::<#secondary_events_ident>(row_old, row_new); if let Some(op) = op { diff --git a/codegen/src/generators/persist/wrapper.rs b/codegen/src/generators/persist/wrapper.rs index a28bd36f..5c0bc922 100644 --- a/codegen/src/generators/persist/wrapper.rs +++ b/codegen/src/generators/persist/wrapper.rs @@ -1,4 +1,4 @@ -use crate::common::name_generator::WorktableNameGenerator; +use crate::common::name_generator::{WorktableNameGenerator, archived_field_is_inline_scalar}; use crate::generators::persist::PersistGenerator; use proc_macro2::TokenStream; use quote::quote; @@ -9,12 +9,38 @@ impl PersistGenerator { let impl_ = self.gen_wrapper_impl(); let storable_impl = self.get_wrapper_storable_impl(); let archived_wrapper_impl = self.get_archived_wrapper_impl(); + let inline_archived_impl = self.get_inline_archived_impl(); quote! { #type_ #impl_ #storable_impl #archived_wrapper_impl + #inline_archived_impl + } + } + + /// Emit `InlineArchived` only when every column is an inline scalar. + /// + /// Gates the zero-copy `select_with` path; see the in-memory generator's + /// copy of this for the reasoning. + fn get_inline_archived_impl(&self) -> TokenStream { + let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); + let wrapper_ident = name_generator.get_wrapper_type_ident(); + + let all_inline = self + .columns + .columns_map + .values() + .all(|ty| archived_field_is_inline_scalar("e! { #ty })); + if !all_inline { + return quote! {}; + } + + quote! { + // SAFETY: every column is a fixed-size scalar, so the archived + // wrapper holds no relative pointers. + unsafe impl worktable::prelude::InlineArchived for #wrapper_ident {} } } @@ -86,10 +112,21 @@ impl PersistGenerator { fn get_archived_wrapper_impl(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); - let row_ident = name_generator.get_archived_wrapper_type_ident(); + let archived_ident = name_generator.get_archived_wrapper_type_ident(); + let row_ident = name_generator.get_row_type_ident(); quote! { - impl ArchivedRowWrapper for #row_ident { + impl ArchivedRowWrapper for #archived_ident { + type Inner = <#row_ident as worktable::prelude::rkyv::Archive>::Archived; + + fn inner(&self) -> &Self::Inner { + &self.inner + } + + fn is_ghosted(&self) -> bool { + self.is_ghosted + } + fn unghost(&mut self) { self.is_ghosted = false; } diff --git a/codegen/src/generators/read_only/locks.rs b/codegen/src/generators/read_only/locks.rs index ffa50040..33d7a894 100644 --- a/codegen/src/generators/read_only/locks.rs +++ b/codegen/src/generators/read_only/locks.rs @@ -144,8 +144,10 @@ impl ReadOnlyGenerator { .map(|i| { let col = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); quote! { - if let Some(lock) = &self.#col { - set.insert(lock.clone()); + if let Some(existing_lock) = &self.#col { + if !set.iter().any(|entry| worktable::prelude::Arc::ptr_eq(entry, existing_lock)) { + set.push(existing_lock.clone()); + } } self.#col = Some(lock.clone()); } @@ -153,9 +155,8 @@ impl ReadOnlyGenerator { .collect(); quote! { - #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { - let mut set = worktable::prelude::HashSet::new(); + fn lock(&mut self, id: u16) -> (Vec>, worktable::prelude::Arc) { + let mut set: Vec> = Vec::new(); let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* @@ -175,8 +176,8 @@ impl ReadOnlyGenerator { if let Some(#col) = &other.#col { if self.#col.is_none() { self.#col = Some(#col.clone()); - } else { - set.insert(#col.clone()); + } else if !set.iter().any(|existing| worktable::prelude::Arc::ptr_eq(existing, #col)) { + set.push(#col.clone()); } } other.#col = self.#col.clone(); @@ -185,9 +186,8 @@ impl ReadOnlyGenerator { .collect(); quote! { - #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { - let mut set = worktable::prelude::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> Vec> { + let mut set: Vec> = Vec::new(); #(#rows)* set } diff --git a/codegen/src/generators/read_only/primary_key.rs b/codegen/src/generators/read_only/primary_key.rs index e7ea6bff..78e6b136 100644 --- a/codegen/src/generators/read_only/primary_key.rs +++ b/codegen/src/generators/read_only/primary_key.rs @@ -55,13 +55,7 @@ impl ReadOnlyGenerator { .expect("should exist as got from definition") }) .collect::>(); - let unsized_derive = if is_unsized_vec(&types.iter().map(|v| v.to_string()).collect::>()) { - quote! { - VariableSizeMeasure, - } - } else { - quote! {} - }; + let is_unsized_key = is_unsized_vec(&types.iter().map(|v| v.to_string()).collect::>()); let (backend_derive, backend_impl) = primary_key_backend_impl(self.columns.primary_index_backend, &ident, types)?; let borrowed_impl = gen_borrowed_primary_key_impl(&ident, types); @@ -117,6 +111,79 @@ impl ReadOnlyGenerator { } }; + // `data_bucket_derive::SizeMeasure` reports only 8-byte field + // alignment, so a generated `u128` newtype under-budgets fixed index + // pages. It also cannot derive `VariableSizeMeasure` for a composite + // `(String, u128)` key because the fixed member does not implement the + // variable-size trait. Model the archived wrapper itself: fixed keys + // are exactly its archived root, while String keys add only their + // out-of-line payload to that root. + let string_fields = types + .iter() + .enumerate() + .filter(|(_, ty)| ty.to_string() == "String") + .map(|(index, ty)| (syn::Index::from(index), *ty)) + .collect::>(); + let string_field_indexes = string_fields.iter().map(|(index, _)| index); + let string_field_types = string_fields.iter().map(|(_, ty)| ty); + let aligned_size_body = if is_unsized_key { + quote! { + let len = core::mem::size_of::< + ::Archived + >() #(+ worktable::prelude::SizeMeasurable::aligned_size( + &self.#string_field_indexes + ).saturating_sub(core::mem::size_of::< + <#string_field_types as worktable::prelude::rkyv::Archive>::Archived + >()))*; + let alignment = core::mem::align_of::< + ::Archived + >(); + let remainder = len % alignment; + if remainder == 0 { len } else { len + alignment - remainder } + } + } else { + quote! { + core::mem::size_of::< + ::Archived + >() + } + }; + let size_measure_impl = quote! { + impl worktable::prelude::SizeMeasurable for #ident { + fn aligned_size(&self) -> usize { + #aligned_size_body + } + + fn align() -> Option { + Some(core::mem::align_of::< + ::Archived + >()) + } + } + }; + + let variable_size_measure_impl = if is_unsized_key { + let string_field_types = string_fields.iter().map(|(_, ty)| ty); + quote! { + impl worktable::prelude::VariableSizeMeasurable for #ident { + fn aligned_size(length: usize) -> usize { + let len = core::mem::size_of::< + ::Archived + >() #(+ <#string_field_types as worktable::prelude::VariableSizeMeasurable> + ::aligned_size(length).saturating_sub(core::mem::size_of::< + <#string_field_types as worktable::prelude::rkyv::Archive>::Archived + >()))*; + let alignment = core::mem::align_of::< + ::Archived + >(); + let remainder = len % alignment; + if remainder == 0 { len } else { len + alignment - remainder } + } + } + } + } else { + quote! {} + }; Ok(quote! { #[derive( Clone, @@ -131,9 +198,7 @@ impl ReadOnlyGenerator { PartialEq, PartialOrd, Ord, - SizeMeasure, MemStat, - #unsized_derive )] #[rkyv(crate = worktable::prelude::rkyv)] #[rkyv(derive(PartialEq, Eq, PartialOrd, Ord, Debug))] @@ -141,6 +206,8 @@ impl ReadOnlyGenerator { #from_impl #into_impl + #size_measure_impl + #variable_size_measure_impl #borrowed_impl #backend_impl diff --git a/codegen/src/generators/read_only/table/impls.rs b/codegen/src/generators/read_only/table/impls.rs index 033d328e..396a3795 100644 --- a/codegen/src/generators/read_only/table/impls.rs +++ b/codegen/src/generators/read_only/table/impls.rs @@ -2,7 +2,9 @@ use convert_case::{Case, Casing}; use proc_macro2::{Ident, Literal, Span, TokenStream}; use quote::quote; -use crate::common::name_generator::{WorktableNameGenerator, is_float, is_unsized_vec}; +use crate::common::name_generator::{ + WorktableNameGenerator, archived_field_is_inline_scalar, is_float, is_unsized_vec, +}; use crate::generators::read_only::ReadOnlyGenerator; impl ReadOnlyGenerator { @@ -287,16 +289,16 @@ impl ReadOnlyGenerator { } async fn load(engine: E) -> worktable::prelude::eyre::Result { - Self::load_with(engine, LoadMode::Strict).await + worktable::prelude::Box::pin(Self::load_with(engine, LoadMode::Strict)).await } async fn load_with(engine: E, mode: LoadMode) -> worktable::prelude::eyre::Result { let table_path = engine.config().table_path().to_owned(); if !std::path::Path::new(&table_path).exists() { - return Self::new(engine).await; + return worktable::prelude::Box::pin(Self::new(engine)).await; }; let table = load_persisted_state(&table_path, async { - let space = #space_ident::parse_file(&table_path).await?; + let space = worktable::prelude::Box::pin(#space_ident::parse_file(&table_path)).await?; Ok::<_, worktable::prelude::eyre::Report>(space.into_worktable_with_mode(&table_path, mode)?) }).await?; Ok(table) @@ -335,11 +337,44 @@ impl ReadOnlyGenerator { let row_type = name_generator.get_row_type_ident(); let primary_key_type = name_generator.get_primary_key_type_ident(); + // `select_with` reads the cell in place with no copy, which is only + // sound when the archived row holds no relative pointers. Emit it only + // for those tables; one with a `String` column simply has no + // `select_with`, so a caller gets "no method named `select_with`" + // instead of a silent copy or a torn pointer. + let select_with_fn = if self + .columns + .columns_map + .values() + .all(|ty| archived_field_is_inline_scalar("e! { #ty })) + { + quote! { + /// Apply `f` to the archived inner row. No cell memcpy; `f` must copy out. + pub fn select_with(&self, pk: Pk, f: F) -> Option + where + #primary_key_type: From, + F: FnMut(&<#row_type as worktable::prelude::rkyv::Archive>::Archived) -> T, + { + self.0.select_with(pk.into(), f) + } + } + } else { + quote! {} + }; + quote! { pub fn select(&self, pk: Pk) -> Option<#row_type> where #primary_key_type: From { self.0.select(pk.into()) } + + /// Pin-guard plus archived inner row. Does not deserialize. + pub fn select_ref(&self, pk: Pk) -> Option> + where #primary_key_type: From { + self.0.select_ref(pk.into()) + } + + #select_with_fn } } diff --git a/codegen/src/generators/read_only/wrapper.rs b/codegen/src/generators/read_only/wrapper.rs index 3a8ead5f..955e3be8 100644 --- a/codegen/src/generators/read_only/wrapper.rs +++ b/codegen/src/generators/read_only/wrapper.rs @@ -1,4 +1,4 @@ -use crate::common::name_generator::WorktableNameGenerator; +use crate::common::name_generator::{WorktableNameGenerator, archived_field_is_inline_scalar}; use crate::generators::read_only::ReadOnlyGenerator; use proc_macro2::TokenStream; use quote::quote; @@ -9,12 +9,38 @@ impl ReadOnlyGenerator { let impl_ = self.gen_wrapper_impl(); let storable_impl = self.get_wrapper_storable_impl(); let archived_wrapper_impl = self.get_archived_wrapper_impl(); + let inline_archived_impl = self.get_inline_archived_impl(); quote! { #type_ #impl_ #storable_impl #archived_wrapper_impl + #inline_archived_impl + } + } + + /// Emit `InlineArchived` only when every column is an inline scalar. + /// + /// Gates the zero-copy `select_with` path; see the in-memory generator's + /// copy of this for the reasoning. + fn get_inline_archived_impl(&self) -> TokenStream { + let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); + let wrapper_ident = name_generator.get_wrapper_type_ident(); + + let all_inline = self + .columns + .columns_map + .values() + .all(|ty| archived_field_is_inline_scalar("e! { #ty })); + if !all_inline { + return quote! {}; + } + + quote! { + // SAFETY: every column is a fixed-size scalar, so the archived + // wrapper holds no relative pointers. + unsafe impl worktable::prelude::InlineArchived for #wrapper_ident {} } } @@ -86,10 +112,21 @@ impl ReadOnlyGenerator { fn get_archived_wrapper_impl(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); - let row_ident = name_generator.get_archived_wrapper_type_ident(); + let archived_ident = name_generator.get_archived_wrapper_type_ident(); + let row_ident = name_generator.get_row_type_ident(); quote! { - impl ArchivedRowWrapper for #row_ident { + impl ArchivedRowWrapper for #archived_ident { + type Inner = <#row_ident as worktable::prelude::rkyv::Archive>::Archived; + + fn inner(&self) -> &Self::Inner { + &self.inner + } + + fn is_ghosted(&self) -> bool { + self.is_ghosted + } + fn unghost(&mut self) { self.is_ghosted = false; } diff --git a/codegen/src/generators/vec_table/mod.rs b/codegen/src/generators/vec_table/mod.rs index 29edd331..86db3358 100644 --- a/codegen/src/generators/vec_table/mod.rs +++ b/codegen/src/generators/vec_table/mod.rs @@ -980,6 +980,10 @@ pub fn expand( } }; + let mutation_api = match queries { + Some(queries) => crate::generators::mutation_builder::vec_mutation_api(&name, &table_ident, &columns, queries)?, + None => quote! {}, + }; let (query_structs, query_methods) = gen_queries(queries, &pk, &pk_type, &columns, &index_columns, &index_unique)?; Ok(quote! { @@ -987,6 +991,8 @@ pub fn expand( #(#query_structs)* + #mutation_api + #row_derives pub struct #row_ident { #(pub #field_names: #field_types,)* @@ -1345,8 +1351,8 @@ pub fn expand( /// Declared `queries:` against a `vec: true` table. /// -/// These are named wrappers, not a new execution path. A declared update is -/// `update(&pk, |row| ..)` with the columns filled in from a generated struct, +/// These are typed-selector wrappers, not a new execution path. A declared update is +/// the existing row edit with the columns filled in from a generated struct, /// and `update` already repairs every index the edit moved a row under — so /// delegating to it is both the shortest implementation and the only one that /// cannot get index repair wrong in a second place. @@ -1442,7 +1448,7 @@ fn gen_queries( // The declared name already carries the key — `AmountById` becomes // `update_amount_by_id` — which is the paged table's convention and the // whole point of generating these. - let method = format_ident!("update_{}", snake_of(name)); + let method = format_ident!("__wt_update_{}", snake_of(name)); let pick = selected(&op.by, unique); let doc = format!( "`update {name}` keyed by `{}`.\n\n\ @@ -1455,7 +1461,7 @@ fn gen_queries( ); methods.push(quote! { #[doc = #doc] - pub fn #method(&mut self, query: #query_ty, key: &#by_type) -> usize { + fn #method(&mut self, query: #query_ty, key: &#by_type) -> usize { #pick let mut touched = 0usize; for found in keys { @@ -1496,40 +1502,50 @@ fn gen_queries( }); } - for (name, op) in &queries.in_place { + for (name, op) in &queries.updates_in_place { let (by_type, unique) = resolve_by(&op.by)?; - if op.columns.len() != 1 { - return Err(syn::Error::new( - name.span(), - "an `in_place` query edits exactly one column through a closure. \ - For several columns at once use an `update` query, which takes a \ - struct of them.", - )); - } - let column = &op.columns[0]; - let column_type = columns - .columns_map - .get(column) - .ok_or_else(|| syn::Error::new(column.span(), format!("no column `{column}`")))?; - let method = format_ident!("update_{}_in_place", snake_of(name)); + let fields = &op.columns; + let field_types = fields + .iter() + .map(|field| { + columns + .columns_map + .get(field) + .ok_or_else(|| syn::Error::new(field.span(), format!("no column `{field}`"))) + }) + .collect::>>()?; + let closure_arg = if field_types.len() == 1 { + let ty = field_types[0]; + quote! { &mut #ty } + } else { + quote! { ( #(&mut #field_types),* ) } + }; + let closure_fields = if fields.len() == 1 { + let field = &fields[0]; + quote! { &mut row.#field } + } else { + quote! { ( #(&mut row.#fields),* ) } + }; + let field_set = fields.iter().map(ToString::to_string).collect::>().join(", "); + let method = format_ident!("__wt_update_in_place_{}", snake_of(name)); let pick = selected(&op.by, unique); let doc = format!( - "`in_place {name}` keyed by `{}`.\n\n\ - Hands a cloned candidate's `{column}` to the closure, then validates \ + "`update_in_place {name}` keyed by `{}`.\n\n\ + Hands a cloned candidate's declared field set ({field_set}) to the closure, then validates \ unique keys before replacing the row. Returns how many rows it reached.", op.by ); methods.push(quote! { #[doc = #doc] - pub fn #method( + fn #method( &mut self, - mut edit: impl FnMut(&mut #column_type), + mut edit: impl FnMut(#closure_arg), key: &#by_type, ) -> usize { #pick let mut touched = 0usize; for found in keys { - if self.update(&found, |row| edit(&mut row.#column)) { + if self.update(&found, |row| edit(#closure_fields)) { touched += 1; } } diff --git a/codegen/src/persist_table/generator/space.rs b/codegen/src/persist_table/generator/space.rs index fb9ba1df..5cb081c4 100644 --- a/codegen/src/persist_table/generator/space.rs +++ b/codegen/src/persist_table/generator/space.rs @@ -32,7 +32,12 @@ impl Generator { let ident = name_generator.get_persistence_engine_ident(); let primary_key_type = name_generator.get_primary_key_type_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); - let disk_capacity = name_generator.get_disk_page_capacity(); + let key_type = quote! { #primary_key_type }; + let disk_capacity = if self.attributes.pk_unsized { + name_generator.get_aligned_disk_page_capacity(&key_type) + } else { + name_generator.get_disk_page_capacity() + }; let page_const_name = name_generator.get_page_size_const_ident(); let const_name = name_generator.get_page_size_const_ident(); let space_primary_index = name_generator.get_space_primary_index_ident(); diff --git a/codegen/src/persist_table/generator/space_file/mod.rs b/codegen/src/persist_table/generator/space_file/mod.rs index cec3a1fa..b832d99b 100644 --- a/codegen/src/persist_table/generator/space_file/mod.rs +++ b/codegen/src/persist_table/generator/space_file/mod.rs @@ -26,8 +26,13 @@ impl Generator { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let index_persisted_ident = name_generator.get_persisted_index_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); - let disk_capacity = name_generator.get_disk_page_capacity(); let pk_type = name_generator.get_primary_key_type_ident(); + let pk_tokens = quote! { #pk_type }; + let disk_capacity = if self.attributes.pk_unsized { + name_generator.get_aligned_disk_page_capacity(&pk_tokens) + } else { + name_generator.get_disk_page_capacity() + }; let space_file_ident = name_generator.get_space_file_ident(); let primary_index = if self.attributes.pk_unsized { quote! { @@ -48,7 +53,7 @@ impl Generator { pub struct #space_file_ident { #primary_index pub indexes: #index_persisted_ident, - pub data: Vec>>, + pub data: Vec>>>, pub data_info: GeneralPage::Generator as PrimaryKeyGeneratorState>::State>>, } } @@ -132,8 +137,13 @@ impl Generator { let index_ident = name_generator.get_index_type_ident(); let task_ident = name_generator.get_persistence_task_ident(); let const_name = name_generator.get_page_inner_size_const_ident(); - let node_capacity = name_generator.get_disk_page_capacity(); let pk_type = name_generator.get_primary_key_type_ident(); + let pk_tokens = quote! { #pk_type }; + let node_capacity = if self.attributes.pk_unsized { + name_generator.get_aligned_disk_page_capacity(&pk_tokens) + } else { + name_generator.get_disk_page_capacity() + }; let lock_type = name_generator.get_lock_type_ident(); let table_name = name_generator.get_work_table_literal_name(); let secondary_index_events = name_generator.get_space_secondary_index_events_ident(); @@ -239,22 +249,24 @@ impl Generator { ) -> Result<#wt_ident, PersistenceLoadError> { let mut page_id = 1; let data = self.data.into_iter().map(|p| { - let mut data = Data::from_data_page(p); - data.set_page_id(page_id.into()); + let mut data = Data::from_data_page_ref_arc(&p); + worktable::prelude::Arc::get_mut(&mut data) + .expect("a newly restored page is uniquely owned") + .set_page_id(page_id.into()); page_id += 1; - worktable::prelude::Arc::new(data) + data }) .collect(); - let data = DataPages::from_data(data) - .with_empty_links(self.data_info.inner.empty_links_list) + let data = DataPages::from_data_arc(data) + .with_empty_links_arc(self.data_info.inner.empty_links_list) .map_err(|error| PersistenceLoadError::corrupt(path, error))?; let indexes = #index_ident::from_persisted(self.indexes); #primary_index_init let table = WorkTable { - data: worktable::prelude::Arc::new(data), + data, primary_index: worktable::prelude::Arc::new(primary_index), indexes: worktable::prelude::Arc::new(indexes), pk_gen: PrimaryKeyGeneratorState::from_state(self.data_info.inner.pk_gen_state), @@ -309,22 +321,24 @@ impl Generator { { let mut page_id = 1; let data = self.data.into_iter().map(|p| { - let mut data = Data::from_data_page(p); - data.set_page_id(page_id.into()); + let mut data = Data::from_data_page_ref_arc(&p); + worktable::prelude::Arc::get_mut(&mut data) + .expect("a newly restored page is uniquely owned") + .set_page_id(page_id.into()); page_id += 1; - worktable::prelude::Arc::new(data) + data }) .collect(); - let data = DataPages::from_data(data) - .with_empty_links(self.data_info.inner.empty_links_list) + let data = DataPages::from_data_arc(data) + .with_empty_links_arc(self.data_info.inner.empty_links_list) .map_err(|error| PersistenceLoadError::corrupt(path, error))?; let indexes = #index_ident::from_persisted(self.indexes); #primary_index_init let table = WorkTable { - data: worktable::prelude::Arc::new(data), + data, primary_index: worktable::prelude::Arc::new(primary_index), indexes: worktable::prelude::Arc::new(indexes), pk_gen: PrimaryKeyGeneratorState::from_state(self.data_info.inner.pk_gen_state), @@ -350,7 +364,12 @@ impl Generator { let pk_type = name_generator.get_primary_key_type_ident(); let page_const_name = name_generator.get_page_size_const_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); - let disk_capacity = name_generator.get_disk_page_capacity(); + let pk_tokens = quote! { #pk_type }; + let disk_capacity = if self.attributes.pk_unsized { + name_generator.get_aligned_disk_page_capacity(&pk_tokens) + } else { + name_generator.get_disk_page_capacity() + }; let persisted_index_name = name_generator.get_persisted_index_ident(); let version_const_name = name_generator.get_version_const_ident(); let index_extension = Literal::string(WT_INDEX_EXTENSION); @@ -416,7 +435,7 @@ impl Generator { let count = file_length.div_ceil(#page_const_name as u64); for page_id in 1..count { let index = parse_data_page::<{ #page_const_name as u32}, { #inner_const_name as usize }, { #page_const_name as u32 }>(&mut data_file, page_id as u32).await?; - data.push(index); + data.push(Box::new(index)); } (data, info) }; diff --git a/codegen/src/persist_table/generator/space_file/worktable_impls.rs b/codegen/src/persist_table/generator/space_file/worktable_impls.rs index 12c92ccb..9822fa29 100644 --- a/codegen/src/persist_table/generator/space_file/worktable_impls.rs +++ b/codegen/src/persist_table/generator/space_file/worktable_impls.rs @@ -184,8 +184,13 @@ impl Generator { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let pk_type = name_generator.get_primary_key_type_ident(); let const_name = name_generator.get_page_inner_size_const_ident(); - let node_capacity = name_generator.get_disk_page_capacity(); - let disk_capacity = name_generator.get_disk_page_capacity(); + let pk_tokens = quote! { #pk_type }; + let node_capacity = if self.attributes.pk_unsized { + name_generator.get_aligned_disk_page_capacity(&pk_tokens) + } else { + name_generator.get_disk_page_capacity() + }; + let disk_capacity = node_capacity.clone(); let page_const_name = name_generator.get_page_size_const_ident(); if self.attributes.pk_congee { // Congee durability is maintained by its native checkpoint/WAL. diff --git a/codegen/src/worktable/mod.rs b/codegen/src/worktable/mod.rs index 06472e03..f7a0e978 100644 --- a/codegen/src/worktable/mod.rs +++ b/codegen/src/worktable/mod.rs @@ -662,7 +662,7 @@ mod tests { .to_string(); let update = output - .split("pub async fn update_balance") + .split("async fn __wt_update_balance") .nth(1) .expect("generated balance update"); assert!( @@ -679,6 +679,113 @@ mod tests { ); } + #[test] + fn opaque_archived_update_rebuilds_for_memory_and_persistence() { + for persist in [false, true] { + let output = expand(quote! { + name: OpaqueArchivedUpdate, + persist: #persist, + columns: { + id: u64 primary_key, + secret: EncryptedSecret, + }, + queries: { + update: { + Secret(secret) by id, + } + } + }) + .unwrap() + .to_string(); + + let update = output + .split("async fn __wt_update_secret") + .nth(1) + .expect("generated opaque-field update"); + assert!( + update.contains("data . update_in_place"), + "opaque fields must be serialized as part of the complete row" + ); + assert!( + update.contains("self . reinsert"), + "a changed serialized length must fall back to reinsert" + ); + assert!( + !update.contains("swap (& mut archived . inner . secret"), + "moving an opaque archived field can retain pointers into the temporary query buffer" + ); + + let full_update = output + .split("async fn update_with_guard") + .nth(1) + .expect("generated full-row update"); + assert!(full_update.contains("data . update_in_place")); + assert!(full_update.contains("self . reinsert")); + assert!(!full_update.contains("swap (& mut archived . inner . secret")); + } + } + + #[test] + fn optional_string_update_rebuilds_for_memory_and_persistence() { + for persist in [false, true] { + let output = expand(quote! { + name: OptionalStringUpdate, + persist: #persist, + columns: { + id: u64 primary_key, + display_name: String optional, + }, + queries: { + update: { + DisplayName(display_name) by id, + } + } + }) + .unwrap() + .to_string(); + + let update = output + .split("async fn __wt_update_display_name") + .nth(1) + .expect("generated optional-string update"); + assert!(update.contains("data . update_in_place")); + assert!(update.contains("self . reinsert")); + assert!(!update.contains("swap (& mut archived . inner . display_name")); + } + } + + #[test] + fn indexed_opaque_update_uses_index_maintaining_reinsert() { + for persist in [false, true] { + let output = expand(quote! { + name: IndexedOpaqueUpdate, + persist: #persist, + columns: { + id: u64 primary_key, + secret: EncryptedSecret, + }, + indexes: { + secret_idx: secret unique using worktables_index, + }, + queries: { + update: { + Secret(secret) by id, + } + } + }) + .unwrap() + .to_string(); + + let update = output + .split("async fn __wt_update_secret") + .nth(1) + .expect("generated indexed opaque-field update"); + assert!(update.contains("self . reinsert")); + assert!(!update.contains("data . update_in_place")); + assert!(!update.contains("swap (& mut archived . inner . secret")); + } + } + #[cfg(feature = "logical-index-persistence")] #[test] fn logical_persistence_wraps_explicit_wti_backends() { @@ -1246,7 +1353,10 @@ mod position_tests { expanded.contains("impl PriceDenseTable"), "the dense payload must be emitted: {expanded}" ); - assert!(expanded.contains("fn update_top_price"), "missing the update query"); + assert!( + expanded.contains("fn __wt_update_top_price"), + "missing the update query" + ); assert!(expanded.contains("fn delete_stale"), "missing the delete query"); } @@ -1297,20 +1407,20 @@ mod position_tests { ); } - /// `in_place` is a synonym here, so it says so rather than generating a + /// `update_in_place` is redundant here, so it says so rather than generating a /// second name for one method. #[test] - fn in_place_on_a_dense_partition_is_refused_as_a_synonym() { + fn update_in_place_on_a_dense_partition_is_refused() { let error = expand(quote! { name: Price, partition_by: symbol_id: u16, partition_max_size: u8, columns: { exchange_id: u8 primary_key, bid: f64 }, queries: { - in_place: { Bump(bid) by exchange_id, } + update_in_place: { Bump(bid) by exchange_id, } } }) - .expect_err("in_place has no meaning on a dense partition") + .expect_err("update_in_place has no meaning on a dense partition") .to_string(); assert!(error.contains("already in place"), "must say why: {error}"); assert!(error.contains("update Bump"), "must name the replacement: {error}"); @@ -1452,7 +1562,7 @@ mod emitted_declarations { delete: { ById() by id, } - in_place: { + update_in_place: { Balance(balance) by id, } } diff --git a/docs/TODO.md b/docs/TODO.md index 1ae18543..309b39bc 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -66,7 +66,7 @@ results were rerun across all three local trees before drawing conclusions. ### WT DSL expansion and trailing commas are deterministic The trailing-comma parser fix was already present on the beta.17 branch and is -covered for `config`, `delete`, `in_place`, block order, and the no-comma form. +covered for `config`, `delete`, `update_in_place`, block order, and the no-comma form. Expansion is now deterministic too. `columns_map`, query maps, and generated unique-type sets preserve declaration order with `IndexMap`/`IndexSet`. The diff --git a/docs/beta17-validation.md b/docs/beta17-validation.md index a3f6d15f..981570ac 100644 --- a/docs/beta17-validation.md +++ b/docs/beta17-validation.md @@ -45,7 +45,7 @@ checked as follows. ## WT DSL findings and fixes The trailing-comma fix already on the branch passes five focused cases: -`config`, `delete`, `in_place`, reordered blocks, and the existing no-comma +`config`, `delete`, `update_in_place`, reordered blocks, and the existing no-comma form. Validation found a separate real beta.17 defect: macro expansion could vary diff --git a/docs/cell-lock-registry.md b/docs/cell-lock-registry.md index d5e4e762..e1ce7575 100644 --- a/docs/cell-lock-registry.md +++ b/docs/cell-lock-registry.md @@ -80,3 +80,41 @@ The corrected stripe implementation must pass the companion performance suite's same-source lock comparison before release. The report records both executable hashes and the exact WorkTable revision so results cannot be mixed with an older scheduler or dependency graph. + +## Point reads without a reader CAS + +A point read does not take the stripe's reader count. It snapshots the stripe, +copies or reads the archived cell, and then checks the stripe again; if a writer +arrived in between, it retries. Incrementing the reader count instead made every +concurrent reader write to the same atomic, and shared `select` scaled 2.4x +across eight workers while the same read on a private table scaled 7.4x. Stripe +states are padded to a cache line each for the same reason: sixteen of them +previously shared one line, so unrelated rows bounced it between cores. + +The snapshot is a monotonic counter, not the stripe's reader/writer word. A +writer releases by storing zero, which is exactly what an idle stripe reads, so +a write that began and finished inside one reader's window would otherwise be +invisible and the reader would keep bytes copied from the middle of it. The +counter advances once per completed write, before the writer bit clears. + +Two read shapes follow from that check. + +`select` and `select_ref` copy the cell into private memory and validate the +copy, so they work for any row. `select_with` instead runs the caller's closure +directly on the page, with no copy, and validates afterwards. That is faster, +about 1.8x at one worker, and it means the closure can observe a value a writer +is currently changing. + +For a fixed-size scalar that is recoverable: the closure reads a torn number and +the retry throws away whatever it computed. For a relative pointer, which is +what an archived `String` or `Vec` field is, it is not: the closure would +dereference the torn pointer before the check runs. So `select_with` is +generated only for tables whose columns are all inline scalars, gated on the +`InlineArchived` marker the macro emits for exactly those. A table with a +`String` column has no `select_with` at all, and a call to it is a compile +error naming the missing method rather than a silent copy or a torn read. Those +tables use the owned `select`. + +The macro refuses opaque user types here, because it cannot inspect a user +type's `Archive::Archived` layout. That costs such a table the zero-copy path; +it never grants one unsoundly. diff --git a/docs/crate.md b/docs/crate.md index 39fd4312..781e52b0 100644 --- a/docs/crate.md +++ b/docs/crate.md @@ -10,6 +10,14 @@ Since 1.9 it also builds without `std`. A consumer with `select_all`; persistence, vacuum and the disk index are the parts that need an operating system, and they are gated out. +For build-once lookup data, [`LinearTable`](crate::LinearTable) keeps duplicate +keys in insertion order and exposes its rows as one contiguous slice. This is +the integrated replacement for `worktable-vec::LinearTable`: callers can sort +while building, freeze behind a shared reference, and use their own +allocation-free binary search. Its optional snapshot operation uses the same +self-contained Vec page codec as generated `vec: true` tables and remains +available in `no_std + alloc` builds. + Three things a declaration can now choose that it could not before: - **The index backend**, with `using`. The default is `arctic`, which takes diff --git a/docs/magic.md b/docs/magic.md index c00ab8b5..92acfc0a 100644 --- a/docs/magic.md +++ b/docs/magic.md @@ -119,7 +119,7 @@ A **fixed, ordered prefix**, then a free-order section list. | 5, required with 4 | `partition_max_size:` | rows per partition, as an index width | | any order | `columns:` | the row and its primary key | | any order | `indexes:` | secondary indexes | -| any order | `queries:` | generated `update` / `delete` / `in_place` | +| any order | `queries:` | generated `update` / `delete` / `update_in_place` | | any order | `config:` | `page_size`, `row_derives` | The prefix is genuinely ordered: `parse_name` reads the first token and errors if @@ -202,7 +202,7 @@ independent of the backend and combines with it. ## Queries -Three kinds. CamelCase in the declaration, snake_case in the generated method. +Three kinds. The declaration names each allowed lookup and field set. ```rust queries: { @@ -212,23 +212,29 @@ queries: { delete: { ByName() by name, }, - in_place: { + update_in_place: { SomeValueById(some_value) by id, } } ``` -**`update`** generates `update_amount_by_id(AmountByIdQuery { amount }, id)`. The -query struct is the name plus `Query`. +**`update`** enables +`update_by_id(id, OrdersColumns::AMOUNT, amount)`. The zero-sized selector is +table-scoped and fixes both the allowed field set and value type. A multi-field +declaration uses one `Columns::FIELD_AND_FIELD` selector and its generated +`Query` value so the field set stays atomic. **`delete`** generates `delete_by_name(name)`. Empty parentheses because a delete names no columns. -**`in_place`** generates `update_some_value_by_id_in_place(id, |value| ...)`, which +**`update_in_place`** enables +`update_in_place_by_id(id, OrdersColumns::SOME_VALUE, |value| ...)`, which mutates without selecting first. Its locking is internal, so it is safe from several threads without the caller holding anything — which is also why it is a -*different concurrency point* from `update`. **Only `by {pk_field}` is -supported.** +*different concurrency point* from `update`. A multi-column declaration uses +one field-set selector such as `OrdersColumns::STATUS_AND_LAST_USED_AT` and a +tuple closure `|(status, last_used_at)| ...`; the declared set changes under one +row lock. **Only `by {pk_field}` is supported.** ## Selects, which are not declared @@ -366,7 +372,7 @@ concurrency points and they want opposite things: | query fan-out | does not exist yet | independent chunks | 1.99x-2.84x on orderbook upsert/delete | The `queries:` sections already group by concurrency point: everything in -`update:` goes through the same lock path, and `in_place:` is a different path by +`update:` goes through the same lock path, and `update_in_place:` is a different path by design. So the annotation belongs on the section. ## Three positions, one keyword @@ -407,7 +413,7 @@ worktable!( Fill(qty) by id, Cancel(qty) by symbol, }, - in_place runtime fast_local: { + update_in_place runtime fast_local: { Bump(qty) by id, }, delete runtime wide: { @@ -524,7 +530,7 @@ runtime does this query use" and it is visible at the place you are reading. **The cost of that choice**, recorded so it is not a surprise: adding a section annotation becomes a breaking change for callers already using `.runtime()`. The window is narrow today — `.runtime()` exists only on select builders, and selects -are not declared in `queries:`, so `update` / `delete` / `in_place` annotations +are not declared in `queries:`, so `update` / `delete` / `update_in_place` annotations can never collide with it. It only bites if a `select` section annotation is added later. diff --git a/docs/paper-2-plan.md b/docs/paper-2-plan.md index 89f97794..36f97ca1 100644 --- a/docs/paper-2-plan.md +++ b/docs/paper-2-plan.md @@ -104,7 +104,7 @@ SIGMOD R4 (2026-10-17). DaMoN 2027 CFP not posted. ## Update 2026-09-11: release evidence and paper scope -The release checkout is WorkTable 1.9.0-alpha1. Its runtime and row-lock +The release checkout is WorkTable 1.9.0-beta1. Its runtime and row-lock implementation differs from the earlier paper: inspect `src/lock/map.rs`, `src/runtime/` and the selected dependency graph at the pinned evaluation commit before describing them. diff --git a/docs/partition-by-one-pager.md b/docs/partition-by-one-pager.md index bbf73d25..ea5f6667 100644 --- a/docs/partition-by-one-pager.md +++ b/docs/partition-by-one-pager.md @@ -72,7 +72,7 @@ let books = OrderBookPartitions::new(); // hot path: array index, no hashing, no allocation let book = books.partition(symbol_id).ok_or(...)?; -book.update_top_price(feed.into(), exchange_id).await?; +book.update_by_exchange_id(exchange_id, OrderBookColumns::BEST_BID_PRICE_AND_BEST_BID_SIZE_AND_BEST_ASK_PRICE_AND_BEST_ASK_SIZE, feed.into()).await?; // across partitions: for sweeps and maintenance, not the hot path let stale = books.select_all().filter(|r| r.ts < cutoff).execute()?; diff --git a/docs/partitioned-tables-worked-example.md b/docs/partitioned-tables-worked-example.md index d8c2879f..aab87adc 100644 --- a/docs/partitioned-tables-worked-example.md +++ b/docs/partitioned-tables-worked-example.md @@ -225,11 +225,11 @@ concept. See section 7. ```rust // today: hashes the symbol string on every tick let book = manager.get_order_book(&feed.symbol).ok_or_else(|| eyre!("unknown symbol"))?; -book.table.update_top_price(feed.into(), row_id).await?; +book.table.update_by_exchange_id(row_id, OrderBookColumns::BEST_BID_PRICE_AND_BEST_BID_SIZE_AND_BEST_ASK_PRICE_AND_BEST_ASK_SIZE, feed.into()).await?; // with partition_by: array index let book = books.partition(feed.symbol_id).ok_or_else(|| eyre!("unknown symbol"))?; -book.update_top_price(feed.into(), row_id).await?; +book.update_by_exchange_id(row_id, OrderBookColumns::BEST_BID_PRICE_AND_BEST_BID_SIZE_AND_BEST_ASK_PRICE_AND_BEST_ASK_SIZE, feed.into()).await?; ``` One indirection fewer, because there is no wrapper struct holding the key diff --git a/docs/pr46-review-findings.md b/docs/pr46-review-findings.md index cae1954b..3f62f79e 100644 --- a/docs/pr46-review-findings.md +++ b/docs/pr46-review-findings.md @@ -7,7 +7,7 @@ Work lands on branch `fix/pr46-review-findings`. ## F1 — Mutation gate held across `.await` (was rated P1) — NOT REPRODUCED `LockMap::mutation_guard` (`src/lock/map.rs`) is a blocking spin/yield ticket -lock. The generated `update`/`in_place`/`delete` paths keep the `MutationGuard` +lock. The generated `update`/`update_in_place`/`delete` paths keep the `MutationGuard` inside `LockGuard` and hold it across `.await` (`update_with_guard(...).await`, `reinsert(...).await`). The concern: two keys colliding on the same 1-of-64 stripe, guard-holder parked at its await while the other task spins. diff --git a/docs/queries.md b/docs/queries.md index b43a2097..0d050dd1 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -1,6 +1,6 @@ # Queries -WorkTable support query definition feature. Users can add custom `update`, `delete` and `update_in_place` queries. +WorkTable supports declared `update`, `delete`, and `update_in_place` queries. ```rust worktable!( @@ -26,7 +26,7 @@ worktable!( delete: { ByName() by name, }, - in_place: { + update_in_place: { SomeValueById(some_value) by id, } } @@ -35,30 +35,49 @@ worktable!( ### `update` queries -`TODO` +An `update` changes only the declared field set. The lookup column is part of +the method name and the generated, table-scoped selector is an explicit +argument: + +```rust +table + .update_by_id(pk, SomethingColumns::AMOUNT, 250) + .await?; +``` + +For one field, the last argument is that field's Rust type. A declaration over +several fields exposes one selector such as `SomethingColumns::NAME_AND_AMOUNT` +and takes the generated query struct, preserving the declaration's atomic field +set. Selector dispatch is sealed, statically typed, and allocation-free. ### `update_in_place` queries -`update_in_place` queries are special update queries that allow you to update field's value +`update_in_place` queries allow you to update a declared field set without need to select it before query. It is useful for counters, as example, because with internal mutation queries locking logic user's don't need to add explicit locks over `WorkTable` object. So you can safely use `update_in_place` queries in multiple threads simultaneously. !!! For now only `by {pk_field}` queries are supported !!! -To declare `update_in_place` query you need to add `in_place` section to `queries`. Query definition is -same to `update`: `{YourQueryNameCamelCase}({fields_you_want_to_update}) by {by_field_name}`. -For example, in declaration above `update_in_place` is declared like this: +To declare an `update_in_place` query, add an `update_in_place` section to `queries`. Its definition is +the same shape as `update`: `{YourQueryNameCamelCase}({fields_you_want_to_update}) by {by_field_name}`. +For example: ``` -in_place: { +update_in_place: { SomeValueById(some_value) by id, + AmountAndSomeValueById(amount, some_value) by id, } ``` -It will generate `update_some_value_by_id_in_place` method for `WorkTable` object (name generation logic is same -as for other queries). It will have two arguments: your `by` field value and closure, where you can use mutable -field value itself. +It enables `update_in_place_by_id` for the generated +`SomethingColumns::SOME_VALUE` selector. The method takes the lookup value, the +selector, and a closure over the mutable archived field value. +For a multi-column declaration the selector preserves the exact atomic field +set and the closure receives a tuple, for example +`update_in_place_by_id(id, SomethingColumns::AMOUNT_AND_SOME_VALUE, +|(amount, some_value)| ...)`. Both fields are edited under one row lock and +persisted as one mutation. ```rust #[tokio::main] @@ -75,7 +94,7 @@ async fn main() -> eyre::Result<()> { let pk = table.insert(row)?; // This will lead to `some_value` field update by adding 100 to it value. table - .update_some_value_by_id_in_place(|some_value| *some_value += 100, pk.0) + .update_in_place_by_id(pk.0, SomethingColumns::SOME_VALUE, |some_value| *some_value += 100) .await?; let row = table.select(pk)?; assert_eq!(row.some_value, 100); @@ -84,8 +103,8 @@ async fn main() -> eyre::Result<()> { } ``` -You can find tests that covers `update_in_place` queries [here](../tests/worktable/in_place.rs). +You can find tests that cover `update_in_place` queries [here](../tests/worktable/in_place.rs). ### `delete` queries -`TODO` \ No newline at end of file +`TODO` diff --git a/docs/why-worktables.typ b/docs/why-worktables.typ index dc9ae9bf..26ec1572 100644 --- a/docs/why-worktables.typ +++ b/docs/why-worktables.typ @@ -92,7 +92,7 @@ as well as memory; memory reclamation does not imply file truncation. Persistence is opt-in for the paged shape, with local disk and an S3-backed tier. Its completion boundaries are explicit. A successful mutation is accepted and queued; orderly `close().await` drains and joins the engine. -The current alpha does not promise transaction journaling or fsync durability. +The current beta does not promise transaction journaling or fsync durability. That makes it a fit for application-owned working state whose recovery contract is designed deliberately. @@ -105,7 +105,7 @@ leaving the consequential choices inspectable. Start with the #link("wt-user-guide.pdf")[WorkTable user guide]: declarations, every storage shape, queries, callsites, runtimes, persistence and lifecycle -examples. It describes 1.9.0-alpha1; use the reviewed checkout until publication. +examples. It describes 1.10.0-beta1; use the reviewed checkout until publication. #v(0.35cm) #text(size: 8pt, fill: rgb("#526873"))[ diff --git a/docs/wt-user-guide.typ b/docs/wt-user-guide.typ index 13ee00bb..46e45709 100644 --- a/docs/wt-user-guide.typ +++ b/docs/wt-user-guide.typ @@ -22,7 +22,7 @@ #text(size: 11pt, style: "italic")[Absolutely not a database.] #v(0.6em) #text(size: 9.5pt)[A user's guide to the `worktable!` macro, its queries, its indexes and its - persistence tier. Written against 1.9.0-alpha1.] + persistence tier. Written against 1.10.0-beta1.] ] #v(1.2em) @@ -39,10 +39,10 @@ See #link()[Persistence].] = Getting started ```sh -cargo add worktable@1.9.0-alpha1 +cargo add worktable@1.10.0-beta1 ``` -Until this alpha is published, depend on the reviewed checkout with +Until this beta is published, depend on the reviewed checkout with `worktable = { path = "../WorkTable" }`. A plain `cargo add worktable` selects the published release and may not include the APIs described here. @@ -151,7 +151,139 @@ let many = table.select_by_country(44).execute()?; // Vec optional column must say `using worktables_index`; Arctic supports `String` keys, but does not support optional keys. -== 5. Declared queries +== 5. Mutations and declared queries + +These operations are deliberately different. Choose by how much of the row the caller +owns and whether an absent key is valid: + +#text(size: 7.5pt)[ +#table( + columns: (1.7fr, 1.05fr, 0.85fr, 2.7fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 5pt, + [*operation*], [*input*], [*key absent*], [*meaning*], + [`insert(row)`], [complete `Row`], [insert], [Create a new row. An existing primary key returns `PrimaryAlreadyExists`; the caller does not authorize replacement.], + [`upsert(row)`], [complete `Row`], [insert], [Insert or replace. The caller declares the complete row authoritative. A row selected earlier can overwrite newer fields if it is later passed here.], + [`replace(row)`], [complete `Row`], [`NotFound`], [Replace every field of an existing row. It never creates a missing row, but the supplied row is still a complete authoritative snapshot.], + [#stack(spacing: 0.22em, [`update_by_(`], [`key, Columns::`], [`FIELD_SET, value)`])], [declared fields], [`NotFound`], [Change only the selector's declared fields. WorkTable rereads under its mutation lock when safe reconstruction needs the complete row, preserving concurrent changes to other fields.], + [#stack(spacing: 0.22em, [`update_in_place_`], [`by_(key,`], [`Columns::FIELD_SET,`], [`closure)`])], [#stack(spacing: 0.22em, [declared mutable], [archived fields])], [`NotFound`], [Directly mutate a declared, unindexed field set of an existing row. This is the lowest-work path and is restricted to primary-key lookup.], +) +] + +#text(size: 7.5pt)[ +#block( + fill: rgb("#f7f7f4"), + inset: 8pt, + radius: 2pt, + width: 100%, + breakable: false, +)[ +*Pays shipping-schema cost, Apple M4 Max, `taskpolicy -b`.* Persisted +`CustomerPayment`: autoincrement `u64` private key, packed 16-character Base62 +`payment_id` public key, four secondary indexes, 32,768 rows, 9 balanced +fresh-process samples. Monetary columns are strings; rerun after typed money. +Base `b1b9546` uses historical `update(row)` / generated query structs. Final +`45c015d` uses `replace` and typed selectors. Strings and custom archived +wrappers take the conservative complete-row `update` path, not in-place. + +#table( + columns: (1.45fr, 0.95fr, 0.95fr, 0.95fr, 0.85fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 4pt, + [*operation*], [*base ns/op*], [*final ns/op*], [*final updates/s*], [*vs in-place*], + [`upsert`], [76175], [74430], [13435], [9.83x], + [`replace`], [71118], [69518], [14385], [9.18x], + [`update`], [11907], [11561], [86500], [1.53x], + [`update_in_place`], [7340], [7571], [132082], [1.00x], +) + +#let bar(label, ns, max-ns) = { + let frac = ns / max-ns + grid( + columns: (3.4cm, 1fr, 1.8cm), + column-gutter: 6pt, + text(size: 7pt, raw(label)), + box(width: 100%, height: 7pt, fill: rgb("#e6e6e1"), + box(width: frac * 100%, height: 7pt, fill: rgb("#3a6ea5"))), + align(right, text(size: 7pt, [#ns ns])), + ) +} +#v(0.3em) +*Final `45c015d` median ns/op* +#v(0.15em) +#bar("upsert", 74430, 74430) +#v(0.1em) +#bar("replace", 69518, 74430) +#v(0.1em) +#bar("update", 11561, 74430) +#v(0.1em) +#bar("update_in_place", 7571, 74430) +] +] + +#text(size: 7.5pt)[ +#block( + fill: rgb("#f7f7f4"), + inset: 8pt, + radius: 2pt, + width: 100%, + breakable: false, +)[ +*Same shipping row, extended campaign, 5 samples, `d4b8aac`.* Indexes on +that table: unique `payment_id` and `app_id` are WTI; non-unique `symbol` and +`endpoint_address` are Arctic. There is no `update_range` primitive. +`range_ids` is consecutive private keys the caller already holds; +`range_scan` is `range_on` then update each. Workers are disjoint keys. +`using fxhash` is refused on a persisted table. + +The first 8-worker numbers were taken under `taskpolicy -b`. Eight tokio +tasks did overlap (`overlap_max=8`) but Darwin background QoS held the +process at ~1.3 cores, so the table looked like "concurrency does almost +nothing". Inherit policy, same JoinSet: + +#table( + columns: (0.7fr, 1.5fr, 1.0fr, 1.1fr, 0.8fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 4pt, + [*workers*], [*mutation*], [*updates/s*], [*cores*], [*vs 1*], + [1], [`replace`], [139920], [2.00], [1.00x], + [8], [`replace`], [177017], [6.14], [1.27x], + [12], [`replace`], [159189], [7.50], [1.14x], + [1], [`update_in_place`], [714567], [1.95], [1.00x], + [8], [`update_in_place`], [552544], [6.74], [0.77x], + [12], [`update_in_place`], [490803], [9.03], [0.69x], +) + +Eight workers burn six cores for 27% more replace/s. In-place gets *worse*. +On a unique `u64` secondary, 1-thread Congee/WTI/Arctic replace are within a +few percent; at 8 workers all three lose ~22% (Congee least-bad, not Arctic). + +#table( + columns: (1.6fr, 1.2fr, 1.2fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 4pt, + [*range (256 keys)*], [*ns/row*], [*updates/s*], + [`range_ids`], [6264], [159634], + [`range_scan`], [58262], [17164], +) + +#table( + columns: (1.1fr, 1.3fr, 1.1fr, 1.2fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 4pt, + [*backend*], [*mutation*], [*ns/op*], [*updates/s*], + [WTI persist], [`replace`], [5636], [177444], + [Arctic persist], [`replace`], [3923], [254934], + [Congee persist], [`replace`], [3938], [253928], + [FxHash `vec`], [`upsert`], [56], [17733214], +) + +FxHash is in-memory only. Arctic/Congee persisted `replace` on a unique +`u64` secondary sit together; WTI `replace` is slower on that fixture. +] +] + +Declare targeted updates, deletes and direct archived-field mutations with the table: ```rust worktable! ( @@ -168,23 +300,56 @@ worktable! ( delete: { ById() by id, // empty parens: names no columns }, - in_place: { + update_in_place: { StateById(state) by id, // only `by ` is supported + AmountAndStateById(amount, state) by id, }, }, ); ``` -CamelCase declared, snake_case generated: +The lookup column names the method and the typed selector names the changed +column. A one-column update takes that column's Rust value directly: ```rust -table.update_amount_by_id(AmountByIdQuery { amount: 900 }, 1).await?; // name + "Query" +table.update_by_id(1, InvoiceColumns::AMOUNT, 900).await?; table.delete_by_id(1).await?; -table.update_state_by_id_in_place(|state| *state = 2.into(), 1).await?; +table.update_in_place_by_id(1, InvoiceColumns::STATE, |state| *state = 2.into()).await?; +table.update_in_place_by_id( + 1, + InvoiceColumns::AMOUNT_AND_STATE, + |(amount, state)| { + *amount = 925.into(); + *state = 3.into(); + }, +).await?; ``` -`update` reads, changes and writes. `in_place` mutates without selecting first and locks -internally, so it is safe from several threads without the caller holding anything. +`InvoiceColumns::AMOUNT` is a generated zero-sized selector. Its sealed dispatch +implementation exists only for the declared `amount by id` combination and its +value type is `u64`, so an undeclared selector/key combination or wrong value +type fails to compile. The selector is monomorphized; it allocates nothing and +uses no dynamic dispatch. A declaration over `name, amount` exposes the atomic +selector `InvoiceColumns::NAME_AND_AMOUNT` and takes its generated query struct. +Selector constants preserve source spelling in uppercase: `attr1` becomes +`ATTR1`, while `some_field` becomes `SOME_FIELD`. + +A declared `update` reads, changes and writes only its named fields. Normal declared +updates accept owned Rust values and are the safe default for strings, options and +application-defined wrappers. An archived string contains a relative pointer, so +WorkTable may reconstruct the complete row rather than move only that field's archived +bytes. It rereads under the full mutation lock first, preserving concurrent changes to +other fields. The macro cannot inspect an external type such as `EncryptedSecret` and +prove whether its archived form contains relative pointers, so unknown custom types take +that conservative path. + +`update_in_place` mutates one declared field set without selecting first and locks internally. +A multi-column declaration passes a tuple of mutable archived fields to one closure, so the +set changes under the same row lock and persistence operation. Use it when the application +can safely edit the archived representation directly, as with scalars or fixed `#[repr(u8)]` +enums. Do not copy an archived string, vector or pointer-bearing wrapper +from another buffer into an `update_in_place` closure. Persisted in-place queries enqueue the +changed slot bytes; they do not skip durability. == 6. Selects you do not declare @@ -350,12 +515,13 @@ column, for a width the key cannot count to (`u16` beside a `u8` key declares 65 into a partition that holds 256), and for `persist: true`, which it has no engine to honour. -`queries:` works. An `update` or `delete` keyed by the primary key generates the same -method name and takes the same `Query` struct as the paged table, so the call reads -the same; it is not `async` and does not return `WorkTableError`, so a call cannot move -between the shapes by accident. A query keyed by any other column is refused, because a +`queries:` works. An `update` keyed by the primary key uses the same +`update_by_(key, Columns::FIELD_SET, value)` form as the paged table; a +multi-column selector takes the same generated query struct. A delete keeps its +declared method name. Dense calls are synchronous and return `Option`, so a call +cannot move between the shapes by accident. A query keyed by any other column is refused, because a dense partition has no secondary index and scanning instead would turn a keyed operation -into a linear one without saying so. `in_place` is refused as a synonym: every update +into a linear one without saying so. `update_in_place` is refused: every update here is already in place. Note that `memory_by_key` and `memory_total` cannot see any of this. They report @@ -426,10 +592,13 @@ error naming what to use instead, rather than being accepted and ignored. Declared `queries` are supported. They use equality on a primary or secondary index, including `fxhash`, and run synchronously through `&mut self`. An update declaration such as `StateById(state) by id` emits -`update_state_by_id(StateByIdQuery { state: 7 }, &id) -> usize`; a delete declaration +`update_by_id(id, TicketColumns::STATE, 7) -> usize`; a delete declaration `ByOwner() by owner` emits `delete_by_owner(&owner) -> usize`. The return value counts -affected rows. `in_place: { Status(state) by id }` emits -`update_status_in_place(|state| *state = 42, &id) -> usize` and accepts one column. +affected rows. `update_in_place: { Status(state) by id }` emits +`update_in_place_by_id(id, TicketColumns::STATE, |state| *state = 42) -> usize` +for one column. A declaration such as `StateAndRevisionById(state, revision) by id` +uses `TicketColumns::STATE_AND_REVISION` and passes `|(state, revision)| ...` to +one closure. These methods belong to the table, not mutable wrappers on the shared partition set. Vec edits validate a cloned candidate before replacing a row. A primary or unique @@ -437,7 +606,7 @@ secondary-key collision panics with that row and its indexes unchanged; a panick edit closure also leaves the stored row unchanged. Replacing an existing row through `upsert` checks unique secondary keys first. Multi-row queries apply one row at a time and are not transactions: earlier successful edits remain if a later edit fails. -Cloning owned fields is part of this mutation cost, including Vec `in_place` queries. +Cloning owned fields is part of this mutation cost, including Vec `update_in_place` queries. === Bytes and back: `unload` and `load` @@ -576,13 +745,13 @@ worktable! { columns: { id: u64 primary_key, total: u64 }, queries: { update runtime scheduled: { TotalById(total) by id }, - in_place runtime scheduled: { TotalById(total) by id }, + update_in_place runtime scheduled: { TotalById(total) by id }, } } let table = Arc::new(OrdersWorkTable::default()); table.insert(OrdersRow { id: 1, total: 10 }).await?; -table.update_total_by_id(TotalByIdQuery { total: 20 }, 1u64).await?; -table.update_total_by_id_in_place(|total| *total = 21.into(), 1u64).await?; +table.update_by_id(1, OrdersColumns::TOTAL, 20).await?; +table.update_in_place_by_id(1, OrdersColumns::TOTAL, |total| *total = 21.into()).await?; let rows = table.select_all() .order_on(OrdersRowFields::Total, Order::Desc) .limit(100).runtime(wide).execute_async().await?; @@ -667,7 +836,7 @@ worktable! ( queries: { update: { ScoreById(score) by id }, delete: { ById() by id }, - in_place: { ScoreById(score) by id }, + update_in_place: { ScoreById(score) by id }, }, config: { page_size: 4096, @@ -911,7 +1080,7 @@ more than halved randomized leaf lookup and was faster for four-client in-memory insertion. Select that tradeoff at the Cargo callsite: ```toml -worktable = { version = "^1.9.0-alpha1", default-features = false, +worktable = { version = "^1.10.0-beta1", default-features = false, features = ["std", "vanilla-index", "wti-predictable-search"] } ``` @@ -943,9 +1112,9 @@ Use `execute()` to materialize those builders. Unique secondary-index selects re was queued, not committed to stable storage. `delete_many(Vec).await` and `delete_range(range).await` return deleted keys and may report a `BatchDeleteError` with partial progress. Range deletion walks the keys present at that walk; it does not promise -to delete concurrent future inserts into the range. `reinsert(old, new).await` is the -explicit row-replacement operation; ordinary updates should use `upsert` or declared -queries so secondary indexes stay synchronized. +to delete concurrent future inserts into the range. Use `replace(row).await` when +the complete row is authoritative, or a declared update when only selected +columns are authoritative; both keep secondary indexes synchronized. With `autoincrement`, get a key from `get_next_pk()`, convert it into the row field, then insert. `reserve_pks(count)` reserves a disjoint range for a bulk producer. Reserved keys @@ -1160,7 +1329,7 @@ opened as ordinary WorkTable space files. `worktable_version!` and `migration_engine!` describe explicit versioned conversions; see `docs/migration.md` and the executable `tests/migration` fixtures for each required trait and transformation. They do not automatically infer data migration from a changed -schema. For this 1.9 alpha, a planned rebuild/data wipe is supported by the release plan; +schema. For this 1.9 beta, a planned rebuild/data wipe is supported by the release plan; do not infer cross-version file compatibility from a successful same-version reopen. `worktable::worktable_dsl` exposes parsing, checking and canonical schema emission for tools; the TypeScript emitter is tested against that Rust source of truth. diff --git a/dsl/Cargo.toml b/dsl/Cargo.toml index a674c168..7aecf63b 100644 --- a/dsl/Cargo.toml +++ b/dsl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "worktable_dsl" -version = "1.0.0-beta.19" +version = "1.10.0-beta1" edition = "2024" license = "MIT" description = "The worktable! schema language: its model and parser, readable outside the proc macro" diff --git a/dsl/src/model/queries.rs b/dsl/src/model/queries.rs index 46e66c08..80f485f7 100644 --- a/dsl/src/model/queries.rs +++ b/dsl/src/model/queries.rs @@ -7,7 +7,7 @@ use crate::model::Operation; pub struct Queries { pub updates: IndexMap, pub deletes: IndexMap, - pub in_place: IndexMap, + pub updates_in_place: IndexMap, /// The profile named by `update runtime :`, when the section was /// annotated. `None` is not a default: it means the section falls back to /// the table's `runtime`, and the table's own default only after that. @@ -19,6 +19,7 @@ pub struct Queries { pub update_runtime: Option, /// The profile named by `delete runtime :`. See `update_runtime`. pub delete_runtime: Option, - /// The profile named by `in_place runtime :`. See `update_runtime`. - pub in_place_runtime: Option, + /// The profile named by `update_in_place runtime :`. See + /// `update_runtime`. + pub update_in_place_runtime: Option, } diff --git a/dsl/src/parser/queries/in_place.rs b/dsl/src/parser/queries/in_place.rs index c10f2f94..35ec6763 100644 --- a/dsl/src/parser/queries/in_place.rs +++ b/dsl/src/parser/queries/in_place.rs @@ -6,17 +6,17 @@ use crate::Parser; use crate::model::Operation; impl Parser { - /// The `in_place` block, and the profile it was annotated with. See + /// The `update_in_place` block, and the profile it was annotated with. See /// [`Parser::parse_updates`] for why the annotation rides beside the /// operations. - pub fn parse_in_place(&mut self) -> syn::Result<(Option, IndexMap)> { + pub fn parse_updates_in_place(&mut self) -> syn::Result<(Option, IndexMap)> { let ident = self.input_iter.next().ok_or(syn::Error::new( self.input.span(), - "Expected `in_place` field in declaration", + "Expected `update_in_place` field in declaration", ))?; if let TokenTree::Ident(ident) = ident { - if ident.to_string().as_str() != "in_place" { - return Err(syn::Error::new(ident.span(), "Expected `in_place` field")); + if ident.to_string().as_str() != "update_in_place" { + return Err(syn::Error::new(ident.span(), "Expected `update_in_place` field")); } } else { return Err(syn::Error::new(ident.span(), "Expected field name identifier.")); @@ -34,7 +34,7 @@ impl Parser { let mut parser = Parser::new(ops.stream()); let operations = parser.parse_operations()?; // Symmetry with `parse_updates`: consume a comma after the block, - // so a `in_place` block is not required to be written last. + // so an `update_in_place` block is not required to be written last. self.try_parse_comma()?; Ok((runtime, operations)) } else { @@ -53,12 +53,12 @@ mod tests { #[test] fn test_update() { let tokens = quote! { - in_place: { + update_in_place: { TestQuery(id) by name, } }; let mut parser = Parser::new(tokens); - let (_, ops) = parser.parse_in_place().unwrap(); + let (_, ops) = parser.parse_updates_in_place().unwrap(); assert_eq!(ops.len(), 1); let op = ops.get(&Ident::new("TestQuery", Span::mixed_site())).unwrap(); diff --git a/dsl/src/parser/queries/mod.rs b/dsl/src/parser/queries/mod.rs index 0664b6da..98ff1eec 100644 --- a/dsl/src/parser/queries/mod.rs +++ b/dsl/src/parser/queries/mod.rs @@ -48,15 +48,27 @@ impl Parser { queries.deletes = deletes; queries.delete_runtime = runtime; } + "update_in_place" => { + let (runtime, updates) = parser.parse_updates_in_place()?; + queries.updates_in_place = updates; + queries.update_in_place_runtime = runtime; + } + // The 1.10 rename. Without this arm the old spelling gets + // the generic "unexpected token" below, which lists the new + // keyword but does not say it is the same thing renamed, so + // the reader has to guess that their table still works. "in_place" => { - let (runtime, in_place) = parser.parse_in_place()?; - queries.in_place = in_place; - queries.in_place_runtime = runtime; + return Err(syn::Error::new( + ident.span(), + "`in_place:` was renamed to `update_in_place:` in WorkTable 1.10; rename the section keyword, the queries inside it are unchanged", + )); } other => { return Err(syn::Error::new( ident.span(), - format!("Unexpected token `{other}`; expected one of `update`, `delete`, `in_place`"), + format!( + "Unexpected token `{other}`; expected one of `update`, `delete`, `update_in_place`" + ), )); } } @@ -83,14 +95,14 @@ mod tests { queries: { update: { Fill(qty) by id }, delete: { BySymbol() by symbol }, - in_place: { Bump(qty) by id }, + update_in_place: { Bump(qty) by id }, } }; let queries = Parser::new(tokens).parse_queries().unwrap(); assert!(queries.update_runtime.is_none()); assert!(queries.delete_runtime.is_none()); - assert!(queries.in_place_runtime.is_none()); + assert!(queries.update_in_place_runtime.is_none()); } #[test] @@ -99,17 +111,17 @@ mod tests { queries: { update runtime fast_local: { Fill(qty) by id }, delete runtime wide: { BySymbol() by symbol }, - in_place runtime bulk: { Bump(qty) by id }, + update_in_place runtime bulk: { Bump(qty) by id }, } }; let queries = Parser::new(tokens).parse_queries().unwrap(); assert_eq!(queries.update_runtime.unwrap(), "fast_local"); assert_eq!(queries.delete_runtime.unwrap(), "wide"); - assert_eq!(queries.in_place_runtime.unwrap(), "bulk"); + assert_eq!(queries.update_in_place_runtime.unwrap(), "bulk"); assert_eq!(queries.updates.len(), 1); assert_eq!(queries.deletes.len(), 1); - assert_eq!(queries.in_place.len(), 1); + assert_eq!(queries.updates_in_place.len(), 1); } #[test] @@ -117,13 +129,13 @@ mod tests { let tokens = quote! { queries: { update runtime fast_local: { Fill(qty) by id }, - in_place: { Bump(qty) by id }, + update_in_place: { Bump(qty) by id }, } }; let queries = Parser::new(tokens).parse_queries().unwrap(); assert_eq!(queries.update_runtime.unwrap(), "fast_local"); - assert!(queries.in_place_runtime.is_none()); + assert!(queries.update_in_place_runtime.is_none()); } #[test] @@ -140,4 +152,21 @@ mod tests { "{error}" ); } + + #[test] + fn legacy_in_place_section_is_rejected() { + let tokens = quote! { + queries: { in_place: { Bump(qty) by id } } + }; + let error = Parser::new(tokens).parse_queries().unwrap_err().to_string(); + + // The generic "Unexpected token" this used to assert is what the + // rename replaced: a table that still says `in_place:` is not using an + // unknown keyword, it is using last version's spelling of this one, and + // the error is only useful if it says so. + assert!( + error.contains("`in_place:` was renamed to `update_in_place:`"), + "{error}" + ); + } } diff --git a/dsl/src/parser/runtime.rs b/dsl/src/parser/runtime.rs index e744be5a..06f4e6eb 100644 --- a/dsl/src/parser/runtime.rs +++ b/dsl/src/parser/runtime.rs @@ -441,13 +441,13 @@ mod tests { queries: { update runtime fast_local: { Fill(qty) by id }, delete runtime wide: { BySymbol() by symbol }, - in_place: { Bump(qty) by id }, + update_in_place: { Bump(qty) by id }, }, ", ); assert_eq!(schema.queries.update_runtime.as_deref(), Some("fast_local")); assert_eq!(schema.queries.delete_runtime.as_deref(), Some("wide")); - assert_eq!(schema.queries.in_place_runtime, None); + assert_eq!(schema.queries.update_in_place_runtime, None); } #[test] diff --git a/dsl/src/schema/emit_dsl.rs b/dsl/src/schema/emit_dsl.rs index fe73a108..6c75459a 100644 --- a/dsl/src/schema/emit_dsl.rs +++ b/dsl/src/schema/emit_dsl.rs @@ -105,9 +105,9 @@ impl Schema { ); write_query_block( &mut out, - "in_place", - self.queries.in_place_runtime.as_deref(), - &self.queries.in_place, + "update_in_place", + self.queries.update_in_place_runtime.as_deref(), + &self.queries.updates_in_place, ); let _ = writeln!(out, "}},"); } diff --git a/dsl/src/schema/emit_uml.rs b/dsl/src/schema/emit_uml.rs index 1f185921..15269fe0 100644 --- a/dsl/src/schema/emit_uml.rs +++ b/dsl/src/schema/emit_uml.rs @@ -56,7 +56,7 @@ impl Schema { for (kind, operations) in [ ("update", &self.queries.updates), ("delete", &self.queries.deletes), - ("in_place", &self.queries.in_place), + ("update_in_place", &self.queries.updates_in_place), ] { for operation in operations { let _ = writeln!( diff --git a/dsl/src/schema/mod.rs b/dsl/src/schema/mod.rs index 701f05e0..2a1b27c4 100644 --- a/dsl/src/schema/mod.rs +++ b/dsl/src/schema/mod.rs @@ -200,22 +200,22 @@ pub struct QueriesSpec { pub updates: Vec, /// `delete:` operations. pub deletes: Vec, - /// `in_place:` operations. - pub in_place: Vec, + /// `update_in_place:` operations. + pub updates_in_place: Vec, /// The profile named by `update runtime :`, if written. Unresolved: /// see [`crate::model::Queries::update_runtime`]. pub update_runtime: Option, /// The profile named by `delete runtime :`, if written. pub delete_runtime: Option, - /// The profile named by `in_place runtime :`, if written. - pub in_place_runtime: Option, + /// The profile named by `update_in_place runtime :`, if written. + pub update_in_place_runtime: Option, } impl QueriesSpec { /// Whether any query was declared. An empty block and an absent one are /// the same thing to the macro, so the emitter writes neither. pub fn is_empty(&self) -> bool { - self.updates.is_empty() && self.deletes.is_empty() && self.in_place.is_empty() + self.updates.is_empty() && self.deletes.is_empty() && self.updates_in_place.is_empty() } } @@ -498,10 +498,10 @@ fn queries_from_model(queries: Queries) -> QueriesSpec { QueriesSpec { update_runtime: queries.update_runtime.map(|profile| profile.to_string()), delete_runtime: queries.delete_runtime.map(|profile| profile.to_string()), - in_place_runtime: queries.in_place_runtime.map(|profile| profile.to_string()), + update_in_place_runtime: queries.update_in_place_runtime.map(|profile| profile.to_string()), updates: convert(queries.updates), deletes: convert(queries.deletes), - in_place: convert(queries.in_place), + updates_in_place: convert(queries.updates_in_place), } } diff --git a/dsl/src/validate.rs b/dsl/src/validate.rs index 44980c1b..3b8d931e 100644 --- a/dsl/src/validate.rs +++ b/dsl/src/validate.rs @@ -87,20 +87,20 @@ pub fn validate_arctic_page_size(columns: &Columns, config: Option<&crate::model Ok(()) } -/// `in_place` queries hand the caller a mutable reference to the archived +/// `update_in_place` queries hand the caller a mutable reference to the archived /// column bytes and bypass all index maintenance, so a column that any index /// is built over cannot be mutated in place: the index would keep resolving /// the old value. pub fn validate_in_place_queries(columns: &Columns, queries: &crate::model::Queries) -> syn::Result<()> { - for (name, op) in &queries.in_place { + for (name, op) in &queries.updates_in_place { for column in &op.columns { if columns.indexes.values().any(|index| &index.field == column) { return Err(syn::Error::new( column.span(), format!( - "in_place query `{name}` mutates column `{column}`, which is covered by an index; \ + "update_in_place query `{name}` mutates column `{column}`, which is covered by an index; \ indexed columns cannot be updated in place because secondary indexes are not \ - maintained on this path. Use an `update` query instead" + maintained on this path. Use an `update` query instead" ), )); } @@ -382,7 +382,7 @@ pub fn validate_query_storage( .update_runtime .as_ref() .or(queries.delete_runtime.as_ref()) - .or(queries.in_place_runtime.as_ref()) + .or(queries.update_in_place_runtime.as_ref()) { return Err(syn::Error::new( profile.span(), @@ -404,19 +404,19 @@ pub fn validate_query_storage( )); } } - for (name, op) in &queries.in_place { + for (name, op) in &queries.updates_in_place { if columns.primary_keys.len() != 1 || columns.primary_keys.first() != Some(&op.by) { return Err(syn::Error::new( op.by.span(), format!( - "in_place query `{name}` requires selection by the single-column primary key; use an update query for an indexed predicate" + "update_in_place query `{name}` requires selection by the single-column primary key; use an update query for an indexed predicate" ), )); } if op.columns.iter().any(|column| columns.primary_keys.contains(column)) { return Err(syn::Error::new( name.span(), - "in_place queries cannot mutate primary key columns; use an update query to maintain indexes", + "update_in_place queries cannot mutate primary key columns; use an update query to maintain indexes", )); } } diff --git a/dsl/tests/query_storage.rs b/dsl/tests/query_storage.rs index 48fa48b6..5b86aa5e 100644 --- a/dsl/tests/query_storage.rs +++ b/dsl/tests/query_storage.rs @@ -4,8 +4,8 @@ use worktable_dsl::check::check; fn paged_mutation_shapes_fail_before_emission() { for query in [ "update: { Change(value) by value }", - "in_place: { Change(value) by value }", - "in_place: { Change(id) by id }", + "update_in_place: { Change(value) by value }", + "update_in_place: { Change(id) by id }", ] { let checked = check(&format!( "name: T, columns: {{ id: u64 primary_key, value: u64 }}, queries: {{ {query} }}" diff --git a/dsl/tests/trailing_commas.rs b/dsl/tests/trailing_commas.rs index e6f27674..a9e4e763 100644 --- a/dsl/tests/trailing_commas.rs +++ b/dsl/tests/trailing_commas.rs @@ -53,14 +53,14 @@ fn a_comma_after_delete_or_in_place_is_accepted() { columns: { id: u64 primary_key, name: String }, queries: { delete: { ByName() by name, }, - in_place: { SetName(name) by id, }, + update_in_place: { SetName(name) by id, }, update: { Renamed(name) by id, } }", ) .expect("`delete` and `in_place` should not have to be written last either"); assert_eq!(schema.queries.deletes.len(), 1); - assert_eq!(schema.queries.in_place.len(), 1); + assert_eq!(schema.queries.updates_in_place.len(), 1); assert_eq!(schema.queries.updates.len(), 1); } diff --git a/examples/guide_check.rs b/examples/guide_check.rs index b853013a..0d192949 100644 --- a/examples/guide_check.rs +++ b/examples/guide_check.rs @@ -53,9 +53,7 @@ async fn main() -> eyre::Result<()> { quantity: 5, }]) .await?; - table - .update_quantity_by_id(QuantityByIdQuery { quantity: 7 }, 100) - .await?; + table.update_by_id(100, OrderColumns::QUANTITY, 7).await?; assert_eq!(table.select(100).unwrap().quantity, 7); assert_eq!(table.select_all().limit(1).execute()?.len(), 1); assert_eq!(table.row_count(), 2); diff --git a/paper-bench/scripts/compile_cost.sh b/paper-bench/scripts/compile_cost.sh index 09f96a13..a224fdbe 100644 --- a/paper-bench/scripts/compile_cost.sh +++ b/paper-bench/scripts/compile_cost.sh @@ -40,7 +40,7 @@ worktable!( indexes: { a_idx_$i: a, }, queries: { update: { UpdA$i(a) by id, }, - in_place: { IncB$i(b) by id, } + update_in_place: { IncB$i(b) by id, } } ); EOF diff --git a/paper-bench/src/bin/ablation.rs b/paper-bench/src/bin/ablation.rs index f9f04b65..c6d1b8b0 100644 --- a/paper-bench/src/bin/ablation.rs +++ b/paper-bench/src/bin/ablation.rs @@ -57,7 +57,7 @@ fn main() { rt.block_on(async { for _ in 0..n { let pk = rng.below(rows); - table.update_upd_b(UpdBQuery { b: pk }, pk).await.unwrap(); + table.update_by_id(pk, BenchColumns::B, pk).await.unwrap(); } }); n @@ -74,7 +74,7 @@ fn main() { rt.block_on(async { for _ in 0..n { let pk = rng.below(rows); - table.update_upd_a(UpdAQuery { a: pk }, pk).await.unwrap(); + table.update_by_id(pk, BenchColumns::A, pk).await.unwrap(); } }); n @@ -91,7 +91,7 @@ fn main() { rt.block_on(async { for _ in 0..n { let pk = rng.below(rows); - table.update_inc_b_in_place(|b| *b += 1, pk).await.unwrap(); + table.update_in_place_by_id(pk, BenchColumns::B, |b| *b += 1).await.unwrap(); } }); n diff --git a/paper-bench/src/bin/contention.rs b/paper-bench/src/bin/contention.rs index dd083c6a..46164297 100644 --- a/paper-bench/src/bin/contention.rs +++ b/paper-bench/src/bin/contention.rs @@ -38,24 +38,24 @@ async fn run(mode: &'static str, tasks: usize) -> f64 { match mode { "disjoint" => { if i % 2 == 0 { - table.update_upd_b(UpdBQuery { b: n }, pk_val).await.unwrap(); + table.update_by_id(pk_val, BenchColumns::B, n).await.unwrap(); } else { - table.update_upd_e(UpdEQuery { e: n }, pk_val).await.unwrap(); + table.update_by_id(pk_val, BenchColumns::E, n).await.unwrap(); } } "overlap" => { - table.update_upd_be(UpdBEQuery { b: n, e: n }, pk_val).await.unwrap(); + table.update_by_id(pk_val, BenchColumns::B_AND_E, UpdBEQuery { b: n, e: n }).await.unwrap(); } "mutex" => { let _g = big_lock.lock().await; if i % 2 == 0 { - table.update_upd_b(UpdBQuery { b: n }, pk_val).await.unwrap(); + table.update_by_id(pk_val, BenchColumns::B, n).await.unwrap(); } else { - table.update_upd_e(UpdEQuery { e: n }, pk_val).await.unwrap(); + table.update_by_id(pk_val, BenchColumns::E, n).await.unwrap(); } } "inplace" => { - table.update_inc_b_in_place(|b| *b += 1, pk_val).await.unwrap(); + table.update_in_place_by_id(pk_val, BenchColumns::B, |b| *b += 1).await.unwrap(); } _ => unreachable!(), } diff --git a/paper-bench/src/lib.rs b/paper-bench/src/lib.rs index 5b19bd88..2063e0e7 100644 --- a/paper-bench/src/lib.rs +++ b/paper-bench/src/lib.rs @@ -28,7 +28,7 @@ worktable!( UpdE(e) by id, UpdBE(b, e) by id, }, - in_place: { + update_in_place: { IncB(b) by id, } } diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index 2976e045..c51a0ff2 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -55,7 +55,7 @@ run cargo test --workspace --all-targets --features versioned-row-publication echo "=== build and test (all-features) ===" run cargo build --workspace --all-targets --all-features -run cargo test --workspace --all-targets --all-features +run env CARGO_PROFILE_TEST_DEBUG=0 cargo test --workspace --all-targets --all-features echo "=== cell-lock concurrency models ===" run env "RUSTFLAGS=--cfg wt_loom" CARGO_TARGET_DIR=target/cell-lock-loom cargo test --release --lib cell_lock_models diff --git a/src/in_memory/data.rs b/src/in_memory/data.rs index 48539796..0fb38a4f 100644 --- a/src/in_memory/data.rs +++ b/src/in_memory/data.rs @@ -1,7 +1,8 @@ -use alloc::vec::Vec; +use alloc::{sync::Arc, vec::Vec}; use core::cell::UnsafeCell; use core::fmt::Debug; use core::marker::PhantomData; +use core::mem::MaybeUninit; use core::ops::{Deref, DerefMut}; #[cfg(not(wt_loom))] use core::sync::atomic::{AtomicU32 as CellState, AtomicUsize as CellOwner}; @@ -25,7 +26,7 @@ use rkyv::{ with::{AtomicLoad, Relaxed, Skip, Unsafe}, }; -use crate::in_memory::ArchivedRowWrapper; +use crate::in_memory::{ArchivedRowWrapper, InlineArchived}; use crate::prelude::Link; #[cfg(all(not(wt_loom), not(any(unix, windows))))] @@ -35,9 +36,30 @@ const CELL_LOCK_SLOTS: usize = 256; const CELL_READER_MASK: u32 = (1_u32 << 31) - 1; const CELL_WRITER: u32 = 1 << 31; +/// One stripe's reader/writer count, alone on a cache line. +/// +/// Packed `[AtomicU32; 256]` put 16 stripes on one 64-byte line. Shared +/// `select` then bounced that line across cores and stalled at ~2.4× while a +/// private table (its own `CellLocks`) and `vec: true` both scaled ~7×. +/// Owners stay packed: the uncontended read path never loads them. +#[repr(align(64))] +#[derive(Debug)] +struct PaddedCellState { + value: CellState, + /// Monotonic write counter, advanced once per completed write. + /// + /// `value` alone cannot carry the seqlock stamp: a writer releases by + /// storing `0`, which is exactly what an idle cell reads, so a write that + /// begins and ends inside one reader's window leaves no trace in it. The + /// reader would then keep bytes copied out of the middle of that write and + /// hand them to `rkyv::access_unchecked`. The counter shares this cache + /// line, which the padding above already reserved. + version: CellState, +} + #[derive(Debug)] struct CellLocks { - states: [CellState; CELL_LOCK_SLOTS], + states: [PaddedCellState; CELL_LOCK_SLOTS], owners: [CellOwner; CELL_LOCK_SLOTS], nested_reads: CellState, } @@ -45,13 +67,55 @@ struct CellLocks { impl Default for CellLocks { fn default() -> Self { Self { - states: core::array::from_fn(|_| CellState::new(0)), + states: core::array::from_fn(|_| PaddedCellState { + value: CellState::new(0), + version: CellState::new(0), + }), owners: core::array::from_fn(|_| CellOwner::new(0)), nested_reads: CellState::new(0), } } } +impl CellLocks { + /// Breaks the build when a field is added to [`CellLocks`] without being + /// added to [`Self::initialize_at`]. See + /// [`Data::every_field_is_initialized`] for why this exists. + #[cfg(test)] + #[expect(dead_code, reason = "compiled for its exhaustiveness check, never called")] + fn every_field_is_initialized(self) { + let Self { + states: _, + owners: _, + nested_reads: _, + } = self; + } + + /// Initialize the lock table directly in its final allocation. + /// + /// Building both fixed-size atomic arrays as a return value makes the + /// compiler reserve another copy on the caller's stack. `Data` uses this + /// when it is created inside an `Arc`, where that copy is unnecessary. + unsafe fn initialize_at(target: *mut Self) { + unsafe { + let states = core::ptr::addr_of_mut!((*target).states).cast::(); + for index in 0..CELL_LOCK_SLOTS { + states.add(index).write(PaddedCellState { + value: CellState::new(0), + version: CellState::new(0), + }); + } + + let owners = core::ptr::addr_of_mut!((*target).owners).cast::(); + for index in 0..CELL_LOCK_SLOTS { + owners.add(index).write(CellOwner::new(0)); + } + + core::ptr::addr_of_mut!((*target).nested_reads).write(CellState::new(0)); + } + } +} + #[inline] fn current_owner() -> usize { #[cfg(wt_loom)] @@ -110,9 +174,91 @@ impl CellLocks { } } - fn read(&self, link: Link) -> Result, ExecutionError> { + /// Snapshot the stripe if no writer is in the critical section. + /// + /// Readers do not CAS. Shared `select` was bouncing packed stripe atomics + /// and stalled at ~2.4×; `vec: true` and a private paged table both scale + /// ~7×. The caller copies bytes, then [`Self::still_stable`]. + fn load_stable(&self, link: Link) -> Result { + let index = Self::start(link); + let state = &self.states[index].value; + let version = &self.states[index].version; + let mut spins = 0; + loop { + let current = state.load(Ordering::Acquire); + if current & CELL_WRITER == 0 { + // Read the counter AFTER observing no writer. A writer that + // starts later bumps it on release, so `still_stable` sees the + // move even though the state word returns to `0`. + return Ok(version.load(Ordering::Acquire)); + } + if self.owners[index].load(Ordering::Acquire) == current_owner() { + let writer_key = current & CELL_READER_MASK; + let requested_key = link.offset.checked_add(1).ok_or(ExecutionError::InvalidLink)?; + if writer_key == requested_key { + return Err(ExecutionError::CellLockReentry); + } + if writer_key != 0 { + // This task is the writer. It sees its own uncommitted + // bytes by definition; the counter is the stamp it will + // compare against and its own release has not run yet. + return Ok(version.load(Ordering::Acquire)); + } + // Reaching here means the writer bit is set and the owner is + // this thread, but no key is published yet. `write` takes the + // bit with a CAS, drains readers, stores the key and only then + // stores the owner, so the owner cannot be us while the key is + // still zero -- unless a previous writer on this stripe left a + // stale owner, which its `Drop` clears before releasing the bit. + // The guard is therefore false in every reachable state today + // and exists to keep this reader honest if that field ordering + // is ever changed: falling through to `wait` is correct for an + // unidentified writer, and returning a stamp would not be. + } + Self::wait(&mut spins); + } + } + + fn still_stable(&self, link: Link, stamp: u32) -> bool { let index = Self::start(link); let state = &self.states[index]; + // Two ways the snapshot dies: a writer is in the critical section now, + // or one completed since [`Self::load_stable`]. The version alone + // catches the second; checking it alone would let a torn copy through + // while a writer is still mid-update. + // + // The exception is this task's own write on a DIFFERENT row of the + // same stripe, which `load_stable` already admits: it is reading rows + // the writer is not touching, and it can never make progress by + // retrying because only it can clear that bit. Rejecting it here + // spins forever (`in_place_callback_can_select_a_colliding_row_ + // without_deadlock`). + let current = state.value.load(Ordering::Acquire); + if current & CELL_WRITER != 0 && !self.writer_is_self_on_other_row(index, link, current) { + return false; + } + state.version.load(Ordering::Acquire) == stamp + } + + /// Is the in-flight writer this same task, working on a different row? + /// + /// Mirrors the reentry arm of [`Self::load_stable`]. A same-row match is + /// `CellLockReentry` there and never reaches a stability check. + fn writer_is_self_on_other_row(&self, index: usize, link: Link, current: u32) -> bool { + if self.owners[index].load(Ordering::Acquire) != current_owner() { + return false; + } + let writer_key = current & CELL_READER_MASK; + // `writer_key != 0` mirrors the same guard in `load_stable`, and for + // the same reason: see the note there. It is unreachable-false given + // the current store order in `write`, and rejecting the snapshot is + // the safe answer if that ever stops holding. + writer_key != 0 && Some(writer_key) != link.offset.checked_add(1) + } + + fn read(&self, link: Link) -> Result, ExecutionError> { + let index = Self::start(link); + let state = &self.states[index].value; let mut spins = 0; loop { let current = state.load(Ordering::Acquire); @@ -148,7 +294,7 @@ impl CellLocks { fn write(&self, link: Link) -> Result, ExecutionError> { let index = Self::start(link); - let state = &self.states[index]; + let state = &self.states[index].value; let owner = &self.owners[index]; let mut spins = 0; loop { @@ -175,6 +321,7 @@ impl CellLocks { Ok(CellWriteGuard { state, owner, + version: &self.states[index].version, _not_send: PhantomData, }) } @@ -184,7 +331,11 @@ impl CellLocks { owner.store(0, Ordering::Relaxed); } for state in &self.states { - state.store(0, Ordering::Release); + // Reset replaces page contents, so it counts as a write for + // snapshot readers: bump before clearing, or an in-flight + // `copy_row_seqlock` would accept bytes from across the reset. + state.version.fetch_add(1, Ordering::Release); + state.value.store(0, Ordering::Release); } self.nested_reads.store(0, Ordering::Relaxed); } @@ -207,6 +358,7 @@ impl Drop for CellReadGuard<'_> { pub(crate) struct CellWriteGuard<'a> { state: &'a CellState, owner: &'a CellOwner, + version: &'a CellState, _not_send: PhantomData<*mut ()>, } @@ -214,6 +366,11 @@ impl Drop for CellWriteGuard<'_> { #[inline] fn drop(&mut self) { self.owner.store(0, Ordering::Relaxed); + // Advance the seqlock counter before clearing the writer bit. A + // snapshot reader that already copied these bytes compares the counter + // and retries; ordering the bump first means it can never observe the + // cell idle at the pre-write version. + self.version.fetch_add(1, Ordering::Release); self.state.store(0, Ordering::Release); } } @@ -224,6 +381,51 @@ pub const DATA_HEADER_LENGTH: usize = 4; /// Length of the inner [`Data`] page part. pub const DATA_INNER_LENGTH: usize = INNER_PAGE_SIZE - DATA_HEADER_LENGTH; +const SEQLOCK_STACK: usize = 256; + +/// Seqlock snapshot of one archived cell. +/// +/// Stack for rows that fit in 256 bytes (the shipping fixture). Larger rows +/// take the cell read lock once and copy onto the heap. +// The variants differ in size by design and `clippy::large_enum_variant`'s fix +// is the defect: boxing `Stack` puts the small-row snapshot on the heap, which +// is the allocation this type exists to avoid. The enum is a local of the +// select path and is never stored in a collection, so the size difference +// costs one stack frame, not memory per row. +#[allow(clippy::large_enum_variant)] +pub(crate) enum ArchivedCopy { + /// Only the first `len` bytes of `buf` are ever written. + /// + /// The storage stays `MaybeUninit`, rather than being `assume_init`ed into + /// an `AlignedBytes<256>`, because the tail beyond `len` is never + /// initialized: a row is at most 256 bytes and usually far less. Calling + /// `assume_init` on the whole array and then moving the `Copy` value into + /// this variant is a 256-byte read of uninitialized memory, which LLVM is + /// entitled to treat as poison even though `as_bytes` never exposes it. + /// Keeping it `MaybeUninit` also makes the move cheaper, not dearer: the + /// initialized prefix is all that has to be meaningful. + Stack { + buf: MaybeUninit>, + len: u16, + }, + Heap(AlignedVec), +} + +impl ArchivedCopy { + #[inline] + pub(crate) fn as_bytes(&self) -> &[u8] { + match self { + // SAFETY: `copy_row_seqlock` is the only constructor of this + // variant and writes exactly `len` bytes at the start of `buf` + // before returning, with `len <= SEQLOCK_STACK`. + Self::Stack { buf, len } => unsafe { + core::slice::from_raw_parts(buf.as_ptr().cast::(), *len as usize) + }, + Self::Heap(bytes) => bytes.as_slice(), + } + } +} + #[derive(Archive, Clone, Copy, Debug, Deserialize, Serialize)] #[repr(C, align(16))] pub struct AlignedBytes(pub [u8; N]); @@ -291,6 +493,90 @@ pub struct Data { unsafe impl Sync for Data {} impl Data { + /// Breaks the build when a field is added to [`Data`] without being added + /// to [`Self::initialize_arc_at`]. + /// + /// That initializer writes each field through `addr_of_mut!` and then + /// `assume_init`s the allocation, so the compiler cannot check it for + /// exhaustiveness the way it checks a struct literal: adding a field and + /// updating only the safe `new` beside it compiles cleanly and hands out an + /// `Arc` with one field uninitialized. Destructuring without `..` is the + /// check the initializer itself cannot have. Keep the binding list here and + /// the writes there in step. + #[cfg(test)] + #[expect(dead_code, reason = "compiled for its exhaustiveness check, never called")] + fn every_field_is_initialized(self) { + let Self { + id: _, + free_offset: _, + access: _, + cell_locks: _, + live_cells: _, + inner_data: _, + _phantom: _, + } = self; + } + + unsafe fn initialize_arc_at(target: *mut Self, id: PageId, free_offset: u32, source: Option<*const u8>) { + unsafe { + core::ptr::addr_of_mut!((*target).id).write(id); + core::ptr::addr_of_mut!((*target).free_offset).write(AtomicU32::new(free_offset)); + core::ptr::addr_of_mut!((*target).access).write(parking_lot::RwLock::new(())); + CellLocks::initialize_at(core::ptr::addr_of_mut!((*target).cell_locks)); + core::ptr::addr_of_mut!((*target).live_cells).write(AtomicU32::new(0)); + + let destination = core::ptr::addr_of_mut!((*target).inner_data).cast::(); + if let Some(source) = source { + core::ptr::copy_nonoverlapping(source, destination, DATA_LENGTH); + } else { + core::ptr::write_bytes(destination, 0, DATA_LENGTH); + } + + core::ptr::addr_of_mut!((*target)._phantom).write(PhantomData); + } + } + + /// Creates a new page directly in its `Arc` allocation. + /// + /// A default page owns a roughly 16 KiB inline byte image. Initializing it + /// through `Arc::new(Data::new(...))` can copy that image through several + /// stack return slots, which compounds when a generated table constructs + /// several indexed components on a normal 2 MiB thread stack. + pub fn new_arc(id: PageId) -> Arc { + let mut page = Arc::::new_uninit(); + let target = Arc::get_mut(&mut page) + .expect("a newly allocated Arc is uniquely owned") + .as_mut_ptr(); + + // SAFETY: every field is initialized exactly once in the allocation + // above. The byte image is an array of u8, for which all-zero is valid. + unsafe { + Self::initialize_arc_at(target, id, 0, None); + page.assume_init() + } + } + + /// Restores a persisted page by reference without moving its inline byte + /// image through the caller's stack. + pub fn from_data_page_ref_arc(page: &GeneralPage>) -> Arc { + let mut restored = Arc::::new_uninit(); + let target = Arc::get_mut(&mut restored) + .expect("a newly allocated Arc is uniquely owned") + .as_mut_ptr(); + + // SAFETY: the boxed page remains alive for the copy, and the helper + // initializes every field before the Arc is exposed. + unsafe { + Self::initialize_arc_at( + target, + page.header.page_id, + page.header.data_length, + Some(page.inner.data.as_ptr()), + ); + restored.assume_init() + } + } + fn validate_link(&self, link: Link) -> Result<(), ExecutionError> { let start = link.offset as usize; let end = start @@ -495,6 +781,111 @@ impl Data { rkyv::deserialize::<_, rkyv::rancor::Error>(row).map_err(|_| ExecutionError::DeserializeError) } + /// Point-read without a reader CAS on the cell stripe. + /// + /// Copies archived bytes, then checks the stripe still has no writer. + /// Deserialize runs only on that private copy. Rows larger than the stack + /// buffer keep the locking path. + pub fn get_row_seqlock(&self, link: Link) -> Result + where + Row: Archive, + ::Archived: Deserialize>, + { + let copy = self.copy_row_seqlock(link)?; + let archived = unsafe { rkyv::access_unchecked::<::Archived>(copy.as_bytes()) }; + rkyv::deserialize::<_, rkyv::rancor::Error>(archived).map_err(|_| ExecutionError::DeserializeError) + } + + /// Seqlock-copy archived bytes without deserializing them. + pub(crate) fn copy_row_seqlock(&self, link: Link) -> Result { + self.validate_link(link)?; + let len = link.length as usize; + if len > SEQLOCK_STACK { + let _guard = self.cell_locks.read(link)?; + let inner = unsafe { &*self.inner_data.get() }; + let start = link.offset as usize; + let mut heap = AlignedVec::with_capacity(len); + heap.extend_from_slice(&inner[start..start + len]); + return Ok(ArchivedCopy::Heap(heap)); + } + // Backoff state for the retry path only. A snapshot that validates on + // the first attempt, which is the overwhelmingly common case, never + // touches this: the counter is incremented only after `still_stable` + // has already rejected a copy. + let mut spins = 0; + loop { + let stamp = self.cell_locks.load_stable(link)?; + let inner = unsafe { &*self.inner_data.get() }; + let start = link.offset as usize; + let mut storage = MaybeUninit::>::uninit(); + // Copy through the raw pointer rather than materializing a + // `&mut [u8; 256]` over uninitialized storage: that reference + // asserts the whole array is initialized, which it is not, and is + // what Stacked/Tree Borrows objects to. Only `..len` is written, + // and only `..len` is ever read back (`ArchivedCopy::as_bytes`). + // + // SAFETY: `len <= SEQLOCK_STACK` is the branch condition above, so + // the destination has room; `validate_link` bounded + // `start..start + len` within the page; and the two regions cannot + // overlap because `storage` is a fresh local. + unsafe { + core::ptr::copy_nonoverlapping( + inner[start..start + len].as_ptr(), + storage.as_mut_ptr().cast::(), + len, + ); + } + if !self.cell_locks.still_stable(link, stamp) { + // A stripe covers many offsets on the page, so a single hot + // writer can invalidate readers of unrelated rows. Without a + // backoff here those readers re-copy as fast as they can and + // hold the line the writer needs; `load_stable` backs off only + // while a writer is *in* the critical section, which is not + // the case a rejected stamp reports. + CellLocks::wait(&mut spins); + continue; + } + return Ok(ArchivedCopy::Stack { + buf: storage, + len: len as u16, + }); + } + } + + /// Run `f` on the archived cell under seqlock, without memcpy. + /// + /// `f` must copy out what it needs and must not stash a reference into the + /// page. A concurrent writer can tear the bytes `f` reads; `f` must not + /// follow pointers or otherwise trap. Retry discards a torn result. + pub(crate) fn with_archived_seqlock(&self, link: Link, mut f: F) -> Result + where + Row: Archive + InlineArchived, + F: FnMut(&::Archived) -> T, + { + // Read in place, with no copy. `Row: InlineArchived` is what makes + // that sound: every archived field is a fixed-size scalar inline in + // the cell, so a concurrent writer can tear a value the closure reads + // but cannot hand it a pointer that refers anywhere else. The retry + // below discards anything computed from a torn read. + // + // Without that bound this would be undefined behaviour, not a wrong + // number: `f` receives `&Archived`, and an archived `String` or `Vec` + // field is a relative pointer, which the closure would dereference + // before `still_stable` ever runs. + self.validate_link(link)?; + // Retry-path backoff only: see the note in `copy_row_seqlock`. + let mut spins = 0; + loop { + let stamp = self.cell_locks.load_stable(link)?; + let archived = self.get_row_ref(link)?; + let result = f(archived); + if self.cell_locks.still_stable(link, stamp) { + return Ok(result); + } + CellLocks::wait(&mut spins); + } + } + /// Validates persisted bytes before deserializing them. /// /// The regular in-memory path only reads bytes written by WorkTable in the @@ -681,6 +1072,37 @@ mod tests { b: u64, } + /// A writer that begins AND completes between `load_stable` and + /// `still_stable` must not be invisible. + /// + /// `CellWriteGuard::drop` stores `0` and `load_stable` returns `0` for an + /// idle cell, so the stamp an uncontended reader captures is the same value + /// the cell returns to after any number of completed writes. Without a + /// counter that moves on release, `still_stable` cannot distinguish "no + /// write happened" from "a whole write happened", and the reader keeps a + /// buffer it memcpy'd out of the middle of that write. + #[test] + fn a_completed_write_between_snapshot_and_check_is_detected() { + let locks = super::CellLocks::default(); + let link = Link { + page_id: 1.into(), + offset: 7, + length: 16, + }; + + let stamp = locks.load_stable(link).expect("idle cell snapshots"); + + // The whole write happens inside the reader's window. + { + let _write = locks.write(link).expect("uncontended writer"); + } + + assert!( + !locks.still_stable(link, stamp), + "a completed write inside the read window must invalidate the snapshot" + ); + } + #[test] fn colliding_rows_keep_using_one_stable_stripe() { let locks = super::CellLocks::default(); @@ -1117,16 +1539,71 @@ mod tests { #[cfg(all(test, wt_loom))] mod cell_lock_models { use super::{CellLocks, Link}; + use alloc::boxed::Box; + use core::sync::atomic::Ordering; + use loom::sync::atomic::AtomicU32; use loom::{cell::UnsafeCell, sync::Arc, thread}; struct Protected { - locks: CellLocks, + locks: Box, value: UnsafeCell<(u64, u64)>, } + /// A lock table built straight into its heap allocation. + /// + /// `CellLocks::default()` returns the two 256-slot atomic arrays by value, + /// so the caller reserves a copy on its own stack. That is affordable on a + /// real thread and is not on a loom coroutine, whose stack is small and + /// whose atomics each carry tracking state: both models overflowed it + /// before reaching their first assertion. `CellLocks::initialize_at` + /// exists for exactly this and is what `Data` uses in production. + fn cell_locks() -> Box { + let mut locks = Box::::new_uninit(); + // SAFETY: `initialize_at` writes every field of `CellLocks`, so the + // allocation is fully initialized when it returns. + unsafe { + CellLocks::initialize_at(locks.as_mut_ptr()); + locks.assume_init() + } + } + // Every access to value below holds the same row's read or write guard. unsafe impl Sync for Protected {} + /// Cell contents for the seqlock models, as two independently published + /// words rather than a `loom::cell::UnsafeCell`. + /// + /// The seqlock reader copies bytes *without* excluding the writer, which is + /// the whole point of the protocol and exactly what loom's `UnsafeCell` + /// reports as a data race. Two atomics are the stand-in: the writer + /// publishes them separately so a snapshot between the two stores is torn, + /// which is the condition `still_stable` has to catch, and loom still + /// explores every interleaving of the four accesses. + /// + /// They are `Release`/`Acquire` rather than `Relaxed`, and the reader takes + /// the later-published half first, because otherwise the model reports its + /// own reordering as a seqlock failure. With relaxed accesses loom is + /// entitled to produce `(0, 1)` -- half 0 stale, half 1 fresh -- even though + /// the writer stores half 0 first, and that outcome says nothing about the + /// protocol. Ordered this way, seeing half 1 set and then half 0 clear is + /// impossible unless `still_stable` accepted a snapshot spanning the write, + /// which is exactly the claim under test. The real path copies plain bytes + /// and has no such ordering to lean on; the model needs it only to keep the + /// assertion about the seqlock rather than about loom. + struct SeqlockProtected { + locks: Box, + halves: (AtomicU32, AtomicU32), + } + + impl SeqlockProtected { + fn new() -> Self { + Self { + locks: cell_locks(), + halves: (AtomicU32::new(0), AtomicU32::new(0)), + } + } + } + #[test] fn colliding_offsets_cannot_split_readers_from_a_writer() { let mut model = loom::model::Builder::new(); @@ -1134,7 +1611,7 @@ mod cell_lock_models { model.max_branches = 10_000; model.check(|| { let protected = Arc::new(Protected { - locks: CellLocks::default(), + locks: cell_locks(), value: UnsafeCell::new((0, 0)), }); let first = Link { @@ -1178,7 +1655,7 @@ mod cell_lock_models { model.max_branches = 10_000; model.check(|| { let protected = Arc::new(Protected { - locks: CellLocks::default(), + locks: cell_locks(), value: UnsafeCell::new((0, 0)), }); let link = Link { @@ -1216,4 +1693,146 @@ mod cell_lock_models { }); }); } + + /// The seqlock snapshot protocol, which `select` takes for every row that + /// fits the stack buffer and which neither model above drives. + /// + /// The reader does not CAS, so it can copy bytes while a writer is midway + /// through publishing them. What makes that sound is the pair + /// `load_stable` / `still_stable`: a snapshot is accepted only if no writer + /// was in the critical section when it started and the version counter has + /// not moved since. This asserts the property the copy path relies on -- + /// **an accepted snapshot is never torn** -- rather than that a retry + /// happens, because a spurious retry is allowed and a missed one is not. + #[test] + fn an_accepted_seqlock_snapshot_is_never_torn() { + let mut model = loom::model::Builder::new(); + model.preemption_bound = Some(2); + model.max_branches = 10_000; + model.check(|| { + let protected = Arc::new(SeqlockProtected::new()); + let link = Link { + page_id: 1.into(), + offset: 64, + length: 16, + }; + + let writer = { + let protected = protected.clone(); + thread::spawn(move || { + let _guard = protected.locks.write(link).unwrap(); + // The two halves are published separately, so any snapshot + // taken between them is torn and must be rejected. + protected.halves.0.store(1, Ordering::Release); + thread::yield_now(); + protected.halves.1.store(1, Ordering::Release); + }) + }; + + // One attempt, not a retry loop: a loop would let the model pass by + // eventually succeeding, where the claim under test is about what a + // single accepted snapshot may contain. + if let Ok(stamp) = protected.locks.load_stable(link) { + // Read the LATER-published half first. With the writer + // publishing half 0 then half 1, observing half 1 set and then + // half 0 clear is impossible under release/acquire, so a torn + // pair here is the seqlock failing rather than the model's own + // atomics being reordered. + let second = protected.halves.1.load(Ordering::Acquire); + let first = protected.halves.0.load(Ordering::Acquire); + let snapshot = (first, second); + if protected.locks.still_stable(link, stamp) { + assert!( + snapshot == (0, 0) || snapshot == (1, 1), + "a snapshot accepted by still_stable was torn: {snapshot:?}" + ); + } + } + + writer.join().unwrap(); + }); + } + + /// A completed write must invalidate a snapshot that started before it, + /// even though the state word is back to `0` by the time the reader + /// re-reads it. + /// + /// This is what the version counter is for, and why the writer's release + /// bumps it *before* clearing the writer bit. Checking the state word + /// alone would accept this interleaving. + #[test] + fn a_write_completed_during_the_snapshot_is_rejected() { + let mut model = loom::model::Builder::new(); + model.preemption_bound = Some(2); + model.max_branches = 10_000; + model.check(|| { + let protected = Arc::new(SeqlockProtected::new()); + let link = Link { + page_id: 1.into(), + offset: 64, + length: 16, + }; + + let stamp = protected.locks.load_stable(link).unwrap(); + + let writer = { + let protected = protected.clone(); + thread::spawn(move || { + let _guard = protected.locks.write(link).unwrap(); + protected.halves.0.store(1, Ordering::Relaxed); + protected.halves.1.store(1, Ordering::Relaxed); + }) + }; + + // Checked concurrently, so the model gets to place this read + // before, during and after the writer's critical section. A read + // that lands before it may legitimately accept; the assertion after + // the join is what must hold in every interleaving. + let _ = protected.locks.still_stable(link, stamp); + + writer.join().unwrap(); + + // The state word is back to `0` here, so a check that read only + // that word would accept a snapshot taken before a write that has + // since completed. The version counter is what rejects it, and the + // writer's release bumps that counter before clearing the bit + // precisely so there is no instant where both look idle at the + // pre-write version. + assert!( + !protected.locks.still_stable(link, stamp), + "a completed write must invalidate every snapshot taken before it" + ); + }); + } + + /// The reader's exemption for its own in-flight write on a *different* row + /// of the same stripe must hold under the model, not just in the + /// single-threaded test. + /// + /// `load_stable` admits this case and `still_stable` must agree with it: + /// rejecting it spins forever, because only this thread can clear the + /// writer bit it is waiting on. + #[test] + fn a_self_write_on_another_row_does_not_reject_the_readers_snapshot() { + let mut model = loom::model::Builder::new(); + model.preemption_bound = Some(2); + model.max_branches = 10_000; + model.check(|| { + let locks = cell_locks(); + let held = Link { + page_id: 1.into(), + offset: 7, + length: 16, + }; + let other = Link { offset: 14, ..held }; + assert_eq!(CellLocks::start(held), CellLocks::start(other)); + + let _write = locks.write(held).unwrap(); + let stamp = locks.load_stable(other).unwrap(); + assert!( + locks.still_stable(other, stamp), + "a thread reading another row under its own write must make progress" + ); + }); + } } diff --git a/src/in_memory/mod.rs b/src/in_memory/mod.rs index 227cd8c7..123f32f5 100644 --- a/src/in_memory/mod.rs +++ b/src/in_memory/mod.rs @@ -3,7 +3,8 @@ mod empty_link_registry; mod pages; mod row; +pub(crate) use data::ArchivedCopy; pub use data::{DATA_INNER_LENGTH, Data, ExecutionError as DataExecutionError}; pub use empty_link_registry::EmptyLinkRegistry; -pub use pages::{DataPages, ExecutionError as PagesExecutionError, ReadGuard as DataPagesReadGuard}; -pub use row::{ArchivedRowWrapper, PublicationSafe, Query, RowWrapper, StorableRow}; +pub use pages::{DataPages, ExecutionError as PagesExecutionError, ReadGuard as DataPagesReadGuard, SelectRef}; +pub use row::{ArchivedRowWrapper, InlineArchived, PublicationSafe, Query, RowWrapper, StorableRow}; diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index ff21d4f3..aeea542a 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -5,6 +5,7 @@ use alloc::{boxed::Box, vec::Vec}; use arc_swap::ArcSwap; use core::fmt::Debug; use core::marker::PhantomData; +use core::ops::Deref; use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize}; use core::sync::atomic::{AtomicU64, Ordering}; use data_bucket::page::PageId; @@ -22,12 +23,13 @@ use rkyv::{ util::AlignedVec, }; +use crate::in_memory::InlineArchived; use crate::in_memory::empty_link_registry::EmptyLinkRegistry; use crate::prelude::ArchivedRowWrapper; use crate::util::epoch::EpochDomain; use crate::{ in_memory::{ - DATA_INNER_LENGTH, Data, DataExecutionError, + ArchivedCopy, DATA_INNER_LENGTH, Data, DataExecutionError, row::{RowWrapper, StorableRow}, }, prelude::Link, @@ -212,7 +214,7 @@ impl PageDirectoryChunk { } } -/// Non-owning, stable page pointers for the first 4,096 pages (64 MiB at the +/// Non-owning, stable page pointers for the first 65,536 pages (1 GiB at the /// default page size). `DataPages::pages` owns every allocation; this directory /// exists solely to avoid shared ArcSwap snapshot accounting on point access. #[derive(Debug)] @@ -225,6 +227,33 @@ struct PageDirectory { } impl PageDirectory { + /// Breaks the build when a field is added to [`PageDirectory`] without + /// being added to [`Self::initialize_at`]. See + /// [`DataPages::every_field_is_initialized`] for why this exists. + #[cfg(test)] + #[expect(dead_code, reason = "compiled for its exhaustiveness check, never called")] + fn every_field_is_initialized(self) { + let Self { roots: _, chunks: _ } = self; + } + + unsafe fn initialize_at(target: *mut Self, pages: &[Arc]) { + unsafe { + let roots = core::ptr::addr_of_mut!((*target).roots).cast::>>(); + for index in 0..PAGE_DIRECTORY_ROOTS { + roots.add(index).write(AtomicPtr::new(core::ptr::null_mut())); + } + core::ptr::addr_of_mut!((*target).chunks).write(Mutex::new(Vec::new())); + } + + // SAFETY: both fields used by `publish` are initialized above, the + // allocation is stable, and it is not observable until its owner is + // returned from construction. + let directory = unsafe { &*target }; + for (index, page) in pages.iter().enumerate() { + directory.publish(index, page); + } + } + fn new(pages: &[Arc]) -> Self { let directory = Self { roots: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())), @@ -298,6 +327,47 @@ pub struct ReadGuard<'a> { marker: PhantomData<&'a ()>, } +/// Point-read that does not deserialize. +/// +/// Holds the epoch pin and a seqlock copy of the archived wrapper. Deref is +/// the archived inner row, so a caller can read fields without building an +/// owned row. Owned [`DataPages::select`] stays for callers that need a `Row`. +/// +/// Not `Send`: the pin belongs to the acquiring thread. +pub struct SelectRef<'a, Row: StorableRow> { + _pin: ReadGuard<'a>, + copy: ArchivedCopy, + _ty: PhantomData, +} + +impl<'a, Row: StorableRow> SelectRef<'a, Row> { + pub(crate) fn new(pin: ReadGuard<'a>, copy: ArchivedCopy) -> Self { + Self { + _pin: pin, + copy, + _ty: PhantomData, + } + } + + #[inline] + fn archived(&self) -> &<::WrappedRow as rkyv::Archive>::Archived { + unsafe { + rkyv::access_unchecked::<<::WrappedRow as rkyv::Archive>::Archived>( + self.copy.as_bytes(), + ) + } + } +} + +impl Deref for SelectRef<'_, Row> { + type Target = <<::WrappedRow as rkyv::Archive>::Archived as ArchivedRowWrapper>::Inner; + + #[inline] + fn deref(&self) -> &Self::Target { + self.archived().inner() + } +} + /// One unit of retired state waiting out its grace period. Reclamation is /// *recycling*, not just freeing: links return to `empty_links` and pages /// return to `empty_pages`. @@ -426,6 +496,87 @@ where Row: StorableRow, ::WrappedRow: RowWrapper, { + /// Breaks the build when a field is added to [`DataPages`] without being + /// added to [`Self::initialize_arc_at`]. + /// + /// That initializer writes each field through `addr_of_mut!` and then + /// `assume_init`s the allocation, so the compiler cannot check it for + /// exhaustiveness the way it checks a struct literal: adding a field and + /// updating only the safe `new` beside it compiles cleanly and hands out an + /// `Arc` with one field uninitialized. Destructuring without `..` is the + /// check the initializer itself cannot have. Keep the binding list here and + /// the writes there in step. + #[cfg(test)] + #[expect(dead_code, reason = "compiled for its exhaustiveness check, never called")] + fn every_field_is_initialized(self) { + let Self { + epoch: _, + retired: _, + reclaimable: _, + pending_retirements: _, + queued_page_retirements: _, + pages: _, + page_directory: _, + pages_write: _, + empty_links: _, + empty_pages: _, + row_count: _, + last_page_id: _, + current_page_id: _, + } = self; + } + + unsafe fn initialize_arc_at( + target: *mut Self, + mut pages: Vec::WrappedRow, DATA_LENGTH>>>, + ) { + if pages.is_empty() { + pages.push(Data::new_arc(1.into())); + } + let last_page_id = pages.len() as u32; + + unsafe { + core::ptr::addr_of_mut!((*target).epoch).write(EpochDomain::new()); + core::ptr::addr_of_mut!((*target).retired).write(Mutex::new(VecDeque::new())); + core::ptr::addr_of_mut!((*target).reclaimable).write(Arc::new(AtomicUsize::new(0))); + core::ptr::addr_of_mut!((*target).pending_retirements).write(AtomicUsize::new(0)); + core::ptr::addr_of_mut!((*target).queued_page_retirements).write(AtomicUsize::new(0)); + PageDirectory::initialize_at(core::ptr::addr_of_mut!((*target).page_directory), &pages); + core::ptr::addr_of_mut!((*target).pages).write(PageList::from_pages(pages)); + core::ptr::addr_of_mut!((*target).pages_write).write(Mutex::new(())); + core::ptr::addr_of_mut!((*target).empty_links).write(EmptyLinkRegistry::::default()); + core::ptr::addr_of_mut!((*target).empty_pages).write(Default::default()); + core::ptr::addr_of_mut!((*target).row_count).write(AtomicU64::new(0)); + core::ptr::addr_of_mut!((*target).last_page_id).write(AtomicU32::new(last_page_id)); + core::ptr::addr_of_mut!((*target).current_page_id).write(AtomicU32::new(last_page_id)); + } + } + + /// Creates the page collection directly in its `Arc` allocation. + /// + /// The fixed page directory deliberately keeps 1,024 roots inline for a + /// pointer-only read path. Constructing `Self` on the stack before moving + /// it into an Arc needlessly reserves that full array in nested generated + /// table-load futures. + pub fn new_arc() -> Arc { + Self::from_data_arc(Vec::new()) + } + + /// Restores a page collection directly in its `Arc` allocation. + pub fn from_data_arc(pages: Vec::WrappedRow, DATA_LENGTH>>>) -> Arc { + let mut collection = Arc::::new_uninit(); + let target = Arc::get_mut(&mut collection) + .expect("a newly allocated Arc is uniquely owned") + .as_mut_ptr(); + + // SAFETY: the helper initializes every field exactly once and the Arc + // remains uniquely owned until initialization is complete. + unsafe { + Self::initialize_arc_at(target, pages); + collection.assume_init() + } + } + fn page_ref( &self, page_id: PageId, @@ -471,12 +622,60 @@ where Portable + Deserialize<::WrappedRow, HighDeserializer>, { let page = self.page_ref(link.page_id)?; - let _cell_guard = page.read_cell(link).map_err(ExecutionError::DataPageError)?; - let wrapped = page.get_row(link).map_err(ExecutionError::DataPageError)?; + let wrapped = page.get_row_seqlock(link).map_err(ExecutionError::DataPageError)?; let flags = Self::publication_flags(&wrapped); Ok((wrapped.get_inner(), flags)) } + /// Seqlock-copy a live (non-ghosted, non-deleted) archived row. + pub(crate) fn copy_non_ghosted(&self, link: Link) -> Result + where + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + { + let page = self.page_ref(link.page_id)?; + let copy = page.copy_row_seqlock(link).map_err(ExecutionError::DataPageError)?; + let archived = unsafe { + rkyv::access_unchecked::<<::WrappedRow as Archive>::Archived>(copy.as_bytes()) + }; + if archived.is_ghosted() { + return Err(ExecutionError::Ghosted); + } + if archived.is_deleted() { + return Err(ExecutionError::Deleted); + } + Ok(copy) + } + + /// Apply `f` to the archived inner row under seqlock, without memcpy. + pub(crate) fn with_non_ghosted(&self, link: Link, mut f: F) -> Result + where + ::WrappedRow: InlineArchived, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + F: FnMut(&<<::WrappedRow as Archive>::Archived as ArchivedRowWrapper>::Inner) -> T, + { + let page = self.page_ref(link.page_id)?; + page.with_archived_seqlock(link, |wrapped| { + if wrapped.is_ghosted() { + Err(ExecutionError::Ghosted) + } else if wrapped.is_deleted() { + Err(ExecutionError::Deleted) + } else { + Ok(f(wrapped.inner())) + } + }) + .map_err(ExecutionError::DataPageError)? + } + + /// Point-read that yields a pin-guard plus archived inner row. + pub fn select_ref(&self, link: Link) -> Result, ExecutionError> + where + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + { + let pin = self.read_guard(); + let copy = self.copy_non_ghosted(link)?; + Ok(SelectRef::new(pin, copy)) + } + pub fn read_guard(&self) -> ReadGuard<'_> { ReadGuard { _guard: self.epoch.pin(), @@ -700,7 +899,7 @@ where } pub fn new() -> Self { - let page = Arc::new(Data::new(1.into())); + let page = Data::new_arc(1.into()); let pages = vec![page]; Self { epoch: EpochDomain::new(), @@ -877,7 +1076,7 @@ where let _write = self.pages_write.lock(); if tried_page == page_id_mapper(self.current_page_id.load(Ordering::Acquire) as usize) { let index = self.last_page_id.fetch_add(1, Ordering::AcqRel) + 1; - let page = Arc::new(Data::new(index.into())); + let page = Data::new_arc(index.into()); self.pages.push(page.clone()); self.publish_page(&page); self.current_page_id.store(index, Ordering::Release); @@ -910,7 +1109,7 @@ where let _write = self.pages_write.lock(); let index = self.last_page_id.fetch_add(1, Ordering::AcqRel) + 1; - let page = Arc::new(Data::new(index.into())); + let page = Data::new_arc(index.into()); self.pages.push(page.clone()); self.publish_page(&page); @@ -1065,13 +1264,12 @@ where /// the current slot (so it fits exactly). /// /// # Persistence - /// This path emits **no** persistence CDC. It is only sound for tables that - /// are not persisted (or on a persistence sink that reconstructs state from - /// the page image on reload). Do NOT route a persisted-table update through - /// this method: the row would change in memory and republish but no change - /// event would reach disk, silently losing durability until reload. The - /// generated persisted update path deliberately keeps the reinsert path for - /// this reason. + /// This storage primitive emits **no** persistence CDC. A persisted caller + /// must read the replacement bytes from this slot and enqueue its durable + /// update before returning success. Omitting that operation changes memory + /// without changing disk, so a cold reload restores the old row. Generated + /// persisted updates follow that contract; memory-only callers need no + /// additional operation. /// /// Serialization and the exact-length check finish before any page byte is /// changed. The exact cell guard excludes readers of this cell during the @@ -1412,6 +1610,18 @@ where Ok(self) } + pub fn with_empty_links_arc(mut self: Arc, links: Vec) -> Result, ExecutionError> { + let registry = EmptyLinkRegistry::default(); + for link in links { + self.page_ref(link.page_id)?.reserve_restored_range(link)?; + registry.push(link); + } + Arc::get_mut(&mut self) + .expect("restored page collection is uniquely owned") + .empty_links = registry; + Ok(self) + } + pub fn current_page_id(&self) -> PageId { self.current_page_id.load(Ordering::Acquire).into() } @@ -1543,6 +1753,16 @@ mod tests { where T: Archive, { + type Inner = T::Archived; + + fn inner(&self) -> &Self::Inner { + &self.inner + } + + fn is_ghosted(&self) -> bool { + self.is_ghosted + } + fn unghost(&mut self) { self.is_ghosted = false } @@ -1595,6 +1815,20 @@ mod tests { assert_eq!(res, row) } + #[test] + fn select_ref_after_unghost() { + let pages = DataPages::::new(); + let row = TestRow { a: 10, b: 20 }; + let link = pages.insert(row).unwrap(); + assert!(pages.select_ref(link).is_err()); + unsafe { + pages.with_mut_ref(link, |archived| archived.unghost()).unwrap(); + } + let view = pages.select_ref(link).unwrap(); + assert_eq!(view.a, 10); + assert_eq!(view.b, 20); + } + #[test] fn select_non_ghosted() { let pages = DataPages::::new(); diff --git a/src/in_memory/row.rs b/src/in_memory/row.rs index 38066fff..d586c0e3 100644 --- a/src/in_memory/row.rs +++ b/src/in_memory/row.rs @@ -5,6 +5,30 @@ pub trait PublicationSafe: Send + Sync + 'static {} impl PublicationSafe for T {} +/// A row whose archived form holds no relative pointers. +/// +/// Every archived field is a fixed-size scalar sitting inline in the cell, so +/// reading one while a writer mutates it can tear a value but cannot produce a +/// pointer that refers anywhere else. That is what makes the zero-copy +/// [`select_with`] path sound: the closure may observe a torn number, and the +/// seqlock retry discards whatever it computed from one. +/// +/// A row with a `String` or `Vec` column has archived relative pointers, and a +/// torn pointer dereferenced inside the closure is undefined behaviour rather +/// than a wrong number. Those rows deliberately do not implement this trait, so +/// they fail to compile against the zero-copy API instead of silently copying, +/// and must use the owned `select`. +/// +/// # Safety +/// +/// Implementors must contain no archived relative pointers. The `worktable!` +/// macro implements this only when every column is a known scalar shape; it is +/// never implemented for an opaque user type, because the macro cannot inspect +/// that type's `Archive::Archived` layout. +/// +/// [`select_with`]: crate::WorkTable::select_with +pub unsafe trait InlineArchived {} + /// Common trait for the `Row`s that can be stored on the [`Data`] page. /// /// [`Data`]: crate::in_memory::data::Data @@ -21,6 +45,10 @@ pub trait RowWrapper { } pub trait ArchivedRowWrapper { + type Inner: ?Sized; + + fn inner(&self) -> &Self::Inner; + fn is_ghosted(&self) -> bool; fn unghost(&mut self); fn set_in_vacuum_process(&mut self); fn delete(&mut self); diff --git a/src/lib.rs b/src/lib.rs index 828d6862..0051ba43 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ mod columnar; pub mod fsx; pub mod in_memory; mod index; +pub mod linear_table; pub mod lock; mod mem_stat; #[cfg(feature = "std")] @@ -47,6 +48,7 @@ pub use columnar::{ ColumnSlotId64, ColumnarColumn, ColumnarRowRef, next_columnar_incarnation, }; pub use index::*; +pub use linear_table::{InsertError as LinearInsertError, LinearTable, VecTable}; #[cfg(feature = "std")] pub use persistence::{ LoadMode, PersistedWorkTable, PersistenceConfig, PersistenceLoadError, UnloadFailure, UnloadReport, @@ -175,10 +177,20 @@ pub mod prelude { pub use hashbrown::{HashMap, HashSet}; pub use crate::atomic_key_table::AtomicKeyTable; - pub use crate::in_memory::{ArchivedRowWrapper, Data, DataPages, Query, RowWrapper, StorableRow}; + pub use crate::in_memory::{ + ArchivedRowWrapper, Data, DataPages, InlineArchived, Query, RowWrapper, SelectRef, StorableRow, + }; pub use crate::lock::FullRowLock; + pub use crate::lock::LockMap; pub use crate::lock::{Lock, RowLock}; - pub use crate::lock::{LockAcquirer, LockGuard, LockMap, PendingLock}; + // `LockAcquirer`, `LockGuard` and `PendingLock` borrow their `LockMap` as a + // raw pointer rather than an owning `Arc` (see the "Borrowed guards" note + // on `LockMap`), so holding one past the map's last `Arc` is undefined + // behaviour. They are not a user-facing API: the only callers are the + // operation bodies this crate generates, which is why they are re-exported + // here for that generated code and `#[doc(hidden)]` at their definitions. + #[doc(hidden)] + pub use crate::lock::{LockAcquirer, LockGuard, PendingLock}; pub use crate::mem_stat::MemStat; pub use crate::partition::{DenseError, DenseRows, MAX_PARTITIONS, PartRef, PartitionError, PartitionSet}; pub use crate::persistence::{AcknowledgeOperation, DeleteOperation, InsertOperation, Operation, OperationId}; @@ -218,6 +230,7 @@ pub mod prelude { }; #[cfg(feature = "s3-support")] pub use crate::{DatabaseS3DiskConfig, DatabaseS3PersistenceEngine, S3Database}; + pub use crate::{LinearInsertError, LinearTable, VecTable}; /// The upstream IndexSet backend, when the `vanilla-index` feature selects it. #[cfg(feature = "vanilla-index")] pub use crate::{UpstreamIndexMap, UpstreamIndexPair}; diff --git a/src/linear_table.rs b/src/linear_table.rs new file mode 100644 index 00000000..074abe9a --- /dev/null +++ b/src/linear_table.rs @@ -0,0 +1,215 @@ +//! Contiguous rows for consumers that build once and search a frozen slice. +//! +//! [`LinearTable`] is deliberately smaller than a generated `vec: true` +//! table. It preserves insertion order and duplicate keys, and [`rows`](LinearTable::rows) +//! exposes the exact contiguous `[(K, V)]` storage. A caller can sort while +//! building, freeze the table behind an immutable reference, and run an +//! allocation-free binary search over that slice. +//! +//! The type is the retired `worktable-vec::LinearTable` moved into WorkTable. +//! Keeping it here lets crash handlers and other `no_std + alloc` consumers +//! use the stable row shape without taking a second table crate or routing a +//! lookup through WorkTable's paged indexes. + +use alloc::vec::Vec; + +use crate::vec_hydrate::{Codec, LoadError, RowTooLarge, from_pages, to_pages}; + +/// Why a uniqueness-checking insert was refused. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum InsertError { + /// The key already belongs to a row. + DuplicateKey(K), + /// Reserved for fixed-capacity table implementations that cannot grow. + /// + /// [`LinearTable`] uses `Vec` growth and does not emit this variant. It is + /// retained for source compatibility with the retired standalone type. + OutOfMemory(K), +} + +/// Ordered, contiguous rows with a linear point lookup. +/// +/// [`push`](Self::push) preserves duplicates. [`insert`](Self::insert) is the +/// explicit uniqueness-checking alternative. The type has the same layout as +/// its row vector, so exposing the immutable slice adds no index, lock, or +/// allocation to a read path. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[repr(transparent)] +pub struct LinearTable { + rows: Vec<(K, V)>, +} + +/// A name for [`LinearTable`] that emphasizes its exact Vec-backed shape. +pub type VecTable = LinearTable; + +impl LinearTable { + /// An empty table. + #[must_use] + pub const fn new() -> Self { + Self { rows: Vec::new() } + } + + /// An empty table with room for at least `capacity` rows. + #[must_use] + pub fn with_capacity(capacity: usize) -> Self { + Self { + rows: Vec::with_capacity(capacity), + } + } + + /// Append a row without rejecting a duplicate key. + /// + /// Returns the row's insertion-order position. + pub fn push(&mut self, key: K, value: V) -> usize { + let row = self.rows.len(); + self.rows.push((key, value)); + row + } + + /// A row by insertion-order position. + #[must_use] + pub fn get_row(&self, row: usize) -> Option<&(K, V)> { + self.rows.get(row) + } + + /// A mutable row by insertion-order position. + pub fn get_row_mut(&mut self, row: usize) -> Option<&mut (K, V)> { + self.rows.get_mut(row) + } + + /// Every row as one contiguous slice. + #[must_use] + pub fn as_slice(&self) -> &[(K, V)] { + self.rows.as_slice() + } + + /// Every row as one mutable contiguous slice. + pub fn as_mut_slice(&mut self) -> &mut [(K, V)] { + self.rows.as_mut_slice() + } + + /// Every row in insertion order. + pub fn iter(&self) -> core::slice::Iter<'_, (K, V)> { + self.rows.iter() + } + + /// Every row mutably, in insertion order. + pub fn iter_mut(&mut self) -> core::slice::IterMut<'_, (K, V)> { + self.rows.iter_mut() + } + + /// Reserve room for at least `additional` more rows. + pub fn reserve(&mut self, additional: usize) { + self.rows.reserve(additional); + } + + /// The current row capacity. + #[must_use] + pub fn capacity(&self) -> usize { + self.rows.capacity() + } + + /// Consume the table and return its rows without copying. + #[must_use] + pub fn into_rows(self) -> Vec<(K, V)> { + self.rows + } + + /// Every row as one contiguous slice. + /// + /// This is the frozen query seam: a caller that has finished building the + /// table can keep only `&LinearTable` and binary-search this slice without + /// allocation or internal mutation. + #[must_use] + pub fn rows(&self) -> &[(K, V)] { + &self.rows + } + + /// Number of rows, including duplicate keys appended with [`push`](Self::push). + #[must_use] + pub fn len(&self) -> usize { + self.rows.len() + } + + /// Whether the table has no rows. + #[must_use] + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } +} + +impl LinearTable +where + K: Eq, +{ + /// Append a row only when its key is absent. + pub fn insert(&mut self, key: K, value: V) -> Result> { + if self.rows.iter().any(|(present, _)| present == &key) { + return Err(InsertError::DuplicateKey(key)); + } + Ok(self.push(key, value)) + } + + /// The first value with this key. + #[inline] + #[must_use] + pub fn select(&self, key: &K) -> Option<&V> { + self.rows + .iter() + .find(|(present, _)| present == key) + .map(|(_, value)| value) + } +} + +impl LinearTable +where + Vec<(K, V)>: Codec, + (K, V): Clone, +{ + /// Encode every row with WorkTable's self-contained Vec page codec. + /// + /// # Errors + /// + /// Refuses a row whose archive does not fit one page body. + pub fn unload(&self) -> Result, RowTooLarge> { + to_pages(&self.rows) + } + + /// Encode rows from `first` onward as an independent append segment. + /// + /// Independent segments restart page numbering at zero. The codec accepts + /// a terminal page followed by such a segment, so callers can concatenate + /// the returned bytes without rewriting the preceding pages. Updates and + /// deletes still require a full snapshot. + /// + /// # Errors + /// + /// Refuses a row whose archive does not fit one page body. + pub fn unload_appending(&self, first: usize) -> Result, RowTooLarge> { + to_pages(&self.rows[first.min(self.rows.len())..]) + } + + /// Rebuild the contiguous rows from WorkTable Vec pages. + /// + /// # Errors + /// + /// Refuses incomplete, corrupt, foreign-version, foreign-schema, or + /// undecodable pages with the codec's page-specific [`LoadError`]. + pub fn load(bytes: &[u8]) -> Result { + Ok(Self { + rows: from_pages(bytes)?, + }) + } +} + +impl From> for LinearTable { + fn from(rows: Vec<(K, V)>) -> Self { + Self { rows } + } +} + +impl AsRef<[(K, V)]> for LinearTable { + fn as_ref(&self) -> &[(K, V)] { + self.as_slice() + } +} diff --git a/src/lock/map.rs b/src/lock/map.rs index 065a7591..fa555918 100644 --- a/src/lock/map.rs +++ b/src/lock/map.rs @@ -1,3 +1,4 @@ +use alloc::boxed::Box; use alloc::sync::Arc; use alloc::vec::Vec; use core::fmt::Debug; @@ -11,12 +12,52 @@ use parking_lot::RwLock; use crate::lock::RowLock; +/// Gates for the synchronous mutation phase. +/// +/// Sixty-four is enough: raising it to 1024 on top of the shard count below +/// moves `paged_in_place` at eight workers from 2.81x to 2.92x, which does not +/// pay for sixteen times the gates. The gate is entered and left in tens of +/// nanoseconds, so collisions on it stay rare at this count. const MUTATION_STRIPE_COUNT: usize = 64; +/// Independent `RwLock` shards of the row-lock map. +/// +/// Far more than the stripe count, because a locked operation costs the map +/// two exclusive shard acquisitions where it costs the gate one short critical +/// section: a disjoint-key writer inserts its entry on the way in and removes +/// it on the way out, and a shard collision parks the loser in the kernel. +/// Measured on `paged_in_place`, eight workers over 16384 keys: +/// +/// | shards | w8 vs w1 | +/// |---:|---:| +/// | 64 | 1.70x | +/// | 256 | 2.39x | +/// | 512 | 2.63x | +/// | 1024 | **2.81x** | +/// | 2048 | 3.07x | +/// +/// 2048 still gains, but it doubles a per-table array for 9%. A shard holds no +/// buckets until a key lands in it, so what this count costs is the array, not +/// the maps. +/// +/// One table-wide `RwLock` serialized every acquire and drop before +/// any of this: 8 disjoint writers then burned ~6 cores for 1.27x replace. +const MAP_SHARD_COUNT: usize = 1024; +/// One mutation stripe: a ticket lock and the lock-label counter for the same +/// set of keys. +/// +/// Aligned to a whole coherency granule. Unpadded, the three fields are 20 +/// bytes, so eight stripes shared one line: a writer taking a ticket for its +/// own stripe invalidated the line seven unrelated stripes were spinning on, +/// and the `serving` spin re-read it every time. Sixty-four stripes behaved +/// like eight. The label counter lives here rather than in its own array so a +/// locked operation touches one line for both, not two. #[derive(Debug, Default)] +#[repr(align(128))] struct MutationStripe { next_ticket: AtomicU64, serving: AtomicU64, + next_label: AtomicU16, } /// Synchronous, task-safe gate for one primary-key mutation stripe. @@ -24,12 +65,33 @@ struct MutationStripe { /// Generated async row locks and synchronous inserts share these gates so a /// synchronous API entry point cannot interleave its multi-structure /// publication with an update or delete of the same key. +#[doc(hidden)] #[derive(Debug)] pub struct MutationGuard { - stripes: Arc<[MutationStripe; MUTATION_STRIPE_COUNT]>, + /// Borrowed, not an `Arc` clone. + /// + /// Cloning the map's stripe array put an atomic increment and a matching + /// decrement on one refcount word into every mutation. That word is shared + /// by every worker on the table and is independent of the key, so striping + /// cannot dilute it and a larger key space does not either: it is the same + /// defect as the `Arc` clones removed from `LockAcquirer` and + /// `PendingLock`, in the one place on the path that still had it. + /// + /// # Safety + /// + /// A guard is a local of the operation that took it, and that operation + /// reached this map through the table's own `Arc`, which it holds + /// for the whole call. The array therefore outlives every guard taken from + /// it. + stripes: *const [MutationStripe; MUTATION_STRIPE_COUNT], stripe: usize, } +// SAFETY: the pointed-to array is `Sync` and outlives the guard (see the field +// note), so a guard is no less safe to move or share than a `&[MutationStripe]`. +unsafe impl Send for MutationGuard {} +unsafe impl Sync for MutationGuard {} + /// Operation-wide activity signal for a chunked bulk mutation. /// /// It does not hold a row or stripe lock. Its only job is to keep background @@ -41,10 +103,20 @@ pub struct BulkMutationGuard { active: Arc, } +/// One shard of the row-lock map. +type LockShard = RwLock>>; + #[derive(Debug)] struct LockEntry { lock: Arc>, - acquirers: Arc, + /// Callers that may still register an operation against this entry. + /// + /// Inline, not an `Arc`. An acquirer used to clone it so it could + /// decrement without touching the map, but its drop then took the shard's + /// write lock anyway, to re-look the entry up for the removal check. The + /// clone bought nothing and cost an allocation and a free on every + /// operation; the decrement now happens under that same write guard. + acquirers: AtomicUsize, } /// A tracked reference to one row-lock entry while an operation registers. @@ -52,6 +124,7 @@ struct LockEntry { /// Dropping this handle, including through async task cancellation, retries /// map cleanup after releasing its lock reference. Clones remain tracked so an /// entry cannot be removed while any caller may still register against it. +#[doc(hidden)] #[derive(Debug)] pub struct LockAcquirer where @@ -59,22 +132,54 @@ where PrimaryKey: Hash + Eq + Debug + Clone, { lock: Option>>, - acquirers: Arc, - lock_map: Arc>, + /// Borrowed, not an `Arc` clone. + /// + /// Cloning the map's `Arc` here put an atomic increment and a matching + /// decrement on **one** refcount word into every operation, and that word + /// is shared by every worker on the table. It is independent of the key, + /// so sharding the map cannot help and a larger key space does not dilute + /// it: measured, `paged_in_place` scales the same at 1k rows and at 262k. + /// Three such clones per operation reproduce the whole negative slope in a + /// twenty-line program with no WorkTable in it (92M ops/s at one worker, + /// 5.4M at eight). + /// + /// # Safety + /// + /// The acquirer is a local of the operation that took it, and that + /// operation reached this map through the table's own + /// `Arc`, which it holds for the whole call. The map therefore + /// outlives every acquirer taken from it. + lock_map: *const LockMap, primary_key: PrimaryKey, } +// SAFETY: `LockMap` is `Sync`, and the pointer is only ever dereferenced while +// the owning `Arc` is alive (see the field note), so an acquirer is no less +// safe to move or share than a `&LockMap` would be. +unsafe impl Send for LockAcquirer +where + LockType: RowLock + Send + Sync, + PrimaryKey: Hash + Eq + Debug + Clone + Send, +{ +} +unsafe impl Sync for LockAcquirer +where + LockType: RowLock + Send + Sync, + PrimaryKey: Hash + Eq + Debug + Clone + Sync, +{ +} + impl Clone for LockAcquirer where LockType: RowLock, PrimaryKey: Hash + Eq + Debug + Clone, { fn clone(&self) -> Self { - self.acquirers.fetch_add(1, Ordering::AcqRel); + // SAFETY: see the field note; the map outlives this acquirer. + unsafe { (*self.lock_map).retain_acquirer(&self.primary_key) }; Self { lock: self.lock.clone(), - acquirers: self.acquirers.clone(), - lock_map: self.lock_map.clone(), + lock_map: self.lock_map, primary_key: self.primary_key.clone(), } } @@ -98,15 +203,17 @@ where PrimaryKey: Hash + Eq + Debug + Clone, { fn drop(&mut self) { - self.acquirers.fetch_sub(1, Ordering::AcqRel); drop(self.lock.take()); - self.lock_map.remove_with_lock_check(&self.primary_key); + // SAFETY: see the field note on `lock_map`; the owning map outlives + // this acquirer. + unsafe { (*self.lock_map).release_acquirer(&self.primary_key) }; } } impl Drop for MutationGuard { fn drop(&mut self) { - self.stripes[self.stripe].serving.fetch_add(1, Ordering::Release); + // SAFETY: see the field note; the map outlives this guard. + unsafe { (*self.stripes)[self.stripe].serving.fetch_add(1, Ordering::Release) }; } } @@ -120,16 +227,73 @@ impl Drop for BulkMutationGuard { /// /// # Sync/async lock boundary /// -/// The `parking_lot` map guard is never returned and never crosses an +/// The `parking_lot` shard guard is never returned and never crosses an /// `.await`. Acquisition clones a tracked `Arc>` before -/// releasing the map guard. Cleanup may synchronously take the short-lived map -/// write guard, but only probes the per-row lock with `try_read`; it never waits -/// on a Tokio lock while holding the map. This one-way boundary prevents a -/// map-lock/per-row-lock cycle during cancellation and `Drop`. +/// releasing the shard guard. Cleanup may synchronously take the short-lived +/// shard write guard, but only probes the per-row lock with `try_read`; it never +/// waits on a Tokio lock while holding the shard. This one-way boundary prevents +/// a map-lock/per-row-lock cycle during cancellation and `Drop`. +/// +/// # Borrowed guards +/// +/// [`LockAcquirer`], [`MutationGuard`] and their callers in [`crate::lock`] +/// hold a raw `*const` to this map or to its stripe array rather than an +/// `Arc` clone or a `&`. The `Arc` clone is what they are avoiding: it is a +/// read-modify-write on one key-independent cache line per operation, and +/// removing three of them took `paged_in_place` from losing throughput with +/// every added worker to holding it. A `&` would be sound and equally fast, +/// but these guards are held across `.await` inside generated operations whose +/// futures must be `'static` to spawn, so a lifetime on the guard becomes a +/// lifetime on the future. +/// +/// What that costs is a precondition the type system cannot state: taking a +/// guard and then dropping the last `Arc` while the guard lives is +/// undefined behaviour. Every in-crate and generated caller takes its guard as +/// a local of an operation that holds the table's own `Arc` for the +/// whole call, which is why this is sound as used. +/// +/// Because that precondition is real and cannot be checked, every entry point +/// that hands out one of these borrowed guards is `unsafe`, and the guard +/// types themselves are `#[doc(hidden)]` and out of the prelude. Leaving them +/// safe would have made the hazard reachable from safe code with no `unsafe` +/// anywhere in the caller, which is the definition of an unsound API: three +/// safe lines that build a map, take a guard, drop the map and drop the guard +/// are enough. They are not a user-facing API: every real caller is generated +/// code inside this crate's own operation bodies. +/// +/// # Sharding +/// +/// The map is `MAP_SHARD_COUNT` independent `RwLock`s. A single +/// table-wide map lock made every `get_or_insert_with` miss and every +/// `LockAcquirer` drop exclusive against every other row. Shards and mutation +/// stripes share one hash of the key but reduce it separately, because the two +/// counts answer to different costs. #[derive(Debug)] pub struct LockMap { - map: RwLock>>, + map: Box<[LockShard; MAP_SHARD_COUNT]>, + /// Table-wide label counter, for the cold callers that have no key in hand + /// (vacuum, and the raw `FullRowLock` helper). Locked operations must use + /// [`Self::next_id_for`] instead: see the note on `mutation_stripes`. next_id: AtomicU16, + /// Per-shard label counters, one to a cache line. + /// + /// `Lock::id` is a diagnostic label. It is not dependency identity, which + /// is `Arc` pointer equality on the lock's `locked` flag, and nothing in + /// the protocol reads it back. Minting it from one table-wide + /// `AtomicU16::fetch_add` nevertheless put a read-modify-write on a single + /// shared cache line into every locked operation, and, being independent + /// of the key, it was a line that eight writers on disjoint rows still + /// fought over. That is the same shape of defect as the `Arc` + /// refcount clone removed just before it, and it survived that fix because + /// the label is minted in generated code rather than in the map. + /// + /// Striping by the key's shard makes the line key-dependent, so disjoint + /// writers stop sharing it. Labels may now repeat across shards sooner + /// than a single counter would repeat, which the type already allows: a + /// `u16` wraps every 65536 operations regardless. + /// + /// The counters live in `mutation_stripes`, one per stripe, because a + /// locked operation already touches its stripe's line. mutation_stripes: Arc<[MutationStripe; MUTATION_STRIPE_COUNT]>, bulk_mutations: Arc, } @@ -137,7 +301,7 @@ pub struct LockMap { impl Default for LockMap { fn default() -> Self { Self { - map: RwLock::new(HashMap::new()), + map: Box::new(core::array::from_fn(|_| RwLock::new(HashMap::new()))), next_id: AtomicU16::default(), mutation_stripes: Arc::new(core::array::from_fn(|_| MutationStripe::default())), bulk_mutations: Arc::default(), @@ -149,24 +313,45 @@ impl LockMap where PrimaryKey: Hash + Eq + Debug + Clone, { + fn shard(&self, key: &PrimaryKey) -> &LockShard { + &self.map[Self::shard_of(key)] + } + + #[cfg(test)] + fn contains_key(&self, key: &PrimaryKey) -> bool { + self.shard(key).read().contains_key(key) + } + /// Inserts a raw lock entry. /// /// A returned or externally retained `Arc` pins cleanup through /// `Arc::strong_count`. Generated operations should prefer /// [`Self::get_or_insert_with`], whose [`LockAcquirer`] makes cancellation /// tracking explicit. + /// + /// Replacing an entry carries its acquirer count over rather than starting + /// the new one at zero. The count is inline in the entry now, not the + /// shared `Arc` an acquirer used to own, so an acquirer of the + /// replaced entry decrements *this* word when it drops: starting from zero + /// wraps it to `usize::MAX` and `remove_if_unused` then never reclaims the + /// key again. Only this public entry point can reach that state, and + /// carrying the count keeps every live acquirer accounted for by the entry + /// it will actually decrement. pub fn insert( &self, key: PrimaryKey, lock: Arc>, ) -> Option>> { - self.map - .write() + let mut shard = self.shard(&key).write(); + let carried = shard + .get(&key) + .map_or(0, |entry| entry.acquirers.load(Ordering::Acquire)); + shard .insert( key, LockEntry { lock, - acquirers: Arc::new(AtomicUsize::new(0)), + acquirers: AtomicUsize::new(carried), }, ) .map(|entry| entry.lock) @@ -175,7 +360,7 @@ where /// Returns an untracked raw lock clone, which keeps the map entry alive /// until that clone is dropped. pub fn get(&self, key: &PrimaryKey) -> Option>> { - self.map.read().get(key).map(|entry| entry.lock.clone()) + self.shard(key).read().get(key).map(|entry| entry.lock.clone()) } /// Returns the lock for `key`, inserting one built by `f` if absent. @@ -187,7 +372,18 @@ where /// can merge into it, but the *winner* already registered its operation on /// a lock that is no longer in the map, so it never waits for the loser and /// both proceed into the row at once. - pub fn get_or_insert_with(self: &Arc, key: PrimaryKey, f: F) -> LockAcquirer + /// + /// # Safety + /// + /// The returned [`LockAcquirer`] borrows this map as a raw pointer and + /// dereferences it on `Drop`, so the caller must keep an `Arc` + /// alive for at least as long as the acquirer and every clone of it. + /// Dropping the last `Arc` first is undefined behaviour. + /// + /// Generated operations satisfy this by construction: the acquirer is a + /// local of a call that reached this map through the table's own `Arc`, + /// which it holds for the whole call. + pub unsafe fn get_or_insert_with(self: &Arc, key: PrimaryKey, f: F) -> LockAcquirer where LockType: RowLock, F: FnOnce() -> LockType, @@ -197,39 +393,72 @@ where // under the guard, so `remove_with_lock_check` (which needs the write // lock) either runs before we looked or sees our extra strong reference // and keeps the entry. - if let Some(entry) = self.map.read().get(&key) { + if let Some(entry) = self.shard(&key).read().get(&key) { entry.acquirers.fetch_add(1, Ordering::AcqRel); return LockAcquirer { lock: Some(entry.lock.clone()), - acquirers: entry.acquirers.clone(), - lock_map: self.clone(), + lock_map: Arc::as_ptr(self), primary_key: key, }; } - let mut map = self.map.write(); + let mut map = self.shard(&key).write(); // Re-check: another task can insert between the read and write guards. let entry = map.entry(key.clone()).or_insert_with(|| LockEntry { lock: Arc::new(nagoya::sync::RwLock::new(f())), - acquirers: Arc::new(AtomicUsize::new(0)), + acquirers: AtomicUsize::new(0), }); entry.acquirers.fetch_add(1, Ordering::AcqRel); LockAcquirer { lock: Some(entry.lock.clone()), - acquirers: entry.acquirers.clone(), - lock_map: self.clone(), + lock_map: Arc::as_ptr(self), primary_key: key, } } pub fn remove(&mut self, key: &PrimaryKey) { - self.map.write().remove(key); + self.shard(key).write().remove(key); + } + + /// Registers one more caller that may still acquire `key`'s entry. + /// + /// Only [`LockAcquirer::clone`] needs this; the acquiring paths increment + /// while they already hold a shard guard. + fn retain_acquirer(&self, key: &PrimaryKey) { + if let Some(entry) = self.shard(key).read().get(key) { + entry.acquirers.fetch_add(1, Ordering::AcqRel); + } + } + + /// Drops one acquirer of `key` and removes the entry if that was the last + /// reason to keep it. + /// + /// The decrement and the removal check share one shard write guard. Split + /// across an atomic and a separate call they were two shared-memory + /// operations where one does, and the check has to take the guard either + /// way. + fn release_acquirer(&self, key: &PrimaryKey) + where + LockType: RowLock, + { + let mut set = self.shard(key).write(); + if let Some(entry) = set.get(key) { + entry.acquirers.fetch_sub(1, Ordering::AcqRel); + } + Self::remove_if_unused(&mut set, key); } pub fn remove_with_lock_check(&self, key: &PrimaryKey) where LockType: RowLock, { - let mut set = self.map.write(); + let mut set = self.shard(key).write(); + Self::remove_if_unused(&mut set, key); + } + + fn remove_if_unused(set: &mut HashMap>, key: &PrimaryKey) + where + LockType: RowLock, + { let should_remove = set.get(key).is_some_and(|entry| { let Some(guard) = entry.lock.try_read() else { return false; @@ -252,12 +481,30 @@ where self.next_id.fetch_add(1, Ordering::Relaxed) } + /// Mints a lock label from `key`'s own shard counter. + /// + /// The hot-path form of [`Self::next_id`]. Generated locked operations + /// call this; see the note on `next_ids` for why the table-wide counter is + /// not acceptable there. + pub fn next_id_for(&self, key: &PrimaryKey) -> u16 { + self.mutation_stripes[Self::stripe_of(key)] + .next_label + .fetch_add(1, Ordering::Relaxed) + } + /// Serializes the synchronous mutation phase for this key. /// /// The holder must not perform a suspending `.await`. Generated locked /// operations acquire this only after their async predecessor wait has /// completed, and the synchronous `insert` path never awaits. - pub fn mutation_guard(&self, key: &PrimaryKey) -> MutationGuard { + /// + /// # Safety + /// + /// The returned [`MutationGuard`] borrows this map's stripe array as a raw + /// pointer and dereferences it on `Drop`, so the caller must keep the map + /// alive for at least as long as the guard. See + /// [`Self::get_or_insert_with`]. + pub unsafe fn mutation_guard(&self, key: &PrimaryKey) -> MutationGuard { self.mutation_guard_for_stripe(Self::stripe_of(key)) } @@ -270,7 +517,11 @@ where /// single-key holders (which never nest stripe acquisitions) cannot form a /// cycle with a batch. The same no-`.await` rule as /// [`Self::mutation_guard`] applies for the whole guard set's lifetime. - pub fn mutation_guards<'a>(&self, keys: impl Iterator) -> Vec + /// + /// # Safety + /// + /// As [`Self::mutation_guard`], for every guard in the returned set. + pub unsafe fn mutation_guards<'a>(&self, keys: impl Iterator) -> Vec where PrimaryKey: 'a, { @@ -283,10 +534,25 @@ where .collect() } - fn stripe_of(key: &PrimaryKey) -> usize { + fn hash_of(key: &PrimaryKey) -> usize { let mut hasher = DefaultHasher::default(); key.hash(&mut hasher); - (hasher.finish() as usize) % MUTATION_STRIPE_COUNT + hasher.finish() as usize + } + + fn stripe_of(key: &PrimaryKey) -> usize { + Self::hash_of(key) % MUTATION_STRIPE_COUNT + } + + /// The row-lock shard for `key`. + /// + /// Derived from the same hash as the mutation stripe but reduced + /// separately: the two counts are tuned against different costs and are + /// not required to match. Folding the stripe index into the shard index + /// instead, as one modulo of the other, silently caps the shard count at + /// the stripe count. + fn shard_of(key: &PrimaryKey) -> usize { + Self::hash_of(key) % MAP_SHARD_COUNT } /// Mutation stripes currently held or being waited on. @@ -351,7 +617,7 @@ where } MutationGuard { - stripes: self.mutation_stripes.clone(), + stripes: Arc::as_ptr(&self.mutation_stripes), stripe, } } @@ -362,6 +628,45 @@ mod tests { use super::*; use crate::lock::FullRowLock; + /// Shards and stripes are reduced from one hash by different moduli, and + /// nothing in the type system keeps them apart: both are a `usize` index. + /// + /// Indexing the map with `stripe_of` compiles, stays within bounds, and is + /// silently wrong - it caps the reachable shard count at the stripe count. + /// That went unnoticed through a whole shard-count sweep, which read as + /// "the shard count does not matter" because shards above 64 were never + /// addressed. + /// + /// This drives `shard()` itself rather than the reduction functions: an + /// earlier version of this test asserted only that `shard_of` and + /// `stripe_of` have the right ranges, which is true no matter which one + /// `shard()` calls, and it passed with the defect injected. + #[test] + fn every_shard_of_the_map_is_reachable() { + const { + assert!( + MAP_SHARD_COUNT > MUTATION_STRIPE_COUNT, + "the defect this guards against is only possible in this direction" + ) + }; + let lock_map: Arc> = Arc::new(LockMap::default()); + let base = lock_map.map.as_ptr(); + let mut reached = alloc::collections::BTreeSet::new(); + for key in 0..(MAP_SHARD_COUNT as u64 * 64) { + // Identify the shard by address, so this measures where `shard()` + // actually lands rather than what a helper returns. + let index = (core::ptr::from_ref(lock_map.shard(&key)) as usize - base as usize) + / core::mem::size_of::>(); + reached.insert(index); + } + assert_eq!( + reached.len(), + MAP_SHARD_COUNT, + "only {} of {MAP_SHARD_COUNT} shards are addressable; a shard index reduced by the stripe count caps it at {MUTATION_STRIPE_COUNT}", + reached.len() + ); + } + /// A batch over more keys than stripes necessarily maps several keys to /// one stripe; acquisition must dedupe instead of deadlocking on the /// second ticket for the same stripe, and everything must be released on @@ -371,12 +676,14 @@ mod tests { let lock_map: LockMap = LockMap::default(); let keys: Vec = (0..1000).collect(); - let guards = lock_map.mutation_guards(keys.iter()); + // SAFETY: `lock_map` outlives the guard set. + let guards = unsafe { lock_map.mutation_guards(keys.iter()) }; assert!(guards.len() <= MUTATION_STRIPE_COUNT); drop(guards); for key in 0..1000u64 { - let _guard = lock_map.mutation_guard(&key); + // SAFETY: `lock_map` outlives the guard. + let _guard = unsafe { lock_map.mutation_guard(&key) }; } } @@ -403,7 +710,8 @@ mod tests { let lock_map: LockMap = LockMap::default(); let before = lock_map.mutation_epoch(); - let guard = lock_map.mutation_guard(&17); + // SAFETY: `lock_map` outlives the guard. + let guard = unsafe { lock_map.mutation_guard(&17) }; assert_eq!(lock_map.mutations_in_flight(), 1); drop(guard); @@ -424,11 +732,13 @@ mod tests { let other_map = lock_map.clone(); let handle = std::thread::spawn(move || { for _ in 0..100 { - let _guards = other_map.mutation_guards(backward.iter()); + // SAFETY: this thread owns an `Arc` clone of the map. + let _guards = unsafe { other_map.mutation_guards(backward.iter()) }; } }); for _ in 0..100 { - let _guards = lock_map.mutation_guards(forward.iter()); + // SAFETY: `lock_map` outlives the guard set. + let _guards = unsafe { lock_map.mutation_guards(forward.iter()) }; } handle.join().unwrap(); } @@ -439,13 +749,38 @@ mod tests { #[test] fn cancelled_acquirer_removes_the_abandoned_entry() { let lock_map: Arc> = Arc::new(LockMap::default()); - let acquirer = lock_map.get_or_insert_with(31, FullRowLock::new); + // SAFETY: `lock_map` outlives `acquirer` in every case below; it is + // dropped at the end of the test. + let acquirer = unsafe { lock_map.get_or_insert_with(31, FullRowLock::new) }; lock_map.remove_with_lock_check(&31); - assert!(lock_map.map.read().contains_key(&31)); + assert!(lock_map.contains_key(&31)); drop(acquirer); - assert!(!lock_map.map.read().contains_key(&31)); + assert!(!lock_map.contains_key(&31)); + } + + /// The acquirer count is inline in the map entry, so replacing an entry + /// through the public `insert` while an acquirer is live must not restart + /// that count at zero. + /// + /// With it restarted, the live acquirer's drop does `fetch_sub` from 0 and + /// wraps to `usize::MAX`. Nothing panics and nothing is unsound, but + /// `remove_if_unused` reads a non-zero count forever and the key is never + /// reclaimed: a silent, permanent entry leak in the row-lock map. + #[test] + fn replacing_an_entry_does_not_strand_a_live_acquirers_count() { + let lock_map: Arc> = Arc::new(LockMap::default()); + // SAFETY: `lock_map` outlives the acquirer. + let acquirer = unsafe { lock_map.get_or_insert_with(57, FullRowLock::new) }; + + lock_map.insert(57, Arc::new(nagoya::sync::RwLock::new(FullRowLock::new()))); + drop(acquirer); + + assert!( + !lock_map.contains_key(&57), + "a replaced entry must still be reclaimable once its last acquirer drops" + ); } /// Cloning the acquisition handle represents two tasks between lookup and @@ -454,22 +789,26 @@ mod tests { #[test] fn cleanup_waits_for_every_acquirer_to_drop() { let lock_map: Arc> = Arc::new(LockMap::default()); - let first = lock_map.get_or_insert_with(33, FullRowLock::new); + // SAFETY: `lock_map` outlives both handles. + let first = unsafe { lock_map.get_or_insert_with(33, FullRowLock::new) }; let second = first.clone(); drop(first); - assert!(lock_map.map.read().contains_key(&33)); + assert!(lock_map.contains_key(&33)); drop(second); - assert!(!lock_map.map.read().contains_key(&33)); + assert!(!lock_map.contains_key(&33)); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cancelling_async_waiter_releases_tracking_without_deadlock() { let lock_map: Arc> = Arc::new(LockMap::default()); - let owner = lock_map.get_or_insert_with(41, FullRowLock::new); + // SAFETY: `lock_map` outlives both acquirers and the spawned task, + // which is awaited before the map is dropped. + let owner = unsafe { lock_map.get_or_insert_with(41, FullRowLock::new) }; let owner_guard = owner.write().await; - let waiter = lock_map.get_or_insert_with(41, FullRowLock::new); + // SAFETY: as above. + let waiter = unsafe { lock_map.get_or_insert_with(41, FullRowLock::new) }; let waiting_task = tokio::spawn(async move { let _guard = waiter.write().await; }); @@ -477,10 +816,35 @@ mod tests { waiting_task.abort(); assert!(waiting_task.await.unwrap_err().is_cancelled()); - assert!(lock_map.map.read().contains_key(&41)); + assert!(lock_map.contains_key(&41)); drop(owner_guard); drop(owner); - assert!(!lock_map.map.read().contains_key(&41)); + assert!(!lock_map.contains_key(&41)); + } + + /// Disjoint keys must not share a map write lock. Eight threads each + /// acquiring and dropping a private key 10_000 times used to serialize on + /// one `RwLock`; they must complete without deadlock. + #[test] + fn disjoint_keys_do_not_share_a_map_write_lock() { + let lock_map: Arc> = Arc::new(LockMap::default()); + let mut handles = Vec::new(); + for worker in 0..8u64 { + let map = lock_map.clone(); + handles.push(std::thread::spawn(move || { + for step in 0..10_000u64 { + let key = worker << 32 | step; + // SAFETY: this thread owns an `Arc` clone of the map for + // the whole loop, so it outlives each acquirer. + let acquirer = unsafe { map.get_or_insert_with(key, FullRowLock::new) }; + drop(acquirer); + assert!(!map.contains_key(&key)); + } + })); + } + for handle in handles { + handle.join().unwrap(); + } } } diff --git a/src/lock/mod.rs b/src/lock/mod.rs index 15c7bced..fafe294a 100644 --- a/src/lock/mod.rs +++ b/src/lock/mod.rs @@ -29,9 +29,18 @@ const MAX_SPINS: u32 = 12; /// /// The guard will also attempt to remove the lock entry from the map on drop /// (preventing memory leaks). +#[doc(hidden)] pub struct LockGuard { lock: Arc, - lock_map: Arc>, + /// Borrowed, not an `Arc` clone: see [`LockAcquirer`]'s field of the same + /// name. Per-operation refcount traffic on the map is the shared-write + /// bottleneck, and it is independent of the key. + /// + /// # Safety + /// + /// Every guard is created inside one operation on a table that holds the + /// map's `Arc` for the duration of that call, so the map outlives it. + lock_map: *const LockMap, primary_key: PrimaryKey, /// Present for single-row operations. Multi-row queries acquire one /// mutation stripe only while processing each row, after all row locks are @@ -41,6 +50,16 @@ pub struct LockGuard { _not_sync: PhantomData>, } +// SAFETY: `LockMap` is `Sync` and the borrowed pointer is live for the guard's +// whole lifetime (see the field note), so a guard is as safe to send as the +// `&LockMap` it stands in for. The type stays `!Sync` via `_not_sync`. +unsafe impl Send for LockGuard +where + LockType: RowLock + Send + Sync, + PrimaryKey: Hash + Eq + Debug + Clone + Send, +{ +} + impl LockGuard where LockType: RowLock, @@ -48,7 +67,28 @@ where { /// Creates a new [`LockGuard`] that will clean up the [`Lock`] entry from /// the [`LockMap`] on [`Drop`]. - pub fn new(lock: Arc, lock_map: Arc>, primary_key: PrimaryKey) -> Self { + /// + /// # Safety + /// + /// The guard borrows the map as a raw pointer and dereferences it on + /// `Drop`, so `lock_map`'s allocation must outlive the returned guard: + /// holding the `Arc` only until this call returns is not enough. See the + /// "Borrowed guards" note on [`LockMap`]. + pub unsafe fn new(lock: Arc, lock_map: &Arc>, primary_key: PrimaryKey) -> Self { + Self::from_raw(lock, Arc::as_ptr(lock_map), primary_key) + } + + /// As [`Self::new`], for a caller that already holds the borrowed map + /// pointer (a [`PendingLock`] being converted). + /// + /// # Safety + /// + /// `lock_map` must outlive the guard; see the field note. + pub(crate) fn from_raw( + lock: Arc, + lock_map: *const LockMap, + primary_key: PrimaryKey, + ) -> Self { Self { lock, lock_map, @@ -60,12 +100,19 @@ where /// Creates a row guard that also serializes the mutation phase with the /// synchronous insert path for the same primary key. - pub fn new_with_mutation( + /// + /// # Safety + /// + /// `lock_map` must point to a live map that outlives the returned guard. + /// It is dereferenced here to take the mutation gate, and again when the + /// guard drops; see the field note on `lock_map`. + pub unsafe fn new_with_mutation( lock: Arc, - lock_map: Arc>, + lock_map: *const LockMap, primary_key: PrimaryKey, ) -> Self { - let mutation_guard = lock_map.mutation_guard(&primary_key); + // SAFETY: guaranteed by this function's own contract. + let mutation_guard = unsafe { (*lock_map).mutation_guard(&primary_key) }; Self { lock, lock_map, @@ -88,7 +135,8 @@ where { fn drop(&mut self) { self.lock.unlock(); - self.lock_map.remove_with_lock_check(&self.primary_key); + // SAFETY: see the field note; the map outlives this guard. + unsafe { (*self.lock_map).remove_with_lock_check(&self.primary_key) }; } } @@ -105,12 +153,30 @@ where /// cleanup; converting it into the final [`LockGuard`] with /// [`Self::into_guard`] or [`Self::into_guard_with_mutation`] defuses that /// cleanup and hands ownership over. +#[doc(hidden)] pub struct PendingLock { lock: Option>, - lock_map: Arc>, + /// Borrowed, not an `Arc` clone: see [`LockAcquirer`]'s field of the same + /// name. Per-operation refcount traffic on the map is the shared-write + /// bottleneck, and it is independent of the key. + /// + /// # Safety + /// + /// Every guard is created inside one operation on a table that holds the + /// map's `Arc` for the duration of that call, so the map outlives it. + lock_map: *const LockMap, primary_key: PrimaryKey, } +// SAFETY: `LockMap` is `Sync` and the pointer is live for the guard's whole +// lifetime (see the field note), so this is as safe to move as a `&LockMap`. +unsafe impl Send for PendingLock +where + LockType: RowLock + Send + Sync, + PrimaryKey: Hash + Eq + Debug + Clone + Send, +{ +} + impl PendingLock where LockType: RowLock, @@ -118,10 +184,15 @@ where { /// Takes ownership of a freshly registered operation lock. Must be called /// synchronously after registration, before the predecessor wait. - pub fn new(lock: Arc, lock_map: Arc>, primary_key: PrimaryKey) -> Self { + /// + /// # Safety + /// + /// As [`LockGuard::new`]: the map's allocation must outlive this value and + /// the [`LockGuard`] it is converted into. + pub unsafe fn new(lock: Arc, lock_map: &Arc>, primary_key: PrimaryKey) -> Self { Self { lock: Some(lock), - lock_map, + lock_map: Arc::as_ptr(lock_map), primary_key, } } @@ -132,7 +203,7 @@ where .lock .take() .expect("pending lock is intact until conversion or drop"); - LockGuard::new(lock, self.lock_map.clone(), self.primary_key.clone()) + LockGuard::from_raw(lock, self.lock_map, self.primary_key.clone()) } /// Defuses the cancellation cleanup and converts into a [`LockGuard`] @@ -147,7 +218,9 @@ where .lock .take() .expect("pending lock is intact until conversion or drop"); - LockGuard::new_with_mutation(lock, self.lock_map.clone(), self.primary_key.clone()) + // SAFETY: a pending lock is a local of the operation that took it, and + // that operation holds the map's `Arc` for its whole call. + unsafe { LockGuard::new_with_mutation(lock, self.lock_map, self.primary_key.clone()) } } } @@ -159,23 +232,32 @@ where fn drop(&mut self) { if let Some(lock) = self.lock.take() { lock.unlock(); - self.lock_map.remove_with_lock_check(&self.primary_key); + // SAFETY: see the field note; the map outlives this guard. + unsafe { (*self.lock_map).remove_with_lock_check(&self.primary_key) }; } } } #[derive(Debug)] pub struct Lock { - // A wrapping diagnostic label, not dependency identity. The existing - // locked allocation stays unique and stable for this lock lifetime. + // A wrapping diagnostic label, not dependency identity. The lock's own + // allocation is what stays unique and stable for its lifetime. id: u16, - locked: Arc, + /// Inline, not an `Arc`. + /// + /// It was separately allocated so a [`LockWait`] could outlive the lock it + /// waits on. A wait can hold an `Arc` instead and keep the whole lock + /// alive, which costs one pointer in a rarely-built future and saves an + /// allocation and a free on **every** locked operation, built or not. + /// Freeing was 30% of the profile on the in-place update path once the + /// map's exclusive acquisitions were out of the way. + locked: AtomicBool, wakers: Mutex>>, } impl PartialEq for Lock { fn eq(&self, other: &Self) -> bool { - Arc::ptr_eq(&self.locked, &other.locked) + core::ptr::eq(self, other) } } @@ -183,7 +265,7 @@ impl Eq for Lock {} impl Hash for Lock { fn hash(&self, state: &mut H) { - Hash::hash(&Arc::as_ptr(&self.locked), state) + Hash::hash(&(self as *const Self), state) } } @@ -197,7 +279,7 @@ impl Lock { pub fn new(id: u16) -> Self { Self { id, - locked: Arc::new(AtomicBool::from(true)), + locked: AtomicBool::new(true), wakers: Mutex::new(vec![]), } } @@ -210,7 +292,7 @@ impl Lock { pub fn new_released(id: u16) -> Self { Self { id, - locked: Arc::new(AtomicBool::new(false)), + locked: AtomicBool::new(false), wakers: Mutex::new(vec![]), } } @@ -236,12 +318,14 @@ impl Lock { self.locked.load(Ordering::Acquire) } - pub fn wait(&self) -> LockWait { + /// Takes `&Arc` because the returned wait keeps the lock alive: the + /// flag it polls lives in the lock now rather than in its own allocation. + pub fn wait(self: &Arc) -> LockWait { let mut guard = self.wakers.lock(); let waker = Arc::new(AtomicWaker::new()); guard.push(waker.clone()); LockWait { - locked: self.locked.clone(), + lock: Arc::clone(self), waker, } } @@ -249,7 +333,7 @@ impl Lock { #[derive(Debug)] pub struct LockWait { - locked: Arc, + lock: Arc, waker: Arc, } @@ -258,21 +342,21 @@ impl Future for LockWait { fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { // Fast path: already unlocked - if !self.locked.load(Ordering::Acquire) { + if !self.lock.locked.load(Ordering::Acquire) { return Poll::Ready(()); } // Spin phase: try up to MAX_SPINS before going async for _ in 0..MAX_SPINS { core::hint::spin_loop(); - if !self.locked.load(Ordering::Acquire) { + if !self.lock.locked.load(Ordering::Acquire) { return Poll::Ready(()); } } // Async phase: register waker and wait self.waker.register(cx.waker()); - if self.locked.load(Ordering::Acquire) { + if self.lock.locked.load(Ordering::Acquire) { Poll::Pending } else { Poll::Ready(()) @@ -307,7 +391,8 @@ mod tests { assert!(lock.is_locked()); { - let _guard = LockGuard::::new(lock.clone(), lock_map.clone(), pk); + // SAFETY: `lock_map` is a local `Arc` that outlives this guard. + let _guard = unsafe { LockGuard::::new(lock.clone(), &lock_map, pk) }; assert!(lock.is_locked()); } @@ -321,7 +406,8 @@ mod tests { let pk = 1u64; assert!(lock.is_locked()); - let guard = LockGuard::::new(lock.clone(), lock_map.clone(), pk); + // SAFETY: `lock_map` is a local `Arc` that outlives this guard. + let guard = unsafe { LockGuard::::new(lock.clone(), &lock_map, pk) }; assert!(lock.is_locked()); guard.unlock(); @@ -337,7 +423,8 @@ mod tests { assert!(lock.is_locked()); let result = std::panic::catch_unwind(AssertUnwindSafe(|| { - let _guard = LockGuard::::new(lock.clone(), lock_map.clone(), pk); + // SAFETY: `lock_map` outlives the unwind this closure triggers. + let _guard = unsafe { LockGuard::::new(lock.clone(), &lock_map, pk) }; panic!("test panic"); })); @@ -358,9 +445,10 @@ mod tests { assert!(lock3.is_locked()); { - let _guard1 = LockGuard::::new(lock1.clone(), lock_map.clone(), 1u64); - let _guard2 = LockGuard::::new(lock2.clone(), lock_map.clone(), 2u64); - let _guard3 = LockGuard::::new(lock3.clone(), lock_map.clone(), 3u64); + // SAFETY: `lock_map` is a local `Arc` that outlives all three guards. + let _guard1 = unsafe { LockGuard::::new(lock1.clone(), &lock_map, 1u64) }; + let _guard2 = unsafe { LockGuard::::new(lock2.clone(), &lock_map, 2u64) }; + let _guard3 = unsafe { LockGuard::::new(lock3.clone(), &lock_map, 3u64) }; assert!(lock1.is_locked()); assert!(lock2.is_locked()); @@ -397,7 +485,8 @@ mod tests { // Create a guard and drop it { - let _guard = LockGuard::new(lock, lock_map.clone(), pk); + // SAFETY: `lock_map` outlives this scope and so outlives the guard. + let _guard = unsafe { LockGuard::new(lock, &lock_map, pk) }; } // Verify the lock entry was removed from the map diff --git a/src/lock/row_lock.rs b/src/lock/row_lock.rs index e9022790..2fa32a49 100644 --- a/src/lock/row_lock.rs +++ b/src/lock/row_lock.rs @@ -1,7 +1,7 @@ use alloc::sync::Arc; +use alloc::vec::Vec; use core::fmt::Debug; use core::hash::Hash; -use hashbrown::HashSet; use crate::lock::{Lock, LockGuard, LockMap, LockWait}; @@ -16,12 +16,18 @@ pub trait RowLock { fn with_lock(id: u16) -> (Self, Arc) where Self: Sized; - /// Locks full [`RowLock`]. - #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (HashSet>, Arc); + /// Locks full [`RowLock`], returning the predecessors to wait on. + /// + /// A `Vec`, not a `HashSet`. The collection holds one entry per column this + /// lock type covers, deduplicated by pointer, which in every shipping + /// schema is a handful. Building a `hashbrown::HashSet` for that seeded a + /// fresh `foldhash` hasher on every operation, which was about a tenth of + /// the profile on the in-place update path, to hash at most a few `Arc` + /// pointers. Linear dedup over a short `Vec` is cheaper and the caller only + /// iterates the result. + fn lock(&mut self, id: u16) -> (Vec>, Arc); /// Merges two [`RowLock`]'s. - #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> HashSet> + fn merge(&mut self, other: &mut Self) -> Vec> where Self: Sized; } @@ -41,12 +47,19 @@ impl FullRowLock { /// Creates a [`LockGuard`] that will automatically unlock this lock when /// dropped. - pub fn guard( + /// + /// # Safety + /// + /// As [`LockGuard::new`]: the guard borrows `lock_map` as a raw pointer + /// and dereferences it on `Drop`, so that allocation must outlive the + /// returned guard. + pub unsafe fn guard( self, - lock_map: Arc>, + lock_map: &Arc>, primary_key: PrimaryKey, ) -> LockGuard { - LockGuard::new(self.l, lock_map, primary_key) + // SAFETY: forwarded to this function's own contract. + unsafe { LockGuard::new(self.l, lock_map, primary_key) } } pub fn wait(&self) -> LockWait { @@ -81,20 +94,19 @@ impl RowLock for FullRowLock { (FullRowLock { l: l.clone() }, l) } - fn lock(&mut self, id: u16) -> (HashSet>, Arc) { - let mut set = HashSet::new(); + fn lock(&mut self, id: u16) -> (Vec>, Arc) { let l = Arc::new(Lock::new(id)); - set.insert(self.l.clone()); + let set = vec![self.l.clone()]; self.l = l.clone(); (set, l) } - fn merge(&mut self, other: &mut Self) -> HashSet> + fn merge(&mut self, other: &mut Self) -> Vec> where Self: Sized, { - let set = HashSet::from_iter([self.l.clone()]); + let set = vec![self.l.clone()]; self.l = other.l.clone(); set } diff --git a/src/persistence/operation/batch.rs b/src/persistence/operation/batch.rs index fd28f970..22903ec1 100644 --- a/src/persistence/operation/batch.rs +++ b/src/persistence/operation/batch.rs @@ -6,8 +6,8 @@ use core::hash::Hash; use core::marker::PhantomData; use hashbrown::HashMap; +use data_bucket::Link; use data_bucket::page::PageId; -use data_bucket::{Link, SizeMeasurable}; use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; use worktable_codegen::{MemStat, worktable}; @@ -261,7 +261,7 @@ where .select_by_pos(old_pos) .ok_or_else(|| eyre::eyre!("batch metadata position {old_pos} is missing during reindex"))?; self.info_wt - .update_pos_by_id(PosByIdQuery { pos: old_pos - 1 }, row.id) + .update_by_id(row.id, BatchInnerColumns::POS, old_pos - 1) .await?; } Ok(()) diff --git a/src/persistence/space/index/mod.rs b/src/persistence/space/index/mod.rs index bc256825..ee25ec89 100644 --- a/src/persistence/space/index/mod.rs +++ b/src/persistence/space/index/mod.rs @@ -87,6 +87,7 @@ where &self, aliases: &page_aliases::PageAliases, event_page_key: &(T, Link), + event_value: Option<&(T, Link)>, ) -> Option<(PageId, Option<(T, Link)>)> { self.table_of_contents .get(event_page_key) @@ -96,6 +97,25 @@ where .resolve(event_page_key) .map(|(page_id, current_key)| (page_id, Some(current_key.clone()))) }) + .or_else(|| { + let event_value = event_value?; + // Page maxima are identities, so removing a maximum at the + // end of one persistence batch can make the next batch's + // recorded identity stale. Insert/remove events carry the + // affected value, which still identifies the ordered page + // range. Structural events do not, and remain hard errors. + // + // `event_page_key` is passed as well as the value: resolving by + // ordering alone always finds *some* page, which would make the + // caller's error branch unreachable and apply a genuinely + // corrupt event to whichever page sorted nearest. Given the + // stale identity, `page_containing` can check that the page it + // found is one that identity could have belonged to, and + // returns nothing when it is not. + self.table_of_contents + .page_containing(event_value, event_page_key) + .map(|(current_key, page_id)| (page_id, Some(current_key))) + }) } pub async fn new>(index_file_path: S, space_id: SpaceId, version: u32) -> eyre::Result { @@ -478,14 +498,16 @@ where let mut page_aliases = page_aliases::PageAliases::default(); for ev in events { match &ev { - ChangeEvent::InsertAt { max_value, .. } | ChangeEvent::RemoveAt { max_value, .. } => { + ChangeEvent::InsertAt { max_value, value, .. } | ChangeEvent::RemoveAt { max_value, value, .. } => { let event_page_key = (max_value.key.clone(), max_value.value); + let event_value = (value.key.clone(), value.value); // A direct TOC hit means the event key is the page's // canonical pre-event identity. An alias hit carries the // current canonical identity captured when the alias was // installed. This lets us compare the actual post-apply // identity without predicting DataBucket's mutation rules. - let Some((page_index, aliased_page_key)) = self.resolve_batch_page(&page_aliases, &event_page_key) + let Some((page_index, aliased_page_key)) = + self.resolve_batch_page(&page_aliases, &event_page_key, Some(&event_value)) else { // Naming the event and the identity it wanted, not // just the sizes. The counts alone say a lookup failed @@ -579,7 +601,8 @@ where split_index, } => { let event_page_key = (max_value.key.clone(), max_value.value); - let Some((page_index, aliased_page_key)) = self.resolve_batch_page(&page_aliases, &event_page_key) + let Some((page_index, aliased_page_key)) = + self.resolve_batch_page(&page_aliases, &event_page_key, None) else { return Err(eyre!( "index split references a missing page (toc_segments={}, buffered_pages={}, aliases={})", diff --git a/src/persistence/space/index/table_of_contents.rs b/src/persistence/space/index/table_of_contents.rs index 064b6081..70295fdd 100644 --- a/src/persistence/space/index/table_of_contents.rs +++ b/src/persistence/space/index/table_of_contents.rs @@ -74,6 +74,63 @@ where None } + /// Finds the page whose ordered range contains `value` and returns its + /// current maximum identity, but only when that page is a plausible + /// successor of the stale identity `stale_maximum`. + /// + /// CDC events name the maximum observed when the event was created, and + /// that identity goes stale at a persistence batch boundary when a + /// preceding max removal re-keys the page. Recovering from it means + /// choosing a page by ordering rather than by identity, which is a guess, + /// so it is fenced to the one shape the staleness can actually take. + /// + /// The page that a removed maximum leaves behind has a *smaller* maximum + /// than the identity the event names, and nothing else can have been + /// inserted between the two: any page whose maximum falls in + /// `(found, stale_maximum]` would itself own that range and would have + /// answered the identity lookup. Both conditions are checked here, so an + /// event whose identity is missing for any other reason -- a torn table of + /// contents, a stream applied out of order, two writers on one file -- + /// finds nothing and stays a hard error rather than being applied to + /// whichever page the ordering happened to pick. + /// + /// Without that fence the tail arm below returns the highest-keyed page for + /// any value above every maximum, so *some* page always matched and the + /// caller's error branch was unreachable for insert and remove events. + pub(crate) fn page_containing(&self, value: &T, stale_maximum: &T) -> Option<(T, PageId)> + where + T: Clone, + { + // One pass. This runs per fallback event inside the persistence batch + // loop, and it is already linear in the table of contents; walking it + // twice to apply the fence below would double that for nothing. + let mut ceiling: Option<(&T, &PageId)> = None; + let mut last: Option<(&T, &PageId)> = None; + // The greatest surviving maximum at or below the stale one. If the + // stale identity was really this page's, nothing survives between them, + // so this ends up being the page found. + let mut greatest_below_stale: Option<&T> = None; + for (maximum, page_id) in self.iter() { + if last.is_none_or(|(current, _)| maximum > current) { + last = Some((maximum, page_id)); + } + if maximum >= value && ceiling.is_none_or(|(current, _)| maximum < current) { + ceiling = Some((maximum, page_id)); + } + if maximum <= stale_maximum && greatest_below_stale.is_none_or(|current| maximum > current) { + greatest_below_stale = Some(maximum); + } + } + let (maximum, page_id) = ceiling.or(last)?; + // The named identity must be one this page could have shed, and no + // surviving page may sit between the two: such a page would own the + // range itself and would have answered the identity lookup. + if maximum >= stale_maximum || greatest_below_stale != Some(maximum) { + return None; + } + Some((maximum.clone(), *page_id)) + } + fn get_current_page_mut(&mut self) -> &mut GeneralPage> { &mut self.pages[self.current_page] } @@ -365,6 +422,55 @@ mod tests { assert_eq!(toc.get(&9), None); } + /// The ordered lookup picks the smallest ceiling, and the tail for a value + /// above every maximum, whenever the stale identity the event named is one + /// the chosen page could have shed. + #[test] + fn page_containing_uses_the_smallest_ceiling_and_the_tail_for_larger_values() { + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + toc.insert(40, 4.into()); + toc.insert(10, 1.into()); + toc.insert(25, 2.into()); + + // Each stale maximum sits above the page found and below the next + // surviving page, which is what a removed maximum leaves behind. + assert_eq!(toc.page_containing(&0, &20), Some((10, 1.into()))); + assert_eq!(toc.page_containing(&10, &20), Some((10, 1.into()))); + assert_eq!(toc.page_containing(&11, &30), Some((25, 2.into()))); + assert_eq!(toc.page_containing(&40, &45), Some((40, 4.into()))); + assert_eq!(toc.page_containing(&41, &45), Some((40, 4.into()))); + } + + /// The positional lookup is a repair for one specific staleness, not a + /// general "nearest page" fallback. + /// + /// Unfenced it returns the tail for any value above every maximum, so it + /// always answered and the caller's error branch was dead: a batch whose + /// identity is missing through real corruption was applied to whichever + /// page sorted nearest, and that wrong write was persisted. + #[test] + fn page_containing_refuses_an_identity_the_page_could_not_have_had() { + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + toc.insert(40, 4.into()); + toc.insert(10, 1.into()); + toc.insert(25, 2.into()); + + // Names a maximum at or below the page that owns the range. A live + // identity would have been found by the identity lookup, so this is a + // stream applied out of order, not a stale maximum. + assert_eq!(toc.page_containing(&11, &25), None); + assert_eq!(toc.page_containing(&11, &20), None); + + // Names a maximum with a surviving page between it and the page found. + // That page owns the range, so the event does not belong here. + assert_eq!(toc.page_containing(&0, &30), None); + assert_eq!(toc.page_containing(&11, &40), None); + + // An empty table of contents has nothing to repair to. + let empty = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + assert_eq!(empty.page_containing(&5, &9), None); + } + #[test] fn growing_key_update_moves_the_entry_instead_of_overflowing_the_segment() { const DATA_LENGTH: u32 = 128; diff --git a/src/persistence/space/index/unsized_.rs b/src/persistence/space/index/unsized_.rs index babafe6b..7ed75bff 100644 --- a/src/persistence/space/index/unsized_.rs +++ b/src/persistence/space/index/unsized_.rs @@ -66,6 +66,7 @@ where &self, aliases: &PageAliases, event_page_key: &(T, Link), + event_value: Option<&(T, Link)>, ) -> Option<(PageId, Option<(T, Link)>)> { self.table_of_contents .get(event_page_key) @@ -75,6 +76,20 @@ where .resolve(event_page_key) .map(|(page_id, current_key)| (page_id, Some(current_key.clone()))) }) + .or_else(|| { + let event_value = event_value?; + // A maximum removed in the preceding persistence batch can + // leave this event's page identity stale. The affected value + // still selects the ordered page range. Structural events do + // not carry such a value and remain hard errors. + // + // The stale identity is passed too, so the positional lookup + // can reject a page that identity could not have named. See the + // sized twin in `super::SpaceIndex::resolve_batch_page`. + self.table_of_contents + .page_containing(event_value, event_page_key) + .map(|(current_key, page_id)| (page_id, Some(current_key))) + }) } fn compact_page_if_needed(page: &mut UnsizedIndexPage) -> eyre::Result<()> { @@ -489,14 +504,16 @@ where let mut page_aliases = PageAliases::default(); for ev in events { match &ev { - ChangeEvent::InsertAt { max_value, .. } | ChangeEvent::RemoveAt { max_value, .. } => { + ChangeEvent::InsertAt { max_value, value, .. } | ChangeEvent::RemoveAt { max_value, value, .. } => { let event_page_key = (max_value.key.clone(), max_value.value); + let event_value = (value.key.clone(), value.value); // A direct TOC hit means the event key is the page's // canonical pre-event identity. An alias hit carries the // current canonical identity captured when the alias was // installed. This lets us compare the actual post-apply // identity without predicting DataBucket's mutation rules. - let Some((page_index, aliased_page_key)) = self.resolve_batch_page(&page_aliases, &event_page_key) + let Some((page_index, aliased_page_key)) = + self.resolve_batch_page(&page_aliases, &event_page_key, Some(&event_value)) else { return Err(eyre!( "unsized index event references a missing page (toc_segments={}, buffered_pages={}, aliases={})", @@ -574,7 +591,8 @@ where } => { let event_page_key = (max_value.key.clone(), max_value.value); - let Some((page_index, aliased_page_key)) = self.resolve_batch_page(&page_aliases, &event_page_key) + let Some((page_index, aliased_page_key)) = + self.resolve_batch_page(&page_aliases, &event_page_key, None) else { return Err(eyre!( "unsized index split references a missing page (toc_segments={}, buffered_pages={}, aliases={})", diff --git a/src/persistence/task.rs b/src/persistence/task.rs index a50ef6f0..fbb70ffa 100644 --- a/src/persistence/task.rs +++ b/src/persistence/task.rs @@ -725,12 +725,16 @@ mod lifecycle_tests { failure: TestFailure, } - #[derive(Clone, Copy)] + #[derive(Clone)] enum TestFailure { None, Engine, IndexCorruption, Panic, + GatedPanic { + entered: Arc, + release: Arc, + }, } impl PersistenceEngine<(), u64, TestEvents, TestIndex> for TestEngine { @@ -753,7 +757,7 @@ mod lifecycle_tests { &mut self, _batch_op: BatchOperation<(), u64, TestEvents, TestIndex>, ) -> eyre::Result<()> { - match self.failure { + match &self.failure { TestFailure::None => {} TestFailure::Engine => return Err(eyre::eyre!("injected batch failure")), TestFailure::IndexCorruption => { @@ -762,6 +766,11 @@ mod lifecycle_tests { ); } TestFailure::Panic => panic!("injected persistence worker panic"), + TestFailure::GatedPanic { entered, release } => { + entered.wait(); + release.wait(); + panic!("injected persistence worker panic"); + } } self.batches.fetch_add(1, Ordering::Relaxed); self.events.lock().push("batch"); @@ -1258,6 +1267,109 @@ mod lifecycle_tests { assert!(Arc::ptr_eq(&wait_error, &intake_error)); } + /// A worker panic must not escape `Drop`. + /// + /// In particular, a destructor that resumes the worker's panic can abort + /// the process when the table is itself being dropped during unwinding. + /// Hold the worker inside the engine until `Drop` has entered `Closing`, + /// proving that the drop path is awaiting this still-busy worker when the + /// injected panic lands. + #[test] + fn busy_drop_contains_a_worker_panic() { + let entered = Arc::new(std::sync::Barrier::new(2)); + let release = Arc::new(std::sync::Barrier::new(2)); + let task = PersistenceTask::run_engine(TestEngine { + batches: Arc::new(AtomicUsize::new(0)), + events: Arc::new(ParkingMutex::new(Vec::new())), + config: TestConfig, + failure: TestFailure::GatedPanic { + entered: entered.clone(), + release: release.clone(), + }, + }); + + task.apply_operation(insert_operation(1)).unwrap(); + entered.wait(); + + let monitor = task.monitor(); + let dropping = + std::thread::spawn(move || std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(task)))); + + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !matches!(monitor.lifecycle.state(), PersistenceState::Closing) { + assert!( + std::time::Instant::now() < deadline, + "Drop did not begin joining the busy persistence worker" + ); + std::thread::yield_now(); + } + release.wait(); + + let drop_result = dropping.join().expect("drop thread itself must remain joinable"); + assert!(drop_result.is_ok(), "a persistence worker panic escaped Drop"); + match monitor.lifecycle.state() { + PersistenceState::Failed(error) => assert_eq!( + error.to_string(), + "persistence engine failed: persistence worker panicked" + ), + state => panic!("worker panic did not become terminal: {state:?}"), + } + } + + /// The busy-drop join must give up on a worker that never finishes. + /// + /// This is the half of the busy-drop change that the panic-containment + /// test does not reach: a hung file or object-store write, an engine + /// future that stalls, or a drop reached from the engine worker itself. + /// `nagoya::block_on` parks with no deadline, so without the bound this + /// hangs the dropping thread -- which on a `current_thread` runtime is the + /// only thread there is -- with no diagnostic. + #[test] + fn a_worker_that_never_finishes_does_not_park_the_dropping_thread_forever() { + let runtime = nagoya::runtime::Runtime::new(1); + let released = Arc::new(AtomicBool::new(false)); + let handle = { + let released = released.clone(); + runtime.spawn(async move { + while !released.load(Ordering::Acquire) { + nagoya::yield_now().await; + } + }) + }; + + let started = std::time::Instant::now(); + let finished = join_with_timeout(handle, Duration::from_millis(50)); + let waited = started.elapsed(); + + assert!(!finished, "a worker still running must not be reported as joined"); + assert!( + waited < Duration::from_secs(5), + "the join must be bounded by its timeout, waited {waited:?}" + ); + // Detached, not cancelled: the worker is still there to finish its + // in-flight write, which is what `JoinHandle::drop` guarantees and what + // this code did before the join existed. + released.store(true, Ordering::Release); + } + + /// The same join must still return promptly when the worker does finish, + /// so the ordinary drain-and-fsync path is not silently paying the timeout. + #[test] + fn a_worker_that_finishes_is_joined_without_waiting_out_the_timeout() { + let runtime = nagoya::runtime::Runtime::new(1); + let handle = runtime.spawn(async {}); + + let started = std::time::Instant::now(); + assert!( + join_with_timeout(handle, Duration::from_secs(30)), + "a completed worker must be reported as joined" + ); + assert!( + started.elapsed() < Duration::from_secs(5), + "joining a finished worker must not wait out the timeout" + ); + } + /// A worker that stops publishes a terminal state instead of leaving its /// waiters parked, and refuses operations afterwards. /// @@ -1722,10 +1834,65 @@ pub struct PersistenceTask, } +/// How long a busy `Drop` waits for the engine worker before detaching it. +/// +/// Long enough that an ordinary drain and fsync finishes inside it, short +/// enough that a wedged write (a stalled object-store request, an engine future +/// that never completes, a drop reached from the engine worker itself) does not +/// hold the dropping thread forever. On expiry the work is not abandoned: the +/// handle is detached and the worker runs on, which is exactly what this code +/// did before the join was introduced. +const BUSY_DROP_JOIN_TIMEOUT: Duration = Duration::from_secs(30); + +/// Runs `handle` to completion on this thread, giving up after `timeout`. +/// +/// Returns whether the task finished. `nagoya::block_on` is the unbounded form +/// and parks on a `Signal` with no deadline, which is not acceptable inside a +/// destructor: see the note on [`PersistenceTask::drop`]. On expiry the handle +/// is dropped, and `nagoya::JoinHandle::drop` detaches rather than cancels, so +/// an in-flight persistence future is never cut in half by this. +fn join_with_timeout(handle: JoinHandle, timeout: Duration) -> bool { + use alloc::task::Wake; + use core::future::Future; + use core::pin::pin; + use core::task::{Context, Waker}; + use std::time::Instant; + + /// Unparks the dropping thread when the engine task makes progress. + struct Unpark(std::thread::Thread); + + impl Wake for Unpark { + fn wake(self: Arc) { + self.0.unpark(); + } + + fn wake_by_ref(self: &Arc) { + self.0.unpark(); + } + } + + let deadline = Instant::now() + timeout; + let waker = Waker::from(Arc::new(Unpark(std::thread::current()))); + let mut context = Context::from_waker(&waker); + let mut handle = pin!(handle); + + loop { + if handle.as_mut().poll(&mut context).is_ready() { + return true; + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + // A wake that landed before this park leaves a permit, so this cannot + // miss one; a spurious return is absorbed by re-polling. + std::thread::park_timeout(remaining); + } +} + impl Drop for PersistenceTask { - /// Aborts the engine task so it cannot outlive the table it persists. + /// Stops the engine task before the table finishes dropping. /// Without this the detached task keeps running on the runtime after the /// table is dropped, and a re-opened table can read the same files while /// the old engine is still writing them. @@ -1733,13 +1900,21 @@ impl Drop /// The abort only happens when the engine is provably idle (queue and /// analyzer empty, no operation in flight) — that is the normal state /// after `wait_for_ops`, and an idle task is parked at the queue pop, an - /// await point where cancellation is clean. Aborting a *busy* engine - /// would cancel persistence futures that are not cancellation-safe (a - /// data page could be left half-written while its index events are - /// abandoned), so a busy engine is left running and reported instead: - /// callers must drain with `wait_for_ops` before dropping. A proper - /// `close()` lifecycle (drain, join, surface terminal errors) is the - /// long-term replacement for this heuristic. + /// await point where cancellation is clean. Aborting a *busy* engine would + /// cancel persistence futures that are not cancellation-safe (a data page + /// could be left half-written while its index events are abandoned). Its + /// worker is private to this task, so a busy drop instead joins that worker + /// after requesting close. This keeps an immediate same-path reopen from + /// racing the last writes. + /// + /// That join is **bounded**. `Drop` runs on whatever thread drops the + /// table, which may be an executor thread (the only one, on a + /// `current_thread` runtime) and may already be unwinding from a panic. An + /// unbounded park there turns a hung file or object-store write into a hung + /// process with no diagnostic. After [`BUSY_DROP_JOIN_TIMEOUT`] the handle + /// is dropped instead, which detaches rather than cancels, so the worker + /// still finishes its in-flight write exactly as it did before this join + /// existed; the difference is that the caller is told rather than stalled. fn drop(&mut self) { match self.engine_task_handle.as_ref() { None => return, @@ -1772,10 +1947,23 @@ impl Drop if let Some(handle) = self.engine_task_handle.take() { handle.cancel(); } - } else { - tracing::error!( - "PersistenceTask dropped with work in flight; the engine task keeps draining detached and then stops, but its errors can no longer be observed. Call close() (or wait_for_ops() before dropping) to guarantee a clean shutdown." - ); + } else if let Some(handle) = self.engine_task_handle.take() { + // The engine owns a dedicated one-worker runtime. Joining here + // cannot occupy the worker that must make this task progress -- + // but note that a drop reached *from* that worker would + // self-deadlock, which the timeout below also bounds. A join + // rethrows a worker panic, which must not escape a destructor (and + // would abort the process if this drop is already unwinding). + // `WorkerCompletionGuard` records that panic in the lifecycle. + let joined = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + join_with_timeout(handle, BUSY_DROP_JOIN_TIMEOUT) + })); + if matches!(joined, Ok(false)) { + tracing::error!( + "PersistenceTask dropped with work in flight and the engine did not finish within {:?}; it keeps draining detached and then stops, but its errors can no longer be observed. Call close() (or wait_for_ops() before dropping) to guarantee a clean shutdown.", + BUSY_DROP_JOIN_TIMEOUT + ); + } } } } diff --git a/src/table/mod.rs b/src/table/mod.rs index 8a76f110..9f98a624 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -4,7 +4,7 @@ pub mod system_info; #[cfg(feature = "std")] pub mod vacuum; -use crate::in_memory::{ArchivedRowWrapper, DataPages, RowWrapper, StorableRow}; +use crate::in_memory::{ArchivedRowWrapper, DataPages, InlineArchived, RowWrapper, SelectRef, StorableRow}; #[cfg(feature = "std")] use crate::persistence::PersistenceLoadError; use crate::persistence::operation::new_operation_uuid; @@ -109,7 +109,7 @@ where { fn default() -> Self { Self { - data: Arc::new(DataPages::new()), + data: DataPages::new_arc(), primary_index: Arc::new(PrimaryIndex::::default()), indexes: Arc::new(SecondaryIndexes::default()), pk_gen: Default::default(), @@ -225,23 +225,35 @@ where self.pk_gen.reserve(count) } - /// Selects `Row` from table identified with provided primary key. Returns `None` if no value presented. - #[cfg_attr(feature = "perf_measurements", performance_measurement(prefix_name = "WorkTable"))] - pub fn select(&self, pk: PrimaryKey) -> Option + /// Resolves `pk` to a link and applies `read`, retrying while the link the + /// index hands back keeps changing under the read. + /// + /// A failed read is ambiguous on its own: the row may genuinely be absent + /// (ghosted, deleted) or it may have just moved, in which case the link is + /// stale rather than wrong. Re-looking the key up distinguishes the two -- + /// an unchanged link means the row really is gone, a changed one means try + /// again at the new address. + /// + /// Shared by [`Self::select`], [`Self::select_ref`] and + /// [`Self::select_with`], which differ only in what they do with the link. + /// The caller takes the epoch pin, because `select_ref` hands that pin out + /// in its return value and the other two only need it held. + /// + /// The bound is the pre-existing one: 64 attempts, after which this gives + /// up and reports absence. A row moving 64 times while one reader looks at + /// it is not a case this distinguishes from deletion. + #[inline] + fn with_link_retry(&self, pk: &PrimaryKey, mut read: F) -> Option where - LockType: 'static, - Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: - Deserialize<::WrappedRow, HighDeserializer>, + F: FnMut(Link) -> Result, { - let _read_guard = self.data.read_guard(); for _ in 0..64 { - let link = self.primary_index.pk_map.lookup_for_select(&pk).map(Into::into)?; - if let Ok(row) = self.data.select_non_ghosted(link) { - return Some(row); + let link = self.primary_index.pk_map.lookup_for_select(pk).map(Into::into)?; + if let Ok(value) = read(link) { + return Some(value); } - let current_link: Option = self.primary_index.pk_map.lookup_for_select(&pk).map(Into::into); + let current_link: Option = self.primary_index.pk_map.lookup_for_select(pk).map(Into::into); if current_link == Some(link) { return None; } @@ -250,6 +262,52 @@ where None } + /// Selects `Row` from table identified with provided primary key. Returns `None` if no value presented. + #[cfg_attr(feature = "perf_measurements", performance_measurement(prefix_name = "WorkTable"))] + pub fn select(&self, pk: PrimaryKey) -> Option + where + LockType: 'static, + Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, + <::WrappedRow as Archive>::Archived: + Deserialize<::WrappedRow, HighDeserializer>, + { + let _read_guard = self.data.read_guard(); + self.with_link_retry(&pk, |link| self.data.select_non_ghosted(link)) + } + + /// Point-read that yields a pin-guard plus archived inner row. + /// + /// Skips rkyv deserialize. The guard must outlive any use of the archived + /// fields. Owned [`Self::select`] stays for callers that need a `Row`. + /// Returning the guard copies the archived cell; [`Self::select_with`] + /// keeps that copy on the stack when the caller only needs a field. + pub fn select_ref(&self, pk: PrimaryKey) -> Option> + where + LockType: 'static, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + { + let pin = self.data.read_guard(); + // The copy is taken under the retry and the pin is attached to it + // afterwards, because the pin is moved into the result and so cannot be + // captured by a closure the loop calls more than once. + let copy = self.with_link_retry(&pk, |link| self.data.copy_non_ghosted(link))?; + Some(SelectRef::new(pin, copy)) + } + + /// Point-read that applies `f` to the archived inner row and returns `f`'s + /// result. No memcpy of the cell; `f` must copy out and not stash a + /// reference into the page. + pub fn select_with(&self, pk: PrimaryKey, mut f: F) -> Option + where + LockType: 'static, + ::WrappedRow: InlineArchived, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + F: FnMut(&<<::WrappedRow as Archive>::Archived as ArchivedRowWrapper>::Inner) -> T, + { + let _pin = self.data.read_guard(); + self.with_link_retry(&pk, |link| self.data.with_non_ghosted(link, &mut f)) + } + #[cfg_attr(feature = "perf_measurements", performance_measurement(prefix_name = "WorkTable"))] pub fn insert(&self, row: Row) -> Result where @@ -268,7 +326,9 @@ where LockType: 'static, { let pk = row.get_primary_key().clone(); - let _mutation_guard = self.lock_manager.mutation_guard(&pk); + // SAFETY: `self.lock_manager` is a field of this table and is borrowed + // for this whole call, so it outlives the guard taken here. + let _mutation_guard = unsafe { self.lock_manager.mutation_guard(&pk) }; self.insert_locked(row) } @@ -416,7 +476,9 @@ where // Stripe-ordered, exactly as `insert_many` takes them, so a batch // delete and a batch insert cannot deadlock against each other. Chunks // release before the next is taken, so that ordering holds across them. - let _mutation_guards = self.lock_manager.mutation_guards(chunk.iter()); + // SAFETY: `self.lock_manager` is a field of this table and is borrowed + // for this whole call, so it outlives every guard in the set. + let _mutation_guards = unsafe { self.lock_manager.mutation_guards(chunk.iter()) }; let _publication = TableSecondaryIndex::::row_publication(&*self.indexes); @@ -533,7 +595,9 @@ where // `DELETE_CHUNK_KEYS`. A range delete is the widest batch this table // takes, so it is the one that most needs not to hold every stripe. for chunk in keys.chunks(DELETE_CHUNK_KEYS) { - let _mutation_guards = self.lock_manager.mutation_guards(chunk.iter()); + // SAFETY: `self.lock_manager` is a field of this table and is borrowed + // for this whole call, so it outlives every guard in the set. + let _mutation_guards = unsafe { self.lock_manager.mutation_guards(chunk.iter()) }; let _publication = TableSecondaryIndex::::row_publication(&*self.indexes); @@ -632,7 +696,9 @@ where return Ok(Vec::new()); } let pks: Vec = rows.iter().map(|row| row.get_primary_key().clone()).collect(); - let _mutation_guards = self.lock_manager.mutation_guards(pks.iter()); + // SAFETY: `self.lock_manager` is a field of this table and is borrowed + // for this whole call, so it outlives every guard in the set. + let _mutation_guards = unsafe { self.lock_manager.mutation_guards(pks.iter()) }; let _publication = TableSecondaryIndex::::row_publication(&*self.indexes); @@ -783,7 +849,9 @@ where PrimaryIndex: TableIndexCdc, { let pk = row.get_primary_key().clone(); - let _mutation_guard = self.lock_manager.mutation_guard(&pk); + // SAFETY: `self.lock_manager` is a field of this table and is borrowed + // for this whole call, so it outlives the guard taken here. + let _mutation_guard = unsafe { self.lock_manager.mutation_guard(&pk) }; let _publication = TableSecondaryIndex::::row_publication(&*self.indexes); @@ -963,7 +1031,9 @@ where return (Vec::new(), Ok(Vec::new())); } let pks: Vec = rows.iter().map(|row| row.get_primary_key().clone()).collect(); - let _mutation_guards = self.lock_manager.mutation_guards(pks.iter()); + // SAFETY: `self.lock_manager` is a field of this table and is borrowed + // for this whole call, so it outlives every guard in the set. + let _mutation_guards = unsafe { self.lock_manager.mutation_guards(pks.iter()) }; let _publication = TableSecondaryIndex::::row_publication(&*self.indexes); @@ -1171,16 +1241,14 @@ where (ops, Ok(pks)) } - /// Reinserts provided row with updating indexes and saving it's data in new - /// place. Is used to not delete and insert because this situation causes - /// a possible gap when row doesn't exist. - /// - /// For reinsert it's ok that part of indexes will lead to old row and other - /// part is for new row. Goal is to make `PrimaryKey` of the row always - /// acceptable. As for reinsert `PrimaryKey` will be same for both old and - /// new [`Link`]'s, goal will be achieved. + /// Internal relocation primitive used by generated replacement code. /// - /// [`Link`]: data_bucket::Link + /// `row_old` must be the exact row currently stored at `row_new`'s primary + /// key, and the caller must participate in the table's mutation protocol. + /// This is not compare-and-replace: the method checks only primary-key + /// equality and uses the caller-supplied old row to repair secondary + /// indexes. Supplying a stale row can therefore leave those indexes wrong. + #[doc(hidden)] pub async fn reinsert(&self, row_old: Row, row_new: Row) -> Result where Row: Archive @@ -1262,6 +1330,7 @@ where } #[allow(clippy::type_complexity)] + #[doc(hidden)] pub fn reinsert_cdc( &self, row_old: Row, diff --git a/src/table/vacuum/pacing.rs b/src/table/vacuum/pacing.rs index 80427e97..a57fae43 100644 --- a/src/table/vacuum/pacing.rs +++ b/src/table/vacuum/pacing.rs @@ -132,6 +132,20 @@ impl VacuumPacing { /// once even on an idle table, so a waiting insert gets the registry /// before vacuum asks for it back. pub(crate) async fn wait_until_quiet(&self, activity: &impl ForegroundActivity, gate: &VacuumGate) { + // Sampled BEFORE the yield, not after, and this ordering is the whole + // content of the fast path below. + // + // Taken after the yield it is the same read the loop makes on its first + // pass, one line apart: the loop has already established that nothing is + // in flight and that the epoch has not moved, so re-asserting both + // against a snapshot taken between them proves nothing, and the guard + // collapses to `if quiet == 1 { return; }` with `quiet_samples` dead. + // Taken here it spans the yield, so it is an actual observation window: + // a mutation that completes while vacuum is off the executor moves the + // epoch and sends this call down the sampling path it exists for. + let entry_epoch = activity.mutation_epoch(); + let stand_downs_seen = gate.stand_downs(); + nagoya::yield_now().await; let mut backoff = self.backoff; @@ -151,6 +165,27 @@ impl VacuumPacing { } quiet += 1; + // An idle table needs no confirmation. The repeated samples exist to + // tell a real lull from the gap between two writes, and that ambiguity + // only exists when something has been writing: if nothing is in flight + // and the epoch has not moved across the entry yield, there is no gap + // to be fooled by. A table that was writing when vacuum arrived fails + // this and pays the full `quiet_samples` buffer. + // + // Paid unconditionally this cost 6 ms a batch (3 samples x 2 ms) on a + // table with no writers at all, and `ghost-vs-drop` measured a 40-page + // sweep at 31 ms of which roughly 36 ms was arithmetic sleep: the + // vacuum was not slow, it was waiting for permission nobody was + // withholding. + // + // `mutations_in_flight() == 0` is deliberately not repeated here: + // the loop head above rejected anything else to reach this line, + // and re-reading it two lines later proves nothing further. The + // stand-down count is not redundant in the same way, because it + // can be moved by a concurrent sweep between entry and now. + if quiet == 1 && current_epoch == entry_epoch && stand_downs_seen == gate.stand_downs() { + return; + } if quiet >= self.quiet_samples { return; } @@ -220,4 +255,120 @@ mod tests { .expect("vacuum should enter after the recheck buffer stays quiet") .unwrap(); } + + /// Foreground activity reported through the epoch alone, driven by the + /// test rather than by being asked. + /// + /// `ActivityBetweenChecks` moves its epoch on every *read*, which forces + /// the busy branch on the first look and so never reaches the fast path. + /// These two tests are about the fast path, so the epoch has to be + /// something the test moves at a chosen moment. + #[derive(Default)] + struct QuietActivity { + epoch: AtomicU64, + } + + impl ForegroundActivity for QuietActivity { + fn mutations_in_flight(&self) -> usize { + 0 + } + + fn mutation_epoch(&self) -> u64 { + self.epoch.load(Ordering::Acquire) + } + } + + /// The point of the fast path: a table nobody is writing to must not pay + /// `quiet_samples` sleeps for permission nobody is withholding. + /// + /// The backoffs are set far above the timeout, so this can only pass by + /// returning without sleeping at all. + #[tokio::test] + async fn an_idle_table_is_entered_without_any_sleep() { + let activity = QuietActivity::default(); + let gate = VacuumGate::default(); + let pacing = VacuumPacing { + backoff: Duration::from_secs(30), + max_backoff: Duration::from_secs(30), + quiet_samples: 3, + ..Default::default() + }; + + tokio::time::timeout(Duration::from_millis(500), pacing.wait_until_quiet(&activity, &gate)) + .await + .expect("an idle table must not sleep through the quiet buffer"); + assert_eq!( + gate.stand_downs(), + 0, + "an idle table gives vacuum nothing to stand down for" + ); + } + + /// Foreground work that completes across the entry yield, and only there. + /// + /// The epoch advances on the read that follows the entry sample and then + /// holds still. Driving it from the read index rather than from a second + /// task is deliberate: `nagoya::yield_now` wakes itself before returning + /// `Pending`, so a current-thread tokio scheduler runs the waiter straight + /// through both reads before the test task is polled again, and a bump + /// placed by the test lands after the loop has already started. There is no + /// window there to race into. + #[derive(Default)] + struct WriteAcrossTheEntryYield { + reads: AtomicU64, + } + + impl ForegroundActivity for WriteAcrossTheEntryYield { + fn mutations_in_flight(&self) -> usize { + 0 + } + + fn mutation_epoch(&self) -> u64 { + // Read 0 is the entry sample and sees epoch 0. Every read after it + // sees epoch 1: one mutation completed while vacuum was off the + // executor, and the table has been quiet ever since. + u64::from(self.reads.fetch_add(1, Ordering::AcqRel) > 0) + } + } + + /// The fast path must be defeated by work that completes across the entry + /// yield, which is the window it claims to observe. + /// + /// Measured by the stand-down count, which is what the two placements of + /// the entry sample actually disagree about here. Sampled before the yield + /// (correct), the pre-loop read and the loop's first read agree, so no + /// stand-down fires; the fast path is refused on `current_epoch != + /// entry_epoch` alone and the call pays the full quiet buffer. Sampled + /// after it, `entry_epoch` becomes the pre-loop read, the loop's first read + /// disagrees with it, and the busy branch fires a stand-down instead. Zero + /// is the fixed behaviour and one is the regression. + #[tokio::test] + async fn a_write_across_the_entry_yield_defeats_the_fast_path() { + let activity = WriteAcrossTheEntryYield::default(); + let gate = VacuumGate::default(); + let pacing = VacuumPacing { + backoff: Duration::from_millis(1), + max_backoff: Duration::from_millis(2), + quiet_samples: 3, + ..Default::default() + }; + + tokio::time::timeout(Duration::from_millis(500), pacing.wait_until_quiet(&activity, &gate)) + .await + .expect("the sampling path must still complete once the table stays quiet"); + + assert_eq!( + gate.stand_downs(), + 0, + "the entry sample must span the yield, so the epoch move is seen as staleness \ + rather than as a fresh mutation arriving mid-loop" + ); + // Three loop iterations, not one: the fast path was refused. Entry + // sample, the pre-loop read, and one read per `quiet_samples`. + assert_eq!( + activity.reads.load(Ordering::Acquire), + 5, + "a mutation completing across the entry yield must cost the full quiet buffer" + ); + } } diff --git a/src/table/vacuum/vacuum.rs b/src/table/vacuum/vacuum.rs index 7e270b18..e675ba0f 100644 --- a/src/table/vacuum/vacuum.rs +++ b/src/table/vacuum/vacuum.rs @@ -579,7 +579,9 @@ where to: PageId, ) -> eyre::Result { let lock = self.full_row_lock(&pk).await; - let _guard = LockGuard::new_with_mutation(lock, self.lock_manager.clone(), pk.clone()); + // SAFETY: the guard is a local of this call, which holds + // `self.lock_manager` for its whole body. + let _guard = unsafe { LockGuard::new_with_mutation(lock, Arc::as_ptr(&self.lock_manager), pk.clone()) }; let current_link: Option = self.primary_index.pk_map.lookup_for_select(&pk).map(Into::into); if current_link != Some(from_link) { @@ -618,9 +620,16 @@ where } async fn full_row_lock(&self, pk: &PrimaryKey) -> Arc { - let lock_id = self.lock_manager.next_id(); + // Striped by the key, as the generated paths are: vacuum takes this + // once per candidate row while foreground writers are running, so the + // table-wide counter would be a shared line it contends for. + let lock_id = self.lock_manager.next_id_for(pk); // One atomic acquire, no check-then-act: see LockMap::get_or_insert_with. - let lock = self.lock_manager.get_or_insert_with(pk.clone(), LockType::new); + // + // SAFETY: `self.lock_manager` is this vacuum's own `Arc`, + // borrowed for the whole call, so it outlives the acquirer, which is + // dropped at the end of this function. + let lock = unsafe { self.lock_manager.get_or_insert_with(pk.clone(), LockType::new) }; let mut lock_guard = lock.write().await; #[allow(clippy::mutable_key_type)] let (locks, op_lock) = lock_guard.lock(lock_id); @@ -908,7 +917,8 @@ mod tests { table.delete(*id).await.unwrap(); } - let mutation = table.0.lock_manager.mutation_guard(&ids[1].into()); + // SAFETY: `table` outlives this guard, and with it the map. + let mutation = unsafe { table.0.lock_manager.mutation_guard(&ids[1].into()) }; let vacuum = create_vacuum(&table).with_pacing(VacuumPacing { batch_pages: 1, backoff: Duration::from_millis(1), @@ -1783,7 +1793,7 @@ mod tests { another: 11, exchange: "updated0".to_string(), }; - table.update(updated_target.clone()).await.unwrap(); + table.replace(updated_target.clone()).await.unwrap(); let current_target_link = table .0 .primary_index diff --git a/tests/linear_table.rs b/tests/linear_table.rs new file mode 100644 index 00000000..06215a8d --- /dev/null +++ b/tests/linear_table.rs @@ -0,0 +1,62 @@ +use worktable::{LinearInsertError, LinearTable}; + +#[derive( + Clone, + Debug, + PartialEq, + worktable::prelude::rkyv::Archive, + worktable::prelude::rkyv::Serialize, + worktable::prelude::rkyv::Deserialize, +)] +#[rkyv(crate = worktable::prelude::rkyv)] +struct SavedRow { + line: u32, +} + +#[test] +fn push_preserves_duplicates_and_exposes_the_exact_contiguous_order() { + let mut table = LinearTable::with_capacity(3); + assert_eq!(table.push(20, "later"), 0); + assert_eq!(table.push(10, "first"), 1); + assert_eq!(table.push(20, "duplicate"), 2); + + assert_eq!(table.rows(), &[(20, "later"), (10, "first"), (20, "duplicate")]); + assert_eq!(table.as_ref(), table.rows()); +} + +#[test] +fn a_frozen_sorted_slice_supports_predecessor_search_without_an_index() { + let table = LinearTable::from(vec![(0_u64, 10_u32), (16, 11), (32, 12)]); + let rows = table.rows(); + let after = rows.partition_point(|(offset, _)| *offset <= 24); + + assert_eq!(after, 2); + assert_eq!(rows.get(after - 1), Some(&(16, 11))); +} + +#[test] +fn insert_is_the_explicit_unique_alternative_to_duplicate_preserving_push() { + let mut table = LinearTable::new(); + assert_eq!(table.insert(7, "first"), Ok(0)); + assert_eq!(table.insert(7, "second"), Err(LinearInsertError::DuplicateKey(7))); + assert_eq!(table.rows(), &[(7, "first")]); +} + +#[test] +fn vec_pages_round_trip_duplicates_and_append_in_insertion_order() { + let first = LinearTable::from(vec![(7_u64, SavedRow { line: 10 }), (7_u64, SavedRow { line: 11 })]); + let second = LinearTable::from(vec![(9_u64, SavedRow { line: 12 })]); + + let mut pages = first.unload().expect("rows fit"); + pages.extend_from_slice(&second.unload_appending(0).expect("rows fit")); + let loaded = LinearTable::::load(&pages).expect("valid pages"); + + assert_eq!( + loaded, + LinearTable::from(vec![ + (7, SavedRow { line: 10 }), + (7, SavedRow { line: 11 }), + (9, SavedRow { line: 12 }), + ]) + ); +} diff --git a/tests/persistence/duplicate_key_index_reload.rs b/tests/persistence/duplicate_key_index_reload.rs index 3a73b531..b275ff6c 100644 --- a/tests/persistence/duplicate_key_index_reload.rs +++ b/tests/persistence/duplicate_key_index_reload.rs @@ -273,7 +273,7 @@ fn test_duplicate_key_secondary_index_survives_reload() { if model.by_score.values().any(|ids| ids.contains(&id)) { let new_score = (id % KEYS) + 1_000; table - .update_score_by_id(ScoreByIdQuery { score: new_score }, id) + .update_by_id(id, DuplicateKeyReloadColumns::SCORE, new_score) .await .unwrap(); model.move_score(id, new_score); @@ -461,7 +461,7 @@ fn test_duplicate_key_mutations_without_reload() { if model.by_score.values().any(|ids| ids.contains(&id)) { let new_score = (id % KEYS) + 1_000; table - .update_score_by_id(ScoreByIdQuery { score: new_score }, id) + .update_by_id(id, DuplicateKeyReloadColumns::SCORE, new_score) .await .unwrap(); model.move_score(id, new_score); diff --git a/tests/persistence/failure/reinsert.rs b/tests/persistence/failure/reinsert.rs index 86f92631..277dcd47 100644 --- a/tests/persistence/failure/reinsert.rs +++ b/tests/persistence/failure/reinsert.rs @@ -56,7 +56,7 @@ fn test_reinsert_pk_mismatch() { unique_b: 200, }; - let result = table.update(new_row).await; + let result = table.replace(new_row).await; assert!(result.is_err()); let valid_row3 = TwoUniqueIdxRow { @@ -148,7 +148,7 @@ fn test_reinsert_two_indexes_first_fail() { unique_b: 500, }; - let result = table.update(updated_row).await; + let result = table.replace(updated_row).await; assert!(result.is_err()); let valid_row3 = TwoUniqueIdxRow { @@ -248,7 +248,7 @@ fn test_reinsert_two_indexes_second_fail() { unique_b: conflict_b, }; - let result = table.update(updated_row).await; + let result = table.replace(updated_row).await; assert!(result.is_err()); let valid_row3 = TwoUniqueIdxRow { @@ -353,7 +353,7 @@ fn test_reinsert_three_indexes_first_fail() { unique_c: 800, }; - let result = table.update(updated_row).await; + let result = table.replace(updated_row).await; assert!(result.is_err()); let valid_row3 = ThreeUniqueIdxRow { @@ -454,7 +454,7 @@ fn test_reinsert_three_indexes_middle_fail() { unique_c: 800, }; - let result = table.update(updated_row).await; + let result = table.replace(updated_row).await; assert!(result.is_err()); let valid_row3 = ThreeUniqueIdxRow { @@ -558,7 +558,7 @@ fn test_reinsert_three_indexes_last_fail() { unique_c: conflict_c, }; - let result = table.update(updated_row).await; + let result = table.replace(updated_row).await; assert!(result.is_err()); let valid_row3 = ThreeUniqueIdxRow { diff --git a/tests/persistence/failure/update.rs b/tests/persistence/failure/update.rs index fce70ad6..91f3ec09 100644 --- a/tests/persistence/failure/update.rs +++ b/tests/persistence/failure/update.rs @@ -64,7 +64,7 @@ fn test_update_unique_secondary_conflict() { unique_b: 500, }; - let result = table.update(updated_row).await; + let result = table.replace(updated_row).await; assert!(result.is_err()); let valid_row3 = TwoUniqueIdxRow { @@ -148,7 +148,7 @@ fn test_update_pk_based_success() { unique_b: 250, }; - let result = table.update(updated_row).await; + let result = table.replace(updated_row).await; assert!(result.is_ok()); let valid_row3 = TwoUniqueIdxRow { diff --git a/tests/persistence/failure/update_non_unique.rs b/tests/persistence/failure/update_non_unique.rs index 109daac8..185350cd 100644 --- a/tests/persistence/failure/update_non_unique.rs +++ b/tests/persistence/failure/update_non_unique.rs @@ -72,7 +72,9 @@ fn test_update_non_unique_middle_fail() { tokio::time::sleep(Duration::from_millis(100)).await; let query = UniqueValueByCategoryQuery { unique_value: 99 }; - let result = table.update_unique_value_by_category(query, 1).await; + let result = table + .update_by_category(1, MixedIdxColumns::UNIQUE_VALUE, (query).unique_value) + .await; assert!(result.is_err()); let valid_row3 = MixedIdxRow { @@ -180,7 +182,9 @@ fn test_update_non_unique_last_fail() { tokio::time::sleep(Duration::from_millis(100)).await; let query = UniqueValueByCategoryQuery { unique_value: 99 }; - let result = table.update_unique_value_by_category(query, 1).await; + let result = table + .update_by_category(1, MixedIdxColumns::UNIQUE_VALUE, (query).unique_value) + .await; assert!(result.is_err()); let valid_row3 = MixedIdxRow { diff --git a/tests/persistence/failure/update_unsized.rs b/tests/persistence/failure/update_unsized.rs index da29b215..9ec7fa2f 100644 --- a/tests/persistence/failure/update_unsized.rs +++ b/tests/persistence/failure/update_unsized.rs @@ -75,7 +75,9 @@ fn test_update_unsized_same_size() { name: "xxx".to_string(), unique_value: 99, }; - let result = table.update_name_and_value_by_category(query, 1).await; + let result = table + .update_by_category(1, NonUniqueUnsizedColumns::NAME_AND_UNIQUE_VALUE, query) + .await; assert!(result.is_err()); let valid_row3 = NonUniqueUnsizedRow { @@ -162,7 +164,9 @@ fn test_update_unsized_larger_all_success() { unique_value: 20, }; - let result = table.update_name_and_value_by_category(query, 1).await; + let result = table + .update_by_category(1, NonUniqueUnsizedColumns::NAME_AND_UNIQUE_VALUE, query) + .await; assert!(result.is_ok()); let valid_row3 = NonUniqueUnsizedRow { @@ -273,7 +277,9 @@ fn test_update_unsized_larger_middle_fail() { unique_value: 99, }; - let result = table.update_name_and_value_by_category(query, 1).await; + let result = table + .update_by_category(1, NonUniqueUnsizedColumns::NAME_AND_UNIQUE_VALUE, query) + .await; assert!(result.is_err()); let valid_row3 = NonUniqueUnsizedRow { @@ -376,7 +382,9 @@ fn test_update_unsized_larger_last_fail() { unique_value: 99, }; - let result = table.update_name_and_value_by_category(query, 1).await; + let result = table + .update_by_category(1, NonUniqueUnsizedColumns::NAME_AND_UNIQUE_VALUE, query) + .await; assert!(result.is_err()); let valid_row3 = NonUniqueUnsizedRow { diff --git a/tests/persistence/in_place_durability.rs b/tests/persistence/in_place_durability.rs index 154dc6b1..e1ab5004 100644 --- a/tests/persistence/in_place_durability.rs +++ b/tests/persistence/in_place_durability.rs @@ -10,11 +10,13 @@ worktable!( columns: { id: u64 primary_key, counter: u64, + revision: u64, note: String, }, queries: { - in_place: { + update_in_place: { CounterById(counter) by id, + CounterAndRevisionById(counter, revision) by id, } } ); @@ -52,12 +54,20 @@ fn in_place_update_survives_reload() { .insert(InPlaceDurabilityRow { id: 1, counter: 10, + revision: 1, note: "row".to_string(), }) .await .unwrap(); table - .update_counter_by_id_in_place(|counter| *counter = 42u64.into(), 1) + .update_in_place_by_id( + 1, + InPlaceDurabilityColumns::COUNTER_AND_REVISION, + |(counter, revision)| { + *counter = 42u64.into(); + *revision = 2u64.into(); + }, + ) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -70,6 +80,7 @@ fn in_place_update_survives_reload() { 42, "the in-place update was not persisted" ); + assert_eq!(table.select(1).unwrap().revision, 2); } remove_dir_if_exists(dir.to_string()).await; diff --git a/tests/persistence/local_write_bandwidth.rs b/tests/persistence/local_write_bandwidth.rs index 5d74f201..80058b23 100644 --- a/tests/persistence/local_write_bandwidth.rs +++ b/tests/persistence/local_write_bandwidth.rs @@ -145,7 +145,7 @@ fn local_write_bandwidth() { // Spread across the whole table rather than a contiguous run. let id = (n * (ROWS / UPDATES)) % ROWS; table - .update(WriteBandwidthRow { + .replace(WriteBandwidthRow { id, payload: replacement.clone(), }) diff --git a/tests/persistence/mod.rs b/tests/persistence/mod.rs index 157b644f..6825ac52 100644 --- a/tests/persistence/mod.rs +++ b/tests/persistence/mod.rs @@ -30,6 +30,8 @@ mod sync; mod toc; mod torn_shutdown; mod tuple_primary_key; +mod u128_primary_index_capacity; +mod uuid_primary_upsert; mod vacuum; #[cfg(feature = "s3-support")] diff --git a/tests/persistence/multi_row_backend_order.rs b/tests/persistence/multi_row_backend_order.rs index cbd71a8d..1a90867e 100644 --- a/tests/persistence/multi_row_backend_order.rs +++ b/tests/persistence/multi_row_backend_order.rs @@ -8,6 +8,7 @@ macro_rules! persisted_multi_row_backend_case { $module:ident, $name:ident, $table:ident, + $columns:ident, $row:ident, $engine:ident, $backend:ident, @@ -56,12 +57,7 @@ macro_rules! persisted_multi_row_backend_case { let replacement = "new-payload".repeat(64); table - .update_payload_by_group( - PayloadByGroupQuery { - payload: replacement.clone(), - }, - 7, - ) + .update_by_group_id(7, $columns::PAYLOAD, replacement.clone()) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -84,6 +80,7 @@ persisted_multi_row_backend_case!( wti, MultiRowWti, MultiRowWtiWorkTable, + MultiRowWtiColumns, MultiRowWtiRow, MultiRowWtiPersistenceEngine, worktables_index, @@ -93,6 +90,7 @@ persisted_multi_row_backend_case!( congee, MultiRowCongee, MultiRowCongeeWorkTable, + MultiRowCongeeColumns, MultiRowCongeeRow, MultiRowCongeePersistenceEngine, congee, @@ -102,6 +100,7 @@ persisted_multi_row_backend_case!( arctic, MultiRowArctic, MultiRowArcticWorkTable, + MultiRowArcticColumns, MultiRowArcticRow, MultiRowArcticPersistenceEngine, arctic, diff --git a/tests/persistence/s3/mod.rs b/tests/persistence/s3/mod.rs index d8c9844a..ae025497 100644 --- a/tests/persistence/s3/mod.rs +++ b/tests/persistence/s3/mod.rs @@ -425,7 +425,7 @@ fn s3_engine_reuses_logical_persistence_for_a_loaded_default_arctic_table() { let table = TestS3WorkTable::load(engine).await.unwrap(); let mut row = table.select(257).expect("persisted row"); row.value = 10_000; - table.update(row).await.unwrap(); + table.replace(row).await.unwrap(); table.wait_for_ops().await.unwrap(); let uploaded_before = s3.puts.lock().unwrap().iter().map(|(_, length)| length).sum::(); @@ -435,7 +435,7 @@ fn s3_engine_reuses_logical_persistence_for_a_loaded_default_arctic_table() { .sum::(); let mut row = table.select(1300).expect("persisted row"); row.value = 20_000; - table.update(row).await.unwrap(); + table.replace(row).await.unwrap(); table.wait_for_ops().await.unwrap(); let uploaded_after = s3.puts.lock().unwrap().iter().map(|(_, length)| length).sum::(); let incremental_bytes = uploaded_after - uploaded_before; diff --git a/tests/persistence/same_size_in_place.rs b/tests/persistence/same_size_in_place.rs index 8c36463f..491d7c5c 100644 --- a/tests/persistence/same_size_in_place.rs +++ b/tests/persistence/same_size_in_place.rs @@ -58,10 +58,7 @@ fn same_size_updates_keep_the_row_link() { let link_before = table.0.primary_index.pk_map.get_value(&pk).unwrap().0; // Fixed-size column update: archived in-place swap, same slot. - table - .update_amount_by_id(AmountByIdQuery { amount: 2 }, 1) - .await - .unwrap(); + table.update_by_id(1, SameSizeInPlaceColumns::AMOUNT, 2).await.unwrap(); let link_after_amount = table.0.primary_index.pk_map.get_value(&pk).unwrap().0; assert_eq!( link_before, link_after_amount, @@ -70,12 +67,7 @@ fn same_size_updates_keep_the_row_link() { // Same-length String update: same-size in-place path, same slot. table - .update_note_by_id( - NoteByIdQuery { - note: "bbbb".to_string(), - }, - 1, - ) + .update_by_id(1, SameSizeInPlaceColumns::NOTE, "bbbb".to_string()) .await .unwrap(); let link_after_note = table.0.primary_index.pk_map.get_value(&pk).unwrap().0; @@ -86,11 +78,10 @@ fn same_size_updates_keep_the_row_link() { // A size-changing String update must still reinsert correctly. table - .update_note_by_id( - NoteByIdQuery { - note: "a-considerably-longer-note".to_string(), - }, + .update_by_id( 1, + SameSizeInPlaceColumns::NOTE, + "a-considerably-longer-note".to_string(), ) .await .unwrap(); diff --git a/tests/persistence/space_index/unsized_write.rs b/tests/persistence/space_index/unsized_write.rs index da84b980..6be6b31e 100644 --- a/tests/persistence/space_index/unsized_write.rs +++ b/tests/persistence/space_index/unsized_write.rs @@ -552,3 +552,66 @@ async fn test_space_index_process_split_node() { "tests/data/expected/space_index_unsized/process_split_node.wt.idx".to_string() )) } + +#[tokio::test] +async fn max_removed_at_batch_boundary_does_not_orphan_the_next_unsized_event() { + let path = "tests/data/space_index_unsized/cross_batch_max.wt.idx"; + remove_file_if_exists(path.to_string()).await; + + let mut space_index = + SpaceIndexUnsized::::new(path, 0.into(), 1) + .await + .unwrap(); + let pair = |key: &str, offset: u32| Pair { + key: key.to_owned(), + value: Link { + page_id: 0.into(), + offset, + length: 24, + }, + }; + + space_index + .process_change_event_batch(vec![ + ChangeEvent::CreateNode { + event_id: 0.into(), + max_value: pair("30", 30), + }, + ChangeEvent::InsertAt { + event_id: 0.into(), + max_value: pair("30", 30), + value: pair("10", 10), + index: 0, + }, + ChangeEvent::InsertAt { + event_id: 0.into(), + max_value: pair("30", 30), + value: pair("20", 20), + index: 1, + }, + ChangeEvent::RemoveAt { + event_id: 0.into(), + max_value: pair("30", 30), + value: pair("30", 30), + index: 2, + }, + ]) + .await + .unwrap(); + + space_index + .process_change_event_batch(vec![ChangeEvent::InsertAt { + event_id: 0.into(), + max_value: pair("30", 30), + value: pair("25", 25), + index: 2, + }]) + .await + .unwrap(); + + let restored = space_index.parse_indexset().await.unwrap(); + for key in ["10", "20", "25"] { + assert!(restored.contains_key(key), "key {key} must survive cross-batch replay"); + } + assert!(!restored.contains_key("30")); +} diff --git a/tests/persistence/space_index/write.rs b/tests/persistence/space_index/write.rs index 81a0befa..8fe4d116 100644 --- a/tests/persistence/space_index/write.rs +++ b/tests/persistence/space_index/write.rs @@ -632,6 +632,79 @@ async fn batch_split_then_max_remove_then_historical_identity_insert_applies() { ); } +/// A page alias cannot be carried forever: persistence deliberately scopes +/// aliases to one batch. The following batch must recover a stale maximum +/// from the affected value's current ordered page range. +#[tokio::test] +async fn max_removed_at_batch_boundary_does_not_orphan_the_next_event() { + remove_file_if_exists("tests/data/space_index/cross_batch_max.wt.idx".to_string()).await; + + let mut space_index = SpaceIndex::::new( + "tests/data/space_index/cross_batch_max.wt.idx", + 0.into(), + 1, + ) + .await + .unwrap(); + + fn pair(key: u32) -> Pair { + Pair { + key, + value: Link { + page_id: 0.into(), + offset: key, + length: 24, + }, + } + } + + space_index + .process_change_event_batch(vec![ + ChangeEvent::CreateNode { + event_id: 0.into(), + max_value: pair(30), + }, + ChangeEvent::InsertAt { + event_id: 0.into(), + max_value: pair(30), + value: pair(10), + index: 0, + }, + ChangeEvent::InsertAt { + event_id: 0.into(), + max_value: pair(30), + value: pair(20), + index: 1, + }, + ChangeEvent::RemoveAt { + event_id: 0.into(), + max_value: pair(30), + value: pair(30), + index: 2, + }, + ]) + .await + .unwrap(); + + // The source CDC stream was created while this page was still identified + // by 30, but the preceding persisted batch re-keyed it to 20. + space_index + .process_change_event_batch(vec![ChangeEvent::InsertAt { + event_id: 0.into(), + max_value: pair(30), + value: pair(25), + index: 2, + }]) + .await + .unwrap(); + + let restored = space_index.parse_indexset().await.unwrap(); + for key in [10u32, 20, 25] { + assert!(restored.contains_key(&key), "key {key} must survive cross-batch replay"); + } + assert!(!restored.contains_key(&30)); +} + /// End-to-end equivalence: one batch carrying a real CDC stream with node /// splits, maximum removals, and re-inserts must replay into exactly the /// source index. This is the sized twin of the unsized alias machinery's diff --git a/tests/persistence/sync/failure.rs b/tests/persistence/sync/failure.rs index 70d11c88..cd8f69b6 100644 --- a/tests/persistence/sync/failure.rs +++ b/tests/persistence/sync/failure.rs @@ -1,7 +1,7 @@ use crate::remove_dir_if_exists; use worktable::prelude::*; -use super::{AnotherByIdQuery, FieldByAnotherQuery, TestSyncPersistenceEngine, TestSyncRow, TestSyncWorkTable}; +use super::{TestSyncColumns, TestSyncPersistenceEngine, TestSyncRow, TestSyncWorkTable}; #[test] fn test_failed_update_by_pk_doesnt_corrupt_persistence() { @@ -43,20 +43,13 @@ fn test_failed_update_by_pk_doesnt_corrupt_persistence() { let engine = TestSyncPersistenceEngine::new(config.clone()).await.unwrap(); let table = TestSyncWorkTable::load(engine).await.unwrap(); - let result = table - .update_another_by_id(AnotherByIdQuery { another: 9999 }, 9999) - .await; + let result = table.update_by_id(9999, TestSyncColumns::ANOTHER, 9999).await; assert!(result.is_err()); assert!(matches!(result.unwrap_err(), WorkTableError::NotFound)); for (i, pk) in pks.iter().enumerate() { table - .update_another_by_id( - AnotherByIdQuery { - another: i as u64 + 1000, - }, - *pk, - ) + .update_by_id(*pk, TestSyncColumns::ANOTHER, i as u64 + 1000) .await .unwrap(); } @@ -116,20 +109,13 @@ fn test_failed_update_by_unique_index_doesnt_corrupt_persistence() { let engine = TestSyncPersistenceEngine::new(config.clone()).await.unwrap(); let table = TestSyncWorkTable::load(engine).await.unwrap(); - let result = table - .update_field_by_another(FieldByAnotherQuery { field: 9999.0 }, 9999) - .await; + let result = table.update_by_another(9999, TestSyncColumns::FIELD, 9999.0).await; assert!(result.is_err()); assert!(matches!(result.unwrap_err(), WorkTableError::NotFound)); for (i, _pk) in pks.iter().enumerate() { table - .update_field_by_another( - FieldByAnotherQuery { - field: i as f64 + 1000.0, - }, - i as u64, - ) + .update_by_another(i as u64, TestSyncColumns::FIELD, i as f64 + 1000.0) .await .unwrap(); } diff --git a/tests/persistence/sync/many_strings.rs b/tests/persistence/sync/many_strings.rs index d9082675..654d813e 100644 --- a/tests/persistence/sync/many_strings.rs +++ b/tests/persistence/sync/many_strings.rs @@ -63,7 +63,10 @@ fn test_space_update_query_pk_sync() { field: "Some field value".to_string(), another: 0, }; - table.update_field_another_by_id(q, pk.clone()).await.unwrap(); + table + .update_by_id(pk.clone(), TestSyncColumns::FIELD_AND_ANOTHER, q) + .await + .unwrap(); table.wait_for_ops().await.unwrap(); } { @@ -122,7 +125,10 @@ fn test_space_update_query_pk_many_times_sync() { field: "Some field value".to_string(), another: i, }; - table.update_field_another_by_id(q, pk.clone()).await.unwrap(); + table + .update_by_id(pk.clone(), TestSyncColumns::FIELD_AND_ANOTHER, q) + .await + .unwrap(); } table.wait_for_ops().await.unwrap(); diff --git a/tests/persistence/sync/mod.rs b/tests/persistence/sync/mod.rs index 38d8ecc5..d32f1f5d 100644 --- a/tests/persistence/sync/mod.rs +++ b/tests/persistence/sync/mod.rs @@ -8,6 +8,7 @@ use worktable::worktable; mod failure; mod failure_multi_index; mod many_strings; +mod opaque_unsized_update; mod option; mod repeated_string_upsert; mod string_primary_index; @@ -185,7 +186,7 @@ fn test_space_update_full_sync() { }; table.insert(row.clone()).await.unwrap(); table - .update(TestSyncRow { + .replace(TestSyncRow { another: 13, non_unique: 0, field: 0.0, @@ -234,10 +235,7 @@ fn test_space_update_query_pk_sync() { id: table.get_next_pk().0, }; table.insert(row.clone()).await.unwrap(); - table - .update_another_by_id(AnotherByIdQuery { another: 13 }, row.id) - .await - .unwrap(); + table.update_by_id(row.id, TestSyncColumns::ANOTHER, 13).await.unwrap(); table.wait_for_ops().await.unwrap(); row.id }; @@ -279,10 +277,7 @@ fn test_space_update_query_unique_sync() { id: table.get_next_pk().0, }; table.insert(row.clone()).await.unwrap(); - table - .update_field_by_another(FieldByAnotherQuery { field: 1.0 }, 42) - .await - .unwrap(); + table.update_by_another(42, TestSyncColumns::FIELD, 1.0).await.unwrap(); table.wait_for_ops().await.unwrap(); row.id }; @@ -325,7 +320,7 @@ fn test_space_update_query_non_unique_sync() { }; table.insert(row.clone()).await.unwrap(); table - .update_another_by_non_unique(AnotherByNonUniqueQuery { another: 13 }, 10) + .update_by_non_unique(10, TestSyncColumns::ANOTHER, 13) .await .unwrap(); table.wait_for_ops().await.unwrap(); diff --git a/tests/persistence/sync/opaque_unsized_update.rs b/tests/persistence/sync/opaque_unsized_update.rs new file mode 100644 index 00000000..4398b4db --- /dev/null +++ b/tests/persistence/sync/opaque_unsized_update.rs @@ -0,0 +1,150 @@ +use crate::remove_dir_if_exists; + +use worktable::prelude::*; +use worktable::worktable; + +#[derive( + worktable::prelude::rkyv::Archive, + Clone, + Debug, + worktable::prelude::rkyv::Deserialize, + MemStat, + PartialEq, + PartialOrd, + worktable::prelude::rkyv::Serialize, +)] +#[rkyv(crate = worktable::prelude::rkyv)] +#[rkyv(derive(Debug, PartialEq, PartialOrd))] +struct WrappedSecret(String); + +worktable!( + name: OpaqueUnsizedUpdate, + version: 1, + persist: true, + columns: { + id: u64 primary_key, + secret: WrappedSecret, + untouched: u64, + }, + queries: { + update: { + SecretById(secret) by id, + } + } +); + +#[test] +fn targeted_update_of_string_wrapper_survives_read_and_reload() { + const DIR: &str = "tests/data/sync/opaque_unsized_update"; + let config = DiskConfig::new_with_table_name( + DIR, + OpaqueUnsizedUpdateWorkTable::name_snake_case(), + OpaqueUnsizedUpdateWorkTable::version(), + ); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_io() + .enable_time() + .build() + .unwrap(); + + runtime.block_on(async { + remove_dir_if_exists(DIR.to_string()).await; + + fn link_of(table: &OpaqueUnsizedUpdateWorkTable, id: u64) -> Link { + table + .0 + .primary_index + .pk_map + .get_value(&OpaqueUnsizedUpdatePrimaryKey::from(id)) + .map(Into::into) + .expect("row must exist") + } + + { + let engine = OpaqueUnsizedUpdatePersistenceEngine::new(config.clone()).await.unwrap(); + let table = OpaqueUnsizedUpdateWorkTable::load(engine).await.unwrap(); + table + .insert(OpaqueUnsizedUpdateRow { + id: 7, + secret: WrappedSecret("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string()), + untouched: 42, + }) + .await + .unwrap(); + let link_before = link_of(&table, 7); + + table + .update_by_id( + 7, + OpaqueUnsizedUpdateColumns::SECRET, + WrappedSecret("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string()), + ) + .await + .unwrap(); + let link_after = link_of(&table, 7); + + let row = table.select(7).unwrap(); + assert_eq!( + row.secret, + WrappedSecret("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string()) + ); + assert_eq!(row.untouched, 42); + assert_eq!(link_before, link_after, "same-size opaque update must retain its slot"); + table.wait_for_ops().await.unwrap(); + } + + { + let engine = OpaqueUnsizedUpdatePersistenceEngine::new(config.clone()).await.unwrap(); + let table = OpaqueUnsizedUpdateWorkTable::load(engine).await.unwrap(); + let row = table.select(7).unwrap(); + assert_eq!( + row.secret, + WrappedSecret("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string()) + ); + assert_eq!(row.untouched, 42); + + table + .update_by_id( + 7, + OpaqueUnsizedUpdateColumns::SECRET, + WrappedSecret("a replacement with a deliberately different serialized length".to_string()), + ) + .await + .unwrap(); + table.wait_for_ops().await.unwrap(); + } + + { + let engine = OpaqueUnsizedUpdatePersistenceEngine::new(config.clone()).await.unwrap(); + let table = OpaqueUnsizedUpdateWorkTable::load(engine).await.unwrap(); + let row = table.select(7).unwrap(); + assert_eq!( + row.secret, + WrappedSecret("a replacement with a deliberately different serialized length".to_string()) + ); + assert_eq!(row.untouched, 42); + + table + .replace(OpaqueUnsizedUpdateRow { + id: 7, + secret: WrappedSecret("full-row replacement after targeted updates".to_string()), + untouched: 84, + }) + .await + .unwrap(); + table.wait_for_ops().await.unwrap(); + } + + { + let engine = OpaqueUnsizedUpdatePersistenceEngine::new(config).await.unwrap(); + let table = OpaqueUnsizedUpdateWorkTable::load(engine).await.unwrap(); + let row = table.select(7).unwrap(); + assert_eq!( + row.secret, + WrappedSecret("full-row replacement after targeted updates".to_string()) + ); + assert_eq!(row.untouched, 84); + } + }); +} diff --git a/tests/persistence/sync/option.rs b/tests/persistence/sync/option.rs index 98e8031a..f22f36a1 100644 --- a/tests/persistence/sync/option.rs +++ b/tests/persistence/sync/option.rs @@ -140,7 +140,7 @@ fn test_option_update_full_sync() { table.insert(row.clone()).await.unwrap(); table - .update(TestOptionSyncRow { + .replace(TestOptionSyncRow { id: row.id, test: Some(100), another: 1, @@ -192,7 +192,7 @@ fn test_option_update_by_id_sync() { table.insert(row.clone()).await.unwrap(); table - .update_test_by_id(TestByIdQuery { test: Some(42) }, row.id) + .update_by_id(row.id, TestOptionSyncColumns::TEST, Some(42)) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -239,7 +239,7 @@ fn test_option_update_none_to_some_sync() { table.insert(row.clone()).await.unwrap(); table - .update_test_by_id(TestByIdQuery { test: Some(55) }, row.id) + .update_by_id(row.id, TestOptionSyncColumns::TEST, Some(55)) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -286,7 +286,7 @@ fn test_option_update_some_to_none_sync() { table.insert(row.clone()).await.unwrap(); table - .update_test_by_id(TestByIdQuery { test: None }, row.id) + .update_by_id(row.id, TestOptionSyncColumns::TEST, None) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -333,7 +333,7 @@ fn test_option_update_by_another_sync() { table.insert(row.clone()).await.unwrap(); table - .update_test_by_another(TestByAnotherQuery { test: Some(77) }, 123) + .update_by_another(123, TestOptionSyncColumns::TEST, Some(77)) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -380,7 +380,7 @@ fn test_option_update_by_exchange_sync() { table.insert(row.clone()).await.unwrap(); table - .update_test_by_exchange(TestByExchangeQuery { test: Some(88) }, 456) + .update_by_exchange(456, TestOptionSyncColumns::TEST, Some(88)) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -436,7 +436,7 @@ fn test_option_multiple_rows_sync() { let pk2 = table.insert(row2).await.unwrap(); table - .update_test_by_id(TestByIdQuery { test: Some(30) }, pk1.clone()) + .update_by_id(pk1.clone(), TestOptionSyncColumns::TEST, Some(30)) .await .unwrap(); @@ -590,7 +590,7 @@ fn test_option_indexed_update_none_to_some_by_id_sync() { table.insert(row.clone()).await.unwrap(); table - .update_index_test_by_id(IndexTestByIdQuery { test: Some(55) }, row.id) + .update_by_id(row.id, TestOptionSyncIndexColumns::TEST, Some(55)) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -637,7 +637,7 @@ fn test_option_indexed_update_some_to_none_by_id_sync() { table.insert(row.clone()).await.unwrap(); table - .update_index_test_by_id(IndexTestByIdQuery { test: None }, row.id) + .update_by_id(row.id, TestOptionSyncIndexColumns::TEST, None) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -684,7 +684,7 @@ fn test_option_indexed_update_by_another_sync() { table.insert(row.clone()).await.unwrap(); table - .update_index_test_by_another(IndexTestByAnotherQuery { test: Some(77) }, 123) + .update_by_another(123, TestOptionSyncIndexColumns::TEST, Some(77)) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -748,12 +748,12 @@ fn test_option_indexed_multiple_rows_sync() { let pk3 = table.insert(row3).await.unwrap(); table - .update_index_test_by_id(IndexTestByIdQuery { test: Some(40) }, pk1.clone()) + .update_by_id(pk1.clone(), TestOptionSyncIndexColumns::TEST, Some(40)) .await .unwrap(); table - .update_index_test_by_id(IndexTestByIdQuery { test: Some(50) }, pk2.clone()) + .update_by_id(pk2.clone(), TestOptionSyncIndexColumns::TEST, Some(50)) .await .unwrap(); @@ -801,7 +801,7 @@ fn test_option_indexed_full_row_update_sync() { table.insert(row.clone()).await.unwrap(); table - .update(TestOptionSyncIndexRow { + .replace(TestOptionSyncIndexRow { id: row.id, test: Some(99), another: 100, diff --git a/tests/persistence/sync/string_primary_index.rs b/tests/persistence/sync/string_primary_index.rs index 58019b28..b887d0c8 100644 --- a/tests/persistence/sync/string_primary_index.rs +++ b/tests/persistence/sync/string_primary_index.rs @@ -145,7 +145,7 @@ fn test_space_update_full_sync() { }; table.insert(row.clone()).await.unwrap(); table - .update(TestSyncRow { + .replace(TestSyncRow { another: 13, non_unique: 0, field: 0.0, @@ -194,7 +194,7 @@ fn test_space_update_query_pk_sync() { }; table.insert(row.clone()).await.unwrap(); table - .update_another_by_id(AnotherByIdQuery { another: 13 }, row.id.clone()) + .update_by_id(row.id.clone(), TestSyncColumns::ANOTHER, 13) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -237,10 +237,7 @@ fn test_space_update_query_unique_sync() { id: "Some string before".to_string(), }; table.insert(row.clone()).await.unwrap(); - table - .update_field_by_another(FieldByAnotherQuery { field: 1.0 }, 42) - .await - .unwrap(); + table.update_by_another(42, TestSyncColumns::FIELD, 1.0).await.unwrap(); table.wait_for_ops().await.unwrap(); row.id }; @@ -282,7 +279,7 @@ fn test_space_update_query_non_unique_sync() { }; table.insert(row.clone()).await.unwrap(); table - .update_another_by_non_unique(AnotherByNonUniqueQuery { another: 13 }, 10) + .update_by_non_unique(10, TestSyncColumns::ANOTHER, 13) .await .unwrap(); table.wait_for_ops().await.unwrap(); diff --git a/tests/persistence/sync/string_re_read.rs b/tests/persistence/sync/string_re_read.rs index fd54f2b2..744961ed 100644 --- a/tests/persistence/sync/string_re_read.rs +++ b/tests/persistence/sync/string_re_read.rs @@ -593,7 +593,7 @@ fn test_toc_not_updated_when_index_value_same_but_link_changes() { let table = StringReReadWorkTable::load(engine).await.unwrap(); table - .update(StringReReadRow { + .replace(StringReReadRow { first: "same_first".to_string(), id: pk1.into(), third: "third_updated".to_string(), @@ -749,7 +749,7 @@ fn test_unique_index_same_value_link_changes() { // Update: same second value, other fields change table - .update(StringReReadRow { + .replace(StringReReadRow { first: "first_updated".to_string(), id: pk1.into(), third: "third_updated".to_string(), diff --git a/tests/persistence/sync/string_secondary_index.rs b/tests/persistence/sync/string_secondary_index.rs index e4b65ffc..bf03744d 100644 --- a/tests/persistence/sync/string_secondary_index.rs +++ b/tests/persistence/sync/string_secondary_index.rs @@ -245,7 +245,7 @@ fn test_space_update_full_sync() { }; table.insert(row.clone()).await.unwrap(); table - .update(TestSyncRow { + .replace(TestSyncRow { another: "Some string to test updated".to_string(), non_unique: 0, field: 0.0, @@ -302,11 +302,10 @@ fn test_space_update_query_pk_sync() { }; table.insert(row.clone()).await.unwrap(); table - .update_another_by_id( - AnotherByIdQuery { - another: "Some string to test updated".to_string(), - }, + .update_by_id( row.id, + TestSyncColumns::ANOTHER, + "Some string to test updated".to_string(), ) .await .unwrap(); @@ -355,7 +354,7 @@ fn test_space_update_query_unique_sync() { }; table.insert(row.clone()).await.unwrap(); table - .update_field_by_another(FieldByAnotherQuery { field: 1.0 }, "Some string before".to_string()) + .update_by_another("Some string before".to_string(), TestSyncColumns::FIELD, 1.0) .await .unwrap(); table.wait_for_ops().await.unwrap(); @@ -400,12 +399,7 @@ fn test_space_update_query_non_unique_sync() { }; table.insert(row.clone()).await.unwrap(); table - .update_another_by_non_unique( - AnotherByNonUniqueQuery { - another: "Some string to test updated".to_string(), - }, - 10, - ) + .update_by_non_unique(10, TestSyncColumns::ANOTHER, "Some string to test updated".to_string()) .await .unwrap(); table.wait_for_ops().await.unwrap(); diff --git a/tests/persistence/sync/string_update_timeout.rs b/tests/persistence/sync/string_update_timeout.rs index 05bab4c0..88090b33 100644 --- a/tests/persistence/sync/string_update_timeout.rs +++ b/tests/persistence/sync/string_update_timeout.rs @@ -83,7 +83,7 @@ fn test_string_update_doesnt_block_persistence() { let engine = UserPersistenceEngine::new(config.clone()).await.unwrap(); let table = UserWorkTable::load(engine).await.unwrap(); - table.update(row.clone()).await.unwrap(); + table.replace(row.clone()).await.unwrap(); let wait_result = timeout(Duration::from_secs(4), table.wait_for_ops()) .await diff --git a/tests/persistence/tuple_primary_key.rs b/tests/persistence/tuple_primary_key.rs index 2f31bc1e..62549c86 100644 --- a/tests/persistence/tuple_primary_key.rs +++ b/tests/persistence/tuple_primary_key.rs @@ -72,7 +72,7 @@ async fn composite_primary_key_survives_mutations_and_reload() { value: 99, ..rows[1].clone() }; - table.update(updated.clone()).await.unwrap(); + table.replace(updated.clone()).await.unwrap(); table.delete((7, 41)).await.unwrap(); assert_eq!(table.select((7, 42)), Some(updated)); assert!(table.select((7, 41)).is_none()); diff --git a/tests/persistence/u128_primary_index_capacity.rs b/tests/persistence/u128_primary_index_capacity.rs new file mode 100644 index 00000000..c0b18e82 --- /dev/null +++ b/tests/persistence/u128_primary_index_capacity.rs @@ -0,0 +1,214 @@ +use worktable::prelude::*; +use worktable::worktable; + +use crate::remove_dir_if_exists; + +worktable!( + name: U128PrimaryIndexCapacity, + persist: true, + columns: { + id: u128 primary_key using worktables_index, + value: u64, + }, +); + +worktable!( + name: U8ThenU128KeyLayout, + columns: { + shard: u8 primary_key using worktables_index, + id: u128 primary_key using worktables_index, + value: u64, + }, +); + +worktable!( + name: U128ThenU8KeyLayout, + columns: { + id: u128 primary_key using worktables_index, + shard: u8 primary_key using worktables_index, + value: u64, + }, +); + +worktable!( + name: StringKeyLayout, + columns: { + id: String primary_key using worktables_index, + value: u64, + }, +); + +worktable!( + name: StringThenU128KeyLayout, + persist: true, + columns: { + tenant: String primary_key using worktables_index, + id: u128 primary_key using worktables_index, + value: u64, + }, +); + +worktable!( + name: U128ThenStringKeyLayout, + persist: true, + columns: { + id: u128 primary_key using worktables_index, + tenant: String primary_key using worktables_index, + value: u64, + }, +); + +#[test] +fn generated_key_size_measure_tracks_archived_alignment_and_dynamic_length() { + use data_bucket::{IndexPage, IndexValue, Persistable}; + + macro_rules! assert_default_index_page_fits { + ($key:ty) => {{ + let capacity = get_index_page_size_from_data_length::<$key>(INNER_PAGE_SIZE); + let page = IndexPage::<$key>::new(IndexValue::default(), capacity); + assert!(page.as_bytes().as_ref().len() <= INNER_PAGE_SIZE); + }}; + } + + assert_default_index_page_fits!(U128PrimaryIndexCapacityPrimaryKey); + assert_default_index_page_fits!(U8ThenU128KeyLayoutPrimaryKey); + assert_default_index_page_fits!(U128ThenU8KeyLayoutPrimaryKey); + + assert_eq!(U128PrimaryIndexCapacityPrimaryKey::align(), Some(16)); + assert_eq!(U8ThenU128KeyLayoutPrimaryKey::align(), Some(16)); + assert_eq!(U128ThenU8KeyLayoutPrimaryKey::align(), Some(16)); + + let empty = StringKeyLayoutPrimaryKey(String::new()).aligned_size(); + let populated = StringKeyLayoutPrimaryKey("a dynamic primary key".to_string()).aligned_size(); + assert!( + populated > empty, + "dynamic key length was lost by the generated wrapper" + ); +} + +#[tokio::test] +async fn mixed_unsized_and_u128_key_orders_survive_persistence() { + let string_first_dir = "tests/data/string_then_u128_primary_index"; + let u128_first_dir = "tests/data/u128_then_string_primary_index"; + remove_dir_if_exists(string_first_dir.to_string()).await; + remove_dir_if_exists(u128_first_dir.to_string()).await; + + let string_first_config = DiskConfig::new_with_table_name( + string_first_dir, + StringThenU128KeyLayoutWorkTable::name_snake_case(), + StringThenU128KeyLayoutWorkTable::version(), + ); + { + let engine = StringThenU128KeyLayoutPersistenceEngine::new(string_first_config.clone()) + .await + .unwrap(); + let table = StringThenU128KeyLayoutWorkTable::load(engine).await.unwrap(); + table + .insert(StringThenU128KeyLayoutRow { + tenant: "tenant-with-a-long-name".to_string(), + id: u128::MAX - 1, + value: 17, + }) + .await + .unwrap(); + table.close().await.unwrap(); + } + { + let engine = StringThenU128KeyLayoutPersistenceEngine::new(string_first_config) + .await + .unwrap(); + let table = StringThenU128KeyLayoutWorkTable::load(engine).await.unwrap(); + assert_eq!( + table + .select(("tenant-with-a-long-name".to_string(), u128::MAX - 1)) + .unwrap() + .value, + 17 + ); + table.close().await.unwrap(); + } + + let u128_first_config = DiskConfig::new_with_table_name( + u128_first_dir, + U128ThenStringKeyLayoutWorkTable::name_snake_case(), + U128ThenStringKeyLayoutWorkTable::version(), + ); + { + let engine = U128ThenStringKeyLayoutPersistenceEngine::new(u128_first_config.clone()) + .await + .unwrap(); + let table = U128ThenStringKeyLayoutWorkTable::load(engine).await.unwrap(); + table + .insert(U128ThenStringKeyLayoutRow { + id: u128::MAX - 2, + tenant: "another-long-tenant-name".to_string(), + value: 23, + }) + .await + .unwrap(); + table.close().await.unwrap(); + } + { + let engine = U128ThenStringKeyLayoutPersistenceEngine::new(u128_first_config) + .await + .unwrap(); + let table = U128ThenStringKeyLayoutWorkTable::load(engine).await.unwrap(); + assert_eq!( + table + .select((u128::MAX - 2, "another-long-tenant-name".to_string())) + .unwrap() + .value, + 23 + ); + table.close().await.unwrap(); + } + + remove_dir_if_exists(string_first_dir.to_string()).await; + remove_dir_if_exists(u128_first_dir.to_string()).await; +} + +/// The generated primary-key newtype must report its 16-byte archived +/// alignment. Under-reporting it makes the WTI node larger than its serialized +/// default page before the first batch is written. +#[test] +fn u128_primary_index_fits_and_survives_default_page_persistence() { + let dir = "tests/data/u128_primary_index_capacity"; + let config = DiskConfig::new_with_table_name( + dir, + U128PrimaryIndexCapacityWorkTable::name_snake_case(), + U128PrimaryIndexCapacityWorkTable::version(), + ); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_io() + .enable_time() + .build() + .unwrap(); + + runtime.block_on(async { + remove_dir_if_exists(dir.to_string()).await; + { + let engine = U128PrimaryIndexCapacityPersistenceEngine::new(config.clone()) + .await + .unwrap(); + let table = U128PrimaryIndexCapacityWorkTable::load(engine).await.unwrap(); + for id in 0..64u128 { + table + .insert(U128PrimaryIndexCapacityRow { id, value: id as u64 }) + .await + .unwrap(); + } + table.wait_for_ops().await.unwrap(); + } + { + let engine = U128PrimaryIndexCapacityPersistenceEngine::new(config.clone()) + .await + .unwrap(); + let table = U128PrimaryIndexCapacityWorkTable::load(engine).await.unwrap(); + for id in 0..64u128 { + assert_eq!(table.select(id).unwrap().value, id as u64); + } + } + remove_dir_if_exists(dir.to_string()).await; + }); +} diff --git a/tests/persistence/uuid_primary_upsert.rs b/tests/persistence/uuid_primary_upsert.rs new file mode 100644 index 00000000..89e79f50 --- /dev/null +++ b/tests/persistence/uuid_primary_upsert.rs @@ -0,0 +1,155 @@ +use rkyv::{Archive, Deserialize, Serialize}; +use uuid::Uuid; +use worktable::prelude::*; +use worktable::worktable; + +use crate::remove_dir_if_exists; + +#[derive( + Archive, + Clone, + Copy, + Debug, + Default, + Deserialize, + Eq, + Hash, + MemStat, + Ord, + PartialEq, + PartialOrd, + Serialize, + SizeMeasure, +)] +#[rkyv(compare(PartialEq), derive(Clone, Debug, PartialOrd, PartialEq, Eq, Ord))] +struct PersistedPaymentId(Uuid); + +#[derive( + Archive, + Clone, + Copy, + Debug, + Default, + Deserialize, + Eq, + Hash, + MemStat, + Ord, + PartialEq, + PartialOrd, + Serialize, + SizeMeasure, +)] +#[rkyv(compare(PartialEq), derive(Debug, PartialOrd, PartialEq, Eq, Ord))] +struct PersistedAppId(Uuid); + +#[derive(Archive, Clone, Copy, Debug, Deserialize, Eq, Hash, MemStat, Ord, PartialEq, PartialOrd, Serialize)] +#[rkyv(compare(PartialEq), derive(Debug))] +#[repr(u8)] +enum PersistedPaymentStatus { + SetUp, + Confirmed, +} + +worktable! { + name: PersistedUuidPrimaryMutation, + persist: true, + columns: { + id: PersistedPaymentId primary_key using worktables_index, + app_id: PersistedAppId, + endpoint_address: String, + symbol: String, + network: u64 optional, + deposited_amount: String, + goal_amount: String, + goal_amount_usd: String, + status: PersistedPaymentStatus, + created_at: i64, + deposit_until: i64 optional, + completed_at: i64 optional, + }, + indexes: { + symbol_idx: symbol, + endpoint_address_idx: endpoint_address, + app_id_idx: app_id using worktables_index, + created_at_idx: created_at, + }, +} + +fn payment_id(index: usize) -> PersistedPaymentId { + PersistedPaymentId(Uuid::from_u128( + 0x0199_45d2_0000_7000_8000_0000_0000_0000 | index as u128, + )) +} + +fn row(index: usize, status: PersistedPaymentStatus) -> PersistedUuidPrimaryMutationRow { + PersistedUuidPrimaryMutationRow { + id: payment_id(index), + app_id: PersistedAppId(Uuid::from_u128( + 0x66c0_af2b_ef09_4255_86db_9c60_0000_0000 | (index % 32) as u128, + )), + endpoint_address: format!("0x{:040x}", index % 32), + symbol: if index.is_multiple_of(2) { + "usdc".to_owned() + } else { + "pol".to_owned() + }, + network: Some(137), + deposited_amount: format!("{index}.000001"), + goal_amount: format!("{}.025", 100 + index), + goal_amount_usd: format!("{}.50", 25 + index), + status, + created_at: 1_789_000_000 + index as i64, + deposit_until: Some(1_789_000_900 + index as i64), + completed_at: None, + } +} + +#[tokio::test] +async fn pays_shaped_uuid_primary_upserts_survive_batch_boundaries_and_reload() { + const ROWS: usize = 256; + + let directory = std::env::temp_dir().join(format!("worktable-uuid-primary-upsert-{}", Uuid::new_v4())); + let root = directory.to_string_lossy().into_owned(); + remove_dir_if_exists(root.clone()).await; + let config = DiskConfig::new_with_table_name( + &root, + PersistedUuidPrimaryMutationWorkTable::name_snake_case(), + PersistedUuidPrimaryMutationWorkTable::version(), + ); + + { + let engine = PersistedUuidPrimaryMutationPersistenceEngine::new(config.clone()) + .await + .unwrap(); + let table = PersistedUuidPrimaryMutationWorkTable::load(engine).await.unwrap(); + for index in 0..ROWS { + table.insert(row(index, PersistedPaymentStatus::SetUp)).await.unwrap(); + } + table.wait_for_ops().await.unwrap(); + + for index in 0..ROWS { + let permuted = (index * 129) & (ROWS - 1); + table + .upsert(row(permuted, PersistedPaymentStatus::Confirmed)) + .await + .unwrap(); + } + table.wait_for_ops().await.unwrap(); + table.close().await.unwrap(); + } + + let engine = PersistedUuidPrimaryMutationPersistenceEngine::new(config) + .await + .unwrap(); + let table = PersistedUuidPrimaryMutationWorkTable::load(engine).await.unwrap(); + assert_eq!(table.select_all().execute().unwrap().len(), ROWS); + for index in [0, ROWS / 2, ROWS - 1] { + assert_eq!( + table.select(payment_id(index)).unwrap().status, + PersistedPaymentStatus::Confirmed + ); + } + table.close().await.unwrap(); + remove_dir_if_exists(root).await; +} diff --git a/tests/runtime_execution.rs b/tests/runtime_execution.rs index 36f438c5..4dc53808 100644 --- a/tests/runtime_execution.rs +++ b/tests/runtime_execution.rs @@ -12,7 +12,7 @@ worktable! { queries: { update runtime scheduled: { ValueById(value) by id }, delete runtime scheduled: { ByGroup() by group }, - in_place runtime scheduled: { ValueById(value) by id }, + update_in_place runtime scheduled: { ValueById(value) by id }, } } @@ -47,19 +47,13 @@ fn generated_profiles_dispatch_mutations_and_owned_selects() { .await .unwrap(); } - table - .update_value_by_id(ValueByIdQuery { value: 100 }, 9u64) - .await - .unwrap(); + table.update_by_id(9, ScheduledColumns::VALUE, 100).await.unwrap(); let caller = std::thread::current().id(); table - .update_value_by_id_in_place( - move |value| { - assert_ne!(std::thread::current().id(), caller); - *value = 101.into(); - }, - 9u64, - ) + .update_in_place_by_id(9, ScheduledColumns::VALUE, move |value| { + assert_ne!(std::thread::current().id(), caller); + *value = 101.into(); + }) .await .unwrap(); assert_eq!(table.select(9u64).unwrap().value, 101); @@ -102,10 +96,7 @@ fn nested_dispatch_progresses_on_one_worker() { }) .await .unwrap(); - table - .update_value_by_id(ValueByIdQuery { value: 4 }, 1u64) - .await - .unwrap(); + table.update_by_id(1, ScheduledColumns::VALUE, 4).await.unwrap(); table.select_all().runtime(scheduled).execute_async().await.unwrap()[0].value }) .await @@ -179,10 +170,7 @@ fn scheduled_mutation_is_persisted_and_reopened() { let engine = ScheduledDiskPersistenceEngine::new(config.clone()).await.unwrap(); let table = Arc::new(ScheduledDiskWorkTable::load(engine).await.unwrap()); table.insert(ScheduledDiskRow { id: 1, value: 2 }).await.unwrap(); - table - .update_disk_value_by_id(DiskValueByIdQuery { value: 99 }, 1u64) - .await - .unwrap(); + table.update_by_id(1, ScheduledDiskColumns::VALUE, 99).await.unwrap(); assert_eq!( table.select_all().runtime(scheduled).execute_async().await.unwrap()[0].value, 99 @@ -205,7 +193,7 @@ mod tokio_execution { name: TokioScheduled, runtime: tokio, columns: { id: u64 primary_key, value: u64 }, - queries: { in_place runtime on_tokio: { TokioValueById(value) by id } } + queries: { update_in_place runtime on_tokio: { TokioValueById(value) by id } } } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn tokio_profiles_dispatch_on_the_entered_runtime() { @@ -213,13 +201,10 @@ mod tokio_execution { let table = Arc::new(TokioScheduledWorkTable::default()); table.insert(TokioScheduledRow { id: 1, value: 0 }).await.unwrap(); table - .update_tokio_value_by_id_in_place( - move |value| { - assert_ne!(std::thread::current().id(), caller); - *value = 8.into(); - }, - 1u64, - ) + .update_in_place_by_id(1, TokioScheduledColumns::VALUE, move |value| { + assert_ne!(std::thread::current().id(), caller); + *value = 8.into(); + }) .await .unwrap(); assert_eq!( @@ -270,7 +255,7 @@ runtimes! { on_spread: nagoya(spread), } worktable! { name: Tunable, columns: { id: u64 primary_key, value: u64 }, - queries: { in_place runtime on_spread: { TunedValue(value) by id } } + queries: { update_in_place runtime on_spread: { TunedValue(value) by id } } } #[test] fn callsites_can_tune_nagoya_without_changing_the_table_default() { @@ -279,13 +264,10 @@ fn callsites_can_tune_nagoya_without_changing_the_table_default() { table.insert(TunableRow { id: 1, value: 2 }).await.unwrap(); let caller = std::thread::current().id(); table - .update_tuned_value_in_place( - move |value| { - assert_ne!(std::thread::current().id(), caller); - *value = 3.into(); - }, - 1u64, - ) + .update_in_place_by_id(1, TunableColumns::VALUE, move |value| { + assert_ne!(std::thread::current().id(), caller); + *value = 3.into(); + }) .await .unwrap(); assert_eq!( diff --git a/tests/slotted_page_requirement.rs b/tests/slotted_page_requirement.rs index 4b039bff..323ba462 100644 --- a/tests/slotted_page_requirement.rs +++ b/tests/slotted_page_requirement.rs @@ -194,3 +194,50 @@ async fn a_store_reopens_without_being_rebuilt() { reopened.close().await.expect("the table closes"); let _ = std::fs::remove_dir_all(dir); } + +/// Dropping a busy table requests a close. That close must finish before the +/// last owner disappears, or an immediate same-path reopen races a detached +/// writer and sees either a partial table or torn page headers. +#[tokio::test] +async fn a_busy_drop_finishes_before_an_immediate_reopen() { + const ACCEPTED: u64 = 300; + let dir = "tests/data/slotted_page/busy_drop_reopen"; + let _ = std::fs::remove_dir_all(dir); + + let engine = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await + .expect("an engine"); + let table = SlottedRowWorkTable::load(engine).await.expect("a table"); + for n in 0..ACCEPTED { + table + .insert(SlottedRowRow { + id: table.get_next_pk().into(), + blob: format!("accepted row {n}"), + }) + .await + .expect("an accepted row"); + } + drop(table); + + let engine = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await + .expect("the immediate reopen does not observe torn files"); + let reopened = SlottedRowWorkTable::load(engine) + .await + .expect("the immediately reopened table"); + assert_eq!( + reopened.select_all().execute().expect("a read").len(), + ACCEPTED as usize, + "every operation accepted before drop is durable when reopen begins" + ); + reopened.close().await.expect("the reopened table closes"); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/tests/ui.rs b/tests/ui.rs index 240194fa..7756b019 100644 --- a/tests/ui.rs +++ b/tests/ui.rs @@ -35,4 +35,9 @@ fn compile_fail() { // Query rules. t.compile_fail("tests/ui/autoincrement_unsupported_key.rs"); t.compile_fail("tests/ui/in_place_over_indexed_column.rs"); + t.compile_fail("tests/ui/update_selector_wrong_value.rs"); + t.compile_fail("tests/ui/update_selector_collision.rs"); + + // Read-path safety rules. + t.compile_fail("tests/ui/select_with_needs_inline_archived.rs"); } diff --git a/tests/ui/in_place_over_indexed_column.rs b/tests/ui/in_place_over_indexed_column.rs index b982ac51..29b20cc1 100644 --- a/tests/ui/in_place_over_indexed_column.rs +++ b/tests/ui/in_place_over_indexed_column.rs @@ -1,4 +1,4 @@ -// Rule: an `in_place` query writes the archived column bytes and maintains no +// Rule: an `update_in_place` query writes the archived column bytes and maintains no // index, so a column any index is built over cannot be mutated on that path. // The index would keep resolving the old value. use worktable::worktable; @@ -14,7 +14,7 @@ worktable! { value_idx: value unique, }, queries: { - in_place: { + update_in_place: { ValueById(value) by id, } }, diff --git a/tests/ui/in_place_over_indexed_column.stderr b/tests/ui/in_place_over_indexed_column.stderr index 9b615ad9..5ea4e79c 100644 --- a/tests/ui/in_place_over_indexed_column.stderr +++ b/tests/ui/in_place_over_indexed_column.stderr @@ -1,4 +1,4 @@ -error: in_place query `ValueById` mutates column `value`, which is covered by an index; indexed columns cannot be updated in place because secondary indexes are not maintained on this path. Use an `update` query instead +error: update_in_place query `ValueById` mutates column `value`, which is covered by an index; indexed columns cannot be updated in place because secondary indexes are not maintained on this path. Use an `update` query instead --> tests/ui/in_place_over_indexed_column.rs:18:23 | 18 | ValueById(value) by id, diff --git a/tests/ui/select_with_needs_inline_archived.rs b/tests/ui/select_with_needs_inline_archived.rs new file mode 100644 index 00000000..03387936 --- /dev/null +++ b/tests/ui/select_with_needs_inline_archived.rs @@ -0,0 +1,22 @@ +// Rule: `select_with` reads the archived cell in place with no copy, so a +// concurrent writer can tear a value the closure observes. That is recoverable +// for an inline scalar, because the seqlock retry discards whatever the closure +// computed. It is undefined behaviour for a relative pointer, which is what an +// archived `String` or `Vec` column is. Tables carrying one therefore do not +// get a `select_with` at all; they use the owned `select`. +use worktable::prelude::*; +use worktable::worktable; + +worktable! { + name: HasString, + persist: false, + columns: { + id: u64 primary_key, + label: String, + }, +} + +fn main() { + let table = HasStringWorkTable::default(); + let _ = table.select_with(1u64, |archived| archived.id); +} diff --git a/tests/ui/select_with_needs_inline_archived.stderr b/tests/ui/select_with_needs_inline_archived.stderr new file mode 100644 index 00000000..32d89ad6 --- /dev/null +++ b/tests/ui/select_with_needs_inline_archived.stderr @@ -0,0 +1,31 @@ +error[E0599]: no method named `select_with` found for struct `HasStringWorkTable` in the current scope + --> tests/ui/select_with_needs_inline_archived.rs:21:19 + | +10 | / worktable! { +11 | | name: HasString, +12 | | persist: false, +13 | | columns: { +... | +16 | | }, +17 | | } + | |_- method `select_with` not found for this struct +... +21 | let _ = table.select_with(1u64, |archived| archived.id); + | ^^^^^^^^^^^ + | +help: there is a method `select` with a similar name, but with different arguments + --> tests/ui/select_with_needs_inline_archived.rs:10:1 + | +10 | / worktable! { +11 | | name: HasString, +12 | | persist: false, +13 | | columns: { +... | +16 | | }, +17 | | } + | |_^ + = note: this error originates in the macro `worktable` (in Nightly builds, run with -Z macro-backtrace for more info) +help: one of the expressions' fields has a method of the same name + | +21 | let _ = table.0.select_with(1u64, |archived| archived.id); + | ++ diff --git a/tests/ui/update_selector_collision.rs b/tests/ui/update_selector_collision.rs new file mode 100644 index 00000000..b6c609eb --- /dev/null +++ b/tests/ui/update_selector_collision.rs @@ -0,0 +1,17 @@ +use worktable::worktable; + +worktable!( + name: SelectorCollision, + columns: { + id: u64 primary_key, + amount: u64, + }, + queries: { + update: { + First(amount) by id, + Second(amount) by id, + } + } +); + +fn main() {} diff --git a/tests/ui/update_selector_collision.stderr b/tests/ui/update_selector_collision.stderr new file mode 100644 index 00000000..3a2a8834 --- /dev/null +++ b/tests/ui/update_selector_collision.stderr @@ -0,0 +1,5 @@ +error: update queries `First` and `Second` both generate `update_by_id(..., Columns::AMOUNT, ...)` + --> tests/ui/update_selector_collision.rs:12:13 + | +12 | Second(amount) by id, + | ^^^^^^ diff --git a/tests/ui/update_selector_wrong_value.rs b/tests/ui/update_selector_wrong_value.rs new file mode 100644 index 00000000..e115c21a --- /dev/null +++ b/tests/ui/update_selector_wrong_value.rs @@ -0,0 +1,18 @@ +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: SelectorTypes, + columns: { + id: u64 primary_key, + amount: u64, + }, + queries: { + update: { AmountById(amount) by id } + } +); + +fn main() { + let table = SelectorTypesWorkTable::default(); + let _ = table.update_by_id(1, SelectorTypesColumns::AMOUNT, "not a u64"); +} diff --git a/tests/ui/update_selector_wrong_value.stderr b/tests/ui/update_selector_wrong_value.stderr new file mode 100644 index 00000000..7e07e5d0 --- /dev/null +++ b/tests/ui/update_selector_wrong_value.stderr @@ -0,0 +1,26 @@ +error[E0308]: mismatched types + --> tests/ui/update_selector_wrong_value.rs:17:65 + | +17 | let _ = table.update_by_id(1, SelectorTypesColumns::AMOUNT, "not a u64"); + | ------------ ^^^^^^^^^^^ expected `u64`, found `&str` + | | + | arguments to this method are incorrect + | +help: the return type of this call is `&'static str` due to the type of the argument passed + --> tests/ui/update_selector_wrong_value.rs:17:13 + | +17 | let _ = table.update_by_id(1, SelectorTypesColumns::AMOUNT, "not a u64"); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^-----------^ + | | + | this argument influences the return type of `update_by_id` +note: method defined here + --> tests/ui/update_selector_wrong_value.rs:4:1 + | + 4 | / worktable!( + 5 | | name: SelectorTypes, + 6 | | columns: { + 7 | | id: u64 primary_key, +... | +13 | | ); + | |_^ + = note: this error originates in the macro `worktable` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/worktable/array.rs b/tests/worktable/array.rs index 3927c4ec..2c2cccb5 100644 --- a/tests/worktable/array.rs +++ b/tests/worktable/array.rs @@ -33,7 +33,7 @@ async fn update() { let row = TestRow { id: 1, test: [1; 20] }; let pk = table.insert(row.clone()).await.unwrap(); let new_row = TestRow { id: 1, test: [2; 20] }; - table.update(new_row.clone()).await.unwrap(); + table.replace(new_row.clone()).await.unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row, new_row); @@ -48,7 +48,7 @@ async fn update_in_a_middle() { let _ = table.insert(row.clone()).await.unwrap(); } let new_row = TestRow { id: 3, test: [1; 20] }; - table.update(new_row.clone()).await.unwrap(); + table.replace(new_row.clone()).await.unwrap(); let selected_row = table.select(3).unwrap(); assert_eq!(selected_row, new_row); @@ -60,7 +60,10 @@ async fn update_query() { let row = TestRow { id: 1, test: [1; 20] }; let pk = table.insert(row.clone()).await.unwrap(); let q = TestByIdQuery { test: [2; 20] }; - table.update_test_by_id(q.clone(), pk.clone()).await.unwrap(); + table + .update_by_id(pk.clone(), TestColumns::TEST, (q.clone()).test) + .await + .unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row.test, q.test); @@ -99,7 +102,7 @@ async fn update_i() { let row = TestIRow { id: 1, test: [1; 20] }; let pk = table.insert(row.clone()).await.unwrap(); let new_row = TestIRow { id: 1, test: [2; 20] }; - table.update(new_row.clone()).await.unwrap(); + table.replace(new_row.clone()).await.unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row, new_row); @@ -114,7 +117,7 @@ async fn update_in_a_middle_i() { let _ = table.insert(row.clone()).await.unwrap(); } let new_row = TestIRow { id: 3, test: [1; 20] }; - table.update(new_row.clone()).await.unwrap(); + table.replace(new_row.clone()).await.unwrap(); let selected_row = table.select(3).unwrap(); assert_eq!(selected_row, new_row); @@ -126,7 +129,10 @@ async fn update_query_i() { let row = TestIRow { id: 1, test: [1; 20] }; let pk = table.insert(row.clone()).await.unwrap(); let q = TestIByIdQuery { test: [2; 20] }; - table.update_test_i_by_id(q.clone(), pk.clone()).await.unwrap(); + table + .update_by_id(pk.clone(), TestIColumns::TEST, (q.clone()).test) + .await + .unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row.test, q.test); diff --git a/tests/worktable/base.rs b/tests/worktable/base.rs index 1cd957a9..7092909b 100644 --- a/tests/worktable/base.rs +++ b/tests/worktable/base.rs @@ -128,7 +128,7 @@ async fn update_spawn() { }; let shared = table.clone(); let shared_updated = updated.clone(); - tokio::spawn(async move { shared.update(shared_updated).await }) + tokio::spawn(async move { shared.replace(shared_updated).await }) .await .unwrap() .unwrap(); @@ -182,7 +182,7 @@ async fn update() { another: 3, exchange: "test".to_string(), }; - table.update(updated.clone()).await.unwrap(); + table.replace(updated.clone()).await.unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row, updated); @@ -206,7 +206,7 @@ async fn update_string() { another: 3, exchange: "much bigger test to make size of new row bigger than previous one".to_string(), }; - table.update(updated.clone()).await.unwrap(); + table.replace(updated.clone()).await.unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row, updated); @@ -238,7 +238,7 @@ async fn update_parallel() { let val = fastrand::u64(..); let id_to_update = fastrand::i64(1..=100); shared - .update_another_by_test(AnotherByTestQuery { another: val }, id_to_update) + .update_by_test(id_to_update, TestColumns::ANOTHER, val) .await .unwrap(); { @@ -253,7 +253,7 @@ async fn update_parallel() { let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..=99); table - .update_another_by_id(AnotherByIdQuery { another: val }, id_to_update) + .update_by_id(id_to_update, TestColumns::ANOTHER, val) .await .unwrap(); { @@ -288,15 +288,13 @@ async fn secondary_update_follows_concurrent_row_relocation() { let writer = tokio::spawn(async move { writer_barrier.wait().await; for revision in 1..=2000 { - writer_table.update_exchange_by_id(ExchangeByIdQuery { - exchange: format!("relocated-{revision}-{}", "x".repeat(revision % 64)), - }, 0).await.unwrap(); + writer_table.update_by_id(0, TestColumns::EXCHANGE, format!("relocated-{revision}-{}", "x".repeat(revision % 64))).await.unwrap(); tokio::task::yield_now().await; } }); barrier.wait().await; for revision in 1..=2000 { - table.update_another_by_test(AnotherByTestQuery { another: revision }, 1) + table.update_by_test(1, TestColumns::ANOTHER, revision) .await.unwrap(); tokio::task::yield_now().await; } @@ -582,6 +580,52 @@ async fn select_multiple_by_exchange() { ) } +#[tokio::test] +async fn select_ref_matches_select() { + let table = TestWorkTable::default(); + let row = TestRow { + id: table.get_next_pk().into(), + test: 1, + another: 1, + exchange: "test".to_string(), + }; + let _ = table.insert(row.clone()).await.unwrap(); + let owned = table.select(row.id).unwrap(); + let view = table.select_ref(row.id).unwrap(); + assert_eq!(owned.another, view.another); + assert_eq!(owned.test, view.test); + assert!(table.select_ref(u64::MAX).is_none()); +} + +worktable! { + name: InlineOnly, + columns: { + id: u64 primary_key, + value: u64, + }, +} + +/// `select_with` exists only where the archived row holds no relative +/// pointers, and it agrees with the owned `select` there. +/// +/// It reads the cell in place with no copy, so a concurrent writer can tear a +/// value the closure sees. That is recoverable for an inline scalar, because +/// the seqlock retry discards it, and is undefined behaviour for a pointer. +/// `TestWorkTable` above has a `String` column and deliberately has no +/// `select_with`; see the `select_with_needs_inline_archived` compile-fail +/// case. +#[tokio::test] +async fn select_with_matches_select_on_inline_rows() { + let table = InlineOnlyWorkTable::default(); + let row = InlineOnlyRow { id: 1, value: 42 }; + let _ = table.insert(row.clone()).await.unwrap(); + + let owned = table.select(row.id).unwrap(); + let via_with = table.select_with(row.id, |archived| archived.value).unwrap(); + assert_eq!(owned.value, u64::from(via_with)); + assert!(table.select_with(u64::MAX, |archived| archived.value).is_none()); +} + #[tokio::test] async fn select_by_test() { let table = TestWorkTable::default(); @@ -1152,7 +1196,7 @@ async fn test_update_by_non_unique() { let _ = table.insert(row2.clone()).await.unwrap(); let row = AnotherByExchangeQuery { another: 3 }; - table.update_another_by_exchange(row, "test".to_string()).await.unwrap(); + table.update_by_exchange("test".to_string(), TestColumns::ANOTHER, (row).another).await.unwrap(); let all = table.select_all().execute().unwrap(); @@ -1189,7 +1233,7 @@ async fn test_update_by_unique() { let _ = table.insert(row.clone()).await.unwrap(); let row = AnotherByTestQuery { another: 3 }; - table.update_another_by_test(row, 1).await.unwrap(); + table.update_by_test(1, TestColumns::ANOTHER, (row).another).await.unwrap(); let row = table.select_by_test(1).unwrap(); @@ -1216,7 +1260,7 @@ async fn test_update_by_pk() { let pk = table.insert(row.clone()).await.unwrap(); let row = AnotherByIdQuery { another: 3 }; - table.update_another_by_id(row, pk).await.unwrap(); + table.update_by_id(pk, TestColumns::ANOTHER, (row).another).await.unwrap(); let row = table.select_by_test(1).unwrap(); diff --git a/tests/worktable/bench.rs b/tests/worktable/bench.rs index d7d24524..ca574766 100644 --- a/tests/worktable/bench.rs +++ b/tests/worktable/bench.rs @@ -66,13 +66,18 @@ async fn _rw_lock_hash_map_vs_wt() { for i in 0..100000u64 { let s: String = Alphanumeric.sample_string(&mut rand::rng(), 8); let q = ValueByIdQuery { value: s }; - task_wt.update_value_by_id(q, (i % 50) * 2).await.unwrap(); + task_wt + .update_by_id((i % 50) * 2, MapColumns::VALUE, (q).value) + .await + .unwrap(); } }); for i in 0..100000u64 { let s: String = Alphanumeric.sample_string(&mut rand::rng(), 8); let q = ValueByIdQuery { value: s }; - wt.update_value_by_id(q, (i % 50) * 2 + 1).await.unwrap(); + wt.update_by_id((i % 50) * 2 + 1, MapColumns::VALUE, (q).value) + .await + .unwrap(); } h.await.unwrap(); println!("wt update in {} μs", wt_start.elapsed().as_micros()); diff --git a/tests/worktable/borrowed_primary_key.rs b/tests/worktable/borrowed_primary_key.rs index aef33b0f..acd8fe60 100644 --- a/tests/worktable/borrowed_primary_key.rs +++ b/tests/worktable/borrowed_primary_key.rs @@ -11,7 +11,7 @@ worktable!( update: { BorrowedValueById(value) by id, } - in_place: { + update_in_place: { BorrowedValueById(value) by id, } } @@ -43,11 +43,11 @@ async fn string_primary_key_accepts_borrowed_forms() { assert_eq!(table.select(&generated), Some(row)); table - .update_borrowed_value_by_id(BorrowedValueByIdQuery { value: 8 }, &id) + .update_by_id(&id, BorrowedStringKeyColumns::VALUE, 8) .await .unwrap(); table - .update_borrowed_value_by_id_in_place(|value| *value += 1, &id) + .update_in_place_by_id(&id, BorrowedStringKeyColumns::VALUE, |value| *value += 1) .await .unwrap(); assert_eq!(table.select(&id).unwrap().value, 9); diff --git a/tests/worktable/cancel_safety.rs b/tests/worktable/cancel_safety.rs index 0ada7eb6..86074571 100644 --- a/tests/worktable/cancel_safety.rs +++ b/tests/worktable/cancel_safety.rs @@ -55,7 +55,7 @@ async fn cancelled_full_row_update_releases_registered_lock() { // blocker; the timeout cancels it at exactly that await. let cancelled = tokio::time::timeout( Duration::from_millis(200), - table.update(CancelSafetyRow { + table.replace(CancelSafetyRow { id: 1, value: 7, other: 7, @@ -73,7 +73,7 @@ async fn cancelled_full_row_update_releases_registered_lock() { // failure instead of a hang. tokio::time::timeout( Duration::from_secs(5), - table.update(CancelSafetyRow { + table.replace(CancelSafetyRow { id: 1, value: 9, other: 9, @@ -109,7 +109,7 @@ async fn cancelled_custom_update_releases_registered_lock() { let cancelled = tokio::time::timeout( Duration::from_millis(200), - table.update_value_by_id(ValueByIdQuery { value: 5 }, 3), + table.update_by_id(3, CancelSafetyColumns::VALUE, 5), ) .await; assert!( @@ -121,7 +121,7 @@ async fn cancelled_custom_update_releases_registered_lock() { tokio::time::timeout( Duration::from_secs(5), - table.update_value_by_id(ValueByIdQuery { value: 11 }, 3), + table.update_by_id(3, CancelSafetyColumns::VALUE, 11), ) .await .expect("update after a cancelled predecessor must not hang") diff --git a/tests/worktable/columnar.rs b/tests/worktable/columnar.rs index b4ea942a..33840d64 100644 --- a/tests/worktable/columnar.rs +++ b/tests/worktable/columnar.rs @@ -43,7 +43,7 @@ worktable!( update: { TemperatureById(temperature) by id, }, - in_place: { + update_in_place: { TimestampById(timestamp) by id, } }, @@ -142,7 +142,7 @@ async fn columnar_fields_and_clustered_index_follow_mutations() { assert_eq!(projected.iter().map(|(_, value)| *value).collect::>(), [1, 2]); table - .update(ColumnarMetricsRow { + .replace(ColumnarMetricsRow { id: 1, host_id: 3, timestamp: 30, @@ -158,13 +158,13 @@ async fn columnar_fields_and_clustered_index_follow_mutations() { assert_eq!(table.columnar_project_temperature(&updated).unwrap()[0].1, 75); table - .update_temperature_by_id(TemperatureByIdQuery { temperature: 76 }, 1) + .update_by_id(1, ColumnarMetricsColumns::TEMPERATURE, 76) .await .unwrap(); assert_eq!(table.columnar_project_temperature(&updated).unwrap()[0].1, 76); table - .update_timestamp_by_id_in_place(|value| *value = 40.into(), 1) + .update_in_place_by_id(1, ColumnarMetricsColumns::TIMESTAMP, |value| *value = 40.into()) .await .unwrap(); assert!(table.columnar_is_dirty()); @@ -198,7 +198,7 @@ async fn concurrent_reinsert_and_columnar_refresh_preserve_row_identity() { tokio::spawn(async move { for value in 1..=200 { table - .update(ColumnarMetricsRow { + .replace(ColumnarMetricsRow { id: 7, host_id: 1, timestamp: value, @@ -308,7 +308,7 @@ async fn columnar_side_indexes_compose_with_congee_and_arctic_using_backends() { [10, 20] ); - table.update($row { id: 1, value: 5 }).await.unwrap(); + table.replace($row { id: 1, value: 5 }).await.unwrap(); assert_eq!(table.columnar_select_value_order(20).unwrap(), []); assert_eq!(table.columnar_select_value_order(5).unwrap().len(), 1); diff --git a/tests/worktable/concurrency.rs b/tests/worktable/concurrency.rs index 75382664..dcb43459 100644 --- a/tests/worktable/concurrency.rs +++ b/tests/worktable/concurrency.rs @@ -494,3 +494,67 @@ macro_rules! backend_suite { backend_suite!(wti, worktables_index, "wti"); backend_suite!(arctic, arctic, "arctic"); backend_suite!(congee, congee, "congee"); + +/// Concurrent in-place updates of ONE key must not lose a write. +/// +/// `update_in_place` takes an uncontended fast path that claims the row with a +/// flag instead of registering an operation and awaiting its predecessor. That +/// path is only correct while it is genuinely uncontended, so the interesting +/// case is the one it must refuse: every writer aimed at the same key, where +/// all but one have to fall back to the chained protocol. +/// +/// The row is an increment counter, so a lost claim is a lost increment and the +/// final value says exactly how many writes survived. +mod same_key_in_place { + use worktable::prelude::*; + use worktable::worktable; + + worktable! { + name: Counter, + columns: { + id: u64 primary_key, + hits: u64, + }, + queries: { + update_in_place: { + HitsById(hits) by id, + }, + }, + } + + #[test] + fn concurrent_in_place_on_one_key_loses_nothing() { + let writers = super::params::env_u64("WT_CONC_WRITERS", 8) as usize; + let per_writer = super::params::env_u64("WT_CONC_PER_WRITER", 500); + + let table = std::sync::Arc::new(CounterWorkTable::default()); + nagoya::block_on(table.insert(CounterRow { id: 1, hits: 0 })).expect("seed"); + + std::thread::scope(|scope| { + for _ in 0..writers { + let table = table.clone(); + scope.spawn(move || { + nagoya::block_on(async { + for _ in 0..per_writer { + table + .update_in_place_by_id(1u64, CounterColumns::HITS, |hits| { + let current: u64 = (*hits).into(); + *hits = current.wrapping_add(1).into(); + }) + .await + .expect("in-place"); + } + }); + }); + } + }); + + let hits: u64 = table.select(1u64).expect("row present").hits; + assert_eq!( + hits, + writers as u64 * per_writer, + "{writers} writers x {per_writer} increments lost writes: the fast path \ + claimed a row another writer already held" + ); + } +} diff --git a/tests/worktable/duplicate_pk_index_update.rs b/tests/worktable/duplicate_pk_index_update.rs new file mode 100644 index 00000000..3702f675 --- /dev/null +++ b/tests/worktable/duplicate_pk_index_update.rs @@ -0,0 +1,37 @@ +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: DuplicatePrimaryIndex, + columns: { + id: u64 primary_key, + value: u64, + }, + indexes: { + id_idx: id unique, + }, + queries: { + update: { + ValueById(value) by id, + } + } +); + +/// An explicit index on the primary-key field has routing precedence in the +/// generated update query. Its hidden method accepts the raw indexed type, so +/// typed selector dispatch must not wrap that key as the table primary key. +#[tokio::test] +async fn update_by_primary_key_duplicated_as_unique_index_uses_raw_key() { + let table = DuplicatePrimaryIndexWorkTable::default(); + table + .insert(DuplicatePrimaryIndexRow { id: 7, value: 1 }) + .await + .unwrap(); + + table + .update_by_id(7, DuplicatePrimaryIndexColumns::VALUE, 9) + .await + .unwrap(); + + assert_eq!(table.select(7).unwrap().value, 9); +} diff --git a/tests/worktable/in_place.rs b/tests/worktable/in_place.rs index 5265e524..d2ae96e5 100644 --- a/tests/worktable/in_place.rs +++ b/tests/worktable/in_place.rs @@ -21,9 +21,10 @@ worktable!( something: u64, }, queries: { - in_place: { + update_in_place: { ValById(val) by id, Val2ById(val2) by id, + ValAndVal2ById(val, val2) by id, } update: { AnotherById(another) by id, @@ -32,6 +33,32 @@ worktable!( } ); +#[tokio::test] +async fn test_update_two_fields_atomically_by_id() -> eyre::Result<()> { + let table = TestWorkTable::default(); + let pk = table + .insert(TestRow { + id: table.get_next_pk().0, + val: 3, + val1: 0, + val2: 5, + another: "another".to_string(), + something: 0, + }) + .await?; + + table + .update_in_place_by_id(pk.0, TestColumns::VAL_AND_VAL2, |(val, val2)| { + *val += 7; + *val2 += 11; + }) + .await?; + + let row = table.select(pk).unwrap(); + assert_eq!((row.val, row.val2), (10, 16)); + Ok(()) +} + #[tokio::test] async fn test_update_val_by_id() -> eyre::Result<()> { let table = TestWorkTable::default(); @@ -45,7 +72,7 @@ async fn test_update_val_by_id() -> eyre::Result<()> { }; let pk = table.insert(row).await?; for _ in 0..10000 { - table.update_val_by_id_in_place(|val| *val += 1, pk.0).await? + table.update_in_place_by_id(pk.0, TestColumns::VAL, |val| *val += 1).await? } let row = table.select(pk).unwrap(); assert_eq!(row.val, 10000); @@ -65,7 +92,7 @@ async fn test_update_val2_by_id() -> eyre::Result<()> { }; let pk = table.insert(row).await?; for _ in 0..100 { - table.update_val_2_by_id_in_place(|val| *val += 1, pk.0).await? + table.update_in_place_by_id(pk.0, TestColumns::VAL2, |val| *val += 1).await? } let row = table.select(pk).unwrap(); assert_eq!(row.val2, 100); @@ -88,13 +115,13 @@ async fn test_update_val_by_id_two_thread() -> eyre::Result<()> { let h = tokio::spawn(async move { for _ in 0..10_000 { shared_table - .update_val_by_id_in_place(|val| *val += 1, pk.0) + .update_in_place_by_id(pk.0, TestColumns::VAL, |val| *val += 1) .await .unwrap() } }); for _ in 0..10_000 { - table.update_val_by_id_in_place(|val| *val += 1, pk.0).await? + table.update_in_place_by_id(pk.0, TestColumns::VAL, |val| *val += 1).await? } h.await?; let row = table.select(pk).unwrap(); @@ -118,7 +145,7 @@ async fn test_update_val_and_val2_by_id_four_thread() -> eyre::Result<()> { let h1 = tokio::spawn(async move { for _ in 0..10_000 { shared_table - .update_val_by_id_in_place(|val| *val += 1, pk.0) + .update_in_place_by_id(pk.0, TestColumns::VAL, |val| *val += 1) .await .unwrap() } @@ -127,7 +154,7 @@ async fn test_update_val_and_val2_by_id_four_thread() -> eyre::Result<()> { let h2 = tokio::spawn(async move { for _ in 0..10_000 { shared_table - .update_val_2_by_id_in_place(|val| *val += 1, pk.0) + .update_in_place_by_id(pk.0, TestColumns::VAL2, |val| *val += 1) .await .unwrap() } @@ -136,13 +163,13 @@ async fn test_update_val_and_val2_by_id_four_thread() -> eyre::Result<()> { let h3 = tokio::spawn(async move { for _ in 0..10_000 { shared_table - .update_val_by_id_in_place(|val| *val += 1, pk.0) + .update_in_place_by_id(pk.0, TestColumns::VAL, |val| *val += 1) .await .unwrap() } }); for _ in 0..10_000 { - table.update_val_2_by_id_in_place(|val| *val += 1, pk.0).await? + table.update_in_place_by_id(pk.0, TestColumns::VAL2, |val| *val += 1).await? } h1.await?; h2.await?; @@ -169,7 +196,7 @@ async fn test_update_val_by_id_four_thread() -> eyre::Result<()> { let h1 = tokio::spawn(async move { for _ in 0..10_000 { shared_table - .update_val_by_id_in_place(|val| *val += 1, pk.0) + .update_in_place_by_id(pk.0, TestColumns::VAL, |val| *val += 1) .await .unwrap() } @@ -178,7 +205,7 @@ async fn test_update_val_by_id_four_thread() -> eyre::Result<()> { let h2 = tokio::spawn(async move { for _ in 0..10_000 { shared_table - .update_val_by_id_in_place(|val| *val += 1, pk.0) + .update_in_place_by_id(pk.0, TestColumns::VAL, |val| *val += 1) .await .unwrap() } @@ -187,13 +214,13 @@ async fn test_update_val_by_id_four_thread() -> eyre::Result<()> { let h3 = tokio::spawn(async move { for _ in 0..10_000 { shared_table - .update_val_by_id_in_place(|val| *val += 1, pk.0) + .update_in_place_by_id(pk.0, TestColumns::VAL, |val| *val += 1) .await .unwrap() } }); for _ in 0..10_000 { - table.update_val_by_id_in_place(|val| *val += 1, pk.0).await? + table.update_in_place_by_id(pk.0, TestColumns::VAL, |val| *val += 1).await? } h1.await?; h2.await?; @@ -227,7 +254,7 @@ async fn test_update_in_place_and_update_sized_multithread() -> eyre::Result<()> let val = fastrand::i64(..); let id_to_update = fastrand::u64(0..=99); shared - .update_val_by_id_in_place(|v| *v = val.into(), id_to_update) + .update_in_place_by_id(id_to_update, TestColumns::VAL, |v| *v = val.into()) .await .unwrap(); { @@ -243,7 +270,7 @@ async fn test_update_in_place_and_update_sized_multithread() -> eyre::Result<()> let val = fastrand::i16(..); let id_to_update = fastrand::u64(0..=99); shared - .update_val_2_by_id_in_place(|v| *v = val.into(), id_to_update) + .update_in_place_by_id(id_to_update, TestColumns::VAL2, |v| *v = val.into()) .await .unwrap(); { @@ -257,7 +284,7 @@ async fn test_update_in_place_and_update_sized_multithread() -> eyre::Result<()> let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..=99); table - .update_something_by_id(SomethingByIdQuery { something: val }, id_to_update) + .update_by_id(id_to_update, TestColumns::SOMETHING, val) .await?; { let mut guard = i_state.lock(); @@ -306,7 +333,7 @@ async fn test_update_in_place_and_update_unsized_multithread() -> eyre::Result<( let val = fastrand::i64(..); let id_to_update = fastrand::u64(0..=99); shared - .update_val_by_id_in_place(|v| *v = val.into(), id_to_update) + .update_in_place_by_id(id_to_update, TestColumns::VAL, |v| *v = val.into()) .await .unwrap(); { @@ -322,7 +349,7 @@ async fn test_update_in_place_and_update_unsized_multithread() -> eyre::Result<( let val = fastrand::i16(..); let id_to_update = fastrand::u64(0..=99); shared - .update_val_2_by_id_in_place(|v| *v = val.into(), id_to_update) + .update_in_place_by_id(id_to_update, TestColumns::VAL2, |v| *v = val.into()) .await .unwrap(); { @@ -336,12 +363,7 @@ async fn test_update_in_place_and_update_unsized_multithread() -> eyre::Result<( let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..=99); table - .update_another_by_id( - AnotherByIdQuery { - another: format!("another_{val}"), - }, - id_to_update, - ) + .update_by_id(id_to_update, TestColumns::ANOTHER, format!("another_{val}")) .await?; { let mut guard = i_state.lock(); diff --git a/tests/worktable/index/mod.rs b/tests/worktable/index/mod.rs index dae8dc52..9cb51faa 100644 --- a/tests/worktable/index/mod.rs +++ b/tests/worktable/index/mod.rs @@ -103,12 +103,13 @@ async fn update_2_idx() { let pk = test_table.insert(row.clone()).await.unwrap(); test_table - .update_all_attr_by_id( + .update_by_id( + pk.clone(), + Test2Columns::ATTR1_AND_ATTR2, AllAttrByIdQuery { attr1: attr1_new.clone(), attr2: attr2_new, }, - pk.clone(), ) .await .unwrap(); @@ -145,7 +146,7 @@ async fn update_2_idx_full_row() { let pk = test_table.insert(row.clone()).await.unwrap(); test_table - .update(Test2Row { + .replace(Test2Row { id: pk.clone().into(), attr1: attr1_new.clone(), attr2: attr2_new, @@ -209,12 +210,7 @@ async fn update_1_idx() { let pk = test_table.insert(row.clone()).await.unwrap(); test_table - .update_attr_1_by_id( - Attr1ByIdQuery { - attr1: attr1_new.clone(), - }, - pk.clone(), - ) + .update_by_id(pk.clone(), TestColumns::ATTR1, attr1_new.clone()) .await .unwrap(); @@ -245,7 +241,7 @@ async fn update_1_idx_full_row() { let pk = test_table.insert(row.clone()).await.unwrap(); test_table - .update(TestRow { + .replace(TestRow { attr2: row.attr2, id: pk.clone().into(), attr1: attr1_new.clone(), diff --git a/tests/worktable/index/update_by_pk.rs b/tests/worktable/index/update_by_pk.rs index d4877ba1..1b08bc08 100644 --- a/tests/worktable/index/update_by_pk.rs +++ b/tests/worktable/index/update_by_pk.rs @@ -1,8 +1,8 @@ use worktable::prelude::SelectQueryExecutor; use crate::worktable::index::{ - Test3NonUniqueRow, Test3NonUniqueWorkTable, Test3UniqueRow, Test3UniqueWorkTable, ThreeAttrByIdQuery, - UniqueThreeAttrByIdQuery, + Test3NonUniqueColumns, Test3NonUniqueRow, Test3NonUniqueWorkTable, Test3UniqueColumns, Test3UniqueRow, + Test3UniqueWorkTable, ThreeAttrByIdQuery, UniqueThreeAttrByIdQuery, }; #[tokio::test] @@ -27,13 +27,14 @@ async fn update_by_pk_unique_indexes() { let pk = test_table.insert(row.clone()).await.unwrap(); test_table - .update_unique_three_attr_by_id( + .update_by_id( + pk.clone(), + Test3UniqueColumns::ATTR1_AND_ATTR2_AND_ATTR3, UniqueThreeAttrByIdQuery { attr1: attr1_new.clone(), attr2: attr2_new, attr3: attr3_new, }, - pk.clone(), ) .await .unwrap(); @@ -77,13 +78,14 @@ async fn update_by_pk_non_unique_indexes() { let pk = test_table.insert(row.clone()).await.unwrap(); test_table - .update_three_attr_by_id( + .update_by_id( + pk.clone(), + Test3NonUniqueColumns::ATTR1_AND_ATTR2_AND_ATTR3, ThreeAttrByIdQuery { attr1: attr1_new.clone(), attr2: attr2_new, attr3: attr3_new, }, - pk.clone(), ) .await .unwrap(); @@ -132,7 +134,7 @@ async fn update_by_pk_with_reinsert_and_secondary_unique_violation() { }; assert!( test_table - .update_unique_three_attr_by_id(update, row1.id) + .update_by_id(row1.id, Test3UniqueColumns::ATTR1_AND_ATTR2_AND_ATTR3, update) .await .is_err() ); @@ -173,7 +175,7 @@ async fn update_by_pk_with_secondary_unique_violation() { }; assert!( test_table - .update_unique_three_attr_by_id(update, row1.id) + .update_by_id(row1.id, Test3UniqueColumns::ATTR1_AND_ATTR2_AND_ATTR3, update) .await .is_err() ); diff --git a/tests/worktable/index/update_full.rs b/tests/worktable/index/update_full.rs index 065ceb27..86a644c5 100644 --- a/tests/worktable/index/update_full.rs +++ b/tests/worktable/index/update_full.rs @@ -23,7 +23,7 @@ async fn update_by_full_row_unique_indexes() { let pk = test_table.insert(row.clone()).await.unwrap(); test_table - .update(Test3UniqueRow { + .replace(Test3UniqueRow { attr1: attr1_new.clone(), id: pk.clone().into(), val: row.val, @@ -72,7 +72,7 @@ async fn update_by_full_row_non_unique_indexes() { let pk = test_table.insert(row.clone()).await.unwrap(); test_table - .update(Test3NonUniqueRow { + .replace(Test3NonUniqueRow { attr1: attr1_new.clone(), id: pk.clone().into(), val: row.val, @@ -122,7 +122,7 @@ async fn update_by_full_row_unique_with_string_update() { let pk = test_table.insert(row.clone()).await.unwrap(); test_table - .update(Test3UniqueRow { + .replace(Test3UniqueRow { attr1: attr1_new.clone(), id: pk.clone().into(), val: row.val, @@ -171,7 +171,7 @@ async fn update_by_full_row_non_unique_with_string_update() { let pk = test_table.insert(row.clone()).await.unwrap(); test_table - .update(Test3NonUniqueRow { + .replace(Test3NonUniqueRow { attr1: attr1_new.clone(), id: pk.clone().into(), val: row.val, @@ -222,7 +222,7 @@ async fn update_by_full_row_with_reinsert_and_primary_key_violation() { let mut update = row1.clone(); update.id = row2.id; update.attr1 = "TEST_______________________1".to_string(); - assert!(test_table.update(update).await.is_err()); + assert!(test_table.replace(update).await.is_err()); assert_eq!(test_table.select(row1.id).unwrap(), row1); assert_eq!(test_table.select_by_attr1(row1.attr1.clone()).unwrap(), row1); @@ -257,7 +257,7 @@ async fn update_by_full_row_with_reinsert_and_secondary_unique_violation() { test_table.insert(row2.clone()).await.unwrap(); let mut update = row1.clone(); update.attr1 = row2.attr1.clone(); - assert!(test_table.update(update).await.is_err()); + assert!(test_table.replace(update).await.is_err()); assert_eq!(test_table.select(row1.id).unwrap(), row1); assert_eq!(test_table.select_by_attr1(row1.attr1.clone()).unwrap(), row1); @@ -292,7 +292,7 @@ async fn update_by_full_row_with_secondary_unique_violation() { test_table.insert(row2.clone()).await.unwrap(); let mut update = row1.clone(); update.attr2 = row2.attr2; - assert!(test_table.update(update).await.is_err()); + assert!(test_table.replace(update).await.is_err()); assert_eq!(test_table.select(row1.id).unwrap(), row1); assert_eq!(test_table.select_by_attr1(row1.attr1.clone()).unwrap(), row1); diff --git a/tests/worktable/index/update_query.rs b/tests/worktable/index/update_query.rs index 6964ec25..ac433340 100644 --- a/tests/worktable/index/update_query.rs +++ b/tests/worktable/index/update_query.rs @@ -1,6 +1,6 @@ use crate::worktable::index::{ - Test3NonUniqueRow, Test3NonUniqueWorkTable, Test3UniqueRow, Test3UniqueWorkTable, TwoAttrByThirdQuery, - UniqueTwoAttrByThirdQuery, + Test3NonUniqueColumns, Test3NonUniqueRow, Test3NonUniqueWorkTable, Test3UniqueColumns, Test3UniqueRow, + Test3UniqueWorkTable, TwoAttrByThirdQuery, UniqueTwoAttrByThirdQuery, }; use worktable::prelude::SelectQueryExecutor; @@ -25,12 +25,13 @@ async fn update_two_via_query_unique_indexes() { let _ = test_table.insert(row.clone()).await.unwrap(); test_table - .update_unique_two_attr_by_third( + .update_by_attr3( + attr3_old, + Test3UniqueColumns::ATTR1_AND_ATTR2, UniqueTwoAttrByThirdQuery { attr1: attr1_new.clone(), attr2: attr2_new, }, - attr3_old, ) .await .unwrap(); @@ -75,7 +76,7 @@ async fn update_with_reinsert_and_secondary_unique_violation() { }; assert!( test_table - .update_unique_two_attr_by_third(update, row1.attr3,) + .update_by_attr3(row1.attr3, Test3UniqueColumns::ATTR1_AND_ATTR2, update) .await .is_err() ); @@ -115,7 +116,7 @@ async fn update_with_secondary_unique_violation() { }; assert!( test_table - .update_unique_two_attr_by_third(update, row1.attr3) + .update_by_attr3(row1.attr3, Test3UniqueColumns::ATTR1_AND_ATTR2, update) .await .is_err() ); @@ -150,12 +151,13 @@ async fn update_two_via_query_non_unique_indexes() { let _ = test_table.insert(row.clone()).await.unwrap(); test_table - .update_two_attr_by_third( + .update_by_attr3( + attr3_old, + Test3NonUniqueColumns::ATTR1_AND_ATTR2, TwoAttrByThirdQuery { attr1: attr1_new, attr2: attr2_new, }, - attr3_old, ) .await .unwrap(); diff --git a/tests/worktable/index_backends.rs b/tests/worktable/index_backends.rs index 09064dfc..7947018d 100644 --- a/tests/worktable/index_backends.rs +++ b/tests/worktable/index_backends.rs @@ -178,7 +178,7 @@ async fn all_unique_backends_support_crud_ranges_and_conflict_rollback() { congee_key: 23, arctic_key: 24, }; - table.update(updated.clone()).await.unwrap(); + table.replace(updated.clone()).await.unwrap(); assert_eq!(table.select(pk), Some(updated.clone())); assert!(table.select_by_wti_key(11).is_none()); assert!(table.select_by_upstream_key(12).is_none()); @@ -223,7 +223,7 @@ async fn alternative_primary_backends_support_point_crud() { id: original.id, value: 2, }; - table.update(updated.clone()).await.unwrap(); + table.replace(updated.clone()).await.unwrap(); assert_eq!(table.select(pk.clone()), Some(updated)); table.delete(original.id).await.unwrap(); @@ -428,7 +428,7 @@ async fn native_art_backends_recover_concurrent_same_row_updates() { barrier.wait().await; for update in 0..UPDATES_PER_WORKER { table - .update(PersistedArcticRow { + .replace(PersistedArcticRow { id, congee_key: 10_000 + worker * UPDATES_PER_WORKER + update, }) @@ -496,7 +496,7 @@ async fn logical_wti_recovers_concurrent_same_row_updates() { barrier.wait().await; for update in 0..UPDATES_PER_WORKER { table - .update(wti::ProviderSwitchRow { + .replace(wti::ProviderSwitchRow { id, unique_key: 10_000 + worker * UPDATES_PER_WORKER + update, }) diff --git a/tests/worktable/leak_probe.rs b/tests/worktable/leak_probe.rs index e0aa521a..e9a1a740 100644 --- a/tests/worktable/leak_probe.rs +++ b/tests/worktable/leak_probe.rs @@ -42,12 +42,7 @@ async fn update_churn_reclaims_under_concurrent_readers() { .unwrap(); for i in 0..100u64 { table - .update_payload( - PayloadQuery { - payload: format!("{:04}", i % 10000), - }, - 1, - ) + .update_by_id(1, LeakProbeColumns::PAYLOAD, format!("{:04}", i % 10000)) .await .unwrap(); } @@ -72,12 +67,7 @@ async fn update_churn_reclaims_under_concurrent_readers() { for i in 0..5_000u64 { table - .update_payload( - PayloadQuery { - payload: format!("{:04}", i % 10000), - }, - 1, - ) + .update_by_id(1, LeakProbeColumns::PAYLOAD, format!("{:04}", i % 10000)) .await .unwrap(); } @@ -114,12 +104,7 @@ async fn update_churn_does_not_grow_storage_unbounded() { let pages_after_warmup = { for i in 0..100u64 { table - .update_payload( - PayloadQuery { - payload: format!("{:04}", i % 10000), - }, - 1, - ) + .update_by_id(1, LeakProbeColumns::PAYLOAD, format!("{:04}", i % 10000)) .await .unwrap(); } @@ -128,12 +113,7 @@ async fn update_churn_does_not_grow_storage_unbounded() { for i in 0..5_000u64 { table - .update_payload( - PayloadQuery { - payload: format!("{:04}", i % 10000), - }, - 1, - ) + .update_by_id(1, LeakProbeColumns::PAYLOAD, format!("{:04}", i % 10000)) .await .unwrap(); } diff --git a/tests/worktable/lock_order.rs b/tests/worktable/lock_order.rs index 519e58e4..abe17c57 100644 --- a/tests/worktable/lock_order.rs +++ b/tests/worktable/lock_order.rs @@ -95,13 +95,9 @@ async fn multi_row_update_locks_in_primary_key_order_not_index_order() { let update_table = table.clone(); let update = tokio::spawn(async move { if use_group_a { - update_table - .update_value_by_group_a(ValueByGroupAQuery { value: 1 }, 1) - .await + update_table.update_by_group_a(1, LockOrderColumns::VALUE, 1).await } else { - update_table - .update_value_by_group_b(ValueByGroupBQuery { value: 1 }, 1) - .await + update_table.update_by_group_b(1, LockOrderColumns::VALUE, 1).await } }); diff --git a/tests/worktable/mod.rs b/tests/worktable/mod.rs index b2cf318b..9569c329 100644 --- a/tests/worktable/mod.rs +++ b/tests/worktable/mod.rs @@ -10,6 +10,7 @@ mod count; mod custom_pk; mod delete; mod delete_many; +mod duplicate_pk_index_update; mod float; mod in_place; mod index; diff --git a/tests/worktable/multi_row_deadlock.rs b/tests/worktable/multi_row_deadlock.rs index 015c23ac..9941da8f 100644 --- a/tests/worktable/multi_row_deadlock.rs +++ b/tests/worktable/multi_row_deadlock.rs @@ -58,7 +58,7 @@ async fn overlapping_multi_row_updates_do_not_deadlock() { "a-much-longer-name-value".to_string() }; table - .update_name_by_group_a(NameByGroupAQuery { name }, 1) + .update_by_group_a(1, MultiRowDeadlockColumns::NAME, name) .await .unwrap(); } @@ -74,7 +74,7 @@ async fn overlapping_multi_row_updates_do_not_deadlock() { "b-much-longer-name-value".to_string() }; table - .update_name_by_group_b(NameByGroupBQuery { name }, 1) + .update_by_group_b(1, MultiRowDeadlockColumns::NAME, name) .await .unwrap(); } diff --git a/tests/worktable/mutation_gate_deadlock.rs b/tests/worktable/mutation_gate_deadlock.rs index 0d716367..d565bed3 100644 --- a/tests/worktable/mutation_gate_deadlock.rs +++ b/tests/worktable/mutation_gate_deadlock.rs @@ -1,7 +1,7 @@ //! Guard test for the synchronous mutation gate (`LockMap::mutation_guard`). //! //! The gate is a blocking spin/yield ticket lock, and the generated -//! `update`/`in_place`/`delete` paths hold the resulting `MutationGuard` inside +//! `update`/`update_in_place`/`delete` paths hold the resulting `MutationGuard` inside //! the `LockGuard` **across `.await`** (e.g. `update_with_guard(...).await`, //! `reinsert(...).await`). A review flagged this as a possible livelock/deadlock //! when two keys collide on the same 1-of-64 stripe on a constrained runtime. @@ -78,7 +78,7 @@ fn concurrent_same_stripe_updates_do_not_deadlock() { let table = table.clone(); tokio::spawn(async move { for i in 0..500u64 { - table.update_val(ValQuery { val: i }, a).await.unwrap(); + table.update_by_id(a, GateBenchColumns::VAL, i).await.unwrap(); } }) }; @@ -86,7 +86,7 @@ fn concurrent_same_stripe_updates_do_not_deadlock() { let table = table.clone(); tokio::spawn(async move { for i in 0..500u64 { - table.update_val(ValQuery { val: i }, b).await.unwrap(); + table.update_by_id(b, GateBenchColumns::VAL, i).await.unwrap(); } }) }; @@ -127,7 +127,7 @@ fn many_same_stripe_updates_do_not_starve_worker_pool() { let key = if worker % 2 == 0 { a } else { b }; handles.push(tokio::spawn(async move { for i in 0..300u64 { - table.update_val(ValQuery { val: i }, key).await.unwrap(); + table.update_by_id(key, GateBenchColumns::VAL, i).await.unwrap(); } })); } diff --git a/tests/worktable/nonunique_arctic.rs b/tests/worktable/nonunique_arctic.rs index 0267b85d..ed394fc0 100644 --- a/tests/worktable/nonunique_arctic.rs +++ b/tests/worktable/nonunique_arctic.rs @@ -93,7 +93,7 @@ async fn update_moves_a_row_between_non_unique_keys() { table.insert(row(&table, SOURCE_A, 2, 20)).await.unwrap(); table - .update_source_by_id(SourceByIdQuery { source_hash: SOURCE_B }, pk.clone()) + .update_by_id(pk.clone(), ArcticAdjacencyColumns::SOURCE_HASH, SOURCE_B) .await .unwrap(); @@ -106,7 +106,7 @@ async fn update_moves_a_row_between_non_unique_keys() { // Updating by the non-unique key touches every row under it. table - .update_weight_by_source(WeightBySourceQuery { weight: 777 }, SOURCE_A) + .update_by_source_hash(SOURCE_A, ArcticAdjacencyColumns::WEIGHT, 777) .await .unwrap(); let rows = table.select_by_weight(777).execute().unwrap(); @@ -298,7 +298,7 @@ mod persisted { source_hash: 6, weight: 101, }; - table.update(moved.clone()).await.unwrap(); + table.replace(moved.clone()).await.unwrap(); table.wait_for_ops().await.unwrap(); drop(table); diff --git a/tests/worktable/option.rs b/tests/worktable/option.rs index 7c82185d..5c59ed60 100644 --- a/tests/worktable/option.rs +++ b/tests/worktable/option.rs @@ -40,7 +40,7 @@ async fn update() { another: 1, exchange: 1, }; - table.update(new_row.clone()).await.unwrap(); + table.replace(new_row.clone()).await.unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row, new_row); } @@ -55,10 +55,7 @@ async fn update_by_another() { exchange: 1, }; let pk = table.insert(row.clone()).await.unwrap(); - table - .update_test_by_another(TestByAnotherQuery { test: Some(1) }, 1) - .await - .unwrap(); + table.update_by_another(1, TestColumns::TEST, Some(1)).await.unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row.test, Some(1)); } @@ -73,10 +70,7 @@ async fn update_by_exchange() { exchange: 1, }; let pk = table.insert(row.clone()).await.unwrap(); - table - .update_test_by_exchange(TestByExchangeQuery { test: Some(1) }, 1) - .await - .unwrap(); + table.update_by_exchange(1, TestColumns::TEST, Some(1)).await.unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row.test, Some(1)); } @@ -94,7 +88,7 @@ async fn update_none_to_some() { assert_eq!(table.select(pk.clone()).unwrap().test, None); table - .update_test_by_id(TestByIdQuery { test: Some(42) }, pk.clone()) + .update_by_id(pk.clone(), TestColumns::TEST, Some(42)) .await .unwrap(); @@ -114,10 +108,7 @@ async fn update_some_to_none() { let pk = table.insert(row.clone()).await.unwrap(); assert_eq!(table.select(pk.clone()).unwrap().test, Some(100)); - table - .update_test_by_id(TestByIdQuery { test: None }, pk.clone()) - .await - .unwrap(); + table.update_by_id(pk.clone(), TestColumns::TEST, None).await.unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row.test, None); @@ -144,7 +135,7 @@ async fn update_multiple_values() { let pk2 = table.insert(row2).await.unwrap(); table - .update_test_by_id(TestByIdQuery { test: Some(30) }, pk1.clone()) + .update_by_id(pk1.clone(), TestColumns::TEST, Some(30)) .await .unwrap(); @@ -190,7 +181,7 @@ async fn custom_update() { another: 1, exchange: 1, }; - table.update(new_row.clone()).await.unwrap(); + table.replace(new_row.clone()).await.unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row, new_row); } @@ -207,7 +198,7 @@ async fn custom_update_by_another() { let pk = table.insert(row.clone()).await.unwrap(); let test_uuid = Uuid::new_v4(); table - .update_custom_test_by_another(CustomTestByAnotherQuery { test: Some(test_uuid) }, 1) + .update_by_another(1, TestCustomColumns::TEST, Some(test_uuid)) .await .unwrap(); let selected_row = table.select(pk).unwrap(); @@ -226,7 +217,7 @@ async fn custom_update_by_exchange() { let pk = table.insert(row.clone()).await.unwrap(); let test_uuid = Uuid::new_v4(); table - .update_custom_test_by_exchange(CustomTestByExchangeQuery { test: Some(test_uuid) }, 1) + .update_by_exchange(1, TestCustomColumns::TEST, Some(test_uuid)) .await .unwrap(); let selected_row = table.select(pk).unwrap(); @@ -247,7 +238,7 @@ async fn custom_update_none_to_some() { let test_uuid = Uuid::new_v4(); table - .update_custom_test_by_id(CustomTestByIdQuery { test: Some(test_uuid) }, pk.clone()) + .update_by_id(pk.clone(), TestCustomColumns::TEST, Some(test_uuid)) .await .unwrap(); @@ -269,7 +260,7 @@ async fn custom_update_some_to_none() { assert_eq!(table.select(pk.clone()).unwrap().test, Some(test_uuid)); table - .update_custom_test_by_id(CustomTestByIdQuery { test: None }, pk.clone()) + .update_by_id(pk.clone(), TestCustomColumns::TEST, None) .await .unwrap(); @@ -301,7 +292,7 @@ async fn custom_update_multiple_uuids() { let uuid3 = Uuid::new_v4(); table - .update_custom_test_by_id(CustomTestByIdQuery { test: Some(uuid3) }, pk1.clone()) + .update_by_id(pk1.clone(), TestCustomColumns::TEST, Some(uuid3)) .await .unwrap(); @@ -462,7 +453,7 @@ async fn indexed_update_indexed_field() { // Update to a new UUID let uuid2 = Uuid::new_v4(); table - .update_index_test_by_id(IndexTestByIdQuery { test: Some(uuid2) }, pk.clone()) + .update_by_id(pk.clone(), TestIndexColumns::TEST, Some(uuid2)) .await .unwrap(); @@ -495,7 +486,7 @@ async fn indexed_update_from_some_to_none() { // Update to None table - .update_index_test_by_id(IndexTestByIdQuery { test: None }, pk.clone()) + .update_by_id(pk.clone(), TestIndexColumns::TEST, None) .await .unwrap(); @@ -528,7 +519,7 @@ async fn indexed_update_from_none_to_some() { // Update to Some UUID let test_uuid = Uuid::new_v4(); table - .update_index_test_by_id(IndexTestByIdQuery { test: Some(test_uuid) }, pk.clone()) + .update_by_id(pk.clone(), TestIndexColumns::TEST, Some(test_uuid)) .await .unwrap(); @@ -558,7 +549,7 @@ async fn indexed_update_via_another_index() { // Update via the unique 'another' index table - .update_index_test_by_another(IndexTestByAnotherQuery { test: Some(uuid2) }, 999) + .update_by_another(999, TestIndexColumns::TEST, Some(uuid2)) .await .unwrap(); @@ -594,7 +585,7 @@ async fn indexed_update_via_non_unique_index() { // Update both rows via the non-unique 'exchange' index table - .update_index_test_by_exchange(IndexTestByExchangeQuery { test: Some(uuid2) }, 100) + .update_by_exchange(100, TestIndexColumns::TEST, Some(uuid2)) .await .unwrap(); diff --git a/tests/worktable/partitioned.rs b/tests/worktable/partitioned.rs index 6522999d..67b09f54 100644 --- a/tests/worktable/partitioned.rs +++ b/tests/worktable/partitioned.rs @@ -205,7 +205,7 @@ async fn updates_are_scoped_to_one_partition() { a.insert(row(7, 100.0)).await.unwrap(); b.insert(row(7, 200.0)).await.unwrap(); - a.update(PriceRow { + a.replace(PriceRow { exchange_id: 7, bid: 999.0, ask: 1000.0, @@ -1021,7 +1021,7 @@ fn a_dense_partition_carries_its_update_queries() { // The same method name and the same query struct the paged table generates, // so the call reads the same. What differs is that there is no `.await`. assert_eq!( - book.update_top_price(TopPriceQuery { bid: 9.0, ask: 10.0 }, &2), + book.update_by_exchange_id(2, QuotedColumns::BID_AND_ASK, TopPriceQuery { bid: 9.0, ask: 10.0 }), Some(()) ); @@ -1030,7 +1030,7 @@ fn a_dense_partition_carries_its_update_queries() { assert_eq!(row.seq, 7, "a column the query does not name is untouched"); assert_eq!( - book.update_top_price(TopPriceQuery { bid: 0.0, ask: 0.0 }, &3), + book.update_by_exchange_id(3, QuotedColumns::BID_AND_ASK, TopPriceQuery { bid: 0.0, ask: 0.0 }), None, "a key holding no row updates nothing" ); diff --git a/tests/worktable/runtime_backends.rs b/tests/worktable/runtime_backends.rs index 61c6bc95..d717b6a9 100644 --- a/tests/worktable/runtime_backends.rs +++ b/tests/worktable/runtime_backends.rs @@ -11,7 +11,7 @@ //! The body covers four things, and each one is here because leaving it out //! would let a broken backend pass: //! -//! 1. `insert` / `select` / `update` / `delete` / `in_place`, so a backend that +//! 1. `insert` / `select` / `update` / `delete` / `update_in_place`, so a backend that //! compiles but cannot drive a mutation is caught. //! 2. A persisted table opened, written, closed and reloaded, so a backend //! whose spawn or timer never reaches the persistence worker is caught. @@ -92,7 +92,7 @@ macro_rules! runtime_backend_suite { // The in-memory table. Carries an indexed column so `update` and // `delete` have index maintenance to do, and a plain one so - // `in_place` has somewhere to write that no index watches. + // `update_in_place` has somewhere to write that no index watches. worktable!( name: RuntimeMatrix, persist: false, @@ -113,7 +113,7 @@ macro_rules! runtime_backend_suite { delete: { ByBucket() by bucket, }, - in_place: { + update_in_place: { CounterById(counter) by id, } } @@ -141,7 +141,7 @@ macro_rules! runtime_backend_suite { update: { PersistBucketById(bucket) by id, }, - in_place: { + update_in_place: { PersistCounterById(counter) by id, } } @@ -241,7 +241,7 @@ macro_rules! runtime_backend_suite { // update, which also has to move the row between index buckets table - .update_bucket_by_id(BucketByIdQuery { bucket: 3 }, ids[0]) + .update_by_id(ids[0], RuntimeMatrixColumns::BUCKET, 3) .await .unwrap(); assert_eq!(table.select(ids[0]).unwrap().bucket, 3); @@ -255,7 +255,7 @@ macro_rules! runtime_backend_suite { // republishing the row for _ in 0..64 { table - .update_counter_by_id_in_place(|counter| *counter += 1u64, ids[1]) + .update_in_place_by_id(ids[1], RuntimeMatrixColumns::COUNTER, |counter| *counter += 1u64) .await .unwrap(); } @@ -351,11 +351,11 @@ macro_rules! runtime_backend_suite { table.insert(row(id)).await.unwrap(); } table - .update_persist_bucket_by_id(PersistBucketByIdQuery { bucket: 3 }, 1) + .update_by_id(1, RuntimeMatrixPersistColumns::BUCKET, 3) .await .unwrap(); table - .update_persist_counter_by_id_in_place(|counter| *counter = 4_242u64.into(), 2) + .update_in_place_by_id(2, RuntimeMatrixPersistColumns::COUNTER, |counter| *counter = 4_242u64.into()) .await .unwrap(); diff --git a/tests/worktable/unique_fixed_unsized.rs b/tests/worktable/unique_fixed_unsized.rs index e81f0c85..87388a1c 100644 --- a/tests/worktable/unique_fixed_unsized.rs +++ b/tests/worktable/unique_fixed_unsized.rs @@ -37,7 +37,7 @@ async fn unique_keyed_fixed_size_update_on_unsized_row_works() { .unwrap(); table - .update_amount_by_code(AmountByCodeQuery { amount: 55 }, 10) + .update_by_code(10, UniqueFixedUnsizedColumns::AMOUNT, 55) .await .unwrap(); diff --git a/tests/worktable/unsized_.rs b/tests/worktable/unsized_.rs index 80f8faa5..0068dfdc 100644 --- a/tests/worktable/unsized_.rs +++ b/tests/worktable/unsized_.rs @@ -45,7 +45,7 @@ async fn test_update_string_full_row() { let first_link = table.0.primary_index.pk_map.get_value(&pk).unwrap(); table - .update(TestRow { + .replace(TestRow { id: row.id, test: 1, another: 1, @@ -87,7 +87,7 @@ async fn test_update_string_by_unique() { let row = ExchangeByTestQuery { exchange: "bigger test to test string update".to_string(), }; - table.update_exchange_by_test(row, 1).await.unwrap(); + table.update_by_test(1, TestColumns::EXCHANGE, (row).exchange).await.unwrap(); let row = table.select_by_test(1).unwrap(); @@ -122,7 +122,7 @@ async fn test_update_string_by_pk() { let row = ExchangeByIdQuery { exchange: "bigger test to test string update".to_string(), }; - table.update_exchange_by_id(row, pk).await.unwrap(); + table.update_by_id(pk, TestColumns::EXCHANGE, (row).exchange).await.unwrap(); let row = table.select_by_test(1).unwrap(); @@ -165,7 +165,7 @@ async fn test_update_string_by_non_unique() { let row = ExchangeByAbotherQuery { exchange: "bigger test to test string update".to_string(), }; - table.update_exchange_by_abother(row, 1).await.unwrap(); + table.update_by_another(1, TestColumns::EXCHANGE, (row).exchange).await.unwrap(); let all = table.select_all().execute().unwrap(); @@ -219,12 +219,7 @@ async fn update_many_times() { let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..=99); table - .update_exchange_by_id( - ExchangeByIdQuery { - exchange: format!("test_{val}"), - }, - id_to_update, - ) + .update_by_id(id_to_update, TestColumns::EXCHANGE, format!("test_{val}")) .await .unwrap(); { @@ -261,12 +256,7 @@ async fn update_parallel() { let val = fastrand::u64(..); let id_to_update = fastrand::i64(1..=100); shared - .update_exchange_by_test( - ExchangeByTestQuery { - exchange: format!("test_{val}"), - }, - id_to_update, - ) + .update_by_test(id_to_update, TestColumns::EXCHANGE, format!("test_{val}")) .await .unwrap(); { @@ -284,12 +274,7 @@ async fn update_parallel() { let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..=99); table - .update_exchange_by_id( - ExchangeByIdQuery { - exchange: format!("test_{val}"), - }, - id_to_update, - ) + .update_by_id(id_to_update, TestColumns::EXCHANGE, format!("test_{val}")) .await .unwrap(); { @@ -356,7 +341,7 @@ async fn test_update_many_strings_by_unique() { exchange: "bigger test to test string update".to_string(), some_string: "some bigger some to test".to_string(), }; - table.update_exchange_and_some_by_test(row, 1).await.unwrap(); + table.update_by_test(1, TestMoreStringsColumns::EXCHANGE_AND_SOME_STRING, row).await.unwrap(); let row = table.select_by_test(1).unwrap(); @@ -396,7 +381,7 @@ async fn test_update_many_strings_by_pk() { exchange: "bigger test to test string update".to_string(), some_string: "some bigger some to test".to_string(), }; - table.update_exchange_and_some_by_id(row, pk).await.unwrap(); + table.update_by_id(pk, TestMoreStringsColumns::EXCHANGE_AND_SOME_STRING, row).await.unwrap(); let row = table.select_by_test(1).unwrap(); @@ -446,7 +431,7 @@ async fn test_update_many_strings_by_non_unique() { exchange: "bigger test to test string update".to_string(), some_string: "some bigger some to test".to_string(), }; - table.update_exchange_and_some_by_another(row, 1).await.unwrap(); + table.update_by_another(1, TestMoreStringsColumns::EXCHANGE_AND_SOME_STRING, row).await.unwrap(); let all = table.select_all().execute().unwrap(); @@ -516,7 +501,7 @@ async fn test_update_many_strings_by_string() { some_string: "some bigger some to test".to_string(), }; table - .update_some_other_by_exchange(row, "test".to_string()) + .update_by_exchange("test".to_string(), TestMoreStringsColumns::SOME_STRING_AND_OTHER_SRTING, row) .await .unwrap(); @@ -582,12 +567,7 @@ async fn update_parallel_more_strings() { let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..=99); shared - .update_exchange_again_by_id( - ExchangeAgainByIdQuery { - exchange: format!("test_{val}"), - }, - id_to_update, - ) + .update_by_id(id_to_update, TestMoreStringsColumns::EXCHANGE, format!("test_{val}")) .await .unwrap(); { @@ -603,12 +583,7 @@ async fn update_parallel_more_strings() { let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..=99); table - .update_some_by_id( - SomeByIdQuery { - some_string: format!("some_{val}"), - }, - id_to_update, - ) + .update_by_id(id_to_update, TestMoreStringsColumns::SOME_STRING, format!("some_{val}")) .await .unwrap(); { @@ -655,12 +630,7 @@ async fn update_parallel_more_strings_more_threads() { let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..=99); shared - .update_exchange_again_by_id( - ExchangeAgainByIdQuery { - exchange: format!("test_{val}"), - }, - id_to_update, - ) + .update_by_id(id_to_update, TestMoreStringsColumns::EXCHANGE, format!("test_{val}")) .await .unwrap(); { @@ -679,7 +649,7 @@ async fn update_parallel_more_strings_more_threads() { let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..=99); shared - .update_another_by_id(AnotherByIdQuery { another: val }, id_to_update) + .update_by_id(id_to_update, TestMoreStringsColumns::ANOTHER, val) .await .unwrap(); { @@ -692,12 +662,7 @@ async fn update_parallel_more_strings_more_threads() { let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..=99); table - .update_some_by_id( - SomeByIdQuery { - some_string: format!("some_{val}"), - }, - id_to_update, - ) + .update_by_id(id_to_update, TestMoreStringsColumns::SOME_STRING, format!("some_{val}")) .await .unwrap(); { @@ -750,12 +715,7 @@ async fn update_parallel_more_strings_with_select_non_unique() { let val = fastrand::u8(0..100); let id_to_update = fastrand::u64(0..1000); shared - .update_exchange_again_by_id( - ExchangeAgainByIdQuery { - exchange: format!("test_{val}"), - }, - id_to_update, - ) + .update_by_id(id_to_update, TestMoreStringsColumns::EXCHANGE, format!("test_{val}")) .await .unwrap(); { @@ -774,7 +734,7 @@ async fn update_parallel_more_strings_with_select_non_unique() { let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..1000); shared - .update_another_by_id(AnotherByIdQuery { another: val }, id_to_update) + .update_by_id(id_to_update, TestMoreStringsColumns::ANOTHER, val) .await .unwrap(); { @@ -889,12 +849,7 @@ async fn update_parallel_more_strings_with_select_unique() { let val = fastrand::u8(0..100); let id_to_update = fastrand::u64(0..1000); shared - .update_exchange_again_by_id( - ExchangeAgainByIdQuery { - exchange: format!("test_{val}"), - }, - id_to_update, - ) + .update_by_id(id_to_update, TestMoreStringsColumns::EXCHANGE, format!("test_{val}")) .await .unwrap(); { @@ -913,7 +868,7 @@ async fn update_parallel_more_strings_with_select_unique() { let val = fastrand::u64(..); let id_to_update = fastrand::u64(0..1000); shared - .update_another_by_id(AnotherByIdQuery { another: val }, id_to_update) + .update_by_id(id_to_update, TestMoreStringsColumns::ANOTHER, val) .await .unwrap(); { diff --git a/tests/worktable/update_delete_race.rs b/tests/worktable/update_delete_race.rs index 7c05ccab..911fb18c 100644 --- a/tests/worktable/update_delete_race.rs +++ b/tests/worktable/update_delete_race.rs @@ -45,7 +45,7 @@ async fn concurrent_update_and_delete_never_panics() { } else { "a-longer-replacement-name".to_string() }; - match table.update_name_by_id(NameByIdQuery { name }, 1).await { + match table.update_by_id(1, UpdateDeleteRaceColumns::NAME, name).await { Ok(()) => {} // Deleted concurrently: legal, retried next iteration. Err(WorkTableError::NotFound) => {} diff --git a/tests/worktable/update_in_place_unsized.rs b/tests/worktable/update_in_place_unsized.rs index 7b1353d7..44ba891d 100644 --- a/tests/worktable/update_in_place_unsized.rs +++ b/tests/worktable/update_in_place_unsized.rs @@ -81,11 +81,10 @@ macro_rules! unsized_in_place_suite { let before = link_of(&table, 1); table - .update_payload( - PayloadQuery { - payload: "12345678".to_string(), // 8 bytes — same length - }, + .update_by_id( 1, + UnsizedUpdateColumns::PAYLOAD, + "12345678".to_string(), // 8 bytes — same length ) .await .unwrap(); @@ -113,23 +112,13 @@ macro_rules! unsized_in_place_suite { .await.unwrap(); table - .update_payload( - PayloadQuery { - payload: "xy".to_string(), - }, - 1, - ) + .update_by_id(1, UnsizedUpdateColumns::PAYLOAD, "xy".to_string()) .await .unwrap(); assert_eq!(table.select(1).unwrap().payload, "xy"); table - .update_payload( - PayloadQuery { - payload: "much longer payload".to_string(), - }, - 1, - ) + .update_by_id(1, UnsizedUpdateColumns::PAYLOAD, "much longer payload".to_string()) .await .unwrap(); assert_eq!(table.select(1).unwrap().payload, "much longer payload"); @@ -159,12 +148,7 @@ macro_rules! unsized_in_place_suite { tokio::spawn(async move { for i in 0..20_000u64 { table - .update_payload( - PayloadQuery { - payload: format!("{:04}", i % 10000), - }, - 1, - ) + .update_by_id(1, UnsizedUpdateColumns::PAYLOAD, format!("{:04}", i % 10000)) .await .unwrap(); } @@ -220,7 +204,7 @@ macro_rules! unsized_in_place_suite { let before = link_of(&table, 1); table - .update_balance(BalanceQuery { balance: 42.5 }, 1) + .update_by_id(1, UnsizedUpdateColumns::BALANCE, 42.5) .await .unwrap(); @@ -240,3 +224,120 @@ macro_rules! unsized_in_place_suite { unsized_in_place_suite!(wti, worktables_index); unsized_in_place_suite!(congee, congee); unsized_in_place_suite!(arctic, arctic); + +mod opaque_wrapper { + use rkyv::{Archive, Deserialize, Serialize}; + use worktable::prelude::*; + use worktable::worktable; + + #[derive(Archive, Clone, Debug, Deserialize, Serialize, PartialEq, PartialOrd, MemStat)] + #[rkyv(compare(PartialEq), derive(Debug))] + struct WrappedString(String); + + worktable!( + name: OpaqueOnly, + persist: false, + columns: { + id: u64 primary_key, + secret: WrappedString, + }, + ); + + worktable!( + name: OpaqueWrapperUpdate, + persist: false, + columns: { + id: u64 primary_key, + padding: String, + nickname: String optional, + secret: WrappedString, + }, + queries: { + update: { + Secret(secret) by id, + Nickname(nickname) by id, + } + } + ); + + #[tokio::test] + async fn custom_archived_string_wrapper_is_rebased_into_row_storage() { + let table = OpaqueWrapperUpdateWorkTable::default(); + table + .insert(OpaqueWrapperUpdateRow { + id: 1, + padding: "keeps the table on variable-size storage".to_string(), + nickname: None, + secret: WrappedString("original out-of-line secret value".to_string()), + }) + .await + .unwrap(); + + table + .update_by_id( + 1, + OpaqueWrapperUpdateColumns::SECRET, + WrappedString("replacement out-of-line secret!!".to_string()), + ) + .await + .unwrap(); + + assert_eq!( + table.select(1).unwrap().secret, + WrappedString("replacement out-of-line secret!!".to_string()) + ); + } + + #[tokio::test] + async fn optional_string_update_is_rebased_into_row_storage() { + let table = OpaqueWrapperUpdateWorkTable::default(); + table + .insert(OpaqueWrapperUpdateRow { + id: 1, + padding: "keeps the table on variable-size storage".to_string(), + nickname: Some("original out-of-line nickname".to_string()), + secret: WrappedString("unchanged out-of-line secret".to_string()), + }) + .await + .unwrap(); + + table + .update_by_id( + 1, + OpaqueWrapperUpdateColumns::NICKNAME, + Some("replacement out-of-line name".to_string()), + ) + .await + .unwrap(); + + assert_eq!( + table.select(1).unwrap().nickname, + Some("replacement out-of-line name".to_string()) + ); + } + + #[tokio::test] + async fn full_row_update_rebases_an_opaque_only_table() { + let table = OpaqueOnlyWorkTable::default(); + table + .insert(OpaqueOnlyRow { + id: 1, + secret: WrappedString("original out-of-line secret value".to_string()), + }) + .await + .unwrap(); + + table + .replace(OpaqueOnlyRow { + id: 1, + secret: WrappedString("replacement out-of-line secret value".to_string()), + }) + .await + .unwrap(); + + assert_eq!( + table.select(1).unwrap().secret, + WrappedString("replacement out-of-line secret value".to_string()) + ); + } +} diff --git a/tests/worktable/upsert_guard.rs b/tests/worktable/upsert_guard.rs index 2d391ecf..9f98c2fb 100644 --- a/tests/worktable/upsert_guard.rs +++ b/tests/worktable/upsert_guard.rs @@ -39,7 +39,10 @@ fn inserting_under_a_held_mutation_gate_completes() { let pk: UpsertGuardPrimaryKey = 1u64.into(); // Exactly what `upsert` holds when it discovers the key is absent. - let _gate = table.0.lock_manager.mutation_guard(&pk); + // + // SAFETY: `table` is an `Arc` held for the rest of the test, so the map + // outlives this guard. + let _gate = unsafe { table.0.lock_manager.mutation_guard(&pk) }; // Detached, not scoped. A scope joins its threads, so when the insert // deadlocks the test would hang at the end of the scope instead of diff --git a/tests/worktable/vec_table.rs b/tests/worktable/vec_table.rs index cbab9cff..4b45b3fe 100644 --- a/tests/worktable/vec_table.rs +++ b/tests/worktable/vec_table.rs @@ -1649,6 +1649,7 @@ worktable!( owner: u64, state: u8, amount: u64, + revision: u64, }, indexes: { owner_idx: owner using fxhash, @@ -1664,8 +1665,9 @@ worktable!( ById() by id, ByOwner() by owner, }, - in_place: { + update_in_place: { Status(state) by id, + StateAndRevisionById(state, revision) by id, }, }, ); @@ -1690,18 +1692,19 @@ fn declared_queries_run_on_a_vec_table() { owner: id % 2, state: 0, amount: 100 + id, + revision: 0, }) .expect("fresh"); } // Keyed by the hash primary key: one row. - assert_eq!(table.update_state_by_id(StateByIdQuery { state: 7 }, &3), 1); + assert_eq!(table.update_by_id(3, TicketColumns::STATE, 7), 1); assert_eq!(table.select(&3).expect("present").state, 7); assert_eq!(table.select(&2).expect("present").state, 0, "only one row moved"); // Keyed by a non-unique hash secondary: every row it names. assert_eq!( - table.update_state_by_owner(StateByOwnerQuery { state: 5 }, &1), + table.update_by_owner(1, TicketColumns::STATE, 5), 3, "owner 1 holds ids 1, 3 and 5" ); @@ -1712,7 +1715,7 @@ fn declared_queries_run_on_a_vec_table() { } assert_eq!(table.select(&0).expect("present").amount, 100, "owner 0 untouched"); - assert_eq!(table.update_amount_by_id(AmountByIdQuery { amount: 999 }, &1), 1); + assert_eq!(table.update_by_id(1, TicketColumns::AMOUNT, 999), 1); // The unique arctic secondary was repaired without stealing another row's key. assert!( table.select_by_amount(&101).is_none(), @@ -1725,8 +1728,20 @@ fn declared_queries_run_on_a_vec_table() { ); // in_place edits one column through a closure. - assert_eq!(table.update_status_in_place(|s| *s = 42, &0), 1); + assert_eq!(table.update_in_place_by_id(0, TicketColumns::STATE, |s| *s = 42), 1); assert_eq!(table.select(&0).expect("present").state, 42); + assert_eq!( + table.update_in_place_by_id(2, TicketColumns::STATE_AND_REVISION, |(state, revision)| { + *state = 8; + *revision = 1; + }), + 1 + ); + assert_eq!( + (table.select(&2).unwrap().state, table.select(&2).unwrap().revision), + (8, 1) + ); + assert_eq!(table.select_by_amount(&102).unwrap().id, 2); // Deletes, by the key and by a non-unique secondary. assert_eq!(table.delete_by_id(&0), 1); @@ -1813,13 +1828,14 @@ fn vec_declared_query_cannot_steal_another_rows_unique_key() { owner: id, state: 0, amount: 100 + id, + revision: 0, }) .unwrap(); } let before = table.unload().unwrap(); assert!( std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - table.update_amount_by_id(AmountByIdQuery { amount: 101 }, &0); + table.update_by_id(0, TicketColumns::AMOUNT, 101); })) .is_err() ); diff --git a/tests/worktable/with_enum.rs b/tests/worktable/with_enum.rs index 4b1e6175..2423f564 100644 --- a/tests/worktable/with_enum.rs +++ b/tests/worktable/with_enum.rs @@ -49,7 +49,7 @@ async fn update() { id: 1, test: SomeEnum::Second, }; - table.update(updated.clone()).await.unwrap(); + table.replace(updated.clone()).await.unwrap(); let selected_row = table.select(pk).unwrap(); assert_eq!(selected_row, updated); diff --git a/tests/worktable/wrong_row_update.rs b/tests/worktable/wrong_row_update.rs index a8ddf35f..806c3b25 100644 --- a/tests/worktable/wrong_row_update.rs +++ b/tests/worktable/wrong_row_update.rs @@ -53,7 +53,7 @@ async fn unique_update_does_not_mutate_a_row_that_stole_the_value() { let update = { let table = table.clone(); - tokio::spawn(async move { table.update_value_by_code(ValueByCodeQuery { value: 99 }, 10).await }) + tokio::spawn(async move { table.update_by_code(10, WrongRowColumns::VALUE, 99).await }) }; // Wait until the update registered its operation lock (it replaces the