From 938dc5e180cc96a4a5bfd8b874984047b83a0df3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 23:17:09 +0000 Subject: [PATCH 1/3] lab(drf-p2): break-even probe + mis-predict/64-switch lease falsifier examples/p2_breakeven.rs times, per 4096-slot focus: a cold gather (build), one fold over the gathered column, and one direct read along the map with lookahead 16, then reports the crossover fold count N* = build / (direct - fold) and the A2 read-hit fraction at N*. Both paths are asserted equal before any number is printed. Reported, not gated. New test p2_mispredicted_focus_and_64_fast_switches_leave_no_lease_and_no_corruption: a predicted-but-unused focus and 63 abandoned switches are reclaimed by evict_unused; the two held foci stay correct. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- crates/drf-focus-probe/Cargo.toml | 3 +- .../drf-focus-probe/examples/p2_breakeven.rs | 131 ++++++++++++++++++ .../tests/integration/focus.rs | 35 +++++ 3 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 crates/drf-focus-probe/examples/p2_breakeven.rs diff --git a/crates/drf-focus-probe/Cargo.toml b/crates/drf-focus-probe/Cargo.toml index f2b33a21b..882432648 100644 --- a/crates/drf-focus-probe/Cargo.toml +++ b/crates/drf-focus-probe/Cargo.toml @@ -1,5 +1,6 @@ # drf-focus-probe — D-DRF-P1, lab only. Immutable DR64 focus, lease by focus load, -# stale/foreign-key refusal, intra-focus prefetch lookahead. No timing claims. +# stale/foreign-key refusal, intra-focus prefetch lookahead. Timings only in +# examples/p2_breakeven.rs, reported, not gated. # Own [workspace] root, excluded from the lance-graph workspace. Run from here: # cargo test [package] diff --git a/crates/drf-focus-probe/examples/p2_breakeven.rs b/crates/drf-focus-probe/examples/p2_breakeven.rs new file mode 100644 index 000000000..8ecc792bc --- /dev/null +++ b/crates/drf-focus-probe/examples/p2_breakeven.rs @@ -0,0 +1,131 @@ +//! D-DRF-P2 break-even: focus (gather once + N folds) vs direct (N in-place reads). +//! +//! Reported, not gated. One focus = 4096 slots; every timing is per call, median of +//! repeated runs, in ns. The crossover N is the smallest fold count at which the +//! focus total (build + N*fold) is below N*direct. +//! cargo run --release --example p2_breakeven +use drf_focus_probe::*; +use std::hint::black_box; +use std::sync::Arc; +use std::time::Instant; + +const OFF: usize = 16; +type MapGen = Box u16>; + +fn xs(s: &mut u64) -> u64 { + *s ^= *s << 13; + *s ^= *s >> 7; + *s ^= *s << 17; + *s +} + +fn median_ns(reps: usize, mut f: impl FnMut()) -> f64 { + let mut v: Vec = (0..reps) + .map(|_| { + let t = Instant::now(); + f(); + t.elapsed().as_nanos() as f64 + }) + .collect(); + v.sort_by(|a, b| a.partial_cmp(b).unwrap()); + v[reps / 2] +} + +fn main() { + let isa = if cfg!(target_feature = "avx512f") { + "avx512f" + } else if cfg!(target_feature = "avx2") { + "avx2" + } else { + "scalar" + }; + println!("isa={isa} slots={SLOTS} row={ROW}B facet=i32@{OFF}"); + println!( + "{:<10} {:>7} {:<10} {:>6} {:>11} {:>9} {:>11} {:>7} {:>12}", + "rows", "MiB", "map", "dens", "build_ns", "fold_ns", "direct_ns", "N*", "A2@N*" + ); + let mut st = 0x9E37_79B9_7F4A_7C15u64; + for nrows in [1024usize, 65536] { + let rows: Vec = (0..nrows) + .map(|i| { + let mut r = [0u8; ROW]; + r[OFF..OFF + 4].copy_from_slice(&((i as i32) * 3 - 7).to_le_bytes()); + r + }) + .collect(); + let soa = Soa { + rows: &rows, + id: SourceId { + dataset: 1, + branch: 1, + version: 1, + index_seal: 7, + }, + }; + let facet = Facet::new(OFF).unwrap(); + let maps: [(&str, MapGen); 2] = [ + ( + "random", + Box::new(move |_, s| (xs(s) % nrows as u64) as u16), + ), + ("clustered", Box::new(move |i, _| (i % nrows) as u16)), + ]; + for (id, (mname, make)) in maps.iter().enumerate() { + let mut ord = Box::new([0u16; SLOTS]); + for (i, o) in ord.iter_mut().enumerate() { + *o = make(i, &mut st); + } + let map = Arc::new(SealedMap::new(MapId(id as u64), 7, ord)); + for dens in [1.0f64, 0.25] { + let mut mask = [0u64; TILE_WORDS]; + for w in mask.iter_mut() { + *w = if dens >= 1.0 { + u64::MAX + } else { + xs(&mut st) & xs(&mut st) + }; + } + let pop: u32 = mask.iter().map(|w| w.count_ones()).sum(); + // build = a cold manager per call, so every rep gathers. + let build = median_ns(201, || { + let mgr = FocusManager::new(); + black_box(mgr.load(&soa, &map, Scope(1), facet).unwrap()); + }); + let mgr = FocusManager::new(); + let focus = mgr.load(&soa, &map, Scope(1), facet).unwrap(); + let fold = median_ns(2001, || { + black_box(focus.sum_where(black_box(&mask))); + }); + let direct = median_ns(201, || { + black_box(direct_sum_where(&soa, &map, facet, black_box(&mask), 16).unwrap()); + }); + // Same answer on both paths, or the timing is meaningless. + assert_eq!( + focus.sum_where(&mask), + direct_sum_where(&soa, &map, facet, &mask, 16).unwrap() + ); + let nstar = if direct > fold { + format!("{}", (build / (direct - fold)).ceil() as u64) + } else { + "never".into() + }; + let a2 = match nstar.parse::() { + Ok(n) => format!("{:.3}", n * pop as f64 / (n * pop as f64 + SLOTS as f64)), + Err(_) => "-".into(), + }; + println!( + "{:<10} {:>7.1} {:<10} {:>6.2} {:>11.0} {:>9.0} {:>11.0} {:>7} {:>12}", + nrows, + (nrows * ROW) as f64 / (1 << 20) as f64, + mname, + pop as f64 / SLOTS as f64, + build, + fold, + direct, + nstar, + a2 + ); + } + } + } +} diff --git a/crates/drf-focus-probe/tests/integration/focus.rs b/crates/drf-focus-probe/tests/integration/focus.rs index b60624e05..8456c8d20 100644 --- a/crates/drf-focus-probe/tests/integration/focus.rs +++ b/crates/drf-focus-probe/tests/integration/focus.rs @@ -309,3 +309,38 @@ fn p1_direct_equals_oracle_equals_focus_for_all_lookaheads() { let s: Vec = patterns.iter().map(|(_, m)| oracle(m, &masks[0])).collect(); assert!(s.windows(2).any(|w| w[0] != w[1])); } + +#[test] +fn p2_mispredicted_focus_and_64_fast_switches_leave_no_lease_and_no_corruption() { + let r = rows(); + let soa = Soa { + rows: &r, + id: sid(1, 1, 1, 9), + }; + let mgr = FocusManager::new(); + let full = [u64::MAX; 64]; + // Predict B, use C: B is loaded and never read. + let b = map_from(100, 9, |s| (s % NROWS) as u16); + let c = map_from(101, 9, |s| ((s * 7) % NROWS) as u16); + drop(mgr.load(&soa, &b, Scope(1), facet()).unwrap()); + let fc = mgr.load(&soa, &c, Scope(1), facet()).unwrap(); + assert_eq!(fc.sum_where(&full), oracle(&c, &full)); + // 64 fast switches; only the last one stays held. + let mut held = None; + for k in 0..64u64 { + let m = map_from(200 + k, 9, |s| ((s + k as usize) % NROWS) as u16); + let f = mgr.load(&soa, &m, Scope(1), facet()).unwrap(); + assert_eq!(f.sum_where(&full), oracle(&m, &full), "switch {k}"); + held = Some((f, m)); + } + assert_eq!(mgr.len(), 66); + // Every unheld entry is reclaimed: B and 63 switches. C and the last stay. + assert_eq!(mgr.evict_unused(), 64); + assert_eq!(mgr.len(), 2); + let (last, lm) = held.unwrap(); + assert_eq!(last.sum_where(&full), oracle(&lm, &full)); + assert_eq!(fc.sum_where(&full), oracle(&c, &full)); + drop((last, fc)); + assert_eq!(mgr.evict_unused(), 2); + assert!(mgr.is_empty()); +} From 09f5380fc3d44de09567d927c419f5a8f61bc1c1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 23:17:27 +0000 Subject: [PATCH 2/3] board: D-DRF-P2 break-even entry Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .../entries/2026-10-24-drf-p2-breakeven.md | 29 +++++++++++++++++++ .claude/board/entries/README.md | 3 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-10-24-drf-p2-breakeven.md diff --git a/.claude/board/entries/2026-10-24-drf-p2-breakeven.md b/.claude/board/entries/2026-10-24-drf-p2-breakeven.md new file mode 100644 index 000000000..fad285711 --- /dev/null +++ b/.claude/board/entries/2026-10-24-drf-p2-breakeven.md @@ -0,0 +1,29 @@ +# D-DRF-P2 — focus vs direct break-even (lab, reported, not gated) + +**Status:** MEASURED (lab). **Probe:** `crates/drf-focus-probe/examples/p2_breakeven.rs`. +**Command:** `cargo run --release --example p2_breakeven` (native AVX-512) and with +`RUSTFLAGS="-C target-cpu=x86-64-v3"` (AVX2). Medians, ns per call, 4096-slot focus, +512 B rows, i32 facet, direct lookahead 16. Both paths asserted equal first. + +| rows (MiB) | map | density | build | fold | direct | N* | +|---|---|---|---|---|---|---| +| 1024 (0.5) | random | 1.00 | 3 827 | 2 341 | 8 198 | 1 | +| 1024 (0.5) | random | 0.25 | 3 824 | 475 | 5 502 | 1 | +| 65536 (32) | random | 1.00 | 20 733 | 2 323 | 20 227 | 2 | +| 65536 (32) | random | 0.25 | 19 924 | 516 | 17 514 | 2 | +| 65536 (32) | clustered | 1.00 | 16 642 | 2 312 | 15 457 | 2 | + +AVX2 rows agree within ~10%. Clustered 1024-row cases match random 1024 (same N*). + +- **Break-even:** the focus pays off from the 1st (L2-resident) or 2nd (32 MiB, DRAM) + repeated fold of the same focus. No win when a focus is folded only once on DRAM. +- **A2** at N*: 0.20–0.67 — below the pre-registered 70–90% band, because N* is so small; + A2 rises with N (N·pop / (N·pop + 4096)). At N=64 full mask, A2 = 0.98. +- **A1, A3:** NOT RUN (no writeback, no learning in this probe). +- **P2 falsifier:** predicted-unused focus + 63 abandoned switches all reclaimed by + `evict_unused`; held foci stay correct. Disable (keep all entries) → red. + +OPEN: the fold cost is `ndarray::simd::masked_sum_i32`, which walks set bits one by one +(`src/simd_masking_ops.rs:814`), ≈0.57 ns per selected slot. A dense SIMD masked sum would +lower `fold` and move N* only further toward the focus. Not changed here. +OPEN: cold page faults, focus churn under a memory budget, and dirty-row writeback (A1, W1–W8). diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 9bbbabd21..bea8ef87a 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,11 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -465 entries, 2026-08-06 .. 2026-10-24. +466 entries, 2026-08-06 .. 2026-10-24. | date | entry id | finding | file | |---|---|---|---| | 2026-10-24 | `tanh-precision-symmetric-i8` | | [2026-10-24-tanh-precision-symmetric-i8.md](2026-10-24-tanh-precision-symmetric-i8.md) | +| 2026-10-24 | `D-DRF-P2` | | [2026-10-24-drf-p2-breakeven.md](2026-10-24-drf-p2-breakeven.md) | | 2026-10-24 | `D-DRF-P1` | | [2026-10-24-drf-p1-focus-lease-probe.md](2026-10-24-drf-p1-focus-lease-probe.md) | | 2026-10-24 | `D-DRF-1` | | [2026-10-24-d-drf-1-focus-lease-foreign-cache.md](2026-10-24-d-drf-1-focus-lease-foreign-cache.md) | | 2026-10-23 | `ps2-compose-chain-preregistration` | | [2026-10-23-ps2-compose-chain-preregistration.md](2026-10-23-ps2-compose-chain-preregistration.md) | From 219069687829debeb518a7c9cd955a2c719fef5f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 23:30:45 +0000 Subject: [PATCH 3/3] board: D-DRF-P2 entry, fold re-measured against ndarray #376 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .claude/board/entries/2026-10-24-drf-p2-breakeven.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.claude/board/entries/2026-10-24-drf-p2-breakeven.md b/.claude/board/entries/2026-10-24-drf-p2-breakeven.md index fad285711..11abf4e8c 100644 --- a/.claude/board/entries/2026-10-24-drf-p2-breakeven.md +++ b/.claude/board/entries/2026-10-24-drf-p2-breakeven.md @@ -23,7 +23,8 @@ AVX2 rows agree within ~10%. Clustered 1024-row cases match random 1024 (same N* - **P2 falsifier:** predicted-unused focus + 63 abandoned switches all reclaimed by `evict_unused`; held foci stay correct. Disable (keep all entries) → red. -OPEN: the fold cost is `ndarray::simd::masked_sum_i32`, which walks set bits one by one -(`src/simd_masking_ops.rs:814`), ≈0.57 ns per selected slot. A dense SIMD masked sum would -lower `fold` and move N* only further toward the focus. Not changed here. +Fold cost: `ndarray::simd::masked_sum_i32` walked set bits one by one, ≈0.57 ns per +selected slot. ndarray #376 (D-MPF-2b hybrid body, threshold 16) re-measured with the same +probe: full-mask fold 2 341 → 650 ns (AVX-512), 2 306 → 559 ns (AVX2); 25 % masks unchanged +(~470 ns). N* stays 1–2; with the cheaper fold it reaches 1 for full masks on AVX2 at 32 MiB. OPEN: cold page faults, focus churn under a memory budget, and dirty-row writeback (A1, W1–W8).