diff --git a/.claude/board/entries/2026-10-25-drav-av1-membership.md b/.claude/board/entries/2026-10-25-drav-av1-membership.md new file mode 100644 index 000000000..c5875ad59 --- /dev/null +++ b/.claude/board/entries/2026-10-25-drav-av1-membership.md @@ -0,0 +1,43 @@ +# D-DRAV-1 AV1 — map-only membership mask (lab, reported not gated) + +**Status:** MEASURED (lab). Contract: [D-DRAV-1](../plans/d-dr-address-first-value-demand-v1.md) +(#1775, draft) AV1. Code: `crates/drf-focus-probe/src/membership.rs`, tests +`tests/integration/membership.rs`, cost probe `examples/av1_membership.rs`. + +`M[i] = contains(S, DR64[i])`: a 4096-bit (512-byte) slot mask from a sealed DR64 ordinal map +and a set of canonical row ordinals in the same attested source domain and scope. + +- **Value independence is structural.** `membership(&BoundMap, &RowSet)` takes no `Soa`, + `Facet` or `Focus`, and neither argument holds a row or a facet. A `compile_fail` doctest + passes a `Soa` and is rejected with E0308; the same snippet with a `BoundMap` compiles + (checked, so the doctest fails for the stated reason). +- **Checks at the binding boundary.** `BoundMap::bind` refuses a seal mismatch and any ordinal + past the attested row count; `RowSet::new` refuses rows past it; `membership` refuses a + source (dataset/branch/version/seal) or scope mismatch. The slot loop does no checks. +- **Tests (6), against a linear-search scalar oracle:** random maps with hits and misses; + 1 000+ slots sharing one ordinal all match (no dedupe); empty and full sets; a seeded slot + shuffle permutes the mask; same ordinals in another dataset, version or scope refuse; stale + seal and out-of-range refuse. +- **Disable runs (red then green):** dedupe slots by ordinal (4 tests red); drop the domain + check (collision test red); reverse bit order within a word (3 red). + +Cost split, warm medians of 2001 calls, random map over 65 536 rows (native; `x86-64-v3` +agrees within noise): + +| step | ns | +|---|---| +| bind (seal + 4096 range checks) | 930-1 370 | +| set prep, k = 64 / 1 024 / 16 384 rows | 140-210 / 540-850 / 8 300-11 800 | +| membership exec (any k) | about 2 000 (0.5 ns/slot) | +| bind + prep + exec, k = 1 024 | 3 450-4 270 | +| value gather from the same map, for reference | 12 600-14 000 | + +The first loop (read-modify-write of the output word per slot) took about 6 000 ns; building +each 64-slot word in a register brought it to about 2 000. Both are scalar. + +Not shown: that real learning steps are map-only (A3 stays NOT RUN); preparation of a row set +from canonical values (the set here is built from an ordinal list); AV2 rehydration, which is +the next PR; joins (AV7/AV11). + +OPEN: AV2 (pinned historical snapshot reads allowed, rebinding to another version, index, +scope or facet refused). diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index baf6fd483..67341ea4b 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -472 entries, 2026-08-06 .. 2026-10-25. +473 entries, 2026-08-06 .. 2026-10-25. | date | entry id | finding | file | |---|---|---|---| @@ -35,6 +35,7 @@ exactly one of them, never both. | 2026-10-25 | `D-DRF-W1` | | [2026-10-25-drf-w1-writeback.md](2026-10-25-drf-w1-writeback.md) | | 2026-10-25 | `D-DRF-G-FACET` | | [2026-10-25-drf-focuskey-facet.md](2026-10-25-drf-focuskey-facet.md) | | 2026-10-25 | `D-DRF-CHURN` | | [2026-10-25-drf-churn-policy.md](2026-10-25-drf-churn-policy.md) | +| 2026-10-25 | `D-DRAV-1` | | [2026-10-25-drav-av1-membership.md](2026-10-25-drav-av1-membership.md) | | 2026-10-24 | `tanh-precision-symmetric-i8` | | [2026-10-24-tanh-precision-symmetric-i8.md](2026-10-24-tanh-precision-symmetric-i8.md) | | 2026-10-24 | `D-DRF-P2` | | [2026-10-24-drf-p2-breakeven.md](2026-10-24-drf-p2-breakeven.md) | | 2026-10-24 | `D-DRF-P1` | | [2026-10-24-drf-p1-focus-lease-probe.md](2026-10-24-drf-p1-focus-lease-probe.md) | diff --git a/crates/drf-focus-probe/examples/av1_membership.rs b/crates/drf-focus-probe/examples/av1_membership.rs new file mode 100644 index 000000000..e8fcefa88 --- /dev/null +++ b/crates/drf-focus-probe/examples/av1_membership.rs @@ -0,0 +1,86 @@ +//! D-DRAV-1 AV1 cost split: set preparation, binding and map-only membership, reported +//! separately and combined, against a value gather from the same map. Reported, not gated. +//! Warm medians of 2001 calls; the set is built from a list of row ordinals (no value read). +//! cargo run --release --example av1_membership +use drf_focus_probe::membership::*; +use drf_focus_probe::*; +use std::hint::black_box; +use std::sync::Arc; +use std::time::Instant; + +const NROWS: usize = 65536; + +fn median_ns(reps: usize, mut f: impl FnMut()) -> f64 { + let mut v: Vec = (0..reps) + .map(|_| { + let t = Instant::now(); + f(); + t.elapsed().as_nanos() as f64 + }) + .collect(); + v.sort_by(|a, b| a.partial_cmp(b).unwrap()); + v[reps / 2] +} + +fn main() { + let id = SourceId { + dataset: 1, + branch: 1, + version: 1, + index_seal: 7, + }; + let mut x = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x + }; + let mut ord = Box::new([0u16; SLOTS]); + ord.iter_mut() + .for_each(|o| *o = (next() % NROWS as u64) as u16); + let map = Arc::new(SealedMap::new(MapId(1), 7, ord)); + let rows_data: Vec = (0..NROWS).map(|_| [0u8; ROW]).collect(); + let soa = Soa { + rows: &rows_data, + id, + }; + let facet = Facet::new(16).unwrap(); + let gather = median_ns(2001, || { + let mgr = FocusManager::new(); + black_box(mgr.load(&soa, &map, Scope(1), facet).unwrap()); + }); + let bound = BoundMap::bind(map.clone(), id, Scope(1), NROWS).unwrap(); + let bind = median_ns(2001, || { + black_box(BoundMap::bind(map.clone(), id, Scope(1), NROWS).unwrap()); + }); + println!("value gather (FocusManager::load, same map): {gather:.0} ns"); + println!("bind (seal + 4096 range checks): {bind:.0} ns"); + println!( + "{:>7} {:>9} {:>9} {:>9} {:>6}", + "set_k", "prep_ns", "exec_ns", "total_ns", "hits" + ); + for k in [64usize, 1024, 16384] { + let list: Vec = (0..k).map(|_| (next() % NROWS as u64) as u16).collect(); + let prep = median_ns(2001, || { + black_box(RowSet::new(id, Scope(1), NROWS, list.iter().copied()).unwrap()); + }); + let set = RowSet::new(id, Scope(1), NROWS, list.iter().copied()).unwrap(); + let exec = median_ns(2001, || { + black_box(membership(black_box(&bound), black_box(&set)).unwrap()); + }); + let hits: u32 = membership(&bound, &set) + .unwrap() + .iter() + .map(|w| w.count_ones()) + .sum(); + println!( + "{:>7} {:>9.0} {:>9.0} {:>9.0} {:>6}", + k, + prep, + exec, + bind + prep + exec, + hits + ); + } +} diff --git a/crates/drf-focus-probe/src/lib.rs b/crates/drf-focus-probe/src/lib.rs index 0db9f46d4..94d1c9385 100644 --- a/crates/drf-focus-probe/src/lib.rs +++ b/crates/drf-focus-probe/src/lib.rs @@ -10,6 +10,7 @@ use std::sync::{Arc, Mutex, OnceLock}; #[cfg(feature = "handoff")] pub mod handoff; +pub mod membership; /// Canonical SoA row ABI. pub const ROW: usize = 512; @@ -107,6 +108,16 @@ pub enum Refusal { ord: u16, offset: usize, }, + /// A map and a row set bound to different sources or scopes (AV1). + DomainMismatch { + map: SourceId, + set: SourceId, + }, + /// Same source, different authorized scope (AV1). + ScopeMismatch { + map: Scope, + set: Scope, + }, } fn read_i32(row: &Row, offset: usize) -> i32 { diff --git a/crates/drf-focus-probe/src/membership.rs b/crates/drf-focus-probe/src/membership.rs new file mode 100644 index 000000000..c5c69d883 --- /dev/null +++ b/crates/drf-focus-probe/src/membership.rs @@ -0,0 +1,125 @@ +//! D-DRAV-1 AV1: map-only membership, `M[i] = contains(S, DR64[i])`. +//! +//! Value independence is structural: [`membership`] takes a [`BoundMap`] and a +//! [`RowSet`], and neither holds a row, a facet or a gathered column. A caller +//! that holds a `Soa` or a `Focus` cannot pass it here: +//! +//! ```compile_fail +//! use drf_focus_probe::{membership::*, *}; +//! fn try_with_rows(soa: &Soa<'_>, set: &RowSet) { +//! let _ = membership(soa, set); // a source with rows is not a BoundMap +//! } +//! ``` +//! +//! All domain, scope, seal and range checks run once, when the map and the set +//! are bound. The slot loop only reads the map and the set. Building a row set +//! from canonical values is preparation and is not part of this claim. + +use crate::{Refusal, Scope, SealedMap, SourceId, TILE_WORDS}; +use std::sync::Arc; + +/// Words in a bitset over the whole u16 ordinal domain. +const SET_WORDS: usize = 1 << 10; + +/// A 4096-bit slot mask, 512 bytes. +pub type SlotMask = [u64; TILE_WORDS]; + +/// A sealed map bound to one attested source domain and scope. Holds no rows. +pub struct BoundMap { + map: Arc, + source: SourceId, + scope: Scope, +} + +impl BoundMap { + /// Bind once: the map's seal must match the source's index seal, and every + /// ordinal must lie inside the attested row count. `n_rows` is source + /// metadata, not a value read. + pub fn bind( + map: Arc, + source: SourceId, + scope: Scope, + n_rows: usize, + ) -> Result { + if map.seal != source.index_seal { + return Err(Refusal::StaleMap { + map_seal: map.seal, + source_seal: source.index_seal, + }); + } + if let Some((slot, &ord)) = map + .ord + .iter() + .enumerate() + .find(|&(_, &o)| o as usize >= n_rows) + { + return Err(Refusal::OrdinalOutOfRange { slot, ord }); + } + Ok(Self { map, source, scope }) + } + + pub fn source(&self) -> SourceId { + self.source + } +} + +/// A set of canonical row ordinals in one attested source domain and scope. +pub struct RowSet { + bits: Box<[u64; SET_WORDS]>, + source: SourceId, + scope: Scope, +} + +impl RowSet { + /// Rows outside the attested row count are refused. Repeated rows are a set + /// member once; that does not dedupe map slots. + pub fn new( + source: SourceId, + scope: Scope, + n_rows: usize, + rows: impl IntoIterator, + ) -> Result { + let mut bits = Box::new([0u64; SET_WORDS]); + for (slot, r) in rows.into_iter().enumerate() { + if r as usize >= n_rows { + return Err(Refusal::OrdinalOutOfRange { slot, ord: r }); + } + bits[r as usize >> 6] |= 1 << (r & 63); + } + Ok(Self { + bits, + source, + scope, + }) + } + + fn contains(&self, r: u16) -> bool { + self.bits[r as usize >> 6] >> (r & 63) & 1 == 1 + } +} + +/// `M[i] = contains(S, DR64[i])` for every slot. Slots sharing an ordinal all +/// match. Refuses when the map and the set are bound to different domains. +pub fn membership(map: &BoundMap, set: &RowSet) -> Result { + if map.source != set.source { + return Err(Refusal::DomainMismatch { + map: map.source, + set: set.source, + }); + } + if map.scope != set.scope { + return Err(Refusal::ScopeMismatch { + map: map.scope, + set: set.scope, + }); + } + let mut m = [0u64; TILE_WORDS]; + for (w, chunk) in m.iter_mut().zip(map.map.ord.as_chunks::<64>().0) { + let mut word = 0u64; + for (bit, &o) in chunk.iter().enumerate() { + word |= (set.contains(o) as u64) << bit; + } + *w = word; + } + Ok(m) +} diff --git a/crates/drf-focus-probe/tests/integration/main.rs b/crates/drf-focus-probe/tests/integration/main.rs index 932dcbfa5..29d11315d 100644 --- a/crates/drf-focus-probe/tests/integration/main.rs +++ b/crates/drf-focus-probe/tests/integration/main.rs @@ -2,3 +2,4 @@ pub mod budget; pub mod focus; #[cfg(feature = "handoff")] pub mod handoff; +pub mod membership; diff --git a/crates/drf-focus-probe/tests/integration/membership.rs b/crates/drf-focus-probe/tests/integration/membership.rs new file mode 100644 index 000000000..d5c9c5dee --- /dev/null +++ b/crates/drf-focus-probe/tests/integration/membership.rs @@ -0,0 +1,166 @@ +//! D-DRAV-1 AV1: map-only membership against an independent scalar oracle. +use drf_focus_probe::membership::*; +use drf_focus_probe::*; +use std::sync::Arc; + +const N: usize = 65536; + +fn src(dataset: u64) -> SourceId { + SourceId { + dataset, + branch: 1, + version: 3, + index_seal: 7, + } +} + +fn xs(mut s: u64) -> impl FnMut() -> u64 { + move || { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + s + } +} + +fn map_from(ord: Box<[u16; SLOTS]>) -> Arc { + Arc::new(SealedMap::new(MapId(1), 7, ord)) +} + +fn random_map(seed: u64, nrows: usize) -> Box<[u16; SLOTS]> { + let mut r = xs(seed); + let mut o = Box::new([0u16; SLOTS]); + o.iter_mut().for_each(|v| *v = (r() % nrows as u64) as u16); + o +} + +/// Independent oracle: linear search over the row list, bit order slot i -> word i/64, bit i%64. +fn oracle(ord: &[u16; SLOTS], rows: &[u16]) -> SlotMask { + let mut m = [0u64; TILE_WORDS]; + for i in 0..SLOTS { + if rows.iter().any(|&r| r == ord[i]) { + m[i / 64] += 1u64 << (i % 64); + } + } + m +} + +fn pop(m: &SlotMask) -> u32 { + m.iter().map(|w| w.count_ones()).sum() +} + +fn run(ord: &[u16; SLOTS], rows: &[u16], nrows: usize) -> SlotMask { + let b = BoundMap::bind(map_from(Box::new(*ord)), src(1), Scope(1), nrows).unwrap(); + let s = RowSet::new(src(1), Scope(1), nrows, rows.iter().copied()).unwrap(); + membership(&b, &s).unwrap() +} + +#[test] +fn av1_random_map_matches_oracle_with_hits_and_misses() { + for seed in 1..=8u64 { + let ord = random_map(seed, 2048); + let mut r = xs(seed * 31); + let rows: Vec = (0..300).map(|_| (r() % 2048) as u16).collect(); + let m = run(&ord, &rows, 2048); + assert_eq!(m, oracle(&ord, &rows), "seed {seed}"); + let p = pop(&m); + assert!(p > 0 && (p as usize) < SLOTS, "anti-vacuity: {p} hits"); + } +} + +#[test] +fn av1_duplicate_ordinals_all_match() { + // 1000 slots point at row 7, the rest at row 9. + let mut ord = Box::new([9u16; SLOTS]); + for (i, o) in ord.iter_mut().enumerate() { + if i % 4 == 1 || i < 24 { + *o = 7; + } + } + let dup = ord.iter().filter(|&&o| o == 7).count() as u32; + assert!(dup > 1000); + let m = run(&ord, &[7], 16); + assert_eq!(pop(&m), dup, "every slot that points at row 7 matches"); + assert_eq!(m, oracle(&ord, &[7])); + // A repeated row in the set changes nothing. + assert_eq!(run(&ord, &[7, 7, 7], 16), m); +} + +#[test] +fn av1_empty_and_full_sets() { + let ord = random_map(5, 300); + assert_eq!(pop(&run(&ord, &[], 300)), 0); + let all: Vec = (0..300).collect(); + assert_eq!(run(&ord, &all, 300), [u64::MAX; TILE_WORDS]); +} + +#[test] +fn av1_shuffled_map_permutes_the_mask() { + let ord = random_map(11, 1000); + let rows: Vec = (0..1000).filter(|r| r % 3 == 0).collect(); + let base = run(&ord, &rows, 1000); + // Seeded permutation p: shuffled[i] = ord[p[i]], so bit i of the new mask = bit p[i] of the old. + let mut p: Vec = (0..SLOTS).collect(); + let mut r = xs(99); + for i in (1..SLOTS).rev() { + p.swap(i, (r() % (i as u64 + 1)) as usize); + } + let mut sh = Box::new([0u16; SLOTS]); + for i in 0..SLOTS { + sh[i] = ord[p[i]]; + } + let m = run(&sh, &rows, 1000); + for i in 0..SLOTS { + let got = m[i / 64] >> (i % 64) & 1; + let want = base[p[i] / 64] >> (p[i] % 64) & 1; + assert_eq!(got, want, "slot {i}"); + } + assert_ne!(m, base, "anti-vacuity: the shuffle moved bits"); +} + +#[test] +fn av1_cross_dataset_ordinal_collision_refuses() { + let ord = random_map(3, 500); + let b = BoundMap::bind(map_from(ord), src(1), Scope(1), 500).unwrap(); + // Same ordinals, other dataset: the row 42 here is a different row. + let other = RowSet::new(src(2), Scope(1), 500, [42u16, 43]).unwrap(); + assert!(matches!( + membership(&b, &other), + Err(Refusal::DomainMismatch { .. }) + )); + let mut v = src(1); + v.version = 4; + let newer = RowSet::new(v, Scope(1), 500, [42u16]).unwrap(); + assert!(matches!( + membership(&b, &newer), + Err(Refusal::DomainMismatch { .. }) + )); + let scoped = RowSet::new(src(1), Scope(2), 500, [42u16]).unwrap(); + assert!(matches!( + membership(&b, &scoped), + Err(Refusal::ScopeMismatch { .. }) + )); + // Control: same domain and scope is answered. + let same = RowSet::new(src(1), Scope(1), 500, [42u16]).unwrap(); + assert!(membership(&b, &same).is_ok()); +} + +#[test] +fn av1_binding_refuses_stale_seal_and_out_of_range() { + let mut s = src(1); + s.index_seal = 8; + assert!(matches!( + BoundMap::bind(map_from(random_map(1, 100)), s, Scope(1), 100), + Err(Refusal::StaleMap { .. }) + )); + assert!(matches!( + BoundMap::bind(map_from(random_map(1, 100)), src(1), Scope(1), 99), + Err(Refusal::OrdinalOutOfRange { .. }) + )); + assert!(matches!( + RowSet::new(src(1), Scope(1), 100, [100u16]), + Err(Refusal::OrdinalOutOfRange { .. }) + )); + // Control: full u16 domain binds. + assert!(BoundMap::bind(map_from(random_map(1, N)), src(1), Scope(1), N).is_ok()); +}