From 3d099f76e00a839ec8a904a857cfcf9be305ae68 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 21:53:31 +0000 Subject: [PATCH 1/7] dir-rmap-probe: D-RMAP-DIR-1 arms A/D/B/B+/F and equivalence tests (lab only) A lab probe over lance-graph-dir-sim's public API; no production file changes. - fixture: deterministic IAM directory (roles, locations, role x location composite groups nested in both, resources with cycles, licenses, teams, divisions that cut the SID chain, grants, a u8 attribute lane). - B: bidirectional membership CSR + nesting CSR, by dir-sim ordinal. - B+: closure rows per group for every chain (all, security, mail), a fixpoint over the nesting CSR; roles carried by the SID chain per group; incremental nesting insertion. - F: 256-slot focus over selected groups, B+ closure rows projected onto the slots; F+users adds one focus row per user. - D: crepe Datalog oracle over fixture ids. Tests: every arm equals dir-sim's walks and the oracle on every user and group, for all three chains, roles and direct membership; the focus answers role x location like both; incremental nesting equals a recompile after an overlay; stale witnesses and unfocused groups are refused; the fixture is checked for cycles, large groups and filtering chains. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- crates/dir-rmap-probe/Cargo.toml | 44 ++ crates/dir-rmap-probe/src/fixture.rs | 338 +++++++++++++ crates/dir-rmap-probe/src/focus.rs | 142 ++++++ crates/dir-rmap-probe/src/lib.rs | 197 ++++++++ crates/dir-rmap-probe/src/oracle.rs | 107 +++++ crates/dir-rmap-probe/src/prep.rs | 526 +++++++++++++++++++++ crates/dir-rmap-probe/tests/equivalence.rs | 268 +++++++++++ 7 files changed, 1622 insertions(+) create mode 100644 crates/dir-rmap-probe/Cargo.toml create mode 100644 crates/dir-rmap-probe/src/fixture.rs create mode 100644 crates/dir-rmap-probe/src/focus.rs create mode 100644 crates/dir-rmap-probe/src/lib.rs create mode 100644 crates/dir-rmap-probe/src/oracle.rs create mode 100644 crates/dir-rmap-probe/src/prep.rs create mode 100644 crates/dir-rmap-probe/tests/equivalence.rs diff --git a/crates/dir-rmap-probe/Cargo.toml b/crates/dir-rmap-probe/Cargo.toml new file mode 100644 index 000000000..2bbf11043 --- /dev/null +++ b/crates/dir-rmap-probe/Cargo.toml @@ -0,0 +1,44 @@ +# dir-rmap-probe — D-RMAP-DIR-1, lab only. +# +# Benchmarks compiled membership closure (B+) and a 256-slot group focus (F) +# against dir-sim's own View walks (A), a prepared CSR (B) and an independent +# crepe Datalog oracle (D). Nothing here is a production path: dir-sim stays +# authoritative and is consumed only through its public API. +# +# Own [workspace] root, excluded from the lance-graph workspace like +# lance-graph-dir-sim (which it depends on). Run from THIS directory: +# cargo test +# cargo run --release --example bench # 8,000 users +# cargo run --release --example bench -- 65000 # large fixture +[package] +name = "dir-rmap-probe" +version = "0.0.0" +edition = "2024" +publish = false +description = "Lab probe D-RMAP-DIR-1: compiled membership closure and a 256-slot group focus over dir-sim, against View walks, a prepared CSR and a crepe Datalog oracle. Results only; no production path." + +[workspace] + +[dependencies] +lance-graph-dir-sim = { path = "../lance-graph-dir-sim" } +ogar-dir-core = { git = "https://github.com/AdaWorldAPI/OGAR", branch = "main" } +ogar-dir-sim = { git = "https://github.com/AdaWorldAPI/OGAR", branch = "main" } +# The mask facade (`mask_or_assign`, `eq_u8_to_mask`, popcount), the same +# kernels #1634's walk_mask uses. Same coordinate as mask-risc, so the graph +# keeps one ndarray package. +ndarray = { git = "https://github.com/AdaWorldAPI/ndarray", branch = "master", default-features = false, features = ["std"] } +# Arm D, the independent oracle: crepe's generated Datalog evaluator over the +# fixture's own integer ids (never dir-sim ordinals). AdaWorldAPI fork, by +# branch (no pins), as in crepe-plan-probe. +crepe = { git = "https://github.com/AdaWorldAPI/crepe", branch = "ccr-d3ce1263-ied9yh" } + +# Same patches as lance-graph-dir-sim: one ndarray identity, and the OGAR +# sibling checkout so a paired OGAR change is tested before it merges. +[patch."https://github.com/AdaWorldAPI/ndarray"] +ndarray = { path = "../../../ndarray" } + +[patch."https://github.com/AdaWorldAPI/OGAR"] +ogar-dir-core = { path = "../../../OGAR/crates/ogar-dir-core" } +ogar-dir-sim = { path = "../../../OGAR/crates/ogar-dir-sim" } +ogar-ad = { path = "../../../OGAR/crates/ogar-ad" } +ogar-az = { path = "../../../OGAR/crates/ogar-az" } diff --git a/crates/dir-rmap-probe/src/fixture.rs b/crates/dir-rmap-probe/src/fixture.rs new file mode 100644 index 000000000..3e08290f8 --- /dev/null +++ b/crates/dir-rmap-probe/src/fixture.rs @@ -0,0 +1,338 @@ +//! A synthetic IAM directory, generated deterministically from a seed. +//! +//! Every group is an ordinary dir-sim group. Role and location are groups, +//! and a role x location combination is a composite group nested in both, so +//! no new membership semantics is needed: inheritance is plain nesting. A +//! conjunction ("in role R and in location L") is a separate question the +//! probe asks of the closure, never a membership. +//! +//! The fixture keeps its own integer ids (`users[i]`, `groups[j]`) and its +//! own edge lists. The oracle reads those; the prepared arms translate them +//! through dir-sim's public ordinals. GUIDs are random, so the fixture order +//! and dir-sim's GUID-sorted ordinal order are unrelated. + +use lance_graph_dir_sim::{Observation, ObservedNode}; +use ogar_dir_core::{DirectoryScope, Guid128}; + +/// What kind of group a fixture group is. Only the generator and the +/// anti-vacuity checks read it; dir-sim never sees it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + /// Every user, directly: the large-group fallback case. + AllUsers, + /// A division: locations are nested in it. Mail-enabled, not security. + Division, + /// The company: divisions are nested in it. + Company, + Role, + Location, + /// A role x location combination, nested in its role and its location. + Composite, + /// A resource group: roles and other resources are nested in it. + Resource, + License, + /// A manual team: users added directly, mail-enabled distribution group. + Team, +} + +/// Sizes of one fixture. +#[derive(Clone, Copy, Debug)] +pub struct Spec { + pub users: usize, + pub divisions: usize, + pub roles: usize, + pub locations: usize, + pub composites: usize, + pub resources: usize, + pub licenses: usize, + pub teams: usize, + /// Percent of users with 1 to 3 extra direct resource memberships. + pub manual_pct: u64, + /// Back edges between resources that close a cycle. + pub cycles: usize, + /// Distinct values of the probe's `u8` attribute lane. + pub attr_values: u8, + /// Distinct role ids granted to groups (at most 64). + pub role_ids: u8, + pub seed: u64, +} + +impl Spec { + /// About 8,000 users and 1,500 groups. + pub fn iam(users: usize) -> Self { + Spec { + users, + divisions: 7, + roles: 300, + locations: 40, + composites: 600, + resources: 500, + licenses: 16, + teams: 30, + manual_pct: 10, + cycles: 6, + attr_values: 12, + role_ids: 48, + seed: 0xD1_5C0D, + } + } + + /// A few hundred users and groups: for the exhaustive equivalence tests. + pub fn small() -> Self { + Spec { + users: 600, + divisions: 3, + roles: 40, + locations: 8, + composites: 60, + resources: 50, + licenses: 5, + teams: 6, + manual_pct: 15, + cycles: 3, + attr_values: 6, + role_ids: 20, + seed: 0x5EED_0001, + } + } +} + +/// splitmix64. +pub struct Rng(u64); + +impl Rng { + pub fn new(seed: u64) -> Self { + Rng(seed) + } + pub fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + pub fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + fn guid(&mut self) -> Guid128 { + let mut b = [0u8; 16]; + b[..8].copy_from_slice(&self.next().to_le_bytes()); + b[8..].copy_from_slice(&self.next().to_le_bytes()); + Guid128(b) + } +} + +/// One generated directory, in fixture ids. +pub struct Fixture { + pub spec: Spec, + pub users: Vec, + pub groups: Vec, + pub kind: Vec, + /// Has a primary SMTP address (dir-sim's `MailEnabled`). + pub mail: Vec, + /// `Some(true)` security-enabled, `Some(false)` not, `None` not read. + pub security: Vec>, + /// `(user, group)`, direct. + pub direct: Vec<(u32, u32)>, + /// `(child group, parent group)`. + pub nested: Vec<(u32, u32)>, + /// `(group, role id)`. + pub grants: Vec<(u32, u8)>, + /// Groups of each kind, in fixture ids. + pub roles: Vec, + pub locations: Vec, + pub teams: Vec, + /// A `u8` attribute per user, in fixture user order (`< attr_values`). + pub attr: Vec, + pub scope: DirectoryScope, +} + +impl Fixture { + pub fn generate(spec: Spec) -> Self { + let mut r = Rng::new(spec.seed); + let mut f = Fixture { + spec, + users: (0..spec.users).map(|_| r.guid()).collect(), + groups: Vec::new(), + kind: Vec::new(), + mail: Vec::new(), + security: Vec::new(), + direct: Vec::new(), + nested: Vec::new(), + grants: Vec::new(), + roles: Vec::new(), + locations: Vec::new(), + teams: Vec::new(), + attr: (0..spec.users) + .map(|_| (r.next() % u64::from(spec.attr_values)) as u8) + .collect(), + scope: DirectoryScope(Guid128([0x5C; 16])), + }; + let add = |f: &mut Fixture, r: &mut Rng, k: Kind, mail: bool, sec: Option| { + f.groups.push(r.guid()); + f.kind.push(k); + f.mail.push(mail); + f.security.push(sec); + (f.groups.len() - 1) as u32 + }; + let all = add(&mut f, &mut r, Kind::AllUsers, true, Some(true)); + let company = add(&mut f, &mut r, Kind::Company, true, Some(true)); + let divisions: Vec = (0..spec.divisions) + .map(|_| add(&mut f, &mut r, Kind::Division, true, Some(false))) + .collect(); + for &d in &divisions { + f.nested.push((d, company)); + } + f.roles = (0..spec.roles) + .map(|_| add(&mut f, &mut r, Kind::Role, false, Some(true))) + .collect(); + // Locations are mail-enabled security groups, nested in a division: + // the SID chain stops at the division, mail continues. + f.locations = (0..spec.locations) + .map(|_| add(&mut f, &mut r, Kind::Location, true, Some(true))) + .collect(); + for (i, &l) in f.locations.clone().iter().enumerate() { + f.nested.push((l, divisions[i % divisions.len()])); + } + let resources: Vec = (0..spec.resources) + .map(|i| { + // One in 20 resources: security flag not read. + let sec = if i % 20 == 7 { None } else { Some(true) }; + add(&mut f, &mut r, Kind::Resource, i % 3 == 0, sec) + }) + .collect(); + // A DAG over resources (child -> later parent), then back edges. + for (i, &c) in resources.iter().enumerate() { + if i + 1 < resources.len() && r.below(10) < 3 { + let p = resources[i + 1 + r.below(resources.len() - i - 1)]; + f.nested.push((c, p)); + } + } + for k in 0..spec.cycles { + let a = resources[(k * 7) % resources.len()]; + let b = resources[(k * 7 + 1) % resources.len()]; + f.nested.push((a, b)); + f.nested.push((b, a)); + } + // Every role is nested in 1 to 3 resources. + for &ro in &f.roles { + for _ in 0..1 + r.below(3) { + f.nested.push((ro, resources[r.below(resources.len())])); + } + } + let licenses: Vec = (0..spec.licenses) + .map(|_| add(&mut f, &mut r, Kind::License, false, Some(true))) + .collect(); + f.teams = (0..spec.teams) + .map(|i| { + // Teams are distribution groups; one in 5 is also security. + add(&mut f, &mut r, Kind::Team, true, Some(i % 5 == 0)) + }) + .collect(); + // Users: one role (skewed) and one location each. A composite group + // exists for the first `composites` distinct pairs met. + let mut composite = std::collections::HashMap::new(); + for u in 0..spec.users as u32 { + let a = r.below(spec.roles); + let b = r.below(spec.roles); + let role = f.roles[a * b / spec.roles]; + let loc = f.locations[r.below(spec.locations)]; + let key = (role, loc); + let c = match composite.get(&key) { + Some(&c) => Some(c), + None if composite.len() < spec.composites => { + let c = add(&mut f, &mut r, Kind::Composite, false, Some(true)); + f.nested.push((c, role)); + f.nested.push((c, loc)); + composite.insert(key, c); + Some(c) + } + None => None, + }; + match c { + Some(c) => f.direct.push((u, c)), + None => { + f.direct.push((u, role)); + f.direct.push((u, loc)); + } + } + f.direct.push((u, all)); + if r.below(10) < 9 { + f.direct.push((u, licenses[r.below(licenses.len())])); + } + if r.below(100) < spec.manual_pct as usize { + for _ in 0..1 + r.below(3) { + f.direct.push((u, resources[r.below(resources.len())])); + } + } + } + // Teams: 20 to 250 direct users each, chosen at random. + for &t in &f.teams.clone() { + for _ in 0..20 + r.below(231) { + f.direct.push((r.below(spec.users) as u32, t)); + } + } + f.direct.sort_unstable(); + f.direct.dedup(); + f.nested.sort_unstable(); + f.nested.dedup(); + // Grants: on half the resources, a fifth of the roles, and on some + // teams (which only count when the team is a security group). + for (j, k) in f.kind.iter().enumerate() { + let p = match k { + Kind::Resource => 50, + Kind::Role => 20, + Kind::Team => 50, + Kind::Location => 25, + _ => 0, + }; + if r.below(100) < p { + f.grants + .push((j as u32, (r.next() % u64::from(spec.role_ids)) as u8)); + } + } + f + } + + /// The dir-sim observation of this fixture. + pub fn observation(&self) -> Observation { + let mut nodes = Vec::with_capacity(self.users.len() + self.groups.len()); + for (i, g) in self.users.iter().enumerate() { + let a = format!("u{i}@probe.test"); + nodes.push((*g, ObservedNode::user(&a, &a))); + } + for (j, g) in self.groups.iter().enumerate() { + let mut n = ObservedNode::group(); + if self.mail[j] { + n.primary_smtp = Some(format!("g{j}@probe.test")); + } + n.security = self.security[j]; + nodes.push((*g, n)); + } + let mut members = Vec::with_capacity(self.direct.len() + self.nested.len()); + members.extend( + self.direct + .iter() + .map(|&(u, g)| (self.users[u as usize], self.groups[g as usize])), + ); + members.extend( + self.nested + .iter() + .map(|&(c, p)| (self.groups[c as usize], self.groups[p as usize])), + ); + Observation { + scope: self.scope, + nodes, + members, + } + } + + /// Groups with more than 256 direct members. + pub fn large_groups(&self) -> usize { + let mut n = vec![0usize; self.groups.len()]; + for &(_, g) in &self.direct { + n[g as usize] += 1; + } + n.iter().filter(|&&c| c > 256).count() + } +} diff --git a/crates/dir-rmap-probe/src/focus.rs b/crates/dir-rmap-probe/src/focus.rs new file mode 100644 index 000000000..f7939889a --- /dev/null +++ b/crates/dir-rmap-probe/src/focus.rs @@ -0,0 +1,142 @@ +//! Arm F: a 256-slot focus over selected group identities, compiled over B+. +//! +//! The focus binds up to 256 group ordinals to local slots once. Every +//! group's B+ closure row (every group, upward) is then projected onto the +//! focus slots: 4 words per group instead of `ceil(groups / 64)`. A user's +//! position in the focus is the OR of its direct groups' projected rows, and +//! a role x location question is a 4-word mask test. +//! +//! Two variants: +//! +//! * [`Focus`]: per-group projected rows only. Each evaluation ORs the +//! user's direct groups' rows (4 words each). +//! * [`FocusUsers`]: additionally one 4-word row per user, computed once. +//! This is the per-user pointer cache the proposal asked to keep out +//! unless measured; it is here so the measurement exists. +//! +//! No per-user 256 x 256 tile is allocated. The projection reads closure +//! rows at the focus's bound ordinals: an address translation, done once in +//! preparation and counted. Its admission is `ISS-ADMISSION-PREP-BINDING`. +//! +//! Reverse lookup (group ordinal -> slot) is a direct `u16` array over the +//! group ordinal space (slot + 1, 0 = not bound): one indexed read, no hash, +//! no search. At 1,500 groups that is 3 KB. + +use crate::prep::{bit, Compiled, Refusal, Witness}; + +pub const SLOTS: usize = 256; +pub type Row = [u64; 4]; + +#[inline] +fn or(a: &mut Row, b: &Row) { + for k in 0..4 { + a[k] |= b[k]; + } +} + +#[inline] +fn set(r: &mut Row, s: usize) { + r[s >> 6] |= 1 << (s & 63); +} + +pub struct Focus { + pub witness: Witness, + /// Slot -> group ordinal. + pub ids: Vec, + /// Group ordinal -> slot + 1; 0 = not bound. + slot_of: Vec, + /// Group ordinal -> its closure row (every group, upward) on the slots. + proj: Vec, +} + +impl Focus { + /// Bind `groups` (distinct, at most 256) and project every group's + /// closure row onto them. + pub fn bind(c: &Compiled, groups: &[u16]) -> Focus { + assert!(groups.len() <= SLOTS, "a focus binds at most 256 groups"); + let n = c.b.groups; + let mut slot_of = vec![0u16; n]; + for (s, &g) in groups.iter().enumerate() { + assert_eq!(slot_of[g as usize], 0, "focus groups must be distinct"); + slot_of[g as usize] = s as u16 + 1; + } + let mut proj = vec![[0u64; 4]; n]; + for (g, row) in proj.iter_mut().enumerate() { + let up = c.row(crate::prep::Chain::All, g); + for (s, &f) in groups.iter().enumerate() { + if bit(up, f as usize) { + set(row, s); + } + } + } + Focus { + witness: c.b.witness, + ids: groups.to_vec(), + slot_of, + proj, + } + } + + pub fn check(&self, current: &Witness) -> Result<(), Refusal> { + if self.witness == *current { + Ok(()) + } else { + Err(Refusal::Stale) + } + } + + pub fn slot(&self, g: u16) -> Option { + match self.slot_of[g as usize] { + 0 => None, + s => Some(s as usize - 1), + } + } + + /// The mask for "in every one of `groups`"; refuses when one is not in + /// the focus, so a caller falls back to B+ instead of reading a bit that + /// was never projected. + pub fn all_of(&self, groups: &[u16]) -> Result { + let mut m = [0u64; 4]; + for &g in groups { + set(&mut m, self.slot(g).ok_or(Refusal::OutOfFocus)?); + } + Ok(m) + } + + /// The user's focus row: OR of its direct groups' projected rows. + #[inline] + pub fn row_of(&self, c: &Compiled, u: u16) -> Row { + let mut r = [0u64; 4]; + for &d in c.b.user_groups.row(u as usize) { + or(&mut r, &self.proj[d as usize]); + } + r + } + + pub fn bytes(&self) -> usize { + self.ids.len() * 2 + self.slot_of.len() * 2 + self.proj.len() * 32 + } +} + +#[inline] +pub fn holds(row: &Row, mask: &Row) -> bool { + (0..4).all(|k| row[k] & mask[k] == mask[k]) +} + +/// [`Focus`] plus one precomputed focus row per user. +pub struct FocusUsers { + pub focus: Focus, + pub rows: Vec, +} + +impl FocusUsers { + pub fn bind(c: &Compiled, groups: &[u16]) -> FocusUsers { + let focus = Focus::bind(c, groups); + let rows = (0..c.b.users as u16).map(|u| focus.row_of(c, u)).collect(); + FocusUsers { focus, rows } + } + + pub fn bytes(&self) -> usize { + self.focus.bytes() + self.rows.len() * 32 + } +} diff --git a/crates/dir-rmap-probe/src/lib.rs b/crates/dir-rmap-probe/src/lib.rs new file mode 100644 index 000000000..12471a5f8 --- /dev/null +++ b/crates/dir-rmap-probe/src/lib.rs @@ -0,0 +1,197 @@ +//! D-RMAP-DIR-1, lab only: compiled membership closure and a 256-slot group +//! focus over `lance-graph-dir-sim`. +//! +//! STATUS: PROBE. Plan: `.claude/plans/d-rmap-dir-1-membership-closure-focus-v1.md`. +//! Nothing here is a production path. dir-sim stays authoritative and is +//! read only through its public API; no production file is changed. +//! +//! Arms: +//! +//! | arm | module | what | +//! |---|---|---| +//! | A | dir-sim `View` | the existing walks (authoritative) | +//! | D | [`oracle`] | crepe Datalog over fixture ids (independent oracle) | +//! | B | [`prep::Prepared`] | bidirectional membership CSR + nesting CSR | +//! | B+ | [`prep::Compiled`] | B + closure rows per group and chain + roles by ordinal | +//! | F | [`focus`] | B+ projected onto a 256-slot focus of selected groups | + +pub mod fixture; +pub mod focus; +pub mod oracle; +pub mod prep; + +use fixture::{Fixture, Kind}; +use lance_graph_dir_sim::{GroupProperty, GroupWhere, View}; +use ogar_dir_core::Guid128; +use prep::Chain; + +/// Arm A: a user's groups through `chain`, as dir-sim answers it. +pub fn a_groups(v: &View<'_>, u: &Guid128, chain: Chain) -> Vec { + match chain { + Chain::All => v.groups_transitive(u), + Chain::Security => v.security_identifiers(u), + Chain::Mail => v.groups_transitive_through(u, &GroupWhere::Is(GroupProperty::MailEnabled)), + } +} + +/// Arm A: a group's users through `chain`. +pub fn a_members(v: &View<'_>, g: &Guid128, chain: Chain) -> Vec { + match chain { + Chain::All => v.members_transitive(g), + Chain::Security => { + v.members_transitive_through(g, &GroupWhere::Is(GroupProperty::SecurityEnabled)) + } + Chain::Mail => v.members_transitive_through(g, &GroupWhere::Is(GroupProperty::MailEnabled)), + } +} + +/// Role x location questions, in fixture ids: the `(role, location)` pairs +/// a composite group joins, in the order the composites were created. +pub fn templates(f: &Fixture) -> Vec<(u32, u32)> { + let mut parents: std::collections::BTreeMap, Option)> = + Default::default(); + for &(c, p) in &f.nested { + if f.kind[c as usize] != Kind::Composite { + continue; + } + let e = parents.entry(c).or_default(); + match f.kind[p as usize] { + Kind::Role => e.0 = Some(p), + Kind::Location => e.1 = Some(p), + _ => {} + } + } + parents + .values() + .filter_map(|&(r, l)| Some((r?, l?))) + .collect() +} + +/// The longest prefix of `templates` whose groups fit in `k` slots, and +/// those groups (first-seen order). +pub fn focus_for(templates: &[(u32, u32)], k: usize) -> (Vec<(u32, u32)>, Vec) { + let mut groups: Vec = Vec::new(); + let mut kept = Vec::new(); + for &(r, l) in templates { + let new = [r, l].iter().filter(|g| !groups.contains(g)).count(); + if groups.len() + new > k { + continue; + } + for g in [r, l] { + if !groups.contains(&g) { + groups.push(g); + } + } + kept.push((r, l)); + } + (kept, groups) +} + +use lance_graph_dir_sim::{Rule, VersionStore}; +use ogar_dir_sim::{Change, EvidenceRef, RuleId, VersionId}; + +/// A fixture observed into a dir-sim store, plus the edge lists of the +/// current version (the fixture's, then every applied change). +pub struct World { + pub f: Fixture, + pub st: VersionStore, + pub version: VersionId, + pub direct: Vec<(u32, u32)>, + pub nested: Vec<(u32, u32)>, +} + +impl World { + pub fn new(f: Fixture) -> Self { + let mut st = VersionStore::new(); + let version = st.observe("probe", 0, f.observation()).expect("fixture builds"); + World { + direct: f.direct.clone(), + nested: f.nested.clone(), + f, + st, + version, + } + } + + pub fn view(&self) -> View<'_> { + self.st.view(self.version).expect("version exists") + } + + /// Apply a change list as a simulated child version, and the same edits + /// to the edge lists the prepared arms and the oracle are built from. + pub fn apply(&mut self, edits: &[Edit]) { + let f = &self.f; + let changes = edits + .iter() + .map(|e| match *e { + Edit::AddDirect(u, g) => Change::AddMembership { + user: f.users[u as usize], + group: f.groups[g as usize], + }, + Edit::RemoveDirect(u, g) => Change::RemoveMembership { + user: f.users[u as usize], + group: f.groups[g as usize], + }, + Edit::AddNested(c, p) => Change::AddMembership { + user: f.groups[c as usize], + group: f.groups[p as usize], + }, + }) + .collect(); + self.version = self + .st + .simulate(self.version, &Propose(changes), &[]) + .expect("change list applies"); + for e in edits { + match *e { + Edit::AddDirect(u, g) => self.direct.push((u, g)), + Edit::RemoveDirect(u, g) => self.direct.retain(|&x| x != (u, g)), + Edit::AddNested(c, p) => self.nested.push((c, p)), + } + } + self.direct.sort_unstable(); + self.direct.dedup(); + self.nested.sort_unstable(); + self.nested.dedup(); + } + + pub fn witness(&self) -> prep::Witness { + prep::Witness::of(self.version, &self.view()) + } + + /// Arm B for the current version. + pub fn prepare(&self) -> prep::Prepared { + let v = self.view(); + let ord = prep::Ordinals::of(&self.f, &v); + prep::Prepared::build( + &v, + prep::Witness::of(self.version, &v), + &ord, + &self.direct, + &self.nested, + &self.f.grants, + ) + } +} + +/// One edit, in fixture ids. +#[derive(Clone, Copy, Debug)] +pub enum Edit { + AddDirect(u32, u32), + RemoveDirect(u32, u32), + AddNested(u32, u32), +} + +struct Propose(Vec); + +impl Rule for Propose { + fn id(&self) -> RuleId { + RuleId { + name: "DirRmapProbe", + version: 1, + } + } + fn propose(&self, _: &View<'_>, _: &[EvidenceRef]) -> Vec { + self.0.clone() + } +} diff --git a/crates/dir-rmap-probe/src/oracle.rs b/crates/dir-rmap-probe/src/oracle.rs new file mode 100644 index 000000000..409e2f4e5 --- /dev/null +++ b/crates/dir-rmap-probe/src/oracle.rs @@ -0,0 +1,107 @@ +//! Arm D: the independent oracle. crepe's generated Datalog evaluator over +//! the fixture's own integer ids. It never sees a dir-sim ordinal, a CSR or +//! a closure row, so it shares no code with the arms it checks. +//! +//! The rules state dir-sim's semantics directly: +//! +//! * `Member`: nesting follows every group. +//! * `SecMember` / `MailMember`: a chain through groups that have the +//! property; the user's direct group and every parent on the way must +//! have it (`groups_transitive_through`, `members_transitive_through`). +//! * `RoleHeld`: the roles granted to the groups whose SID the user holds. +//! * `Eligible`: a role x location question, the AND of two memberships. +//! It is a question asked of the closure, not a membership. + +#![allow(clippy::all)] + +use crate::fixture::Fixture; +use crepe::crepe; +use std::collections::{BTreeSet, HashSet}; + +crepe! { + @input + struct Direct(u32, u32); + @input + struct Nested(u32, u32); + @input + struct Sec(u32); + @input + struct Mail(u32); + @input + struct Grant(u32, u32); + @input + struct Template(u32, u32); + + @output + struct Member(u32, u32); + @output + struct SecMember(u32, u32); + @output + struct MailMember(u32, u32); + @output + struct RoleHeld(u32, u32); + @output + struct Eligible(u32, u32, u32); + + Member(u, g) <- Direct(u, g); + Member(u, p) <- Member(u, c), Nested(c, p); + + SecMember(u, g) <- Direct(u, g), Sec(g); + SecMember(u, p) <- SecMember(u, c), Nested(c, p), Sec(p); + + MailMember(u, g) <- Direct(u, g), Mail(g); + MailMember(u, p) <- MailMember(u, c), Nested(c, p), Mail(p); + + RoleHeld(u, r) <- SecMember(u, g), Grant(g, r); + + Eligible(u, r, l) <- Template(r, l), Member(u, r), Member(u, l); +} + +/// The oracle's answers, in fixture ids. +pub struct Answers { + pub member: BTreeSet<(u32, u32)>, + pub sec: BTreeSet<(u32, u32)>, + pub mail: BTreeSet<(u32, u32)>, + pub roles: BTreeSet<(u32, u32)>, + pub eligible: BTreeSet<(u32, u32, u32)>, +} + +/// Run the program. `direct` and `nested` are passed separately so an +/// overlay's edge lists can be checked too. +pub fn run( + f: &Fixture, + direct: &[(u32, u32)], + nested: &[(u32, u32)], + templates: &[(u32, u32)], +) -> Answers { + let mut rt = Crepe::new(); + rt.extend(direct.iter().map(|&(u, g)| Direct(u, g))); + rt.extend(nested.iter().map(|&(c, p)| Nested(c, p))); + // dir-sim's `SecurityEnabled` is "known and true"; `MailEnabled` is + // "has a primary SMTP address". + rt.extend( + (0..f.groups.len() as u32) + .filter(|&g| f.security[g as usize] == Some(true)) + .map(Sec), + ); + rt.extend((0..f.groups.len() as u32).filter(|&g| f.mail[g as usize]).map(Mail)); + rt.extend(f.grants.iter().map(|&(g, r)| Grant(g, u32::from(r)))); + rt.extend(templates.iter().map(|&(r, l)| Template(r, l))); + let (member, sec, mail, roles, eligible): ( + HashSet, + HashSet, + HashSet, + HashSet, + HashSet, + ) = rt.run(); + Answers { + member: member.into_iter().map(|Member(u, g)| (u, g)).collect(), + sec: sec.into_iter().map(|SecMember(u, g)| (u, g)).collect(), + mail: mail.into_iter().map(|MailMember(u, g)| (u, g)).collect(), + roles: roles.into_iter().map(|RoleHeld(u, r)| (u, r)).collect(), + eligible: eligible + .into_iter() + .map(|Eligible(u, r, l)| (u, r, l)) + .collect(), + } +} diff --git a/crates/dir-rmap-probe/src/prep.rs b/crates/dir-rmap-probe/src/prep.rs new file mode 100644 index 000000000..1d87d13a9 --- /dev/null +++ b/crates/dir-rmap-probe/src/prep.rs @@ -0,0 +1,526 @@ +//! Arms B and B+: membership laid out by ordinal, prepared once. +//! +//! **B** ([`Prepared`]): user -> direct groups and group -> direct users as +//! CSR, the nesting pairs as child -> parents and parent -> children CSR, and +//! the group property planes. Walks read a CSR row at a reached coordinate. +//! +//! **B+** ([`Compiled`]): B plus one closure row per group and chain (every +//! group, the security chain, the mail chain), computed once by a fixpoint +//! over the nesting CSR in the style of #1634's walk_mask, and the roles a +//! group's SID chain carries. A user's effective groups are then the OR of +//! the closure rows of the user's direct groups. +//! +//! Source of the edges: dir-sim's membership lanes are `pub(crate)`, and +//! this probe changes no production code, so B is built from the same +//! records dir-sim ingested (plus the same change list), translated through +//! dir-sim's public ordinals. A production B would read the lanes; its +//! build would skip that translation, so the build times here are an upper +//! bound. The equivalence tests check B against dir-sim's own walks. +//! +//! Address translation (GUID -> ordinal, fixture id -> ordinal) happens only +//! in preparation, counted. Whether preparation may translate at all is +//! `ISS-ADMISSION-PREP-BINDING`, still open. + +use crate::fixture::Fixture; +use lance_graph_dir_sim::{groups_where, GroupProperty, GroupWhere, View}; +use ndarray::simd::mask_or_assign; +use ogar_dir_sim::VersionId; + +pub fn words(n: usize) -> usize { + n.div_ceil(64) +} + +#[inline] +pub fn bit(p: &[u64], i: usize) -> bool { + p[i >> 6] >> (i & 63) & 1 == 1 +} + +#[inline] +pub fn set(p: &mut [u64], i: usize) { + p[i >> 6] |= 1 << (i & 63); +} + +/// The set bits of a plane, ascending. +pub fn ones(p: &[u64]) -> impl Iterator + '_ { + p.iter().enumerate().flat_map(|(wi, &w)| { + let mut b = w; + std::iter::from_fn(move || { + if b == 0 { + return None; + } + let i = b.trailing_zeros() as usize; + b &= b - 1; + Some(wi * 64 + i) + }) + }) +} + +/// What a prepared structure was built against. Ordinals are valid inside +/// one version only, so any difference refuses. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Witness { + pub version: VersionId, + pub users: usize, + pub groups: usize, +} + +impl Witness { + pub fn of(version: VersionId, v: &View<'_>) -> Self { + Witness { + version, + users: v.users_len(), + groups: v.groups_len(), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refusal { + /// Built against another version. + Stale, + /// The question names a group outside the focus. + OutOfFocus, +} + +/// A compressed row table: `row(i)` is sorted. +#[derive(Clone, Debug)] +pub struct Csr { + off: Vec, + val: Vec, +} + +impl Csr { + /// From `(row, value)` pairs; duplicates are kept once. + pub fn build(rows: usize, pairs: &mut [(u16, u16)]) -> Csr { + pairs.sort_unstable(); + let mut off = vec![0u32; rows + 1]; + let mut val = Vec::with_capacity(pairs.len()); + let mut last = None; + for &(r, v) in pairs.iter() { + if last == Some((r, v)) { + continue; + } + last = Some((r, v)); + off[r as usize + 1] += 1; + val.push(v); + } + for i in 0..rows { + off[i + 1] += off[i]; + } + Csr { off, val } + } + #[inline] + pub fn row(&self, i: usize) -> &[u16] { + &self.val[self.off[i] as usize..self.off[i + 1] as usize] + } + pub fn bytes(&self) -> usize { + self.off.len() * 4 + self.val.len() * 2 + } +} + +/// Fixture id -> dir-sim ordinal, read once through the public API. +pub struct Ordinals { + pub user: Vec, + pub group: Vec, +} + +impl Ordinals { + pub fn of(f: &Fixture, v: &View<'_>) -> Self { + Ordinals { + user: f + .users + .iter() + .map(|g| v.user_ordinal(g).expect("fixture user exists").0) + .collect(), + group: f + .groups + .iter() + .map(|g| v.group_ordinal(g).expect("fixture group exists").0) + .collect(), + } + } +} + +/// Which nesting chain a walk follows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Chain { + /// Every group (`members_transitive`, `groups_transitive`). + All, + /// Security groups only (`security_identifiers`). + Security, + /// Mail-enabled groups only. + Mail, +} + +pub const CHAINS: [Chain; 3] = [Chain::All, Chain::Security, Chain::Mail]; + +/// Arm B. +pub struct Prepared { + pub witness: Witness, + pub users: usize, + pub groups: usize, + pub user_groups: Csr, + pub group_users: Csr, + /// `(child, parent)`, sorted: the source of `up` and `down`. + pub nested: Vec<(u16, u16)>, + pub up: Csr, + pub down: Csr, + /// Group property planes, from dir-sim's own `groups_where`. + pub security: Vec, + pub mail: Vec, + /// Roles granted to each group, as a bit set (role ids < 64). + pub grants: Vec, +} + +impl Prepared { + pub fn build( + v: &View<'_>, + witness: Witness, + ord: &Ordinals, + direct: &[(u32, u32)], + nested: &[(u32, u32)], + grants: &[(u32, u8)], + ) -> Self { + let (users, groups) = (v.users_len(), v.groups_len()); + let mut ug: Vec<(u16, u16)> = direct + .iter() + .map(|&(u, g)| (ord.user[u as usize], ord.group[g as usize])) + .collect(); + let mut gu: Vec<(u16, u16)> = ug.iter().map(|&(u, g)| (g, u)).collect(); + let mut nest: Vec<(u16, u16)> = nested + .iter() + .map(|&(c, p)| (ord.group[c as usize], ord.group[p as usize])) + .collect(); + nest.sort_unstable(); + nest.dedup(); + let plane = |p: GroupProperty| { + let mut out = vec![0u64; words(groups)]; + for i in groups_where(v, &GroupWhere::Is(p)).rows() { + set(&mut out, i); + } + out + }; + let mut g = vec![0u64; groups]; + for &(gr, role) in grants { + g[ord.group[gr as usize] as usize] |= 1 << role; + } + let (up, down) = nest_csr(groups, &nest); + Prepared { + witness, + users, + groups, + user_groups: Csr::build(users, &mut ug), + group_users: Csr::build(groups, &mut gu), + nested: nest, + up, + down, + security: plane(GroupProperty::SecurityEnabled), + mail: plane(GroupProperty::MailEnabled), + grants: g, + } + } + + pub fn check(&self, current: &Witness) -> Result<(), Refusal> { + if self.witness == *current { + Ok(()) + } else { + Err(Refusal::Stale) + } + } + + pub fn admit(&self, chain: Chain) -> Option<&[u64]> { + match chain { + Chain::All => None, + Chain::Security => Some(&self.security), + Chain::Mail => Some(&self.mail), + } + } + + /// Direct membership: one binary search in the user's row. + pub fn is_member(&self, u: u16, g: u16) -> bool { + self.user_groups.row(u as usize).binary_search(&g).is_ok() + } + + /// Groups `u` is in through `chain`, as a plane over group ordinals. + pub fn groups_of(&self, u: u16, chain: Chain, out: &mut [u64], queue: &mut Vec) { + let admit = self.admit(chain); + let ok = |g: u16| admit.is_none_or(|a| bit(a, g as usize)); + out.fill(0); + queue.clear(); + for &d in self.user_groups.row(u as usize) { + if ok(d) && !bit(out, d as usize) { + set(out, d as usize); + queue.push(d); + } + } + while let Some(g) = queue.pop() { + for &p in self.up.row(g as usize) { + if ok(p) && !bit(out, p as usize) { + set(out, p as usize); + queue.push(p); + } + } + } + } + + /// Users in `g` through `chain`, as a plane over user ordinals. + pub fn members_of( + &self, + g: u16, + chain: Chain, + reached: &mut [u64], + out: &mut [u64], + queue: &mut Vec, + ) { + let admit = self.admit(chain); + let ok = |g: u16| admit.is_none_or(|a| bit(a, g as usize)); + reached.fill(0); + out.fill(0); + queue.clear(); + if !ok(g) { + return; + } + set(reached, g as usize); + queue.push(g); + while let Some(h) = queue.pop() { + for &u in self.group_users.row(h as usize) { + set(out, u as usize); + } + for &c in self.down.row(h as usize) { + if ok(c) && !bit(reached, c as usize) { + set(reached, c as usize); + queue.push(c); + } + } + } + } + + /// Roles through the SID chain. + pub fn roles_of(&self, u: u16, scratch: &mut [u64], queue: &mut Vec) -> u64 { + self.groups_of(u, Chain::Security, scratch, queue); + ones(scratch).fold(0, |a, g| a | self.grants[g]) + } + + /// Rebuild the user and group membership tables after a change list. + pub fn set_direct(&mut self, ord: &Ordinals, direct: &[(u32, u32)]) { + let mut ug: Vec<(u16, u16)> = direct + .iter() + .map(|&(u, g)| (ord.user[u as usize], ord.group[g as usize])) + .collect(); + let mut gu: Vec<(u16, u16)> = ug.iter().map(|&(u, g)| (g, u)).collect(); + self.user_groups = Csr::build(self.users, &mut ug); + self.group_users = Csr::build(self.groups, &mut gu); + } + + /// Add one nesting pair and rebuild the nesting tables. + pub fn add_nesting(&mut self, c: u16, p: u16) { + if let Err(i) = self.nested.binary_search(&(c, p)) { + self.nested.insert(i, (c, p)); + } + let (up, down) = nest_csr(self.groups, &self.nested); + self.up = up; + self.down = down; + } + + pub fn bytes(&self) -> usize { + self.user_groups.bytes() + + self.group_users.bytes() + + self.nested.len() * 4 + + self.up.bytes() + + self.down.bytes() + + (self.security.len() + self.mail.len() + self.grants.len()) * 8 + } +} + +fn nest_csr(groups: usize, nested: &[(u16, u16)]) -> (Csr, Csr) { + let mut up: Vec<(u16, u16)> = nested.to_vec(); + let mut down: Vec<(u16, u16)> = nested.iter().map(|&(c, p)| (p, c)).collect(); + (Csr::build(groups, &mut up), Csr::build(groups, &mut down)) +} + +/// Arm B+. +pub struct Compiled { + pub b: Prepared, + /// Words per closure row. + pub w: usize, + /// Per chain: row `g` = the groups `g` reaches upward, `g` included + /// when it passes the chain. + pub up: [Vec; 3], + /// Per chain: row `g` = the groups reachable downward from `g`. + pub down: [Vec; 3], + /// Roles carried by the SID chain starting at each group. + pub role_sec: Vec, + /// Fixpoint passes of the last full compile, per chain (up, down). + pub passes: [(usize, usize); 3], + /// `b.up` / `b.down` no longer match `b.nested` (after `add_nesting`). + pub b_nest_stale: bool, +} + +fn ci(chain: Chain) -> usize { + chain as usize +} + +/// Reflexive closure over `adj`, restricted to admitted groups: a fixpoint +/// of `row[g] |= row[n]` for every admitted neighbour `n` of an admitted `g`. +/// Rows are read only at coordinates named by the CSR, as #1634 reads +/// `delta` at the adjacency's set bits. +fn closure(n: usize, w: usize, adj: &Csr, admit: Option<&[u64]>) -> (Vec, usize) { + let ok = |g: usize| admit.is_none_or(|a| bit(a, g)); + let mut c = vec![0u64; n * w]; + for g in 0..n { + if ok(g) { + set(&mut c[g * w..g * w + w], g); + } + } + let mut passes = 0; + loop { + passes += 1; + let mut changed = false; + for g in 0..n { + if !ok(g) { + continue; + } + for &p in adj.row(g) { + let p = p as usize; + if p == g || !ok(p) { + continue; + } + for k in 0..w { + let v = c[p * w + k]; + let old = c[g * w + k]; + if old | v != old { + c[g * w + k] = old | v; + changed = true; + } + } + } + } + if !changed { + return (c, passes); + } + } +} + +impl Compiled { + pub fn compile(b: Prepared) -> Self { + let n = b.groups; + let w = words(n); + let mut up: [Vec; 3] = Default::default(); + let mut down: [Vec; 3] = Default::default(); + let mut passes = [(0, 0); 3]; + for chain in CHAINS { + let admit = b.admit(chain); + let (u, pu) = closure(n, w, &b.up, admit); + let (d, pd) = closure(n, w, &b.down, admit); + up[ci(chain)] = u; + down[ci(chain)] = d; + passes[ci(chain)] = (pu, pd); + } + let mut c = Compiled { + b, + w, + up, + down, + role_sec: vec![0; n], + passes, + b_nest_stale: false, + }; + for g in 0..n { + c.refresh_roles(g); + } + c + } + + fn refresh_roles(&mut self, g: usize) { + let w = self.w; + let row = &self.up[ci(Chain::Security)][g * w..g * w + w]; + self.role_sec[g] = ones(row).fold(0, |a, h| a | self.b.grants[h]); + } + + pub fn row(&self, chain: Chain, g: usize) -> &[u64] { + &self.up[ci(chain)][g * self.w..g * self.w + self.w] + } + + pub fn down_row(&self, chain: Chain, g: usize) -> &[u64] { + &self.down[ci(chain)][g * self.w..g * self.w + self.w] + } + + /// A user's groups through `chain`: OR of its direct groups' rows. + pub fn groups_of(&self, u: u16, chain: Chain, out: &mut [u64]) { + out.fill(0); + for &d in self.b.user_groups.row(u as usize) { + mask_or_assign(out, self.row(chain, d as usize)); + } + } + + /// Users in `g` through `chain`: the direct users of every group in + /// `g`'s downward closure row. + pub fn members_of(&self, g: u16, chain: Chain, out: &mut [u64]) { + out.fill(0); + for h in ones(self.down_row(chain, g as usize)) { + for &u in self.b.group_users.row(h) { + set(out, u as usize); + } + } + } + + pub fn roles_of(&self, u: u16) -> u64 { + self.b + .user_groups + .row(u as usize) + .iter() + .fold(0, |a, &d| a | self.role_sec[d as usize]) + } + + /// Add one nesting pair without recompiling: for every chain both ends + /// pass, each descendant of `c` gains the ancestors of `p` and each + /// ancestor of `p` gains the descendants of `c` (the transitive-closure + /// insertion, exact for reflexive rows, cycles included). + pub fn add_nesting(&mut self, c: u16, p: u16) { + let (c, p) = (c as usize, p as usize); + let w = self.w; + for chain in CHAINS { + if let Some(a) = self.b.admit(chain) { + if !bit(a, c) || !bit(a, p) { + continue; + } + } + let k = ci(chain); + let anc_p = self.up[k][p * w..p * w + w].to_vec(); + let desc_c = self.down[k][c * w..c * w + w].to_vec(); + for x in ones(&desc_c) { + mask_or_assign(&mut self.up[k][x * w..x * w + w], &anc_p); + } + for y in ones(&anc_p) { + mask_or_assign(&mut self.down[k][y * w..y * w + w], &desc_c); + } + if chain == Chain::Security { + for x in ones(&desc_c) { + self.refresh_roles(x); + } + } + } + if let Err(i) = self.b.nested.binary_search(&(c as u16, p as u16)) { + self.b.nested.insert(i, (c as u16, p as u16)); + } + // B+ never reads B's nesting CSR after compiling; it is rebuilt from + // `nested` by `recompile`. Until then it is stale, so B's own walks + // must use their own `Prepared`, not `self.b`. + self.b_nest_stale = true; + } + + /// Full recompile from the current pairs: the reference the incremental + /// insertion is checked against, and the only path for a removal. + pub fn recompile(mut self) -> Self { + let (up, down) = nest_csr(self.b.groups, &self.b.nested); + self.b.up = up; + self.b.down = down; + Compiled::compile(self.b) + } + + /// Bytes of the closure rows and role table, B's tables excluded. + pub fn closure_bytes(&self) -> usize { + (self.up.iter().chain(self.down.iter()).map(Vec::len).sum::() + self.role_sec.len()) + * 8 + } +} diff --git a/crates/dir-rmap-probe/tests/equivalence.rs b/crates/dir-rmap-probe/tests/equivalence.rs new file mode 100644 index 000000000..feeeb9830 --- /dev/null +++ b/crates/dir-rmap-probe/tests/equivalence.rs @@ -0,0 +1,268 @@ +//! Every prepared arm equals dir-sim's own walks (A) and the independent +//! crepe oracle (D), on every user and group of a small fixture, before and +//! after an overlay. Plus the refusals and the anti-vacuity checks that keep +//! the comparisons meaningful. + +use dir_rmap_probe::fixture::{Fixture, Kind, Spec}; +use dir_rmap_probe::focus::{holds, Focus, FocusUsers}; +use dir_rmap_probe::prep::{ones, words, Chain, Compiled, Ordinals, Prepared, Refusal, CHAINS}; +use dir_rmap_probe::{a_groups, a_members, focus_for, oracle, templates, Edit, World}; +use ogar_dir_core::Guid128; +use std::collections::{BTreeSet, HashMap}; + +fn world() -> World { + World::new(Fixture::generate(Spec::small())) +} + +fn sorted(mut v: Vec) -> Vec { + v.sort_unstable(); + v +} + +/// A plane over group ordinals, as GUIDs. +fn groups_of_plane(w: &World, p: &[u64]) -> Vec { + let v = w.view(); + sorted( + ones(p) + .map(|g| v.group_guid(lance_graph_dir_sim::GroupOrdinal(g as u16)).unwrap()) + .collect(), + ) +} + +fn users_of_plane(w: &World, p: &[u64]) -> Vec { + let v = w.view(); + sorted( + ones(p) + .map(|u| v.user_guid(lance_graph_dir_sim::UserOrdinal(u as u16)).unwrap()) + .collect(), + ) +} + +fn oracle_rel(a: &oracle::Answers, chain: Chain) -> &BTreeSet<(u32, u32)> { + match chain { + Chain::All => &a.member, + Chain::Security => &a.sec, + Chain::Mail => &a.mail, + } +} + +/// Roles as dir-sim answers them: the grants of the SIDs it reports, keyed +/// by GUID, the shape `ViewActors::for_principal` reads. +fn a_roles(w: &World, u: &Guid128) -> u64 { + let by_guid: HashMap = w.f.grants.iter().fold(HashMap::new(), |mut m, &(g, r)| { + *m.entry(w.f.groups[g as usize]).or_default() |= 1 << r; + m + }); + a_groups(&w.view(), u, Chain::Security) + .iter() + .fold(0, |a, g| a | by_guid.get(g).copied().unwrap_or(0)) +} + +/// The full comparison of A, B, B+ and D for one version. +fn check_all(w: &World, b: &Prepared, c: &Compiled) { + let v = w.view(); + let ord = Ordinals::of(&w.f, &v); + let t = templates(&w.f); + let d = oracle::run(&w.f, &w.direct, &w.nested, &t); + let (gw, uw) = (words(v.groups_len()), words(v.users_len())); + let (mut p, mut q, mut r) = (vec![0; gw], vec![0; gw], vec![0; uw]); + let mut queue = Vec::new(); + for chain in CHAINS { + let rel = oracle_rel(&d, chain); + for (i, ug) in w.f.users.iter().enumerate() { + let a = sorted(a_groups(&v, ug, chain)); + let uo = ord.user[i]; + b.groups_of(uo, chain, &mut p, &mut queue); + c.groups_of(uo, chain, &mut q); + let dd = sorted( + rel.range((i as u32, 0)..=(i as u32, u32::MAX)) + .map(|&(_, g)| w.f.groups[g as usize]) + .collect(), + ); + assert_eq!(groups_of_plane(w, &p), a, "B groups, {chain:?}, user {i}"); + assert_eq!(groups_of_plane(w, &q), a, "B+ groups, {chain:?}, user {i}"); + assert_eq!(dd, a, "oracle groups, {chain:?}, user {i}"); + } + let mut by_group: HashMap> = HashMap::new(); + for &(u, g) in rel { + by_group.entry(g).or_default().push(w.f.users[u as usize]); + } + for (j, gg) in w.f.groups.iter().enumerate() { + let a = sorted(a_members(&v, gg, chain)); + let go = ord.group[j]; + b.members_of(go, chain, &mut q, &mut r, &mut queue); + assert_eq!(users_of_plane(w, &r), a, "B members, {chain:?}, group {j}"); + c.members_of(go, chain, &mut r); + assert_eq!(users_of_plane(w, &r), a, "B+ members, {chain:?}, group {j}"); + let dd = sorted(by_group.remove(&(j as u32)).unwrap_or_default()); + assert_eq!(dd, a, "oracle members, {chain:?}, group {j}"); + } + } + let mut d_roles = vec![0u64; w.f.users.len()]; + for &(u, role) in &d.roles { + d_roles[u as usize] |= 1 << role; + } + for (i, ug) in w.f.users.iter().enumerate() { + let a = a_roles(w, ug); + assert_eq!(b.roles_of(ord.user[i], &mut p, &mut queue), a, "B roles, user {i}"); + assert_eq!(c.roles_of(ord.user[i]), a, "B+ roles, user {i}"); + assert_eq!(d_roles[i], a, "oracle roles, user {i}"); + } + // Direct membership, every (user, group) pair. + for (i, ug) in w.f.users.iter().enumerate() { + for (j, gg) in w.f.groups.iter().enumerate() { + assert_eq!( + b.is_member(ord.user[i], ord.group[j]), + v.is_member(ug, gg), + "B is_member ({i}, {j})" + ); + } + } +} + +#[test] +fn every_arm_equals_dir_sim_and_the_oracle() { + let w = world(); + let b = w.prepare(); + let c = Compiled::compile(w.prepare()); + check_all(&w, &b, &c); +} + +#[test] +fn the_focus_answers_role_and_location_like_the_oracle_and_dir_sim() { + let w = world(); + let v = w.view(); + let ord = Ordinals::of(&w.f, &v); + let c = Compiled::compile(w.prepare()); + let all = templates(&w.f); + let d = oracle::run(&w.f, &w.direct, &w.nested, &all); + let mut checked = 0; + for k in [16, 64, 256] { + let (ts, gs) = focus_for(&all, k); + assert!(!ts.is_empty() && gs.len() <= k); + let groups: Vec = gs.iter().map(|&g| ord.group[g as usize]).collect(); + let f = Focus::bind(&c, &groups); + let fu = FocusUsers::bind(&c, &groups); + for (i, ug) in w.f.users.iter().enumerate() { + let a = a_groups(&v, ug, Chain::All); + let row = f.row_of(&c, ord.user[i]); + for &(r, l) in &ts { + let m = f + .all_of(&[ord.group[r as usize], ord.group[l as usize]]) + .unwrap(); + let want = a.contains(&w.f.groups[r as usize]) && a.contains(&w.f.groups[l as usize]); + assert_eq!(holds(&row, &m), want, "F k={k} user {i} ({r}, {l})"); + assert_eq!(holds(&fu.rows[ord.user[i] as usize], &m), want, "F+users"); + assert_eq!(d.eligible.contains(&(i as u32, r, l)), want, "oracle"); + checked += 1; + } + } + } + assert!(checked > 10_000, "the comparison covered {checked} questions"); +} + +#[test] +fn overlay_incremental_equals_recompile_dir_sim_and_the_oracle() { + let mut w = world(); + let mut c = Compiled::compile(w.prepare()); + let f = &w.f; + let team = f.teams[0]; + let role = f.roles[3]; + let removed = *f.direct.iter().find(|&&(_, g)| f.kind[g as usize] == Kind::Composite).unwrap(); + // A nesting that closes a cycle through a role and a resource, and one + // that links a team (not security) into a role. + let res = f.nested.iter().find(|&&(c, _)| c == role).unwrap().1; + let edits = [ + Edit::AddDirect(5, team), + Edit::AddDirect(7, role), + Edit::RemoveDirect(removed.0, removed.1), + Edit::AddNested(res, role), + Edit::AddNested(team, f.roles[9]), + ]; + w.apply(&edits); + let v = w.view(); + let ord = Ordinals::of(&w.f, &v); + // Ordinals are unchanged by membership edits, so the incremental path + // may reuse them; the witness still moves to the new version. + c.b.set_direct(&ord, &w.direct); + for e in &edits { + if let Edit::AddNested(ch, p) = *e { + c.add_nesting(ord.group[ch as usize], ord.group[p as usize]); + } + } + c.b.witness = w.witness(); + let full = Compiled::compile(w.prepare()); + for chain in CHAINS { + assert_eq!(c.up[chain as usize], full.up[chain as usize], "{chain:?} up"); + assert_eq!(c.down[chain as usize], full.down[chain as usize], "{chain:?} down"); + } + assert_eq!(c.role_sec, full.role_sec); + check_all(&w, &w.prepare(), &full); +} + +#[test] +fn a_stale_witness_and_an_unfocused_group_are_refused() { + let mut w = world(); + let b = w.prepare(); + assert_eq!(b.check(&w.witness()), Ok(())); + let f = &w.f; + let (u, g) = (11, f.teams[1]); + w.apply(&[Edit::AddDirect(u, g)]); + assert_eq!(b.check(&w.witness()), Err(Refusal::Stale)); + let c = Compiled::compile(w.prepare()); + let v = w.view(); + let ord = Ordinals::of(&w.f, &v); + let focus = Focus::bind(&c, &[ord.group[w.f.roles[0] as usize]]); + assert_eq!(focus.check(&w.witness()), Ok(())); + assert!(focus.all_of(&[ord.group[w.f.roles[0] as usize]]).is_ok()); + assert_eq!( + focus.all_of(&[ord.group[w.f.roles[1] as usize]]), + Err(Refusal::OutOfFocus) + ); +} + +/// The fixture must exercise what the comparisons claim to cover. +#[test] +fn the_fixture_is_not_vacuous() { + let w = world(); + let f = &w.f; + let cyclic = f.nested.iter().filter(|&&(a, b)| f.nested.binary_search(&(b, a)).is_ok()).count(); + assert!(cyclic >= 2, "the nesting has cycles"); + assert!(f.large_groups() >= 1, "some group exceeds 256 direct members"); + let c = Compiled::compile(w.prepare()); + assert!(c.passes[Chain::All as usize].0 > 1, "the closure needs more than one pass"); + let v = w.view(); + let ord = Ordinals::of(&w.f, &v); + let gw = words(v.groups_len()); + let (mut all, mut sec, mut mail) = (vec![0; gw], vec![0; gw], vec![0; gw]); + let (mut sec_differs, mut mail_differs, mut roles) = (0, 0, 0); + let (mut sec_some, mut mail_some) = (0, 0); + for u in 0..f.users.len() { + let uo = ord.user[u]; + c.groups_of(uo, Chain::All, &mut all); + c.groups_of(uo, Chain::Security, &mut sec); + c.groups_of(uo, Chain::Mail, &mut mail); + sec_differs += usize::from(sec != all); + mail_differs += usize::from(mail != all); + sec_some += usize::from(sec.iter().any(|&w| w != 0)); + mail_some += usize::from(mail.iter().any(|&w| w != 0)); + roles += usize::from(c.roles_of(uo) != 0); + } + let n = f.users.len(); + // Each chain removes something for many users and empties it for few: + // a filter that changed nothing, or kept nothing, would not test the + // chain semantics. + assert!(sec_differs * 10 > n, "the SID chain filters: {sec_differs}/{n}"); + assert!(sec_some * 10 > n * 9, "the SID chain keeps groups: {sec_some}/{n}"); + assert!(mail_differs * 10 > n, "the mail chain filters: {mail_differs}/{n}"); + assert!(mail_some * 10 > n * 9, "the mail chain keeps groups: {mail_some}/{n}"); + assert!(roles * 2 > n, "most users hold a role: {roles}/{n}"); + let t = templates(f); + let d = oracle::run(f, &w.direct, &w.nested, &t); + let total = n * t.len(); + assert!( + d.eligible.len() * 50 > total / 50 && d.eligible.len() * 2 < total, + "role x location is neither empty nor everything: {}/{total}", + d.eligible.len() + ); +} From a1125eea289840a503f456d4372700900c6fa4da Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 22:02:15 +0000 Subject: [PATCH 2/7] dir-rmap-probe: benchmark, plan, CI and board for D-RMAP-DIR-1 - examples/bench.rs: W1-W9 for arms A, D, B, B+, F and F+users at 8,000 and 65,000 users; focus sizes 16/64/128/256; 1 to 10,000 evaluations per prepared focus; memory; cold vs warm. Every timed answer is checked against A (and D) first. - Clippy fixes (Rng::next_u64, collapsed if-let); the closure's parent admit check is marked an optimisation: its disable run stayed green because an unadmitted row is never seeded or written. - Plan d-rmap-dir-1-membership-closure-focus-v1.md with the measured results; STATUS_BOARD rows D-RMAP-DIR-1 (MEASURED) and D-RMAP-DIR-2 (PROPOSED, not authorized); entry, INTEGRATION_PLANS, entries index. - CI dir-rmap-probe.yml: fmt, clippy, tests, and the 8,000-user benchmark as an equivalence smoke run (timings not gated). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .claude/board/INTEGRATION_PLANS.md | 4 + .claude/board/STATUS_BOARD.md | 2 + ...0-d-rmap-dir-1-membership-closure-focus.md | 12 + .claude/board/entries/README.md | 3 +- ...-rmap-dir-1-membership-closure-focus-v1.md | 156 ++++ .github/workflows/dir-rmap-probe.yml | 90 ++ crates/dir-rmap-probe/examples/bench.rs | 799 ++++++++++++++++++ crates/dir-rmap-probe/src/fixture.rs | 12 +- crates/dir-rmap-probe/src/focus.rs | 2 +- crates/dir-rmap-probe/src/lib.rs | 4 +- crates/dir-rmap-probe/src/oracle.rs | 6 +- crates/dir-rmap-probe/src/prep.rs | 21 +- crates/dir-rmap-probe/tests/equivalence.rs | 91 +- 13 files changed, 1164 insertions(+), 38 deletions(-) create mode 100644 .claude/board/entries/2026-10-20-d-rmap-dir-1-membership-closure-focus.md create mode 100644 .claude/plans/d-rmap-dir-1-membership-closure-focus-v1.md create mode 100644 .github/workflows/dir-rmap-probe.yml create mode 100644 crates/dir-rmap-probe/examples/bench.rs diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index 558fbc938..9776f75bb 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -1,3 +1,7 @@ +## 2026-10-20 — d-rmap-dir-1-membership-closure-focus-v1 — compiled membership closure vs a 256-slot focus over dir-sim → `.claude/plans/d-rmap-dir-1-membership-closure-focus-v1.md` + +**Status:** D-RMAP-DIR-1 MEASURED, lab only (`crates/dir-rmap-probe`); D-RMAP-DIR-2 PROPOSED, not authorized. Sharing compiled closure rows across users (B+) is where the acceleration comes from (thousands of times over dir-sim's per-call scan); a 256-slot focus (F) adds at most 2× and only for small foci reused about 1,000 times. Entry `2026-10-20-d-rmap-dir-1-membership-closure-focus.md`. + ## 2026-10-20 — thought-git-basics-v1 — the basics of thinking with a git API → `.claude/plans/thought-git-basics-v1.md` **Status:** D-TGB-0..6 PROPOSED, lab, test-first; nothing built. From the ideal state: a slot is taken only at the Rubicon crossing (`Planning → CognitiveWork`) and freed at `Commit` / `Plan` / `Prune`, so no stale branch can exist; squash, prune and GC are kanban outcomes; every thought is replayable from its receipt. `Commit` lands one sparse batch through `LanceCycleWriter`, which gives `KanbanColumn::Commit` its declared-but-missing calcify step. Lance 13 supplies `Ref::Version(branch, version)`, branches, tags, `delta()`, `read_version_transaction`; MemWAL is an optional arm (D-WAL, D-LNC-7). Parent: `thought-git-api-v1.md` correction (#1628). diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index de03cf737..8420dbc1b 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -54,6 +54,8 @@ These PRs merged with no board rows. Every row below is a PROPOSAL (documents an | D-RPR-3 | MeTTa-facing parity of a chain's answers | BLOCKED on D-ASM-4 (not authorized) | — | | D-MPS-8 | native join over a versioned relationship table: R and S laid out by `y` slot, leapfrog intersection as one mask-risc `And`, per-side generations and reseal, per-thought filter planes (`mork-pathmap-schnellverschluss-v1.md` §7b; `crates/mork-pathmap-probe/src/join.rs`) | MEASURED, probe only; not admitted (D-RMAP-1 BIND membrane review) | 8 join tests + native arms in the differential suite vs the oracle; 7 guards disable-verified; cyclic bodies open | | D-RMAP-2 | context-declaration reuse and GraphBLAS/BF16 tile probe (`d-rmap-2-context-tile-algebra-probe-v1.md`) | LAB ONLY, not admitted to production; PARTIAL: §5a MEASURED on the polyfill tier and on the AMX tile tier (§5a-AMX); `VDPBF16PS`-only tier and GraphBLAS arm not run | arms 1–5 in the plan; 297× (D-GTM-0j) is not evidence | +| D-RMAP-DIR-1 | dir-sim membership: A (View) vs D (crepe oracle) vs B (bidirectional CSR) vs B+ (compiled closure rows + roles by ordinal) vs F (256-slot focus over B+) (`d-rmap-dir-1-membership-closure-focus-v1.md`; `crates/dir-rmap-probe`, CI `dir-rmap-probe.yml`) | MEASURED, lab only; nothing admitted, no production file changed | 5 tests: every arm = A = D on every user/group/chain; 7 guards disable-verified red, 1 shown redundant; B removes A's per-call full scan (W2 ~4,000×), B+ adds 2–3× (groups) and 12–15× (roles), F ≤ 2× only for ≤ 64 focus groups after ~1,000 reuses, none near 256 | +| D-RMAP-DIR-2 | B built in `Snapshot::build` from the lanes; B+ closure as a lazily compiled, version-witnessed cache behind the existing `View` methods; F not proposed | PROPOSED; not authorized; waits on `ISS-ADMISSION-PREP-BINDING` (hot path reads rows at a user's direct-group values) | dir-sim tests + D-RMAP-DIR-1 oracle as the equivalence gate | | D-SRAM-0 | inventory Soufflé AST/ast2ram/RAM passes against D-ASM/D-RA/D-TGA owners | Queued | evidence per mapped pass; no code imported | | D-SRAM-1 | bound AST → RAM-like schedule for pure conjunctive selection | Queued | same answers and read sets on hand-built fixtures; unsupported ops typed-refused | | D-SRAM-2 | cold optimizer rewrites (SCC/strata, magic sets, ordering, dedup) | Queued (research until D-ASM AST placement and D-RA-2); lab spike measured 2026-10-20, `entries/2026-10-20-crepe-rule-optimizer-spike.md`, not authorization | optimized == unoptimized == oracle; rewrite off removes only the plan shape; disabling a semantic guard fails | diff --git a/.claude/board/entries/2026-10-20-d-rmap-dir-1-membership-closure-focus.md b/.claude/board/entries/2026-10-20-d-rmap-dir-1-membership-closure-focus.md new file mode 100644 index 000000000..07cae826e --- /dev/null +++ b/.claude/board/entries/2026-10-20-d-rmap-dir-1-membership-closure-focus.md @@ -0,0 +1,12 @@ +# D-RMAP-DIR-1: compiled membership closure vs a 256-slot focus over dir-sim (2026-10-20) + +**Status:** MEASURED, lab only. **Basis:** `crates/dir-rmap-probe` (`tests/equivalence.rs` 5/5, `examples/bench.rs` at 8,000 and 65,000 users, 3 runs at 8,000). Plan `d-rmap-dir-1-membership-closure-focus-v1.md`. + +- **Arms:** A dir-sim `View` walks; D crepe Datalog over fixture ids; B membership and nesting CSR; B+ closure rows per group and chain plus roles by ordinal; F B+ projected onto a 256-slot focus (and F+users, one focus row per user). +- **Equivalence:** B, B+, D equal A on every user, group and chain of the small fixture; B+ equals D on all 79,229 (8,000) and 611,965 (65,000) memberships before timing. 7 guards disable-verified red; the closure's parent-admit check stayed green and is marked an optimisation. +- **A's cost is a full membership scan per call:** 0.4 ms at 35,000 rows, 0.7 ms at 263,000. +- **B:** W2 user → groups 92–124 ns vs 408–466 µs; all-user reconciliation 7.1 s → 0.9 ms (8,000 users). +- **B+:** 32–49 ns per user's groups, 11–15 ns per user's roles; compile 1.6 ms independent of user count; one nesting pair inserted incrementally in 2–2.5 µs. +- **F:** ≤ 2× faster evaluation than B+ with ≤ 64 focus groups, break-even ~1,000 evaluations per prepared focus; 1.3× at 128; none near 256 and none cold. The focus saves the OR stage only; question tests dominate past a few hundred questions. + +OPEN: hot-path read pattern of B/B+/F (reads at a user's direct-group values) under `ISS-ADMISSION-PREP-BINDING`; membership row patching; ordinal remap after a snapshot rebuild; real directory data; D-RMAP-DIR-2 not authorized. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index c131c948c..a6778bfd0 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -389 entries, 2026-08-06 .. 2026-10-20. +390 entries, 2026-08-06 .. 2026-10-20. | date | entry id | finding | file | |---|---|---|---| @@ -39,6 +39,7 @@ exactly one of them, never both. | 2026-10-20 | `D-TGA-SLOT-AXIS` | thoughts are 64 slots, and the slot number is CE64's witness pointer | [2026-10-20-d-tga-slot-axis-witness-pointer.md](2026-10-20-d-tga-slot-axis-witness-pointer.md) | | 2026-10-20 | `D-RPR` | | [2026-10-20-d-rpr-consolidation-1624.md](2026-10-20-d-rpr-consolidation-1624.md) | | 2026-10-20 | `D-RPR-1` | | [2026-10-20-d-rpr-1-handoff-lab-demo.md](2026-10-20-d-rpr-1-handoff-lab-demo.md) | +| 2026-10-20 | `D-RMAP-DIR-1` | | [2026-10-20-d-rmap-dir-1-membership-closure-focus.md](2026-10-20-d-rmap-dir-1-membership-closure-focus.md) | | 2026-10-20 | `D-RMAP-2` | | [2026-10-20-d-rmap-2-tile-gemm-cell.md](2026-10-20-d-rmap-2-tile-gemm-cell.md) | | 2026-10-20 | `D-RMAP-2-AMX` | | [2026-10-20-d-rmap-2-amx-tier.md](2026-10-20-d-rmap-2-amx-tier.md) | | 2026-10-20 | `D-RMAP-2-AMX-RERUN` | §5a-AMX re-measured on a second AMX host | [2026-10-20-d-rmap-2-amx-remeasured.md](2026-10-20-d-rmap-2-amx-remeasured.md) | diff --git a/.claude/plans/d-rmap-dir-1-membership-closure-focus-v1.md b/.claude/plans/d-rmap-dir-1-membership-closure-focus-v1.md new file mode 100644 index 000000000..0ec7bdd87 --- /dev/null +++ b/.claude/plans/d-rmap-dir-1-membership-closure-focus-v1.md @@ -0,0 +1,156 @@ +# D-RMAP-DIR-1: compiled membership closure vs a 256-slot focus over dir-sim + +> **Status: MEASURED, lab only (2026-10-20).** Code: `crates/dir-rmap-probe` (own workspace, CI `dir-rmap-probe.yml`). Nothing is admitted; no production file changed. dir-sim stays authoritative and is read only through its public API. Related: D-RMAP-1/1A (#1617), the `Focus`/`Csr`/`Tile` probe (#1621), D-RPR (#1624), walk_mask (#1634), `ISS-ADMISSION-PREP-BINDING`, the 2026-10-16 no-gather ruling. + +## 1. Question + +Does `lance-graph-dir-sim` gain more from sharing compiled closure results across users (B+), from reusing a local 256-slot geometry across evaluations (F), or from both? Each optimisation is judged on its own: + +- **B:** does a prepared CSR remove the dominant membership-scan cost? +- **B+:** does a compiled group closure add a substantial gain when many users share inheritance? +- **F:** does a reusable 256-slot focus beat B+ once binding and amortisation are counted? + +## 2. Arms + +| arm | code | what | +|---|---|---| +| A | dir-sim `View` | `groups_transitive`, `members_transitive`, `security_identifiers`, `_through`, `is_member` (authoritative) | +| D | `oracle.rs` | crepe Datalog over the fixture's own integer ids (never dir-sim ordinals) | +| B | `prep::Prepared` | user → direct groups and group → direct users as CSR; nesting child → parents and parent → children as CSR; property planes from dir-sim's own `groups_where` | +| B+ | `prep::Compiled` | B + one closure row per group for each chain (every group, security, mail), up and down, by a fixpoint over the nesting CSR; roles carried by each group's SID chain; incremental insertion of a nesting pair | +| F | `focus::Focus` | B+ closure rows projected onto up to 256 bound groups (4 words per group); reverse map is a direct `u16` array over group ordinals | +| F+users | `focus::FocusUsers` | F + one 4-word focus row per user (the per-user pointer cache, measured so it is not argued) | + +Role and location are ordinary groups. A role × location combination is a composite group nested in both, so no new membership semantics exists. "In role R and in location L" is asked of the closure (W5), never stored as a membership. + +**Edge source.** dir-sim's membership lanes are `pub(crate)` and this probe changes no production file, so B is built from the same records dir-sim ingested (plus the same change list), translated through dir-sim's public ordinals. A production B would read the lanes; the build times here are an upper bound. + +## 3. Fixture + +Deterministic (`Spec::iam(n)`): 1,495 groups. These are an all-users group, a company group, 7 divisions, 300 role groups and 40 location groups. There are also 600 role × location composites, 500 resources (a DAG plus 6 two-cycles, 1 in 20 with an unread security flag), 16 license groups and 30 teams (distribution groups, 1 in 5 also security). Grants go on resources, roles, locations and teams. There is a `u8` attribute lane with 12 values. Divisions are mail-enabled but not security, so they cut the SID chain for every user and keep the mail chain. + +At 8,000 users this gives 35,314 direct rows, 1,996 nesting pairs and 17 groups with more than 256 direct members. At 65,000 users it gives 263,470 direct rows and 138 such groups. + +## 4. Correctness + +- `tests/equivalence.rs`, 5 tests on `Spec::small()` (600 users): + - B, B+ and D equal A on every user and group, for all three chains, roles and direct membership (every user × group pair); + - F and F+users equal A and D on every role × location question for 16, 64 and 256 focus groups (more than 10,000 questions); + - after an overlay (membership adds and removals, a nesting pair closing a cycle, a team nested in a role), incremental B+ equals a full recompile, and both equal A and D; + - a stale witness refuses, and an unfocused group refuses; + - anti-vacuity: cycles, groups over 256 members, closure passes > 1; the security and mail chains each filter for > 10 % of users and keep groups for > 90 %; most users hold a role; role × location is neither empty nor everything. +- The benchmark asserts, at full scale and before any timing: + - B+ = A on a 2,000-user sample (500 at 65,000); + - B+ = D on every membership (79,229 at 8,000; 611,965 at 65,000); + - roles B+ = A on the sample; + - F = F+users = B+ on every user and question; + - W6 and W7 answers equal A. +- **Disable runs** (committed first, each restored with `git checkout`): + +| guard removed | result | +|---|---| +| closure: skip unadmitted group `g` | red (2 tests) | +| closure: seed admitted groups only | red (2 tests) | +| incremental insertion: update descendants' down rows | red (overlay test) | +| incremental insertion: refresh SID-chain roles | red (overlay test) | +| witness check | red (refusal test) | +| focus refusal for an unbound group | red (refusal test) | +| oracle: `Sec(p)` on the recursive security rule | red (2 tests) | +| closure: skip unadmitted *parent* | **green**: it is an optimisation only. An unadmitted row is never seeded or written, so ORing it adds nothing. Marked so in the code. | + +## 5. Results + +**Measurement setup.** Release builds on a 4-core Xeon @ 2.10 GHz with AVX-512, 48 KiB L1d and 2 MiB L2, on a quiet machine. Figures are medians. Per-operation numbers are a timed batch divided by its size. Three runs at 8,000 users agreed within the ranges shown. Arm A is measured on a sample because each call scans every membership row. Cold means after touching 64 MiB of unrelated memory. + +### 5.1 Construction (8,000 / 65,000 users) + +| step | 8,000 | 65,000 | +|---|---|---| +| dir-sim snapshot (`observe`) | 26 ms | 282 ms | +| B prepare (with GUID translation) | 2.8 ms | 23.8 ms | +| B+ compile (3 chains × up/down) | 1.6 ms | 1.6 ms | +| D crepe, all relations | 76 ms | 920 ms | + +### 5.2 Hot workloads, warm, per operation + +| workload | A | B | B+ | 8,000: B+ vs A | 65,000: A / B / B+ | +|---|---|---|---|---|---| +| W1 direct membership | 307 ns | 10 ns | (B's CSR) | 31× | 510 / 9 ns | +| W2 user → effective groups | 408–466 µs | 92–124 ns | 32–49 ns | ~9,000–14,000× | 671 µs / 88 / 32 ns | +| W2 emitted as GUIDs | — | — | 299 ns | ~1,400× | 306 ns | +| W3 group → transitive members | 441 µs | 114 ns | 143 ns | ~3,100× | 820 µs / 505 / 515 ns | +| W4 roles via the SID chain | 412–431 µs | 181–192 ns | 13–15 ns | ~28,000–33,000× | 673 µs / 186 / 11 ns | +| W6 team members ∩ attribute (per team) | 460 µs | — | 0.3 µs | ~1,500× | 799 / 2.5 µs | + +**A's cost is a full scan per call.** The cost of `members_walk` and `groups_reach` follows the number of membership rows, not the answer: about 0.4 ms at 35,000 rows, about 0.7 ms at 263,000. + +### 5.3 W5: role × location, B+ vs F (8,000 users) + +One evaluation is one user against every question in the focus. + +| focus groups | questions | B+ eval | F eval | F+users eval | F prepare | F+users prepare | break-even vs B+ (F) | +|---|---|---|---|---|---|---|---| +| 16 | 11 | 46–47 ns | 24–25 ns | 7 ns | 22 µs | 113–126 µs | ~1,000 evaluations (measured: equal at N = 1,000; F 1.8× faster at N = 10,000) | +| 64 | 135 | 155–186 ns | 74–88 ns | 57–70 ns | 80–145 µs | 180–279 µs | ~1,000 (F 1.8–2.1× faster at N = 10,000) | +| 128 | 364 | 347–367 ns | 268–282 ns | 229–295 ns | 162–227 µs | 260–327 µs | ~2,000–2,500 (1.3–1.4× at N = 10,000) | +| 237–243 | 600 | 536–558 ns | 533–650 ns | 482–523 ns | 294–334 µs | 395–405 µs | none reproducible: F's evaluation equals or trails B+ | + +- At 65,000 users the B+/F figures are within 10 % of these. F+users preparation grows to 582–866 µs, because it holds one row per user. +- Cold single evaluation (256 slots, 600 questions): + - 8,000 users: B+ 2.6–3.2 µs, F 3.6–4.0 µs; + - 65,000 users: B+ 4.0 µs, F 3.7 µs. +- **The focus saves only the OR stage** (4 words per direct group instead of 24). Once there are a few hundred questions, the question tests dominate and cost the same in both arms. The 256 × 256 tile was not built: B+ has already compiled the closure, so a tile would recompute it. + +### 5.4 Overlay, reconciliation, rebuild (8,000 / 65,000 users) + +| step | 8,000 | 65,000 | +|---|---|---| +| W7 all-user reconciliation, A (two versions) | 7.1 s | ~87 s, extrapolated from 500 users | +| W7 same, B+ (two compiled versions; the compiles are 2 × 1.6 ms more) | 0.9 ms | 4.4 ms | +| W8 B full rebuild | 2.7 ms | 23 ms | +| W8 membership edits: rebuild the two membership CSRs, closures untouched | 1.9 ms | 15 ms | +| W8 one nesting pair, incremental closure insertion | 2.1 µs | 2.5 µs | +| W8 B + B+ full recompile | 4.4 ms | 25 ms | +| W8 F (256 slots) rebind | 318 µs | 315 µs | +| W9 snapshot with one user inserted first | every user ordinal moves; the old B+ refuses (witness); snapshot 23 ms + B and B+ rebuild 4.1 ms | 248 ms + 24 ms | + +### 5.5 Memory (8,000 users) + +| structure | size | +|---|---| +| B (CSR tables and planes) | 214 KiB (1.3 MiB at 65,000) | +| B+ closures | 1.65 MiB: 3 chains × up/down × 287 KiB (one dense chain is 1,495 × 24 words = 281 KiB), independent of user count | +| F (256 slots) | 50 KiB | +| F+users | 300 KiB (2.0 MiB at 65,000) | + +## 6. Decisions on the criteria + +- **B: yes.** The dominant cost in A is a scan of every membership row on every call. B removes it, with a 31× to ~4,000× gain per call. +- **B+: yes, when users share inheritance.** + - On top of B: 2–3× for a user's effective groups and 12–15× for roles. + - All-user reconciliation drops from seconds to about 1 ms. + - The compile costs 1.6 ms, independent of user count. + - A nesting change is a 2 µs incremental insertion; a removal is a full recompile. + - Group → members gains nothing over B, since both read the same direct-user rows. +- **F: not a meaningful general advantage.** Its gain is bounded: up to 2× per evaluation with 64 or fewer focus groups, and only after about 1,000 evaluations reuse one prepared focus, on warm caches. At 128 groups it is 1.3×. Near 256 groups there is no gain, and none when cold. F+users trades about 33 bytes per user for the lowest evaluation cost on small foci; its break-even is about 3,000 evaluations. +- **Answer to the question:** sharing compiled closure results across users is where the acceleration comes from. Reusing a local geometry adds a small, conditional gain on top. A dense global closure is already compact at this directory size: 24 words per row, resident in L2. + +## 7. Boundaries and what this does not settle + +- **The no-gather rule is not satisfied by B, B+ or F's hot path either**, and this probe does not claim it is. Each reads a CSR row or closure row at an address taken from a user's direct-group list, which is a value in another table. The improvement is in count: a handful of reads per call instead of a scan of every membership row. The co-addressed form would iterate the set bits of a user × group bit row, as #1634's walk_mask does with its adjacency rows. That is the same reads under coordinate addressing, and whether it counts as admissible is part of `ISS-ADMISSION-PREP-BINDING` / the gather decision. +- **Preparation translates addresses** (GUID → ordinal, closure projection onto the focus), counted. It is legal only in the lab until that issue is ruled. +- **The data is synthetic.** The fixture shape is a modelling choice, not a measured directory. +- **Not built:** + - patching single membership rows (rebuilding the CSRs takes 1.9 ms at 8,000 users); + - remapping old ordinals after a snapshot rebuild; + - incremental removal of a nesting pair; + - a focus that computes its own closure (tile). + +## 8. Proposed next step (not authorized) + +**D-RMAP-DIR-2:** +- B built inside `Snapshot::build` from the lanes: CSR by user and by group, and the nesting pairs as ordinals. +- B+ closure as a lazily compiled, version-witnessed cache behind the existing `View` methods. +- dir-sim's own tests and this probe's oracle as the equivalence gate. + +It waits on two things: a ruling on the hot-path read pattern (§7) and explicit authorization. F is not proposed for production. diff --git a/.github/workflows/dir-rmap-probe.yml b/.github/workflows/dir-rmap-probe.yml new file mode 100644 index 000000000..ca278fea2 --- /dev/null +++ b/.github/workflows/dir-rmap-probe.yml @@ -0,0 +1,90 @@ +name: dir-rmap probe +# `crates/dir-rmap-probe` (D-RMAP-DIR-1, lab only) is workspace-excluded (own +# [workspace] root, like lance-graph-dir-sim, which it depends on), so no +# other job builds it. It compares compiled membership closure and a 256-slot +# focus against dir-sim's own walks and a crepe oracle; the benchmark run also +# asserts equivalence at 8,000 users before it times anything. +on: + pull_request: + paths: + - crates/dir-rmap-probe/** + - crates/lance-graph-dir-sim/** + - crates/lance-graph-quack/** + - crates/lance-graph-mask-risc/** + - .github/workflows/dir-rmap-probe.yml + - rust-toolchain.toml + push: + branches: + - main + paths: + - crates/dir-rmap-probe/** + - crates/lance-graph-dir-sim/** + - crates/lance-graph-quack/** + - crates/lance-graph-mask-risc/** + - .github/workflows/dir-rmap-probe.yml + - rust-toolchain.toml + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_PROFILE_TEST_DEBUG: "0" + RUST_BACKTRACE: "1" + +jobs: + probe: + name: dir-rmap probe + runs-on: ubuntu-24.04 + timeout-minutes: 45 + defaults: + run: + working-directory: lance-graph + steps: + - uses: actions/checkout@v4 + with: + path: lance-graph + persist-credentials: false + - name: Checkout AdaWorldAPI/ndarray (sibling dependency) + uses: actions/checkout@v4 + with: + repository: AdaWorldAPI/ndarray + path: ndarray + persist-credentials: false + - name: Checkout AdaWorldAPI/OGAR (sibling dependency) + uses: actions/checkout@v4 + with: + repository: AdaWorldAPI/OGAR + path: OGAR + persist-credentials: false + - name: Setup rust toolchain (pinned by rust-toolchain.toml) + run: rustup show + # `.cargo/config.toml` builds with target-cpu=native; spell it out as + # this runner's feature list so a cached artifact never runs on a CPU + # without its features (same step as dir-sim-persist.yml). + - name: Resolve target-cpu=native for this runner + id: cpu + run: | + flags=$(.github/scripts/resolve-native-rustflags.sh) + echo "RUSTFLAGS=$flags" >> "$GITHUB_ENV" + hash=$(printf %s "$flags" | sha256sum | cut -c1-16) + echo "flags=$hash" >> "$GITHUB_OUTPUT" + - uses: Swatinem/rust-cache@v2 + with: + key: cpu-${{ steps.cpu.outputs.flags }} + workspaces: lance-graph/crates/dir-rmap-probe + - name: fmt + run: cargo fmt --manifest-path crates/dir-rmap-probe/Cargo.toml --check + - name: clippy + run: cargo clippy --manifest-path crates/dir-rmap-probe/Cargo.toml --all-targets -- -D warnings + - name: test + run: cargo test --release --manifest-path crates/dir-rmap-probe/Cargo.toml + - name: benchmark (asserts equivalence at 8,000 users; timings are not gated) + run: cargo run --release --manifest-path crates/dir-rmap-probe/Cargo.toml --example bench diff --git a/crates/dir-rmap-probe/examples/bench.rs b/crates/dir-rmap-probe/examples/bench.rs new file mode 100644 index 000000000..74846e64f --- /dev/null +++ b/crates/dir-rmap-probe/examples/bench.rs @@ -0,0 +1,799 @@ +//! D-RMAP-DIR-1 benchmark: arms A (dir-sim View), D (crepe), B (CSR), +//! B+ (compiled closure) and F (256-slot focus), on one synthetic IAM +//! directory. Run in release from this crate's directory: +//! +//! ```text +//! cargo run --release --example bench # 8,000 users +//! cargo run --release --example bench -- 65000 # large fixture +//! ``` +//! +//! Every timed answer is checked against arm A (and D where it applies) +//! before it is timed. Medians of repeated runs; per-operation figures are +//! a timed batch divided by its size. "Cold" means after touching 64 MiB of +//! unrelated memory, so the prepared structures are out of L1/L2. + +use dir_rmap_probe::fixture::{Fixture, Spec}; +use dir_rmap_probe::focus::{Focus, FocusUsers, holds}; +use dir_rmap_probe::prep::{Chain, Compiled, Ordinals, bit, words}; +use dir_rmap_probe::{Edit, World, a_groups, a_members, focus_for, oracle, templates}; +use lance_graph_dir_sim::{GroupOrdinal, UserOrdinal}; +use ndarray::simd::{eq_u8_to_mask, mask_and_assign, popcount_batch_u64}; +use ogar_dir_core::Guid128; +use std::collections::HashMap; +use std::hint::black_box; +use std::time::{Duration, Instant}; + +fn median(mut v: Vec) -> Duration { + v.sort(); + v[v.len() / 2] +} + +/// Median wall time of `reps` runs of `f`. +fn time(reps: usize, mut f: impl FnMut() -> T) -> Duration { + let mut t = Vec::with_capacity(reps); + for _ in 0..reps { + let s = Instant::now(); + black_box(f()); + t.push(s.elapsed()); + } + median(t) +} + +fn ns(d: Duration, n: usize) -> f64 { + d.as_nanos() as f64 / n as f64 +} + +fn us(d: Duration) -> f64 { + d.as_secs_f64() * 1e6 +} + +/// Touch 64 MiB so the next run starts with cold caches. +fn evict(buf: &mut [u64]) { + for (i, x) in buf.iter_mut().enumerate() { + *x = x.wrapping_add(i as u64); + } + black_box(&buf); +} + +fn cache_sizes() -> String { + let mut out = Vec::new(); + for i in 0..4 { + let base = format!("/sys/devices/system/cpu/cpu0/cache/index{i}"); + let (Ok(l), Ok(t), Ok(s)) = ( + std::fs::read_to_string(format!("{base}/level")), + std::fs::read_to_string(format!("{base}/type")), + std::fs::read_to_string(format!("{base}/size")), + ) else { + continue; + }; + out.push(format!("L{} {} {}", l.trim(), t.trim(), s.trim())); + } + out.join(", ") +} + +fn cpu() -> String { + std::fs::read_to_string("/proc/cpuinfo") + .ok() + .and_then(|s| { + s.lines() + .find(|l| l.starts_with("model name")) + .map(|l| l.split(':').nth(1).unwrap_or("").trim().to_string()) + }) + .unwrap_or_default() +} + +fn main() { + let users: usize = std::env::args() + .nth(1) + .map(|s| s.parse().expect("user count")) + .unwrap_or(8_000); + println!("# D-RMAP-DIR-1, {users} users\n"); + println!("cpu: {} ", cpu()); + println!("caches: {} ", cache_sizes()); + println!( + "avx512f: {}, avx2: {}\n", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx2") + ); + let mut evict_buf = vec![0u64; 8 << 20]; + + // ---- construction --------------------------------------------------- + let s = Instant::now(); + let fx = Fixture::generate(Spec::iam(users)); + let t_gen = s.elapsed(); + let s = Instant::now(); + let mut w = World::new(fx); + let t_snap = s.elapsed(); + let v0 = w.version; + let (n_users, n_groups) = (w.view().users_len(), w.view().groups_len()); + let t_b = time(5, || w.prepare()); + let b = w.prepare(); + let t_c = time(5, || { + let p = w.prepare(); + let s = Instant::now(); + let c = Compiled::compile(p); + (s.elapsed(), c.passes) + }); + // `time` measured prepare + compile; compile alone is measured inside. + let mut compile_only = Vec::new(); + for _ in 0..5 { + let p = w.prepare(); + let s = Instant::now(); + black_box(Compiled::compile(p)); + compile_only.push(s.elapsed()); + } + let t_compile = median(compile_only); + let c = Compiled::compile(w.prepare()); + let all_templates = templates(&w.f); + let s = Instant::now(); + let d = oracle::run(&w.f, &w.direct, &w.nested, &all_templates); + let t_d = s.elapsed(); + let v = w.view(); + let ord = Ordinals::of(&w.f, &v); + println!("## Construction\n"); + println!("| step | time |\n|---|---|"); + println!( + "| fixture generation | {:.1} ms |", + t_gen.as_secs_f64() * 1e3 + ); + println!( + "| dir-sim snapshot (`observe`: Snapshot::build) | {:.1} ms |", + t_snap.as_secs_f64() * 1e3 + ); + println!( + "| B: CSR prepare (incl. GUID translation) | {:.2} ms |", + t_b.as_secs_f64() * 1e3 + ); + println!( + "| B+: closure compile, 3 chains x up/down (B excluded) | {:.2} ms |", + t_compile.as_secs_f64() * 1e3 + ); + println!("| B + B+ together | {:.2} ms |", t_c.as_secs_f64() * 1e3); + println!( + "| D: crepe oracle, all relations | {:.1} ms |", + t_d.as_secs_f64() * 1e3 + ); + println!( + "\n{n_users} users, {n_groups} groups, {} direct rows, {} nesting pairs, {} groups > 256 direct members; closure passes (up, down) all/sec/mail: {:?}\n", + w.direct.len(), + w.nested.len(), + w.f.large_groups(), + c.passes + ); + + // ---- memory ----------------------------------------------------------- + let gw = words(n_groups); + println!("## Memory\n"); + println!("| structure | bytes |\n|---|---|"); + println!("| B: CSR tables + planes | {} |", b.bytes()); + println!( + "| B+: closure rows (3 chains x up/down) + role table | {} |", + c.closure_bytes() + ); + println!( + "| one dense closure row ({gw} words) / one chain, up only | {} / {} |", + gw * 8, + n_groups * gw * 8 + ); + let (_, g256) = focus_for(&all_templates, 256); + let focus_ids: Vec = g256.iter().map(|&g| ord.group[g as usize]).collect(); + let f256 = Focus::bind(&c, &focus_ids); + let fu256 = FocusUsers::bind(&c, &focus_ids); + println!( + "| F (256 slots): projected rows + reverse map | {} |", + f256.bytes() + ); + println!( + "| F+users (256 slots): + one row per user | {} |", + fu256.bytes() + ); + println!("| a 256 x 256 Boolean tile (not built) | 8192 |\n"); + + // ---- correctness at scale --------------------------------------------- + // Arm A scans every membership row per call, so it is measured on a + // sample: 2,000 users, or 500 above 20,000 users. + let a_n = if users > 20_000 { 500 } else { 2_000 }; + let sample: Vec = (0..w.f.users.len()) + .step_by((w.f.users.len() / a_n).max(1)) + .collect(); + let mut row = vec![0u64; gw]; + for &i in &sample { + c.groups_of(ord.user[i], Chain::All, &mut row); + let got: Vec = dir_rmap_probe::prep::ones(&row) + .map(|g| v.group_guid(GroupOrdinal(g as u16)).unwrap()) + .collect(); + let mut a = a_groups(&v, &w.f.users[i], Chain::All); + a.sort_unstable(); + let mut got = got; + got.sort_unstable(); + assert_eq!(got, a, "B+ != A for user {i}"); + } + let total_b: usize = (0..w.f.users.len()) + .map(|i| { + c.groups_of(ord.user[i], Chain::All, &mut row); + popcount_batch_u64(&row) as usize + }) + .sum(); + assert_eq!( + total_b, + d.member.len(), + "B+ and the oracle disagree on membership size" + ); + println!( + "Checked: B+ = A on {} users; B+ = D on all {} memberships.\n", + sample.len(), + d.member.len() + ); + + // ---- W1 direct membership -------------------------------------------- + let mut rng = dir_rmap_probe::fixture::Rng::new(7); + let pairs: Vec<(usize, usize)> = (0..10_000) + .map(|_| (rng.below(w.f.users.len()), rng.below(w.f.groups.len()))) + .collect(); + let a1 = time(9, || { + pairs + .iter() + .filter(|&&(u, g)| v.is_member(&w.f.users[u], &w.f.groups[g])) + .count() + }); + let b1 = time(9, || { + pairs + .iter() + .filter(|&&(u, g)| b.is_member(ord.user[u], ord.group[g])) + .count() + }); + // ---- W2 user -> effective groups -------------------------------------- + let mut queue = Vec::new(); + let a2 = time(3, || { + sample + .iter() + .map(|&i| a_groups(&v, &w.f.users[i], Chain::All).len()) + .sum::() + }); + let all_users: Vec = (0..w.f.users.len()).collect(); + let mut prow = vec![0u64; gw]; + let b2 = time(9, || { + for &i in &all_users { + b.groups_of(ord.user[i], Chain::All, &mut prow, &mut queue); + } + prow[0] + }); + let c2 = time(9, || { + for &i in &all_users { + c.groups_of(ord.user[i], Chain::All, &mut prow); + } + prow[0] + }); + let c2e = time(9, || { + let mut n = 0; + for &i in &all_users { + c.groups_of(ord.user[i], Chain::All, &mut prow); + let out: Vec = dir_rmap_probe::prep::ones(&prow) + .map(|g| v.group_guid(GroupOrdinal(g as u16)).unwrap()) + .collect(); + n += out.len(); + } + n + }); + // ---- W3 group -> transitive members ----------------------------------- + let uw = words(n_users); + let (mut reached, mut members) = (vec![0u64; gw], vec![0u64; uw]); + let gsample: Vec = (0..w.f.groups.len()).collect(); + let a3 = time(3, || { + gsample + .iter() + .map(|&j| a_members(&v, &w.f.groups[j], Chain::All).len()) + .sum::() + }); + let b3 = time(9, || { + for &j in &gsample { + b.members_of( + ord.group[j], + Chain::All, + &mut reached, + &mut members, + &mut queue, + ); + } + members[0] + }); + let c3 = time(9, || { + for &j in &gsample { + c.members_of(ord.group[j], Chain::All, &mut members); + } + members[0] + }); + // ---- W4 roles and SIDs ------------------------------------------------- + let by_guid: HashMap = + w.f.grants.iter().fold(HashMap::new(), |mut m, &(g, r)| { + *m.entry(w.f.groups[g as usize]).or_default() |= 1 << r; + m + }); + let a4 = time(3, || { + sample + .iter() + .map(|&i| { + a_groups(&v, &w.f.users[i], Chain::Security) + .iter() + .fold(0u64, |a, g| a | by_guid.get(g).copied().unwrap_or(0)) + }) + .fold(0, |a, r| a ^ r) + }); + for &i in &sample { + let a = a_groups(&v, &w.f.users[i], Chain::Security) + .iter() + .fold(0u64, |a, g| a | by_guid.get(g).copied().unwrap_or(0)); + assert_eq!(c.roles_of(ord.user[i]), a, "B+ roles != A"); + } + let b4 = time(9, || { + all_users + .iter() + .map(|&i| b.roles_of(ord.user[i], &mut prow, &mut queue)) + .fold(0, |a, r| a ^ r) + }); + let c4 = time(9, || { + all_users + .iter() + .map(|&i| c.roles_of(ord.user[i])) + .fold(0, |a, r| a ^ r) + }); + let c4s = time(9, || { + for &i in &all_users { + c.groups_of(ord.user[i], Chain::Security, &mut prow); + } + prow[0] + }); + + println!("## Hot workloads (per operation, warm)\n"); + println!("| workload | A (dir-sim) | B (CSR) | B+ (closure) | B+ / A |\n|---|---|---|---|---|"); + let row_fmt = |name: &str, a: f64, bb: Option, cc: f64| { + println!( + "| {name} | {a:.0} ns | {} | {cc:.0} ns | {:.0}x faster |", + bb.map_or("-".into(), |x| format!("{x:.0} ns")), + a / cc + ); + }; + row_fmt( + "W1 direct membership", + ns(a1, pairs.len()), + Some(ns(b1, pairs.len())), + ns(b1, pairs.len()), + ); + row_fmt( + "W2 user -> effective groups (plane)", + ns(a2, sample.len()), + Some(ns(b2, all_users.len())), + ns(c2, all_users.len()), + ); + row_fmt( + "W2 same, emitted as GUIDs", + ns(a2, sample.len()), + None, + ns(c2e, all_users.len()), + ); + row_fmt( + "W3 group -> transitive members", + ns(a3, gsample.len()), + Some(ns(b3, gsample.len())), + ns(c3, gsample.len()), + ); + row_fmt( + "W4 roles via SID chain", + ns(a4, sample.len()), + Some(ns(b4, all_users.len())), + ns(c4, all_users.len()), + ); + println!( + "| W4 SIDs only (security closure OR) | - | - | {:.0} ns | |", + ns(c4s, all_users.len()) + ); + println!( + "\nA columns are measured on {} sampled users / all {} groups; B and B+ on all users.\n", + sample.len(), + gsample.len() + ); + + // ---- W5 role x location: B+ vs F --------------------------------------- + println!("## W5: role x location questions, B+ vs F\n"); + println!( + "One evaluation = one user, every template question in the focus. N = evaluations reusing one prepared focus (users cycled). F prepare = bind + projection of every group's closure row. Totals measured directly.\n" + ); + println!( + "| focus groups | questions | A eval | B+ eval | F eval | F+users eval | F prepare | F+users prepare | B+ N=1 / F / F+u | N=10 | N=100 | N=1000 | N=10000 |" + ); + println!("|---|---|---|---|---|---|---|---|---|---|---|---|---|"); + for k in [16usize, 64, 128, 256] { + let (ts, gs) = focus_for(&all_templates, k); + let groups: Vec = gs.iter().map(|&g| ord.group[g as usize]).collect(); + let tq: Vec<(usize, usize)> = ts + .iter() + .map(|&(r, l)| { + ( + ord.group[r as usize] as usize, + ord.group[l as usize] as usize, + ) + }) + .collect(); + let f = Focus::bind(&c, &groups); + let fu = FocusUsers::bind(&c, &groups); + let masks: Vec<_> = tq + .iter() + .map(|&(r, l)| f.all_of(&[r as u16, l as u16]).unwrap()) + .collect(); + let tq_guid: Vec<(Guid128, Guid128)> = ts + .iter() + .map(|&(r, l)| (w.f.groups[r as usize], w.f.groups[l as usize])) + .collect(); + // Equal answers on every user before timing. + for &i in &all_users { + let u = ord.user[i]; + c.groups_of(u, Chain::All, &mut prow); + let fr = f.row_of(&c, u); + for (q, &(r, l)) in tq.iter().enumerate() { + let want = bit(&prow, r) && bit(&prow, l); + assert_eq!(holds(&fr, &masks[q]), want); + assert_eq!(holds(&fu.rows[u as usize], &masks[q]), want); + } + } + let users_ord: Vec = all_users.iter().map(|&i| ord.user[i]).collect(); + let eval_bp = |n: usize, prow: &mut Vec| { + let mut hits = 0usize; + for e in 0..n { + let u = users_ord[e % users_ord.len()]; + c.groups_of(u, Chain::All, prow); + for &(r, l) in &tq { + hits += usize::from(bit(prow, r) && bit(prow, l)); + } + } + hits + }; + let eval_f = |f: &Focus, n: usize| { + let mut hits = 0usize; + for e in 0..n { + let u = users_ord[e % users_ord.len()]; + let r = f.row_of(&c, u); + for m in &masks { + hits += usize::from(holds(&r, m)); + } + } + hits + }; + let eval_fu = |fu: &FocusUsers, n: usize| { + let mut hits = 0usize; + for e in 0..n { + let r = &fu.rows[users_ord[e % users_ord.len()] as usize]; + for m in &masks { + hits += usize::from(holds(r, m)); + } + } + hits + }; + let m = users_ord.len(); + let t_a = time(3, || { + sample + .iter() + .map(|&i| { + let a = a_groups(&v, &w.f.users[i], Chain::All); + tq_guid + .iter() + .filter(|(r, l)| a.binary_search(r).is_ok() && a.binary_search(l).is_ok()) + .count() + }) + .sum::() + }); + let t_bp = time(9, || eval_bp(m, &mut prow)); + let t_f = time(9, || eval_f(&f, m)); + let t_fu = time(9, || eval_fu(&fu, m)); + let t_fprep = time(9, || Focus::bind(&c, &groups)); + let t_fuprep = time(5, || FocusUsers::bind(&c, &groups)); + let mut totals = Vec::new(); + for n in [1usize, 10, 100, 1_000, 10_000] { + let reps = if n <= 100 { 51 } else { 9 }; + let bp = time(reps, || eval_bp(n, &mut prow)); + let ff = time(reps, || { + let f = Focus::bind(&c, &groups); + eval_f(&f, n) + }); + let fuu = time(reps, || { + let fu = FocusUsers::bind(&c, &groups); + eval_fu(&fu, n) + }); + totals.push(format!("{:.1} / {:.1} / {:.1} µs", us(bp), us(ff), us(fuu))); + } + println!( + "| {} | {} | {:.0} ns | {:.0} ns | {:.0} ns | {:.0} ns | {:.1} µs | {:.1} µs | {} |", + gs.len(), + ts.len(), + ns(t_a, sample.len()), + ns(t_bp, m), + ns(t_f, m), + ns(t_fu, m), + us(t_fprep), + us(t_fuprep), + totals.join(" | ") + ); + } + // Cold vs warm for one evaluation, 256-slot focus. + { + let (ts, gs) = focus_for(&all_templates, 256); + let groups: Vec = gs.iter().map(|&g| ord.group[g as usize]).collect(); + let f = Focus::bind(&c, &groups); + let masks: Vec<_> = ts + .iter() + .map(|&(r, l)| { + f.all_of(&[ord.group[r as usize], ord.group[l as usize]]) + .unwrap() + }) + .collect(); + let tq: Vec<(usize, usize)> = ts + .iter() + .map(|&(r, l)| { + ( + ord.group[r as usize] as usize, + ord.group[l as usize] as usize, + ) + }) + .collect(); + let mut cold_bp = Vec::new(); + let mut cold_f = Vec::new(); + for e in 0..41 { + let u = ord.user[(e * 197) % w.f.users.len()]; + evict(&mut evict_buf); + let s = Instant::now(); + c.groups_of(u, Chain::All, &mut prow); + black_box( + tq.iter() + .filter(|&&(r, l)| bit(&prow, r) && bit(&prow, l)) + .count(), + ); + cold_bp.push(s.elapsed()); + evict(&mut evict_buf); + let s = Instant::now(); + let r = f.row_of(&c, u); + black_box(masks.iter().filter(|m| holds(&r, m)).count()); + cold_f.push(s.elapsed()); + } + println!( + "\nCold single evaluation (256-slot focus, {} questions, after evicting 64 MiB): B+ {:.0} ns, F {:.0} ns.\n", + ts.len(), + us(median(cold_bp)) * 1e3, + us(median(cold_f)) * 1e3 + ); + } + + // ---- W6 team membership x attribute ----------------------------------- + let mut lane = vec![0u8; n_users]; + for (i, &a) in w.f.attr.iter().enumerate() { + lane[ord.user[i] as usize] = a; + } + let teams: Vec = w.f.teams.iter().map(|&t| t as usize).collect(); + let mut attr_mask = vec![0u64; uw]; + let a6 = time(5, || { + let mut n = 0; + for &t in &teams { + for g in a_members(&v, &w.f.groups[t], Chain::All) { + let o = v.user_ordinal(&g).unwrap().0 as usize; + n += usize::from(lane[o] == 3); + } + } + n + }); + let c6 = time(9, || { + let mut n = 0; + for &t in &teams { + c.members_of(ord.group[t], Chain::All, &mut members); + eq_u8_to_mask(&lane, 3, &mut attr_mask); + mask_and_assign(&mut members, &attr_mask); + n += popcount_batch_u64(&members); + } + n + }); + let check_a = { + let mut n = 0; + for &t in &teams { + for g in a_members(&v, &w.f.groups[t], Chain::All) { + let o = v.user_ordinal(&g).unwrap().0 as usize; + n += usize::from(lane[o] == 3); + } + } + n as u64 + }; + let check_c = { + let mut n = 0; + for &t in &teams { + c.members_of(ord.group[t], Chain::All, &mut members); + eq_u8_to_mask(&lane, 3, &mut attr_mask); + mask_and_assign(&mut members, &attr_mask); + n += popcount_batch_u64(&members); + } + n + }; + assert_eq!(check_a, check_c); + println!("## W6: team transitive members AND attribute = value\n"); + println!( + "{} teams: A {:.1} µs per team, B+ with `eq_u8_to_mask` + AND + popcount {:.1} µs per team. F does not apply: the focus selects groups, the attribute is per user.\n", + teams.len(), + us(a6) / teams.len() as f64, + us(c6) / teams.len() as f64 + ); + + // ---- W7/W8 overlay: reconciliation and refresh ------------------------ + let c0 = Compiled::compile(w.prepare()); + let mut rng = dir_rmap_probe::fixture::Rng::new(11); + let mut edits = Vec::new(); + for _ in 0..(w.f.users.len() / 100).max(1) { + let u = rng.below(w.f.users.len()) as u32; + let t = w.f.teams[rng.below(w.f.teams.len())]; + if w.direct.binary_search(&(u, t)).is_err() { + edits.push(Edit::AddDirect(u, t)); + } + } + let removals: Vec<(u32, u32)> = w + .direct + .iter() + .step_by(97) + .copied() + .take(edits.len()) + .collect(); + edits.extend(removals.iter().map(|&(u, g)| Edit::RemoveDirect(u, g))); + let membership_edits = edits.len(); + let nest = (w.f.teams[0], w.f.roles[5]); + edits.push(Edit::AddNested(nest.0, nest.1)); + w.apply(&edits); + let v1 = w.version; + let view0 = w.st.view(v0).unwrap(); + let view1 = w.view(); + let ord1 = Ordinals::of(&w.f, &view1); + let full1 = Compiled::compile(w.prepare()); + // A: both versions, on every user up to 20,000 users, else on the + // sample (time extrapolated to every user, labelled). + let a7_users: Vec = if users > 20_000 { + sample.clone() + } else { + all_users.clone() + }; + let a7s = Instant::now(); + let mut changed_a = 0; + for &i in &a7_users { + let u = &w.f.users[i]; + changed_a += usize::from(view0.groups_transitive(u) != view1.groups_transitive(u)); + } + let a7 = a7s + .elapsed() + .mul_f64(w.f.users.len() as f64 / a7_users.len() as f64); + let (mut r0, mut r1) = (vec![0u64; gw], vec![0u64; gw]); + let c7 = time(9, || { + let mut changed = 0; + for i in 0..w.f.users.len() { + c0.groups_of(ord.user[i], Chain::All, &mut r0); + full1.groups_of(ord1.user[i], Chain::All, &mut r1); + changed += usize::from(r0 != r1); + } + changed + }); + let mut changed_c = 0; + for &i in &a7_users { + c0.groups_of(ord.user[i], Chain::All, &mut r0); + full1.groups_of(ord1.user[i], Chain::All, &mut r1); + changed_c += usize::from(r0 != r1); + } + assert_eq!(changed_a, changed_c, "reconciliation disagrees"); + // Refresh paths. + let t_full_b = time(5, || w.prepare()); + let t_full_c = time(3, || Compiled::compile(w.prepare())); + let t_direct = time(9, || { + let mut p = w.prepare(); + let s = Instant::now(); + p.set_direct(&ord1, &w.direct); + s.elapsed() + }); + let mut set_direct_only = Vec::new(); + for _ in 0..9 { + let mut p = w.prepare(); + let s = Instant::now(); + p.set_direct(&ord1, &w.direct); + set_direct_only.push(s.elapsed()); + black_box(p); + } + let mut inc = Vec::new(); + for _ in 0..9 { + let mut c = Compiled::compile({ + let mut p = w.prepare(); + p.nested + .retain(|&x| x != (ord1.group[nest.0 as usize], ord1.group[nest.1 as usize])); + let (u, d) = { + let mut up: Vec<(u16, u16)> = p.nested.clone(); + let mut dn: Vec<(u16, u16)> = p.nested.iter().map(|&(c, p)| (p, c)).collect(); + ( + dir_rmap_probe::prep::Csr::build(p.groups, &mut up), + dir_rmap_probe::prep::Csr::build(p.groups, &mut dn), + ) + }; + p.up = u; + p.down = d; + p + }); + let s = Instant::now(); + c.add_nesting(ord1.group[nest.0 as usize], ord1.group[nest.1 as usize]); + inc.push(s.elapsed()); + for chain in [Chain::All, Chain::Security, Chain::Mail] { + assert_eq!( + c.up[chain as usize], full1.up[chain as usize], + "incremental != full" + ); + } + } + let t_rebind = time(9, || { + Focus::bind(&full1, &focus_ids_for(&full1, &all_templates, &ord1)) + }); + println!("## W7/W8: overlay ({membership_edits} membership edits + 1 nesting edge)\n"); + println!("| step | time |\n|---|---|"); + println!( + "| W7 reconciliation, all users, A (2 versions) | {:.1} ms{} ({changed_a} of {} checked users changed) |", + a7.as_secs_f64() * 1e3, + if a7_users.len() < w.f.users.len() { + ", extrapolated from the sample" + } else { + "" + }, + a7_users.len() + ); + println!( + "| W7 reconciliation, all users, B+ (2 compiled versions) | {:.1} ms |", + c7.as_secs_f64() * 1e3 + ); + println!( + "| W8 B full rebuild | {:.2} ms |", + t_full_b.as_secs_f64() * 1e3 + ); + println!( + "| W8 B + B+ full recompile | {:.2} ms |", + t_full_c.as_secs_f64() * 1e3 + ); + println!( + "| W8 membership edits: rebuild the two membership CSRs only (closures untouched) | {:.2} ms |", + median(set_direct_only).as_secs_f64() * 1e3 + ); + let _ = t_direct; + println!( + "| W8 one nesting edge: incremental closure insertion | {:.1} µs |", + us(median(inc)) + ); + println!( + "| W8 F (256 slots) rebind after the change | {:.1} µs |\n", + us(t_rebind) + ); + + // ---- W9 snapshot rebuild with shifted ordinals ------------------------ + let mut fx2 = Fixture::generate(Spec::iam(users)); + fx2.users.insert(0, Guid128([0; 16])); + fx2.attr.insert(0, 0); + for e in fx2.direct.iter_mut() { + e.0 += 1; + } + let s = Instant::now(); + let w2 = World::new(fx2); + let t_snap2 = s.elapsed(); + let ord2 = Ordinals::of(&w2.f, &w2.view()); + let shifted = (0..w.f.users.len()) + .filter(|&i| ord2.user[i + 1] != ord.user[i]) + .count(); + let stale = c.b.check(&w2.witness()).is_err(); + assert!(stale, "a prepared structure must refuse a rebuilt snapshot"); + let t_rebuild = time(3, || Compiled::compile(w2.prepare())); + println!("## W9: snapshot rebuild with one user inserted first\n"); + println!( + "{shifted} of {} user ordinals moved; the old B+ refuses (witness: version and size). Snapshot {:.1} ms, B + B+ rebuild {:.2} ms. Remapping old ordinals was not built.\n", + w.f.users.len(), + t_snap2.as_secs_f64() * 1e3, + t_rebuild.as_secs_f64() * 1e3 + ); + let _ = v1; + let _ = UserOrdinal(0); +} + +fn focus_ids_for(c: &Compiled, templates: &[(u32, u32)], ord: &Ordinals) -> Vec { + let _ = c; + let (_, gs) = focus_for(templates, 256); + gs.iter().map(|&g| ord.group[g as usize]).collect() +} diff --git a/crates/dir-rmap-probe/src/fixture.rs b/crates/dir-rmap-probe/src/fixture.rs index 3e08290f8..10e1abb65 100644 --- a/crates/dir-rmap-probe/src/fixture.rs +++ b/crates/dir-rmap-probe/src/fixture.rs @@ -104,7 +104,7 @@ impl Rng { pub fn new(seed: u64) -> Self { Rng(seed) } - pub fn next(&mut self) -> u64 { + pub fn next_u64(&mut self) -> u64 { self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); let mut z = self.0; z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); @@ -112,12 +112,12 @@ impl Rng { z ^ (z >> 31) } pub fn below(&mut self, n: usize) -> usize { - (self.next() % n as u64) as usize + (self.next_u64() % n as u64) as usize } fn guid(&mut self) -> Guid128 { let mut b = [0u8; 16]; - b[..8].copy_from_slice(&self.next().to_le_bytes()); - b[8..].copy_from_slice(&self.next().to_le_bytes()); + b[..8].copy_from_slice(&self.next_u64().to_le_bytes()); + b[8..].copy_from_slice(&self.next_u64().to_le_bytes()); Guid128(b) } } @@ -164,7 +164,7 @@ impl Fixture { locations: Vec::new(), teams: Vec::new(), attr: (0..spec.users) - .map(|_| (r.next() % u64::from(spec.attr_values)) as u8) + .map(|_| (r.next_u64() % u64::from(spec.attr_values)) as u8) .collect(), scope: DirectoryScope(Guid128([0x5C; 16])), }; @@ -288,7 +288,7 @@ impl Fixture { }; if r.below(100) < p { f.grants - .push((j as u32, (r.next() % u64::from(spec.role_ids)) as u8)); + .push((j as u32, (r.next_u64() % u64::from(spec.role_ids)) as u8)); } } f diff --git a/crates/dir-rmap-probe/src/focus.rs b/crates/dir-rmap-probe/src/focus.rs index f7939889a..f6df42f70 100644 --- a/crates/dir-rmap-probe/src/focus.rs +++ b/crates/dir-rmap-probe/src/focus.rs @@ -22,7 +22,7 @@ //! group ordinal space (slot + 1, 0 = not bound): one indexed read, no hash, //! no search. At 1,500 groups that is 3 KB. -use crate::prep::{bit, Compiled, Refusal, Witness}; +use crate::prep::{Compiled, Refusal, Witness, bit}; pub const SLOTS: usize = 256; pub type Row = [u64; 4]; diff --git a/crates/dir-rmap-probe/src/lib.rs b/crates/dir-rmap-probe/src/lib.rs index 12471a5f8..2a57813a6 100644 --- a/crates/dir-rmap-probe/src/lib.rs +++ b/crates/dir-rmap-probe/src/lib.rs @@ -103,7 +103,9 @@ pub struct World { impl World { pub fn new(f: Fixture) -> Self { let mut st = VersionStore::new(); - let version = st.observe("probe", 0, f.observation()).expect("fixture builds"); + let version = st + .observe("probe", 0, f.observation()) + .expect("fixture builds"); World { direct: f.direct.clone(), nested: f.nested.clone(), diff --git a/crates/dir-rmap-probe/src/oracle.rs b/crates/dir-rmap-probe/src/oracle.rs index 409e2f4e5..109e383b9 100644 --- a/crates/dir-rmap-probe/src/oracle.rs +++ b/crates/dir-rmap-probe/src/oracle.rs @@ -84,7 +84,11 @@ pub fn run( .filter(|&g| f.security[g as usize] == Some(true)) .map(Sec), ); - rt.extend((0..f.groups.len() as u32).filter(|&g| f.mail[g as usize]).map(Mail)); + rt.extend( + (0..f.groups.len() as u32) + .filter(|&g| f.mail[g as usize]) + .map(Mail), + ); rt.extend(f.grants.iter().map(|&(g, r)| Grant(g, u32::from(r)))); rt.extend(templates.iter().map(|&(r, l)| Template(r, l))); let (member, sec, mail, roles, eligible): ( diff --git a/crates/dir-rmap-probe/src/prep.rs b/crates/dir-rmap-probe/src/prep.rs index 1d87d13a9..8c853cb53 100644 --- a/crates/dir-rmap-probe/src/prep.rs +++ b/crates/dir-rmap-probe/src/prep.rs @@ -22,7 +22,7 @@ //! `ISS-ADMISSION-PREP-BINDING`, still open. use crate::fixture::Fixture; -use lance_graph_dir_sim::{groups_where, GroupProperty, GroupWhere, View}; +use lance_graph_dir_sim::{GroupProperty, GroupWhere, View, groups_where}; use ndarray::simd::mask_or_assign; use ogar_dir_sim::VersionId; @@ -382,6 +382,9 @@ fn closure(n: usize, w: usize, adj: &Csr, admit: Option<&[u64]>) -> (Vec, u } for &p in adj.row(g) { let p = p as usize; + // A skip only: an unadmitted row is never seeded or written, + // so ORing it adds nothing (disable-checked). Correctness + // rests on the seeding and the `!ok(g)` skip above. if p == g || !ok(p) { continue; } @@ -480,10 +483,10 @@ impl Compiled { let (c, p) = (c as usize, p as usize); let w = self.w; for chain in CHAINS { - if let Some(a) = self.b.admit(chain) { - if !bit(a, c) || !bit(a, p) { - continue; - } + if let Some(a) = self.b.admit(chain) + && (!bit(a, c) || !bit(a, p)) + { + continue; } let k = ci(chain); let anc_p = self.up[k][p * w..p * w + w].to_vec(); @@ -520,7 +523,13 @@ impl Compiled { /// Bytes of the closure rows and role table, B's tables excluded. pub fn closure_bytes(&self) -> usize { - (self.up.iter().chain(self.down.iter()).map(Vec::len).sum::() + self.role_sec.len()) + (self + .up + .iter() + .chain(self.down.iter()) + .map(Vec::len) + .sum::() + + self.role_sec.len()) * 8 } } diff --git a/crates/dir-rmap-probe/tests/equivalence.rs b/crates/dir-rmap-probe/tests/equivalence.rs index feeeb9830..0bc3a2aa7 100644 --- a/crates/dir-rmap-probe/tests/equivalence.rs +++ b/crates/dir-rmap-probe/tests/equivalence.rs @@ -4,9 +4,9 @@ //! the comparisons meaningful. use dir_rmap_probe::fixture::{Fixture, Kind, Spec}; -use dir_rmap_probe::focus::{holds, Focus, FocusUsers}; -use dir_rmap_probe::prep::{ones, words, Chain, Compiled, Ordinals, Prepared, Refusal, CHAINS}; -use dir_rmap_probe::{a_groups, a_members, focus_for, oracle, templates, Edit, World}; +use dir_rmap_probe::focus::{Focus, FocusUsers, holds}; +use dir_rmap_probe::prep::{CHAINS, Chain, Compiled, Ordinals, Prepared, Refusal, ones, words}; +use dir_rmap_probe::{Edit, World, a_groups, a_members, focus_for, oracle, templates}; use ogar_dir_core::Guid128; use std::collections::{BTreeSet, HashMap}; @@ -24,7 +24,10 @@ fn groups_of_plane(w: &World, p: &[u64]) -> Vec { let v = w.view(); sorted( ones(p) - .map(|g| v.group_guid(lance_graph_dir_sim::GroupOrdinal(g as u16)).unwrap()) + .map(|g| { + v.group_guid(lance_graph_dir_sim::GroupOrdinal(g as u16)) + .unwrap() + }) .collect(), ) } @@ -33,7 +36,10 @@ fn users_of_plane(w: &World, p: &[u64]) -> Vec { let v = w.view(); sorted( ones(p) - .map(|u| v.user_guid(lance_graph_dir_sim::UserOrdinal(u as u16)).unwrap()) + .map(|u| { + v.user_guid(lance_graph_dir_sim::UserOrdinal(u as u16)) + .unwrap() + }) .collect(), ) } @@ -49,10 +55,11 @@ fn oracle_rel(a: &oracle::Answers, chain: Chain) -> &BTreeSet<(u32, u32)> { /// Roles as dir-sim answers them: the grants of the SIDs it reports, keyed /// by GUID, the shape `ViewActors::for_principal` reads. fn a_roles(w: &World, u: &Guid128) -> u64 { - let by_guid: HashMap = w.f.grants.iter().fold(HashMap::new(), |mut m, &(g, r)| { - *m.entry(w.f.groups[g as usize]).or_default() |= 1 << r; - m - }); + let by_guid: HashMap = + w.f.grants.iter().fold(HashMap::new(), |mut m, &(g, r)| { + *m.entry(w.f.groups[g as usize]).or_default() |= 1 << r; + m + }); a_groups(&w.view(), u, Chain::Security) .iter() .fold(0, |a, g| a | by_guid.get(g).copied().unwrap_or(0)) @@ -104,7 +111,11 @@ fn check_all(w: &World, b: &Prepared, c: &Compiled) { } for (i, ug) in w.f.users.iter().enumerate() { let a = a_roles(w, ug); - assert_eq!(b.roles_of(ord.user[i], &mut p, &mut queue), a, "B roles, user {i}"); + assert_eq!( + b.roles_of(ord.user[i], &mut p, &mut queue), + a, + "B roles, user {i}" + ); assert_eq!(c.roles_of(ord.user[i]), a, "B+ roles, user {i}"); assert_eq!(d_roles[i], a, "oracle roles, user {i}"); } @@ -150,7 +161,8 @@ fn the_focus_answers_role_and_location_like_the_oracle_and_dir_sim() { let m = f .all_of(&[ord.group[r as usize], ord.group[l as usize]]) .unwrap(); - let want = a.contains(&w.f.groups[r as usize]) && a.contains(&w.f.groups[l as usize]); + let want = + a.contains(&w.f.groups[r as usize]) && a.contains(&w.f.groups[l as usize]); assert_eq!(holds(&row, &m), want, "F k={k} user {i} ({r}, {l})"); assert_eq!(holds(&fu.rows[ord.user[i] as usize], &m), want, "F+users"); assert_eq!(d.eligible.contains(&(i as u32, r, l)), want, "oracle"); @@ -158,7 +170,10 @@ fn the_focus_answers_role_and_location_like_the_oracle_and_dir_sim() { } } } - assert!(checked > 10_000, "the comparison covered {checked} questions"); + assert!( + checked > 10_000, + "the comparison covered {checked} questions" + ); } #[test] @@ -168,7 +183,11 @@ fn overlay_incremental_equals_recompile_dir_sim_and_the_oracle() { let f = &w.f; let team = f.teams[0]; let role = f.roles[3]; - let removed = *f.direct.iter().find(|&&(_, g)| f.kind[g as usize] == Kind::Composite).unwrap(); + let removed = *f + .direct + .iter() + .find(|&&(_, g)| f.kind[g as usize] == Kind::Composite) + .unwrap(); // A nesting that closes a cycle through a role and a resource, and one // that links a team (not security) into a role. let res = f.nested.iter().find(|&&(c, _)| c == role).unwrap().1; @@ -193,8 +212,14 @@ fn overlay_incremental_equals_recompile_dir_sim_and_the_oracle() { c.b.witness = w.witness(); let full = Compiled::compile(w.prepare()); for chain in CHAINS { - assert_eq!(c.up[chain as usize], full.up[chain as usize], "{chain:?} up"); - assert_eq!(c.down[chain as usize], full.down[chain as usize], "{chain:?} down"); + assert_eq!( + c.up[chain as usize], full.up[chain as usize], + "{chain:?} up" + ); + assert_eq!( + c.down[chain as usize], full.down[chain as usize], + "{chain:?} down" + ); } assert_eq!(c.role_sec, full.role_sec); check_all(&w, &w.prepare(), &full); @@ -226,11 +251,21 @@ fn a_stale_witness_and_an_unfocused_group_are_refused() { fn the_fixture_is_not_vacuous() { let w = world(); let f = &w.f; - let cyclic = f.nested.iter().filter(|&&(a, b)| f.nested.binary_search(&(b, a)).is_ok()).count(); + let cyclic = f + .nested + .iter() + .filter(|&&(a, b)| f.nested.binary_search(&(b, a)).is_ok()) + .count(); assert!(cyclic >= 2, "the nesting has cycles"); - assert!(f.large_groups() >= 1, "some group exceeds 256 direct members"); + assert!( + f.large_groups() >= 1, + "some group exceeds 256 direct members" + ); let c = Compiled::compile(w.prepare()); - assert!(c.passes[Chain::All as usize].0 > 1, "the closure needs more than one pass"); + assert!( + c.passes[Chain::All as usize].0 > 1, + "the closure needs more than one pass" + ); let v = w.view(); let ord = Ordinals::of(&w.f, &v); let gw = words(v.groups_len()); @@ -252,10 +287,22 @@ fn the_fixture_is_not_vacuous() { // Each chain removes something for many users and empties it for few: // a filter that changed nothing, or kept nothing, would not test the // chain semantics. - assert!(sec_differs * 10 > n, "the SID chain filters: {sec_differs}/{n}"); - assert!(sec_some * 10 > n * 9, "the SID chain keeps groups: {sec_some}/{n}"); - assert!(mail_differs * 10 > n, "the mail chain filters: {mail_differs}/{n}"); - assert!(mail_some * 10 > n * 9, "the mail chain keeps groups: {mail_some}/{n}"); + assert!( + sec_differs * 10 > n, + "the SID chain filters: {sec_differs}/{n}" + ); + assert!( + sec_some * 10 > n * 9, + "the SID chain keeps groups: {sec_some}/{n}" + ); + assert!( + mail_differs * 10 > n, + "the mail chain filters: {mail_differs}/{n}" + ); + assert!( + mail_some * 10 > n * 9, + "the mail chain keeps groups: {mail_some}/{n}" + ); assert!(roles * 2 > n, "most users hold a role: {roles}/{n}"); let t = templates(f); let d = oracle::run(f, &w.direct, &w.nested, &t); From af1bb2864e89c148eb913ea25a0ab56a16f279cb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 22:09:49 +0000 Subject: [PATCH 3/7] dir-rmap-probe: per-chain focus, split preparation, refresh, refusals, composite vs conjunction fixture F now binds one chain (all/security/mail) and projects the complete B+ closure through down rows; a focus-local reference projection is kept for the equality test. Preparation is split (slots, projection, eager or lazy per-user rows), refresh is incremental after membership edits and nesting pairs, and a focus refuses a stale B+, >256 groups and duplicates. New tests: per-chain F vs A and D, two-hop paths through unfocused groups, refreshed focus vs fresh bind after an overlay with a cycle, composite membership vs policy conjunction (composites=0 fixture). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- crates/dir-rmap-probe/src/fixture.rs | 18 ++ crates/dir-rmap-probe/src/focus.rs | 272 +++++++++++++--- crates/dir-rmap-probe/src/lib.rs | 44 +++ crates/dir-rmap-probe/src/prep.rs | 4 + crates/dir-rmap-probe/tests/equivalence.rs | 355 ++++++++++++++++++++- 5 files changed, 652 insertions(+), 41 deletions(-) diff --git a/crates/dir-rmap-probe/src/fixture.rs b/crates/dir-rmap-probe/src/fixture.rs index 10e1abb65..fe89bbfca 100644 --- a/crates/dir-rmap-probe/src/fixture.rs +++ b/crates/dir-rmap-probe/src/fixture.rs @@ -77,6 +77,24 @@ impl Spec { } } + /// [`Spec::iam`] with the role, resource and composite counts multiplied + /// by `factor`: the group-count sweep that pushes the B+ closure past L2. + pub fn iam_scaled(users: usize, factor: usize) -> Self { + let mut s = Spec::iam(users); + s.roles *= factor; + s.resources *= factor; + s.composites *= factor; + s + } + + /// The same shape with no composite groups: every user holds its role + /// and its location directly, and role x location exists only as a + /// question (the policy-conjunction fixture). + pub fn without_composites(mut self) -> Self { + self.composites = 0; + self + } + /// A few hundred users and groups: for the exhaustive equivalence tests. pub fn small() -> Self { Spec { diff --git a/crates/dir-rmap-probe/src/focus.rs b/crates/dir-rmap-probe/src/focus.rs index f6df42f70..83c1ce7ea 100644 --- a/crates/dir-rmap-probe/src/focus.rs +++ b/crates/dir-rmap-probe/src/focus.rs @@ -1,28 +1,56 @@ //! Arm F: a 256-slot focus over selected group identities, compiled over B+. //! -//! The focus binds up to 256 group ordinals to local slots once. Every -//! group's B+ closure row (every group, upward) is then projected onto the -//! focus slots: 4 words per group instead of `ceil(groups / 64)`. A user's -//! position in the focus is the OR of its direct groups' projected rows, and -//! a role x location question is a 4-word mask test. +//! A focus binds up to 256 group ordinals to local slots for ONE chain +//! (every group, the security chain or the mail chain). It then projects the +//! complete B+ closure of that chain onto the slots: 4 words per group, +//! instead of `ceil(groups / 64)`. A user's position in the focus is the OR +//! of its direct groups' projected rows, and a role x location question is a +//! 4-word mask test. //! -//! Two variants: +//! **Completeness.** The projection reads the B+ closure, never a closure +//! restricted to the focus. A path that runs through groups outside the +//! focus is therefore represented: `x -> h -> f` with `h` unfocused still +//! sets `f`'s slot in `x`'s row. The test +//! `a_path_through_unfocused_groups_is_represented` checks this, and its +//! disable run (a focus-local closure) goes red. A question F cannot answer +//! completely is refused, never truncated: //! -//! * [`Focus`]: per-group projected rows only. Each evaluation ORs the -//! user's direct groups' rows (4 words each). -//! * [`FocusUsers`]: additionally one 4-word row per user, computed once. -//! This is the per-user pointer cache the proposal asked to keep out -//! unless measured; it is here so the measurement exists. +//! * a question naming a group outside the focus: [`Refusal::OutOfFocus`]; +//! * a focus or B+ built against another version: [`Refusal::Stale`]; +//! * more than 256 groups, or a group named twice: [`Refusal::TooLarge`], +//! [`Refusal::Duplicate`]. +//! +//! **Preparation is split** so each phase is timed and counted alone: +//! +//! 1. [`Focus::slots`]: slot -> ordinal and the reverse map (focus +//! construction); +//! 2. [`Focus::project`]: every group's closure row onto the slots (closure +//! projection). It walks each focus group's DOWNWARD closure row and sets +//! that slot in every group the row names, so the cost is the size of the +//! focus groups' closures, not `groups x 256`. `project_by_test` is the +//! reference (test every slot of every group's upward row); a test checks +//! the two agree; +//! 3. per-user rows: [`FocusUsers`] (eager, every user at once) or +//! [`FocusLazy`] (a user's row on first use). +//! +//! **Refresh.** After a nesting pair is added to B+ ([`Compiled::add_nesting`]), +//! [`Focus::refresh_nesting`] re-projects the child's downward closure (the +//! only groups whose upward rows can have changed). After membership edits +//! the projected rows are unchanged; only the edited users' rows are +//! recomputed ([`FocusUsers::refresh_users`], [`FocusLazy::refresh_users`]). +//! Every refresh takes its witness from the B+ it read, so a focus refreshed +//! against a stale B+ keeps refusing. //! //! No per-user 256 x 256 tile is allocated. The projection reads closure -//! rows at the focus's bound ordinals: an address translation, done once in -//! preparation and counted. Its admission is `ISS-ADMISSION-PREP-BINDING`. +//! rows at the focus's bound ordinals: an address translation done in +//! preparation and counted. Its admission is `ISS-ADMISSION-PREP-BINDING`, +//! still open. //! //! Reverse lookup (group ordinal -> slot) is a direct `u16` array over the //! group ordinal space (slot + 1, 0 = not bound): one indexed read, no hash, -//! no search. At 1,500 groups that is 3 KB. +//! no search. -use crate::prep::{Compiled, Refusal, Witness, bit}; +use crate::prep::{Chain, Compiled, Refusal, Witness, bit, ones}; pub const SLOTS: usize = 256; pub type Row = [u64; 4]; @@ -41,40 +69,105 @@ fn set(r: &mut Row, s: usize) { pub struct Focus { pub witness: Witness, + pub chain: Chain, /// Slot -> group ordinal. pub ids: Vec, /// Group ordinal -> slot + 1; 0 = not bound. slot_of: Vec, - /// Group ordinal -> its closure row (every group, upward) on the slots. + /// Group ordinal -> its closure row through `chain`, on the slots. proj: Vec, } impl Focus { - /// Bind `groups` (distinct, at most 256) and project every group's - /// closure row onto them. - pub fn bind(c: &Compiled, groups: &[u16]) -> Focus { - assert!(groups.len() <= SLOTS, "a focus binds at most 256 groups"); - let n = c.b.groups; - let mut slot_of = vec![0u16; n]; + /// Phase 1: bind `groups` to slots. Refuses a stale B+, more than 256 + /// groups, and a group named twice. No projection yet. + pub fn slots( + c: &Compiled, + chain: Chain, + groups: &[u16], + current: &Witness, + ) -> Result { + c.b.check(current)?; + if groups.len() > SLOTS { + return Err(Refusal::TooLarge); + } + let mut slot_of = vec![0u16; c.b.groups]; for (s, &g) in groups.iter().enumerate() { - assert_eq!(slot_of[g as usize], 0, "focus groups must be distinct"); - slot_of[g as usize] = s as u16 + 1; + let e = &mut slot_of[g as usize]; + if *e != 0 { + return Err(Refusal::Duplicate); + } + *e = s as u16 + 1; } - let mut proj = vec![[0u64; 4]; n]; - for (g, row) in proj.iter_mut().enumerate() { - let up = c.row(crate::prep::Chain::All, g); - for (s, &f) in groups.iter().enumerate() { + Ok(Focus { + witness: c.b.witness, + chain, + ids: groups.to_vec(), + slot_of, + proj: Vec::new(), + }) + } + + /// Phase 2: project the complete closure of `self.chain` onto the slots. + pub fn project(&mut self, c: &Compiled) { + self.proj = vec![[0u64; 4]; c.b.groups]; + for s in 0..self.ids.len() { + let f = self.ids[s] as usize; + for g in ones(c.down_row(self.chain, f)) { + set(&mut self.proj[g], s); + } + } + self.witness = c.b.witness; + } + + /// The reference projection: test every slot in every group's upward + /// closure row. Same answer as [`Focus::project`], `groups x slots` work. + pub fn project_by_test(&mut self, c: &Compiled) { + self.proj = vec![[0u64; 4]; c.b.groups]; + for (g, row) in self.proj.iter_mut().enumerate() { + let up = c.row(self.chain, g); + for (s, &f) in self.ids.iter().enumerate() { if bit(up, f as usize) { set(row, s); } } } - Focus { - witness: c.b.witness, - ids: groups.to_vec(), - slot_of, - proj, + self.witness = c.b.witness; + } + + /// Phases 1 and 2. + pub fn bind( + c: &Compiled, + chain: Chain, + groups: &[u16], + current: &Witness, + ) -> Result { + let mut f = Focus::slots(c, chain, groups, current)?; + f.project(c); + Ok(f) + } + + /// After `c.add_nesting(child, _)`: re-project the groups whose upward + /// closure can have changed, which are exactly `child`'s downward + /// closure in this chain (empty when the child does not pass it). + pub fn refresh_nesting(&mut self, c: &Compiled, child: u16) { + for g in ones(c.down_row(self.chain, child as usize)) { + let up = c.row(self.chain, g); + let mut row = [0u64; 4]; + for (s, &f) in self.ids.iter().enumerate() { + if bit(up, f as usize) { + set(&mut row, s); + } + } + self.proj[g] = row; } + self.witness = c.b.witness; + } + + /// After membership edits only: the projected rows do not depend on + /// direct membership, so only the witness moves (taken from `c`). + pub fn refresh_membership(&mut self, c: &Compiled) { + self.witness = c.b.witness; } pub fn check(&self, current: &Witness) -> Result<(), Refusal> { @@ -104,6 +197,7 @@ impl Focus { } /// The user's focus row: OR of its direct groups' projected rows. + /// Unchecked hot path: the caller has checked the witness. #[inline] pub fn row_of(&self, c: &Compiled, u: u16) -> Row { let mut r = [0u64; 4]; @@ -113,6 +207,28 @@ impl Focus { r } + /// The projected row of group `g` (tests compare projections). + pub fn projected(&self, g: usize) -> Row { + self.proj[g] + } + + /// A checked question: is `u` in every one of `groups` through the + /// focus's chain? Refuses rather than answers from a stale or partial + /// focus. + pub fn ask( + &self, + c: &Compiled, + current: &Witness, + u: u16, + groups: &[u16], + ) -> Result { + self.check(current)?; + c.b.check(current)?; + let m = self.all_of(groups)?; + Ok(holds(&self.row_of(c, u), &m)) + } + + /// Bytes retained: slot list, reverse map, projected rows. pub fn bytes(&self) -> usize { self.ids.len() * 2 + self.slot_of.len() * 2 + self.proj.len() * 32 } @@ -123,20 +239,104 @@ pub fn holds(row: &Row, mask: &Row) -> bool { (0..4).all(|k| row[k] & mask[k] == mask[k]) } -/// [`Focus`] plus one precomputed focus row per user. +/// [`Focus`] plus one precomputed focus row per user (eager). pub struct FocusUsers { pub focus: Focus, pub rows: Vec, } impl FocusUsers { - pub fn bind(c: &Compiled, groups: &[u16]) -> FocusUsers { - let focus = Focus::bind(c, groups); + pub fn from_focus(c: &Compiled, focus: Focus) -> FocusUsers { let rows = (0..c.b.users as u16).map(|u| focus.row_of(c, u)).collect(); FocusUsers { focus, rows } } + pub fn bind( + c: &Compiled, + chain: Chain, + groups: &[u16], + current: &Witness, + ) -> Result { + Ok(FocusUsers::from_focus( + c, + Focus::bind(c, chain, groups, current)?, + )) + } + + /// Recompute the rows of `users` (after their direct memberships + /// changed, or after `refresh_nesting` changed projected rows they read). + pub fn refresh_users(&mut self, c: &Compiled, users: &[u16]) { + for &u in users { + self.rows[u as usize] = self.focus.row_of(c, u); + } + } + + /// After `c.add_nesting(child, _)`: re-project, then recompute the rows + /// of the users that read a re-projected row (direct members of the + /// child's downward closure). + pub fn refresh_nesting(&mut self, c: &Compiled, child: u16) { + self.focus.refresh_nesting(c, child); + let mut users = Vec::new(); + for g in ones(c.down_row(self.focus.chain, child as usize)) { + users.extend_from_slice(c.b.group_users.row(g)); + } + self.refresh_users(c, &users); + } + pub fn bytes(&self) -> usize { self.focus.bytes() + self.rows.len() * 32 } } + +/// [`Focus`] plus per-user rows computed on first use (lazy). `have` marks +/// the rows that are filled; a refresh clears the marks of the users it +/// names instead of recomputing them. +pub struct FocusLazy { + pub focus: Focus, + rows: Vec, + have: Vec, + pub filled: usize, +} + +impl FocusLazy { + pub fn from_focus(c: &Compiled, focus: Focus) -> FocusLazy { + FocusLazy { + focus, + rows: vec![[0; 4]; c.b.users], + have: vec![0; c.b.users.div_ceil(64)], + filled: 0, + } + } + + #[inline] + pub fn row(&mut self, c: &Compiled, u: u16) -> Row { + let i = u as usize; + if self.have[i >> 6] >> (i & 63) & 1 == 0 { + self.rows[i] = self.focus.row_of(c, u); + self.have[i >> 6] |= 1 << (i & 63); + self.filled += 1; + } + self.rows[i] + } + + pub fn refresh_users(&mut self, users: &[u16]) { + for &u in users { + let i = u as usize; + if self.have[i >> 6] >> (i & 63) & 1 == 1 { + self.have[i >> 6] &= !(1 << (i & 63)); + self.filled -= 1; + } + } + } + + /// Drop every cached row (after a nesting refresh, any user may read a + /// changed projected row). + pub fn invalidate(&mut self) { + self.have.fill(0); + self.filled = 0; + } + + pub fn bytes(&self) -> usize { + self.focus.bytes() + self.rows.len() * 32 + self.have.len() * 8 + } +} diff --git a/crates/dir-rmap-probe/src/lib.rs b/crates/dir-rmap-probe/src/lib.rs index 2a57813a6..8513ceeb1 100644 --- a/crates/dir-rmap-probe/src/lib.rs +++ b/crates/dir-rmap-probe/src/lib.rs @@ -67,6 +67,50 @@ pub fn templates(f: &Fixture) -> Vec<(u32, u32)> { .collect() } +/// Policy conjunctions, in fixture ids: the distinct `(role, location)` +/// pairs users hold, directly or through a composite, in first-seen user +/// order. A conjunction is a question asked of the closure ("in R and in +/// L"); it is never stored as a membership. On a fixture without composites +/// this is the only form role x location takes. +pub fn conjunctions(f: &Fixture) -> Vec<(u32, u32)> { + let mut parents: std::collections::HashMap, Option)> = + Default::default(); + for &(c, p) in &f.nested { + if f.kind[c as usize] == Kind::Composite { + let e = parents.entry(c).or_default(); + match f.kind[p as usize] { + Kind::Role => e.0 = Some(p), + Kind::Location => e.1 = Some(p), + _ => {} + } + } + } + let mut held: Vec<(Option, Option)> = vec![(None, None); f.users.len()]; + for &(u, g) in &f.direct { + let h = &mut held[u as usize]; + match f.kind[g as usize] { + Kind::Role => h.0 = Some(g), + Kind::Location => h.1 = Some(g), + Kind::Composite => *h = parents[&g], + _ => {} + } + } + let mut seen = std::collections::HashSet::new(); + held.iter() + .filter_map(|&(r, l)| Some((r?, l?))) + .filter(|p| seen.insert(*p)) + .collect() +} + +/// Materialized composite groups, in fixture ids: "is the user in this +/// composite" is one membership bit, a different question from the +/// conjunction of its role and location. +pub fn composite_groups(f: &Fixture) -> Vec { + (0..f.groups.len() as u32) + .filter(|&g| f.kind[g as usize] == Kind::Composite) + .collect() +} + /// The longest prefix of `templates` whose groups fit in `k` slots, and /// those groups (first-seen order). pub fn focus_for(templates: &[(u32, u32)], k: usize) -> (Vec<(u32, u32)>, Vec) { diff --git a/crates/dir-rmap-probe/src/prep.rs b/crates/dir-rmap-probe/src/prep.rs index 8c853cb53..cc60fa53b 100644 --- a/crates/dir-rmap-probe/src/prep.rs +++ b/crates/dir-rmap-probe/src/prep.rs @@ -80,6 +80,10 @@ pub enum Refusal { Stale, /// The question names a group outside the focus. OutOfFocus, + /// A focus binds at most 256 groups. + TooLarge, + /// A focus names a group twice. + Duplicate, } /// A compressed row table: `row(i)` is sorted. diff --git a/crates/dir-rmap-probe/tests/equivalence.rs b/crates/dir-rmap-probe/tests/equivalence.rs index 0bc3a2aa7..726c0bb47 100644 --- a/crates/dir-rmap-probe/tests/equivalence.rs +++ b/crates/dir-rmap-probe/tests/equivalence.rs @@ -4,9 +4,11 @@ //! the comparisons meaningful. use dir_rmap_probe::fixture::{Fixture, Kind, Spec}; -use dir_rmap_probe::focus::{Focus, FocusUsers, holds}; +use dir_rmap_probe::focus::{Focus, FocusLazy, FocusUsers, SLOTS, holds}; use dir_rmap_probe::prep::{CHAINS, Chain, Compiled, Ordinals, Prepared, Refusal, ones, words}; -use dir_rmap_probe::{Edit, World, a_groups, a_members, focus_for, oracle, templates}; +use dir_rmap_probe::{ + Edit, World, a_groups, a_members, composite_groups, conjunctions, focus_for, oracle, templates, +}; use ogar_dir_core::Guid128; use std::collections::{BTreeSet, HashMap}; @@ -152,8 +154,8 @@ fn the_focus_answers_role_and_location_like_the_oracle_and_dir_sim() { let (ts, gs) = focus_for(&all, k); assert!(!ts.is_empty() && gs.len() <= k); let groups: Vec = gs.iter().map(|&g| ord.group[g as usize]).collect(); - let f = Focus::bind(&c, &groups); - let fu = FocusUsers::bind(&c, &groups); + let f = Focus::bind(&c, Chain::All, &groups, &w.witness()).unwrap(); + let fu = FocusUsers::bind(&c, Chain::All, &groups, &w.witness()).unwrap(); for (i, ug) in w.f.users.iter().enumerate() { let a = a_groups(&v, ug, Chain::All); let row = f.row_of(&c, ord.user[i]); @@ -237,7 +239,13 @@ fn a_stale_witness_and_an_unfocused_group_are_refused() { let c = Compiled::compile(w.prepare()); let v = w.view(); let ord = Ordinals::of(&w.f, &v); - let focus = Focus::bind(&c, &[ord.group[w.f.roles[0] as usize]]); + let focus = Focus::bind( + &c, + Chain::All, + &[ord.group[w.f.roles[0] as usize]], + &w.witness(), + ) + .unwrap(); assert_eq!(focus.check(&w.witness()), Ok(())); assert!(focus.all_of(&[ord.group[w.f.roles[0] as usize]]).is_ok()); assert_eq!( @@ -313,3 +321,340 @@ fn the_fixture_is_not_vacuous() { d.eligible.len() ); } + +/// `SLOTS` group ordinals spread over the whole ordinal range. +fn spread(groups: usize) -> Vec { + let step = groups.div_ceil(SLOTS).max(1); + (0..groups) + .step_by(step) + .take(SLOTS) + .map(|g| g as u16) + .collect() +} + +/// For every chain, a focus over 256 spread groups answers every +/// (user, slot) pair like dir-sim's own chain walk and the oracle, through +/// the eager and the lazy per-user rows alike; and the down-row projection +/// equals the reference projection. +#[test] +fn a_focus_on_every_chain_equals_dir_sim_and_the_oracle() { + let w = world(); + let v = w.view(); + let ord = Ordinals::of(&w.f, &v); + let c = Compiled::compile(w.prepare()); + let d = oracle::run(&w.f, &w.direct, &w.nested, &templates(&w.f)); + let ids = spread(v.groups_len()); + // Fixture id of every group ordinal, to read the oracle. + let mut fid = vec![0u32; v.groups_len()]; + for (j, &o) in ord.group.iter().enumerate() { + fid[o as usize] = j as u32; + } + let mut yes = [0usize; 3]; + for chain in CHAINS { + let f = Focus::bind(&c, chain, &ids, &w.witness()).unwrap(); + let mut reference = Focus::slots(&c, chain, &ids, &w.witness()).unwrap(); + reference.project_by_test(&c); + for g in 0..v.groups_len() { + assert_eq!( + f.projected(g), + reference.projected(g), + "{chain:?} group {g}" + ); + } + let fu = FocusUsers::bind(&c, chain, &ids, &w.witness()).unwrap(); + let mut fl = FocusLazy::from_focus(&c, Focus::bind(&c, chain, &ids, &w.witness()).unwrap()); + let rel = oracle_rel(&d, chain); + for (i, ug) in w.f.users.iter().enumerate() { + let a = a_groups(&v, ug, chain); + let u = ord.user[i]; + let row = f.row_of(&c, u); + assert_eq!(fu.rows[u as usize], row, "eager row"); + assert_eq!(fl.row(&c, u), row, "lazy row"); + for (s, &g) in ids.iter().enumerate() { + let m = f.all_of(&[g]).unwrap(); + let want = a.contains(&v.group_guid(lance_graph_dir_sim::GroupOrdinal(g)).unwrap()); + assert_eq!(holds(&row, &m), want, "F {chain:?} user {i} slot {s}"); + assert_eq!( + rel.contains(&(i as u32, fid[g as usize])), + want, + "oracle {chain:?}" + ); + yes[chain as usize] += usize::from(want); + } + } + assert_eq!(fl.filled, w.f.users.len()); + } + // Anti-vacuity: every chain has positive answers, and the filtered + // chains answer fewer of them than the unfiltered one. + assert!(yes.iter().all(|&y| y > 1_000), "{yes:?}"); + assert!(yes[1] < yes[0] && yes[2] < yes[0], "{yes:?}"); +} + +/// A path that runs through groups outside the focus is represented: for +/// users whose membership in a focus group needs at least two nesting hops +/// (the group is neither a direct group of the user nor a direct parent of +/// one), the focus holding only that group still answers yes. A +/// focus-local closure would answer no. +#[test] +fn a_path_through_unfocused_groups_is_represented() { + let w = world(); + let v = w.view(); + let ord = Ordinals::of(&w.f, &v); + let c = Compiled::compile(w.prepare()); + let gw = words(v.groups_len()); + let mut row = vec![0u64; gw]; + let mut checked = [0usize; 3]; + for chain in CHAINS { + for i in (0..w.f.users.len()).step_by(7) { + let u = ord.user[i]; + let direct = c.b.user_groups.row(u as usize); + c.groups_of(u, chain, &mut row); + let far: Vec = ones(&row) + .map(|g| g as u16) + .filter(|g| !direct.contains(g)) + .filter(|&g| !direct.iter().any(|&d| c.b.up.row(d as usize).contains(&g))) + .take(4) + .collect(); + for g in far { + let f = Focus::bind(&c, chain, &[g], &w.witness()).unwrap(); + assert_eq!( + f.ask(&c, &w.witness(), u, &[g]), + Ok(true), + "{chain:?} user {i} group {g}" + ); + checked[chain as usize] += 1; + } + } + } + assert!( + checked.iter().all(|&n| n >= 20), + "two-hop cases per chain: {checked:?}" + ); +} + +/// After an overlay (membership adds and removals, a nesting pair that +/// closes a cycle, a team nested in a role), a focus refreshed +/// incrementally equals a fresh bind, dir-sim and the oracle, on every +/// chain, eager and lazy rows alike. Before the refresh it refuses. +#[test] +fn a_refreshed_focus_equals_a_fresh_bind_dir_sim_and_the_oracle() { + let mut w = world(); + let v0 = w.view(); + let ord = Ordinals::of(&w.f, &v0); + drop(v0); + let mut c = Compiled::compile(w.prepare()); + let ids = spread(c.b.groups); + let w0 = w.witness(); + let mut fs: Vec = CHAINS + .iter() + .map(|&ch| FocusUsers::bind(&c, ch, &ids, &w0).unwrap()) + .collect(); + let mut fl: Vec = CHAINS + .iter() + .map(|&ch| FocusLazy::from_focus(&c, Focus::bind(&c, ch, &ids, &w0).unwrap())) + .collect(); + for (k, l) in fl.iter_mut().enumerate() { + for u in 0..c.b.users as u16 { + let _ = l.row(&c, u); + } + assert_eq!(l.filled, c.b.users, "chain {k} warmed"); + } + let f = &w.f; + let role = f.roles[3]; + let res = f.nested.iter().find(|&&(c, _)| c == role).unwrap().1; + let removed = *f + .direct + .iter() + .find(|&&(_, g)| f.kind[g as usize] == Kind::Composite) + .unwrap(); + let edits = [ + Edit::AddDirect(5, f.teams[0]), + Edit::AddDirect(7, role), + Edit::RemoveDirect(removed.0, removed.1), + Edit::AddNested(res, role), + Edit::AddNested(f.teams[0], f.roles[9]), + ]; + w.apply(&edits); + let w1 = w.witness(); + for fu in &fs { + assert_eq!( + fu.focus.check(&w1), + Err(Refusal::Stale), + "refuses before refresh" + ); + } + // Incremental: membership CSRs, then each nesting pair. + c.b.set_direct(&ord, &w.direct); + c.b.witness = w1; + let edited: Vec = edits + .iter() + .filter_map(|e| match *e { + Edit::AddDirect(u, _) | Edit::RemoveDirect(u, _) => Some(ord.user[u as usize]), + Edit::AddNested(..) => None, + }) + .collect(); + for (fu, l) in fs.iter_mut().zip(fl.iter_mut()) { + fu.focus.refresh_membership(&c); + fu.refresh_users(&c, &edited); + l.focus.refresh_membership(&c); + l.refresh_users(&edited); + } + for e in &edits { + if let Edit::AddNested(ch, p) = *e { + let (ch, p) = (ord.group[ch as usize], ord.group[p as usize]); + c.add_nesting(ch, p); + for (fu, l) in fs.iter_mut().zip(fl.iter_mut()) { + fu.refresh_nesting(&c, ch); + l.focus.refresh_nesting(&c, ch); + l.invalidate(); + } + } + } + let full = Compiled::compile(w.prepare()); + let v = w.view(); + let d = oracle::run(&w.f, &w.direct, &w.nested, &templates(&w.f)); + let mut fid = vec![0u32; v.groups_len()]; + for (j, &o) in ord.group.iter().enumerate() { + fid[o as usize] = j as u32; + } + for (k, &chain) in CHAINS.iter().enumerate() { + let fresh = Focus::bind(&full, chain, &ids, &w1).unwrap(); + assert_eq!(fs[k].focus.check(&w1), Ok(())); + for g in 0..v.groups_len() { + assert_eq!( + fs[k].focus.projected(g), + fresh.projected(g), + "{chain:?} group {g}" + ); + } + let rel = oracle_rel(&d, chain); + for (i, ug) in w.f.users.iter().enumerate() { + let u = ord.user[i]; + let row = fresh.row_of(&full, u); + assert_eq!(fs[k].rows[u as usize], row, "eager {chain:?} user {i}"); + assert_eq!(fl[k].row(&c, u), row, "lazy {chain:?} user {i}"); + let a = a_groups(&v, ug, chain); + for &g in &ids { + let want = a.contains(&v.group_guid(lance_graph_dir_sim::GroupOrdinal(g)).unwrap()); + assert_eq!(holds(&row, &fresh.all_of(&[g]).unwrap()), want); + assert_eq!(rel.contains(&(i as u32, fid[g as usize])), want); + } + } + } +} + +/// The two forms of role x location stay distinct questions. On the +/// composite fixture, "is the user in composite C" is one membership bit; +/// on the conjunction fixture (no composites), "in role R and in location +/// L" is a mask test of two bits. Each agrees with dir-sim and the oracle, +/// and neither creates a membership the other fixture does not have. +#[test] +fn composite_membership_and_policy_conjunction_are_distinct_questions() { + // Composite fixture: one bit per composite. + let w = world(); + let v = w.view(); + let ord = Ordinals::of(&w.f, &v); + let c = Compiled::compile(w.prepare()); + let comps = composite_groups(&w.f); + assert!(comps.len() >= 20); + let ids: Vec = comps + .iter() + .take(SLOTS) + .map(|&g| ord.group[g as usize]) + .collect(); + let f = Focus::bind(&c, Chain::All, &ids, &w.witness()).unwrap(); + let d = oracle::run(&w.f, &w.direct, &w.nested, &[]); + let mut yes = 0; + for (i, ug) in w.f.users.iter().enumerate() { + let a = a_groups(&v, ug, Chain::All); + let row = f.row_of(&c, ord.user[i]); + for (&cg, &o) in comps.iter().zip(&ids) { + let want = a.contains(&w.f.groups[cg as usize]); + assert_eq!(holds(&row, &f.all_of(&[o]).unwrap()), want); + assert_eq!(d.member.contains(&(i as u32, cg)), want); + yes += usize::from(want); + } + } + let total = w.f.users.len() * ids.len(); + assert!( + yes > 0 && yes * 4 < total, + "composite membership: {yes}/{total}" + ); + + // Conjunction fixture: no composite exists; the question is two bits. + let wc = World::new(Fixture::generate(Spec::small().without_composites())); + assert!(composite_groups(&wc.f).is_empty()); + let vc = wc.view(); + let ordc = Ordinals::of(&wc.f, &vc); + let cc = Compiled::compile(wc.prepare()); + let all = conjunctions(&wc.f); + let (ts, gs) = focus_for(&all, SLOTS); + assert!(ts.len() >= 20, "conjunction questions: {}", ts.len()); + let fc = Focus::bind( + &cc, + Chain::All, + &gs.iter() + .map(|&g| ordc.group[g as usize]) + .collect::>(), + &wc.witness(), + ) + .unwrap(); + let dc = oracle::run(&wc.f, &wc.direct, &wc.nested, &ts); + let mut yes = 0; + for (i, ug) in wc.f.users.iter().enumerate() { + let a = a_groups(&vc, ug, Chain::All); + let row = fc.row_of(&cc, ordc.user[i]); + for &(r, l) in &ts { + let m = fc + .all_of(&[ordc.group[r as usize], ordc.group[l as usize]]) + .unwrap(); + let want = a.contains(&wc.f.groups[r as usize]) && a.contains(&wc.f.groups[l as usize]); + assert_eq!(holds(&row, &m), want); + assert_eq!(dc.eligible.contains(&(i as u32, r, l)), want); + yes += usize::from(want); + } + } + let total = wc.f.users.len() * ts.len(); + assert!(yes > 0 && yes * 4 < total, "conjunctions: {yes}/{total}"); + // The conjunction fixture's memberships are exactly its direct and + // nesting edges: the conjunction question added none. + assert_eq!(dc.member.len(), { + let gw = words(vc.groups_len()); + let mut p = vec![0u64; gw]; + (0..wc.f.users.len()) + .map(|i| { + cc.groups_of(ordc.user[i], Chain::All, &mut p); + p.iter().map(|w| w.count_ones() as usize).sum::() + }) + .sum::() + }); +} + +/// A focus refuses what it cannot answer completely: more than 256 groups, +/// a duplicate, a B+ from another version, and (through `ask`) a stale +/// focus or an unfocused group. +#[test] +fn a_focus_refuses_what_it_cannot_answer_completely() { + let mut w = world(); + let c = Compiled::compile(w.prepare()); + let w0 = w.witness(); + let many: Vec = (0..=SLOTS as u16).collect(); + assert_eq!( + Focus::bind(&c, Chain::All, &many, &w0).err(), + Some(Refusal::TooLarge) + ); + assert_eq!( + Focus::bind(&c, Chain::All, &[3, 4, 3], &w0).err(), + Some(Refusal::Duplicate) + ); + let f = Focus::bind(&c, Chain::Security, &[3, 4], &w0).unwrap(); + assert!(f.ask(&c, &w0, 0, &[3]).is_ok()); + assert_eq!(f.ask(&c, &w0, 0, &[5]), Err(Refusal::OutOfFocus)); + w.apply(&[Edit::AddDirect(11, w.f.teams[1])]); + let w1 = w.witness(); + assert_eq!( + Focus::bind(&c, Chain::All, &[3], &w1).err(), + Some(Refusal::Stale) + ); + assert_eq!(f.ask(&c, &w1, 0, &[3]), Err(Refusal::Stale)); +} From 63e990847e73842cc2047ac65b8efa81a8231c23 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 22:11:05 +0000 Subject: [PATCH 4/7] dir-rmap-probe: refuse a stale focus over a current B+ (test) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- crates/dir-rmap-probe/tests/equivalence.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/dir-rmap-probe/tests/equivalence.rs b/crates/dir-rmap-probe/tests/equivalence.rs index 726c0bb47..c8a8ddd15 100644 --- a/crates/dir-rmap-probe/tests/equivalence.rs +++ b/crates/dir-rmap-probe/tests/equivalence.rs @@ -657,4 +657,9 @@ fn a_focus_refuses_what_it_cannot_answer_completely() { Some(Refusal::Stale) ); assert_eq!(f.ask(&c, &w1, 0, &[3]), Err(Refusal::Stale)); + // A current B+ does not make an old focus current: the focus's own + // witness refuses. + let c1 = Compiled::compile(w.prepare()); + assert_eq!(c1.b.check(&w1), Ok(())); + assert_eq!(f.ask(&c1, &w1, 0, &[3]), Err(Refusal::Stale)); } From dec59037fbee7eed0a64211a6d47a6a08134b666 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 22:19:55 +0000 Subject: [PATCH 5/7] dir-rmap-probe: allocation accounting, amortization, composite vs conjunction workloads, working-set sweep Bench: counting global allocator; per-phase timing and allocations for focus construction, projection (down rows vs reference), eager and lazy per-user rows, refresh after membership edits and after a narrow and a wide nesting pair; computed and measured crossovers (T_F(N) vs T_B+(N)), a rebuild-per-question arm and a question-aware B+ (needed words only); W5a conjunction / W5b composite membership / W5c conjunction on a fixture without composites; group-count sweep against L1/L2 with bytes touched and working sets, hot/warm/cold. CI runs the sweep at 1,2. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .github/workflows/dir-rmap-probe.yml | 3 + crates/dir-rmap-probe/examples/bench.rs | 1050 +++++++++++++++++++---- 2 files changed, 880 insertions(+), 173 deletions(-) diff --git a/.github/workflows/dir-rmap-probe.yml b/.github/workflows/dir-rmap-probe.yml index ca278fea2..7dbdcbdfa 100644 --- a/.github/workflows/dir-rmap-probe.yml +++ b/.github/workflows/dir-rmap-probe.yml @@ -87,4 +87,7 @@ jobs: - name: test run: cargo test --release --manifest-path crates/dir-rmap-probe/Cargo.toml - name: benchmark (asserts equivalence at 8,000 users; timings are not gated) + env: + # The full sweep (1,2,4,8) is a local run; CI keeps the smaller two. + DIR_RMAP_SWEEP: "1,2" run: cargo run --release --manifest-path crates/dir-rmap-probe/Cargo.toml --example bench diff --git a/crates/dir-rmap-probe/examples/bench.rs b/crates/dir-rmap-probe/examples/bench.rs index 74846e64f..cedf1e2a3 100644 --- a/crates/dir-rmap-probe/examples/bench.rs +++ b/crates/dir-rmap-probe/examples/bench.rs @@ -13,9 +13,11 @@ //! unrelated memory, so the prepared structures are out of L1/L2. use dir_rmap_probe::fixture::{Fixture, Spec}; -use dir_rmap_probe::focus::{Focus, FocusUsers, holds}; -use dir_rmap_probe::prep::{Chain, Compiled, Ordinals, bit, words}; -use dir_rmap_probe::{Edit, World, a_groups, a_members, focus_for, oracle, templates}; +use dir_rmap_probe::focus::{Focus, FocusLazy, FocusUsers, Row, holds}; +use dir_rmap_probe::prep::{Chain, Compiled, Ordinals, Witness, bit, ones, words}; +use dir_rmap_probe::{ + Edit, World, a_groups, a_members, composite_groups, conjunctions, focus_for, oracle, templates, +}; use lance_graph_dir_sim::{GroupOrdinal, UserOrdinal}; use ndarray::simd::{eq_u8_to_mask, mask_and_assign, popcount_batch_u64}; use ogar_dir_core::Guid128; @@ -149,6 +151,19 @@ fn main() { t_compile.as_secs_f64() * 1e3 ); println!("| B + B+ together | {:.2} ms |", t_c.as_secs_f64() * 1e3); + let (_, a_b) = counted(|| w.prepare()); + let p = w.prepare(); + let (cc, a_comp) = counted(|| Compiled::compile(p)); + drop(cc); + println!( + "| allocations: B prepare {} ({} allocated, {} retained); B+ compile {} ({} allocated, {} retained) | |", + a_b.allocs, + kib(a_b.bytes as isize), + kib(a_b.retained), + a_comp.allocs, + kib(a_comp.bytes as isize), + kib(a_comp.retained) + ); println!( "| D: crepe oracle, all relations | {:.1} ms |", t_d.as_secs_f64() * 1e3 @@ -177,8 +192,8 @@ fn main() { ); let (_, g256) = focus_for(&all_templates, 256); let focus_ids: Vec = g256.iter().map(|&g| ord.group[g as usize]).collect(); - let f256 = Focus::bind(&c, &focus_ids); - let fu256 = FocusUsers::bind(&c, &focus_ids); + let f256 = Focus::bind(&c, Chain::All, &focus_ids, &w.witness()).unwrap(); + let fu256 = FocusUsers::bind(&c, Chain::All, &focus_ids, &w.witness()).unwrap(); println!( "| F (256 slots): projected rows + reverse map | {} |", f256.bytes() @@ -393,173 +408,8 @@ fn main() { gsample.len() ); - // ---- W5 role x location: B+ vs F --------------------------------------- - println!("## W5: role x location questions, B+ vs F\n"); - println!( - "One evaluation = one user, every template question in the focus. N = evaluations reusing one prepared focus (users cycled). F prepare = bind + projection of every group's closure row. Totals measured directly.\n" - ); - println!( - "| focus groups | questions | A eval | B+ eval | F eval | F+users eval | F prepare | F+users prepare | B+ N=1 / F / F+u | N=10 | N=100 | N=1000 | N=10000 |" - ); - println!("|---|---|---|---|---|---|---|---|---|---|---|---|---|"); - for k in [16usize, 64, 128, 256] { - let (ts, gs) = focus_for(&all_templates, k); - let groups: Vec = gs.iter().map(|&g| ord.group[g as usize]).collect(); - let tq: Vec<(usize, usize)> = ts - .iter() - .map(|&(r, l)| { - ( - ord.group[r as usize] as usize, - ord.group[l as usize] as usize, - ) - }) - .collect(); - let f = Focus::bind(&c, &groups); - let fu = FocusUsers::bind(&c, &groups); - let masks: Vec<_> = tq - .iter() - .map(|&(r, l)| f.all_of(&[r as u16, l as u16]).unwrap()) - .collect(); - let tq_guid: Vec<(Guid128, Guid128)> = ts - .iter() - .map(|&(r, l)| (w.f.groups[r as usize], w.f.groups[l as usize])) - .collect(); - // Equal answers on every user before timing. - for &i in &all_users { - let u = ord.user[i]; - c.groups_of(u, Chain::All, &mut prow); - let fr = f.row_of(&c, u); - for (q, &(r, l)) in tq.iter().enumerate() { - let want = bit(&prow, r) && bit(&prow, l); - assert_eq!(holds(&fr, &masks[q]), want); - assert_eq!(holds(&fu.rows[u as usize], &masks[q]), want); - } - } - let users_ord: Vec = all_users.iter().map(|&i| ord.user[i]).collect(); - let eval_bp = |n: usize, prow: &mut Vec| { - let mut hits = 0usize; - for e in 0..n { - let u = users_ord[e % users_ord.len()]; - c.groups_of(u, Chain::All, prow); - for &(r, l) in &tq { - hits += usize::from(bit(prow, r) && bit(prow, l)); - } - } - hits - }; - let eval_f = |f: &Focus, n: usize| { - let mut hits = 0usize; - for e in 0..n { - let u = users_ord[e % users_ord.len()]; - let r = f.row_of(&c, u); - for m in &masks { - hits += usize::from(holds(&r, m)); - } - } - hits - }; - let eval_fu = |fu: &FocusUsers, n: usize| { - let mut hits = 0usize; - for e in 0..n { - let r = &fu.rows[users_ord[e % users_ord.len()] as usize]; - for m in &masks { - hits += usize::from(holds(r, m)); - } - } - hits - }; - let m = users_ord.len(); - let t_a = time(3, || { - sample - .iter() - .map(|&i| { - let a = a_groups(&v, &w.f.users[i], Chain::All); - tq_guid - .iter() - .filter(|(r, l)| a.binary_search(r).is_ok() && a.binary_search(l).is_ok()) - .count() - }) - .sum::() - }); - let t_bp = time(9, || eval_bp(m, &mut prow)); - let t_f = time(9, || eval_f(&f, m)); - let t_fu = time(9, || eval_fu(&fu, m)); - let t_fprep = time(9, || Focus::bind(&c, &groups)); - let t_fuprep = time(5, || FocusUsers::bind(&c, &groups)); - let mut totals = Vec::new(); - for n in [1usize, 10, 100, 1_000, 10_000] { - let reps = if n <= 100 { 51 } else { 9 }; - let bp = time(reps, || eval_bp(n, &mut prow)); - let ff = time(reps, || { - let f = Focus::bind(&c, &groups); - eval_f(&f, n) - }); - let fuu = time(reps, || { - let fu = FocusUsers::bind(&c, &groups); - eval_fu(&fu, n) - }); - totals.push(format!("{:.1} / {:.1} / {:.1} µs", us(bp), us(ff), us(fuu))); - } - println!( - "| {} | {} | {:.0} ns | {:.0} ns | {:.0} ns | {:.0} ns | {:.1} µs | {:.1} µs | {} |", - gs.len(), - ts.len(), - ns(t_a, sample.len()), - ns(t_bp, m), - ns(t_f, m), - ns(t_fu, m), - us(t_fprep), - us(t_fuprep), - totals.join(" | ") - ); - } - // Cold vs warm for one evaluation, 256-slot focus. - { - let (ts, gs) = focus_for(&all_templates, 256); - let groups: Vec = gs.iter().map(|&g| ord.group[g as usize]).collect(); - let f = Focus::bind(&c, &groups); - let masks: Vec<_> = ts - .iter() - .map(|&(r, l)| { - f.all_of(&[ord.group[r as usize], ord.group[l as usize]]) - .unwrap() - }) - .collect(); - let tq: Vec<(usize, usize)> = ts - .iter() - .map(|&(r, l)| { - ( - ord.group[r as usize] as usize, - ord.group[l as usize] as usize, - ) - }) - .collect(); - let mut cold_bp = Vec::new(); - let mut cold_f = Vec::new(); - for e in 0..41 { - let u = ord.user[(e * 197) % w.f.users.len()]; - evict(&mut evict_buf); - let s = Instant::now(); - c.groups_of(u, Chain::All, &mut prow); - black_box( - tq.iter() - .filter(|&&(r, l)| bit(&prow, r) && bit(&prow, l)) - .count(), - ); - cold_bp.push(s.elapsed()); - evict(&mut evict_buf); - let s = Instant::now(); - let r = f.row_of(&c, u); - black_box(masks.iter().filter(|m| holds(&r, m)).count()); - cold_f.push(s.elapsed()); - } - println!( - "\nCold single evaluation (256-slot focus, {} questions, after evicting 64 MiB): B+ {:.0} ns, F {:.0} ns.\n", - ts.len(), - us(median(cold_bp)) * 1e3, - us(median(cold_f)) * 1e3 - ); - } + w5_all(&w, &c, &ord, &mut evict_buf, &sample); + sweep(users, &mut evict_buf); // ---- W6 team membership x attribute ----------------------------------- let mut lane = vec![0u8; n_users]; @@ -724,7 +574,12 @@ fn main() { } } let t_rebind = time(9, || { - Focus::bind(&full1, &focus_ids_for(&full1, &all_templates, &ord1)) + Focus::bind( + &full1, + Chain::All, + &focus_ids_for(&full1, &all_templates, &ord1), + &w.witness(), + ) }); println!("## W7/W8: overlay ({membership_edits} membership edits + 1 nesting edge)\n"); println!("| step | time |\n|---|---|"); @@ -797,3 +652,852 @@ fn focus_ids_for(c: &Compiled, templates: &[(u32, u32)], ord: &Ordinals) -> Vec< let (_, gs) = focus_for(templates, 256); gs.iter().map(|&g| ord.group[g as usize]).collect() } + +// --------------------------------------------------------------------------- +// Allocation accounting: a counting global allocator. Every allocation the +// process makes goes through it; `counted` reports the allocations, bytes +// allocated and bytes still live (retained) across one closure. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicIsize, AtomicUsize, Ordering::Relaxed}; + +struct Counting; +static ALLOCS: AtomicUsize = AtomicUsize::new(0); +static BYTES: AtomicUsize = AtomicUsize::new(0); +static LIVE: AtomicIsize = AtomicIsize::new(0); + +// SAFETY: every method forwards to `System` with the caller's arguments +// unchanged; the counters are side effects only. +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + ALLOCS.fetch_add(1, Relaxed); + BYTES.fetch_add(l.size(), Relaxed); + LIVE.fetch_add(l.size() as isize, Relaxed); + // SAFETY: forwarded unchanged; the caller upholds `alloc`'s contract. + unsafe { System.alloc(l) } + } + unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { + ALLOCS.fetch_add(1, Relaxed); + BYTES.fetch_add(l.size(), Relaxed); + LIVE.fetch_add(l.size() as isize, Relaxed); + // SAFETY: forwarded unchanged. + unsafe { System.alloc_zeroed(l) } + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + LIVE.fetch_sub(l.size() as isize, Relaxed); + // SAFETY: `p` came from this allocator with layout `l`. + unsafe { System.dealloc(p, l) } + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { + ALLOCS.fetch_add(1, Relaxed); + BYTES.fetch_add(new, Relaxed); + LIVE.fetch_add(new as isize - l.size() as isize, Relaxed); + // SAFETY: forwarded unchanged; `p` came from this allocator. + unsafe { System.realloc(p, l, new) } + } +} + +#[global_allocator] +static GLOBAL: Counting = Counting; + +#[derive(Clone, Copy, Default)] +struct Acct { + allocs: usize, + bytes: usize, + retained: isize, +} + +/// Run `f` once, counting what it allocates; the result is kept alive while +/// `retained` is read, so retained = what the result holds. +fn counted(f: impl FnOnce() -> T) -> (T, Acct) { + let (a0, b0, l0) = ( + ALLOCS.load(Relaxed), + BYTES.load(Relaxed), + LIVE.load(Relaxed), + ); + let t = f(); + let a = Acct { + allocs: ALLOCS.load(Relaxed) - a0, + bytes: BYTES.load(Relaxed) - b0, + retained: LIVE.load(Relaxed) - l0, + }; + (t, a) +} + +fn kib(b: isize) -> String { + if b.abs() < 10_240 { + format!("{b} B") + } else { + format!("{:.1} KiB", b as f64 / 1024.0) + } +} + +// --------------------------------------------------------------------------- +// W5: questions over group ordinals. A question of one group is a +// membership bit (a materialized composite); of two, a conjunction. + +/// The longest prefix of `qs` whose groups fit in `k` slots, and those +/// groups (first-seen order). +fn take_k(qs: &[Vec], k: usize) -> (Vec>, Vec) { + let mut groups: Vec = Vec::new(); + let mut kept = Vec::new(); + for q in qs { + let new = q.iter().filter(|g| !groups.contains(g)).count(); + if groups.len() + new > k { + continue; + } + for &g in q { + if !groups.contains(&g) { + groups.push(g); + } + } + kept.push(q.clone()); + } + (kept, groups) +} + +/// B+: the user's full closure row (every group), then the question bits. +fn e_bp(c: &Compiled, u: u16, qs: &[Vec], row: &mut [u64]) -> usize { + c.groups_of(u, Chain::All, row); + qs.iter() + .filter(|q| q.iter().all(|&g| bit(row, g as usize))) + .count() +} + +/// B+ reading only the closure words that hold a question group: the global +/// closure, question-aware, with no focus. +fn e_bpw(c: &Compiled, u: u16, qs: &[Vec], wds: &[usize], acc: &mut [u64]) -> usize { + for &k in wds { + acc[k] = 0; + } + for &d in c.b.user_groups.row(u as usize) { + let r = c.row(Chain::All, d as usize); + for &k in wds { + acc[k] |= r[k]; + } + } + qs.iter() + .filter(|q| q.iter().all(|&g| bit(acc, g as usize))) + .count() +} + +/// A question compiled onto focus slots: the slots it needs set. +type Slots = Vec; + +#[inline] +fn slot_bit(r: &Row, s: u8) -> bool { + r[(s >> 6) as usize] >> (s & 63) & 1 == 1 +} + +#[inline] +fn answer(r: &Row, qs: &[Slots]) -> usize { + qs.iter() + .filter(|q| q.iter().all(|&s| slot_bit(r, s))) + .count() +} + +/// F: the user's focus row (OR of its direct groups' projected rows), then +/// one 4-word mask test per question on the 256-bit row. +fn e_f(f: &Focus, c: &Compiled, u: u16, masks: &[Row]) -> usize { + let r = f.row_of(c, u); + masks.iter().filter(|m| holds(&r, m)).count() +} + +fn e_fu(fu: &FocusUsers, u: u16, masks: &[Row]) -> usize { + let r = &fu.rows[u as usize]; + masks.iter().filter(|m| holds(r, m)).count() +} + +/// F with the questions as slot bits instead of masks: the same work per +/// question as B+'s bit tests, on the focus row. +fn e_f_bits(f: &Focus, c: &Compiled, u: u16, qs: &[Slots]) -> usize { + answer(&f.row_of(c, u), qs) +} + +fn slots_of(f: &Focus, qs: &[Vec]) -> Vec { + qs.iter() + .map(|q| q.iter().map(|&g| f.slot(g).unwrap() as u8).collect()) + .collect() +} + +/// Per-evaluation figures of one (workload, k). +struct Evals { + bp: f64, + bpw: f64, + f: f64, + fu: f64, +} + +/// Run `n` evaluations of each arm, users cycled. +#[allow(clippy::too_many_arguments)] +fn time_evals( + c: &Compiled, + users: &[u16], + qs: &[Vec], + wds: &[usize], + f: &Focus, + fu: &FocusUsers, + masks: &[Row], + n: usize, + reps: usize, +) -> (Evals, [usize; 4]) { + let mut row = vec![0u64; c.w]; + let mut acc = vec![0u64; c.w]; + let pick = |e: usize| users[e % users.len()]; + let mut allocs = [0usize; 4]; + let ((), a) = counted(|| { + for e in 0..n { + black_box(e_bp(c, pick(e), qs, &mut row)); + } + }); + allocs[0] = a.allocs; + let ((), a) = counted(|| { + for e in 0..n { + black_box(e_bpw(c, pick(e), qs, wds, &mut acc)); + } + }); + allocs[1] = a.allocs; + let ((), a) = counted(|| { + for e in 0..n { + black_box(e_f(f, c, pick(e), masks)); + } + }); + allocs[2] = a.allocs; + let ((), a) = counted(|| { + for e in 0..n { + black_box(e_fu(fu, pick(e), masks)); + } + }); + allocs[3] = a.allocs; + let bp = time(reps, || { + (0..n) + .map(|e| e_bp(c, pick(e), qs, &mut row)) + .sum::() + }); + let bpw = time(reps, || { + (0..n) + .map(|e| e_bpw(c, pick(e), qs, wds, &mut acc)) + .sum::() + }); + let tf = time(reps, || { + (0..n).map(|e| e_f(f, c, pick(e), masks)).sum::() + }); + let tfu = time(reps, || { + (0..n).map(|e| e_fu(fu, pick(e), masks)).sum::() + }); + ( + Evals { + bp: ns(bp, n), + bpw: ns(bpw, n), + f: ns(tf, n), + fu: ns(tfu, n), + }, + allocs, + ) +} + +/// Median cold time (after evicting 64 MiB) of one evaluation per arm. +#[allow(clippy::too_many_arguments)] +fn time_cold( + c: &Compiled, + users: &[u16], + qs: &[Vec], + wds: &[usize], + f: &Focus, + fu: &FocusUsers, + masks: &[Row], + evict_buf: &mut [u64], +) -> [f64; 4] { + let mut row = vec![0u64; c.w]; + let mut acc = vec![0u64; c.w]; + let mut t: [Vec; 4] = Default::default(); + for e in 0..31 { + let u = users[(e * 197) % users.len()]; + evict(evict_buf); + let s = Instant::now(); + black_box(e_bp(c, u, qs, &mut row)); + t[0].push(s.elapsed()); + evict(evict_buf); + let s = Instant::now(); + black_box(e_bpw(c, u, qs, wds, &mut acc)); + t[1].push(s.elapsed()); + evict(evict_buf); + let s = Instant::now(); + black_box(e_f(f, c, u, masks)); + t[2].push(s.elapsed()); + evict(evict_buf); + let s = Instant::now(); + black_box(e_fu(fu, u, masks)); + t[3].push(s.elapsed()); + } + t.map(|v| ns(median(v), 1)) +} + +/// Distinct closure-word indices holding a focus group. +fn words_of(groups: &[u16]) -> Vec { + let mut k: Vec = groups.iter().map(|&g| g as usize >> 6).collect(); + k.sort_unstable(); + k.dedup(); + k +} + +/// One W5 workload over `all_qs`, for each focus size. +#[allow(clippy::too_many_arguments)] +fn w5( + title: &str, + c: &Compiled, + witness: &Witness, + fresh: &dyn Fn() -> Compiled, + edges: &[(u16, u16)], + edited: &[u16], + all_qs: &[Vec], + evict_buf: &mut [u64], +) { + let users: Vec = (0..c.b.users as u16).collect(); + let m = users.len(); + println!("### {title}\n"); + let mut evals_tbl = Vec::new(); + let mut acct_tbl = Vec::new(); + let mut amort_tbl = Vec::new(); + for k in [16usize, 64, 128, 256] { + let (qs, groups) = take_k(all_qs, k); + if qs.is_empty() { + continue; + } + let wds = words_of(&groups); + // Preparation, phase by phase, counted. + let (mut f, a_slots) = counted(|| Focus::slots(c, Chain::All, &groups, witness).unwrap()); + let ((), a_proj) = counted(|| f.project(c)); + let mut fref = Focus::slots(c, Chain::All, &groups, witness).unwrap(); + let ((), a_ref) = counted(|| fref.project_by_test(c)); + let masks: Vec = qs.iter().map(|q| f.all_of(q).unwrap()).collect(); + let sl = slots_of(&f, &qs); + let (fu, a_users) = counted(|| { + FocusUsers::from_focus(c, Focus::bind(c, Chain::All, &groups, witness).unwrap()) + }); + // Every arm agrees with B+ on every user before anything is timed. + let mut row = vec![0u64; c.w]; + let mut acc = vec![0u64; c.w]; + for &u in &users { + let want = e_bp(c, u, &qs, &mut row); + assert_eq!(e_bpw(c, u, &qs, &wds, &mut acc), want); + assert_eq!(e_f(&f, c, u, &masks), want); + assert_eq!(e_fu(&fu, u, &masks), want); + assert_eq!(e_f_bits(&f, c, u, &sl), want); + } + let t_bits = time(9, || { + (0..m) + .map(|e| e_f_bits(&f, c, users[e], &sl)) + .sum::() + }); + for g in 0..c.b.groups { + assert_eq!(f.projected(g), fref.projected(g)); + } + let t_slots = time(15, || { + Focus::slots(c, Chain::All, &groups, witness).unwrap() + }); + let t_proj = { + let mut v = Vec::new(); + for _ in 0..15 { + let mut g = Focus::slots(c, Chain::All, &groups, witness).unwrap(); + let s = Instant::now(); + g.project(c); + v.push(s.elapsed()); + black_box(g); + } + median(v) + }; + let t_ref = { + let mut v = Vec::new(); + for _ in 0..7 { + let mut g = Focus::slots(c, Chain::All, &groups, witness).unwrap(); + let s = Instant::now(); + g.project_by_test(c); + v.push(s.elapsed()); + } + median(v) + }; + let t_users = { + let mut v = Vec::new(); + for _ in 0..9 { + let g = Focus::bind(c, Chain::All, &groups, witness).unwrap(); + let s = Instant::now(); + black_box(FocusUsers::from_focus(c, g)); + v.push(s.elapsed()); + } + median(v) + }; + // Lazy: cache allocation, then first touch of every user, then warm. + let (mut fl, a_lazy) = counted(|| { + FocusLazy::from_focus(c, Focus::bind(c, Chain::All, &groups, witness).unwrap()) + }); + let t_lazy_alloc = time(15, || { + FocusLazy::from_focus(c, Focus::bind(c, Chain::All, &groups, witness).unwrap()) + }) + .saturating_sub(time(15, || { + Focus::bind(c, Chain::All, &groups, witness).unwrap() + })); + let s = Instant::now(); + for &u in &users { + black_box(fl.row(c, u)); + } + let t_lazy_first = s.elapsed(); + assert_eq!(fl.filled, m); + let t_lazy_warm = time(9, || { + users + .iter() + .map(|&u| { + let r = fl.row(c, u); + masks.iter().filter(|mm| holds(&r, mm)).count() + }) + .sum::() + }); + // Refresh after membership edits (the edited users' rows). + let mut fu2 = + FocusUsers::from_focus(c, Focus::bind(c, Chain::All, &groups, witness).unwrap()); + let ((), a_ru) = counted(|| fu2.refresh_users(c, edited)); + let t_ru = time(15, || fu2.refresh_users(c, edited)); + let line = |phase: &str, t: f64, a: Acct| { + format!( + "| {} | {phase} | {t:.1} µs | {} | {} | {} |", + groups.len(), + a.allocs, + kib(a.bytes as isize), + kib(a.retained) + ) + }; + // Refresh after one nesting pair: B+, then F, then F+users, for a + // narrow edge and a wide one (a child with a large downward closure). + let mut nest_rows = Vec::new(); + for &edge in edges { + let (mut t_bn, mut t_fn, mut t_fun) = (Vec::new(), Vec::new(), Vec::new()); + let (mut a_bn, mut a_fn, mut a_fun) = + (Acct::default(), Acct::default(), Acct::default()); + for _ in 0..5 { + let mut cc = fresh(); + let mut fx = Focus::bind(&cc, Chain::All, &groups, witness).unwrap(); + let mut fux = FocusUsers::from_focus( + &cc, + Focus::bind(&cc, Chain::All, &groups, witness).unwrap(), + ); + let s = Instant::now(); + let ((), a) = counted(|| cc.add_nesting(edge.0, edge.1)); + t_bn.push(s.elapsed()); + a_bn = a; + let s = Instant::now(); + let ((), a) = counted(|| fx.refresh_nesting(&cc, edge.0)); + t_fn.push(s.elapsed()); + a_fn = a; + let s = Instant::now(); + let ((), a) = counted(|| fux.refresh_nesting(&cc, edge.0)); + t_fun.push(s.elapsed()); + a_fun = a; + let check = Focus::bind(&cc, Chain::All, &groups, witness).unwrap(); + for g in 0..cc.b.groups { + assert_eq!(fx.projected(g), check.projected(g), "refresh != rebind"); + } + for &u in &users { + assert_eq!(fux.rows[u as usize], check.row_of(&cc, u), "users refresh"); + } + } + let refreshed = ones(c.down_row(Chain::All, edge.0 as usize)).count(); + nest_rows.push(line( + &format!("refresh after 1 nesting pair (child closure {refreshed} groups): B+ closure insertion"), + us(median(t_bn)), + a_bn, + )); + nest_rows.push(line( + " … F re-projects those groups", + us(median(t_fn)), + a_fn, + )); + nest_rows.push(line( + " … F+users: re-project + recompute their direct users", + us(median(t_fun)), + a_fun, + )); + } + // Evaluations. + let (ev, allocs) = time_evals(c, &users, &qs, &wds, &f, &fu, &masks, m, 9); + let cold = time_cold(c, &users, &qs, &wds, &f, &fu, &masks, evict_buf); + evals_tbl.push(format!( + "| {} | {} | {} | {:.0} | {:.0} | {:.0} | {:.0} | {:.0} | {:.0} | {:.0} / {:.0} / {:.0} / {:.0} | {:?} |", + groups.len(), + qs.len(), + wds.len(), + ev.bp, + ev.bpw, + ev.f, + ns(t_bits, m), + ev.fu, + ns(t_lazy_warm, m), + cold[0], + cold[1], + cold[2], + cold[3], + allocs + )); + acct_tbl.push(line( + "focus construction (slots + reverse map)", + us(t_slots), + a_slots, + )); + acct_tbl.push(line("closure projection (down rows)", us(t_proj), a_proj)); + acct_tbl.push(line( + "closure projection by test (reference)", + us(t_ref), + a_ref, + )); + acct_tbl.push(line("eager per-user rows, all users", us(t_users), { + let mut a = a_users; + // `a_users` includes a bind; subtract its slots and projection. + a.allocs -= a_slots.allocs + a_proj.allocs; + a.bytes -= a_slots.bytes + a_proj.bytes; + a.retained -= a_slots.retained + a_proj.retained; + a + })); + acct_tbl.push(line( + &format!( + "lazy per-user rows: allocate the cache (then first touch {:.0} ns per user, filled on use)", + ns(t_lazy_first, m) + ), + us(t_lazy_alloc), + { + let mut a = a_lazy; + a.allocs -= a_slots.allocs + a_proj.allocs; + a.bytes -= a_slots.bytes + a_proj.bytes; + a.retained -= a_slots.retained + a_proj.retained; + a + }, + )); + acct_tbl.push(line( + &format!( + "refresh after membership edits ({} users), eager rows", + edited.len() + ), + us(t_ru), + a_ru, + )); + acct_tbl.extend(nest_rows); + // Amortization: computed crossover and measured crossover. + let p_bind = ns(t_slots, 1); + let p_proj = ns(t_proj, 1); + let p_users = ns(t_users, 1); + let nstar = |p: f64, base: f64, e: f64| { + if base > e { + format!("{:.0}", p / (base - e)) + } else { + "never".into() + } + }; + let mut measured = |arm: u8| -> String { + for n in [ + 1usize, 2, 5, 10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000, 20_000, + 50_000, + ] { + let reps = if n <= 1_000 { 21 } else { 5 }; + let base = time(reps, || { + (0..n) + .map(|e| e_bp(c, users[e % m], &qs, &mut row)) + .sum::() + }); + let t = match arm { + 0 => time(reps, || { + let f = Focus::bind(c, Chain::All, &groups, witness).unwrap(); + (0..n) + .map(|e| e_f(&f, c, users[e % m], &masks)) + .sum::() + }), + _ => time(reps, || { + let fu = FocusUsers::bind(c, Chain::All, &groups, witness).unwrap(); + (0..n) + .map(|e| e_fu(&fu, users[e % m], &masks)) + .sum::() + }), + }; + if t <= base { + return format!("{n}"); + } + } + "> 50000".into() + }; + // Rebuilding a focus for each question: bind + project + 1 evaluation. + let t_rebuild = time(9, || { + (0..20) + .map(|e| { + let f = Focus::bind(c, Chain::All, &groups, witness).unwrap(); + e_f(&f, c, users[(e * 31) % m], &masks) + }) + .sum::() + }); + amort_tbl.push(format!( + "| {} | {:.1} | {:.1} | {:.1} | {:.0} | {:.0} | {:.0} | {:.0} | {} | {} | {} | {} | {} | {:.1} µs |", + groups.len(), + p_bind / 1e3, + p_proj / 1e3, + p_users / 1e3, + ev.bp, + ev.bpw, + ev.f, + ev.fu, + nstar(p_bind + p_proj, ev.bp, ev.f), + nstar(p_bind + p_proj, ev.bpw, ev.f), + nstar(p_bind + p_proj + p_users, ev.bp, ev.fu), + measured(0), + measured(1), + us(t_rebuild) / 20.0, + )); + } + println!( + "Per evaluation (one user, every question), warm, users cycled; cold = after evicting 64 MiB. ns. `words` = closure words that hold a focus group. Allocations counted over {m} evaluations per arm (B+, B+w, F, F+u).\n" + ); + println!( + "| focus groups | questions | words | B+ | B+w | F | F (slot bits) | F+users | F lazy (warm) | cold B+ / B+w / F / F+u | allocs |" + ); + println!("|---|---|---|---|---|---|---|---|---|---|---|"); + for r in &evals_tbl { + println!("{r}"); + } + println!( + "\nPreparation accounting (each phase alone; bytes = allocated, retained = still live after the phase):\n" + ); + println!( + "| focus groups | phase | time | allocs | bytes | retained |\n|---|---|---|---|---|---|" + ); + for r in &acct_tbl { + println!("{r}"); + } + println!( + "\nAmortization. T_B+(N) = N·e_B+; T_F(N) = P_bind + P_proj + N·e_F; T_F+u(N) = P_bind + P_proj + P_users + N·e_F+u. B+ compile is shared by every arm and excluded. N* = P / (e_B+ − e_F), computed; \"measured\" = smallest N in {{1, 2, 5, ..., 50000}} at which the directly timed total of F (bind included) is ≤ the timed B+ total. Rebuild = bind + project + one evaluation, per question.\n" + ); + println!( + "| focus groups | P_bind µs | P_proj µs | P_users µs | e_B+ ns | e_B+w ns | e_F ns | e_F+u ns | N* F vs B+ | N* F vs B+w | N* F+u vs B+ | measured F | measured F+u | rebuild per question |" + ); + println!("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"); + for r in &amort_tbl { + println!("{r}"); + } + println!(); +} + +/// W5 on the three question shapes. +fn w5_all(w: &World, c: &Compiled, ord: &Ordinals, evict_buf: &mut [u64], _sample: &[usize]) { + println!("## W5: role x location and composite questions, B+ vs F\n"); + let witness = w.witness(); + let fresh = || Compiled::compile(w.prepare()); + let f = &w.f; + // The nesting pair used for the refresh figures: a team into the first + // role of the focus (absent before). + let edge = ( + ord.group[f.teams[0] as usize], + ord.group[f.roles[0] as usize], + ); + assert!(c.b.nested.binary_search(&edge).is_err()); + let edited: Vec = (0..(f.users.len() / 100).max(1)) + .map(|i| ord.user[(i * 97) % f.users.len()]) + .collect(); + // (a) Policy conjunctions asked over the composite fixture. + let conj: Vec> = conjunctions(f) + .iter() + .map(|&(r, l)| vec![ord.group[r as usize], ord.group[l as usize]]) + .collect(); + let edge_c = (edge.0, conj[0][0]); + w5( + "W5a: policy conjunction (in role R and in location L), composite fixture", + c, + &witness, + &fresh, + &[edge_c, (wide_child(c, edge_c.1), edge_c.1)], + &edited, + &conj, + evict_buf, + ); + // (b) Materialized composites: one membership bit per question. + let comp: Vec> = composite_groups(f) + .iter() + .map(|&g| vec![ord.group[g as usize]]) + .collect(); + w5( + "W5b: materialized composite membership (in composite C), composite fixture", + c, + &witness, + &fresh, + &[ + (edge.0, comp[0][0]), + (wide_child(c, comp[0][0]), comp[0][0]), + ], + &edited, + &comp, + evict_buf, + ); + // (c) Policy conjunctions on the fixture with no composites. + let wc = World::new(Fixture::generate( + Spec::iam(f.users.len()).without_composites(), + )); + let vc = wc.view(); + let ordc = Ordinals::of(&wc.f, &vc); + let cc = Compiled::compile(wc.prepare()); + let qc: Vec> = conjunctions(&wc.f) + .iter() + .map(|&(r, l)| vec![ordc.group[r as usize], ordc.group[l as usize]]) + .collect(); + // B+ on this fixture equals the oracle (membership count) and dir-sim + // on a sample, before W5c is timed. + let d = oracle::run(&wc.f, &wc.direct, &wc.nested, &[]); + let mut row = vec![0u64; cc.w]; + let total: usize = (0..wc.f.users.len()) + .map(|i| { + cc.groups_of(ordc.user[i], Chain::All, &mut row); + popcount_batch_u64(&row) as usize + }) + .sum(); + assert_eq!(total, d.member.len()); + for i in (0..wc.f.users.len()).step_by(37) { + cc.groups_of(ordc.user[i], Chain::All, &mut row); + let mut got: Vec = ones(&row) + .map(|g| vc.group_guid(GroupOrdinal(g as u16)).unwrap()) + .collect(); + got.sort_unstable(); + let mut a = a_groups(&vc, &wc.f.users[i], Chain::All); + a.sort_unstable(); + assert_eq!(got, a); + } + let editedc: Vec = (0..(wc.f.users.len() / 100).max(1)) + .map(|i| ordc.user[(i * 97) % wc.f.users.len()]) + .collect(); + let freshc = || Compiled::compile(wc.prepare()); + let edgec = (ordc.group[wc.f.teams[0] as usize], qc[0][0]); + println!( + "Fixture without composites: {} groups, {} conjunction questions; B+ = oracle on all {} memberships, = dir-sim on a sample.\n", + vc.groups_len(), + qc.len(), + d.member.len() + ); + w5( + "W5c: policy conjunction, fixture without composites", + &cc, + &wc.witness(), + &freshc, + &[edgec, (wide_child(&cc, edgec.1), edgec.1)], + &editedc, + &qc, + evict_buf, + ); +} + +// --------------------------------------------------------------------------- +// Working-set sweep: more groups, so the B+ closure outgrows L2. + +fn sweep(users: usize, evict_buf: &mut [u64]) { + let factors: Vec = std::env::var("DIR_RMAP_SWEEP") + .unwrap_or_else(|_| "1,2,4,8".into()) + .split(',') + .map(|s| s.trim().parse().expect("sweep factor")) + .collect(); + println!("## Working-set sweep ({users} users, conjunction questions, focus of 64 groups)\n"); + println!( + "Caches: {}. A focus row is 32 B; a closure row is `words`·8 B. \"touched\" = bytes read by one evaluation (direct rows + output); \"set\" = bytes of rows read when every user is evaluated in turn (distinct direct groups × row size). Hot = one user repeated; warm = users cycled; cold = after evicting 64 MiB. ns per evaluation.\n", + cache_sizes() + ); + println!( + "| groups | words | closure, one chain up | B+ set | F set | F+u set | avg direct groups | touched B+ / B+w / F | hot B+ / F | warm B+ / B+w / F / F+u | cold B+ / B+w / F / F+u | compile ms | F prep µs | N* F vs B+ | N* F vs B+w |" + ); + println!("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"); + for factor in factors { + let w = World::new(Fixture::generate(Spec::iam_scaled(users, factor))); + let v = w.view(); + let ord = Ordinals::of(&w.f, &v); + let witness = w.witness(); + let s = Instant::now(); + let c = Compiled::compile(w.prepare()); + let t_comp = s.elapsed(); + let all: Vec> = conjunctions(&w.f) + .iter() + .map(|&(r, l)| vec![ord.group[r as usize], ord.group[l as usize]]) + .collect(); + let (qs, groups) = take_k(&all, 64); + let wds = words_of(&groups); + let f = Focus::bind(&c, Chain::All, &groups, &witness).unwrap(); + let fu = + FocusUsers::from_focus(&c, Focus::bind(&c, Chain::All, &groups, &witness).unwrap()); + let masks: Vec = qs.iter().map(|q| f.all_of(q).unwrap()).collect(); + let users_o: Vec = (0..c.b.users as u16).collect(); + let mut row = vec![0u64; c.w]; + for &u in &users_o { + let want = e_bp(&c, u, &qs, &mut row); + assert_eq!(e_f(&f, &c, u, &masks), want); + assert_eq!(e_fu(&fu, u, &masks), want); + } + let n = c.b.groups; + let total_deg: usize = users_o + .iter() + .map(|&u| c.b.user_groups.row(u as usize).len()) + .sum(); + let avg = total_deg as f64 / users_o.len() as f64; + let mut distinct = vec![false; n]; + for &u in &users_o { + for &d in c.b.user_groups.row(u as usize) { + distinct[d as usize] = true; + } + } + let dg = distinct.iter().filter(|&&x| x).count(); + let set_bp = dg * c.w * 8; + let set_f = dg * 32; + let set_fu = users_o.len() * 32; + let touched = ( + (avg + 1.0) * c.w as f64 * 8.0, + avg * wds.len() as f64 * 8.0, + avg * 32.0 + masks.len() as f64 * 32.0, + ); + let (ev, _) = time_evals(&c, &users_o, &qs, &wds, &f, &fu, &masks, users_o.len(), 7); + let one = [users_o[users_o.len() / 2]]; + let (hot, _) = time_evals(&c, &one, &qs, &wds, &f, &fu, &masks, 20_000, 7); + let cold = time_cold(&c, &users_o, &qs, &wds, &f, &fu, &masks, evict_buf); + let t_prep = time(9, || { + Focus::bind(&c, Chain::All, &groups, &witness).unwrap() + }); + let p = ns(t_prep, 1); + let nstar = |base: f64, e: f64| { + if base > e { + format!("{:.0}", p / (base - e)) + } else { + "never".into() + } + }; + println!( + "| {n} | {} | {} | {} | {} | {} | {avg:.1} | {} / {} / {} | {:.0} / {:.0} | {:.0} / {:.0} / {:.0} / {:.0} | {:.0} / {:.0} / {:.0} / {:.0} | {:.0} | {:.1} | {} | {} |", + c.w, + kib((n * c.w * 8) as isize), + kib(set_bp as isize), + kib(set_f as isize), + kib(set_fu as isize), + kib(touched.0 as isize), + kib(touched.1 as isize), + kib(touched.2 as isize), + hot.bp, + hot.f, + ev.bp, + ev.bpw, + ev.f, + ev.fu, + cold[0], + cold[1], + cold[2], + cold[3], + t_comp.as_secs_f64() * 1e3, + us(t_prep), + nstar(ev.bp, ev.f), + nstar(ev.bpw, ev.f), + ); + } + println!(); +} + +/// The group with the largest downward closure that is not yet nested in +/// `parent`: the widest refresh a single nesting pair into `parent` causes. +fn wide_child(c: &Compiled, parent: u16) -> u16 { + (0..c.b.groups as u16) + .filter(|&g| g != parent && c.b.nested.binary_search(&(g, parent)).is_err()) + .max_by_key(|&g| ones(c.down_row(Chain::All, g as usize)).count()) + .unwrap() +} From 79e802c51b4741fe731ba4a3ee825237d1358bcd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 22:27:11 +0000 Subject: [PATCH 6/7] dir-rmap-probe: witness names the snapshot; user ranges survive 65,536 users Codex review on #1635: (1) a witness of (version, users, groups) passes across two stores, since every VersionStore numbers versions from 0; it now also holds a Weak to the version's Snapshot, compared by pointer (the Weak keeps the address from being reused). New test: a cache from one store refuses another store's witness over the same data. (2) user ranges iterate as usize, so 65,536 users no longer wrap to an empty range. Bench: the W8 rebind time no longer includes choosing the focus. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- crates/dir-rmap-probe/examples/bench.rs | 33 +++++++++++++--------- crates/dir-rmap-probe/src/focus.rs | 13 +++++---- crates/dir-rmap-probe/src/lib.rs | 4 +-- crates/dir-rmap-probe/src/prep.rs | 31 ++++++++++++++++++-- crates/dir-rmap-probe/tests/equivalence.rs | 20 +++++++++++-- 5 files changed, 75 insertions(+), 26 deletions(-) diff --git a/crates/dir-rmap-probe/examples/bench.rs b/crates/dir-rmap-probe/examples/bench.rs index cedf1e2a3..9dd1d7d6e 100644 --- a/crates/dir-rmap-probe/examples/bench.rs +++ b/crates/dir-rmap-probe/examples/bench.rs @@ -573,13 +573,9 @@ fn main() { ); } } + let rebind_ids = focus_ids_for(&full1, &all_templates, &ord1); let t_rebind = time(9, || { - Focus::bind( - &full1, - Chain::All, - &focus_ids_for(&full1, &all_templates, &ord1), - &w.witness(), - ) + Focus::bind(&full1, Chain::All, &rebind_ids, &w.witness()) }); println!("## W7/W8: overlay ({membership_edits} membership edits + 1 nesting edge)\n"); println!("| step | time |\n|---|---|"); @@ -953,7 +949,7 @@ fn w5( all_qs: &[Vec], evict_buf: &mut [u64], ) { - let users: Vec = (0..c.b.users as u16).collect(); + let users: Vec = (0..c.b.users).map(|u| u as u16).collect(); let m = users.len(); println!("### {title}\n"); let mut evals_tbl = Vec::new(); @@ -1388,21 +1384,25 @@ fn w5_all(w: &World, c: &Compiled, ord: &Ordinals, evict_buf: &mut [u64], _sampl // --------------------------------------------------------------------------- // Working-set sweep: more groups, so the B+ closure outgrows L2. +const SWEEP_QUESTIONS: usize = 150; + fn sweep(users: usize, evict_buf: &mut [u64]) { let factors: Vec = std::env::var("DIR_RMAP_SWEEP") .unwrap_or_else(|_| "1,2,4,8".into()) .split(',') .map(|s| s.trim().parse().expect("sweep factor")) .collect(); - println!("## Working-set sweep ({users} users, conjunction questions, focus of 64 groups)\n"); + println!( + "## Working-set sweep ({users} users, conjunction questions, focus of 64 groups, {SWEEP_QUESTIONS} questions)\n" + ); println!( "Caches: {}. A focus row is 32 B; a closure row is `words`·8 B. \"touched\" = bytes read by one evaluation (direct rows + output); \"set\" = bytes of rows read when every user is evaluated in turn (distinct direct groups × row size). Hot = one user repeated; warm = users cycled; cold = after evicting 64 MiB. ns per evaluation.\n", cache_sizes() ); println!( - "| groups | words | closure, one chain up | B+ set | F set | F+u set | avg direct groups | touched B+ / B+w / F | hot B+ / F | warm B+ / B+w / F / F+u | cold B+ / B+w / F / F+u | compile ms | F prep µs | N* F vs B+ | N* F vs B+w |" + "| groups | questions | F masks | words | closure, one chain up | B+ set | F set | F+u set | avg direct groups | touched B+ / B+w / F | hot B+ / F | warm B+ / B+w / F / F+u | cold B+ / B+w / F / F+u | compile ms | F prep µs | N* F vs B+ | N* F vs B+w |" ); - println!("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"); + println!("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"); for factor in factors { let w = World::new(Fixture::generate(Spec::iam_scaled(users, factor))); let v = w.view(); @@ -1415,13 +1415,18 @@ fn sweep(users: usize, evict_buf: &mut [u64]) { .iter() .map(|&(r, l)| vec![ord.group[r as usize], ord.group[l as usize]]) .collect(); - let (qs, groups) = take_k(&all, 64); + // A fixed number of questions, so the rows differ only in group + // count (the first 64-group focus holds a different number of + // conjunctions at each scale). + let (mut qs, groups) = take_k(&all, 64); + assert!(qs.len() >= SWEEP_QUESTIONS, "{} questions", qs.len()); + qs.truncate(SWEEP_QUESTIONS); let wds = words_of(&groups); let f = Focus::bind(&c, Chain::All, &groups, &witness).unwrap(); let fu = FocusUsers::from_focus(&c, Focus::bind(&c, Chain::All, &groups, &witness).unwrap()); let masks: Vec = qs.iter().map(|q| f.all_of(q).unwrap()).collect(); - let users_o: Vec = (0..c.b.users as u16).collect(); + let users_o: Vec = (0..c.b.users).map(|u| u as u16).collect(); let mut row = vec![0u64; c.w]; for &u in &users_o { let want = e_bp(&c, u, &qs, &mut row); @@ -1465,7 +1470,9 @@ fn sweep(users: usize, evict_buf: &mut [u64]) { } }; println!( - "| {n} | {} | {} | {} | {} | {} | {avg:.1} | {} / {} / {} | {:.0} / {:.0} | {:.0} / {:.0} / {:.0} / {:.0} | {:.0} / {:.0} / {:.0} / {:.0} | {:.0} | {:.1} | {} | {} |", + "| {n} | {} | {} | {} | {} | {} | {} | {} | {avg:.1} | {} / {} / {} | {:.0} / {:.0} | {:.0} / {:.0} / {:.0} / {:.0} | {:.0} / {:.0} / {:.0} / {:.0} | {:.0} | {:.1} | {} | {} |", + qs.len(), + kib((masks.len() * 32) as isize), c.w, kib((n * c.w * 8) as isize), kib(set_bp as isize), diff --git a/crates/dir-rmap-probe/src/focus.rs b/crates/dir-rmap-probe/src/focus.rs index 83c1ce7ea..c8fb4a4d2 100644 --- a/crates/dir-rmap-probe/src/focus.rs +++ b/crates/dir-rmap-probe/src/focus.rs @@ -100,7 +100,7 @@ impl Focus { *e = s as u16 + 1; } Ok(Focus { - witness: c.b.witness, + witness: c.b.witness.clone(), chain, ids: groups.to_vec(), slot_of, @@ -117,7 +117,7 @@ impl Focus { set(&mut self.proj[g], s); } } - self.witness = c.b.witness; + self.witness = c.b.witness.clone(); } /// The reference projection: test every slot in every group's upward @@ -132,7 +132,7 @@ impl Focus { } } } - self.witness = c.b.witness; + self.witness = c.b.witness.clone(); } /// Phases 1 and 2. @@ -161,13 +161,13 @@ impl Focus { } self.proj[g] = row; } - self.witness = c.b.witness; + self.witness = c.b.witness.clone(); } /// After membership edits only: the projected rows do not depend on /// direct membership, so only the witness moves (taken from `c`). pub fn refresh_membership(&mut self, c: &Compiled) { - self.witness = c.b.witness; + self.witness = c.b.witness.clone(); } pub fn check(&self, current: &Witness) -> Result<(), Refusal> { @@ -247,7 +247,8 @@ pub struct FocusUsers { impl FocusUsers { pub fn from_focus(c: &Compiled, focus: Focus) -> FocusUsers { - let rows = (0..c.b.users as u16).map(|u| focus.row_of(c, u)).collect(); + // `users` may be 65,536 (the full u16 ordinal space): iterate as usize. + let rows = (0..c.b.users).map(|u| focus.row_of(c, u as u16)).collect(); FocusUsers { focus, rows } } diff --git a/crates/dir-rmap-probe/src/lib.rs b/crates/dir-rmap-probe/src/lib.rs index 8513ceeb1..4699ff972 100644 --- a/crates/dir-rmap-probe/src/lib.rs +++ b/crates/dir-rmap-probe/src/lib.rs @@ -202,7 +202,7 @@ impl World { } pub fn witness(&self) -> prep::Witness { - prep::Witness::of(self.version, &self.view()) + prep::Witness::of(&self.st, self.version) } /// Arm B for the current version. @@ -211,7 +211,7 @@ impl World { let ord = prep::Ordinals::of(&self.f, &v); prep::Prepared::build( &v, - prep::Witness::of(self.version, &v), + prep::Witness::of(&self.st, self.version), &ord, &self.direct, &self.nested, diff --git a/crates/dir-rmap-probe/src/prep.rs b/crates/dir-rmap-probe/src/prep.rs index cc60fa53b..3826d6429 100644 --- a/crates/dir-rmap-probe/src/prep.rs +++ b/crates/dir-rmap-probe/src/prep.rs @@ -22,9 +22,12 @@ //! `ISS-ADMISSION-PREP-BINDING`, still open. use crate::fixture::Fixture; +use lance_graph_dir_sim::snapshot::Snapshot; +use lance_graph_dir_sim::store::VersionStore; use lance_graph_dir_sim::{GroupProperty, GroupWhere, View, groups_where}; use ndarray::simd::mask_or_assign; use ogar_dir_sim::VersionId; +use std::sync::{Arc, Weak}; pub fn words(n: usize) -> usize { n.div_ceil(64) @@ -56,20 +59,42 @@ pub fn ones(p: &[u64]) -> impl Iterator + '_ { } /// What a prepared structure was built against. Ordinals are valid inside -/// one version only, so any difference refuses. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +/// one snapshot only, so any difference refuses. +/// +/// `VersionId` alone does not identify a snapshot: every `VersionStore` +/// numbers its versions from zero, so a store restored from an image, or a +/// second store over other data, can reuse an id with the same user and +/// group counts. The witness therefore also holds a `Weak` to the version's +/// `Snapshot` and compares it by pointer. The `Weak` keeps the allocation +/// alive, so its address is never reused while the witness exists. +#[derive(Clone, Debug)] pub struct Witness { pub version: VersionId, pub users: usize, pub groups: usize, + snapshot: Weak, +} + +impl PartialEq for Witness { + fn eq(&self, o: &Self) -> bool { + self.version == o.version + && self.users == o.users + && self.groups == o.groups + && Weak::ptr_eq(&self.snapshot, &o.snapshot) + } } +impl Eq for Witness {} + impl Witness { - pub fn of(version: VersionId, v: &View<'_>) -> Self { + pub fn of(st: &VersionStore, version: VersionId) -> Self { + let snap = st.snapshot(version).expect("version exists"); + let v = st.view(version).expect("version exists"); Witness { version, users: v.users_len(), groups: v.groups_len(), + snapshot: Arc::downgrade(snap), } } } diff --git a/crates/dir-rmap-probe/tests/equivalence.rs b/crates/dir-rmap-probe/tests/equivalence.rs index c8a8ddd15..e7921bfa4 100644 --- a/crates/dir-rmap-probe/tests/equivalence.rs +++ b/crates/dir-rmap-probe/tests/equivalence.rs @@ -454,7 +454,7 @@ fn a_refreshed_focus_equals_a_fresh_bind_dir_sim_and_the_oracle() { .map(|&ch| FocusLazy::from_focus(&c, Focus::bind(&c, ch, &ids, &w0).unwrap())) .collect(); for (k, l) in fl.iter_mut().enumerate() { - for u in 0..c.b.users as u16 { + for u in (0..c.b.users).map(|u| u as u16) { let _ = l.row(&c, u); } assert_eq!(l.filled, c.b.users, "chain {k} warmed"); @@ -485,7 +485,7 @@ fn a_refreshed_focus_equals_a_fresh_bind_dir_sim_and_the_oracle() { } // Incremental: membership CSRs, then each nesting pair. c.b.set_direct(&ord, &w.direct); - c.b.witness = w1; + c.b.witness = w1.clone(); let edited: Vec = edits .iter() .filter_map(|e| match *e { @@ -663,3 +663,19 @@ fn a_focus_refuses_what_it_cannot_answer_completely() { assert_eq!(c1.b.check(&w1), Ok(())); assert_eq!(f.ask(&c1, &w1, 0, &[3]), Err(Refusal::Stale)); } + +/// Two stores over the same data both number their first version 0 and +/// have the same counts; a prepared structure from one must still refuse +/// the other's witness (it names another snapshot). +#[test] +fn a_witness_from_another_store_is_refused() { + let a = world(); + let b = world(); + assert_eq!(a.version, b.version, "both stores start at the same id"); + assert_eq!(a.view().users_len(), b.view().users_len()); + let c = Compiled::compile(a.prepare()); + assert_eq!(c.b.check(&a.witness()), Ok(())); + assert_eq!(c.b.check(&b.witness()), Err(Refusal::Stale)); + let f = Focus::bind(&c, Chain::All, &[3], &a.witness()).unwrap(); + assert_eq!(f.check(&b.witness()), Err(Refusal::Stale)); +} From cb9906d958d30e5f080dabfc3eda24c9ebe7e170 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 22:29:57 +0000 Subject: [PATCH 7/7] =?UTF-8?q?D-RMAP-DIR-1=20round=202:=20results,=20plan?= =?UTF-8?q?=20=C2=A79,=20board?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plan §9: completeness tests and disable runs, the working-set sweep against L1/L2, composite vs conjunction, per-phase preparation accounting and the amortization table, revised F verdict. Board: entry for round 2, STATUS_BOARD row, regenerated entries index. Bench: W8 rebind time no longer includes building the witness. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G22yT6htkcdyXsihxxXdrg --- .claude/board/STATUS_BOARD.md | 2 +- ...-20-d-rmap-dir-1-round-2-focus-geometry.md | 23 +++ .claude/board/entries/README.md | 3 +- ...-rmap-dir-1-membership-closure-focus-v1.md | 170 +++++++++++++++++- crates/dir-rmap-probe/examples/bench.rs | 5 +- 5 files changed, 194 insertions(+), 9 deletions(-) create mode 100644 .claude/board/entries/2026-10-20-d-rmap-dir-1-round-2-focus-geometry.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 8420dbc1b..3e7f4aa87 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -54,7 +54,7 @@ These PRs merged with no board rows. Every row below is a PROPOSAL (documents an | D-RPR-3 | MeTTa-facing parity of a chain's answers | BLOCKED on D-ASM-4 (not authorized) | — | | D-MPS-8 | native join over a versioned relationship table: R and S laid out by `y` slot, leapfrog intersection as one mask-risc `And`, per-side generations and reseal, per-thought filter planes (`mork-pathmap-schnellverschluss-v1.md` §7b; `crates/mork-pathmap-probe/src/join.rs`) | MEASURED, probe only; not admitted (D-RMAP-1 BIND membrane review) | 8 join tests + native arms in the differential suite vs the oracle; 7 guards disable-verified; cyclic bodies open | | D-RMAP-2 | context-declaration reuse and GraphBLAS/BF16 tile probe (`d-rmap-2-context-tile-algebra-probe-v1.md`) | LAB ONLY, not admitted to production; PARTIAL: §5a MEASURED on the polyfill tier and on the AMX tile tier (§5a-AMX); `VDPBF16PS`-only tier and GraphBLAS arm not run | arms 1–5 in the plan; 297× (D-GTM-0j) is not evidence | -| D-RMAP-DIR-1 | dir-sim membership: A (View) vs D (crepe oracle) vs B (bidirectional CSR) vs B+ (compiled closure rows + roles by ordinal) vs F (256-slot focus over B+) (`d-rmap-dir-1-membership-closure-focus-v1.md`; `crates/dir-rmap-probe`, CI `dir-rmap-probe.yml`) | MEASURED, lab only; nothing admitted, no production file changed | 5 tests: every arm = A = D on every user/group/chain; 7 guards disable-verified red, 1 shown redundant; B removes A's per-call full scan (W2 ~4,000×), B+ adds 2–3× (groups) and 12–15× (roles), F ≤ 2× only for ≤ 64 focus groups after ~1,000 reuses, none near 256 | +| D-RMAP-DIR-1 | dir-sim membership: A (View) vs D (crepe oracle) vs B (bidirectional CSR) vs B+ (compiled closure rows + roles by ordinal) vs F (256-slot focus over B+) (`d-rmap-dir-1-membership-closure-focus-v1.md`; `crates/dir-rmap-probe`, CI `dir-rmap-probe.yml`) | MEASURED, lab only; nothing admitted, no production file changed | 5 tests: every arm = A = D on every user/group/chain; 7 guards disable-verified red, 1 shown redundant; B removes A's per-call full scan (W2 ~4,000×), B+ adds 2–3× (groups) and 12–15× (roles), F ≤ 2× only for ≤ 64 focus groups after ~1,000 reuses, none near 256. **Round 2 (§9):** 11 tests; F per chain, refuses incomplete answers; projection through down rows moves F's break-even to 2–100 reuses; F 2–5× warm / ~2× cold while the question masks fit L1, its gain growing with group count (2.9× at 1.5k groups → 5.5× at 11k); slower than B+ once ~4,300 conjunction masks exceed L1; rebuilding per question never pays; not promoted | | D-RMAP-DIR-2 | B built in `Snapshot::build` from the lanes; B+ closure as a lazily compiled, version-witnessed cache behind the existing `View` methods; F not proposed | PROPOSED; not authorized; waits on `ISS-ADMISSION-PREP-BINDING` (hot path reads rows at a user's direct-group values) | dir-sim tests + D-RMAP-DIR-1 oracle as the equivalence gate | | D-SRAM-0 | inventory Soufflé AST/ast2ram/RAM passes against D-ASM/D-RA/D-TGA owners | Queued | evidence per mapped pass; no code imported | | D-SRAM-1 | bound AST → RAM-like schedule for pure conjunctive selection | Queued | same answers and read sets on hand-built fixtures; unsupported ops typed-refused | diff --git a/.claude/board/entries/2026-10-20-d-rmap-dir-1-round-2-focus-geometry.md b/.claude/board/entries/2026-10-20-d-rmap-dir-1-round-2-focus-geometry.md new file mode 100644 index 000000000..ea83f36a3 --- /dev/null +++ b/.claude/board/entries/2026-10-20-d-rmap-dir-1-round-2-focus-geometry.md @@ -0,0 +1,23 @@ +# D-RMAP-DIR-1 round 2: when 256-slot focus geometry beats compiled closure (2026-10-20) + +**Status:** MEASURED, lab only; nothing promoted. **Basis:** `crates/dir-rmap-probe` (`tests/equivalence.rs` 11/11; `examples/bench.rs`, 4 runs at 8,000 users plus 1 at 65,000). Plan `d-rmap-dir-1-membership-closure-focus-v1.md` §9. + +- **Completeness:** + - F binds one chain and projects the full B+ closure through down rows, so paths through unfocused groups are represented (tested on two-hop cases for every chain); + - it refuses >256 groups, duplicates, a stale B+ or focus, and unfocused groups; + - the witness now compares the `Snapshot` by pointer, so two stores both starting at `VersionId(0)` refuse each other; + - 8 guards disable-verified red. +- **Preparation:** + - down-row projection costs 1.6–2.7 µs at 64 groups, against 72–82 µs for round 1's per-slot test, and that moves F's break-even from ~1,000 reuses to 2–100; + - eager per-user rows cost 84–95 µs and 32 B per user; lazy rows fill at ~16 ns per user; + - every arm's hot path makes 0 allocations. +- **Working set:** + - B+'s closure passing L2 cost it only 5–17 % (warm vs hot); it slows with row width (218 → 436 ns from 1.5k to 11k groups); + - F stays at 74–79 ns, a 2.9× → 5.5× advantage; + - cold: F ~2×, F+users ~4×. +- **Question shape decides:** + - composite membership (one bit): F 1.7–2.9× at every focus size; + - conjunctions: 2.4–5.4× up to ~650 questions, but at 256 groups (~4,300 conjunctions, 135 KiB of masks, more than L1d) F's mask form is 1.5–1.7× *slower* than B+. +- **Rebuilding a focus per question never pays;** question-aware B+ (needed words only) never helps. + +OPEN: real directories' question-to-group ratio; other CPUs (small L3); the gather admission (`ISS-ADMISSION-PREP-BINDING`); D-RMAP-DIR-2 not authorized. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index a6778bfd0..bbc63a231 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -390 entries, 2026-08-06 .. 2026-10-20. +391 entries, 2026-08-06 .. 2026-10-20. | date | entry id | finding | file | |---|---|---|---| @@ -39,6 +39,7 @@ exactly one of them, never both. | 2026-10-20 | `D-TGA-SLOT-AXIS` | thoughts are 64 slots, and the slot number is CE64's witness pointer | [2026-10-20-d-tga-slot-axis-witness-pointer.md](2026-10-20-d-tga-slot-axis-witness-pointer.md) | | 2026-10-20 | `D-RPR` | | [2026-10-20-d-rpr-consolidation-1624.md](2026-10-20-d-rpr-consolidation-1624.md) | | 2026-10-20 | `D-RPR-1` | | [2026-10-20-d-rpr-1-handoff-lab-demo.md](2026-10-20-d-rpr-1-handoff-lab-demo.md) | +| 2026-10-20 | `D-RMAP-DIR-1` | | [2026-10-20-d-rmap-dir-1-round-2-focus-geometry.md](2026-10-20-d-rmap-dir-1-round-2-focus-geometry.md) | | 2026-10-20 | `D-RMAP-DIR-1` | | [2026-10-20-d-rmap-dir-1-membership-closure-focus.md](2026-10-20-d-rmap-dir-1-membership-closure-focus.md) | | 2026-10-20 | `D-RMAP-2` | | [2026-10-20-d-rmap-2-tile-gemm-cell.md](2026-10-20-d-rmap-2-tile-gemm-cell.md) | | 2026-10-20 | `D-RMAP-2-AMX` | | [2026-10-20-d-rmap-2-amx-tier.md](2026-10-20-d-rmap-2-amx-tier.md) | diff --git a/.claude/plans/d-rmap-dir-1-membership-closure-focus-v1.md b/.claude/plans/d-rmap-dir-1-membership-closure-focus-v1.md index 0ec7bdd87..86537b3d9 100644 --- a/.claude/plans/d-rmap-dir-1-membership-closure-focus-v1.md +++ b/.claude/plans/d-rmap-dir-1-membership-closure-focus-v1.md @@ -1,6 +1,6 @@ # D-RMAP-DIR-1: compiled membership closure vs a 256-slot focus over dir-sim -> **Status: MEASURED, lab only (2026-10-20).** Code: `crates/dir-rmap-probe` (own workspace, CI `dir-rmap-probe.yml`). Nothing is admitted; no production file changed. dir-sim stays authoritative and is read only through its public API. Related: D-RMAP-1/1A (#1617), the `Focus`/`Csr`/`Tile` probe (#1621), D-RPR (#1624), walk_mask (#1634), `ISS-ADMISSION-PREP-BINDING`, the 2026-10-16 no-gather ruling. +> **Status: MEASURED, lab only (2026-10-20); round 2 (constraints 1–7) measured the same day, §9.** Nothing is promoted. Code: `crates/dir-rmap-probe` (own workspace, CI `dir-rmap-probe.yml`). Nothing is admitted; no production file changed. dir-sim stays authoritative and is read only through its public API. Related: D-RMAP-1/1A (#1617), the `Focus`/`Csr`/`Tile` probe (#1621), D-RPR (#1624), walk_mask (#1634), `ISS-ADMISSION-PREP-BINDING`, the 2026-10-16 no-gather ruling. ## 1. Question @@ -20,6 +20,10 @@ Does `lance-graph-dir-sim` gain more from sharing compiled closure results acros | B+ | `prep::Compiled` | B + one closure row per group for each chain (every group, security, mail), up and down, by a fixpoint over the nesting CSR; roles carried by each group's SID chain; incremental insertion of a nesting pair | | F | `focus::Focus` | B+ closure rows projected onto up to 256 bound groups (4 words per group); reverse map is a direct `u16` array over group ordinals | | F+users | `focus::FocusUsers` | F + one 4-word focus row per user (the per-user pointer cache, measured so it is not argued) | +| F lazy | `focus::FocusLazy` | F + per-user rows computed on first use (round 2) | +| B+w | bench only | B+ reading only the closure words that hold a question group (question-aware B+, no focus; round 2) | + +Round 2: F is bound per chain (all, security, mail), and its preparation is split into slot binding, closure projection and per-user rows, each timed and counted. Role and location are ordinary groups. A role × location combination is a composite group nested in both, so no new membership semantics exists. "In role R and in location L" is asked of the closure (W5), never stored as a membership. @@ -33,7 +37,7 @@ At 8,000 users this gives 35,314 direct rows, 1,996 nesting pairs and 17 groups ## 4. Correctness -- `tests/equivalence.rs`, 5 tests on `Spec::small()` (600 users): +- `tests/equivalence.rs`, 5 tests on `Spec::small()` (600 users); round 2 brings it to 11, see §9.2: - B, B+ and D equal A on every user and group, for all three chains, roles and direct membership (every user × group pair); - F and F+users equal A and D on every role × location question for 16, 64 and 256 focus groups (more than 10,000 questions); - after an overlay (membership adds and removals, a nesting pair closing a cycle, a team nested in a role), incremental B+ equals a full recompile, and both equal A and D; @@ -123,7 +127,7 @@ One evaluation is one user against every question in the focus. | F (256 slots) | 50 KiB | | F+users | 300 KiB (2.0 MiB at 65,000) | -## 6. Decisions on the criteria +## 6. Decisions on the criteria (round 1; F revised in §9.6) - **B: yes.** The dominant cost in A is a scan of every membership row on every call. B removes it, with a 31× to ~4,000× gain per call. - **B+: yes, when users share inheritance.** @@ -132,7 +136,7 @@ One evaluation is one user against every question in the focus. - The compile costs 1.6 ms, independent of user count. - A nesting change is a 2 µs incremental insertion; a removal is a full recompile. - Group → members gains nothing over B, since both read the same direct-user rows. -- **F: not a meaningful general advantage.** Its gain is bounded: up to 2× per evaluation with 64 or fewer focus groups, and only after about 1,000 evaluations reuse one prepared focus, on warm caches. At 128 groups it is 1.3×. Near 256 groups there is no gain, and none when cold. F+users trades about 33 bytes per user for the lowest evaluation cost on small foci; its break-even is about 3,000 evaluations. +- **F (round 1, superseded by §9.6):** not a meaningful general advantage. Its gain is bounded: up to 2× per evaluation with 64 or fewer focus groups, and only after about 1,000 evaluations reuse one prepared focus, on warm caches. At 128 groups it is 1.3×. Near 256 groups there is no gain, and none when cold. F+users trades about 33 bytes per user for the lowest evaluation cost on small foci; its break-even is about 3,000 evaluations. - **Answer to the question:** sharing compiled closure results across users is where the acceleration comes from. Reusing a local geometry adds a small, conditional gain on top. A dense global closure is already compact at this directory size: 24 words per row, resident in L2. ## 7. Boundaries and what this does not settle @@ -154,3 +158,161 @@ One evaluation is one user against every question in the focus. - dir-sim's own tests and this probe's oracle as the equivalence gate. It waits on two things: a ruling on the hot-path read pattern (§7) and explicit authorization. F is not proposed for production. + +## 9. Round 2: the seven constraints (2026-10-20) + +Asked: is reusable 256-slot relationship geometry worth anything beyond compiled global closure, and under which workload conditions? Lab only. Nothing is promoted; no production file, canonical SoA or identity authority changes; the no-gather rule stands; `ISS-ADMISSION-PREP-BINDING` stays open. + +Setup is as in §5: Xeon @ 2.10 GHz with AVX-512, L1d 48 KiB, L1i 32 KiB, L2 2 MiB, L3 260 MiB, 4 cores. Four full runs at 8,000 users agreed on every pattern below. Figures are the last run. Ranges are given where runs differed by more than 15 %. + +### 9.1 What changed in the code + +- **F per chain.** `Focus::bind(c, chain, groups, witness)` projects the closure of the chain it is bound to. F is no longer limited to every-group membership. +- **Projection from down rows.** For each focus group, F walks that group's downward closure row and sets its slot in every group the row names. The cost is the size of the focus groups' closures. Round 1 tested every slot of every group's upward row (`project_by_test`, kept as the reference). The two agree on every group. At 64 groups the down-row walk takes 1.6–2.7 µs and the reference 72–82 µs, about 30×. **This is what moved F's break-even**; see §9.5. +- **Split preparation:** slots, then projection, then per-user rows. Eager per-user rows are `FocusUsers`; lazy ones are `FocusLazy`. +- **Incremental refresh.** After a nesting pair, F re-projects the child's downward closure. After membership edits, F recomputes only the edited users' rows. Every refresh takes its witness from the B+ it read. +- **Refusals:** + - more than 256 groups or a duplicate group; + - a stale B+; + - a stale focus, even over a current B+; + - a group outside the focus. +- **The witness names the snapshot.** It now holds a `Weak`, compared by pointer. This answers a Codex review finding: two stores both number their first version 0. + +### 9.2 Constraint 2: closure correctness, and a refusal for anything incomplete + +New tests in `tests/equivalence.rs`, 11 in total: + +| test | what | +|---|---| +| `a_focus_on_every_chain_equals_dir_sim_and_the_oracle` | 256 spread groups, every chain, every user × slot. F, eager rows and lazy rows each equal A (dir-sim's own chain walks) and D (crepe). The down-row projection equals the reference. Anti-vacuity: each chain has more than 1,000 positive answers, and the filtered chains have fewer than the all-groups chain. | +| `a_path_through_unfocused_groups_is_represented` | For every chain, at least 20 users whose membership needs two or more hops. The focus holds only the target group, so every intermediate group is outside the focus, and it still answers yes. | +| `a_refreshed_focus_equals_a_fresh_bind_dir_sim_and_the_oracle` | Overlay: membership adds and removals, a nesting pair that closes a cycle, a team nested in a role. The focus refuses before the refresh. After the incremental refresh, every chain, both eager and lazy rows, equals a fresh bind, A and D. | +| `composite_membership_and_policy_conjunction_are_distinct_questions` | Constraint 5, see §9.4. | +| `a_focus_refuses_what_it_cannot_answer_completely` | Refusals: more than 256 groups, a duplicate, a stale B+, an unfocused group, and a stale focus over a current B+. | +| `a_witness_from_another_store_is_refused` | Two stores over the same data share `VersionId(0)` and the same counts, yet still refuse each other. | + +Disable runs. The work was committed first, then each guard was removed and the file restored with `git checkout`. Every one went red: + +| guard removed | result | +|---|---| +| projection walks the full closure; disabled: walks the focus group and its direct children only (a focus-local closure) | red, 3 tests: the unfocused-path, per-chain and refresh tests | +| refresh re-projects the child's closure | red (refresh test) | +| eager rows recomputed for refreshed users | red (refresh test) | +| lazy cache invalidated on nesting refresh | red (refresh test) | +| bind checks the B+ witness | red (refusal test) | +| bind refuses more than 256 groups | red: panics past the slot range | +| `ask` checks the focus's own witness | **green at first**: the test only asked through a stale B+, so the B+ check refused before the focus check mattered. A current-B+/stale-focus case was added; then red. | +| witness compares the snapshot pointer | red (`a_witness_from_another_store_is_refused`) | + +### 9.3 Constraint 1: L1/L2 geometry and the measured working set + +**Group-count sweep.** 8,000 users, the same 150 conjunction questions in every row, a focus of 64 groups. The first version of this table let the question count fall with scale, from 658 to 176. That confounded the trend and was caught before any conclusion was drawn. ns per evaluation. + +| groups | closure words | one chain up | B+ working set (all users) | F working set | touched per eval: B+ / B+w / F | warm B+ / B+w / F / F+u | hot (one user) B+ / F | cold B+ / B+w / F / F+u | +|---|---|---|---|---|---|---|---|---| +| 1,495 | 24 | 280 KiB | 272 KiB | 45 KiB | 1.0 / 0.8 / 4.8 KiB | 218 / 279 / 74 / 58 | 208 / 60 | 2,103 / 2,290 / 1,121 / 621 | +| 2,895 | 46 | 1.0 MiB | 941 KiB | 82 KiB | 1.9 / 1.2 / 4.8 KiB | 318 / 363 / 76 / 57 | 301 / 61 | 2,161 / 2,364 / 1,170 / 654 | +| 5,695 | 89 | 3.9 MiB | 3.1 MiB | 143 KiB | 3.6 / 1.6 / 4.8 KiB | 344 / 407 / 76 / 57 | 308 / 64 | 2,786 / 3,168 / 1,334 / 703 | +| 11,295 | 177 | 15.3 MiB | 10.1 MiB | 235 KiB | 6.7 / 1.5 / 4.8 KiB | 436 / 466 / 79 / 57 | 372 / 62 | 2,602 / 3,075 / 1,264 / 669 | + +What the measurements show, as distinct from what the sizes suggest: + +- **B+'s working set passes L2 (2 MiB) between 2,895 and 5,695 groups, and that costs it little.** Warm (users cycled) is within 5–17 % of hot (one user repeated) at every size. B+ grows from 218 to 436 ns because its row grows from 24 to 177 words, not because of misses. A single evaluation reads 1–7 KiB, sequentially, and L3 is 260 MiB. +- **F is flat at 74–79 ns, and F+users at 57 ns, across a 7.5× range of groups.** Its working set stays under 240 KiB, inside L2. The per-evaluation advantage over B+ grows with group count: 2.9× at 1,495 groups, 5.5× at 11,295. +- **Cold, both arms pay a few misses.** F is 1.9–2.2× faster than B+, and F+users 3.4–3.9×. +- **The 8 KiB focus "fits in L1" only for the focus row itself.** What decides the speed is the question set. At 256 focus groups the composite fixture yields 4,312 conjunctions. As 4-word masks that is 135 KiB, beyond L1d. F with masks then takes 8.6–10.1 µs per evaluation, *slower* than B+ at 5.7–6.1 µs. The same focus with each question as two slot bits takes 4.5–4.9 µs. At 64 groups (658 questions, 21 KiB of masks) the mask form is the fastest, at 275–332 ns against 663–676 ns for slot bits. So the cost is set by the question representation crossing L1, not by the 256-slot geometry. +- **B+w (question-aware B+, reading only the needed closure words) never helps.** It reads fewer bytes but gathers scattered words, and is equal to or slower than B+ in every row. + +### 9.4 Constraint 5: composite membership vs policy conjunction + +Two questions, kept separate in three fixtures (§3 plus `Spec::without_composites()`): + +- **W5b, materialized composite** ("is the user in composite C"): one membership bit per question. The question count equals the focus size. +- **W5a, policy conjunction on the composite fixture** ("in role R and in location L"): two bits. The question set is the distinct (role, location) pairs that users hold. +- **W5c, policy conjunction on a fixture without composites** (895 groups, 4,757 pairs): every user holds its role and location directly. The test checks that the conjunction question adds no membership: the oracle's membership count equals the direct and nesting closure. + +Neither form adds dir-sim semantics: a composite is an ordinary nested group, and a conjunction is an AND asked of the closure. + +Per evaluation, warm, ns, 8,000 users: + +| focus groups | W5b composite B+ / F / F+u | W5a conjunction B+ / F / F+u (questions) | W5c conjunction B+ / F / F+u (questions) | +|---|---|---|---| +| 16 | 49 / 24 / 8 | 79 / 33 / 18 (41) | 196 / 36 / 22 (52) | +| 64 | 107–191 / 53–66 / 36–44 | 805–1,188 / 275–332 / 337–440 (658) | 888–914 / 260–308 / 303–408 (644) | +| 128 | 176 / 76–105 / 60–78 | 2,810–2,933 / 2,215–2,449 / 2,291–2,557 (2,101) | 2,728–3,110 / 1,977–2,128 / 2,027–2,049 (1,954) | +| 256 | 320–331 / 192–202 / 189–232 | 5,672–6,101 / 8,605–10,097 / 8,936–10,124 (4,312) | 5,708–5,961 / 7,907–10,366 / 8,370–9,687 (4,208) | + +- Composite membership is a cheap question. F's gain over B+ stays at 1.7–2.9× across every focus size, because the number of questions is at most 256. +- Conjunctions multiply questions as the focus grows. F gains 2.4–5.4× up to 64 groups, then the question loop dominates, and past L1 the mask form loses (§9.3). + +### 9.5 Constraints 3 and 4: preparation accounting and amortization + +**Preparation at 64 focus groups, 8,000 users** (W5a, composite fixture). Each phase alone; allocations are counted by a global counting allocator. + +| phase | time | allocations | bytes allocated | retained | +|---|---|---|---|---| +| B prepare (CSR tables and planes) | 2.55 ms | 52 | 562 KiB | 214 KiB | +| B+ compile (3 chains × up/down, roles) | 1.57 ms | 7 | 1.65 MiB | 1.65 MiB | +| F focus construction (slots + reverse map) | 0.2 µs | 2 | 3.1 KiB | 3.1 KiB | +| F closure projection (down rows) | 1.6–2.7 µs | 1 | 46.7 KiB | 46.7 KiB | +| F closure projection by test (round 1's method) | 72–82 µs | 1 | 46.7 KiB | 46.7 KiB | +| eager per-user rows, all users | 84–95 µs | 1 | 250 KiB | 250 KiB | +| lazy per-user rows: allocate the cache | 5.4–5.9 µs | 2 | 251 KiB | 251 KiB | +| lazy per-user rows: first touch | 15–18 ns per user | 0 | 0 | 0 | +| refresh after 80 membership edits, eager rows | 0.3 µs | 0 | 0 | 0 | +| one narrow nesting pair (child closure 1 group): B+ insertion / F / F+users | 2.3–3.1 / 0.4 / 2.1 µs | 5 / 0 / 1 | 16 KiB / 0 / 212 B | 8 KiB / 0 / 0 | +| one wide nesting pair (child closure 648 groups): B+ / F / F+users | 10.4 / 33–45 / 143–153 µs | 5 / 0 / 9 | 16 KiB / 0 / 46 KiB | 8 KiB / 0 / 0 | + +The hot path of every arm allocates nothing: 0 allocations over 8,000 evaluations per arm, in every W5 row. Eager rows cost 32 B per user. The lazy cache costs the same memory but fills rows at about 16 ns each on first use. It pays off only when a small fraction of users is asked about. + +**Amortization.** The B+ compile is shared by every arm, so it drops out of the comparison: + +- T_B+(N) = N · e_B+ +- T_F(N) = P_bind + P_proj + N · e_F +- T_F+u(N) = P_bind + P_proj + P_users + N · e_F+u +- N* = P / (e_B+ − e_F), when e_B+ > e_F; otherwise there is no crossover. + +"Measured" is the smallest N in {1, 2, 5, 10, …, 50,000} at which the directly timed total of F, bind included, is at or below the timed total of B+. + +| workload, focus groups | P_bind + P_proj | e_B+ / e_F | N* computed | N* measured | F+users: P_users, N* computed / measured | rebuild per question (bind + project + 1 eval) | +|---|---|---|---|---|---|---| +| W5a, 16 | 1.3 µs | 79 / 33 ns | 27–36 | 50 | 93 µs, 1,539–1,730 / 2,000 | 1.3–1.8 µs | +| W5a, 64 | 2.6–2.8 µs | 805–1,188 / 275–332 ns | 3–5 | 10 | 91–95 µs, 131–201 / 200 | 2.9–3.8 µs | +| W5a, 128 | 4.0–4.9 µs | 2,810–2,933 / 2,215–2,449 ns | 6–9 | 5 | 90–102 µs, 168–271 / 200–1,000 | 6.9–7.0 µs | +| W5a, 256 | 6.5–7.0 µs | 5,672–6,101 / 8,605–10,097 ns | none | none ≤ 50,000 | none | 18.5–26 µs | +| W5b, 16–256 | 1.3–5.1 µs | 49–331 / 24–202 ns | 16–62 | 50–100 | 85–96 µs, 594–2,351 / 1,000–5,000 | 1.2–5.1 µs | +| W5c, 16–128 | 0.6–2.0 µs | 196–3,110 / 36–2,128 ns | 2–4 | 2–5 | 84–93 µs, 89–518 / 100–500 | 0.5–4.5 µs | +| W5c, 256 | 3.3 µs | 5,708–5,961 / 7,907–10,366 ns | none | none ≤ 50,000 | none | 14–17 µs | + +- **Reuse over one prepared population pays after 2–100 evaluations.** Round 1 put this at ~1,000. The whole difference is the projection: round 1 tested every group's row for every slot. +- **Rebuilding a focus for every question never pays.** Per question it costs 0.5–26 µs, against 0.05–6 µs for a B+ evaluation, in every row. +- **F+users needs 100–5,000 evaluations**, because it builds 8,000 rows up front. It wins per evaluation only at small foci, and lazy rows cover the case where few users are asked about. +- The gaps between computed and measured N* come from the grid of N and from noise at very small N. The measured value is the one to rely on. + +### 9.6 Answer and decisions + +**Does reusable 256-slot relationship geometry add anything beyond compiled global closure? Yes, under conditions, and none of it changes B or B+.** + +- **Conditions under which F gains:** + - the questions fit L1 as masks (in this fixture, up to about 650 conjunctions or any number of single-bit composite questions); + - the focus is reused at least 2–100 times; + - the directory has more groups than a closure row comfortably covers. + + Then F is 2–5× faster per evaluation warm and about 2× cold (F+users about 4×). The gain grows with group count, from 2.9× at 1,495 groups to 5.5× at 11,295, because F's cost does not depend on the closure row width. +- **Conditions under which it does not gain:** + - one question per focus, since rebuilding never pays; + - question sets past L1: about 4,300 conjunctions over 256 groups, where F's mask form is 1.5–1.7× slower than B+ and the slot-bit form only 1.2–1.3× faster; + - group → members (W3), where F does not apply. +- **The L1 picture.** The nominal 8 KiB tile fits L1 but decides nothing. The working set that matters is the question set plus the focus rows of the users' direct groups (45–235 KiB for every group, in L2). B+'s own working set passing L2 cost it 5–17 % at most here. +- **Round 1's F verdict is revised, not reversed.** The "~1,000-evaluation break-even" was an artifact of the projection method. The remaining caveats are unchanged: the gain is bounded, conditional on reuse and on question shape, and F is still not proposed for production. Promotion was not asked for and is not made. +- **Unchanged:** + - B is yes; + - B+ is yes; + - D-RMAP-DIR-2 is not authorized; + - the hot-path reads in B, B+ and F are still gathers (§7), and `ISS-ADMISSION-PREP-BINDING` stays open. + +### 9.7 What round 2 does not settle + +- The data is still synthetic. Real directories may have a different question-to-group ratio, and that ratio decides the conjunction rows. +- Only one machine was measured: a 4-core VM with a 260 MiB L3. On a CPU with a small L3, B+'s past-L2 rows would cost more than measured here. +- Not built: a dense register over the question set itself (for example, questions grouped by role, so one row test answers many), and a lazy focus that projects only the groups a user actually reaches. diff --git a/crates/dir-rmap-probe/examples/bench.rs b/crates/dir-rmap-probe/examples/bench.rs index 9dd1d7d6e..d7c5203c7 100644 --- a/crates/dir-rmap-probe/examples/bench.rs +++ b/crates/dir-rmap-probe/examples/bench.rs @@ -574,9 +574,8 @@ fn main() { } } let rebind_ids = focus_ids_for(&full1, &all_templates, &ord1); - let t_rebind = time(9, || { - Focus::bind(&full1, Chain::All, &rebind_ids, &w.witness()) - }); + let w_now = w.witness(); + let t_rebind = time(9, || Focus::bind(&full1, Chain::All, &rebind_ids, &w_now)); println!("## W7/W8: overlay ({membership_edits} membership edits + 1 nesting edge)\n"); println!("| step | time |\n|---|---|"); println!(