From d58295c11f7cda9ecfedb5e6b70bbf238c3df8f5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 07:44:55 +0000 Subject: [PATCH 1/3] Query: saved views with edge sort over Quack; views through the IAM gate (D-QRY-3) A SavedView is data: object, filter groups, one sort key, columns. page() returns one page in the view's order without ORDER BY: storage order with no key; for an I32 key the values at the page's first and last rank are found by Count-threshold bisection and only the band between them is sorted; for a code key one GroupReduce Count picks the codes overlapping the page. Ties go by ordinal, records without a value last. hubspo-compat::views::open_view refuses a view whose filter or sort reads a hidden property and drops hidden columns. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PPcakjk7ioQsnasmNDtc5z --- crates/hubspo-compat/src/lib.rs | 3 + crates/hubspo-compat/src/views.rs | 177 ++++++++++++ crates/hubspo-query/src/lib.rs | 3 + crates/hubspo-query/src/view.rs | 349 +++++++++++++++++++++++ crates/hubspo-query/tests/view_oracle.rs | 319 +++++++++++++++++++++ 5 files changed, 851 insertions(+) create mode 100644 crates/hubspo-compat/src/views.rs create mode 100644 crates/hubspo-query/src/view.rs create mode 100644 crates/hubspo-query/tests/view_oracle.rs diff --git a/crates/hubspo-compat/src/lib.rs b/crates/hubspo-compat/src/lib.rs index 3dd59a9..188bff5 100644 --- a/crates/hubspo-compat/src/lib.rs +++ b/crates/hubspo-compat/src/lib.rs @@ -7,5 +7,8 @@ //! - [`contacts`]: the first authorized output path (D-IAM-3): one contact //! by id, and the gate a search's filters and sorts pass before they run. //! The REST wire format around it lands with D-API-1. +//! - [`views`]: a saved view (D-QRY-3) through the same gate: a hidden +//! filter or sort refuses it, a hidden column is dropped. pub mod contacts; +pub mod views; diff --git a/crates/hubspo-compat/src/views.rs b/crates/hubspo-compat/src/views.rs new file mode 100644 index 0000000..1d69bf8 --- /dev/null +++ b/crates/hubspo-compat/src/views.rs @@ -0,0 +1,177 @@ +//! A saved view through the D-IAM-3 gate. +//! +//! A view is shared data: whoever opens it gets its filters, sort and +//! columns. The request's access decides what that may reveal: +//! +//! - a filter or the sort on a property the request may not read refuses +//! the whole view, since which records match (or their order) would leak +//! the hidden values; +//! - a column the request may not read is dropped from the output, the same +//! way [`crate::contacts::lookup`] drops a hidden property: absent, not +//! blanked. + +use crate::contacts::{SearchRefusal, check_search}; +use hubspo_iam::access::RequestAccess; +use hubspo_iam::fields::{Schema, Stamp}; +use hubspo_query::view::SavedView; + +/// Check `view` against the request's access; on success, the columns the +/// request may see, in the view's order. +/// +/// # Errors +/// +/// See [`SearchRefusal`]. +pub fn open_view( + access: &RequestAccess, + current: &Stamp, + schema: &Schema, + view: &SavedView, +) -> Result, SearchRefusal> { + let sorts: Vec<&str> = view.sort.iter().map(|s| s.property.as_str()).collect(); + check_search(access, current, schema, &view.groups, &sorts)?; + let projection = access.projection(current).map_err(SearchRefusal::Refused)?; + Ok(view + .columns + .iter() + .filter_map(|name| { + let ordinal = schema.ordinal(name)?; + let p = schema.property(ordinal)?; + projection.admits(current, schema, name).then_some(p.name) + }) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + use hubspo_generated::standard_properties::{CONTACT, PACKED_RUN_POSITION, Slot}; + use hubspo_iam::fields::{ + OrdinalSet, PolicyVersion, Refusal, RoleVersion, SchemaVersion, SeatVersion, + }; + use hubspo_iam::scope::{OwnerScope, RowScope}; + use hubspo_iam::tenant::{ + Assignee, PermissionSet, PermissionSetId, Seat, TenantId, TenantPolicy, + }; + use hubspo_port::ROLE_VIEW_ONLY; + use hubspo_query::filter::{FilterGroup, Operator, PropertyFilter}; + use hubspo_query::view::{Direction, Sort}; + use lance_graph_contract::class_view::WideFieldMask; + use lance_graph_contract::rbac::{ClassGrant, OpMask}; + use ogar_dir_core::Guid128; + use ogar_vocab::class_ids; + + const T: TenantId = TenantId(3); + const C: u16 = class_ids::CRM_CONTACT; + + fn contact() -> Schema { + Schema { + concept: C, + version: SchemaVersion(1), + properties: CONTACT, + } + } + + fn seat() -> Seat { + Seat { + version: SeatVersion(1), + concepts: vec![C], + actions: Vec::new(), + } + } + + /// Reads email, firstname and city. + fn access() -> (RequestAccess, Stamp) { + let lanes = CONTACT.iter().filter_map(|p| match p.slot { + Slot::Lane(n) => Some(n), + Slot::Packed => None, + }); + let every = WideFieldMask::from_positions( + &std::iter::once(PACKED_RUN_POSITION) + .chain(lanes) + .collect::>(), + ); + let mut props = OrdinalSet::new(); + for n in ["email", "firstname", "city"] { + props.insert(contact().ordinal(n).unwrap()); + } + let mut p = TenantPolicy::new(T, PolicyVersion(1)); + p.add_set( + PermissionSetId(1), + PermissionSet { + grants: vec![ClassGrant::new(C, OpMask::READ)], + scopes: vec![(C, OwnerScope::Everything)], + fields: vec![(C, every)], + properties: vec![(C, props)], + ..PermissionSet::default() + }, + ) + .unwrap(); + p.assign(Assignee::Role(ROLE_VIEW_ONLY), PermissionSetId(1)) + .unwrap(); + let a = RequestAccess::compile( + &p, + T, + &seat(), + &Guid128([1; 16]), + &[ROLE_VIEW_ONLY], + RoleVersion(1), + &contact(), + |_| RowScope::All, + ) + .unwrap(); + let now = RequestAccess::current_stamp(&p, &contact(), &seat(), RoleVersion(1)); + (a, now) + } + + fn view(filter_on: &str, sort_on: &str, columns: &[&str]) -> SavedView { + SavedView { + object: "contacts".into(), + groups: vec![FilterGroup { + filters: vec![PropertyFilter::new(filter_on, Operator::Eq, "x")], + }], + sort: Some(Sort { + property: sort_on.into(), + direction: Direction::Ascending, + }), + columns: columns.iter().map(|c| (*c).to_string()).collect(), + } + } + + #[test] + fn a_hidden_column_is_dropped_and_the_rest_keep_their_order() { + let (a, now) = access(); + let v = view( + "email", + "city", + &["city", "date_of_birth", "email", "no_such", "firstname"], + ); + assert_eq!( + open_view(&a, &now, &contact(), &v), + Ok(vec!["city", "email", "firstname"]) + ); + } + + #[test] + fn a_filter_or_sort_on_a_hidden_property_refuses_the_view() { + let (a, now) = access(); + let dob = contact().ordinal("date_of_birth").unwrap(); + assert_eq!( + open_view( + &a, + &now, + &contact(), + &view("date_of_birth", "city", &["email"]) + ), + Err(SearchRefusal::Refused(Refusal::Hidden(dob))) + ); + assert_eq!( + open_view( + &a, + &now, + &contact(), + &view("email", "date_of_birth", &["email"]) + ), + Err(SearchRefusal::Refused(Refusal::Hidden(dob))) + ); + } +} diff --git a/crates/hubspo-query/src/lib.rs b/crates/hubspo-query/src/lib.rs index 45c1a71..d92372a 100644 --- a/crates/hubspo-query/src/lib.rs +++ b/crates/hubspo-query/src/lib.rs @@ -9,9 +9,12 @@ //! mask programs. //! - [`board`] (D-QRY-3, board half): count and amount per pipeline stage //! as two grouped programs. +//! - [`view`] (D-QRY-3, saved-view half): a saved view as data and one page +//! of it in sort order, sorted at the edge over Quack programs. //! //! [`Filter`]: lance_graph_quack::Filter pub mod board; pub mod filter; pub mod list; +pub mod view; diff --git a/crates/hubspo-query/src/view.rs b/crates/hubspo-query/src/view.rs new file mode 100644 index 0000000..fae4f70 --- /dev/null +++ b/crates/hubspo-query/src/view.rs @@ -0,0 +1,349 @@ +//! D-QRY-3 (saved-view half): a saved view as data, and one page of it in +//! sort order over Quack programs. +//! +//! A [`SavedView`] is what HubSpot stores for an index page: the object +//! type, filter groups, at most one sort key and the columns shown. Nothing +//! here owns or shares a view; who may use one, and which of its columns a +//! request may read, is decided at the output edge (`hubspo-compat`). +//! +//! ## Sorting without `ORDER BY` +//! +//! Quack keeps storage order and has no `ORDER BY`, so [`page`] finds a +//! page's records with ordinary programs and sorts only what it must: +//! +//! - **No sort key:** the kept records in storage order (`Agg::Rows`). +//! - **`I32` key:** the values at the page's first and last rank are found +//! by bisection over the value domain, each step one `Agg::Count` with a +//! threshold (≤ 33 programs per bound). Only the band between the two +//! values is materialised and sorted: the page plus the ties at its ends. +//! - **Code key:** one `GroupReduce Count` over the code lane gives every +//! code's count; only the codes that overlap the page are materialised. +//! Codes sort by their numeric identity (the codebook's order). +//! +//! The order is total and fixed: by value, ties by record ordinal ascending +//! in both directions, and records without a value (a clear validity bit) +//! last in both directions, in storage order. The oracle in +//! `tests/view_oracle.rs` states the same order as a plain sort. +//! +//! **Cost.** Materialised rows are the page plus every record sharing a +//! value with the page's first or last record (I32), or every record of a +//! code that overlaps the page (code). A page inside one huge code or one +//! huge tie materialises that whole group; if a page-cost probe shows that +//! dominates, `ORDER BY` is the upstream ask, not before. + +use crate::filter::{CompileError, FilterGroup, compile}; +use lance_graph_quack::bind::{Binder, FieldKind}; +use lance_graph_quack::{Agg, Cmp, Col, Filter, GroupAddr, GroupAgg, Query, normalize_group_sink}; + +/// The direction of a sort. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Direction { + /// Smallest value first. + Ascending, + /// Largest value first. + Descending, +} + +/// A view's sort key: one property. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Sort { + /// The property's internal name. + pub property: String, + /// The direction. + pub direction: Direction, +} + +/// A saved view: data only. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct SavedView { + /// The object type (`contacts`, `deals`, ...). + pub object: String, + /// The filter groups; none keeps every live record. + pub groups: Vec, + /// The sort key; `None` is storage order. + pub sort: Option, + /// The properties shown, in display order. + pub columns: Vec, +} + +/// Executes the programs [`page`] builds. +pub trait Runner { + /// Why execution failed. + type Error; + /// Run an `Agg::Count` query. + /// + /// # Errors + /// + /// The executor's error. + fn count(&mut self, query: &Query) -> Result; + /// Run an `Agg::Rows` query; the kept ordinals, ascending. + /// + /// # Errors + /// + /// The executor's error. + fn rows(&mut self, query: &Query) -> Result, Self::Error>; + /// Run an `Agg::GroupReduce` query into a raw `i64` sink of `groups` + /// slots (not normalised). + /// + /// # Errors + /// + /// The executor's error. + fn groups(&mut self, query: &Query, groups: u32) -> Result, Self::Error>; + /// The size of a code lane's domain (codes are `0..size`). + fn code_domain(&self, col: Col) -> u32; + /// The value of an `I32` lane at `row`. + fn i32_at(&self, col: Col, row: usize) -> i32; +} + +/// Why a page could not be produced. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PageError { + /// The view's filter did not compile. + Compile(CompileError), + /// The sort property does not exist on the object. + UnknownSortProperty(String), + /// Execution failed. + Run(E), +} + +impl From for PageError { + fn from(e: CompileError) -> Self { + PageError::Compile(e) + } +} + +/// The ordinals of `limit` records starting at rank `offset` of `view`, in +/// the view's order. +/// +/// # Errors +/// +/// See [`PageError`]. +pub fn page( + view: &SavedView, + binder: &dyn Binder, + offset: usize, + limit: usize, + runner: &mut R, +) -> Result, PageError> { + let filter = compile(&view.object, &view.groups, binder)?; + let Some(sort) = &view.sort else { + let rows = runner.rows(&rows_query(&filter)).map_err(PageError::Run)?; + return Ok(rows.into_iter().skip(offset).take(limit).collect()); + }; + let table = binder + .table(&view.object) + .ok_or_else(|| PageError::Compile(CompileError::UnknownObject(view.object.clone())))?; + let field = binder + .field(table, &sort.property) + .ok_or_else(|| PageError::UnknownSortProperty(sort.property.clone()))?; + + let (valued, nulls) = match field.validity { + Some(valid) => ( + Filter::and([filter.clone(), Filter::plane(valid)]), + Some(Filter::and([ + filter.clone(), + Filter::negate(Filter::plane(valid)), + ])), + ), + None => (filter.clone(), None), + }; + let n_valued = usize::try_from( + runner + .count(&count_query(&valued)) + .map_err(PageError::Run)?, + ) + .unwrap_or(usize::MAX); + + let end = offset.saturating_add(limit); + let mut out = Vec::with_capacity(limit.min(n_valued.max(1))); + if offset < n_valued { + let r1 = end.min(n_valued); + let ranked = match field.kind { + FieldKind::I32 => i32_page(&valued, field.col, sort.direction, offset, r1, runner)?, + FieldKind::Code => code_page(&valued, field.col, sort.direction, offset, r1, runner)?, + }; + out.extend(ranked); + } + if let Some(nulls) = nulls + && end > n_valued + { + let skip = offset.saturating_sub(n_valued); + let take = end - n_valued.max(offset); + let rows = runner.rows(&rows_query(&nulls)).map_err(PageError::Run)?; + out.extend(rows.into_iter().skip(skip).take(take)); + } + Ok(out) +} + +fn count_query(filter: &Filter) -> Query { + Query { + filter: filter.clone(), + agg: Agg::Count, + } +} + +fn rows_query(filter: &Filter) -> Query { + Query { + filter: filter.clone(), + agg: Agg::Rows, + } +} + +fn with(filter: &Filter, col: Col, cmp: Cmp) -> Filter { + Filter::and([filter.clone(), Filter::cmp(col, cmp)]) +} + +/// The value at rank `r` (0-based) among `valued` records in `direction`. +fn value_at( + valued: &Filter, + col: Col, + direction: Direction, + r: usize, + runner: &mut R, +) -> Result> { + let r = r as u64; + let mut count = |cmp| { + runner + .count(&count_query(&with(valued, col, cmp))) + .map_err(PageError::Run) + }; + // Ascending: the smallest t with |{v ≤ t}| > r. + // Descending: the largest t with |{v ≥ t}| > r. + let (mut lo, mut hi) = (i64::from(i32::MIN), i64::from(i32::MAX)); + while lo < hi { + match direction { + Direction::Ascending => { + let mid = lo + (hi - lo) / 2; + if count(Cmp::LeI32(mid as i32))? > r { + hi = mid; + } else { + lo = mid + 1; + } + } + Direction::Descending => { + let mid = lo + (hi - lo + 1) / 2; + if count(Cmp::GeI32(mid as i32))? > r { + lo = mid; + } else { + hi = mid - 1; + } + } + } + } + Ok(lo as i32) +} + +fn i32_page( + valued: &Filter, + col: Col, + direction: Direction, + r0: usize, + r1: usize, + runner: &mut R, +) -> Result, PageError> { + let first = value_at(valued, col, direction, r0, runner)?; + let last = value_at(valued, col, direction, r1 - 1, runner)?; + let (before_cmp, band) = match direction { + Direction::Ascending => ( + Cmp::LtI32(first), + Filter::and([ + valued.clone(), + Filter::cmp(col, Cmp::GeI32(first)), + Filter::cmp(col, Cmp::LeI32(last)), + ]), + ), + Direction::Descending => ( + Cmp::GtI32(first), + Filter::and([ + valued.clone(), + Filter::cmp(col, Cmp::LeI32(first)), + Filter::cmp(col, Cmp::GeI32(last)), + ]), + ), + }; + let before = runner + .count(&count_query(&with(valued, col, before_cmp))) + .map_err(PageError::Run)?; + let mut band: Vec<(i32, usize)> = runner + .rows(&rows_query(&band)) + .map_err(PageError::Run)? + .into_iter() + .map(|row| (runner.i32_at(col, row), row)) + .collect(); + match direction { + Direction::Ascending => band.sort_unstable(), + Direction::Descending => band.sort_unstable_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))), + } + let skip = r0 - usize::try_from(before).unwrap_or(usize::MAX).min(r0); + Ok(band + .into_iter() + .skip(skip) + .take(r1 - r0) + .map(|(_, row)| row) + .collect()) +} + +fn code_page( + valued: &Filter, + col: Col, + direction: Direction, + r0: usize, + r1: usize, + runner: &mut R, +) -> Result, PageError> { + let domain = runner.code_domain(col); + let mut sink = runner + .groups( + &Query { + filter: valued.clone(), + agg: Agg::GroupReduce { + key: GroupAddr::Local(col), + agg: GroupAgg::Count, + }, + }, + domain, + ) + .map_err(PageError::Run)?; + let present = normalize_group_sink(GroupAgg::Count.fold(), &mut sink); + let count_of = |c: u32| -> usize { + let g = c as usize; + if present[g / 64] >> (g % 64) & 1 == 1 { + usize::try_from(sink[g].unsigned_abs()).unwrap_or(usize::MAX) + } else { + 0 + } + }; + let codes: Box> = match direction { + Direction::Ascending => Box::new(0..domain), + Direction::Descending => Box::new((0..domain).rev()), + }; + let mut out = Vec::with_capacity(r1 - r0); + let mut start = 0usize; + for code in codes { + if start >= r1 { + break; + } + let n = count_of(code); + let stop = start + n; + if n > 0 && stop > r0 { + let rows = runner + .rows(&rows_query(&with(valued, col, Cmp::EqU32(code)))) + .map_err(PageError::Run)?; + let skip = r0.saturating_sub(start); + let take = r1.min(stop) - start.max(r0); + out.extend(rows.into_iter().skip(skip).take(take)); + } + start = stop; + } + Ok(out) +} + +/// The view's filter, compiled; for callers that run their own programs +/// over the same records (a board, a count badge). +/// +/// # Errors +/// +/// The first [`CompileError`]. +pub fn filter(view: &SavedView, binder: &dyn Binder) -> Result { + compile(&view.object, &view.groups, binder) +} diff --git a/crates/hubspo-query/tests/view_oracle.rs b/crates/hubspo-query/tests/view_oracle.rs new file mode 100644 index 0000000..f19420c --- /dev/null +++ b/crates/hubspo-query/tests/view_oracle.rs @@ -0,0 +1,319 @@ +//! D-QRY-3 saved-view half against a plain sort: every page of every view +//! (no sort, an `I32` key and a code key, both directions, nullable and not, +//! filtered and not) equals the same records sorted in Rust by the order +//! `view` documents — value, then ordinal ascending, records without a value +//! last in storage order. + +use std::cell::Cell; + +use hubspo_query::filter::{FilterGroup, Operator, PropertyFilter}; +use hubspo_query::view::{Direction, Runner, SavedView, Sort, page}; +use hubspo_store::catalog::{Catalog, Codebook, FieldSpec}; +use lance_graph_mask_risc::{ + Foreign, LaneRef, Out, Planes, Scratch, Value, execute_into, materialize_rows, words_for, +}; +use lance_graph_quack::bind::{FieldKind, create_table}; +use lance_graph_quack::{Col, Mask, Query, lower}; + +const N: usize = 3_000; +const STAGES: &[&str] = &["lead", "opportunity", "customer", "evangelist", "other"]; + +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +/// Lanes: 0 stage (code), 1 amount (i32, nullable via plane 1), 2 score +/// (i32, never null, wide range), 3 owner (code, nullable via plane 2). +/// Planes: 0 live, 1 amount valid, 2 owner valid. +struct Data { + live: Vec, + amount_valid: Vec, + owner_valid: Vec, + stage: Vec, + amount: Vec, + score: Vec, + owner: Vec, + programs: Cell, +} + +fn set(w: &mut [u64], i: usize) { + w[i / 64] |= 1 << (i % 64); +} +fn get(w: &[u64], i: usize) -> bool { + w[i / 64] >> (i % 64) & 1 == 1 +} + +fn data() -> Data { + let mut r = Rng(0x5A7E_D71E); + let mut d = Data { + live: vec![0; words_for(N)], + amount_valid: vec![0; words_for(N)], + owner_valid: vec![0; words_for(N)], + stage: Vec::new(), + amount: Vec::new(), + score: Vec::new(), + owner: Vec::new(), + programs: Cell::new(0), + }; + for i in 0..N { + if r.below(100) >= 5 { + set(&mut d.live, i); + } + // Narrow range: many ties at every page edge. + if r.below(100) >= 20 { + set(&mut d.amount_valid, i); + } + if r.below(100) >= 10 { + set(&mut d.owner_valid, i); + } + // Stage 4 ("other") is never used: an empty code must be skipped. + d.stage.push(r.below(4) as u32); + d.amount.push(r.below(41) as i32 - 20); + d.score.push(r.next() as i32); + d.owner.push(r.below(3) as u32); + } + // The extremes of the domain must sort correctly too. + d.score[3] = i32::MIN; + d.score[4] = i32::MAX; + set(&mut d.live, 3); + set(&mut d.live, 4); + d +} + +fn catalog() -> Catalog { + let mut c = Catalog::new(); + let t = create_table("contacts") + .width(8) + .register(&mut c) + .expect("register") + .id; + let fields = [ + ( + "lifecyclestage", + Col(0), + FieldKind::Code, + None, + Some(STAGES), + ), + ("amount", Col(1), FieldKind::I32, Some(Mask(1)), None), + ("score", Col(2), FieldKind::I32, None, None), + ( + "hubspot_owner_id", + Col(3), + FieldKind::Code, + Some(Mask(2)), + Some(&["ana", "ben", "cy"][..]), + ), + ]; + for (name, col, kind, validity, codes) in fields { + c.declare( + t, + FieldSpec { + name: name.into(), + col, + kind, + validity, + codebook: codes.map(|c| Codebook::new(c.iter().copied()).expect("codebook")), + }, + ) + .expect("declare"); + } + c +} + +impl Data { + fn run(&self, q: &Query, out: Out<'_>) -> Value { + self.programs.set(self.programs.get() + 1); + let masks: [&[u64]; 3] = [&self.live, &self.amount_valid, &self.owner_valid]; + let lanes = [ + LaneRef::U32(&self.stage), + LaneRef::I32(&self.amount), + LaneRef::I32(&self.score), + LaneRef::U32(&self.owner), + ]; + let planes = Planes { + n_rows: N, + masks: &masks, + lanes: &lanes, + }; + let program = lower(q).expect("lower"); + let mut scratch = Scratch::for_program(&program, N).expect("scratch"); + execute_into(&program, &planes, &Foreign::NONE, &mut scratch, out).expect("execute") + } +} + +impl Runner for Data { + type Error = (); + fn count(&mut self, q: &Query) -> Result { + match self.run(q, Out::None) { + Value::Count(n) => Ok(n as u64), + other => panic!("count gave {other:?}"), + } + } + fn rows(&mut self, q: &Query) -> Result, ()> { + let mut kept = vec![0u64; words_for(N)]; + self.run(q, Out::Mask(&mut kept)); + Ok(materialize_rows(&kept, N)) + } + fn groups(&mut self, q: &Query, groups: u32) -> Result, ()> { + let mut sink = vec![0i64; groups as usize]; + self.run(q, Out::I64(&mut sink)); + Ok(sink) + } + fn code_domain(&self, col: Col) -> u32 { + match col { + Col(0) => STAGES.len() as u32, + Col(3) => 3, + _ => panic!("not a code lane"), + } + } + fn i32_at(&self, col: Col, row: usize) -> i32 { + match col { + Col(1) => self.amount[row], + Col(2) => self.score[row], + _ => panic!("not an i32 lane"), + } + } +} + +/// The documented order, by a plain sort. +fn oracle(d: &Data, keep: &dyn Fn(usize) -> bool, sort: Option<&Sort>) -> Vec { + let rows: Vec = (0..N).filter(|&i| get(&d.live, i) && keep(i)).collect(); + let Some(sort) = sort else { return rows }; + let key = |i: usize| -> Option { + match sort.property.as_str() { + "lifecyclestage" => Some(i64::from(d.stage[i])), + "amount" => get(&d.amount_valid, i).then(|| i64::from(d.amount[i])), + "score" => Some(i64::from(d.score[i])), + "hubspot_owner_id" => get(&d.owner_valid, i).then(|| i64::from(d.owner[i])), + p => panic!("{p}"), + } + }; + let (mut valued, nulls): (Vec, Vec) = + rows.into_iter().partition(|&i| key(i).is_some()); + valued.sort_by(|&a, &b| { + let (ka, kb) = (key(a).unwrap(), key(b).unwrap()); + match sort.direction { + Direction::Ascending => ka.cmp(&kb), + Direction::Descending => kb.cmp(&ka), + } + .then(a.cmp(&b)) + }); + valued.extend(nulls); + valued +} + +fn every_page(d: &mut Data, view: &SavedView, keep: &dyn Fn(usize) -> bool) { + let cat = catalog(); + let want = oracle(d, keep, view.sort.as_ref()); + assert!(want.len() > 1_000, "the fixture must be non-trivial"); + for limit in [1usize, 7, 50, 333] { + let mut offset = 0; + while offset <= want.len() + limit { + let got = page(view, &cat, offset, limit, d).expect("page"); + let end = (offset + limit).min(want.len()); + let expect = if offset < want.len() { + &want[offset..end] + } else { + &[][..] + }; + assert_eq!(got, expect, "{view:?} offset {offset} limit {limit}"); + offset += limit.max(97); + } + } +} + +fn view(sort: Option<(&str, Direction)>, groups: Vec) -> SavedView { + SavedView { + object: "contacts".into(), + groups, + sort: sort.map(|(p, direction)| Sort { + property: p.into(), + direction, + }), + columns: vec!["email".into()], + } +} + +#[test] +fn storage_order_without_a_sort() { + let mut d = data(); + every_page(&mut d, &view(None, vec![]), &|_| true); +} + +#[test] +fn every_sort_key_both_directions_matches_a_plain_sort() { + let mut d = data(); + for property in ["amount", "score", "lifecyclestage", "hubspot_owner_id"] { + for direction in [Direction::Ascending, Direction::Descending] { + every_page(&mut d, &view(Some((property, direction)), vec![]), &|_| { + true + }); + } + } +} + +#[test] +fn a_filtered_view_sorts_only_the_kept_records() { + let mut d = data(); + let groups = vec![ + FilterGroup { + filters: vec![PropertyFilter::new( + "lifecyclestage", + Operator::Eq, + "customer", + )], + }, + FilterGroup { + filters: vec![PropertyFilter::new("score", Operator::Gt, "0")], + }, + ]; + let stage = d.stage.clone(); + let score = d.score.clone(); + let keep = move |i: usize| stage[i] == 2 || score[i] > 0; + for direction in [Direction::Ascending, Direction::Descending] { + every_page( + &mut d, + &view(Some(("amount", direction)), groups.clone()), + &keep, + ); + } +} + +#[test] +fn a_page_costs_programs_independent_of_its_offset() { + let mut d = data(); + let cat = catalog(); + let v = view(Some(("score", Direction::Ascending)), vec![]); + let mut costs = Vec::new(); + for offset in [0usize, 1_000, 2_000] { + d.programs.set(0); + page(&v, &cat, offset, 25, &mut d).expect("page"); + costs.push(d.programs.get()); + } + // count + two bisections (≤ 33 each) + before-count + band rows. + assert!(costs.iter().all(|&c| c <= 1 + 2 * 33 + 2), "{costs:?}"); +} + +#[test] +fn an_unknown_sort_property_is_refused() { + let mut d = data(); + let cat = catalog(); + let v = view(Some(("no_such", Direction::Ascending)), vec![]); + assert_eq!( + page(&v, &cat, 0, 10, &mut d), + Err(hubspo_query::view::PageError::UnknownSortProperty( + "no_such".into() + )) + ); +} From 0c64f677a0c1923798b65a42d8c5ddb86f845ba0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 07:45:38 +0000 Subject: [PATCH 2/3] Board: D-QRY-3 saved-view half Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PPcakjk7ioQsnasmNDtc5z --- .claude/board/LATEST_STATE.md | 35 ++++++++++++++++++++++++++++++++--- .claude/board/STATUS_BOARD.md | 2 +- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index b6f6ac9..99b2e1d 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -187,9 +187,38 @@ universe accepted (1 red), empty stages reported (1 red — the narrow-board test, added after this disable first stayed green: every earlier fixture occupied all 21 stages). -- Open (the saved-view half): view definitions as data, and sorting. - Quack has no `ORDER BY` (order is storage order), so a table view sorted - by a property needs its own decision. +- *Done 2026-10-21, see the next section:* the saved-view half. + +## Saved views (D-QRY-3, saved-view half) + +- `hubspo-query::view::SavedView`: object, filter groups, at most one sort + key, columns. Data only; sharing and column visibility are decided at the + output edge. +- `view::page(view, binder, offset, limit, runner)` returns one page of + ordinals in the view's order without `ORDER BY`: + - no key: `Agg::Rows`, storage order; + - `I32` key: the values at the page's first and last rank by + `Count`-threshold bisection (≤ 33 programs each), then only the band + between them (page + ties at its ends) is materialised and sorted; + - code key: one `GroupReduce Count`, then only codes overlapping the page + are materialised. Codes sort by numeric identity. + - Order: value, ties by ordinal ascending, records without a value last in + both directions. + - `Runner` is the caller's executor (count, rows, groups, code domain, + `i32_at`). +- `tests/view_oracle.rs`: every page (sizes 1, 7, 50, 333) of no-sort, two + `I32` keys (one narrow-range nullable, one full-range with `i32::MIN/MAX`) + and two code keys (one nullable, one with an empty code), both directions, + filtered and not, equals a plain Rust sort over 3,000 records. A page at + offsets 0 / 1,000 / 2,000 costs the same bounded number of programs. +- `hubspo-compat::views::open_view`: a filter or the sort on a hidden + property refuses the view; hidden or unknown columns are dropped. +- Eight disable runs measured red: descending tie-break reversed, + `before` count ignored, nulls dropped, bisection off by one, code order + not reversed, band missing the view's filter, sort left out of the gate, + hidden column kept. +- Open: a page-cost probe at 1M+ rows (the `ORDER BY` question waits on + it), multi-key sorts, view sharing (owner/team/everyone) and persistence. ## Lists (D-QRY-2) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 4d3da00..9d87bf0 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -49,7 +49,7 @@ |---|---|---|---|---| | D-QRY-1 | Filter compiler: HubSpot filter groups (AND within group, OR across groups; operators EQ/NEQ/LT/…/HAS_PROPERTY/CONTAINS_TOKEN/IN/BETWEEN) → Quack program | L | Shipped | 2026-10-14: lance-graph #1546 `lance-graph-relalg` (D-RA-1) normalizes a filter before it runs and answers a contradictory one as `Empty` without a program; D-CMR-1 moves that set algebra into the contract. Adopt it there, not from the prototype crate; `hubspo-store::catalog` + `hubspo-query::filter`; 600 random filter sets equal a plain-loop oracle on mask-risc; #19 | | D-QRY-2 | Lists: active (incremental on version delta) + static (stored mask); membership history | L | Shipped | `hubspo-query::list` (refresh, static add/remove/members, added/removed); tests vs full recomputation. P-LIST-1 2.6 ms at 10M; P-SPARSE-1 1.93 ms, 257 KiB flat at 10M. Storage of list masks lands with D-STO-1 | -| D-QRY-3 | Saved views + board view (report pivot over pipeline × stage) | M | In progress | board half shipped (#21): `hubspo-query::board` (count + amount per stage as two `GroupReduce` programs); saved-view half open: `SavedView` value type + sort at the edge over existing Quack programs (`Agg::Rows` storage order, `Count` threshold bisection for I32, `GroupReduce Count` for coded lanes); ORDER BY is asked upstream only if a page-cost probe fails | +| D-QRY-3 | Saved views + board view (report pivot over pipeline × stage) | M | In progress | board half shipped (#21): `hubspo-query::board` (count + amount per stage as two `GroupReduce` programs); saved-view half shipped 2026-10-21: `hubspo-query::view` (`SavedView`, `page` with edge sort: storage order, `Count`-threshold bisection for I32, `GroupReduce Count` for codes; ties by ordinal, nulls last), oracle over every page, `hubspo-compat::views::open_view` through the D-IAM-3 gate; 8 disable runs red. Open: page-cost probe at 1M+ rows (ORDER BY asked upstream only if it fails), multi-key sort, sharing, persistence | | D-QRY-4 | Quack lowering of callcenter's `ParsedQuery` (the upstream DataFusion `parsed_query_to_plan` is an unimplemented stub) | M | Queued | | | D-QRY-5 | Association-traversing filters ("contacts whose company has an open deal"): Quack `Semijoin` over edge lanes for one hop; multi-hop needs cypher-quack shapes that do not exist yet → upstream request first | L | Queued | 2026-10-20: lance-graph #1632/#1633 (D-SRAM-2 lab spike, `crepe-plan-probe`) measured a semijoin reduction before preparation and an addressed semijoin over attested lanes with masked Lance 13 reads; lab only, not authorization; 2026-10-14: the hop is D-CMR-5, preimage `f⁻¹(S)` under a proven mapping (lance-graph `d-cmr-concept-chain-normalization-v1`), not chained key reads (withdrawn in #1524 / ndarray #359). One hop is ready on the existing Quack `Semijoin`; multi-hop waits on D-CMR-5 rather than on new cypher-quack shapes | | D-QRY-6 | Reports: single-object, cross-object, funnel, time-series, attribution skeleton on `lance-graph-report` | XL | Queued | 2026-10-21 ruling: also owns forecast types (`crm-forecast-types`, 2 functions) and reads D-FLOW-2's custom behavioural events. | From 7681fe7658ba977c9841a7fe45cd32b95953a9a3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 07:50:09 +0000 Subject: [PATCH 3/3] view: empty page for limit 0; open_view binds the view's table to the authorized concept Codex on #45: a zero-length I32 page underflowed r1 - 1, and open_view never checked that the view's object is the concept the access was compiled for, so an unfiltered view of another object passed. The store now answers which concept a table holds, and anything else is refused. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PPcakjk7ioQsnasmNDtc5z --- crates/hubspo-compat/src/contacts.rs | 3 ++ crates/hubspo-compat/src/views.rs | 52 ++++++++++++++++++++++-- crates/hubspo-query/src/view.rs | 3 ++ crates/hubspo-query/tests/view_oracle.rs | 20 +++++++++ 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/crates/hubspo-compat/src/contacts.rs b/crates/hubspo-compat/src/contacts.rs index d45bd7a..f4bf4fa 100644 --- a/crates/hubspo-compat/src/contacts.rs +++ b/crates/hubspo-compat/src/contacts.rs @@ -101,6 +101,9 @@ pub enum SearchRefusal { SchemaMismatch, /// The request holds no read on the concept. NoRead, + /// The view's object is not the concept the access was compiled for, or + /// its table resolves to no concept. + ObjectMismatch, } /// Check a search's filter groups and sort properties against the request's diff --git a/crates/hubspo-compat/src/views.rs b/crates/hubspo-compat/src/views.rs index 1d69bf8..c77e80a 100644 --- a/crates/hubspo-compat/src/views.rs +++ b/crates/hubspo-compat/src/views.rs @@ -18,6 +18,12 @@ use hubspo_query::view::SavedView; /// Check `view` against the request's access; on success, the columns the /// request may see, in the view's order. /// +/// `table_concept` is the store's answer to which concept a table holds. +/// The view's table must resolve to exactly the concept `access` was +/// compiled for, or the view is refused before anything else is read: an +/// unfiltered view of another object would otherwise pass every property +/// check and be paged against a concept nobody authorized. +/// /// # Errors /// /// See [`SearchRefusal`]. @@ -26,7 +32,11 @@ pub fn open_view( current: &Stamp, schema: &Schema, view: &SavedView, + table_concept: impl Fn(&str) -> Option, ) -> Result, SearchRefusal> { + if table_concept(&view.object) != Some(access.stamp().concept) { + return Err(SearchRefusal::ObjectMismatch); + } let sorts: Vec<&str> = view.sort.iter().map(|s| s.property.as_str()).collect(); check_search(access, current, schema, &view.groups, &sorts)?; let projection = access.projection(current).map_err(SearchRefusal::Refused)?; @@ -60,6 +70,15 @@ mod tests { use ogar_dir_core::Guid128; use ogar_vocab::class_ids; + /// The store's table → concept answer in these tests. + fn tables(name: &str) -> Option { + match name { + "contacts" => Some(class_ids::CRM_CONTACT), + "deals" => Some(class_ids::CRM_DEAL), + _ => None, + } + } + const T: TenantId = TenantId(3); const C: u16 = class_ids::CRM_CONTACT; @@ -146,11 +165,36 @@ mod tests { &["city", "date_of_birth", "email", "no_such", "firstname"], ); assert_eq!( - open_view(&a, &now, &contact(), &v), + open_view(&a, &now, &contact(), &v, tables), Ok(vec!["city", "email", "firstname"]) ); } + #[test] + fn a_view_of_another_or_unknown_object_is_refused() { + let (a, now) = access(); + // No filter, no sort: nothing for the property gate to catch. + let mut v = SavedView { + object: "deals".into(), + columns: vec!["email".into()], + ..SavedView::default() + }; + assert_eq!( + open_view(&a, &now, &contact(), &v, tables), + Err(SearchRefusal::ObjectMismatch) + ); + v.object = "no_such_table".into(); + assert_eq!( + open_view(&a, &now, &contact(), &v, tables), + Err(SearchRefusal::ObjectMismatch) + ); + v.object = "contacts".into(); + assert_eq!( + open_view(&a, &now, &contact(), &v, tables), + Ok(vec!["email"]) + ); + } + #[test] fn a_filter_or_sort_on_a_hidden_property_refuses_the_view() { let (a, now) = access(); @@ -160,7 +204,8 @@ mod tests { &a, &now, &contact(), - &view("date_of_birth", "city", &["email"]) + &view("date_of_birth", "city", &["email"]), + tables ), Err(SearchRefusal::Refused(Refusal::Hidden(dob))) ); @@ -169,7 +214,8 @@ mod tests { &a, &now, &contact(), - &view("email", "date_of_birth", &["email"]) + &view("email", "date_of_birth", &["email"]), + tables ), Err(SearchRefusal::Refused(Refusal::Hidden(dob))) ); diff --git a/crates/hubspo-query/src/view.rs b/crates/hubspo-query/src/view.rs index fae4f70..4395b53 100644 --- a/crates/hubspo-query/src/view.rs +++ b/crates/hubspo-query/src/view.rs @@ -126,6 +126,9 @@ pub fn page( runner: &mut R, ) -> Result, PageError> { let filter = compile(&view.object, &view.groups, binder)?; + if limit == 0 { + return Ok(Vec::new()); + } let Some(sort) = &view.sort else { let rows = runner.rows(&rows_query(&filter)).map_err(PageError::Run)?; return Ok(rows.into_iter().skip(offset).take(limit).collect()); diff --git a/crates/hubspo-query/tests/view_oracle.rs b/crates/hubspo-query/tests/view_oracle.rs index f19420c..5178c85 100644 --- a/crates/hubspo-query/tests/view_oracle.rs +++ b/crates/hubspo-query/tests/view_oracle.rs @@ -305,6 +305,26 @@ fn a_page_costs_programs_independent_of_its_offset() { assert!(costs.iter().all(|&c| c <= 1 + 2 * 33 + 2), "{costs:?}"); } +#[test] +fn a_zero_length_page_is_empty_for_every_key() { + let mut d = data(); + let cat = catalog(); + let sorts = [ + None, + Some(("amount", Direction::Ascending)), + Some(("score", Direction::Descending)), + Some(("lifecyclestage", Direction::Ascending)), + ]; + for sort in sorts { + for offset in [0usize, 5, 10_000] { + d.programs.set(0); + let got = page(&view(sort, vec![]), &cat, offset, 0, &mut d).expect("page"); + assert!(got.is_empty(), "{sort:?} {offset}"); + assert_eq!(d.programs.get(), 0, "an empty page runs nothing"); + } + } +} + #[test] fn an_unknown_sort_property_is_refused() { let mut d = data();