From 5b931355b908a53dd813a405dd29550c561c8740 Mon Sep 17 00:00:00 2001 From: pasta Date: Tue, 27 Oct 2026 19:40:36 +0000 Subject: [PATCH 1/2] refactor: drop the protocol-13 paging workarounds Forge's contracts need protocol 14, where Drive pads a startAfter page off the primary key (DriveDocumentQuery::pads_cursor_page, gated on non_primary_key_path_query >= 1). Rows sharing the cursor's $createdAt are no longer skipped and pages are no longer one row short, and descending pages after a cursor prove and verify under the pinned v5.0.0-rc.8 SDK. So the complete readers in forge-core and forge-web page on the cursor alone, in the requested order: - remove the $createdAt boundary tie probe (tie_probe_allowed / tieProbeAllowed, cursorPadded) and its tests - remove the descending-to-ascending rewrite (ascending_equivalent / ascendingEquivalent), so a capped descending read can stop early and a descending page after a cursor is one request again - chrome reads its timeline deltas on every network - keep the short-page completeness rule, the page cap and the no-cursor error Co-Authored-By: Claude Opus 5.5 --- crates/forge-core/src/platform/mod.rs | 374 +++---------------------- docs/BUILDING.md | 22 +- forge-web/lib/repo/chrome.test.ts | 3 +- forge-web/lib/repo/chrome.ts | 6 +- forge-web/lib/repo/list-budget.test.ts | 6 +- forge-web/lib/repo/packs.test.ts | 4 +- forge-web/lib/repo/pagination.test.ts | 130 +-------- forge-web/lib/repo/pull-index.test.ts | 7 +- forge-web/lib/repo/source.test.ts | 27 +- forge-web/lib/sdk/index.ts | 3 - forge-web/lib/sdk/query.test.ts | 26 +- forge-web/lib/sdk/query.ts | 114 +------- 12 files changed, 79 insertions(+), 643 deletions(-) diff --git a/crates/forge-core/src/platform/mod.rs b/crates/forge-core/src/platform/mod.rs index e5c2e94c9..84ea876b3 100644 --- a/crates/forge-core/src/platform/mod.rs +++ b/crates/forge-core/src/platform/mod.rs @@ -1226,31 +1226,11 @@ impl PlatformClient { limit: u32, start_after: Option<&str>, ) -> Result> { - // A descending page after a cursor cannot be proved on protocol 13 under the 4.2 - // verifier (see [`ascending_equivalent`]). When an ascending equivalent exists, read - // the whole set that way (complete, reversed to the requested order) and slice out - // the page after the cursor. This costs a complete read per page; the listings that - // page descending (issues, PRs, stars) are small. - if let (Some(after), Some(_)) = (start_after, ascending_equivalent(filters, order)) { - if limit == 0 { - return Err(Error::Config("query limit must be greater than 0".into())); - } - let all = self - .query_all_documents(contract, document_type, filters, order) - .await?; - let Some(at) = all.iter().position(|d| d.id == after) else { - return Err(Error::Config(format!( - "start_after document {after} is not in the {document_type} result set" - ))); - }; - return Ok(all.into_iter().skip(at + 1).take(limit as usize).collect()); - } self.query_page(contract, document_type, filters, order, limit, start_after) .await } - /// One page exactly as requested — the network half of [`Self::query_documents`], which - /// the complete reader calls directly (it only ever pages ascending after a cursor). + /// One page exactly as requested, refreshing a stale contract once. async fn query_page( &self, contract: &LoadedContract, @@ -1359,8 +1339,6 @@ impl PlatformClient { /// [`Self::query_all_documents`], stopping once more than `max` rows are held: the rows in /// `order`, at least `max + 1` of them when more match (the caller truncates and says so). - /// A descending read that is paged ascending and reversed (see [`ascending_equivalent`]) - /// cannot stop early (its first rows are the last read), so it reads every match. pub async fn query_documents_up_to( &self, contract: &LoadedContract, @@ -1369,19 +1347,9 @@ impl PlatformClient { order: &[QueryOrder], max: usize, ) -> Result> { - // Page a descending read in ascending order and reverse it (see - // [`ascending_equivalent`]): the rs-sdk 4.2 verifier rejects the proof a protocol-13 - // node returns for a descending page that starts after a cursor, so every read past - // the first 100 rows would fail on testnet. - let ascending = ascending_equivalent(filters, order); - let order = ascending.as_deref().unwrap_or(order); - // See [`tie_probe_allowed`]: when the index ends in `$createdAt`, every page boundary - // is followed by a read of the boundary timestamp, so same-block rows are not lost. - let tie_safe = tie_probe_allowed(filters, order); - let max = if ascending.is_some() { usize::MAX } else { max }; - // `async move` so the futures own their inputs; returning a future that borrows the + // `async move` so the future owns its input; returning a future that borrows the // closure's parameter would not outlive the call. - let mut documents = page_until( + page_until( document_type, |start_after: Option| async move { self.query_page( @@ -1394,22 +1362,9 @@ impl PlatformClient { ) .await }, - tie_safe.then_some(|created_at: u64| async move { - let mut tie = filters.to_vec(); - tie.push(QueryFilter::eq( - "$createdAt", - FieldValue::uint64(created_at), - )); - self.query_page(contract, document_type, &tie, order, PAGE_SIZE, None) - .await - }), max, ) - .await?; - if ascending.is_some() { - documents.reverse(); - } - Ok(documents) + .await } /// [`Self::query_all_documents`] for documents of ~15 KB each (pack `chunk`s), whose @@ -1426,8 +1381,8 @@ impl PlatformClient { /// not banned for it (the deadline is ours, not their fault). Rows already read are kept: /// a slow page costs that page, not the read. /// - /// `order` must be all-ascending with no `$createdAt` tie handling needed (chunks are - /// read by `seq`); any other read goes through [`Self::query_all_documents`]. + /// `order` must be all-ascending (chunks are read by `seq`); any other read goes through + /// [`Self::query_all_documents`]. pub async fn query_all_large_documents( &self, contract: &LoadedContract, @@ -1436,7 +1391,7 @@ impl PlatformClient { order: &[QueryOrder], ) -> Result> { // (Its pages refresh and retry one by one.) - if !order.iter().all(|o| o.ascending) || tie_probe_allowed(filters, order) { + if !order.iter().all(|o| o.ascending) { return self .query_all_documents(contract, document_type, filters, order) .await; @@ -3530,88 +3485,21 @@ pub fn decode_identifier(base58: &str) -> Result<[u8; 32]> { /// Rows per page. Drive's default and maximum are both 100. const PAGE_SIZE: u32 = 100; -/// The all-ascending order whose traversal, reversed, is exactly `order`'s traversal — or -/// `None` when `order` is already ascending or has no such equivalent. -/// -/// A complete read ([`PlatformClient::query_all_documents`]) pages with a `start_after` -/// cursor. The grovedb verifier in rs-sdk 4.2 checks that every proof op matches the walk -/// direction. Protocol-13 nodes (testnet today) answer a descending page after a cursor -/// with a proof that fails that check. That page is only needed when a read passes one page, -/// so it surfaced as `packManifest` reads failing on a repo with 101 manifests. Ascending -/// pages verify, and Drive's descending walk is the exact reverse of its ascending walk, -/// so an ascending read reversed returns the same rows in the same order. -/// -/// Reversing the whole result also reverses any clause that was already ascending. That is -/// only harmless when an equality filter pins that clause's field to one value, so any -/// other mixed order returns `None` and is read as requested. -fn ascending_equivalent(filters: &[QueryFilter], order: &[QueryOrder]) -> Option> { - if order.iter().all(|o| o.ascending) { - return None; - } - let pinned = |field: &str| { - filters - .iter() - .any(|f| f.op == QueryOp::Eq && f.field == field) - }; - if order.iter().any(|o| o.ascending && !pinned(&o.field)) { - return None; - } - Some( - order - .iter() - .map(|o| QueryOrder::asc(o.field.clone())) - .collect(), - ) -} - /// Hard safety cap on total rounds, matching forge-web's `queryAllDocuments`. /// 1000 pages x 100 rows = 100k documents. const MAX_PAGES: usize = 1000; -/// Whether a complete read can make its page boundaries tie-safe: its order ends in -/// `$createdAt` ascending, every earlier order field is pinned by an `==` filter, and every -/// filter is an `==` on some other field. The index then ends in `$createdAt`, so an -/// equality on the boundary timestamp is a valid query on the same index. -/// -/// Why: on protocol 13, a `start_after` cursor excludes the cursor's whole `$createdAt` -/// key, not just the rows up to the cursor document. Documents created in the same block -/// as a page's last row that sort after it would be silently skipped. `$createdAt` is the -/// block time, so these ties are real. Protocol 14 bounds the cursor by document id and -/// does not drop them. -/// -/// Every forge-v2 type read with an order ending in `$createdAt` requires it, so a proved -/// query returns the boundary row's -/// timestamp. -fn tie_probe_allowed(filters: &[QueryFilter], order: &[QueryOrder]) -> bool { - let Some((last, rest)) = order.split_last() else { - return false; - }; - let pinned = |field: &str| { - filters - .iter() - .any(|f| f.op == QueryOp::Eq && f.field == field) - }; - last.field == "$createdAt" - && last.ascending - && rest.iter().all(|o| o.ascending && pinned(&o.field)) - && filters - .iter() - .all(|f| f.op == QueryOp::Eq && f.field != "$createdAt") -} - -/// Page a `$id`-cursored query to exhaustion, given a page fetcher and, optionally, a -/// boundary-tie reader. +/// Page a `$id`-cursored query to exhaustion, given a page fetcher. /// /// Transport-free so the loop itself is testable: `query_all_documents` owns a live `Sdk`, /// which made the one piece of logic that decides whether a read is complete the one piece /// with no test. `fetch` receives the `start_after` cursor (`None` for the first page) and /// returns one page. /// -/// `tie_probe`, when given, returns every row whose `$createdAt` equals its argument, in -/// traversal order (see [`tie_probe_allowed`]). After each full page it is called with the -/// last row's `$createdAt`. Rows not yet held are appended, and the cursor moves to the last -/// tied row. A tie of a full page or more cannot be proved complete and fails the read. -/// Rows are deduplicated by `$id` throughout. +/// The cursor alone is enough on protocol 14, the only protocol Forge's contracts run on: +/// Drive continues a `startAfter` page within the cursor's index key by document id, so rows +/// sharing the cursor's `$createdAt` are not skipped (`DriveDocumentQuery::pads_cursor_page`). +/// Rows are deduplicated by `$id`. /// /// A **short page is the only accepted proof** that the end was reached. If the cursor /// cannot advance, or the page cap is hit, this returns [`Error::IncompleteRead`] rather @@ -3619,33 +3507,24 @@ fn tie_probe_allowed(filters: &[QueryFilter], order: &[QueryOrder]) -> bool { /// different points on identical data would itself be the cross-client divergence these /// reads exist to prevent. #[cfg(test)] -async fn page_to_exhaustion( - document_type: &str, - fetch: F, - tie_probe: Option

, -) -> Result> +async fn page_to_exhaustion(document_type: &str, fetch: F) -> Result> where F: FnMut(Option) -> Fut, Fut: std::future::Future>>, - P: FnMut(u64) -> PFut, - PFut: std::future::Future>>, { - page_until(document_type, fetch, tie_probe, usize::MAX).await + page_until(document_type, fetch, usize::MAX).await } /// [`page_to_exhaustion`] that also stops, successfully, once more than `max` rows are held /// (a capped read: the caller knows more may match). -async fn page_until( +async fn page_until( document_type: &str, mut fetch: F, - mut tie_probe: Option

, max: usize, ) -> Result> where F: FnMut(Option) -> Fut, Fut: std::future::Future>>, - P: FnMut(u64) -> PFut, - PFut: std::future::Future>>, { let incomplete = |fetched: usize, reason: String| Error::IncompleteRead { document_type: document_type.to_string(), @@ -3654,23 +3533,20 @@ where }; let mut out: Vec = Vec::new(); let mut held: std::collections::HashSet = std::collections::HashSet::new(); - let mut take = |out: &mut Vec, rows: Vec| { - for d in rows { - if held.insert(d.id.clone()) { - out.push(d); - } - } - }; let mut start_after: Option = None; for _ in 0..MAX_PAGES { let page = fetch(start_after.clone()).await?; let n = page.len(); - let last = page.last().map(|d| (d.id.clone(), d.created_at)); - take(&mut out, page); + let last = page.last().map(|d| d.id.clone()); + for d in page { + if held.insert(d.id.clone()) { + out.push(d); + } + } if n < PAGE_SIZE as usize || out.len() > max { return Ok(out); } - let Some((mut cursor, created_at)) = last else { + let Some(cursor) = last else { // A full page with no last element is impossible, but treating it as "done" // would silently truncate; treat it as unprovable instead. return Err(incomplete( @@ -3678,22 +3554,6 @@ where "a full page yielded no cursor document".to_string(), )); }; - if let (Some(probe), Some(t)) = (tie_probe.as_mut(), created_at) { - let tied = probe(t).await?; - if tied.len() >= PAGE_SIZE as usize { - return Err(incomplete( - out.len(), - format!( - "{PAGE_SIZE} or more documents share $createdAt {t}; the page \ - boundary tie cannot be read completely" - ), - )); - } - if let Some(last_tied) = tied.last() { - cursor.clone_from(&last_tied.id); - } - take(&mut out, tied); - } start_after = Some(cursor); } Err(Error::IncompleteRead { @@ -5111,10 +4971,10 @@ mod tests { } use super::{ - ascending_equivalent, is_contract_missing, is_stale_contract, is_transient_node_error, - page_to_exhaustion, page_until, retry_with_backoff, tie_probe_allowed, FetchedDocument, - FieldValue, JournalStore, PushJournal, QueryFilter, QueryOrder, SignedTransition, - WriteIntent, WriteOp, MAX_PAGES, MAX_READ_ATTEMPTS, NONCE_MASK, PAGE_SIZE, + is_contract_missing, is_stale_contract, is_transient_node_error, page_to_exhaustion, + page_until, retry_with_backoff, FetchedDocument, FieldValue, JournalStore, PushJournal, + SignedTransition, WriteIntent, WriteOp, MAX_PAGES, MAX_READ_ATTEMPTS, NONCE_MASK, + PAGE_SIZE, }; use crate::error::{Error, Result}; use crate::network::{Network, NetworkTarget}; @@ -5414,41 +5274,6 @@ mod tests { assert!(!same_field(None, s(FieldValue::text("x")).as_ref())); } - fn fields(order: &[QueryOrder]) -> Vec<(String, bool)> { - order - .iter() - .map(|o| (o.field.clone(), o.ascending)) - .collect() - } - - #[test] - fn a_descending_complete_read_is_paged_ascending() { - let asc = ascending_equivalent(&[], &[QueryOrder::desc("$createdAt")]).unwrap(); - assert_eq!(fields(&asc), [("$createdAt".to_string(), true)]); - } - - #[test] - fn an_ascending_read_is_left_alone() { - assert!(ascending_equivalent(&[], &[QueryOrder::asc("$createdAt")]).is_none()); - assert!(ascending_equivalent(&[], &[]).is_none()); - } - - #[test] - fn a_mixed_order_flips_only_when_the_ascending_prefix_is_pinned() { - let order = [QueryOrder::asc("kind"), QueryOrder::desc("$createdAt")]; - // `kind` pinned by an equality filter: reversing it is a no-op, so the read flips. - let pinned = [QueryFilter::eq("kind", FieldValue::integer(1))]; - let asc = ascending_equivalent(&pinned, &order).unwrap(); - assert_eq!( - fields(&asc), - [("kind".to_string(), true), ("$createdAt".to_string(), true)] - ); - // Unpinned (or only range-filtered): reversing would reorder `kind`, so it stays. - assert!(ascending_equivalent(&[], &order).is_none()); - let ranged = [QueryFilter::gt("kind", FieldValue::integer(1))]; - assert!(ascending_equivalent(&ranged, &order).is_none()); - } - fn doc(i: usize) -> FetchedDocument { FetchedDocument { id: format!("d-{i:06}"), @@ -5480,132 +5305,17 @@ mod tests { } } - type Ready = std::future::Ready>>; - - fn no_probe() -> Option Ready> { - None - } - - /// A document at `created_at` with id `d-`. - fn doc_at(i: usize, created_at: u64) -> FetchedDocument { - FetchedDocument { - created_at: Some(created_at), - ..doc(i) - } - } - - /// Rows sorted by ($createdAt, $id), served with PROTOCOL-13 cursor semantics: a - /// `start_after` cursor excludes every row whose `$createdAt` is <= the cursor's, so rows - /// sharing the cursor's timestamp but sorting after it are skipped (the real Drive - /// behaviour this guards against). - fn protocol_13(rows: Vec) -> impl FnMut(Option) -> Ready { - move |start_after| { - let from_t = start_after.map(|id| { - rows.iter() - .find(|d| d.id == id) - .and_then(|d| d.created_at) - .unwrap() - }); - let page = rows - .iter() - .filter(|d| from_t.is_none_or(|t| d.created_at.unwrap() > t)) - .take(PAGE_SIZE as usize) - .cloned() - .collect(); - std::future::ready(Ok(page)) - } - } - - /// 99 rows at distinct times, then `tied` rows sharing one timestamp: row 100 is the - /// first of the tie, so the page boundary falls inside it. - fn straddling_tie(tied: usize) -> Vec { - let mut rows: Vec = (0..99).map(|i| doc_at(i, i as u64)).collect(); - rows.extend((99..99 + tied).map(|i| doc_at(i, 1_000))); - rows.push(doc_at(99 + tied, 2_000)); - rows - } - - #[tokio::test] - async fn protocol_13_drops_same_block_rows_at_a_page_boundary_without_the_probe() { - let rows = straddling_tie(3); - let got = page_to_exhaustion("event", protocol_13(rows.clone()), no_probe()) - .await - .unwrap(); - // The mock reproduces the gap: the two tied rows after the boundary are lost. - assert_eq!(got.len(), rows.len() - 2); - } - - #[tokio::test] - async fn the_tie_probe_recovers_same_block_rows_at_a_page_boundary() { - let rows = straddling_tie(3); - let tie_rows = rows.clone(); - let probe = move |t: u64| { - let tied = tie_rows - .iter() - .filter(|d| d.created_at == Some(t)) - .cloned() - .collect(); - std::future::ready(Ok(tied)) - }; - let got = page_to_exhaustion("event", protocol_13(rows.clone()), Some(probe)) - .await - .unwrap(); - let ids: Vec<&str> = got.iter().map(|d| d.id.as_str()).collect(); - let want: Vec<&str> = rows.iter().map(|d| d.id.as_str()).collect(); - assert_eq!(ids, want, "every row, once, in traversal order"); - } - - #[tokio::test] - async fn a_boundary_tie_of_a_full_page_is_refused() { - let rows = straddling_tie(PAGE_SIZE as usize); - let tie_rows = rows.clone(); - let probe = move |t: u64| { - let tied = tie_rows - .iter() - .filter(|d| d.created_at == Some(t)) - .take(PAGE_SIZE as usize) - .cloned() - .collect(); - std::future::ready(Ok(tied)) - }; - let got = page_to_exhaustion("event", protocol_13(rows), Some(probe)).await; - assert!(matches!(got, Err(Error::IncompleteRead { .. })), "{got:?}"); - } - - #[test] - fn the_tie_probe_runs_only_where_created_at_ends_the_index() { - let owner = || QueryFilter::eq("$ownerId", FieldValue::integer(1)); - assert!(tie_probe_allowed(&[], &[QueryOrder::asc("$createdAt")])); - assert!(tie_probe_allowed( - &[owner()], - &[QueryOrder::asc("$ownerId"), QueryOrder::asc("$createdAt")] - )); - // Not ending in $createdAt, an unpinned prefix, a range filter, or descending. - assert!(!tie_probe_allowed(&[], &[QueryOrder::asc("seq")])); - assert!(!tie_probe_allowed( - &[], - &[QueryOrder::asc("$ownerId"), QueryOrder::asc("$createdAt")] - )); - assert!(!tie_probe_allowed( - &[QueryFilter::gt("seq", FieldValue::integer(1))], - &[QueryOrder::asc("$createdAt")] - )); - assert!(!tie_probe_allowed(&[], &[QueryOrder::desc("$createdAt")])); - } - #[tokio::test] async fn pages_past_the_first_page_boundary() { // 101 rows: the case the whole change exists for. A single page would stop at 100. - let got = page_to_exhaustion("event", serve(101), no_probe()) - .await - .unwrap(); + let got = page_to_exhaustion("event", serve(101)).await.unwrap(); assert_eq!(got.len(), 101); assert_eq!(got[100].id, "d-000100"); } #[tokio::test] async fn a_short_first_page_is_the_end() { - let capped = page_until("event", serve(usize::MAX), no_probe(), 150) + let capped = page_until("event", serve(usize::MAX), 150) .await .expect("a capped read stops once past the cap"); assert_eq!( @@ -5613,9 +5323,7 @@ mod tests { 200, "two full pages: the second passes the cap" ); - let got = page_to_exhaustion("event", serve(7), no_probe()) - .await - .unwrap(); + let got = page_to_exhaustion("event", serve(7)).await.unwrap(); assert_eq!(got.len(), 7); } @@ -5624,15 +5332,11 @@ mod tests { // Exactly PAGE_SIZE rows: the first page is full, so the end is NOT yet proven and // a second (empty, therefore short) page must be requested. let mut calls = 0usize; - let got = page_to_exhaustion( - "event", - |start_after| { - calls += 1; - let mut f = serve(PAGE_SIZE as usize); - f(start_after) - }, - no_probe(), - ) + let got = page_to_exhaustion("event", |start_after| { + calls += 1; + let mut f = serve(PAGE_SIZE as usize); + f(start_after) + }) .await .unwrap(); assert_eq!(got.len(), PAGE_SIZE as usize); @@ -5643,7 +5347,7 @@ mod tests { async fn refuses_to_return_a_partial_answer_at_the_page_cap() { // A cursor that always advances and pages that are always full: the end is never // proven, so this must fail rather than hand back 100k rows as if complete. - let got = page_to_exhaustion("event", serve(usize::MAX), no_probe()).await; + let got = page_to_exhaustion("event", serve(usize::MAX)).await; match got { Err(Error::IncompleteRead { document_type, @@ -5659,11 +5363,9 @@ mod tests { #[tokio::test] async fn propagates_a_fetch_error() { - let got = page_to_exhaustion( - "event", - |_| std::future::ready(Err(Error::Platform("boom".into()))), - no_probe(), - ) + let got = page_to_exhaustion("event", |_| { + std::future::ready(Err(Error::Platform("boom".into()))) + }) .await; assert!(matches!(got, Err(Error::Platform(_)))); } diff --git a/docs/BUILDING.md b/docs/BUILDING.md index 15548d665..8fba0fda4 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -158,20 +158,14 @@ Protocol 14 changes three things that a client has to handle: Drive rejects. forge-web's `documentForCreate` builds documents through `Document.fromObject`, which keeps bytes as bytes. -Protocol 13 under the 4.2 verifier has two paging gotchas, which both clients handle in their -complete readers (`query_all_documents` / `queryAllDocuments`): - -- **Descending page after a cursor.** The 4.2 grovedb verifier rejects the proof a - protocol-13 node returns for one ("Proof op family does not match the query direction"). - A descending read is paged ascending and reversed, which gives the same rows in the same - order. -- **Same-block ties.** A `start_after` cursor excludes the cursor's whole `$createdAt`, so - documents created in the same block as a page's last row, and sorting after it, are - skipped. When the index ends in `$createdAt`, the readers also query the boundary - timestamp with `==`. This needs the boundary row's `$createdAt`, which history-keeping - types (`packManifest`, `event`, `refUpdate`, `issue`, ...) do not return from a proved - query on protocol 13. forge-v2 runs only on protocol 14, where the readers bound the - cursor by document id. +The complete readers (`query_all_documents` / `queryAllDocuments`) page on the `startAfter` +cursor alone, in the requested order. On protocol 14 Drive lowers a `startAfter` off the +primary key as a `startAt` on the cursor's own index key, continues within that key by +document id and drops the cursor row (`DriveDocumentQuery::pads_cursor_page`). So documents +created in the same block as a page's last row are not skipped, and pages are not one row +short. Descending pages after a cursor prove and verify like ascending ones. Protocol 13 got +both wrong, and the readers used to work around it; forge-v2 needs protocol 14, so they no +longer do. ## Networks diff --git a/forge-web/lib/repo/chrome.test.ts b/forge-web/lib/repo/chrome.test.ts index 3b452b2f8..385d45e43 100644 --- a/forge-web/lib/repo/chrome.test.ts +++ b/forge-web/lib/repo/chrome.test.ts @@ -16,7 +16,7 @@ vi.mock('../constants', async (importOriginal) => { }) import type { ForgeIds } from '../deployments' -import { base64ToHex, bytesToBase64, hexToBase64, setPlatformVersion, type DocumentQuery } from '../sdk' +import { base64ToHex, bytesToBase64, hexToBase64, type DocumentQuery } from '../sdk' import { CHROME_KEYSET_SPLITS } from './refs' import { loadRepoHome } from '../view/repo-view' import { cachedDpnsName, clearDpnsCache } from '../view/dpns' @@ -184,7 +184,6 @@ function fakeSdk( const REF: RepoRef = { forge: FORGE, repoId: REPO, ownerId: OWNER, name: 'demo', visibility: 'public' } beforeEach(() => { - setPlatformVersion(14) resetRepoTimelines() clearDpnsCache() invalidateMembers(REF, 'devnet') diff --git a/forge-web/lib/repo/chrome.ts b/forge-web/lib/repo/chrome.ts index 37e56832b..8e4e73d9e 100644 --- a/forge-web/lib/repo/chrome.ts +++ b/forge-web/lib/repo/chrome.ts @@ -36,7 +36,7 @@ import type { EvoSDK } from '@dashevo/evo-sdk' import { DEFAULT_NETWORK, NETWORKS, type Network } from '../constants' import type { ForgeIds } from '../deployments' import { compositeOf, countsAt, docsAt, queryComposite, type CompositeSub } from '../sdk/composite' -import { base64ToHex, cursorPadded, queryAllDocuments, shareInFlight, type PlainDocument, type WhereClause } from '../sdk' +import { base64ToHex, queryAllDocuments, shareInFlight, type PlainDocument, type WhereClause } from '../sdk' import { DOC, type RepoRef } from './contract' import { membersGeneration, membershipsFromDocs, seedMemberships } from './members' import { noteForkOf } from './fork-parent' @@ -382,9 +382,7 @@ async function readChrome( const generation = generationOf(key) const membersAtStart = membersGeneration() const held = touch(key) - // Only on protocol 14 is a delta page continued safely: a `startAfter` cursor below it drops - // rows tied with the cursor, and the `>=` bound rules out the tie probe. Read whole there. - const known = cursorPadded() ? held?.settled : undefined + const known = held?.settled const q = compositeOf( { dataContractId: forge.core, documentTypeName: DOC.repo, where: [['$ownerId', '==', ownerId], ['name', '==', name]] }, 1, diff --git a/forge-web/lib/repo/list-budget.test.ts b/forge-web/lib/repo/list-budget.test.ts index 6ec896c6f..c409198fa 100644 --- a/forge-web/lib/repo/list-budget.test.ts +++ b/forge-web/lib/repo/list-budget.test.ts @@ -13,11 +13,11 @@ */ import { sha256 } from '@noble/hashes/sha2.js' -import { beforeAll, describe, expect, it, vi } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { base58Encode } from '../auth/base58' import type { ForgeIds } from '../deployments' -import { bytesToBase64, hexToBase64, setPlatformVersion } from '../sdk' +import { bytesToBase64, hexToBase64 } from '../sdk' import type { RepoRef } from './contract' import { mockSdk, newSeen, type Doc, type Seen, type Store } from './drive-mock' import { queryIssues, type IssueListPage, type IssueSelection } from './issue-index' @@ -202,8 +202,6 @@ async function settledIssues(r: Repo, q: IssueSelection): Promise<{ page: IssueL return { page, loads } } -beforeAll(() => setPlatformVersion(14)) - describe('list request budget on a dash-sized repo (QW2-002, QW3-003)', () => { for (const items of [5615, 7700, 15_000]) { for (const state of ['merged', 'closed', 'all', 'unmerged'] as const) { diff --git a/forge-web/lib/repo/packs.test.ts b/forge-web/lib/repo/packs.test.ts index 22cfd2906..47d0ac091 100644 --- a/forge-web/lib/repo/packs.test.ts +++ b/forge-web/lib/repo/packs.test.ts @@ -72,8 +72,8 @@ describe('packed byteArray parsing (tips / supersedes)', () => { ]), REPO, ) - // The read's row order is the reader's concern (it pages ascending and reverses); look - // the rows up by pack instead of by position. + // The read's row order is the reader's concern (it reads newest first); look the rows up + // by pack instead of by position. const byPack = (b: number) => docs.find((d) => d.packHash === hashHex(b)) expect(byPack(0xaa)?.uris).toEqual(['https://m/p.pack']) expect(byPack(0xaa)?.supersedes).toEqual([]) diff --git a/forge-web/lib/repo/pagination.test.ts b/forge-web/lib/repo/pagination.test.ts index 2c2d08a93..ebda267cd 100644 --- a/forge-web/lib/repo/pagination.test.ts +++ b/forge-web/lib/repo/pagination.test.ts @@ -13,13 +13,7 @@ import type { EvoSDK } from '@dashevo/evo-sdk' import { sha256 } from '@noble/hashes/sha2.js' import { describe, expect, it } from 'vitest' -import { - ascendingEquivalent, - bytesToBase64, - IncompleteReadError, - queryAllDocuments, - tieProbeAllowed, -} from '../sdk' +import { bytesToBase64, IncompleteReadError, queryAllDocuments } from '../sdk' import { readConfigBundle } from './config' import { DOC, type RepoRef } from './contract' import { readEvents, readIssue, readReviews } from './issues' @@ -187,19 +181,12 @@ describe('ref history across a page boundary', () => { // The query SHAPE, not just the row count: the filter must be carried on every page, // and page 2 must resume from the last `$id` of page 1 rather than re-reading page 1. - // Between them sits the boundary-tie read (`$createdAt ==` the last row's), which the - // tie-safe pager issues after every full page of a `(prefix, $createdAt)` read. const refQueries = seen.filter((q) => q.documentTypeName === DOC.refUpdate) - expect(refQueries).toHaveLength(3) + expect(refQueries).toHaveLength(2) expect(refQueries[0]?.where).toEqual([SCOPE, ['refNameHash', '==', REF_HASH_B64]]) expect(refQueries[0]?.startAfter).toBeUndefined() - expect(refQueries[1]?.where).toEqual([ - SCOPE, - ['refNameHash', '==', REF_HASH_B64], - ['$createdAt', '==', expect.any(Number)], - ]) - expect(refQueries[2]?.where).toEqual([SCOPE, ['refNameHash', '==', REF_HASH_B64]]) - expect(refQueries[2]?.startAfter).toBe(`u-${String(PAGE - 1).padStart(4, '0')}`) + expect(refQueries[1]?.where).toEqual([SCOPE, ['refNameHash', '==', REF_HASH_B64]]) + expect(refQueries[1]?.startAfter).toBe(`u-${String(PAGE - 1).padStart(4, '0')}`) }) it('resolves the tip to the newest push, not the last one on page 1', async () => { @@ -413,116 +400,7 @@ describe('pack manifests across a page boundary', () => { expect(read).toHaveLength(PAGE + 1) expect(read[0]?.documentId).toBe(`m-${String(PAGE).padStart(4, '0')}`) expect(read[read.length - 1]?.documentId).toBe('m-0000') - // No descending page after a cursor: the proof a protocol-13 node returns for one fails - // evo-sdk 4.2's verifier, so the newest-first read is paged ascending and reversed. expect(seen.length).toBeGreaterThan(1) - expect( - seen.filter((q) => q.startAfter !== undefined && q.orderBy?.some(([, d]) => d === 'desc')), - ).toEqual([]) - }) -}) - -describe('same-block ties at a page boundary (protocol 13)', () => { - /** - * PROTOCOL-13 cursor semantics: `startAfter` excludes every row whose `$createdAt` is <= - * the cursor's, so rows sharing the cursor's timestamp but sorting after it are skipped — - * the real Drive behaviour. An `==` on `$createdAt` is served normally. - */ - function protocol13Sdk(rows: Record[]): EvoSDK { - return { - documents: { - query: (q: QueryLike): Promise> => { - let out = [...rows] - for (const [field, op, value] of q.where ?? []) { - if (op === '==') out = out.filter((d) => d[field] === value) - } - if (q.startAfter !== undefined) { - const t = rows.find((d) => d['$id'] === q.startAfter)?.['$createdAt'] as number - out = out.filter((d) => (d['$createdAt'] as number) > t) - } - out = out.slice(0, Math.min(q.limit ?? PAGE, PAGE)) - return Promise.resolve(new Map(out.map((d) => [String(d['$id']), d]))) - }, - }, - } as unknown as EvoSDK - } - - /** 99 rows at distinct times, then `tied` rows in one block, then one more row. */ - function straddlingTie(tied: number): Record[] { - const rows: Record[] = Array.from({ length: 99 }, (_, i) => ({ - $id: `r-${String(i).padStart(4, '0')}`, - $createdAt: i, - })) - for (let i = 99; i < 99 + tied; i++) { - rows.push({ $id: `r-${String(i).padStart(4, '0')}`, $createdAt: 1_000 }) - } - rows.push({ $id: `r-${String(99 + tied).padStart(4, '0')}`, $createdAt: 2_000 }) - return rows - } - - const q = { dataContractId: 'c', documentTypeName: 'issue', orderBy: [['$createdAt', 'asc']] as const } - - it('recovers the rows the cursor alone would skip, once each and in order', async () => { - const rows = straddlingTie(3) - // The mock reproduces the gap: without the tie read, the two tied rows after row 100 - // are lost (paging a range-filtered read skips the probe). - const lossy = await queryAllDocuments(protocol13Sdk(rows), { - ...q, - where: [['$createdAt', '>', -1]] as never, - }) - expect(lossy).toHaveLength(rows.length - 2) - - const read = await queryAllDocuments(protocol13Sdk(rows), q) - expect(read.map((d) => d['$id'])).toEqual(rows.map((d) => d['$id'])) - }) - - it('refuses a boundary tie of a full page rather than guessing', async () => { - await expect(queryAllDocuments(protocol13Sdk(straddlingTie(PAGE)), q)).rejects.toThrow( - IncompleteReadError, - ) - }) - - it('probes only where $createdAt ends the index', () => { - const base = { dataContractId: 'c', documentTypeName: 'x' } - expect(tieProbeAllowed({ ...base, orderBy: [['$createdAt', 'asc']] })).toBe(true) - expect( - tieProbeAllowed({ - ...base, - where: [['$ownerId', '==', 'me']], - orderBy: [ - ['$ownerId', 'asc'], - ['$createdAt', 'asc'], - ], - }), - ).toBe(true) - expect(tieProbeAllowed({ ...base, orderBy: [['seq', 'asc']] })).toBe(false) - expect(tieProbeAllowed({ ...base, orderBy: [['$createdAt', 'desc']] })).toBe(false) - expect( - tieProbeAllowed({ ...base, where: [['kind', '>', 1]], orderBy: [['$createdAt', 'asc']] }), - ).toBe(false) - }) -}) - -describe('ascendingEquivalent', () => { - const base = { dataContractId: 'c', documentTypeName: DOC.packManifest } - it('flips a descending order, leaves an ascending one alone', () => { - expect(ascendingEquivalent({ ...base, orderBy: [['$createdAt', 'desc']] })).toEqual([ - ['$createdAt', 'asc'], - ]) - expect(ascendingEquivalent({ ...base, orderBy: [['$createdAt', 'asc']] })).toBeNull() - expect(ascendingEquivalent(base)).toBeNull() - }) - it('flips a mixed order only when its ascending fields are pinned by ==', () => { - const orderBy = [ - ['$ownerId', 'asc'], - ['$createdAt', 'desc'], - ] as const - expect(ascendingEquivalent({ ...base, orderBy, where: [['$ownerId', '==', 'me']] })).toEqual([ - ['$ownerId', 'asc'], - ['$createdAt', 'asc'], - ]) - expect(ascendingEquivalent({ ...base, orderBy })).toBeNull() - expect(ascendingEquivalent({ ...base, orderBy, where: [['$ownerId', '>', 'me']] })).toBeNull() }) }) diff --git a/forge-web/lib/repo/pull-index.test.ts b/forge-web/lib/repo/pull-index.test.ts index d2261337e..9c3a9666f 100644 --- a/forge-web/lib/repo/pull-index.test.ts +++ b/forge-web/lib/repo/pull-index.test.ts @@ -7,11 +7,11 @@ */ import { sha256 } from '@noble/hashes/sha2.js' -import { beforeAll, describe, expect, it } from 'vitest' +import { describe, expect, it } from 'vitest' import { base58Encode } from '../auth/base58' import type { ForgeIds } from '../deployments' -import { bytesToBase64, hexToBase64, setPlatformVersion, type DocumentQuery } from '../sdk' +import { bytesToBase64, hexToBase64, type DocumentQuery } from '../sdk' import { invalidateRepoFeed } from './issues' import { queryIssues } from './issue-index' import { openPullsOnBranch, pullsLinking, queryPulls, type PullSelection } from './pull-index' @@ -113,9 +113,6 @@ function fresh(n: number, opts?: { churn?: boolean }) { const plainOf = (seen: Seen, type: string): DocumentQuery[] => seen.queries.filter((q) => q.documentTypeName === type) const requests = (seen: Seen): number => seen.composites.length + seen.queries.length + seen.counts.length + seen.sums.length -// Protocol 14 (moutai, bonsia): a paged read continues past its cursor without a tie probe. -beforeAll(() => setPlatformVersion(14)) - describe('pull index', () => { it('lists the first page from one composite and one state sum, with the proved Open / Merged / Closed counts', async () => { const { sdk, seen, repo } = fresh(203) diff --git a/forge-web/lib/repo/source.test.ts b/forge-web/lib/repo/source.test.ts index 4737abd26..b102ffa22 100644 --- a/forge-web/lib/repo/source.test.ts +++ b/forge-web/lib/repo/source.test.ts @@ -1,9 +1,6 @@ /** - * The v2 `repoId ==` prefix must keep complete reads tie-safe and descending reads pageable: - * `queryAllDocuments` probes a page boundary's `$createdAt` only when every earlier order - * field is pinned by `==` (`tieProbeAllowed`), and pages a descending read ascending - * (`ascendingEquivalent`). The prefix is an `==` on an index's leading field, so both still - * apply to every scoped complete read the readers make. + * `repoSource`: which Forge contract holds each document type, and that the layout names + * exactly the types each contract schema declares. */ import { readFileSync } from 'node:fs' @@ -11,7 +8,6 @@ import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' -import { ascendingEquivalent, tieProbeAllowed } from '../sdk' import type { RepoRef } from './contract' import { COLLAB_TYPES, COMMUNITY_TYPES, CORE_TYPES } from '../layout' import { repoSource } from './source' @@ -23,7 +19,6 @@ const DEMO: RepoRef = { name: 'n', visibility: 'public', } -const s = repoSource(DEMO) describe('which contract holds a type', () => { const three = repoSource({ ...DEMO, forge: { ...DEMO.forge, community: 'COMMUNITY' } }) @@ -55,21 +50,3 @@ describe('which contract holds a type', () => { expect(declared.sort()).toEqual(Object.keys(schema.documentSchemas).sort()) }) }) - -describe('repoId-scoped complete reads', () => { - it('stay tie-safe', () => { - expect(tieProbeAllowed(s.repoQuery('config', { orderBy: [['$createdAt', 'asc']] }))).toBe(true) - expect(tieProbeAllowed(s.repoQuery('event', { orderBy: [['$createdAt', 'asc']] }))).toBe(true) - expect( - tieProbeAllowed( - s.repoQuery('refUpdate', { where: [['refNameHash', '==', 'h']], orderBy: [['$createdAt', 'asc']] }), - ), - ).toBe(true) - }) - - it('page a newest-first read ascending', () => { - expect(ascendingEquivalent(s.repoQuery('packManifest', { orderBy: [['$createdAt', 'desc']] }))).toEqual([ - ['$createdAt', 'asc'], - ]) - }) -}) diff --git a/forge-web/lib/sdk/index.ts b/forge-web/lib/sdk/index.ts index fde1cb1f6..2b57f9e1e 100644 --- a/forge-web/lib/sdk/index.ts +++ b/forge-web/lib/sdk/index.ts @@ -72,7 +72,6 @@ export { type CostPreview, } from './cost' export { - ascendingEquivalent, base64ToBytes, base64ToHex, bytesToBase64, @@ -82,7 +81,6 @@ export { uintGroupKey, uintOfGroupKey, type GroupedQuery, - cursorPadded, hexToBase64, normalizeDocument, setPlatformVersion, @@ -94,7 +92,6 @@ export { rankedDocuments, retryOnStaleContract, shareInFlight, - tieProbeAllowed, type DocumentQuery, type RankedEntry, type RankedPage, diff --git a/forge-web/lib/sdk/query.test.ts b/forge-web/lib/sdk/query.test.ts index 5bee75489..ef737845d 100644 --- a/forge-web/lib/sdk/query.test.ts +++ b/forge-web/lib/sdk/query.test.ts @@ -1,7 +1,7 @@ import type { EvoSDK } from '@dashevo/evo-sdk' import { afterEach, describe, expect, it, vi } from 'vitest' -import { countDocuments, countDocumentsGrouped, noteSdkWrite, queryAllDocuments, queryDocuments, retryOnStaleContract, setPlatformVersion, setStaleContractHandler, staleContractCause, staleDocumentVersion, sumDocumentsGrouped, ungroupedRangeProblem, type DocumentQuery } from './query' +import { countDocuments, countDocumentsGrouped, noteSdkWrite, queryAllDocuments, queryDocuments, retryOnStaleContract, setStaleContractHandler, staleContractCause, staleDocumentVersion, sumDocumentsGrouped, ungroupedRangeProblem, type DocumentQuery } from './query' const CONTRACT = 'C1zHeeG7EUudXdB5ZyDQnXVU35hrCfvd1fRCybXEqaPS' @@ -165,32 +165,24 @@ describe('identical reads in flight are joined (S-1)', () => { }) }) -describe('the page-boundary tie read is for protocol 13 only', () => { +describe('a complete read pages on the cursor alone', () => { // Two full pages, then a short one. const rows = Array.from({ length: 201 }, (_, i) => ({ $id: `r-${String(i).padStart(4, '0')}`, $createdAt: i })) - function pagedSdk(version: number, seen: unknown[]): EvoSDK { - const query = async (q: { where?: unknown[][]; startAfter?: string; limit?: number }) => { + function pagedSdk(seen: { startAfter?: string }[]): EvoSDK { + const query = async (q: { startAfter?: string; limit?: number }) => { seen.push(q) let out = [...rows] - for (const [f, op, v] of q.where ?? []) if (op === '==' && f === '$createdAt') out = out.filter((d) => d.$createdAt === v) if (q.startAfter !== undefined) out = out.slice(out.findIndex((d) => d.$id === q.startAfter) + 1) return new Map(out.slice(0, q.limit ?? 100).map((d) => [d.$id, d])) } - return { documents: { query }, version: () => version } as unknown as EvoSDK + return { documents: { query }, version: () => 14 } as unknown as EvoSDK } const q = { dataContractId: CONTRACT, documentTypeName: 'event', where: [['repoId', '==', 'r']] as const, orderBy: [['$createdAt', 'asc']] as const } - afterEach(() => setPlatformVersion(14)) - it('protocol 14 pages on the cursor alone (Drive pads the tie itself)', async () => { - const seen: unknown[] = [] - expect(await queryAllDocuments(pagedSdk(14, seen), q)).toHaveLength(201) - expect(seen).toHaveLength(3) - }) - - it('protocol 13 still reads each boundary timestamp', async () => { - const seen: unknown[] = [] - expect(await queryAllDocuments(pagedSdk(13, seen), q)).toHaveLength(201) - expect(seen).toHaveLength(5) + it('one request per page, each after the last row (Drive keeps rows tied with the cursor)', async () => { + const seen: { startAfter?: string }[] = [] + expect(await queryAllDocuments(pagedSdk(seen), q)).toHaveLength(201) + expect(seen.map((s) => s.startAfter)).toEqual([undefined, 'r-0099', 'r-0199']) }) }) diff --git a/forge-web/lib/sdk/query.ts b/forge-web/lib/sdk/query.ts index 3d1fc6555..e9155f8b9 100644 --- a/forge-web/lib/sdk/query.ts +++ b/forge-web/lib/sdk/query.ts @@ -96,23 +96,6 @@ export type PlainDocument = Record // (the value only selects DPP's serialization rules, which are stable for our field types). let platformVersion = 1 -/** The protocol version from which a `startAfter` page never skips rows tied with its cursor. */ -const CURSOR_PADDED_FROM = 14 - -/** - * Whether the network pages tie-safely by itself (protocol {@link CURSOR_PADDED_FROM}+), as far - * as the SDK has seen: the version is learned from the first proved reply, which every paged - * read has had by the time it needs its second page. - * - * Drive pads a cursor page only off the primary key and for types that are not `indexOnly` - * (`pads_cursor_page`). Every tie-probed read here orders by `$createdAt` (never the primary key), - * and none of the `indexOnly` forge types (`star`, `starBeat`, `follow`, `watch`) has an index - * ending in `$createdAt`, the only shape {@link tieProbeAllowed} admits: the version alone decides. - */ -export function cursorPadded(): boolean { - return platformVersion >= CURSOR_PADDED_FROM -} - /** Pin the DPP platform version used by {@link normalizeDocument}'s `toJSON`. */ export function setPlatformVersion(version: number): void { if (Number.isInteger(version) && version > 0) platformVersion = version @@ -485,28 +468,6 @@ export async function rankedDocuments(sdk: EvoSDK, query: RankedQuery): Promise< } } -/** - * The all-ascending `orderBy` whose traversal, reversed, is exactly `query`'s — or null when - * the order is already ascending or has no such equivalent. Parity: forge-core - * `platform::ascending_equivalent`. - * - * A complete read pages with a `startAfter` cursor. The grovedb verifier in evo-sdk 4.2 checks - * that every proof op matches the walk direction. Protocol-13 nodes answered a - * descending page after a cursor with a proof that fails that check. It surfaced as - * `packManifest` reads failing once a repo passed 100 manifests. Ascending pages verify, and - * Drive's descending walk is the exact reverse of its ascending walk. Reversing also - * reverses any clause that was already ascending, which is only harmless when an `==` filter - * pins that clause's field, so any other mixed order is left as requested. - */ -export function ascendingEquivalent(query: DocumentQuery): OrderByClause[] | null { - const orderBy = query.orderBy ?? [] - if (orderBy.every(([, direction]) => direction === 'asc')) return null - const pinned = (field: string): boolean => - (query.where ?? []).some(([f, op]) => f === field && op === '==') - if (orderBy.some(([field, direction]) => direction === 'asc' && !pinned(field))) return null - return orderBy.map(([field]) => [field, 'asc'] as const) -} - /** Thrown when a read that must be complete could not be proven complete. */ export class IncompleteReadError extends Error { constructor( @@ -536,40 +497,12 @@ export class IncompleteReadError extends Error { * genuinely want a bounded window should use {@link queryDocumentsWithProof} with a `limit` * and present it as a window. * - * `pageLimit` bounds each round-trip; `maxPages` is a hard safety cap on total rounds. - */ -/** - * Whether a complete read can make its page boundaries tie-safe. Parity: forge-core - * `platform::tie_probe_allowed`. The conditions: the order ends in `$createdAt` ascending, - * every earlier order field is pinned by an `==` filter, and every filter is an `==` on some - * other field. The index then ends in `$createdAt`, so an equality on the boundary timestamp - * is a valid query on the same index. + * The cursor alone is enough on protocol 14, the only protocol Forge's contracts run on: Drive + * continues a `startAfter` page within the cursor's index key by document id, so rows sharing + * the cursor's `$createdAt` are not skipped (`rs-drive` `DriveDocumentQuery::pads_cursor_page`). * - * Why: on protocol 13, a `startAfter` cursor excludes the cursor's whole `$createdAt` key. - * Documents created in the same block as a page's last row that sort after it would be - * silently skipped. Protocol 14 bounds the cursor by document id and does not drop them. - * - * It needs the boundary row's `$createdAt`, which every forge-v2 history type requires. - * - * Only below protocol 14 ({@link CURSOR_PADDED_FROM}): there Drive lowers a `startAfter` as a - * `startAt` on the cursor's own index key, continues within it by document id and strips the - * cursor row (`rs-drive` `DriveDocumentQuery::pads_cursor_page`, gated on - * `non_primary_key_path_query >= 1`, first set in protocol 14; proved the same way, and - * covered by `compound_cursor_tests.rs`). The probe would be one more request per full page for - * rows the next page returns anyway. + * `pageLimit` bounds each round-trip; `maxPages` is a hard safety cap on total rounds. */ -export function tieProbeAllowed(query: DocumentQuery): boolean { - const orderBy = query.orderBy ?? [] - const last = orderBy[orderBy.length - 1] - if (last === undefined || last[0] !== '$createdAt' || last[1] !== 'asc') return false - const where = query.where ?? [] - const pinned = (field: string): boolean => where.some(([f, op]) => f === field && op === '==') - return ( - orderBy.slice(0, -1).every(([field, direction]) => direction === 'asc' && pinned(field)) && - where.every(([field, op]) => op === '==' && field !== '$createdAt') - ) -} - export async function queryAllDocuments( sdk: EvoSDK, query: DocumentQuery, @@ -579,18 +512,13 @@ export async function queryAllDocuments( /** * The query's first page, already read elsewhere (a composite's sibling sub-query with * the same query and a `pageLimit` limit): a short one is the whole answer, a full one is - * continued from its last row instead of being read again. Ascending queries only. + * continued from its last row instead of being read again. */ readonly firstPage?: readonly PlainDocument[] } = {}, ): Promise { const pageLimit = opts.pageLimit ?? 100 const maxPages = opts.maxPages ?? 1000 - // Page a descending read ascending and reverse it — see `ascendingEquivalent`. - const ascending = ascendingEquivalent(query) - if (opts.firstPage !== undefined && ascending !== null) throw new Error('firstPage continues an ascending query only') - const paged = ascending === null ? query : { ...query, orderBy: ascending } - const tieShape = tieProbeAllowed(paged) const out: PlainDocument[] = [] const held = new Set() const take = (rows: PlainDocument[]): void => { @@ -603,17 +531,15 @@ export async function queryAllDocuments( out.push(d) } } - const done = (): PlainDocument[] => (ascending === null ? out : out.reverse()) let startAfter: string | undefined for (let page = 0; page < maxPages; page++) { const documents = page === 0 && opts.firstPage !== undefined ? [...opts.firstPage] - : (await queryDocumentsWithProof(sdk, { ...paged, limit: pageLimit, startAfter })).documents + : (await queryDocumentsWithProof(sdk, { ...query, limit: pageLimit, startAfter })).documents take(documents) - if (documents.length < pageLimit) return done() - const last = documents[documents.length - 1] - const lastId = last?.['$id'] + if (documents.length < pageLimit) return out + const lastId = documents[documents.length - 1]?.['$id'] if (typeof lastId !== 'string') { throw new IncompleteReadError( query.documentTypeName, @@ -621,29 +547,7 @@ export async function queryAllDocuments( 'a full page ended on a result with no id, so the next page cannot be read', ) } - let cursor: string = lastId - // Same-block rows past the boundary: read the boundary timestamp in full (see - // `tieProbeAllowed`), then continue after the last of them. - const createdAt = last?.['$createdAt'] - // Checked per boundary: the version is known once the first proved page has come back. - if (tieShape && !cursorPadded() && typeof createdAt === 'number') { - const { documents: tied } = await queryDocumentsWithProof(sdk, { - ...paged, - where: [...(paged.where ?? []), ['$createdAt', '==', createdAt]], - limit: pageLimit, - }) - if (tied.length >= pageLimit) { - throw new IncompleteReadError( - query.documentTypeName, - out.length, - `${pageLimit} or more results share the creation time ${createdAt}, so the page boundary cannot be read completely`, - ) - } - const lastTied = tied[tied.length - 1]?.['$id'] - if (typeof lastTied === 'string') cursor = lastTied - take(tied) - } - startAfter = cursor + startAfter = lastId } throw new IncompleteReadError( query.documentTypeName, From 6520dfc59c5ef33c257d947a97ef3012ec7d4dc3 Mon Sep 17 00:00:00 2001 From: pasta Date: Tue, 27 Oct 2026 19:47:58 +0000 Subject: [PATCH 2/2] docs: note the primary-key and indexOnly cursor cases The readers' doc comments and BUILDING.md now say that the primary-key path needs no padding and that indexOnly types refuse a cursor. The pack-manifest paging test asserts the descending query shape, and the parity spec no longer mentions tie probes. Co-Authored-By: Claude Opus 5.5 --- crates/forge-core/src/platform/mod.rs | 2 ++ docs/BUILDING.md | 5 ++++- docs/design/platform-parity-spec.md | 4 ++-- forge-web/lib/repo/pagination.test.ts | 5 ++++- forge-web/lib/sdk/index.ts | 2 +- forge-web/lib/sdk/query.ts | 2 ++ 6 files changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/forge-core/src/platform/mod.rs b/crates/forge-core/src/platform/mod.rs index 84ea876b3..9f31449fa 100644 --- a/crates/forge-core/src/platform/mod.rs +++ b/crates/forge-core/src/platform/mod.rs @@ -3499,6 +3499,8 @@ const MAX_PAGES: usize = 1000; /// The cursor alone is enough on protocol 14, the only protocol Forge's contracts run on: /// Drive continues a `startAfter` page within the cursor's index key by document id, so rows /// sharing the cursor's `$createdAt` are not skipped (`DriveDocumentQuery::pads_cursor_page`). +/// The primary-key path is not padded and needs no padding, since ids are unique. An `indexOnly` +/// type refuses a cursor outright, so a read of one fails at its second page. /// Rows are deduplicated by `$id`. /// /// A **short page is the only accepted proof** that the end was reached. If the cursor diff --git a/docs/BUILDING.md b/docs/BUILDING.md index 8fba0fda4..3b40660e8 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -163,7 +163,10 @@ cursor alone, in the requested order. On protocol 14 Drive lowers a `startAfter` primary key as a `startAt` on the cursor's own index key, continues within that key by document id and drops the cursor row (`DriveDocumentQuery::pads_cursor_page`). So documents created in the same block as a page's last row are not skipped, and pages are not one row -short. Descending pages after a cursor prove and verify like ascending ones. Protocol 13 got +short. Descending pages after a cursor prove and verify like ascending ones. The primary-key +path is not padded and needs no padding, because ids are unique. `indexOnly` types (`star`, +`watch`, `follow`) refuse a cursor outright, so their reads must stay within one page or page +by a range clause. Protocol 13 got both wrong, and the readers used to work around it; forge-v2 needs protocol 14, so they no longer do. diff --git a/docs/design/platform-parity-spec.md b/docs/design/platform-parity-spec.md index 170313d05..0d0657a57 100644 --- a/docs/design/platform-parity-spec.md +++ b/docs/design/platform-parity-spec.md @@ -236,7 +236,7 @@ CLI (audit of `origin/master`; every read proof-verified; `fetch_contract` is ** | Command | Queries | N+1 | |---|---|---| -| `git clone` / `git fetch` | identity 1; resolve 2; `read_refs` = contract + `⌈Cfg/100⌉` + `2⌈U/100⌉` (`repo.rs:454-472`, `refs.rs:156-190`); default branch 2; manifests `1+⌈P/100⌉` (+ tie probes); members `2⌈M/100⌉`; per pack (8 in flight) copies in reader order, Platform fallback `⌈chunks/100⌉` | **fetch re-downloads every kind-0 pack whenever anything is new** (`helper.rs:242-316`); partial-clone promisor fetches start a new helper (each pays the whole prologue) | +| `git clone` / `git fetch` | identity 1; resolve 2; `read_refs` = contract + `⌈Cfg/100⌉` + `2⌈U/100⌉` (`repo.rs:454-472`, `refs.rs:156-190`); default branch 2; manifests `1+⌈P/100⌉`; members `2⌈M/100⌉`; per pack (8 in flight) copies in reader order, Platform fallback `⌈chunks/100⌉` | **fetch re-downloads every kind-0 pack whenever anything is new** (`helper.rs:242-316`); partial-clone promisor fetches start a new helper (each pays the whole prologue) | | `git push` (N refs, C chunks) | `read_refs` ×3 (list, push, converge ×1–6, `helper.rs:368, 501-537`); precheck roles 4 + protected patterns 2; `read_pack_copies` 2; **C sequential chunk writes each awaiting a proof** (`repo.rs:695-715`; only `PlatformBackend::put` pipelines 8, `backends/platform.rs:260-273`); manifest 2; browse index: manifests again + members + locator (pipelined) + manifest; **per ref: contract + full config history + write** (`repo.rs:392-432`) | 0.5–0.7 chunks/s (D-910) is the sequential proof wait; config re-read per ref | | `dg pr list` | session 3; one page `patch` (cap 100); members 3; **per PR: `patch_view` (target log 2, base ref tips 2 + config, contract ×3) + reviews 2 ≈ 9** (`pr.rs:286-300`, `collab/v2.rs:1028-1046`) | 915 requests / 117 s for 100 PRs (D-500) | | `dg pr view` | ≈ 20 round trips, 7 of them `getDataContract` | | @@ -282,7 +282,7 @@ CLI: `dg pr list` = one `DocumentQuery::with_sub_queries` (rs-sdk; changelog dev ### 3.4 Caches (the delta-cache invariant) -A type is delta-cacheable iff it is immutable **and** non-deletable (then a `$createdAt > cursor` read on a `(repoId, $createdAt)` index is exact): `refUpdate`, `protectedRefUpdate`, `config`, `packManifest`, `manifestPart`, `chunk`, `event`, `authorEvent`, `issue`/`patch` creation rows (the documents are mutable — cache by `$updatedAt`, re-read the page you show), `policy`, `repoKey`. Not cacheable without a TTL: `comment`, `review`, `release`, `label`, `checkRun`, `webhook`, `star`/`follow` (deletable). IDB stores: `refs`, `config`, `feed`, `manifests`, `fragments`, `objects` (LRU 64 MB), keyed by network + repoId, with the cursor = the last row's `$createdAt` (ties are re-read, `$createdAt >=` then dedupe by `$id`; the `$createdAt` tie probe already exists in `query.ts:253-263`). The CLI mirrors this in `~/.cache/dash-forge///*.jsonl` and in `.git/dash/`. Storage objects: HTTP `Cache-Control: immutable` on content-addressed keys is a bucket setting we document in the storage guide; the browser cache then serves fragments and ranges for free. +A type is delta-cacheable iff it is immutable **and** non-deletable (then a `$createdAt > cursor` read on a `(repoId, $createdAt)` index is exact): `refUpdate`, `protectedRefUpdate`, `config`, `packManifest`, `manifestPart`, `chunk`, `event`, `authorEvent`, `issue`/`patch` creation rows (the documents are mutable — cache by `$updatedAt`, re-read the page you show), `policy`, `repoKey`. Not cacheable without a TTL: `comment`, `review`, `release`, `label`, `checkRun`, `webhook`, `star`/`follow` (deletable). IDB stores: `refs`, `config`, `feed`, `manifests`, `fragments`, `objects` (LRU 64 MB), keyed by network + repoId, with the cursor = the last row's `$createdAt` (ties are re-read, `$createdAt >=` then dedupe by `$id`). The CLI mirrors this in `~/.cache/dash-forge///*.jsonl` and in `.git/dash/`. Storage objects: HTTP `Cache-Control: immutable` on content-addressed keys is a bucket setting we document in the storage guide; the browser cache then serves fragments and ranges for free. Contracts: seed evo-sdk from the deployment file's contracts (`wasm-sdk` "let apps seed the contracts they already hold", #4746) so no page fetches `getDataContract`; forge-core memoises `fetch_contract` per process and persists it under `~/.cache/dash-forge/contracts/.bin`, validated by `getDataContractsLatestVersions` (one request for all ids, #4739) once per hour. diff --git a/forge-web/lib/repo/pagination.test.ts b/forge-web/lib/repo/pagination.test.ts index ebda267cd..154b1a635 100644 --- a/forge-web/lib/repo/pagination.test.ts +++ b/forge-web/lib/repo/pagination.test.ts @@ -400,7 +400,10 @@ describe('pack manifests across a page boundary', () => { expect(read).toHaveLength(PAGE + 1) expect(read[0]?.documentId).toBe(`m-${String(PAGE).padStart(4, '0')}`) expect(read[read.length - 1]?.documentId).toBe('m-0000') - expect(seen.length).toBeGreaterThan(1) + // Paged newest first: page 2 continues after page 1's oldest row, in the same order. + expect(seen).toHaveLength(2) + expect(seen[1]?.orderBy).toEqual([['$createdAt', 'desc']]) + expect(seen[1]?.startAfter).toBe('m-0001') }) }) diff --git a/forge-web/lib/sdk/index.ts b/forge-web/lib/sdk/index.ts index 2b57f9e1e..ed89ef792 100644 --- a/forge-web/lib/sdk/index.ts +++ b/forge-web/lib/sdk/index.ts @@ -3,7 +3,7 @@ * * Trusted-mode connections (`testnetTrusted()` / `mainnetTrusted()` / a devnet's) + `*WithProof` * reads (the only WASM-viable path, S0.3), with the base64 byteArray operand encoding and the - * complete, tie-safe paging that active-repo correctness depends on. + * complete paging that active-repo correctness depends on. */ export { ensureSdk, evoSdkService, preloadContractIds, type EvoSdkConfig, type SdkStatus } from './service' diff --git a/forge-web/lib/sdk/query.ts b/forge-web/lib/sdk/query.ts index e9155f8b9..50ddb09b9 100644 --- a/forge-web/lib/sdk/query.ts +++ b/forge-web/lib/sdk/query.ts @@ -500,6 +500,8 @@ export class IncompleteReadError extends Error { * The cursor alone is enough on protocol 14, the only protocol Forge's contracts run on: Drive * continues a `startAfter` page within the cursor's index key by document id, so rows sharing * the cursor's `$createdAt` are not skipped (`rs-drive` `DriveDocumentQuery::pads_cursor_page`). + * The primary-key path is not padded and needs no padding, since ids are unique. An `indexOnly` + * type refuses a cursor outright, so a read of one throws at its second page. * * `pageLimit` bounds each round-trip; `maxPages` is a hard safety cap on total rounds. */