diff --git a/.github/workflows/testnet-nightly.yml b/.github/workflows/testnet-nightly.yml
index 0412d8184..d1c3813f5 100644
--- a/.github/workflows/testnet-nightly.yml
+++ b/.github/workflows/testnet-nightly.yml
@@ -8,15 +8,17 @@ name: Testnet Nightly
# frozen push, no-token push, third-party verify, depth+filter
#
# They split by what they need:
-# * READ-ONLY specs need only a network path to testnet DAPI and the seeded fixture repo,
-# so they run unconditionally and their failure fails this workflow.
+# * READ-ONLY specs need only a network path to testnet DAPI and the read fixture repo
+# (seeded by `seed-read-fixture` when the secrets exist; e2e/README.md reserves it), so
+# they run unconditionally and their failure fails this workflow.
# * The CLI suite and the write specs need FUNDED fixture identities, which cannot live in
# a fork's PR context. A separate `fixtures` probe job decides whether they run, so when
# the secrets are absent the suite job's conclusion is literally `skipped` rather than a
# green job with everything gated out — a distinction any badge or alert rule can see.
#
# Fixture identity secrets (e2e/cli/config.sh names the roles):
-# FORGE_TEST_IDENTITY_DEPLOYER - owns the reused m1 repo, grants tokens
+# FORGE_TEST_IDENTITY_DEPLOYER - owns the CLI suite's repo and the read fixture, grants
+# tokens (which repo each suite may write: e2e/README.md)
# FORGE_TEST_IDENTITY_COLLAB - holds an unfrozen WRITE token
# FORGE_TEST_IDENTITY_CONTRIB - holds no token (negative push case)
# FORGE_TEST_IDENTITY_FROZEN - holds a frozen WRITE token (negative push case)
@@ -35,6 +37,11 @@ concurrency:
jobs:
web-read-paths:
name: playwright (read-only, live testnet)
+ # Reads the dedicated read fixture (e2e/README.md), so it waits for the seed job — but
+ # still runs when that job is skipped (no secrets: the fixture is read as it stands) or
+ # failed (the specs then say what is wrong with it).
+ needs: seed-read-fixture
+ if: ${{ !cancelled() }}
runs-on: ubuntu-latest
timeout-minutes: 30
defaults:
@@ -76,6 +83,8 @@ jobs:
runs-on: ubuntu-latest
outputs:
present: ${{ steps.probe.outputs.present }}
+ # The read-fixture seeder signs with DEPLOYER alone.
+ deployer: ${{ steps.probe.outputs.deployer }}
steps:
- name: Probe
id: probe
@@ -85,6 +94,11 @@ jobs:
FORGE_TEST_IDENTITY_CONTRIB: ${{ secrets.FORGE_TEST_IDENTITY_CONTRIB }}
FORGE_TEST_IDENTITY_FROZEN: ${{ secrets.FORGE_TEST_IDENTITY_FROZEN }}
run: |
+ if [ -n "$FORGE_TEST_IDENTITY_DEPLOYER" ]; then
+ echo "deployer=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "deployer=false" >> "$GITHUB_OUTPUT"
+ fi
missing=()
# Report the SECRET names an operator has to create, not internal aliases.
for name in FORGE_TEST_IDENTITY_DEPLOYER FORGE_TEST_IDENTITY_COLLAB \
@@ -108,6 +122,46 @@ jobs:
echo "present=true" >> "$GITHUB_OUTPUT"
fi
+ # Make the browser specs' read fixture exist and hold exactly the expected commit. It is
+ # its own repo, written only by this job, so nothing the CLI suite or an ad-hoc storage run
+ # stores can reach what the browser reads (e2e/README.md). Idempotent: a no-op once seeded.
+ seed-read-fixture:
+ name: seed the read fixture
+ needs: fixtures
+ if: needs.fixtures.outputs.deployer == 'true'
+ runs-on: ubuntu-latest
+ timeout-minutes: 45
+ env:
+ ID_DEPLOYER_JSON: ${{ secrets.FORGE_TEST_IDENTITY_DEPLOYER }}
+ steps:
+ - uses: actions/checkout@v5
+ - name: Install protoc
+ env:
+ PROTOC_VERSION: '28.3'
+ run: |
+ curl -sSLo /tmp/protoc.zip \
+ "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip"
+ sudo unzip -q -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*'
+ sudo chmod +x /usr/local/bin/protoc
+ - uses: dtolnay/rust-toolchain@stable
+ - uses: Swatinem/rust-cache@v2
+ with:
+ shared-key: e2e-cli
+ - name: Materialize the DEPLOYER identity
+ run: |
+ dir="${HOME}/.config/dash-forge/test-identities"
+ mkdir -p "$dir" && chmod 700 "$dir"
+ printf '%s' "$ID_DEPLOYER_JSON" > "$dir/DEPLOYER.identity.json"
+ # harness_init checks for the whole role set; only DEPLOYER signs here.
+ for r in COLLAB CONTRIB; do cp "$dir/DEPLOYER.identity.json" "$dir/$r.identity.json"; done
+ chmod 600 "$dir"/*.identity.json
+ - name: Build CLI binaries
+ timeout-minutes: 30
+ run: cargo build --locked -p dg -p git-remote-dash
+ - name: Seed
+ timeout-minutes: 12
+ run: bash e2e/cli/seed-read-fixture.sh
+
cli-suite:
name: cli e2e (live testnet)
needs: fixtures
@@ -140,6 +194,8 @@ jobs:
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
+ with:
+ shared-key: e2e-cli
- name: Materialize fixture identities
run: |
diff --git a/Makefile b/Makefile
index 93e5b994a..e5ee50712 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@ SHELL := /bin/bash
COMPOSE_FILE := infra/docker-compose.yml
-.PHONY: check check-rust check-web build build-rust build-web infra-up infra-down e2e devnet-identities devnet-identities-verify storage-it storage-e2e
+.PHONY: check check-rust check-web build build-rust build-web infra-up infra-down e2e e2e-fixture devnet-identities devnet-identities-verify storage-it storage-e2e
## check: run rust + web lint/test suites; tolerant of dirs that don't exist yet
check: check-rust check-web
@@ -54,13 +54,17 @@ infra-up:
infra-down:
docker compose -f $(COMPOSE_FILE) down -v
-## e2e: run the CLI end-to-end suite (LIVE testnet) against the reused m1 repo.
+## e2e: run the CLI end-to-end suite (LIVE testnet) against its reserved repo (e2e/README.md).
## Builds the binaries if needed, then drives real git push/clone through the
## dash:// helper. See e2e/cli/README-less run.sh header for env knobs
## (RUN_ID, E2E_TIMEOUT, E2E_NO_CLEANUP, subset args). Exits non-zero on any FAIL.
e2e: build-rust
@bash e2e/cli/run.sh
+## e2e-fixture: seed the browser specs' read fixture (idempotent; see e2e/README.md).
+e2e-fixture: build-rust
+ @bash e2e/cli/seed-read-fixture.sh
+
## devnet-identities: mint (or resume) the 9-role identity pool on a devnet,
## funded from the devnet's faucet wallet key, then verify every identity on
## Platform. The key is read from dash-network-configs at runtime (process
diff --git a/crates/forge-core/src/collab.rs b/crates/forge-core/src/collab.rs
index 73e6205f9..063ed4ef6 100644
--- a/crates/forge-core/src/collab.rs
+++ b/crates/forge-core/src/collab.rs
@@ -42,8 +42,6 @@ const DOC_EVENT: &str = "event";
const DOC_REVIEW: &str = "review";
const DOC_LABEL: &str = "label";
const DOC_RELEASE: &str = "release";
-const DOC_REF_UPDATE: &str = "refUpdate";
-const DOC_PROTECTED_REF_UPDATE: &str = "protectedRefUpdate";
// Registry-contract document types.
const DOC_STAR: &str = "star";
const DOC_FOLLOW: &str = "follow";
@@ -926,46 +924,30 @@ impl<'a> PullRequestService<'a> {
}
/// Collect every oid that was ever a tip of `base_ref_name` (the monotonic merge-
- /// reachability set) plus the newest such tip. Walks the full `refUpdate` +
- /// `protectedRefUpdate` history for the ref (paginated), taking every non-null `newOid`.
+ /// reachability set) plus the newest such tip. Walks the ref's full `refUpdate` +
+ /// `protectedRefUpdate` history — [`crate::refs::read_ref_history`], an equality read on
+ /// one `refNameHash`, so its cost is this ref's pushes, not the repo's — taking every
+ /// non-null `newOid`.
async fn base_ref_tips(
&self,
contract: &LoadedContract,
base_ref_name: &str,
) -> Result<(std::collections::BTreeSet, Option)> {
let ref_name_hash = sha256(base_ref_name.as_bytes());
+ let updates = crate::refs::read_ref_history(self.client, contract, ref_name_hash).await?;
let mut tips: std::collections::BTreeSet = std::collections::BTreeSet::new();
let mut newest: Option<(u64, String, String)> = None; // (created_at, id, oid)
- for doc_type in [DOC_REF_UPDATE, DOC_PROTECTED_REF_UPDATE] {
- let docs = self
- .client
- .query_all_documents(
- contract,
- doc_type,
- &[QueryFilter::eq(
- "refNameHash",
- FieldValue::bytes32(ref_name_hash),
- )],
- &[QueryOrder::asc("$createdAt")],
- )
- .await?;
- for d in &docs {
- let Some(oid) = d.field_hex("newOid") else {
- continue;
- };
- if oid.is_empty() || oid.bytes().all(|b| b == b'0') {
- continue; // null oid = ref deletion, never a reachable tip
- }
- tips.insert(oid.clone());
- let created_at = d.created_at.unwrap_or(0);
- let candidate = (created_at, d.id.clone(), oid);
- let better = match &newest {
- None => true,
- Some(n) => (n.0, &n.1) < (candidate.0, &candidate.1),
- };
- if better {
- newest = Some(candidate);
- }
+ for u in updates {
+ if u.new_oid.is_empty() || u.new_oid.bytes().all(|b| b == b'0') {
+ continue; // null oid = ref deletion, never a reachable tip
+ }
+ tips.insert(u.new_oid.clone());
+ let better = match &newest {
+ None => true,
+ Some(n) => (n.0, &n.1) < (u.created_at, &u.id),
+ };
+ if better {
+ newest = Some((u.created_at, u.id, u.new_oid));
}
}
Ok((tips, newest.map(|(_, _, oid)| oid)))
diff --git a/crates/forge-core/src/lib.rs b/crates/forge-core/src/lib.rs
index 0311631eb..bae2c945e 100644
--- a/crates/forge-core/src/lib.rs
+++ b/crates/forge-core/src/lib.rs
@@ -8,6 +8,8 @@
//! `WriteEngine` document create/delete lifecycle + idempotent-retry journal types.
//! - [`repo`] — `RepoService`: the repo-lifecycle API (`create_repo` / `resolve_repo` /
//! ref + pack-manifest + chunk read/write) `git-remote-dash` calls.
+//! - [`refs`] — complete ref-update reads (keyset scan + completeness fallback) shared by
+//! ref listing and PR base-tip resolution.
//! - [`tokens`] — `TokenService`: the collaborator ACL (grant/suspend/revoke = token
//! mint/freeze/destroy; balances = the on-chain collaborator list).
//! - [`collab`] — issue / PR / review / release / label services + the registry social
@@ -32,6 +34,7 @@ pub mod keystore;
pub mod network;
pub mod pack;
pub mod platform;
+pub mod refs;
pub mod repo;
pub mod rules;
pub mod storage;
diff --git a/crates/forge-core/src/refs.rs b/crates/forge-core/src/refs.rs
new file mode 100644
index 000000000..f34d54fdb
--- /dev/null
+++ b/crates/forge-core/src/refs.rs
@@ -0,0 +1,658 @@
+//! Ref-update reads: every ref's complete `refUpdate` + `protectedRefUpdate` history.
+//!
+//! Shared by [`crate::repo::RepoService::read_refs`] (list every ref) and the PR base-tip
+//! reader in [`crate::collab`] (one ref). forge-web's `lib/repo/refs.ts` implements the same
+//! reads the same way; the two must agree, because the fold over their output
+//! ([`crate::rules::resolve_ref`]) is only as parity-safe as its input.
+//!
+//! ## Why a keyset scan, and never a cursor over the `refState` index
+//!
+//! The `refState` index is `(refNameHash, $createdAt)`. Paging it with a `startAfter`
+//! document cursor LOSES rows on testnet (protocol 13): measured on the nightly repo, 32 of
+//! 229 updates never came back. Drive's v0 path-query lowering applies the cursor
+//! document's lower-level bounds (its `$createdAt`, then its `$id`) to every *sibling*
+//! `refNameHash` branch instead of only the cursor's own branch, so a page silently omits
+//! rows from later refs and can come back short — and a short page is the only end-of-data
+//! signal a pager has, so everything after it vanishes too. This is the multi-branch cursor
+//! family dashpay/platform#4396 fixed for `in` queries at protocol 14; the `orderBy`-only
+//! shape used here is one it lists as still unfixed.
+//!
+//! A range *where-clause* has no cursor document to leak, so this module never sends one on
+//! a multi-branch query. It pages by key instead:
+//!
+//! 1. `refNameHash > last` ordered `(refNameHash, $createdAt)`, `limit 100`. Every ref on
+//! a full page except the last one is complete; the next page starts after the last
+//! complete ref, re-reading the one that may have been cut off.
+//! 2. A ref that fills a whole page by itself is read on its own with an equality query
+//! (`refNameHash == h`), which is single-branch — the cursor bug needs sibling branches —
+//! and the scan resumes after it.
+//!
+//! Each round must strictly advance `last` — the scan checks it — so it terminates without
+//! leaning on a page cap, and a one-ref read costs pages of that ref only, not of the whole
+//! repo.
+//!
+//! ## When the scan is not trusted
+//!
+//! The scan is abandoned, and every update of both types is re-read in `$createdAt`
+//! (`reflog`) order instead — the result then comes from that read ALONE, deduplicated by
+//! `$id` — when either:
+//!
+//! * a page comes back out of `refNameHash` order, holds a row at or below its
+//! `refNameHash > last` bound, or would not advance `last` (the node did not honor the
+//! query, so nothing it returned is trusted); or
+//! * the completeness check fails. Every non-null `prevOid` a pusher records is the tip it
+//! saw, which is some earlier update's `newOid` for the same ref; an update whose
+//! `prevOid` matches nothing is evidence of a missing row. A dangling `prevOid` can also
+//! be written on purpose (by a WRITE holder), which costs the fallback's extra reads but
+//! never changes the answer.
+//!
+//! ## What the completeness check cannot see
+//!
+//! It finds a gap only in the MIDDLE of a chain. A missing newest update, or a ref missing
+//! entirely, leaves no dangling `prevOid`, so the ref would quietly resolve to an older tip
+//! (or not be listed). The scan itself does not drop rows that way — it is cursor-free,
+//! which is the point — so this is a limit of the safety net, not a known failure.
+//!
+//! One path does still use a cursor: a ref with more than a page of updates is read with
+//! `refNameHash == h` paged by `startAfter`. That is single-branch, so the sibling-branch
+//! drop cannot happen, but protocol 13 still skips rows sharing the page boundary's
+//! `$createdAt` (docs/BUILDING.md, "same-block ties"), and repo-v1 ref updates do not return
+//! `$createdAt` from a proved query, so the tie probe cannot repair it. That read is the one
+//! `base_ref_tips` always used; it goes away with forge-v2 on protocol 14, whose cursor is
+//! bounded by document id. The mock's `ref_history` is exact, so no test here covers it.
+
+use std::collections::{BTreeMap, BTreeSet};
+
+use crate::error::{Error, Result};
+use crate::platform::{
+ FetchedDocument, FieldValue, LoadedContract, PlatformClient, QueryFilter, QueryOrder,
+};
+use crate::rules::RefUpdate;
+
+/// The plain ref-update document type.
+pub(crate) const DOC_REF_UPDATE: &str = "refUpdate";
+/// The MAINTAIN-gated ref-update document type.
+pub(crate) const DOC_PROTECTED_REF_UPDATE: &str = "protectedRefUpdate";
+
+/// Both ref-update types, each with the `protected` flag its updates carry into the fold.
+const REF_UPDATE_TYPES: [(&str, bool); 2] =
+ [(DOC_REF_UPDATE, false), (DOC_PROTECTED_REF_UPDATE, true)];
+
+/// Rows per keyset page — Drive's per-query maximum.
+pub(crate) const KEYSET_PAGE: u32 = 100;
+
+/// A backstop against a node that never lets the scan finish. Every round advances past at
+/// least one ref, so this is a bound on distinct refs per type (with every ref on its own
+/// round), not on updates.
+const MAX_KEYSET_ROUNDS: usize = 100_000;
+
+/// Every ref's history, keyed by the raw `refNameHash`.
+pub type RefHistories = BTreeMap<[u8; 32], Vec>;
+
+/// The three reads the scan is built from. A trait so the scan (the part that decides
+/// whether a read is complete) runs against a mock in tests; [`PlatformRefSource`] is the
+/// live implementation.
+pub(crate) trait RefDocSource {
+ /// One page of `doc_type` ordered `(refNameHash, $createdAt)`, restricted to
+ /// `refNameHash > after` when `after` is set, at most [`KEYSET_PAGE`] rows. No cursor.
+ async fn keyset_page(
+ &self,
+ doc_type: &str,
+ after: Option<[u8; 32]>,
+ ) -> Result>;
+
+ /// Every `doc_type` update of one ref (an equality read, paged to exhaustion).
+ async fn ref_history(&self, doc_type: &str, hash: [u8; 32]) -> Result>;
+
+ /// Every `doc_type` update in `$createdAt` order — the fallback.
+ async fn full_scan(&self, doc_type: &str) -> Result>;
+}
+
+/// [`RefDocSource`] over a live Platform connection.
+pub(crate) struct PlatformRefSource<'a> {
+ pub(crate) client: &'a PlatformClient,
+ pub(crate) contract: &'a LoadedContract,
+}
+
+impl RefDocSource for PlatformRefSource<'_> {
+ async fn keyset_page(
+ &self,
+ doc_type: &str,
+ after: Option<[u8; 32]>,
+ ) -> Result> {
+ let filters: Vec = after
+ .map(|h| QueryFilter::gt("refNameHash", FieldValue::bytes32(h)))
+ .into_iter()
+ .collect();
+ self.client
+ .query_documents(
+ self.contract,
+ doc_type,
+ &filters,
+ &[
+ QueryOrder::asc("refNameHash"),
+ QueryOrder::asc("$createdAt"),
+ ],
+ KEYSET_PAGE,
+ None,
+ )
+ .await
+ }
+
+ async fn ref_history(&self, doc_type: &str, hash: [u8; 32]) -> Result> {
+ self.client
+ .query_all_documents(
+ self.contract,
+ doc_type,
+ &[QueryFilter::eq("refNameHash", FieldValue::bytes32(hash))],
+ &[QueryOrder::asc("$createdAt")],
+ )
+ .await
+ }
+
+ async fn full_scan(&self, doc_type: &str) -> Result> {
+ self.client
+ .query_all_documents(
+ self.contract,
+ doc_type,
+ &[],
+ &[QueryOrder::asc("$createdAt")],
+ )
+ .await
+ }
+}
+
+/// Read every ref's complete history from a live repo contract. See the module docs.
+pub async fn read_all_ref_updates(
+ client: &PlatformClient,
+ contract: &LoadedContract,
+) -> Result {
+ read_all_with(&PlatformRefSource { client, contract }).await
+}
+
+/// Read one ref's complete history (both types) from a live repo contract.
+pub async fn read_ref_history(
+ client: &PlatformClient,
+ contract: &LoadedContract,
+ ref_name_hash: [u8; 32],
+) -> Result> {
+ ref_history_with(&PlatformRefSource { client, contract }, ref_name_hash).await
+}
+
+pub(crate) async fn ref_history_with(
+ src: &impl RefDocSource,
+ hash: [u8; 32],
+) -> Result> {
+ let hash_hex = hex::encode(hash);
+ let mut out = Vec::new();
+ for (doc_type, protected) in REF_UPDATE_TYPES {
+ for d in src.ref_history(doc_type, hash).await? {
+ out.push(ref_update_from_doc(&d, &hash_hex, protected));
+ }
+ }
+ Ok(out)
+}
+
+/// The scan behind [`read_all_ref_updates`], over any [`RefDocSource`].
+pub(crate) async fn read_all_with(src: &impl RefDocSource) -> Result {
+ let mut docs: Vec<(Vec, bool, &str)> = Vec::new();
+ let mut misbehaved = false;
+ for (doc_type, protected) in REF_UPDATE_TYPES {
+ let Some(rows) = keyset_scan(src, doc_type).await? else {
+ misbehaved = true;
+ break;
+ };
+ docs.push((rows, protected, doc_type));
+ }
+ if misbehaved {
+ tracing::warn!(
+ "a ref-update keyset page came back out of order or out of range; re-reading \
+ every update in reflog order"
+ );
+ } else {
+ let by_hash = group(&docs)?;
+ let dangling = by_hash.values().filter(|u| has_missing_parent(u)).count();
+ if dangling == 0 {
+ return Ok(by_hash);
+ }
+ tracing::warn!(
+ refs_with_missing_parent = dangling,
+ "ref-update keyset scan is missing parents; re-reading every update in reflog order"
+ );
+ }
+ // The fallback stands alone: a scan that misbehaved or lost rows is not trusted for any
+ // of them, and the reflog read is complete on its own.
+ let mut full = Vec::with_capacity(REF_UPDATE_TYPES.len());
+ for (doc_type, protected) in REF_UPDATE_TYPES {
+ full.push((dedupe(src.full_scan(doc_type).await?), protected, doc_type));
+ }
+ group(&full)
+}
+
+/// Drop repeated `$id`s, keeping the first occurrence.
+fn dedupe(rows: Vec) -> Vec {
+ let mut seen = BTreeSet::new();
+ rows.into_iter()
+ .filter(|d| seen.insert(d.id.clone()))
+ .collect()
+}
+
+/// Page one type by key (see the module docs). `None` when the node did not honor the
+/// query: a page out of order, a row at or below the `refNameHash > after` bound, or a round
+/// that would not move `after` forward. The caller then discards the scan entirely.
+async fn keyset_scan(
+ src: &impl RefDocSource,
+ doc_type: &str,
+) -> Result>> {
+ let mut out: Vec = Vec::new();
+ let mut after: Option<[u8; 32]> = None;
+ for _ in 0..MAX_KEYSET_ROUNDS {
+ let page = src.keyset_page(doc_type, after).await?;
+ let hashes = page
+ .iter()
+ .map(|d| ref_hash(d, doc_type))
+ .collect::>>()?;
+ let in_order = hashes.windows(2).all(|w| w[0] <= w[1]);
+ let in_range = after.is_none_or(|a| hashes.iter().all(|h| *h > a));
+ if !in_order || !in_range {
+ return Ok(None);
+ }
+
+ if page.len() < KEYSET_PAGE as usize {
+ out.extend(page);
+ return Ok(Some(dedupe(out)));
+ }
+ let last = *hashes.last().expect("a full page is non-empty");
+ let cut = hashes.iter().position(|h| *h == last).unwrap_or(0);
+ let next = if cut == 0 {
+ // One ref filled the page: read it on its own, then move past it.
+ out.extend(src.ref_history(doc_type, last).await?);
+ last
+ } else {
+ // Every ref before the last is whole; the last may be cut off, so it is
+ // re-read from its start on the next page.
+ out.extend(page.into_iter().take(cut));
+ hashes[cut - 1]
+ };
+ // In-range pages already imply progress; this guards the invariant the loop's
+ // termination rests on rather than the node's behavior.
+ if after.is_some_and(|a| next <= a) {
+ return Ok(None);
+ }
+ after = Some(next);
+ }
+ Err(Error::IncompleteRead {
+ document_type: doc_type.to_string(),
+ fetched: out.len(),
+ reason: format!("the ref keyset scan did not finish in {MAX_KEYSET_ROUNDS} rounds"),
+ })
+}
+
+/// Group rows per ref. Within a ref, plain updates come before protected ones and each
+/// source keeps its read order; the fold re-sorts by `(createdAt, id)` regardless.
+fn group(docs: &[(Vec, bool, &str)]) -> Result {
+ let mut by_hash = RefHistories::new();
+ for (rows, protected, doc_type) in docs {
+ for d in rows {
+ let hash = ref_hash(d, doc_type)?;
+ by_hash.entry(hash).or_default().push(ref_update_from_doc(
+ d,
+ &hex::encode(hash),
+ *protected,
+ ));
+ }
+ }
+ Ok(by_hash)
+}
+
+/// The document's `refNameHash`. A row without one cannot be attributed to a ref; the schema
+/// forbids it, and skipping it would make the answer wrong rather than partial.
+fn ref_hash(d: &FetchedDocument, doc_type: &str) -> Result<[u8; 32]> {
+ d.field_bytes("refNameHash")
+ .and_then(|b| <[u8; 32]>::try_from(b).ok())
+ .ok_or_else(|| Error::Platform(format!("{doc_type} {} has no 32-byte refNameHash", d.id)))
+}
+
+/// Whether some update's non-null `prevOid` is no update's `newOid` in the same ref — a
+/// parent that should have been read and was not. Parity: forge-web `hasMissingParent`.
+pub fn has_missing_parent(updates: &[RefUpdate]) -> bool {
+ let tips: BTreeSet<&str> = updates.iter().map(|u| u.new_oid.as_str()).collect();
+ updates.iter().any(|u| {
+ let null = u.prev_oid.is_empty() || u.prev_oid.bytes().all(|b| b == b'0');
+ !null && !tips.contains(u.prev_oid.as_str())
+ })
+}
+
+/// Flatten a ref-update document to the [`RefUpdate`] shape the fold consumes.
+pub(crate) fn ref_update_from_doc(
+ d: &FetchedDocument,
+ hash_hex: &str,
+ protected: bool,
+) -> RefUpdate {
+ RefUpdate {
+ id: d.id.clone(),
+ ref_name_hash: hash_hex.to_string(),
+ ref_name: d.field_str("refName").unwrap_or_default(),
+ prev_oid: d.field_hex("prevOid").unwrap_or_default(),
+ new_oid: d.field_hex("newOid").unwrap_or_default(),
+ force: d.field_bool("force"),
+ protected,
+ author: d.owner_id.clone(),
+ created_at: d.created_at.unwrap_or(0),
+ }
+}
+
+#[cfg(test)]
+// The mocks below answer synchronously; `async fn` keeps them shaped like the live source.
+#[allow(clippy::unused_async_trait_impl)]
+mod tests {
+ use super::{
+ has_missing_parent, read_all_with, ref_history_with, RefDocSource, DOC_REF_UPDATE,
+ KEYSET_PAGE,
+ };
+ use crate::error::Result;
+ use crate::platform::{FetchedDocument, FieldValue};
+ use std::cell::{Cell, RefCell};
+ use std::collections::{BTreeMap, BTreeSet};
+
+ /// A refUpdate row. `ref_seed` picks the ref (its hash is `[ref_seed; 32]`), `id` its
+ /// `$id`; oids are one byte repeated so a chain is easy to spell.
+ fn row(ref_seed: u8, id: u32, prev: u8, new: u8) -> FetchedDocument {
+ let mut fields = BTreeMap::new();
+ fields.insert("refNameHash".into(), FieldValue::Bytes32([ref_seed; 32]));
+ fields.insert(
+ "refName".into(),
+ FieldValue::Text(format!("refs/heads/r{ref_seed}")),
+ );
+ fields.insert("newOid".into(), FieldValue::Bytes(vec![new; 20]));
+ if prev != 0 {
+ fields.insert("prevOid".into(), FieldValue::Bytes(vec![prev; 20]));
+ }
+ FetchedDocument {
+ id: format!("id{id:06}"),
+ owner_id: "pusher".into(),
+ // The deployed repo-v1 type never recorded `$createdAt` (design-freeze-2 §3).
+ created_at: None,
+ fields,
+ }
+ }
+
+ /// A ref with `n` linear updates (1 → 2 → … → n), ids drawn from `ids`.
+ fn chain(ref_seed: u8, n: u8, ids: &mut u32) -> Vec {
+ (1..=n)
+ .map(|k| {
+ *ids += 1;
+ row(ref_seed, *ids, k - 1, k)
+ })
+ .collect()
+ }
+
+ fn hash_of(d: &FetchedDocument) -> [u8; 32] {
+ <[u8; 32]>::try_from(d.field_bytes("refNameHash").unwrap()).unwrap()
+ }
+
+ /// An in-memory Drive for one document type, in `refState` order
+ /// (`refNameHash`, then — `$createdAt` being absent — `$id`).
+ ///
+ /// `cursor_page` reproduces the protocol-13 multi-branch cursor lowering: the cursor
+ /// document's `$id` bound leaks into every later `refNameHash` branch, so rows there
+ /// with a smaller `$id` are skipped. `drop_on_keyset` removes one id from keyset pages,
+ /// standing in for a node that answers a correct query incompletely.
+ struct MockDrive {
+ rows: Vec,
+ drop_on_keyset: Option,
+ /// Serve keyset pages ignoring `refNameHash > after` (a node not honoring the query).
+ ignore_range: bool,
+ keyset_calls: Cell,
+ history_calls: RefCell>,
+ full_scans: Cell,
+ }
+
+ impl MockDrive {
+ fn new(mut rows: Vec) -> Self {
+ rows.sort_by(|a, b| hash_of(a).cmp(&hash_of(b)).then_with(|| a.id.cmp(&b.id)));
+ Self {
+ rows,
+ drop_on_keyset: None,
+ ignore_range: false,
+ keyset_calls: Cell::new(0),
+ history_calls: RefCell::new(Vec::new()),
+ full_scans: Cell::new(0),
+ }
+ }
+
+ /// The `startAfter` page the old reader requested, with the v0 sibling-branch drop.
+ fn cursor_page(&self, after_id: Option<&str>) -> Vec {
+ let Some(after_id) = after_id else {
+ return self
+ .rows
+ .iter()
+ .take(KEYSET_PAGE as usize)
+ .cloned()
+ .collect();
+ };
+ let cursor = self.rows.iter().find(|d| d.id == after_id).unwrap();
+ let pos = self.rows.iter().position(|d| d.id == after_id).unwrap();
+ self.rows[pos + 1..]
+ .iter()
+ .filter(|d| hash_of(d) == hash_of(cursor) || d.id > cursor.id)
+ .take(KEYSET_PAGE as usize)
+ .cloned()
+ .collect()
+ }
+ }
+
+ impl RefDocSource for MockDrive {
+ async fn keyset_page(
+ &self,
+ _doc_type: &str,
+ after: Option<[u8; 32]>,
+ ) -> Result> {
+ self.keyset_calls.set(self.keyset_calls.get() + 1);
+ Ok(self
+ .rows
+ .iter()
+ .filter(|d| self.ignore_range || after.is_none_or(|a| hash_of(d) > a))
+ .filter(|d| self.drop_on_keyset.as_deref() != Some(d.id.as_str()))
+ .take(KEYSET_PAGE as usize)
+ .cloned()
+ .collect())
+ }
+
+ async fn ref_history(
+ &self,
+ _doc_type: &str,
+ hash: [u8; 32],
+ ) -> Result> {
+ self.history_calls.borrow_mut().push(hash);
+ Ok(self
+ .rows
+ .iter()
+ .filter(|d| hash_of(d) == hash)
+ .cloned()
+ .collect())
+ }
+
+ async fn full_scan(&self, _doc_type: &str) -> Result> {
+ self.full_scans.set(self.full_scans.get() + 1);
+ Ok(self.rows.clone())
+ }
+ }
+
+ /// Serves only the plain type; protected reads come back empty.
+ struct PlainOnly(MockDrive);
+
+ impl RefDocSource for PlainOnly {
+ async fn keyset_page(
+ &self,
+ t: &str,
+ after: Option<[u8; 32]>,
+ ) -> Result> {
+ if t == DOC_REF_UPDATE {
+ self.0.keyset_page(t, after).await
+ } else {
+ Ok(Vec::new())
+ }
+ }
+ async fn ref_history(&self, t: &str, hash: [u8; 32]) -> Result> {
+ if t == DOC_REF_UPDATE {
+ self.0.ref_history(t, hash).await
+ } else {
+ Ok(Vec::new())
+ }
+ }
+ async fn full_scan(&self, t: &str) -> Result> {
+ if t == DOC_REF_UPDATE {
+ self.0.full_scan(t).await
+ } else {
+ Ok(Vec::new())
+ }
+ }
+ }
+
+ /// 60 refs × 1..4 updates with ids interleaved across refs — the shape of the nightly
+ /// repo, where every run pushes several refs and ids are random with respect to hashes.
+ fn nightly_like() -> Vec {
+ let mut ids = 0u32;
+ let mut rows = Vec::new();
+ for r in 1..=60u8 {
+ rows.extend(chain(r, 1 + r % 4, &mut ids));
+ }
+ // Scramble ids against hash order: reverse them.
+ let n = rows.len();
+ for (i, d) in rows.iter_mut().enumerate() {
+ d.id = format!("id{:06}", n - i);
+ }
+ rows
+ }
+
+ #[test]
+ fn the_mock_reproduces_the_cursor_drop() {
+ // The old reader: refState order, paged by `startAfter`, stop at a short page.
+ let drive = MockDrive::new(nightly_like());
+ let total = drive.rows.len();
+ assert!(total > KEYSET_PAGE as usize, "needs more than one page");
+ let mut got = 0;
+ let mut after: Option = None;
+ loop {
+ let page = drive.cursor_page(after.as_deref());
+ got += page.len();
+ if page.len() < KEYSET_PAGE as usize {
+ break;
+ }
+ after = page.last().map(|d| d.id.clone());
+ }
+ assert!(
+ got < total,
+ "the cursor read should lose rows ({got} of {total})"
+ );
+ }
+
+ #[tokio::test]
+ async fn keyset_scan_reads_every_update_without_a_cursor() {
+ let drive = PlainOnly(MockDrive::new(nightly_like()));
+ let total = drive.0.rows.len();
+ let got = read_all_with(&drive).await.unwrap();
+ assert_eq!(got.len(), 60);
+ assert_eq!(got.values().map(Vec::len).sum::(), total);
+ assert_eq!(
+ drive.0.full_scans.get(),
+ 0,
+ "a consistent scan needs no fallback"
+ );
+ // ~2 pages for ~150 rows, not one round-trip per ref.
+ assert!(
+ drive.0.keyset_calls.get() <= 3,
+ "{} pages",
+ drive.0.keyset_calls.get()
+ );
+ }
+
+ #[tokio::test]
+ async fn a_ref_that_fills_a_page_is_read_on_its_own() {
+ let mut ids = 0u32;
+ let mut rows = chain(5, 3, &mut ids);
+ rows.extend(chain(7, 150, &mut ids)); // one hot ref, more than a page
+ rows.extend(chain(9, 2, &mut ids));
+ let drive = PlainOnly(MockDrive::new(rows));
+ let got = read_all_with(&drive).await.unwrap();
+ assert_eq!(got[&[7; 32]].len(), 150);
+ assert_eq!(got[&[5; 32]].len(), 3);
+ assert_eq!(got[&[9; 32]].len(), 2);
+ assert_eq!(*drive.0.history_calls.borrow(), vec![[7u8; 32]]);
+ assert_eq!(drive.0.full_scans.get(), 0);
+ }
+
+ #[tokio::test]
+ async fn an_out_of_range_page_abandons_the_scan_for_the_full_read() {
+ // A node that ignores `refNameHash > after` serves the first page forever: the scan
+ // must stop at once (not loop, not merge) and answer from the reflog read alone.
+ let mut drive = MockDrive::new(nightly_like());
+ drive.ignore_range = true;
+ let drive = PlainOnly(drive);
+ let got = read_all_with(&drive).await.unwrap();
+ assert_eq!(drive.0.keyset_calls.get(), 2, "stops on the first bad page");
+ assert_eq!(drive.0.full_scans.get(), 1);
+ assert_eq!(
+ got.values().map(Vec::len).sum::(),
+ drive.0.rows.len()
+ );
+ }
+
+ #[tokio::test]
+ async fn a_missing_parent_falls_back_to_the_full_scan_alone() {
+ let mut rows = nightly_like();
+ // Drop the middle update of a 3-update ref from keyset pages only.
+ let victim = rows
+ .iter()
+ .find(|d| hash_of(d) == [2; 32] && d.field_hex("newOid") == Some("02".repeat(20)))
+ .unwrap()
+ .id
+ .clone();
+ rows.sort_by(|a, b| a.id.cmp(&b.id));
+ let mut drive = MockDrive::new(rows);
+ drive.drop_on_keyset = Some(victim);
+ let drive = PlainOnly(drive);
+ let got = read_all_with(&drive).await.unwrap();
+ assert_eq!(drive.0.full_scans.get(), 1);
+ assert_eq!(got[&[2; 32]].len(), 3, "the dropped update is recovered");
+ assert_eq!(
+ got.values().map(Vec::len).sum::(),
+ drive.0.rows.len(),
+ "and nothing is counted twice"
+ );
+ }
+
+ #[tokio::test]
+ async fn ref_history_reads_both_types() {
+ let mut ids = 0u32;
+ let drive = MockDrive::new(chain(3, 4, &mut ids));
+ // The mock serves the same rows for both types; each is tagged with its source.
+ let got = ref_history_with(&drive, [3; 32]).await.unwrap();
+ assert_eq!(got.len(), 8);
+ assert_eq!(got.iter().filter(|u| u.protected).count(), 4);
+ let ids: BTreeSet<&str> = got.iter().map(|u| u.id.as_str()).collect();
+ assert_eq!(ids.len(), 4);
+ }
+
+ #[test]
+ fn missing_parent_detection() {
+ let u = |prev: &str, new: &str| crate::rules::RefUpdate {
+ id: new.into(),
+ ref_name_hash: "h".into(),
+ ref_name: "refs/heads/x".into(),
+ prev_oid: prev.into(),
+ new_oid: new.into(),
+ force: false,
+ protected: false,
+ author: "a".into(),
+ created_at: 0,
+ };
+ assert!(!has_missing_parent(&[u("", "aa"), u("aa", "bb")]));
+ assert!(
+ !has_missing_parent(&[u("0000", "aa"), u("aa", "0000")]),
+ "create + delete"
+ );
+ assert!(has_missing_parent(&[u("", "aa"), u("cc", "dd")]));
+ }
+}
diff --git a/crates/forge-core/src/repo.rs b/crates/forge-core/src/repo.rs
index 70a7109d1..1628f1421 100644
--- a/crates/forge-core/src/repo.rs
+++ b/crates/forge-core/src/repo.rs
@@ -25,10 +25,10 @@ use crate::backends::{ByteRange, PackBackend, PackMeta, PlatformBackend, Uri};
use crate::error::{Error, Result};
use crate::keystore::BridgeIdentity;
use crate::platform::{
- self, FetchedDocument, FieldValue, JournalStore, LoadedContract, LoadedIdentity,
- PlatformClient, PushJournal, QueryFilter, QueryOrder, WriteEngine, WriteIntent,
+ self, FieldValue, JournalStore, LoadedContract, LoadedIdentity, PlatformClient, PushJournal,
+ QueryFilter, QueryOrder, WriteEngine, WriteIntent,
};
-use crate::rules::{self, ConfigDoc, RefState, RefUpdate};
+use crate::rules::{self, ConfigDoc, RefState};
use crate::storage::{PackReader, Replication, StorageTarget};
/// The repo-v1 contract template (2 tokens + 15 doc types), embedded at build time.
@@ -44,8 +44,7 @@ const REPO_V1_TEMPLATE: &str = include_str!(concat!(
// Document type names (repo contract).
const DOC_CONFIG: &str = "config";
-const DOC_REF_UPDATE: &str = "refUpdate";
-const DOC_PROTECTED_REF_UPDATE: &str = "protectedRefUpdate";
+use crate::refs::{DOC_PROTECTED_REF_UPDATE, DOC_REF_UPDATE};
const DOC_PACK_MANIFEST: &str = "packManifest";
const DOC_MANIFEST_PART: &str = "manifestPart";
const DOC_CHUNK: &str = "chunk";
@@ -712,17 +711,19 @@ impl<'a> RepoService<'a> {
/// Enumerate every ref and its resolved [`RefState`].
///
- /// Both ref-update types are read whole, paged in `$createdAt` order, and grouped by
- /// `refNameHash` locally; each ref's combined update history + the repo's `config`
- /// history is folded by [`crate::rules::resolve_ref`].
+ /// Every ref's history comes from [`crate::refs::read_all_ref_updates`] — a keyset scan
+ /// over the `refState` index, ⌈updates/100⌉ queries per type plus one per ref that fills
+ /// a page by itself, with a `prevOid` completeness check — and each ref's combined
+ /// update history + the repo's `config` history is folded by
+ /// [`crate::rules::resolve_ref`].
///
- /// This used to discover refs with the S0.8 skip-scan (one `limit 1` query per ref per
- /// type) and then read each ref's history separately: about four sequential round-trips
- /// per ref, *including deleted ones*, since a delete is just another update. Every
- /// `git` command on a `dash://` remote reads refs at least once, so on the nightly's
- /// test repo (≈80 refs ever pushed, 170 updates) that was ~300 queries and over a
- /// minute per command — the reason a partial clone, which reads refs twice, outlived
- /// its command timeout. Paging costs ⌈updates/100⌉ queries per type instead.
+ /// Two earlier readers failed in opposite ways. The S0.8 skip-scan (one `limit 1` query
+ /// per ref per type, then one history read per ref) cost about four sequential
+ /// round-trips per ref ever pushed, deleted ones included: over a minute per `git`
+ /// command on the nightly repo. Paging the `refState` index with a `startAfter` cursor
+ /// was fast but silently lost rows on protocol 13 (see the `refs` module docs). Paging
+ /// the whole `reflog` index was correct but read every update of every ref on every
+ /// command, so it is kept only as the fallback the completeness check falls to.
///
/// The ancestry predicate is reflexive-only here (M1 has no read-side commit graph):
/// fast-forward supersession via `prevOid` still resolves, but descend-detection is
@@ -732,7 +733,7 @@ impl<'a> RepoService<'a> {
let repo_contract = self.client.fetch_contract(&repo.repo_contract_id).await?;
let configs = self.fetch_config_history(&repo_contract).await?;
- let by_hash = self.read_all_ref_updates(&repo_contract).await?;
+ let by_hash = crate::refs::read_all_ref_updates(self.client, &repo_contract).await?;
let mut out = Vec::with_capacity(by_hash.len());
for (hash, updates) in &by_hash {
@@ -1976,70 +1977,6 @@ impl<'a> RepoService<'a> {
})
.collect())
}
-
- /// Read every ref's full update history, grouped by `refNameHash`: both ref-update types
- /// paged to exhaustion in `$createdAt` order. Within a ref, plain updates come before
- /// protected ones and each source is `$createdAt asc` — the order the per-ref fold has
- /// always consumed (the fold itself re-sorts by `(createdAt, id)`).
- ///
- /// Paged to exhaustion: `resolve_ref` folds the whole causal chain, so stopping at one
- /// page would pin a branch at a stale tip — `list` would advertise it, clones would get
- /// a stale HEAD, and `plan_pushes` would call correct fast-forwards non-fast-forward.
- ///
- /// A document without a 32-byte `refNameHash` cannot be attributed to a ref; the
- /// schema forbids it, and skipping it silently would make the answer wrong rather than
- /// partial, so it fails the read.
- async fn read_all_ref_updates(
- &self,
- repo_contract: &LoadedContract,
- ) -> Result>> {
- let mut by_hash: BTreeMap<[u8; 32], Vec> = BTreeMap::new();
- for (doc_type, protected) in [(DOC_REF_UPDATE, false), (DOC_PROTECTED_REF_UPDATE, true)] {
- let docs = self
- .client
- .query_all_documents(
- repo_contract,
- doc_type,
- &[],
- // The `reflog` index. NOT `refState` (`refNameHash, $createdAt`): on
- // testnet, `start_after` paging over that compound index dropped 5 of
- // 168 updates at the page boundary (one ref vanished from `ls-remote`),
- // while `$createdAt` paging returns all of them.
- &[QueryOrder::asc("$createdAt")],
- )
- .await?;
- for d in &docs {
- let hash = d
- .field_bytes("refNameHash")
- .and_then(|b| <[u8; 32]>::try_from(b).ok())
- .ok_or_else(|| {
- Error::Platform(format!("{doc_type} {} has no 32-byte refNameHash", d.id))
- })?;
- by_hash.entry(hash).or_default().push(ref_update_from_doc(
- d,
- &hex::encode(hash),
- protected,
- ));
- }
- }
- Ok(by_hash)
- }
-}
-
-/// Flatten a `refUpdate` / `protectedRefUpdate` document to the [`RefUpdate`] shape
-/// [`crate::rules::resolve_ref`] consumes.
-fn ref_update_from_doc(d: &FetchedDocument, hash_hex: &str, protected: bool) -> RefUpdate {
- RefUpdate {
- id: d.id.clone(),
- ref_name_hash: hash_hex.to_string(),
- ref_name: d.field_str("refName").unwrap_or_default(),
- prev_oid: d.field_hex("prevOid").unwrap_or_default(),
- new_oid: d.field_hex("newOid").unwrap_or_default(),
- force: d.field_bool("force"),
- protected,
- author: d.owner_id.clone(),
- created_at: d.created_at.unwrap_or(0),
- }
}
/// The initial `config` document properties (`defaultBranch`, empty protected patterns,
diff --git a/crates/forge-core/src/rules.rs b/crates/forge-core/src/rules.rs
index b6ed4edfb..b39f275ec 100644
--- a/crates/forge-core/src/rules.rs
+++ b/crates/forge-core/src/rules.rs
@@ -110,7 +110,7 @@ impl Ancestry {
// ===========================================================================
/// A single append-only `refUpdate` / `protectedRefUpdate` document, flattened to the
-/// fields resolution needs. Callers fetch these (via the §2.3 skip-scan + §3
+/// fields resolution needs. Callers fetch these (via the §2.3 keyset scan, `crate::refs`, + §3
/// completeness fallback) and hand the slice in.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
diff --git a/crates/git-remote-dash/src/helper.rs b/crates/git-remote-dash/src/helper.rs
index 8dbc39371..88f041962 100644
--- a/crates/git-remote-dash/src/helper.rs
+++ b/crates/git-remote-dash/src/helper.rs
@@ -436,17 +436,25 @@ impl Helper {
const MAX_ATTEMPTS: usize = 6;
let conn = self.conn.as_ref().expect("connected before finalize");
let svc = RepoService::new(&conn.client, &conn.identity, &conn.bridge);
- let expected: Vec<(&str, &str)> = planned
+ // `None` = a delete, converged once the ref reads as gone. Deletes wait too: a node
+ // that has not applied the delete yet would otherwise make a landed delete read as
+ // "did not take effect" (the nightly's 03 scenario hit exactly that).
+ let expected: Vec<(&str, Option<&str>)> = planned
.iter()
.filter(|p| p.reject.is_none())
- .filter_map(|p| p.new_oid.as_deref().map(|oid| (p.spec.dst.as_str(), oid)))
+ .map(|p| (p.spec.dst.as_str(), p.new_oid.as_deref()))
.collect();
let mut last = svc.read_refs(&conn.repo).await?;
for attempt in 1..=MAX_ATTEMPTS {
- let converged = expected.iter().all(|(dst, oid)| {
- matches!(last.iter().find(|(n, _)| n == dst),
- Some((_, RefState::Resolved { oid: got, .. })) if got == oid)
+ let converged = expected.iter().all(|(dst, want)| {
+ let state = last.iter().find(|(n, _)| n == dst).map(|(_, s)| s);
+ match want {
+ Some(oid) => {
+ matches!(state, Some(RefState::Resolved { oid: got, .. }) if got == oid)
+ }
+ None => matches!(state, None | Some(RefState::Unborn)),
+ }
});
if converged || expected.is_empty() {
break;
@@ -923,6 +931,16 @@ async fn publish_browse_index(
replication: &Replication,
externals: &[ExternalTarget],
) {
+ // `DASH_FORGE_NO_BROWSE_INDEX=1` skips it on purpose: the nightly's read fixture
+ // (e2e/cli/seed-read-fixture.sh) must stay unindexed so the web app's fallback clone is
+ // what the browser specs exercise.
+ if matches!(
+ std::env::var("DASH_FORGE_NO_BROWSE_INDEX").as_deref(),
+ Ok("1" | "true")
+ ) {
+ tracing::info!("DASH_FORGE_NO_BROWSE_INDEX set; not publishing a browse-index fragment");
+ return;
+ }
let chain = replication.has_platform().then(|| {
PlatformChunkTarget::new(ctx.svc, ctx.repo, forge_core::storage::PLATFORM_PROFILE)
});
diff --git a/docs/contracts/data-contracts.md b/docs/contracts/data-contracts.md
index 65a2f7341..6bbc662c1 100644
--- a/docs/contracts/data-contracts.md
+++ b/docs/contracts/data-contracts.md
@@ -103,7 +103,18 @@ Why the non-deletables: a deletable `refUpdate` lets whoever authored the curren
Precise "supersedes" rule (pinned in `FORGE_RULES_V1`, Stage 2): the resolver tracks *live heads*. A head is superseded by a strictly-newer update that (a) is a delete or `force`, (b) fast-forwards off it (`prevOid == that head's newOid`), or (c) descends from it. **Consequence to note:** fast-forwarding only *one* racing head leaves the other still live → the ref stays `Diverged` (the strict reading of "supersedes both", not "newest wins"). Total-order tiebreak everywhere is `($createdAt, then $id)` — including which of two same-`$createdAt` configs applies (greatest `$id`). A merge event whose `oid` is unreachable from the base (or with no base tip supplied) is treated as inert.
-Branch/tag enumeration (no distinct-values query on Platform): **skip-scan with `limit 1` hops** — query `refNameHash > ` orderBy `refNameHash` **limit 1** to seek directly to the next distinct name (a `limit 100` page here would return *rows*, and one hyperactive ref with thousands of updates would make enumeration O(total pushes), not O(refs)); then batch tip lookups (`in` on ≤ 100 hashes with the §3 completeness fallback). Cost: one cheap seek per distinct ref — bounded by real branch/tag counts. **Skip-scan verified in S0.8 ✅** — a flat O(log n) seek regardless of duplicate-hash count. **Load-bearing:** `byteArray` where-operands (`refNameHash`, `packHash`, any oid/hash tip query) must be passed as **base64 strings** (not Uint8Array, not base58 — base58 is for identifiers only); results also return base64.
+Branch/tag enumeration (no distinct-values query on Platform) is a **keyset scan over the `refState` index**, per ref-update type (forge-core `refs.rs`, forge-web `lib/repo/refs.ts`; they behave the same):
+
+1. Query `refNameHash > `, ordered `(refNameHash, $createdAt)`, `limit 100`, **with no `startAfter` cursor**. On a full page, every ref except the last one is complete. The next page starts after the last complete ref, so a ref the page cut off is read again from its start.
+2. If a single ref fills a whole page, read it on its own with `refNameHash == h` (paged to exhaustion) and resume after it.
+3. **Completeness check:** every non-null `prevOid` must be some `newOid` of the same ref. A dangling parent, or a page that comes back out of order, triggers the fallback: read every update in `$createdAt` (`reflog`) order and union the result by `$id`.
+
+Cost: ⌈updates/100⌉ queries per type, plus one per ref with more than 100 updates. On the nightly repo that is 3 queries for 229 updates across 129 refs, and it returns the same 229 updates as the full reflog read. Two earlier designs failed, one on cost and one on correctness:
+
+- **`limit 1` skip-scan (S0.8):** one sequential seek per distinct ref, then a history read per ref. That meant about four round-trips per ref ever pushed, deleted refs included, and more than a minute per `git` command at about 80 refs.
+- **`startAfter` cursor over `refState`:** it silently loses rows on protocol 13. On the nightly repo it returned 197 of 229 updates. Drive's v0 path-query lowering applies the cursor document's lower index levels to *every* sibling `refNameHash` branch, not just the cursor's own branch, so later refs lose rows and a page can come back short, which a pager reads as the end. This is the same family as dashpay/platform#4396, which fixed the `in`-clause shape at protocol 14 and lists this `orderBy`-only shape as still unfixed. **Never page a multi-branch index with a document cursor.** An equality prefix (one branch) or a key range with no cursor is safe.
+
+A single ref's history, for PR base tips, is an equality read (`refNameHash == h`), so its cost scales with that ref's pushes, not the repo's. **Load-bearing:** `byteArray` where-operands (`refNameHash`, `packHash`, any oid/hash tip query) must be passed as **base64 strings** (not Uint8Array, not base58 — base58 is for identifiers only); results also return base64.
**`packManifest`** — `packHash` hash32; `kind` integer (**0 = git pack, 1 = objectLocator, 2 = flatIndex** — browse-plane artifacts share the entire pack storage/transport machinery); `sizeBytes`, `objectCount`, `chunkCount` integers; `storage` 0 platform / 1 external; `uris` list ≤ 8 of ≤ 300; `tips` list ≤ 16 oid (for kind 2: the tip commit it indexes); `supersedes` list ≤ 32 hash32; `offsetIndexParts` integer. **The list fields (`uris`, `tips`, `supersedes`) serialize as JSON-string / packed byteArray, not native arrays (§0).** Indices: unique `(packHash)`; `($createdAt desc)`; `(kind, $createdAt desc)` — readers grab newest locator/flatIndex in one query.
@@ -152,7 +163,7 @@ Count trees cost per-index storage overhead; they are assigned **only** where a
| release count | plain list query — newest-wins supersedes make raw doc counts wrong, and release pages are small | O(releases) |
| chunk availability audit | countable `chunk(packHash, seq)` vs `manifest.chunkCount` | O(1) |
| **open vs closed issue/PR counts** | **not natively countable — by design, not omission**: state is a fold of `event` docs (mutation ownership forbids an authoritative indexed `state` field on the author-owned doc). Strategy: list pages fold events per page (one `in` query on ≤ 100 targetIds); results cached in IndexedDB keyed by newest-event cursor; tabs render "Issues (N)" instantly and "open/closed" splits hydrate. Optional v1.1: MAINTAIN-posted `stateSummary` cache doc, explicitly non-authoritative, always corrected by fold. | fold |
-| branch/tag counts | skip-scan enumeration (cached); no count tree (distinct-count ≠ doc-count) | O(refs) |
+| branch/tag counts | keyset-scan enumeration (§2.3); no count tree (distinct-count ≠ doc-count) | O(updates / 100) |
Not countable (no story pays for the overhead): `refUpdate`, `event`, `manifestPart`, `checkRun`, `label`, `release`, `webhook`, `profile`. Flag validity for the unique+countable combinations is a named S0.6 check.
diff --git a/docs/design-freeze-2.md b/docs/design-freeze-2.md
index caad83c40..ab4b98d2e 100644
--- a/docs/design-freeze-2.md
+++ b/docs/design-freeze-2.md
@@ -20,7 +20,7 @@ The thesis is proven, not asserted:
5. **Trusted-connect proof-verifies plain reads** — the explicit `*WithProof` facade is redundant *and* browser-buggy in evo-sdk 4.0.0; forge-web uses plain `query()`/`get()` (still verified in trusted mode).
6. **Repo-v1 economics: instantiation ~1.18 DASH** (2 tokens + 15 doc types + count-trees), registry ~0.68 DASH — materially above the early <0.02 estimate; count-tree indices are the main driver. tokenCost payments **recirculate to the repo owner**, not burned.
7. **Contract templates: solo-owner by default.** A MainGroup needs ≥2 members; the source template now uses `ContractOwner` admin (org multi-sig is an opt-in v2 variant). Positions must be top-level-contiguous. Deployed contracts predate these and work via name-addressing + runtime compat shims; only new repos inherit the fixes.
-8. **Query completeness is mandatory, not optional** — `in`-batch starvation and the 100-row default truncate authz/fold reads; every fold/authz path paginates to exhaustion + the per-key completeness fallback. Ref enumeration uses flat `limit-1` skip-scan.
+8. **Query completeness is mandatory, not optional** — `in`-batch starvation and the 100-row default truncate authz/fold reads; every fold/authz path paginates to exhaustion + the per-key completeness fallback. Ref enumeration is a cursor-free keyset scan over `refState` (`refNameHash > last`, 100-row pages) with a `prevOid` completeness check that falls back to the full reflog read. **Superseded:** it was the `limit-1` skip-scan (one round-trip per ref, too slow), then briefly a `startAfter`-paged `refState` read, which drops sibling-branch rows on protocol 13 (data-contracts §2.3).
9. **Shallow clone dropped; partial clone kept** (S0.9) — a fetch/push helper has no depth reply channel; `--depth` fails loudly, `--filter=blob:none` works via `.promisor`. **jj works unmodified** (gitoxide).
10. **objectLocator: 36-byte fixed-stride rows** (deltaChainSpan fixed 4-byte, not varint; +1-byte delta-depth hint); single-span read for blobs, per-base walk for deep-delta trees. Cold browse loads root-tree-via-locator, not the O(files) flatIndex.
11. **CI reality**: per-push CI runs the web app + the 70-vector TS parity suite; the Rust workspace's large Platform dependency tree does not fit a per-push job for unrelated changes, so `rust.yml` gates pull requests and pushes that touch Rust paths, plus nightly. **Superseded in part:** the Platform SDK is no longer a path dep on a sibling checkout — it is a git dependency pinned to an immutable tag, so nothing clones the monorepo and `rust.yml` is the Rust gate rather than a backstop.
diff --git a/docs/design/style-guide.md b/docs/design/style-guide.md
index 1f9d0e87f..4f71e31d7 100644
--- a/docs/design/style-guide.md
+++ b/docs/design/style-guide.md
@@ -19,17 +19,22 @@ colors: {
50:'#fafaf9',100:'#f5f5f4',200:'#e7e5e4',300:'#d6d3d1',400:'#a8a29e',
500:'#78716c',600:'#57534e',700:'#44403c',750:'#3a3835',800:'#292524',850:'#211e1c',900:'#1c1917',950:'#0f0d0c'
},
- verify: '#16a34a', // proof/hash verified
+ verify: { // proof/hash verified
+ DEFAULT:'#16a34a', // icons, borders, tints
+ 700:'#15803d' // solid fill behind white text
+ },
caution:'#d97706', // degraded availability
danger: '#dc2626', // force-push, delete, failed verification
dash: { // Dash brand blue — reserved for identity/credits/network UI only
DEFAULT:'#008de4', // fills, tints, icons
400:'#4aaef0', // TEXT on dark surfaces
- 600:'#006bb0' // TEXT on light surfaces
+ 600:'#006bb0', // TEXT on light surfaces
+ 700:'#005a94' // solid fill behind white text
}
}
```
- **Dash-blue text is `text-dash-600 dark:text-dash-400`**, never plain `text-dash`: the brand value is under WCAG AA's 4.5:1 as text on every surface in both themes (4.28:1 on `anvil-800`). `forge-web/lib/design/contrast.test.ts` pins the ratios and fails on a raw `text-dash` that is not an icon.
+- **White text sits on `-700` fills**: `bg-dash-700`, `bg-verify-700`, `bg-forge-700`, and a hover darkens (`hover:bg-forge-800`) rather than brightens. The base `dash` (3.54:1) and `verify` (3.3:1) values and `forge-600` (3.56:1) fail AA behind white. The same test checks every `bg-*` (hover:/dark: variants included) behind `text-white`, and fails on any fill it cannot resolve. The identity pill's avatar fill is darkened per hue until its white initial clears 4.5:1 (`lib/design/avatar.ts`).
- **Dark mode is the primary theme** (class-based, `next-themes`); light mode fully supported. Backgrounds: `anvil-950/900/850` layered surfaces (dark), `anvil-50/white` (light).
- Semantic colors are *meaningful*, never decorative: green = cryptographically verified, amber = availability risk, red = destructive/unverified, dash-blue = platform identity & credits. Don't repurpose.
diff --git a/e2e/README.md b/e2e/README.md
index c60cd9f37..ab16b7fe4 100644
--- a/e2e/README.md
+++ b/e2e/README.md
@@ -1,17 +1,32 @@
# End-to-end suites
-- `cli/`: the CLI suite, run against **live testnet** (`make e2e`). Configuration is in `cli/config.sh`.
+- `cli/run.sh`: the CLI suite, run against **live testnet** (`make e2e`). Its configuration is in `cli/config.sh`.
+- `cli/seed-read-fixture.sh`: seeds the read fixture that the browser specs read (`make e2e-fixture`). The nightly runs it before Playwright.
+- `../forge-web/e2e/`: the Playwright specs. The read specs only read the read fixture.
- `cli/storage-byo.sh`: bring-your-own storage (`make storage-e2e`). A real `git push` / `git clone` whose packs go to the local MinIO + kubo from `infra/docker-compose.yml`.
## Reserved fixture repos (testnet)
-Each fixture repo belongs to one suite. Don't push to a repo that another suite owns.
-Packs stored on a repo's local-only storage (MinIO/kubo on `127.0.0.1`) can't be read by anyone else, so a repo shared with other suites would make their clones depend on your laptop.
+Each fixture repo belongs to one suite. Don't push to a repo that another suite owns, and don't run ad-hoc experiments on any of them.
-| Repo (owner = DEPLOYER `8hJmcHWT…`) | Used by | Notes |
-|---|---|---|
-| `m1-75299` | `cli/scenarios/*` (`make e2e`) | The shared M1 repo; Platform-stored packs only. |
-| `storage-e2e-a` | `cli/storage-byo.sh` steps 1–4 | Created by the script on its first run (repo-v1, about 1.18 tDASH once). Each run uses a fresh `e2e//byo` branch and deletes it afterwards. |
-| `storage-e2e-b` | `cli/storage-byo.sh` steps 5–6 | Same, plus the step-5 "copy deleted" scenario. Step 6 restores that copy, so the repo stays clonable. |
+Why this matters: a pack is only as readable as the storage its manifest names. Packs stored on local-only storage (MinIO/kubo on `127.0.0.1`) can't be read by anyone else. On 2026-09-25 a BYO-storage run pushed such packs into the CLI suite's repo, and every browser spec that browsed that repo failed with "no external URI served the range". A repo shared between suites turns one suite's storage choices into another suite's failures.
-Override the names with `STORAGE_E2E_REPO` / `STORAGE_E2E_REPO_B`.
+All of these repos are owned by DEPLOYER (`8hJmcHWTsdvkHyCrk4UgjbyugDAmE7QfuCTQXpXAc7nB`).
+
+| Repo | Written by | Read by | Notes |
+|---|---|---|---|
+| `m1-5124` | `cli/seed-read-fixture.sh` only (never the write spec) | `forge-web/e2e/*` (read-paths, fallback-browse, zero-backend, a11y) | The read fixture. `main` holds one deterministic commit (`README.md`, `src/`, `lib/`) with its pack stored on Platform and no browse index published, so the browser takes the in-browser fallback clone. The seeder is idempotent: it force-pushes `main` only when `main` is not at that commit. Override the name with `NIGHTLY_FIXTURE_REPO` (seeder) and `E2E_FIXTURE_NAME` (Playwright). |
+| `m1-75299` | `cli/scenarios/*` (`make e2e`); `forge-web/e2e/auth-write.spec.ts` with `E2E_WRITE=1` (issues only) | the same | The CLI suite's repo, and the opt-in web write spec's fixture (`WRITE_FIXTURE` in `forge-web/e2e/helpers.ts`, override `E2E_WRITE_FIXTURE_NAME`). It holds Platform-stored packs only. Each run pushes fresh `e2e//…` refs and deletes them afterwards. Override with `E2E_REPO_NAME`. It still holds seven stale external packs (MinIO/IPFS on `127.0.0.1`) from the 2026-09-25 incident; clones skip them with a warning. |
+| `storage-e2e-a` | `cli/storage-byo.sh` steps 1–4 | the same script | Created by the script on its first run (repo-v1, about 1.18 tDASH once). Each run uses a fresh `e2e//byo` branch and deletes it afterwards. |
+| `storage-e2e-b` | `cli/storage-byo.sh` steps 5–6 | the same script | Same, plus the step-5 "copy deleted" scenario. Step 6 restores that copy, so the repo stays clonable. |
+
+Override the storage repo names with `STORAGE_E2E_REPO` / `STORAGE_E2E_REPO_B`. When you add a suite that writes, give it its own repo and add a row here. `dg repo create ` costs about 1.18 tDASH once.
+
+## The partial-clone rule both clients follow
+
+A live pack whose storage cannot be reached does not fail a whole clone or browse:
+
+- `git-remote-dash` skips an external pack whose mirrors are down, and git's connectivity check then fails the fetch only if a wanted object was in it.
+- The web app's in-browser clone tries every recorded URI and the shared IPFS gateway list (`forge-contracts/config/storage-defaults.json`). If none of them serves the pack's exact sha256, it skips that pack. It then shows "N packs could not be fetched from their storage; some objects may be missing", marks the trust panel's content link `partial`, and names the missing pack and its hosts in any view that needs an object from it.
+
+Packs stored on Platform are never skipped: if one can't be read, the clone fails loudly.
diff --git a/e2e/cli/config.sh b/e2e/cli/config.sh
index b998bf458..3159ac22a 100644
--- a/e2e/cli/config.sh
+++ b/e2e/cli/config.sh
@@ -12,11 +12,14 @@
: "${DASH_FORGE_NETWORK:=testnet}"
export DASH_FORGE_NETWORK
-# --- the reused test repo ----------------------------------------------------
-# DEPLOYER is the owner + token granter for this repo.
+# --- the CLI suite's repo ----------------------------------------------------
+# DEPLOYER is the owner + token granter. Reserved for `run.sh` (e2e/README.md): every
+# scenario pushes Platform-stored packs to fresh `e2e//…` refs and deletes them.
+# An ad-hoc run that stores packs anywhere else (local MinIO/kubo) must use its own repo —
+# set E2E_REPO_NAME — so the nightly's clones never depend on someone's laptop.
export E2E_OWNER_ID="8hJmcHWTsdvkHyCrk4UgjbyugDAmE7QfuCTQXpXAc7nB"
-export E2E_REPO_NAME="m1-75299"
-export E2E_REPO_CONTRACT="5rrwgjjVUqMghnessfiXPXubpiM2QLNNXH142Hv4PDyX"
+: "${E2E_REPO_NAME:=m1-75299}"
+export E2E_REPO_NAME
export E2E_REMOTE="dash://${E2E_OWNER_ID}/${E2E_REPO_NAME}"
# --- dedicated bring-your-own-storage repos (e2e/cli/storage-byo.sh) ------------
@@ -27,6 +30,14 @@ export E2E_REMOTE="dash://${E2E_OWNER_ID}/${E2E_REPO_NAME}"
: "${STORAGE_E2E_REPO_B:=storage-e2e-b}"
export STORAGE_E2E_REPO STORAGE_E2E_REPO_B
+# --- the nightly's read fixture ----------------------------------------------
+# The repo the browser (Playwright) specs read: DEPLOYER-owned, written ONLY by
+# e2e/cli/seed-read-fixture.sh, which pins `main` to one deterministic commit and publishes
+# no browse index (so the in-browser fallback clone is what gets exercised). forge-web's
+# e2e/helpers.ts names the same repo.
+: "${NIGHTLY_FIXTURE_REPO:=m1-5124}"
+export NIGHTLY_FIXTURE_REPO
+
# --- fixture identity files --------------------------------------------------
: "${E2E_IDENTITY_DIR:=${HOME}/.config/dash-forge/test-identities}"
export E2E_IDENTITY_DIR
diff --git a/e2e/cli/run.sh b/e2e/cli/run.sh
index bf567996c..5d8e97a47 100755
--- a/e2e/cli/run.sh
+++ b/e2e/cli/run.sh
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
# run.sh — Dash Forge CLI end-to-end suite driver.
#
-# Runs every scenario against LIVE testnet, reusing the DEPLOYER-owned m1 repo
+# Runs every scenario against LIVE testnet, on the CLI suite's reserved DEPLOYER-owned repo
# (config.sh). Prints a PASS/FAIL/SKIP matrix and exits non-zero if ANY scenario
# FAILs. A scenario SKIPs only when a check flaked on every retry; one SKIP is reported
# but tolerated, more than E2E_MAX_SKIPS fails the run.
diff --git a/e2e/cli/seed-read-fixture.sh b/e2e/cli/seed-read-fixture.sh
new file mode 100644
index 000000000..c86ac37ac
--- /dev/null
+++ b/e2e/cli/seed-read-fixture.sh
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# seed-read-fixture.sh — make the nightly's browser read fixture exist and hold exactly
+# the expected content. Idempotent: a run that finds `main` already at the fixture commit
+# writes nothing.
+#
+# The fixture (NIGHTLY_FIXTURE_REPO, config.sh; reserved in e2e/README.md) is what
+# forge-web's Playwright specs read. It is written only here, so no other suite's refs,
+# packs or storage choices can reach what the browser sees:
+# * `main` = one deterministic commit (fixed author, committer and dates) holding a
+# README, a `src/` and a `lib/` directory — the entries the tree-browse spec looks for;
+# * its pack is stored on Platform (chunk documents), never external storage;
+# * no browse index is published (DASH_FORGE_NO_BROWSE_INDEX), so the web app takes the
+# in-browser fallback clone, which fallback-browse.spec.ts exercises.
+#
+# The repo is created (repo-v1, ~1.18 tDASH, once) only if absent. Exit 0 = seeded or
+# already seeded; 1 = could not seed.
+SCENARIO_NAME="seed the nightly read fixture (${NIGHTLY_FIXTURE_REPO:-m1-5124})"
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib.sh"
+harness_init
+
+FIX_REPO="${E2E_OWNER_ID}/${NIGHTLY_FIXTURE_REPO}"
+FIX_REMOTE="dash://${FIX_REPO}"
+SRC="${WORKROOT}/fixture-src"
+LOG="${WORKROOT}/fixture"
+
+step "fixture repo ${FIX_REPO}"
+if dg_read_retry "$ID_DEPLOYER" "$LOG-view.json" "$LOG-view.err" --json repo view "$FIX_REPO"; then
+ info "reusing ${FIX_REPO}"
+elif grep -qiE 'not found|no such|does not exist|unknown repo' "$LOG-view.err"; then
+ info "creating ${FIX_REPO} (one-time, ~1.18 tDASH)"
+ dg_as "$ID_DEPLOYER" --yes --json repo create "$NIGHTLY_FIXTURE_REPO" \
+ --description "Dash Forge nightly read fixture (reserved; see e2e/README.md)" \
+ >"$LOG-create.out" 2>"$LOG-create.err" || {
+ cat "$LOG-create.err" >&2; bad "could not create ${FIX_REPO}"; finish_scenario
+ }
+else
+ cat "$LOG-view.err" >&2; bad "could not read ${FIX_REPO}"; finish_scenario
+fi
+
+step "build the deterministic fixture commit"
+rm -rf "$SRC"; mkdir -p "$SRC/src" "$SRC/lib"
+git init -q -b main "$SRC"
+cat >"$SRC/README.md" <<'EOF'
+# Dash Forge nightly read fixture
+
+This repository is the fixture the Dash Forge web app's browser tests read. It is written
+only by `e2e/cli/seed-read-fixture.sh`; do not push to it.
+EOF
+printf 'fn main() {\n println!("hello from the nightly fixture");\n}\n' >"$SRC/src/main.rs"
+printf 'pub fn answer() -> u32 {\n 42\n}\n' >"$SRC/lib/answer.rs"
+export GIT_AUTHOR_NAME="Dash Forge Fixture" GIT_AUTHOR_EMAIL="fixture@dash-forge.test"
+export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+export GIT_AUTHOR_DATE="2026-09-25T00:00:00Z" GIT_COMMITTER_DATE="2026-09-25T00:00:00Z"
+git -C "$SRC" -c commit.gpgsign=false add -A
+git -C "$SRC" -c commit.gpgsign=false commit -q -m "nightly read fixture"
+WANT="$(git -C "$SRC" rev-parse HEAD)"
+info "fixture commit ${WANT}"
+
+remote_main() { # prints the remote main oid, or nothing
+ git_dash_retry "$ID_DEPLOYER" "$1" ls-remote "$FIX_REMOTE" refs/heads/main || return 1
+ awk '$2 == "refs/heads/main" { print $1 }' "$1.out"
+}
+
+# The stored packs, as `dg storage status` sees them: prints "ok" when every manifest is a
+# kind-0 git pack stored on Platform (no external copy, no browse index), else what is wrong.
+packs_ok() { # packs_ok
+ dg_read_retry "$ID_DEPLOYER" "$1.json" "$1.err" --json storage status "$FIX_REPO" || return 1
+ python3 - "$1.json" <<'PY'
+import json, sys
+packs = json.load(open(sys.argv[1])).get("packs", [])
+bad = [f"{p.get('packHash','?')[:12]} kind={p.get('kind')} tier={p.get('storageTier')}"
+ for p in packs if p.get("kind") != 0 or p.get("storageTier") != "platform"]
+print("ok" if packs and not bad else ("no packs" if not packs else "; ".join(bad)))
+PY
+}
+
+step "is ${FIX_REMOTE} main already at the fixture commit, on Platform, unindexed?"
+if ! HAVE="$(remote_main "$LOG-ls1")"; then
+ cat "$LOG-ls1.err" >&2; bad "could not list the fixture's refs"; finish_scenario
+fi
+if [[ "$HAVE" == "$WANT" ]]; then
+ if ! PACKS="$(packs_ok "$LOG-packs1")"; then
+ cat "$LOG-packs1.err" >&2; bad "could not read the fixture's pack manifests"; finish_scenario
+ fi
+ if [[ "$PACKS" == "ok" ]]; then
+ ok "already seeded (main = ${WANT}, Platform-stored, no browse index); nothing written"
+ else
+ # A re-push cannot fix this: the right pack is already recorded and the helper will not
+ # store it again. The repo needs its storage torn down (dg repo delete) and reseeding.
+ bad "main is at the fixture commit but its storage is wrong (${PACKS}); tear the fixture's storage down and reseed"
+ fi
+ finish_scenario
+fi
+info "main is ${HAVE:-absent}; force-pushing the fixture commit"
+
+step "push the fixture (Platform storage only, no browse index)"
+# `-c` is git's command scope, which the helper ranks above any global or repo setting, so
+# a developer's `dash.storage=minio` (or a per-remote override) cannot redirect this pack.
+if ! DASH_FORGE_NO_BROWSE_INDEX=1 git_dash_retry "$ID_DEPLOYER" "$LOG-push" \
+ -C "$SRC" -c dash.storage=platform -c dash.platformFallback=false \
+ -c "remote.origin.dashStorage=platform" \
+ push "$FIX_REMOTE" "+refs/heads/main:refs/heads/main"; then
+ cat "$LOG-push.err" >&2; bad "fixture push failed"; finish_scenario
+fi
+if HAVE="$(remote_main "$LOG-ls2")" && [[ "$HAVE" == "$WANT" ]]; then
+ ok "seeded: main = ${WANT}"
+else
+ bad "after the push, main is ${HAVE:-unreadable}, not ${WANT}"
+fi
+if PACKS="$(packs_ok "$LOG-packs2")" && [[ "$PACKS" == "ok" ]]; then
+ ok "fixture packs are Platform-stored with no browse index"
+else
+ bad "fixture storage after the push: ${PACKS:-unreadable}"
+fi
+finish_scenario
diff --git a/forge-web/components/repo/browse-boundary.tsx b/forge-web/components/repo/browse-boundary.tsx
index 56fca557a..8aa24d3d6 100644
--- a/forge-web/components/repo/browse-boundary.tsx
+++ b/forge-web/components/repo/browse-boundary.tsx
@@ -10,13 +10,51 @@
*/
import type { ReactNode } from 'react'
-import { HardDriveDownload, PackageOpen } from 'lucide-react'
+import { AlertTriangle, HardDriveDownload, PackageOpen } from 'lucide-react'
import { Button } from '@/components/ui/button'
import { EmptyState, ErrorState, LoadingBlock } from '@/components/ui/states'
import { useBrowseReader } from '@/hooks/use-browse-reader'
import type { BrowseReader } from '@/lib/browse'
import type { RepoRef } from '@/lib/repo'
-import { formatBytes } from '@/lib/view'
+import { formatBytes, type UnavailablePack } from '@/lib/view'
+
+/**
+ * The honest caveat for a partial in-browser clone: some live packs live in external storage
+ * nobody could serve, so what is shown is checked but not the whole repo.
+ */
+function UnavailablePacksNotice({ packs }: { packs: readonly UnavailablePack[] }): JSX.Element {
+ const n = packs.length
+ const corrupt = packs.filter((p) => p.corrupt).length
+ return (
+
+
+
+
+ {n} {n === 1 ? 'pack' : 'packs'} could not be fetched from {n === 1 ? 'its' : 'their'} storage; some
+ objects may be missing. Everything shown was still hash-checked.
+
+ {corrupt > 0 ? (
+
+ {corrupt === 1 ? 'A mirror' : 'Mirrors'} served bad data for {corrupt}{' '}
+ {corrupt === 1 ? 'pack' : 'packs'}: bytes that do not match the sha256 in the proof-checked
+ manifest. They were refused.
+
+ ) : null}
+
+ {packs.map((p) => (
+
+ {p.packHash.slice(0, 12)}… — {p.hosts.length > 0 ? p.hosts.join(', ') : 'no browser-fetchable mirror'}
+ {p.corrupt ? ' (served bad data)' : ''}
+
+ ))}
+
+
+
+ )
+}
export function BrowseBoundary({
repo,
@@ -48,6 +86,7 @@ export function BrowseBoundary({
Viewing a copy loaded into your browser — this repo's browse index{' '}
{state.behind ? "doesn't cover everything stored" : "hasn't been published"} yet.
+ {state.unavailable.length > 0 ? : null}
{children(state.reader)}
)
diff --git a/forge-web/components/repo/issue-content.tsx b/forge-web/components/repo/issue-content.tsx
index 756ade7e7..e6109f8a1 100644
--- a/forge-web/components/repo/issue-content.tsx
+++ b/forge-web/components/repo/issue-content.tsx
@@ -107,7 +107,7 @@ export function IssueContent({
{issue.title || '(untitled)'} #{issue.number}
-
+
{open ? : }
{open ? 'Open' : 'Closed'}
diff --git a/forge-web/components/repo/pull-content.tsx b/forge-web/components/repo/pull-content.tsx
index 8cf2e3efc..15fb41e8b 100644
--- a/forge-web/components/repo/pull-content.tsx
+++ b/forge-web/components/repo/pull-content.tsx
@@ -81,10 +81,10 @@ export function PullContent({ home, addr, number }: { home: RepoHome; addr: Repo
const base = pull.baseRefName || 'the base branch'
const status = merged
- ? { label: 'Merged', icon: , bg: 'bg-dash' }
+ ? { label: 'Merged', icon: , bg: 'bg-dash-700' }
: !open
? { label: 'Closed', icon: , bg: 'bg-danger' }
- : { label: pull.state.draft ? 'Draft' : 'Open', icon: , bg: pull.state.draft ? 'bg-anvil-500' : 'bg-verify' }
+ : { label: pull.state.draft ? 'Draft' : 'Open', icon: , bg: pull.state.draft ? 'bg-anvil-500' : 'bg-verify-700' }
const postComment = async (): Promise => {
if (!identity || !signer) {
diff --git a/forge-web/components/repo/pull-diff.tsx b/forge-web/components/repo/pull-diff.tsx
index 54f6c60cd..d9e885ec7 100644
--- a/forge-web/components/repo/pull-diff.tsx
+++ b/forge-web/components/repo/pull-diff.tsx
@@ -250,7 +250,7 @@ export function PullDiff({ pull, home }: { pull: PullView; home: RepoHome }): JS
href={original}
target="_blank"
rel="noopener noreferrer"
- className="inline-flex h-7 items-center rounded-md bg-forge-700 px-2.5 text-dense font-medium text-white hover:bg-forge-600"
+ className="inline-flex h-7 items-center rounded-md bg-forge-700 px-2.5 text-dense font-medium text-white hover:bg-forge-800"
>
View original diff
diff --git a/forge-web/components/ui/button.tsx b/forge-web/components/ui/button.tsx
index 09485ced1..0b4314a44 100644
--- a/forge-web/components/ui/button.tsx
+++ b/forge-web/components/ui/button.tsx
@@ -15,9 +15,10 @@ const button = cva(
variants: {
variant: {
// forge-700 (#c2410c) with white text = 5.18:1, clears WCAG AA (4.5:1); forge-600 as
- // text-on-orange only reaches ~3.6:1 and fails. Hover brightens to forge-600 for lift.
+ // text-on-orange only reaches ~3.6:1 and fails — as a hover fill too, so hover darkens
+ // to forge-800 (7.31:1) instead of brightening. lib/design/contrast.test.ts checks it.
primary:
- 'bg-forge-700 text-white hover:bg-forge-600 dark:bg-forge-700 dark:hover:bg-forge-600',
+ 'bg-forge-700 text-white hover:bg-forge-800 dark:bg-forge-700 dark:hover:bg-forge-800',
outline:
'border border-anvil-300 bg-transparent text-anvil-800 hover:bg-anvil-100 ' +
'dark:border-anvil-700 dark:text-anvil-100 dark:hover:bg-anvil-800',
diff --git a/forge-web/components/ui/identity-pill.tsx b/forge-web/components/ui/identity-pill.tsx
index ae423204c..32f84f697 100644
--- a/forge-web/components/ui/identity-pill.tsx
+++ b/forge-web/components/ui/identity-pill.tsx
@@ -1,3 +1,4 @@
+import { avatarFill, avatarHue } from '@/lib/design/avatar'
import { abbreviate, cn } from '@/lib/utils'
/**
@@ -19,15 +20,6 @@ export interface IdentityPillProps {
className?: string
}
-/** Deterministic hue from the identity id so the stub avatar is stable per identity. */
-function hueFor(id: string): number {
- let hash = 0
- for (let i = 0; i < id.length; i += 1) {
- hash = (hash * 31 + id.charCodeAt(i)) % 360
- }
- return hash
-}
-
export function IdentityPill({
identityId,
name,
@@ -35,7 +27,7 @@ export function IdentityPill({
className,
}: IdentityPillProps): JSX.Element {
const initial = (name ?? identityId).charAt(0).toUpperCase()
- const hue = hueFor(identityId)
+ const fill = avatarFill(avatarHue(identityId))
return (
{initial}
diff --git a/forge-web/e2e/auth-write.spec.ts b/forge-web/e2e/auth-write.spec.ts
index c16129f29..756868b04 100644
--- a/forge-web/e2e/auth-write.spec.ts
+++ b/forge-web/e2e/auth-write.spec.ts
@@ -2,17 +2,18 @@ import { test, expect } from '@playwright/test'
import { existsSync } from 'node:fs'
import { homedir } from 'node:os'
import { join } from 'node:path'
-import { repoUrl, waitForRepoResolved, shot, M1 } from './helpers'
+import { repoUrl, waitForRepoResolved, shot, WRITE_FIXTURE } from './helpers'
/**
* OPTIONAL auth / write scenario (lower priority — read paths are the gate).
*
* Gated behind E2E_AUTH=1 so the default CI run stays key-free and hermetic. When enabled it
- * loads the DEPLOYER bridge identity (which owns the m1 repo) from the local, gitignored
+ * loads the DEPLOYER bridge identity (which owns the write fixture) from the local, gitignored
* ~/.config/dash-forge/test-identities/, drives the login modal's identity-file upload via
* setInputFiles (keys never leave this machine), and verifies the session is established.
*
- * With E2E_WRITE=1 it goes further and creates a cheap issue on m1 (a real testnet write).
+ * With E2E_WRITE=1 it goes further and creates a cheap issue on the WRITE fixture (helpers.ts
+ * WRITE_FIXTURE, never the read fixture the read specs assert on) — a real testnet write.
* Anything flaky here self-skips rather than failing the suite.
*/
@@ -27,8 +28,8 @@ test.describe('auth + write (opt-in via E2E_AUTH=1)', () => {
test.skip(!AUTH_ENABLED, 'set E2E_AUTH=1 to run the local-key login/write scenario')
test.skip(!existsSync(IDENTITY_PATH), `DEPLOYER identity not found at ${IDENTITY_PATH}`)
- test('login via identity file, then optionally create an issue on m1', async ({ page }) => {
- await page.goto(repoUrl('issues'), { waitUntil: 'domcontentloaded' })
+ test('login via identity file, then optionally create an issue on the write fixture', async ({ page }) => {
+ await page.goto(repoUrl('issues', WRITE_FIXTURE), { waitUntil: 'domcontentloaded' })
await waitForRepoResolved(page)
// Open the login modal from the header sign-in affordance.
@@ -54,7 +55,7 @@ test.describe('auth + write (opt-in via E2E_AUTH=1)', () => {
return
}
- // Create a real (cheap) issue on m1.
+ // Create a real (cheap) issue on the write fixture.
const marker = `e2e-web smoke ${new Date().toISOString()}`
await page.getByRole('button', { name: /new issue/i }).first().click()
await page.getByLabel(/title/i).fill(marker)
@@ -66,6 +67,6 @@ test.describe('auth + write (opt-in via E2E_AUTH=1)', () => {
await expect(page.getByText(marker)).toBeVisible({ timeout: 60_000 })
await shot(page, '09-issue-created')
- expect(M1.name).toBeTruthy()
+ expect(WRITE_FIXTURE.name).toBeTruthy()
})
})
diff --git a/forge-web/e2e/helpers.ts b/forge-web/e2e/helpers.ts
index 2801fa59e..658ba6160 100644
--- a/forge-web/e2e/helpers.ts
+++ b/forge-web/e2e/helpers.ts
@@ -2,15 +2,34 @@ import { type Page, type ConsoleMessage } from '@playwright/test'
import { mkdirSync } from 'node:fs'
import { join } from 'node:path'
-/** Real testnet fixture: the m1 repo seeded by the CLI e2e suite. */
+/**
+ * Real testnet fixture: the nightly READ fixture, written only by
+ * `e2e/cli/seed-read-fixture.sh` (reserved in e2e/README.md). `main` holds one deterministic
+ * commit (README.md, src/, lib/) stored on Platform with no browse index, so the fallback
+ * clone is what the browse specs exercise. Never point these specs at a repo another suite
+ * writes: the storage e2e once left packs on a laptop's MinIO in the shared CLI repo, and
+ * every browse spec failed on them. Override with E2E_FIXTURE_OWNER / E2E_FIXTURE_NAME.
+ */
export const M1 = {
- owner: '8hJmcHWTsdvkHyCrk4UgjbyugDAmE7QfuCTQXpXAc7nB',
- name: 'm1-75299',
- contract: '5rrwgjjVUqMghnessfiXPXubpiM2QLNNXH142Hv4PDyX',
+ owner: process.env['E2E_FIXTURE_OWNER'] ?? '8hJmcHWTsdvkHyCrk4UgjbyugDAmE7QfuCTQXpXAc7nB',
+ // The seeder's override (NIGHTLY_FIXTURE_REPO) applies here too, so a renamed fixture is
+ // seeded and read as the same repo.
+ name: process.env['E2E_FIXTURE_NAME'] ?? process.env['NIGHTLY_FIXTURE_REPO'] ?? 'm1-5124',
+} as const
+
+/**
+ * The repo the opt-in WRITE spec (auth-write, E2E_WRITE=1) creates issues on — never the read
+ * fixture above, which only its seeder may write (its issues page is asserted on). This is
+ * the CLI suite's DEPLOYER-owned repo, which test runs already write to. Override with
+ * E2E_WRITE_FIXTURE_NAME. See e2e/README.md.
+ */
+export const WRITE_FIXTURE = {
+ owner: M1.owner,
+ name: process.env['E2E_WRITE_FIXTURE_NAME'] ?? 'm1-75299',
} as const
-export function repoUrl(path = ''): string {
- const q = `owner=${M1.owner}&name=${M1.name}`
+export function repoUrl(path = '', repo: { owner: string; name: string } = M1): string {
+ const q = `owner=${repo.owner}&name=${repo.name}`
if (path === '') return `/repo/?${q}`
return `/repo/${path}/?${q}`
}
diff --git a/forge-web/e2e/read-paths.spec.ts b/forge-web/e2e/read-paths.spec.ts
index 8fc9759d2..75f96df7a 100644
--- a/forge-web/e2e/read-paths.spec.ts
+++ b/forge-web/e2e/read-paths.spec.ts
@@ -90,7 +90,7 @@ test.describe('logged-out read paths', () => {
const openClosed = page.getByText(/\bOpen\b|\bClosed\b/i).first()
const emptyState = page.getByText(/no (open )?issues/i).first()
- await expect(openClosed.or(emptyState).or(readErrorBanner(page))).toBeVisible({
+ await expect(openClosed.or(emptyState).or(readErrorBanner(page)).first()).toBeVisible({
timeout: 30_000,
})
await shot(page, '03-issues')
@@ -101,7 +101,7 @@ test.describe('logged-out read paths', () => {
'rejects with a wasm-bindgen error). Open/closed issue folding could not be asserted.',
)
}
- await expect(openClosed.or(emptyState)).toBeVisible()
+ await expect(openClosed.or(emptyState).first()).toBeVisible()
})
test('4. tree browse reaches an honest terminal state (best-effort)', async ({ page }) => {
diff --git a/forge-web/hooks/use-browse-reader.ts b/forge-web/hooks/use-browse-reader.ts
index 26b7977d4..a8d79e24c 100644
--- a/forge-web/hooks/use-browse-reader.ts
+++ b/forge-web/hooks/use-browse-reader.ts
@@ -14,7 +14,7 @@ import { useBrowse } from '@/hooks/use-browse'
import { AUTO_LOAD_MAX_BYTES, useFallbackBrowse } from '@/hooks/use-fallback-browse'
import type { BrowseReader } from '@/lib/browse'
import type { RepoRef } from '@/lib/repo'
-import { formatBytes, type FallbackProgress } from '@/lib/view'
+import { formatBytes, type FallbackProgress, type UnavailablePack } from '@/lib/view'
export type BrowseReaderState =
| { readonly kind: 'loading'; readonly label: string }
@@ -24,6 +24,8 @@ export type BrowseReaderState =
/** Served from an in-browser clone because the published index is missing or behind. */
readonly local: boolean
readonly behind: boolean
+ /** Live external packs the in-browser clone could not fetch (empty: nothing skipped). */
+ readonly unavailable: readonly UnavailablePack[]
}
| { readonly kind: 'error'; readonly title?: string; readonly message: string; readonly retry: () => void }
| { readonly kind: 'no-packs' }
@@ -50,13 +52,19 @@ export function useBrowseReader(repo: RepoRef | null): BrowseReaderState {
if (error) return { kind: 'error', message: error, retry: reload }
if (data === null || data.kind === 'no-packs') return { kind: 'no-packs' }
if (data.kind === 'ready') {
- return { kind: 'ready', reader: data.context.reader, local: false, behind: false }
+ return { kind: 'ready', reader: data.context.reader, local: false, behind: false, unavailable: [] }
}
// No usable published index — the in-browser fallback clone takes over.
const behind = data.reason === 'index-behind'
if (fallback.status === 'ready' && fallback.context !== null) {
- return { kind: 'ready', reader: fallback.context.reader, local: true, behind }
+ return {
+ kind: 'ready',
+ reader: fallback.context.reader,
+ local: true,
+ behind,
+ unavailable: fallback.context.unavailable ?? [],
+ }
}
if (fallback.status === 'error') {
return {
diff --git a/forge-web/lib/browse/reader.ts b/forge-web/lib/browse/reader.ts
index 764d4f58a..22f7888f0 100644
--- a/forge-web/lib/browse/reader.ts
+++ b/forge-web/lib/browse/reader.ts
@@ -50,6 +50,11 @@ export interface BrowseReaderOptions {
* read). This is how the UI learns what was actually checked rather than assuming it.
*/
readonly onObject?: (verdict: ObjectVerdict) => void
+ /**
+ * The error for an OID the locator does not index. Defaults to `object not in locator`; a
+ * reader built over an incomplete pack set supplies one that names what is missing.
+ */
+ readonly missingObject?: (oidHex: string) => Error
}
/** Per-reader object-memo budget — readers live for the session (cached browse context). */
@@ -117,7 +122,9 @@ export class BrowseReader {
if (cached !== undefined) return cached
const entry = this.locate(oidHex)
- if (entry === null) throw new Error(`object not in locator: ${oidHex}`)
+ if (entry === null) {
+ throw this.opts.missingObject?.(oidHex) ?? new Error(`object not in locator: ${oidHex}`)
+ }
const obj = singleReadAdvised(entry)
? await this.readSpan(entry)
diff --git a/forge-web/lib/constants.ts b/forge-web/lib/constants.ts
index faf6c617e..88cbb060c 100644
--- a/forge-web/lib/constants.ts
+++ b/forge-web/lib/constants.ts
@@ -18,6 +18,7 @@ import {
type DeploymentFile,
type ForgeIds,
} from './deployments'
+import storageDefaults from '../../forge-contracts/config/storage-defaults.json'
// ---------------------------------------------------------------------------
// Network
@@ -233,6 +234,15 @@ export const QUORUM_KEY_ENDPOINT: Readonly> = {
devnet: quorumEndpoint(NETWORKS.devnet),
}
+/**
+ * Public IPFS gateways an `ipfs://` pack URI is fetched through, in order, after
+ * whatever `http(s)` mirrors the manifest itself records — the list every client shares
+ * (`forge-contracts/config/storage-defaults.json`, which forge-core embeds). Byte sources
+ * only: every external pack is sha256-checked against its proof-read manifest, so a gateway
+ * that lies or is down costs a retry, never integrity.
+ */
+export const IPFS_GATEWAYS: readonly string[] = storageDefaults.ipfsGateways
+
/** The config of the network this build targets. */
export const ACTIVE_NETWORK: NetworkConfig = NETWORKS[DEFAULT_NETWORK]
diff --git a/forge-web/lib/design/avatar.ts b/forge-web/lib/design/avatar.ts
new file mode 100644
index 000000000..ffc94b7d8
--- /dev/null
+++ b/forge-web/lib/design/avatar.ts
@@ -0,0 +1,54 @@
+/**
+ * The identity pill's stub avatar fill: a deterministic hue per identity, darkened until the
+ * white initial on it clears WCAG AA (4.5:1).
+ *
+ * A fixed `hsl(h 45% 45%)` failed for yellow-green hues (2.58:1 at hue 60): at equal HSL
+ * lightness, yellows and greens are far brighter than blues. So the lightness starts at 45%
+ * and steps down per hue until the contrast holds. `lib/design/contrast.test.ts` checks all
+ * 360 hues.
+ */
+
+/** Deterministic hue from the identity id so the stub avatar is stable per identity. */
+export function avatarHue(id: string): number {
+ let hash = 0
+ for (let i = 0; i < id.length; i += 1) {
+ hash = (hash * 31 + id.charCodeAt(i)) % 360
+ }
+ return hash
+}
+
+const SATURATION = 0.45
+const START_LIGHTNESS = 0.45
+const AA_TEXT = 4.5
+
+/** HSL (h in degrees, s/l in 0..1) to 8-bit sRGB. */
+export function hslToRgb(h: number, s: number, l: number): [number, number, number] {
+ const c = (1 - Math.abs(2 * l - 1)) * s
+ const hp = (((h % 360) + 360) % 360) / 60
+ const x = c * (1 - Math.abs((hp % 2) - 1))
+ const [r1, g1, b1] =
+ hp < 1 ? [c, x, 0] : hp < 2 ? [x, c, 0] : hp < 3 ? [0, c, x] : hp < 4 ? [0, x, c] : hp < 5 ? [x, 0, c] : [c, 0, x]
+ const m = l - c / 2
+ return [r1 + m, g1 + m, b1 + m].map((v) => Math.round(v * 255)) as [number, number, number]
+}
+
+/** WCAG relative luminance of an 8-bit sRGB colour. */
+export function luminance([r, g, b]: readonly [number, number, number]): number {
+ const lin = (c: number): number => {
+ const v = c / 255
+ return v <= 0.03928 ? v / 12.92 : ((v + 0.055) / 1.055) ** 2.4
+ }
+ return 0.2126 * lin(r) + 0.7152 * lin(g) + 0.0722 * lin(b)
+}
+
+/** Contrast ratio of white text on `rgb`. */
+export function whiteContrast(rgb: readonly [number, number, number]): number {
+ return 1.05 / (luminance(rgb) + 0.05)
+}
+
+/** The avatar fill for `hue`, as a CSS `hsl()` string, with white text at ≥ 4.5:1. */
+export function avatarFill(hue: number): string {
+ let l = START_LIGHTNESS
+ while (l > 0.05 && whiteContrast(hslToRgb(hue, SATURATION, l)) < AA_TEXT) l -= 0.01
+ return `hsl(${hue} ${SATURATION * 100}% ${Math.round(l * 100)}%)`
+}
diff --git a/forge-web/lib/design/contrast.test.ts b/forge-web/lib/design/contrast.test.ts
index 0c0240589..183c64710 100644
--- a/forge-web/lib/design/contrast.test.ts
+++ b/forge-web/lib/design/contrast.test.ts
@@ -2,6 +2,7 @@ import { readdirSync, readFileSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { describe, expect, it } from 'vitest'
import tailwindConfig from '@/tailwind.config.js'
+import { avatarFill, hslToRgb, whiteContrast } from './avatar'
/**
* WCAG 2 AA contrast for the design tokens, checked without a browser.
@@ -14,11 +15,23 @@ import tailwindConfig from '@/tailwind.config.js'
// The config's JSDoc type makes every theme key optional and possibly a function; this
// config is a plain object literal, so read it as one.
-const { anvil, dash } = (
+const colors = (
tailwindConfig as unknown as {
- theme: { extend: { colors: Record<'anvil' | 'dash', Record> } }
+ theme: { extend: { colors: Record> } }
}
).theme.extend.colors
+const anvil = colors['anvil'] as Record
+const dash = colors['dash'] as Record
+
+/** A Tailwind color class suffix (`dash-700`, `verify`, `anvil-500`) to its hex, if known. */
+function tokenHex(token: string): string | undefined {
+ if (token === 'white') return '#ffffff'
+ const m = /^([a-z]+)(?:-(\d+))?$/.exec(token)
+ if (m === null) return undefined
+ const entry = colors[m[1] as string]
+ if (typeof entry === 'string') return m[2] === undefined ? entry : undefined
+ return entry?.[m[2] ?? 'DEFAULT']
+}
type Rgb = [number, number, number]
@@ -121,3 +134,124 @@ describe('no component renders text in the raw brand blue', () => {
expect(flagged('{n} ')).toBe(false)
})
})
+
+describe('identity-pill avatar fills keep the white initial at AA', () => {
+ it('holds for every hue', () => {
+ for (let hue = 0; hue < 360; hue++) {
+ const m = /hsl\((\d+) (\d+)% (\d+)%\)/.exec(avatarFill(hue))
+ expect(m).not.toBeNull()
+ const rgbFill = hslToRgb(Number(m?.[1]), Number(m?.[2]) / 100, Number(m?.[3]) / 100)
+ expect(whiteContrast(rgbFill), `hue ${hue}`).toBeGreaterThanOrEqual(AA_TEXT)
+ }
+ })
+
+ it('the old fixed fill failed where the report said', () => {
+ // hsl(60 45% 45%) — the yellow-green the review measured at 2.58:1.
+ expect(whiteContrast(hslToRgb(60, 0.45, 0.45))).toBeLessThan(AA_TEXT)
+ })
+})
+
+describe('white text on solid fills meets WCAG AA', () => {
+ const root = resolve(__dirname, '../..')
+
+ function sources(dir: string): string[] {
+ return readdirSync(dir, { withFileTypes: true }).flatMap((e) => {
+ const p = join(dir, e.name)
+ if (e.isDirectory()) return sources(p)
+ return /\.tsx$/.test(e.name) ? [p] : []
+ })
+ }
+
+ // Every `bg-…` class, including `hover:` / `dark:` / `focus:` variants (a hover fill sits
+ // behind the same white text), arbitrary values and `/NN` tints. The token captured is
+ // what follows `bg-`; anything this test cannot resolve to a solid colour FAILS, so a new
+ // shape of fill cannot slip past it unchecked.
+ const SOLID_BG = /(? text.slice(0, offset).split('\n').length
+ const scan = (chunk: string, base: number): void => {
+ for (const m of chunk.matchAll(SOLID_BG)) {
+ out.push({ token: m[1] as string, line: lineAt(base + (m.index ?? 0)) })
+ }
+ }
+ // Whole class expressions, however many lines they span: a `className="…"` string, a
+ // `className={…}` expression (template, `cn(…)` call — braces balanced), or a
+ // `cva`-style quoted string. Each that contains `text-white` is scanned as a unit.
+ const CLASS_EXPR = /className=(?:"[^"]*"|'[^']*'|\{)/g
+ for (const m of text.matchAll(CLASS_EXPR)) {
+ const start = m.index ?? 0
+ let end = start + m[0].length
+ if (m[0].endsWith('{')) {
+ let depth = 1
+ while (end < text.length && depth > 0) {
+ const c = text[end]
+ if (c === '{') depth += 1
+ else if (c === '}') depth -= 1
+ end += 1
+ }
+ }
+ const expr = text.slice(start, end)
+ if (expr.includes('text-white')) scan(expr, start)
+ }
+ // Class strings defined away from the element (a variants table): any quoted string
+ // holding both `text-white` and a solid fill.
+ for (const m of text.matchAll(/'[^'\n]*'|"[^"\n]*"/g)) {
+ if (m[0].includes('text-white') && !text.slice(Math.max(0, (m.index ?? 0) - 10), m.index).includes('className=')) {
+ scan(m[0], m.index ?? 0)
+ }
+ }
+ // A badge whose background is computed (`${status.bg}`): every `bg:` value the file
+ // assigns can end up behind the white text.
+ if (/text-white[\s\S]{0,200}?\$\{[\w.]*\bbg\}/.test(text)) {
+ for (const m of text.matchAll(/\bbg:\s*[^\n]*/g)) scan(m[0], m.index ?? 0)
+ }
+ return [...new Map(out.map((b) => [`${b.line}:${b.token}`, b])).values()]
+ }
+
+ it('checks the Merged / Open / Closed / Draft badges and every other white-text fill', () => {
+ const failures: string[] = []
+ let checked = 0
+ for (const file of ['app', 'components'].flatMap((d) => sources(join(root, d)))) {
+ for (const { token, line } of whiteTextBackgrounds(readFileSync(file, 'utf8'))) {
+ const hex = tokenHex(token)
+ if (hex === undefined) {
+ // Not a token from this config (a Tailwind default like `green-600`, an arbitrary
+ // `[#…]`, a `/NN` tint): this test cannot vouch for it, so it is a failure, not a pass.
+ failures.push(`${file.slice(root.length + 1)}:${line} bg-${token} (unknown to the contrast test)`)
+ continue
+ }
+ checked += 1
+ const ratio = contrast(rgb('#ffffff'), rgb(hex))
+ if (ratio < AA_TEXT) {
+ failures.push(`${file.slice(root.length + 1)}:${line} bg-${token} ${ratio.toFixed(2)}:1`)
+ }
+ }
+ }
+ expect(checked).toBeGreaterThanOrEqual(6)
+ expect(failures).toEqual([])
+ })
+
+ it('catches the regressions it exists for', () => {
+ const tokens = (src: string): string[] => whiteTextBackgrounds(src).map((b) => b.token)
+ // The Merged badge before the fix: brand blue behind white is 3.54:1.
+ const merged = "const s = { bg: 'bg-dash' }\n"
+ expect(tokens(merged)).toEqual(['dash'])
+ expect(contrast(rgb('#ffffff'), rgb(tokenHex('dash') as string))).toBeLessThan(AA_TEXT)
+ expect(contrast(rgb('#ffffff'), rgb(tokenHex('verify') as string))).toBeLessThan(AA_TEXT)
+ // The hover fill sits behind the same white text, so it is checked too.
+ expect(tokens('')).toEqual(['forge-700', 'forge-600'])
+ expect(tokens('')).toEqual(['[#123456]', 'green-600'])
+ expect(tokenHex('green-600')).toBeUndefined() // → reported as unknown, not skipped
+ // text-white and the fill on different lines of one class expression.
+ expect(tokens('')).toEqual(['dash'])
+ expect(tokens('')).toEqual(['verify'])
+ expect(tokens('')).toEqual([])
+ })
+})
diff --git a/forge-web/lib/repo/index.ts b/forge-web/lib/repo/index.ts
index 8026f8756..a74184df7 100644
--- a/forge-web/lib/repo/index.ts
+++ b/forge-web/lib/repo/index.ts
@@ -38,7 +38,8 @@ export {
} from './config'
export {
branchesOf,
- enumerateRefHashes,
+ hasMissingParent,
+ readAllRefUpdates,
readRefUpdates,
readRefs,
resolveRefByHash,
diff --git a/forge-web/lib/repo/refs.test.ts b/forge-web/lib/repo/refs.test.ts
index b3ab1b936..a7cc27e90 100644
--- a/forge-web/lib/repo/refs.test.ts
+++ b/forge-web/lib/repo/refs.test.ts
@@ -1,7 +1,10 @@
/**
- * readRefs paths: the one-page fast path (small repos resolve from two parallel queries,
- * grouped locally) and the skip-scan fallback (a full page means the update set may be
- * incomplete, so enumeration must go back through `> last limit 1` hops + per-ref reads).
+ * readRefs: the keyset scan over the `refState` index, against a mock Drive that serves
+ * `refNameHash > x` pages exactly and reproduces the protocol-13 cursor bug for `startAfter`
+ * (the cursor's `$id` bound leaks into every later `refNameHash` branch). The reader must
+ * never send a cursor on that multi-branch query, must read a page-filling ref on its own,
+ * and must fall back to the reflog read when a `prevOid` has no parent in what came back.
+ * Parity: forge-core `refs::tests`.
*/
import type { EvoSDK } from '@dashevo/evo-sdk'
@@ -9,9 +12,9 @@ import { sha256 } from '@noble/hashes/sha2.js'
import { bytesToHex } from '@noble/hashes/utils.js'
import { describe, expect, it } from 'vitest'
-import { bytesToBase64 } from '../sdk'
+import { base64ToHex, bytesToBase64 } from '../sdk'
import { DOC, type RepoRef } from './contract'
-import { readRefs } from './refs'
+import { hasMissingParent, readRefs } from './refs'
const REPO: RepoRef = { contractId: 'contract', ownerId: 'owner' }
@@ -20,129 +23,215 @@ function refHashBytes(seed: number): Uint8Array {
return sha256(new TextEncoder().encode(`refs/heads/ref-${seed}`))
}
-function hashB64(seed: number): string {
- return bytesToBase64(refHashBytes(seed))
-}
-
function refHashHex(seed: number): string {
return bytesToHex(refHashBytes(seed))
}
-function hashHex(seed: number): string {
- return Array.from({ length: 32 }, () => seed.toString(16).padStart(2, '0')).join('')
+function oidHex(seed: number): string {
+ return Array.from({ length: 20 }, () => seed.toString(16).padStart(2, '0')).join('')
}
-let nextId = 0
-function updateDoc(
- hashSeed: number,
- oidSeed: number,
- createdAt: number,
- prevOidSeed: number | null = null,
-): Record {
- nextId += 1
+type Doc = Record
+
+/** A refUpdate row; `$createdAt` is absent, as on the deployed repo-v1 type. */
+function updateDoc(id: string, refSeed: number, newSeed: number, prevSeed = 0): Doc {
return {
- $id: `doc-${nextId}`,
+ $id: id,
$ownerId: 'pusher',
- $createdAt: createdAt,
- refNameHash: hashB64(hashSeed),
- refName: `refs/heads/ref-${hashSeed}`,
- prevOid: prevOidSeed === null ? null : bytesToBase64(new Uint8Array(32).fill(prevOidSeed)),
- newOid: bytesToBase64(new Uint8Array(32).fill(oidSeed)),
+ refNameHash: bytesToBase64(refHashBytes(refSeed)),
+ refName: `refs/heads/ref-${refSeed}`,
+ prevOid: prevSeed === 0 ? null : bytesToBase64(new Uint8Array(20).fill(prevSeed)),
+ newOid: bytesToBase64(new Uint8Array(20).fill(newSeed)),
force: false,
}
}
+/** A ref with `n` linear updates 1 → 2 → … → n. */
+function chain(refSeed: number, n: number, nextId: () => string): Doc[] {
+ return Array.from({ length: n }, (_, k) => updateDoc(nextId(), refSeed, k + 1, k))
+}
+
interface QueryLike {
documentTypeName: string
where?: readonly (readonly [string, string, unknown])[]
orderBy?: readonly (readonly [string, string])[]
limit?: number
+ startAfter?: string
}
-/** Mock SDK whose `documents.query` is routed through `dispatch`; records every query. */
-function mockSdk(
- dispatch: (q: QueryLike) => Record[],
- seen: QueryLike[],
-): EvoSDK {
- return {
+const hexOf = (d: Doc): string => base64ToHex(d['refNameHash'] as string)
+
+/**
+ * In-memory Drive for `refUpdate` (protected is empty). Rows are kept in `refState` order
+ * (`refNameHash`, then `$id` — `$createdAt` being absent). `dropOnKeyset` removes one `$id`
+ * from keyset pages only, standing in for a node answering a correct query incompletely.
+ */
+function mockDrive(rows: Doc[], opts: { dropOnKeyset?: string; ignoreRange?: boolean } = {}) {
+ const sorted = [...rows].sort((a, b) =>
+ hexOf(a) < hexOf(b) ? -1 : hexOf(a) > hexOf(b) ? 1 : String(a['$id']) < String(b['$id']) ? -1 : 1,
+ )
+ const seen: QueryLike[] = []
+ const page = (docs: Doc[], q: QueryLike): Doc[] => {
+ let from = 0
+ if (q.startAfter !== undefined) {
+ const cursor = docs.find((d) => d['$id'] === q.startAfter) as Doc
+ from = docs.indexOf(cursor) + 1
+ // The protocol-13 lowering: the cursor's `$id` bound applies in sibling branches too.
+ if (q.orderBy?.[0]?.[0] === 'refNameHash') {
+ docs = docs.filter(
+ (d, i) => i < from || hexOf(d) === hexOf(cursor) || String(d['$id']) > String(cursor['$id']),
+ )
+ from = docs.indexOf(cursor) + 1
+ }
+ }
+ return docs.slice(from, from + (q.limit ?? 100))
+ }
+ const sdk = {
documents: {
query: (q: QueryLike): Promise> => {
seen.push(q)
- const docs = dispatch(q)
- return Promise.resolve(new Map(docs.map((d) => [String(d['$id']), d])))
+ let out: Doc[] = []
+ if (q.documentTypeName === DOC.config) out = [{ $id: 'cfg', $createdAt: 1 }]
+ else if (q.documentTypeName === DOC.refUpdate) {
+ const where = q.where ?? []
+ const gt = where.find((w) => w[1] === '>')
+ const eq = where.find((w) => w[1] === '==')
+ let docs = sorted
+ if (gt && !opts.ignoreRange) docs = docs.filter((d) => hexOf(d) > base64ToHex(gt[2] as string))
+ if (eq) docs = docs.filter((d) => hexOf(d) === base64ToHex(eq[2] as string))
+ if (gt && opts.dropOnKeyset) docs = docs.filter((d) => d['$id'] !== opts.dropOnKeyset)
+ if (!gt && !eq && q.orderBy?.[0]?.[0] === 'refNameHash' && opts.dropOnKeyset) {
+ docs = docs.filter((d) => d['$id'] !== opts.dropOnKeyset)
+ }
+ if (q.orderBy?.[0]?.[0] === '$createdAt') {
+ docs = [...docs].sort((a, b) => (String(a['$id']) < String(b['$id']) ? -1 : 1))
+ }
+ out = page(docs, q)
+ }
+ return Promise.resolve(new Map(out.map((d) => [String(d['$id']), d])))
},
},
} as unknown as EvoSDK
+ return { sdk, seen, sorted }
}
-const isFastPathPage = (q: QueryLike): boolean => q.orderBy?.length === 2 && q.limit === 100
-const isSkipScanHop = (q: QueryLike): boolean => q.limit === 1 && q.orderBy?.length === 1
-
-describe('readRefs fast path (whole update set in one page per type)', () => {
- it('resolves every ref from the two page queries with no skip-scan hops', async () => {
- const seen: QueryLike[] = []
- const sdk = mockSdk((q) => {
- if (q.documentTypeName === DOC.config) return [{ $id: 'cfg', $createdAt: 1 }]
- if (!isFastPathPage(q)) throw new Error(`unexpected query: ${JSON.stringify(q)}`)
- if (q.documentTypeName === DOC.refUpdate) {
- return [updateDoc(1, 0xaa, 10), updateDoc(1, 0xbb, 20, 0xaa), updateDoc(2, 0xcc, 15)]
- }
- return [] // protectedRefUpdate: none
- }, seen)
+/** 60 refs × 1..4 updates, `$id`s running against hash order (the nightly repo's shape). */
+function nightlyLike(): Doc[] {
+ let n = 1000
+ const nextId = (): string => `id${String(n--).padStart(6, '0')}`
+ return Array.from({ length: 60 }, (_, r) => chain(r + 1, 1 + ((r + 1) % 4), nextId)).flat()
+}
+
+const isKeysetPage = (q: QueryLike): boolean =>
+ q.documentTypeName === DOC.refUpdate && q.orderBy?.[0]?.[0] === 'refNameHash'
+
+describe('the mock reproduces the protocol-13 cursor drop', () => {
+ it('loses rows when refState is paged with startAfter', async () => {
+ const { sdk, sorted } = mockDrive(nightlyLike())
+ const got: Doc[] = []
+ let startAfter: string | undefined
+ for (;;) {
+ const res = (await (sdk as unknown as { documents: { query: (q: QueryLike) => Promise> } }).documents.query({
+ documentTypeName: DOC.refUpdate,
+ orderBy: [
+ ['refNameHash', 'asc'],
+ ['$createdAt', 'asc'],
+ ],
+ limit: 100,
+ startAfter,
+ }))
+ const page = [...res.values()]
+ got.push(...page)
+ if (page.length < 100) break
+ startAfter = String(page[page.length - 1]?.['$id'])
+ }
+ expect(sorted.length).toBeGreaterThan(100)
+ expect(got.length).toBeLessThan(sorted.length)
+ })
+})
+describe('readRefs keyset scan', () => {
+ it('reads every update with range pages and never a cursor on the multi-branch query', async () => {
+ const { sdk, seen } = mockDrive(nightlyLike())
const refs = await readRefs(sdk, REPO)
- const byName = new Map(refs.map((r) => [r.refName, r]))
- expect([...byName.keys()].sort()).toEqual(['refs/heads/ref-1', 'refs/heads/ref-2'])
- const ref1 = byName.get('refs/heads/ref-1')
- expect(ref1?.refNameHash).toBe(refHashHex(1))
- expect(ref1?.state).toMatchObject({ state: 'resolved', oid: hashHex(0xbb) })
- expect(byName.get('refs/heads/ref-2')?.state).toMatchObject({
+ expect(refs).toHaveLength(60)
+ for (let r = 1; r <= 60; r++) {
+ const ref = refs.find((x) => x.refNameHash === refHashHex(r))
+ expect(ref?.state).toMatchObject({ state: 'resolved', oid: oidHex(1 + (r % 4)) })
+ }
+ const pages = seen.filter(isKeysetPage)
+ expect(pages.every((q) => q.startAfter === undefined)).toBe(true)
+ expect(pages.length).toBeLessThanOrEqual(3)
+ // A consistent scan never falls back to the reflog read.
+ expect(seen.some((q) => q.documentTypeName === DOC.refUpdate && q.orderBy?.[0]?.[0] === '$createdAt' && !q.where?.length)).toBe(false)
+ })
+
+ it('reads a page-filling ref on its own, then moves past it', async () => {
+ let n = 0
+ const nextId = (): string => `id${String(n++).padStart(6, '0')}`
+ const rows = [...chain(5, 3, nextId), ...chain(7, 150, nextId), ...chain(9, 2, nextId)]
+ const { sdk, seen } = mockDrive(rows)
+ const refs = await readRefs(sdk, REPO)
+
+ expect(refs.find((r) => r.refNameHash === refHashHex(7))?.state).toMatchObject({
state: 'resolved',
- oid: hashHex(0xcc),
+ oid: oidHex(150),
})
- expect(seen.some(isSkipScanHop)).toBe(false)
+ expect(refs).toHaveLength(3)
+ const eqReads = seen.filter((q) => q.where?.some((w) => w[1] === '==') && q.documentTypeName === DOC.refUpdate)
+ expect(eqReads.map((q) => base64ToHex(q.where?.[0]?.[2] as string))).toContain(refHashHex(7))
})
- it('merges protected updates into the same ref group', async () => {
- const sdk = mockSdk((q) => {
- if (q.documentTypeName === DOC.config) return [{ $id: 'cfg', $createdAt: 1 }]
- if (q.documentTypeName === DOC.refUpdate) return [updateDoc(1, 0xaa, 10)]
- return [updateDoc(1, 0xdd, 30, 0xaa)] // protectedRefUpdate, newer, supersedes 0xaa
- }, [])
-
+ it('falls back to the reflog read alone when a prevOid has no parent', async () => {
+ const rows = nightlyLike()
+ // Ref 2 has 3 updates; hide its middle one from keyset pages.
+ const victim = rows.find(
+ (d) => hexOf(d) === refHashHex(2) && base64ToHex(d['newOid'] as string) === oidHex(2),
+ )?.['$id'] as string
+ const { sdk, seen } = mockDrive(rows, { dropOnKeyset: victim })
const refs = await readRefs(sdk, REPO)
- expect(refs).toHaveLength(1)
- expect(refs[0]?.state).toMatchObject({ state: 'resolved', oid: hashHex(0xdd) })
+ expect(seen.some((q) => q.documentTypeName === DOC.refUpdate && q.orderBy?.[0]?.[0] === '$createdAt' && !q.where?.length)).toBe(true)
+ expect(refs.find((r) => r.refNameHash === refHashHex(2))?.state).toMatchObject({
+ state: 'resolved',
+ oid: oidHex(3),
+ })
+ expect(refs).toHaveLength(60)
})
-})
-
-describe('readRefs skip-scan fallback (a full page = possibly incomplete)', () => {
- it('falls back to enumeration + per-ref reads when the page overflows', async () => {
- // 100 refUpdate docs on one ref — the page is full, so the fast path must not trust it.
- const fullPage = Array.from({ length: 100 }, (_, i) => updateDoc(1, 0xaa, 10 + i))
- const seen: QueryLike[] = []
- const sdk = mockSdk((q) => {
- if (q.documentTypeName === DOC.config) return [{ $id: 'cfg', $createdAt: 1 }]
- if (isFastPathPage(q)) return q.documentTypeName === DOC.refUpdate ? fullPage : []
- if (isSkipScanHop(q)) {
- if (q.documentTypeName !== DOC.refUpdate) return []
- // hop 1: no where → first hash; hop 2: > hash(1) → done
- return q.where?.length ? [] : [fullPage[0] as Record]
- }
- // per-ref history read (`refNameHash ==`, `$createdAt asc`)
- if (q.where?.[0]?.[1] === '==' && q.documentTypeName === DOC.refUpdate) {
- return [updateDoc(1, 0xee, 500)]
- }
- return []
- }, seen)
+ it('abandons the scan on an out-of-range page and answers from the reflog read alone', async () => {
+ // A node ignoring `refNameHash > last` serves the first page again: stop at once.
+ const { sdk, seen } = mockDrive(nightlyLike(), { ignoreRange: true })
const refs = await readRefs(sdk, REPO)
+ expect(seen.filter(isKeysetPage).filter((q) => q.documentTypeName === DOC.refUpdate)).toHaveLength(2)
+ expect(refs).toHaveLength(60)
+ for (let r = 1; r <= 60; r++) {
+ expect(refs.find((x) => x.refNameHash === refHashHex(r))?.state).toMatchObject({
+ state: 'resolved',
+ oid: oidHex(1 + (r % 4)),
+ })
+ }
+ })
+})
- expect(refs).toHaveLength(1)
- expect(refs[0]?.refNameHash).toBe(refHashHex(1))
- expect(refs[0]?.state).toMatchObject({ state: 'resolved', oid: hashHex(0xee) })
- expect(seen.some(isSkipScanHop)).toBe(true)
+describe('hasMissingParent', () => {
+ const u = (prevOid: string, newOid: string) => ({
+ id: newOid,
+ refNameHash: 'h',
+ refName: 'refs/heads/x',
+ prevOid,
+ newOid,
+ force: false,
+ protected: false,
+ author: 'a',
+ createdAt: 0,
+ })
+ it('accepts a linked chain, a create and a delete', () => {
+ expect(hasMissingParent([u('', 'aa'), u('aa', 'bb')])).toBe(false)
+ expect(hasMissingParent([u('0000', 'aa'), u('aa', '0000')])).toBe(false)
+ })
+ it('flags a prevOid no update produced', () => {
+ expect(hasMissingParent([u('', 'aa'), u('cc', 'dd')])).toBe(true)
})
})
diff --git a/forge-web/lib/repo/refs.ts b/forge-web/lib/repo/refs.ts
index 530f76cc9..35992aff5 100644
--- a/forge-web/lib/repo/refs.ts
+++ b/forge-web/lib/repo/refs.ts
@@ -1,13 +1,36 @@
/**
* Ref reads — branch/tag enumeration + tip resolution (data-contracts §2.3, §4).
*
- * Small repos (update set ≤ one query page per type) load everything in two parallel
- * queries and resolve locally. Larger repos enumerate via **skip-scan**
- * (`refNameHash > last limit 1` hops) across both the `refUpdate` and `protectedRefUpdate`
- * types — O(distinct refs), not O(total pushes). Tip resolution folds a ref's full update
- * history (both types, with the `protected` flag set per source) through
- * {@link resolveRef}, honoring as-of protected-pattern config.
+ * Every ref's complete update history comes from a **keyset scan** over the `refState` index
+ * (`refNameHash, $createdAt`): pages of `refNameHash > last`, 100 rows each, with no
+ * `startAfter` cursor. Every ref on a full page except the last is complete; the next page
+ * starts after the last complete ref. A ref that fills a page by itself is read on its own
+ * (an equality query, single-branch) and the scan moves past it. Cost: ⌈updates/100⌉
+ * queries per type plus one per such ref. Parity: forge-core `refs::read_all_ref_updates`.
*
+ * WHY NO CURSOR: paging `refState` with `startAfter` loses rows on testnet (protocol 13) —
+ * 32 of 229 updates on the nightly repo. Drive's v0 lowering applies the cursor document's
+ * lower-level bounds to every sibling `refNameHash` branch, not only the cursor's own, so a
+ * page omits rows from later refs and can come back short (the only end-of-data signal a
+ * pager has). Same family as dashpay/platform#4396; the orderBy-only shape is still unfixed
+ * there. A range where-clause carries no cursor document, so nothing leaks.
+ *
+ * WHEN THE SCAN IS NOT TRUSTED: the scan is abandoned and the answer comes from the
+ * `$createdAt` (`reflog`) read of both types ALONE, deduplicated by `$id`, when a page is out
+ * of `refNameHash` order, holds a row at or below its `> last` bound, or would not advance
+ * `last` (the node did not honor the query), or when the completeness check fails: every
+ * non-null `prevOid` a pusher records is some earlier update's `newOid` in the same ref, so a
+ * dangling one means a row is missing (or was written dangling on purpose — which costs the
+ * extra read, never the answer).
+ *
+ * LIMITS (same as forge-core `refs`, whose module doc has the detail): the check sees only
+ * mid-chain gaps — a missing newest update or a wholly missing ref leaves no dangling
+ * `prevOid`. And a ref with more than a page of updates is read by `==` paged with
+ * `startAfter`: single-branch, so no sibling drop, but protocol 13's same-`$createdAt`
+ * boundary skip still applies to repo-v1 ref updates until forge-v2 on protocol 14.
+ *
+ * Tip resolution folds a ref's full update history (both types, with the `protected` flag
+ * set per source) through {@link resolveRef}, honoring as-of protected-pattern config.
* Divergence resolution that turns on real commit ancestry (a merge superseding both racing
* heads) needs a commit-graph predicate from the browse plane; callers may pass one via
* `isAncestor`. Without it, linear/fast-forward/force/delete cases still resolve correctly
@@ -18,13 +41,20 @@ import type { EvoSDK } from '@dashevo/evo-sdk'
import {
displayRefName,
+ isNullOid,
resolveRef,
type ConfigDoc,
type IsAncestor,
type RefState,
type RefUpdate,
} from '../rules'
-import { base64ToHex, queryAllDocuments, queryDocumentsWithProof, skipScanDistinct } from '../sdk'
+import {
+ base64ToHex,
+ IncompleteReadError,
+ queryAllDocuments,
+ queryDocumentsWithProof,
+ type PlainDocument,
+} from '../sdk'
import { DOC, toRefUpdate, type RepoRef } from './contract'
import { readConfigHistory } from './config'
@@ -33,6 +63,15 @@ const NO_ANCESTRY: IsAncestor = () => false
/** One query page — also the Platform per-query document cap. */
const PAGE = 100
+/** Backstop against a node that never lets the scan finish (each round passes ≥ 1 ref). */
+const MAX_KEYSET_ROUNDS = 100_000
+
+/** Both ref-update types, each with the `protected` flag its updates carry into the fold. */
+const REF_UPDATE_TYPES: readonly (readonly [string, boolean])[] = [
+ [DOC.refUpdate, false],
+ [DOC.protectedRefUpdate, true],
+]
+
/** A resolved ref for list views. */
export interface ResolvedRef {
/** The ref name, e.g. `refs/heads/main`. */
@@ -43,21 +82,172 @@ export interface ResolvedRef {
readonly state: RefState
}
-/** Enumerate the distinct `refNameHash` values across both ref document types (base64). */
-export async function enumerateRefHashes(sdk: EvoSDK, repo: RepoRef): Promise {
- const [plain, protectedHashes] = await Promise.all([
- skipScanDistinct(sdk, {
- dataContractId: repo.contractId,
- documentTypeName: DOC.refUpdate,
- keyField: 'refNameHash',
- }),
- skipScanDistinct(sdk, {
+/** The row's `refNameHash` as lowercase hex — hex order is byte order; base64 order is not. */
+function refHashHexOf(doc: PlainDocument, documentTypeName: string): string {
+ const raw = doc['refNameHash']
+ let hex = ''
+ try {
+ hex = typeof raw === 'string' ? base64ToHex(raw) : ''
+ } catch {
+ /* reported below */
+ }
+ if (hex.length !== 64) {
+ // A row that cannot be attributed to a ref would make the answer wrong, not partial.
+ throw new Error(`${documentTypeName} ${String(doc['$id'])} has no 32-byte refNameHash`)
+ }
+ return hex
+}
+
+/** One ref's updates of one type (an equality read — single-branch, so cursor-safe). */
+function readOneRef(
+ sdk: EvoSDK,
+ repo: RepoRef,
+ documentTypeName: string,
+ refNameHashB64: string,
+): Promise {
+ return queryAllDocuments(sdk, {
+ dataContractId: repo.contractId,
+ documentTypeName,
+ where: [['refNameHash', '==', refNameHashB64]],
+ orderBy: [['$createdAt', 'asc']],
+ })
+}
+
+/** Drop repeated `$id`s, keeping the first occurrence. */
+function dedupeById(rows: readonly PlainDocument[]): PlainDocument[] {
+ const seen = new Set()
+ return rows.filter((d) => {
+ const id = String(d['$id'])
+ if (seen.has(id)) return false
+ seen.add(id)
+ return true
+ })
+}
+
+/**
+ * Page one type by key (see the module doc). `null` when the node did not honor the query —
+ * a page out of order, a row at or below the `> last` bound, or a round that would not
+ * advance `last` — and the caller then discards the scan entirely.
+ */
+async function keysetScan(
+ sdk: EvoSDK,
+ repo: RepoRef,
+ documentTypeName: string,
+): Promise {
+ const rows: PlainDocument[] = []
+ let after = null as { hex: string; b64: string } | null
+ for (let round = 0; round < MAX_KEYSET_ROUNDS; round++) {
+ const floorHex = after?.hex
+ const { documents: page } = await queryDocumentsWithProof(sdk, {
dataContractId: repo.contractId,
- documentTypeName: DOC.protectedRefUpdate,
- keyField: 'refNameHash',
- }),
- ])
- return [...new Set([...plain, ...protectedHashes])]
+ documentTypeName,
+ where: after === null ? [] : [['refNameHash', '>', after.b64]],
+ orderBy: [
+ ['refNameHash', 'asc'],
+ ['$createdAt', 'asc'],
+ ],
+ limit: PAGE,
+ })
+ const hashes = page.map((d) => refHashHexOf(d, documentTypeName))
+ const inOrder = hashes.every((h, i) => i === 0 || (hashes[i - 1] as string) <= h)
+ const inRange = floorHex === undefined || hashes.every((h) => h > floorHex)
+ if (!inOrder || !inRange) return null
+
+ if (page.length < PAGE) {
+ rows.push(...page)
+ return dedupeById(rows)
+ }
+ const last = hashes[hashes.length - 1] as string
+ const cut = hashes.indexOf(last)
+ let next: { hex: string; b64: string }
+ if (cut === 0) {
+ // One ref filled the page: read it on its own, then move past it.
+ const b64 = (page[0] as PlainDocument)['refNameHash'] as string
+ rows.push(...(await readOneRef(sdk, repo, documentTypeName, b64)))
+ next = { hex: last, b64 }
+ } else {
+ // Every ref before the last is whole; the last may be cut off and is re-read next page.
+ const prev = page[cut - 1] as PlainDocument
+ next = { hex: hashes[cut - 1] as string, b64: prev['refNameHash'] as string }
+ rows.push(...page.slice(0, cut))
+ }
+ // In-range pages already imply progress; this guards the invariant termination rests on.
+ if (floorHex !== undefined && next.hex <= floorHex) return null
+ after = next
+ }
+ throw new IncompleteReadError(
+ documentTypeName,
+ rows.length,
+ `the ref keyset scan did not finish in ${MAX_KEYSET_ROUNDS} rounds`,
+ )
+}
+
+/**
+ * Whether some update's non-null `prevOid` is no update's `newOid` in the same ref — a parent
+ * that should have been read and was not. Parity: forge-core `refs::has_missing_parent`.
+ */
+export function hasMissingParent(updates: readonly RefUpdate[]): boolean {
+ const tips = new Set(updates.map((u) => u.newOid))
+ return updates.some((u) => u.prevOid !== undefined && !isNullOid(u.prevOid) && !tips.has(u.prevOid))
+}
+
+interface TypeScan {
+ readonly type: string
+ readonly isProtected: boolean
+ readonly rows: PlainDocument[]
+}
+
+/** Group rows per ref: plain before protected, each in read order (the fold re-sorts). */
+function groupByRef(scans: readonly TypeScan[]): Map {
+ const byHash = new Map()
+ for (const { type, isProtected, rows } of scans) {
+ for (const doc of rows) {
+ const hex = refHashHexOf(doc, type)
+ const update = toRefUpdate(doc, isProtected)
+ const group = byHash.get(hex)
+ if (group === undefined) byHash.set(hex, [update])
+ else group.push(update)
+ }
+ }
+ return byHash
+}
+
+/**
+ * Every ref's complete update history, keyed by `refNameHash` hex. Keyset scan per type, then
+ * the `prevOid` completeness check; when the scan misbehaves or fails the check, the answer
+ * is the `reflog` read of both types alone (see the module doc).
+ */
+export async function readAllRefUpdates(
+ sdk: EvoSDK,
+ repo: RepoRef,
+): Promise> {
+ const scanned = await Promise.all(
+ REF_UPDATE_TYPES.map(async ([type, isProtected]) => ({
+ type,
+ isProtected,
+ rows: await keysetScan(sdk, repo, type),
+ })),
+ )
+ if (scanned.every((s) => s.rows !== null)) {
+ const grouped = groupByRef(scanned as TypeScan[])
+ if (![...grouped.values()].some(hasMissingParent)) return grouped
+ }
+
+ // The fallback stands alone: a scan that misbehaved or lost rows is not trusted for any.
+ const full = await Promise.all(
+ REF_UPDATE_TYPES.map(async ([type, isProtected]): Promise => ({
+ type,
+ isProtected,
+ rows: dedupeById(
+ await queryAllDocuments(sdk, {
+ dataContractId: repo.contractId,
+ documentTypeName: type,
+ orderBy: [['$createdAt', 'asc']],
+ }),
+ ),
+ })),
+ )
+ return groupByRef(full)
}
/**
@@ -67,7 +257,7 @@ export async function enumerateRefHashes(sdk: EvoSDK, repo: RepoRef): Promise {
const [plain, prot] = await Promise.all([
- queryAllDocuments(sdk, {
- dataContractId: repo.contractId,
- documentTypeName: DOC.refUpdate,
- where: [['refNameHash', '==', refNameHashB64]],
- orderBy: [['$createdAt', 'asc']],
- }),
- queryAllDocuments(sdk, {
- dataContractId: repo.contractId,
- documentTypeName: DOC.protectedRefUpdate,
- where: [['refNameHash', '==', refNameHashB64]],
- orderBy: [['$createdAt', 'asc']],
- }),
+ readOneRef(sdk, repo, DOC.refUpdate, refNameHashB64),
+ readOneRef(sdk, repo, DOC.protectedRefUpdate, refNameHashB64),
])
return [
...plain.map((d) => toRefUpdate(d, false)),
@@ -140,51 +320,10 @@ export async function resolveRefByHash(
}
/**
- * Fast path: fetch ONE page of each ref-update type ordered by the `refState` index
- * (`refNameHash asc, $createdAt asc`). A short page (< {@link PAGE}) proves the type's
- * update set is complete, so the per-ref histories can be grouped locally with zero further
- * queries. Returns null when either type overflows the page — callers fall back to skip-scan.
- *
- * This matters because skip-scan costs one sequential round-trip per distinct ref (~450ms
- * each on testnet — measured 8s for 18 refs), while a small repo's ENTIRE update set fits
- * in two parallel queries (~0.5s).
- */
-async function readAllRefUpdatesOnePage(
- sdk: EvoSDK,
- repo: RepoRef,
-): Promise | null> {
- const pageOf = (documentTypeName: string): ReturnType =>
- queryDocumentsWithProof(sdk, {
- dataContractId: repo.contractId,
- documentTypeName,
- orderBy: [
- ['refNameHash', 'asc'],
- ['$createdAt', 'asc'],
- ],
- limit: PAGE,
- })
- const [plain, prot] = await Promise.all([pageOf(DOC.refUpdate), pageOf(DOC.protectedRefUpdate)])
- if (plain.documents.length >= PAGE || prot.documents.length >= PAGE) return null
-
- // Group per ref, plain updates before protected — the same per-ref order
- // readRefUpdates produces (each source is already `$createdAt asc`).
- const byHash = new Map()
- const add = (update: RefUpdate): void => {
- const group = byHash.get(update.refNameHash)
- if (group === undefined) byHash.set(update.refNameHash, [update])
- else group.push(update)
- }
- for (const doc of plain.documents) add(toRefUpdate(doc, false))
- for (const doc of prot.documents) add(toRefUpdate(doc, true))
- return byHash
-}
-
-/**
- * Read every ref of a repo. Small repos (whole update set ≤ one query page per type) resolve
- * from two parallel queries; larger repos fall back to skip-scan enumeration + per-ref
- * resolution (parallelized). Fetches the config history once and reuses it across refs —
- * or accepts the caller's in-flight fetch (`configHistoryPromise`) so a composed read like
- * `loadRepoHome` issues only ONE config query total.
+ * Read every ref of a repo: {@link readAllRefUpdates}, each ref folded locally. Fetches the
+ * config history once and reuses it across refs — or accepts the caller's in-flight fetch
+ * (`configHistoryPromise`) so a composed read like `loadRepoHome` issues only ONE config
+ * query total.
*/
export async function readRefs(
sdk: EvoSDK,
@@ -192,23 +331,15 @@ export async function readRefs(
isAncestor: IsAncestor = NO_ANCESTRY,
configHistoryPromise?: Promise,
): Promise {
- const [complete, configHistory] = await Promise.all([
- readAllRefUpdatesOnePage(sdk, repo),
+ const [byHash, configHistory] = await Promise.all([
+ readAllRefUpdates(sdk, repo),
configHistoryPromise ?? readConfigHistory(sdk, repo),
])
- if (complete !== null) {
- return [...complete]
- .map(([refNameHashHex, updates]) =>
- toResolvedRef(updates, configHistory, refNameHashHex, isAncestor),
- )
- .filter((r): r is ResolvedRef => r !== null)
- }
-
- const hashes = await enumerateRefHashes(sdk, repo)
- const resolved = await Promise.all(
- hashes.map((h) => resolveRefByHash(sdk, repo, h, configHistory, isAncestor)),
- )
- return resolved.filter((r): r is ResolvedRef => r !== null)
+ return [...byHash]
+ .map(([refNameHashHex, updates]) =>
+ toResolvedRef(updates, configHistory, refNameHashHex, isAncestor),
+ )
+ .filter((r): r is ResolvedRef => r !== null)
}
/** Filter helper: only `refs/heads/*` branches. */
diff --git a/forge-web/lib/sdk/query.ts b/forge-web/lib/sdk/query.ts
index eef6d75b4..65a458a01 100644
--- a/forge-web/lib/sdk/query.ts
+++ b/forge-web/lib/sdk/query.ts
@@ -12,7 +12,7 @@
* orderBy-traversal order, so one hot key starves all siblings (measured 9/9 starved).
* So a multi-key read is done per key, in parallel, not as one `in` batch.
*
- * Plus **skip-scan** ref enumeration ({@link skipScanDistinct}): `> lastKey` orderBy key
+ * Plus **skip-scan** distinct-key enumeration ({@link skipScanDistinct}, used for pushers): `> lastKey` orderBy key
* `limit 1` hops to the next distinct key — O(log n) per distinct ref, not O(total pushes).
*/
diff --git a/forge-web/lib/view/browse-fallback.test.ts b/forge-web/lib/view/browse-fallback.test.ts
index bf6baa931..1ec5e7290 100644
--- a/forge-web/lib/view/browse-fallback.test.ts
+++ b/forge-web/lib/view/browse-fallback.test.ts
@@ -9,14 +9,16 @@ import type { EvoSDK } from '@dashevo/evo-sdk'
import { sha256 } from '@noble/hashes/sha2.js'
import { bytesToHex } from '@noble/hashes/utils.js'
import { zlibSync } from 'fflate'
-import { describe, expect, it } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
import { gitOidHex } from '../browse'
import {
T_BLOB,
T_OFS_DELTA,
+ T_REF_DELTA,
concat,
copyInsertDelta,
+ hexToBytes,
objHeader,
ofsBase,
packFrame,
@@ -25,6 +27,9 @@ import { CHUNK_PAYLOAD_MAX } from '../constants'
import type { PackManifest, RepoRef } from '../repo'
import { base64ToHex, bytesToBase64 } from '../sdk'
import { cachedFallback, startFallback, type FallbackProgress } from './browse-fallback'
+import { externalFetchUrls, resetExternalFetchState } from './browse-source'
+import { contentChecks, resetContentChecks } from './content-checks'
+import { deriveTrust } from './trust'
/** Mock SDK serving each pack's bytes as `chunk` docs split at CHUNK_PAYLOAD_MAX. */
function mockSdk(packsByHash: Map): EvoSDK {
@@ -121,3 +126,242 @@ describe('startFallback', () => {
await expect(startFallback(sdk, repo, [manifest])).rejects.toThrow(/header claims/)
})
})
+
+/** A single-blob pack (no deltas) holding `text`. */
+function blobPack(text: string): { pack: Uint8Array; oid: string } {
+ const body = new TextEncoder().encode(text)
+ return { pack: packFrame(concat(objHeader(T_BLOB, body.length), zlibSync(body))), oid: gitOidHex('blob', body) }
+}
+
+/** Route `fetch` by URL: a handler returns bytes, or throws for a dead host. */
+function stubFetch(routes: Record Uint8Array>): string[] {
+ const calls: string[] = []
+ vi.stubGlobal('fetch', (url: string) => {
+ calls.push(url)
+ const route = routes[url]
+ if (route === undefined) return Promise.reject(new TypeError('fetch failed: connection refused'))
+ const bytes = route()
+ return Promise.resolve(new Response(new Blob([bytes as BlobPart]), { status: 200 }))
+ })
+ return calls
+}
+
+describe('startFallback with external-storage packs', () => {
+ afterEach(() => {
+ vi.unstubAllGlobals()
+ vi.useRealTimers()
+ resetContentChecks()
+ resetExternalFetchState()
+ })
+
+ it('skips an external pack no mirror serves, reports it, and still serves the rest', async () => {
+ const plat = blobPack('on-chain content\n')
+ const ext = blobPack('content only the dead mirror had\n')
+ const platform = manifestFor(plat.pack, 1, { createdAt: 1, documentId: 'a' })
+ const external = manifestFor(ext.pack, 1, {
+ storage: 1,
+ chunkCount: 0,
+ uris: ['http://127.0.0.1:9000/forge-byo/pack', 's3://forge-byo/pack'],
+ createdAt: 2,
+ documentId: 'b',
+ })
+ const repo: RepoRef = { contractId: 'fallback-partial', ownerId: 'owner' }
+ const calls = stubFetch({})
+ const ctx = await startFallback(mockSdk(new Map([[platform.packHash, plat.pack]])), repo, [platform, external])
+
+ expect(calls).toEqual(['http://127.0.0.1:9000/forge-byo/pack']) // s3:// is not browser-fetchable
+ expect(ctx.unavailable).toHaveLength(1)
+ expect(ctx.unavailable?.[0]?.packHash).toBe(external.packHash)
+ expect(ctx.unavailable?.[0]?.hosts).toEqual(['127.0.0.1:9000'])
+ expect(Array.from((await ctx.reader.readObject(plat.oid)).bytes)).toEqual(
+ Array.from(new TextEncoder().encode('on-chain content\n')),
+ )
+ // A needed object from the skipped pack gets a per-view error naming pack and storage.
+ await expect(ctx.reader.readObject(ext.oid)).rejects.toThrow(
+ new RegExp(`${external.packHash.slice(0, 12)}.*127\\.0\\.0\\.1:9000`),
+ )
+ // The trust ledger reports the gap: content is partial, never verified.
+ const checks = contentChecks(repo.contractId)
+ expect(checks.unavailablePacks).toEqual([external.packHash])
+ const trust = deriveTrust({
+ network: 'testnet',
+ connection: 'trusted',
+ tip: 'missing',
+ checks,
+ configuredBackend: 'platform',
+ })
+ expect(trust.content.state).toBe('partial')
+ expect(trust.content.detail).toMatch(/1 pack could not be fetched from its storage; some objects may be missing|could not be fetched/)
+ })
+
+ it('reports a pack named by several manifests once', async () => {
+ const plat = blobPack('kept\n')
+ const ext = blobPack('gone\n')
+ const platform = manifestFor(plat.pack, 1, { createdAt: 1, documentId: 'a' })
+ const external = (documentId: string): PackManifest =>
+ manifestFor(ext.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'], createdAt: 2, documentId })
+ stubFetch({})
+ const ctx = await startFallback(
+ mockSdk(new Map([[platform.packHash, plat.pack]])),
+ { contractId: 'fallback-dup', ownerId: 'owner' },
+ [platform, external('b'), external('c')],
+ )
+ expect(ctx.unavailable).toHaveLength(1)
+ })
+
+ it('fetches an ipfs:// pack through a gateway and verifies it', async () => {
+ const ext = blobPack('pinned on ipfs\n')
+ const external = manifestFor(ext.pack, 1, { storage: 1, chunkCount: 0, uris: ['ipfs://bafkreitest'] })
+ const repo: RepoRef = { contractId: 'fallback-ipfs', ownerId: 'owner' }
+ const [first, second] = externalFetchUrls(external.uris)
+ expect(first).toBe('https://ipfs.io/ipfs/bafkreitest')
+ // The first gateway is down; the second serves the right bytes.
+ stubFetch({ [second as string]: () => ext.pack })
+ const ctx = await startFallback(mockSdk(new Map()), repo, [external])
+ expect(ctx.unavailable).toEqual([])
+ expect((await ctx.reader.readObject(ext.oid)).type).toBe('blob')
+ expect(contentChecks(repo.contractId).sources).toEqual(['dweb.link'])
+ })
+
+ it('treats a mirror serving the wrong bytes as unavailable, not as content', async () => {
+ const plat = blobPack('real\n')
+ const ext = blobPack('expected\n')
+ const platform = manifestFor(plat.pack, 1, { createdAt: 1, documentId: 'a' })
+ const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['https://mirror.example/p'], createdAt: 2, documentId: 'b' })
+ const repo: RepoRef = { contractId: 'fallback-liar', ownerId: 'owner' }
+ // Same length as the real pack, different bytes: only the sha256 check can catch it.
+ stubFetch({ 'https://mirror.example/p': () => blobPack('forgery!\n').pack })
+ const ctx = await startFallback(mockSdk(new Map([[platform.packHash, plat.pack]])), repo, [platform, external])
+ expect(ctx.unavailable?.[0]?.reason).toMatch(/sha256/)
+ expect(ctx.unavailable?.[0]?.corrupt).toBe(true)
+ expect(contentChecks(repo.contractId).corruptMirrorPacks).toEqual([external.packHash])
+ await expect(ctx.reader.readObject(ext.oid)).rejects.toThrow(/could not be fetched/)
+ })
+
+ it('still fails loudly when an on-chain pack cannot be read', async () => {
+ const plat = blobPack('on-chain\n')
+ const platform = manifestFor(plat.pack, 1)
+ const repo: RepoRef = { contractId: 'fallback-platform-missing', ownerId: 'owner' }
+ // The chunk documents are absent: platform storage must not be skipped.
+ await expect(startFallback(mockSdk(new Map()), repo, [platform])).rejects.toThrow(/missing chunk/)
+ })
+
+ it('fails with the reasons when no live pack at all could be fetched', async () => {
+ const ext = blobPack('x\n')
+ const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'] })
+ stubFetch({})
+ await expect(
+ startFallback(mockSdk(new Map()), { contractId: 'fallback-none', ownerId: 'owner' }, [external]),
+ ).rejects.toThrow(/none of this repo's 1 live packs could be fetched/)
+ })
+
+ it('never hits one origin with more than 4 requests at once', async () => {
+ const packs = Array.from({ length: 10 }, (_, i) => blobPack(`pack ${i}\n`))
+ const manifests = packs.map((p, i) =>
+ manifestFor(p.pack, 1, { storage: 1, uris: [`https://one.example/p${i}`], createdAt: i, documentId: `d${i}` }),
+ )
+ let active = 0
+ let peak = 0
+ vi.stubGlobal('fetch', async (url: string) => {
+ active += 1
+ peak = Math.max(peak, active)
+ await new Promise((r) => setTimeout(r, 5))
+ active -= 1
+ const i = Number(url.slice(url.lastIndexOf('p') + 1))
+ return new Response(new Blob([packs[i]?.pack as BlobPart]), { status: 200 })
+ })
+ const ctx = await startFallback(mockSdk(new Map()), { contractId: 'fallback-origin', ownerId: 'o' }, manifests)
+ expect(ctx.unavailable).toEqual([])
+ expect(peak).toBeLessThanOrEqual(4)
+ })
+
+ it('retries a timed-out mirror once before calling the pack unavailable', async () => {
+ const ext = blobPack('slow once\n')
+ const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['https://slow.example/p'] })
+ let calls = 0
+ vi.useFakeTimers()
+ vi.stubGlobal('fetch', (_url: string, init: RequestInit) => {
+ calls += 1
+ if (calls === 1) {
+ // Never answers; the 15 s deadline aborts it.
+ return new Promise((_, reject) =>
+ init.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError'))),
+ )
+ }
+ return Promise.resolve(new Response(new Blob([ext.pack as BlobPart]), { status: 200 }))
+ })
+ const run = startFallback(mockSdk(new Map()), { contractId: 'fallback-retry', ownerId: 'o' }, [external])
+ await vi.advanceTimersByTimeAsync(16_000)
+ vi.useRealTimers()
+ const ctx = await run
+ expect(calls).toBe(2)
+ expect(ctx.unavailable).toEqual([])
+ })
+
+ it('remembers a dead mirror for the session instead of asking it again', async () => {
+ const plat = blobPack('base\n')
+ const platform = manifestFor(plat.pack, 1, { createdAt: 0, documentId: 'a' })
+ const dead = (i: number): PackManifest => {
+ const p = blobPack(`gone ${i}\n`)
+ return manifestFor(p.pack, 1, { storage: 1, uris: ['http://127.0.0.1:9000/p'], createdAt: i + 1, documentId: `x${i}` })
+ }
+ const calls = stubFetch({})
+ const sdk = mockSdk(new Map([[platform.packHash, plat.pack]]))
+ await startFallback(sdk, { contractId: 'fallback-dead', ownerId: 'o' }, [platform, dead(1)])
+ await startFallback(sdk, { contractId: 'fallback-dead-2', ownerId: 'o' }, [platform, dead(2)])
+ expect(calls).toEqual(['http://127.0.0.1:9000/p'])
+ })
+
+ it('cancels external downloads when an on-chain pack fails', async () => {
+ const plat = blobPack('on-chain\n')
+ const ext = blobPack('big external\n')
+ const platform = manifestFor(plat.pack, 1, { createdAt: 0, documentId: 'a' })
+ const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['https://slow.example/big'], createdAt: 1, documentId: 'b' })
+ let aborted = false
+ vi.stubGlobal(
+ 'fetch',
+ (_url: string, init: RequestInit) =>
+ new Promise((_, reject) =>
+ init.signal?.addEventListener('abort', () => {
+ aborted = true
+ reject(new DOMException('aborted', 'AbortError'))
+ }),
+ ),
+ )
+ await expect(
+ startFallback(mockSdk(new Map()), { contractId: 'fallback-cancel', ownerId: 'o' }, [platform, external]),
+ ).rejects.toThrow(/missing chunk/)
+ expect(aborted).toBe(true)
+ })
+
+ it("names the skipped pack when a thin pack's REF_DELTA base was in it", async () => {
+ // Pack A (external, unreachable) holds the base blob; pack B (platform) holds a REF_DELTA
+ // onto it — the shape an imported thin pack can have.
+ const base = new TextEncoder().encode('the quick brown fox jumps over the lazy dog\n')
+ const target = new TextEncoder().encode('the quick brown fox jumps over the lazy cat\n')
+ const baseOid = gitOidHex('blob', base)
+ const packA = packFrame(concat(objHeader(T_BLOB, base.length), zlibSync(base)))
+ const delta = copyInsertDelta(base.length, target.length, 40, new TextEncoder().encode('cat\n'))
+ const packB = packFrame(concat(objHeader(T_REF_DELTA, delta.length), hexToBytes(baseOid), zlibSync(delta)))
+ const a = manifestFor(packA, 1, { storage: 1, uris: ['http://127.0.0.1:9000/a'], createdAt: 0, documentId: 'a' })
+ const b = manifestFor(packB, 1, { createdAt: 1, documentId: 'b' })
+ stubFetch({})
+ await expect(
+ startFallback(mockSdk(new Map([[b.packHash, packB]])), { contractId: 'fallback-thin', ownerId: 'o' }, [a, b]),
+ ).rejects.toThrow(new RegExp(`${a.packHash.slice(0, 12)}.*127\\.0\\.0\\.1:9000`))
+ })
+
+ it('expires a partial clone so a later view retries the mirrors', async () => {
+ vi.useFakeTimers()
+ const plat = blobPack('kept\n')
+ const platform = manifestFor(plat.pack, 1, { createdAt: 0, documentId: 'a' })
+ const ext = blobPack('later\n')
+ const external = manifestFor(ext.pack, 1, { storage: 1, uris: ['https://flaky.example/p'], createdAt: 1, documentId: 'b' })
+ const repo: RepoRef = { contractId: 'fallback-expiry', ownerId: 'o' }
+ vi.stubGlobal('fetch', () => Promise.resolve(new Response('nope', { status: 503 })))
+ await startFallback(mockSdk(new Map([[platform.packHash, plat.pack]])), repo, [platform, external])
+ expect(cachedFallback(repo.contractId, [platform, external])).not.toBeNull()
+ await vi.advanceTimersByTimeAsync(61_000)
+ expect(cachedFallback(repo.contractId, [platform, external])).toBeNull()
+ })
+})
diff --git a/forge-web/lib/view/browse-fallback.ts b/forge-web/lib/view/browse-fallback.ts
index cda7d034f..685e68380 100644
--- a/forge-web/lib/view/browse-fallback.ts
+++ b/forge-web/lib/view/browse-fallback.ts
@@ -3,7 +3,9 @@
* objectLocator.
*
* Downloads the repo's live kind-0 packs whole (each sha256-verified against its
- * consensus-proven `packManifest.packHash`, mirroring `git-remote-dash::fetch`), indexes
+ * consensus-proven `packManifest.packHash`, mirroring `git-remote-dash::fetch`: platform
+ * packs from `chunk` documents, external packs from any of their mirrors or IPFS gateways —
+ * an external pack none serves is skipped and reported, never silently), indexes
* them client-side (`lib/browse/indexer` — dynamically imported so pako stays out of the
* main bundles), and assembles the same {@link BrowseContext} the locator path produces,
* so every downstream view works unchanged.
@@ -21,7 +23,12 @@ import { bytesToHex } from '@noble/hashes/utils.js'
import { BrowseReader, ObjectLocator } from '../browse'
import type { PackManifest, RepoRef } from '../repo'
-import { loadArtifactBytesProgress, type BrowseContext } from './browse-source'
+import {
+ loadArtifactBytesProgress,
+ PackUnavailableError,
+ type BrowseContext,
+ type UnavailablePack,
+} from './browse-source'
import { noteContentCheck, objectObserver } from './content-checks'
import {
deleteStoredFallback,
@@ -48,12 +55,28 @@ interface CacheEntry {
const cache = new Map()
const restores = new Map>()
+/**
+ * How long a PARTIAL clone (some external packs skipped) is reused before the next view
+ * tries the mirrors again. A mirror that was down or slow a minute ago may be back; holding
+ * the partial context for the whole session would keep hiding its objects. A complete clone
+ * is kept for the session (and persisted), as before.
+ */
+export const PARTIAL_FALLBACK_TTL_MS = 60_000
+
function remember(contractId: string, manifestKey: string, promise: Promise): Promise {
const entry: CacheEntry = { manifestKey, promise }
cache.set(contractId, entry)
- promise.catch(() => {
- if (cache.get(contractId) === entry) cache.delete(contractId)
- })
+ promise.then(
+ (ctx) => {
+ if ((ctx.unavailable?.length ?? 0) === 0) return
+ setTimeout(() => {
+ if (cache.get(contractId) === entry) cache.delete(contractId)
+ }, PARTIAL_FALLBACK_TTL_MS)
+ },
+ () => {
+ if (cache.get(contractId) === entry) cache.delete(contractId)
+ },
+ )
return promise
}
@@ -162,6 +185,92 @@ function contextFromStored(repo: RepoRef, stored: StoredFallback): Promise `${p.packHash.slice(0, 12)}… (${p.hosts.length > 0 ? p.hosts.join(', ') : 'no fetchable mirror'})`)
+ .join('; ')
+ return new Error(
+ `object ${oidHex.slice(0, 12)}… is not in any pack this browser could load. ` +
+ `${unavailable.length === 1 ? 'One pack' : `${unavailable.length} packs`} could not be fetched from ` +
+ `${unavailable.length === 1 ? 'its' : 'their'} external storage and may hold it: ${where}. ` +
+ 'Cloning with dash:// reads the same packs; if their mirrors are down it will fail the same way.',
+ )
+}
+
+/** A downloaded live pack, or the record of why it could not be. */
+type PackOutcome =
+ | { readonly manifest: PackManifest; readonly bytes: Uint8Array }
+ | { readonly manifest: PackManifest; readonly unavailable: UnavailablePack }
+
+/**
+ * Download every live pack. Platform packs (storage 0) are the repo's own chain data: they
+ * download one after another and any failure fails the clone. External packs race their
+ * mirrors concurrently from the start, so dead mirrors cost one timeout in total, not one per
+ * pack; one no mirror serves authentically is skipped and reported, mirroring the dash://
+ * helper (a clone is not held hostage by one dead mirror, and git's connectivity check —
+ * here, the reader's missing-object error — still fails anything that truly needed it).
+ */
+async function downloadPacks(
+ sdk: EvoSDK,
+ repo: RepoRef,
+ livePacks: readonly PackManifest[],
+ report: (bytesFetched: number) => void,
+): Promise {
+ let fetched = 0
+ // Cancels every external download when the clone fails on a Platform pack: nothing will
+ // use those bytes, and a large pack would otherwise keep downloading in the background.
+ const abandon = new AbortController()
+ const external = new Map(
+ livePacks
+ .filter((m) => m.storage !== 0)
+ .map((m) => {
+ const outcome = loadArtifactBytesProgress(sdk, repo, m, undefined, abandon.signal).then(
+ (bytes): PackOutcome => {
+ fetched += m.sizeBytes
+ report(fetched)
+ return { manifest: m, bytes }
+ },
+ (e: unknown): PackOutcome => {
+ if (!(e instanceof PackUnavailableError)) throw e
+ return {
+ manifest: m,
+ unavailable: { packHash: m.packHash, hosts: e.hosts, reason: e.message, corrupt: e.corrupt },
+ }
+ },
+ )
+ // Observed here so an outcome no one awaits (the clone failed first) is never an
+ // unhandled rejection.
+ outcome.catch(() => undefined)
+ return [m, outcome] as const
+ }),
+ )
+ try {
+ const outcomes: PackOutcome[] = []
+ for (const manifest of livePacks) {
+ const ext = external.get(manifest)
+ if (ext !== undefined) {
+ outcomes.push(await ext)
+ continue
+ }
+ const base = fetched
+ const bytes = await loadArtifactBytesProgress(sdk, repo, manifest, (done) => report(base + done))
+ fetched += manifest.sizeBytes
+ outcomes.push({ manifest, bytes })
+ }
+ return outcomes
+ } catch (e) {
+ abandon.abort()
+ throw e
+ }
+}
+
async function runFallback(
sdk: EvoSDK,
repo: RepoRef,
@@ -179,33 +288,69 @@ async function runFallback(
...p,
})
- const packs: Uint8Array[] = []
- let fetchedBefore = 0
- for (const manifest of livePacks) {
- const bytes = await loadArtifactBytesProgress(sdk, repo, manifest, (done) =>
- report({ phase: 'download', bytesFetched: fetchedBefore + done }),
+ const outcomes = await downloadPacks(sdk, repo, livePacks, (bytesFetched) =>
+ report({ phase: 'download', bytesFetched }),
+ )
+ const got = outcomes.filter((o): o is Extract => 'bytes' in o)
+ // Keyed by packHash: a repo can carry several manifests for one pack (a re-push, a
+ // re-announced mirror), and a pack is missing once however many documents name it.
+ const unavailable = [
+ ...new Map(
+ outcomes.flatMap((o) =>
+ 'unavailable' in o ? [[o.unavailable.packHash.toLowerCase(), o.unavailable] as const] : [],
+ ),
+ ).values(),
+ ]
+ for (const u of unavailable) {
+ noteContentCheck(repo.contractId, { unavailablePack: u.packHash, corruptMirror: u.corrupt })
+ }
+ if (got.length === 0) {
+ throw new Error(
+ `none of this repo's ${livePacks.length} live packs could be fetched from their storage: ` +
+ unavailable.map((u) => u.reason).join('; '),
)
- fetchedBefore += manifest.sizeBytes
- packs.push(bytes)
}
+
+ const packs = got.map((o) => o.bytes)
+ const manifests = got.map((o) => o.manifest)
try {
- validatePacks(packs, livePacks)
+ validatePacks(packs, manifests)
} catch (e) {
// A downloaded pack that does not match its proof-read manifest is a content-check
// failure the trust panel must report, not only an error on this page.
noteContentCheck(repo.contractId, { packsFailed: 1 })
throw e
}
- noteContentCheck(repo.contractId, { packsVerified: livePacks.length })
+ noteContentCheck(repo.contractId, { packsVerified: packs.length })
const { indexPacks, serializeLocator, memoryPackSource } = await import('../browse/indexer')
- const objects = await indexPacks(packs, (objectsIndexed, objectsTotal) =>
- report({ phase: 'index', bytesFetched: bytesTotal, objectsIndexed, objectsTotal }),
- )
+ let objects: Awaited>
+ try {
+ objects = await indexPacks(packs, (objectsIndexed, objectsTotal) =>
+ report({ phase: 'index', bytesFetched: bytesTotal, objectsIndexed, objectsTotal }),
+ )
+ } catch (e) {
+ // A thin pack (imported or third-party) can REF_DELTA a base that only a skipped pack
+ // holds. Say so, naming the skipped packs, instead of a bare indexer error.
+ const base = /REF_DELTA base not found in live packs: ([0-9a-f]+)/.exec(
+ e instanceof Error ? e.message : '',
+ )?.[1]
+ if (base !== undefined && unavailable.length > 0) throw missingObjectError(base, unavailable)
+ throw e
+ }
const locatorBytes = serializeLocator(objects)
const locator = ObjectLocator.parse(locatorBytes)
+ // The synthesized locator's packRef space is exactly the packs that downloaded.
const packSource = memoryPackSource(packs)
- const reader = new BrowseReader(locator, packSource, { onObject: objectObserver(repo.contractId) })
- await storeFallback(repo.contractId, livePacks, { locator: locatorBytes, packs })
- return { locator, packs: packSource, reader }
+ const reader = new BrowseReader(locator, packSource, {
+ onObject: objectObserver(repo.contractId),
+ missingObject:
+ unavailable.length > 0 ? (oid) => missingObjectError(oid, unavailable) : undefined,
+ })
+ // Only a complete clone is persisted. A skipped pack's mirror may come back, and a reload
+ // is the natural moment to try it again; a persisted partial clone would never retry.
+ if (unavailable.length === 0) {
+ await storeFallback(repo.contractId, livePacks, { locator: locatorBytes, packs })
+ }
+ return { locator, packs: packSource, reader, unavailable }
}
diff --git a/forge-web/lib/view/browse-source.ts b/forge-web/lib/view/browse-source.ts
index 73876471b..db78299be 100644
--- a/forge-web/lib/view/browse-source.ts
+++ b/forge-web/lib/view/browse-source.ts
@@ -15,7 +15,10 @@
import type { EvoSDK } from '@dashevo/evo-sdk'
-import { CHUNK_PAYLOAD_MAX, PACK_KIND } from '../constants'
+import { sha256 } from '@noble/hashes/sha2.js'
+import { bytesToHex } from '@noble/hashes/utils.js'
+
+import { CHUNK_PAYLOAD_MAX, IPFS_GATEWAYS, PACK_KIND } from '../constants'
import {
BrowseReader,
FlatIndex,
@@ -236,30 +239,348 @@ async function fetchPlatformRange(
return out
}
+// ---------------------------------------------------------------------------
+// External (storage 1) artifacts
+// ---------------------------------------------------------------------------
+
/**
- * Fetch a contiguous range of an external artifact via HTTP Range. `onServed` is told which
- * URI answered, so the trust panel can name the host bytes actually came from.
+ * Budget for one request to one external mirror. The mirror may be a dead host (a manifest
+ * written by a push to someone's local MinIO) or a public gateway searching the IPFS network
+ * for a CID nobody pins; neither may hold a browse hostage.
*/
-async function fetchExternalRange(
+const EXTERNAL_FETCH_TIMEOUT_MS = 15_000
+
+/**
+ * An external artifact that no mirror served authentically. Carries what a view needs to
+ * say *which* pack is missing and *where* it was looked for.
+ */
+export class PackUnavailableError extends Error {
+ constructor(
+ readonly packHash: string,
+ /** Hosts actually tried (empty: the manifest records no browser-fetchable mirror). */
+ readonly hosts: readonly string[],
+ /** Whether some mirror answered with bytes that failed the sha256 check. */
+ readonly corrupt: boolean,
+ reason: string,
+ ) {
+ super(
+ `pack ${packHash.slice(0, 12)}… could not be fetched from its storage (${
+ hosts.length > 0 ? hosts.join(', ') : 'no browser-fetchable mirror recorded'
+ }): ${reason}`,
+ )
+ this.name = 'PackUnavailableError'
+ }
+}
+
+/**
+ * The HTTP(S) URLs an external artifact can be fetched from, in order: the manifest's own
+ * `http(s)` mirrors, then each `ipfs://` through every gateway in `gateways`. Schemes a
+ * browser cannot fetch (`s3://`, `platform://`) are skipped — an `s3://` locator always
+ * travels with the bucket's public `https` URL, which is listed separately.
+ */
+export function externalFetchUrls(
uris: readonly string[],
+ gateways: readonly string[] = IPFS_GATEWAYS,
+): string[] {
+ const out: string[] = []
+ for (const uri of uris) {
+ if (/^https?:\/\//i.test(uri)) out.push(uri)
+ const ipfs = /^ipfs:\/\/(.+)$/i.exec(uri)
+ if (ipfs !== null) {
+ for (const gw of gateways) out.push(`${gw.replace(/\/+$/, '')}/ipfs/${ipfs[1]}`)
+ }
+ }
+ return [...new Set(out)]
+}
+
+/** Requests in flight per origin. Every external pack starts at once; one host must not be
+ * hit with all of them (nor hold the browser's per-host connection pool hostage). */
+const PER_ORIGIN_CONCURRENCY = 4
+
+/** How many of an artifact's URLs are raced at once (an `ipfs://` fans out per gateway). */
+const MIRROR_RACE_WIDTH = 3
+
+/** Why a URL failed: a timeout is worth one more sequential try, anything else is not. */
+class FetchFailure extends Error {
+ constructor(
+ message: string,
+ readonly timedOut: boolean,
+ ) {
+ super(message)
+ }
+}
+
+/** A tiny per-origin semaphore. */
+const originSlots = new Map void)[] }>()
+
+async function withOriginSlot(url: string, run: () => Promise): Promise {
+ let origin: string
+ try {
+ origin = new URL(url).origin
+ } catch {
+ origin = url
+ }
+ let slot = originSlots.get(origin)
+ if (slot === undefined) {
+ slot = { active: 0, waiting: [] }
+ originSlots.set(origin, slot)
+ }
+ const s = slot
+ if (s.active >= PER_ORIGIN_CONCURRENCY) await new Promise((resolve) => s.waiting.push(resolve))
+ s.active += 1
+ try {
+ return await run()
+ } finally {
+ s.active -= 1
+ s.waiting.shift()?.()
+ }
+}
+
+/**
+ * URLs that failed this session for a reason other than a timeout (connection refused, 404,
+ * wrong bytes). A later pack naming the same mirror skips them rather than paying for the
+ * same failure again. Keyed by URL, not host: one gateway may lack one CID and hold another.
+ */
+const deadUrls = new Set()
+
+/** Test hook: forget the dead-URL list and the per-origin queues. */
+export function resetExternalFetchState(): void {
+ deadUrls.clear()
+ originSlots.clear()
+}
+
+/**
+ * GET `url` and read the whole body. Two deadlines, both {@link EXTERNAL_FETCH_TIMEOUT_MS}:
+ * one for the response to begin (a dead or silent host), then an IDLE deadline re-armed on
+ * every chunk, started only once the response has begun — a slow mirror streaming a large
+ * pack is fine, a stalled one is not. A whole-body deadline would make every pack larger
+ * than bandwidth × deadline permanently "unavailable" from healthy mirrors. The per-origin
+ * queue wait is not counted against either. A 206 is accepted alongside 2xx.
+ */
+async function fetchBody(
+ url: string,
+ init: RequestInit = {},
+ opts: { readonly cancel?: AbortSignal; readonly maxBytes?: number } = {},
+): Promise {
+ return withOriginSlot(url, async () => {
+ if (opts.cancel?.aborted) throw new FetchFailure('another mirror served it first', false)
+ const controller = new AbortController()
+ let timedOut = false
+ let timer: ReturnType | undefined
+ const arm = (): void => {
+ clearTimeout(timer)
+ timer = setTimeout(() => {
+ timedOut = true
+ controller.abort()
+ }, EXTERNAL_FETCH_TIMEOUT_MS)
+ }
+ const onCancel = (): void => controller.abort()
+ opts.cancel?.addEventListener('abort', onCancel)
+ arm() // the response must begin within the deadline
+ try {
+ const resp = await fetch(url, { ...init, signal: controller.signal })
+ if (!resp.ok && resp.status !== 206) throw new FetchFailure(`HTTP ${resp.status}`, false)
+ if (resp.body === null) return new Uint8Array(await resp.arrayBuffer())
+ const reader = resp.body.getReader()
+ const parts: Uint8Array[] = []
+ let total = 0
+ for (;;) {
+ arm() // idle deadline: re-armed per chunk once the body is streaming
+ const { done, value } = await reader.read()
+ if (done) break
+ total += value.length
+ // A mirror streaming more than the manifest says cannot be serving this pack.
+ if (opts.maxBytes !== undefined && total > opts.maxBytes) {
+ controller.abort()
+ throw new FetchFailure('served more bytes than the manifest records', false)
+ }
+ parts.push(value)
+ }
+ const out = new Uint8Array(total)
+ let at = 0
+ for (const p of parts) {
+ out.set(p, at)
+ at += p.length
+ }
+ return out
+ } catch (e) {
+ if (timedOut) throw new FetchFailure(`no data for ${EXTERNAL_FETCH_TIMEOUT_MS / 1000}s`, true)
+ if (opts.cancel?.aborted) throw new FetchFailure('another mirror served it first', false)
+ if (e instanceof FetchFailure) throw e
+ throw new FetchFailure(e instanceof Error ? e.message : String(e), false)
+ } finally {
+ clearTimeout(timer)
+ opts.cancel?.removeEventListener('abort', onCancel)
+ }
+ })
+}
+
+function hostsOf(urls: readonly string[]): string[] {
+ return [...new Set(urls.map(externalSourceName))]
+}
+
+/**
+ * Fetch a contiguous range of an external artifact via HTTP Range, trying each fetchable
+ * mirror in turn. `onServed` is told which URL answered, so the trust panel can name the host
+ * bytes actually came from. A range cannot be hashed on its own: the reader re-hashes every
+ * object it reconstructs from it.
+ */
+async function fetchExternalRange(
+ manifest: PackManifest,
start: number,
end: number,
onServed?: (uri: string) => void,
): Promise {
- let lastErr: unknown
- for (const uri of uris) {
+ const urls = externalFetchUrls(manifest.uris)
+ let lastErr: unknown = 'no browser-fetchable mirror'
+ for (const url of urls.filter((u) => !deadUrls.has(u))) {
try {
- const resp = await fetch(uri, { headers: { Range: `bytes=${start}-${end - 1}` } })
- if (!resp.ok && resp.status !== 206) throw new Error(`HTTP ${resp.status}`)
- const buf = new Uint8Array(await resp.arrayBuffer())
- onServed?.(uri)
+ const buf = await fetchBody(url, { headers: { Range: `bytes=${start}-${end - 1}` } })
+ onServed?.(url)
// Some hosts ignore Range and return the whole body — slice defensively.
return buf.length > end - start ? buf.subarray(start, end) : buf
} catch (e) {
+ if (e instanceof FetchFailure && !e.timedOut) deadUrls.add(url)
lastErr = e
}
}
- throw new Error(`no external URI served the range: ${String(lastErr)}`)
+ throw new PackUnavailableError(manifest.packHash, hostsOf(urls), false, errorText(lastErr))
+}
+
+function errorText(e: unknown): string {
+ return e instanceof Error ? e.message : String(e)
+}
+
+/**
+ * Fetch a whole external artifact; the first body whose size and sha256 match the
+ * proof-read manifest wins. Mirrors are availability, never authority: one answering with
+ * other bytes is treated as down and flagged `corrupt` (reported distinctly — it is a
+ * misbehaving host, not an outage).
+ *
+ * The URLs are raced {@link MIRROR_RACE_WIDTH} at a time (skipping this session's dead URLs),
+ * and the losers are cancelled as soon as one wins. URLs that only timed out get ONE more
+ * sequential try each before the pack is declared unavailable: a gateway resolving a cold
+ * CID is often slow once and fast after. `cancel` aborts everything (the clone gave up).
+ */
+async function fetchExternalWhole(
+ manifest: PackManifest,
+ onServed?: (uri: string) => void,
+ cancel?: AbortSignal,
+): Promise {
+ const urls = externalFetchUrls(manifest.uris)
+ const want = manifest.packHash.toLowerCase()
+ const live = urls.filter((u) => !deadUrls.has(u))
+ if (live.length === 0) {
+ throw new PackUnavailableError(
+ manifest.packHash,
+ hostsOf(urls),
+ false,
+ urls.length === 0 ? 'nothing to try' : 'every mirror already failed this session',
+ )
+ }
+ let corrupt = false
+ const reasons: string[] = []
+ const timedOut: string[] = []
+ // Cancels the losing mirrors once one has served the pack (or the whole clone gave up):
+ // an ipfs:// URI fans out to one request per gateway, and each would otherwise download
+ // (and hold) the whole pack.
+ const winner = new AbortController()
+ const onCancel = (): void => winner.abort()
+ cancel?.addEventListener('abort', onCancel)
+ if (cancel?.aborted) winner.abort()
+
+ const attempt = async (url: string): Promise<{ url: string; bytes: Uint8Array }> => {
+ try {
+ const bytes = await fetchBody(url, {}, { cancel: winner.signal, maxBytes: manifest.sizeBytes })
+ if (bytes.length !== manifest.sizeBytes || bytesToHex(sha256(bytes)) !== want) {
+ corrupt = true
+ throw new FetchFailure('served bytes that do not match the manifest sha256', false)
+ }
+ return { url, bytes }
+ } catch (e) {
+ if (!winner.signal.aborted) {
+ reasons.push(`${externalSourceName(url)}: ${errorText(e)}`)
+ if (e instanceof FetchFailure && e.timedOut) timedOut.push(url)
+ else deadUrls.add(url)
+ }
+ throw e
+ }
+ }
+
+ try {
+ const got = (await raceBounded(live, MIRROR_RACE_WIDTH, attempt, winner.signal)) ??
+ (await firstSequential(timedOut, attempt, winner.signal))
+ if (got !== null) {
+ winner.abort()
+ onServed?.(got.url)
+ return got.bytes
+ }
+ if (cancel?.aborted) throw new Error('the in-browser clone was cancelled')
+ throw new PackUnavailableError(manifest.packHash, hostsOf(urls), corrupt, reasons.join('; '))
+ } finally {
+ cancel?.removeEventListener('abort', onCancel)
+ }
+}
+
+/**
+ * Run `attempt` over `items` with at most `width` in flight; resolve with the first success,
+ * or null once every item has failed (or `stop` fired).
+ */
+function raceBounded(
+ items: readonly T[],
+ width: number,
+ attempt: (item: T) => Promise,
+ stop: AbortSignal,
+): Promise {
+ return new Promise((resolve) => {
+ let next = 0
+ let running = 0
+ let settled = false
+ const launch = (): void => {
+ while (!settled && !stop.aborted && running < width && next < items.length) {
+ const item = items[next++] as T
+ running += 1
+ attempt(item).then(
+ (r) => {
+ if (!settled) {
+ settled = true
+ resolve(r)
+ }
+ },
+ () => {
+ running -= 1
+ if (settled) return
+ if ((next >= items.length || stop.aborted) && running === 0) {
+ settled = true
+ resolve(null)
+ } else launch()
+ },
+ )
+ }
+ if (!settled && running === 0) {
+ settled = true
+ resolve(null)
+ }
+ }
+ launch()
+ })
+}
+
+/** Try `items` one at a time; the first success, or null. */
+async function firstSequential(
+ items: readonly T[],
+ attempt: (item: T) => Promise,
+ stop: AbortSignal,
+): Promise {
+ for (const item of [...items]) {
+ if (stop.aborted) return null
+ try {
+ return await attempt(item)
+ } catch {
+ /* next */
+ }
+ }
+ return null
}
/** Record in the repo's content-check ledger where an artifact's bytes came from. */
@@ -277,7 +598,7 @@ export function artifactRangeFetch(
): RangeFetch {
return async (start: number, end: number) => {
if (manifest.storage !== 0) {
- return fetchExternalRange(manifest.uris, start, end, (uri) => noteSource(repo, uri))
+ return fetchExternalRange(manifest, start, end, (uri) => noteSource(repo, uri))
}
const bytes = await fetchPlatformRange(sdk, repo.contractId, manifest.packHash, start, end)
noteSource(repo)
@@ -311,21 +632,22 @@ const DOWNLOAD_WINDOW = CHUNK_QUERY_MAX * CHUNK_PAYLOAD_MAX
/**
* Load a whole artifact with download progress — the fallback-clone path for full git
* packs, which can exceed the single-query chunk window. Platform storage downloads in
- * `DOWNLOAD_WINDOW` strides; external storage fetches the body whole (progress reported
- * only at completion).
+ * `DOWNLOAD_WINDOW` strides; external storage races its mirrors for the whole body, which
+ * must match the manifest's size and sha256 (progress reported only at completion), and
+ * throws {@link PackUnavailableError} when none does.
*/
export async function loadArtifactBytesProgress(
sdk: EvoSDK,
repo: RepoRef,
manifest: PackManifest,
onProgress?: (bytesFetched: number, bytesTotal: number) => void,
+ cancel?: AbortSignal,
): Promise {
const total = manifest.sizeBytes
if (total <= 0) return new Uint8Array(0)
onProgress?.(0, total)
if (manifest.storage !== 0) {
- const bytes = await fetchExternalRange(manifest.uris, 0, total, (uri) => noteSource(repo, uri))
- if (bytes.length !== total) throw new Error('external artifact length mismatch')
+ const bytes = await fetchExternalWhole(manifest, (uri) => noteSource(repo, uri), cancel)
onProgress?.(total, total)
return bytes
}
@@ -424,11 +746,26 @@ export function buildPackSource(
}
}
+/** A live pack the browser could not obtain from its external storage. */
+export interface UnavailablePack {
+ readonly packHash: string
+ /** Hosts tried (empty: the manifest records no browser-fetchable mirror). */
+ readonly hosts: readonly string[]
+ readonly reason: string
+ /** Some mirror served bytes that failed the sha256 check — a misbehaving host, not an outage. */
+ readonly corrupt: boolean
+}
+
/** The assembled browse context for a repo, or a reason it is unavailable. */
export interface BrowseContext {
readonly locator: ObjectLocator
readonly packs: PackSource
readonly reader: BrowseReader
+ /**
+ * External packs the in-browser clone skipped because no mirror served them. Objects only
+ * those packs hold are absent from this context; empty/absent means nothing was skipped.
+ */
+ readonly unavailable?: readonly UnavailablePack[]
}
/**
diff --git a/forge-web/lib/view/content-checks.test.ts b/forge-web/lib/view/content-checks.test.ts
index bc0b7c945..cbe53533c 100644
--- a/forge-web/lib/view/content-checks.test.ts
+++ b/forge-web/lib/view/content-checks.test.ts
@@ -58,6 +58,14 @@ describe('content-check ledger', () => {
unsubscribe()
})
+ it('records each unavailable pack once, case-insensitively', () => {
+ noteContentCheck('repo-a', { unavailablePack: 'AB'.repeat(32) })
+ const snap = contentChecks('repo-a')
+ noteContentCheck('repo-a', { unavailablePack: 'ab'.repeat(32) })
+ expect(contentChecks('repo-a')).toBe(snap)
+ expect(snap.unavailablePacks).toEqual(['ab'.repeat(32)])
+ })
+
it('names an external source by its host', () => {
expect(externalSourceName('https://ipfs.io/ipfs/bafy')).toBe('ipfs.io')
expect(externalSourceName('https://bucket.s3.amazonaws.com/p.pack')).toBe('bucket.s3.amazonaws.com')
diff --git a/forge-web/lib/view/content-checks.ts b/forge-web/lib/view/content-checks.ts
index 43fb8367f..ce1fcd473 100644
--- a/forge-web/lib/view/content-checks.ts
+++ b/forge-web/lib/view/content-checks.ts
@@ -29,6 +29,17 @@ export interface ContentChecks {
readonly packsFailed: number
/** Where bytes actually came from: `platform`, `browser cache`, or an external host. */
readonly sources: readonly string[]
+ /**
+ * Live external packs (hash hex) the in-browser clone skipped because no mirror served
+ * them. What is shown was checked, but it is not the whole repo: objects only those packs
+ * hold are missing.
+ */
+ readonly unavailablePacks: readonly string[]
+ /**
+ * Of those, the packs some mirror answered with bytes that failed the sha256 check — a host
+ * serving bad data, reported distinctly from an outage.
+ */
+ readonly corruptMirrorPacks: readonly string[]
}
export const NO_CONTENT_CHECKS: ContentChecks = {
@@ -38,12 +49,22 @@ export const NO_CONTENT_CHECKS: ContentChecks = {
packsVerified: 0,
packsFailed: 0,
sources: [],
+ unavailablePacks: [],
+ corruptMirrorPacks: [],
}
-type Counter = Exclude
+type Counter = Exclude
-/** A change to one repo's ledger: counter increments and/or a byte source seen. */
-export type ContentCheckDelta = Partial> & { readonly source?: string }
+/**
+ * A change to one repo's ledger: counter increments, a byte source seen, and/or a live pack
+ * that could not be fetched (hash hex).
+ */
+export type ContentCheckDelta = Partial> & {
+ readonly source?: string
+ readonly unavailablePack?: string
+ /** With `unavailablePack`: a mirror served bytes that failed the sha256 check. */
+ readonly corruptMirror?: boolean
+}
const ledger = new Map()
const listeners = new Set<() => void>()
@@ -60,11 +81,17 @@ export function noteContentCheck(contractId: string, delta: ContentCheckDelta):
'packsFailed',
]
const bumped = counters.some((k) => (delta[k] ?? 0) > 0)
- if (!newSource && !bumped) return
+ const missing = delta.unavailablePack?.toLowerCase()
+ const newMissing = missing !== undefined && !prev.unavailablePacks.includes(missing)
+ const newCorrupt =
+ missing !== undefined && delta.corruptMirror === true && !prev.corruptMirrorPacks.includes(missing)
+ if (!newSource && !bumped && !newMissing && !newCorrupt) return
const next: { -readonly [K in keyof ContentChecks]: ContentChecks[K] } = { ...prev }
for (const k of counters) next[k] = prev[k] + Math.max(0, delta[k] ?? 0)
if (newSource && delta.source !== undefined) next.sources = [...prev.sources, delta.source]
+ if (newMissing) next.unavailablePacks = [...prev.unavailablePacks, missing]
+ if (newCorrupt) next.corruptMirrorPacks = [...prev.corruptMirrorPacks, missing]
ledger.set(contractId, next)
for (const l of listeners) l()
}
diff --git a/forge-web/lib/view/index.ts b/forge-web/lib/view/index.ts
index b15774781..a64e11d49 100644
--- a/forge-web/lib/view/index.ts
+++ b/forge-web/lib/view/index.ts
@@ -67,8 +67,10 @@ export {
loadFlatIndex,
orderGitPacks,
peekBrowseState,
+ PackUnavailableError,
type BrowseContext,
type BrowseState,
+ type UnavailablePack,
} from './browse-source'
export {
cachedFallback,
diff --git a/forge-web/lib/view/trust.test.ts b/forge-web/lib/view/trust.test.ts
index d6401b65f..74e59c032 100644
--- a/forge-web/lib/view/trust.test.ts
+++ b/forge-web/lib/view/trust.test.ts
@@ -123,6 +123,25 @@ describe('deriveTrust — content hashes', () => {
expect(pack.content.detail).toMatch(/1 pack did not match its manifest/)
})
+ it('is partial, never verified, when a live pack could not be fetched', () => {
+ const read = deriveTrust(
+ inputs({ checks: checks({ objectsVerified: 5, packsVerified: 3, unavailablePacks: ['ab'.repeat(32)] }) }),
+ )
+ expect(read.content.state).toBe('partial')
+ expect(read.content.summary).toBe('1 pack missing')
+ expect(read.content.detail).toMatch(/1 pack could not be fetched from its storage, so some objects may be missing/)
+ expect(read.overall).toBe('partial')
+
+ // Known before any object is read, so not `pending` either.
+ const none = deriveTrust(inputs({ checks: checks({ unavailablePacks: ['ab'.repeat(32), 'cd'.repeat(32)] }) }))
+ expect(none.content.state).toBe('partial')
+ expect(none.content.summary).toBe('2 packs missing')
+
+ // A hash failure still wins.
+ const bad = deriveTrust(inputs({ checks: checks({ objectsFailed: 1, unavailablePacks: ['ab'.repeat(32)] }) }))
+ expect(bad.content.state).toBe('failed')
+ })
+
it('names every source bytes actually came from', () => {
const r = deriveTrust(inputs({ checks: checks({ objectsVerified: 1, sources: ['platform', 'ipfs.io'] }) }))
expect(r.source.summary).toBe('2 sources')
diff --git a/forge-web/lib/view/trust.ts b/forge-web/lib/view/trust.ts
index 7f7d2ddf6..dfc3b1038 100644
--- a/forge-web/lib/view/trust.ts
+++ b/forge-web/lib/view/trust.ts
@@ -170,6 +170,27 @@ function deriveRefs(input: TrustInputs): TrustLink {
}
function deriveContent(checks: ContentChecks): TrustLink {
+ const link = deriveReadContent(checks)
+ const missing = checks.unavailablePacks.length
+ if (missing === 0 || link.state === 'failed') return link
+ // Everything shown passed its check, but the answer is incomplete: objects only the
+ // skipped packs hold cannot be shown at all. That is at best `partial`, never `verified`
+ // — and never `pending` either, since the skip is known before any object is read.
+ const corrupt = checks.corruptMirrorPacks.length
+ const bad =
+ corrupt > 0
+ ? ` A mirror served bad data (bytes that fail the manifest sha256) for ${plural(corrupt, 'pack')}; it was refused.`
+ : ''
+ const note = `${plural(missing, 'pack')} could not be fetched from ${missing === 1 ? 'its' : 'their'} storage, so some objects may be missing.${bad}`
+ return {
+ state: link.state === 'unverified' ? 'unverified' : 'partial',
+ summary: `${missing} ${missing === 1 ? 'pack' : 'packs'} missing`,
+ detail: `${note} ${link.detail}`,
+ }
+}
+
+/** The content link from what was read, before accounting for packs that were skipped. */
+function deriveReadContent(checks: ContentChecks): TrustLink {
const failed = checks.objectsFailed + checks.packsFailed
if (failed > 0) {
const parts: string[] = []
diff --git a/forge-web/tailwind.config.js b/forge-web/tailwind.config.js
index fb1c88e1b..ec8669404 100644
--- a/forge-web/tailwind.config.js
+++ b/forge-web/tailwind.config.js
@@ -43,7 +43,10 @@ module.exports = {
950: '#0f0d0c',
},
// Semantic colors are MEANINGFUL, never decorative. Do not repurpose.
- verify: '#16a34a', // proof/hash verified
+ verify: {
+ DEFAULT: '#16a34a', // proof/hash verified — icons, borders, tints
+ 700: '#15803d', // solid fill behind white text (5.02:1; the base value is 3.3:1)
+ },
caution: '#d97706', // degraded availability
danger: '#dc2626', // force-push, delete, failed verification
// Dash brand blue — identity/credits/network UI only. The brand value is for fills,
@@ -54,6 +57,7 @@ module.exports = {
DEFAULT: '#008de4',
400: '#4aaef0', // text on dark surfaces (anvil-950…800)
600: '#006bb0', // text on light surfaces (white, anvil-50/100)
+ 700: '#005a94', // solid fill behind white text (7.27:1; the brand value is 3.54:1)
},
},
fontFamily: {