From 603115aa0b5330d06828303070756aebbae4ca34 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 14:50:48 +0000 Subject: [PATCH 1/7] dt64: BindStore::check_route, the read-only form of route (D-TGL-PUB step 1) check_route applies route's two refusals (UnknownClass, OutsideReservation) without a route event or a new RouteVersion. route runs the same check before it writes (RouteLog::check), so the two cannot disagree. A checked intent stays valid: classes are only added and the answer does not read the route log. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-dt64/src/dr.rs | 10 ++- crates/lance-graph-dt64/src/lib.rs | 27 ++++++- crates/lance-graph-dt64/tests/bind_store.rs | 82 +++++++++++++++++++++ 3 files changed, 115 insertions(+), 4 deletions(-) diff --git a/crates/lance-graph-dt64/src/dr.rs b/crates/lance-graph-dt64/src/dr.rs index c1af50857..efffaf6ad 100644 --- a/crates/lance-graph-dt64/src/dr.rs +++ b/crates/lance-graph-dt64/src/dr.rs @@ -258,6 +258,14 @@ impl RouteLog { class: u32, identity: Axis, ) -> Result { + self.check(position, class, identity)?; + Ok(self.push(position, Some(class))) + } + + /// Whether [`RouteLog::route`] would accept routing `position` to + /// `class`, without routing it: the same refusal, no event, no version. + /// `route` runs this exact check, so the two cannot disagree. + pub fn check(&self, position: u16, class: u32, identity: Axis) -> Result<(), RouteRefusal> { let reserved = identity.reserved(); if usize::from(position) >= reserved { return Err(RouteRefusal::OutsideReservation { @@ -266,7 +274,7 @@ impl RouteLog { reserved, }); } - Ok(self.push(position, Some(class))) + Ok(()) } /// Remove `position`'s route; returns the new version. diff --git a/crates/lance-graph-dt64/src/lib.rs b/crates/lance-graph-dt64/src/lib.rs index 9f5b7ccc6..68494c8c4 100644 --- a/crates/lance-graph-dt64/src/lib.rs +++ b/crates/lance-graph-dt64/src/lib.rs @@ -737,13 +737,34 @@ impl BindStore { /// Route `position` to the bound `class`; returns the new version. A /// position past the class's identity reservation is refused. pub fn route(&mut self, position: u16, class: u32) -> Result { + let identity = self.route_identity(position, class)?; + Ok(self.routes.route(position, class, identity)?) + } + + /// Whether [`BindStore::route`] would accept routing `position` to + /// `class`, without routing it: the same two refusals + /// ([`Refusal::UnknownClass`], [`Refusal::OutsideReservation`]), no route + /// event, no new [`RouteVersion`]. `route` runs this exact check before + /// it writes, so the two cannot disagree. + /// + /// A route intent that passes stays valid: classes are only ever added + /// (a bound class is never removed or changed, D5) and the answer does + /// not depend on the route log. A prepared land (G10/G11) can therefore + /// check its route intents up front and apply them later without an + /// error path. + pub fn check_route(&self, position: u16, class: u32) -> Result<(), Refusal> { + self.route_identity(position, class).map(|_| ()) + } + + /// The class's identity reservation, after the route check. + fn route_identity(&self, position: u16, class: u32) -> Result { let entry = self .classes .get(&class) .ok_or(Refusal::UnknownClass(class))?; - Ok(self - .routes - .route(position, class, entry.schema.extent.identity)?) + let identity = entry.schema.extent.identity; + self.routes.check(position, class, identity)?; + Ok(identity) } /// What `class` has claimed of its reservation at `version`: the identity diff --git a/crates/lance-graph-dt64/tests/bind_store.rs b/crates/lance-graph-dt64/tests/bind_store.rs index 403dfca93..d200f9531 100644 --- a/crates/lance-graph-dt64/tests/bind_store.rs +++ b/crates/lance-graph-dt64/tests/bind_store.rs @@ -1099,3 +1099,85 @@ fn a_view_that_does_not_implement_lens_fields_still_compiles_and_refuses_behavio Refusal::UnknownLensBasis(RELALG_LENS) ); } + +/// A store for the `check_route` tests: the catalogue plus ONNX with a +/// one-tile reservation (256 identity positions). +fn routing_store() -> BindStore { + let mut s = catalogue(); + s.register(ONNX, schema_in(1, 4, Extent::DR256_SQ_X64), vec![node(1)]) + .unwrap(); + s +} + +/// `check_route` answers exactly what `route` would, on every case: an +/// unknown class, the last position inside the reservation, the first +/// outside it, and a position far past it. Red if it skips either of +/// `route`'s two refusals (disables: drop the class lookup; drop the +/// reservation check from `check_route` only). +#[test] +fn check_route_agrees_with_route_on_every_case() { + let s = routing_store(); + let cases = [ + (7, 0xBEEF_0001), + (0, ONNX), + (255, ONNX), + (256, ONNX), + (7_619, ONNX), + ]; + let (mut accepted, mut unknown, mut outside) = (0, 0, 0); + for (position, class) in cases { + let checked = s.check_route(position, class); + let routed = s.clone().route(position, class).map(|_| ()); + assert_eq!(checked, routed, "position {position}, class {class:#x}"); + match checked { + Ok(()) => accepted += 1, + Err(Refusal::UnknownClass(_)) => unknown += 1, + Err(Refusal::OutsideReservation { .. }) => outside += 1, + Err(other) => panic!("unexpected refusal {other:?}"), + } + } + // Anti-vacuity: both outcomes and both refusals occur. + assert_eq!((accepted, unknown, outside), (2, 1, 2)); +} + +/// `check_route` writes nothing: no route event, no new `RouteVersion`, no +/// claimed tile, for accepted and refused intents alike. Twin: `route` on the +/// same intent does move all three, so the comparison can fail. +#[test] +fn check_route_writes_nothing() { + let mut s = routing_store(); + let v = s.current_version(); + let claim = s.claimed(ONNX, v).unwrap(); + for (position, class) in [(3, ONNX), (256, ONNX), (3, 0xBEEF_0001)] { + let _ = s.check_route(position, class); + } + assert_eq!(s.current_version(), v, "no version issued"); + assert_eq!(s.routed_at(s.current_version(), 3), Ok(None)); + assert_eq!(s.claimed(ONNX, s.current_version()).unwrap(), claim); + + s.route(3, ONNX).unwrap(); + assert_ne!(s.current_version(), v, "route does move the version"); + assert_eq!(s.routed_at(s.current_version(), 3), Ok(Some(ONNX))); +} + +/// A checked route intent stays valid while the store grows: other classes +/// are bound and other positions routed or retracted, and the same intent is +/// still accepted, then applied without a refusal (the property a prepared +/// land relies on to apply with no error path). +#[test] +fn a_checked_route_intent_stays_valid_as_the_store_grows() { + let mut s = routing_store(); + assert_eq!(s.check_route(42, ONNX), Ok(())); + s.register( + JINA, + schema_in(1, 4, Extent::DR256_GLOBAL_X64), + vec![node(1)], + ) + .unwrap(); + s.route(42, JINA).unwrap(); + s.route(7, ONNX).unwrap(); + s.retract(7); + assert_eq!(s.check_route(42, ONNX), Ok(())); + let v = s.route(42, ONNX).expect("a checked intent applies"); + assert_eq!(s.routed_at(v, 42), Ok(Some(ONNX))); +} From 987c5e7a2aa7ad12fb36cc1df60fbbc3b5815d6d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 14:52:02 +0000 Subject: [PATCH 2/7] board: D-TGL-PUB step 1 (check_route) shipped, lab-ready Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- .claude/board/STATUS_BOARD.md | 2 +- ...-02-d-tgl-release-fence-and-g10-g11-boundaries.md | 12 ++++++++++++ .claude/plans/g10-g11-atomic-publication-lab-v1.md | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 8880b04e2..6f368e96c 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -249,7 +249,7 @@ Plan: [d-codegraph-0-resident-codegraph-v1.md](../plans/d-codegraph-0-resident-c | **D-TGL-G10** | atomic publication: prepare all, durable commit first, then infallible in-memory apply from the land record, replay after crash | OPEN (production); named gap: no prepare/validate route API on `BindStore` | a `Fenced` leaves no published partial state | | **D-TGL-G11** | branch-local routing; no branch-minted DR position as durable routing key | OPEN (production) | a branch route leaves `BindStore::current_version()` unchanged until land | | **D-TGL-G12** | identities apart: W-slot, generation, ThoughtCommitId, DurableRef, RouteVersion, Lance version, source SHA (provenance only) | OPEN (production) | joins only through receipt / CommitWitness / MetaKey / land record | -| **D-TGL-PUB** | G10 × G11 lab: prepared land (Codegraph + DT64 route intent), fenced durable commit through a test `WalSink`, crash recovery from the durable record, branch-local routing | SPEC: plan `g10-g11-atomic-publication-lab-v1.md`; boundaries B1–B5 located; needs `BindStore::check_route` (read-only, missing) | L1–L6, each with a disable; L4 shows route re-application ticks `RouteVersion` without a watermark | +| **D-TGL-PUB** | G10 × G11 lab: prepared land (Codegraph + DT64 route intent), fenced durable commit through a test `WalSink`, crash recovery from the durable record, branch-local routing | SPEC: plan `g10-g11-atomic-publication-lab-v1.md`; boundaries B1–B5 located. Step 1 SHIPPED (lab-ready): `BindStore::check_route` (read-only; `route` runs the same `RouteLog::check`), 3 tests, 3 disables red. Steps 2 (lab `WalSink`) and 3 (L1–L6) open | L1–L6, each with a disable; L4 shows route re-application ticks `RouteVersion` without a watermark | | **D-TGL-EXO148** | `DurableRef` + never-reused `ThoughtCommitId` before any durable land, replay or PR record | OPEN (production, EXO-148) | reuse test (D-RST-1) | | **D-DT64-AUTH** | DT64 / F### as the one method authority: census of production words with a two-way exception ratchet; parallel identities (`MethodSignature`, tract `Registry`, XLOG `Word`, ogar-loco `FnIndex`) stay as listed `LAB` adapters | QUEUED: plan `d-dt64-method-authority-census-v1.md` (operator decision 2026-11-02); egglog pinned to one commit in `egglog-r2il-probe`, `dr-math-binding-probe`, `xlog-relcomp-probe` | undeclared production word → red; stale exception → red; current tree green | | **D-CODEGRAPH-0-D** | local stdio MCP over C | QUEUED | only after B and C gates | diff --git a/.claude/board/entries/2026-11-02-d-tgl-release-fence-and-g10-g11-boundaries.md b/.claude/board/entries/2026-11-02-d-tgl-release-fence-and-g10-g11-boundaries.md index 3686c79e3..8c685fcc4 100644 --- a/.claude/board/entries/2026-11-02-d-tgl-release-fence-and-g10-g11-boundaries.md +++ b/.claude/board/entries/2026-11-02-d-tgl-release-fence-and-g10-g11-boundaries.md @@ -28,3 +28,15 @@ - **The smallest common falsifier set is L1–L6.** Each has its own disable. The decisive one is L4: without the watermark, a second recovery ticks `RouteVersion` while the Codegraph stays unchanged. OPEN: `check_route`, the lab `WalSink`, and the L1–L6 test are three separate reviewable steps (plan §4). None is implemented. + +## Step 1 done: `BindStore::check_route` (2026-11-02) +- **Code:** + - `BindStore::check_route(&self, position, class) -> Result<(), Refusal>` applies `route`'s two refusals and writes nothing. + - `route` runs the same check (`route_identity`, then `RouteLog::check`) before it pushes, so the two cannot disagree. +- **Tests** (`crates/lance-graph-dt64/tests/bind_store.rs`): + - agreement with `route` on 5 cases: 2 accepted, 1 `UnknownClass`, 2 `OutsideReservation` at the 255/256 boundary and beyond; + - writes nothing: version, `routed_at` and `claimed` unchanged; the twin `route` does move them; + - a checked intent stays valid as the store grows: other classes bound, other routes and retracts, then applied without a refusal. +- **Disables (each red on the agreement test):** always accept; skip the reservation; skip the class lookup. + - "Writes nothing" holds by the type (`&self`), so it has no disable. Its twin shows the comparison can fail. +- **Gates:** dt64 suites green (25 + 29); clippy and fmt clean. dt64 already runs in `rust-test.yml`. diff --git a/.claude/plans/g10-g11-atomic-publication-lab-v1.md b/.claude/plans/g10-g11-atomic-publication-lab-v1.md index 9317c15de..33369ead8 100644 --- a/.claude/plans/g10-g11-atomic-publication-lab-v1.md +++ b/.claude/plans/g10-g11-atomic-publication-lab-v1.md @@ -78,7 +78,7 @@ The in-memory publication is one infallible step (Codegraph `commit_refresh` + t ## 4. What has to exist before the lab runs (each its own reviewable step) -1. `BindStore::check_route` (read-only; the two refusals of `route`, no write). Its own falsifiers: +1. **DONE (2026-11-02):** `BindStore::check_route` (read-only; the two refusals of `route`, no write; `route` runs the same check). Its own falsifiers: - it agrees with `route` on accept and refuse for every case; - it leaves `current_version()` unchanged. 2. A lab-only test `WalSink` beside the Codegraph test (dev-dependency on `lance-graph-planner`; test `WalSink` impls exist in ten places, for example `lance-graph-planner/examples/blw_fusion.rs` `MemWal`). No production edge from Codegraph to the planner. From 7b58c93dadbe5c58d001eab4335ffdd440241c16 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 14:52:40 +0000 Subject: [PATCH 3/7] board: post-merge record for #1924; carry check_route made after its merge point Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- .claude/board/LATEST_STATE.md | 5 +++++ .claude/board/PR_ARC_INVENTORY.md | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 83cb4cc88..cf074d64a 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,8 @@ +## 2026-11-02 — main after #1924 (merged `78f9996af`) + +- Codegraph refresh splits into a fenced prepare / commit (crate-private); the F-LAND / F-DISCARD lab tests pin it, in debug and release. +- Board: the D-TGL lifecycle contract, the G8 bag contract, production gates G9–G12 + EXO-148 (open), and the G10 × G11 lab spec D-TGL-PUB. + ## 2026-11-02 — main after #1922 and #1923 (merged `2e19a5adc`) - `lance-graph-codegraph` opens stores of any size on a multi-tile xlog cube. diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 1a3bd5ccd..f1d8b79a7 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -1,3 +1,17 @@ +## 2026-11-02 — lance-graph PR #1924 (merged `78f9996af`, branch `claude/clone-repositories-71a5sw`) — F-LAND / F-DISCARD lab, D-TGL lifecycle contract, G10 × G11 lab spec + +- **Added:** + - Codegraph `prepare_refresh` / `commit_refresh` with a generation fence (`RefreshRefusal::Stale`), crate-private. + - Lab tests F-LAND (second branch recomputed against the current W0 == clean open, call sites included) and F-DISCARD. + - The post-merge record for #1923. +- **Locked:** a preparation made against an older graph generation is refused before any change. Verified in release, where the stale change would otherwise be accepted silently. +- **Docs:** + - `thought-git-lifecycle-contract-v1.md` (D-TGL: GitHub-analogue lifecycle, G8 bag contract, gates G9–G12 + EXO-148); + - `g10-g11-atomic-publication-lab-v1.md` (D-TGL-PUB: boundaries B1–B5, falsifiers L1–L6). +- **Deferred:** the lab `WalSink`, the L1–L6 test, all production gates. +- **Confidence:** 7 lib tests in debug and release; 2 disables red in debug, 1 in release. +- **Stranded:** `check_route` (two commits after the merge point), carried by the next PR. + ## 2026-11-02 — lance-graph PR #1923 (merged `2e19a5adc`, branch `claude/clone-repositories-71a5sw`) — D-CODEGRAPH-0 wide closure, atomic refresh, versioned DR index (D-DYNBAG-4) - **Added:** From 6a689fbaf134eac39840fb040cb1f1f7b9a0f051 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 14:54:08 +0000 Subject: [PATCH 4/7] codegraph: commit drops a closure cache filled between prepare and commit (codex P2 on #1924) A read between prepare_refresh and commit_refresh (impact, closure take &self) can fill a closure cache the preparation did not see. It was built from the pre-commit edges, and commit kept it, so impact answered from the old graph. When the batch changes edges, commit now drops every cache it has no prepared refresh for. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-codegraph/src/graph.rs | 53 +++++++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/crates/lance-graph-codegraph/src/graph.rs b/crates/lance-graph-codegraph/src/graph.rs index 41c7ab1c0..65738cb45 100644 --- a/crates/lance-graph-codegraph/src/graph.rs +++ b/crates/lance-graph-codegraph/src/graph.rs @@ -838,14 +838,27 @@ impl Codegraph { } None => self.cube.frame().clone(), }; + let changed = !staged.is_empty(); self.cube.commit(&self.table, &frame, staged); for (slot, cell) in [(0usize, &mut self.rconf), (1, &mut self.rposs)] { - if let Some(next) = closures[slot].take() { - cell.take(); - if let Some((rows, st)) = next { + match closures[slot].take() { + // Prepared against the cache that was there: replace it. + Some(Some((rows, st))) => { + cell.take(); stats.rows_recomputed += st.rows_recomputed; let _ = cell.set((rows, st)); } + // The cache was there but could not be refreshed: drop it. + Some(None) => { + cell.take(); + } + // No cache at preparation. A read between prepare and commit + // may have filled one (`closure` / `impact` take `&self`) from + // the pre-commit edges; the edges change now, so it is stale. + None if changed => { + cell.take(); + } + None => {} } } @@ -1439,6 +1452,40 @@ mod lifecycle { ); } + /// Codex P2 on #1924. A closure cache filled by a read between prepare and + /// commit (`impact` takes `&self`) was built from the pre-commit edges. + /// Red if commit keeps it (disable: drop the `None if changed` arm in + /// `commit_refresh`): `impact` would then answer from the old graph. + #[test] + fn a_closure_cached_between_prepare_and_commit_is_not_served_stale() { + let w0 = store("w0", &m_v1(), &n_v1()); + let branch_a = store("a", &m_v2(), &n_v1()); + let mut g = Codegraph::open(&w0).expect("open W0"); + assert!(g.rconf.get().is_none() && g.rposs.get().is_none()); + let a = g + .prepare_refresh(&branch_a, "m") + .expect("A") + .expect("A changes"); + let b = g.pos("m::b").expect("m::b"); + let before = g.impact(b).expect("impact"); + // Anti-vacuity: the read did fill a cache the preparation did not see. + assert!(g.rconf.get().is_some() || g.rposs.get().is_some()); + g.commit_refresh(a).expect("commit"); + let after = g.impact(b).expect("impact"); + let x = g.pos("m::x").expect("m::x"); + let has = |m: &[u64]| { + m.get(x as usize / 64) + .is_some_and(|w| w >> (x % 64) & 1 == 1) + }; + assert!(!has(&before.confirmed), "m::x did not exist before"); + assert!( + has(&after.confirmed), + "impact of m::b must include its new caller m::x" + ); + let mut clean = Codegraph::open(&branch_a).expect("clean"); + assert_eq!(answers(&mut g), answers(&mut clean)); + } + /// F-DISCARD. A prepared branch that is dropped leaves W0 exactly as it /// was: interner, positions, `last_owner`, lanes, digests, the map /// table's version and history, the cube's frame and logs, masks, From f473448a18655ef2cc9cf87a7e10159b2353bcae Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 14:54:28 +0000 Subject: [PATCH 5/7] board: record the codex P2 fix (stale closure cache between prepare and commit) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- .../2026-11-02-p4-operator-decisions-lifecycle-lab.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.claude/board/entries/2026-11-02-p4-operator-decisions-lifecycle-lab.md b/.claude/board/entries/2026-11-02-p4-operator-decisions-lifecycle-lab.md index 3cfe5fd23..00be9d872 100644 --- a/.claude/board/entries/2026-11-02-p4-operator-decisions-lifecycle-lab.md +++ b/.claude/board/entries/2026-11-02-p4-operator-decisions-lifecycle-lab.md @@ -36,3 +36,10 @@ - The Codegraph has no `BindStore`, so F-DISCARD's "global `RouteVersion` unchanged" holds by construction. DT64 routing staging is open (G10/G11: no prepare/validate route API on `BindStore`). - `PreparedRefresh` / `commit_refresh` are crate-private. Nothing publishes, persists or routes. - Lab results do not satisfy G9–G12 or EXO-148. + +## Correction (codex P2 on #1924, fixed in #1925) +- **Defect:** a read between `prepare_refresh` and `commit_refresh` (`impact` and `closure` take `&self`) could fill a closure cache the preparation had not seen. That cache was built from the pre-commit edges. `commit_refresh` kept it, because the prepared plan said "no cache", and the generation fence does not move on a read. `impact` then answered from the old graph: `m::x` was missing as a caller of `m::b`. +- **Fix:** when the batch changes edges, `commit_refresh` drops every closure cache it has no prepared refresh for. +- **Test:** `lifecycle::a_closure_cached_between_prepare_and_commit_is_not_served_stale`, which compares the result with a clean open. +- **Disable:** dropping the new arm turns it red in debug and release. +- **Why the lab tests missed it:** they never read between prepare and commit. Interleaved reads are part of the lifecycle (a PR is read while it waits), so the fixture now covers that case. From 6684c9b22d521810818fee04965656c51b56a270 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 14:58:04 +0000 Subject: [PATCH 6/7] codegraph: lab WalSink for the G10 x G11 publication lab (D-TGL-PUB step 2) An in-memory WalSink, test-only, that honours the production writer contract: fence against the head (Fenced, nothing written), one version per commit, reconcile a re-submitted (cycle, batch_hash), HashConflict for another batch of a durable cycle, payload-free scan_sealed. Lab seams: injected faults (I/O before publish, Ambiguous after publish) and a read of the durable image, where recovery must find land records. lance-graph-planner is a dev-dependency only. The crate's own workspace patches the ndarray git coordinate to the sibling checkout, as the root workspace does, so the graph keeps one ndarray package. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-codegraph/Cargo.toml | 15 + crates/lance-graph-codegraph/src/lab_wal.rs | 293 ++++++++++++++++++++ crates/lance-graph-codegraph/src/lib.rs | 2 + 3 files changed, 310 insertions(+) create mode 100644 crates/lance-graph-codegraph/src/lab_wal.rs diff --git a/crates/lance-graph-codegraph/Cargo.toml b/crates/lance-graph-codegraph/Cargo.toml index a6d9d01e1..f70dc454e 100644 --- a/crates/lance-graph-codegraph/Cargo.toml +++ b/crates/lance-graph-codegraph/Cargo.toml @@ -14,3 +14,18 @@ lance-graph-dt64 = { path = "../lance-graph-dt64" } lance-graph-quack = { path = "../lance-graph-quack" } lance-graph-mask-risc = { path = "../lance-graph-mask-risc" } ndarray = { git = "https://github.com/AdaWorldAPI/ndarray", branch = "master", default-features = false, features = ["std"] } + +[dev-dependencies] +# Lab only (D-TGL-PUB): the durable writer contract (`WalSink`, `CommitOutcome`, +# `CommitError`, `DetachedCycleBatch`) for the G10 x G11 publication lab. Never +# a production edge from the codegraph to the planner. +lance-graph-planner = { path = "../lance-graph-planner" } +lance-graph-contract = { path = "../lance-graph-contract" } +tokio = { version = "1", features = ["rt"] } + +# One ndarray (CARGO-COMPUTE-SUBSTRATE rule 9): the planner names the sibling +# checkout by path, this crate and its xlog/mask-risc deps by the git +# coordinate. Point the coordinate at the same checkout, as the lance-graph +# root workspace does, so the graph holds one ndarray package. +[patch."https://github.com/AdaWorldAPI/ndarray"] +ndarray = { path = "../../../ndarray" } diff --git a/crates/lance-graph-codegraph/src/lab_wal.rs b/crates/lance-graph-codegraph/src/lab_wal.rs new file mode 100644 index 000000000..7a945a38d --- /dev/null +++ b/crates/lance-graph-codegraph/src/lab_wal.rs @@ -0,0 +1,293 @@ +//! Lab only (D-TGL-PUB step 2): an in-memory [`WalSink`] for the G10 × G11 +//! publication lab. It honours the production writer contract +//! (`lance-graph-planner/src/persist_sink.rs`) and adds two lab seams: injected +//! faults, and a read of the durable image (`scan_sealed` returns landings +//! without payload, so recovery must read land records from the image). +//! +//! Not a store: it holds what the test wrote, in memory, for one test. + +use std::collections::BTreeMap; + +use lance_graph_contract::scheduler::DatasetVersion; +use lance_graph_planner::persist_sink::{ + CommitError, CommitOutcome, CycleId, DetachedCycleBatch, FrameMeta, LandedSlot, SweepSlot, + WalSink, WriteFailed, +}; + +/// A fault the next `commit_cycle` injects. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Fault { + /// I/O failed before anything was published. + IoBeforePublish, + /// The batch was published, then the acknowledgement was lost. + AmbiguousAfterPublish, +} + +struct Sealed { + frame_base: DatasetVersion, + cycle: CycleId, + batch_hash: u64, + landings: Vec, + image: BTreeMap>, +} + +/// The lab writer. One logical writer (`&mut self` on commit), like the real one. +pub(crate) struct LabWal { + head: DatasetVersion, + sealed: Vec, + fault: Option, + /// Durable appends performed (a reconcile performs none). + pub(crate) appends: u64, +} + +impl LabWal { + pub(crate) fn new(head: DatasetVersion) -> Self { + Self { + head, + sealed: Vec::new(), + fault: None, + appends: 0, + } + } + + pub(crate) fn head(&self) -> DatasetVersion { + self.head + } + + /// Inject a fault into the next `commit_cycle` only. + pub(crate) fn inject(&mut self, fault: Fault) { + self.fault = Some(fault); + } + + /// The durable image of `cycle` (row → payload): where recovery reads + /// land records, because `scan_sealed` carries no payload. + pub(crate) fn image(&self, cycle: CycleId) -> Option<&BTreeMap>> { + self.sealed + .iter() + .find(|s| s.cycle == cycle) + .map(|s| &s.image) + } +} + +impl WalSink for LabWal { + async fn commit_cycle( + &mut self, + batch: DetachedCycleBatch, + ) -> Result { + let fault = self.fault.take(); + if fault == Some(Fault::IoBeforePublish) { + return Err(CommitError::Io(WriteFailed("lab: injected I/O".into()))); + } + // Reconciliation first: an already durable (cycle, hash) is success, a + // durable cycle with another hash fails closed. + if let Some(rec) = self.sealed.iter().find(|s| s.cycle == batch.frame.cycle) { + return if rec.batch_hash == batch.batch_hash { + Ok(CommitOutcome::Reconciled { + current_head: self.head, + cycle: batch.frame.cycle, + batch_hash: batch.batch_hash, + }) + } else { + Err(CommitError::HashConflict { + cycle: batch.frame.cycle, + stored_hash: rec.batch_hash, + offered_hash: batch.batch_hash, + }) + }; + } + // The fence: the batch must target the current head. + if batch.frame.base_version != self.head { + return Err(CommitError::Fenced { + current_head: self.head, + }); + } + self.appends += 1; + self.head = DatasetVersion(self.head.0 + 1); + let (cycle, batch_hash) = (batch.frame.cycle, batch.batch_hash); + self.sealed.push(Sealed { + frame_base: batch.frame.base_version, + cycle, + batch_hash, + landings: batch.landings, + image: batch.image, + }); + if fault == Some(Fault::AmbiguousAfterPublish) { + return Err(CommitError::Ambiguous { + cycle, + batch_hash, + cause: "lab: acknowledgement lost after publish".into(), + }); + } + Ok(CommitOutcome::Committed { + version: self.head, + cycle, + batch_hash, + }) + } + + async fn scan_sealed( + &self, + after_cycle: Option, + ) -> Result, WriteFailed> { + // Stored order, payload-free: the production projection, not better. + Ok(self + .sealed + .iter() + .filter(|s| after_cycle.is_none_or(|c| s.cycle > c)) + .flat_map(|s| { + s.landings.iter().map(|slot| LandedSlot { + cycle: s.cycle, + slot: SweepSlot { + payload: Vec::new(), + ..slot.clone() + }, + }) + }) + .collect()) + } + + async fn timeline(&self) -> Result, WriteFailed> { + Ok(self + .sealed + .iter() + .map(|s| FrameMeta { + cycle: s.cycle, + base_version: s.frame_base, + batch_hash: s.batch_hash, + }) + .collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use lance_graph_planner::persist_sink::CycleFrame; + + fn run(f: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .build() + .expect("runtime") + .block_on(f) + } + + fn batch(cycle: u64, base: u64, payload: &[u8]) -> DetachedCycleBatch { + DetachedCycleBatch::freeze( + CycleFrame::new(CycleId(cycle), DatasetVersion(base)), + vec![SweepSlot { + cycle: CycleId(cycle), + stream_position: cycle, + owner: 1, + row: 7, + paired_move: None, + payload: payload.to_vec(), + }], + ) + } + + /// Everything the writer exposes, for "nothing was written" checks. + fn state(w: &LabWal) -> (DatasetVersion, u64, usize, usize) { + let t = run(w.timeline()).expect("timeline").len(); + let s = run(w.scan_sealed(None)).expect("scan").len(); + (w.head(), w.appends, t, s) + } + + /// A batch against a stale base is `Fenced` and writes nothing. Red if + /// the fence is skipped (disable: drop the `base_version != head` check). + #[test] + fn a_stale_base_is_fenced_and_writes_nothing() { + let mut w = LabWal::new(DatasetVersion(5)); + let before = state(&w); + let r = run(w.commit_cycle(batch(1, 4, b"land"))); + assert_eq!( + r, + Err(CommitError::Fenced { + current_head: DatasetVersion(5) + }) + ); + assert_eq!(state(&w), before); + } + + /// A commit against the head publishes exactly one version. `scan_sealed` + /// carries no payload; the image does. Red if scan leaks the payload + /// (disable: return the stored slot unchanged), which would let a + /// recovery read from the wrong place and pass in the lab only. + #[test] + fn a_commit_publishes_one_version_and_the_payload_lives_in_the_image() { + let mut w = LabWal::new(DatasetVersion(5)); + let b = batch(1, 5, b"land"); + let hash = b.batch_hash; + let r = run(w.commit_cycle(b)); + assert_eq!( + r, + Ok(CommitOutcome::Committed { + version: DatasetVersion(6), + cycle: CycleId(1), + batch_hash: hash + }) + ); + let scanned = run(w.scan_sealed(None)).expect("scan"); + assert_eq!(scanned.len(), 1); + assert!( + scanned[0].slot.payload.is_empty(), + "scan carries no payload" + ); + assert_eq!( + w.image(CycleId(1)) + .and_then(|i| i.get(&7)) + .map(Vec::as_slice), + Some(&b"land"[..]) + ); + } + + /// Re-submitting the same frozen batch reconciles with no second append; + /// another batch for a durable cycle fails closed. Red if the + /// reconciliation lookup is skipped (disable): the re-submit is then + /// `Fenced` (head moved) and a retry after a lost ack cannot confirm. + #[test] + fn a_resubmit_reconciles_and_a_different_batch_conflicts() { + let mut w = LabWal::new(DatasetVersion(5)); + let b = batch(1, 5, b"land"); + run(w.commit_cycle(b.clone())).expect("commit"); + let after = state(&w); + assert!(matches!( + run(w.commit_cycle(b)), + Ok(CommitOutcome::Reconciled { .. }) + )); + assert_eq!(state(&w), after, "no second append"); + assert!(matches!( + run(w.commit_cycle(batch(1, 5, b"other"))), + Err(CommitError::HashConflict { .. }) + )); + assert_eq!(state(&w), after); + } + + /// B3: the acknowledgement is lost after publication. The batch IS + /// durable; re-submitting it reconciles. An injected I/O fault before + /// publication writes nothing. Twin pair: the two faults must leave + /// different durable states, or the injection is not doing anything. + #[test] + fn faults_before_and_after_publish_leave_different_durable_states() { + let mut w = LabWal::new(DatasetVersion(5)); + let before = state(&w); + w.inject(Fault::IoBeforePublish); + assert!(matches!( + run(w.commit_cycle(batch(1, 5, b"land"))), + Err(CommitError::Io(_)) + )); + assert_eq!(state(&w), before, "I/O before publish writes nothing"); + + let b = batch(1, 5, b"land"); + w.inject(Fault::AmbiguousAfterPublish); + assert!(matches!( + run(w.commit_cycle(b.clone())), + Err(CommitError::Ambiguous { .. }) + )); + assert_ne!(state(&w), before, "published despite the lost ack"); + assert!(matches!( + run(w.commit_cycle(b)), + Ok(CommitOutcome::Reconciled { .. }) + )); + assert_eq!(w.appends, 1); + } +} diff --git a/crates/lance-graph-codegraph/src/lib.rs b/crates/lance-graph-codegraph/src/lib.rs index bcab215cb..1c20cd76f 100644 --- a/crates/lance-graph-codegraph/src/lib.rs +++ b/crates/lance-graph-codegraph/src/lib.rs @@ -6,6 +6,8 @@ pub mod artifact; pub mod diag; pub mod graph; +#[cfg(test)] +mod lab_wal; pub mod query; pub use graph::{Codegraph, IntakeRefusal, Pos, RefreshRefusal, RefreshStats, RunStatus}; From 03481f98b04659a4b1a20ad0a2fd5b19af756159 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 14:59:07 +0000 Subject: [PATCH 7/7] board: D-TGL-PUB step 2 (lab WalSink) shipped Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- .claude/board/STATUS_BOARD.md | 2 +- ...gl-release-fence-and-g10-g11-boundaries.md | 22 +++++++++++++++++++ .../g10-g11-atomic-publication-lab-v1.md | 2 +- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 6f368e96c..5b5717066 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -249,7 +249,7 @@ Plan: [d-codegraph-0-resident-codegraph-v1.md](../plans/d-codegraph-0-resident-c | **D-TGL-G10** | atomic publication: prepare all, durable commit first, then infallible in-memory apply from the land record, replay after crash | OPEN (production); named gap: no prepare/validate route API on `BindStore` | a `Fenced` leaves no published partial state | | **D-TGL-G11** | branch-local routing; no branch-minted DR position as durable routing key | OPEN (production) | a branch route leaves `BindStore::current_version()` unchanged until land | | **D-TGL-G12** | identities apart: W-slot, generation, ThoughtCommitId, DurableRef, RouteVersion, Lance version, source SHA (provenance only) | OPEN (production) | joins only through receipt / CommitWitness / MetaKey / land record | -| **D-TGL-PUB** | G10 × G11 lab: prepared land (Codegraph + DT64 route intent), fenced durable commit through a test `WalSink`, crash recovery from the durable record, branch-local routing | SPEC: plan `g10-g11-atomic-publication-lab-v1.md`; boundaries B1–B5 located. Step 1 SHIPPED (lab-ready): `BindStore::check_route` (read-only; `route` runs the same `RouteLog::check`), 3 tests, 3 disables red. Steps 2 (lab `WalSink`) and 3 (L1–L6) open | L1–L6, each with a disable; L4 shows route re-application ticks `RouteVersion` without a watermark | +| **D-TGL-PUB** | G10 × G11 lab: prepared land (Codegraph + DT64 route intent), fenced durable commit through a test `WalSink`, crash recovery from the durable record, branch-local routing | SPEC: plan `g10-g11-atomic-publication-lab-v1.md`; boundaries B1–B5 located. Step 1 SHIPPED (lab-ready): `BindStore::check_route` (read-only; `route` runs the same `RouteLog::check`), 3 tests, 3 disables red. Step 2 SHIPPED (lab): `LabWal` in `lance-graph-codegraph/src/lab_wal.rs` (test-only `WalSink`: fence, one version per commit, reconcile, `HashConflict`, payload-free scan, image read, injected I/O / Ambiguous faults), 4 tests, 4 disables red; planner is a dev-dependency, one ndarray via `[patch]`. Step 3 (L1–L6) open | L1–L6, each with a disable; L4 shows route re-application ticks `RouteVersion` without a watermark | | **D-TGL-EXO148** | `DurableRef` + never-reused `ThoughtCommitId` before any durable land, replay or PR record | OPEN (production, EXO-148) | reuse test (D-RST-1) | | **D-DT64-AUTH** | DT64 / F### as the one method authority: census of production words with a two-way exception ratchet; parallel identities (`MethodSignature`, tract `Registry`, XLOG `Word`, ogar-loco `FnIndex`) stay as listed `LAB` adapters | QUEUED: plan `d-dt64-method-authority-census-v1.md` (operator decision 2026-11-02); egglog pinned to one commit in `egglog-r2il-probe`, `dr-math-binding-probe`, `xlog-relcomp-probe` | undeclared production word → red; stale exception → red; current tree green | | **D-CODEGRAPH-0-D** | local stdio MCP over C | QUEUED | only after B and C gates | diff --git a/.claude/board/entries/2026-11-02-d-tgl-release-fence-and-g10-g11-boundaries.md b/.claude/board/entries/2026-11-02-d-tgl-release-fence-and-g10-g11-boundaries.md index 8c685fcc4..3c3d22aa8 100644 --- a/.claude/board/entries/2026-11-02-d-tgl-release-fence-and-g10-g11-boundaries.md +++ b/.claude/board/entries/2026-11-02-d-tgl-release-fence-and-g10-g11-boundaries.md @@ -40,3 +40,25 @@ OPEN: `check_route`, the lab `WalSink`, and the L1–L6 test are three separate - **Disables (each red on the agreement test):** always accept; skip the reservation; skip the class lookup. - "Writes nothing" holds by the type (`&self`), so it has no disable. Its twin shows the comparison can fail. - **Gates:** dt64 suites green (25 + 29); clippy and fmt clean. dt64 already runs in `rust-test.yml`. + +## Step 2 done: the lab `WalSink` (2026-11-02) +- **Code:** `LabWal` in `crates/lance-graph-codegraph/src/lab_wal.rs`, `#[cfg(test)]` only. It is an in-memory `WalSink` that follows the production contract: + - `Fenced` against the head, with nothing written; + - one version per commit; + - reconciliation of a re-submitted `(cycle, batch_hash)`; + - `HashConflict` for another batch of a durable cycle; + - a payload-free `scan_sealed`. +- **Lab seams:** + - injected faults: I/O before publish, and `Ambiguous` after publish (B3); + - `image(cycle)`, the durable payload read that recovery must use. +- **Dependencies:** + - `lance-graph-planner`, `lance-graph-contract` and `tokio` (rt) are dev-dependencies only; `cargo tree -e normal` shows no planner. + - The crate's own workspace patches the ndarray git coordinate to the sibling checkout, as the root workspace does. `cargo tree -i ndarray` shows one package; `-d` shows no duplicate. + - CI's `test` job already checks out ndarray as a sibling. +- **Tests:** + - stale base fenced, nothing written; + - a commit publishes one version, and the payload lives only in the image; + - a re-submit reconciles, a different batch conflicts; + - a fault before publish and one after publish leave different durable states. +- **Disables (each red on its own test):** drop the fence; leak the payload through `scan`; skip the reconcile lookup; make `Ambiguous` publish nothing. +- **Gates:** codegraph suites green (12 lib tests); clippy and fmt clean. diff --git a/.claude/plans/g10-g11-atomic-publication-lab-v1.md b/.claude/plans/g10-g11-atomic-publication-lab-v1.md index 33369ead8..39e2c31fe 100644 --- a/.claude/plans/g10-g11-atomic-publication-lab-v1.md +++ b/.claude/plans/g10-g11-atomic-publication-lab-v1.md @@ -81,7 +81,7 @@ The in-memory publication is one infallible step (Codegraph `commit_refresh` + t 1. **DONE (2026-11-02):** `BindStore::check_route` (read-only; the two refusals of `route`, no write; `route` runs the same check). Its own falsifiers: - it agrees with `route` on accept and refuse for every case; - it leaves `current_version()` unchanged. -2. A lab-only test `WalSink` beside the Codegraph test (dev-dependency on `lance-graph-planner`; test `WalSink` impls exist in ten places, for example `lance-graph-planner/examples/blw_fusion.rs` `MemWal`). No production edge from Codegraph to the planner. +2. **DONE (2026-11-02):** a lab-only test `WalSink` (`LabWal`, `crates/lance-graph-codegraph/src/lab_wal.rs`) beside the Codegraph test (dev-dependency on `lance-graph-planner`; test `WalSink` impls exist in ten places, for example `lance-graph-planner/examples/blw_fusion.rs` `MemWal`). No production edge from Codegraph to the planner. 3. The lab test itself, in-crate, because `prepare_refresh` / `commit_refresh` are crate-private. Out of scope: `LanceCycleWriter` (needs the 512-byte ABI and real Lance), `ThoughtCommitId`/`DurableRef` (EXO-148), MergeQueue, review gates.