diff --git a/crates/omnigraph-dst/README.md b/crates/omnigraph-dst/README.md index 81226a193..0ad735cfa 100644 --- a/crates/omnigraph-dst/README.md +++ b/crates/omnigraph-dst/README.md @@ -133,7 +133,9 @@ and `tests/integration/lane_b.rs`. exercises the same constructors as the hunt and fleet, including the hunt's probe pass. Returned table errors are attributed; native-fork reconciliation failures reported only through tracing are outside this - deferral channel. + deferral channel. Windows no universe can cross are named with their + reasons in `harness::REACH_EXPECTED_DARK`: the reach census asserts that + list, and the hunt and the fleet do not schedule its windows. - **Injected storage faults**: `Scenario::faults` installs a seeded `FaultPlan` on the adapter and optionally Lance. Latency uses virtual time. Lance supports errors, latency and lost acknowledgements; adapter text corruption and diff --git a/crates/omnigraph-dst/src/catalog.rs b/crates/omnigraph-dst/src/catalog.rs index 3cde7a7aa..1adb60df3 100644 --- a/crates/omnigraph-dst/src/catalog.rs +++ b/crates/omnigraph-dst/src/catalog.rs @@ -69,6 +69,21 @@ pub const CRASH_WINDOWS: [&str; 53] = [ mod tests { use super::CRASH_WINDOWS; + /// Every window the reach census expects dark is a catalog window, so a + /// renamed or removed window cannot linger in that list. + #[test] + fn expected_dark_windows_are_catalog_windows() { + let stale: Vec<&str> = crate::harness::REACH_EXPECTED_DARK + .iter() + .map(|(window, _)| *window) + .filter(|window| !CRASH_WINDOWS.contains(window)) + .collect(); + assert!( + stale.is_empty(), + "REACH_EXPECTED_DARK names windows the catalog does not hold: {stale:?}" + ); + } + /// The catalog's names-guard (module doc): every entry must be a seam /// the engine catalog declares. #[test] diff --git a/crates/omnigraph-dst/src/harness.rs b/crates/omnigraph-dst/src/harness.rs index b00cd2759..109c1df1c 100644 --- a/crates/omnigraph-dst/src/harness.rs +++ b/crates/omnigraph-dst/src/harness.rs @@ -791,10 +791,8 @@ enum Milestone { /// Delete a base-fixture person on the milestone branch — puts a DELETE /// in the branch's merge delta (the rewrite-with-deletes merge route). DeleteFixtureOnBranch, - /// `ensure_indices` on main (the non-fork ensure_indices windows). + /// `ensure_indices` on main. EnsureIndicesMain, - /// `ensure_indices` on the milestone branch after a data op there. - EnsureIndicesBranch, /// `cleanup` on main (branch-snapshot resolution / fork reconciliation / /// table GC run against whatever state earlier milestones built). CleanupMain, @@ -829,13 +827,12 @@ fn milestone_steps(window: &str) -> Vec { } // Diverge main after the branch insert so this reaches the general // rewrite publisher, whose insert commit precedes the update phase. - // The hook also fires when that phase has no update payload. - "branch_merge.rewrite_after_insert_pre_update" => { + // The hook also fires when that phase has no update payload. A merge + // into an unchanged main only switches pins, so the rewrite is also + // the route that stages a table effect. + "branch_merge.rewrite_after_insert_pre_update" | "branch_merge.post_table_effect" => { return vec![EnsureBranch, DataOnBranch, MutateMain, MergeBranch]; } - "ensure_indices.post_table_effect" => { - return vec![EnsureBranch, DataOnBranch, EnsureIndicesBranch]; - } // cleanup with state to work on: a live branch fork. "cleanup.resolve_branch_snapshot" => { return vec![EnsureBranch, DataOnBranch, CleanupMain]; @@ -844,7 +841,7 @@ fn milestone_steps(window: &str) -> Vec { return vec![MutateMain, DeleteFixtureOnMain, CleanupMain]; } // A dead staging for cleanup to reclaim: `setup_crash_for`'s crash - // stops the index build at its pre-manifest window, stranding a + // stops the index build after its first table effect, stranding a // detached commit whose witness records main's head; the second // insert moves that head on, so the next cleanup judges the staging // dead and deletes its files before its manifest. @@ -862,6 +859,10 @@ fn milestone_steps(window: &str) -> Vec { "load" if window == "load.post_branch_create_pre_stage" => vec![ForkLoad], "load" => vec![DataOnBranch], // LoadMerge/Append on main "mutation" | "graph_publish" | "publish" | "classify" => vec![MutateMain], + // Index maintenance builds only missing indexes, and so does an + // Optimize; the sampler holds Optimize back while a recipe still + // owes its index build, so an EnsureIndices is the first index build + // and has the fixture's unbuilt indexes to build. "ensure_indices" => vec![MutateMain, EnsureIndicesMain], "cleanup" => vec![MutateMain, CleanupMain], "optimize" => vec![MutateMain], @@ -870,19 +871,75 @@ fn milestone_steps(window: &str) -> Vec { } /// The setup crash a window needs (`Scenario::setup_crash`), shared by the -/// reach census, the hunt and the fleet. +/// reach census, the hunt and the fleet. The setup window must be crossed +/// only once its aftermath exists, since a recorded setup crash arms the +/// target. /// - the window before the dead stagings' manifests: an index build crashed -/// before its manifest commit leaves the unpublished staging that the -/// window's recipe then makes dead and reclaims. +/// after its first table effect leaves the unpublished staging that the +/// window's recipe then makes dead and reclaims. Its pre-manifest window +/// would not do: an index build with nothing to build crosses that too. pub fn setup_crash_for(window: &str) -> Option<(&'static str, usize)> { match window { - "cleanup.sweep_pre_dead_manifest_delete" => { - Some(("ensure_indices.post_phase_b_pre_manifest_commit", 0)) - } + "cleanup.sweep_pre_dead_manifest_delete" => Some(("ensure_indices.post_table_effect", 0)), _ => None, } } +/// The windows no seeded universe can cross, each with its reason. The +/// reach census fails, naming the window, when another window stays dark or +/// a listed one crosses; the hunt and the fleet do not schedule these. +pub const REACH_EXPECTED_DARK: &[(&str, &str)] = &[ + ( + "init.after_coordinator_init", + "graph birth precedes the probe; dst_birth_contract_sweep owns it", + ), + ( + "init.post_manifest_create", + "graph birth precedes the probe; dst_birth_contract_sweep owns it", + ), + ( + "blob_read.post_capture", + "the DST schema has no Blob property", + ), + ( + "mutation.post_no_effect_pre_gate", + "crossed only by a no-effect mutation that carries an expected graph head; \ + the workload sends none", + ), + ( + "branch_merge.between_delete_chunks", + "needs a merge that deletes more than one chunk of rows", + ), + ( + "classify.fresh_read", + "crossed only for a table with native fork trees, a table fork from before format v11", + ), + ( + "cleanup.resolve_branch_snapshot", + "crossed only for a table with native fork trees, a table fork from before format v11", + ), + ( + "branch_merge.adopt_after_append_pre_upsert", + "a merge adopts a delta only from a forked source table (adopt_advances_head)", + ), + ( + "branch_merge.adopt_after_upsert_pre_delete", + "a merge adopts a delta only from a forked source table (adopt_advances_head)", + ), + ( + "branch_merge.adopt_between_insert_chunks", + "a merge adopts a delta only from a forked source table (adopt_advances_head)", + ), +]; + +/// Why `window` is in [`REACH_EXPECTED_DARK`], if it is. +pub fn expected_dark_reason(window: &str) -> Option<&'static str> { + REACH_EXPECTED_DARK + .iter() + .find(|(dark, _)| *dark == window) + .map(|(_, reason)| *reason) +} + /// Emit the next pending milestone op if it is due (seeded coin, or forced /// when the remaining op budget is tight), else `None` to fall through to /// ordinary sampling. `progress` is advanced as steps are emitted. @@ -1031,15 +1088,6 @@ fn milestone_op( branch: "main".to_string(), op: Op::EnsureIndices, }, - Milestone::EnsureIndicesBranch => { - if !world.branches.contains_key(MILESTONE_BRANCH) { - return None; // wait for EnsureBranch - } - WorldOp::Data { - branch: MILESTONE_BRANCH.to_string(), - op: Op::EnsureIndices, - } - } Milestone::CleanupMain => WorldOp::Data { branch: "main".to_string(), op: Op::Cleanup, @@ -5558,16 +5606,34 @@ impl UniverseScenario for Scenario { &mut next_ver, ) .unwrap_or_else(|| { - sample_world_op( - &mut rng, - &world, - &mut next_ver, - sc.hostile, - sc.wide, - sc.schema_ops, - &mut schema_extras, - &mut fresh_load, - ) + // An Optimize builds missing indexes too: while the recipe + // still owes its index build, a sampled one would leave that + // build nothing to do, so the sampler draws again. + let owes_index_build = + reach_steps[reach_progress..].contains(&Milestone::EnsureIndicesMain); + loop { + let wop = sample_world_op( + &mut rng, + &world, + &mut next_ver, + sc.hostile, + sc.wide, + sc.schema_ops, + &mut schema_extras, + &mut fresh_load, + ); + if !(owes_index_build + && matches!( + wop, + WorldOp::Data { + op: Op::Optimize, + .. + } + )) + { + break wop; + } + } }); if matches!( wop, diff --git a/crates/omnigraph-dst/tests/scenarios.rs b/crates/omnigraph-dst/tests/scenarios.rs index acd14451f..668a2419d 100644 --- a/crates/omnigraph-dst/tests/scenarios.rs +++ b/crates/omnigraph-dst/tests/scenarios.rs @@ -135,9 +135,11 @@ fn dst_harness_crash_recovery_deterministic() { /// 873 universes misses by construction. A window whose precondition is an /// earlier crash's aftermath gets that setup crash first /// (`setup_crash_for`), and a hit is the window's own verdict -/// (`window_hit`), never the setup's. Families the workload has no op -/// for are reported as "unschedulable" so the coverage report says WHY -/// each dark window is dark. Any panic is a find. +/// (`window_hit`), never the setup's. Windows in +/// `harness::REACH_EXPECTED_DARK` are not scheduled; the reach census owns +/// them. Families the workload has no op for are reported as +/// "unschedulable" so the coverage report says WHY each dark window is +/// dark. Any panic is a find. #[cfg(feature = "failpoints")] #[test] #[ignore = "hunt: targeted-scheduling catalog run; run explicitly with -- --ignored"] @@ -151,8 +153,14 @@ fn dst_hunt_crash_window_sweep() { let mut hit: Vec<&str> = Vec::new(); let mut never_reached: Vec<&str> = Vec::new(); let mut unschedulable: Vec<&str> = Vec::new(); + let mut expected_dark: Vec<&str> = Vec::new(); let mut universes = 0usize; for (w, window) in catalog::CRASH_WINDOWS.iter().enumerate() { + // The reach census proves these dark; scheduling them only burns cells. + if omnigraph_dst::harness::expected_dark_reason(window).is_some() { + expected_dark.push(window); + continue; + } if !omnigraph_dst::harness::workload_can_reach(window) { unschedulable.push(window); continue; @@ -213,16 +221,18 @@ fn dst_hunt_crash_window_sweep() { } println!( - "HUNT COMPLETE ({universes} universes): {} hit, {} crossed-but-absorbed, {} never reached, {} unschedulable", + "HUNT COMPLETE ({universes} universes): {} hit, {} crossed-but-absorbed, {} never reached, {} unschedulable, {} expected dark", hit.len(), absorbed.len(), dark.len(), - unschedulable.len() + unschedulable.len(), + expected_dark.len() ); println!("hit: {hit:?}"); println!("crossed-but-absorbed (self-healing verified live): {absorbed:?}"); println!("never reached (workload/path gap): {dark:?}"); println!("unschedulable (need schema/init workload ops): {unschedulable:?}"); + println!("expected dark (harness::REACH_EXPECTED_DARK, not scheduled): {expected_dark:?}"); assert!( !hit.is_empty(), "coverage assertion: the hunt must actually reach SOME windows" @@ -1941,11 +1951,6 @@ fn dst_dead_staging_sweep_window_is_reached_and_retried() { &format!("shared-memory://dst-dead-{label}-stop-a"), &stopped, ); - assert!( - window_hit(&a, window), - "{label} must stop the cleanup at {window}: {:?}", - a.reconcile_verdicts - ); assert!( a.reconcile_verdicts.iter().any(|(tag, verdict, channel)| { tag.starts_with(&format!("deferred:{window}@")) @@ -1997,6 +2002,50 @@ fn dst_setup_crash_waits_for_a_crossed_window() { omnigraph_dst::harness::assert_strict_replay(&a, &b, "uncrossed setup"); } +/// A recorded setup crash arms its target, so the setup window must be one +/// that is crossed only once the aftermath exists. In hunt cell seed 2 an +/// Optimize builds the missing indexes before the first EnsureIndices, which +/// then has nothing to build: it still crosses the index build's +/// pre-manifest window, but not `ensure_indices.post_table_effect`, the +/// setup `setup_crash_for` names, so no setup is recorded and the +/// dead-staging target stays unarmed. +#[cfg(feature = "failpoints")] +#[test] +#[serial] +fn dst_setup_crash_needs_its_aftermath() { + let _s = omnigraph::seams::FailScenario::setup(); + let window = omnigraph::seams::catalog::CLEANUP_SWEEP_PRE_DEAD_MANIFEST_DELETE.name(); + let pre_manifest = Scenario { + setup_crash: Some(( + omnigraph::seams::catalog::ENSURE_INDICES_POST_PHASE_B_PRE_MANIFEST_COMMIT.name(), + 0, + )), + ..hunt_window_scenario(2, window, 0) + }; + let a = run_universe( + "shared-memory://dst-setup-aftermath-pre-manifest", + &pre_manifest, + ); + assert_eq!( + (a.crashes, window_hit(&a, window)), + (1, false), + "the cell's index build has nothing to build yet crosses the pre-manifest \ + window, so a setup there arms the target with no staging: {:?}", + a.reconcile_verdicts + ); + + let named = hunt_window_scenario(2, window, 0); + let a = run_universe("shared-memory://dst-setup-aftermath-a", &named); + assert_eq!( + a.crashes, 0, + "an index build with nothing to build must not record the setup: {:?}", + a.reconcile_verdicts + ); + assert!(!window_hit(&a, window)); + let b = run_universe("shared-memory://dst-setup-aftermath-b", &named); + omnigraph_dst::harness::assert_strict_replay(&a, &b, "setup without its aftermath"); +} + #[cfg(feature = "failpoints")] fn returned_cleanup_error_scenario(keep_handle: bool) -> Scenario { Scenario { @@ -3753,14 +3802,16 @@ fn dst_predict_born_on_both_person_probe() { } /// MILESTONE REACH PROBE: the "one seeded universe per window, -/// can we catch all 66?" experiment. For each catalog window, ONE universe -/// with the window's milestone plan woven into the seeded stream + -/// `crash_on_match` probe_only (record-only crossing). Reports per-window -/// crossed/not and the tally. Success target: every milestone-reachable -/// window crosses in its single universe; the residue (recovery internals, -/// init/open) is expected dark for NAMED reasons. (The schema families -/// left the dark list when the `schema_ops` face requalified them.) -/// cargo test -p omnigraph-dst dst_window_reach_probe -- --ignored --nocapture +/// can we catch them all?" experiment. For each catalog window, up to three +/// seeded universes with the window's milestone plan woven into the stream, +/// its setup crash (`setup_crash_for`) when it has one, and a record-only +/// probe over the whole universe (`probe_window`). Reports per-window +/// crossed/not and the tally. Every window must cross on one of its seeds +/// except those in `harness::REACH_EXPECTED_DARK`; the first cross wins, so +/// a seed that trips an oracle fails the census only when no seed of that +/// window crosses (`errored`), since the census asks whether the window is +/// reachable. +/// cd crates/omnigraph-dst && cargo test --locked --test scenarios dst_window_reach_probe -- --ignored --nocapture #[test] #[serial] #[ignore = "instrument: one-universe-each milestone reach census over the crash-window catalog"] @@ -3816,17 +3867,16 @@ fn dst_window_reach_probe() { "errored (milestone tripped an oracle — separate finding) ({}): {errored:?}", errored.len() ); - // Non-regression floor: the census measured 50/66 on 2026-08-12; the - // then-16 non-crossing windows all carried NAMED reasons (schema - // quarantine — since requalified by the `schema_ops` face, whose - // windows now cross — birth-owned init, #473-blocked adopts, chunk - // thresholds, branch-from-branch first-touch shapes, kill-territory - // orphan discard). Floor 48 leaves margin for benign seed sensitivity; - // a bigger drop = a recipe-mechanism regression. + // Each universe is a pure function of its seed, so a change in reach + // comes from a code change; the message names the window it moved. + let expected_dark = + |window: &&str| omnigraph_dst::harness::expected_dark_reason(window).is_some(); + let newly_dark: Vec<&&str> = dark.iter().filter(|w| !expected_dark(w)).collect(); + let listed_but_crossed: Vec<&&str> = crossed.iter().filter(|w| expected_dark(w)).collect(); assert!( - crossed.len() >= 48, - "milestone reach regressed below the measured 50 ({} crossed)", - crossed.len() + newly_dark.is_empty() && listed_but_crossed.is_empty() && errored.is_empty(), + "milestone reach changed: newly dark {newly_dark:?}, listed dark but crossed \ + {listed_but_crossed:?}, errored {errored:?}" ); } @@ -3980,15 +4030,20 @@ fn dst_fleet() { }, &mut failures, ); - // Crash-window arm: round-robin over the catalog so a 66+-seed - // pass draws every window. `reach_target` weaves the - // window's milestone recipe into the seeded stream — guarantee from - // milestones, diversity from the seed — and skip stays 0 so the - // recipe-built op is the one that carries the crash. Windows the + // Crash-window arm: round-robin over the catalog so a pass with at + // least as many seeds as windows draws every window. `reach_target` + // weaves the window's milestone recipe into the seeded stream — + // guarantee from milestones, diversity from the seed — and skip + // stays 0, so the crash lands on the first matching op; for a window + // with a setup crash (`setup_crash_for`), that is the first matching + // op after the setup crash is recorded. Windows the // workload has no op for are skipped (the hunt's coverage ledger - // owns naming those as unschedulable). + // owns naming those as unschedulable), and so are those in + // `harness::REACH_EXPECTED_DARK`, which no universe can cross. let window = catalog::CRASH_WINDOWS[(seed as usize) % catalog::CRASH_WINDOWS.len()]; - if omnigraph_dst::harness::workload_can_reach(window) { + if omnigraph_dst::harness::workload_can_reach(window) + && omnigraph_dst::harness::expected_dark_reason(window).is_none() + { universes += 1; run( seed,