diff --git a/.claude/board/entries/2026-10-22-probes-pd4-pe2-pd3.md b/.claude/board/entries/2026-10-22-probes-pd4-pe2-pd3.md index 1e2c9e8e1..08d9d8db5 100644 --- a/.claude/board/entries/2026-10-22-probes-pd4-pe2-pd3.md +++ b/.claude/board/entries/2026-10-22-probes-pd4-pe2-pd3.md @@ -1,6 +1,8 @@ # 2026-10-22: probes P-D4, P-E2, P-D3 — measured -**Status:** MEASURED (lab probes; no production code changed). These are the +**Status:** MEASURED (lab probes; no production code changed). **Corrected +the same day** (codex review of #1724): P-D4's post-hoc pass does not hold, so +P-D4 fails as well (see point 2 below). These are the top three probes pre-registered in `2026-10-22-coresearch-pal256-evidence-probes.md`. ## P-E2: count each source once (`crates/causal-edge/tests/p_e2_stamp_gate_probe.rs`) @@ -57,7 +59,7 @@ Results: - the top candidate has ≥ 2 sources; - the support flag is clear. - Brier(always-answer) − Brier(gate) = **0.0242 [0.0079, 0.0409]** at 44% coverage. This clears the 0.02 bar, but it is post-hoc, so it is weaker evidence than a pre-registered pass. + Brier(always-answer) − Brier(gate) = 0.0242 at 44% coverage. ~~[0.0079, 0.0409]~~: that interval held the base rate π fixed, even though π is estimated from the same blocks being resampled. **With π refitted in every draw, the interval is [−0.0062, 0.0552]. It includes 0, so the kill fires.** `post_hoc_gate_gain_is_not_established` pins this. Fixing π on a separate calibration set would be the alternative. 3. **Label-blind control.** The pre-registered shuffled-label Brier control cannot stay silent: replacing p ≠ π with π lowers the Brier score on any labels. It is replaced by within-block selective accuracy: | labels | answered − abstained accuracy | @@ -83,4 +85,6 @@ Results: - The calibration claim needs real held-out labelled outcomes, not a synthetic world. - CI runs `bgz-tensor` with `--lib` only, so P-D3 runs locally, not in CI. -Closeout: `STATUS: measured | OUTCOME: P-E2 and P-D3 pass as registered; P-D4's registered margin is vacuous and its post-hoc gate passes on the margin alone | OPEN: separating world for P-D4, G5, real labels` +The ablation gains in point 4 used the same fixed-π interval, so they also overstate certainty. + +Closeout: `STATUS: measured | OUTCOME: P-E2 and P-D3 pass as registered; P-D4 fails: the registered margin is vacuous, and the post-hoc gate's gain is not established once π's error is included | OPEN: a separating world with a separate calibration set for π, G5, real labels` diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 64396aca6..07f8d22be 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -440 entries, 2026-08-06 .. 2026-10-22. +441 entries, 2026-08-06 .. 2026-10-22. | date | entry id | finding | file | |---|---|---|---| diff --git a/crates/causal-edge/tests/p_d4_decision_gate_probe.rs b/crates/causal-edge/tests/p_d4_decision_gate_probe.rs index c5d758f74..c65ade069 100644 --- a/crates/causal-edge/tests/p_d4_decision_gate_probe.rs +++ b/crates/causal-edge/tests/p_d4_decision_gate_probe.rs @@ -31,6 +31,11 @@ //! cannot clear with 3–9 observations. The rule is killed by vacuity, and //! `preregistered_interval_margin_is_vacuous` pins that. //! +//! CORRECTION (codex review of #1724): π is estimated from the same blocks +//! being resampled, so it is refitted inside every bootstrap draw. With that, +//! the post-hoc gain is 0.0242 [−0.0062, 0.0552]: the interval includes 0 and +//! the kill fires. The earlier [0.0079, 0.0409] held π fixed. +//! //! POST-HOC arm (b′), declared after that run and before any other: the //! margin is on evidence counted once — the top candidate needs at least 2 //! more distinct sources than the runner-up. The other two conditions are @@ -224,6 +229,19 @@ fn base_rate(os: &[Outcome], parity: usize) -> f64 { other.iter().map(|o| o.y_a).sum::() / other.len() as f64 } +/// The same base rate refitted on the drawn blocks (with multiplicity), so a +/// bootstrap interval carries the base rate's own estimation error. +fn base_rate_in(os: &[Outcome], blocks: &[usize], parity: usize) -> f64 { + let (mut s, mut n) = (0.0_f64, 0.0_f64); + for &blk in blocks.iter().filter(|&&b| b % 2 != parity) { + for o in os.iter().filter(|o| o.block == blk) { + s += o.y_a; + n += 1.0; + } + } + s / n.max(1.0) +} + fn brier_diff(os: &[Outcome], blocks: &[usize], pi: [f64; 2]) -> f64 { let (mut a, mut b, mut n) = (0.0, 0.0, 0.0); for &blk in blocks { @@ -312,21 +330,31 @@ fn preregistered_interval_margin_is_vacuous() { } #[test] -fn gate_beats_always_answer_on_brier_with_a_block_interval() { +/// Codex review of #1724: with π refitted inside every bootstrap draw (π is +/// estimated from the same blocks), the interval includes 0. The post-hoc +/// gate therefore does NOT clear the kill condition; this pins that. The +/// earlier fixed-π interval [0.0079, 0.0409] omitted π's estimation error. +fn post_hoc_gate_gain_is_not_established() { let os = evaluate(&world(), Margin::Sources); let coverage = os.iter().filter(|o| o.answered).count() as f64 / os.len() as f64; let acc_a = os.iter().map(|o| o.y_a).sum::() / os.len() as f64; let pi = [base_rate(&os, 0), base_rate(&os, 1)]; - let (d, lo, hi) = block_ci(|b| brier_diff(&os, b, pi)); + let (_, flo, fhi) = block_ci(|b| brier_diff(&os, b, pi)); + let (d, lo, hi) = + block_ci(|b| brier_diff(&os, b, [base_rate_in(&os, b, 0), base_rate_in(&os, b, 1)])); println!( "coverage {coverage:.3}, always-answer accuracy {acc_a:.3}, pi {pi:.3?}, \ - Brier(a)-Brier(b) {d:.4} [{lo:.4}, {hi:.4}]" + Brier(a)-Brier(b) {d:.4}, refitted-pi interval [{lo:.4}, {hi:.4}] \ + (fixed-pi interval [{flo:.4}, {fhi:.4}], which omits pi's error)" ); assert!( (0.2..0.95).contains(&coverage), "gate is vacuous: coverage {coverage}" ); - assert!(d >= 0.02 && lo > 0.0, "kill: gain {d} [{lo}, {hi}]"); + assert!( + lo <= 0.0, + "the gain is now established: {d} [{lo}, {hi}] — re-pin P-D4" + ); } #[test]