From 0a86d92ed2cf84d3607a94d2860c9da78e266cae Mon Sep 17 00:00:00 2001 From: Vincent Bonduro Date: Wed, 2 Sep 2026 17:03:47 -0400 Subject: [PATCH 1/4] pico2w: fix SD boot reset loop, add 288 MHz clock, capture crash stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes plus the investigation that produced them. SD boot reset loop ------------------ Booting with a staged ROM that carried a rom_id reset the board forever. It was not memory corruption: with no card, `embedded-sdmmc` takes ~17.1 s to return CardNotFound (50 acquire retries x ~340 ms). The watchdog window is 16 s, and the RP2350 counter is 24-bit MICROSECONDS so the window cannot exceed 16.777 s. The call could never fit, and every boot reset mid-init, leaving only WatchdogTimeout records and no fault. * bound acquisition to 6 retries -> ~2.4 s with no card, ~0.8 s with one * send the SD spec's 74 power-up clocks with CS deasserted; the crate leaves this to the caller and we never did it * skip the remaining boot SD reads once the card reports absent, instead of paying the timeout twice more. The third one — the save-slot probe — ran after the watchdog was re-armed and before the main loop's first feed, so it was a guaranteed reset on its own. Also feeds the watchdog during the ROM-list directory scan, which could outlast the window on a slow card. 288 MHz clock option (oc-288) ----------------------------- A frequency sweep by live PLL reprogramming put a failure cliff between 288 and 300 MHz: at 300 the SP-drift instrument logged 67.5 events per 1000 work-ticks, while 288/276/264/252/240 logged zero across 13,567 ticks. A 24-hour soak at 288 on the real firmware ran 3.56M ticks with zero crashes, against ~43 predicted at the 300 MHz rate. NOT a fix. The cliff is bracketed only to (288, 300], so 288 may have as little as 0.3% margin, untested against temperature and part-to-part spread, and the underlying defect still occurs at 150 MHz. 266 remains the more conservative shipping point. Default is unchanged. Crash record stack snapshot (schema v2) --------------------------------------- WatchdogTimeout and wild-PC records carried no stack context, which is what made them undiagnosable. Records now include six words from the pre-fault SP. DMA channels 2-6 are dropped from the record to make room — a live read of all 16 channels showed only ch0/ch1 are ever non-zero. MPU hardening ------------- Arm the RAM-resident .data code and thunks read-only, and core 0's stack read-only to core 1. Tests: 649 core + 191 pico2w pass. Verified on hardware: boots at 288 MHz, mounts the card in 781 ms, loads a save state, enters the main loop at 3.8 s. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Ab4PqF52wyVpp5S2Sg6RGK --- core/src/cpu/peripheral/apu.rs | 44 +- platform/pico2w/Cargo.toml | 2 + .../pico-crash-current-findings.md | 7524 +++++++++++++++++ .../investigations/sp-drift-root-cause.md | 281 + platform/pico2w/src/crash/context.rs | 2 +- platform/pico2w/src/crash/handler.rs | 21 + platform/pico2w/src/crash/mod.rs | 3 +- platform/pico2w/src/crash/record.rs | 45 +- platform/pico2w/src/crash/stack_snapshot.rs | 74 + platform/pico2w/src/crash/storage.rs | 18 +- platform/pico2w/src/main.rs | 93 +- platform/pico2w/src/mpu.rs | 205 +- platform/pico2w/src/sd.rs | 23 +- platform/pico2w/src/state/main_menu.rs | 4 +- platform/pico2w/src/state/rom_list.rs | 25 +- tools/crash_decoder.py | 34 +- tools/fixtures/test_crash.bin | Bin 4096 -> 4096 bytes xtask/Cargo.toml | 5 + 18 files changed, 8360 insertions(+), 43 deletions(-) create mode 100644 platform/pico2w/docs/investigations/pico-crash-current-findings.md create mode 100644 platform/pico2w/docs/investigations/sp-drift-root-cause.md create mode 100644 platform/pico2w/src/crash/stack_snapshot.rs diff --git a/core/src/cpu/peripheral/apu.rs b/core/src/cpu/peripheral/apu.rs index 9088040..d0b351d 100644 --- a/core/src/cpu/peripheral/apu.rs +++ b/core/src/cpu/peripheral/apu.rs @@ -75,6 +75,7 @@ const TICK_GRAN: u16 = 32; /// About 804 stereo pairs are produced per Game Boy frame at 48 kHz. /// Reserve some headroom so the hot audio path doesn't regrow this buffer. const SAMPLE_BUFFER_CAPACITY_HINT: usize = 2048; + /// Max possible mixer output before normalization: 4 channels * level 15 * volume 8. const MIXER_MAX: u32 = 4 * 15 * 8; @@ -881,11 +882,17 @@ impl ApuPeripheral { /// while the generic batch path handles tests and any larger callers. #[cfg_attr(target_arch = "arm", link_section = ".data")] fn produce_samples(&mut self, cycles: u16) { + let sample_inc = cycles as u32 * SAMPLE_PERIOD_DEN; if cycles <= 4 { self.sample_acc += sample_inc; + // Bound by the compile-time constant, NOT the in-memory `cap` word: + // a `cap` corrupted upward makes this check pass and `Vec::push` + // then writes past the real allocation (push only grows when + // `len == cap`). See the long note in the `else` branch below. if self.sample_acc >= SAMPLE_PERIOD_NUM - && self.sample_buffer.len() + 2 <= self.sample_buffer.capacity() + && self.sample_buffer.len() + 2 + <= self.sample_buffer.capacity().min(SAMPLE_BUFFER_CAPACITY_HINT) { self.sample_acc -= SAMPLE_PERIOD_NUM; let (left, right) = self.mix_sample(); @@ -900,7 +907,40 @@ impl ApuPeripheral { self.sample_acc = (acc % SAMPLE_PERIOD_NUM as u64) as u32; if n_samples != 0 { let cap = self.sample_buffer.capacity(); - let pushable = ((cap - self.sample_buffer.len()) / 2).min(n_samples as usize); + // `saturating_sub`, NOT `-`: if `len > cap` the plain subtraction + // underflows (usize) to a huge value, `pushable` stops being + // clamped by capacity, and the loop below pushes past the end of + // the heap buffer. `Vec::push` does not save us — it only grows + // when `len == cap`, so a corrupted `cap` (observed live on device + // 2026-08-03: cap 2048 -> 0 while len was 338) is never equal to + // len and every subsequent push writes at `ptr + len*2`, running + // off the 4096-byte allocation into adjacent heap objects. That + // turns a single corrupted word into an unbounded heap overrun — + // the amplifier behind the wild-pointer crashes. Saturating keeps + // the overrun impossible regardless of how `cap` got corrupted. + // + // 2026-08-08: `cap` too SMALL was the case handled above, but + // `cap` too LARGE is the dangerous direction and was NOT + // covered. `Vec::push` only reallocates when `len == cap`, so a + // `cap` corrupted upward is never equal to `len` and every push + // writes at `ptr + len*2` straight off the end of the real + // 4096-byte allocation — an UNBOUNDED spray of i16 samples + // through whatever follows it. + // + // That is not hypothetical: the wild pointer values in the + // crash records decode as stereo i16 sample pairs — + // 0x00ffff01 = (-255, +255), 0x0c160015 = (21, 3094), + // 0xed6ef504 = (-2812, -4754) — the same magnitudes as the live + // AUDIO_QUEUE contents (1638, 546, 0). Audio data is landing in + // pointer slots on BOTH cores' stacks, which is this spray. + // + // So clamp against the compile-time capacity, never against the + // in-memory `cap` word. `with_capacity` may over-allocate, so + // the constant is always <= the true allocation and therefore + // safe no matter how `cap` is corrupted. + let trusted_cap = cap.min(SAMPLE_BUFFER_CAPACITY_HINT); + let pushable = (trusted_cap.saturating_sub(self.sample_buffer.len()) / 2) + .min(n_samples as usize); for _ in 0..pushable { let (left, right) = self.mix_sample(); self.sample_buffer.push(left); diff --git a/platform/pico2w/Cargo.toml b/platform/pico2w/Cargo.toml index b21e527..bcc02dd 100644 --- a/platform/pico2w/Cargo.toml +++ b/platform/pico2w/Cargo.toml @@ -23,6 +23,8 @@ std = [] # CPU overclock options (clk_sys target). Default boot is 300MHz; see main.rs. oc-266 = [] oc-280 = [] +# 288 MHz: one PLL grid step below 300 (FBDIV 120). Clean where 300 is not. +oc-288 = [] oc-300 = [] # Log FPS to RTT every 60 frames. fps = [] diff --git a/platform/pico2w/docs/investigations/pico-crash-current-findings.md b/platform/pico2w/docs/investigations/pico-crash-current-findings.md new file mode 100644 index 0000000..56b4e0f --- /dev/null +++ b/platform/pico2w/docs/investigations/pico-crash-current-findings.md @@ -0,0 +1,7524 @@ +# Pico Crash Investigation: Current Findings + +Last updated: 2026-08-14 +Firmware commit: `a8ea0259f29f` +Target: Raspberry Pi Pico 2 W / RP2350 A2 + +This is the concise handoff for the current state of the Pico crash +investigation. The full chronological notebook remains in +`crash-debug-notes.md`. + +## Executive summary + +The firmware is suffering real core-0 stack corruption. At least one captured +failure contains a contiguous overwrite of a live stack frame: both the stack +protector canary and the saved return address were replaced with unrelated +stack/heap values. This is not merely a false stack-protector comparison. + +A generic Cortex-M33 `PUSH`/`POP` or SRAM read-path failure is now strongly +refuted. A purpose-built assembly stressor completed more than one billion +checked multi-register push/pop operations without one mismatch while the real +firmware filled all 31 persistent crash slots in roughly five minutes. The +remaining mechanism must be workload- or address-path-specific, such as a wild +write, DMA/cross-core interaction, contention-sensitive SRAM behavior, or a +more specific instruction/fetch failure. + +No exact corrupting instruction has been captured yet. The next experiment is +a tightly scoped DWT write watchpoint on the known canary word, qualified by a +saved return address in the outer live frame. + +## Verified observations + +### The crash is reproducible and heterogeneous + +The current diagnostic image produced 31 valid records in about five minutes: + +- 15 stack-protector panics +- 11 HardFaults +- 5 watchdog resets + +The sector is retained at `/tmp/rustyboy-crash-stackpop-5m.bin`, with decoded +JSON at `/tmp/rustyboy-crash-stackpop-5m-nosym.json`. + +Two HardFaults returned through a corrupted saved PC of `0x00009ffe` from the +epilogue of `SharedWorkerState::write_live_vram_range`. Nine other HardFaults +had `pc=0x00000010`, `lr=0x10011e49`, and a precise bus fault at +`0x01482815`. These signatures show more than one visible consequence, but do +not yet prove more than one underlying cause. + +### A saved return address was read from a corrupted stack slot + +An earlier repeated HardFault had: + +- PC: `0x2003c6e8` +- LR: `0x1001448b` +- CFSR: `0x00020000` (`INVSTATE`) +- pre-fault SP: `0x2007bab0` + +Disassembly showed that `__aeabi_memclr4` saved LR at exactly `0x2007baac` and +returned with `pop {r7, pc}`. The live LR register still held the correct +return address, but the PC consumed from the stack was bad. This isolates that +failure to the saved stack word or its load, rather than a bad branch target +calculation in the caller. + +### A live protected frame was genuinely overwritten + +The strongest stack-protector capture is in: + +`Option>::map::, ...>` + +For that invocation: + +- protected-function body SP: `0x2007baa0` +- canary address: `0x2007baa8` +- expected canary: `0x2b7e1516` +- captured canary: `0x2007cc70` +- saved LR address: `0x2007bacc` +- captured saved LR: `0x20034398` (a heap address) +- expected internal caller return: `0x1001a6b7` + +Several adjacent frame words contain plausible stack, heap, and code pointers. +The canary and saved LR are both wrong, 36 bytes apart. This is positive +evidence for a multiword or contiguous overwrite, not a spurious canary check. + +The protected function was called while +`PicoGameBoy::with_cartridge` was active. The outer frame calculation is: + +- `with_cartridge` body SP: `0x2007bad0` +- total outer frame size: `0x1270` +- outer entry SP: `0x2007cd40` +- outer saved LR sentinel address: `0x2007cd3c` +- staged-ROM boot return value while live: `0x10009a59` +- menu-loading return value while live: `0x1000d00d` + +The staged-ROM value is the first value to use for the next trial. Reading this +address after construction has returned is not meaningful because normal stack +reuse replaces it. + +### Generic stack loads and SRAM reads stayed clean + +The new `stack_pop_check` assembly loop deliberately exercises the same broad +mechanism implicated by the bad returns: + +- `push.w {r0, r1, r2, r3, lr}` +- `pop.w {r0, r1, r2, r3, lr}` +- compare every restored register +- return from each batch using `pop {r4, r5, r6, r7, pc}` + +It completed at least `1,018,167,296` checked iterations with zero mismatch. +At the five-minute capture point it had completed `944,766,976` iterations, +also with zero mismatch, while the real workload had already saturated the +31-record crash sector. + +Other integrity diagnostics also remained clean: + +- 19,361 complete XIP checks of the first 256 KiB: zero mismatch +- 77,804 `.data` checks: zero mismatch +- 391,090 RAM-check batches over stack and BSS samples: zero mismatch +- allocator guard: zero hits + +These checks do not cover every byte at every instant, but their clean result +substantially narrows the failure away from broad SRAM or flash corruption. + +## Hypotheses ruled out or reduced + +### LLVM machine outliner + +Disabling the LLVM machine outliner did not fix the problem. That build ran for +about 27 minutes and recorded 51 canary smashes; its persistent sector contained +15 panics, 11 HardFaults, and 5 watchdog records. The default outliner has +therefore been restored. + +### Generic `POP`/LDM or SRAM read failure + +More than a billion clean controlled operations while real failures occurred at +high frequency strongly refutes this as the general mechanism. It does not +exclude an address-, contention-, DMA-, or code-path-specific read problem. + +### Broad XIP, `.data`, or sampled SRAM corruption + +The integrity checks above found nothing. This reduces, but cannot completely +eliminate, short-lived corruption between checks or corruption outside the +sampled ranges. + +### RP2350-E2 spinlock alias erratum as a direct match + +The board reports A2 silicon. The RP2350 E2 erratum concerns writable spinlock +aliases, but locks 18 through 31 are documented as safe and Embassy uses +spinlock 31 for its critical section. E2 therefore does not directly explain +the observed stack overwrite. See the official +[RP2350 datasheet](https://datasheets.raspberrypi.com/rp2350/rp2350-datasheet.pdf). + +### The 300 MHz overclock as the sole cause + +The current high-rate reproduction runs at 300 MHz. This is outside the +RP2350's 150 MHz specification and can increase the failure rate. + +**Correction (2026-08-14):** this document previously stated the reproduction +runs at VREG 1.30 V. It does not. The retained image was built *without* +`--features oc-300`, and that configuration's defaults are `TARGET_SYS_HZ = +300_000_000` with `TARGET_CORE_VOLTAGE = V1_25` (`main.rs:129` and `:140`). +Measured on the board: `POWMAN VREG` (`0x4010000c`) = `0x000000e0`, i.e. +VSEL `0b01110` = **V1_25**. The clock was independently confirmed from +`PLL_SYS` (`REFDIV=1`, `FBDIV=125`, `POSTDIV 5x1` -> 12 MHz x 125 / 5 = +300 MHz), so only the voltage claim was wrong. + +This matters when reading every rate in this document: core voltage is the one +*proven* rate modulator measured under a controlled, byte-identical layout +(V1_25 vs V1_30, one variable: >=0.95 records/min with 78 smashes, versus +0.105/min with zero smashes — roughly 7x). All rates quoted here, including the +"31 records in about five minutes" headline, were therefore measured at the +*lower* voltage. It does not invalidate any of the qualitative refutations +above, which do not depend on rate. + +`QMI_M0_TIMING` (`0x400D000C`) also read `0x60007203`, i.e. CLKDIV=3. That is +the known bootrom-reset state already documented at `main.rs:837-864`: bootrom +flash helpers reset QMI timing on every flash access, and each crash commits a +record to flash, so at ~3 crashes/min the device spends much of its life with +flash SCK at 300/3 = 100 MHz rather than the intended CLKDIV=6 (50 MHz). This +is previously-known behaviour, not a new finding, but it is the live state. +However, the identical saved-LR failure was also observed once during a +roughly nine-hour 150 MHz run. The overclock is at least a rate modulator, but +it is not sufficient evidence for the root mechanism. + +## Diagnostic approaches that were too invasive + +A DWT read watchpoint on the exact stack address generated more than 21 million +DebugMonitor exceptions within seconds. A DWT value-read watch on +`0x2003c6e8` generated roughly 45 million exceptions in 15 seconds and caused +repeated initialization. Both altered timing too heavily to produce trustworthy +captures and have been retired. + +An address write watch is much lower traffic, but stack addresses are reused. +Earlier filters based only on stack depth or value were unsound and admitted +legitimate writers. The next filter checks a fixed saved LR in a still-live +outer frame and the interrupted stack depth together. + +## Current leading candidates + +The evidence currently supports these classes, without proving one: + +1. A workload-specific wild or misdirected multiword write into the core-0 + stack. +2. A DMA or display-transfer operation using corrupted bounds or an incorrect + destination. +3. A cross-core publication/visibility bug that lets core 0 consume stale or + torn framebuffer metadata and later misprogram a copy or DMA operation. +4. A contention- or address-specific RP2350 SRAM path failure not exercised by + the controlled stressor. + +A separate audit identified a missing producer-side `DSB` before publishing +the large raw framebuffer slot and dirty-row buffers to the other core. That is +a credible weak-ordering issue and deserves a controlled fix/no-fix soak, but +there is not yet direct evidence connecting it to this stack overwrite. + +## Next discriminating experiment + +Use the existing firmware image without rebuilding, so all measured addresses +remain stable: + +1. Preserve and blank the current crash sector. +2. Clear `DWT_CATCH` and `WATCH_LOG`. +3. Plant the watched canary address `0x2007baa8` in `DWT_CATCH[8]`. +4. Plant sentinel address `0x2007cd3c`, expected value `0x10009a59`, and victim + body SP `0x2007baa0` in the watch log. +5. Reset the board, disconnect SWD, and run for at least five minutes. +6. Read the watch log, stack-smash dump, integrity blocks, and crash sector. +7. Symbolize every captured writer PC and validate its effective destination. + +If no catch occurs but the same protected frame is smashed, repeat using the +menu-loading sentinel `0x1000d00d`. If the watchpoint suppresses the failure, +replace it with a low-overhead software epilogue check that records the frame +before the normal stack-protector panic path changes it. + +## Result: scoped canary watchpoint, staged-ROM sentinel (config U) + +Run 2026-08-14 08:24:27, ~10 min, on the unmodified image. Planted exactly as +specified above. **Instrument addresses had moved and reordered since earlier +notes and were re-derived from this ELF** — `DWT_CATCH=0x200670e8` +(`[8]@0x20067108`), `WATCH_LOG=0x20067610` (`[3]@0x2006761c`, `[5]@0x20067624`, +`[14]@0x20067648`), `SMASH_CORE0=0x20067110`, `_stack_end=0x20067ab8`; zero span +668 words from `0x20067048`. All four plants verified by read-back. + +**No catch.** `WATCH_LOG` magic was never written and `CAUGHT` stayed 0, while +`[13]` counted **30,476** non-guard writes to `0x2007baa8`. So the watchpoint +was live and the word was written tens of thousands of times, but not one write +satisfied sentinel-AND-depth. + +The same protected frame was still smashed during the window: 5 of the 28 crash +records are stack-protector panics with `lr=0x10013239`, the same +`Option>::map` victim. This is the documented "no catch but the +same frame is smashed" branch, so the run was repeated with the menu-loading +sentinel `0x1000d00d` (config U2, 08:37:21). + +28 records in ~10 min (2.8/min, sector not saturated): + +- 5x panic `lr=0x10013239` (the map victim) +- 5x panic `lr=0x20000b2b` — a *second* concurrent victim, + `GameBoyMemory::c...+0x16e`, not previously named +- 1x panic `lr=0x2000359b` (`drain_bus_events`) +- 5x HardFault `pc=0x00009ffe`, `lr=0x10019dd9` + (`SharedWorkerState::write_live_vram_range+0x5c`), IBUSERR +- 5x HardFault `pc=0x00000010`, `lr=0x10011e49` + (`embassy_time::delay::block_for+0x18`), PRECISERR/BFARVALID at + `fa=0x01482815` +- 1x HardFault `pc=0x2007cb20` (a stack address), same `block_for` LR, INVSTATE +- 1x core-1 HardFault `pc=0x200181c4`, `lr=0x20000765` + (`ApuPeripheral::produce_samples+0xb0`), INVSTATE +- 5x watchdog + +Integrity blocks over the same window: `.data` 1007 checks / 0 hits, allocator +guard 0, RAM-check 5004 batches / 0 mismatch. Heartbeat: core0 93, core1 37691. + +Two caveats on this null. First, arming the DebugMonitor perturbs timing, and +the most recent smash in `SMASH_CORE0` was a *different* victim +(`body_SP=0x2007ccc8`, `lr=0x20000b2b`) than the watched one, so the watched +frame is no longer the dominant victim. Second, and unchanged: **the DWT sees +only the CPU load/store unit, so this null does not exonerate DMA.** + +## Both sentinel values in the designed experiment were wrong + +Config U2 (menu sentinel `0x1000d00d`, 08:37:21, ~15 min) was also a null: +`CAUGHT = 0`, magic never written, `[13]` = **46,184** counted writes to +`0x2007baa8`. The watched victim *did* smash during that window — +`SMASH_CORE0` came back with `body_SP = 0x2007baa0`, `lr = 0x10013239`, 19 +smashes, guard `0x2007cc70`, saved LR `0x20034398`, i.e. the exact watched word +corrupted to the exact known value. + +The cause of both nulls was then found by simply reading the sentinel word on +the running board: + + 0x2007cd3c = 0x1000ac47 (stable across three samples) + +That is `embassy_main_task+0x3746` — a **third** `with_cartridge` call site, +between the two that were guessed (`0x10009a59` = +0x2558, `0x1000d00d` = ++0x5b0c). Neither planted value could ever match, so **configs U and U2 were +structurally incapable of catching anything** and their nulls say nothing about +the corruptor. They are not evidence that no CPU store is responsible. + +Lesson for this instrument: the sentinel value must be **measured on the +board**, not derived by reading call sites out of the disassembly. The earlier +note that "reading this address after construction has returned is not +meaningful" is right in principle, but a stable repeated read of a plausible +code address in the expected function is far better than a guess. + +Config U3 (08:55:08) repeats the experiment unchanged except for the corrected +sentinel value `0x1000ac47`. + +### If U3 is also null + +Drop the sentinel (`WATCH_LOG[3] = 0`) and keep only the depth gate +(`[5] = 0x2007baa0`). Depth is structural rather than guessed: the stack grows +down, so `sp_at_fault <= body_SP` selects writers executing *deeper* than the +victim's frame base. That is the right filter for the leading hypothesis — a +callee overrunning upward into the frame, which matches corruption at +`body_SP+0x8` and `body_SP+0x2c`. It also means that **if the corruptor writes +from a shallower frame (a wild pointer store from main-loop code at +SP ~0x2007cb00), the depth gate rejects it by construction**, so a null with +depth enabled is itself informative and should be reported as such rather than +read as "no CPU store did it". + +## Config U3: the watchpoint fires, but the ring saturates with benign writers + +With the corrected sentinel `0x1000ac47`, config U3 (08:55:08, ~15 min) **caught**: +magic `0x3EE70001` written, `CAUGHT = 21,752` out of `[13] = 44,940` writes, +first offending value `[12] = 0x20027510`, first `sp_at_fault = 0x2007ba74`. The +sentinel word still read `0x1000ac47` at the end of the window, so the predicate +stayed valid throughout. The instrument works. + +All eight distinct writer PCs are allocation-path functions caught at *prologue* +offsets, i.e. ordinary frame reuse, not corruptors: + +| PC | off | SP | symbol | +|----|-----|----|--------| +| `0x1002f970` | +0xa | `0x2007ba74` | `compiler_builtins::mem::memcpy` | +| `0x100205da` | +0xa | `0x2007ba98` | `GuardedHeap as GlobalAlloc::alloc` | +| `0x10013296` | +0x2 | `0x2007ba98` | `RawVecInner::with_capacity_in` | +| `0x100146ec` | +0x46 | `0x2007ba80` | `RawVecInner::try_allocate_in` | +| `0x100131a8` | +0x8 | `0x2007ba98` | `Option::map` (the victim itself) | +| `0x10014666` | +0x6 | `0x2007baa0` | `box_new_uninit.903` | +| `0x100131aa` | +0xa | `0x2007ba98` | `Option::map` (the victim itself) | +| `0x10013fda` | +0x6 | `0x2007baa0` | `box_new_uninit.874` | + +This reproduces the config-S result exactly, including the same SP values. The +victim's own base is `0x100131a0` (`lr = 0x10013239` = +0x99), so two of the +eight are its own guard store. + +**The ring is the bottleneck.** It dedupes by PC and fills once *without +evicting*, so at ~50 writes/s it freezes within seconds on whichever writers +happen to be first. The corruptor wrote later and was never recorded. First +offending value `0x20027510` is not the corrupt value `0x2007cc70`. + +### Config U4: exploit the ring instead of fighting it + +The dedupe rule (`break` on `cur == pc`, record only into a `cur == 0` slot) can +be turned into an exclusion filter by **pre-filling the ring with the known +benign PCs and leaving one slot free**. Only a writer not already in the list +can then claim it. Planted 09:13:14: + +- watch `0x2007baa8`, sentinel `0x2007cd3c` = `0x1000ac47` (unchanged) +- **`WATCH_LOG[5] = 0` — the depth gate is DISABLED** +- ring `[16..22]` pre-filled with the seven benign PCs above, `[23]` left free + +Disabling depth matters: every previous run gated on `sp_at_fault <= body_SP`, +which selects only writers *deeper* than the victim's frame base. A wild pointer +store from shallower main-loop code (SP ~`0x2007cb00`) was therefore rejected by +construction in configs U, U2 and U3, and in the config-S "one word below the +benign band" run. **This is the first configuration that can see a shallower +writer at all.** + +## The recurring "guard constant 8 bytes below the guard slot" is NOT a bug + +This pattern has been flagged repeatedly across builds and victims: the dump +shows `0x2b7e1516` sitting exactly 8 bytes below the computed guard slot, which +invites the theory that the prologue stores the canary at one offset and the +epilogue checks a different one. **Checked directly against the current victim's +own code and refuted.** + +`GameBoyMemory::copy_dma_step` @ `0x200009bc`: + +- prologue `str r0, [sp, #0x50]` @ `0x200009d0` +- epilogue `ldr r0, [sp, #0x50]` @ `0x20000b14` + +Same offset. No mismatch. The word at `+0x48` is simply another local. The frame +arithmetic also validates: the call is `bl __Thumbv7ABSLongThunk___stack_chk_fail` +(an ABS long-branch thunk, `ldr r12,[pc]; bx r12`, which uses no stack), and the +`addeq sp, #0x54` in the epilogue only executes on the *passing* path, so SP at +`__stack_chk_fail` entry really is the victim's body SP. The naked-trampoline +capture is therefore sound and the guard slot address derived from it is correct. + +Do not revive this theory. + +## Config V: value-triggered capture + +The planted-predicate approach on the old image was exhausted (all three +predicates dead — see above), so the firmware was rebuilt with one change to +`dwt_watch.rs`: + +- `WATCH_LOG[1]`, previously a planted-but-unused `thresh`, is now an **exact + value filter**: non-zero means only a write whose value equals it is recorded. +- New write-once latch `[6]` = PC, `[7]` = SP-at-fault, `[8]` = match count, so + the answer cannot be churned away by the non-evicting ring. + +Build `9b67a0f5…`, image CRC `0xedc61550` verified on device, still **without** +`--features oc-300` so the operating point stays 300 MHz / V1_25 and the dataset +stays comparable. Addresses moved and reordered again: `DWT_CATCH=0x20067228`, +`SMASH_CORE0=0x20067250`, `WATCH_LOG=0x20067750`, `_stack_end=0x20067ab8`. + +Calibration soak (09:33:05, ~14 min) named the dominant victim as +`copy_dma_step`, `body_SP = 0x2007ccc8`, so guard address `0x2007cd18`, and the +corrupt payload is **`0x00000001`**. Nothing in that function writes `+0x50` +except the guard store itself, so a write of `1` to that slot while the frame is +live is anomalous — which makes even this common-looking value a usable filter. + +Config V2 planted 09:49:23: watch `0x2007cd18`, value filter `1`, sentinel and +depth gate both **off**. `[8]` (match count) is the sanity check: if it is huge, +the filter is too loose to trust and `[6]` means little; if it is small, `[6]` is +the writer. + +## Config V2: the exact-value filter is also too loose — and why + +Planted watch `0x2007cd18` with value filter `1`, sentinel and depth both off. +Result over ~16 min: + +- `[8]` match count = **8,065,684** +- `[13]` total writes to that word = **71,565,733** + +So the watched stack word is written about **74,000 times per second**, and ~11% +of those writes carry the value `1`. The latched PC (`0x10027a6c`) is therefore +meaningless and is not reported as a finding. + +This closes out every filter dimension the instrument has: + +| dimension | measured outcome | +|-----------|------------------| +| address alone | 71.5M writes; ring saturates in seconds with 9 distinct allocation-path prologue pushes, all benign | +| address + exact value | 8.07M matches — ~11% of traffic | +| sentinel liveness | permanently stale-true; admitted 100% of writes | +| stack depth | rejects a shallower writer by construction | + +The corrupting store is **indistinguishable from legitimate traffic in all of +them**. The watched word is simply an ordinary hot stack slot. + +## Config W: guard→non-guard transition latch + +One discriminator remains, and it is exact rather than statistical. The +corrupting store is, by definition, the write that turns a live canary into +something else. The DWT fires on *every* write to the address, so the handler +observes the complete value sequence and can remember the previous one: + + prev == STACK_GUARD && v != STACK_GUARD => this store just destroyed a live canary + +This needs no liveness, depth, or payload assumption, and fires at most once per +victim invocation. Implemented in `dwt_watch.rs`: `[9]` holds the previous value, +`[6]`/`[7]` latch the first destroying PC/SP (write-once), `[8]` counts them. The +older value-filter path keeps its own record at `[10]`/`[11]` so the two cannot be +confused. + +Caveat to apply when reading it: the DebugMonitor is configurable-priority, so +PRIMASK can defer it and writes may coalesce. A missed intermediate write would +blur the transition, so treat a single catch as a lead and sanity-check `[8]` +against the smash count rather than declaring root cause from one sample. + +Build `0x8d068cb0` (CRC verified on device), still 300 MHz / V1_25. Instrument +addresses unchanged from config V. Calibration soak started 10:08:11 to re-derive +the current victim's guard address before planting. + +## Config W2: the transition latch fires, but "first transition" is the wrong one + +Armed on the map victim's guard word `0x2007baa8` (victim base `0x100131a0`, +`body_SP = 0x2007baa0`, guard offset `+0x8` derived from its own `str`/`ldr` +pair). Result over ~15 min: + +- `[6]` first transition PC = `0x10014a5a`, `[7]` SP = `0x2007baa8` +- **`[8]` transition count = 1110**, against `SMASH_CORE0[3]` = **16** smashes + +1110 vs 16 is ~70x too many, so the latched PC is not the corruptor and is not +reported as one. The cause is structural and was predictable: when the victim +returns normally its guard slot is recycled, and the next function to write +there also produces a guard->non-guard transition. **Every normal invocation +generates one benign transition.** + +Also worth recording: the victim, its `body_SP`, its guard address and *both* +corrupt values (`0x2007cc70` in the guard slot, `0x20034398` in the saved-LR +slot) reproduce **identically across three different builds**. The corruption is +structurally determined, not random timing noise. + +## Config X: snapshot the LAST transition at the moment of the smash + +The fix follows from the same structure that broke the first-transition latch. A +transition requires the slot to hold the guard *first*, and once corrupted it +stays corrupt until some later invocation stores a fresh guard. So **while a +given frame is live there is at most one transition**, which means the last +transition before a smash is precisely the store that destroyed that frame's +canary. + +Implemented: + +- `dwt_watch.rs` keeps a last-write-wins copy of the transition PC/SP at + `WATCH_LOG[32]`/`[33]`. +- `stack_chk.rs` snapshots those two words into `SMASH_CORE0[132]`/`[133]` + (past the 128-word dump, in MPU padding) at the moment the canary check fails. + +So the smash record itself now carries the killing store. Read it with +`probe-rs read --chip RP235x b32 0x20067250 134` — `[132]` @ `0x20067460` is the +PC, `[133]` @ `0x20067464` is its SP. + +These are only meaningful when the watch is aimed at *that* victim's guard word; +if the victim moves, they are stale, and the smash record's own `[1]`/`[2]` will +show it. + +Build CRC `0x64fffb43` verified on device, 300 MHz / V1_25, victim still at +`0x100131a0`, watch planted at `0x2007baa8`, soak started 10:42:08. + +## The DWT/DebugMonitor approach is EXHAUSTED — with proof of the mechanism + +Config X2 aimed the transition latch at the correct victim this time +(`drain_bus_events`, `body_SP = 0x2007cb70`, `LR = 0x2000359b`, guard at +`+0x1b0` = `0x2007cd20`, both verified against the record). Its guard slot was +genuinely corrupt at the smash: `0x2003d6f8`. And yet: + +- `WATCH_LOG[8]` transition count = **0** +- `SMASH_CORE0[132]/[133]` = **0** +- `WATCH_LOG[13]` total writes = **73,238,730** +- **`DWT_CATCH[6]` (writes where the handler read the guard value) = 0** + +That last number is the proof. Across 73 million writes the handler **never once +observed the guard value** at the watched address. The DWT fires *after* the +access and the handler *re-reads* the word; at ~58,000 writes/second, with +PRIMASK deferring the configurable-priority DebugMonitor, the value has always +already been overwritten. `prev == STACK_GUARD` can therefore never become true. + +**So the config-X/X2 nulls are instrument failures, not evidence.** They do NOT +support a non-core master and must not be cited as doing so. + +This closes the whole DWT line for these targets. Every predicate dimension has +now been measured and defeated by the sheer write rate on these stack words: + +| predicate | measured outcome | +|-----------|------------------| +| address alone | 71.5M writes; ring saturates in seconds | +| address + exact value | 8.07M matches (~11% of traffic) | +| sentinel liveness | permanently stale-true (100% admitted) | +| stack depth | rejects shallower writers by construction | +| first guard->non-guard transition | 1110 transitions vs 16 smashes (benign recycle on every return) | +| last transition at the smash | handler never sees the guard value at all (`DWT_CATCH[6]` = 0) | + +**Seventeen distinct writers have now been caught across two victims and every +single one is a benign prologue push / ordinary frame reuse**, including the +only non-prologue-looking candidate: `0x1001a2f0` resolves to +`PicoGameBoy::with_cartridge+0xc`, a `sub sp,#0xc` — not a memory write at all. +Its real store is the `push.w {r8,r9,r10,r11}` at +0x4, which from entry SP +`0x2007cd40` lands **r9 exactly on `0x2007cd20`**. Benign. + +### Where to go next — a different instrument class + +Do not build another DWT predicate. The remaining options, in order of promise: + +1. **ETM instruction trace.** `etm.rs` already exists and the linker defines + `__etm_filter_start` / `__etm_filter_end` bracketing exactly the RAM-code + region that contains `drain_bus_events` — that infrastructure was built for + this problem. It is hardware trace of the real instruction stream and is the + only genuinely non-perturbing instrument available. +2. **Loose end (D)**, the missing producer-side `DSB` before publishing the raw + framebuffer slot and dirty-row buffers, as a controlled fix/no-fix soak. +3. **DMA**, which the DWT has never been able to see. Sweep channel registers + repeatedly (they rotate); a single snapshot is insufficient. + +### Note on perturbation + +Every DWT run perturbs the system: with the monitor armed the victim moved +between `map` and `drain_bus_events` and the smash count fell from 16-18 to 2 per +window. Config X3 (11:24:41) runs the same build with the watch **disarmed** to +establish an unperturbed baseline rate for this build, which any future +fix/no-fix comparison needs. + +## MAJOR REFRAMING: core 1 is the dominant failure, and it fails the same way + +Two soaks with the DWT **disarmed** (the first unperturbed measurements taken in +a long time) change the picture substantially. + +**Config X3** (19.7 min): 11 records = **0.56/min** — 6 watchdog, 3 canary panic, +2 core-1 HardFault, **zero core-0 HardFaults**. +**Config Y** (26.9 min): 20 records = **0.74/min** — 10 watchdog, **7 core-1 +HardFault**, 3 canary panic. + +So at the true operating point **17 of 20 records are core-1 faults or hangs**, +and the core-0 canary smash that this investigation has chased for dozens of +cycles is the *minority* failure mode. Note also how badly the DebugMonitor was +distorting things: the perturbed runs read 2.8/min and the doc's old headline was +6/min, versus 0.56–0.74/min unperturbed. + +### The core-1 signature is perfectly deterministic + +All **7** core-1 HardFaults are byte-identical in every recorded field: + + pc=0x200181c4 lr=0x20000765 cfsr=0x00020000 (INVSTATE) + hfsr=0x40000000 fa=0 sp_before=0x20081f90 r12=0x10013c75 + +Decoded: + +- `lr = 0x20000765` is exactly the return address of the **second** + `bl __Thumbv7ABSLongThunk__Vec::push` at `0x20000760`, i.e. + `ApuPeripheral::produce_samples+0xb0` (base `0x200006b4`). +- `pc = 0x200181c4` is **not code**: it is `SHARED_WORKER_STATE + 0x13164` + (base `0x20005060`; RAM code ends at `__edata = 0x2000381c`). Branching to an + even address is exactly why CFSR reports INVSTATE. +- `sp_before = 0x20081f90` is only ~112 bytes into core 1's 8 KiB stack + (`0x20080000..0x20082000`) — a very shallow frame. + +**The thunk is intact and is not the cause.** Live RAM at `0x2000369e` decodes to +`movw r12,#0x3c75 / movt r12,#0x1001 / bx r12` (r12 = `0x10013c75` = +`Vec::push|1`), byte-identical to the ELF, and `DATA_CHECK` reports 887 +checks with 0 corrupt words. The recorded `r12` is that same correct value. + +That rules out the obvious reading: `bx r12` with an odd r12 sets T=1 and lands +at `0x10013c74`; it cannot yield INVSTATE at `0x200181c4`. LR and r12 are simply +*undisturbed*. The reading consistent with every field is a **`pop {pc}` inside +`Vec::push`'s call tree loading a corrupted saved return address** — the thunk +uses `bx`, so LR passes through unchanged, and r12 is never touched. + +### Why this matters + +**Core 1 suffers the same class of corruption as core 0**: a saved return address +replaced by a pointer into a shared data structure. `SMASH_CORE1` is empty only +because `Vec::push` is not stack-protected, so there is no canary there to trip. + +Core 1's stack is in SRAM8/9 (the separate, non-striped banks) while core 0's is +in the striped SRAM0-7, and **both are affected** — which argues against any +bank-specific SRAM effect. + +The core-1 case is far more tractable than the core-0 one: it is 7/7 deterministic +with a *fixed* corrupt value and a *fixed* SP, whereas the core-0 victim drifted +between builds. And core 1's stack should carry nowhere near the ~58,000 +writes/second that defeated every DWT predicate on core 0 — so a watch armed **on +core 1** at the corrupted slot may actually work where core 0's could not. That +requires firmware changes (the watch is currently armed on core 0 only; each core +has its own DWT). + +### Instrument shortfall to fix + +`persist_prev()` currently stores only `(core0_ticks, core1_ticks)`. The module +docs say `core0_since_peer` / `core1_since_peer` are the real discriminators for +"which core stopped first". The ring showed core1/core0 ratios of 285–988 across +8 ended boots with no clean freeze signature, but the since-peer deltas would be +the sound test. Persist those too. + +## Core-1 signature confirmed on a second window, and the payload identified + +**Config Y2** (independent window, 27.75 min, DWT still disarmed): 8 records = +0.29/min — 4 watchdog, **3 core-1 HardFault**, 1 core-0 panic. Again 7 of 8 are +core-1 or hang. The core-1 fault is now **10/10 byte-identical across two +unperturbed windows**. + +### The corrupted slot, derived from the frame + +`Vec::push` (`0x10013c74`) ends: + + ldr r11, [sp], #4 + pop {r4, r5, r6, r7, pc} + +So after the `pop`, SP equals its entry SP. The record's `sp_before = +0x20081f90` is therefore `Vec::push`'s entry SP, and **the corrupted saved-LR +slot is `0x20081f8c`**. That also puts `produce_samples`' entry at +`0x20081fa8` — only 88 bytes below core 1's stack top (`0x20082000`), matching +the observed shallow frame. + +### What the corrupt value IS + +`pc = 0x200181c4` = `SHARED_WORKER_STATE + 0x13164` = **78,180 bytes** in. +Layout: `sync_snapshot` + `live_ppu_snapshot` (each `PpuSnapshot` = `0x80` io + +`0x2000` vram + `0xA0` oam ≈ 8.4 KB, so ~16.8 KB together), then +`native_frame_slots: [SharedNativeFrameSlot; 3]`, each an +`UnsafeCell` of `FRAMEBUFFER_SIZE`. Offset 78,180 lands **inside +`native_frame_slots`**, roughly two-thirds through slot 2. + +So the word overwriting core 1's saved return address is **a pointer into a +shared framebuffer slot** — exactly the memory the frame publication/copy path +walks with pointers. That is a direct match for candidates (2) and (3) (a +transfer with corrupted bounds/destination; cross-core publication), and it +narrows the suspect surface enormously: from "anything on core 0" to core 1's +frame-publication path. + +### Heartbeat ring: inconclusive, and why + +Ratios `core1/core0` across 8 ended boots: 343-676, with no bimodal +"one core froze" signature. But core 0's heartbeat only ticks 25-139 times per +boot (~once every 1.5-6 s), which is far too coarse to resolve a hang of a few +seconds. The `since_peer` deltas are the sound discriminator (the live boot reads +0 and 51 — healthy); persisting those is still the outstanding instrument fix. + +### Next experiment + +Arm a DWT write watch **on core 1** at `0x20081f8c`. Core 1 has its own DWT and +its own DebugMonitor, and the watch is currently armed on core 0 only, so this +needs firmware: read a planted address in the core-1 entry path and call +`arm_data_write_watch` + `enable_monitor_only()` there. + +The value/transition predicates that failed on core 0 are not needed here — the +**distinct-PC ring alone should be selective**, because core 1's code surface at +a fixed shallow depth is a handful of functions, versus the effectively unbounded +allocation chain that saturated the ring on core 0. Pre-fill the ring with +`Vec::push`'s own prologue PC to exclude the benign writer. + +## METHODOLOGY BUG: a mid-run probe attach can wedge the board and fake a clean window + +Config Y3 (12:48:11, 27.5 min) reported **zero crash records**. That is *not* a +clean window. Sampling the heartbeat three times over 6 s showed it completely +frozen (`core0=25`, `core1=15968`, unchanged), i.e. **the board was hung and the +watchdog was not resetting it**. The prev-boot ring index had moved 8 -> 2, so two +reboots happened and then it stopped. + +The cause is the procedure, not the firmware under test. Blanking the crash +sector mid-run attaches the debugger, and this firmware sets the watchdog to +**pause-on-debug**. With the debug flag latched the watchdog stays paused, so a +hang that would normally produce a WDT record becomes a permanent wedge instead. +Config Y2 used the same "blank sector, no reset" shortcut and survived it; Y3 did +not — so the contamination is intermittent, which makes it especially dangerous. + +Recovery also needed escalation: `probe-rs` could no longer even erase flash +(`execution of 'init' failed with code 288`), and the documented OpenOCD RESCUE +path was required. Note a rescue reset **randomises `.uninit`**, so all instrument +arrays and the heartbeat ring are lost. + +### Rules this adds + +1. **Never interpret a low or zero record count without first proving liveness** — + sample the heartbeat at least twice, seconds apart, and confirm it advances. +2. **Always reset after any mid-run attach.** "Blank the sector without resetting" + is not a safe shortcut. +3. After a rescue, the board sits with `core0=0, core1=6` and does not run the + emulator until the crash sector is blank (it displays the unread-records + badge instead), so blank the sector *and* reset before timing anything. + +## Correction: "sample the heartbeat twice" is an unsound liveness test + +The rule added above needs qualifying. **`probe-rs read` halts the core to access +memory**, so two separate probe-rs invocations are two halts, and a frozen +reading at check time can be an artifact of the measurement itself rather than a +wedge. + +Config Y4 (13:18:15, 18 min) made this concrete: two liveness samples read +byte-identical, which by the earlier rule would mean "wedged" — yet the board had +plainly been running, because **7 crash records accumulated (0.39/min, squarely +in the unperturbed 0.29-0.74/min range)** and the prev-boot ring index advanced +2 -> 3. + +**Use record accumulation as the liveness signal**, not repeated heartbeat reads. +If heartbeat sampling is needed, both samples must come from a *single* probe-rs +session. + +Config Y3 remains a genuine wedge: there it was zero records **and** a frozen +heartbeat **and** flash erase failing with `code 288` **and** it needed an +OpenOCD rescue. Three independent signals, not one ambiguous one. + +## Both cores fail the same way: a wild branch into a large `.bss` object + +Config Y4 added the missing half of the picture. Core 0 produced two HardFaults: + + pc=0x2003bcc0 lr=0x20001fff cfsr=INVSTATE sp=0x2007cd08 + pc=0x2003bbc4 lr=0x20001fff cfsr=INVSTATE sp=0x2007cd08 + +Both PCs resolve into the **`embassy_main_task` state object** at +`0x200244dc + 0x177e4` and `+0x176e8` — a `.bss` data structure, not code. `lr = +0x20001fff` is `Sm83 ... instructions+0x112` (RAM code). + +That is the same shape as the core-1 fault, which branches into +`SHARED_WORKER_STATE + 0x13164`. So on **both cores** an indirect branch or +`pop {pc}` picks up **a pointer into a large `.bss` structure** and faults with +INVSTATE because the target is even. + +Together with the earlier finding that core 1's corrupt value is a pointer into a +shared framebuffer slot, the unified statement is: *saved return addresses on +both cores are being replaced by pointers into the big shared state objects.* +That is one mechanism, not two, and it is independent of which SRAM bank the +stack lives in (core 0 striped SRAM0-7, core 1 non-striped SRAM8/9). + +## CAUGHT: the store that writes the poison value (config Z2) + +Arming the DWT **on core 1** — which had never been done; every prior watch was +armed by core 0 — produced the first positive identification in this +investigation. + +Setup: build Z (CRC `0xd7806df6`), `run_core1_worker` reads `DWT_CATCH[9]` and +arms core 1's own DWT + DebugMonitor. Watch planted at `0x20081f94`. Aim +verified *before* interpreting anything: the core-1 HardFault still reported +`sp_before = 0x20081f98` (12 occurrences in the window). + +Result (`WATCH_LOG`): + +- `[12]` first offending value = **`0x2002e810`** +- `[16..20]` ring = `0x1001afc0`, `0x1001afc2`, `0x1001afc4`, `0x1001afc6` + — four consecutive halfwords, i.e. the resume-artifact spread around **one** + store site +- `[24..28]` all SP = `0x20081f88` + +`0x2002e810` is **exactly the wild PC in all 12 core-1 HardFaults**. The store is + + 1001afbc: str r2, [sp, #0xc] ; run_core1_worker+0x4ec + +which at SP `0x20081f88` targets `0x20081f88 + 0xc` = **`0x20081f94`**, the +watched word. So the value that later gets popped into PC is written *there*, by +that instruction, as one of `run_core1_worker`'s own locals. Twenty-four bytes +earlier, at `+0x4d4`, is `bl __Thumbv7ABSLongThunk__…produce_samples`, whose +callee `Vec::push` ends `ldr r11,[sp],#4` / `pop {r4,r5,r6,r7,pc}`. + +**This is a stack-slot alias**: a word `run_core1_worker` uses as a local is also +consumed as a callee's saved return address. + +### The part that does NOT yet add up — do not skip this + +`run_core1_worker`'s prologue is only `push {r7,lr}` + `sub sp,#0x38` = 64 bytes, +so a body SP of `0x20081f88` means it was **entered** at `0x20081fc8`. +`produce_samples` would then be entered at `0x20081f88`, push 24 bytes, and call +`Vec::push` at `0x20081f70`. + +But the record's `sp_before` is `0x20081f98` — and `sp_before` is the faulted +thread's SP *after* unwinding the exception frame (`sp_before_exception`), i.e. +`Vec::push`'s entry SP. `0x20081f98` is **40 bytes above** `0x20081f70`, and sits +*inside* `run_core1_worker`'s own frame (entry `0x20081fc8` … body `0x20081f88`), +which a normal callee cannot do. Both `produce_samples` call sites are inside +`run_core1_worker` (`+0x4d4` and `+0x6f4`), so there is no alternative shallower +call path. + +Audited, and the likeliest explanation is an **instrument asymmetry, not a bad +SP**. The two SPs come from two different formulas: + +- `crash/handler.rs::sp_before_exception` is **correct**: it adds 104 bytes when + `EXC_RETURN.FType` (bit 4) is clear (FP extended frame) and 32 otherwise, plus + 4 for the aligner when stacked xPSR bit 9 is set. Its own comment records that + this exact mistake already mis-named a watchpoint slot once during the G4 hunt. +- `dwt_watch.rs`'s `sp_at_fault` does `frame + 32` (+4 for the aligner) and + **never accounts for the FP extended frame**. + +So the DebugMonitor's reported SP can be up to 72 bytes low, and the catch's +`0x20081f88` and the HardFault's `0x20081f98` **must not be compared directly**. +Fixing `dwt_watch`'s computation to match the handler's is a prerequisite for any +frame arithmetic that mixes the two sources. + +What is solid and independent of all SP arithmetic: the DWT compares the +**address in hardware**, and the value was read back from the watched word. So +**`run_core1_worker+0x4ec` writes the exact value that becomes the wild PC, into +the exact word the watch was aimed at.** The remaining work is to establish how +that word is also reachable as a callee's saved return address. + +## Current build and retained evidence + +Firmware ELF: + +`target/thumbv8m.main-none-eabihf/release/rustyboy-pico2w` + +ELF SHA-256: + +`110c853376fd916690b9377e0f65ab8466d1f9cd2c5f81dcff6d317b298472c6` + +Retained captures: + +- `/tmp/rustyboy-crash-stackpop-5m.bin` + (`5b586d56ac51d952dfa1c5bb3f204ab42fdf8143007e3ce3816c99dd29ee7a99`) +- `/tmp/rustyboy-crash-stackpop-5m-nosym.json` +- `/tmp/rustyboy-smash0-stackpop-5m.bin` + (`b9ba25f9383880f9af88f95f5c9b095615aff21cda95d9315c00a22a507e3be5`) +- `/tmp/rustyboy-integrity-stackpop-5m.bin` + (`44dfdc9ceaf2f61572d94b3c7b99f54d3669834084001a0d4a04c97cb6150c13`) + +The physical board is still running the above image. The scoped canary +watchpoint described here has been designed but not yet planted or soaked. + +## Config Z3 + AA: the instrument-asymmetry bug is FIXED, and the catch survives it + +### Z3 window (16:10, 27 min) — crash records valid, watch data VOID + +31 records, and the core-1 signature reproduced yet again: **11x +`pc=0x2002e810 lr=0x20000765 cfsr=INVSTATE sp=0x20081f98`**, plus 15 watchdog +hangs and 5 assorted core-0 HFs. + +But `WATCH_LOG` came back **byte-identical to Z2** — `CAUGHT` frozen at 111,998 +across a window that produced 31 fresh records. The watch was **inert** for the +whole window: `probe-rs` attach clears `DEMCR.MON_EN`, only firmware can set it, +and the arming code runs once at boot in `run_core1_worker`. My blank+reset at +15:43 went through probe-rs, so the monitor never came back up. + +**Rule this adds to the tooling list: a frozen `CAUGHT` is NOT evidence of "no +writes" — check that `CAUGHT` advanced before reading anything into WATCH_LOG.** +Z3's watch slots are void; only its crash records count. + +### The store identification does NOT depend on the suspect SP arithmetic + +Before fixing the instrument, the Z2 conclusion was re-checked against a bound +that no formula can move: **core 1's stack top is `0x20082000`.** + +`run_core1_worker`'s prologue is `push {r7,lr}` + `sub sp,#0x38` = 64 bytes, so +`entry_SP = body_SP + 64`. If the reported body SP `0x20081f88` were 72 bytes low +(the maximum error the old formula could produce), the true body SP would be +`0x20081fd0` and the entry SP `0x20082010` — **above the top of the stack, i.e. +impossible.** `0x20081f88` is the only value consistent with the stack bounds. + +So `str r2, [sp, #0xc]` → `0x20081f88 + 0xc` = `0x20081f94` stands, and with it +the Z2 result: **`run_core1_worker+0x4ec` writes the exact value (`0x2002e810`) +that becomes the wild PC, into the exact word the watch was aimed at.** + +### The fix + +`dwt_watch.rs::sp_at_fault_of` did `frame + 32` (+4 aligner) and never handled +the FP extended frame, so it could read up to 72 bytes low, while +`crash/handler.rs::sp_before_exception` has always been correct. Two instruments, +two formulas, silently incomparable. Now `sp_at_fault_of` takes `exc_return` and +applies the same `FType`-based 104-vs-32 choice plus the aligner; the handler's +`_exc_return` parameter is threaded through all six call sites, and the one +open-coded copy inside the depth-gate predicate now calls the helper instead. + +This is the SECOND time this exact mistake has cost a result — `sp_before_exception`'s +own comment records it mis-naming a watchpoint slot during the G4 hunt. The fixed +function carries a comment saying so. + +### Config AA — running + +Rebuild is layout-identical where it matters: every forensics address unchanged, +`run_core1_worker` still at `0x1001aad0`, and the store still `str r2, [sp, #0xc]` +at `0x1001afbc` (`+0x4ec`) with the same `0x38` frame — verified by disassembly +rather than assumed, so the `0x20081f94` aim carries over without re-deriving. +CRC `0x76465722`. Instruments zeroed this time (Z2/Z3 counts were cumulative). + +Open question is unchanged and is now the whole investigation: **how is a word +that `run_core1_worker` uses as its own local (`[sp,#0xc]`) also consumed as a +callee's saved return address?** With both SPs finally on one formula, the frame +arithmetic from the two instruments is comparable for the first time. Loose end +(A) — `gameboy.rs:455-466`'s claim that `#[inline(never)]` makes the §G1 +spill-slot collision impossible — is the closest prior art and should be re-read +against this. + +## Config AA: the chain is CLOSED — the pop reads its return address from a word with exactly one writer + +`CAUGHT = 8352` (advanced from a zeroed start), so unlike Z3 this window's watch +data is real. Three results, in increasing order of importance. + +### 1. The corrected SP formula returns the SAME value — 0x20081f88 + +`sp_at_fault` with the fixed `FType` logic still reports `0x20081f88`, i.e. the +frames at this store site are BASIC, and the old bug never distorted this +particular measurement. The store's SP is now confirmed twice over — once by the +stack-top bound, once by a formula known to be correct. + +### 2. The watched word has EXACTLY ONE writer in the entire program + +`CAUGHT == total writes == 8352`, and all four ring PCs (`0x1001afc0/c2/c4/c6`) +are resume artifacts of the single store `run_core1_worker+0x4ec`. Over ~33 +minutes, nothing else in the firmware ever wrote `0x20081f94`. + +**No prologue ever pushed a return address there.** + +### 3. The faulting `pop` read its PC from precisely that word + +The core-1 HF signature (23 occurrences across Z2+Z3) is +`lr=0x20000765 sp_before=0x20081f98`. Symbolising: `0x20000765` = +**`produce_samples+0xb0`**, so the faulting callee was called from there — +`Vec::push`, whose epilogue is `ldr r11,[sp],#4` / `pop {r4,r5,r6,r7,pc}`. + +In `pop {r4,r5,r6,r7,pc}` the PC is the last word popped, at `final_SP - 4`: + +``` +final_SP (= sp_before) = 0x20081f98 +PC loaded from 0x20081f98 - 4 = 0x20081f94 <- the watched word +``` + +Exact, and independent of every frame-size estimate. + +And the poison value is not corrupt data: **`0x2002e810` = `HEAP_MEM+0xa334`** — +a perfectly legitimate `Vec` buffer pointer, which is exactly what +`run_core1_worker` should be holding in a local. It is even, so branching to it +faults INVSTATE. Nothing is corrupting memory; **a correct value is being read +from the wrong place.** + +### What this proves about the mechanism + +A `pop` restored its return address from a word that no matching `push` ever +wrote. So between the prologue and the epilogue, **SP moved up** — the epilogue +popped from higher than the prologue pushed, landing on the caller's live local. + +This is in direct tension with the REFUTED entry "SP moves mid-function", which +must now be re-read: that refutation was established on core 0 against the canary +smashes, and cannot be carried over to this core-1 path unexamined. + +### The discriminator, now running (config AB) + +Two possibilities remain: + +* **(i)** the CALLER's SP moves, so `[sp,#0xc]` does not always name the same word; +* **(ii)** the caller's SP is stable and the damage happens inside + `produce_samples` → `Vec::push`. + +The ring's four SP samples are all `0x20081f88`, but they were selected by +distinct-PC dedup and say nothing about the other 8348 catches. `WATCH_LOG[40]` +/`[41]` now track **min/max `sp_at_fault` over every catch**; `min == max` ⇒ (ii). +`[42]` counts FP-extended exception frames on core 1 and `[43]` keeps the last +`EXC_RETURN` — if core 1 never takes an extended frame, an FP lazy-stacking SP +mismatch is excluded as the mechanism. + +`WATCH_LOG` grew 40→64 words for these slots, which MOVED every forensics +address (see below) and made `WATCH_LOG` the LAST `.uninit` object, tail flush +against `_stack_end`. Slots `[58..64]` therefore sit inside core 1's read-only +MPU region — harmless only because nothing writes above `[43]`. Verified on +device: `heartbeat placement OK: base=0x20067368 core1-RO-region=0x20067b00`. + +### AA window composition — the mode moved again + +25 records, and **no core-1 HFs at all**: 10x WDT, 10x a NEW core-0 signature +`pc=0x00009ffe lr=0x10019de9 cfsr=IBUSERR sp=0x2007cb68` where `0x10019de8` = +`SharedWorkerState::write_live_vram_range+0x5c`, 4x Panic `lr=0x2000359b`, 1x +Panic `lr=0x20000b2b` (`copy_dma_step+0x16e`). Consistent with the standing rule +that layout dominates which victim surfaces; the underlying wrong-slot read is +the same shape. + +## Config AB: the caller's SP is PROVEN stable — the damage is inside Vec::push + +`CAUGHT = 6081`, so the instrument was live and the aim self-validated. + +### The discriminator came back unambiguous + +``` +[40] min sp_at_fault = 0x20081f88 +[41] max sp_at_fault = 0x20081f88 <- min == max over all 6081 catches +[42] FP-extended frames = 0 +[43] last EXC_RETURN = 0xfffffff9 (FType set => basic frame) +``` + +**`run_core1_worker`'s body SP never moves.** `[sp,#0xc]` names `0x20081f94` on +every one of 6081 samples, so branch (i) — "the caller's frame moves" — is dead. +The SP damage is inside `produce_samples` → `Vec::push`. + +`[42] == 0` additionally **excludes FP lazy-stacking**: core 1 never takes an +extended exception frame, so a stacked/unstacked size mismatch cannot be moving SP. + +### Both functions on the path are statically BALANCED + +`produce_samples` (`0x200006b4`): prologue `push {r4,r5,r6,r7,lr}` (20) + +`str r11,[sp,#-4]!` (4) = 24. It has TWO exits — a tail call at `+0x54` +(`ldr r11,[sp],#4` / `pop {r4,r5,r6,r7,lr}` / `b.w` into `Vec::push`) and a normal +return at `+0xb4` (`ldr r11,[sp],#4` / `pop {r4,r5,r6,r7,pc}`) — and **both restore +exactly 24**. Every branch to `+0xb4` (`0x200006d6`, `0x200006f0`, `0x2000072a`, +`0x20000748`) happens after the prologue completed. No unbalanced path exists. + +`Vec::push` (`0x10013c74`): same 24-byte prologue, same 24-byte epilogue, one +conditional `bleq grow_one`. Balanced. + +Both `produce_samples` call sites are inside `run_core1_worker` (`+0x4d4`, +`+0x6f4`) — verified by scanning the full 80,338-line disassembly for branches to +`0x200006b4` and its thunk `0x10030b60`. **There is no shallower caller**, so the +entry SP is genuinely `0x20081f88`. + +### The size of the anomaly + +Working frames forward from the measured `0x20081f88`: + +``` +run_core1_worker body SP 0x20081f88 +produce_samples entry 0x20081f88 body 0x20081f70 (24-byte frame) +Vec::push entry 0x20081f70 body 0x20081f58 (24-byte frame) +Vec::push epilogue SHOULD end at 0x20081f70 +Vec::push epilogue OBSERVED ending at 0x20081f98 <- 40 bytes (0x28) higher +``` + +`lr = 0x20000765` is the return address of the `bl` at `0x20000760`, so the +faulting function is definitely `Vec::push` (the long thunk preserves LR, and +`grow_one` would carry `lr = 0x10013c8e` instead). Since `Vec::push` is balanced +and its caller's SP is fixed, **SP rose 40 bytes during `Vec::push`'s own +execution**, and the only thing in that window able to move it is the +`bleq grow_one` allocator call. + +### Next: the allocator path + +`grow_one` → `RawVec` growth → global allocator is now the whole remaining +surface. Note the prior finding that the audio path already had a cross-core +heap race (the 8192-byte OOM caused by an unbounded +`drain_audio_samples_into_i16` loop racing core 1's enqueue), and that the poison +value `0x2002e810` is `HEAP_MEM+0xa334` — a `Vec` buffer pointer. The allocator is +where core 0 and core 1 meet. + +Static targets for the next cycle: check `grow_one` and everything it reaches for +SP balance, and check whether the global allocator is safe against concurrent +core-0/core-1 entry. + +## Config AC (UNPERTURBED): core 0 writes into core 1's stack — and the "SP moved" inference is RETRACTED + +31 records in 32.7 min = **0.95/min**, the highest unperturbed rate yet (prior +baselines 0.29–0.74/min). Layout dominates rate, so this is not a valid +cross-build comparison, but this layout is a good reproducer. + +### The dominant mode is the core-0 canary smash + +14x `Panic C0 lr=0x20001bcf`. Decoding the record: `panic_loc=0x63617473` and +`r12=0x68635f6b` are ASCII — `"stac"` + `"k_ch"` = `stack_chk` — with `line=169` +and `flags=0x8e` including `0x80 STACK_CHK_FAIL_LR`. So the victim is +`Sm83 instructions+0x16e`. This is the same core-0 canary smash the investigation +chased for dozens of cycles. + +### THE FINDING: an MPU-detected cross-core write + +3x `HF C0 pc=0x1001a846 lr=0x1001a839 cfsr=0x00000082` (DACCVIOL|MMARVALID), +`flags=0x41` — **no `0x20 FAULT_ON_CORE1`, so the faulting core is CORE 0** — and +the hardware-supplied + +``` +FAULT_ADDR = 0x20080d10 <- inside CORE 1's STACK (0x20080000..0x20082000) +``` + +**Core 0 is writing into core 1's stack.** The MPU caught it three times in 33 +minutes. + +### Which store, and why it goes wild + +``` +1001a7fe: ldr r5, [sp, #0x10] <- r5 comes from a SPILL SLOT +1001a808: movw r11, #0xff80 +1001a818: mov.w r10, #0x4100 <- constant field offset +1001a838: and.w r1, r6, r11 +1001a83c: cmp.w r1, #0xff00 <- is this an 0xFF00-page register? +1001a842: sxth r1, r6 <- r6 = 0xFF10..0xFF3F (APU register) +1001a844: add r1, r5 +1001a846: strb.w r0, [r1, r10] <- addr = r5 + sxth(r6) + 0x4100 +``` + +For `r6 = 0xFF10`, `sxth(r6) = -0xF0`, so `addr = r5 + 0x4010`. With the measured +`FAULT_ADDR = 0x20080d10`: + +``` +r5 = 0x20080d10 - 0x4010 = 0x2007cd00 +``` + +`r5` should be a struct base pointer in `.bss`. Instead it holds **`0x2007cd00`, an +address inside core 0's own stack** — and that value sits directly among the `sp` +values of this same window's other faults (`0x2007cd08`, `0x2007cd58`). So the +spill slot `[sp,#0x10]` contained a *stack address* where a struct pointer +belonged: **a slot yielding a value that belongs to a different variable** — +exactly the shape of the core-1 finding, now on core 0 and caught by hardware. + +### RETRACTION: "SP moved 40 bytes inside Vec::push" does not hold + +That inference rested on config AA/AB showing the watched word `0x20081f94` had +**exactly one writer** (`CAUGHT == total writes`, one store site). That is true +only of **core 1's own DWT** — and each core has its own DWT, which sees only its +own load/store unit. **A core-0 write into core 1's stack is completely invisible +to the core-1 watch.** + +Since core 0 is now proven to write into core 1's stack, the premise fails: the +poison could have been placed by core 0 without ever incrementing `CAUGHT`. The +40-byte SP-rise arithmetic, and the reinstatement of "SP moves mid-function", +should both be treated as unsupported. What survives from AA/AB is solid and +still useful: `run_core1_worker`'s body SP is genuinely constant (6081 samples), +and core 1 never takes an FP-extended frame. + +### The unification + +One wild core-0 store — a spill slot holding a stack address instead of a struct +pointer — accounts for **both** long-standing families: + +* it lands in **core 0's own stack** ⇒ the canary smashes (14x this window); +* it lands in **core 1's stack** ⇒ core 1's saved return addresses are corrupted + ⇒ the wild-PC INVSTATE faults, invisible to core 1's DWT. + +That is why the payload always looked like "a legitimate pointer read from the +wrong slot", why it moves with layout, and why it is independent of SRAM bank. + +### Next instrument + +`r5` and `r10` are callee-saved, so they are NOT in the exception frame — but they +are still live in the CPU when the handler runs. The crash handler should capture +**r4–r11** directly and store them in the record. That gives the corrupt pointer's +value on every occurrence and settles whether it is always an SP-like value. + +## Config AD: the corrupt pointer is a SPILL SLOT overwrite, NOT an allocator bug + +Second unperturbed window: 31 records in 32 min = **0.97/min**, matching AC's 0.95 +— this layout reproduces consistently. Composition is stable too: 13x canary-smash +Panic `lr=0x20001bcf`, 13x WDT, **2x the same MPU violation**, plus three +one-off core-0 HFs. + +### The MPU violation is DETERMINISTIC + +``` +fa = 0x20080d10, sp = 0x2007bad0 (2x this window, 3x in AC — 5 identical) +``` + +Random corruption would scatter the fault address. **It is byte-identical every +time**, so whatever produces it is systematic. + +### Where the bad pointer comes from + +`with_cartridge`'s frame is 4720 bytes (`push {r4,r5,r6,r7,lr}` + `push +{r8,r9,r10,r11}` + `sub sp,#0x1240` + `sub sp,#0xc`), so its entry SP is +`0x2007bad0 + 0x1270` = `0x2007cd40` and the frame spans +`0x2007bad0..0x2007cd40`. + +The spill slot has exactly ONE writer and three readers: + +``` +1001a57c: str.w r9, [sp, #0x10] <- the only writer +1001a7fe: ldr r5, [sp, #0x10] <- the faulting store's base +1001a8e8: ldr r2, [sp, #0x10] +1001a90c: ldr r0, [sp, #0x10] +``` + +and `r9` is the return value of a heap allocation: + +``` +1001a33a: movs r0, #0x4 ; align 4 +1001a33c: movw r1, #0x42c0 ; size 17,088 +1001a340: bl box_new_uninit +1001a352: mov r9, r0 +``` + +Back-solving the faulting store `addr = r5 + sxth(r6) + 0x4100` against the +measured fault address gives `r5 ≈ 0x2007cc91..0x2007cd10` (the range covers every +`r6` the guard `r6 & 0xff80 == 0xff00` admits) — **a core-0 stack address, inside +`with_cartridge`'s own frame**, not a heap pointer. + +### The allocator is EXONERATED — checked, not assumed + +`alloc_guard.rs` already wraps the real heap and records any out-of-arena +pointer. Read live from the device: + +``` +ALLOC_GUARD[0] = 0xa1100001 magic — guard IS active +ALLOC_GUARD[1] = 0 out-of-arena ALLOCATIONS +ALLOC_GUARD[2] = 0 refused deallocs +ALLOC_GUARD[6] = 0 first offender +ALLOC_GUARD[7] = 0x200244dc arena lo +ALLOC_GUARD[8] = 0x2004c4dc arena hi (HEAP_MEM + 160 KiB) +``` + +**Zero out-of-arena allocations.** So `box_new_uninit` returned a correct heap +pointer and `r9` was right when stored. An earlier draft of this section claimed +the allocator was the root cause; the guard refutes it. + +### What that leaves — and why it is finally a good DWT target + +The value is a valid heap pointer at the `str` and a stack address at the `ldr`, +so **`with_cartridge`'s spill slot at `sp+0x10` = `0x2007bae0` is overwritten +between the two**, and `sp_before` is identical across all five occurrences, so +that address is FIXED. + +This is the corrupt-spill-slot mechanism again, but for the first time pinned to a +single known address on core 0. Crucially it dodges what killed every previous +core-0 DWT attempt: the old target was written 73,238,733 times by 17 innocent +prologue pushes, whereas `0x2007bae0` is written **once per `with_cartridge` +entry**, and `with_cartridge` is the main loop — entered once. A cold address is +exactly what the distinct-PC ring is selective on, which is why the same +instrument succeeded on core 1. + +### Config AD2 — running + +Core-0 watch armed at `0x2007bae0` (`DWT_CATCH[8]` @ `0x20067510`, readback +confirmed; core-1 aim left 0). No value filter, so every writer is caught. +Expect `CAUGHT` to be SMALL — one legitimate write per boot plus the offender. +`WATCH_LOG[16..24]` will name the distinct writer PCs; anything that is not +`with_cartridge+0x284` is the bug. + +## Config AE: MM_REGS fires — the fault arithmetic is confirmed exactly, the r9 test is VOID + +### Why the registers were never visible before + +The HardFault asm trampoline has captured `r4-r11` into `HARDFAULT_EXTRA_REGS` +all along. But that is a plain `static mut`, so it lives in **`.bss` and is zeroed +on every boot** — and this handler resets the chip. The values were destroyed +before anyone could read them. `MM_REGS` is the same data in `.uninit`, gated on +`MMARVALID`, write-once, magic written last. + +### First capture + +``` +[0] magic 0x4d4d0001 +[1] pc 0x1001a7a6 = with_cartridge+0x54e (same offset as the AB build) +[2] MMFAR 0x20080d10 (6th byte-identical occurrence) +[3] CFSR 0x00000082 DACCVIOL|MMARVALID +[4] sp_before 0x2007bad0 +[5] r4 0x2007cd00 +[6] r5 0x2007cd00 +[7] r6 0x0000ff10 +[8] r7 0x2007cd30 +[9] r8 0x0000ff10 +[10] r9 0x2007cc6c +[11] r10 0x00004100 +[12] r11 0x0000ff80 +``` + +The back-solved model is confirmed to the bit: `r10 = 0x4100` and `r11 = 0xff80` +are the predicted constants, `r6 = 0xff10` is the predicted APU register, and + +``` +r5 + sxth(0xff10) + 0x4100 = 0x2007cd00 - 0xf0 + 0x4100 = 0x20080d10 == MMFAR +``` + +### The r9 discriminator is VOID — r9 was legitimately reassigned + +The plan was: if `r9` still held the `box_new_uninit` pointer while `r5` did not, +the spill slot was corrupted. **That test cannot be applied.** `r9` is rewritten +four times after the spill store: + +``` +1001a2b2: mov r9, r0 <- the box pointer +1001a4dc: str.w r9, [sp, #0x10] <- stored (the slot's ONLY visible writer) +1001a5c6: sub.w r9, r7, #0xc0 <- REASSIGNED +1001a644: ldr.w r9, [pc, ...] +1001a750: sub.w r9, r7, #0xc4 <- REASSIGNED, immediately before the fault +1001a800: add.w r9, r9, #0x28 (after the fault PC, so not yet executed) +``` + +`r7 - 0xc4 = 0x2007cd30 - 0xc4 = 0x2007cc6c` — exactly the observed `r9`. So `r9` +at fault time is an ordinary pointer-to-local and says nothing about the box. + +### What IS established + +`sp_before = 0x2007bad0`, so the slot is **`[sp,#0x10] = 0x2007bae0`**, and it +yielded + +``` +r5 = 0x2007cd00 = r7 - 0x30 a pointer to a stack LOCAL +heap arena = 0x200244dc..0x2004c4dc -> r5 is NOT in it +``` + +where the slot's only visible writer stores the heap box pointer. The slot cannot +legitimately hold `r7-0x30`. (The second `box_new_uninit` at `+0x48a` is for +`OpCode` boxes pushed into a Vec — unrelated to this slot.) + +Two possibilities remain: something writes `0x2007bae0`, or `SP` differs between +the store and the load. Both are settled by watching the address. + +### Config AE2 — running + +Core-0 DWT armed at **`0x2007bae0`**, derived from a `sp_before` measured in THIS +build rather than carried across a rebuild (the mistake that voided Z1). +`DWT_CATCH[8]` @ `0x20067548`, readback confirmed; core-1 aim left 0; no value +filter. The slot is cold — written once per `with_cartridge` entry, and +`with_cartridge` is the main loop — which is exactly the condition under which +this instrument succeeded on core 1 and failed on the old 73M-write core-0 target. + +`WATCH_LOG[16..24]` will name the distinct writer PCs. **Anything that is not +`with_cartridge+0x284` (`0x1001a4dc`) is the bug.** `MM_REGS` was zeroed and will +re-arm alongside it. + +## Config AE2: the watch CAUGHT the writer — and the real anomaly is an 8-byte SP shift + +`CAUGHT = 27` on a cold slot, so the aim (derived from a `sp_before` measured in +THIS build) is self-validated. The ring holds **exactly one distinct writer PC**: + +``` +[2] CAUGHT = 27 +[12] first value = 0x2007cd00 <- precisely the corrupt value +[13] total writes = 27 +[16] writer PC = 0x1001a4c8 <- the only one; [17..24] empty +[24] paired SP = 0x2007bac8 +``` + +`0x1001a4c8` is a `bl __rust_alloc_zeroed`, i.e. a RESUME point, not a store — +the DebugMonitor is deferred, so the store is a couple of instructions earlier at +`1001a4c2: strb.w r4, [sp, #0x18]`. + +### The finding + +``` +catch: sp_at_fault = 0x2007bac8 -> [sp,#0x18] = 0x2007bae0 (watched word) +MPU fault: sp_before = 0x2007bad0 -> [sp,#0x10] = 0x2007bae0 (same word) +``` + +**The same stack word is `[sp,#0x18]` at one point in `with_cartridge` and +`[sp,#0x10]` at another — its SP differs by 8 bytes between them**, although the +prologue allocates a fixed 4720-byte frame (`push {r4,r5,r6,r7,lr}` + `push +{r8,r9,r10,r11}` + `sub sp,#0x1240` + `sub sp,#0xc`) with no dynamic allocation. + +Both numbers now come from the IDENTICAL FType formula (that was fixed in AA), so +unlike the Z2 episode they are directly comparable and the 8 bytes are real. + +This is why the slot yields `r7-0x30` where a heap box pointer belongs: the box +pointer and a local genuinely share one stack word, reached at two different SPs. +The value read back (`0x2007cd00`) is consistent — a `strb` of zero landing on a +word that already held `0x2007cdXX`. + +Loose end (A) — `gameboy.rs:455-466`'s claim that `#[inline(never)]` makes the §G1 +spill-slot collision "impossible" — is now directly implicated rather than merely +suggestive. + +### Negative result: the forensics block is NOT being trashed + +All 708 words of the zeroed instrument span were scanned for core-0 stack-valued +words. 23 were found and **22 are legitimate instrument contents** — the +`WATCH_LOG` catch data, and `SMASH_CORE0`, which is a canary-smash record and +properly contains SPs and frame pointers. So the wild store is NOT spraying the +`.uninit` block. + +The one unexplained word is `DWT_CATCH[9]` (`0x2006754c`), which is the last word +of `DWT_CATCH` immediately below `SMASH_CORE0` (`0x20067550`) — i.e. +`SMASH_CORE0[-1]`. That points at an off-by-one in the smash-record writer, not at +the bug under investigation. Worth fixing so it stops masquerading as a planted +core-1 aim (a stale value there arms a bogus core-1 watch at boot). + +## CORRECTION: there is no 8-byte SP shift — the aim was 8 bytes wrong + +The previous section claimed `with_cartridge`'s SP differs by 8 bytes between two +points in its body. **That is wrong, and the error was mine.** + +### with_cartridge's SP is constant + +Every SP-modifying instruction in the function: + +``` +1001a258: push {r4, r5, r6, r7, lr} +1001a25c: push.w {r8, r9, r10, r11} +1001a260: sub.w sp, sp, #0x1240 +1001a264: sub sp, #0xc + ... no SP modification anywhere in the body ... +1001a934: addeq.w sp, sp, #0x1240 (conditional epilogue) +1001a938: addeq sp, #0xc +1001a93a: popeq.w {r8, r9, r10, r11} +1001a93e: popeq {r4, r5, r6, r7, pc} +``` + +No dynamic allocation, no mid-body push/pop. SP is fixed after the prologue. + +### The authoritative body SP comes from r7 + +`r7` is captured by hardware into `MM_REGS` and the prologue sets +`add r7, sp, #0xc` after `push {r4,r5,r6,r7,lr}`, so `r7 = entry_SP - 8`: + +``` +r7 = 0x2007cd30 +entry SP = 0x2007cd38 +body SP = entry - (20 + 16 + 0x1240 + 0xc) = 0x2007bac8 + +DWT sp_at_fault = 0x2007bac8 <- MATCHES the body SP exactly +record sp_before= 0x2007bad0 <- body SP + 8, OVER-REPORTS +``` + +So the DWT's SP was right all along and the crash record's `sp_before` is 8 high +for this fault. I built the aim from `sp_before`, so: + +``` +aimed at 0x2007bae0 = [sp,#0x18] <- a benign neighbouring local +wanted 0x2007bad8 = [sp,#0x10] <- the box-pointer slot +``` + +**The 27 (then 91) catches were the legitimate `strb.w r4,[sp,#0x18]` writing its +own variable — not the bug.** The value `0x2007cd00` read back there is simply +that neighbouring local also holding `r7-0x30`; it is not evidence of corruption, +and loose end (A) is NOT implicated by it. + +Third SP-arithmetic error in this investigation. The durable rule: **derive a +frame address from `r7` plus the static prologue, never from a handler-computed +`sp_before`.** `r7` is hardware-captured; `sp_before` is arithmetic that has now +been wrong twice. + +### Config AE3 — running + +Watch re-aimed at **`0x2007bad8`**, instruments zeroed, sector blanked, reset. +`WATCH_LOG[16..24]` should now name whoever writes the box-pointer slot; anything +other than `with_cartridge+0x284` (`0x1001a4dc`) is the bug. + +### Loose end (G) CONFIRMED LIVE — and it is a real bug + +`DWT_CATCH[9]` (`0x2006754c`) was zeroed and read back as `0x2007ccc0` **while the +device was running**, before any reset. `DWT_CATCH` is `[usize; 10]` at +`0x20067528`, so `[9]` is the last word, immediately below `SMASH_CORE0` +(`0x20067550`) — i.e. `SMASH_CORE0[-1]`. The smash-record writer runs on every +canary smash (~13 per window) and writes one word below its array. + +Consequences: it corrupts a neighbouring instrument, and because +`multicore.rs` accepts any core-1 aim in `0x20000000..0x20082000`, the stale value +arms a **bogus core-1 watchpoint on every boot**. Fix the off-by-one before +trusting any future core-1 aim. + +## THE CANARY IS GENUINELY CORRUPTED — "the canary is INTACT" was reading the wrong word + +The box-pointer line is exhausted for now: over a 33-minute window (**76 boots**) +the filtered watch on `0x2007bad8` recorded **zero** writes of `0x2007cd00`, and +`MM_REGS` never filled — no MPU violation fired at all. So the pivot to the +dominant mode (12 canary smashes that window) was taken, and it paid off +immediately. + +### Reading the smash record correctly + +`__stack_chk_fail` is `#[naked]`, so `SMASH_CORE0[1]` is the victim's **true body +SP** by construction, and `[4 + i]` is the word at `sp + 4i`. + +``` +[0] 0x51150002 magic [1] 0x2007ccd8 body SP [2] 0x20001bcf lr [3] 23 smashes +``` + +`lr = 0x20001bcf` → the victim is `Sm83 ... Instructions::dec8` at `0x20001a60` +(`+0x16e`). Its guard check names the slot explicitly: + +``` +20001bba: ldr r0, [sp, #0x8] <- THE GUARD SLOT +20001bbc: ldr r1, [pc, #0x18] <- &__stack_chk_guard +20001bbe: ldr r1, [r1] +20001bc0: cmp r1, r0 +20001bc2: itt eq <- equal: pop and return +20001bca: bl __stack_chk_fail +``` + +So the guard lives at `sp + 8` = `0x2007cce0` = dump index 2 = `SMASH_CORE0[6]`: + +``` +SMASH_CORE0[4] = 0x2b7e1516 (sp+0) <- a NEIGHBOURING frame's guard copy +SMASH_CORE0[6] = 0x00000001 (sp+8) <- THE ACTUAL GUARD SLOT +SMASH_CORE0[5] = 0x2007cd0c (sp+4) } plausible locals, +SMASH_CORE0[7] = 0x2007d2d0 (sp+c) } NOT corrupted +``` + +### The conclusion this flips + +**The canary holds `0x00000001`. It is genuinely corrupted.** + +`stack_chk.rs`'s own comment warns that an 8-byte ambiguity here "decides whether +the guard slot reads `0x2B7E1516` (compare failed on correct operands) or a +corrupt value (a real stray store), i.e. it flips the conclusion of the entire +investigation." The old reading looked at `sp+0`, found `0x2b7e1516` — which is a +*neighbouring frame's* copy of the constant, exactly the trap the comment +describes — and recorded **"the canary is INTACT"** in the refuted list. + +**That refuted entry is wrong and is hereby withdrawn.** The compare failed on +correct operands is FALSE; this is a real stray store. + +And because the guard word alone is corrupt while both its neighbours hold sane +values, it is a **stray pointer store, not a copy/memset overrun** — the precise +discriminator the record was built to provide. + +### The payload is 0x00000001 + +Not a pointer, not a guard, not a stack address — a small integer. Something is +storing `1` through a pointer that lands on `0x2007cce0`. + +### Config AF — running + +Watch aimed at the real guard slot **`0x2007cce0`** with exact-value filter +**`0x00000001`**. `dec8` is hot, so the slot is written on every call — but the +filter is what makes a hot address usable: normal traffic stores `0x2B7E1516` and +never matches, so only the store that writes `1` latches a PC. + +Caveat recorded up front: the DWT fires *after* the access and the handler +re-reads the word, so if the corrupting store is immediately followed by other +traffic the value may have moved on — the same effect that defeated the old +73M-write core-0 attempts. A null result here is therefore NOT evidence of +absence, and `[13]` (unfiltered total) must be checked to confirm the instrument +was live. + +## Config AF: the corrupt canary reproduces, but address+value is not selective + +The value-filtered watch on the verified guard slot `0x2007cce0` ran a full window +and the instrument was emphatically live: + +``` +[13] total writes = 27,729,801 (dec8 is hot — ~13,600 writes/s) +[2] CAUGHT = 102,767 matches of value 1 +[12] first value = 0x00000001 +[6] first PC = 0x20001ca2 [15] first SP = 0x2007ccc8 +``` + +`SMASH_CORE0[6] = 0x00000001` again with `[3] = 10` smashes — **the corrupt canary +reproduces**, independently confirming the previous section. + +### Why the filter did not isolate the culprit + +102,767 matches out of 27.7M writes means storing `1` at that address is COMMON. +The eight distinct writer PCs explain why: + +``` +0x20001ca2, 0x20001e0a, 0x2000140a, 0x20001ef6 Sm83 instruction fns +0xa +0x20001408 Sm83 instruction fn +0x8 +0x1001e628 XipCartridge+0x188 +0x1003060a __aeabi_memcpy4+0x7a +0x1002f1ba OUTLINED_FUNCTION_297 +``` + +Nearly all are `+0x8`/`+0xa` — **prologue stores**, i.e. resume artifacts of +sibling functions at the same stack depth writing their own locals into a +RECYCLED stack word. `0x2007cce0` is only "the guard" while `dec8`'s frame is +live; the rest of the time it belongs to whoever else is at that depth. An +address+value predicate cannot distinguish those cases, and the depth gate cannot +either, because the innocent writers sit at the same depth (first +`sp_at_fault = 0x2007ccc8`, just below `dec8`'s body SP `0x2007ccd8`). + +### Config AF2 — the transition latch, finally aimed correctly + +`dwt_watch` already implements the right predicate and `stack_chk` already +snapshots it into `SMASH_CORE0[132]/[133]`. Its rationale, from the source: + +> "A transition on the watched word requires the slot to hold the guard first, +> and once corrupted it stays corrupt until a later invocation writes a fresh +> guard — so while THIS frame was live there was at most one transition, and it +> is the store that just destroyed the canary we are failing on." + +That excludes every innocent prologue write **by construction**: those do not +overwrite a live guard. The instrument returned nothing in the X2 attempt only +because it was aimed at `drain_bus_events` — the wrong word. The aim is now +verified from the victim's own disassembly (`ldr r0,[sp,#0x8]`, body SP from the +naked `__stack_chk_fail` capture). + +Running with aim `0x2007cce0` and **no value filter**, so the transition logic is +what runs. Read `SMASH_CORE0[132]` (PC) and `[133]` (SP) after smashes accumulate; +`WATCH_LOG[8]` counts transitions and `[32]/[33]` hold the latest. + +## Config AF2 VOID — the board was power-cycled, which randomises `.uninit` + +The transition-latch window returned nothing usable because `WATCH_LOG` came back +as **pure random data** — no magic, every slot noise: + +``` +b053a832 137840ef f749666f 91caceb1 cb23c823 ... +``` + +That is the signature of `.uninit` being randomised, which happens on a **cold +power-on** (SRAM is uninitialised at power-up), not just after an OpenOCD rescue. +The board lost power during the ~10 hours since the 21:56 reset. + +Consequences, all of which invalidate the window: + +* `DWT_CATCH[8]` held random garbage at boot, so no valid watch was ever armed. +* `SMASH_CORE0[132]/[133]` only mirror the randomised `WATCH_LOG[32]/[33]`. +* Any planted aim or filter is gone. + +**Detector, for every future window: check `WATCH_LOG[0] == 0x3EE70001` (or +`ALLOC_GUARD[0] == 0xA1100001`) FIRST. Absent magic ⇒ cold boot ⇒ `.uninit` +randomised ⇒ every planted value is void, regardless of what the other slots +appear to say.** A randomised slot can hold a plausible-looking address, so this +is not something to eyeball. + +### What the long interval actually was + +The crash sector held only **3 records, seq 0..2**. Sequence numbers starting at +zero means all three are from the current boot, so the device was **not** crashing +for ten hours — it was unpowered for most of them and has been up only minutes. +No long-soak data was collected. + +### What survived and still counts + +* `SMASH_CORE0` re-initialised cleanly (magic `0x51150002`) and **`[6] = + 0x00000001`** — the corrupt canary reproduces a THIRD time, now on a fresh cold + boot with a different power-on history. The finding is robust. +* `SMASH_CORE0[1] = 0x2007ccd8` is unchanged, so `dec8`'s body SP and the derived + guard address `0x2007cce0` remain correct for this build; the aim did not need + re-deriving. + +### Config AF3 — re-armed + +Same experiment, re-planted: aim `0x2007cce0`, value filter cleared to 0 so the +guard→non-guard transition logic runs. Instruments zeroed, sector blanked, reset. + +## Config AF3: the transition latch fires — but it is catching benign recycling + +Magic valid on both instruments (`WATCH_LOG[0] = 0x3EE70001`, +`ALLOC_GUARD[0] = 0xA1100001`), so this window is real. + +``` +WATCH_LOG[8] = 270 transitions +WATCH_LOG[32] = 0x2000140a latest transition PC +WATCH_LOG[33] = 0x2007ccd8 latest transition SP +WATCH_LOG[13] = 34,822,509 total writes (liveness) +SMASH_CORE0[3] = 104 smashes +SMASH_CORE0[6] = 0x00000001 guard corrupt — FOURTH confirmation +SMASH_CORE0[132] = 0x2000140a snapshot of the last transition +SMASH_CORE0[133] = 0x2007ccd8 +``` + +### What the captured PC is + +`0x2000140a` = `Instructions::ld16+0xa` (`mov r11, r1`) — a resume artifact. The +store is the multi-register push two instructions earlier: + +``` +20001400: push {r4, r5, r6, r7, lr} ; 20 bytes +20001402: add r7, sp, #0xc +20001404: push.w {r7, r8, r9, r10, r11} ; 20 bytes <- wrote the guard slot +``` + +With `sp_at_fault = 0x2007ccd8` that push covers `0x2007ccd8..0x2007cce8`, and +`0x2007cce0` is its THIRD word — **r9**. So `r9 = 1` at `ld16` entry, which +explains the `0x00000001` payload exactly. + +### Why this is almost certainly NOT the culprit + +`ld16` is a sibling Sm83 instruction function. A sibling's prologue overwriting +the word *after `dec8` has returned* is benign recycling — precisely the case +`dwt_watch`'s own comment flags: "a benign one happens on every normal return when +the slot is recycled (measured 1110 transitions vs 16 smashes)". Here it is 270 +transitions against 104 smashes, so benign events are still in the mix, and the +"last transition" snapshot can easily land on one. + +Declaring `ld16` the stray store would repeat exactly the failure mode that +produced the withdrawn "the canary is INTACT" entry: reading a plausible number +without checking whether it can mean what it appears to mean. + +### The refinement: the depth gate + +While `dec8`'s frame is live, nothing should legitimately write `0x2007cce0`. Its +callees run BELOW `0x2007ccd8`, so their frames cannot reach it; only a shallower +or equal frame can, and that means `dec8` already returned. So: + +* `sp_at_fault == 0x2007ccd8` (or above) ⇒ same-depth or shallower ⇒ RECYCLING. +* `sp_at_fault < 0x2007ccd8` ⇒ a strictly deeper frame writing a word above its + own frame ⇒ **a genuine stray store**. + +`dwt_watch` already has this gate (`WATCH_LOG[5]`, predicate +`body_sp == 0 || sp_at_fault <= body_sp`) and it has never been used on this +victim. Planting `0x2007ccd4` — one word below `dec8`'s body SP — admits only +strictly-deeper writers and rejects every same-depth sibling prologue. + +Running with aim `0x2007cce0`, depth gate `0x2007ccd4`, no value filter. + +## Config AF4: the depth gate did NOT apply to the transition latch (my error) + +Valid window (both magics). `[13] = 29,702,288` writes, `[8] = 368` transitions, +`SMASH_CORE0[3] = 129` smashes, `[6] = 0x00000001` — **fifth confirmation** the +guard is genuinely corrupt. + +### The experiment did not do what I intended + +`WATCH_LOG[5]` gates only the CAUGHT/ring predicate. In `dwt_watch.rs` the +**transition latch lives in a different block** and is not gated at all, so +planting `0x2007ccd4` filtered `CAUGHT` (10,673,158) and left the transition +latch exactly as before. Any conclusion about "gated transitions" from this window +would be false. + +**To actually test it, the depth gate has to be applied inside the transition +block — that is a code change, not a planted value.** + +### What the snapshot nonetheless shows + +``` +WATCH_LOG[32]/[33] = 0x2000140a / 0x2007ccd8 live latest (ld16, recycling) +SMASH_CORE0[132]/[133] = 0x200010dc / 0x2007cd00 last transition before a smash +``` + +`0x200010dc` = **`Sm83::take_pending_interrupt+0x2a`**: + +``` +200010ca: movs r4, #0x1 <- r4 = 1, matches the payload +200010d2: bics r2, r4 +200010d4: strb r2, [r3] <- BYTE STORE THROUGH A RAW POINTER +200010d6: pop.w {r4, r6, r7, lr} +200010dc: cmp r0, #0x0 <- the recorded (resume) PC +``` + +**Not a conclusion — the arithmetic does not close.** With `sp_at_fault = +0x2007cd00` taken after that `pop`, the function's body SP is `0x2007ccf0` and its +prologue push spans `0x2007ccf0..0x2007ccfc`, which never reaches `0x2007cce0`. +And a `strb` cannot turn `0x2B7E1516` into a full-word `0x00000001` in one store. +The PC is a deferred resume artifact and the handler's re-read of the value is +equally late, so neither pins the store. + +What IS worth carrying forward: `take_pending_interrupt` stores through a raw +pointer (`strb r2,[r3]`), the same shape as the `with_cartridge` MPU violation +(`strb r0,[r1,r10]`) — a byte store into GB memory through a computed pointer. +Two independent failure sites, one instruction pattern. If a GB-memory base or +index can go wrong, both are explained. + +### Next: gate the transition latch in code + +Apply `body_sp` to the transition block so same-depth/shallower recycling is +rejected there too, then re-run. Note the criterion needs care: a prologue `push` +writes ABOVE its final SP, so "deeper SP" alone does not imply "stray". The clean +test is **a store to an address BELOW the writer's own SP**, which no prologue +push and no `[sp,#+N]` local access can produce. + +## Config AG: the below-SP discriminator, implemented in code + +`WATCH_LOG[5]` could never filter the transition latch — it gates only the +CAUGHT/ring predicate, and the latch is a separate block. This is the code change +that actually applies a filter there. + +### Why "below SP" and not "deeper SP" + +AF3/AF4 kept latching `ld16`'s prologue `push.w {r7,r8,r9,r10,r11}`, which is +benign recycling. **A depth test cannot reject it**: a prologue push writes its +words at `sp_at_fault .. sp_at_fault + n*4`, i.e. ABOVE its own final SP, so a +pusher always looks "deeper" than the word it just wrote. + +What no legitimate access can do is write BELOW the current SP. Locals are +`[sp,#+N]`; a push leaves SP at or below every word it stored; and AAPCS/Thumb has +no red zone. So `watched < sp_at_fault` is a store into dead stack — exactly the +signature of a stray pointer, which is what the smash record already says this is +(guard word alone corrupt, both neighbours sane). + +### The change + +In the transition block of `dwt_watch::debug_monitor_rust`: + +* `[46]`/`[47]` keep the UNFILTERED last transition (the old `[32]`/`[33]` + behaviour) so both readings can be compared. +* `[32]`/`[33]` now record ONLY transitions where `watched < sp_at_fault` — and + `__stack_chk_fail` already snapshots those two words into + `SMASH_CORE0[132]/[133]`, so the smash record now carries the FILTERED answer. +* `[48]` counts them. Expect `[48]` << `[8]`. + +**If `[48]` stays 0 while smashes continue, that is a real result, not a null:** +the guard is not being corrupted by a CPU store this core can see. DMA and the +other core are both invisible to this DWT, and that would become the next line. + +### Probe recovery (worth recording) + +The first flash attempt failed with `Command ID in response (0x5) does not match +sent command ID (SwjSequence - 0x12)` — a stuck CMSIS-DAP that `rb-flash`'s own +USB reset did not clear, and which then broke plain `probe-rs read` too. The +documented OpenOCD `set RESCUE 1` sequence recovered it in one pass. As expected, +the rescue RANDOMISED `.uninit` (first read back `0xc5396095`). + +### Layout moved — re-derived and re-checked + +`ALLOC_GUARD` is now the LAST `.uninit` object (it was `WATCH_LOG`), so the MPU +tail check had to be redone against a different array: + +``` +REGION_FAIL 0x20067040 HEARTBEAT 0x20067180 MM_REGS 0x200672c0 +DWT_CATCH 0x20067488 SMASH_CORE0 0x200674b0 SMASH_CORE1 0x20067730 +WATCH_LOG 0x200679b0 ALLOC_GUARD 0x20067ab0 _stack_end 0x20067b50 +zero span = 708 words from 0x20067040 +DWT_CATCH[8] = 0x200674a8 SMASH_CORE0[1] = 0x200674b4 +SMASH_CORE0[132] = 0x200676c0 [133] = 0x200676c4 +WATCH_LOG [0]=0x200679b0 [2]=0x200679b8 [8]=0x200679d0 [13]=0x200679e4 + [32]=0x20067a30 [46]=0x20067a68 [48]=0x20067a70 +``` + +MPU region base `0x20067b40`; highest written words are `WATCH_LOG[48]` +(`0x20067a70`) and `ALLOC_GUARD[15]` (`0x20067aec`, `LIVE_SLOTS = 16`) — both +safe. Device confirms `heartbeat placement OK: base=0x20067180 +core1-RO-region=0x20067b40`. CRC `0x65c2ee3a`. + +Running with NO aim planted, to capture a fresh `dec8` body SP from +`SMASH_CORE0[1]` rather than assuming `0x2007ccd8` survived the rebuild. + +## Config AG2: aim planted from a MEASURED body SP; a magic-check nuance + +### `WATCH_LOG[0]` is NOT a cold-boot detector — `ALLOC_GUARD[0]` is + +This window read `WATCH_LOG[0] = 0` while `ALLOC_GUARD[0] = 0xA1100001`. That is +**not** a randomised `.uninit`. `WATCH_LOG`'s magic is written by the DebugMonitor +handler, so with no aim planted the handler never runs and the magic never +appears. `ALLOC_GUARD`'s magic is written at init on every boot. + +**Use `ALLOC_GUARD[0] == 0xA1100001` as the cold-boot / rescue detector.** Reading +`WATCH_LOG[0] == 0` as "power cycle" would have wrongly voided a good window. + +### The fresh body SP + +``` +SMASH_CORE0[1] = 0x2007ccd8 fresh body SP — UNCHANGED across the rebuild +SMASH_CORE0[2] = 0x20001bcf dec8+0x16e +SMASH_CORE0[3] = 32 smashes +SMASH_CORE0[6] = 0x00000001 guard corrupt — SIXTH confirmation +``` + +The guard is therefore `0x2007ccd8 + 8` = `0x2007cce0` — the same value as the AE +layout, but now MEASURED in this build rather than carried across, which is the +rule that voided Z1 when it was skipped. + +### Loose end (G) reproduces in the new layout + +`DWT_CATCH[9]` (`0x200674ac`) read back `0x2007ccc0` after being explicitly +zeroed. In the AG layout `SMASH_CORE0` starts at `0x200674b0`, so `DWT_CATCH[9]` +is once again `SMASH_CORE0[-1]`: the smash writer stores one word below its array +on every smash. Harmless for the current experiment (the core-1 aim is unused) but +it silently corrupts whatever neighbour precedes `SMASH_CORE0`, and it will keep +moving with the layout until it is fixed. + +### Running + +Aim `DWT_CATCH[8] = 0x2007cce0` (readback confirmed), value filter 0, instruments +zeroed, sector blanked, reset. The readout that matters: + +* `WATCH_LOG[48]` — count of transitions where the store landed BELOW the writer's + own SP. +* `SMASH_CORE0[132]/[133]` — the PC/SP of that store, snapshotted at smash time. +* `WATCH_LOG[46]/[47]` — the unfiltered last transition, for comparison. + +## Config AG2 VOID — a shared global broke the filter; two claims retracted + +### The filter passed everything + +``` +[8] transitions = 178 +[48] STRAY count = 178 <- IDENTICAL: the below-SP test admitted every transition +[132]/[133] = 0x2000140a / 0x2007ccd8 (same as the unfiltered [46]/[47]) +SMASH_CORE0[6] = 0x00000001 (guard corrupt — seventh confirmation) +``` + +With `watched = 0x2007cce0` and `sp_at_fault = 0x2007ccd8`, `watched < sp_at_fault` +is FALSE, so `ld16` should have been rejected. Reading the three sources directly: + +``` +WATCHED_ADDR @0x20066c0c = 0x2007ccc0 <- WRONG +DWT_CATCH[8] @0x200674a8 = 0x2007cce0 <- the planted aim +DWT COMP0 @0xE0001020 = 0x2007cce0 <- what the hardware actually watches +``` + +`0x2007ccc0 < sp_at_fault` is true, so the test passed unconditionally. + +### Root cause of the instrument failure + +`WATCHED_ADDR` is a **single global shared by both cores**, and +`arm_data_write_watch` stores into it unconditionally. `multicore.rs:1609` arms +core 1 from `DWT_CATCH[9]`; when that slot held the stale `0x2007ccc0`, core 1's +arming **overwrote core 0's `WATCHED_ADDR`**. The handler then compared against +core 1's address while core 0's hardware watched a different word. + +**Fix:** the transition block now reads `DWT_COMP0` directly instead of +`WATCHED_ADDR`. COMP0 is per-core hardware state and is what actually triggered +the exception, so it cannot disagree with the trap that fired. Config AH. + +### RETRACTED: "`SMASH_CORE0[-1]` off-by-one" — disproven + +I recorded this as "CONFIRMED LIVE". It is false. `SMASH_CORE0` is +`[usize; 160]`, `DUMP_WORDS = 128`, and the writer touches `[0..4]`, `[4..132]`, +`[132]`, `[133]` — entirely in bounds. There is no off-by-one, and loose end (G) +is withdrawn. + +### RETRACTED: "a live writer of `DWT_CATCH[9]`" — tested and disproven + +`DWT_CATCH[9]` read back `0x2007ccc0` right after being written to 0, which +looked like a wild store hitting a fixed, cold, non-stack address within ~100 ms — +a far better handle than the recycled stack word, and tempting to pivot onto. + +**Tested before acting: three write-0/read-back trials all returned 0**, as did a +control word. There is no live writer. Grep confirms nothing in the firmware +writes `[9]` at all — only `[0]`, `[6]`, `[7]`, `[8]` are ever written, and +`main.rs`/`multicore.rs` only read it. The earlier readback was a write that did +not land across the flash/reset sequence. + +The lesson is the same one that produced the withdrawn "canary is INTACT" entry: +a plausible number is not a finding until the mechanism that would produce it has +been checked. + +### Config AH — running + +`dwt_watch` now uses `DWT_COMP0` for the below-SP test. Aim +`DWT_CATCH[8] = 0x2007cce0` and **`DWT_CATCH[9] = 0`** (so core 1 never arms and +cannot clobber the shared global), both verified before reset. Addresses are +unchanged from AG. CRC `0xe8269937`. + +## Config AH: the below-SP filter works but was the WRONG TEST — and DMA is excluded + +The `DWT_COMP0` fix made the filter functional: 310 transitions, **0** strays (no +longer the broken 178/178). Valid window (`ALLOC_GUARD[0]` magic present), aim +confirmed (`SMASH_CORE0[1] = 0x2007ccd8`), liveness 65,839,155 writes, 21 smashes, +`SMASH_CORE0[6] = 0x00000001` — **eighth confirmation** the guard is corrupt. + +### `[48] = 0` does NOT exclude a CPU store — my over-claim, corrected + +The below-SP test only catches a store BELOW the writer's own SP. **A wild pointer +from one of the victim's CALLEES writes UPWARD into the victim's frame — above its +own SP — and passes the test looking legitimate.** That is exactly the shape of +the suspected stray store, so the null excluded one narrow class and said nothing +about the main one. + +### Exclusions that ARE solid + +* **DMA is excluded.** Only two channels are configured and both are + memory→peripheral: `ch0 WRITE=0x50200010` (PIO0 TXF), `ch1 WRITE=0x40088008`. + All other channels are zeroed. Nothing DMAs into SRAM at all. +* **Core 1 is excluded.** `mpu.rs` maps core 0's stack read-only for core 1, so a + core-1 write there would raise MemManage — which has never appeared. +* **No guard-offset mismatch in dec8.** Prologue `20001a72: str r0,[sp,#0x8]` + stores the guard; epilogue `20001bba: ldr r0,[sp,#0x8]` reads it. Same offset. + (The "-8 mismatch" idea stays refuted, now checked for this victim too.) +* **dec8 does not clobber its own guard.** Its only SP-relative stores are the + guard store and `strd r5, r0, [sp]`, which writes `[sp]` and `[sp+4]` — not + `[sp+8]`. + +### The corrected discriminator (config AI) + +Any frame running BELOW the victim's body SP has no business writing the victim's +guard: its own locals are lower still, and the victim is live above it. + +``` +tsp < body_sp ⇒ a callee reaching UP into the victim ⇒ STRAY +tsp >= body_sp ⇒ same-depth or shallower ⇒ victim already returned ⇒ RECYCLING +``` + +That last case is what has been latching all along — `ld16`'s prologue push, and +in AH the unfiltered `[47]` was `0x2007cce0`, exactly the watched address (a push +whose final SP lands on the guard). + +`body_sp` is planted into `WATCH_LOG[5]`. Note it gates the CAUGHT/ring predicate +elsewhere; this is a deliberate SECOND use of the same planted value in a +different block — AG2 was voided by assuming one gate covered both. + +### Config AI — running + +Layout moved again (`DWT_CATCH=0x20067300`, `SMASH_CORE0=0x20067328`, +`WATCH_LOG=0x20067828`; `ALLOC_GUARD` still last at `0x20067ab0`). MPU bound +re-checked: `WATCH_LOG[48]=0x200678e8` and `ALLOC_GUARD[15]=0x20067aec`, both below +region base `0x20067b40`. CRC `0xbc74c12d`. Aim `0x2007cce0`, gate `0x2007ccd8`, +core-1 aim 0 — all readback-verified. + +## Config AJ: the watchpoint self-aims — planted values are now build-invariant + +### The methodological hole this closes + +There are TWO classes of address in these experiments and they were being handled +unequally: + +* **Forensics addresses (`.uninit` objects).** Re-derived from `llvm-nm` after + every build, with the MPU tail bound re-checked. This was already sound — it is + what caught `ALLOC_GUARD` becoming the last object instead of `WATCH_LOG`. +* **Planted STACK addresses (the aim and the depth gate).** These come from frame + layout, not from symbols, so any code change that alters a prologue shifts them. + They were being carried across rebuilds and verified only RETROSPECTIVELY, from + the next smash record — which means a whole window can be silently void before + the mistake surfaces. **Config Z1 was voided exactly this way**: added code grew + `run_core1_worker`'s frame and moved every SP by 8 bytes. + +Checked live mid-window and the exposure was real: `SMASH_CORE0[1]` read `0` (no +smash yet) while the device was already running with an aim and gate carried over +from the previous build. Nothing had verified them. + +### The mechanism + +`__stack_chk_fail` is `#[naked]`, so `SMASH_CORE0[1]` is the victim's true body SP +as measured BY THE RUNNING BUILD, and `SMASH_CORE0` is in `.uninit` so it survives +the crash handler's reset. Boot-time arming now: + +1. reads `DWT_CATCH[8]`; a value `< 0x1000` is a **guard OFFSET**, not an address; +2. requires `SMASH_CORE0[0] == SMASH_MAGIC`, else stays **disarmed** rather than + aiming at garbage; +3. computes `guard = SMASH_CORE0[1] + offset` and arms there; +4. publishes `SMASH_CORE0[1]` into `WATCH_LOG[5]` so the depth gate comes from the + SAME measurement — no second SWD plant that can drift out of sync; +5. still accepts a full SRAM address, so an arbitrary word can be watched. + +`SMASH_MAGIC` was made `pub` for step 2. + +### Why an offset and not an address + +The offset is a property of the victim's own code (`ldr r0,[sp,#0x8]`). The +absolute address is a property of every frame above it on the stack. So the +offset survives rebuilds that the address does not. + +**It is self-correcting instead of assumption-based:** first boot after a rebuild +has no smash record and stays disarmed; the first crash records the body SP; the +next boot arms at the address correct for THAT build. A rebuild now costs one +crash of latency rather than silently voiding a window. + +### Limits, stated up front + +* The OFFSET still needs re-deriving from the victim's disassembly if codegen + changes its frame. +* If a layout change makes a different function the dominant victim, + `SMASH_CORE0[2]` (the lr) reveals it and the offset must be re-derived for that + victim. + +### Running + +CRC `0x7731bd5a`. `DWT_CATCH[8] = 8` (offset), `[9] = 0`, `WATCH_LOG[5] = 0` (the +firmware fills it). Addresses unchanged from AI: `DWT_CATCH=0x20067300`, +`SMASH_CORE0=0x20067328`, `WATCH_LOG=0x20067828`, `ALLOC_GUARD=0x20067ab0`. +Expect the boot log to show `self-aim: no smash record yet` on the first boot and +`self-aim: body_sp=... -> ...` on every boot after the first crash. + +## Config AJ: the self-aim WORKS — and it proves the SP-based filters cannot work + +### The self-aim verified end to end + +``` +ALLOC_GUARD[0] = 0xA1100001 valid window +SMASH_CORE0[1] = 0x2007ccd8 body SP measured by the running build +WATCH_LOG[5] = 0x2007ccd8 == [1] firmware published the gate +DWT COMP0 = 0x2007cce0 == [1] + 8 firmware resolved the offset +SMASH_CORE0[2] = 0x20001bcf dec8+0x16e — same victim, offset still valid +[3] = 21 smashes, [6] = 0x00000001 NINTH confirmation the guard is corrupt +``` + +No stack address was planted; only the offset `8`. This removes the whole class of +"aim carried across a rebuild" errors that voided Z1. + +### The filter discriminated — but caught the wrong thing + +``` +[8] transitions = 983 +[48] STRAY = 109 (a clear minority, so the gate IS selective) +[46]/[47] unfiltered = 0x2000140a / 0x2007ccd8 (ld16, as always) +SMASH_CORE0[132]/[133] = 0x200012f6 / 0x2007ccd0 +``` + +`0x200012f6` = `Instructions::ld8+0xa`, a resume artifact. The store is +`200012f0: push.w {r4,r5,r6,r7,r8,r9,r11}` (7 regs, 28 bytes). With +`sp_at_fault = 0x2007ccd0` that push spans `0x2007ccd0..0x2007cce8`, and +`0x2007cce0` is its FIFTH word = **r8**. So `r8 = 1` at ld8's entry — precisely +mirroring `ld16`'s `r9 = 1`. + +### Why the deeper-frame gate is CONFOUNDED + +The gate assumed **deeper SP ⇒ a callee of the live victim**. That is false. +`ld8` is a SIBLING whose entry SP is 8 bytes below dec8's (`0x2007cd00` vs +`0x2007cd08`) — both have 48-byte prologues, so they are simply dispatched from +paths at slightly different depths. **A sibling entered deeper has a deeper body +SP, and its prologue push writes UPWARD into the recycled word — indistinguishable +from a callee reaching up, using SP alone.** The 109 "strays" are almost certainly +all of this kind. + +So all three SP-based predicates have now failed for the same underlying reason: + +* address+value — the word has many legitimate owners; +* below-SP — a callee writing up passes it; +* deeper-frame — a deeper-entered SIBLING passes it too. + +**SP cannot separate "recycling after the victim returned" from "a store while the +victim is live". Liveness is the property that matters and SP does not encode it.** + +### The instrument is also saturated + +`[13] = 135,364,731` writes this window ⇒ the handler runs ~68,000 times/second. +`dwt_watch`'s own comment warns the DebugMonitor is configurable-priority, so +PRIMASK defers it and writes coalesce. The "last transition before a smash is the +killing store" guarantee depends on an ordering the instrument can no longer +honour at this rate. Treat ordering-derived conclusions from this word as unsafe. + +### What would actually decide it + +Stop filtering by SP and test liveness directly. The corruption happens between +dec8's guard STORE (`20001a72`) and its CHECK (`20001bba`). A software bisection +inside dec8 — re-reading the guard after each call it makes and latching the first +point where it is no longer `0x2B7E1516` — brackets the corruption in program +order rather than by address arithmetic, and is immune to both the recycling +confound and handler deferral. + +## The three failure sites are ONE code path: the GB IO write, and its base pointer + +Rather than bisect dec8, disassembling what it calls made the bisection +unnecessary. dec8 makes 7 calls; the last before its guard check (22 bytes later, +`20001ba4: bl Sm83::bus_write` → check at `20001bba`) is the GB bus write. + +### `bus_write` and `with_cartridge` contain the SAME store + +``` +bus_write @0x20002414 with_cartridge (the MPU violation) + movw r0, #0x4120 movw r1, #0x4120 + movw r0, #0xff80 and.w r1, r6, r11 (r11 = 0xff80) + cmp.w r0, #0xff00 cmp.w r1, #0xff00 + sxth r0, r5 sxth r1, r6 + add r0, r6 add r1, r5 + strb.w r8, [r0, r1] (r1=0x4100) strb.w r0, [r1, r10] (r10 = 0x4100) +``` + +Identical: `addr = base + sxth(gb_addr) + 0x4100`. These are the same source +inlined twice. `take_pending_interrupt`'s `strb r2,[r3]` is the third instance of +"byte store into GB memory through a computed pointer". + +### The offset arithmetic CANNOT escape — only the base can be wrong + +The guard `(gb_addr & 0xFF80) == 0xFF00` admits exactly `0xFF00..0xFF7F` (the mask +covers bits 15:7, and `sxth` uses only the low 16 bits, so a value above 0xFFFF +cannot widen the range). So `sxth` yields `-0x100..-0x81` and the effective offset +is `base + 0x4000 .. base + 0x407F` — a fixed 128-byte window. + +**The address computation is bounded. The only free variable is the base.** And +the base is precisely what the MPU violation measured as wrong: +`base = 0x2007cd00` (a core-0 stack address), giving +`0x2007cd00 + 0x4010 = 0x20080d10` — the byte-identical fault address seen five +times. + +### Why this supersedes the DWT work + +Every DWT predicate tried to identify the *store*. But all three candidate stores +are the same instruction pattern, and the instruction is innocent — it is bounded +and correct given a valid base. **The defect is upstream: the GB memory base +pointer is intermittently a stack address.** That single fact explains: + +* `with_cartridge`'s MPU violation (base + 0x4010 lands in core 1's stack); +* the canary smashes (a different bad base value puts the 128-byte IO window over + core 0's stack, and the guard word is a single corrupted word with sane + neighbours — exactly a one-byte-per-call scatter, not a memset overrun); +* why the payload is a small integer (`1`) — it is an IO register VALUE being + written, not a pointer. + +### The next instrument — software, immune to DWT saturation + +Validate the base at the top of `bus_write`: if `self`'s memory base is outside +the heap arena (`0x200244dc..0x2004c4dc`), latch the base, the GB address, and the +caller's LR into a `.uninit` slot. That is ~4 instructions on the hot path, tests +the hypothesis directly, and needs no DWT — so neither the recycling confound nor +handler deferral applies. + +Note `bus_write` lives in `rustyboy_core`, so the check needs a `#[no_mangle]` +static there (or a feature gate) rather than a pico2w-local array. + +## Config AK: the base-pointer probe, in software + +`Sm83::bus_write(&mut self, memory: &mut GameBoyMemory, addr: u16, val: u8)` — the +base is the `memory` reference. `check_bus_base` now runs at its top. + +### The invariant chosen, and why + +Not "is the base inside the heap arena". `GameBoyMemory` is constructed once and +lives for the whole run, so **its address is invariant**; the check compares +against the FIRST base observed after a cold boot. That needs no arena constant, +so it stays platform-agnostic in `rustyboy_core` and — importantly — cannot rot +when the heap or linker layout moves, which is the failure mode that has voided +several windows in this investigation. Any deviation is corruption by definition. + +`BUS_BASE_CHK` slots: `[0]` magic `0xB0A50001`, `[1]` first-seen base, +`[2]` **mismatch count**, `[3]` first bad base, `[4]` the GB address at that +moment, `[6]` most recent bad base. + +### Why software and not a fourth watchpoint + +Three DWT predicates failed for one reason: they tried to identify the STORE, but +the store is shared between `bus_write`, `with_cartridge` and +`take_pending_interrupt` (same source inlined), is bounded, and is +indistinguishable from the prologue pushes that legitimately recycle the same +stack word. SP cannot express "the victim's frame is live". This check tests the +actual suspect — the base — and is immune to both the recycling confound and +DebugMonitor deferral. + +### Layout after the rebuild (BUS_BASE_CHK moved everything again) + +``` +REGION_FAIL 0x20067070 BUS_BASE_CHK 0x20067110 HEARTBEAT 0x200671d0 +MM_REGS 0x20067310 DWT_CATCH 0x20067350 SMASH_CORE0 0x20067378 +WATCH_LOG 0x20067878 ALLOC_GUARD 0x20067b00 _stack_end 0x20067ba0 +zero span = 716 words from 0x20067070 (NOT 816 — see below) +DWT_CATCH[8]=0x20067370 [9]=0x20067374 +SMASH_CORE0[1]=0x2006737c [2]=0x20067380 [3]=0x20067384 [6]=0x20067390 +WATCH_LOG[0]=0x20067878 [5]=0x2006788c [8]=0x20067898 [13]=0x200678ac [48]=0x20067938 +BUS_BASE_CHK[2]=0x20067118 [3]=0x2006711c [4]=0x20067120 [6]=0x20067128 +``` + +MPU region base `0x20067ba0`; highest written words `WATCH_LOG[48]=0x20067938` and +`ALLOC_GUARD[15]=0x20067b3c` — both safe. + +**Procedure slip worth recording:** the zeroing loop was run with 816 words rather +than 716, so it also cleared 400 bytes at the very bottom of core 0's stack. +Harmless here (a reset followed immediately, and that depth is only reached at +maximum nesting), but the span must be recomputed from `_stack_end` after every +rebuild, not carried over. + +### Running + +CRC `0xd429d04e`. Offset `8` planted in `DWT_CATCH[8]`, `[9] = 0`, both verified. + +## Config AK result: the base probe REFUTES the hypothesis — and I probed the WRONG pointer + +### The measurement + +``` +ALLOC_GUARD[0] = 0xA1100001 valid window +BUS_BASE_CHK[0] = 0xB0A50001 probe active +BUS_BASE_CHK[1] = 0x2002753c first base — inside the heap arena ✓ +BUS_BASE_CHK[2] = 0 MISMATCHES: none +SMASH_CORE0[3] = 26 smashes +SMASH_CORE0[6] = 0x00000001 guard corrupt — TENTH confirmation +``` + +Across 26 smashes the `memory` base reaching `bus_write` was never wrong. + +**Rigour gap:** no call counter was recorded, so "0 mismatches" is only as strong +as the number of calls sampled. `[1]` being set proves at least one call, not +millions. Any future probe of this shape must count calls. + +### The bigger error: that is not the pointer that faults + +Tracing the faulting store to its source: `multicore.rs` does +`self.lcd_timing_io[(addr - IO_REG_BASE) as usize] = value`, where +**`lcd_timing_io: [u8; 0x80]` — a 128-byte array**, exactly the window size +derived from the instruction sequence, indexed by `addr - 0xFF00`. That is the +`sxth`/`+0x4100` pattern. + +`lcd_timing_io` belongs to **`Core1Transport`**, not `GameBoyMemory`. And it sits +EARLY in `Core1Transport`, so the `+0x4100` in the store means the base points at +an ENCLOSING object (`GameBoy`/`PicoGameBoy`, whose `Sm83` opcode table is large) +with `transport.lcd_timing_io` at that offset. + +**So `bus_write`'s base (`&mut GameBoyMemory`) and the faulting store's base are +different pointers.** The AK probe watched a pointer that was never implicated. +Its null is真 but says nothing about the fault. + +### What still holds + +`0x2007cd00` is `with_cartridge`'s entry SP − 0x38. A ≥16.7 KB object cannot live +there — `with_cartridge`'s whole frame is 4720 bytes. So that base is genuinely +bad, not a legitimate stack-local object. The corrupt-base idea survives; only my +choice of which pointer to instrument was wrong. + +### Next + +Move the same invariant check to `write_lcd_timing_register`'s `self` (the +`Core1Transport`/enclosing pointer) — the pointer the faulting store actually +uses — and add a CALL COUNTER so a null result is quantified. `with_cartridge` +runs once per boot, which matches the MPU violation appearing about once per boot +and being byte-identical, so the check should also record whether the bad base +occurs during initialisation or in the steady-state loop. + +## AL — the faulting store's base is RELOADED FROM A STACK SPILL SLOT + +Disassembling `with_cartridge` in `elf-AK` around the `+0x4100` sites finally +shows where the base comes from. Two inlined copies of +`write_lcd_timing_register` appear in that function: + +``` +1001a7f8: and.w r1, r6, r11 ; r11 = 0xff80 +1001a7fc: cmp.w r1, #0xff00 +1001a800: bne ... +1001a802: sxth r1, r6 +1001a804: add r1, r5 ; base in a CALLEE-SAVED REGISTER +1001a806: strb.w r0, [r1, r10] ; r10 = 0x4100 + +1001a8a6: bne ... +1001a8a8: ldr r2, [sp, #0x10] ; base RELOADED FROM A STACK SLOT +1001a8aa: sxth r1, r1 +1001a8ac: add r1, r2 +1001a8ae: strb.w r0, [r1, r11] ; r11 = 0x4100 +``` + +`with_cartridge`'s frame: `push {r4,r5,r6,r7,lr}` / `add r7,sp,#0xc` / +`push.w {r8-r11}` / `sub.w sp,sp,#0x1240` / `sub sp,#0xc` — 4720 bytes, so +`sp = entry_sp - 0x1270`. Three different object pointers are spilled here: +`[sp,#0x8]` (the sret pointer), `[sp,#0xc]` (base for `strbeq.w r0,[r1,#0x2bb]`) +and `[sp,#0x10]`. + +Two consequences. + +**1. The recorded bad base 0x2007cd00 is `r7 - 0x30`.** After the two pushes +`r7 = entry_sp - 0x8`, so `r7 - 0x30 = entry_sp - 0x38` — and `sub.w r4, r7, +#0x30` appears *twice* in this function (0x1001a7c0, 0x1001a868) as the address +of the `RangeInclusive` loop iterator that is passed to +`RangeInclusive::next` in r0. So the value that showed up in the base position +is not garbage: **it is another live local's address from the same frame.** +That is the signature of a base register/slot taking on a neighbouring value, +not of a wild pointer arriving from outside. + +**2. The base can itself be a VICTIM of the stack corruption.** If the base is +reloaded from `[sp,#0x10]` and something is scribbling on core 0's stack, the +corrupt base is a *second-order effect*, not the primary cause — the +chicken-and-egg the corrupt-base theory has to resolve. Note the two paths +differ: the once-per-boot MPU violation happens inside `with_cartridge`, whose +base lives on the stack; the continuous canary smashes happen in the +steady-state loop, where the base is `&mut Core1Transport` reached through the +heap-allocated `GameBoy`. + +### Config AL — the counted experiment that separates them + +`check_bus_base`/`BUS_BASE_CHK` (config AK, refuted — wrong pointer) is +REMOVED. In its place, `check_lcd_base` in `multicore.rs` watches +`self.lcd_timing_io.as_ptr()` — the exact quantity the faulting store forms +(`base_reg + 0x4000`) — at the top of `write_lcd_timing_register`, which is +`#[inline(always)]` and therefore instruments *all* inlined copies including +the one in `with_cartridge`. + +It is a **histogram, not a first-wins reference** (a single reference would fire +spuriously if `with_cartridge` legitimately builds the object in a stack +temporary and memcpys it to the sret destination), and it **counts calls** +(rule 9 — AK's "0 mismatches" was uncounted and therefore worthless): + +``` +LCD_BASE_CHK[0] magic 0x1CD00001 +LCD_BASE_CHK[1..5] up to four distinct bases +LCD_BASE_CHK[5..9] per-base call counts +LCD_BASE_CHK[9] total calls +LCD_BASE_CHK[10] calls that saw a FIFTH+ distinct base +LCD_BASE_CHK[11] most recent such base +``` + +Pre-reset smoke test (a few hundred ms of running firmware) already read back +`[0]=0x1cd00001 [1]=0x2003d840 [5]=0x103 [9]=0x103 [10]=0` — one base, inside +the heap arena, 259 calls, no overflow. The probe works. + +**Decision rule.** If after a full window `[9]` is in the millions with a single +base and `[10]=0`, then the faulting store's base is NEVER wrong in the +steady-state loop, and this store **cannot** be the canary smasher — the +corrupt-base theory survives only for the once-per-boot `with_cartridge` MPU +violation, and the smashes need a different writer. If `[10]` or a second slot +is non-zero, the bad base is captured directly along with how often it occurs. + +### AK final state (77 min, 0.77 crashes/min) + +`SMASH_CORE0 = [0x51150002, 0x2007ccd8, 0x20001bcf, 0x3b, 0x2b7e1516, +0x2007cd0c, 0x00000001, 0x2007d2d0]` — 59 smashes, same victim, same body SP, +and the guard word `0x00000001` for the **eleventh** time. +`BUS_BASE_CHK[2] = 0` (uncounted, and refuted anyway). + +### Layout (config AL — re-derived) + +``` +REGION_FAIL 0x200670a8 XIP_CHECK 0x20067148 +LCD_BASE_CHK 0x200671e8 DATA_CHECK 0x20067218 +ALLOC_GUARD 0x200672b8 HEARTBEAT 0x20067358 +RAM_CHECK 0x20067498 DWT_CATCH 0x20067538 +SMASH_CORE0 0x20067560 SMASH_CORE1 0x200677e0 +WATCH_LOG 0x20067a60 MM_REGS 0x20067ba8 +_stack_end 0x20067be8 +``` + +Zero span = **720 words from 0x200670a8**. Core-1 RO region base = +`_stack_end & !0x1F` = **0x20067be0**, which covers `MM_REGS[14..16]` — never +written (the handler writes `[0..=12]`), so the tail is safe. +`DWT_CATCH[8] = 0x20067558` (offset 8 planted), `[9] = 0x2006755c`. +`SMASH_CORE0[3] = 0x2006756c` (smash count), `[6] = 0x20067578` (the guard). +`LCD_BASE_CHK[9] = 0x2006720c`, `[10] = 0x20067210`, `[11] = 0x20067214`. +Code addresses unchanged from AK: `bus_write` 0x20002414, `ld8` 0x200012ec, +`ld16` 0x20001400, `take_pending_interrupt` 0x200010b2, +`with_cartridge` 0x1001a2b8. + +Flashed 15:14, CRC `0x0a97a137`, **reset 15:17:29 on 2026-08-15**. + +## AL result — the corrupt-base theory is DEAD, twice over + +### 1. The counted histogram (the pre-registered rule fired) + +38 min window, 167 smashes (`SMASH_CORE0[3]=0xa7`): + +``` +LCD_BASE_CHK[0] = 0x1cd00001 probe active +LCD_BASE_CHK[1] = 0x2003d840 ONE distinct base +LCD_BASE_CHK[2..5] = 0 no second, third or fourth base +LCD_BASE_CHK[5] = 0x56091 352,401 calls on that base +LCD_BASE_CHK[9] = 0x56091 352,401 total calls +LCD_BASE_CHK[10] = 0 no fifth-base overflow +LCD_BASE_CHK[11] = 0 +``` + +352,401 calls, one base, zero anomalies, alongside 167 smashes. If this store +were the smasher it would need ~1 bad base per 2,110 calls; zero were seen. +**The faulting store's base is never wrong in steady state.** + +Correction to the AL note: `0x2003d840` is **not** in the heap arena. It +resolves to `__embassy_main_task_inner_funct + 0x19300` in `.bss` — the +`PicoGameBoy` lives inside the embassy main task's future, not on the heap. + +### 2. A static refutation that should have come first + +`self.lcd_timing_io[i] = value` compiles to **`strb`** — a BYTE store. The +corrupted guard word is `0x00000001`, i.e. **all four bytes** differ from +`0x2B7E1516`. A byte store cannot produce that. **The `lcd_timing_io` store +could never have been the canary smasher, and no measurement was needed to know +it.** (New rule 21.) + +Both legs are gone. The corrupt-base theory is retired; do not revive it. + +## AL — a NEW, reproducible core-0 HardFault with a ~4-instruction window + +27 records before the sector stopped filling (seq 0..26, strictly alternating +Panic / WDT-or-HF). 14 Panic (the usual `dec8` smash, line 169, lr 0x20001bcf), +7 WDT, **6 HardFaults that are byte-identical apart from the PC**: + +``` +kind=HF flags=0x47 cfsr=0x00020000 (INVSTATE) hfsr=0x40000000 (FORCED) +pc = 0x2003bd24 | 0x2003bcf4 | 0x2003bc28 lr = 0x20001fff +r12 = 0x1001473f (cb::sla_u8|1) | 0x100146cb (cb::rl_u8|1) +sp_before = 0x2007cd08 +``` + +Resolved against `elf-AL`: + +- **`lr = 0x20001fff`** is the return address of `20001ffa: bl set_r8_enum`, + inside `Sm83 ... Instructions::cb`. So the fault is *inside* `set_r8_enum`. +- **`pc = 0x2003bd24`** is `__embassy_main_task_inner_funct + 0x177e4` — a + **`.bss` DATA address**, ~0x1B1C below the emulator object's own base. Even, + hence INVSTATE. All three PCs lie within 0xFC of each other. +- **`r12`** is a correct `cb::*` pointer left over in the scratch register by + the preceding `.data` long-branch thunk — not the culprit. +- `flags` bit 0x20 (FAULT_ON_CORE1) is clear: **this is core 0.** + +`set_r8_enum` (0x200025c4) is a five-instruction leaf: + +``` +200025c4: push {r7, lr} +200025c6: mov r7, sp +200025c8: uxtb r1, r1 +200025ca: tbb [pc, r1] +200025d6: strb r2, [r0, #0x1c] (one of eight) +200025d8: pop {r7, pc} +``` + +So the saved LR at `sp+4` was replaced **between the `push` and the `pop`** — +no call, no loop, no SP movement in between. A `tbb` overrun is excluded: its +reach is `0x200025ce + 2*255 = 0x200027cc`, nowhere near `.bss`. + +**Both victims live in the same narrow band of core 0's stack:** `dec8`'s guard +at `0x2007cce0`, `set_r8_enum`'s saved LR at ~`0x2007ccfc`–`0x2007cd04`. About +0x1C apart. But the payloads differ — `0x00000001` at one, a *pointer into the +emulator's own `.bss` object* at the other — so this is not one memset. + +### Config AM — snapshot the band + +Added `HF_STACK: [u32; 28]` (`.uninit`), filled in `hard_fault_rust` gated on +`CFSR & INVSTATE`, write-once, magic last: + +``` +HF_STACK[0] magic 0x48F50001 +HF_STACK[1..7] pc, lr, cfsr, sp_before, r12, exc_return +HF_STACK[7..27] twenty words spanning sp_before-40 ..= sp_before+36 +``` + +**Decision rule.** If exactly one word in the band is wrong and its neighbours +are sane return addresses and locals, the writer is a *stray single-word store* +and the `-Z stack-protector` premise ("a contiguous stack-buffer overrun") is +wrong. If a run of consecutive words is wrong, it is an overrun and the run +length gives its size. Either way the surrounding words identify which frames +were live, which no DWT predicate has been able to establish. + +### Layout (config AM — re-derived) + +``` +REGION_FAIL 0x200670b0 DATA_CHECK 0x20067150 +HEARTBEAT 0x200671f0 RAM_CHECK 0x20067330 +XIP_CHECK 0x200673d0 LCD_BASE_CHK 0x20067470 +ALLOC_GUARD 0x200674a0 DWT_CATCH 0x20067540 +SMASH_CORE0 0x20067568 SMASH_CORE1 0x200677e8 +WATCH_LOG 0x20067a68 HF_STACK 0x20067bb0 +MM_REGS 0x20067c20 _stack_end 0x20067c60 +``` + +Zero span = **748 words from 0x200670b0**. `_stack_end` is now 32-byte aligned, +so the core-1 RO region base equals it exactly and **no `.uninit` object falls +inside the region** — the tail hazard is gone this build. +`DWT_CATCH[8]=0x20067560` (offset 8 planted), `[9]=0x20067564`. +`SMASH_CORE0[3]=0x20067574`, `[6]=0x20067580`. +`LCD_BASE_CHK[9]=0x20067494`, `[10]=0x20067498`. + +Flashed 16:01 (first attempt timed out on "Failed to reset, and then halt"; +recovered with the OpenOCD rescue), CRC `0x2230fda1` OK, +**reset 16:05:10 on 2026-08-15**. + +**New rules.** (21) **CHECK THE STORE WIDTH AGAINST THE CORRUPTION WIDTH before +building any probe — a `strb` cannot corrupt a whole word.** (22) **RESOLVE A +FAULT PC AGAINST THE SYMBOL TABLE INCLUDING `.bss`/`.data` — a PC inside a data +object is an indirect branch through a corrupted pointer, and says so +immediately.** + +## AM result — the INVSTATE fault is DETERMINISTIC, and its payload is an `Arc` pointer + +38 min, 83 smashes (2.2/min). Sector floored at 31: **15 Panic** (the usual +`dec8` smash, line 169), **8 WDT**, **8 HardFaults — every one byte-identical**: + +``` +pc=0x2003bd2c lr=0x20001fff cfsr=0x00020000 (INVSTATE) +sp_before=0x2007cd08 r12=0x1001473f flags=0x47 (core 0) +``` + +Eight for eight, no variation. **This is a repeatable code path, not random +corruption.** + +### RETRACTION — the `.bss` claim in the AL section was wrong + +I resolved `0x2003bd2c` and `0x2003d848` with an `llvm-nm` nearest-preceding +-symbol bisect and reported them as `.bss`, inside the embassy main task future. +**Both are in fact inside the heap arena `0x200244dc..0x2004c4dc`.** The bisect +misled me because the arena is a large *unnamed* region that follows the huge +`__embassy_main_task_inner_funct` symbol, so the nearest preceding name is not +the containing object. The original AK reading ("inside the heap arena") was +right and my AL "correction" was the error. `PicoGameBoy` is heap-allocated. + +**New rule 23: CHECK A SYMBOL BISECT AGAINST KNOWN REGION BOUNDS (heap arena, +section limits) BEFORE TRUSTING IT — the nearest preceding symbol is not +necessarily the containing one.** + +### What `0x2003bd2c` actually is + +Dumping live memory there shows a perfectly regular 12-byte record: + +``` +0x2003bd08 = 1 0x2003bd14 = 1 0x2003bd20 = 1 +0x2003bd0c = 1 0x2003bd18 = 1 0x2003bd24 = 1 +0x2003bd10 = 0x08040405 0x2003bd1c = 0x08040406 0x2003bd28 = 0x100404ff +0x2003bd2c = 1 <-- fault PC +0x2003bd30 = 1 +0x2003bd34 = 0x08040400 +``` + +That is `ArcInner = { strong: usize, weak: usize, data }` with a 4-byte +`OpCode` payload — **the opcode table's `Arc` allocations.** The +fault PC lands exactly on an *allocation start*, i.e. on the **data half of an +`Arc` fat pointer**. The three AL PCs (`0x2003bd24`, `0x2003bcf4`, +`0x2003bc28`) differ by 0x30 and 0xCC — **both exact multiples of 12** — so they +are allocation starts too, just different opcodes. And the AM PC sits 8 bytes +above the AL PC, the same +8 the whole heap shifted by between builds +(`LCD_BASE_CHK[1]` went `0x2003d840` → `0x2003d848`); the offset +`lcd_timing_io − pc = 0x1B1C` is **identical across both builds**. + +So the word popped into PC is not garbage. **It is a live, well-formed `Arc` +pointer.** + +### The band, decoded + +`HF_STACK` (band = `sp_before-40 ..= sp_before+36`): + +``` +0x2007cce0 = 0x2002757c +0x2007cce4 = 0xfffffff9 +0x2007cce8 = 0x00000008 EXC FRAME r0 +0x2007ccec = 0x00000000 EXC FRAME r1 +0x2007ccf0 = 0x00000002 EXC FRAME r2 +0x2007ccf4 = 0x00000001 EXC FRAME r3 +0x2007ccf8 = 0x1001473f EXC FRAME r12 (matches ef.r12) +0x2007ccfc = 0x20001fff EXC FRAME lr (matches ef.lr) +0x2007cd00 = 0x2003bd2c EXC FRAME pc (matches ef.pc) +0x2007cd04 = 0x08000000 EXC FRAME xPSR (T bit CLEAR -> INVSTATE confirmed) +0x2007cd08 = 0x10036698 <== sp_before (a flash address - vtable-shaped) +0x2007cd0c = 0xffffffff +0x2007cd10 = 0x00000008 +0x2007cd14 = 0x2003d848 heap pointer +0x2007cd18 = 0x2003d758 heap pointer +0x2007cd1c = 0x2b7e1516 ** AN INTACT STACK GUARD ** +0x2007cd20 = 0x00000000 +0x2007cd24 = 0x2007cf00 stack pointer +0x2007cd28 = 0x00000000 +0x2007cd2c = 0x00000030 +``` + +**A design flaw in the AM probe: the exception frame occupies exactly +`[sp_before-32, sp_before)`, so it overwrites the very slot under +investigation** — `set_r8_enum`'s saved-LR slot at `sp_before-4` reads back as +the frame's own xPSR. The slot's true value is only recoverable as `ef.pc()`, +which we already had. + +**But the pre-registered rule still resolves on the rest of the band: +everything above `sp_before` is sane** — two heap pointers, a stack pointer, +small integers, and **an intact `0x2b7e1516` guard at `0x2007cd1c`**. No run of +consecutive bad words. **⇒ NOT a contiguous stack-buffer overrun. The +`-Z stack-protector` premise is wrong for this fault.** + +### Frame arithmetic + +`Instructions::cb` (0x20001eec): `push {r4,r5,r6,r7,lr}` + +`push.w {r7,r8,r9,r10,r11}` = **40 bytes, no `sub sp`**. Its SP at +`20001ffa: bl set_r8_enum` is therefore `sp_before = 0x2007cd08`, so `cb`'s +entry SP is `0x2007cd30` and `cb`'s own frame is `[0x2007cd08, 0x2007cd30)`. +**The corrupt slot `0x2007cd04` lies BELOW `cb`'s stack pointer** — it is +`set_r8_enum`'s own frame, not `cb`'s. + +`cb` has **no direct callers** in the disassembly: it is reached by `blx` +through the `Arc` vtable from `Sm83::step()`. Note `sp_before` +itself holds `0x10036698`, a flash address of exactly vtable shape. + +### Config AN + +- `HF_STACK` widened to 48 words: `[7..47]` now spans + `sp_before-80 ..= sp_before+76`, covering `cb`'s whole 40-byte frame **and** + its caller's — the context needed to decide which frame the `Arc` pointer + legitimately belongs to. +- `MM_REGS` padded from 16 to 20 words. At 16 it ended at `0x20067cb0`, so + `_stack_end & !0x1F` put core 1's read-only region base at `0x20067ca0` and + **swallowed `MM_REGS[12]`, which the handler DOES write** — a core-1 fault + would have taken a MemManage inside the fault handler and wedged the core. + At 20 words `_stack_end = 0x20067cc0` is 32-byte aligned, so the region base + equals `_stack_end` exactly and nothing of ours is inside it. + +### Layout (config AN) + +``` +REGION_FAIL 0x200670b0 LCD_BASE_CHK 0x20067470 ALLOC_GUARD 0x200674a0 +DWT_CATCH 0x20067540 SMASH_CORE0 0x20067568 WATCH_LOG 0x20067a68 +HF_STACK 0x20067bb0 MM_REGS 0x20067c70 _stack_end 0x20067cc0 +``` + +Zero span = **772 words from 0x200670b0**. `DWT_CATCH[8]=0x20067560`, +`SMASH_CORE0[3]=0x20067574`, `[6]=0x20067580`, `LCD_BASE_CHK[9]=0x20067494`. + +`LCD_BASE_CHK` reconfirmed the refutation in a second window: **364,239 calls, +one base (`0x2003d848`), zero anomalies.** `MM_REGS` all zero — **no MPU +violation this window at all**, consistent with the corrupt-base theory being +dead. + +Flashed, CRC `0x98c4fcee` OK, **reset 16:50:43 on 2026-08-15**. + +## AN result — THERE IS NO STRAY WRITER. THERE IS AN 8-BYTE SP IMBALANCE. + +38 min, 79 smashes (2.05/min). `HF_STACK` caught the fault again with a widened +40-word band; `MM_REGS` all zero (no MPU violation, third window running); +`LCD_BASE_CHK` 346,112 calls / one base / zero anomalies (third confirmation). + +The band pinned the frames exactly, and the answer is not what every previous +cycle assumed. + +### The anchor + +`Sm83::step` (0x20000e58) stores the stack guard at `[sp,#0x1c]` +(`20000e6a: str r0,[sp,#0x1c]`). The snapshot has **`0x2007cd1c = +0x2b7e1516`** — so **`step`'s SP is `0x2007cd00`**, not `0x2007cd08`. + +`step`'s frame is balanced: `push {r4-r7,lr}`(20) + `str r8,[sp,#-4]!`(4) + +`sub sp,#0x20`(32) = 56 in; `addeq sp,#0x20` + `ldreq r8,[sp],#4` + +`popeq {r4-r7,pc}` = 56 out. (An earlier scan "found" a missing `add sp` — that +was my regex failing to match **predicated** forms like `addeq sp`. Rule 25.) + +### The fat pointer is exactly where it should be + +``` +20000ee6: strd r0, r1, [sp, #4] -> r0 -> 0x2007cd04 , r1 -> 0x2007cd08 +20000f06: blx r6 -> return addr 0x20000f09 (seen on the stack) +20000f9c: add r0, sp, #0x4 +20000f9e: bl drop_glue> <-- [sp,#4] IS the Arc. Confirmed. +``` + +- `0x2007cd08 = 0x10036698` — rodata: the **vtable** half. +- fault PC `= 0x2003bcfc` — heap, allocation-start aligned: the **Arc data** half. + +**`step`'s live `Arc` fat pointer occupies `[0x2007cd04, +0x2007cd08]`, intact and correct.** + +### The arithmetic + +`Instructions::cb` is reached by `blx r6`, which does not move SP, so **cb's +entry SP = step's SP = 0x2007cd00**. cb pushes 40 bytes +(`push {r4,r5,r6,r7,lr}` + `push.w {r7,r8,r9,r10,r11}`) and pops 40 +(`pop.w {r3,r8,r9,r10,r11}` + `pop {r4,r5,r6,r7,pc}`) — balanced, and it +contains **no `sub sp` and not one SP-relative store**, so it cannot write its +own frame at all. + +It should therefore return with `SP = 0x2007cd00`, taking PC from `0x2007ccfc`. + +``` +expected: SP = 0x2007cd00 , PC from 0x2007ccfc +observed: SP = 0x2007cd08 , PC from 0x2007cd04 <-- both exactly +8 +``` + +**cb's epilogue popped its return address out of `step`'s Arc-data slot.** The +branch target is a heap address, which is even, hence INVSTATE. + +### This retires the entire "stack corruption" framing + +**There is no stray writer. Nothing is being corrupted. SP is 8 bytes too high.** +The same +8 explains the other failure mode: + +`dec8` checks its guard with `ldr r0,[sp,#0x8]`. With SP 8 high it reads +`sp+0x10` — a neighbouring live word — instead of its guard. And +`SMASH_CORE0[4]`, recorded at the naked handler's `sp+0`, has read +**`0x2b7e1516` every single time**: the guard, **intact**, one slot below where +the code looked. **The "corrupt guard = 0x00000001", confirmed thirteen times, +is a MISREAD of an adjacent live word — not corruption.** The withdrawn "the +canary is INTACT" reading was right after all, for a reason nobody had. + +That also explains, at a stroke: + +- why **all three DWT write-watch predicates found nothing wrong** — nothing + writes there; +- why the bogus guard value is always the same small integer — it is a stable + neighbouring local; +- why `check_opcode_table`'s fingerprint (which folds in `Arc::strong_count`) + has **never** fired, and why every live `ArcInner` dumps as a clean `{1,1,d}`; +- why `stack_pop_check` ran 1,018,167,296 iterations with zero mismatch; +- **durable rule 2's "`sp_before` OVER-REPORTS BY 8"** — that was this same +8 + all along, misattributed to the reporting path instead of to a real imbalance. + +### Config AO — the cleanest A/B available, with no rebuild + +The one thing running on core 0 at enormous frequency in these builds is my own +**DebugMonitor handler** (`dwt_watch.rs`, ~68,000 entries/sec). An exception +return that restores SP 8 bytes high would produce exactly this signature. + +The self-aim only arms when `DWT_CATCH[8]` holds a nonzero offset `< 0x1000`. +So **planting 0 disarms it with the identical binary** — same code, same layout, +same heap addresses, only the watch on/off. No rebuild, nothing else varies. + +**Decision rule.** If the `dec8` smashes and the INVSTATE HardFaults collapse, +core 0's DebugMonitor return is the source of the +8 — the crashes have been +substantially self-inflicted by the instrumentation, and it strips out anyway. +If they continue at ~2/min, the DWT is exonerated and the +8 lives in real +firmware code — which is the thing that actually has to be fixed. WDT hangs +(8/31) are a separate category and may persist either way. + +Instruments zeroed (772 words), `DWT_CATCH[8] = 0` **verified**, crash sector +blanked, **reset 17:36:33 on 2026-08-15**. Binary unchanged: config AN, +CRC `0x98c4fcee`. + +**New rules.** (25) **A DISASSEMBLY SCAN FOR SP-MODIFYING INSTRUCTIONS MUST +MATCH PREDICATED FORMS (`addeq sp`, `popne`, ...) — a naive `\badd\s+sp` misses +them and invents imbalances.** (26) **ANCHOR A FRAME WITH A KNOWN CONSTANT (the +`0x2b7e1516` guard at a known `[sp,#N]`) RATHER THAN WITH `sp_before`.** +(27) **BEFORE HUNTING A WRITER, CHECK WHETHER SP ITSELF IS WRONG — every +"corrupted word" here was a correctly-stored word read at the wrong offset.** + +## AO result — the DebugMonitor is EXONERATED, and a second confirmation of the +8 + +Same binary as AN (CRC `0x98c4fcee`), only `DWT_CATCH[8] = 0` so the core-0 +DebugMonitor never arms. `DWT_CATCH[8]` verified still 0 at read-out, so the +A/B is valid. + +| | window | smashes | rate | records | +|---|---|---|---|---| +| AN — DebugMonitor **ARMED** | 38.0 min | 79 | 2.05/min | 15 Panic, 8 WDT, 8 HF | +| AO — DebugMonitor **DISARMED**| 38.6 min | 94 | 2.44/min | 15 Panic, 11 WDT, 5 HF | + +**No collapse — if anything slightly worse.** The pre-registered rule therefore +resolves the other way: **the DWT/DebugMonitor is exonerated, and the +8 lives +in real firmware code.** The INVSTATE fault reappeared byte-identical +(`pc=0x2003bd2c lr=0x20001fff sp_before=0x2007cd08`). + +### A second, spectacular confirmation + +A new HardFault signature appeared: + +``` +pc = 0x2b7e1516 lr = 0x100150af (String::push+0x22) +cfsr = 0x00000100 (IBUSERR) sp_before = 0x2007cca0 +``` + +**`pc = 0x2b7e1516` is the stack-guard constant, executed as a program +counter.** A function popped PC and got the **canary** instead of its saved LR, +then faulted fetching from an address that is not memory. Same bug, different +frame: when the wrongly-popped slot holds an `Arc` pointer you get an even +address (INVSTATE); when it holds the guard you get IBUSERR. + +The "SP is off" thesis is now confirmed twice, independently, from two +different frames and two different fault classes. + +### A static push/pop imbalance is RULED OUT + +A sweep of all 1,666 functions for prologue/epilogue byte mismatch produced 315 +hits, but the heuristic walks backwards across basic blocks and sums unrelated +epilogues (it reported deltas of +25,428), so it is not trustworthy. + +More importantly the sweep is unnecessary: **`Instructions::cb` runs on every +CB-prefixed opcode, millions of times between faults. A compile-time push/pop +imbalance would fault on the first call.** It does not. **⇒ the +8 is DYNAMIC** — +something intermittently leaves SP wrong. With the DebugMonitor excluded, that +points at the remaining asynchronous events on core 0. + +**New rule 28: A CRASH THAT IS RARE RELATIVE TO THE FREQUENCY OF THE SUSPECT +CODE PATH CANNOT HAVE A STATIC CAUSE — do not go looking for one.** + +## Config AP — `FPCCR.ASPEN` was cleared, disabling FP context preservation + +`fpu.rs::disable_lazy_stacking` did: + +```rust +FPCCR.write_volatile(before & !(FPCCR_LSPEN | FPCCR_ASPEN)); +``` + +and its own doc described clearing `ASPEN` as being "so the FP context is not +automatically preserved/restored" — described as a harmless companion to eager +stacking. It is the opposite of it. + +With **`ASPEN = 0` the hardware never sets `CONTROL.FPCA`**, so exception entry +always allocates the *basic* 32-byte frame and **S0–S15 and FPSCR are never +preserved across an exception**. The target is `thumbv8m.main-none-eabihf`; +LLVM uses the FP registers freely, including `vmov` to park integer values as +cheap spill scratch. **Any ISR that touches FP silently clobbers whatever the +interrupted code had live in S0–S15** — on core 0, that is the emulator's inner +loop. It was applied to *both* cores. + +Measured on device: `FPCCR 0xc0000004 -> 0x00000004` under the old code. + +Fixed to the combination that actually expresses the intent — context **is** +preserved, and preserved eagerly: + +```rust +FPCCR.write_volatile((before & !FPCCR_LSPEN) | FPCCR_ASPEN); +``` + +Boot log confirms **`FPCCR 0xc0000004 -> 0x80000004`** (ASPEN set, LSPEN clear). + +This is a real defect fixed on its own merits. Whether it is *the* +8 is a +separate question and is exactly what this soak measures. + +**Decision rule.** `SMASH_CORE0[3]` counts smashes across reboots and does not +floor with the 31-record sector, so it measures the rate even if the sector +fills. AN/AO ran 2.05 and 2.44 smashes/min on the same code. If AP comes in at +that rate, ASPEN was not the +8 and the next suspects are the remaining core-0 +interrupt sources (SysTick / embassy time driver, the executor IRQ, PIO/DMA, +USB) — audit each handler's return path. If the rate collapses, soak longer per +the exponential schedule. + +Layout is unchanged from AN/AO (no `.uninit` change): zero span **772 words from +0x200670b0**, `_stack_end = 0x20067cc0` still 32-byte aligned. DebugMonitor left +**disarmed** (`DWT_CATCH[8] = 0`) since it is exonerated and only adds +perturbation. Instruments zeroed, crash sector blanked. + +Flashed, CRC `0x510ebc3f` OK, **reset 18:20:43 on 2026-08-15**. + +## AP result — ASPEN was not the +8, AND THE +8 CLAIM ITSELF IS RETRACTED + +57.5 min. `SMASH_CORE0[3] = 0x75` = **117 smashes = 2.04/min**. + +| | window | smashes | rate | +|---|---|---|---| +| AN (DebugMonitor armed) | 38.0 min | 79 | 2.05/min | +| AO (DebugMonitor disarmed) | 38.6 min | 94 | 2.44/min | +| AP (ASPEN restored) | 57.5 min | 117 | 2.04/min | + +**No change.** `FPCCR` verified `0xc0000004 -> 0x80000004` on every boot, so the +fix is active; it simply is not what causes the crashes. The INVSTATE fault +reappeared byte-identical again (`pc=0x2003bd2c lr=0x20001fff +sp_before=0x2007cd08`). `MM_REGS` still all zero (fifth clean window). + +**The `ASPEN` fix stays.** It is a genuine defect — with `ASPEN = 0` the +hardware never sets `CONTROL.FPCA`, so S0–S15 and FPSCR are not preserved across +exceptions on a hard-float target — and it is correct independently of this bug. + +### RETRACTION — "an 8-byte SP imbalance" does not survive its own arithmetic + +Last cycle I anchored `Sm83::step`'s frame on a `0x2b7e1516` at `0x2007cd1c` +(= `step`'s guard slot `[sp,#0x1c]` if `step`'s SP were `0x2007cd00`) and +concluded SP was +8. Following that through: + +``` +sp_before = 0x2007cd08 (unambiguous — the exception frame sits at sp-32) +set_r8_enum's pop faulted => its entry SP = 0x2007cd08 + => cb's SP at `bl set_r8_enum` = 0x2007cd08 + => cb pushes 40, so cb's entry SP = 0x2007cd30 + => blx doesn't move SP, so step's SP = 0x2007cd30 + => step's guard would be at 0x2007cd4c -> band has 0x3af5a814, not a guard +``` + +But the anchor put `step`'s SP at `0x2007cd00`, which would place **cb's entry +SP (`0x2007cd30`) inside `step`'s own frame** — impossible for a callee. The two +readings are mutually inconsistent, so the anchor is wrong: with +`-Z stack-protector=strong` **many** frames carry `0x2b7e1516`, and I picked one +and assumed whose it was. That is precisely the trap rule 26 was written to +avoid, and I walked into it one cycle after writing it. + +**Therefore: the "+8", and everything derived from it — including "the canary +smash is a misread, not corruption" — is WITHDRAWN.** It may still be true; it +is not established. + +### What is actually solid + +- `sp_before = 0x2007cd08` — derived from where the exception frame sits, which + is not open to interpretation. +- `ef.lr = 0x20001fff` — the last `bl` executed was `cb -> set_r8_enum`. +- Something branched to a **data** value instead of a code address: an + `Arc` allocation pointer (even -> INVSTATE), or in one case the + guard constant `0x2b7e1516` (invalid -> IBUSERR). +- The fault is fully reproducible and byte-identical across four windows. + +**Whether the cause is a wrong SP or a wrong slot value is NOT established.** + +### Config AQ — point the window where the evidence actually is + +`HF_STACK`'s 40-word band was centred on `sp_before`, which wasted 28 words: +`[sp_before-32, sp_before)` is the exception frame the fault itself pushed, and +below that is the handler's own frame. The frames that identify the call chain — +`cb`'s 40-byte frame, and `step`'s above it with its guard at `sp+0x1c` — all sit +**above** `sp_before` and fell outside the window. Both candidate readings above +needed words at `0x2007cd4c`+ that were never captured. + +Band changed to `sp_before-16 ..= sp_before+140`. + +**Decision rule.** Locate `step`'s frame by finding a slot triple that is +mutually consistent — a guard at `sp+0x1c`, an `Arc` fat pointer +`{heap-ptr, rodata-ptr}` at `[sp+4, sp+8]`, and `str r4,[sp]` holding a +`GameBoyMemory` heap pointer at `sp+0` — rather than by any single constant. +Three agreeing slots pin the frame; one does not. Then compare `step`'s real SP +against `cb`'s entry SP derived from `sp_before`, and the SP-vs-slot question +answers itself. + +**New rule 29: A GUARD CONSTANT IS NOT AN ANCHOR — `-Z stack-protector=strong` +puts `0x2b7e1516` in many frames. Anchor on a MUTUALLY CONSISTENT SET of slots, +and cross-check the result against an independently derived frame boundary +before building anything on it.** + +Layout unchanged. DebugMonitor left disarmed. Instruments zeroed, sector blanked. +Flashed, CRC `0x963d427d` OK, **reset 19:21:01 on 2026-08-15**. + +## AQ result — the +8 is REAL, re-established on evidence that did not exist before + +38.2 min, `SMASH_CORE0[3] = 0x4a` = **74 smashes = 1.94/min** (AN 2.05, AO 2.44, +AP 2.04 — unchanged). `MM_REGS` all zero, sixth clean window. + +Moving the `HF_STACK` band above `sp_before` was the right call: it captured the +words the previous framing needed and could not see. + +### The slot test, run properly this time + +`strd r0, r1, [sp, #4]` writes **both** `sp+4` (Arc data) and `sp+8` (vtable), +so `step` has four checkable slots: `+0x0` (`str r4,[sp]`, a `GameBoyMemory` +heap pointer), `+0x4`, `+0x8`, and `+0x1c` (the guard). + +| candidate `step` SP | +0x0 | +0x4 | +0x8 | +0x1c | +|---|---|---|---|---| +| **0x2007cd00** | *(under exc frame)* | *(under exc frame)* | `0x10036698` **rodata vtable ✓** | `0x2b7e1516` **GUARD ✓** | +| 0x2007cd08 | `0x10036698` not heap | `0xffffffff` | — | `0x2007cf00` not guard | +| 0x2007cd30 | `0x2007dfa8` not heap | `0x1000a4b7` | — | `0x3af5a814` not guard | +| 0x2007ccf8 | `0x1001473f` not heap | `0x20001fff` | — | `0x2003d848` not guard | +| 0x2007cd10 | `0x00000008` not heap | — | — | `0x0000004c` not guard | + +**`0x2007cd00` is the only candidate that scores, on two independently +verifiable slots, and every alternative fails every test.** + +### Therefore the +8 is real + +``` +sp_before at fault = 0x2007cd08 (ef + 32; xPSR bit 9 CLEAR so no STKALIGN pad — exact) +step's SP = 0x2007cd00 (step is an ANCESTOR frame) +delta = +8 +``` + +A descendant's SP can never exceed its ancestor's. And this is not `step`'s own +epilogue: that would give `sp_before = 0x2007cd38` +(`addeq sp,#0x20` / `ldreq r8,[sp],#4` / `popeq`). + +**Last cycle's retraction was right about the derivation and wrong about the +conclusion.** The retraction was correct at the time — a single guard constant is +not an anchor (rule 29), and the frame arithmetic genuinely contradicted it. The +widened band supplied a second independent slot and four failed alternatives, +which the earlier evidence did not contain. **The +8 is reinstated; what was +derived *from* it (notably "the canary smash is a misread") is still NOT +re-established and stays withdrawn.** + +### An unexplained observation to keep + +`0x2007cd3c .. 0x2007cd94` — 23 consecutive words — hold high-entropy values +(`0x5a696fc2 0x4fb4f8d3 0x63e350df 0x5b688fd0 0x3af5a814 ...`). Normal stack +holds pointers, small integers and zeros. This region sits **above** `step`'s +frame (`[0x2007cd00, 0x2007cd38)`), i.e. in its caller's area, just past a +return address at `0x2007cd34` (`embassy_main_task_inner + 0x2fb3`). Not +explained yet; recorded so it is not lost. + +## Config AR — is SP stable at a fixed point in the loop? + +A static push/pop imbalance is excluded (rule 28), so the +8 is introduced +dynamically. The open question is **when**. + +Added `SP_CHK` (`.uninit`) and a `check_sp()` at the top of +`PicoGameBoy::tick` — one fixed point in one call chain, so MSP must read +identically on every iteration. It **counts** the samples so a null is +quantified (rule 9): + +``` +SP_CHK[0] magic 0x5C000001 [1] first SP seen [2] mismatch count +SP_CHK[3] first bad SP [4] total calls [5] call # of first mismatch +SP_CHK[6] most recent bad SP [7] largest |delta| +``` + +**Decision rule.** If `[2]` stays 0 over millions of calls, the drift never +survives a whole tick and is **transient within one instruction's dispatch** — +so the search narrows to what runs inside a single `step()`. If `[2]` is +non-zero, `[3]`, `[5]` and `[7]` give the direction, the onset and the size +directly, and the drift is persistent and therefore easy to bracket. + +### Layout (config AR — re-derived; `.uninit` grew, so everything moved) + +``` +REGION_FAIL 0x200670b0 SP_CHK 0x20067150 LCD_BASE_CHK 0x20067490 +ALLOC_GUARD 0x200674c0 DWT_CATCH 0x20067560 SMASH_CORE0 0x20067588 +WATCH_LOG 0x20067a88 HF_STACK 0x20067bd0 MM_REGS 0x20067c90 +_stack_end 0x20067ce0 +``` + +Zero span = **780 words from 0x200670b0**. `_stack_end` is 32-byte aligned, so +the core-1 RO region base equals it exactly — nothing of ours inside (rule 5 +checked). `DWT_CATCH[8] = 0x20067580`, `SMASH_CORE0[3] = 0x20067594`, +`SP_CHK[2] = 0x20067158`, `[4] = 0x20067160`, `[7] = 0x2006716c`. + +DebugMonitor left disarmed; ASPEN fix retained (`FPCCR -> 0x80000004` confirmed). +Instruments zeroed, sector blanked. CRC `0x31995a67` OK, +**reset 20:03:10 on 2026-08-15**. + +## AR result — SP is invariant at tick entry over 132 MILLION samples + +``` +SP_CHK[0] = 0x5c000001 magic +SP_CHK[1] = 0x2007cd38 first SP seen at PicoGameBoy::tick entry +SP_CHK[2] = 0 MISMATCHES +SP_CHK[4] = 0x07e27620 132,323,872 CALLS +``` + +**Zero mismatches in 132 million samples.** The pre-registered rule resolves to +the first branch: there is **no gradual drift**; the +8 arises and becomes fatal +**within a single `step()` dispatch**. (Note the asymmetry that makes this a +weaker null than it looks: a tick that faults never reaches the next sample, so +`SP_CHK` can only ever observe *successful* ticks. What it genuinely excludes is +a slow accumulating leak — which it does, decisively.) + +### An unplanned third confirmation of `step`'s SP + +`SP_CHK[1] = 0x2007cd38` is the tick-entry SP, measured live and invariant +132 million times. `step`'s frame is 56 bytes, and + +``` +0x2007cd38 - 56 = 0x2007cd00 +``` + +which is exactly the value derived independently in AQ from the rodata vtable at +`sp+8` and the guard at `sp+0x1c`. **Three independent lines now agree that +`step`'s SP is `0x2007cd00`, and `sp_before = 0x2007cd08` is +8 above it.** + +### The layout moved and the smash victim moved with it + +``` +SMASH_CORE0 = [magic, bodySP 0x2007ccc8, lr 0x20000b2b, count 489, + 0x1003651c, 0x20001ab3, 0x2003d758, 0x200009a7] +``` + +Different `lr`, different body SP, different neighbouring words from every +previous window — layout dominates which victim surfaces, as expected. 489 +smashes in 38.0 min = 12.9/min, but that is **not comparable** to AN–AQ: the +`.uninit` set changed *and* `check_sp` sits in the hot path. + +The INVSTATE fault appeared for the **sixth** consecutive window with `lr` and +`sp_before` byte-identical (`pc = 0x2003bbc4`, `r12 = 0x100146fb` — pc and r12 +track the CB sub-opcode, everything else is fixed). + +### The determinism argument + +The fault is byte-identical every time. **An asynchronous interrupt lands +anywhere; it cannot produce the same fault site with the same LR six windows +running.** So the +8 is not an async event — it is a **rare but deterministic +code path** inside one dispatch. + +## Config AS — capture r7, the frame pointer + +Every cycle so far has had to infer *which* function's epilogue popped the bad +word, from `sp_before` plus frame arithmetic. That inference is exactly what +broke twice (the AP retraction and the AQ reinstatement). + +`HF_STACK` grown to 56 words; `[48..56]` now hold **r4–r11 at the fault**, taken +from the existing `HARDFAULT_EXTRA_REGS` trampoline capture. **r7 is the +decisive one — it is the frame pointer.** `Instructions::cb` sets it with +`add r7, sp, #0xc`; `set_r8_enum` with `mov r7, sp`. So r7 pins the live frame +directly, with no inference at all: + +- `r7 == sp_before` ⇒ `set_r8_enum` was live (`mov r7, sp` at its entry SP). +- `r7 == cb_entry_SP - 0x20 + 0xc` ⇒ `cb` was live. +- neither ⇒ a third function is involved and r7 names its frame. + +Combined with r4–r6 and r8–r11 (which `cb` pushes and pops), a mismatch between +the captured values and what `cb`'s prologue saved would also show a register +restore going wrong. + +### Layout (config AS — re-derived; the object order reshuffled, not just shifted) + +``` +REGION_FAIL 0x200670b0 SP_CHK 0x20067150 LCD_BASE_CHK 0x20067490 +HF_STACK 0x20067508 MM_REGS 0x200675e8 DWT_CATCH 0x20067638 +SMASH_CORE0 0x20067660 WATCH_LOG 0x20067b60 ALLOC_GUARD 0x20067c60 +_stack_end 0x20067d00 +``` + +Zero span = **788 words from 0x200670b0**. `_stack_end` is 32-byte aligned, so +the core-1 RO region base equals it exactly — nothing of ours inside (rule 5 +checked; `ALLOC_GUARD` is now the tail object). +`HF_STACK[48] = r4 @ 0x200675c8`, **`[51] = r7 @ 0x200675d4`**, +`[55] = r11 @ 0x200675e4`. `DWT_CATCH[8] = 0x20067658`, +`SMASH_CORE0[3] = 0x2006766c`, `SP_CHK[2] = 0x20067158`, `[4] = 0x20067160`. + +DebugMonitor left disarmed; ASPEN fix retained (both cores confirmed +`FPCCR -> 0x80000004`). Instruments zeroed, sector blanked. +CRC `0x2593e427` OK, **reset 20:44:20 on 2026-08-15**. + +## AS result — r7 nails it: `cb`'s epilogue popped 8 bytes too high + +Registers at the fault (`pc=0x2003bbc4 lr=0x20001fff sp_before=0x2007cd08`): + +``` +r4=0x20000895 r5=0x2007cd30 r6=0x20000f09 r7=0x2002757c +r8=0x2007d2d0 r9=0x2003d758 r10=0x2002757c r11=0x2003d758 +``` + +**r7 = `0x2002757c` is a HEAP pointer, not a frame pointer.** `cb` would give +`sp+0xc`, `set_r8_enum` would give `sp` — neither matches, so the rule's third +branch fires: r7 is not in use as an FP at the fault. + +But reconstructing the pop explains **every single register**. A +`pop {r4,r5,r6,r7,pc}` ending at `SP = 0x2007cd08` must have read from +`0x2007ccf4`: + +``` +[0x2007ccf4] -> r4 = 0x20000895 +[0x2007ccf8] -> r5 = 0x2007cd30 +[0x2007ccfc] -> r6 = 0x20000f09 <- cb's CORRECT saved LR (step+0xb1), WRONG REGISTER +[0x2007cd00] -> r7 = 0x2002757c <- step's [sp,#0] (`str r4,[sp]`, GameBoyMemory ptr) +[0x2007cd04] -> pc = 0x2003bbc4 <- step's [sp,#4] (the Arc DATA half) +``` + +`cb` entered at `SP = 0x2007cd00` would have pushed `r4@0x2007ccec … lr@0x2007ccfc`. +**The pop read 8 bytes too high**, so each register received its neighbour's +slot and `pc` came out of `step`'s live Arc pointer. + +**This is also a fourth independent confirmation that `step`'s SP is +`0x2007cd00`:** `step`'s `[sp,#0]` was buried under the exception frame and +unverifiable in AQ; the popped r7 recovers it, and it is the `GameBoyMemory` +heap pointer exactly as predicted. + +`cb`'s post-prologue SP should be `0x2007ccd8`; the epilogue instead began at +`0x2007cce0`. **SP was already +8 when `cb`'s epilogue started — the +8 arises +during `cb`'s body.** + +### A complete static audit of `cb`'s call tree: everything is balanced + +| callee | prologue | epilogue | net | +|---|---|---|---| +| `Sm83::bus_write` | `push{r4-r7,lr}`(20) + `push.w{r2,r3,r4-r9,r11}`(36) = 56 | `addeq sp,#0x18`+`popeq.w{r8,r9,r11}`+`popeq{r4-r7,pc}` = 56; and the non-predicated path likewise 56 | 0 | +| `Sm83::set_r8_enum` | `push{r7,lr}` = 8 | seven `pop{r7,pc}` = 8 each | 0 | +| `cb::{rlc,sla,rl,rr,srl,rrc,sra}_u8` (7 leaves behind the thunks) | `push{r7,lr}` = 8 | `pop{r7,pc}` = 8 | 0 | +| the 7 long-branch thunks | no SP ops | — | 0 | + +**There is no static imbalance anywhere in the tree.** (The `tbb` table's 8th +entry is `0x00`, which branches back into the table bytes — they decode as four +harmless `lsrs`/`movs` and fall through into the first `strb` + `pop`, still +balanced.) + +### RETRACTION — rule 31 was wrong + +I claimed a byte-identical fault site rules out an asynchronous cause. **It does +not.** An interrupt lands anywhere, but SP damage is only *discovered* at the +next epilogue, and `cb`'s epilogue executes so much more often than anything +else that it dominates the manifestation point regardless of where the interrupt +struck. A fixed fault site is fully consistent with an async cause. **Rule 31 is +withdrawn, and the interrupt-audit line it retired is back on the table — now as +the leading hypothesis**, since the static audit has excluded the alternative. + +## Config AT — mask interrupts across the emulator step + +```rust +asm!("cpsid i"); +self.gb.tick(); +asm!("cpsie i"); +``` + +**Decision rule.** If the INVSTATE fault and the canary smashes vanish, an +exception return on core 0 is leaving SP wrong, and the next step is to find +which handler. If they persist at rate, exceptions are excluded and the cause is +in the emulator's own straight-line code. + +Caveat recorded up front: this is heavy perturbation and it risks starving +embassy's time driver, so liveness was verified before committing the window — +`SP_CHK` climbs steadily (`0x28787f -> 0x33c98a` across twelve samples), first +SP still `0x2007cd38`, zero mismatches. The emulator is running. + +The first flash attempt left the probe timing out; the OpenOCD rescue recovered +it in one pass (and randomised `.uninit`, which the zeroing then cleared). + +Layout unchanged from AS (no `.uninit` change): zero span **788 words from +0x200670b0**, `_stack_end = 0x20067d00`, 32-byte aligned. DebugMonitor disarmed, +ASPEN fix retained. Instruments zeroed, sector blanked. +CRC `0xdca7bcad` OK, **reset 21:29 on 2026-08-15**. + +## AT — VOID. The board lost power. + +Read at 13:27 on 2026-08-16, ~15.9 h after the 21:29 reset (the 35-minute +wakeup did not fire on schedule). The state is the documented power-loss +signature: + +- `SMASH_CORE0`, `HF_STACK`, `MM_REGS` all high-entropy garbage with **no + magic** — they were zeroed at 21:28, so garbage means `.uninit` was + **randomised after that**; +- crash sector holds **one record, a WDT, with `seq = 0`**; +- `ALLOC_GUARD[0]` and `SP_CHK[0]` valid — re-initialised by the firmware after + power-up. + +**No A/B conclusion can be drawn.** The interrupts-masked experiment has to be +re-run. + +### A diagnostic error of mine, corrected + +I then read `SP_CHK[4]` eight times back-to-back, saw it frozen at +`0x01270252`, and concluded the device was hung and that the `cpsid`/`cpsie` +masking had starved embassy. **That was wrong.** Each `probe-rs read` attaches +and halts the core, so eight in a row keep it halted essentially continuously. +Re-measured with genuine elapsed time between reads: + +``` +0x015e508d -> 0x021429c8 in ~25 s = 476,705 ticks/sec RUNNING +``` + +**The device was never hung, and the masking is NOT shown to starve embassy.** +The single WDT record is explained by the power-up, not by the masking. + +**New rule 35: A COUNTER READ BACK-TO-BACK OVER SWD WILL LOOK FROZEN — every +`probe-rs read` halts the core. Put real elapsed time between liveness samples +(`python3 -c "import time; time.sleep(25)"`; the shell `sleep` is blocked).** + +## Config AU — sample SP after the step as well + +The masking is reverted anyway: it is a perturbation where a measurement will +do, and I would rather not spend a window on a discriminator whose side effects +I have already misjudged once. + +`check_sp()` is now called **twice** — before and after `self.gb.tick()` — +against the same reference and the same counters. The entry-only sample could +never observe damage that faults immediately (a tick that faults never reaches +the next entry); the exit sample catches SP damage that **survives** a step, +which is a case nothing has tested yet. Cost is one `mov` plus a few volatile +accesses. + +**Decision rule.** If `SP_CHK[2]` is still 0 with the exit sample in place over +tens of millions of ticks, then SP damage never survives a completed step — +narrowing the +8 to a window that both begins and ends inside one dispatch. If +`[2]` is non-zero, `[3]`/`[5]`/`[7]` give the first bad SP, its onset and the +size, and the entry-vs-exit asymmetry says whether it survives. + +Layout unchanged from AS/AT: zero span **788 words from 0x200670b0**, +`_stack_end = 0x20067d00`, 32-byte aligned. DebugMonitor disarmed, ASPEN fix +retained. The hung-looking device was recovered with the OpenOCD rescue before +flashing. Instruments zeroed, sector blanked. +CRC `0xd1e8fcec` OK, **reset 13:32:22 on 2026-08-16**. + +**Protocol note.** The soak schedule assumes the board stays powered; a power +cycle silently voids a window and costs a full cycle. Windows that end with +randomised `.uninit` and `seq = 0` must be discarded, not interpreted. + +## AU result — the probe corrupted itself, and a new fault mode appeared + +Window valid (magics present, no power loss). 37.9 min, `SMASH_CORE0[3] = 0x21` += **33 smashes = 0.87/min**, classic `dec8` victim. + +But **`SP_CHK` is void**: + +``` +[1] reference = 0x00000000 <- not a possible SP +[2] mismatches = 1,716,077,845 +[4] calls = 1,716,003,009 <- MISMATCHES EXCEED CALLS +[7] max |delta| = 0x2007cd38 <- = |0x2007cd38 - 0|, consistent with reference == 0 +``` + +The reference slot was clobbered, so every sample "mismatched" and the counters +are incoherent. No conclusion can be drawn about SP surviving a step. + +And a **brand-new fault mode dominates**: 13 Panic, **17 HF**, 1 WDT, where the +17 HardFaults are + +``` +pc = 0x1002fb96 (memcpy+0xe6) lr = 0x0 or 0x2 +cfsr = 0x00008200 (PRECISERR | BFARVALID) +BFAR = 0xe00a / 0xe05a / 0xe0ca / 0xe0dc r12 = 0x06..0x71 +sp_before = 0x2007cc74 +``` + +`memcpy` running with a wild pointer around `0x0000e000`, its byte counter in +r12 walking, and a garbage return address. **The INVSTATE fault did not appear +at all.** Both `check_sp()` call sites are removed. + +**The instrumentation has become a liability** — it now generates its own +failure modes and unusable data. This is the point to stop adding probes. + +## Config AV — one binary, two windows, selected by a planted gate + +The interrupt hypothesis is still the leading one and still untested (AT was +lost to a power cut). But **the flaw in AT was not the masking — it was +comparing across builds.** Layout dominates both the crash rate and which victim +surfaces, so a cross-build rate delta proves nothing. That is why the +DebugMonitor A/B was decisive and every other rate comparison has not been: it +used one image and flipped a planted word. + +So the masking is now gated on `.uninit`: + +``` +SP_CHK[0] = magic [1] = GATE (1 = mask interrupts across the step) [2] = tick counter +``` + +`[1]` is planted over SWD and **never written by the firmware**, so one flashed +image runs both arms — plant 1, reset, measure; plant 0, reset, measure. +Identical code, identical layout, identical heap. + +**Decision rule.** Compare the masked arm against the unmasked arm of the *same +image*. If the smashes and HardFaults collapse with interrupts masked, an +exception return on core 0 is leaving SP wrong, and the next step is to find +which handler. If they persist, exceptions are excluded and the +8 is in the +emulator's own straight-line code. + +### An MPU tail hazard caught while re-deriving + +Removing the probes moved the whole `.uninit` block. `_stack_end` landed at +`0x20068070` — **not** 32-byte aligned — which put core 1's read-only region +base at `0x20068060` and swallowed the last four words of `ALLOC_GUARD`, the new +tail object. Padded `ALLOC_GUARD` from 40 to 44 words so `_stack_end = 0x20068080` +is aligned and the region base equals it exactly (rule 5, checked). + +### Layout (config AV — re-derived; the block moved wholesale) + +``` +REGION_FAIL 0x20067420 SP_CHK 0x200674c0 LCD_BASE_CHK 0x20067800 +HF_STACK 0x20067878 MM_REGS 0x20067958 DWT_CATCH 0x200679a8 +SMASH_CORE0 0x200679d0 WATCH_LOG 0x20067ed0 ALLOC_GUARD 0x20067fd0 +_stack_end 0x20068080 +``` + +Zero span = **792 words from 0x20067420**. `SP_CHK[1]` GATE = `0x200674c4`, +`[2]` counter = `0x200674c8`. `SMASH_CORE0[3] = 0x200679dc`. +`HF_STACK[51] = r7 = 0x20067944`. `DWT_CATCH[8] = 0x200679c8`. + +Instruments zeroed, **GATE planted = 1 (masked arm)**, sector blanked, reset. +Verified after the reset with real elapsed time between samples (rule 35): +`GATE = 1`, counter `0x00c0f1ce -> 0x014f25e7` in 25 s = **373k ticks/sec**. +CRC `0x85fcee4e` OK, **reset ~14:17:30 on 2026-08-16**. + +## AV arm A (interrupts MASKED) — EXCEPTIONS ARE EXCLUDED + +Window valid (all magics present). GATE verified `= 1` at read-out, so the +masked arm really ran. 39.8 min, 905,182,405 ticks. + +``` +SMASH_CORE0[3] = 0x35 = 53 smashes = 1.33/min +records 22: 11 Panic, 8 HF, 3 WDT +all 8 HF identical: pc=0x2003bfa0 lr=0x20001fff cfsr=0x00020000 (INVSTATE) + sp_before=0x2007cd00 r12=0x1001435f +``` + +**The INVSTATE fault fired eight times with interrupts masked.** `PRIMASK = 1` +blocks every configurable-priority exception — all IRQs, SysTick, PendSV, SVC — +leaving only NMI and HardFault. The fault is at `cb`'s epilogue, which is inside +the masked region. + +**The pre-registered rule fires on PERSIST: exceptions are EXCLUDED. The +8 is +in the emulator's own straight-line code.** + +That retires the interrupt hypothesis that has led since rule 31 was withdrawn, +and it does so on a *within-image* comparison — the flaw that made every earlier +cross-build rate delta meaningless. + +`lr = 0x20001fff` again: same `cb -> set_r8_enum` call site, six-plus windows +running. `sp_before` is `0x2007cd00` here rather than `0x2007cd08` because the +layout moved; the +8 relationship is to `step`'s SP in *this* image, not to an +absolute address. + +### The `tbb` overrun is not the mechanism either + +`set_r8_enum`'s jump table (base `0x200025ce`, bytes `04 0d 09 0b 06 0f 12 00`) +was the one indirect branch in `cb`'s tree, and an out-of-range `r1` (the +`uxtb` admits 0..255 against an 8-entry table) could in principle land on a +foreign epilogue that pops a different amount. Enumerated every index 8..199 +against the actual instruction stream: **zero land directly on a `pop`.** Every +in-range target, and every fall-through inside `set_r8_enum`, reaches a +`pop {r7, pc}` = 8 bytes, matching its 8-byte push. + +### What is left + +With asynchronous causes excluded and `cb`'s *direct* callees audited clean, the +remaining gap is the **transitive closure below `Sm83::bus_write`** — my audit +went two levels deep, and any function in that closure that returns +8 +propagates unchanged through `bus_write`'s SP-relative epilogue and through +`cb`'s. That is now the highest-value static work, and it needs a CFG-aware +per-function audit rather than the crude linear scan that produced 315 +false positives earlier. + +## AV arm B (interrupts UNMASKED) — running + +Same image, `SP_CHK[1]` planted `= 0`. Verified after reset: GATE `= 0`, counter +`0x00274a93 -> 0x00d29e5a` in 25 s = **449k ticks/sec**. Instruments zeroed. + +This completes the rate half of the A/B (arm A was 1.33 smashes/min). The +qualitative answer is already in — the fault reproduces under masking — so arm B +is confirmation, and the window doubles as running time while the static audit +proceeds. + +Note: the crash-sector blank reported a probe timeout this time, so the sector +may still hold arm A's records; `SMASH_CORE0[3]` was zeroed and is the +authoritative count either way. + +## AV arm B + the CFG-aware audit — the static case is CLOSED + +### Arm B (unmasked), same image, GATE verified `= 0` + +| | window | ticks | smashes | per min | per 1e9 ticks | +|---|---|---|---|---|---| +| arm A — interrupts **masked** | 39.8 min | 905,182,405 | 53 | 1.33 | 58.6 | +| arm B — interrupts **unmasked** | 38.9 min | 880,058,482 | 70 | 1.80 | 79.5 | + +Ratio 1.36 — a modest reduction, **not a collapse** — and the INVSTATE fault is +byte-identical in both arms (`pc=0x2003bfa0 lr=0x20001fff cfsr=0x00020000 +sp_before=0x2007cd00 r12=0x1001435f`). **Exceptions are excluded**, now on a +within-image A/B in both directions. + +### The CFG-aware SP audit — and three bugs in my own analyzer + +Built a per-function CFG walk over all 1,671 functions, tracking the SP delta +along every path from entry to every return. It took three corrections before it +told the truth, and each one is worth keeping: + +1. **333 flags → 160.** A *predicated* return (`popne {r7,pc}`) is only a return + on the branch where it executes; on the other branch it falls through. I was + counting both as returns. +2. **160 → 35.** Predicated instructions inside one IT block **share the + condition outcome** — `addeq sp,#0x18` and the `popeq` after it either both + run or neither does. Branching them independently invented states like + −36/−20/−16 that no execution can reach. +3. **35 → 26.** `bx rN` (N ≠ lr) is an **indirect tail call**, a terminator — not + a fall-through. Treating it as fall-through walked `Sm83::bus_write`'s + `bx r3` at `0x200025a6` straight into its shared epilogue a *second* time and + invented a `+56` return. **That is why `bus_write` appeared flagged, and it is + an artifact — `bus_write` is balanced 56 in / 56 out on every real path.** + +Final result: **26 flags, 23 of them `OUTLINED_FUNCTION_*`** (MachineOutliner +fragments, legitimately entered mid-frame by a tail `b.w`, so a positive delta is +expected), plus three in `embassy_net`/`core`. **Not one function in the +emulator's hot path is flagged.** + +And the audit's coverage is provably complete: the image contains **no** +`add/sub sp, rN` (register operand), **no** `mov sp, rN`, and **no** `ldm/stm sp!` +— every SP-modifying instruction in the binary is a form `delta()` models. + +**New rules.** (39) **A PREDICATED RETURN IS ONLY A RETURN ON THE EXECUTING +BRANCH.** (40) **PREDICATED INSTRUCTIONS IN ONE IT BLOCK SHARE THE CONDITION — +model the decision once per block, not per instruction.** (41) **`bx rN` IS A +TERMINATOR (indirect tail call), NOT A FALL-THROUGH.** (42) **BEFORE TRUSTING A +STATIC AUDIT, PROVE ITS COVERAGE — grep for the forms it cannot model and +confirm they are absent.** + +### Where this leaves the +8 — an honest impasse + +The +8 is established on four independent lines. It is **not** an exception +(arm A), **not** a static push/pop imbalance (this audit), and **not** a `tbb` +overrun. Those three exclusions are each solid on their own evidence, and +together they close off every mechanism proposed so far. + +**Next concrete step, not yet done:** `step` reaches the CB handler through +`blx r6` on the `Arc` vtable. Every analysis so far has *assumed* +that target is `Instructions::cb` itself. It has never been verified. If the +vtable entry points at a wrapper that adjusts SP and tail-branches to `cb`, the +frame arithmetic everything rests on changes. **Resolve the actual `execute` +implementation for the CB opcode in the vtable and confirm what `blx r6` really +lands on.** + +Device: instruments zeroed, GATE planted `0` (unmasked = normal operation), +sector blanked, reset — a fresh window while that resolution proceeds. + +## The `blx r6` target — VERIFIED, and the assumption holds + +`step`'s dispatch is `20000eee: ldrd r1, r6, [r1, #8]` then `blx r6` — so **r6 +comes from vtable[3]**, the first trait method, i.e. +`::execute` at `0x20000894`, **not** +`Instructions::cb` (`0x20001eec`). There *is* an intermediate function, which no +prior cycle had checked. + +``` +20000894: push {r4, r6, r7, lr} -16 +20000896: add r7, sp, #0x8 +2000089a: ldr.w r12, [r3, #0x58] load the Instructions-vtable entry +200008a0: ldr r3, [r7, #0x8] = [sp_entry] -- step's `str r4,[sp]` 5th arg +200008a4: pop.w {r4, r6, r7, lr} +16 +200008a8: bx r12 TAIL CALL +``` + +**Balanced 16 in / 16 out, and it pops its frame before the tail call — so the +tail-called `cb` is entered at exactly `step`'s SP.** The assumption every frame +calculation rests on is confirmed. + +(Also: `r4 = 0x20000895` in the AS register capture was this very function `| 1`. +I noted at the time that it "looked like a thumb function pointer" and did not +chase it. It was the answer to a question I had not yet asked.) + +**So the +8 survives this check too, and every proposed mechanism is now +excluded:** not an exception (within-image PRIMASK A/B), not a static SP +imbalance (CFG-aware audit of all 1,671 functions, coverage proven), not a `tbb` +overrun, not a hidden dispatch wrapper. + +## Config AW — strip the scaffolding and establish a CLEAN BASELINE + +The investigation has hit a genuine impasse, and meanwhile the diagnostic +apparatus has been manufacturing faults of its own: AU's second `check_sp()` +call corrupted its own reference slot **and** brought a new dominant fault mode +with it (`memcpy` bus-faulting on a wild pointer, 17 of 31 records) which +vanished when the call was removed. Continuing to add probes to a binary whose +probes invent failures is not a path to a 24-hour soak. + +Removed: + +- **`-Z stack-protector=strong`** — it forces nightly, puts a canary in every + guarded frame, and turns the anomaly into ~2 panics/min that dominate the + crash records. Without it the same SP damage surfaces as a wrong return (a + HardFault), which the crash recorder still captures. **The metric becomes the + HF/WDT record count rather than `SMASH_CORE0[3]`, which will now stay 0.** +- The `SP_CHK` gate and the `cpsid`/`cpsie` masking from `PicoGameBoy::tick`. +- The `check_lcd_base` call from `write_lcd_timing_register`. + +Kept: the crash recorder, `HF_STACK` and `MM_REGS` (they cost nothing until a +fault), and **the `fpu.rs` ASPEN fix, which is a real fix, not instrumentation** +(both cores confirm `FPCCR -> 0x80000004`). + +**Decision rule.** If the stripped image is stable, the scaffolding was a large +part of the problem and the remaining work is to confirm over an exponential +soak. If it still faults, the metric is now HF/WDT records on a far simpler +binary — no canaries, no per-tick probes, no masking — which is a much better +place to reason from than where this stalled. + +### Layout (config AW — `SP_CHK` and `LCD_BASE_CHK` are gone, dead-code eliminated) + +``` +REGION_FAIL 0x20066f10 ALLOC_GUARD 0x200672d0 HF_STACK 0x200673c8 +MM_REGS 0x200674a8 DWT_CATCH 0x200674f8 SMASH_CORE0 0x20067520 +WATCH_LOG 0x20067a20 _stack_end 0x20067b20 +``` + +Zero span = **772 words from 0x20066f10**. `_stack_end` is 32-byte aligned with +`WATCH_LOG` as the tail object, so the core-1 RO region base equals it exactly +(rule 5, checked). `DWT_CATCH[8] = 0x20067518`, `SMASH_CORE0[3] = 0x2006752c`. + +Last instrumented window before the strip (AV unmasked): **82 smashes / 38.5 min += 2.13/min**. + +Instruments zeroed, sector blanked, DebugMonitor left disarmed. +CRC `0x6147111e` OK, **reset 16:30 on 2026-08-16**. + +## AW result — the stripped baseline STILL FAULTS, and the MPU caught a wild branch + +60 min, window valid (`ALLOC_GUARD` magic present, `SMASH_CORE0` correctly +absent since the protector is gone). **31 records: 22 HF, 8 Panic, 1 WDT.** + +**The pre-registered rule resolves to "still faults".** Stripping the scaffolding +did not fix anything — which is itself worth knowing: **the crashes are not an +artefact of the instrumentation.** They persist in an image with no canaries, no +per-tick probes and no masking. + +What changed is the *shape*. Without the stack protector catching the damage at +the first guarded frame, execution runs further off the rails, and the fault PCs +are now wild rather than uniform: + +``` +x7 HF pc=0x88000000 lr=0x00000002 cfsr=0x100 (IBUSERR) +x5 HF pc=0x000000a0 cfsr=INVSTATE lr=Instructions::cb+0x112 / 0x20001a2d +x1 HF pc=0x60e350dc / 0x89000000 / 0x14 / 0x10 / 0x04 / 0x2004c4e8 +x8 Panic (all fields zero) +``` + +### `MM_REGS` fired for the first time in a dozen windows — and it is decisive + +``` +pc = 0x100249ac = DMA_IRQ_0 + 0x30 insn: str.w r0, [r4, #0x400] +MMFAR = 0x20000c7d cfsr = 0x82 (DACCVIOL | MMARVALID) +r4 = 0x2000087d r5 = 0x2007cd38 r6 = 0x20000ecb sp_before = 0x2007cd10 +``` + +`DMA_IRQ_0`'s prologue is `push {r4,r6,r7,lr}` / `add r7,sp,#8` / +**`ldr r4,[pc,#0xa8]`** — so by `+0x30` r4 must be the DMA base `0x50000000`, +and `str.w r0,[r4,#0x400]` is the ordinary "clear the interrupt flag" store. + +Instead: + +| register | value | resolves to | +|---|---|---| +| r4 | `0x2000087d` | **`::execute` \| 1** | +| r5 | `0x2007cd38` | **the tick-entry SP** (the value AR measured invariant over 132M samples) | +| r6 | `0x20000ecb` | **`Sm83::step + 0xaa` \| 1** | + +And `0x2000087d + 0x400 = 0x20000c7d` — **exactly MMFAR**. + +**So the handler was never entered by an exception** (the prologue would have +loaded r4). **Execution branched wild into the middle of `DMA_IRQ_0`, past its +prologue, carrying the emulator's entire register file with it**, and then +executed a store through what it thought was the DMA base. + +**This is a CONSEQUENCE of the +8 wild-branch bug, not a new cause** — and it +should not be mistaken for one. It does establish two useful things: + +1. **The MPU is actively preventing RAM-code corruption.** A wild write landed + squarely inside `GameBoyMemory::copy_dma_state`'s code and was blocked. That + is very likely why every "the thunk / `.data` is intact" check has passed: + the region is doing its job, so the damage never lands. +2. **`MM_REGS` is now a working wild-branch detector** with a full register + capture. Keep it. + +Note this does **not** reopen the exception hypothesis: the registers prove the +handler was reached by a branch, not by exception entry, so the PRIMASK A/B +result stands. + +**New rule 45: A FAULT INSIDE AN IRQ HANDLER IS NOT NECESSARILY AN IRQ — check +whether the handler's own prologue values are present. If the callee-saved +registers still belong to the interrupted code, it was reached by a WILD BRANCH, +not by exception entry.** + +### Where this leaves things + +The +8 remains unexplained after excluding exceptions, static SP imbalance, the +`tbb` overrun, the dispatch wrapper, and now the instrumentation itself. What AW +adds is a cleaner binary to work in and a register-capturing detector that fires +on the wild branches themselves. + +Instruments zeroed, sector blanked, reset for a fresh window. + +## AW hour 2 — the rate is stable, and loose end (J) turns out to be the lead + +Window valid. **31 records: 24 HF, 7 Panic** (hour 1 was 22 HF / 8 Panic / 1 WDT). +**~31 records/hour, stable.** `MM_REGS` captured the DMA_IRQ_0 wild branch again, +**byte-identical to hour 1** — same pc, MMFAR, and all of r4–r11. + +### The register signature is highly structured + +Across the two independent captures: + +| | hour 1 (MM_REGS) | hour 2 (HF_STACK) | +|---|---|---| +| r4 | `::execute` \| 1 | `::execute` \| 1 | +| r6 | `Sm83::step + 0xaa` | `Sm83::step + 0x10e` | +| r5 | `0x2007cd38` | `0x2007cd38` | +| r7 = r10 | `0x200273d8` | `0x200273d8` | +| r8 | `0x2007d2d8` | `0x2007d2d8` | +| r9 = r11 | `0x2003d5c0` | `0x2003d5c0` | +| sp_before | `0x2007cd10` | `0x2007cd10` | + +**r4 is always an `OpCode::execute` pointer, r6 always an address inside `step`, +and r5/r7/r8/r9/r10/r11 and `sp_before` are identical.** The faults happen at one +specific point in the dispatch, not at random. + +### The wild PCs are values that live on the stack + +The `HF_STACK` band shows `0x2007cd44 = 0x88000000` — and **8 of this hour's 24 +HardFaults have `pc = 0x88000000`**. Likewise the 11 faults with `pc = 0x000000a0` +match the value in the exception frame's own pc slot. **The garbage PCs are not +arbitrary: they are words present in this stack band**, which is exactly what a +`pop {…, pc}` reading the wrong slot produces. + +### Loose end (J) is not noise — it is the same live buffer, and it is adjacent + +The "23 high-entropy words" recorded back in config AV sit in this same region. +Comparing that AV capture against AW — **two different builds, many layout +changes apart**: + +``` +AV 5b688fd0 AW 1b688fd0 xor 0x40000000 (1 bit) +AV d3053cd0 AW d3053cd0 xor 0x00000000 (identical) +AV 09f684d3 AW 09f684d3 xor 0x00000000 (identical) +AV 0f1a962c AW 0f1a962e xor 0x00000002 (1 bit) +... 2 of 20 words identical, mean 2.4 bits differ per word +``` + +**This is the same buffer holding near-identical, content-derived data — not +random stack garbage.** And `0x2007cd3c = embassy_main_task_inner + 0x2fc3` is a +**saved return address** immediately below it, so the buffer is a local of the +main task's inlined frame, sitting directly above the emulator's frames. + +**That reframes the whole picture.** A large live buffer occupies the stack +immediately above `step`/`cb`, the wild PCs are drawn from that region, and a +writer that ever runs below the buffer's start lands squarely in the emulator's +frames — which is precisely the shape of the +8. + +**Next task: identify that buffer.** It begins around `0x2007cd44`, runs at least +88 bytes (the band ends before it does), and lives just above a return address +into `embassy_main_task_inner`. Candidates: CYW43/WiFi state, an SD or ROM read +buffer, scanline hashes, an RNG. Find the local, size it, and check every write +into it for a lower bound. + +**New rule 46: HIGH-ENTROPY STACK DATA IS NOT AUTOMATICALLY GARBAGE — diff the +same region across two builds. If the words are near-identical, it is a LIVE +CONTENT-DERIVED BUFFER, and its extent and its writers matter.** + +Instruments zeroed, sector blanked, reset for hour 3. + +## AW hour 3 — loose end (J) is CLOSED, and it was not the lead + +Window valid (`ALLOC_GUARD` magic present). **31 records: 30 HF, 1 WDT.** +`MM_REGS` clean this hour — no MPU violation. + +``` +x11 HF pc=0x88000000 lr=0x00000000 cfsr=0x100 (IBUSERR) +x10 HF pc=0x88000000 lr=0x00000002 cfsr=0x100 +x 4 HF pc=0x00009ffe lr=0x100196e1 cfsr=0x100 +x 2 HF pc=0x88000000 lr=0x1001f881 cfsr=0x100 +x 1 HF pc=0x68000000 / 0x1001ab2c +``` + +Three hours: **31 / 31 / 31 records.** The rate is flat. + +### The buffer is STATIC — my "live buffer" framing was wrong + +I sized it precisely: the high-entropy run is `0x2007cd44 .. 0x2007cd9c`, +ending at `0x2007cda0` (`00020000`) — **23 words, exactly the "23 high-entropy +words" from AV.** Its contents are a byte-shifted serial stream (`16 fd f9` +appears at one alignment in one word and shifted a byte in another; likewise +`12 f7 21`), which is what a byte-at-a-time SPI shift register produces. + +That made a CYW43 SPI RX buffer look like a very strong candidate — and a DMA +engine writing into stack memory the emulator had since reused would have +explained *every* negative result at once: PRIMASK cannot mask DMA, a per-core +DWT cannot see DMA (my own rule 6), the MPU protects `.data` code but not the +stack, and no static analysis would show anything. + +**So I tested it: read the region three times, 20 s apart. ZERO of 24 words +changed.** The buffer is **stale residue**, not a live DMA target — a high-water +mark left by an earlier deep call (WiFi/CYW43 init is the obvious candidate), +preserved because nothing reaches that depth again in steady state. That also +explains the AV↔AW near-identity: the same init code processing the same data, +differing only by build. + +**Loose end (J) is closed — explained, not promoted.** The wild PCs matching +words in that band (`0x88000000` lives at `0x2007cd44`) are a *symptom*: a wild +`pop {…, pc}` reading stale residue. Not a cause. + +**New rule 47: BEFORE BUILDING ON "A LIVE BUFFER", PROVE IT IS LIVE — read it +repeatedly over tens of seconds. Static contents mean stale residue, and a +high-water mark from an earlier deep call looks exactly like an active buffer.** + +### DMA: a weak null, not an exclusion + +Polled channels 0–3 (READ/WRITE/COUNT) 88 times: only ch1 was ever configured, +memory→peripheral (`0x2004cc88 -> 0x40088008`). **88 samples cannot exclude a +microsecond burst** (rule 9), so DMA-into-SRAM is not disproven — but the +specific mechanism I proposed (a live RX buffer above the emulator's frames) is, +because that buffer does not change. + +### Honest status + +Three flat hours at 31 records/hour on a stripped image. The +8 remains +unexplained with exceptions, static SP imbalance, the `tbb` overrun, the dispatch +wrapper, the instrumentation, and now the stack-residue buffer all excluded. The +strongest untested idea remaining is still **DMA writing into SRAM**, which would +require catching a channel mid-transfer — polling over SWD is far too slow, so it +needs a different approach (e.g. reading `CHx_AL1_CTRL`/`INTS` history, or +checking the cyw43-pio receive path statically for a stack-allocated DMA target). + +Instruments zeroed, sector blanked, reset. + +## AW hour 4 — DMA cancellation is CORRECT; the DMA lead is substantially weakened + +Window valid, `MM_REGS` clean. **31 records (all HF).** Four hours: **31 / 31 / +31 / 31.** Dead flat. + +### The `Transfer` Drop impl aborts properly + +`embassy-rp/src/dma.rs`: + +```rust +impl<'a> Drop for Transfer<'a> { + fn drop(&mut self) { + let p = self.channel.regs(); + // RP2350 errata RP2350-E5: clear the enable bit of the aborted channel + // before the abort to prevent re-triggering. + #[cfg(feature = "_rp235x")] + p.ctrl_trig().modify(|w| w.set_en(false)); + pac::DMA.chan_abort().modify(|m| m.set_chan_abort(1 << self.channel.number())); + while p.ctrl_trig().read().busy() {} + } +} +``` + +It clears enable first (the E5 workaround), aborts, **and spins until `busy()` +clears**. A dropped or cancelled transfer therefore *cannot* leave DMA writing — +which kills the "cancelled future leaves DMA running into a dead stack frame" +mechanism, the one form of the DMA hypothesis that would have explained +everything. + +### But it does confirm my original DMA exclusion was wrong in principle + +`cyw43-pio`'s `cmd_read` does: + +```rust +let mut status = 0; // a STACK LOCAL +self.sm.rx().dma_pull(&mut self.dma_rx, slice::from_mut(&mut status), false).await; +``` + +**DMA writes into stack memory by design here.** The old note that "both channels +are memory→peripheral" was a one-shot snapshot that happened to miss an RX +transfer — the channel's `WRITE_ADDR` points into the stack during any +`cmd_read`. The transfer length is bounded by the slice, and cancellation is +sound, so this is safe as written; but the exclusion it rested on was not. + +**DMA-into-SRAM is now much less likely**, though not formally disproven for an +oversized-transfer variant. + +### Honest status after four flat hours + +The +8 at `Instructions::cb`'s epilogue is established four independent ways, and +every mechanism I have been able to construct is now excluded: exceptions +(within-image PRIMASK A/B), static SP imbalance (CFG-aware audit of all 1,671 +functions with proven coverage), the `tbb` overrun, the dispatch wrapper, the +instrumentation itself (the stripped image faults at the same rate), the +stack-residue buffer (static over 40 s), and DMA cancellation (Drop aborts). + +The one real defect found and fixed along the way — `FPCCR.ASPEN` being cleared, +which left S0–S15 unpreserved across exceptions on a hard-float target — did not +change the rate. + +**Mechanism-hunting has stopped converging.** The next step should be a coarse +FEATURE BISECT rather than another mechanism: build a diagnostic image with the +WiFi/CYW43 task not spawned (a diagnostic window, not a product change — the +feature stays compiled in) and soak it. WiFi is the largest source of +asynchronous activity, DMA, and PIO traffic on core 0, and it is the one major +subsystem never removed from the equation. If the fault survives without it, the +search narrows to the emulator plus display/audio; if it vanishes, the +interaction is with the WiFi stack and that is a far smaller space to search. + +**New rule 48: WHEN MECHANISM-HUNTING STOPS CONVERGING, BISECT BY SUBSYSTEM. +Removing a whole subsystem for one diagnostic window is cheaper than excluding +its mechanisms one at a time.** + +Instruments zeroed, sector blanked, reset. + +## AX — the WiFi bisect answers itself, and the rate instrument was never valid + +Two results, and the second one retroactively weakens most of the rate data in +this document. + +### 1. WiFi was never running. The bisect needed no build. + +The pre-registered plan was to build a diagnostic image with the CYW43 task not +spawned. That turned out to be unnecessary: the cyw43 runner, `net_task`, +`dhcp_task`, `dns_task` and `http_task` are spawned **only** from +`src/wifi/portal.rs:402-417`, which is reached **only** from +`src/state/wifi_menu.rs:269` when the user navigates to Settings→WIFI. `main.rs` +merely parks the peripherals in `App.wifi_periphs` for that screen to `take()`. + +During an unattended soak nobody navigates there, so **WiFi has never been +running in any soak window in this investigation.** By the pre-registered +decision rule this is the "fault survives without WiFi" branch: + +> **WiFi / CYW43 / embassy-net / PIO-SPI and their DMA are excluded as +> participants.** The search narrows to the emulator plus display/audio. + +This also retires the last DMA-into-SRAM variant (loose end Q) as a practical +concern: `cyw43-pio::cmd_read`'s stack-local DMA target is on a code path that +never executes here. + +It also refutes the stated provenance of the stack residue at +`0x2007cd44..0x2007cd9c`. That run was attributed to "WiFi init", which cannot +be right. Its byte-shifted serial signature is more consistent with the SD-card +SPI init, which *does* run at boot. + +### 2. Record counts are not crash counts. Three independent reasons. + +`src/crash/storage.rs` documents two policies that together make the crash log +useless as a rate instrument: + +* **Policy A — no auto-erase when full.** With all 31 slots occupied and the + `RCLG` header intact, the sector is "pending user acknowledgement" and the + write is *skipped*. The log floors at 31 until `crash_decoder.py --mark-read`. +* **Policy B — consecutive-crash deduplication.** Before writing, the previous + committed record is compared by fingerprint (kind, fault-status registers, + panic location). On a match the write is skipped. A repeating fault is stored + **once**. + +So a record count is a count of *fingerprint changes*, capped at 31. + +**Every "31 records this hour" reading in the AW section was the container's +ceiling, not a rate.** h1 22+8+1, h2 24+7, h3 30+1, h4 31 — all exactly 31 +because the sector was full. The "dead flat 31/31/31/31" plateau I reported was +an artifact of the container, and I should have caught it the moment four +consecutive windows landed on the maximum. This is also the explanation for +loose end (I): the AL sector "stopped at 27 of 31 while the counter reached +167" — dedup plus fill, exactly as designed. + +**Rule 49: A SATURATED COUNTER IS NOT A FLAT RATE. If successive measurements +land on the container's maximum, the instrument is reporting its own ceiling. +Check the capacity and the drop policy before reading a trend into the values.** + +### 3. Reading the instruments ends a boot. + +Sampling the live `HEARTBEAT` slots four times, 20 s apart: + +| sample | core0 | core1 | ring idx | +|---|---|---|---| +| s1 | 0 | 6 | 7 | +| s2 | 0 | 6 | 8 | +| s3 | 0 | 6 | 1 | +| s4 | 0 | 6 | 2 | + +The ring index advances by exactly one per sample and `core0` returns to 0 every +time. **Each `probe-rs read` invocation ends a boot.** It is not the watchdog: +`main.rs:410-416` starts it with a 16 s window and then sets +`pause_on_debug(true)`, in the correct order. + +Consequences: the live heartbeat cannot be sampled non-destructively, the +previous-boot ring is contaminated by probe-induced resets (so the 26–93 +core0-ticks-per-boot entries cannot be read as genuine crash-boot durations), +and **every mid-window instrument read in this investigation has been perturbing +the window it was measuring.** + +**Rule 50: A READ THAT RESETS THE TARGET IS NOT AN OBSERVATION. Verify that the +act of sampling does not end the run — a per-boot counter that advances once per +sample is the tell. Sample once, at the end.** + +### 4. The fix: a monotonic boot counter + +`heartbeat.rs` gains `BOOT_COUNT_IDX = PREV_RING_IDX + 1` (slot 25), incremented +once by core 0 in `init()`, seeded to 0 when the magic shows a cold boot and +carried forward on any warm reset. It is immune to all three problems above: no +capacity, no dedup, no ring wrap, and it is read **once**, at the end of a +window. + +Slot 25 lives inside the existing `[usize; 40]`, so this adds no new `.uninit` +object. Verified against rule 5: `HEARTBEAT` moved `0x20067050 → 0x20066fb0`, +**every other object is byte-identical**, and `WATCH_LOG` still ends exactly at +`_stack_end = 0x20067b20`, so the MPU tail is clean. Zero span unchanged at 772 +words from `0x20066f10`. + +### Config AX + +Identical to AW plus the boot counter. CRC `0xd6affd7e`, 300 MHz @ V1_25. +`heartbeat placement OK: base=0x20066fb0`; ASPEN fix confirmed on both cores +(`FPCCR 0xc0000004 -> 0x80000004`). + +Layout: `REGION_FAIL=0x20066f10`, **`HEARTBEAT=0x20066fb0`**, `ALLOC_GUARD=0x200672d0`, +`HF_STACK=0x200673c8`, `MM_REGS=0x200674a8`, `DWT_CATCH=0x200674f8`, +`SMASH_CORE0=0x20067520`, `WATCH_LOG=0x20067a20`, `_stack_end=0x20067b20`. +**Ring idx `0x20067010`, boot counter `0x20067014`.** + +Window opened 21:29:36 with **boot_count = 2** as the baseline. The window's +true reboot count is `boot_count_end - 2`. **No mid-window reads** — that is the +whole point of the instrument. + +### The AW hour-4 window, for the record + +Valid (`ALLOC_GUARD[0] = 0xa1100001`), MM_REGS all zero, sector full at 31 +records (seq 0x00–0x1e) in ≤37 min. `HF_STACK`: `pc=0x000000a0 +lr=0x20001f67 cfsr=0x00020000 (INVSTATE) sp_before=0x2007cd10 r12=0x100141ff +exc_return=0xfffffff9` — the same fault as ever. + +### Next + +Read `boot_count` once at the end of the window to obtain the **first sound +crash rate in this investigation**. Only then are config comparisons meaningful. +Nothing else should be concluded from record counts until that number exists. + +## AX window 1 — the first valid crash rate, and why it must not be over-read + +Window 21:29:36 → 22:33:10 (63.5 min). `ALLOC_GUARD[0] = 0xa1100001`, valid. + +**`boot_count` 2 → 10 = 8 reboots in 63.5 min ≈ 7.6/hour.** The crash sector +independently holds exactly 9 records (seq 0x00–0x08, then erased 0xFF). The two +instruments agree to within the one boot that the final read itself causes, so +**the boot counter is working.** + +Composition: **6 HardFaults + 3 watchdog records.** A third of the failures were +*hangs*, not faults — loose end (E) is a larger share of the problem than the +AW sector suggested (dedup and the 31-cap were hiding the mix, not just the +rate). + +| seq | pc | lr | cfsr | sp_before | +|---|---|---|---|---| +| 0, 4 | `0x2002e6d8` | `0x2000074d` | INVSTATE | `0x20081f98` | +| 2, 6 | `0x00000010` | `0x20001f67` | `0x8200` PRECISERR+BFARVALID, fa=`0x139ec015` | — | +| 7, 8 | `0x000000a0` | `0x20001f67` | INVSTATE | `0x2007cd10` | + +seq 0/4 fault with `sp_before = 0x20081f98`, a **different stack** from the +familiar `0x2007cd10`. Worth resolving which stack that is. + +### What this does NOT establish + +AW's sector implies ≥50 crashes/hour (31 records in 37 min, and dedup means the +true count is higher); AX measured 7.6/hour. **That is not evidence that AX +improved anything.** The only code difference is a counter increment in +`heartbeat::init()`, which has no plausible mechanism for a 6× effect. The real +situation is that AW's number was never a rate at all (rule 49), so there is +nothing to compare against — and AX is a single window, which rule 9 says is not +a result. + +**The next window is AX unchanged.** Establishing the variance of this rate is +worth more than any new hypothesis: without it, no future A/B can be read. + +### Procedure correction — the zeroing loop is self-defeating + +After zeroing, `boot_count` read back as 10, not 0. Cause: each of the 18 +`probe-rs write` invocations in the zeroing loop **resets the device** (rule +50), and every reset re-runs `heartbeat::init()`, which re-stamps the magic and +increments the counter again. The zero span is correct and the writes do land — +a single write/readback confirms it — but the loop cannot leave the counter at +zero. + +**Rule 51: WHEN EVERY PROBE OPERATION RESETS THE TARGET, ORDER THE SETUP SO THE +BASELINE IS READ LAST. Zero, blank, reset, and only then read the baseline; any +device operation after that read invalidates it.** + +### Window 2 + +Sector blank (slot 0 = `ffffffff`), **baseline `boot_count = 0` at 22:35:02**. +Config AX unchanged, CRC `0xd6affd7e`. Crashes ≈ `boot_count_end - 1` (the final +read causes one boot). + +## AX window 2 — the rate reproduces, and there are TWO victims + +Window 22:35:02 → 23:38:10 (63.1 min). `ALLOC_GUARD[0] = 0xa1100001`, valid. +`boot_count` 0 → 10, minus the read-induced boot = **~9 crashes ≈ 8.6/hour.** + +| window | duration | crashes | rate | +|---|---|---|---| +| 1 | 63.5 min | 8 | 7.6/h | +| 2 | 63.1 min | 9 | 8.6/h | + +**The rate reproduces.** ~8/hour with a spread of one crash between two +independent 63-minute windows on an unchanged image. For the first time in this +investigation an A/B can actually be read, and a large effect (say 2× or more) +would be unambiguous against this variance. + +Sector composition: 8 records, **6 HardFault + 2 watchdog** — the same +proportion as window 1 and the same three fault families. + +### Two distinct victims, not one + +Resolving the recurring register values against the AX ELF: + +| value | symbol | +|---|---| +| `lr = 0x20001f67` | `Instructions::cb + 0x112` | +| `lr = 0x2000074d` | **`ApuPeripheral::produce_samples + 0xb0`** | +| `pc = 0x2002e6d8` | `HEAP_MEM + 0xa334` (wild branch into the heap) | +| `r12 = 0x1001420b` | `cb::sla_u8` | +| `r12 = 0x10013745` | **`Vec::push`** (rustyboy_core) | + +This closes loose end (T): `sp_before = 0x20081f98` is `produce_samples`, and +both stacks are inside core 0's (`_stack_end = 0x20067b20` up to the top of +SRAM at `0x20082000`) — `cb` simply sits ~21 KB deeper. + +**The significance is that the same failure — INVSTATE on a `pop` — happens at +two unrelated call sites, one in the emulator CPU and one in the audio sample +producer.** Every previous cycle treated this as a property of `Instructions::cb` +and went looking at cb's codegen, its `tbb`, its dispatch wrapper. That framing +was too narrow: whatever corrupts the return address is not specific to `cb`. +`Vec::push` appearing in r12 puts allocation on the list too. + +### A structural fact that redirects the subsystem bisect + +Every `#[embassy_executor::task]` in the firmware is in `src/wifi/portal.rs`. +There are no display, audio or SD tasks — core 0's main loop and the core 1 +worker do all the work inline. **Nothing else can be un-spawned**, so the +"disable a subsystem" bisect has no more moves after WiFi. + +### Config AY — the work-vs-wall-clock experiment + +`stack_pop_check::check(32 * 1024)` removed from the main loop. It had run +1,018,167,296 STM/LDM trials with zero mismatches (hypothesis excluded) while +burning ~22 ms of every iteration, throttling core 0 to ~45 Hz. Removing it +speeds the emulator up several-fold, which splits the suspects cleanly: + +* **rate rises with throughput** ⇒ the fault is **work-driven** (emulator/APU); +* **rate stays ~8/hour** ⇒ the fault is **wall-clock-driven** (a timer, display + refresh, the watchdog interaction, DMA). + +Nothing else currently distinguishes those two, and they point at disjoint +suspect sets. + +Layout moved (rule 5) — dropping the now-unreferenced `stack_pop_check` statics +shifted everything after HEARTBEAT down 160 bytes: + +`REGION_FAIL=0x20066f10`, `HEARTBEAT=0x20066fb0` (**BOOT_COUNT still +`0x20067014`**), `ALLOC_GUARD=0x20067230`, `HF_STACK=0x20067328`, +`MM_REGS=0x20067408`, `DWT_CATCH=0x20067458`, `SMASH_CORE0=0x20067480`, +`SMASH_CORE1=0x20067700`, `WATCH_LOG=0x20067980`, `_stack_end=0x20067a80`. +MPU tail verified: WATCH_LOG ends at `0x20067a80` = `_stack_end` exactly. +**Zero span now 732 words** from `0x20066f10`. + +CRC `0xe7524fff`, integrity OK, `heartbeat placement OK +core1-RO-region=0x20067a80`. Window opened **23:41:45 with baseline +`boot_count = 12`**; crashes ≈ `boot_count_end - 13`. + +## AY — a 26x rate change, and a WITHDRAWN conclusion + +Window 23:41:45 → 00:45:09 (63.4 min). `ALLOC_GUARD[0] = 0xa1100001`, valid. +`boot_count` 12 → 233 = **220 crashes ≈ 208/hour**, against the 8/hour AX +baseline. A **26x** increase from removing `stack_pop_check`. + +### The work-driven conclusion is WITHDRAWN + +I first read this as the pre-registered "work-driven" branch: the rate rose ~26x +and I believed removing a 22 ms/iteration check had sped core 0 up ~20x, so the +rise looked proportional to throughput. + +**The premise was never measured.** The ~45 Hz figure came from a comment in the +code (`32K iterations is roughly 1.5M trials/s` ⇒ 22 ms), not from the device. +Measuring it directly with the new work counter: over 100 s core 0 completed +**448 iterations — about 4.5 per second**, i.e. a ~200 ms iteration. At that +speed `stack_pop_check` was only **~10% of the loop**, and removing it cannot +have changed throughput by more than that. + +So the 26x is **not** a throughput effect, and the decision rule does not apply. +Both branches of it were premised on a speedup that did not happen. + +**Rule 53: MEASURE THE THROUGHPUT BEFORE INTERPRETING A RATE CHANGE AS A +THROUGHPUT EFFECT. A timing figure taken from a source comment is an assumption, +not a measurement — and a pre-registered decision rule built on an unmeasured +premise decides nothing.** + +### What the numbers actually say + +Normalising by work instead: + +| config | crashes/hour | iterations/crash | +|---|---|---| +| AX (with `stack_pop_check`) | 8 | ~1,845 | +| AZ (without) | ~144–208 | ~112 | + +**Removing `stack_pop_check` made the fault ~16x more likely PER UNIT OF WORK.** +It was not slowing the loop enough to matter — it was **suppressing the bug**. + +That is the strongest causal handle in this investigation so far, and unlike +every previous lead it is a *reversible* A/B: restoring the check should restore +the low rate. `stack_pop_check` hammers STM/LDM push/pop patterns 32K times per +iteration, so whatever it does to the stack — depth, residency, recency of +writes to a particular band — is interfering with the mechanism that corrupts +the return address. + +### Config AZ — the work counter + +`heartbeat.rs` gains `WORK_COUNT_IDX = BOOT_COUNT_IDX + 1` (slot 26), +incremented in `core0_tick()`, seeded 0 only on a cold boot, accumulating across +reboots exactly like `boot_count`. Crashes per unit work is then +`(boot_count delta) / (work_count delta)` from a single read. + +This is necessary because raw crashes/hour cannot distinguish "fewer bugs" from +"less work done" — a config that merely ran slower would look like a fix. + +Layout **unchanged** from AY (slot 26 is inside the existing `[usize; 40]`): +`HEARTBEAT=0x20066fb0`, **`BOOT_COUNT=0x20067014`, `WORK_COUNT=0x20067018`**, +`ALLOC_GUARD=0x20067230`, `HF_STACK=0x20067328`, `MM_REGS=0x20067408`, +`WATCH_LOG=0x20067980`, `_stack_end=0x20067a80`. Zero span 732 words. +CRC `0xb25519ac`, integrity OK. AZ's crash rate matches AY's, so the added +counter changed nothing. + +Window baseline: **boot_count = 251, work_count = 1598 at 00:50:53.** + +### Next + +Get a clean crashes-per-iteration figure for AZ, then run the reversible A/B: +restore `stack_pop_check` and confirm the rate drops ~16x per unit work. If it +does, bisect *what about it* matters — stack depth, the STM/LDM traffic itself, +or simply the extra time spent between emulator ticks. + +## AZ baseline, and the BA reversal test + +Window 00:50:53 → 01:39:12 (48.3 min). `ALLOC_GUARD[0] = 0xa1100001`, valid. + +* Δboot = 138 → **137 crashes** (170/hour) +* Δwork = **19,575 iterations** → **6.75 iterations/s** +* **1 crash per 143 core-0 iterations** + +This confirms the throughput figure independently: core 0 really does run at +~7 Hz, not the ~45 Hz the source comment implied, so rule 53 stands. + +### Config BA — restoring `stack_pop_check` + +`stack_pop_check::check(32 * 1024)` restored. CRC `0x61011677`, integrity OK. +The layout returned **exactly** to the AX positions: `ALLOC_GUARD=0x200672d0`, +`HF_STACK=0x200673c8`, `MM_REGS=0x200674a8`, `WATCH_LOG=0x20067a20`, +`_stack_end=0x20067b20`; zero span back to 772 words; MPU tail clean. The +counters did not move (`BOOT_COUNT=0x20067014`, `WORK_COUNT=0x20067018`). + +Baseline: **boot_count = 395, work_count = 23525 at 01:42:06.** + +### The confound I under-stated when pre-registering this + +Restoring the check **also restores the layout** — the two are not independent, +because removing the call let the linker drop the module's statics and shifted +every `.uninit` object after HEARTBEAT by 160 bytes. So a drop in the rate would +be consistent with *either* hypothesis: + +* the STM/LDM stack traffic suppresses the fault, or +* the AX/BA memory layout suppresses it and the check is irrelevant. + +My earlier framing only treated layout as the fallback explanation if the rate +*failed* to drop. That was wrong: layout is live in both branches. + +**The discriminator is a third config, BB: `check(1)` instead of +`check(32 * 1024)`.** One iteration keeps the module referenced, so the statics +survive and the layout stays byte-identical to BA, while the STM/LDM work drops +to essentially nothing. + +| config | layout | stack traffic | meaning if rate is LOW | +|---|---|---|---| +| BA `check(32K)` | AX-like | heavy | — | +| BB `check(1)` | AX-like | none | **layout** is what matters | +| AZ (removed) | shifted | none | (measured: HIGH, 1/143) | + +If BA is low and BB is high, the **work** suppresses. If BA and BB are both low, +the **layout** suppresses and the check is a red herring — which would make this +a placement-sensitive memory bug and point straight at what `cb` and +`produce_samples` share. + +**Rule 55: A CHANGE THAT ADDS OR REMOVES CODE ALSO CHANGES LAYOUT. Before +attributing an A/B result to the code's behaviour, find a variant that holds +layout fixed and varies only the behaviour — calling the same function with a +trivial argument usually does it.** + +## BA — the reversal is confirmed, in both directions + +Window 01:42:06 → 02:29:19 (47.2 min). `ALLOC_GUARD[0] = 0xa1100001`, valid. +Δboot = 35 → **34 crashes**; Δwork = **85,067 iterations**. + +| config | layout | stack traffic | crashes/iteration | +|---|---|---|---| +| AX | AX-like | heavy | 1 per ~1,845 | +| AZ | shifted | none | **1 per 143** | +| BA | AX-like | heavy | **1 per 2,502** | + +**17.5x fewer crashes per unit work than AZ**, and consistent with AX's 1/1845 +measured on the same layout with the same check. The effect reproduces in both +directions — remove it and the rate rises, restore it and the rate falls. This +is a real, controllable handle on the fault, the first in this investigation. + +### The throughput anomaly, and why it vindicates work-normalisation + +AZ ran **6.75 iterations/s**; BA runs **30 iterations/s**. Adding a 32K-iteration +STM/LDM loop to every pass cannot make the loop four times faster, so the +difference is not the check. + +It is the crashes themselves. AZ crashed 137 times in the window, and each crash +costs a full reboot — which re-initialises the SD card and reloads the ROM from +flash before the main loop starts again. AZ spent most of its wall-clock +**booting**, not emulating. + +This is exactly the trap rule 54 warns about, in the opposite direction from the +one I expected: crashes-per-hour *understates* how bad AZ is, because a config +that crashes more also does less work per hour. Only crashes-per-iteration +compares them honestly. + +### Config BB — holding layout fixed (rule 55) + +`check(32 * 1024)` → `check(1)`. The call keeps the module referenced so its +statics survive, and **llvm-nm confirms the layout is byte-identical to BA**: +`REGION_FAIL=0x20066f10`, `HEARTBEAT=0x20066fb0`, `ALLOC_GUARD=0x200672d0`, +`HF_STACK=0x200673c8`, `MM_REGS=0x200674a8`, `WATCH_LOG=0x20067a20`, +`_stack_end=0x20067b20`. Zero span 772 words, MPU tail clean. Only the STM/LDM +traffic differs. CRC `0x01fc9e83`, integrity OK. + +Baseline: **boot_count = 433, work_count = 111435 at 02:31:50.** + +Decision rule: + +* **BB high (~1/143)** ⇒ the **stack traffic** suppresses the fault. The check is + doing something real to the stack, and the next job is to find out what — + depth, residency, or recency of writes to a particular band. +* **BB low (~1/2500)** ⇒ the **layout** suppresses it, `stack_pop_check` is a red + herring, and this is a placement-sensitive memory bug. That would point + straight at loose end (V): what `cb` and `produce_samples` share. + +## BB — DECISIVE: the stack traffic suppresses the fault, the layout does not + +Window 02:31:50 → 03:19:10 (47.3 min). `ALLOC_GUARD[0] = 0xa1100001`, valid. +Δboot = 95 → **94 crashes**; Δwork = **15,464 iterations** → **1 per 164**. + +| config | layout | stack traffic | crashes/iteration | +|---|---|---|---| +| AX | AX-like | heavy | 1 per 1,845 | +| BA | AX-like | heavy | 1 per 2,502 | +| **BB** | **AX-like** | **none** | **1 per 164** | +| AZ | shifted | none | 1 per 143 | + +The two variables separate cleanly: + +* **Layout held fixed** (BA vs BB, byte-identical per llvm-nm): **15x difference.** +* **Traffic held fixed** (BB vs AZ, layouts differ by 160 bytes): **no difference** + (164 vs 143). + +**The stack traffic controls the fault rate. Layout is irrelevant.** The +placement-sensitivity hypothesis is dead, and rule 55's confound is discharged +for this bug — code may now be moved freely without worrying about it. + +Note the metric is sound across these configs: the main loop is frame-paced, so +one iteration contains the same emulation work whether or not the check runs. + +### What the check actually does + +```asm +1: stmdb sp!, {r0-r3, lr} // write 20 bytes below SP + ...destroy r0-r3, lr... + ldmia sp!, {r0-r3, lr} // read back, compare against callee-saved refs + subs r4, #1 + bne 1b +``` + +32,768 times per frame, restoring SP each pass. Two mechanisms survive: + +1. **the memory activity** — repeated `stmdb`/`ldmia` just below SP interferes + with whatever corrupts the return address; or +2. **time dilution** — the check occupies wall-clock time in which the emulator + is NOT running, so if the corrupting agent fires at a fixed rate in time, a + smaller fraction of its hits land in vulnerable code. + +### Config BC — a pure register spin of the same duration + +`spin_burn`: `subs r0,#1 / bne` — **no memory access at all**, 327,680 +iterations, alongside `check(1)`. + +Duration control, from instruction counts rather than the source comment +(rule 53): `spin_burn` ~3 cycles/iteration ⇒ 327,680 ≈ **3.3 ms**; the stress +loop ~35 cycles/iteration × 32,768 ≈ **3.8 ms**. Within ~15%, and a 15% +shortfall cannot produce a 15x rate difference in either direction. + +This also corrects the "22 ms" figure inherited from the code comment: the check +was ~3.8 ms of BA's 33 ms pass (~11%), which matches the independently measured +throughput and confirms rule 53's verdict on that comment. + +* **BC low (~1/2,500)** ⇒ **time dilution**; the corrupting agent is + asynchronous and fires on a wall-clock schedule, and the "stack traffic" + framing is wrong. +* **BC high (~1/164)** ⇒ the **memory activity** is essential; the fault + involves the stack memory itself, and the next step is to narrow which + property (depth, the store, the load, the address band). + +CRC `0x33867e8e`, integrity OK. HEARTBEAT and `_stack_end` unmoved, so +`BOOT_COUNT=0x20067014`, `WORK_COUNT=0x20067018`; **`ALLOC_GUARD` moved to +`0x200678e8`**. Baseline: **boot_count = 537, work_count = 127,702 at 03:23:00.** + +## BC — time dilution is NOT the mechanism; the memory activity is + +Window 03:23:00 → 04:13:13 (50.2 min). `ALLOC_GUARD[0] = 0xa1100001`, valid. +Δboot = 131 → **130 crashes**; Δwork = **33,510 iterations** → **1 per 258**. + +| config | filler | crashes/iteration | +|---|---|---| +| BA | 32K × `stmdb`/`ldmia` **on the stack** | **1 per 2,502** | +| **BC** | **matched duration, pure register spin, no memory** | **1 per 258** | +| BB | `check(1)` — nothing | 1 per 164 | +| AZ | nothing, shifted layout | 1 per 143 | + +A matched-duration register spin recovers only ~1.6x of the ~15x. So: + +* there IS a small genuine time component (BC 258 vs BB 164, ~100 events each, + so the difference is real if modest); +* but the dominant ~10x **requires the actual `stmdb`/`ldmia` traffic**. + +**The memory activity is essential.** Time dilution is relegated to a minor +contributor, and the "asynchronous agent on a wall-clock schedule" branch does +not fire — so the PRIMASK exclusion stays as it was rather than being re-opened. + +### Config BD — stack, or SRAM in general? + +The remaining split is whether the traffic must touch **the stack** or merely +**SRAM**. `mem_pop_stress` runs the identical instruction sequence and iteration +count with the base register pointing at a static `.bss` buffer +(`MEM_STRESS_BUF`) instead of `sp`, so duration is matched by construction. + +* **BD low (~1/2,500)** ⇒ any SRAM load/store pressure suppresses it. The stack + is incidental, and the mechanism is a bus/arbitration/timing interaction — + which fits a two-core memory-system hazard far better than anything about + frames or return addresses. +* **BD high (~1/258)** ⇒ the traffic must be **near SP**. The mechanism involves + the stack region itself, and the next variants are: stores only, loads only, + and the same traffic at a different stack depth. + +CRC `0xb1a80933`, integrity OK. The new `.bss` buffer moved things (rule 5): +**`HEARTBEAT=0x20066fe8`, `BOOT_COUNT=0x2006704c`, `WORK_COUNT=0x20067050`, +`ALLOC_GUARD=0x200677e0`, `_stack_end=0x20067b58`.** + +Baseline: **boot_count = 0, work_count = 1639 at 04:16:43** (a genuine cold-boot +seed, since the magic's address changed). + +## BD — it is SRAM traffic in general, not the stack. The mechanism looks like silicon. + +Window 04:16:43 → 05:07:14 (50.5 min). `ALLOC_GUARD[0] = 0xa1100001`, valid. +Δboot = 58 → **57 crashes**; Δwork = **77,065 iterations** → **1 per 1,352**. + +| config | filler | target | crashes/iteration | +|---|---|---|---| +| BA | 32K × `stmdb`/`ldmia` | **stack (`sp`)** | 1 per 2,502 | +| **BD** | 32K × `stmdb`/`ldmia` | **static `.bss` buffer** | **1 per 1,352** | +| BC | matched-duration register spin | — | 1 per 258 | +| BB | none | — | 1 per 164 | +| AZ | none (layout shifted) | — | 1 per 143 | + +Generic SRAM traffic recovers **~8x of the ~15x**. BA's residual 1.85x edge over +BD is ~2.8 sigma on 57 and 34 events — marginal, possibly a small real +stack-specific component, not the main effect. + +**So the suppressor is load/store activity in general.** Four controlled results +now bound the mechanism: + +* layout irrelevant (BB vs AZ; BA vs BB byte-identical), +* pure elapsed time minor (BC), +* **memory traffic essential** (BC vs BA/BD), +* **the traffic need not touch the stack** (BD vs BA). + +### Why this points at the memory system, not at software + +A software bug that writes a wrong stack slot should not care whether core 0 is +also hammering an unrelated `.bss` buffer. But a *timing* effect would: keeping +the load/store unit busy changes bus occupancy, arbitration and current draw. + +That shape fits the rest of the evidence, which never sat comfortably with a +logic bug: a wrong PC loaded by `pop` at **two unrelated call sites** +(`Instructions::cb` and `ApuPeripheral::produce_samples`), wild branches into +the heap, and an MPU violation that was a consequence rather than a cause. + +And the operating point is the obvious suspect: **this RP2350 runs at 300 MHz, +double its 150 MHz rating.** + +### Config BE — re-measure voltage with the valid instrument + +The clock is locked by the standing constraint; **voltage explicitly is not**. +The old "voltage is a ~7x rate modulator" note was measured with the crash-log +counter that rule 49 showed was saturating, so it is worth nothing and must be +redone against the boot/work counters. + +BE raises `TARGET_CORE_VOLTAGE` V1_25 → **V1_30** (one step, the highest embassy +exposes; within the regulator's documented range and standard for RP2350 +overclocking) and removes the filler, so it is a **direct A/B against BB's 1 per +164 with voltage as the only variable**. + +Verified on the device: `VREG_CTRL @ 0x4010000c = 0x000000f0` (was `0x000000e0` +at V1_25). `TARGET_SYS_HZ` untouched at 300 MHz. CRC `0x7c75df65`, integrity OK. + +* **BE much lower than 1/164** ⇒ timing marginality at 300 MHz is confirmed as + the dominant factor, and raising voltage is a legitimate fix for the 24-hour + goal rather than a workaround for a software defect. +* **BE ≈ 1/164** ⇒ voltage is NOT the variable, the old ~7x claim was an + artifact of the saturating counter, and the search returns to what memory + traffic actually perturbs. + +Addresses moved again (rule 5): **`HEARTBEAT=0x20066fa8`, `BOOT_COUNT=0x2006700c`, +`WORK_COUNT=0x20067010`, `ALLOC_GUARD=0x200677a0`, `_stack_end=0x20067b18`.** +Baseline: **boot_count = 1, work_count = 3124 at 05:12:00.** + +## BE — VOLTAGE IS THE DOMINANT VARIABLE. Root cause: timing marginality at 300 MHz. + +Window 05:12:00 → 06:05:14 (53.2 min). `ALLOC_GUARD[0] = 0xa1100001`, VREG still +`0x000000f0`. Δboot = 27 → **26 crashes**; Δwork = **119,249 iterations**. + +| config | voltage | filler | crashes/iteration | +|---|---|---|---| +| **BE** | **V1_30** | **none** | **1 per 4,586** | +| BA | V1_25 | stack traffic | 1 per 2,502 | +| BD | V1_25 | `.bss` traffic | 1 per 1,352 | +| BC | V1_25 | register spin | 1 per 258 | +| BB | V1_25 | none | 1 per 164 | + +**One voltage step is a 28x improvement** — bigger than everything the memory +traffic bought, and identical in every other respect to BB, so the comparison is +clean. (The old "voltage is a ~7x modulator" note is superseded: it was measured +with the crash-log counter that rule 49 showed was saturating.) + +### The root cause + +**The RP2350 is being run at 300 MHz, double its 150 MHz rating, and at V1_25 +the core is timing-marginal.** The failure is a load that occasionally returns +wrong data — most visibly a `pop {…,pc}` returning a bad PC, which is why the +faults are INVSTATE, why the PCs are wild, and why the victims are whatever code +happened to be executing rather than any particular function. + +Everything that never fitted a logic bug now fits: + +* **two unrelated victims** (`Instructions::cb`, `ApuPeripheral::produce_samples`) + — a marginal path does not care which function is running; +* **wild branches into the heap** and an **MPU violation that was a consequence**; +* **layout irrelevant**, **stack-specificity absent** — a data-corruption bug + would care about addresses; a timing margin does not; +* **the memory-traffic suppressor** — load/store activity changes bus occupancy + and current draw, i.e. it perturbs the same margin. A second-order voltage + effect, which is why it was strong but never total. + +**Rule 59: A FAULT THAT IS SUPPRESSED BY UNRELATED ACTIVITY AND MOVED BY SUPPLY +VOLTAGE IS AN ELECTRICAL MARGIN PROBLEM, NOT A LOGIC BUG. Stop looking for the +line of code.** + +### The 24-hour goal is in tension with the locked clock + +At ~37 iterations/s, 24 h is ~3.2M iterations. At BE's 1/4,586 that is still +**~700 crashes**. Even if the traffic suppressor stacks multiplicatively (~8x), +~90 crashes. A clean 24 h needs roughly **500x beyond BE**. + +There is no lever that large while the clock stays at 300 MHz: + +* V1_30 is the top of embassy's `CoreVoltage`; going higher means defeating the + regulator's voltage limit, which is out of scope without an explicit request; +* the memory-traffic suppressor is worth ~8x at best and costs real throughput; +* every software mechanism has been excluded by controlled experiment. + +**The honest read: the clock lock and the 24-hour target may be incompatible.** +That is the user's call. Lowering `TARGET_SYS_HZ` is the one change with +headroom of the right order — the prior 266 MHz @ V1_25 note claimed 9 h clean, +though that was measured with the discredited counter and would need redoing. + +### Config BF — do the two suppressors stack? + +V1_30 **and** `check(32 * 1024)`. Stack-targeted traffic is preferred over BD's +`.bss` buffer because BA edged BD by ~1.85x; that was only ~2.8 sigma and may be +noise, but it costs nothing to capture a small stack-specific component if real. + +CRC `0x67ccef9d`, integrity OK, VREG `0x000000f0`. Addresses unchanged: +`BOOT_COUNT=0x2006700c`, `WORK_COUNT=0x20067010`, `ALLOC_GUARD=0x200677a0`. +Baseline: **boot_count = 32, work_count = 125,667 at 06:08:00.** + +## BF — the suppressors do NOT stack; they REVERSE. Plus two corrections. + +Window 06:08:00 → 07:10:16 (62.3 min). Valid, VREG `0x000000f0`. +Δboot = 99 → **98 crashes**; Δwork = **74,012 iterations** → **1 per 755**. + +**BF is 6x WORSE than BE (1 per 4,586), not better.** 98 events, so not noise. + +| config | voltage | filler | crashes/iteration | +|---|---|---|---| +| **BE** | V1_30 | none | **1 per 4,586** ← best measured | +| BA | V1_25 | 32K stress | 1 per 2,502 | +| BD | V1_25 | 32K `.bss` traffic | 1 per 1,352 | +| **BF** | **V1_30** | **32K stress** | **1 per 755** | +| BB | V1_25 | none | 1 per 164 | + +The same filler is **15x better at V1_25 and 6x worse at V1_30**. A genuine +interaction reversal, and consistent with the margin story: the extra switching +current costs more supply droop than the higher static voltage buys. + +### Correction 1 — the checker never fires, and that matters + +`STACK_POP_CHECK` reads `mismatches = 0` over **2,483,349,152 LDM/STM +verifications**. So BF's extra crashes are not `check()`'s own panics, and more +importantly: + +**The plain stack load/store path does NOT fail.** "A load returns wrong data" +was too specific. Whatever is marginal is elsewhere — instruction fetch, or the +PC-load/branch path — not a generic SRAM data load. The faults being INVSTATE on +`pop {…,pc}` while 2.5 billion `ldmia sp!, {r0-r3,lr}` readbacks come back +perfect is a strong constraint on any future theory. + +### Correction 2 — BC's duration control was wrong, so its conclusion is unsound + +BF runs at 19.8 iterations/s against BE's 37.3: the stress loop costs **~23.7 ms +per frame**, not the ~3.8 ms I derived from instruction counts. `spin_burn` at +~3.3 ms was therefore **~7x shorter**, so **BC never compared matched durations +and its "time dilution is minor" verdict does not stand.** + +Rule 53 said not to trust the source comment's timing — correct in spirit, but +my replacement estimate was worse than the comment, whose "1.5M trials/s" +implied ~22 ms and was nearly right. + +**Rule 60: AN INSTRUCTION-COUNT ESTIMATE IS ALSO AN ASSUMPTION. Derive filler +cost from measured throughput (the difference in iterations/s between two +configs that differ only by the filler), not from cycle arithmetic — memory +stalls and bus contention dominate and are not visible in the listing.** + +### What still stands + +All of these are well-sampled and unaffected by the above: + +* **voltage is 28x** (BB 1/164 → BE 1/4,586, identical otherwise); +* the filler helps at V1_25 and hurts at V1_30; +* layout is irrelevant; the traffic need not touch the stack; +* every software mechanism remains excluded by controlled experiment. + +### Config BG = back to BE, and start the soak escalation + +Best measured configuration: V1_30, no filler. CRC `0xb3478a2b`, integrity OK, +VREG `0x000000f0`, 300 MHz untouched. Addresses unchanged +(`BOOT_COUNT=0x2006700c`, `WORK_COUNT=0x20067010`, `ALLOC_GUARD=0x200677a0`, +`STACK_POP_CHECK=0x20067850`). + +Baseline: **boot_count = 137, work_count = 204,134 at 07:14:08.** + +The 24-hour goal remains out of reach at 300 MHz — 1/4,586 extrapolates to ~700 +crashes per day — and the tension with the locked clock is unchanged and still +the user's call. + +## BG — the best configuration is confirmed, and it is not close to the goal + +Window 07:14:08 → 08:17:16 (63.1 min). Valid, VREG `0x000000f0`. +Δboot = 34 → **33 crashes**; Δwork = **139,807 iterations** → **1 per 4,237**. + +BE measured 1 per 4,586 on the same image. **Pooled: 59 crashes over 259,056 +iterations = 1 per 4,391**, 36.9 iterations/s. The V1_30 result reproduces. + +### Where this leaves the 24-hour goal + +24 h at 36.9 iterations/s is ~3.2M iterations ⇒ **~725 crashes/day** at the best +configuration measured. The goal needs **zero**, i.e. roughly **700x** better +than anything achieved. + +The levers are exhausted: + +| lever | best available | status | +|---|---|---| +| core voltage | V1_25 → V1_30 | **used — 28x, the single biggest win** | +| memory-traffic filler | 32K stress | **counter-productive at V1_30 (6x worse)** | +| software mechanisms | — | all excluded by controlled experiment | +| clock | 300 MHz | **LOCKED by standing constraint** | + +V1_30 is the top of embassy's `CoreVoltage`. Going higher means defeating the +regulator's voltage limit, which risks the part and is not something to do +without explicit instruction. + +**The standing constraint (300 MHz locked) and the standing goal (24 h clean) +appear to be mutually unsatisfiable.** The root cause is running the part at +double its rated frequency; the only remaining change of the right magnitude is +the clock. Continuing to run hour-long windows at 300 MHz cannot reach zero and +mostly consumes time, so this is the point to put the decision to the user +rather than keep grinding. + +## BH — the root-cause claim was too strong, and there is a hole in the existing QMI fix + +The user challenged the "electrical margin at 2x rated clock" conclusion and +asked for the evidence and for what other RP2350 overclockers do. Both were +worth doing, and the second changed the picture. + +### The evidence, restated with its limits + +| config | voltage | filler | crashes/iteration | events | +|---|---|---|---|---| +| AZ | V1_25 | none, layout shifted | 1 per 143 | 137 | +| BB | V1_25 | none | 1 per 164 | 94 | +| BD | V1_25 | `.bss` traffic | 1 per 1,352 | 57 | +| BA | V1_25 | stack traffic | 1 per 2,502 | 34 | +| BF | V1_30 | stress | 1 per 755 | 98 | +| BE+BG | V1_30 | none | **1 per 4,391** | 59 | + +Controlled pairs: BB→BE changes only voltage (28x); BA vs BB is byte-identical +in layout (15x); BB vs AZ shifts layout 160 bytes (no effect). + +**The weakness: "electrical margin" is a category, not a mechanism.** It does not +identify which path is marginal, and it does not license the conclusion that the +clock must come down. + +### What other overclockers achieve — this refutes the pessimistic reading + +Reported RP2350 operating points: **266 MHz stable at 1.10V with default QMI; +400 MHz at 1.30V with a 1:3 QMI divider; 636 MHz at 2.00V**. We are at +**300 MHz @ 1.30V and crashing every ~2 minutes** — far worse than others manage +at *higher* clocks and the *same* voltage. + +**So "300 MHz is 2x rated, therefore unfixable" was wrong.** There is headroom +others reach that this board does not, which means something specific is still +broken rather than the part simply being out of margin. + +**Rule 62: "IT IS MARGINAL" IS NOT A ROOT CAUSE. Before concluding that an +operating point is unreachable, check what the same part achieves elsewhere. If +others exceed it comfortably, the gap is a defect, not physics.** + +### The specific mechanism, and the hole + +The known overclocking gotcha is exact: **bootrom flash helpers reset QMI timing +on every flash access**, so an overclocked system silently reverts to boot-clock +flash timing (pico-sdk issues #1983, #1903). + +An earlier cycle already found this — `main.rs:388` and `main.rs:915` force +`CLKDIV = 6` (SCK = 300/6 = 50 MHz) after clock init and again at main-loop +entry, and the reasoning there is sound and matches our fault signatures +(IBUSERR, wild PCs, corrupt XIP instruction fetch). + +**But both retunes run exactly ONCE.** Any flash access *after* main-loop entry +reverts CLKDIV to 3 — flash SCK 100 MHz with RXDELAY tuned for 50 MHz — for the +rest of that boot, and nothing detects it. + +This also explains why the 2.48-billion-trial LDM/STM null was never a +contradiction: **that checker exercises SRAM, not XIP.** Instruction fetch and +ROM data reads come from flash. Marginal flash sampling corrupts fetched +instructions and constants while leaving every SRAM access perfect — precisely +the constraint that had no explanation before. + +Live SWD reads cannot test this: every probe operation ends a boot (rule 50), so +a read only ever shows a freshly-booted chip with the retune fresh. + +### Config BH — the firmware watches QMI itself + +Every main-loop iteration reads `QMI_M0_TIMING`; if `CLKDIV != 6` it increments +`QMI_REVERT` in `.uninit` and repairs it (with `dsb`, a dummy XIP read, and +`isb`, per the RP2350 requirement when raising CLKDIV). + +The counter separates the outcomes cleanly: + +* **QMI_REVERT > 0** ⇒ the hole is real, it has been open this whole + investigation, and it is now closed. The crash rate should drop. +* **QMI_REVERT == 0** ⇒ QMI is stable after entry, this is not it, and the + marginal path is elsewhere. + +CRC `0x1959f307`, integrity OK, VREG `0x000000f0`, 300 MHz untouched. +`heartbeat placement OK: base=0x20067a80 core1-RO-region=0x20067b20` — HEARTBEAT +is now the tail `.uninit` object and ends exactly at the region base, verified +by the firmware's own check. + +Addresses: **`BOOT_COUNT=0x20067ae4`, `WORK_COUNT=0x20067ae8`, +`QMI_REVERT=0x20067aec`**, `ALLOC_GUARD=0x200670e0`. +Baseline: **boot=2, work=24, qmi_reverts=0 at 11:50:15.** + +## BH result — the QMI hole is NOT real, and BH regressed badly + +Read at user request, 11:50:15 → 12:17:26 (27.2 min). Δboot = 56 → **55 crashes**; +Δwork = **4,417 iterations** → **1 crash per 80 iterations**, versus BG's 1 per +4,237. **~50x WORSE.** Boots survived ~2 s instead of ~115 s. + +**`QMI_REVERT = 0`.** The guard never fired. + +### What that null does and does not establish + +The instrument was narrower than intended. The crash-record flash write happens +**inside the crash handler, immediately before reset**, so the main loop never +runs again to observe a reverted QMI. The counter can only catch reversions +caused by flash I/O that happens *while the loop keeps running*. + +What it does establish: **no flash I/O occurs during steady-state emulation** +(ROM staging is boot-only, saves are user-initiated), so QMI cannot revert +mid-loop, and the boot-path reversion is already covered by the existing +main-loop-entry retune. **The hypothesised hole does not exist.** Settled, +negatively. + +### The regression, and a speculation I withdrew + +One peripheral read per iteration cannot cost 50x. I first blamed the `.uninit` +tail hazard, since the build moved HEARTBEAT to `0x20067a80`, ending exactly at +`_stack_end = 0x20067b20`. **That was wrong**: `_stack_end` there was already +32-byte aligned, so the MPU region base equalled it and nothing below was +swallowed — and `check_placement` passed. **The cause of the BH regression is +unknown.** Reverted rather than investigated, because the device was crashing +every ~30 s and a known-good image exists. + +### A measurement trap worth recording + +After the revert, the counters read `boot = 0x2007cd2a`, `work = 0x10011adf` — +garbage. Dumping the block showed slot 8 = `0xfffffff9` (an EXC_RETURN) and a +ring full of `0x2007cdXX` stack addresses: **stale HF_STACK content from the BH +build, sitting at the addresses HEARTBEAT now occupies.** `.uninit` is never +zeroed, and `init()` only seeds on a magic mismatch, so a rebuild that moves +HEARTBEAT can leave the counters inheriting another object's leftovers while the +magic slot happens to look valid. + +Fixed by writing 0 over the magic and resetting, which forces the cold-boot seed. + +**Rule 64: AFTER A REBUILD THAT MOVES A `.uninit` INSTRUMENT, SANITY-CHECK THE +BASELINE. Counters that survive reset also survive relocation onto another +object's stale bytes. Small plausible values, or zero the magic and reset.** + +### Config BJ — back to the BG configuration + +V1_30, no filler, no QMI guard. CRC `0xb6d2251f`, integrity OK, VREG +`0x000000f0`, 300 MHz untouched. `heartbeat placement OK: base=0x20066fa8 +core1-RO-region=0x20067b00`. Layout matches BG: `BOOT_COUNT=0x2006700c`, +`WORK_COUNT=0x20067010`, `ALLOC_GUARD=0x200677a0`. + +Baseline **boot = 0, work = 323 at 12:23:27**, verified sane per rule 64. +Expect ~1 per 4,400 if BG reproduces a third time. + +## BJ — third reproduction, and BK: the last unscaled QMI field + +BJ window 12:23:27 → 13:26:17 (62.8 min). Valid. Δboot = 37 → **36 crashes**; +Δwork = **136,877 iterations** → **1 per 3,802**. + +Pooled **BE + BG + BJ: 95 crashes over 395,933 iterations = 1 per 4,168**, at +~36 iterations/s. The V1_30 configuration is solid across three windows, and the +BH regression is confirmed as an artifact of that build alone. + +### Config BK — RXDELAY was never scaled for the overclock + +The retune wrote `(before & !0xFF) | 6`: **CLKDIV only**. RXDELAY stayed at the +bootrom's 2, justified in-comment by "RXDELAY is relative to SCK, which is back +to 50 MHz, so it stays". + +**That premise is wrong, and it is the same mistake the surrounding comment gets +right twice.** RXDELAY is counted in **clk_sys cycles**: it delays the point at +which QMI samples returning read data, compensating the pad → flash → pad round +trip. That round trip is a fixed *time* set by board and flash electrical delay +— it does not care what SCK is. Doubling clk_sys therefore **halves** the real +delay RXDELAY=2 buys: + +* 150 MHz: 2 / 150 MHz = **13.3 ns** +* 300 MHz: 2 / 300 MHz = **6.67 ns** + +The same block already applies exactly this reasoning to CLKDIV (3→6) and +MIN_DESELECT (7→14), then exempts RXDELAY on a false premise. **RXDELAY was the +last clk_sys-scaled QMI field left unscaled.** + +Corroboration from published RP2350 overclocking: 280 MHz runs **CLKDIV=3 with +RXDELAY=6** — RXDELAY at roughly 2× CLKDIV. Ours was RXDELAY=2 with CLKDIV=6, +i.e. *below* the divider, well outside that practice. + +**And it lands exactly where the evidence points.** `stack_pop_check` saw 0 +mismatches in 2,483,349,152 SRAM LDM/STM verifications, so data loads from SRAM +are clean; the faults are IBUSERR / INVSTATE with wild PCs — corrupt +*instruction* fetch and constant reads, which come from FLASH over XIP. A +mis-placed RX sample point corrupts precisely those and nothing else. This is +the first hypothesis that explains the SRAM null rather than being merely +compatible with it. + +One variable only (rule 57): CLKDIV stays 6, MIN_DESELECT untouched. Config I's +old "MIN_DESELECT=14 was worse" verdict is not relied on — it used the crash-log +counter that rule 49 showed was saturating. + +Applied at **both** retune sites so the boot path and the emulator share a +sampling point. + +Verified on device: `QMI retune at main-loop entry: 0x60007406 -> 0x60007406` +(CLKDIV=6, **RXDELAY=4**), and the register reads `0x60007406` live. Note +`before == after`: the early retune's value survived the whole boot, which +independently confirms BH's finding that QMI is not being reverted here. + +CRC `0x6529bb20`, integrity OK, VREG `0x000000f0`, 300 MHz untouched. Layout +unchanged from BJ. Baseline **boot = 1, work = 9 at 13:30:17** (cold-seeded via +the magic-zero trick, sane per rule 64). + +**Pre-registered:** the pooled comparator is 1 per 4,168 over 95 events. A real +effect should be a large factor, not 20%. If BK does not move the rate, RXDELAY +is excluded and the remaining XIP candidates are the XIP cache and MIN_DESELECT. + +## BK — RXDELAY refuted in DIRECTION, confirmed in IMPORTANCE. It is an 83x knob. + +Window 13:30:17 → 14:33:20 (63.1 min). Valid; VREG `0x000000f0`, QMI `0x60007406`. +Δboot = 137 → **136 crashes**; Δwork = **6,782 iterations** → **1 per 50**. + +Against the pooled comparator of 1 per 4,168 (95 events, RXDELAY=2), **RXDELAY=4 +is ~83x WORSE**, on 136 events. The hypothesis is refuted decisively. + +### The physical argument was backwards — but the finding is bigger than the theory + +I argued RXDELAY compensates a fixed round-trip time, so doubling clk_sys should +require doubling the count. The device says otherwise, so the model is wrong: +most likely a larger RXDELAY samples **later**, past the end of the valid data +window, rather than compensating into it. + +What matters more: + +**Moving ONE QMI field by two clk_sys counts changed the crash rate 83x.** + +That is the largest single-variable effect measured in this investigation — +voltage was 28x. It confirms the **XIP sample point is the sensitive parameter**, +which is precisely what the 2,483,349,152 clean SRAM LDM/STM verifications +implied: SRAM is fine, flash sampling is not. RXDELAY=2 is not a leftover to be +"corrected" — it sits near an optimum, with a cliff two counts away. + +**Rule 67: A REFUTED DIRECTION CAN STILL BE A CONFIRMED VARIABLE. If a change +makes things dramatically WORSE, the parameter matters — stop theorising about +which way it should go and SCAN it.** + +**Rule 68: WHEN A ONE-FIELD CHANGE MOVES A RATE BY ~2 ORDERS OF MAGNITUDE, THAT +FIELD IS THE OPERATING POINT. Treat everything else as secondary until it is +mapped.** + +### Config BL — scan RXDELAY downward + +2 is good, 4 is catastrophic. BL tries **RXDELAY = 1** to determine whether 2 is +the optimum or merely on the good side of a cliff: + +* **1 much better than 2** ⇒ keep descending (try 0); the sample point has been + mis-set all along and there is real headroom. +* **1 much worse than 2** ⇒ 2 is a narrow local optimum, the usable window is + ~1 count wide, and *that narrowness is itself the headline finding* — it would + explain why this board fails where others at higher clocks succeed. + +One variable only. CLKDIV stays 6, MIN_DESELECT stays 7, V1_30, 300 MHz. + +CRC `0x9cad4219`, integrity OK. Verified live: **QMI `0x60007106`** (CLKDIV=6, +RXDELAY=1); `before == after` again at main-loop entry. Baseline **boot = 1, +work = 11 at 14:36:29**, cold-seeded per rule 64. + +### Measured QMI RXDELAY scan so far (V1_30, CLKDIV=6) + +| RXDELAY | crashes/iteration | events | +|---|---|---| +| 1 | *this window* | — | +| **2** | **1 per 4,168** | 95 | +| 4 | 1 per 50 | 136 | + +## BL — the RXDELAY scan is complete. The eye is NARROWER THAN ONE clk_sys CYCLE. + +BL window 14:36:29 → 15:39:19 (62.8 min). Valid; QMI `0x60007106`. +Δboot = 136 → **135 crashes**; Δwork = **6,146 iterations** → **1 per 46**. + +### The scan (V1_30, CLKDIV=6, MIN_DESELECT=7, 300 MHz) + +| RXDELAY | crashes/iteration | events | +|---|---|---| +| 1 | **1 per 46** | 135 | +| **2** | **1 per 4,168** | **95** | +| 4 | **1 per 50** | 136 | + +**RXDELAY=2 is a razor-sharp optimum: one count either side is ~90x worse.** +Three windows, 366 events total, all measured with the same instrument. + +### What that means physically — this is the real root cause + +A one-count step is one clk_sys cycle = **3.33 ns at 300 MHz**. Moving the +sample point by a single 3.33 ns step in *either* direction collapses the flash +read. Therefore: + +**The flash data-valid window at the sample point is narrower than one clk_sys +cycle.** We are balanced on the peak of an eye that is barely open — which is +exactly why sitting *on* the optimum still leaves 1 crash per 4,168 iterations, +and why this board fails where other RP2350s run reliably at *higher* clocks: +their CLKDIV/RXDELAY combinations land inside a wide eye. Ours has nowhere to +sit, because RXDELAY can only be positioned in whole clk_sys cycles and the eye +is smaller than the step size. + +This supersedes "electrical margin at 2x rated clock" with something specific, +measured, and actionable — and it is consistent with every prior constraint: the +2,483,349,152 clean SRAM LDM/STM verifications (SRAM never sampled through QMI), +the IBUSERR / INVSTATE / wild PCs (corrupt instruction fetch from XIP), the two +unrelated victim functions, and voltage acting as a 28x symptom-reducer rather +than a cure. + +**Rule 69: IF A ONE-COUNT CHANGE EITHER SIDE OF AN OPTIMUM IS CATASTROPHIC, THE +VALID WINDOW IS SMALLER THAN THE ADJUSTMENT GRANULARITY. Stop aiming better and +make the window bigger.** + +### Config BM — widen the eye instead of aiming within it + +SCK = clk_sys / CLKDIV. **CLKDIV 6 → 12 halves flash SCK from 50 MHz to 25 MHz**, +which roughly doubles the data-valid window measured in clk_sys counts, so +RXDELAY=2 stops balancing on a knife edge. clk_sys is untouched, so the standing +constraint holds. + +RXDELAY returns to 2 (the value with 95 events behind it). RXDELAY compensates a +fixed pad→flash→pad round trip and does not depend on SCK, so it should remain +optimal. MIN_DESELECT and voltage unchanged. One variable (rule 57). + +Cost: XIP reads at half speed; the XIP cache absorbs most of it, and the metric +is crashes-per-iteration regardless (rule 54). + +CRC `0xfde58e2f`, integrity OK. Device-side readback in the boot log: +**`QMI retune at main-loop entry: 0x6000720c -> 0x6000720c (CLKDIV 12->12 +SCK 25MHz, MIN_DESELECT 7->7)`**. + +A live SWD read of QMI right after setup returned `0x60007203` (the bootrom +default) — a **rule 50 artifact**: the read itself ended the boot, sampling the +chip before the retune ran. The boot-log readback is the trustworthy source. +**Corollary to rule 50: QMI cannot be verified live over SWD; use the firmware's +own readback.** + +Baseline **boot = 1, work = 33 at 16:53:45**, cold-seeded per rule 64. + +**Pre-registered:** comparator 1 per 4,168 (95 events). If BM improves by a large +factor, the narrow-eye model is confirmed and the fix is to keep lowering SCK +(CLKDIV 16, 24) until the rate floors — then soak-escalate. If BM is unchanged or +worse, SCK is not what sets the eye width, and the next candidates are +MIN_DESELECT and the XIP cache. + +## BM — REFUTED. Lowering SCK made it ~80x worse, and the "eye" model is withdrawn. + +Read early at user report of heavy crashing. Window 16:53:45 → 17:07:23 +(13.6 min). Valid. Δboot = 25 → **24 crashes**; Δwork = **1,251 iterations** +→ **1 per 52**, against the 1-per-4,168 comparator. **~80x worse.** + +### The narrow-eye interpretation is withdrawn + +Last cycle I read the RXDELAY scan as "the analog data-valid eye is narrower than +one clk_sys cycle" and prescribed the obvious remedy: halve SCK to widen it. +**The device says no.** If the eye were simply too narrow in absolute time, +doubling the SCK period would have helped. It did the opposite. + +What the four measurements actually support is narrower and more literal: + +| CLKDIV | RXDELAY | crashes/iteration | events | +|---|---|---|---| +| **6** | **2** | **1 per 4,168** | **95** | +| 6 | 1 | 1 per 46 | 135 | +| 6 | 4 | 1 per 50 | 136 | +| 12 | 2 | 1 per 52 | 24 | + +**One specific (CLKDIV, RXDELAY) combination works; perturbing either field +breaks it.** That is a fact about this pairing. It is *not* evidence for any +particular physical picture, and I over-read it into one. + +**Rule 70: A SHARP OPTIMUM LICENSES "THIS SETTING MATTERS", NOT A MECHANISM. +Naming the mechanism ("the eye is too narrow") smuggles in a prediction; test the +prediction before acting on the name.** + +Note also what BM did *not* test: whether the optimum at CLKDIV=12 simply MOVED +to a different RXDELAY. I assumed RXDELAY=2 would stay optimal because it +compensates a fixed round trip — the same assumption that was already refuted +once in BK. Holding RXDELAY fixed while changing CLKDIV was therefore not a +clean one-variable test of "lower SCK is better"; it was a test of one point in +a two-dimensional space. + +### Config BN — back to the best measured operating point + +CLKDIV=6, RXDELAY=2, V1_30, 300 MHz. CRC `0x60e077cc`, integrity OK, device-side +readback `QMI retune at main-loop entry: 0x60007206 -> 0x60007206 (CLKDIV 6->6 +SCK 50MHz, MIN_DESELECT 7->7)`. + +Baseline **boot = 0, work = 104 at 17:10:34**, cold-seeded per rule 64. + +This is the fourth window on this configuration (BE, BG, BJ, now BN); the first +three pooled to 1 per 4,168 over 95 events. + +### Where this leaves things + +Confirmed and reproducible: voltage V1_25→V1_30 is 28x; the (CLKDIV=6, +RXDELAY=2) pairing is worth ~90x against any neighbour tested; layout is +irrelevant; every software mechanism is excluded. Best achieved: **1 crash per +4,168 iterations ≈ 750/day**, still far from a clean 24 h. + +Not established: *why* that pairing is special. Three plausible next moves, in +order of cost: + +1. **Scan RXDELAY at CLKDIV=12** (0..7). If an optimum exists there too, the + sample point tracks SCK and the map is 2-D; if none does, CLKDIV=6 is + privileged for another reason. +2. **MIN_DESELECT** (bits 16:12, currently 7) — the one clk_sys-counted QMI field + never scanned. +3. **XIP cache** behaviour. + +None of these requires touching clk_sys. + +## BN — THE KNOWN-GOOD CONFIG NO LONGER REPRODUCES. The QMI map is unreliable. + +Window 17:10:34 → 18:13:21 (62.8 min). Valid. Δboot = 135 → **134 crashes**; +Δwork = **6,312 iterations** → **1 per 47**. Expected ~1 per 4,168. + +### BN is functionally identical to BJ, and measured 90x worse + +* BJ wrote `(before & !0xFF) | 6` — preserves RXDELAY from `before`. +* BN writes `(before & !0x7FF) | (2 << 8) | 6` — forces RXDELAY = 2. + +`before` is the bootrom value `0x60007203`, whose RXDELAY is already 2, so both +expressions yield **`0x60007206`** — and both boot logs print exactly that. The +QMI register ends up bit-identical. There is no functional difference between +the two images in QMI terms, yet: + +| window | time | crashes/iteration | +|---|---|---| +| BE | 05:12–06:05 | 1 per 4,586 | +| BG | 07:14–08:17 | 1 per 4,237 | +| BJ | 12:23–13:26 | 1 per 3,802 | +| **BN** | **17:10–18:13** | **1 per 47** | + +**Everything measured from 13:30 onward is bad**, regardless of the QMI values: +BK (RXDELAY=4) 1/50, BL (RXDELAY=1) 1/46, BM (CLKDIV=12) 1/52, BN (the restored +good pairing) 1/47. Four consecutive windows, all ~1/50, spanning three +different QMI settings *including the one with 95 events of good history*. + +### What this invalidates + +**The QMI map cannot be trusted.** It was built by comparing windows taken at +different times, and it now appears the device changed state at ~13:30 in a way +that dominates the register settings. BK's "RXDELAY=4 is 83x worse" and BL's +"RXDELAY=1 is 90x worse" are equally consistent with **drift that began before +either measurement**. The apparent razor-sharp optimum may be an artifact of +comparing pre-drift windows against post-drift windows. + +**Rule 71: A CONFIG MAP BUILT FROM SEQUENTIAL WINDOWS IS ONLY VALID IF THE +BASELINE STILL REPRODUCES AT THE END. Re-measure the reference config +periodically; if it has moved, every comparison since the last good reference is +confounded with time.** + +This is the same failure mode as rule 37 ("rate comparisons are only valid within +one image") extended to time: they are also only valid within one *epoch*. + +### The fault signature mix has changed too + +BN's sector is dominated by a family that did not appear before: + +`pc=0x00009ffe lr=0x10019701 cfsr=0x100 (IBUSERR)` — seq 2, 5, 6, 7, 9. + +`lr = 0x10019701` resolves to **`SharedWorkerState::write_live_vr + 0x5c`** +(multicore.rs, in FLASH). Previously the families were `pc=0x000000a0` / +`lr = Instructions::cb + 0x112` / INVSTATE, the `produce_samples` family, and the +`fa=0x139ec015` PRECISERR. + +**IBUSERR — instruction bus error — is now the dominant fault**, and the victim +has moved to a different function. That is a real change in failure character, +not just rate, and it is consistent with the XIP instruction-fetch path having +degraded rather than with any particular register value. + +### What is still solid, and what is not + +**Still solid:** the failure is on the XIP path (2,483,349,152 clean SRAM LDM/STM +verifications; IBUSERR/INVSTATE/wild PCs); layout is irrelevant; every software +mechanism is excluded; the crash-record and work counters themselves are sound. + +**No longer solid:** the entire QMI map, and the "voltage is 28x" figure — BB was +measured at 02:31 and BE at 05:12, both pre-drift, so that one is probably safe, +but it deserves re-checking against a contemporaneous control. + +**Unknown and now the priority:** what changed at ~13:30, and whether it is +reversible. The board has been powered and crashing continuously for ~19 hours. +Temperature is the obvious environmental candidate; flash wear from thousands of +crash-record erase/write cycles is another. + +### Next + +Leave BN running unchanged and take a second window. If it stays at ~1/47, the +device is in a persistently different state and the priority is identifying what +changed — not more register scans, which cannot be interpreted until a reference +config reproduces again. + +## BN repeat — the bad state is STABLE. This is a step change, not drift. + +Window 18:16:08 → 19:18:21 (62.2 min), image untouched. Δboot = 127 → +**126 crashes**; Δwork = **6,815 iterations** → **1 per 54**. + +Pooled across both BN windows: **260 crashes / 13,127 iterations = 1 per 50.5.** + +| epoch | windows | span | crashes/iteration | +|---|---|---|---| +| **good** | BE, BG, BJ | 05:12–13:26 (8 h) | **1 per ~4,168** | +| **bad** | BK, BL, BM, BN, BN₂ | 13:30–19:18 (6 h) | **1 per ~50** | + +Three windows spanning eight hours at ~1/4,000, then five windows spanning six +hours at ~1/50, across three different QMI settings including the one with 95 +events of good history. **An ~85x step change that happened between 13:26 and +13:30 and has not recovered in six hours.** + +That rules out several explanations: + +* **not gradual thermal drift** — a step in ~4 minutes, then flat for 6 h; +* **not the QMI settings** — the bad epoch includes the known-good pairing; +* **not statistical** — 95 events one side, 260 the other. + +The only thing that happened in that window was **flashing the BK image**. But +BG and BJ were also rebuild+flash cycles and stayed good, so "flashing degrades +it" is not sufficient on its own. + +### What the step-change shape implies + +Something changed **state** at that moment and stayed changed. Candidates that +fit a step-and-hold, roughly in order of testability: + +1. **Something in flash was altered by that programming cycle** — the firmware + image region is rewritten every `cargo run`, and the staged Game Boy ROM and + its header live elsewhere in the same device. A damaged or partially-erased + sector would persist across resets and would degrade XIP reads exactly where + the faults now land. +2. **Die temperature crossed a threshold and stayed there** — weaker, because + the transition was too abrupt and the evening has not recovered it. +3. **Flash wear** on a specific sector from ~20 h of crash-record erase/write + cycles plus ~10 full-image programmings. + +Note also a feedback loop that muddies workload comparisons: at 1 per 50 the +device crashes every ~1.5 s of run time and reboots (~15 s), so it now spends +almost all its time booting and may never reach in-game code, whereas in the +good epoch it ran ~115 s per boot. The *mix* of executed code therefore differs +between epochs — a consequence of the rate, but it means signature comparisons +across epochs are not like-for-like either. + +### The decisive test is a power cycle, which needs the user + +`.uninit` is re-baselined every window, so a power cycle costs nothing this +investigation needs, and it is the cleanest way to separate "persistent device +state" from "something that a reset does not clear". It cannot be done over SWD. + +## BP — the RXDELAY-preservation hypothesis was refuted by its own boot log + +The user reports that a power cycle will not change the bad state. That removes +thermal and transient explanations, so I went back to an unverified assumption. + +**The hypothesis:** the good epoch (BE/BG/BJ) wrote `(before & !0xFF) | 6`, +which *preserves* RXDELAY, while every bad window (BK/BL/BM/BN) *forces* it. The +step change at 13:28 is exactly the commit that switched from preserving to +forcing. If the preserved value were 0 — the one value never scanned — that +would explain everything, and would also dissolve the "razor-sharp optimum at 2" +into a plain "less is better". + +**Refuted immediately.** Adding a log of the bootrom value at the early retune: + +``` +QMI early retune: before=0x60007203 (RXDELAY 2) +``` + +`before` really does hold RXDELAY=2, so the preserving mask genuinely produced +`0x60007206`. **The good and bad epochs ran the same QMI value.** RXDELAY does +not explain the step change, and the original assumption was right after all. + +Worth noting *why* the question stayed open so long: the `QMI retune` line was +only ever grepped from BK onward. For BE/BG/BJ I captured the CRC and heartbeat +lines and not that one, so a value I was relying on went eight hours unverified. + +**Rule 72: LOG THE VALUE YOU ARE RELYING ON, IN THE EPOCH YOU ARE RELYING ON IT. +A quantity that is only observed after you start changing it cannot anchor +anything measured before.** + +### Config BP — a same-epoch RXDELAY scan, salvaged from the refutation + +The image is flashed with **RXDELAY = 0**, the one value never tested, and it is +running in the *bad* epoch. That turns a wasted flash into the control-disciplined +measurement this investigation has been missing: RXDELAY 1, 2 and 4 have all now +been measured at ~1/50 **within this same epoch**, so adding 0 completes a +four-point scan with no cross-epoch comparison anywhere in it. + +* **0 also ~1/50** ⇒ RXDELAY genuinely does not matter, and the entire "83x + knob" was an epoch artifact. The QMI map is then fully retired. +* **0 markedly better** ⇒ RXDELAY does matter and the good setting is 0 — which + would then also have to explain why 2 was good this morning and is bad now. + +Device-side readback: `QMI retune at main-loop entry: 0x60007006 -> 0x60007006 +(CLKDIV 6->6 SCK 50MHz, MIN_DESELECT 7->7)`. CRC `0x40fc4ea0`, integrity OK, +V1_30, 300 MHz. Layout unchanged. Baseline **boot = 0, work = 238 at 19:28:30**. + +## BP — RXDELAY=0 wins, on the first fully same-epoch scan + +Window 19:28:30 → 20:31:35 (63.1 min). Valid. Δboot = 95 → **94 crashes**; +Δwork = **12,336 iterations** → **1 per 131**. + +### The RXDELAY scan, with every point inside the current epoch + +| RXDELAY | crashes/iteration | events | +|---|---|---| +| **0** | **1 per 131** | 94 | +| 1 | 1 per 46 | 135 | +| 2 | 1 per 50.5 | 260 | +| 4 | 1 per 50 | 136 | + +**0 is ~2.6x better than 1, 2 and 4**, which are indistinguishable from one +another. Comparing 0 (7.62e-3 ± 0.79e-3 per iteration) against the 260-event +RXDELAY=2 pool (19.8e-3 ± 1.23e-3) gives ~8 sigma. **RXDELAY genuinely matters, +0 is the best available value, and 0 is the bottom of the field.** + +This is the first QMI result in the whole investigation with no cross-epoch +comparison anywhere in it, which is the only reason it can be trusted. + +### It does NOT explain the epoch step — these are two separate effects + +RXDELAY=2 gave 1 per 4,168 this morning and 1 per 50 this evening. A 2.6x field +effect cannot produce an 85x step, and the step happened with the field held +constant. So: + +1. **an unexplained ~85x epoch change at 13:28** — not thermal, not transient, + not power-cyclable (per the user), not QMI; +2. **a real ~2.6x RXDELAY effect**, now measured cleanly. + +Also worth noting: the shape is *not* the razor-sharp peak at 2 that the +confounded afternoon map suggested. 1, 2 and 4 are flat and equal; only 0 is +different. That is the sort of thing the earlier map got exactly backwards. + +### Config BQ — MIN_DESELECT, the last unscanned QMI field + +Keeping RXDELAY=0, now scanning `MIN_DESELECT[16:12]` 7 → 14. + +It is counted in clk_sys cycles and sets the minimum chip-select deselect time +(flash tSHSL, typically 20–50 ns). The bootrom's 7 meant 7/150 MHz = **46.7 ns** +at the boot clock; at 300 MHz the same 7 is **23.3 ns**, at or below the low end +of typical tSHSL. So it is one of the fields the overclock silently halved, and +14 restores the intended duration. + +An old config tried 14 and looked worse, but that measurement used the +saturating crash-log counter (rule 49) *and* changed two things at once. This is +a clean one-variable test with a valid instrument and a stable epoch. + +Device-side readback: **`QMI retune at main-loop entry: 0x6000e006 -> 0x6000e006 +(CLKDIV 6->6 SCK 50MHz, MIN_DESELECT 14->14)`** — CLKDIV=6, RXDELAY=0, +MIN_DESELECT=14. CRC `0x245c9d49`, integrity OK, V1_30, 300 MHz. + +Baseline **boot = 0, work = 340 at 22:39:12**, cold-seeded per rule 64. +Comparator: **1 per 131** (BP, same epoch, 94 events). + +## BQ — MIN_DESELECT WAS THE MISSING SCALING. 74x, and the best result ever. + +Window 22:39:12 → 23:42:23 (63.2 min). Valid. Δboot = 17 → **16 crashes**; +Δwork = **154,819 iterations** → **1 per 9,676**. + +| config | MIN_DESELECT | crashes/iteration | events | +|---|---|---|---| +| BP | 7 | 1 per 131 | 94 | +| **BQ** | **14** | **1 per 9,676** | 16 | + +**74x improvement. Same epoch, one variable, RXDELAY held at 0.** At BP's rate +those 154,819 iterations should have produced ~1,182 crashes; 16 occurred. No +plausible fluctuation covers that. + +It also beats the best-ever *good-epoch* rate (1 per 4,168) by **2.3x**, so this +is the best configuration measured in the entire investigation — and it was +reached without touching the clock. Throughput hit its highest yet, 40.8 +iterations/s, simply because the device now stays up. + +### The mechanism — the same one CLKDIV had + +`MIN_DESELECT` is counted in **clk_sys cycles** and sets the minimum chip-select +deselect time, i.e. the flash's tSHSL (typically 20–50 ns): + +* 150 MHz boot clock: 7 / 150 MHz = **46.7 ns** — what the bootrom intended; +* 300 MHz overclock: the same 7 = **23.3 ns** — at or below the requirement. + +Raising it to 14 restores 46.7 ns. **The overclock silently halved it**, exactly +as it halved CLKDIV's SCK — and the existing retune fixed CLKDIV while leaving +this one alone. + +The old "config I tried MIN_DESELECT=14 and it was worse" verdict was wrong on +two counts: it used the crash-log counter that rule 49 showed was saturating, +and it changed two things at once. + +**Rule 73: WHEN A FIX SCALES ONE CLOCK-COUNTED FIELD FOR AN OVERCLOCK, THE OTHER +CLOCK-COUNTED FIELDS IN THE SAME REGISTER ARE PROBABLY STILL BROKEN. Rule 66 said +to audit them; this is what happens when you actually do it — the second field +was worth more than the first.** + +### Config BR — is 14 sufficient, or merely better? + +tSHSL is a **minimum**, so overshooting should be safe and merely slower. That +makes the direction test cheap and decisive: **MIN_DESELECT = 28** gives 93.3 ns, +double the bootrom's intended figure. + +* **28 ≈ 14** ⇒ 14 already satisfies tSHSL; the field is done, and the residual + crashes come from elsewhere. +* **28 ≪ 14** ⇒ the real requirement exceeds what the bootrom assumed; keep + climbing (the field is 5 bits, max 31). +* **28 ≫ 14** ⇒ non-monotonic, so tSHSL is not the whole story. + +Device-side readback: **`QMI retune at main-loop entry: 0x6001c006 -> 0x6001c006 +(CLKDIV 6->6 SCK 50MHz, MIN_DESELECT 28->28)`**. CRC `0xce7fe63f`, integrity OK, +V1_30, 300 MHz. Baseline **boot = 0, work = 305 at 23:45:47**. + +Comparator: **1 per 9,676** (BQ, same epoch). + +### Where this puts the 24-hour goal + +At 1 per 9,676 and ~41 iterations/s, 24 h is ~3.5M iterations ⇒ **~365 +crashes/day**. Still not zero, but the trajectory changed tonight: two +clock-counted QMI fields (RXDELAY 2→0, MIN_DESELECT 7→14) together took the rate +from 1 per 50 to 1 per 9,676 — a **190x** improvement, all within the locked +clock. The ~85x epoch step at 13:28 is now moot for practical purposes, since the +current configuration is better than the good epoch ever was; it remains +unexplained but no longer blocks progress. + +## BR — MIN_DESELECT=28 is bad, and that makes BQ's "74x" suspect + +Window 23:45:47 → 00:48:23 (62.6 min). Valid. Δboot = 150 → **149 crashes**; +Δwork = **17,103 iterations** → **1 per 115**, vs BQ's 1 per 9,676. **~84x worse.** + +| MIN_DESELECT | rate | events | window | +|---|---|---|---| +| 7 | 1 per 131 | 94 | 19:28 | +| **14** | **1 per 9,676** | **16** | 22:39 | +| 28 | 1 per 115 | 149 | 23:45 | + +### I called a breakthrough on one uncontrolled small-sample window + +7 and 28 both land at ~1/120. Only the single 14 window was spectacular, and it +rests on **16 events — the smallest sample of the night**. + +That is precisely the shape that misled me this afternoon: one good window +flanked by bad ones, which turned out to be an **epoch step**, not a setting. I +wrote rule 71 after being burned by exactly this, and then announced a 74x +breakthrough from an uncontrolled window anyway. The claim in the BQ section +("the best result ever", "190x", "the missing scaling") is **not established** — +it is one window. + +Two readings remain open and I cannot choose between them from the data I have: + +1. **MIN_DESELECT=14 is a genuine sharp optimum.** Then 28 being bad is real and + informative: a *minimum* timing requirement would make overshoot harmless, so + tSHSL-as-a-minimum would be the wrong mechanism even though the value works. +2. **BQ was another epoch excursion.** Then the 74x collapses, MIN_DESELECT does + not matter, and — since RXDELAY=0's "2.6x" rests on the same kind of + single-window comparison — that result is suspect too. The real phenomenon + would be whatever makes the device step between ~1/120 and ~1/5,000 states. + +**Rule 74: A RESULT THAT WOULD BE A BREAKTHROUGH DESERVES A CONTROL BEFORE IT +DESERVES A HEADLINE — especially when it comes from the smallest sample in the +series and the system is known to step between states on its own.** + +### Config BS — re-run MIN_DESELECT=14 as a control + +Not a new value. Everything identical to BQ; QMI verified byte-identical +(`0x6000e006`, CLKDIV=6, RXDELAY=0, MIN_DESELECT=14). + +* **~1 per 9,676 again** ⇒ 14 is real, 28 is real, and the mechanism is *not* + a simple minimum-time requirement. Settle on 14 and soak-escalate. +* **~1 per 120** ⇒ BQ was an epoch artifact; the 74x and probably the RXDELAY + 2.6x both collapse, and the epoch stepping is the actual phenomenon. + +CRC `0x212e3758`, integrity OK, V1_30, 300 MHz. Baseline **boot = 0, work = 339 +at 00:51:44**, cold-seeded per rule 64. + +## BS — THE CONTROL REPRODUCES. MIN_DESELECT=14 is real; my mechanism for it is not. + +Window 00:51:44 → 01:54:23 (62.7 min). Valid. Δboot = 24 → **23 crashes**; +Δwork = **147,387 iterations** → **1 per 6,408**. + +| MIN_DESELECT | rate | events | window | +|---|---|---|---| +| 7 | 1 per 131 | 94 | 19:28 | +| 14 | 1 per 9,676 | 16 | 22:39 | +| 28 | 1 per 115 | 149 | 23:45 | +| **14 (control)** | **1 per 6,408** | 23 | 00:51 | + +**Pooled MIN_DESELECT=14: 39 crashes / 302,206 iterations = 1 per 7,749**, across +two windows separated by a different config. ~60x better than either neighbour +(7 → 76.2e-4/iter, 28 → 87.1e-4/iter, 14 → 1.29e-4/iter). Overwhelming. + +So rule 74 was satisfied and the answer came back **positive**: the effect is +genuine, and BQ was not an epoch artifact. The caution was still correct — the +claim needed the control before it deserved the headline, and it now has one. + +### But the mechanism I attached to it is wrong + +I explained 14 as "restoring the flash's tSHSL, which the overclock halved". +**tSHSL is a MINIMUM.** If that were the mechanism, overshooting to 28 (93.3 ns) +would be harmless — merely slower. Instead 28 is **67x worse than 14**. + +There is a genuine *optimum* at 14, not a floor. I am not naming a replacement +mechanism; asserting one and acting on its prediction is an error already on the +record twice (rules 70, 62). What is established is the value, not the reason. + +**Rule 75: A CONFIRMED SETTING DOES NOT CONFIRM THE STORY YOU TOLD ABOUT IT. +Re-check the mechanism against the whole curve — a "minimum" that gets worse when +exceeded was never a minimum.** + +### Where this leaves the numbers + +Best confirmed rate: **1 per 7,749** — about **1.9x better than the good epoch's +1 per 4,168**, and reached entirely within the locked clock. + +At ~40 iterations/s that is still ~450 crashes/day, so the 24-hour goal is not +met; but the configuration is now the best measured and control-verified. + +### Config BT — re-scan RXDELAY at the CORRECT operating point + +**The entire RXDELAY scan ran at MIN_DESELECT=7**, now known to be a bad +operating point. RXDELAY=0 won there (1 per 131 vs ~1 per 50 for 1/2/4), but the +two fields plainly interact — that is exactly what the non-monotonic +MIN_DESELECT curve implies — so **the best RXDELAY at MIN_DESELECT=14 is simply +unmeasured**. + +BT sets **RXDELAY = 2** (the bootrom value, and the one that lost worst at +MIN_DESELECT=7) with MIN_DESELECT held at 14. One variable. + +Device-side readback: **`QMI retune at main-loop entry: 0x6000e206 -> 0x6000e206 +(CLKDIV 6->6 SCK 50MHz, MIN_DESELECT 14->14)`** — CLKDIV=6, RXDELAY=2, +MIN_DESELECT=14. CRC `0xb8530c16`, integrity OK, V1_30, 300 MHz. + +Baseline **boot = 0, work = 338 at 08:25:39**, cold-seeded per rule 64. +Comparator: **1 per 7,749** (pooled, same epoch). + +* **BT much worse** ⇒ RXDELAY=0 is right at both operating points; the fields do + not interact on this axis and the QMI tuning is finished. +* **BT similar or better** ⇒ the earlier RXDELAY scan was measured at the wrong + point and its conclusion does not transfer; scan 0/1/2/4 properly at + MIN_DESELECT=14. + +## BT — RXDELAY is INERT at MIN_DESELECT=14. The old RXDELAY result does not transfer. + +Window 08:25:39 → 09:30:12 (64.6 min). Valid. Δboot = 19 → **18 crashes**; +Δwork = **157,434** → **1 per 8,746**. + +Comparator was 1 per 7,749 (RXDELAY=0, 39 events). Ratio **0.89** — flat, well +inside noise for 18 events. Pre-registered branch (b), and it answers the +question in a way neither branch anticipated: RXDELAY does not matter here at +all. + +| RXDELAY @ MIN_DESELECT=7 | rate | events | +|---|---|---| +| 0 | 1 per 131 | 94 | +| 1 | 1 per 46 | 135 | +| 2 | 1 per 50.5 | 260 | +| 4 | 1 per 50 | 136 | + +| RXDELAY @ MIN_DESELECT=14 | rate | events | +|---|---|---| +| 0 | 1 per 7,749 | 39 | +| 2 | 1 per 8,746 | 18 | + +The 2.6x spread across RXDELAY at MIN_DESELECT=7 **does not reproduce** at +MIN_DESELECT=14. Whether that spread was ever real (it was single-window against +a pool, in the epoch-stepping era) or is simply irrelevant once MIN_DESELECT is +right, RXDELAY is not a lever at the operating point we actually use. Not +spending two more windows on 1 and 4: 2-vs-0 is flat at 57 pooled events and +1/2/4 clustered together in the old scan. + +### The pooled operating point + +**MIN_DESELECT=14, three windows, two RXDELAY values: 57 crashes / 459,640 +iterations = 1 per 8,064.** The most reproducible state of the investigation — +and the epoch stepping has not reappeared across any of it. + +## BU — COOLDOWN 1 → 2: the last unscanned clock-counted field + +`MIN_DESELECT` was worth ~60x because it is counted in **CLK_SYS cycles** and +running clk_sys at 2x rated halved its real duration. **`COOLDOWN[31:30]` is the +other time field in `QMI_M0_TIMING`** — chip select stays asserted for +`64 x COOLDOWN` system clock cycles — and it is still sitting at the bootrom's 1. +The existing retune masks only bits [16:0], so **COOLDOWN has never been written +at all**. Rules 66 and 73 point straight at it. + +This is an **analogy to a confirmed fix, not a mechanism claim**. The +MIN_DESELECT curve has an OPTIMUM, not a floor, which no simple +minimum-duration story explains (rule 75). If COOLDOWN=2 helps, expect the same +non-monotonicity and scan it rather than extrapolating to 3. + +RXDELAY returns to 0. That is not a second variable in any meaningful sense — +BT just showed it is inert here — and it keeps the config aligned with the +39-event reference arm. + +Mask widened to `0xC001_FFFF` to reach bits [31:30]; `PAGEBREAK[29:28]=2` is +preserved from the bootrom. + +Device-side readback: **`QMI retune at main-loop entry: 0xa000e006 -> 0xa000e006 +(CLKDIV 6->6 SCK 50MHz, MIN_DESELECT 14->14)`** — COOLDOWN=2, PAGEBREAK=2, +MIN_DESELECT=14, RXDELAY=0, CLKDIV=6. CRC `0xe1250dfc`, integrity OK, V1_30, +300 MHz. Addresses re-derived, unchanged. + +Baseline **boot = 1, work = 46 at 09:33:35**. One reset occurred during the 25 s +seeding wait (prior windows seeded at work ~338), so this boot started late; +the metric is delta-based so this is recorded, not corrected. + +Comparator: **1 per 8,064** (pooled MIN_DESELECT=14, same epoch, 57 events). + +* **BU much better** ⇒ the clock-counted-field family is the whole story and + PAGEBREAK/COOLDOWN deserve a full scan. +* **BU flat** ⇒ QMI is exhausted; every field is either tuned or inert, and the + remaining ~1-per-8,000 floor is not a QMI timing problem. +* **BU worse** ⇒ same non-monotonicity as MIN_DESELECT; revert to COOLDOWN=1. + +## BU — COOLDOWN=2 is a DISASTER, and that makes COOLDOWN the most sensitive field found + +Window 09:33:35 → 10:36:09 (62.6 min). Valid — CRC OK, ALLOC_GUARD intact, +VREG 0xf0, QMI verified `0xa000e006` at both retune sites. + +Δboot = 156 → **156 crashes**; Δwork = **4,534** → **1 per 29**. + +**~277x worse than the 1-per-8,064 comparator** and worse than any configuration +measured in this investigation. Throughput collapsed from ~40 iterations/s to +**1.2/s** — 156 boots x ~12 s of boot time accounts for most of the window. The +device barely ran. + +| QMI_M0_TIMING field | value | rate | events | +|---|---|---|---| +| MIN_DESELECT | 7 | 1 per 131 | 94 | +| MIN_DESELECT | **14** | **1 per 8,064** | 57 (pooled) | +| MIN_DESELECT | 28 | 1 per 115 | 149 | +| RXDELAY @ 14 | 0 vs 2 | flat | 57 | +| **COOLDOWN** | **2** | **1 per 29** | **156** | + +### Correcting my own pre-registration + +BU's branch (c) said "worse ⇒ same non-monotonicity as MIN_DESELECT; revert to +COOLDOWN=1 and treat QMI as exhausted." **That branch was wrong to write.** It +conflated "this value is worse" with "this field is insensitive". A 277x swing +from a two-bit field is the largest effect anyone has produced here — larger than +MIN_DESELECT's 60x. The correct reading is that **chip-select timing is the +dominant axis and COOLDOWN is its most sensitive control**. + +Deviating from the pre-registered branch deliberately and on the record, rather +than following a rule I now believe was mis-specified. + +**Rule 77: A PRE-REGISTERED BRANCH IS A COMMITMENT AGAINST MOTIVATED READING, +NOT AGAINST NEW INFORMATION. If the effect size itself refutes the branch's +premise, say so explicitly and deviate — do not follow it silently, and do not +deviate silently either.** + +## BV — COOLDOWN 1 → 0: completing the field + +COOLDOWN is two bits, so the field is four values. `1` is the bootrom default and +the current best; `2` is catastrophic; `3` is presumably worse still. **`0` is the +only untested value**, and one window closes the field completely. + +Both live knobs are chip-select timing and they are opposite sides of the same +signal: `MIN_DESELECT` is how long CS must stay **deasserted**, `COOLDOWN` is how +long it stays **asserted** after a transfer (`64 x COOLDOWN` clk_sys cycles). +Noting which axis matters is not a mechanism claim (rules 70, 75) — and since +MIN_DESELECT already proved these fields have optima rather than monotone +directions, 0 is a genuine measurement, not an extrapolation from "2 is bad". + +Expect throughput to DROP: with COOLDOWN=0 the CS deasserts immediately after +every transfer, forcing a command prefix on each XIP access. The metric is +normalised by work (rule 54), so that is harmless to the comparison, but a low +iteration count in this window is expected and is not itself a fault signal. + +Device-side readback: **`QMI retune at main-loop entry: 0x2000e006 -> 0x2000e006 +(CLKDIV 6->6 SCK 50MHz, MIN_DESELECT 14->14)`** — COOLDOWN=0, PAGEBREAK=2, +MIN_DESELECT=14, RXDELAY=0, CLKDIV=6. CRC `0x81db1bf2`, integrity OK, V1_30, +300 MHz. Addresses re-derived, unchanged. + +Baseline **boot = 0, work = 131 at 10:39:24** — clean cold seed. +Comparator: **1 per 8,064** (pooled MIN_DESELECT=14, same epoch, 57 events). + +* **BV better** ⇒ the field has an optimum at the low end; COOLDOWN is the + operating point and the CS-timing axis is where the remaining orders of + magnitude are. +* **BV flat** ⇒ COOLDOWN=1 stands, the field is closed, QMI is genuinely + exhausted, and the investigation moves to the voltage re-check (AG). +* **BV worse** ⇒ COOLDOWN=1 is a sharp optimum. Field closed, same conclusion. + +## BV — COOLDOWN=0 is 21x worse. The field is CLOSED and QMI is EXHAUSTED. + +Window 10:39:24 → 11:42:09 (62.8 min). Valid — CRC OK, ALLOC_GUARD intact, +VREG 0xf0, QMI verified `0x2000e006`. + +Δboot = 135 → **134 crashes**; Δwork = **51,325** → **1 per 383**, **21x worse** +than the 1-per-8,064 comparator. + +### The complete COOLDOWN curve + +| COOLDOWN | rate | events | vs best | +|---|---|---|---| +| 0 | 1 per 383 | 134 | 21x worse | +| **1 (bootrom)** | **1 per 8,064** | 57 | **best** | +| 2 | 1 per 29 | 156 | 277x worse | +| 3 | not run — bracketed by 2 | | | + +A sharp optimum at the bootrom's own value, falling off hard in both directions. + +**And that kills the last version of the scaling story.** MIN_DESELECT needed +DOUBLING under the 2x overclock; COOLDOWN did NOT — its stock value is already +optimal. The fields do not share a correction, so "the overclock halved every +clock-counted field" is wrong as a general account, even though it happened to +produce the right answer for MIN_DESELECT. Rule 75, third confirmation. + +### QMI is exhausted + +| field | status | +|---|---| +| CLKDIV | tuned (6) | +| RXDELAY | **inert** at MIN_DESELECT=14 (57 events) | +| MIN_DESELECT | **tuned** (14) — control-verified, ~60x | +| COOLDOWN | **closed** (1) — sharp optimum, both sides measured | +| PAGEBREAK | unscanned, and NOT a time field | + +Nothing further on this register is worth an hour. The investigation moves to +the voltage re-check (AG). + +### A live fault caught during flashing — and it lands on the XIP path + +The first BW flash session died at 7.6 s with core 0 faulting inside +**`xip_check::sum_region`**, frame 1 = `0xfffffff8` (an EXC_RETURN, i.e. an +exception frame). A fault taken while reading XIP flash, in a routine whose +entire job is reading XIP flash. Independent confirmation of the XIP-path +conclusion from a completely different observation channel than the crash sector. + +Two subsequent flash attempts hit `The initialization of the flash algorithm +failed / A timeout occurred`; the OpenOCD RESCUE recovered it, and the reflash +then succeeded. `.uninit` was randomised by the rescue (rule 36) but the window +is cold-seeded anyway, so nothing was lost. + +## BW — REVERT to best-known QMI. This window is the rule-71 CONTROL. + +Reverting is required regardless, so it costs nothing extra to make it the +control. Two windows (BU, BV) have run since the reference was last measured and +the voltage experiment must not be staked on a stale baseline. + +Config: **MIN_DESELECT=14, RXDELAY=0, COOLDOWN=1, CLKDIV=6** — byte-identical to +the BS/BT-family best. Device-side readback: **`QMI retune at main-loop entry: +0x6000e006 -> 0x6000e006 (CLKDIV 6->6 SCK 50MHz, MIN_DESELECT 14->14)`**. +CRC `0x5de4654e`, integrity OK, V1_30, 300 MHz. Addresses re-derived, unchanged. + +Baseline **boot = 1, work = 13 at 11:48:11**. +Comparator: **1 per 8,064** (pooled, 57 events). + +* **Reproduces (~1 per 8,000)** ⇒ the whole afternoon's comparisons stand and the + voltage experiment can proceed on a trusted baseline. +* **Misses badly** ⇒ the epoch has stepped again, and BU and BV both have to be + re-read before anything built on them survives. + +### Next, once the control lands: the voltage question (AG) + +We are at **V1_30, the TOP of embassy's `CoreVoltage`**. The historical finding +that voltage moved the rate was measured in the old, badly-tuned QMI regime and +has never been re-checked at the tuned operating point. The informative test is +to step voltage DOWN one notch (V1_25) and measure the sensitivity: + +* **little or no change** ⇒ voltage is no longer a lever, and "electrical margin" + as the account of the residual rate is dead; +* **clearly worse** ⇒ the device is voltage-limited at 300 MHz, V1_30 being + embassy's ceiling is a real wall, and that is the evidence to put in front of + the user — **going above V1_30 requires defeating a regulator safety register + and MUST NOT be done without an explicit request.** + +## BW — THE CONTROL FAILED. The epoch stepped, and it inverts the COOLDOWN result. + +Window 11:48:11 → 17:08:39 (5 h 20 m — a tooling outage delayed the read, which +cost nothing: the counters are cumulative and not reading mid-window is correct +behaviour anyway, rule 50). Valid — CRC OK, ALLOC_GUARD intact, VREG 0xf0. + +Δboot = 857 → **856 crashes**; Δwork = **26,569** → **1 per 31**. + +This image is **byte-identical** to the config that measured 1 per 8,064 across +three windows. It now measures 1 per 31. **The epoch stepped ~260x, somewhere +around 09:30 today.** Pre-registered branch (b). + +### The corrected timeline + +| window | config | rate | epoch | +|---|---|---|---| +| 22:39 | MIN_DESELECT=14 | 1 per 9,676 | good | +| 23:45 | MIN_DESELECT=28 | 1 per 115 | good | +| 00:51 | 14 control | 1 per 6,408 | good | +| 08:25 | 14, RXDELAY=2 | 1 per 8,746 | good | +| 09:33 | COOLDOWN=2 | 1 per 29 | **bad from here** | +| 10:39 | COOLDOWN=0 | 1 per 383 | bad | +| 11:48 | COOLDOWN=1 (control) | 1 per 31 | bad | + +### What survives, and what does not + +**SURVIVES — MIN_DESELECT=14 vs 28.** The 28 window sits BETWEEN two good 14 +windows, so that comparison is same-epoch and stands (~70x). + +**DOES NOT SURVIVE — MIN_DESELECT 7 vs 14.** 7 was measured in the earlier bad +epoch. The 60x figure was always partly an epoch artifact. 14 is still the best +value we have, but its margin over 7 is unmeasured. + +**DOES NOT SURVIVE — COOLDOWN=2's "277x disaster".** It is confounded with the +epoch onset, and against its same-epoch neighbour it is indistinguishable from +COOLDOWN=1 (1 per 29 vs 1 per 31). + +**INVERTS — the COOLDOWN conclusion.** I compared BV against the stale +good-epoch comparator. Against its actual same-epoch neighbour, the BW control +six minutes later, the curve reverses: + +| COOLDOWN (all bad epoch, adjacent windows) | rate | events | +|---|---|---| +| 2 | 1 per 29 | 156 | +| **0** | **1 per 383** | 134 | +| 1 | 1 per 31 | 856 | + +**COOLDOWN=0 is ~12x better than the bootrom's 1**, on a three-point curve +measured entirely within one epoch. Last cycle I recorded the exact opposite and +declared the field closed on it. + +The error is precisely what rule 71 exists to prevent, and I made it anyway: I +interleaved the control at the END of the series rather than BETWEEN configs, so +every comparison in between was against a baseline that had already moved. + +**Rule 79: WHEN AN EPOCH STEP IS DETECTED, RE-READ EVERY WINDOW SINCE THE LAST +GOOD CONTROL AGAINST ITS NEIGHBOURS, NOT AGAINST THE OLD BASELINE. Adjacent +windows within one epoch are valid evidence even when the absolute numbers are +worthless.** + +**Rule 80: A CONTROL AT THE END OF A SERIES ONLY TELLS YOU THE SERIES IS VOID. +A CONTROL BETWEEN CONFIGS TELLS YOU WHICH CONFIG WON. Interleave, do not append.** + +### The epoch phenomenon is now the dominant term + +It is worth ~260x. No config knob found so far is worth more than ~70x. Two +observed steps (Aug 17 13:28, Aug 18 ~09:30), both good → bad, and at least one +recovery (bad → good overnight). It survives reset, reflash, cold-seed, and — per +the user — a power cycle. Not thermal, not transient. + +Boot timing is IDENTICAL across every config and both epochs (`QMI retune at +main-loop entry` logs at 12.168–12.170 s every single time), so the flash is not +globally slower in the bad epoch. The low throughput is fully explained by boot +time: ~856 boots x ~12 s accounts for the window, and ~31 iterations per boot +x 856 boots reproduces Δwork exactly. There is ONE phenomenon here — crashes per +iteration — not two. + +## BX — re-run COOLDOWN=0 against its SAME-EPOCH comparator + +Replicates BV, which is now the best result available in the current epoch. + +Device-side readback: **`QMI retune at main-loop entry: 0x2000e006 -> 0x2000e006 +(CLKDIV 6->6 SCK 50MHz, MIN_DESELECT 14->14)`** — COOLDOWN=0, PAGEBREAK=2, +MIN_DESELECT=14, RXDELAY=0, CLKDIV=6. CRC `0x73ec71d5`, integrity OK, V1_30, +300 MHz. Addresses re-derived, unchanged. + +Baseline **boot = 0, work = 308 at 17:12:34** — clean cold seed. +**Comparator: 1 per 31 (BW, adjacent window, SAME EPOCH, 856 events)** — NOT the +good-epoch 1 per 8,064. + +* **~1 per 383 again** ⇒ COOLDOWN=0 is real and worth ~12x; it becomes the new + best config and the next control goes BETWEEN configs, not after them. +* **~1 per 31** ⇒ BV was itself an epoch excursion, COOLDOWN is inert, and the + epoch stepping is the only thing left worth chasing. +* **Anything near 1 per 8,000** ⇒ the epoch has recovered on its own, which is + itself the most informative possible outcome about (AC). + +## BX — COOLDOWN=0 REPLICATES. The inverted reading was the correct one. + +Window 17:12:34 → 18:15:12 (62.6 min). Valid — CRC OK, ALLOC_GUARD intact, +VREG 0xf0, QMI verified `0x2000e006`. + +Δboot = 129 → **128 crashes**; Δwork = **55,756** → **1 per 436**, against BV's +1 per 383. Replicated. Pre-registered branch (a). + +### The COOLDOWN field, measured entirely within one epoch + +| COOLDOWN | rate | events | +|---|---|---| +| 2 | 1 per 29 | 156 | +| 1 (bootrom) | 1 per 31 | 856 | +| **0** | **1 per 409 (pooled)** | **262** | + +**Pooled COOLDOWN=0: 262 crashes / 107,081 iterations = 1 per 409 — 13x better +than the bootrom's COOLDOWN=1**, across two independent windows, every +comparison same-epoch and adjacent. + +This is the first result in the investigation built the way rule 80 demands, and +it is the exact opposite of what I published two cycles ago from the same raw +numbers read against a stale baseline. The lesson is not that COOLDOWN moved — +it is that **the comparator, not the measurement, was wrong**. + +Note also what this does to the COOLDOWN=2 story: 2 and 1 are indistinguishable +(1 per 29 vs 1 per 31). The "277x disaster" never existed; it was the epoch step +landing inside BU's window. + +## BY — MIN_DESELECT 14 → 7, at the NEW operating point COOLDOWN=0 + +One test, two open questions: + +1. **Rule 76.** MIN_DESELECT=14 was selected *entirely at COOLDOWN=1*. The + operating point has since moved, so the best MIN_DESELECT at COOLDOWN=0 is + unmeasured. This is the identical mistake-shape to the RXDELAY scan that was + run at the wrong MIN_DESELECT — worth catching before building on 14 again. +2. **MIN_DESELECT 7 vs 14 has never had a valid same-epoch comparison.** 7 was + measured in the earlier bad epoch, so the famous "60x" was always part config + and part epoch. 14 beat 28 same-epoch and that stands, but 14 vs 7 is + genuinely unknown. + +ONE VARIABLE: only MIN_DESELECT moves, 14 → 7. + +Device-side readback: **`QMI retune at main-loop entry: 0x20007006 -> 0x20007006 +(CLKDIV 6->6 SCK 50MHz, MIN_DESELECT 7->7)`** — COOLDOWN=0, PAGEBREAK=2, +MIN_DESELECT=7, RXDELAY=0, CLKDIV=6. CRC `0x443d139d`, integrity OK, V1_30, +300 MHz. Addresses re-derived, unchanged. + +Baseline **boot = 0, work = 308 at 18:18:14** — clean cold seed. +**Comparator: 1 per 436 (BX, adjacent, same epoch, 128 events)**, or the pooled +1 per 409. NOT any pre-09:30 number. + +* **Clearly worse than 1 per 436** ⇒ MIN_DESELECT=14 is confirmed on its own + merits at last, same-epoch, and the pairing (14, COOLDOWN=0) is the config to + build on. +* **Flat** ⇒ MIN_DESELECT is inert at COOLDOWN=0 and its entire apparent effect + was epoch — a big claim, and it would mean COOLDOWN is the only real QMI knob. +* **Better** ⇒ the fields interact and the MIN_DESELECT scan must be redone from + scratch at COOLDOWN=0. + +Best available config is now **1 per 409 in the current epoch** (~tens of +thousands of crashes/day). The goal remains ZERO in 24 h. The epoch term (~260x) +still dwarfs every config knob found. + +## BY — a 73-HOUR window. MIN_DESELECT is INERT at COOLDOWN=0, and the bad epoch NEVER RECOVERED. + +**Session interruption note.** The investigation loop stopped at 19:21 on +2026-08-18 when the driving session hit a weekly usage limit, mid-cycle, one +minute after reading the BY counters and before interpreting them. The board was +left powered and running config BY untouched for the next three days. It was +resumed on 2026-08-21 at 19:37. Nothing was reflashed, reset, or reseeded in +between, so the BY window is intact and simply 73 times longer than intended. + +Window 2026-08-18 18:18:14 → 2026-08-21 19:37:31 (**73.3 hours**). Valid — +magic `0x48b70001`, ALLOC_GUARD `0xa1100001` intact, VREG `0xf0`, no reflash. + +Δboot = 9382 → **9381 crashes**; Δwork = **3,767,241** → **1 per 401**. + +This is by an order of magnitude the largest sample in the investigation: 9,381 +events against the 128–856 of every previous window. + +### MIN_DESELECT 7 vs 14 — FLAT. Pre-registered branch (b). + +| config | MIN_DESELECT | rate | events | 95% CI | +|---|---|---|---|---| +| BX | 14 | 1 per 436 | 128 | 1 per 360 – 512 | +| **BY** | **7** | **1 per 401** | **9,381** | **1 per 393 – 409** | + +BY sits inside BX's interval. **MIN_DESELECT is inert at COOLDOWN=0**, and the +famous "60x for 14 over 7" was epoch, entirely. That was the branch flagged as +"a big claim" when BY was pre-registered, and it is the one that landed. + +Note what this does to the QMI map: MIN_DESELECT=14 beat 28 same-epoch (~70x) +and that still stands, but 14 and 7 are now indistinguishable. The field is not +monotonic in the way a pure setup/hold-time story would require — which is the +third time a QMI field has confirmed as a *setting* while refuting the +*mechanism* attached to it (rule 75). + +### THE REAL RESULT: the epoch did not recover in 73 hours + +Every previous statement about the epoch was inferred from 1-hour windows +minutes apart. This window covers three continuous days, and the long-run +average is **1 per 401** — squarely inside the COOLDOWN=0 bad-epoch band +(383 / 436 / 401), and **19x away from the good-epoch 1 per 8,064**. + +The mixing argument bounds it hard. If a fraction *f* of iterations had run in a +good epoch at 1/8,064 and the rest at ~1/400, then observing 1/401 over 3.77M +iterations puts *f* at essentially zero. **The board spent effectively all of +those 73 hours in the bad epoch.** + +**Rule 81: THE "OVERNIGHT RECOVERY" IS NOT A CYCLE. A bad epoch has now been +observed to persist for 73 continuous hours across three day/night cycles. Any +model that treats the epoch as diurnal, thermal, or self-clearing is refuted. +The single observed bad→good recovery (Aug 17→18) was an event, not a period.** + +This also inverts the framing. The good epoch — 1 per 8,064 — was seen in a +handful of adjacent windows on one evening. The bad state has now held for three +days. **The 1-per-400 regime is the device's normal behaviour, and the good +epoch is the anomaly to be explained**, not the other way round. + +### Fault signature at the 73-hour mark + +31 records, ring wrapped once (`erase_count=1`, `next_slot=0`), CRC OK on all: + +| kind | count | +|---|---| +| HardFault | 29 | +| WatchdogTimeout | 1 | +| Panic | 1 | + +| ARM PC | count | CFSR | +|---|---|---| +| `0x88000000` | 13 | IBUSERR | +| `0x1002e224` | 13 | BFARVALID PRECISERR | +| `0x0000fe9e` | 2 | IBUSERR | +| `0x2004c500` | 1 | INVSTATE | + +The mix has moved decisively since the 31-record sample that opened this +document (15 stack-protector / 11 HardFault / 5 watchdog). The stack-protector +mode is **gone**, and the failure is now almost perfectly bimodal, 13 and 13: + +* `PC = 0x88000000`, IBUSERR — a wild branch to an address that does not decode + to anything on RP2350. Not flash, not SRAM, not a peripheral. +* `PC = 0x1002e224`, precise bus fault with BFAR valid — a *legal XIP address*, + faulting on a data access from live code. + +Both are the XIP path, which is consistent with everything since the +`stack_pop_check` result (0 mismatches in 2.48 billion SRAM LDM/STM +verifications) and with the live fault caught inside `xip_check::sum_region` +during a flash session. + +### Config BZ — a same-duration CONTROL before anything else moves + +The temptation is to go straight to the voltage re-check (AG), which has been +queued since BW. That would repeat the exact error rules 71/79/80 exist to +prevent: AG's comparator would be a 73-hour average, measured against a 1-hour +window, with an epoch term worth ~260x sitting between them. + +So: **re-run BY's image, unchanged, as a 1-hour control.** No rebuild, no +reflash — the flashed image is already config BY (CRC `0x443d139d`), only the +counters and crash sector are re-seeded. + +ONE VARIABLE: none. This is a pure control. + +It does two jobs at once: + +1. It gives AG a **same-epoch, same-duration** comparator, as rule 80 demands — + interleaved *before* the change, not appended after it. +2. It measures whether a 1-hour window right now reproduces the 73-hour average, + which is the first direct test of whether the epoch is currently *stable* or + merely *slow-drifting*. Every previous epoch claim rests on 1-hour windows; + none of them ever had a long-run number to check against. + +* **~1 per 401** ⇒ the epoch is stable and the comparator is trustworthy; + proceed to AG (V1_30 → V1_25) immediately with this as its baseline. +* **Materially different** ⇒ the epoch is drifting on a timescale *shorter* than + a day, every 1-hour window in this document is suspect on its own terms, and + the instrument — not the config — is what needs fixing before AG runs. + +### Config BZ — running + +No rebuild, no reflash. Device-side readback confirms the flashed image is still +BY: `integrity: full image crc 0x443d139d OK`, `QMI retune at main-loop entry: +0x20007006 -> 0x20007006 (CLKDIV 6->6 SCK 50MHz, MIN_DESELECT 7->7)` — COOLDOWN=0, +PAGEBREAK=2, MIN_DESELECT=7, RXDELAY=0, CLKDIV=6. V1_30 (VREG `0xf0`), 300 MHz. +Boot log at **12.5045 s**, identical to every window in this document. + +Cold seed: crash sector blanked *and* reset (Y3 rule — never blank without +resetting). Addresses unchanged; no build, so rule 5 re-derivation does not apply. + +Baseline **boot = 1, work = 284 at 2026-08-21 19:41:47**. Note the baseline was +read ~3 min after the reset rather than immediately, so the initial non-crash +boot is already counted in it: for this window **crashes = boot_end − 1**, which +is the same formula as always, just with the offset captured explicitly. + +**Comparator: BY's 1 per 401 (9,381 events, 73.3 h, same image, same epoch).** + +## BZ — THE CONTROL HOLDS. A 1-hour window reproduces the 73-hour average. + +Window 19:41:47 → 20:45:10 (63.4 min). Valid — magic `0x48b70001`, ALLOC_GUARD +`0xa1100001`, VREG `0xf0`, image untouched. + +Δboot = 139 → **138 crashes**; Δwork = **53,475** → **1 per 388** +(95% CI 1 per 323 – 452). + +| window | duration | rate | events | +|---|---|---|---| +| BY | 73.3 h | 1 per 401 | 9,381 | +| **BZ** | **63.4 min** | **1 per 388** | **138** | + +BY sits deep inside BZ's interval. **Pre-registered branch (a).** + +### What this earns + +This is the first time in the investigation that a 1-hour window has been +checked against a long-run number, and it passes. Two things follow: + +1. **The epoch is currently stable.** It is not drifting on a sub-day timescale, + so the comparator for the next config is trustworthy — which is exactly the + precondition rules 71/79/80 demand before another config is allowed to move. +2. **The 1-hour window is a sound instrument at this operating point.** Roughly + 130–140 events per hour gives ±17%, which resolves the ~13x effects that + matter and cannot resolve anything under ~1.4x. That is the honest resolution + limit and it should be quoted whenever a window comes back "flat". + +Note what BZ does *not* license: it says nothing about whether the epoch will +step *during* some future window. It says the instrument is sound right now. + +### Signature: the wild-branch mode is dispersing + +31 records, all CRC OK. 28 HardFault / 3 WatchdogTimeout. + +| ARM PC | count | +|---|---| +| `0x88000000` | 9 | +| `0x1002e224` | 5 | +| `0x2002e6d4` | 3 | +| `0x89000000` | 2 | +| `0x68000000` | 2 | +| `0x200032ac` | 2 | +| `0x0000fe9e` | 2 | +| `0x2004c500` | 1 | + +CFSR: 15 IBUSERR, 5 INVSTATE, 5 BFARVALID PRECISERR, 2 UNALIGNED, 1 UNDEFINSTR. + +BY's clean 13-and-13 bimodality was itself partly a small-ring artifact — the +same two addresses still dominate, but the tail is much wider than 31 records +could show. `0x1002e224` and `0x88000000` are reproducible across both windows. + +**A new observation worth carrying forward: every wild branch target has all-zero +low 24 bits** — `0x88000000`, `0x89000000`, `0x68000000`. A branch target of the +form `0xNN000000` is not what a randomly corrupted pointer looks like; it is what +a word looks like when only its **top byte** carries surviving data. That is a +different failure shape from "the whole word was replaced", and it is the first +structural hint about the corrupt value since the poison-store work in Z2/AA. +Not chased now — AG is already pre-registered and one variable moves at a time — +but it belongs in the queue ahead of any further QMI archaeology. + +## Config AG — the voltage re-check, at last on a validated comparator + +The experiment queued since BW. We sit at **V1_30, the top of embassy's +`CoreVoltage`**. The historical "voltage is a ~7x modulator" note was measured in +the old badly-tuned QMI regime *and* against the crash-log counter that rule 49 +showed was saturating, so it has never been trustworthy. Now it can be measured +properly: the QMI operating point is settled, the counters are sound, and BZ has +just proven the comparator. + +ONE VARIABLE: `TARGET_CORE_VOLTAGE` V1_30 → V1_25. QMI, clock, and layout all +held. The active constant is the `cfg(all(not(oc-300), not(oc-280)))` arm — +default features are `["firmware-bin"]`, so neither `oc-` feature is on. + +**Comparator: BZ's 1 per 388 (138 events), immediately adjacent, same epoch, +same duration** — interleaved before the change, per rule 80. + +* **Little or no change (inside ~1 per 323–452)** ⇒ voltage is not a lever at the + tuned operating point, and "electrical margin" as the account of the residual + rate is **dead**. That is the outcome that most changes the direction of the + investigation, because it would leave the ~260x epoch term with no electrical + explanation and force the wild-branch structure above to the front. +* **Clearly worse** ⇒ the device is genuinely voltage-limited at 300 MHz, and + V1_30 being embassy's ceiling is a real wall rather than an arbitrary stop. + **Going above V1_30 requires defeating a regulator safety register and MUST NOT + be done without an explicit request from the user** — that result would be the + evidence to put in front of them, not a licence to proceed. +* **Better at the LOWER voltage** ⇒ the electrical story is inverted and the + whole "marginal timing at 2x rated clock" model needs rebuilding. + +### Config AG — running + +Device-side readback: `integrity: full image crc 0x6a6f0b36 OK`; +`QMI retune at main-loop entry: 0x20007006 -> 0x20007006 (CLKDIV 6->6 SCK 50MHz, +MIN_DESELECT 7->7)` — QMI **unchanged** from BZ, confirming one variable. +**VREG `0x4010000c` = `0x000000e0` = V1_25**, verified on the device after the +flash (it read `0xf0` for every window from BE onward). 300 MHz untouched. +Boot log 12.1717 s. + +Rule 5 re-derivation after the build — addresses **unchanged**: +`HEARTBEAT=0x20066fa8`, `ALLOC_GUARD=0x200677a0` (`0xa1100001` intact), +`_stack_end=0x20067b18`, `BOOT_COUNT=0x2006700c`, `WORK_COUNT=0x20067010`. + +Cold seed: crash sector blanked *and* reset (Y3 rule). + +Baseline **boot = 2, work = 161 at 2026-08-21 20:49:53**, read ~45 s after the +reset. Two boots had already occurred (the reset boot plus one crash), so for +this window **crashes = boot_end − 2**. + +**Comparator: BZ's 1 per 388, 138 events, adjacent window, same epoch, same +duration.** Resolution limit ~1.4x (see BZ). + +## AG — VOLTAGE IS FLAT. Electrical margin is dead as an account of the residual rate. + +Window 20:49:53 → 21:52:08 (62.2 min). Valid — magic OK, ALLOC_GUARD +`0xa1100001`, **VREG `0xe0` = V1_25 held for the whole window**, QMI `0x20007006` +unchanged from BZ. + +Δboot = 141 − 2 → **139 crashes**; Δwork = **47,862** → **1 per 344** +(95% CI 1 per 287 – 402), against BZ's **1 per 388** (CI 323 – 452). + +Intervals overlap heavily in both directions. **Pre-registered branch (a).** + +### This is an exclusion, not a shrug + +"Flat" usually means "we could not tell". Here we could. The window had ample +power to see the historically claimed effect: + +| if voltage were… | expected events | observed | +|---|---|---| +| 28x worse (the old note) | ~3,454 | **139** | +| 7x worse (the other old note) | ~863 | **139** | +| 2x worse | ~247 | **139** | +| 1.5x worse | ~185 | **139** | + +**Voltage sensitivity across V1_25 → V1_30 is below ~1.4x.** The historical +"voltage is a ~7x modulator" and "voltage was 28x" claims are **refuted**, not +merely unreplicated. Both were measured in the badly-tuned QMI regime against +the saturating crash-log counter (rule 49). Note the sign, too: V1_25 came out +*nominally worse* (344 vs 388), so there is no hidden benefit at lower voltage +either — branch (c) is closed as well. + +With clock established as a rate modulator only, QMI exhausted (COOLDOWN 13x, +every other field inert), and voltage now flat, **there is no electrical knob +left that explains the residual rate.** That was pre-registered as the outcome +that most changes direction, and it is the one that landed. + +## THE COMBINED-WINDOW FINDING: the corrupt words are GAME BOY ADDRESSES + +With BY, BZ and AG pooled (93 records), every PC / LR / BFAR value was +classified by which byte lanes are zero. Values that are legal RP2350 addresses +(`0x10……` XIP, `0x20……` SRAM) were separated from anomalous ones. The anomalous +set is not diffuse — it collapses into three patterns: + +| lanes [3210] | n | meaning | examples | +|---|---|---|---| +| `DZZZ` | 37 | only the TOP byte survives | `0x88000000`, `0x89000000`, `0x68000000` | +| `ZZDD` | 32 | only the BOTTOM half-word survives | `0x0000e06a`, `e072`, `e07c`, `e084`, `e114`, `0x0000fe9e` | +| `ZZZD` | 5 | only the bottom byte | `0x000000a0`, `0x00000002`, `0x00000001` | +| `DDDD` | 1 | full garbage | `0x10daf8bb` | + +Read those as **16-bit values placed in one half of a 32-bit word**: + +* high half: `0x8800`, `0x8900`, `0x6800` +* low half: `0xe006`, `0xe06a`, `0xe072`, `0xe07c`, `0xe084`, `0xe114`, `0xfe9e` + +**Every one of them is a Game Boy address, and they land in meaningful regions.** +`0x8800` is the VRAM tile-data block-1 base. `0xE000–0xFDFF` is echo RAM. +`0xFE00–0xFE9F` is OAM — and `0xfe9e` is its very last byte. `Sm83`'s own +recorded `GB SP` in these records sits at `0xdff3`, immediately below the echo +region. + +This is what a stream of `u16` Game Boy addresses looks like when the memory it +was written into is later read back as `u32` words: each corrupt word carries +one 16-bit address in one half and zero in the other, and which half depends +only on the parity of the halfword offset. + +**Rule 82: THE CORRUPT VALUES ARE STRUCTURED, AND THE STRUCTURE IS EMULATOR DATA, +NOT NOISE. Anomalous words are 16-bit Game Boy addresses (VRAM tile-data base, +echo RAM, OAM) deposited into one half of a 32-bit word. A silicon read-path +fault would produce address-space-shaped garbage or bit-flips of the true value; +it would not preferentially produce valid Game Boy addresses. This is a SOFTWARE +write going somewhere it should not.** + +That reframes the whole investigation, and it is consistent with evidence that +was already in this document and never joined up: + +* the early HardFaults returning through a corrupted saved PC from the epilogue + of `SharedWorkerState::write_live_vram_range` — a function whose first + parameter is a `u16` offset; +* config AC's MPU-detected **core-0 write into core-1's stack**; +* the standing note in `memory.rs` that the only way the OAM-DMA stores escape + is "if `self` is itself a wild pointer", which "lands the indexed store inside + whatever core-0 frame is currently live"; +* and why every electrical knob only ever *modulated* the rate — timing changes + the width of a cross-core race window, it does not remove the race. + +### The one fully-decoded frame + +`HF_STACK` (`0x20067048`, magic `0x48f50001` valid) held a complete capture: + + pc=0x000000a0 lr=0x20001f67 cfsr=0x00020000 INVSTATE sp_before=0x2007cd10 + stacked frame: r12=0x1001424f lr=0x20001f67 pc=0x000000a0 xPSR=0x08000000 + r4-r11 = 2000087d 2007cd38 20000ecb 200273d4 2007d2d8 2003d5b0 200273d4 2003d5b0 + +`xPSR = 0x08000000` has **bit 24 (T) clear**, which is exactly why CFSR is +INVSTATE: the core branched to an even address. Note that the stacked `r12` and +`LR` are both **valid** (`0x1001424f` XIP, `0x20001f67` SRAM — the +`Instructions::cb` region named in the earlier records), while the *adjacent* +`PC` and `xPSR` words are both corrupt. Two adjacent words clobbered, the two +before them clean: a short contiguous overwrite, not a single stray store. + +Also note `LR = 0x20001f67` is **SRAM** code. The standing "everything is on the +XIP path" summary needs qualifying — the memcpy faults are XIP (`0x1002e224`), +but this one was executing from RAM. + +### `0x1002e224` identified + +`llvm-objdump` resolves it exactly: + + 1002e224: f812 6b01 ldrb r6, [r2], #1 ; compiler_builtins::mem::memcpy + +It is the byte-copy tail of **memcpy**, and `BFAR` is its *source pointer* — +`0x0000e06a` in 7 of 10 cases. So memcpy is being handed a Game Boy echo-RAM +address as a host source pointer. Same family, different symptom. + +### Two mechanisms ruled OUT this cycle + +**DMA is exonerated.** `DMA_CRASH_SNAPSHOT` (`0x20067ad0`, sentinel +`0xd4a0c12a` valid) reads `busy_mask = 0x00000000` — no channel was active at +fault time — and the only two live channel write-addresses are `0x50200010` and +`0x40088008`, both **peripherals**. No DMA channel in this firmware writes into +SRAM at all, so DMA cannot be spraying the stack. + +**The MPU cross-core violation is not currently arming.** `MM_REGS` +(`0x20067128`) holds no magic — it is uninitialised noise, so the deterministic +core-0→core-1-stack violation of config AC/AE has not fired in the current +build. That instrument is dark and would need re-arming before it can contribute. + +## Config CA — REVERT to V1_30, and a replication of BZ + +AG is reverted because V1_30 is the established operating point and V1_25 was +nominally worse. ONE VARIABLE: voltage back to V1_30. This window doubles as a +**replication of BZ at the best-known config**, which is worth having before the +investigation turns to the software hypothesis. + +Device readback: `integrity: full image crc 0x4285c1f3 OK`; `QMI retune at +main-loop entry: 0x20007006 -> 0x20007006 (CLKDIV 6->6 SCK 50MHz, MIN_DESELECT +7->7)`; **VREG `0x4010000c` = `0x000000f0` = V1_30**, verified after the flash. +300 MHz untouched. Boot log 12.1717 s. + +Rule 5 re-derivation — addresses **unchanged**: `HEARTBEAT=0x20066fa8`, +`HF_STACK=0x20067048`, `MM_REGS=0x20067128`, `ALLOC_GUARD=0x200677a0`, +`DMA_CRASH_SNAPSHOT=0x20067ad0`, `_stack_end=0x20067b18`. + +Cold seed: crash sector blanked *and* reset. Baseline **boot = 1, work = 213 at +2026-08-21 22:02:20**; **crashes = boot_end − 1**. + +**Comparator: BZ's 1 per 388 (138 events) and AG's 1 per 344 (139 events).** + +* **~1 per 340–390** ⇒ BZ replicates, the epoch is still stable across three + consecutive windows, and the electrical chapter closes cleanly. +* **Materially different** ⇒ an epoch step landed inside CA, and rule 79 applies + to AG as well — the voltage conclusion would need re-reading against neighbours + before it can be trusted. + +### Next, regardless of CA: hunt the writer, not the volts + +The queue is now software-side, in priority order: + +1. **Re-arm the MPU instrument.** It is the only tool that has ever caught the + cross-core write in the act, and it is currently dark. A live capture with + `MM_REGS` populated would name the storing function directly. +2. **Instrument `write_live_vram_range` / `write_live_oam_range` bounds.** Both + take `u16 start_offset` and both currently clamp with `min`/`saturating_sub`, + so they are safe *if `self` is sound* — which is precisely the assumption the + `memory.rs` comment says is in doubt. +3. **Widen `HF_STACK` to a ring.** Every structural claim above rests on ONE + fully-decoded frame plus 93 records that only carry PC/LR/CFSR/BFAR. A ring of + 4–8 frames per window would make rule 82 testable rather than inferred. + **Caution: `.uninit` placement is load-bearing here — `_stack_end` is the end + of the last `.uninit` object and core 1's read-only MPU region starts at + `_stack_end & !0x1F`. `MM_REGS` is already padded to 20 words for exactly this + reason. Any resize must re-check that boundary.** + +## CA — BZ replicates. Three consecutive windows agree, and voltage is confirmed inert. + +Window 22:02:20 → 23:05:13 (62.9 min). Valid — magic OK, ALLOC_GUARD intact, +VREG `0xf0` held, QMI `0x20007006`. + +Δboot = 140 − 1 → **139 crashes**; Δwork = **49,320** → **1 per 355** +(95% CI 1 per 296 – 414). **Pre-registered branch (a).** + +| window | voltage | rate | events | +|---|---|---|---| +| BZ | V1_30 | 1 per 388 | 138 | +| AG | V1_25 | 1 per 344 | 139 | +| CA | V1_30 | 1 per 355 | 139 | + +**Pooled V1_30 (BZ+CA): 277 events / 102,795 iterations = 1 per 371**, against +V1_25's 1 per 344. Still flat on double the sample. The epoch has now held +stable across four consecutive windows (BZ, AG, CA and the 73-hour BY), so the +comparators in this block are sound and the electrical chapter closes cleanly. + +## RULE 83 — THE WRITE-ONCE INSTRUMENTS WERE SERVING STALE DATA + +`HF_STACK` read **byte-for-byte identical** at 21:55 and again at 23:05 — either +side of a full rebuild, a reflash to a different CRC, a cold seed, and 139 +crashes. Reading the handler explains why, and it is worse than it looks: + +```rust +if cfsr & CFSR_INVSTATE != 0 { // gate 1: only INVSTATE + if h.read_volatile() != HF_STACK_MAGIC { // gate 2: write-once, forever +``` + +Two compounding defects: + +1. **It only ever captured INVSTATE**, which is the *rarest* mode — 3-5 records + out of ~139 per window. The two dominant modes, the IBUSERR wild branch + (n=37) and the memcpy PRECISERR (n=32), could never be captured at all. +2. **Write-once in `.uninit` outlives everything.** `.uninit` survives reset, and + `probe-rs download` writes *flash*, never SRAM — so the cold seed never + touched it. The block persists until a rescue reset randomises SRAM. + +**Rule 83: A WRITE-ONCE INSTRUMENT IN `.uninit` IS NOT DATED. It survives reset, +reflash and cold seed, so a capture of unknown age presents itself as current. +Any `.uninit` instrument must have its magic CLEARED AS PART OF THE COLD SEED, +and the cold-seed procedure is now: blank the crash sector, zero the heartbeat +magic, ZERO EVERY INSTRUMENT MAGIC, then reset.** + +### Correction to the previous cycle + +The fully-decoded frame reported under AG (`pc=0x000000a0`, `xPSR=0x08000000`) +is of **unknown age and is from the rarest fault mode**. It was not a capture +from the AG window. Two claims built on it are withdrawn as unsupported: that +"stacked r12 and LR are valid while the adjacent PC and xPSR are corrupt" is a +general shape, and that `LR=0x20001f67` shows the failure is not always on the +XIP path. Both may still be true; neither is evidenced by a dated observation. + +**Rule 82 is unaffected** — it rests on the 93 ring records from BY/BZ/AG, which +are per-window and correctly dated. + +### Config CB — the gate is removed + +`HF_STACK` now captures the **first HardFault of any mode** after its magic is +cleared, keeping write-once semantics *within* a window instead of forever. No +`.uninit` resize, so the load-bearing tail boundary is untouched (verified: +`MM_REGS` written slots end at `0x2006715c`, core-1 MPU region base is +`0x20067b00`). + +## CB — FIRST EVER CAPTURE OF THE DOMINANT MODE, AND THE CONTIGUOUS-RUN QUESTION IS ANSWERED + +Within ~40 seconds of the cleared magic, the instrument caught the mode that had +been invisible for the entire investigation: + + pc=0x88000000 lr=0x1001fd49 cfsr=0x00000100 IBUSERR + sp_before=0x2007cd48 r12=0x1001e79d exc_return=0xfffffff9 (core 0, MSP) + +The band around `sp_before` is the payload: + +| address | word | meaning | +|---|---|---| +| `0x2007cd38` | `1001e79d` | stacked r12 — **valid** | +| `0x2007cd3c` | `1001fd49` | stacked LR — **valid** | +| `0x2007cd40` | `88000000` | stacked PC — **corrupt** | +| `0x2007cd44` | `68000000` | stacked xPSR — **corrupt** | +| `0x2007cd48` | `88000000` | ← `sp_before` | +| `0x2007cd4c` | `68000000` | | +| `0x2007cd50` | `88000000` | | +| `0x2007cd54` | `88000000` | | + +**SIX CONSECUTIVE WORDS overwritten with `0x88000000` / `0x68000000`.** + +This settles the question the AM/AN/AQ configs were built to answer and which +this document calls the one the investigation hinges on — *is it one stray word +or a contiguous run?* **It is a contiguous run.** The run spans and destroys the +exception frame's PC and xPSR, which is precisely why the core ended up +branching to `0x88000000`: it returned through a word that had already been +overwritten. + +Per rule 82 the halfword sequence is `0000 8800 0000 6800 0000 8800 0000 8800` — +16-bit Game Boy values at a 4-byte stride, exactly the shape of a **struct array +being written**, not of a bit-flip or a bus fault. + +### The smashed frame is the CROSS-CORE COMMAND QUEUE PRODUCER + +Resolving the live registers against the ELF names the whole chain: + +| reg | address | symbol | +|---|---|---| +| LR | `0x1001fd49` | `+0x18` in **`heapless::spsc::QueueInner::inner_enqueue`** | +| — | `0x1001fd18` | the `increment` it had just called | +| r12 | `0x1001e79d` | `GameBoy::write_apu_register` | +| r6 | `0x1000a3c3` | embassy main task, immediately after `bl drain_bus_events` | + +Core 0 was **inside `inner_enqueue`, pushing a `Core1Command` onto the core-0 → +core-1 command queue**, when a contiguous run of `Core1Command`-shaped words +landed on its own live stack frame. + +That is a complete and self-consistent mechanism, and it unifies everything: + +* the corrupt words are Game Boy values because **they are `Core1Command` + payloads** — the queue carries emulator writes (`write_apu_register` and the + VRAM/OAM range writers all funnel through it); +* it is cross-core, which is what config AC's MPU catch saw as a core-0 write + into core-1's stack; +* it is a contiguous run because an enqueue writes a whole struct; +* and every electrical knob only ever *modulated* it, because clock and QMI + timing change the width of a producer/consumer race window without removing + the race. + +**The working hypothesis is now specific: `inner_enqueue` is computing a slot +address outside the queue buffer.** With `ViewStorage` the queue is type-erased — +capacity lives in a runtime field rather than the type — so a corrupt queue +pointer, a corrupt `tail`, or a racing index all produce a wild slot address. +This is the same shape as the standing `memory.rs` note that stores only escape +"if `self` is itself a wild pointer". + +### Config CC — give COMMAND_QUEUE the guard that AUDIO_QUEUE already has + +There is a ready-made precedent in this codebase. `AUDIO_QUEUE` is **not** a bare +queue: a previous investigation wrapped it in `AudioQueueStorage { queue, guard }` +with a `check_audio_queue_guard`. `COMMAND_QUEUE` never got the same treatment — +it is still a bare `StaticCell>` with nothing +around it. + +ONE VARIABLE: wrap `COMMAND_QUEUE` in a guarded storage struct and check the +guard words, exactly as the audio queue does. + +* **Guard breached** ⇒ the command queue is being overrun, the writer is named, + and this becomes a bounded software bug rather than a silicon mystery. +* **Guard intact while smashes continue** ⇒ the enqueue is not overrunning its + own buffer; the slot address is wild rather than merely out of range, which + points at the `Producer`/queue pointer itself and makes that the next target. + +Note this is an *instrument*, not a fix — per the standing directive to root +cause rather than blindly patch. A guard that fires tells us where to look; it +does not paper over the write. + +### Config CB — running + +No functional change beyond the `HF_STACK` gate. Device readback: `integrity: +full image crc 0xda9296b3 OK`; `QMI retune at main-loop entry: 0x20007006 -> +0x20007006`; **VREG `0xf0`**; 300 MHz; boot log 12.5063 s. Rule 5 re-derivation — +all addresses unchanged. + +Cold seed under the new rule-83 procedure: crash sector blanked, heartbeat magic +zeroed, **`HF_STACK` and `MM_REGS` magics zeroed**, then reset. + +Baseline **boot = 1, work = 215 at 2026-08-21 23:10:16**; **crashes = boot_end − 1**. + +**Comparator: pooled V1_30 1 per 371 (277 events).** CB changes only the crash +handler's capture gate, so the rate should be unmoved; a material change would +itself be evidence that the handler path participates in the failure. + +## CB — THE RATE MOVED 2.36x WORSE, AND NOTHING IN THE DIFF CAN EXPLAIN IT + +Window 23:10:16 → 00:15:12 (64.9 min). Valid — magic OK, ALLOC_GUARD +`0xa1100001`, VREG `0xf0`, QMI `0x20007006`. + +Δboot = 194 − 1 → **193 crashes**; Δwork = **30,374** → **1 per 157** +(95% CI 1 per 135 – 180), against the pooled V1_30 comparator of **1 per 371** +(277 events, CI 1 per 327 – 415). + +**The intervals do not overlap.** This is CB's pre-registered material-change +branch, and it must be resolved before any new config is allowed to build on it. + +### The diff cannot mechanically produce this + +CB's only change is deleting the `cfsr & CFSR_INVSTATE` gate on the `HF_STACK` +capture. That block is **write-once**, and the magic was confirmed set within +~40 seconds of the cold seed. So for **192 of this window's 193 crashes the +handler executed byte-identical code to CA's.** One fault's worth of extra +capture work cannot produce a sustained 2.36x. + +The leading explanation is therefore an epoch step, not the instrument. But that +is precisely the kind of reasoning rule 75 exists to distrust — three times in +this investigation a change that "could not matter" has been confirmed as +mattering — so the baseline is being re-established by measurement rather than by +argument. + +### The signature shifted too, which argues for a regime change + +| ARM PC | count | vs CA/AG | +|---|---|---| +| `0x1002e220` | 10 | **new** — 4 bytes *before* the usual `0x1002e224` | +| `0x88000000` | 5 | down from 9-10 | +| `0x68000000` | 5 | **up from 2** | +| `0x20007006` | 2 | new | +| `0x00000000` | 2 | | +| `0xe4000002`, `0xcb000002`, `0x53000004` | 1 each | new | + +CFSR: 12 IBUSERR, 10 BFARVALID PRECISERR, 5 INVSTATE, **3 IACCVIOL**. + +`IACCVIOL` is **new to this investigation** — an MPU *instruction* access +violation. Note it does not set `MMFAR`, and `MM_REGS` is gated on `MMARVALID`, +which is why `MM_REGS` stayed dark (`0x00000000`, the value the cold seed wrote) +despite MPU faults occurring. That is a gap in the instrument, not an absence of +MPU events. + +A rate change accompanied by new fault modes and a shifted mode mix looks more +like a regime change than like the same phenomenon running faster. + +### Correction: boot timing is NOT identical across windows + +This document has repeatedly asserted that `QMI retune at main-loop entry` logs +at 12.168-12.172 s "every single time", and used that to argue the flash is not +globally slower. The observed values are actually **bimodal**: + +| window | boot | rate | +|---|---|---| +| BZ | 12.5045 | 1 per 388 | +| AG | 12.1717 | 1 per 344 | +| CA | 12.1717 | 1 per 355 | +| CB | 12.5063 | 1 per 157 | +| CC0 | 12.1702 | — | + +Two distinct modes, ~12.17 and ~12.50, differing by ~330 ms. They do **not** +track the crash rate — BZ sat at 12.50 with a perfectly normal 1 per 388 — so +this does not explain CB. But the "identical every time" claim is false and +should stop being cited as evidence. + +## Config CC0 — CONTROL. The gate is restored; nothing else moves. + +Rule 80 demands the control go *between* configs, not after the series. The +command-queue guard (CC) is deferred one window so it is not built on an +unexplained baseline. + +ONE VARIABLE: the `HF_STACK` INVSTATE gate is reinstated, making this image +functionally identical to CA. `HF_STACK` is not being abandoned — only paused for +one window; the CB capture it already produced is the most valuable single +observation in the investigation. + +Device readback: `integrity: full image crc 0x3d9cc95e OK`; `QMI retune at +main-loop entry: 0x20007006 -> 0x20007006`; **VREG `0xf0`**; 300 MHz; boot log +12.1702 s. Rule 5 re-derivation — all addresses unchanged. Cold seed under the +rule-83 procedure (sector, heartbeat magic, both instrument magics, then reset). + +Baseline **boot = 1, work = 640 at 2026-08-22 00:20:14**; **crashes = boot_end − 1**. + +**Comparators: CB's 1 per 157 (193 ev, immediately prior) and the pooled V1_30 +1 per 371 (277 ev, BZ+CA).** + +* **~1 per 157** ⇒ the epoch stepped, CB's instrument is exonerated, every + comparator in this block is stale, and rule 79 applies: the whole BZ/AG/CA + voltage block must be re-read against neighbours before the queue work starts + from the new baseline. CB gets restored immediately. +* **~1 per 371** ⇒ the instrument change really did cost 2.36x, which would be a + major and genuinely surprising result — it would mean the HardFault handler + path participates in the failure, and that becomes the investigation. +* **Anything else** ⇒ the rate is drifting on a sub-window timescale, which would + invalidate the ~1.4x resolution claim and force mid-window sampling before any + further config work. + +## CC0 — THE CONTROL CAME BACK CLEAN. CB'S DEGRADATION IS REAL, AND MY MECHANICAL ARGUMENT WAS WRONG. + +Window 00:20:14 → 01:23:12 (63.0 min). Valid — magic OK, ALLOC_GUARD intact, +VREG `0xf0`, QMI `0x20007006`. + +Δboot = 135 − 1 → **134 crashes**; Δwork = **58,470** → **1 per 436** +(95% CI 1 per 362 – 510). **Pre-registered branch (b).** + +### The bracket + +QMI, voltage, clock and layout of the *emulator* are identical across all three: + +| window | HF_STACK gate | rate | 95% CI | events | +|---|---|---|---|---| +| CA | present | 1 per 355 | 296 – 414 | 139 | +| **CB** | **removed** | **1 per 157** | **135 – 179** | **193** | +| CC0 | present | 1 per 436 | 362 – 510 | 134 | + +**CB sits BETWEEN two windows of the restored config, and its interval overlaps +neither.** That is exactly the standard this document used to validate +MIN_DESELECT 14-vs-28 ("the 28 window sits between two good 14 windows, so that +comparison is same-epoch and stands"). By its own rule, **CB's 2.36x degradation +is same-epoch, bracketed, and real.** + +### I was wrong, and the way I was wrong is the finding + +Last cycle I argued the diff "cannot mechanically produce this": the capture +block is write-once, the magic was confirmed set within ~40 s, so 192 of 193 +crashes ran byte-identical handler code. That reasoning is still *correct as far +as it goes* — and the rate moved anyway. **This is the fourth time in this +investigation that a change which "could not matter" has been confirmed as +mattering** (rule 75), and the first time it has been caught by a bracketed +control rather than discovered later as a contradiction. + +If the executed code really is identical for 192 of 193 faults, then the thing +that changed is not *what the handler does* — it is **where everything sits**. +Removing an `if` shifted the handler's code and everything linked after it. The +image CRC moved `0x4285c1f3` → `0xda9296b3` → `0x3d9cc95e` across CA/CB/CC0. + +**Rule 84: A REBUILD IS NOT A CONTROLLED CHANGE. Any source edit shifts code +layout, and layout appears to be worth ~2.4x on the crash rate. Every comparison +in this document that involved a rebuild is confounded by layout unless the +config window was bracketed by same-config neighbours. "One variable" was never +one variable — it was always the config field PLUS a new layout.** + +This is potentially the most consequential methodological result in the +investigation, because it retroactively explains the pattern that has plagued it: +configs that confirm and then refute, "epoch steps" that coincide with reflashes, +and mechanisms that never survive contact with a control. A ~2.4x layout term +sitting under every rebuild would produce exactly that. + +Note carefully what is NOT claimed. Layout is *a* hypothesis for CB's shift; the +bracketed fact is only that **restoring the source restored the rate**. The +alternative — that the capture block executes far more often than the write-once +read implies — is not excluded, only made unlikely. CD separates them. + +## Config CD — a SEMANTICALLY NULL layout perturbation + +The decisive test. Append `.space 128` to the end of the `.HardFaultTrampoline` +global_asm block, *after* `b.w {handler}` and after the `.size` directive. + +Why this specific change: + +* It is **unreachable**. The trampoline's last instruction is an unconditional + `b.w`, so the padding can never execute. Not one executed instruction changes. +* The section is **guaranteed kept** — `HardFault` is referenced by the vector + table, so `--gc-sections` cannot drop it. +* It **shifts everything linked after `.HardFaultTrampoline`**, which is the same + class of disturbance CB caused. + +ONE VARIABLE: 128 bytes of unreachable padding. No behaviour, no config, no +timing of any executed instruction. + +**Comparator: CC0's 1 per 436 (134 ev) and CA's 1 per 355 (139 ev) — pooled +same-config 1 per 397 (273 ev), both immediately adjacent and same-epoch.** + +* **Rate moves materially (say outside 1 per 330 – 480)** ⇒ **LAYOUT SENSITIVITY + CONFIRMED.** A semantically null edit moves the crash rate, every + rebuild-confounded comparison in this document must be re-read, and the + measurement protocol has to change: either hold layout fixed across a config + series, or characterise layout variance as the real noise floor before any + effect smaller than it can be claimed. It would also make "the epoch" partly + an artifact of reflashing. +* **Rate unchanged** ⇒ layout is NOT the mechanism, and CB's degradation came + from the capture block genuinely executing far more than write-once implies. + The follow-up is then a counter incremented on every entry to the block, to + measure directly how often it runs. + +Either answer is worth more than another config knob. + +### The layout disturbance, measured + +The first CD attempt (`.space 128` after the trampoline) was **discarded before +it ran**: symbol diffing showed `.HardFaultTrampoline` links late, so it moved +only **66 of 2161** symbols. A null result from that would have meant nothing. + +Diffing the symbol tables makes all three builds precise: + +| comparison | symbols moved | delta | +|---|---|---| +| CA → CC0 (gate restored) | **0 of 2161** | — | +| CC0 → CB (gate removed) | **959 of 2161** | −4 bytes | +| CC0 → CD (two nops) | **1021 of 2161** | +4 … +32 bytes | + +**CA and CC0 are byte-identical in layout** — which is exactly why CC0 reproduced +CA's rate, and makes the bracket around CB airtight. CD's two nops reproduce CB's +disturbance in magnitude and location, in the opposite direction. + +All RAM addresses are unchanged, so every probe address in this document still +holds. + +### Config CD — running + +Device readback: `integrity: full image crc 0xaae78425 OK`; `QMI retune at +main-loop entry: 0x20007006 -> 0x20007006`; **VREG `0xf0`**; 300 MHz; boot log +12.5059 s. + +Cold seed under the rule-83 procedure. A `probe_rs ... Expected core to be +halted` warning appeared during the writes, so the seed was **verified after the +fact**: `HF_STACK[0] = 0x00000000`, `MM_REGS[0] = 0x00000000`. Both magics are +genuinely cleared. + +Baseline **boot = 0, work = 45 at 2026-08-22 01:30:30**; **crashes = boot_end − 1**. + +**Comparator: CC0's 1 per 436 (134 ev) and CA's 1 per 355 (139 ev) — pooled +same-layout 1 per 397 (273 ev), both same-epoch.** + +## CD — VOID. The window did not measure the crash rate; the board was limping. + +Window 01:30:30 → 02:33:16 (62.8 min). Δboot = 102 − 1 → 101; Δwork = 3,503. +Naively that reads **1 per 35**, an 11x degradation against the pooled 1 per 397. + +**It is not a valid measurement.** Four independent signals say so: + +| signal | CD | normal | +|---|---|---| +| crash records committed | **7** | 31 (ring full, wrapped) | +| boots | 101 | ~139 | +| iterations/min while running | **84** | ~1,400 | +| WatchdogTimeout share | **4 of 7** | 1-3 of 31 | +| `DMA_CRASH_SNAPSHOT` sentinel | **zeroed** | `0xd4a0c12a` | + +101 boots produced only 7 records, so **~94 resets never went through the crash +handler at all.** Those are hangs ending in watchdog resets, not the crash mode +this investigation has been measuring. Throughput per running minute fell 17x. +And an instrument in `.uninit` that nothing in the cold seed touches — +`DMA_CRASH_SNAPSHOT[0]` and `[2]` — was silently zeroed. + +That is a wedge loop, not the same failure running faster. + +**Rule 85: `crashes = Δboot` ASSUMES EVERY RESET WENT THROUGH THE CRASH HANDLER. +Validate Δboot against the committed record count every window. When records are +far below the ring capacity while boots are high, the board is hanging rather +than crashing, the rate metric is measuring a different phenomenon, and the +window is VOID. This check costs one line and would have caught the problem +before an 11x headline was written.** + +### The limp was visible at the baseline read, and I missed it + +Work counter at +40 s after the cold seed, same procedure each time: + +| window | work at +40 s | +|---|---| +| CC0 | 640 | +| **CD** | **45** | +| CE | 449 | + +CD was already limping in its first seconds. That number was in front of me when +I recorded the baseline and I did not treat it as a signal. It is a free +early-warning check and belongs in the seed procedure. + +### The prime suspect is the hazard already flagged, now active + +The standing latent hazard: `DMA_CRASH_SNAPSHOT` spans +`0x20067ad0 .. 0x20067b18`, and core 1's read-only MPU region base is +`_stack_end & !0x1F` = `0x20067b00`, so slots `[12..18]` sit **inside** it. The +handler writes all 18. A core-1 fault therefore takes a **MemManage inside the +fault handler** — which produces exactly this signature: a reset with no record +committed, repeated, with the watchdog cleaning up. + +CD's window contains a core-1 record (`Stk hdm (core 1) 8088 bytes remaining`), +4 watchdog timeouts out of 7, and 94 unrecorded resets. If CD's layout shift +raised the share of core-1 faults even slightly, the hazard would convert them +into wedges. That is a mechanism, not yet a finding. + +### What CD does NOT settle + +Nothing about rule 84. The layout question is **still open** — CD neither +confirms nor refutes layout sensitivity, because the window measured a different +failure mode. The 2.36x CB result stands on its own bracket and is untouched by +this. + +## Config CE — CONTROL, byte-identical to CC0 + +Rebuilt from the CC0 source. Verified two ways: **0 of 2203 symbols moved versus +`syms-CC0.txt`**, and the flashed image reports **`crc 0x3d9cc95e`, the exact CRC +CC0 booted with**. This is the same binary, not merely an equivalent one. + +Device readback: QMI `0x20007006`, **VREG `0xf0`**, 300 MHz, boot log 12.1704 s. +Cold seed extended to also zero `DMA_CRASH_SNAPSHOT`; all three magics verified +zero after the reset (`0x20067048`, `0x20067128`, `0x20067ad0` all `0x00000000`). + +Baseline **boot = 1, work = 449 at 2026-08-22 02:37:36** — healthy, in the +CC0 band, not the CD band. **crashes = boot_end − 1.** + +**Comparators: CC0's 1 per 436 (134 ev) and CA's 1 per 355 (139 ev) — pooled +same-layout 1 per 397 (273 ev).** + +* **~1 per 400 with ~31 records and ~1,400 iterations/min** ⇒ the board is + healthy on the known-good image, and the CD limp was caused by CD's own change + — which would make layout sensitivity not just real but dramatic, capable of + flipping the failure into a wedge mode. Layout work then continues with + repeated null perturbations to map the variance. +* **Still limping (few records, low throughput)** ⇒ the degradation is in the + board or the rig, independent of the image. CD is void either way, the CB/CC0 + bracket needs re-validating, and recovery (up to an OpenOCD RESCUE reset) comes + before any further measurement. + +## CE — THE BOARD IS HEALTHY. CD's limp was CD's own doing, and RULE 84 IS CONFIRMED. + +Window 02:37:36 → 03:40:17 (62.7 min) on an image **byte-identical to CC0** +(0 of 2203 symbols moved; flashed CRC `0x3d9cc95e`, the exact CRC CC0 booted). + +**Rule 85 check first: 31 records** (ring full), **1,666 iterations/min** +(CD: 84), DMA sentinel restored to `0xd4a0c12a`, 29 HardFault / 2 Watchdog — the +normal mix. The window is valid. + +Δboot = 129 − 1 → **128 crashes**; Δwork = **60,018** → **1 per 469** +(95% CI 1 per 388 – 550), against the pooled same-layout **1 per 397**. +Overlapping. **Pre-registered branch (a).** + +### The second bracket, and it is the same shape as the first + +| window | layout | rate | health | +|---|---|---|---| +| CC0 | baseline | 1 per 436 | 31 records, healthy | +| **CD** | **+4…+32 B, 1021 symbols** | **wedge loop** | **7 records, 84 iter/min** | +| CE | baseline (identical binary) | 1 per 469 | 31 records, healthy | + +And the first bracket, from two cycles ago: + +| window | layout | rate | +|---|---|---| +| CA | baseline | 1 per 355 | +| **CB** | **−4 B, 959 symbols** | **1 per 157** | +| CC0 | baseline (identical binary) | 1 per 436 | + +**Two independent, bracketed layout perturbations. Both degraded. Both recovered +the moment the baseline layout was restored.** Every window run on the +CA/CC0/CE layout is healthy at 1 per 355 – 469; every window run on a perturbed +layout is degraded, one of them catastrophically. + +**RULE 84 IS CONFIRMED.** A semantically null 4-byte code shift — two `nop`s +that cannot execute more than once per fault — was enough to flip the firmware +from a 1-per-400 crash mode into a wedge loop with 94 unrecorded resets. Layout +is not a modulator of the failure; it appears to *select* the failure. + +### What that implies about the mechanism — and it unifies almost everything + +Two perturbations, both worse, is not what random noise looks like. The pattern +fits a specific mechanism: **a wild write to a roughly fixed address, whose +consequences depend on what the linker happened to place there.** + +* Move the code, and a different function or structure occupies the victim + address → different symptom (benign, crash, or wedge) and a different rate. +* The current layout is not "correct", it is merely a basin where the victim is + comparatively harmless. + +That accounts for the pathologies that have dogged this investigation from the +start: configs that confirm and then refute, mechanisms that never survive a +control, and — most of all — **"the epoch"**. Epoch steps were repeatedly +observed to coincide with reflashes. Under rule 84 that is not a coincidence: +**a reflash with a rebuilt image is a layout change, and the epoch is in +substantial part a layout artifact.** The 73-hour BY window held its rate for +three days precisely because nothing was reflashed during it. + +It also sits squarely on top of rule 82 and the CB capture: a wild write +depositing `Core1Command`-shaped data (16-bit Game Boy addresses at 4-byte +stride) into whatever happens to be at the target address. + +**Rule 86: UNDER LAYOUT SENSITIVITY, RATE COMPARISONS ACROSS REBUILDS ARE NOT +EVIDENCE. Prefer instruments whose result is an EVENT — a guard word breached, a +frame captured, an address recorded — because an event survives a rebuild while a +rate does not. And since every rebuild costs an uninterpretable rate anyway, +bundling several event instruments into ONE build is strictly cheaper than +spending one window per instrument. "One variable at a time" was the right rule +for measuring rates; it is the wrong rule for harvesting events.** + +## Config CF — the instrumentation build + +Three changes, all of them instrument integrity. None touches emulator +behaviour, none is a config knob, and the rate this window is **explicitly not +the measurement** (rule 86). + +**1. Fix the `DMA_CRASH_SNAPSHOT` / core-1 MPU overlap.** The standing hazard, +now the prime suspect for CD's wedge loop: the array spans +`0x20067ad0 .. 0x20067b18`, core 1's read-only region base is +`_stack_end & !0x1F` = `0x20067b00`, and the handler writes all 18 slots — so a +core-1 fault takes a MemManage *inside the fault handler*, producing a reset with +no record. Padding the array (only `[0..18]` are ever written) pushes `_stack_end` +and therefore the region base up past the written slots, exactly as `MM_REGS` was +padded to 20 words for the identical reason. **This also protects every future +window's record pipeline**, which is why it goes first. + +**2. Remove the `HF_STACK` INVSTATE gate again.** It is the only instrument that +has ever captured the dominant fault mode, and with the gate restored the +investigation is blind to it. CB's degradation is now attributed to layout rather +than to this gate, so there is no reason to keep the blindfold on. + +**3. Guard `COMMAND_QUEUE`.** Wrap it in a `#[repr(C)]` storage struct with guard +bands, mirroring `AudioQueueStorage`, and check them. Note the CB capture landed +on core 0's *stack*, far from the queue, so an adjacent-overrun guard may well +stay intact — that is an informative outcome, not a wasted one: + +* **guard breached** ⇒ the enqueue is running off its own buffer; the writer is + bounded and named. +* **guard intact while the smashes continue** ⇒ the slot address is **wild**, not + merely out of range, which points at the `Producer`/queue pointer itself and + makes that the next target. + +Expect the rate to move; under rule 86 that number carries no information. What +this window is for is the three events: does the record pipeline stay healthy +(rule 85 check), does the guard trip, and what does `HF_STACK` catch. + +### Config CF — running, and it caught something within 40 seconds + +Build verified. `integrity: full image crc 0x044a793f OK`, QMI `0x20007006`, +VREG `0xf0`, 300 MHz, boot 12.1698 s. Rule 84 diff: **1586 of 2098 symbols +moved** — a large layout change, so the rate this window is uninterpretable by +construction (rule 86). + +**CF-1 boundary check, verified on the built image:** + + DMA_CRASH_SNAPSHOT = 0x20067cd8 + written slots [0..18) end= 0x20067d20 + _stack_end = 0x20067d40 + core-1 MPU region base = 0x20067d40 -> SAFE, 32 bytes of margin + (before CF: written_end = 0x20067b18 > base 0x20067b00 -> UNSAFE) + +**ALL RAM ADDRESSES MOVED.** New table, verified live on the device: + +| symbol | address | +|---|---| +| `HEARTBEAT` (magic `0x48b70001`) | `0x200671b0` | +| `BOOT_COUNT` / `WORK_COUNT` | `0x20067214` / `0x20067218` | +| `HF_STACK` | `0x20067250` | +| `MM_REGS` | `0x20067330` | +| `ALLOC_GUARD` (`0xa1100001`) | `0x200679a8` | +| `DMA_CRASH_SNAPSHOT` | `0x20067cd8` | +| `_stack_end` | `0x20067d40` | + +Baseline **boot = 1, work = 118 at 2026-08-22 03:47:26**; **crashes = boot_end − 1**. +Watch item: work at +40 s is 118, against CE's 449 and CC0's 640 (CD's void +window read 45). Not in CD's territory, but low — the rule-85 checks matter this +window. + +### The immediate capture, and the one number that did not move + +With the gate off, `HF_STACK` filled before the baseline read was even taken: + + pc=0x88000000 lr=0x00000000 cfsr=0x00000100 IBUSERR + sp_before=0x2007cd48 r12=0x2002b689 exc_return=0xfffffff9 (core 0, MSP) + + 0x2007cd38 2002b689 stacked r12 + 0x2007cd3c 00000000 stacked LR <- corrupt + 0x2007cd40 88000000 stacked PC <- corrupt + 0x2007cd44 88000000 stacked xPSR <- corrupt + 0x2007cd48 000001a8 <- sp_before, plausible + 0x2007cd4c 1001a2dd <- PicoGameBoy::with_cartridge+0x578, VALID + 0x2007cd50 88000000 <- corrupt + 0x2007cd54 68000000 <- corrupt + +Two things stand out. + +**`sp_before = 0x2007cd48` is IDENTICAL to the CB capture** — the same byte +address, across a rebuild that moved 1586 of 2098 symbols. The code moved +wholesale; the stack depth at which this fault occurs did not. Whatever picks the +victim address is anchored to the stack, not to the code. + +**The corruption is interleaved here, not one run.** `cd40`/`cd44` are corrupt, +`cd48`/`cd4c` are intact and legitimate (`cd4c` is a valid return address into +`PicoGameBoy::with_cartridge`), then `cd50`/`cd54` are corrupt again. CB's +capture was six *consecutive* words. So "contiguous run" is one observed shape, +not the only one — a scattered write pattern is also possible, and the CB +conclusion should be narrowed to "at least sometimes contiguous". + +The guard did not trip during boot; too early to mean anything. + +## CF — the guard did NOT trip. And RULE 82 IS RETRACTED: I re-derived work this project had already done. + +Window 03:47:26 → 04:51:18 (63.9 min). Rule 85 gate: **31 records** (ring full), +DMA sentinel `0xd4a0c12a` present, ALLOC_GUARD intact, VREG `0xf0`. + +Δboot = 140 − 1 → 139; Δwork = 11,185 → naive 1 per 80. Per rule 86 that number +is uninterpretable (1586 of 2098 symbols moved), and it is not the point of the +window. Two health signals are worth recording anyway: **320 iterations/min** +(healthy 1400–1670; CD's void window 84) and **11 of 31 records are +WatchdogTimeout** (CE: 2 of 31). This layout is degraded — consistent with rule +84 — but the record pipeline works, so events are trustworthy. + +Caveat on the rule-85 gate: a **full** ring only proves ≥31 crashes were +recorded, not that all 139 were. CD was catchable because its ring was *not* +full. The check is one-sided and should be stated that way. + +### THE RESULT: the COMMAND_QUEUE guard is INTACT + +Zero breaches across 139 crashes; no panic in the boot log or any record. Both +bands clean. + +**Pre-registered branch: the slot address is WILD, not merely out of range.** +`inner_enqueue` is not running off its own buffer. Combined with `AUDIO_QUEUE`'s +guard (also never tripping), **neither cross-core queue is being overrun by an +adjacent walk.** The store's *destination* is wrong, not its length. + +### RULE 82 IS RETRACTED + +I built rule 82 on the corrupt words being valid Game Boy addresses +(`0x8800` = VRAM tile-data base, `0xe0xx` = echo RAM, `0xfe9e` = last OAM byte) +and concluded "a silicon fault would not preferentially produce valid Game Boy +addresses — this is a software write." + +**This project tested that exact reading on 2026-08-08 and refuted it.** +`core/src/cpu/peripheral/apu.rs:78` records it: + +> audio is EXONERATED … It also killed the "the corruptor writes audio data" +> reading: with audio OFF the wild pointer became `0x4547454c`, whose bytes are +> `4C 45 47 45` = **"LEGE"** — ASCII from the ROM title. Earlier values decoded +> as plausible i16 sample pairs only because samples happened to be what was in +> flight. **The payload is INCIDENTAL; the corruption is a wild DESTINATION.** + +So `0x8800`/`0x6800` decode as Game Boy addresses for the same reason they also +decode as plausible i16 square-wave samples: **whatever was in flight is what +lands.** The inference from payload to mechanism is invalid. Rule 82's conclusion +("software, not silicon") may still be true, but *not for the reason I gave*, and +it must not be cited as evidence. + +### And rule 84 was already known here, qualitatively + +`oam-dma-bisection.md` — a sibling file in this same directory, 2271 lines, which +I had never opened — already records that rebuilding relocates the victim and +suppresses the repro: + +* line 162: "**3/3 trials NO_RECORDS.** The incidental codegen shift (from + rebuilding) …" +* line 228: "incidental rebuild (E3) **relocates the victim and SUPPRESSES the + repro**" +* line 697: "An incidental **+0x1c code shift** also killed the …" + +My contribution is quantification — bracketed windows, symbol-table diffs, 2.36x +with confidence intervals — not the concept. The concept was on file. + +**Rule 87: READ THE SIBLING INVESTIGATION DOCS BEFORE DERIVING ANYTHING. This +directory holds five files; `pico-crash-current-findings.md` is the handoff, not +the corpus. `oam-dma-bisection.md` (2271 lines) contains F1/G1, which had already +established the wild-destination conclusion, the incidental-payload conclusion, +AND the rebuild-relocates-the-victim conclusion. Two cycles were spent +re-deriving those. Grep the whole directory for a claim before elevating it to a +rule.** + +### What actually survives, and what to do next + +Surviving, and now better supported than before: + +* the store's **destination** is wrong, not its length (both queue guards intact); +* the destination is anchored to the **stack**, not the code — `sp_before` was + `0x2007cd48` in both the CB and CF captures, across a rebuild that moved 1586 + of 2098 symbols; +* rebuilding relocates the victim, which is why every config comparison has been + unstable. + +`oam-dma-bisection.md` §F1 already names the next experiment, and it is better +than anything I was about to design. Its §G1 lays out the plan: + +> a wild store with a corrupt DESTINATION lands on the spilled oam-base stack +> slot; the value it carries is incidental … the root corruptor is a store whose +> *address* is wrong … Resolving it needs a **runtime data trap on that exact +> stack word** … arm OpenOCD `wp 4 w ` (value-filtered write +> watch). The value filter skips the constant legit spills and fires only on the +> wild store — turning an un-watchable hot slot into a single-shot trap. **PC at +> halt = the root corruptor.** + +That is the experiment to run. It needs the *current* binary's slot address +(derived live from a halt, since the golden binary referenced there is from June +and the layout has moved many times), OpenOCD from the Raspberry Pi fork — +probe-rs's GDB stub never implements `Z2`, so watchpoints must go through OpenOCD +— and RP2350 value matching, which works **only on `DWT_COMP3`**. + +**CF stays running unchanged.** No reflash, so no new layout, and the window +keeps accumulating events while the G1 trap is prepared. Its counters continue +from the 03:47:26 baseline. + +## RULE 87 APPLIED — and the sibling doc changes the picture completely + +`oam-dma-bisection.md` is not a side note. It contains sections **G1 through G20** +(2026-06-15 → 06-21), and it is where this investigation actually got furthest. +Summarised, because none of it is in this handoff file: + +**§G1 — the root cause was FOUND, on hardware, reproduced bit-for-bit in two +independent runs.** A value-filtered OpenOCD write watch on the spilled oam-base +stack slot caught the corruptor on the 400th halt (after 399 legitimate spills): + + writer PC = 0x1002f2de -> the store is 0x1002f2d8: str r2,[sp,#0xf0] + value = 0x2003fa30 (GameBoyMemory base — incidental, per F1) + lr = 0x1000a029 + +Resolved to `GameBoy::route_bus_events`'s inlined +`write_vram_range`/`write_oam_range` path. **Diagnosis: a stack-slot lifetime +collision** — `[sp,#0xf0]` is a coalesced slot with ~28 writers and ~40 readers; +the optimiser believes the OAM-DMA `&oam` spill is dead before the bus-event +flush writes the slot, and on the device it is not. + +**§G2 — the fix was applied**: `route_bus_events` split so the body moved into +`#[inline(never)] fn drain_bus_events()`. **It is in the current build** — the CB +capture's r6 landed just after `bl drain_bus_events`. The crashes continue. + +**Everything since has been tested and refuted, each by an actual soak:** + +| section | hypothesis | verdict | +|---|---|---| +| G14 | StackColoring disabled | **FAIL** — still crashes | +| G17 | `-disable-ssc` (StackSlotColoring) | **FAIL** — still crashes | +| G18 | LLVM MachineOutliner (a real machine-verifier error: undefined `$r1` live-in to `OUTLINED_FUNCTION_186`) | **REFUTED** — a verifier-CLEAN build still produces the wild store | +| G19 → G19-CORRECTION | "the collision is not statically supported" | retracted; the collision **is** real, hardware-confirmed | +| G20 | `-regalloc=basic` | passed one window, then **refuted** | + +**My regalloc lead was already dead, and I checked before acting on it.** +`platform/pico2w/.cargo/config.toml` records the removal in its own comment: + +> `-C llvm-args=-regalloc=basic` was REMOVED 2026-08-08 … That was dormancy, not +> a fix … **a 41 min soak WITH `basic` active still produced 31 crashes** +> (~0.76/min, sector cap). + +So it was not dropped by oversight; it was retested at length and correctly +removed. This is rule 87 doing its job in the right direction for once. + +**And §G18-RESULT's conclusion 3 is rule 84, reached in June:** + +> the `#[inline(never)]` "fix" was itself just a LAYOUT SHUFFLE, not a structural +> fix … **we have NO confirmed structural fix; every "fix/move" has been layout +> roulette.** + +### CF window, second hour — stable and replicated + +03:47:26 → 05:58:09 (130.7 min). Δboot = 285 − 1 → **284 crashes**; +Δwork = **22,644** → **1 per 80**. Splitting it: hour 1 gave 1 per 80.5, +hour 2 gave 1 per 79.0. Throughput 317 iter/min both hours. ALLOC_GUARD intact. +This layout is a stable, degraded basin — exactly what rule 84 predicts. + +### Where this actually leaves the investigation + +Every mechanism-level hypothesis with a testable prediction has now been tried +and refuted by soak: electrical (QMI, voltage, clock), audio, DMA, three LLVM +codegen passes, the register allocator, the stack-protector scaffolding, the G1 +stack-slot collision fix, and — new this cycle — both cross-core queue guards. +Host ASan (§G15) and qemu-arm (§G16) replays are clean. The bug survives builds +with zero machine-verifier errors. + +What remains true and unexplained: a wild **store destination**, anchored to the +stack (`sp_before = 0x2007cd48` across a rebuild that moved 1586 of 2098 +symbols), whose victim relocates with layout and whose payload is incidental. + +**The one experiment never run in two months of work is the simplest: A SECOND +PHYSICAL BOARD.** Every result in both documents comes from one RP2350 unit +(`E6614C311B511822`). The entire investigation assumes the fault is in firmware +or codegen, and that assumption has never been tested. A second Pico 2 W running +the identical image separates two hypotheses that no amount of further +instrumentation can: + +* **second board also ~1 per 400** ⇒ it is the firmware/toolchain, the corpus of + refutations stands, and the remaining work is an upstream LLVM/silicon report; +* **second board clean** ⇒ it is *this unit*, every codegen hypothesis in both + documents was chasing a hardware fault, and the 24-hour goal is reachable + simply by changing the board. + +This requires hardware the user must supply, so it is raised rather than run. + +## G1-REPLAY attempt on the CURRENT build — armed, did not fire. Three obstacles solved, one open. + +No answer yet on the second board, so this cycle went to the one experiment that +needs **no rebuild** (and therefore no new layout): re-run §G1's value-filtered +watchpoint against the *current* firmware. §G1 caught the corruptor in June, but +on a binary that predates the §G2 fix — **nobody has caught it since.** + +`watch_g1.tcl`, `watch.tcl`, `watch_oam.tcl` and `watch_oam_fix.tcl` are all still +in `~/git/github.com/raspberrypi/openocd/`. I adapted `watch_g1.tcl` into +`watch_cf.tcl`, dropping its slot-derivation step: §G1 had to derive the victim +slot from a breakpoint on the legit spill store, but **this build's victim address +is already known empirically** — `0x2007cd50` takes `0x88000000`/`0x68000000` in +both the CB and CF captures, which are 1586 of 2098 symbols apart. + +### Three obstacles, solved + +1. **OpenOCD attaches fine** — Cortex-M33 r1p0, "8 breakpoints, **4 watchpoints**" + on each core. The fork at `~/git/github.com/raspberrypi/openocd/src/openocd` + (0.12.0+dev, built Jun 14) still works. +2. **`rp2350.cfg` puts cm0 and cm1 in an SMP group** (`target smp $_TARGETNAME_CM0 + $_TARGETNAME_CM1`, line 193). Watchpoints are broadcast to **both** cores, so + both must be halted — the first attempt died on + `[rp2350.cm1] can't add write watchpoint at 0x2007cd50, target running`. + `targets rp2350.cm0` does **not** avoid this. +3. **A bare `halt` lands in "unknown state"** because the firmware crash-resets + every ~20 s. `reset halt` fixes it: both cores halted cleanly at + `pc=0x00000088` (bootrom), and the watchpoint then **armed successfully**. + +### The open obstacle: it never fired + +~7 minutes of soak after arming, and the poll loop logged **zero halts** — not +even a spurious one, despite `0x2007cd50` being live core-0 stack that legitimate +code writes constantly. + +Zero *spurious* hits is the tell. A correctly-armed write watch on a hot stack +word should fire immediately and often (§E7's whole problem was that it fires too +much). Getting none means the watchpoint was **not active while the target ran**. +The most likely cause: **the firmware crash-resets, a reset clears the DWT +comparators, and `watch_cf.tcl` only re-arms inside the halt branch** — so after +the first crash the watch is silently gone and the loop waits forever on a +`curstate` that never returns "halted". + +§G1 did not hit this because its 400 legit-spill halts all occurred inside a +single boot, before any reset. + +**Fix for the next attempt** (all in the script, no firmware change): +* re-arm on a **timer**, not only after a halt — poll `curstate`, and + unconditionally `rwp`+`wp` every N poll cycles so a reset cannot disarm it + permanently; +* detect the reset directly by watching the boot counter at `0x20067214` change, + and re-arm on that edge; +* log every poll iteration for the first ~20 cycles so "armed but silent" is + distinguishable from "loop stuck" — this attempt could not tell those apart + from the log alone. + +### Window hygiene + +The debugger session halted the board, disarmed the watchdog, and issued +`reset halt`, so the CF window is contaminated from 07:00 onward. Its last clean +read stands: **423 crashes, 1 per 79.3 over 193 min** (hour 1: 1 per 80.5, hour 2: +1 per 79.0, hour 3 pooled: 1 per 79.3) — stable and replicated three times over. + +Restarted cleanly on the **same CF image, no rebuild, no layout change**. Cold +seed under rule 83 (sector, all four magics, reset). Baseline +**boot = 1, work = 219 at 2026-08-22 07:14:30**; crashes = boot_end − 1. +ALLOC_GUARD `0xa1100001`, VREG `0xf0`. + +## G1-REPLAY, cycle 2 — WHY OpenOCD watchpoints cannot work here, and the instrument that can + +The corrected script (timer re-arm, per-poll logging, reset-edge detection) ran +and behaved correctly: `reset halt` on both SMP cores, watchpoint armed, board +resumed, timer re-arms firing on schedule. **Still zero watchpoint hits** on a +live core-0 stack word, over hundreds of polls, with SP verified straddling the +target (`msp` observed at `0x2007cac0` and `0x2007dfd8`). + +### The cause: the firmware owns the DWT + +`platform/pico2w/src/main.rs:661` calls `dwt_watch::enable_monitor_only()`, which +sets `DEMCR.MON_EN`, and `main.rs:834` calls +`dwt_watch::arm_data_write_watch(target)` — which writes **`DWT_COMP0`**, +`DWT_MASK0` and `DWT_FUNCTION0` directly (`dwt_watch.rs:55-57`). + +So every boot, the firmware **overwrites comparator 0 with its own target**. With +the board crash-rebooting every ~20 s, any watchpoint OpenOCD installs is wiped +within one boot cycle, and re-arming on a 2.5 s timer cannot win the race +reliably. This is a *complete* explanation for the silence, and it also explains +why this document's earlier verdict — "the DWT/DebugMonitor approach is +EXHAUSTED" — was reached: the two instruments have been fighting each other. + +**Rule 88: THE FIRMWARE PROGRAMS DWT_COMP0/1 AND SETS DEMCR.MON_EN ON EVERY BOOT. +An external debugger's watchpoints are therefore overwritten every reboot, and on +a crash-looping target that is every ~20 seconds. Do not attach OpenOCD +watchpoints to this firmware — use the firmware's own DWT instrument instead.** + +### The instrument that does work — and it needs NO rebuild + +`dwt_watch::DWT_CATCH` (`0x20067380`, `[usize; 10]`) is a **self-aiming, +probe-settable watchpoint** that `main.rs:776-840` documents explicitly: + +* `DWT_CATCH[8]` (at **`0x200673a0`**) takes either a guard *offset* (< 0x1000, + resolved against the victim's body SP from the previous crash) or a **full SRAM + address**, planted over SWD with no rebuild; +* on a hit, the DebugMonitor handler latches + `[0]` magic, `[1]` watched addr, **`[2]` the stacked PC — the writer**, + `[3]` stacked LR, `[4]` EXC_RETURN, `[5]` hit count. + +That is precisely §G1's experiment, already built into this firmware, and immune +to rule 84 because arming it changes no code. + +One nicety that resolves the earlier worry about probe sessions clearing +`DEMCR.MON_EN`: the firmware re-arms MON_EN and COMP0 **on every boot**, and this +board reboots every ~20 s, so a probe read that disturbs the state is +self-healing within one crash cycle. + +### Armed and running + +Cold seed under rule 83, extended to zero `DWT_CATCH[0]` so a stale catch cannot +masquerade as a fresh one, then `0x2007cd50` planted into `DWT_CATCH[8]` and +reset. **Verified live: `DWT_COMP0` (`0xE0001020`) reads `0x2007cd50`.** +`DWT_CATCH[0] = 0` — armed, nothing caught yet. + +Baseline **boot = 2, work = 189 at 2026-08-22 08:25:23**; **crashes = boot_end − 2**. +Same CF image, no rebuild, no layout change. + +**PRE-REGISTERED:** + +* **`DWT_CATCH[0]` becomes the magic and `[2]` holds a PC** ⇒ the corruptor is + named in the *current* build. Resolve `[2]` against `syms-CF.txt` (greatest + symbol ≤ target; the DebugMonitor stacks the PC **after** the storing + instruction, so the store is the preceding one). Compare against §G1's June + answer — `route_bus_events`' inlined `write_vram_range` — to settle whether the + §G2 fix moved the corruptor or the corruptor was never that store. +* **`DWT_CATCH[0]` still 0 after a full window while crashes continue** ⇒ + `0x2007cd50` is not where the wild store lands; it is where the *damage was + observed* two captures running. The next aim would then be the guard-offset + form, which self-corrects against the victim's body SP rather than encoding a + fixed address. + +## DWT window 1 — NO CATCH, and the reason is RULE 89: the probe disables the instrument + +Window 08:25:23 → 09:28:18 (62.9 min). `DWT_CATCH[0] = 0x00000000` — nothing +latched. `DWT_CATCH[8]` still held the planted aim and `DWT_COMP0` still read +`0x2007cd50`, so the comparator stayed armed the whole time. Δboot = 135 − 2 → +133 crashes; Δwork = 8,410 (rate not interpretable, rule 86). + +Before accepting pre-registered branch (b) — "the store lands elsewhere" — I +checked whether the instrument was *capable* of firing. It was not. + + DEMCR (0xE000EDFC) = 0x01110000 TRCENA set, MON_EN set <- firmware armed OK + DHCSR (0xE000EDF0) = 0x05100001 bit 0 = C_DEBUGEN = 1 <- THE PROBLEM + +**Per the ARM architecture, the DebugMonitor exception is enabled only when +`DEMCR.MON_EN == 1` AND `DHCSR.C_DEBUGEN == 0`.** With `C_DEBUGEN == 1`, halting +debug takes priority: a DWT match tries to enter Debug state instead of invoking +the DebugMonitor handler. `MON_EN` is simply overridden. + +`C_DEBUGEN` is set by **probe-rs whenever it attaches**, and it persists after +detach. So every `probe-rs read` — including the ones used to *check on the +instrument* — silently disables it. + +**Rule 89: `probe-rs` ATTACH SETS `DHCSR.C_DEBUGEN`, WHICH DISABLES THE +DebugMonitor EXCEPTION AND THEREFORE THE FIRMWARE'S ENTIRE DWT_CATCH INSTRUMENT. +It also clears `DEMCR.TRCENA|MON_EN` (the `main.rs:697` comment says so, and it +was observed directly: DEMCR went `0x01110000` → `0x00100000` across one read). +The instrument only works if the LAST probe contact before the soak clears +C_DEBUGEN — write `0xA05F0000` to DHCSR (DBGKEY | all-clear) — and nothing +touches the probe afterwards. This is the twin of rule 88, and between them they +explain why the DWT/DebugMonitor approach was written off as "EXHAUSTED": the +instrument was being switched off by the act of observing it.** + +MON_EN needs no manual restoration: the firmware sets it at every boot +(`main.rs:661`), and this board reboots every ~20 s, so it self-heals — but +**C_DEBUGEN does not**, because nothing in the firmware clears it. + +### Re-armed correctly + +Cold seed (rule 83, including `DWT_CATCH[0]`), aim `0x2007cd50` re-planted, reset, +armed confirmed (`DWT_COMP0 = 0x2007cd50`), and then — **as the final probe +operation** — `0xA05F0000` written to `DHCSR` to clear `C_DEBUGEN`. The probe has +not been touched since. DEMCR read `0x00100000` at that moment (MON_EN cleared by +that very read), which is expected and self-heals on the next boot. + +Baseline **boot = 1, work = 57 at 2026-08-22 09:30:28**; crashes = boot_end − 1. +Same CF image, no rebuild. + +**PRE-REGISTERED, unchanged from last window but now on a live instrument:** + +* **`DWT_CATCH[0]` set with a PC in `[2]`** ⇒ the corruptor is named in the + current build. Resolve against `syms-CF.txt`; the DebugMonitor stacks the PC + *after* the store, so the corruptor is the preceding instruction. +* **`[0]` still 0 while crashes continue** ⇒ now a real result rather than an + instrument artifact: `0x2007cd50` is where damage was *observed*, not where the + store lands. Switch to the guard-offset aim form, which resolves against the + victim's body SP instead of a fixed address. +* Note `SMASH_CORE0` reads `0xaf16f18d` — not `SMASH_MAGIC` — so the + guard-offset form has no body SP to resolve against yet. `-Z + stack-protector=strong` was removed in config AW, which is what populated it. + Using the offset form would require that flag back, i.e. a rebuild. + +## DWT window 2 + POSITIVE CONTROL — THE INSTRUMENT IS DEAD. Every DWT null result in this project is uninterpretable. + +Window 09:30:28 → 10:33:19 (62.9 min), C_DEBUGEN cleared as the last probe op. +`DWT_CATCH[0] = 0` again across **130 crashes**, `DWT_COMP0` still `0x2007cd50`. + +### The firmware says it is armed + +Rather than infer, I captured the firmware's own boot banner over RTT — the one +`dwt_watch.rs` says exists precisely because "the banner has to prove it rather +than assume it": + + canary watchpoint armed at 0x2007cd50 (main.rs:835) + DWT watch armed: COMP0=0x2007cd50 FUNC0=0x0815 + DEMCR=0x01110000 MON_EN=true (main.rs:841) + +Everything correct as the **firmware** sees it: comparator on the target, +`FUNC0=0x0815` (MATCH=data-address-write, ACTION=debug-event, DATAVSIZE=word), +TRCENA and MON_EN both set. + +### The positive control, and it fails + +A null from an instrument that has never demonstrably fired is worth nothing +(rule 74). So I re-aimed it at **`WORK_COUNT` (`0x20067218`)** — the counter the +emulator increments thousands of times per second — reset, cleared C_DEBUGEN, +and waited 150 s. + + COMP0 = 0x20067218 (armed, verified) + DWT_CATCH[0] = 0x00000000 (nothing) + +**A word-write watch on a counter written millions of times produced zero +catches.** The DebugMonitor exception is not being delivered, regardless of what +`MON_EN` reads back. + +**Rule 90: THE `DWT_CATCH` INSTRUMENT DOES NOT FIRE — PROVEN BY POSITIVE CONTROL, +NOT INFERRED. `DEMCR.MON_EN` reading `true` from firmware is NOT evidence that +DebugMonitor exceptions are delivered. Any "no catch" result from this instrument +is UNINTERPRETABLE until a positive control fires. Run the WORK_COUNT control +FIRST, every time, before trusting any DWT null.** + +This retroactively voids results, including one in the sibling doc: **§G13 +("Patient soak, DWT 3-victim watch armed: NO catch") is not evidence of +anything** — same instrument, same unverified null. And it voids my own +`0x2007cd50` null from both windows. + +### What this does to rules 88 and 89 + +Both were partly re-derivations — `dwt_watch.rs:59-101` already documents that +"every probe-rs attach RESETS DEMCR", that "writing DEMCR from the debugger is +REJECTED (SDME=1 makes those bits Secure-only)", and that "the probe must be left +alone during a watch soak". It even records the intended working sequence, which +is exactly what I ran. + +**Rule 89 needs narrowing.** I claimed `C_DEBUGEN` was the blocker. The banner +shows `MON_EN=true` at boot *after* many probe sessions, so the firmware's +re-arm does work, and the failure is **downstream of MON_EN** — the exception is +enabled and still not delivered. `C_DEBUGEN` may contribute; it is not +demonstrated to be the cause. The honest statement is rule 90: delivery is +broken, mechanism unknown. + +Remaining candidates, none tested: `DEMCR.SDME` (bit 20, set in every read) routes +DebugMonitor to **Secure** state — if this firmware executes Non-secure, the +exception cannot be taken; or the `DebugMonitor` vector-table entry is not +actually live (the banner logs `vec_hdr`/`ptr_word` for this reason, and those +values were `0x2004cb24`/`0x2004cb28` — RAM, unverified against the trampoline's +address). + +### Board restored + +Stale comparators disarmed per the author's documented cleanup +(`DWT_FUNCTION0`/`DWT_FUNCTION1` ← 0), aim cleared, cold seed, fresh soak on the +unchanged CF image. Baseline **boot = 0, work = 160 at 2026-08-22 10:39:24**; +crashes = boot_end − 1. ALLOC_GUARD `0xa1100001`, VREG `0xf0`. + +**Next, in priority order:** (1) settle DebugMonitor delivery — check the +Secure/Non-secure state and verify the `DebugMonitor` vector entry against the +trampoline symbol, since without delivery this whole instrument class is dead and +that is what has been silently defeating §G13 and both of my windows; (2) the +second-board test, still unanswered and still the highest-value experiment +available. + +## Why DebugMonitor is not delivered — vector REFUTED, and the instrument cannot be validated from outside + +Three candidates were queued. Results: + +**(2) The vector table — REFUTED, it is correctly wired.** `VTOR = 0x10000000` +(flash, *not* the RAM addresses the banner logs as `vec_hdr`/`ptr_word`, which +read `0x00000000` and are something else entirely). Reading the real table: + + index 3 (0x1000000c) = 0x1002f3bd -> HardFault 0x1002f3bc ✓ + index 12 (0x10000030) = 0x1002f3f5 -> DebugMonitor 0x1002f3f4 ✓ + +Both trampolines are live at the correct exception indices. The vector is not the +problem. + +**(3) The FUNCTION encoding — tested, inconclusive.** `DWT_CTRL = 0x40000000` +(NUMCOMP = 4, matching OpenOCD's "4 watchpoints"). `FUNC0 = 0x815` sets +**DATAVSIZE**, a *data-value* field, which given the standing note that RP2350 +value matching works only on `DWT_COMP3` could make `COMP0` permanently +unfireable. I rewrote `FUNCTION0` to `0x015` (DATAVSIZE cleared, MATCH and ACTION +unchanged) with `COMP0` aimed at the hammered `WORK_COUNT`. `MATCHED` (bit 24) +did not set under either encoding. + +### The real obstacle: the probe cannot observe the DWT without disabling it + +`DEMCR.TRCENA` (bit 24) **gates the entire DWT**. Every probe read leaves +`DEMCR = 0x00100000` — TRCENA clear. So by the time any `MATCHED` bit is read, +the DWT has already been powered down, and it stays down until the next boot. + +**Rule 91: `MATCHED` CANNOT BE TESTED THROUGH THE PROBE. Reading any debug +register clears `DEMCR.TRCENA`, which gates the whole DWT, so a probe-observed +`MATCHED = 0` is exactly as meaningless as a probe-observed "no catch". The DWT +can only be validated by the FIRMWARE reporting on itself — which needs a build +that logs `DWT_FUNCTION0` / `DWT_CATCH` over RTT from inside.** + +This is the same trap as rule 90, one level down, and it means candidate (3) is +**not settled** — the DATAVSIZE hypothesis is neither confirmed nor refuted. + +### The board wedged, and that is a clue + +During the FUNCTION0 test the counters froze completely (`boot/work` identical +across 20 s — the rule-85 liveness check). With `C_DEBUGEN` set by probe-rs, +halting debug takes priority over DebugMonitor: **a DWT match halts the core into +Debug state instead of raising the exception.** A wedge at exactly the moment a +comparator was armed at a hammered address is consistent with the comparator +working correctly and being routed to a halt that nothing services. + +That is suggestive, not proven — a coincidental wedge cannot be excluded. But it +does partially rehabilitate the original (later narrowed) rule 89: `C_DEBUGEN` +looks like a genuine part of the failure after all. + +Recovered by disarming both comparators first, then cold seed and reset. +Liveness re-proved by counter accumulation (boot 0 → 2, work 0x29 → 0xe6 over +25 s). + +### Honest status of this instrument class + +To make the DWT usable, the firmware itself must report `MATCHED` and the arming +state periodically over RTT, because every external observation destroys the +thing being observed. That is a rebuild, and worth doing — but it is instrument +work, not progress on the bug, and this class has now consumed several cycles. + +**Board state:** unchanged CF image, comparators disarmed, aim cleared, cold +seeded. Baseline **boot = 2, work = 230 at 2026-08-22 11:47:34**; +crashes = boot_end − 2. ALLOC_GUARD `0xa1100001`. + +**The second-board test remains unanswered and remains the highest-value +experiment available** — one hour, no instrument work, and it splits the +hypothesis space in half. Everything in this section is elaborate machinery for +deciding *where* in the firmware a wild store originates, on the unexamined +assumption that the firmware is where it originates at all. + +## Holding window, and the arithmetic nobody has written down + +Window 11:47:34 → 12:51:06 (63.5 min), CF image, comparators disarmed, no +instrumentation. Rule 85 gate: **31 records** (ring full), ALLOC_GUARD intact. +Δboot = 140 − 2 → **138 crashes**; Δwork = **9,955** → **1 per 72**, 286 +iterations/min. Signature: 19 HardFault / 12 WatchdogTimeout; PCs `0x88000000` +(×10) and `0x1002e3b4` (×9). Stable in the CF basin. + +### How far is the 24-hour goal, actually + +This has never been stated numerically in either document, and it should be: + +| layout | rate | throughput | crashes/min | crashes/day | +|---|---|---|---|---| +| CE (best basin measured) | 1 per 469 | 1,666 iter/min | 3.6 | **~5,100** | +| CF (current) | 1 per 72 | 286 iter/min | 4.4 | **~6,400** | + +**A 24-hour zero-crash soak requires roughly a 5,000x improvement.** + +Against that: the best knob ever found in this investigation is COOLDOWN=0 at +**13x**; the largest effect ever *claimed* (before same-epoch correction) was +~70x; and the layout term — which selects rather than tunes — spans about 6x +between the CE and CF basins. Every one of those is three orders of magnitude +short, and they do not compose. + +So the goal is not reachable by tuning. It needs the mechanism removed, which +means either finding the wild store (the firmware hypothesis) or establishing +that the firmware is not where it originates (the hardware hypothesis). Only the +second has never been tested. + +### Second-board test, prepared and ready + +Written to `tools/second_board_test.sh` so it is zero-effort the moment a second +Pico 2 W is available. It flashes the identical image, cold-seeds under rule 83, +soaks, and applies the rule-85 gate before reporting a rate. + +Pre-registered, so the answer cannot be read motivatedly afterwards: + +* **second board ~1 per 72–470** ⇒ the fault follows the firmware. Every + refutation in both documents stands, the corpus is sound, and the remaining + work is an upstream LLVM/silicon report rather than more soaks here. +* **second board clean, or orders of magnitude better** ⇒ the fault is **this + unit**. Two months of codegen bisection were chasing a hardware defect, the + layout sensitivity is explained as a marginal part being poked differently by + each build, and the 24-hour goal is reachable by swapping the board. +* **second board different but still bad** ⇒ both contribute; the firmware work + retains value but the absolute numbers in both documents are board-specific. + +**Board state:** unchanged CF image, no instrumentation armed, left soaking from +the 11:47:34 baseline. diff --git a/platform/pico2w/docs/investigations/sp-drift-root-cause.md b/platform/pico2w/docs/investigations/sp-drift-root-cause.md new file mode 100644 index 0000000..1306dc5 --- /dev/null +++ b/platform/pico2w/docs/investigations/sp-drift-root-cause.md @@ -0,0 +1,281 @@ +# Bug #5: the `memcpy` fault family is a +8 SP drift, not memory corruption + +**2026-08-30.** Status: **fault chain CLOSED. Drift mechanism OPEN.** + +This document supersedes the "stray write / wild pointer" reading of the +`memcpy` fault family. Nothing writes to memory. `copy_dma_step` reads its own +**intact** frame at the wrong offsets because the stack pointer moved. + +--- + +## 1. The chain + +SP is `0x2007cb18` for every *store* in `copy_dma_step` and `0x2007cb20` for +every *load*. Each load therefore reads the slot 8 bytes high: + +| load | intended | actually read | +|---|---|---| +| `ldr r0,[sp,#0x14]` | oam base `0x2004cb0c` | `count` = `1` | +| `ldr r0,[sp,#0x10]` | `self` `0x20048a8c` | `actual_src` = `0xC065` | + +The WRAM branch then computes `wram_base = r0 + 0x2000`: + +``` +index(0xC065 + 0x2000, off=0x65, end=0x66) -> 0xE065 + 0x65 = 0x0000E0CA +index_mut(1, off=0x65, end=0x66) -> dst = 0x66 +memcpy(dst=0x66, src=0x0000e0ca, len=1) -> precise bus error +``` + +**`0x0000e0ca` is NOT `ECHO_BASE + 0xCA`.** It is `0xC065 + 0x2000 + 0x65`. +Both pointers handed to `memcpy` are garbage, not just the source. + +Generalising, since `actual_src = 0xC000 + progress` and the `+0x2000` is +applied to it instead of to `self`: + +> **BFAR = 0xE000 + 2 x progress** +> **r12 = count + progress** + +and because `memcpy` does `sub.w lr,r2,#1`, the stacked **LR = len - 1**, so +`count = LR + 1`. + +--- + +## 2. The falsifier + +The relation above was derived from the disassembly **before** any crash data +was consulted, and predicts a *different* `r12` for different `(LR, BFAR)`. +One violating record kills it. + +Tested against the on-board crash sector (`0x103FF000`, dumped with OpenOCD +`dump_image`, decoded via `tools/crash_decoder.py --json`; fields `arm.lr`, +`arm.fault_addr`, `ext_regs.r12`, `ext_regs.r4`): + +``` +# LR BFAR prog cnt pred obs verdict pre_r4 +2 0x0 0xe0ca 0x65 1 0x66 0x66 PASS 0x2007cacc +4 0x0 0xe0ca 0x65 1 0x66 0x66 PASS 0x2007cacc +6 0x0 0xe0ca 0x65 1 0x66 0x66 PASS 0x2007cacc +7 0x2 0xe0cc 0x66 3 0x69 0x69 PASS 0x2007cacc +8 0x0 0xe0ca 0x65 1 0x66 0x66 PASS 0x2007cacc +10 0x0 0xe0ca 0x65 1 0x66 0x66 PASS 0x2007cacc + +RESULT: 6 PASS / 0 FAIL +``` + +**Record #7 is the load-bearing one**: distinct inputs (`LR=2` so `count=3`, +`BFAR=0xe0cc` so `progress=0x66`), predicted `r12=0x69`, observed `0x69`. + +**Independent corroboration the formula never uses:** all six records carry +`pre_r4 = 0x2007cacc`, exactly the `memcpy` SP the +8 theory requires +(`0x2007cb20 - 0x54`). The no-drift counterfactual gives `0x2007cac4`. + +--- + +## 3. Frame decode (the live catch) + +SP is authoritative from `DWT_CATCH[9]`, which `copy_dma_step` writes with its +own SP. Prologue: `push {r4-r7,lr}` / `add r7,sp,#0xc` / `push.w {r8-r11}` / +`sub sp,#0x4c`. SP never moves again until the epilogue — the *entire* function +contains exactly two SP writes. + +``` +[sp,#0x10] self 0x2007cb28 = 0x20048a8c valid heap ptr +[sp,#0x14] oam base 0x2007cb2c = 0x2004cb0c == self+0x4080 EXACT +[sp,#0x18] actual_src 0x2007cb30 = 0x0000c065 == 0xC000+0x65 +[sp,#0x1c] count 0x2007cb34 = 0x00000001 +[sp,#0x20] progress 0x2007cb38 = 0x00000065 +[sp,#0x44] oam(prolog)0x2007cb5c = 0x2004cb0c +[sp,#0x48] oam latch 0x2007cb60 = 0x2004cb0c +``` + +Every slot intact and self-consistent. `strd r11,r8,[sp,#16]` at `0x200026b8` +initialises both suspect slots (a byte scan for `STR`/`STR.W` misses `STRD` +`0xe9cd` — that mistake was made and corrected). + +**Drift window:** `0x200026b8` (strd, SP=0x2007cb18) .. `0x2000270e` (ldr, +SP=0x2007cb20). Entry SP `0x2007cb88` is independently anchored by the saved +`r7` chain, so the caller is clean and the drift is confined to the body. + +The only `bl` in that window is `RangeInclusive::contains` @ `0x1001599a`: +`push {r7,lr}` ... `pop {r7,pc}` @ `0x100159c0`. Statically net-zero on every +path; observed net **+8**. + +**The sharpest fact:** `contains` is called *twice* in this same invocation — +VRAM (`0x200026b4`) and WRAM (`0x200026f6`) — executing the byte-identical +sequence and exiting through the same unconditional `pop`. Call 1 returned with +SP correct (proved by the `strd` landing at `0x2007cb28/2c`). Call 2 returned ++8, twelve instructions later. **Non-deterministic at the granularity of a +single `pop {r7,pc}`.** + +--- + +## 4. Mechanism: two survivors, indistinguishable from a halted board + +Control flow through the window was correct (the WRAM branch ran, `memcpy` +executed), so **SP changed while PC did not**. That kills every push/pop +mismatch variant: a skipped push or restarted pop takes PC from `[0x2007cb1c]` +and returns to `0x2000277a`. + +| candidate | status | +|---|---| +| duplicate `POP {r7,pc}` base-register writeback | survives by elimination only; **no positive evidence**; no published erratum for M33 r1p0 (`CPUID=0x411fd210`) | +| transient instruction mis-fetch | survives; fits identically; better prior at 300 MHz on a 150 MHz-rated part; `cbz`(0xB1xx) and `add sp,#8`(0xB002) are adjacent encodings and the window holds two `cbz`s | + +**ETM cannot separate them** — it records control flow, not SP or register +writes. Both leave identical visible flow. (It is also feature-gated off in the +shipped image, and this ETM has `TRCIDR4.NUMACPAIRS == 0`, so trace cannot be +filtered.) + +--- + +## 5. Ruled out (do not re-derive) + +- **Hardware DMA** — ch0 `AUDIO_BUF_A -> 0x50200010` (PIO TX FIFO), ch1 + `CORE0_SCALE_BUF -> 0x40088008` (SPI1 DR), ch2-15 all zero. Both + memory->peripheral; no engine writes RAM. +- **Core 1** — MPU R0 enabled, `base 0x20067d80 limit 0x2007ffff AP=RO-priv` + covering all of core-0's stack; core-1 `CFSR=0 HFSR=0`, it never faulted, so + it never attempted. +- **Allocator free-list poisoning** — `ALLOC_GUARD` clean, 0 bad allocs, 0 bad + deallocs. **This refutes the premise `platform/pico2w/src/alloc_guard.rs` was + built on.** +- **Audio `Vec` spray** — no `(ptr, cap=0x800, len)` triple in POOL/stack/ + statics; `AUDIO_QUEUE` empty. The `cap` clamp is sound regardless. +- **Stored-image corruption** — full RAM `.data` diff vs ELF (3,500 words): one + mismatch, `_SEGGER_RTT+0x24`, a ring write pointer. Flash sums match the ELF. + *Note this rules out stored bytes only, NOT transient mis-fetch.* +- **NMI** — RP2350's per-core NMI mask is in the M33 EPPB block at `0xE0080000`, + **not** SYSCFG (that is the RP2040 layout). `NMI_MASK0 = NMI_MASK1 = 0`. No + IRQ is routed to NMI, so the historical `cpsid i` A/B had nothing to be blind to. +- **TrustZone stack banking** — zero `bxns`/`blxns`/`sg` image-wide. +- **CONTROL / MSP / PSP switching** — no `msr control`/`msp`/`psp` image-wide. +- **FPU lazy stacking** — `FPCCR.LSPEN=0`, `CONTROL.FPCA=0`, + `EXC_RETURN=0xfffffff9`, zero `vpush`/`vpop`/`vldm`/`vstm` image-wide. +- **IT-block `pophi {r7,pc}` hazard** — the `itt hi` never fires for these + operands (`cmp 0xC000,0xC065` makes HI false); both calls exit via the + unconditional pop. The cb-site callee `set_r8_enum` has no IT blocks at all. + +--- + +## 6. Retractions of earlier findings in this corpus + +- **"COMBINED-WINDOW FINDING" (rule 82) is WRONG.** `0x0000e06a`, `e072`, + `e07c`, `e084`, `e114` are not Game Boy echo-RAM addresses deposited by a + stray write. They are `0xE000 + 2*progress` — arithmetic, not data. Every one + implies a legal `progress <= 0x9F`. +- **`0x0000fe9e` was never a BFAR.** It is a wild PC with `CFSR=0x00000100` + (IBUSERR), a different family. Not a counterexample to the formula. +- **The `regalloc=basic` comment in `core/src/memory/memory.rs` is STALE** — it + claims that flag as the root-cause fix; the flag was removed 2026-08-08 as + dormancy-not-fix. +- **`data_check::init()` is dead code**, never called anywhere in the tree, so + `DATA_CHECK` counters are cumulative across boots, not per-boot. The `[2]=0` + negative still holds (every live slot is a monotonic counter or first-failure + latch, so cross-epoch mixing cannot manufacture a zero). +- **`XIP_CHECK` is void.** Its reference sum `0x98e89352` was captured under a + different flashed image (true current `0xa58a51da`), so 100% of its 28,008 + mismatches are artifacts. + +--- + +## 7. Scope — do not overclaim + +This explains the **`memcpy` / `BFAR ~ 0xE0xx` family only**. It does **not** +explain: + +- the wild-PC / IBUSERR family (`CFSR=0x00000100`): `pc=0x89000000`, + `0x13fc21b4`, `0x0000fe9e` +- `BFAR=0x00008000` (a store through `r8` under `-Z stack-protector=strong`) + +Crash records #1/#3/#5/#9 in the tested sector are these other families, and the +formula correctly does not apply to them. + +--- + +## 8. Next step + +The right instrument is a **register-only SP tripwire at the return boundary**, +giving positive evidence instead of elimination. Immediately after the WRAM +`bl contains`, before the existing `cbz`: + +```asm +mov.w r12, r7 +sub.w r12, r12, #0x68 ; copy_dma_step body SP = r7 - 0x68 +cmp.w r12, sp +bne.w sp_shift_capture +``` + +`sp_shift_capture` must be naked, stackless, make no Rust calls, record +SP/expected/LR/PC/xPSR/ICI into `.uninit`, then halt. **A gated positive control +in the same image is mandatory** — without it a null result is +indistinguishable from a broken tripwire. + +Caveat: this diagnostic moved `_stack_end` `0x20067d90 -> 0x20067e28`, shifting +every frame by 0x98. Given this bug's dormancy history, failure to reproduce is +not evidence of absence. + +If it fires, the follow-up is a fixed-binary variant matrix (native `pop` vs +split `ldr/ldr/add sp/bx`, XIP vs SRAM, selected by a planted gate) to separate +duplicate-writeback from mis-fetch. + +--- + +# 24-HOUR PRODUCTION SOAK AT 288 MHz — PASSED (2026-09-01 → 2026-09-02) + +``` +elapsed 1441 min (24.02 h) +work 3,560,324 emulator ticks +drift 0 +reboots 0 +``` +287 sample blocks, zero read failures, work counter linear throughout +(min 12,381 / max 12,422 / mean 12,405 per 5-min block, 41.4 ticks/s, 0.3% spread). + +**Configuration:** real firmware, NO `test-rom` feature. Zelda: Link's Awakening +(512 KiB, 32 banks) staged in flash. Compiled-in 288 MHz via the new `oc-288` +feature (`TARGET_SYS_HZ = 288_000_000`, PLL FBDIV 120 / VCO 1440 / POSTDIV 5), +V1_30, QMI CLKDIV 6 → SCK 48 MHz. Image CRC `0x733173ac`. + +**Arithmetic.** Baseline is the corpus's controlled series (`main.rs` clock table): +300 MHz @ V1_25 produced 4 crash records in 2h15m = **1.78 crashes/hour**. Over +24.02 h that predicts **42.7 crashes**. Observed **0** → **≥14.2×** reduction at +95% confidence (expected/3); P(0 | null) = e^-42.7 ≈ 2.7e-19. + +## What this does NOT establish + +1. **Product validation, not a controlled experiment.** No 300 MHz control of + comparable duration was run on this build. The comparison is against a + historical baseline from a different build and workload. +2. **The cliff is bracketed only to (288, 300].** 288 may have as little as 0.3% + margin. Untested against temperature and part-to-part spread — silicon timing + moves more than that on its own across a commercial temperature range. +3. **The defect still exists at 150 MHz.** The corpus records a 150 MHz crash + identical in every field (PC 0x2003bd98, LR 0x2000204d, r4 0x2007ccb0, + r12 0x100143fb). This is RATE REDUCTION, not elimination. +4. **One board, one voltage (V1_30), one ambient temperature.** +5. **Zero drift is not a null result here.** The ~95 events/min drift rate came + from the synthetic fast-DMA ROM, which did nothing but issue OAM DMA. Zelda + issues ~1 DMA per frame, so the drift counter is far less exposed on this + workload and a zero reading is expected. (An earlier reading of a 4-minute + zero-drift window at 300 MHz as evidence of dormancy was WRONG: at 1.78/h + that window predicts 0.12 events, so zero was the expected outcome either way.) + +## BLOCKER: staged ROM cannot carry a rom_id + +The staged ROM had to be written with **`rom_id` omitted** (header bytes +[96..128] left 0xFF). A valid `rom_id` triggers the boot save-state restore, +which is the documented core-1 boot-crash path (see memory +`project_wifi_menu_crash_repro`). With `rom_id` present the board wedged on boot: +core 0 spinning in `embassy_time::delay::block_for`, core 1 parked in +`run_core1_worker+0x1a4`, no faults, and "entering main loop" never reached. + +**This is a SEPARATE, unresolved bug, and it currently blocks shipping a staged +ROM with saves enabled.** It needs its own investigation. + +## Recommendation + +**Ship 266 MHz (FBDIV 111), not 288.** 266 sits 8–11% below the observed cliff +versus 288's 0–4%, and the corpus already has **9 h clean at 266 with LOWER +voltage (V1_25)**. 288 passed 24 h here, but it is riding an edge whose exact +location is unknown, measured on one part at one temperature. The 11% frame-rate +cost against 300 buys margin for temperature, regulator tolerance and lot spread. diff --git a/platform/pico2w/src/crash/context.rs b/platform/pico2w/src/crash/context.rs index 3697dd5..364e120 100644 --- a/platform/pico2w/src/crash/context.rs +++ b/platform/pico2w/src/crash/context.rs @@ -290,7 +290,7 @@ impl ScratchRegs { panic_loc[4..8].copy_from_slice(&wd[15].to_le_bytes()); CrashRecord { - schema_ver: 1, + schema_ver: 2, crash_kind, flags: flags_byte, slot_seq, diff --git a/platform/pico2w/src/crash/handler.rs b/platform/pico2w/src/crash/handler.rs index 205bbd1..943b5b9 100644 --- a/platform/pico2w/src/crash/handler.rs +++ b/platform/pico2w/src/crash/handler.rs @@ -307,6 +307,27 @@ fn commit_crash_and_reset( let gb_pc = ctx.as_ref().map(|c| c.gb_pc).unwrap_or(0); let packed7 = ((rom_bank as u32) << 16) | (gb_pc as u32); + // Capture a stack window for the crash record. `commit_crash_and_reset` does + // not build the record here — it stashes state in the watchdog/POWMAN + // scratch registers and resets, and the record is assembled on the NEXT + // boot — so the window goes to `.uninit`, which survives the reset the same + // way the scratch registers do. + // + // For a HardFault `sp` is MSP with the 8-word exception frame already + // pushed, so the interesting region starts above it; the +4 accounts for the + // STKALIGN pad, which the stacked xPSR bit 9 reports. A panic has no + // exception frame, so read from `sp` directly. + #[cfg(target_arch = "arm")] + unsafe { + let base = if matches!(kind, CrashKind::HardFault) { + let stacked_xpsr = core::ptr::read_volatile((sp + 0x1c) as *const u32); + sp + 0x20 + if stacked_xpsr & (1 << 9) != 0 { 4 } else { 0 } + } else { + sp + }; + crate::crash::stack_snapshot::capture_crash_stack(base); + } + write_watchdog_scratch( arm_pc, arm_lr, diff --git a/platform/pico2w/src/crash/mod.rs b/platform/pico2w/src/crash/mod.rs index fd0974d..cc54be6 100644 --- a/platform/pico2w/src/crash/mod.rs +++ b/platform/pico2w/src/crash/mod.rs @@ -21,6 +21,7 @@ //! never write the magic sentinel, so they are never confused with crashes. // Re-export sub-modules. +pub mod stack_snapshot; pub mod storage; #[cfg(target_arch = "arm")] @@ -94,7 +95,7 @@ mod tests { fn sample_record(slot: u8) -> CrashRecord { CrashRecord { - schema_ver: 1, + schema_ver: 2, crash_kind: CrashKind::HardFault as u8, flags: flags::HAS_ARM_REGS | flags::HAS_GB_STATE | flags::HAS_ROM_INFO, slot_seq: slot, diff --git a/platform/pico2w/src/crash/record.rs b/platform/pico2w/src/crash/record.rs index e2d0e0a..26e4446 100644 --- a/platform/pico2w/src/crash/record.rs +++ b/platform/pico2w/src/crash/record.rs @@ -123,8 +123,19 @@ pub mod flags { // if HAS_STACK_OVERFLOW set → overflow depth, // else → remaining stack headroom // [84..88] dma_busy_mask bitmask of DMA channels with BUSY=1 at crash time -// [88..116] dma_write_addrs WRITE_ADDR for DMA channels 0-6 (7 × u32 LE) -// [116..120] _reserved zero-filled (schema v2+; 0 in v1 records) +// [88..96] dma_write_addrs WRITE_ADDR for DMA channels 0-1 (2 × u32 LE) +// Channels 2-6 were dropped in schema v2: a live +// read of all 16 channels showed only ch0 (audio +// -> PIO TX FIFO) and ch1 (scale buf -> SPI1 DR) +// are ever non-zero. The reclaimed 20 bytes hold +// the stack snapshot below. +// [96..120] stack_snapshot 6 × u32 LE: stack words starting at the +// PRE-FAULT SP (i.e. above the 8-word exception +// frame the fault pushed). For a wild-PC fault +// out of a `pop {rlist, pc}` this window spans +// the return-address slot and slot+8, which is +// what the +8 SP-drift test needs. +// Zero for schema v1 records. // [120..124] crc32 CRC32 of bytes [0..120] // [124..128] _pad4 // @@ -171,9 +182,16 @@ pub struct CrashRecord { /// Bitmask of DMA channels (0-7) that had CTRL_TRIG.BUSY=1 at crash time. /// Zero if DMA snapshot was not captured (schema v1 records). pub dma_busy_mask: u32, - /// WRITE_ADDR register snapshot for DMA channels 0-6 at crash time. + /// WRITE_ADDR register snapshot for DMA channels 0-1 at crash time. /// Zeros for schema v1 records or channels that were not snapshotted. - pub dma_write_addrs: [u32; 7], + /// Channels 2-6 were dropped in schema v2 — a live read of all 16 channels + /// found them permanently zero, so they were 20 bytes of dead record. + pub dma_write_addrs: [u32; 2], + /// Stack words starting at the PRE-FAULT SP (above the exception frame). + /// Spans the return-address slot and slot+8 for a `pop {rlist, pc}` fault, + /// which is what testing the +8 SP-drift prediction requires. + /// Zero for schema v1 records. + pub stack_snapshot: [u32; 6], } /// Opaque 128-byte wire representation. @@ -220,6 +238,10 @@ impl CrashRecord { buf[80..82].copy_from_slice(&self.panic_line.to_le_bytes()); buf[82..84].copy_from_slice(&self.stack_headroom.to_le_bytes()); buf[84..88].copy_from_slice(&self.dma_busy_mask.to_le_bytes()); + for (i, &w) in self.stack_snapshot.iter().enumerate() { + let off = 96 + i * 4; + buf[off..off + 4].copy_from_slice(&w.to_le_bytes()); + } for (i, &addr) in self.dma_write_addrs.iter().enumerate() { let off = 88 + i * 4; buf[off..off + 4].copy_from_slice(&addr.to_le_bytes()); @@ -281,13 +303,26 @@ impl CrashRecord { stack_headroom: u16::from_le_bytes(buf[82..84].try_into().unwrap()), dma_busy_mask: u32::from_le_bytes(buf[84..88].try_into().unwrap()), dma_write_addrs: { - let mut addrs = [0u32; 7]; + let mut addrs = [0u32; 2]; for (i, addr) in addrs.iter_mut().enumerate() { let off = 88 + i * 4; *addr = u32::from_le_bytes(buf[off..off + 4].try_into().unwrap()); } addrs }, + // Schema v1 wrote DMA channels 2-6 here; v2 reuses the space for the + // stack snapshot. v1 records decode as garbage in this field, which + // is why the reader must gate on `schema_ver >= 2`. + stack_snapshot: { + let mut w = [0u32; 6]; + if buf[4] >= 2 { + for (i, word) in w.iter_mut().enumerate() { + let off = 96 + i * 4; + *word = u32::from_le_bytes(buf[off..off + 4].try_into().unwrap()); + } + } + w + }, }) } diff --git a/platform/pico2w/src/crash/stack_snapshot.rs b/platform/pico2w/src/crash/stack_snapshot.rs new file mode 100644 index 0000000..dc5f408 --- /dev/null +++ b/platform/pico2w/src/crash/stack_snapshot.rs @@ -0,0 +1,74 @@ +//! Stack window captured at fault time and carried across the reset. +//! +//! `commit_crash_and_reset` does not build a `CrashRecord` — it stashes state in +//! the watchdog/POWMAN scratch registers and resets, and the record is assembled +//! on the NEXT boot. There is no room in scratch for a stack window, so it lives +//! here in `.uninit`, which survives the reset the same way the scratch +//! registers do. +//! +//! For a wild-PC fault out of a `pop {rlist, pc}` this window spans the return +//! slot, which is what makes such a record diagnosable at all. + +/// `.uninit` backing store. Padded well clear of `_stack_end` so its tail cannot +/// land inside core 1's read-only MPU region (see `heartbeat` for the same +/// hazard, which silently killed core 1 when it was first hit). +#[no_mangle] +#[link_section = ".uninit.CRASH_STACK_SNAP"] +pub static mut CRASH_STACK_SNAP: [usize; 16] = [0; 16]; + +const CRASH_SNAP_BASE: usize = 0; +const CRASH_SNAP_MAGIC: usize = 0x5CAF_0001; + +/// Number of stack words carried in a crash record. Must match +/// `CrashRecord::stack_snapshot`. +pub const CRASH_SNAP_WORDS: usize = 6; + +/// Capture `CRASH_SNAP_WORDS` stack words starting at `base` into `.uninit`, +/// for the next boot to fold into the crash record. +/// +/// # Safety +/// Called from a fault/panic handler with interrupts effectively quiesced. +/// `base` is range-checked against the core-0 stack before any dereference, so +/// a wild SP cannot turn this into an out-of-bounds read. +pub unsafe fn capture_crash_stack(base: usize) { + let f = &raw mut CRASH_STACK_SNAP; + (*f)[CRASH_SNAP_BASE] = CRASH_SNAP_MAGIC; + (*f)[CRASH_SNAP_BASE + 1] = base; + // Only read what is provably inside the core-0 stack. A drifted or wild SP + // is exactly the condition this instrument exists to record, so it must not + // fault while recording it. + let lo = 0x2000_0000usize; + let hi = 0x2008_2000usize; + for i in 0..CRASH_SNAP_WORDS { + let a = base.wrapping_add(i * 4); + (*f)[CRASH_SNAP_BASE + 2 + i] = if a >= lo && a + 4 <= hi && a % 4 == 0 { + core::ptr::read_volatile(a as *const usize) + } else { + 0 + }; + } +} + +/// Read back the snapshot captured before the last reset. +/// Returns `None` if no snapshot was recorded. +pub unsafe fn last_crash_stack() -> Option<(usize, [u32; CRASH_SNAP_WORDS])> { + let f = &raw const CRASH_STACK_SNAP; + if (*f)[CRASH_SNAP_BASE] != CRASH_SNAP_MAGIC { + return None; + } + let base = (*f)[CRASH_SNAP_BASE + 1]; + let mut w = [0u32; CRASH_SNAP_WORDS]; + for (i, out) in w.iter_mut().enumerate() { + *out = (*f)[CRASH_SNAP_BASE + 2 + i] as u32; + } + Some((base, w)) +} + +/// Clear the snapshot so a stale one cannot be folded into a later record. +/// +/// # Safety +/// Call once at boot, after `last_crash_stack()` has been consumed. +pub unsafe fn clear_crash_stack() { + let f = &raw mut CRASH_STACK_SNAP; + (*f)[CRASH_SNAP_BASE] = 0; +} diff --git a/platform/pico2w/src/crash/storage.rs b/platform/pico2w/src/crash/storage.rs index 737a0bf..a5dae87 100644 --- a/platform/pico2w/src/crash/storage.rs +++ b/platform/pico2w/src/crash/storage.rs @@ -62,11 +62,23 @@ pub fn check_and_commit(flash: &mut OnboardFlash<'_>) -> bool { match write_record_to_flash(flash, |slot| { let mut rec = snap.to_crash_record(slot); + // Bug #5: fold in the stack window captured by the fault handler before + // the reset. `.uninit` carries it across the reset the same way the + // watchdog scratch registers carry the register state. + unsafe { + if let Some((_base, words)) = crate::crash::stack_snapshot::last_crash_stack() { + rec.stack_snapshot = words; + crate::crash::stack_snapshot::clear_crash_stack(); + } + } unsafe { use crate::crash::handler::{DMA_CRASH_SNAPSHOT, DMA_SNAPSHOT_SENTINEL}; if DMA_CRASH_SNAPSHOT[0] == DMA_SNAPSHOT_SENTINEL { rec.dma_busy_mask = DMA_CRASH_SNAPSHOT[1]; - for i in 0..7usize { + // Schema v2 keeps only ch0/ch1 in the record: a live read of all + // 16 channels showed ch2-15 permanently zero. Higher channels are + // still logged via defmt below. + for i in 0..rec.dma_write_addrs.len() { rec.dma_write_addrs[i] = DMA_CRASH_SNAPSHOT[2 + i]; } // Log high channels (ch7-ch15) via defmt — not stored in flash record. @@ -229,7 +241,7 @@ impl ResetReasonSnapshot { fn to_crash_record(self, slot_seq: u8) -> CrashRecord { let mut record = CrashRecord { - schema_ver: 1, + schema_ver: 2, crash_kind: if self.watchdog_timer() { super::CrashKind::WatchdogTimeout as u8 } else { @@ -394,7 +406,7 @@ fn clear_watchdog_reason() { #[cfg(target_arch = "arm")] fn build_watchdog_record(slot_seq: u8) -> super::CrashRecord { super::CrashRecord { - schema_ver: 1, + schema_ver: 2, crash_kind: super::CrashKind::WatchdogTimeout as u8, flags: 0, slot_seq, diff --git a/platform/pico2w/src/main.rs b/platform/pico2w/src/main.rs index 8685916..2f8d74a 100644 --- a/platform/pico2w/src/main.rs +++ b/platform/pico2w/src/main.rs @@ -60,12 +60,31 @@ const TARGET_SYS_HZ: u32 = 300_000_000; // 16 s gives worst-case frame/flash-sync stalls plenty of headroom without false-tripping; // the multicore livelock that caused the old 5 s freeze→reset is separately fixed by WFE backpressure. const WATCHDOG_WINDOW_MS: u64 = 16_000; -#[cfg(all(not(feature = "oc-300"), feature = "oc-280"))] +#[cfg(all(not(feature = "oc-300"), feature = "oc-288"))] +// 288 MHz = PLL FBDIV 120 (VCO 1440) / POSTDIV 5 — one grid step below 300 on +// the stock PLL config, and the fastest clock measured free of the SP-drift +// fault. A frequency sweep (live PLL reprogramming, no rebuild) put the failure +// cliff between 288 and 300: at 300 MHz the drift instrument logged 67.5 events +// per 1000 work-ticks, while 288/276/264/252/240 logged ZERO across 13,567 +// ticks. A 24 h soak at 288 on the real firmware logged 0 crashes. +// +// NOT proven safe: the cliff is bracketed only to (288, 300], so 288 may have as +// little as 0.3% margin, and that is untested against temperature and +// part-to-part spread. 266 is the more conservative shipping point. +// See docs/investigations/sp-drift-root-cause.md. +const TARGET_SYS_HZ: u32 = 288_000_000; +#[cfg(all(not(feature = "oc-300"), not(feature = "oc-288"), feature = "oc-280"))] const TARGET_SYS_HZ: u32 = 280_000_000; -#[cfg(all(not(feature = "oc-300"), not(feature = "oc-280"), feature = "oc-266"))] +#[cfg(all( + not(feature = "oc-300"), + not(feature = "oc-288"), + not(feature = "oc-280"), + feature = "oc-266" +))] const TARGET_SYS_HZ: u32 = 266_000_000; #[cfg(all( not(feature = "oc-300"), + not(feature = "oc-288"), not(feature = "oc-280"), not(feature = "oc-266") ))] @@ -340,10 +359,34 @@ async fn main(spawner: Spawner) { // SD card — always initialised so the ROM list is available. let mut spi_cfg = spi::Config::default(); spi_cfg.frequency = 400_000; - let spi_bus = Spi::new_blocking(p.SPI0, p.PIN_6, p.PIN_7, p.PIN_4, spi_cfg); + let mut spi_bus = Spi::new_blocking(p.SPI0, p.PIN_6, p.PIN_7, p.PIN_4, spi_cfg); rp_pac::PADS_BANK0.gpio(4).modify(|w| w.set_pue(true)); - let spi_dev = ExclusiveDevice::new(spi_bus, Output::new(p.PIN_5, Level::High), Delay); - let sdcard = SdCard::new(spi_dev, Delay); + + // SD power-up clocks. The SD spec requires >=74 clock cycles at 400 kHz with + // CS DEASSERTED before the card will enter SPI mode. `embedded-sdmmc` does + // not do this — its `SdCard` docs leave it to the caller ("some [cards] do, + // some don't") — and this firmware previously went straight from + // construction to CMD0. CS is therefore created FIRST, held high for the + // clocks, then moved into the ExclusiveDevice. 10 bytes = 80 clocks. + let cs = Output::new(p.PIN_5, Level::High); + if let Err(e) = spi_bus.blocking_write(&[0xFFu8; 10]) { + defmt::warn!("SD power-up clocks failed: {:?}", defmt::Debug2Format(&e)); + } + + let spi_dev = ExclusiveDevice::new(spi_bus, cs, Delay); + // Bounded card acquisition. With no card each attempt costs ~340 ms (a CMD0 + // that times out, 255 flush bytes at 400 kHz, and a delay), so the crate + // default of 50 retries takes ~17.1 s to return `CardNotFound`. The watchdog + // window is 16 s and the RP2350 counter is 24-bit MICROSECONDS, capping the + // window at 16.777 s — so a no-card boot could never fit and reset the board + // mid-init, every boot, producing only `WatchdogTimeout` records. 6 retries + // puts a no-card failure at ~2.4 s; a card that is present answers CMD0 + // within the first few attempts (measured: 781 ms to a successful mount). + let sd_opts = embedded_sdmmc::sdcard::AcquireOpts { + acquire_retries: 6, + ..Default::default() + }; + let sdcard = SdCard::new_with_options(spi_dev, Delay, sd_opts); let mut sd_mgr = PicoSdMgr::new(sdcard, DummyClock); // Check for unread crash records so menus can show the crash-report badge. @@ -393,6 +436,9 @@ async fn main(spawner: Spawner) { let mut audio_samples = Vec::with_capacity(2048); // Build initial state: Running if a ROM is staged, else MainMenu. + // Cleared as soon as any boot SD access reports the card missing. Each SD + // call costs a full acquisition timeout, so one failure is enough to know. + let mut sd_available = true; let (initial_state, gameboy_init, staged_rom_name, staged_rom_id, core1_token) = match staged { Some((info, name)) => { info!( @@ -424,17 +470,24 @@ async fn main(spawner: Spawner) { // avoids holding two large SD blobs at once during boot. let slot = SaveSlot::new(0).expect("slot 0 is valid"); watchdog.feed(Duration::from_millis(WATCHDOG_WINDOW_MS)); - let save_state_blob = - sd_mgr.read_save_state(&rom_id, slot).unwrap_or_else(|e| { - defmt::warn!( - "boot save state read failed: {:?}", - defmt::Debug2Format(&e) - ); - None - }); + let save_state_res = sd_mgr.read_save_state(&rom_id, slot); + // One attempt is enough to know. If the card did not + // answer, every further read costs another full + // acquisition timeout and can only fail the same way. + let sd_alive = save_state_res.is_ok(); + if !sd_alive { + sd_available = false; + } + let save_state_blob = save_state_res.unwrap_or_else(|e| { + defmt::warn!( + "boot save state read failed: {:?}", + defmt::Debug2Format(&e) + ); + None + }); watchdog.feed(Duration::from_millis(WATCHDOG_WINDOW_MS)); let has_save_state = save_state_blob.is_some(); - let battery_data = if has_save_state { + let battery_data = if has_save_state || !sd_alive { None // save state includes cart RAM; skip battery read } else { watchdog.feed(Duration::from_millis(WATCHDOG_WINDOW_MS)); @@ -547,7 +600,13 @@ async fn main(spawner: Spawner) { spawner, wifi_periphs, }; - refresh_save_slot_available(&mut app, &sd_mgr); + // Runs after the watchdog is armed and before the main loop's first feed, so + // on a dead card its acquisition timeout would itself trip the watchdog. + if sd_available { + refresh_save_slot_available(&mut app, &sd_mgr); + } else { + app.save_slot_available = false; + } #[cfg(feature = "fps")] let mut tracker = perf::PerfTracker::new(); @@ -591,13 +650,13 @@ async fn main(spawner: Spawner) { AppState::MainMenu(menu_state) => { menu_state - .tick(&mut app, &mut game_disp, &mut input, &mut sd_mgr) + .tick(&mut app, &mut game_disp, &mut input, &mut sd_mgr, &mut watchdog) .await; } AppState::RomList(rom_list_state) => { rom_list_state - .tick(&mut app, &mut game_disp, &mut input, &mut sd_mgr) + .tick(&mut app, &mut game_disp, &mut input, &mut sd_mgr, &mut watchdog) .await; } diff --git a/platform/pico2w/src/mpu.rs b/platform/pico2w/src/mpu.rs index be5651f..8d568c6 100644 --- a/platform/pico2w/src/mpu.rs +++ b/platform/pico2w/src/mpu.rs @@ -19,6 +19,7 @@ pub unsafe fn setup_core0_data_mpu() { unsafe extern "C" { static _SEGGER_RTT: u8; + static __edata: u8; } const MPU_TYPE: *mut u32 = 0xE000_ED90 as *mut u32; @@ -52,6 +53,83 @@ pub unsafe fn setup_core0_data_mpu() { MPU_RBAR.write_volatile(0x2000_001C); MPU_RLAR.write_volatile(data_limit as u32 | 1); + // Region 1: the REST of `.data`, above the RTT control block. + // + // Region 0's comment claims deriving the bound from `_SEGGER_RTT` stops + // the end of `.data` being silently writable. It does the opposite: the + // linker places `_SEGGER_RTT` in the MIDDLE of `.data`, so region 0 ends + // at 0x2000373f while `.data` actually runs to `__edata` (0x20003920) — + // leaving ~480 bytes unprotected, and those bytes are exactly the + // `__Thumbv7ABSLongThunk_*` long-branch table. That is the one part of + // `.data` we have direct crash evidence against (a fault whose branch + // target register held the CORRECT address for `cb::sla_u8`, whose thunk + // lives at 0x200037d4, yet landed at a wild heap PC — i.e. the thunk's + // own instructions/literal pool were corrupted). It is also why every + // "Core N never writes .data" result so far was vacuous for that range. + // + // Start above the RTT control block (SEGGER control block is 0x30 bytes + // and IS legitimately written by defmt), rounded up to the 32-byte MPU + // granule; end at `__edata`. + if dregion > 1 { + let thunks_base = ((rtt_addr + 0x30 + 0x1F) & !0x1F) as u32; + let edata = core::ptr::addr_of!(__edata) as u32; + // RLAR's LIMIT is INCLUSIVE and its low 5 bits are forced to 0x1F, so + // the region always ends at `limit | 0x1F` — i.e. it rounds UP to the + // enclosing 32-byte granule. `(edata - 1) & !0x1F` therefore covers + // PAST `__edata` and marks the first bytes of `.bss` read-only, since + // `__edata` is essentially never 32-byte aligned. That is a real fault + // generator, not a theoretical one: with `-Z stack-protector=strong` + // (`__edata = 0x20003ab4`) it made the firmware unbootable — a genuine + // DACCVIOL, MMFAR = 0x20003ab8 = `__sbss` exactly, faulting in an + // ordinary early `atomic_store::` into `.bss`. + // Step back a full granule so the region always ends BELOW `__edata`, + // giving up at most 32 bytes of thunk coverage to stay off `.bss`. + let thunks_limit = edata.saturating_sub(0x20) & !0x1F; + if thunks_limit > thunks_base { + MPU_RNR.write_volatile(1); + MPU_RBAR.write_volatile(thunks_base | 0x1C); // SH=11, AP=10 priv-RO, XN=0 + MPU_RLAR.write_volatile(thunks_limit | 1); + defmt::info!( + "core0 MPU region 1 armed: .data thunks RO [{=u32:#010x}..={=u32:#010x}]", + thunks_base, + thunks_limit + 0x1F + ); + } + } + + // Region 2: CORE 1's STACK, privileged-read-only *for core 0*. + // + // Core 0 must never write core 1's stack. The existing regions fence + // core 1 out of core 0's stack, but the symmetric direction was never + // covered — core 0 could scribble on CORE1_STACK with nothing to trap it. + // + // Evidence this matters (recovered 2026-08-05 from the WATCHDOG scratch + // registers, an uncommitted core-1 crash record that the normal crash log + // could never show because only CORE 1 reset, so no boot ever ran + // check_and_commit): core 1 took a precise bus fault + // (CFSR=0x00008200 BFARVALID|PRECISERR) at address 0x0c164015 with + // LR=0x20000325 = ApuPeripheral::produce_samples. The sample_buffer Vec + // was INTACT (cap=2048, ptr=0x2002b7bc), and ptr+len*2=0x2002b960 is + // nowhere near the faulting address — so the buffer was fine and `self` + // itself was garbage, i.e. the `worker` reference held on CORE 1's STACK + // had been clobbered. Same fault-address family as two earlier core-1 + // crashes (0x0c0e0015, 0x0c160015). + // + // If core 0 is the writer, this region turns that silent clobber into an + // immediate DACCVIOL with MMFAR = the exact address and the stacked PC = + // the exact storing instruction. If it never fires, core 0 is excluded + // and the writer is core 1 itself or DMA. + // + // XN=1 here (unlike the .data regions): core 0 never executes from core + // 1's stack. CORE1_STACK is 0x20080000..0x20082000 (8 KiB, SRAM8/9), both + // ends already 32-byte aligned so no granule rounding is needed. + // NOTE: this region is NOT armed here — see `arm_core1_stack_fence()`. + // Core 0 legitimately writes core 1's stack while setting up its initial + // frame in `spawn_core1`, so arming it this early makes boot fault. That + // is not hypothetical: doing so produced an immediate + // CFSR=0x00000082 (MMARVALID|DACCVIOL), MMFAR=0x20081ffc (top word of + // CORE1_STACK), PC=core::ptr::write — which also proves the region works. + // ENABLE=1, PRIVDEFENA=1 (other addresses use default map), HFNMIENA=0 // (MPU disabled during fault handlers so the crash handler can read freely). MPU_CTRL.write_volatile(0x0000_0005); @@ -65,14 +143,36 @@ pub unsafe fn setup_core0_data_mpu() { } #[cfg_attr(target_arch = "arm", link_section = ".data")] -/// Configure Core 1's PMSAv8-M MPU to mark Core 0's stack as privileged-read-only. +/// Configure Core 1's PMSAv8-M MPU with two privileged-read-only regions. /// -/// Region 0: 0x20066B60–0x2007FFFF, AP=10 (priv RO), XN=1, SH=inner-shareable. +/// Region 0: Core 0's MSP stack, AP=10 (priv RO), XN=1, SH=inner-shareable. +/// Region 1: `.data` RAM code + long-branch thunk table, AP=10 (priv RO), **XN=0** +/// (Core 1 *executes* `.data`-resident code such as +/// `PpuPeripheral::render_scanline`, so execution must stay permitted — +/// XN=1 here would fault instantly). /// PRIVDEFENA=1 leaves all other addresses with full default access. /// -/// When Core 1 writes anywhere in this range → MemManage fault (CFSR.DACCVIOL=1, +/// When Core 1 writes anywhere in either range → MemManage fault (CFSR.DACCVIOL=1, /// MMFAR=faulting address) → escalates to HardFault → existing handler records /// stacked PC = the exact corrupt store instruction. +/// +/// # Why region 1 exists (bug #5) +/// +/// Core 0 has protected `.data` since the region-0 setup in +/// [`setup_core0_data_mpu`], and has never once tripped DACCVIOL — so Core 0 is +/// not the writer. Core 1, however, was only ever fenced off from *Core 0's +/// stack*; `.data` was left fully writable from Core 1, so the 2026-06-12 +/// "Core 1 definitively ruled out" result only ruled it out for stack writes and +/// never covered `.data` at all. +/// +/// That gap matters because `.data` (all code + literal pools, no mutable +/// statics — any post-startup write to it is by definition the bug) holds the +/// `__Thumbv7ABSLongThunk_*` long-branch table at its top. A live capture +/// (git a8ea0259, cycle ~2.58B) faulted with `LR` = `Sm83::cb_reg`, `r12` = +/// `cb::sla_u8` — i.e. the branch *target register was correct* — yet landed at +/// `PC = 0x2002b5ec` (heap, INVSTATE). A correct target register with a wrong +/// destination means the thunk's own instructions/literal pool were corrupted, +/// not the register: the writer hit the thunk table itself. #[cfg(target_arch = "arm")] pub unsafe fn setup_core1_mpu() { const MPU_TYPE: *mut u32 = 0xE000_ED90 as *mut u32; @@ -120,6 +220,60 @@ pub unsafe fn setup_core1_mpu() { base ); + // Region 1: `.data` RAM code + thunk table, priv-RO but still executable. + // Same encoding and same `_SEGGER_RTT`-derived limit as Core 0's region 0 + // (see `setup_core0_data_mpu`) so both cores fence exactly the same bytes. + // RBAR: BASE=0x20000000 | SH=11 | AP=10 (priv-RO) | XN=0 → 0x2000_001C + // RLAR: LIMIT | EN=1 + let data_limit = if dregion > 1 { + unsafe extern "C" { + static _SEGGER_RTT: u8; + static __edata: u8; + } + let rtt_addr = core::ptr::addr_of!(_SEGGER_RTT) as usize; + let limit = ((rtt_addr - 1) & !0x1F) as u32; + MPU_RNR.write_volatile(1); + MPU_RBAR.write_volatile(0x2000_001C); + MPU_RLAR.write_volatile(limit | 1); + + // Region 2: the thunk table above the RTT control block — see the long + // rationale on Core 0's region 1 in `setup_core0_data_mpu`. Region 1 + // stops at the RTT block (which defmt legitimately writes), leaving the + // `__Thumbv7ABSLongThunk_*` table unprotected; that gap is why "Core 1 + // never writes .data" did not actually cover the range we have crash + // evidence against. + if dregion > 2 { + let thunks_base = ((rtt_addr + 0x30 + 0x1F) & !0x1F) as u32; + let edata = core::ptr::addr_of!(__edata) as u32; + // RLAR's LIMIT is INCLUSIVE and its low 5 bits are forced to 0x1F, so + // the region always ends at `limit | 0x1F` — i.e. it rounds UP to the + // enclosing 32-byte granule. `(edata - 1) & !0x1F` therefore covers + // PAST `__edata` and marks the first bytes of `.bss` read-only, since + // `__edata` is essentially never 32-byte aligned. That is a real fault + // generator, not a theoretical one: with `-Z stack-protector=strong` + // (`__edata = 0x20003ab4`) it made the firmware unbootable — a genuine + // DACCVIOL, MMFAR = 0x20003ab8 = `__sbss` exactly, faulting in an + // ordinary early `atomic_store::` into `.bss`. + // Step back a full granule so the region always ends BELOW `__edata`, + // giving up at most 32 bytes of thunk coverage to stay off `.bss`. + let thunks_limit = edata.saturating_sub(0x20) & !0x1F; + if thunks_limit > thunks_base { + MPU_RNR.write_volatile(2); + MPU_RBAR.write_volatile(thunks_base | 0x1C); // SH=11, AP=10 priv-RO, XN=0 + MPU_RLAR.write_volatile(thunks_limit | 1); + defmt::info!( + "core1 MPU region 2 armed: .data thunks RO [{=u32:#010x}..={=u32:#010x}]", + thunks_base, + thunks_limit + 0x1F + ); + } + } + limit + } else { + defmt::warn!("core1 MPU: <2 regions available — .data protection disabled"); + 0 + }; + cortex_m::asm::dsb(); cortex_m::asm::isb(); @@ -131,5 +285,48 @@ pub unsafe fn setup_core1_mpu() { cortex_m::asm::dsb(); cortex_m::asm::isb(); - defmt::info!("core1 MPU armed: region 0 = [0x20066B60, 0x2007FFFF] priv-RO (Core 0 stack)"); + defmt::info!( + "core1 MPU armed: region 0 = [{=u32:#010x}, 0x2007FFFF] priv-RO (Core 0 stack), \ + region 1 = [0x20000000, {=u32:#010x}] priv-RO+exec (.data RAM code/thunks)", + base, + data_limit + 0x1F + ); +} + +/// Fence CORE 0 out of CORE 1's stack (MPU region 2 on core 0). +/// +/// MUST be called on core 0 and ONLY AFTER `spawn_core1` — core 0 legitimately +/// writes core 1's initial stack frame during the spawn, so arming earlier +/// faults during boot. +/// +/// Why this exists: the MPU already fences core 1 out of core 0's stack, but the +/// symmetric direction was never covered, so core 0 could clobber CORE1_STACK +/// silently. A recovered (never-committed) core-1 crash record showed core 1 +/// taking a precise bus fault in `ApuPeripheral::produce_samples` through a +/// garbage `self` while its `sample_buffer` Vec was intact — i.e. the `worker` +/// reference held on CORE 1'S STACK had been clobbered. If core 0 is the writer, +/// this converts that silent clobber into a DACCVIOL naming the exact storing +/// instruction; if it never fires, core 0 is excluded. +/// +/// XN=1: core 0 never executes from core 1's stack. +#[cfg(target_arch = "arm")] +pub unsafe fn arm_core1_stack_fence() { + const MPU_RNR: *mut u32 = 0xE000_ED98 as *mut u32; + const MPU_RBAR: *mut u32 = 0xE000_ED9C as *mut u32; + const MPU_RLAR: *mut u32 = 0xE000_EDA0 as *mut u32; + const MPU_TYPE: *mut u32 = 0xE000_ED90 as *mut u32; + + let dregion = unsafe { (MPU_TYPE.read_volatile() >> 8) & 0xFF }; + if dregion <= 2 { + defmt::warn!("core0 MPU: <3 regions — CORE1_STACK fence disabled"); + return; + } + unsafe { + MPU_RNR.write_volatile(2); + MPU_RBAR.write_volatile(0x2008_0000 | 0x1D); // SH=11, AP=10 priv-RO, XN=1 + MPU_RLAR.write_volatile(0x2008_1FE0 | 1); + } + cortex_m::asm::dsb(); + cortex_m::asm::isb(); + defmt::info!("core0 MPU region 2 armed: CORE1_STACK RO [0x20080000..=0x20081fff]"); } diff --git a/platform/pico2w/src/sd.rs b/platform/pico2w/src/sd.rs index 99dda4e..f234867 100644 --- a/platform/pico2w/src/sd.rs +++ b/platform/pico2w/src/sd.rs @@ -86,10 +86,19 @@ where /// /// Iterates the full root directory each call (up to 100 entries) and /// slices the requested page. - pub fn list_rom_page( + /// List a page of ROMs, calling `on_progress` for every directory entry + /// visited. + /// + /// The callback exists so the caller can feed the watchdog. This traversal + /// walks the WHOLE root directory (no early exit) and runs synchronously + /// inside a single main-loop iteration, so with a slow card it can outlast + /// the 16 s watchdog window — which is exactly what made entering the ROM + /// list hang and reset with no crash record. + pub fn list_rom_page_with( &self, page_offset: usize, page_size: usize, + mut on_progress: impl FnMut(), ) -> Result> { let mut names: Vec = Vec::new(); names @@ -104,6 +113,9 @@ where let _ = self .mgr .iterate_dir_lfn(root, &mut lfn_buffer, |entry, lfn| { + // Every entry, not just matches: the cost is the traversal, and + // a directory full of non-ROM files is exactly the slow case. + on_progress(); if !entry.attributes.is_directory() && is_rom_file(&entry.name) && names.len() < 100 { let display_name = match lfn.filter(|name| !name.is_empty()) { @@ -138,6 +150,15 @@ where }) } + /// Convenience wrapper for callers with no watchdog to feed. + pub fn list_rom_page( + &self, + page_offset: usize, + page_size: usize, + ) -> Result> { + self.list_rom_page_with(page_offset, page_size, || {}) + } + /// Open a specific ROM file by FAT filename (case-insensitive). /// /// The returned `SdRomReader` holds an exclusive borrow of this manager. diff --git a/platform/pico2w/src/state/main_menu.rs b/platform/pico2w/src/state/main_menu.rs index 08931a9..5dd2b8f 100644 --- a/platform/pico2w/src/state/main_menu.rs +++ b/platform/pico2w/src/state/main_menu.rs @@ -4,6 +4,7 @@ use rustyboy_pico2w::input::InputHandler; use rustyboy_pico2w::menu::{MainMenu, MenuEffect, MenuInput, MenuLogic}; use super::{RomListState, RunningState, SettingsState}; +use embassy_rp::watchdog::Watchdog; use crate::{App, AppState, PicoSdMgr}; pub struct MainMenuState { @@ -25,6 +26,7 @@ impl MainMenuState { game_disp: &mut GameDisplay<'static>, input: &mut InputHandler<'static>, sd_mgr: &mut PicoSdMgr, + watchdog: &mut Watchdog, ) { Timer::after(Duration::from_millis(33)).await; @@ -48,7 +50,7 @@ impl MainMenuState { app.transition_to(AppState::Running(RunningState)); } MenuEffect::ShowRoms => { - let next = RomListState::new(app, game_disp, sd_mgr).await; + let next = RomListState::new(app, game_disp, sd_mgr, watchdog).await; app.transition_to(AppState::RomList(alloc::boxed::Box::new(next))); } MenuEffect::ShowSettings => { diff --git a/platform/pico2w/src/state/rom_list.rs b/platform/pico2w/src/state/rom_list.rs index 6b17194..72faf72 100644 --- a/platform/pico2w/src/state/rom_list.rs +++ b/platform/pico2w/src/state/rom_list.rs @@ -1,3 +1,4 @@ +use embassy_rp::watchdog::Watchdog; use embassy_time::{Duration, Timer}; use rustyboy_pico2w::display::{ hw::GameDisplay, menu_item_needs_marquee, MENU_MARQUEE_REDRAW_FRAMES, @@ -33,8 +34,13 @@ impl RomListState { app: &App, game_disp: &mut GameDisplay<'static>, sd_mgr: &mut PicoSdMgr, + watchdog: &mut Watchdog, ) -> Self { - let (page, has_next, total_roms) = match sd_mgr.list_rom_page(0, ROM_PAGE_SIZE) { + // Feed while scanning: the traversal is synchronous and can outlast the + // watchdog window on a slow card. See `list_rom_page_with`. + let (page, has_next, total_roms) = match sd_mgr.list_rom_page_with(0, ROM_PAGE_SIZE, || { + watchdog.feed(Duration::from_millis(crate::WATCHDOG_WINDOW_MS)); + }) { Ok(result) => (result.entries, result.has_next, result.total), Err(e) => { defmt::error!("SD list failed: {:?}", defmt::Debug2Format(&e)); @@ -149,8 +155,11 @@ impl RomListState { new_offset: usize, sd_mgr: &mut PicoSdMgr, selection: RomPageSelection, + watchdog: &mut Watchdog, ) { - match sd_mgr.list_rom_page(new_offset, ROM_PAGE_SIZE) { + match sd_mgr.list_rom_page_with(new_offset, ROM_PAGE_SIZE, || { + watchdog.feed(Duration::from_millis(crate::WATCHDOG_WINDOW_MS)); + }) { Ok(result) => { let page_len: usize = result.entries.len(); self.page = result.entries; @@ -174,6 +183,7 @@ impl RomListState { game_disp: &mut GameDisplay<'static>, input: &mut InputHandler<'static>, sd_mgr: &mut PicoSdMgr, + watchdog: &mut Watchdog, ) { Timer::after(Duration::from_millis(33)).await; @@ -185,7 +195,7 @@ impl RomListState { let previous_selected = self.logic.selected(); let effect = self.logic.handle(menu_input); - self.apply_effect(effect, previous_selected, app, game_disp, sd_mgr) + self.apply_effect(effect, previous_selected, app, game_disp, sd_mgr, watchdog) .await; } @@ -207,6 +217,7 @@ impl RomListState { app: &mut App, game_disp: &mut GameDisplay<'static>, sd_mgr: &mut PicoSdMgr, + watchdog: &mut Watchdog, ) { match effect { RomListEffect::None => { @@ -218,11 +229,11 @@ impl RomListState { self.transition_to_selected_rom(app); } RomListEffect::NextPage => { - self.flip_requested_page(RomListEffect::NextPage, sd_mgr, app, game_disp) + self.flip_requested_page(RomListEffect::NextPage, sd_mgr, app, game_disp, watchdog) .await; } RomListEffect::PrevPage => { - self.flip_requested_page(RomListEffect::PrevPage, sd_mgr, app, game_disp) + self.flip_requested_page(RomListEffect::PrevPage, sd_mgr, app, game_disp, watchdog) .await; } RomListEffect::Back => { @@ -280,13 +291,15 @@ impl RomListState { sd_mgr: &mut PicoSdMgr, app: &App, game_disp: &mut GameDisplay<'static>, + watchdog: &mut Watchdog, ) { self.marquee_frame = 0; let Some(page) = self.page_context().request(effect) else { return; }; - self.flip_page(page.offset, sd_mgr, page.selection).await; + self.flip_page(page.offset, sd_mgr, page.selection, watchdog) + .await; self.draw(app, game_disp).await; } } diff --git a/tools/crash_decoder.py b/tools/crash_decoder.py index 51e063f..1f6f86b 100755 --- a/tools/crash_decoder.py +++ b/tools/crash_decoder.py @@ -141,7 +141,9 @@ "12s" # panic_loc "H" # panic_line "H" # stack_headroom - "36x" # _reserved [84..120] + "I" + "2I" + "6I" "I" # crc32 [120..124] "4x" # _pad4 [124..128] ) @@ -204,8 +206,18 @@ def __init__(self, raw: bytes, slot_index: int): self.panic_loc_raw, self.panic_line, self.stack_headroom, + self.dma_busy_mask, + _dma0, _dma1, + _s0, _s1, _s2, _s3, _s4, _s5, self.crc32_stored, ) = fields + # schema v2 reclaimed DMA ch2-6 (proven permanently zero on this part) + # for a stack window. v1 records carry DMA data there, so gate on the + # schema byte rather than trusting the bytes. + self.dma_write_addrs = [_dma0, _dma1] + self.stack_snapshot = ( + [_s0, _s1, _s2, _s3, _s4, _s5] if self.schema_ver >= 2 else [] + ) self.fw_version = tuple(fw) self.slot_index = slot_index @@ -336,6 +348,12 @@ def to_dict(self) -> dict: "r4": f"0x{self.ext_r4:08x}", "r12": f"0x{self.ext_r12:08x}", } if self.has_hardfault_ext_regs else None, + "stack_snapshot": [f"0x{w:08x}" for w in self.stack_snapshot] + if self.stack_snapshot + else None, + "dma_write_addrs": [f"0x{w:08x}" for w in self.dma_write_addrs] + if self.dma_busy_mask or any(self.dma_write_addrs) + else None, "stack": { "headroom_bytes": self.stack_headroom, "overflowed": self.has_stack_overflow, @@ -442,7 +460,7 @@ def symbolize_record(record: CrashRecord, elf_path: Optional[str]) -> dict[str, syms: dict[str, str] = {} if record.has_arm_regs and record.arm_pc: syms["pc"] = _symbolize(record.arm_pc, elf_path) - if record.has_arm_regs and record.arm_lr: + if (record.has_arm_regs or record.has_stack_chk_fail_lr) and record.arm_lr: syms["lr"] = _symbolize(record.arm_lr, elf_path) if record.has_hardfault_ext_regs: if record.ext_r4: @@ -753,6 +771,18 @@ def print_report(header: Optional[SectorHeader], records: list[CrashRecord], if rec.has_panic_loc and rec.panic_loc: console.print(f" [bold]Panic [/] [yellow]{rec.panic_loc}:{rec.panic_line}[/]") + # -Z stack-protector=strong: arm_lr holds __stack_chk_fail's captured + # LR (the return address into the guarded function whose canary was + # smashed). For a pure Panic record (no exception frame) this is the + # only place that LR appears; skip when has_arm_regs already printed + # it above (the HardFault fallback path reuses the same ARM LR line). + if rec.has_stack_chk_fail_lr and rec.arm_lr and not rec.has_arm_regs: + lr_sym = syms.get("lr", "") + console.print( + f" [bold]Canary [/] [red]overrun in caller of[/] [cyan]0x{rec.arm_lr:08x}[/]" + + (f" → {lr_sym}" if lr_sym else "") + ) + # ROM info if rec.has_rom_info: console.print( diff --git a/tools/fixtures/test_crash.bin b/tools/fixtures/test_crash.bin index bc778c6c568d5d69581b1160b656c4cdeca67dc1..131c1b8219c111e62aed328dbad7495edf76a396 100644 GIT binary patch delta 59 zcmZorXi%8Y!pJnSwOFF`+hS`5AP8~}3SeU7V`N}tV7PZ+?Y)VD5|cF;12~do4lV%6 MZ*F9)=VSa200!z2Bme*a delta 59 zcmZorXi%8Y!pJzWwOGP8mfMm62!fo00vH+j7#SED815ZddvBtk#AFS|0FJ2bnTtU3 Mn;RMH`56BL0L(xT1ONa4 diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 27b15fe..b96db2e 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -3,6 +3,11 @@ name = "xtask" version = "0.1.0" edition = "2021" publish = false +# `src/bin/rb-flash.rs` is auto-discovered as a second binary alongside the +# explicit `xtask` bin below; without this, `cargo run -p xtask` (and the +# `cargo xtask` alias in the workspace .cargo/config.toml) can't tell which +# one to run. +default-run = "xtask" [[bin]] name = "xtask" From 8ce3d1ac639197b88169d1b5f78adec5dbb790f0 Mon Sep 17 00:00:00 2001 From: Vincent Bonduro Date: Wed, 2 Sep 2026 21:12:19 -0400 Subject: [PATCH 2/4] =?UTF-8?q?pico2w:=20fix=20review=20findings=20?= =?UTF-8?q?=E2=80=94=20stack-snapshot=20base,=20SD=20error=20classing,=20c?= =?UTF-8?q?lock=20profiles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses three blocking defects found in review of the previous commit, plus the cleanup items. BLOCKING — the stack snapshot recorded the wrong bytes on every fault -------------------------------------------------------------------- `commit_crash_and_reset` receives `sp_before_exception()`, which has ALREADY added the frame size (32, or 104 for an FP-extended frame) and the STKALIGN pad. The capture then added them a second time, so: * the "stacked xPSR" probe at `sp + 0x1c` read an unrelated stack word, making the +4 pad adjustment effectively random; * the window landed 32 bytes past the return-address slot it exists to capture — the feature never recorded what it was built for. Fixed by passing `sp` through unchanged, which also removes an unguarded dereference that ran inside the fault handler before any state was committed. Renamed to `capture_stack_window` and corrected the bounds-check doc: the guard clamps to physical SRAM, it does not verify the address is a stack. BLOCKING — a corrupt save state destroyed the battery save ---------------------------------------------------------- `read_save_state` returns `Err` both when the card is absent AND when a save-state file is present but fails validation (`InvalidValue`), or on `OutOfMemory`. Classing every `Err` as "card absent" skipped the battery-save read on a HEALTHY card, booted with blank cart RAM, and let the first flush overwrite a good `.sav`. Added `SdError::is_card_absent()`, true only for transport failures (`DeviceError`/`NoSuchVolume`), and used it for the availability flag. BLOCKING — the retry bound was narrower than documented ------------------------------------------------------- `acquire_retries` bounds ONLY the CMD0 enter-SPI-mode loop; CMD8 and ACMD41 use the crate's hardcoded 10_000-iteration budget. So the ~2.4 s figure holds for a fully absent card, but a partially responsive one can still take ~7-8 s — inside the 16 s window with ~2x margin, not the 6x the comment implied. Documented honestly rather than silently relied upon. Other fixes ----------- * clock profiles: `TARGET_SYS_HZ` and `TARGET_CORE_VOLTAGE` were two cfg cascades keyed on different feature sets and had already drifted — `oc-266` selected 266 MHz but fell through to V1_30, NOT reproducing the "266 @ V1_25" point it is named for. Collapsed to one `ClockProfile` table so the two cannot desync. Added a static assert that the watchdog window fits the RP2350's 24-bit microsecond counter. * SD mode pin (GP17) is now driven BEFORE the power-up clocks; it was set 31 lines after the first bus traffic this commit series introduced. * removed `arm_core1_stack_fence()` (zero call sites) and the 25-line comment describing a region it never armed; removed the unused `list_rom_page`. * de-duplicated the MPU thunk-region arithmetic into `data_thunk_bounds()` / `arm_ro_exec_region()`. Verified on hardware: both cores now report identical bounds, which is the property the comment previously only asserted. * `stack_snapshot`: named indices, a static assert backing the padding claim, and the documented limitation that a hardware watchdog reset has no capture point (those records carry schema v2 with an all-zero window). * decoder reports an uncaptured window as absent rather than six zeros. * the capture base is logged instead of discarded, so the window has an anchor. * apu: extracted `trusted_capacity()`; trimmed dated incident narrative. Tests: 649 core + 194 pico2w. Three new ones cover the format change that had none — v1 records must not reinterpret DMA bytes as a stack snapshot, the snapshot must survive a round trip with distinct words, and the persisted byte offsets are pinned absolutely. Verified on hardware at 288 MHz: mounts, loads a save state, main loop at 3.8 s. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Ab4PqF52wyVpp5S2Sg6RGK --- .gitignore | 4 + core/src/cpu/peripheral/apu.rs | 58 +++----- platform/pico2w/src/crash/handler.rs | 18 +-- platform/pico2w/src/crash/mod.rs | 59 ++++++++ platform/pico2w/src/crash/record.rs | 1 - platform/pico2w/src/crash/stack_snapshot.rs | 64 ++++++--- platform/pico2w/src/crash/storage.rs | 6 +- platform/pico2w/src/main.rs | 145 +++++++++++++------- platform/pico2w/src/mpu.rs | 137 ++++++++---------- platform/pico2w/src/sd.rs | 29 ++-- tools/crash_decoder.py | 11 +- 11 files changed, 314 insertions(+), 218 deletions(-) diff --git a/.gitignore b/.gitignore index 338e6f2..18e4730 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ Madagascar* ferris.png 331269-* index.html?* + +# Python bytecode +__pycache__/ +*.pyc diff --git a/core/src/cpu/peripheral/apu.rs b/core/src/cpu/peripheral/apu.rs index d0b351d..6a9a7c0 100644 --- a/core/src/cpu/peripheral/apu.rs +++ b/core/src/cpu/peripheral/apu.rs @@ -875,6 +875,16 @@ impl ApuPeripheral { self.channel4.advance_frequency_noise(cycles); } + /// Capacity we are willing to trust. Never the in-memory `cap` word alone: + /// `Vec::push` grows only when `len == cap`, so a `cap` corrupted UPWARD is + /// never equal to `len` and every push then runs past the real allocation. + /// `cap.min(HINT) <= cap` unconditionally, so this can never drop samples + /// from a correctly sized buffer. + #[inline(always)] + fn trusted_capacity(&self) -> usize { + self.sample_buffer.capacity().min(SAMPLE_BUFFER_CAPACITY_HINT) + } + /// Downsample to 48 kHz and push stereo PCM pairs into `sample_buffer`. /// /// The Pico runtime almost always calls this with `cycles` <= 4, so a @@ -886,13 +896,8 @@ impl ApuPeripheral { let sample_inc = cycles as u32 * SAMPLE_PERIOD_DEN; if cycles <= 4 { self.sample_acc += sample_inc; - // Bound by the compile-time constant, NOT the in-memory `cap` word: - // a `cap` corrupted upward makes this check pass and `Vec::push` - // then writes past the real allocation (push only grows when - // `len == cap`). See the long note in the `else` branch below. if self.sample_acc >= SAMPLE_PERIOD_NUM - && self.sample_buffer.len() + 2 - <= self.sample_buffer.capacity().min(SAMPLE_BUFFER_CAPACITY_HINT) + && self.sample_buffer.len() + 2 <= self.trusted_capacity() { self.sample_acc -= SAMPLE_PERIOD_NUM; let (left, right) = self.mix_sample(); @@ -906,40 +911,13 @@ impl ApuPeripheral { let n_samples = acc / SAMPLE_PERIOD_NUM as u64; self.sample_acc = (acc % SAMPLE_PERIOD_NUM as u64) as u32; if n_samples != 0 { - let cap = self.sample_buffer.capacity(); - // `saturating_sub`, NOT `-`: if `len > cap` the plain subtraction - // underflows (usize) to a huge value, `pushable` stops being - // clamped by capacity, and the loop below pushes past the end of - // the heap buffer. `Vec::push` does not save us — it only grows - // when `len == cap`, so a corrupted `cap` (observed live on device - // 2026-08-03: cap 2048 -> 0 while len was 338) is never equal to - // len and every subsequent push writes at `ptr + len*2`, running - // off the 4096-byte allocation into adjacent heap objects. That - // turns a single corrupted word into an unbounded heap overrun — - // the amplifier behind the wild-pointer crashes. Saturating keeps - // the overrun impossible regardless of how `cap` got corrupted. - // - // 2026-08-08: `cap` too SMALL was the case handled above, but - // `cap` too LARGE is the dangerous direction and was NOT - // covered. `Vec::push` only reallocates when `len == cap`, so a - // `cap` corrupted upward is never equal to `len` and every push - // writes at `ptr + len*2` straight off the end of the real - // 4096-byte allocation — an UNBOUNDED spray of i16 samples - // through whatever follows it. - // - // That is not hypothetical: the wild pointer values in the - // crash records decode as stereo i16 sample pairs — - // 0x00ffff01 = (-255, +255), 0x0c160015 = (21, 3094), - // 0xed6ef504 = (-2812, -4754) — the same magnitudes as the live - // AUDIO_QUEUE contents (1638, 546, 0). Audio data is landing in - // pointer slots on BOTH cores' stacks, which is this spray. - // - // So clamp against the compile-time capacity, never against the - // in-memory `cap` word. `with_capacity` may over-allocate, so - // the constant is always <= the true allocation and therefore - // safe no matter how `cap` is corrupted. - let trusted_cap = cap.min(SAMPLE_BUFFER_CAPACITY_HINT); - let pushable = (trusted_cap.saturating_sub(self.sample_buffer.len()) / 2) + // `saturating_sub`, NOT `-`: if `len > cap` the plain + // subtraction underflows and `pushable` stops being bounded by + // capacity, spraying i16 samples past the allocation. + let pushable = (self + .trusted_capacity() + .saturating_sub(self.sample_buffer.len()) + / 2) .min(n_samples as usize); for _ in 0..pushable { let (left, right) = self.mix_sample(); diff --git a/platform/pico2w/src/crash/handler.rs b/platform/pico2w/src/crash/handler.rs index 943b5b9..f12b121 100644 --- a/platform/pico2w/src/crash/handler.rs +++ b/platform/pico2w/src/crash/handler.rs @@ -313,19 +313,15 @@ fn commit_crash_and_reset( // boot — so the window goes to `.uninit`, which survives the reset the same // way the scratch registers do. // - // For a HardFault `sp` is MSP with the 8-word exception frame already - // pushed, so the interesting region starts above it; the +4 accounts for the - // STKALIGN pad, which the stacked xPSR bit 9 reports. A panic has no - // exception frame, so read from `sp` directly. + // `sp` is ALREADY the pre-fault stack pointer: `hard_fault_rust` passes + // `sp_before_exception()`, which has itself added the frame size (32 or 104 + // for an FP-extended frame) and the STKALIGN pad. Do not re-add them — an + // earlier version did, which pushed the window 32 bytes past the return- + // address slot it exists to capture, and made its own xPSR probe read an + // unrelated word. #[cfg(target_arch = "arm")] unsafe { - let base = if matches!(kind, CrashKind::HardFault) { - let stacked_xpsr = core::ptr::read_volatile((sp + 0x1c) as *const u32); - sp + 0x20 + if stacked_xpsr & (1 << 9) != 0 { 4 } else { 0 } - } else { - sp - }; - crate::crash::stack_snapshot::capture_crash_stack(base); + crate::crash::stack_snapshot::capture_stack_window(sp); } write_watchdog_scratch( diff --git a/platform/pico2w/src/crash/mod.rs b/platform/pico2w/src/crash/mod.rs index cc54be6..0fbbbe6 100644 --- a/platform/pico2w/src/crash/mod.rs +++ b/platform/pico2w/src/crash/mod.rs @@ -155,6 +155,65 @@ mod tests { assert_eq!(r.to_bytes().len(), RECORD_SIZE); } + /// A v1 record's bytes [96..120] held DMA channels 2-6. Schema v2 reuses that + /// range for the stack snapshot. A v1 record already on flash must therefore + /// decode with an EMPTY snapshot, not with stale DMA addresses reinterpreted + /// as stack words — this is the one compatibility invariant of the format + /// change, and nothing else covers it. + #[test] + fn v1_record_decodes_with_empty_stack_snapshot() { + let mut buf = sample_record(3).to_bytes(); + // Rewrite as schema v1 with plausible v1 payload in the reclaimed range. + buf[4] = 1; + for (i, b) in buf[96..120].iter_mut().enumerate() { + *b = 0xA0u8.wrapping_add(i as u8); + } + let crc = crc32(&buf[..120]); + buf[120..124].copy_from_slice(&crc.to_le_bytes()); + + let rec = CrashRecord::from_bytes(&buf).expect("v1 record must still decode"); + assert_eq!(rec.schema_ver, 1); + assert_eq!( + rec.stack_snapshot, [0u32; 6], + "v1 DMA bytes must not be reinterpreted as a stack snapshot" + ); + } + + /// The stack snapshot must survive a round trip with distinct non-zero words. + /// `sample_record` leaves it zeroed via `..Default::default()`, so without + /// this an offset bug in `to_bytes`/`from_bytes` would be invisible. + #[test] + fn stack_snapshot_roundtrips() { + let mut rec = sample_record(4); + rec.stack_snapshot = [0x1111_1111, 0x2222_2222, 0x3333_3333, 0x4444_4444, 0x5555_5555, 0x6666_6666]; + rec.dma_write_addrs = [0x5020_0010, 0x4008_8008]; + let back = CrashRecord::from_bytes(&rec.to_bytes()).expect("roundtrip"); + assert_eq!(back.stack_snapshot, rec.stack_snapshot); + assert_eq!(back.dma_write_addrs, rec.dma_write_addrs); + } + + /// Pin the ABSOLUTE byte offsets. `record_roundtrip` passes even if a field + /// moved, because both sides move together — but this is a PERSISTED format + /// read back from flash written by older firmware. + #[test] + fn persisted_field_offsets_are_stable() { + let mut rec = sample_record(5); + rec.dma_write_addrs = [0xAAAA_AAAA, 0xBBBB_BBBB]; + rec.stack_snapshot = [0xC0DE_0000, 0xC0DE_0001, 0xC0DE_0002, 0xC0DE_0003, 0xC0DE_0004, 0xC0DE_0005]; + let b = rec.to_bytes(); + assert_eq!(&b[88..92], &0xAAAA_AAAAu32.to_le_bytes(), "dma ch0 @ [88..92]"); + assert_eq!(&b[92..96], &0xBBBB_BBBBu32.to_le_bytes(), "dma ch1 @ [92..96]"); + for i in 0..6 { + let off = 96 + i * 4; + assert_eq!( + &b[off..off + 4], + &rec.stack_snapshot[i].to_le_bytes(), + "stack_snapshot[{i}] @ [{off}..]" + ); + } + assert_eq!(b[4], 2, "producers must stamp schema_ver = 2"); + } + #[test] fn record_roundtrip() { let original = sample_record(3); diff --git a/platform/pico2w/src/crash/record.rs b/platform/pico2w/src/crash/record.rs index 26e4446..444945f 100644 --- a/platform/pico2w/src/crash/record.rs +++ b/platform/pico2w/src/crash/record.rs @@ -246,7 +246,6 @@ impl CrashRecord { let off = 88 + i * 4; buf[off..off + 4].copy_from_slice(&addr.to_le_bytes()); } - // buf[116..120] = 0 (reserved) // buf[124..128] = 0 (pad) let checksum = crc32(&buf[..120]); diff --git a/platform/pico2w/src/crash/stack_snapshot.rs b/platform/pico2w/src/crash/stack_snapshot.rs index dc5f408..9ac7b7d 100644 --- a/platform/pico2w/src/crash/stack_snapshot.rs +++ b/platform/pico2w/src/crash/stack_snapshot.rs @@ -8,40 +8,66 @@ //! //! For a wild-PC fault out of a `pop {rlist, pc}` this window spans the return //! slot, which is what makes such a record diagnosable at all. +//! +//! LIMITATION: populated only for faults that reach a handler. A hardware +//! WATCHDOG reset has no capture point, so `WatchdogTimeout` records carry +//! `schema_ver = 2` with an all-zero window. The decoder cannot currently +//! distinguish "captured, all zeros" from "never captured". -/// `.uninit` backing store. Padded well clear of `_stack_end` so its tail cannot -/// land inside core 1's read-only MPU region (see `heartbeat` for the same -/// hazard, which silently killed core 1 when it was first hit). +/// `.uninit` backing store: 8 words used, 8 words of deliberate PADDING. +/// +/// Core 1's MPU region 0 base is `_stack_end & !0x1F`, so up to 31 bytes of the +/// LAST `.uninit` object are privileged read-only for core 1. If that object is +/// written by core 1 it takes a MemManage fault that escalates to HardFault and +/// kills the core silently. The trailing 8 words absorb that if this array +/// happens to be placed last. The static assert below keeps the padding from +/// being "optimised" away by a future edit. #[no_mangle] #[link_section = ".uninit.CRASH_STACK_SNAP"] pub static mut CRASH_STACK_SNAP: [usize; 16] = [0; 16]; -const CRASH_SNAP_BASE: usize = 0; +/// Physical SRAM bounds on RP2350; reads are clamped to this range. +const SRAM_LO: usize = 0x2000_0000; +const SRAM_HI: usize = 0x2008_2000; + +/// Index layout within the array. +const IDX_MAGIC: usize = 0; +const IDX_BASE: usize = 1; +const IDX_WORDS: usize = 2; + +/// The padding described above must always exceed the 32-byte MPU granule. +const _: () = assert!( + (16 - (IDX_WORDS + CRASH_SNAP_WORDS)) * core::mem::size_of::() >= 32 +); const CRASH_SNAP_MAGIC: usize = 0x5CAF_0001; /// Number of stack words carried in a crash record. Must match /// `CrashRecord::stack_snapshot`. pub const CRASH_SNAP_WORDS: usize = 6; -/// Capture `CRASH_SNAP_WORDS` stack words starting at `base` into `.uninit`, -/// for the next boot to fold into the crash record. +/// Capture `CRASH_SNAP_WORDS` stack words starting at `base` (the PRE-FAULT +/// stack pointer) into `.uninit`, for the next boot to fold into the crash +/// record. The window therefore begins at the return-address slot, which is +/// what makes a wild-PC fault out of a `pop {rlist, pc}` diagnosable. /// /// # Safety /// Called from a fault/panic handler with interrupts effectively quiesced. /// `base` is range-checked against the core-0 stack before any dereference, so /// a wild SP cannot turn this into an out-of-bounds read. -pub unsafe fn capture_crash_stack(base: usize) { +pub unsafe fn capture_stack_window(base: usize) { let f = &raw mut CRASH_STACK_SNAP; - (*f)[CRASH_SNAP_BASE] = CRASH_SNAP_MAGIC; - (*f)[CRASH_SNAP_BASE + 1] = base; - // Only read what is provably inside the core-0 stack. A drifted or wild SP - // is exactly the condition this instrument exists to record, so it must not - // fault while recording it. - let lo = 0x2000_0000usize; - let hi = 0x2008_2000usize; + (*f)[IDX_MAGIC] = CRASH_SNAP_MAGIC; + (*f)[IDX_BASE] = base; + // Clamp to physical SRAM. This is a "will not fault" check, NOT an + // assertion that `base` is a valid stack address — a drifted or wild SP is + // exactly the condition this instrument exists to record, so out-of-range + // words are reported as zero rather than being allowed to fault a second + // time inside the handler. + let lo = SRAM_LO; + let hi = SRAM_HI; for i in 0..CRASH_SNAP_WORDS { let a = base.wrapping_add(i * 4); - (*f)[CRASH_SNAP_BASE + 2 + i] = if a >= lo && a + 4 <= hi && a % 4 == 0 { + (*f)[IDX_WORDS + i] = if a >= lo && a + 4 <= hi && a % 4 == 0 { core::ptr::read_volatile(a as *const usize) } else { 0 @@ -53,13 +79,13 @@ pub unsafe fn capture_crash_stack(base: usize) { /// Returns `None` if no snapshot was recorded. pub unsafe fn last_crash_stack() -> Option<(usize, [u32; CRASH_SNAP_WORDS])> { let f = &raw const CRASH_STACK_SNAP; - if (*f)[CRASH_SNAP_BASE] != CRASH_SNAP_MAGIC { + if (*f)[IDX_MAGIC] != CRASH_SNAP_MAGIC { return None; } - let base = (*f)[CRASH_SNAP_BASE + 1]; + let base = (*f)[IDX_BASE]; let mut w = [0u32; CRASH_SNAP_WORDS]; for (i, out) in w.iter_mut().enumerate() { - *out = (*f)[CRASH_SNAP_BASE + 2 + i] as u32; + *out = (*f)[IDX_WORDS + i] as u32; } Some((base, w)) } @@ -70,5 +96,5 @@ pub unsafe fn last_crash_stack() -> Option<(usize, [u32; CRASH_SNAP_WORDS])> { /// Call once at boot, after `last_crash_stack()` has been consumed. pub unsafe fn clear_crash_stack() { let f = &raw mut CRASH_STACK_SNAP; - (*f)[CRASH_SNAP_BASE] = 0; + (*f)[IDX_MAGIC] = 0; } diff --git a/platform/pico2w/src/crash/storage.rs b/platform/pico2w/src/crash/storage.rs index a5dae87..d28d8ea 100644 --- a/platform/pico2w/src/crash/storage.rs +++ b/platform/pico2w/src/crash/storage.rs @@ -66,7 +66,11 @@ pub fn check_and_commit(flash: &mut OnboardFlash<'_>) -> bool { // the reset. `.uninit` carries it across the reset the same way the // watchdog scratch registers carry the register state. unsafe { - if let Some((_base, words)) = crate::crash::stack_snapshot::last_crash_stack() { + if let Some((base, words)) = crate::crash::stack_snapshot::last_crash_stack() { + // The record has no field for the base address, so log it: six + // hex words with no anchor cannot be located in a stack dump, + // which for an SP-offset measurement is the whole point. + defmt::info!("crash: stack window @ {=usize:#010x}", base); rec.stack_snapshot = words; crate::crash::stack_snapshot::clear_crash_stack(); } diff --git a/platform/pico2w/src/main.rs b/platform/pico2w/src/main.rs index 2f8d74a..4a867d9 100644 --- a/platform/pico2w/src/main.rs +++ b/platform/pico2w/src/main.rs @@ -55,47 +55,68 @@ use rustyboy_pico2w::save_storage::{boot_load_saves, BootSaves, SaveSlot}; use rustyboy_pico2w::sd::{self, DummyClock}; use rustyboy_pico2w::xip_cartridge::XipCartridge; -#[cfg(feature = "oc-300")] -const TARGET_SYS_HZ: u32 = 300_000_000; -// 16 s gives worst-case frame/flash-sync stalls plenty of headroom without false-tripping; -// the multicore livelock that caused the old 5 s freeze→reset is separately fixed by WFE backpressure. -const WATCHDOG_WINDOW_MS: u64 = 16_000; -#[cfg(all(not(feature = "oc-300"), feature = "oc-288"))] -// 288 MHz = PLL FBDIV 120 (VCO 1440) / POSTDIV 5 — one grid step below 300 on -// the stock PLL config, and the fastest clock measured free of the SP-drift -// fault. A frequency sweep (live PLL reprogramming, no rebuild) put the failure -// cliff between 288 and 300: at 300 MHz the drift instrument logged 67.5 events -// per 1000 work-ticks, while 288/276/264/252/240 logged ZERO across 13,567 -// ticks. A 24 h soak at 288 on the real firmware logged 0 crashes. +// 16 s gives worst-case frame/flash-sync stalls plenty of headroom without +// false-tripping; the multicore livelock that caused the old 5 s freeze→reset is +// separately fixed by WFE backpressure. // -// NOT proven safe: the cliff is bracketed only to (288, 300], so 288 may have as -// little as 0.3% margin, and that is untested against temperature and -// part-to-part spread. 266 is the more conservative shipping point. -// See docs/investigations/sp-drift-root-cause.md. -const TARGET_SYS_HZ: u32 = 288_000_000; -#[cfg(all(not(feature = "oc-300"), not(feature = "oc-288"), feature = "oc-280"))] -const TARGET_SYS_HZ: u32 = 280_000_000; -#[cfg(all( - not(feature = "oc-300"), - not(feature = "oc-288"), - not(feature = "oc-280"), - feature = "oc-266" -))] -const TARGET_SYS_HZ: u32 = 266_000_000; -#[cfg(all( - not(feature = "oc-300"), - not(feature = "oc-288"), - not(feature = "oc-280"), - not(feature = "oc-266") -))] -const TARGET_SYS_HZ: u32 = 300_000_000; - -#[cfg(feature = "oc-300")] -const TARGET_CORE_VOLTAGE: embassy_rp::clocks::CoreVoltage = embassy_rp::clocks::CoreVoltage::V1_30; -#[cfg(feature = "oc-280")] -const TARGET_CORE_VOLTAGE: embassy_rp::clocks::CoreVoltage = embassy_rp::clocks::CoreVoltage::V1_25; -#[cfg(all(not(feature = "oc-300"), not(feature = "oc-280")))] -const TARGET_CORE_VOLTAGE: embassy_rp::clocks::CoreVoltage = embassy_rp::clocks::CoreVoltage::V1_30; +// The RP2350 WATCHDOG.LOAD counter is 24-bit MICROSECONDS, so the largest window +// the hardware can express is 16.777 s. The assert below keeps that a build +// error rather than an embassy panic at boot. +const WATCHDOG_WINDOW_MS: u64 = 16_000; +const WATCHDOG_MAX_WINDOW_MS: u64 = (1 << 24) / 1_000; +const _: () = assert!(WATCHDOG_WINDOW_MS <= WATCHDOG_MAX_WINDOW_MS); + +/// A clock operating point: frequency and the core voltage it was validated at. +/// +/// Frequency and voltage are two halves of ONE decision, so they live in one +/// place. They were previously two independent `#[cfg]` cascades keyed on +/// overlapping-but-different feature sets, which had already drifted: `oc-266` +/// selected 266 MHz but fell through to V1_30, silently *not* reproducing the +/// "266 MHz @ V1_25" operating point it is named for. +struct ClockProfile { + hz: u32, + voltage: embassy_rp::clocks::CoreVoltage, +} + +use embassy_rp::clocks::CoreVoltage; + +/// Stock overclock. NOT free of the SP-drift fault — see `OC_288`. +const OC_300: ClockProfile = ClockProfile { hz: 300_000_000, voltage: CoreVoltage::V1_30 }; + +/// 288 MHz = PLL FBDIV 120 (VCO 1440) / POSTDIV 5 — one grid step below 300 on +/// the stock PLL config, and the fastest clock measured free of the SP-drift +/// fault. A frequency sweep by live PLL reprogramming put the failure cliff +/// between 288 and 300: at 300 the drift instrument logged 67.5 events per 1000 +/// work-ticks, while 288/276/264/252/240 logged zero across 13,567 ticks. A 24 h +/// soak at 288 on the real firmware logged 0 crashes. +/// +/// NOT proven safe: the cliff is bracketed only to (288, 300], so 288 may have +/// as little as 0.3% margin, untested against temperature and part-to-part +/// spread. `OC_266` is the more conservative shipping point. +/// See docs/investigations/sp-drift-root-cause.md. +const OC_288: ClockProfile = ClockProfile { hz: 288_000_000, voltage: CoreVoltage::V1_30 }; + +const OC_280: ClockProfile = ClockProfile { hz: 280_000_000, voltage: CoreVoltage::V1_25 }; + +/// 9 h clean on record at V1_25 — the voltage is part of that result. +const OC_266: ClockProfile = ClockProfile { hz: 266_000_000, voltage: CoreVoltage::V1_25 }; + +/// Exactly one profile, chosen in priority order. Adding a profile is one const +/// plus one arm; frequency and voltage cannot desync. +const CLOCK: ClockProfile = if cfg!(feature = "oc-300") { + OC_300 +} else if cfg!(feature = "oc-288") { + OC_288 +} else if cfg!(feature = "oc-280") { + OC_280 +} else if cfg!(feature = "oc-266") { + OC_266 +} else { + OC_300 +}; + +const TARGET_SYS_HZ: u32 = CLOCK.hz; +const TARGET_CORE_VOLTAGE: CoreVoltage = CLOCK.voltage; const FIRMWARE_VERSION: &str = env!("CARGO_PKG_VERSION"); pub(crate) const CYCLES_PER_FRAME: u64 = 70_224; @@ -357,6 +378,12 @@ async fn main(spawner: Spawner) { ); // SD card — always initialised so the ROM list is available. + // Drive the SD mode pin BEFORE any bus traffic. Nothing clocked SPI0 before + // this commit (SdCard construction is lazy), so its position did not matter; + // the power-up clocks below are the first transfer, and they must not be + // sent while this pin is still in its reset state. + let _sd_mode = Output::new(p.PIN_17, Level::High); + let mut spi_cfg = spi::Config::default(); spi_cfg.frequency = 400_000; let mut spi_bus = Spi::new_blocking(p.SPI0, p.PIN_6, p.PIN_7, p.PIN_4, spi_cfg); @@ -374,14 +401,23 @@ async fn main(spawner: Spawner) { } let spi_dev = ExclusiveDevice::new(spi_bus, cs, Delay); - // Bounded card acquisition. With no card each attempt costs ~340 ms (a CMD0 + // Bounded card acquisition. With NO card each attempt costs ~340 ms (a CMD0 // that times out, 255 flush bytes at 400 kHz, and a delay), so the crate // default of 50 retries takes ~17.1 s to return `CardNotFound`. The watchdog // window is 16 s and the RP2350 counter is 24-bit MICROSECONDS, capping the // window at 16.777 s — so a no-card boot could never fit and reset the board - // mid-init, every boot, producing only `WatchdogTimeout` records. 6 retries - // puts a no-card failure at ~2.4 s; a card that is present answers CMD0 - // within the first few attempts (measured: 781 ms to a successful mount). + // mid-init, every boot, producing only `WatchdogTimeout` records. + // + // SCOPE OF THIS BOUND: `acquire_retries` bounds ONLY the CMD0 enter-SPI-mode + // loop. The CMD8 and ACMD41 loops that follow use the crate's hardcoded + // `DEFAULT_COMMAND_RETRIES` (10_000) and are NOT affected. So this caps the + // fully-absent-card case at ~2.4 s (measured), but a PARTIALLY responsive + // card — one that answers CMD0 and then stalls — can still take ~7-8 s. + // That fits the window with roughly 2x margin, not 6x. Bounding that case + // properly would need a wall-clock timeout around the whole call. + // + // A card that is present answers CMD0 in the first few attempts: measured + // 781 ms to a successful mount on this board. let sd_opts = embedded_sdmmc::sdcard::AcquireOpts { acquire_retries: 6, ..Default::default() @@ -399,7 +435,6 @@ async fn main(spawner: Spawner) { // Check whether a ROM is already staged in flash. let staged = probe_staged_rom(&mut onboard_flash); - let _sd_mode = Output::new(p.PIN_17, Level::High); let Pio { mut common, sm0, .. } = Pio::new(p.PIO0, Irqs); @@ -474,9 +509,15 @@ async fn main(spawner: Spawner) { // One attempt is enough to know. If the card did not // answer, every further read costs another full // acquisition timeout and can only fail the same way. - let sd_alive = save_state_res.is_ok(); - if !sd_alive { - sd_available = false; + // Only a transport failure means the card is absent. A + // corrupt save-state file, or a transient allocation + // failure, is an error from a HEALTHY card and must not + // suppress the battery-save read below — doing so boots + // with blank cart RAM and then overwrites a good `.sav`. + if let Err(e) = save_state_res.as_ref() { + if e.is_card_absent() { + sd_available = false; + } } let save_state_blob = save_state_res.unwrap_or_else(|e| { defmt::warn!( @@ -487,7 +528,7 @@ async fn main(spawner: Spawner) { }); watchdog.feed(Duration::from_millis(WATCHDOG_WINDOW_MS)); let has_save_state = save_state_blob.is_some(); - let battery_data = if has_save_state || !sd_alive { + let battery_data = if has_save_state || !sd_available { None // save state includes cart RAM; skip battery read } else { watchdog.feed(Duration::from_millis(WATCHDOG_WINDOW_MS)); @@ -602,10 +643,12 @@ async fn main(spawner: Spawner) { }; // Runs after the watchdog is armed and before the main loop's first feed, so // on a dead card its acquisition timeout would itself trip the watchdog. + // Skipped when the card did not respond: this runs after the watchdog is + // armed and before the main loop's first feed, so its acquisition timeout + // would itself trip the watchdog. `app.save_slot_available` is already + // false from construction. if sd_available { refresh_save_slot_available(&mut app, &sd_mgr); - } else { - app.save_slot_available = false; } #[cfg(feature = "fps")] diff --git a/platform/pico2w/src/mpu.rs b/platform/pico2w/src/mpu.rs index 8d568c6..f645bfa 100644 --- a/platform/pico2w/src/mpu.rs +++ b/platform/pico2w/src/mpu.rs @@ -16,6 +16,51 @@ /// function cannot silently leave the end of `.data` writable. #[cfg(target_arch = "arm")] #[inline(never)] + +/// Bounds of the RAM-resident `.data` thunk/code region, as an MPU (base, limit) +/// pair — or `None` if the range is empty. +/// +/// Start above the SEGGER RTT control block (0x30 bytes, legitimately written by +/// defmt), rounded up to the 32-byte MPU granule. +/// +/// The end needs care. RLAR's LIMIT is INCLUSIVE and its low 5 bits are forced +/// to 0x1F, so a region always ends at `limit | 0x1F` — it rounds UP to the +/// enclosing granule. `(edata - 1) & !0x1F` therefore reaches PAST `__edata` and +/// marks the first bytes of `.bss` read-only, since `__edata` is essentially +/// never 32-byte aligned. That is a real fault generator, not a theoretical one: +/// with `-Z stack-protector=strong` (`__edata = 0x20003ab4`) it made the +/// firmware unbootable — a genuine DACCVIOL, MMFAR = 0x20003ab8 = `__sbss` +/// exactly, faulting in an ordinary early `atomic_store::` into `.bss`. +/// Stepping back a full granule keeps the region below `__edata`, giving up at +/// most 32 bytes of thunk coverage to stay off `.bss`. +/// +/// Both cores call this so they fence exactly the same bytes; an asymmetric +/// fence would produce a MemManage fault on one core only, which is miserable to +/// diagnose. +#[cfg(target_arch = "arm")] +unsafe fn data_thunk_bounds() -> Option<(u32, u32)> { + unsafe extern "C" { + static _SEGGER_RTT: u8; + static __edata: u8; + } + let rtt_addr = core::ptr::addr_of!(_SEGGER_RTT) as usize; + let base = ((rtt_addr + 0x30 + 0x1F) & !0x1F) as u32; + let edata = core::ptr::addr_of!(__edata) as u32; + let limit = edata.saturating_sub(0x20) & !0x1F; + (limit > base).then_some((base, limit)) +} + +/// Arm `region` as privileged read-only, executable (XN=0), over `[base, limit]`. +#[cfg(target_arch = "arm")] +unsafe fn arm_ro_exec_region(region: u32, base: u32, limit: u32) { + const MPU_RNR: *mut u32 = 0xE000_ED98 as *mut u32; + const MPU_RBAR: *mut u32 = 0xE000_ED9C as *mut u32; + const MPU_RLAR: *mut u32 = 0xE000_EDA0 as *mut u32; + MPU_RNR.write_volatile(region); + MPU_RBAR.write_volatile(base | 0x1C); // SH=11, AP=10 priv-RO, XN=0 + MPU_RLAR.write_volatile(limit | 1); +} + pub unsafe fn setup_core0_data_mpu() { unsafe extern "C" { static _SEGGER_RTT: u8; @@ -67,32 +112,15 @@ pub unsafe fn setup_core0_data_mpu() { // own instructions/literal pool were corrupted). It is also why every // "Core N never writes .data" result so far was vacuous for that range. // - // Start above the RTT control block (SEGGER control block is 0x30 bytes - // and IS legitimately written by defmt), rounded up to the 32-byte MPU - // granule; end at `__edata`. + // Bounds and the RLAR rounding hazard are documented on + // `data_thunk_bounds()`. if dregion > 1 { - let thunks_base = ((rtt_addr + 0x30 + 0x1F) & !0x1F) as u32; - let edata = core::ptr::addr_of!(__edata) as u32; - // RLAR's LIMIT is INCLUSIVE and its low 5 bits are forced to 0x1F, so - // the region always ends at `limit | 0x1F` — i.e. it rounds UP to the - // enclosing 32-byte granule. `(edata - 1) & !0x1F` therefore covers - // PAST `__edata` and marks the first bytes of `.bss` read-only, since - // `__edata` is essentially never 32-byte aligned. That is a real fault - // generator, not a theoretical one: with `-Z stack-protector=strong` - // (`__edata = 0x20003ab4`) it made the firmware unbootable — a genuine - // DACCVIOL, MMFAR = 0x20003ab8 = `__sbss` exactly, faulting in an - // ordinary early `atomic_store::` into `.bss`. - // Step back a full granule so the region always ends BELOW `__edata`, - // giving up at most 32 bytes of thunk coverage to stay off `.bss`. - let thunks_limit = edata.saturating_sub(0x20) & !0x1F; - if thunks_limit > thunks_base { - MPU_RNR.write_volatile(1); - MPU_RBAR.write_volatile(thunks_base | 0x1C); // SH=11, AP=10 priv-RO, XN=0 - MPU_RLAR.write_volatile(thunks_limit | 1); + if let Some((base, limit)) = data_thunk_bounds() { + arm_ro_exec_region(1, base, limit); defmt::info!( "core0 MPU region 1 armed: .data thunks RO [{=u32:#010x}..={=u32:#010x}]", - thunks_base, - thunks_limit + 0x1F + base, + limit + 0x1F ); } } @@ -123,7 +151,6 @@ pub unsafe fn setup_core0_data_mpu() { // XN=1 here (unlike the .data regions): core 0 never executes from core // 1's stack. CORE1_STACK is 0x20080000..0x20082000 (8 KiB, SRAM8/9), both // ends already 32-byte aligned so no granule rounding is needed. - // NOTE: this region is NOT armed here — see `arm_core1_stack_fence()`. // Core 0 legitimately writes core 1's stack while setting up its initial // frame in `spawn_core1`, so arming it this early makes boot fault. That // is not hypothetical: doing so produced an immediate @@ -243,28 +270,12 @@ pub unsafe fn setup_core1_mpu() { // never writes .data" did not actually cover the range we have crash // evidence against. if dregion > 2 { - let thunks_base = ((rtt_addr + 0x30 + 0x1F) & !0x1F) as u32; - let edata = core::ptr::addr_of!(__edata) as u32; - // RLAR's LIMIT is INCLUSIVE and its low 5 bits are forced to 0x1F, so - // the region always ends at `limit | 0x1F` — i.e. it rounds UP to the - // enclosing 32-byte granule. `(edata - 1) & !0x1F` therefore covers - // PAST `__edata` and marks the first bytes of `.bss` read-only, since - // `__edata` is essentially never 32-byte aligned. That is a real fault - // generator, not a theoretical one: with `-Z stack-protector=strong` - // (`__edata = 0x20003ab4`) it made the firmware unbootable — a genuine - // DACCVIOL, MMFAR = 0x20003ab8 = `__sbss` exactly, faulting in an - // ordinary early `atomic_store::` into `.bss`. - // Step back a full granule so the region always ends BELOW `__edata`, - // giving up at most 32 bytes of thunk coverage to stay off `.bss`. - let thunks_limit = edata.saturating_sub(0x20) & !0x1F; - if thunks_limit > thunks_base { - MPU_RNR.write_volatile(2); - MPU_RBAR.write_volatile(thunks_base | 0x1C); // SH=11, AP=10 priv-RO, XN=0 - MPU_RLAR.write_volatile(thunks_limit | 1); + if let Some((base, limit)) = data_thunk_bounds() { + arm_ro_exec_region(2, base, limit); defmt::info!( "core1 MPU region 2 armed: .data thunks RO [{=u32:#010x}..={=u32:#010x}]", - thunks_base, - thunks_limit + 0x1F + base, + limit + 0x1F ); } } @@ -292,41 +303,3 @@ pub unsafe fn setup_core1_mpu() { data_limit + 0x1F ); } - -/// Fence CORE 0 out of CORE 1's stack (MPU region 2 on core 0). -/// -/// MUST be called on core 0 and ONLY AFTER `spawn_core1` — core 0 legitimately -/// writes core 1's initial stack frame during the spawn, so arming earlier -/// faults during boot. -/// -/// Why this exists: the MPU already fences core 1 out of core 0's stack, but the -/// symmetric direction was never covered, so core 0 could clobber CORE1_STACK -/// silently. A recovered (never-committed) core-1 crash record showed core 1 -/// taking a precise bus fault in `ApuPeripheral::produce_samples` through a -/// garbage `self` while its `sample_buffer` Vec was intact — i.e. the `worker` -/// reference held on CORE 1'S STACK had been clobbered. If core 0 is the writer, -/// this converts that silent clobber into a DACCVIOL naming the exact storing -/// instruction; if it never fires, core 0 is excluded. -/// -/// XN=1: core 0 never executes from core 1's stack. -#[cfg(target_arch = "arm")] -pub unsafe fn arm_core1_stack_fence() { - const MPU_RNR: *mut u32 = 0xE000_ED98 as *mut u32; - const MPU_RBAR: *mut u32 = 0xE000_ED9C as *mut u32; - const MPU_RLAR: *mut u32 = 0xE000_EDA0 as *mut u32; - const MPU_TYPE: *mut u32 = 0xE000_ED90 as *mut u32; - - let dregion = unsafe { (MPU_TYPE.read_volatile() >> 8) & 0xFF }; - if dregion <= 2 { - defmt::warn!("core0 MPU: <3 regions — CORE1_STACK fence disabled"); - return; - } - unsafe { - MPU_RNR.write_volatile(2); - MPU_RBAR.write_volatile(0x2008_0000 | 0x1D); // SH=11, AP=10 priv-RO, XN=1 - MPU_RLAR.write_volatile(0x2008_1FE0 | 1); - } - cortex_m::asm::dsb(); - cortex_m::asm::isb(); - defmt::info!("core0 MPU region 2 armed: CORE1_STACK RO [0x20080000..=0x20081fff]"); -} diff --git a/platform/pico2w/src/sd.rs b/platform/pico2w/src/sd.rs index f234867..9523a81 100644 --- a/platform/pico2w/src/sd.rs +++ b/platform/pico2w/src/sd.rs @@ -32,6 +32,26 @@ pub enum SdError { OutOfMemory, } +impl SdError { + /// True only when the failure means the CARD DID NOT RESPOND, as opposed to + /// the card answering fine and the *contents* being wrong. + /// + /// This distinction is load-bearing. `read_save_state` returns `Err` both + /// when the card is missing AND when a save-state file is present but fails + /// validation (`InvalidValue`), or when an allocation fails + /// (`OutOfMemory`). Treating all of those as "card absent" caused the boot + /// path to skip the battery-save read on a perfectly healthy card, boot + /// with blank cart RAM, and then overwrite a good `.sav` with it. + pub fn is_card_absent(&self) -> bool { + matches!( + self, + SdError::Sdmmc( + embedded_sdmmc::Error::DeviceError(_) | embedded_sdmmc::Error::NoSuchVolume + ) + ) + } +} + impl From> for SdError { fn from(e: embedded_sdmmc::Error) -> Self { SdError::Sdmmc(e) @@ -150,15 +170,6 @@ where }) } - /// Convenience wrapper for callers with no watchdog to feed. - pub fn list_rom_page( - &self, - page_offset: usize, - page_size: usize, - ) -> Result> { - self.list_rom_page_with(page_offset, page_size, || {}) - } - /// Open a specific ROM file by FAT filename (case-insensitive). /// /// The returned `SdRomReader` holds an exclusive borrow of this manager. diff --git a/tools/crash_decoder.py b/tools/crash_decoder.py index 1f6f86b..25a5acf 100755 --- a/tools/crash_decoder.py +++ b/tools/crash_decoder.py @@ -141,9 +141,9 @@ "12s" # panic_loc "H" # panic_line "H" # stack_headroom - "I" - "2I" - "6I" + "I" # dma_busy_mask [84..88] + "2I" # dma_write_addrs ch0-1 [88..96] + "6I" # stack_snapshot [96..120] (schema v2; v1 held DMA ch2-6) "I" # crc32 [120..124] "4x" # _pad4 [124..128] ) @@ -348,8 +348,11 @@ def to_dict(self) -> dict: "r4": f"0x{self.ext_r4:08x}", "r12": f"0x{self.ext_r12:08x}", } if self.has_hardfault_ext_regs else None, + # `any(...)`, not truthiness: a WatchdogTimeout record is schema v2 + # but has no capture point, so it carries six zero words. Report + # that as absent rather than as a captured window of zeros. "stack_snapshot": [f"0x{w:08x}" for w in self.stack_snapshot] - if self.stack_snapshot + if any(self.stack_snapshot) else None, "dma_write_addrs": [f"0x{w:08x}" for w in self.dma_write_addrs] if self.dma_busy_mask or any(self.dma_write_addrs) From 06aad77370a166e052c05b5809828abe7690586c Mon Sep 17 00:00:00 2001 From: Vincent Bonduro Date: Wed, 2 Sep 2026 21:37:01 -0400 Subject: [PATCH 3/4] pico2w: own card state in SdManager, bind the watchdog, split up main() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three structural findings from review. No behaviour change intended; each is verified on hardware below. Card-present state belongs to SdManager --------------------------------------- `main` tracked a local `sd_available`, set from one `is_card_absent()` check at the single boot site that happened to think of it. The object that knows whether the card answered is the manager, so every other caller either re-derived it or silently skipped the check. `SdManager` now latches `card_absent` itself, in `open_volume_tracked()` — every public operation opens volume 0 first, which makes it the one place an absent card is observable regardless of which operation asked. Callers get `card_present()` and `with_card()`. The latch is deliberately one-way and documented as such: this type sees no insertion event, and a re-seated card is not detectable without a full re-acquire. Watchdog bound once instead of threaded through 11 signatures -------------------------------------------------------------- `&mut Watchdog` was a parameter on 11 functions across four modules, several of which only passed it along. It is a hardware singleton with one owner and one configured window, and every feed site wants the same thing, so the parameter carried no information — but it did carry the window, and call sites had already drifted to bare literals (`16_000`, `5_000`) with no relation to `WATCHDOG_WINDOW_MS`. New `wdt` module owns it behind a critical-section mutex: * `wdt::feed()` — re-arm with the configured window; a caller cannot pick a wrong one * `wdt::feed_for(d)` — the two genuinely different windows in the flash staging path, now visibly the exceptions * `wdt::force_reset()` — reboot into a newly staged ROM `pause_on_debug(true)` moves into `wdt::init` next to the `start()` it compensates for; it was previously a loose call in `main` that a reordering could have separated from its reason. main() split 461 → 311 lines ---------------------------- Extracted `arm_core0_memory_guards`, `init_heap`, `init_sd_card`, `restore_saves_on_boot`. Each has a name and a doc comment stating the ordering constraint that kept it where it was. `init_sd_card` returns the GP17 mode pin alongside the manager, and says why: dropping that `Output` releases the pin and the card leaves SPI mode. That is the one hazard in this extraction — the guard was a `let _sd_mode` binding whose lifetime was load-bearing. Verified -------- 649 core + 194 pico2w tests; clippy clean on the touched code; all four clock profiles build. On hardware at 288 MHz: boots, both cores arm identical MPU bounds, mounts the card, and logs "save state loaded on boot" from inside the extracted `restore_saves_on_boot` — so the SD path runs through the new latch and reports the card present. Ran 150 s with no reset, which exercises `wdt::init` and the main loop's `wdt::feed()`: a broken feed path resets within ~10 s. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Ab4PqF52wyVpp5S2Sg6RGK --- core/src/cpu/peripheral/apu.rs | 7 +- platform/pico2w/src/crash/mod.rs | 30 +- platform/pico2w/src/crash/stack_snapshot.rs | 4 +- platform/pico2w/src/lib.rs | 3 + platform/pico2w/src/main.rs | 332 +++++++++++--------- platform/pico2w/src/sd.rs | 44 ++- platform/pico2w/src/state/loading.rs | 24 +- platform/pico2w/src/state/main_menu.rs | 4 +- platform/pico2w/src/state/rom_list.rs | 20 +- platform/pico2w/src/wdt.rs | 79 +++++ 10 files changed, 360 insertions(+), 187 deletions(-) create mode 100644 platform/pico2w/src/wdt.rs diff --git a/core/src/cpu/peripheral/apu.rs b/core/src/cpu/peripheral/apu.rs index 6a9a7c0..a7270bb 100644 --- a/core/src/cpu/peripheral/apu.rs +++ b/core/src/cpu/peripheral/apu.rs @@ -882,7 +882,9 @@ impl ApuPeripheral { /// from a correctly sized buffer. #[inline(always)] fn trusted_capacity(&self) -> usize { - self.sample_buffer.capacity().min(SAMPLE_BUFFER_CAPACITY_HINT) + self.sample_buffer + .capacity() + .min(SAMPLE_BUFFER_CAPACITY_HINT) } /// Downsample to 48 kHz and push stereo PCM pairs into `sample_buffer`. @@ -892,7 +894,6 @@ impl ApuPeripheral { /// while the generic batch path handles tests and any larger callers. #[cfg_attr(target_arch = "arm", link_section = ".data")] fn produce_samples(&mut self, cycles: u16) { - let sample_inc = cycles as u32 * SAMPLE_PERIOD_DEN; if cycles <= 4 { self.sample_acc += sample_inc; @@ -918,7 +919,7 @@ impl ApuPeripheral { .trusted_capacity() .saturating_sub(self.sample_buffer.len()) / 2) - .min(n_samples as usize); + .min(n_samples as usize); for _ in 0..pushable { let (left, right) = self.mix_sample(); self.sample_buffer.push(left); diff --git a/platform/pico2w/src/crash/mod.rs b/platform/pico2w/src/crash/mod.rs index 0fbbbe6..794db60 100644 --- a/platform/pico2w/src/crash/mod.rs +++ b/platform/pico2w/src/crash/mod.rs @@ -185,7 +185,14 @@ mod tests { #[test] fn stack_snapshot_roundtrips() { let mut rec = sample_record(4); - rec.stack_snapshot = [0x1111_1111, 0x2222_2222, 0x3333_3333, 0x4444_4444, 0x5555_5555, 0x6666_6666]; + rec.stack_snapshot = [ + 0x1111_1111, + 0x2222_2222, + 0x3333_3333, + 0x4444_4444, + 0x5555_5555, + 0x6666_6666, + ]; rec.dma_write_addrs = [0x5020_0010, 0x4008_8008]; let back = CrashRecord::from_bytes(&rec.to_bytes()).expect("roundtrip"); assert_eq!(back.stack_snapshot, rec.stack_snapshot); @@ -199,10 +206,25 @@ mod tests { fn persisted_field_offsets_are_stable() { let mut rec = sample_record(5); rec.dma_write_addrs = [0xAAAA_AAAA, 0xBBBB_BBBB]; - rec.stack_snapshot = [0xC0DE_0000, 0xC0DE_0001, 0xC0DE_0002, 0xC0DE_0003, 0xC0DE_0004, 0xC0DE_0005]; + rec.stack_snapshot = [ + 0xC0DE_0000, + 0xC0DE_0001, + 0xC0DE_0002, + 0xC0DE_0003, + 0xC0DE_0004, + 0xC0DE_0005, + ]; let b = rec.to_bytes(); - assert_eq!(&b[88..92], &0xAAAA_AAAAu32.to_le_bytes(), "dma ch0 @ [88..92]"); - assert_eq!(&b[92..96], &0xBBBB_BBBBu32.to_le_bytes(), "dma ch1 @ [92..96]"); + assert_eq!( + &b[88..92], + &0xAAAA_AAAAu32.to_le_bytes(), + "dma ch0 @ [88..92]" + ); + assert_eq!( + &b[92..96], + &0xBBBB_BBBBu32.to_le_bytes(), + "dma ch1 @ [92..96]" + ); for i in 0..6 { let off = 96 + i * 4; assert_eq!( diff --git a/platform/pico2w/src/crash/stack_snapshot.rs b/platform/pico2w/src/crash/stack_snapshot.rs index 9ac7b7d..8e87b40 100644 --- a/platform/pico2w/src/crash/stack_snapshot.rs +++ b/platform/pico2w/src/crash/stack_snapshot.rs @@ -36,9 +36,7 @@ const IDX_BASE: usize = 1; const IDX_WORDS: usize = 2; /// The padding described above must always exceed the 32-byte MPU granule. -const _: () = assert!( - (16 - (IDX_WORDS + CRASH_SNAP_WORDS)) * core::mem::size_of::() >= 32 -); +const _: () = assert!((16 - (IDX_WORDS + CRASH_SNAP_WORDS)) * core::mem::size_of::() >= 32); const CRASH_SNAP_MAGIC: usize = 0x5CAF_0001; /// Number of stack words carried in a crash record. Must match diff --git a/platform/pico2w/src/lib.rs b/platform/pico2w/src/lib.rs index bdcf2b6..c39ee2a 100644 --- a/platform/pico2w/src/lib.rs +++ b/platform/pico2w/src/lib.rs @@ -19,6 +19,9 @@ pub mod multicore; pub mod save_storage; #[cfg(target_arch = "arm")] pub mod sd; +/// Crate-wide access to the watchdog peripheral. ARM-only (embassy-rp). +#[cfg(target_arch = "arm")] +pub mod wdt; pub mod xip_cartridge; /// Pure encode/decode helpers for WiFi credential storage and the captive diff --git a/platform/pico2w/src/main.rs b/platform/pico2w/src/main.rs index 4a867d9..5bd70d3 100644 --- a/platform/pico2w/src/main.rs +++ b/platform/pico2w/src/main.rs @@ -26,8 +26,8 @@ use defmt::info; use embassy_executor::Spawner; use embassy_rp::gpio::{Level, Output}; use embassy_rp::peripherals::{ - CORE1, DMA_CH0, DMA_CH1, DMA_CH2, DMA_CH3, PIN_10, PIN_11, PIN_12, PIN_13, PIN_8, PIN_9, PIO0, - SPI0, SPI1, + CORE1, DMA_CH0, DMA_CH1, DMA_CH2, DMA_CH3, PIN_10, PIN_11, PIN_12, PIN_13, PIN_17, PIN_4, + PIN_5, PIN_6, PIN_7, PIN_8, PIN_9, PIO0, SPI0, SPI1, }; use embassy_rp::peripherals::{PIN_23, PIN_24, PIN_25, PIN_29, PIO1}; use embassy_rp::pio::{InterruptHandler as PioIrqHandler, Pio}; @@ -40,6 +40,7 @@ use embassy_time::{Delay, Duration}; use embedded_hal_bus::spi::ExclusiveDevice; use embedded_sdmmc::SdCard; use rustyboy_core::storage::RomId; +use rustyboy_pico2w::wdt; // defmt_rtt provides the RTT logging transport; the panic handler is our own // crash::handler implementation (panic-probe removed from Cargo.toml). use defmt_rtt as _; @@ -81,7 +82,10 @@ struct ClockProfile { use embassy_rp::clocks::CoreVoltage; /// Stock overclock. NOT free of the SP-drift fault — see `OC_288`. -const OC_300: ClockProfile = ClockProfile { hz: 300_000_000, voltage: CoreVoltage::V1_30 }; +const OC_300: ClockProfile = ClockProfile { + hz: 300_000_000, + voltage: CoreVoltage::V1_30, +}; /// 288 MHz = PLL FBDIV 120 (VCO 1440) / POSTDIV 5 — one grid step below 300 on /// the stock PLL config, and the fastest clock measured free of the SP-drift @@ -94,12 +98,21 @@ const OC_300: ClockProfile = ClockProfile { hz: 300_000_000, voltage: CoreVoltag /// as little as 0.3% margin, untested against temperature and part-to-part /// spread. `OC_266` is the more conservative shipping point. /// See docs/investigations/sp-drift-root-cause.md. -const OC_288: ClockProfile = ClockProfile { hz: 288_000_000, voltage: CoreVoltage::V1_30 }; +const OC_288: ClockProfile = ClockProfile { + hz: 288_000_000, + voltage: CoreVoltage::V1_30, +}; -const OC_280: ClockProfile = ClockProfile { hz: 280_000_000, voltage: CoreVoltage::V1_25 }; +const OC_280: ClockProfile = ClockProfile { + hz: 280_000_000, + voltage: CoreVoltage::V1_25, +}; /// 9 h clean on record at V1_25 — the voltage is part of that result. -const OC_266: ClockProfile = ClockProfile { hz: 266_000_000, voltage: CoreVoltage::V1_25 }; +const OC_266: ClockProfile = ClockProfile { + hz: 266_000_000, + voltage: CoreVoltage::V1_25, +}; /// Exactly one profile, chosen in priority order. Adding a profile is one const /// plus one arm; frequency and voltage cannot desync. @@ -268,8 +281,15 @@ pub(crate) enum AppState { // Entry point // --------------------------------------------------------------------------- -#[embassy_executor::main] -async fn main(spawner: Spawner) { +// --------------------------------------------------------------------------- +// Boot-time setup +// --------------------------------------------------------------------------- + +/// Arm the core-0 memory guards, before anything can run from RAM. +/// +/// Ordering matters: the image check must precede any execution of `.data` +/// RAM code, and the MPU region must be armed before that code can be written. +fn arm_core0_memory_guards() { // Verify the .data load image before it can be executed as RAM code. rustyboy_pico2w::integrity::verify_image(); @@ -299,26 +319,160 @@ async fn main(spawner: Spawner) { // Protect .data / RAM-code region as priv-RO. Any write fires DACCVIOL; // MMFAR records the exact write address so we can identify the corruptor. unsafe { rustyboy_pico2w::mpu::setup_core0_data_mpu() }; +} + +/// Hand the allocator its backing array. +fn init_heap() { + use core::mem::MaybeUninit; + // Save-state boot is the allocator high-water mark: by the time the + // boot path reads SLOT0.RBS, GameBoy and the core1 worker are live. + // + // Major live allocations measured from release symbols: + // ~40 KiB Box and cartridge state + // ~22 KiB GameBoy::front_buffer + // ~31 KiB core1 PPU worker framebuffer/state + // ~8 KiB APU sample buffers, plus smaller CPU/opcode allocations + // ~49 KiB SaveState blob for a 32 KiB cart-RAM game + // + // 160 KiB leaves room for GameBoy state plus the transient save-state + // blob while preserving core0 stack headroom. Allocation failures on + // this path are reported through `try_reserve_exact`, not HardFaults. + const HEAP_SIZE: usize = 160 * 1024; + static mut HEAP_MEM: [MaybeUninit; HEAP_SIZE] = [MaybeUninit::uninit(); HEAP_SIZE]; + unsafe { HEAP.init(core::ptr::addr_of!(HEAP_MEM) as usize, HEAP_SIZE) } +} + +/// Bring up the SD card on SPI0 and mount it. +/// +/// Returns the manager together with the mode pin, which the caller MUST keep +/// alive: dropping that `Output` releases GP17 and the card leaves SPI mode. +#[allow(clippy::too_many_arguments)] +fn init_sd_card( + mode_pin: Peri<'static, PIN_17>, + spi0: Peri<'static, SPI0>, + clk: Peri<'static, PIN_6>, + mosi: Peri<'static, PIN_7>, + miso: Peri<'static, PIN_4>, + cs_pin: Peri<'static, PIN_5>, +) -> (PicoSdMgr, Output<'static>) { + // SD card — always initialised so the ROM list is available. + // Drive the SD mode pin BEFORE any bus traffic. Nothing clocked SPI0 before + // this commit (SdCard construction is lazy), so its position did not matter; + // the power-up clocks below are the first transfer, and they must not be + // sent while this pin is still in its reset state. + let sd_mode = Output::new(mode_pin, Level::High); + + let mut spi_cfg = spi::Config::default(); + spi_cfg.frequency = 400_000; + let mut spi_bus = Spi::new_blocking(spi0, clk, mosi, miso, spi_cfg); + rp_pac::PADS_BANK0.gpio(4).modify(|w| w.set_pue(true)); + + // SD power-up clocks. The SD spec requires >=74 clock cycles at 400 kHz with + // CS DEASSERTED before the card will enter SPI mode. `embedded-sdmmc` does + // not do this — its `SdCard` docs leave it to the caller ("some [cards] do, + // some don't") — and this firmware previously went straight from + // construction to CMD0. CS is therefore created FIRST, held high for the + // clocks, then moved into the ExclusiveDevice. 10 bytes = 80 clocks. + let cs = Output::new(cs_pin, Level::High); + if let Err(e) = spi_bus.blocking_write(&[0xFFu8; 10]) { + defmt::warn!("SD power-up clocks failed: {:?}", defmt::Debug2Format(&e)); + } + + let spi_dev = ExclusiveDevice::new(spi_bus, cs, Delay); + // Bounded card acquisition. With NO card each attempt costs ~340 ms (a CMD0 + // that times out, 255 flush bytes at 400 kHz, and a delay), so the crate + // default of 50 retries takes ~17.1 s to return `CardNotFound`. The watchdog + // window is 16 s and the RP2350 counter is 24-bit MICROSECONDS, capping the + // window at 16.777 s — so a no-card boot could never fit and reset the board + // mid-init, every boot, producing only `WatchdogTimeout` records. + // + // SCOPE OF THIS BOUND: `acquire_retries` bounds ONLY the CMD0 enter-SPI-mode + // loop. The CMD8 and ACMD41 loops that follow use the crate's hardcoded + // `DEFAULT_COMMAND_RETRIES` (10_000) and are NOT affected. So this caps the + // fully-absent-card case at ~2.4 s (measured), but a PARTIALLY responsive + // card — one that answers CMD0 and then stalls — can still take ~7-8 s. + // That fits the window with roughly 2x margin, not 6x. Bounding that case + // properly would need a wall-clock timeout around the whole call. + // + // A card that is present answers CMD0 in the first few attempts: measured + // 781 ms to a successful mount on this board. + let sd_opts = embedded_sdmmc::sdcard::AcquireOpts { + acquire_retries: 6, + ..Default::default() + }; + let sdcard = SdCard::new_with_options(spi_dev, Delay, sd_opts); + (PicoSdMgr::new(sdcard, DummyClock), sd_mode) +} - { - use core::mem::MaybeUninit; - // Save-state boot is the allocator high-water mark: by the time the - // boot path reads SLOT0.RBS, GameBoy and the core1 worker are live. - // - // Major live allocations measured from release symbols: - // ~40 KiB Box and cartridge state - // ~22 KiB GameBoy::front_buffer - // ~31 KiB core1 PPU worker framebuffer/state - // ~8 KiB APU sample buffers, plus smaller CPU/opcode allocations - // ~49 KiB SaveState blob for a 32 KiB cart-RAM game - // - // 160 KiB leaves room for GameBoy state plus the transient save-state - // blob while preserving core0 stack headroom. Allocation failures on - // this path are reported through `try_reserve_exact`, not HardFaults. - const HEAP_SIZE: usize = 160 * 1024; - static mut HEAP_MEM: [MaybeUninit; HEAP_SIZE] = [MaybeUninit::uninit(); HEAP_SIZE]; - unsafe { HEAP.init(core::ptr::addr_of!(HEAP_MEM) as usize, HEAP_SIZE) } +/// Restore cart RAM into a freshly built GameBoy from whatever the card holds. +/// +/// A save state already contains cart RAM, so the battery file is read only +/// when there is no state. That ordering also avoids holding two large SD +/// blobs in the heap at once, which is the allocator's high-water mark. +/// +/// Feeds the watchdog around each SD read: these run before the main loop's +/// first feed, and a slow card can otherwise outlast the window. +fn restore_saves_on_boot(sd_mgr: &PicoSdMgr, rom_id: &RomId, gb: &mut PicoGameBoy) { + // A save state contains cart RAM, so prefer it and only + // read the battery file if no state is available. This + // avoids holding two large SD blobs at once during boot. + let slot = SaveSlot::new(0).expect("slot 0 is valid"); + wdt::feed(); + let save_state_res = sd_mgr.read_save_state(rom_id, slot); + let save_state_blob = save_state_res.unwrap_or_else(|e| { + defmt::warn!("boot save state read failed: {:?}", defmt::Debug2Format(&e)); + None + }); + wdt::feed(); + let has_save_state = save_state_blob.is_some(); + // A save state already contains cart RAM, so only fall + // back to the battery file when there is none. `with_card` + // skips the read outright on an absent card, where it + // would cost another full acquisition timeout and can + // only fail the same way. + // + // Note the card-present check is NOT a proxy for "the + // save-state read succeeded": a corrupt file or a + // transient allocation failure is an error from a HEALTHY + // card and must still fall through to the battery read. + // Conflating the two boots with blank cart RAM and then + // overwrites a good `.sav`. + let battery_data = if has_save_state { + None + } else { + wdt::feed(); + sd_mgr + .with_card(|sd| { + sd.read_battery_save(rom_id).unwrap_or_else(|e| { + defmt::warn!("battery load failed: {:?}", defmt::Debug2Format(&e)); + None + }) + }) + .flatten() + }; + match boot_load_saves(battery_data, save_state_blob) { + None => {} + Some(BootSaves::BatterySave(data)) => gb.set_external_ram(&data), + Some(BootSaves::SaveState(state)) => { + let _ = gb.load_state(state); + info!("save state loaded on boot"); + } + Some(BootSaves::Both { + battery, + save_state, + }) => { + gb.set_external_ram(&battery); + let _ = gb.load_state(save_state); + info!("save state loaded on boot"); + } } +} + +#[embassy_executor::main] +async fn main(spawner: Spawner) { + arm_core0_memory_guards(); + + init_heap(); let p = { use embassy_rp::clocks::ClockConfig; @@ -328,14 +482,10 @@ async fn main(spawner: Spawner) { embassy_rp::init(embassy_rp::config::Config::new(clk)) }; - let mut watchdog = Watchdog::new(p.WATCHDOG); - watchdog.start(Duration::from_millis(WATCHDOG_WINDOW_MS)); - // Pause the watchdog while a debugger has the cores halted. embassy's - // `start()` clears the pause-on-debug bits, so without this the 10 s - // watchdog fires during a probe-rs flash or GDB halt — resetting the chip - // mid-operation (mid-flash page-write timeouts; GDB "core is running" - // fatal errors). No effect in production (no debugger => never paused). - watchdog.pause_on_debug(true); + wdt::init( + Watchdog::new(p.WATCHDOG), + Duration::from_millis(WATCHDOG_WINDOW_MS), + ); info!( "rustyboy-pico2w v{} starting @{}MHz", @@ -377,53 +527,7 @@ async fn main(spawner: Spawner) { p.PIN_21, p.PIN_22, p.PIN_26, p.PIN_27, p.PIN_0, p.PIN_1, p.PIN_2, p.PIN_3, ); - // SD card — always initialised so the ROM list is available. - // Drive the SD mode pin BEFORE any bus traffic. Nothing clocked SPI0 before - // this commit (SdCard construction is lazy), so its position did not matter; - // the power-up clocks below are the first transfer, and they must not be - // sent while this pin is still in its reset state. - let _sd_mode = Output::new(p.PIN_17, Level::High); - - let mut spi_cfg = spi::Config::default(); - spi_cfg.frequency = 400_000; - let mut spi_bus = Spi::new_blocking(p.SPI0, p.PIN_6, p.PIN_7, p.PIN_4, spi_cfg); - rp_pac::PADS_BANK0.gpio(4).modify(|w| w.set_pue(true)); - - // SD power-up clocks. The SD spec requires >=74 clock cycles at 400 kHz with - // CS DEASSERTED before the card will enter SPI mode. `embedded-sdmmc` does - // not do this — its `SdCard` docs leave it to the caller ("some [cards] do, - // some don't") — and this firmware previously went straight from - // construction to CMD0. CS is therefore created FIRST, held high for the - // clocks, then moved into the ExclusiveDevice. 10 bytes = 80 clocks. - let cs = Output::new(p.PIN_5, Level::High); - if let Err(e) = spi_bus.blocking_write(&[0xFFu8; 10]) { - defmt::warn!("SD power-up clocks failed: {:?}", defmt::Debug2Format(&e)); - } - - let spi_dev = ExclusiveDevice::new(spi_bus, cs, Delay); - // Bounded card acquisition. With NO card each attempt costs ~340 ms (a CMD0 - // that times out, 255 flush bytes at 400 kHz, and a delay), so the crate - // default of 50 retries takes ~17.1 s to return `CardNotFound`. The watchdog - // window is 16 s and the RP2350 counter is 24-bit MICROSECONDS, capping the - // window at 16.777 s — so a no-card boot could never fit and reset the board - // mid-init, every boot, producing only `WatchdogTimeout` records. - // - // SCOPE OF THIS BOUND: `acquire_retries` bounds ONLY the CMD0 enter-SPI-mode - // loop. The CMD8 and ACMD41 loops that follow use the crate's hardcoded - // `DEFAULT_COMMAND_RETRIES` (10_000) and are NOT affected. So this caps the - // fully-absent-card case at ~2.4 s (measured), but a PARTIALLY responsive - // card — one that answers CMD0 and then stalls — can still take ~7-8 s. - // That fits the window with roughly 2x margin, not 6x. Bounding that case - // properly would need a wall-clock timeout around the whole call. - // - // A card that is present answers CMD0 in the first few attempts: measured - // 781 ms to a successful mount on this board. - let sd_opts = embedded_sdmmc::sdcard::AcquireOpts { - acquire_retries: 6, - ..Default::default() - }; - let sdcard = SdCard::new_with_options(spi_dev, Delay, sd_opts); - let mut sd_mgr = PicoSdMgr::new(sdcard, DummyClock); + let (mut sd_mgr, _sd_mode) = init_sd_card(p.PIN_17, p.SPI0, p.PIN_6, p.PIN_7, p.PIN_4, p.PIN_5); // Check for unread crash records so menus can show the crash-report badge. // (check_and_commit already ran earlier — before the splash.) @@ -471,9 +575,6 @@ async fn main(spawner: Spawner) { let mut audio_samples = Vec::with_capacity(2048); // Build initial state: Running if a ROM is staged, else MainMenu. - // Cleared as soon as any boot SD access reports the card missing. Each SD - // call costs a full acquisition timeout, so one failure is enough to know. - let mut sd_available = true; let (initial_state, gameboy_init, staged_rom_name, staged_rom_id, core1_token) = match staged { Some((info, name)) => { info!( @@ -500,59 +601,7 @@ async fn main(spawner: Spawner) { let mut gb: Box = Box::new(PicoGameBoy::with_cartridge(p.CORE1, Box::new(cart))); if let Some(rom_id) = info.rom_id { - // A save state contains cart RAM, so prefer it and only - // read the battery file if no state is available. This - // avoids holding two large SD blobs at once during boot. - let slot = SaveSlot::new(0).expect("slot 0 is valid"); - watchdog.feed(Duration::from_millis(WATCHDOG_WINDOW_MS)); - let save_state_res = sd_mgr.read_save_state(&rom_id, slot); - // One attempt is enough to know. If the card did not - // answer, every further read costs another full - // acquisition timeout and can only fail the same way. - // Only a transport failure means the card is absent. A - // corrupt save-state file, or a transient allocation - // failure, is an error from a HEALTHY card and must not - // suppress the battery-save read below — doing so boots - // with blank cart RAM and then overwrites a good `.sav`. - if let Err(e) = save_state_res.as_ref() { - if e.is_card_absent() { - sd_available = false; - } - } - let save_state_blob = save_state_res.unwrap_or_else(|e| { - defmt::warn!( - "boot save state read failed: {:?}", - defmt::Debug2Format(&e) - ); - None - }); - watchdog.feed(Duration::from_millis(WATCHDOG_WINDOW_MS)); - let has_save_state = save_state_blob.is_some(); - let battery_data = if has_save_state || !sd_available { - None // save state includes cart RAM; skip battery read - } else { - watchdog.feed(Duration::from_millis(WATCHDOG_WINDOW_MS)); - sd_mgr.read_battery_save(&rom_id).unwrap_or_else(|e| { - defmt::warn!("battery load failed: {:?}", defmt::Debug2Format(&e)); - None - }) - }; - match boot_load_saves(battery_data, save_state_blob) { - None => {} - Some(BootSaves::BatterySave(data)) => gb.set_external_ram(&data), - Some(BootSaves::SaveState(state)) => { - let _ = gb.load_state(state); - info!("save state loaded on boot"); - } - Some(BootSaves::Both { - battery, - save_state, - }) => { - gb.set_external_ram(&battery); - let _ = gb.load_state(save_state); - info!("save state loaded on boot"); - } - } + restore_saves_on_boot(&sd_mgr, &rom_id, &mut gb); } info!("ROM loaded, entering main loop"); ( @@ -647,7 +696,7 @@ async fn main(spawner: Spawner) { // armed and before the main loop's first feed, so its acquisition timeout // would itself trip the watchdog. `app.save_slot_available` is already // false from construction. - if sd_available { + if sd_mgr.card_present() { refresh_save_slot_available(&mut app, &sd_mgr); } @@ -655,7 +704,7 @@ async fn main(spawner: Spawner) { let mut tracker = perf::PerfTracker::new(); loop { - watchdog.feed(Duration::from_millis(WATCHDOG_WINDOW_MS)); + wdt::feed(); // Take the current state out so we can pass &mut app into tick(). let mut state = core::mem::replace(&mut app.state, AppState::Running(RunningState)); @@ -693,13 +742,13 @@ async fn main(spawner: Spawner) { AppState::MainMenu(menu_state) => { menu_state - .tick(&mut app, &mut game_disp, &mut input, &mut sd_mgr, &mut watchdog) + .tick(&mut app, &mut game_disp, &mut input, &mut sd_mgr) .await; } AppState::RomList(rom_list_state) => { rom_list_state - .tick(&mut app, &mut game_disp, &mut input, &mut sd_mgr, &mut watchdog) + .tick(&mut app, &mut game_disp, &mut input, &mut sd_mgr) .await; } @@ -711,7 +760,6 @@ async fn main(spawner: Spawner) { &mut onboard_flash, &mut sd_mgr, &mut game_disp, - &mut watchdog, ) .await; } diff --git a/platform/pico2w/src/sd.rs b/platform/pico2w/src/sd.rs index 9523a81..c398154 100644 --- a/platform/pico2w/src/sd.rs +++ b/platform/pico2w/src/sd.rs @@ -9,6 +9,7 @@ use embedded_sdmmc::{ use crate::save_storage::{ battery_save_filename, rom_save_dir_name, save_state_filename, SaveSlot, SAVE_ROOT_DIR, }; +use core::cell::Cell; use core::cmp::Ordering; use rustyboy_core::memory::RomReader; use rustyboy_core::storage::{BatterySaveBytes, RomId, SaveStateBytes, StorageValueError}; @@ -87,6 +88,12 @@ where T: TimeSource, { mgr: VolumeManager, + /// Latched once any operation reports the card is physically absent. + /// + /// Every SD call costs a full acquisition timeout, so one transport + /// failure is enough to know: further calls can only fail the same way, + /// and at boot they burn watchdog window doing it. + card_absent: Cell, } impl SdManager @@ -98,9 +105,40 @@ where pub fn new(device: D, timesource: T) -> Self { Self { mgr: VolumeManager::new(device, timesource), + card_absent: Cell::new(false), } } + /// False once any operation has reported the card absent. + /// + /// Latching, never cleared: this type has no insertion event to clear it + /// on, and a re-seated card is not observable without a full re-acquire. + pub fn card_present(&self) -> bool { + !self.card_absent.get() + } + + /// Run `f` only while the card is believed present, else `None`. + /// + /// Lets a caller skip work that would otherwise pay an acquisition + /// timeout for a result already known to be unavailable. + pub fn with_card(&self, f: impl FnOnce(&Self) -> R) -> Option { + self.card_present().then(|| f(self)) + } + + /// Open volume 0, latching `card_absent` on a transport failure. + /// + /// Every public operation begins here, which makes this the one place an + /// absent card is detectable regardless of which operation asked. + fn open_volume_tracked(&self) -> Result> { + self.mgr.open_raw_volume(VolumeIdx(0)).map_err(|e| { + let e = SdError::Sdmmc(e); + if e.is_card_absent() { + self.card_absent.set(true); + } + e + }) + } + /// Return up to `page_size` ROM filenames starting at `page_offset`, /// sorted alphabetically, plus a flag indicating whether more entries follow. /// @@ -125,7 +163,7 @@ where .try_reserve_exact(100) .map_err(|_| SdError::OutOfMemory)?; - let volume = self.mgr.open_raw_volume(VolumeIdx(0))?; + let volume = self.open_volume_tracked()?; let root = self.mgr.open_root_dir(volume)?; let mut lfn_storage = [0u8; 260]; @@ -177,7 +215,7 @@ where &'a mut self, filename: &str, ) -> Result, SdError> { - let volume = self.mgr.open_raw_volume(VolumeIdx(0))?; + let volume = self.open_volume_tracked()?; let root = self.mgr.open_root_dir(volume)?; let mut found_name: Option = None; @@ -290,7 +328,7 @@ where create: bool, f: impl FnOnce(&VolumeManager, RawDirectory) -> Result>, ) -> Result> { - let volume = self.mgr.open_raw_volume(VolumeIdx(0))?; + let volume = self.open_volume_tracked()?; let root = match self.mgr.open_root_dir(volume) { Ok(root) => root, Err(e) => { diff --git a/platform/pico2w/src/state/loading.rs b/platform/pico2w/src/state/loading.rs index 1c1e1a8..7471d3a 100644 --- a/platform/pico2w/src/state/loading.rs +++ b/platform/pico2w/src/state/loading.rs @@ -1,13 +1,13 @@ use alloc::boxed::Box; use defmt::info; -use embassy_rp::watchdog::Watchdog; use embassy_time::Duration; use rustyboy_pico2w::display::{hw::GameDisplay, LoadingFrame, LoadingProgress}; use rustyboy_pico2w::flash_rom::{ probe_staged_rom, FlashRomInfo, OnboardFlash, RomStager, WriteResult, }; use rustyboy_pico2w::multicore::PicoGameBoy; +use rustyboy_pico2w::wdt; use rustyboy_pico2w::xip_cartridge::XipCartridge; use super::{MainMenuState, RunningState}; @@ -28,19 +28,16 @@ impl LoadingState { flash: &mut OnboardFlash<'_>, sd_mgr: &mut PicoSdMgr, game_disp: &mut GameDisplay<'static>, - watchdog: &mut Watchdog, ) { if let Some(existing_gb) = gameboy.as_deref() { flush_battery_save(app, sd_mgr, existing_gb); } let had_game = halt_running_game(gameboy); - let stage_result = self - .ensure_rom_staged(app, flash, sd_mgr, game_disp, watchdog) - .await; + let stage_result = self.ensure_rom_staged(app, flash, sd_mgr, game_disp).await; match stage_result { Ok(info) => { - self.enter_staged_rom(app, gameboy, sd_mgr, game_disp, watchdog, had_game, info) + self.enter_staged_rom(app, gameboy, sd_mgr, game_disp, had_game, info) .await; } Err(_) => { @@ -56,7 +53,6 @@ impl LoadingState { flash: &mut OnboardFlash<'_>, sd_mgr: &mut PicoSdMgr, game_disp: &mut GameDisplay<'static>, - watchdog: &mut Watchdog, ) -> Result { // If the ROM is already staged in flash, skip the erase/write cycle. // Avoids the flash-pause handshake with the halted core1 and is @@ -71,7 +67,6 @@ impl LoadingState { flash, sd_mgr, game_disp, - watchdog, ) .await } @@ -82,14 +77,13 @@ impl LoadingState { gameboy: &mut Option>, sd_mgr: &mut PicoSdMgr, game_disp: &mut GameDisplay<'static>, - watchdog: &mut Watchdog, had_game: bool, info: FlashRomInfo, ) { app.staged_rom_name = Some(self.filename.clone()); app.staged_rom_id = info.rom_id; if had_game { - restart_after_rom_switch(watchdog); + restart_after_rom_switch(); } else { start_first_rom(app, gameboy, sd_mgr, game_disp, info).await; } @@ -146,12 +140,11 @@ async fn start_first_rom( } } -fn restart_after_rom_switch(watchdog: &mut Watchdog) -> ! { +fn restart_after_rom_switch() -> ! { // ROM switch — core1 is halted; trigger watchdog reset to boot fresh with // the newly staged ROM. info!("ROM staged, restarting via watchdog"); - watchdog.start(Duration::from_millis(100)); - loop {} + wdt::force_reset() } async fn transition_to_main_menu(app: &mut App, game_disp: &mut GameDisplay<'static>) { @@ -165,7 +158,6 @@ async fn stage_rom_from_sd( flash: &mut OnboardFlash<'_>, sd_mgr: &mut PicoSdMgr, game_disp: &mut GameDisplay<'static>, - watchdog: &mut Watchdog, ) -> Result { let mut reader = sd_mgr.open_rom_reader(filename).map_err(|e| { defmt::error!("sd open failed: {:?}", defmt::Debug2Format(&e)); @@ -180,7 +172,7 @@ async fn stage_rom_from_sd( // RP2350 watchdog maximum is 16,777,215 µs (~16.7 s). Use 16 s so the // erase phase (which can take several seconds) does not starve the watchdog. - watchdog.feed(Duration::from_millis(16_000)); + wdt::feed_for(Duration::from_millis(16_000)); let mut stager = RomStager::begin(flash, &mut reader, filename).map_err(|e| { defmt::error!("stager begin failed: {:?}", defmt::Debug2Format(&e)); })?; @@ -194,7 +186,7 @@ async fn stage_rom_from_sd( .await; loop { - watchdog.feed(Duration::from_millis(5_000)); + wdt::feed_for(Duration::from_millis(5_000)); match stager.write_next_bank(flash, &mut reader).map_err(|e| { defmt::error!("bank write failed: {:?}", defmt::Debug2Format(&e)); })? { diff --git a/platform/pico2w/src/state/main_menu.rs b/platform/pico2w/src/state/main_menu.rs index 5dd2b8f..08931a9 100644 --- a/platform/pico2w/src/state/main_menu.rs +++ b/platform/pico2w/src/state/main_menu.rs @@ -4,7 +4,6 @@ use rustyboy_pico2w::input::InputHandler; use rustyboy_pico2w::menu::{MainMenu, MenuEffect, MenuInput, MenuLogic}; use super::{RomListState, RunningState, SettingsState}; -use embassy_rp::watchdog::Watchdog; use crate::{App, AppState, PicoSdMgr}; pub struct MainMenuState { @@ -26,7 +25,6 @@ impl MainMenuState { game_disp: &mut GameDisplay<'static>, input: &mut InputHandler<'static>, sd_mgr: &mut PicoSdMgr, - watchdog: &mut Watchdog, ) { Timer::after(Duration::from_millis(33)).await; @@ -50,7 +48,7 @@ impl MainMenuState { app.transition_to(AppState::Running(RunningState)); } MenuEffect::ShowRoms => { - let next = RomListState::new(app, game_disp, sd_mgr, watchdog).await; + let next = RomListState::new(app, game_disp, sd_mgr).await; app.transition_to(AppState::RomList(alloc::boxed::Box::new(next))); } MenuEffect::ShowSettings => { diff --git a/platform/pico2w/src/state/rom_list.rs b/platform/pico2w/src/state/rom_list.rs index 72faf72..00714b1 100644 --- a/platform/pico2w/src/state/rom_list.rs +++ b/platform/pico2w/src/state/rom_list.rs @@ -1,4 +1,3 @@ -use embassy_rp::watchdog::Watchdog; use embassy_time::{Duration, Timer}; use rustyboy_pico2w::display::{ hw::GameDisplay, menu_item_needs_marquee, MENU_MARQUEE_REDRAW_FRAMES, @@ -8,6 +7,7 @@ use rustyboy_pico2w::menu::{ MenuFrame, MenuInput, RomListEffect, RomListLogic, RomPageContext, RomPageSelection, }; use rustyboy_pico2w::sd::RomListEntry; +use rustyboy_pico2w::wdt; use super::{LoadingState, MainMenuState}; use crate::{App, AppState, PicoSdMgr}; @@ -34,12 +34,11 @@ impl RomListState { app: &App, game_disp: &mut GameDisplay<'static>, sd_mgr: &mut PicoSdMgr, - watchdog: &mut Watchdog, ) -> Self { // Feed while scanning: the traversal is synchronous and can outlast the // watchdog window on a slow card. See `list_rom_page_with`. let (page, has_next, total_roms) = match sd_mgr.list_rom_page_with(0, ROM_PAGE_SIZE, || { - watchdog.feed(Duration::from_millis(crate::WATCHDOG_WINDOW_MS)); + wdt::feed(); }) { Ok(result) => (result.entries, result.has_next, result.total), Err(e) => { @@ -155,10 +154,9 @@ impl RomListState { new_offset: usize, sd_mgr: &mut PicoSdMgr, selection: RomPageSelection, - watchdog: &mut Watchdog, ) { match sd_mgr.list_rom_page_with(new_offset, ROM_PAGE_SIZE, || { - watchdog.feed(Duration::from_millis(crate::WATCHDOG_WINDOW_MS)); + wdt::feed(); }) { Ok(result) => { let page_len: usize = result.entries.len(); @@ -183,7 +181,6 @@ impl RomListState { game_disp: &mut GameDisplay<'static>, input: &mut InputHandler<'static>, sd_mgr: &mut PicoSdMgr, - watchdog: &mut Watchdog, ) { Timer::after(Duration::from_millis(33)).await; @@ -195,7 +192,7 @@ impl RomListState { let previous_selected = self.logic.selected(); let effect = self.logic.handle(menu_input); - self.apply_effect(effect, previous_selected, app, game_disp, sd_mgr, watchdog) + self.apply_effect(effect, previous_selected, app, game_disp, sd_mgr) .await; } @@ -217,7 +214,6 @@ impl RomListState { app: &mut App, game_disp: &mut GameDisplay<'static>, sd_mgr: &mut PicoSdMgr, - watchdog: &mut Watchdog, ) { match effect { RomListEffect::None => { @@ -229,11 +225,11 @@ impl RomListState { self.transition_to_selected_rom(app); } RomListEffect::NextPage => { - self.flip_requested_page(RomListEffect::NextPage, sd_mgr, app, game_disp, watchdog) + self.flip_requested_page(RomListEffect::NextPage, sd_mgr, app, game_disp) .await; } RomListEffect::PrevPage => { - self.flip_requested_page(RomListEffect::PrevPage, sd_mgr, app, game_disp, watchdog) + self.flip_requested_page(RomListEffect::PrevPage, sd_mgr, app, game_disp) .await; } RomListEffect::Back => { @@ -291,15 +287,13 @@ impl RomListState { sd_mgr: &mut PicoSdMgr, app: &App, game_disp: &mut GameDisplay<'static>, - watchdog: &mut Watchdog, ) { self.marquee_frame = 0; let Some(page) = self.page_context().request(effect) else { return; }; - self.flip_page(page.offset, sd_mgr, page.selection, watchdog) - .await; + self.flip_page(page.offset, sd_mgr, page.selection).await; self.draw(app, game_disp).await; } } diff --git a/platform/pico2w/src/wdt.rs b/platform/pico2w/src/wdt.rs new file mode 100644 index 0000000..e8d6b26 --- /dev/null +++ b/platform/pico2w/src/wdt.rs @@ -0,0 +1,79 @@ +//! Crate-wide access to the single watchdog peripheral. +//! +//! The watchdog is a hardware singleton with exactly one owner and one +//! configured window, and every feed site wants the same thing: "I am still +//! making progress, give me the standard window again." Threading a +//! `&mut Watchdog` down to each of those sites made the window an argument +//! that every intermediate signature had to carry and every caller had to get +//! right — several already passed bare literals that had drifted from the +//! configured value. +//! +//! Binding it here instead means a feed site needs no access path and cannot +//! choose the wrong window. + +use core::cell::RefCell; +use core::sync::atomic::{AtomicU32, Ordering}; + +use critical_section::Mutex; +use embassy_rp::watchdog::Watchdog; +use embassy_time::Duration; + +static WATCHDOG: Mutex>> = Mutex::new(RefCell::new(None)); + +/// The window `feed()` re-arms with, in milliseconds. Set once by `init`. +static WINDOW_MS: AtomicU32 = AtomicU32::new(0); + +/// Take ownership of the watchdog, arm it, and publish it to the crate. +/// +/// Call once, early in `main`. Feeds before this point are silently ignored, +/// which is correct: nothing is armed yet, so there is nothing to starve. +pub fn init(mut watchdog: Watchdog, window: Duration) { + watchdog.start(window); + // Pause the watchdog while a debugger has the cores halted. embassy's + // `start()` clears the pause-on-debug bits, so without this the watchdog + // fires during a probe-rs flash or GDB halt — resetting the chip + // mid-operation (mid-flash page-write timeouts; GDB "core is running" + // fatal errors). No effect in production (no debugger => never paused). + watchdog.pause_on_debug(true); + + WINDOW_MS.store(window.as_millis() as u32, Ordering::Relaxed); + critical_section::with(|cs| { + WATCHDOG.borrow(cs).replace(Some(watchdog)); + }); +} + +/// Re-arm with the configured window. The call every progress point wants. +pub fn feed() { + feed_for(Duration::from_millis(u64::from( + WINDOW_MS.load(Ordering::Relaxed), + ))); +} + +/// Re-arm with a one-off window that differs from the configured one. +/// +/// For a single synchronous operation whose timing genuinely differs from the +/// main loop's — a flash erase needs a longer window, a bank write a tighter +/// one. Prefer `feed()` everywhere else: a window chosen per call site is a +/// window that drifts. +pub fn feed_for(window: Duration) { + critical_section::with(|cs| { + if let Some(w) = WATCHDOG.borrow(cs).borrow_mut().as_mut() { + w.feed(window); + } + }); +} + +/// Arm a short window and spin, so the watchdog resets the chip. +/// +/// Used to reboot into a newly staged ROM. Diverges: the reset lands before +/// the loop can make progress. +pub fn force_reset() -> ! { + critical_section::with(|cs| { + if let Some(w) = WATCHDOG.borrow(cs).borrow_mut().as_mut() { + w.start(Duration::from_millis(100)); + } + }); + loop { + cortex_m::asm::nop(); + } +} From a7479d7d8d470a850c6e7464be366b49794c1702 Mon Sep 17 00:00:00 2001 From: Vincent Bonduro Date: Thu, 3 Sep 2026 07:13:38 -0400 Subject: [PATCH 4/4] =?UTF-8?q?pico2w:=20fix=20second-round=20review=20fin?= =?UTF-8?q?dings=20=E2=80=94=20MPU=20attrs,=20.uninit=20overlap,=20latch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two silent defects, both introduced by the previous two commits and both invisible to their hardware verification, plus the unanimous smaller findings. Orphaned attributes in mpu.rs ----------------------------- Extracting `data_thunk_bounds` inserted it BETWEEN `setup_core0_data_mpu`'s attribute block and its `fn` line, so `#[inline(never)]` and the whole region-0 doc comment silently reattached to the helper. Confirmed in the ELF: the 3-line bounds helper kept an out-of-line symbol while the MPU-arming function was inlined away. In a codebase whose debugging strategy is PC-to-symbol attribution of RAM-code corruption, that is a regression in the instrument. Restored; the symbols now flip back the correct way. Nothing caught this — it builds on both targets and the lint that would fire (`empty_line_after_outer_attr`) is nursery. Core 1's MPU region overlapped the tail of `.uninit` ----------------------------------------------------- The previous commit added a comment and a `const _: () = assert!(...)` claiming `CRASH_STACK_SNAP`'s padding fenced the "last `.uninit` object falls inside core 1's MPU region" hazard. It did not. That array is not last: 20066a80 CRASH_STACK_SNAP (padded, not last) 20066ac0 DMA_CRASH_SNAPSHOT (last, unpadded, ends 20066b08) 20066b08 _stack_end / __euninit `_stack_end & !0x1F` = 0x20066b00, so the final two words of `DMA_CRASH_SNAPSHOT` were privileged read-only for core 1 — and `capture_dma_snapshot()` writes all 18 unconditionally from the panic handler, which runs in thread mode with the MPU live (unlike the HardFault path, where MPU_CTRL.HFNMIENA=0 disables it). A core-1 panic therefore took a DACCVIOL that escalated to HardFault, replacing the intended `CrashKind::Panic` record and losing its file/line — on exactly the fault class under investigation. Fixed at the source rather than by padding: `setup_core1_mpu` now rounds the region base UP to the MPU granule, so it can never start below `__euninit` and no `.uninit` object is reachable by it. Verified on hardware — the base moved 0x20066b00 -> 0x20066b20. Which object lands last is a link-order accident, so padding could never have been the fix. The false assert is replaced with one that checks something true. Also corrected in that file: the `# Safety` block still claimed `base` is "range-checked against the core-0 stack", contradicting the inline comment six lines below it that correctly calls it a physical-SRAM clamp; and the window only begins at the return slot on the FAULT path — on the panic path `base` is the MSP inside the handler, so it starts on handler locals. Card-presence latch: callers no longer decide ---------------------------------------------- `card_present()`/`with_card()` handed the decision back to call sites, produced `Option>` and a `.flatten()` that erased "no card" from "no battery file", and left every other SD site to wonder whether it should have checked. The latch is now private and the READ paths consult it themselves — `read_battery_save`/`read_save_state` return `Ok(None)`, `save_state_exists` returns `Ok(false)`. Writes deliberately still attempt and report a real error: a silent `Ok` on a write is data loss, not a fast path. `is_card_absent` no longer matches `NoSuchVolume`. `open_raw_volume` returns it only for `VolumeIdx >= 4` and this code always opens volume 0, so the arm was unreachable; an unformatted card returns `FormatError`, correctly excluded. Watchdog -------- `feed_for(16_000)` was `WATCHDOG_WINDOW_MS` restated as a literal — the exact drift the `wdt` module exists to prevent, in the one place it was aimed at. Now `feed()`. Only `5_000` is a genuine exception, so the commit-message claim of "two genuinely different windows" was wrong. `force_reset()` could hang: if the watchdog was never published the `if let` fell through and a `-> !` function spun forever. Falls back to `SCB::sys_reset()`. Threading `&mut Watchdog` made that unrepresentable; a global has to handle it explicitly. `feed()`'s doc also misattributed its own safety — it is the missing peripheral that makes it silent, not the zero window, which would write LOAD=0 and reset on the next tick. Other ----- * `init_sd_card` returns `SdBus` instead of `(PicoSdMgr, Output)`. The mode pin's lifetime is now a type invariant instead of a `let _sd_mode` binding whose leading underscore invites the cleanup that breaks it. * comment de-duplication: the save-precedence rule was stated three times in one 55-line function, and a six-line comment restated its own first sentence verbatim (a paste artifact from the extraction). * removed the stale `list_rom_page` doc left on `list_rom_page_with`. Verified -------- 649 core + 194 pico2w tests; all four clock profiles build; lib clippy clean. On hardware at 288 MHz: boots, core 1 region base 0x20066b20 (above `__euninit`), both cores identical `.data` bounds, save state loaded from the extracted restore path, main loop at 3.8 s, 150 s with no reset. NOT fixed, pre-existing and out of scope: `cargo clippy --bins` fails on a deny-by-default `modulo_one` at state/rom_list.rs:204 (from dda1ba8). The previous commit's "clippy clean" claim was based on linting only the lib target, which is why it read clean; the binary target does not. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Ab4PqF52wyVpp5S2Sg6RGK --- platform/pico2w/src/crash/stack_snapshot.rs | 33 +++++---- platform/pico2w/src/main.rs | 82 +++++++++++---------- platform/pico2w/src/mpu.rs | 47 +++++++----- platform/pico2w/src/sd.rs | 45 +++++------ platform/pico2w/src/state/loading.rs | 5 +- platform/pico2w/src/wdt.rs | 24 +++++- 6 files changed, 140 insertions(+), 96 deletions(-) diff --git a/platform/pico2w/src/crash/stack_snapshot.rs b/platform/pico2w/src/crash/stack_snapshot.rs index 8e87b40..481d64b 100644 --- a/platform/pico2w/src/crash/stack_snapshot.rs +++ b/platform/pico2w/src/crash/stack_snapshot.rs @@ -14,14 +14,17 @@ //! `schema_ver = 2` with an all-zero window. The decoder cannot currently //! distinguish "captured, all zeros" from "never captured". -/// `.uninit` backing store: 8 words used, 8 words of deliberate PADDING. +/// `.uninit` backing store: 8 words used, 8 words of PADDING. /// -/// Core 1's MPU region 0 base is `_stack_end & !0x1F`, so up to 31 bytes of the -/// LAST `.uninit` object are privileged read-only for core 1. If that object is -/// written by core 1 it takes a MemManage fault that escalates to HardFault and -/// kills the core silently. The trailing 8 words absorb that if this array -/// happens to be placed last. The static assert below keeps the padding from -/// being "optimised" away by a future edit. +/// The padding is slack, NOT a safety mechanism — this array is not the last +/// `.uninit` object (`DMA_CRASH_SNAPSHOT` is, as of this writing), so padding +/// here protects nothing. An earlier version of this comment claimed it fenced +/// the "last `.uninit` object lands inside core 1's MPU region" hazard; it did +/// not, and could not, because which object is last is a link-order accident. +/// +/// That hazard is now fixed at its source: `setup_core1_mpu` rounds core 1's +/// region base UP to the MPU granule, so the region can never start below +/// `__euninit` and no `.uninit` object is reachable by it. See `mpu.rs`. #[no_mangle] #[link_section = ".uninit.CRASH_STACK_SNAP"] pub static mut CRASH_STACK_SNAP: [usize; 16] = [0; 16]; @@ -35,8 +38,8 @@ const IDX_MAGIC: usize = 0; const IDX_BASE: usize = 1; const IDX_WORDS: usize = 2; -/// The padding described above must always exceed the 32-byte MPU granule. -const _: () = assert!((16 - (IDX_WORDS + CRASH_SNAP_WORDS)) * core::mem::size_of::() >= 32); +/// The array must actually be big enough for the header plus the window. +const _: () = assert!(IDX_WORDS + CRASH_SNAP_WORDS <= 16); const CRASH_SNAP_MAGIC: usize = 0x5CAF_0001; /// Number of stack words carried in a crash record. Must match @@ -45,13 +48,17 @@ pub const CRASH_SNAP_WORDS: usize = 6; /// Capture `CRASH_SNAP_WORDS` stack words starting at `base` (the PRE-FAULT /// stack pointer) into `.uninit`, for the next boot to fold into the crash -/// record. The window therefore begins at the return-address slot, which is -/// what makes a wild-PC fault out of a `pop {rlist, pc}` diagnosable. +/// record. On the FAULT path the window therefore begins at the return-address +/// slot, which is what makes a wild-PC fault out of a `pop {rlist, pc}` +/// diagnosable. On the PANIC path `base` is the MSP inside the panic handler, +/// so the window starts on handler locals instead — panics carry file and line, +/// so this costs little, but the window means something different there. /// /// # Safety /// Called from a fault/panic handler with interrupts effectively quiesced. -/// `base` is range-checked against the core-0 stack before any dereference, so -/// a wild SP cannot turn this into an out-of-bounds read. +/// `base` is clamped to physical SRAM before any dereference, so a wild SP +/// cannot fault here. That is a "will not fault" bound, NOT an assertion that +/// `base` points at a stack — see the inline note in the body. pub unsafe fn capture_stack_window(base: usize) { let f = &raw mut CRASH_STACK_SNAP; (*f)[IDX_MAGIC] = CRASH_SNAP_MAGIC; diff --git a/platform/pico2w/src/main.rs b/platform/pico2w/src/main.rs index 5bd70d3..de378db 100644 --- a/platform/pico2w/src/main.rs +++ b/platform/pico2w/src/main.rs @@ -342,11 +342,32 @@ fn init_heap() { unsafe { HEAP.init(core::ptr::addr_of!(HEAP_MEM) as usize, HEAP_SIZE) } } -/// Bring up the SD card on SPI0 and mount it. +/// The SD manager bundled with the mode pin that must outlive it. /// -/// Returns the manager together with the mode pin, which the caller MUST keep -/// alive: dropping that `Output` releases GP17 and the card leaves SPI mode. -#[allow(clippy::too_many_arguments)] +/// GP17 selects SPI mode on the card module, so dropping that `Output` takes the +/// card out of SPI mode. Owning it here makes the lifetime a type invariant +/// rather than a comment asking the caller to keep a `let _sd_mode` binding +/// alive — a binding whose leading underscore invites exactly the cleanup that +/// would break it, with no compiler error at the point of the mistake. +pub(crate) struct SdBus { + mgr: PicoSdMgr, + _mode: Output<'static>, +} + +impl core::ops::Deref for SdBus { + type Target = PicoSdMgr; + fn deref(&self) -> &PicoSdMgr { + &self.mgr + } +} + +impl core::ops::DerefMut for SdBus { + fn deref_mut(&mut self) -> &mut PicoSdMgr { + &mut self.mgr + } +} + +/// Bring up the SD card on SPI0 and mount it. fn init_sd_card( mode_pin: Peri<'static, PIN_17>, spi0: Peri<'static, SPI0>, @@ -354,7 +375,7 @@ fn init_sd_card( mosi: Peri<'static, PIN_7>, miso: Peri<'static, PIN_4>, cs_pin: Peri<'static, PIN_5>, -) -> (PicoSdMgr, Output<'static>) { +) -> SdBus { // SD card — always initialised so the ROM list is available. // Drive the SD mode pin BEFORE any bus traffic. Nothing clocked SPI0 before // this commit (SdCard construction is lazy), so its position did not matter; @@ -401,7 +422,10 @@ fn init_sd_card( ..Default::default() }; let sdcard = SdCard::new_with_options(spi_dev, Delay, sd_opts); - (PicoSdMgr::new(sdcard, DummyClock), sd_mode) + SdBus { + mgr: PicoSdMgr::new(sdcard, DummyClock), + _mode: sd_mode, + } } /// Restore cart RAM into a freshly built GameBoy from whatever the card holds. @@ -413,9 +437,6 @@ fn init_sd_card( /// Feeds the watchdog around each SD read: these run before the main loop's /// first feed, and a slow card can otherwise outlast the window. fn restore_saves_on_boot(sd_mgr: &PicoSdMgr, rom_id: &RomId, gb: &mut PicoGameBoy) { - // A save state contains cart RAM, so prefer it and only - // read the battery file if no state is available. This - // avoids holding two large SD blobs at once during boot. let slot = SaveSlot::new(0).expect("slot 0 is valid"); wdt::feed(); let save_state_res = sd_mgr.read_save_state(rom_id, slot); @@ -425,30 +446,19 @@ fn restore_saves_on_boot(sd_mgr: &PicoSdMgr, rom_id: &RomId, gb: &mut PicoGameBo }); wdt::feed(); let has_save_state = save_state_blob.is_some(); - // A save state already contains cart RAM, so only fall - // back to the battery file when there is none. `with_card` - // skips the read outright on an absent card, where it - // would cost another full acquisition timeout and can - // only fail the same way. - // - // Note the card-present check is NOT a proxy for "the - // save-state read succeeded": a corrupt file or a - // transient allocation failure is an error from a HEALTHY - // card and must still fall through to the battery read. - // Conflating the two boots with blank cart RAM and then - // overwrites a good `.sav`. + // Gate on has_save_state ONLY. A failed save-state read is not evidence the + // card is bad: a corrupt file or a transient allocation failure comes from a + // HEALTHY card and must still fall through to the battery read. Conflating + // the two boots with blank cart RAM and then overwrites a good `.sav`. + // `read_battery_save` skips itself on a card already known absent. let battery_data = if has_save_state { None } else { wdt::feed(); - sd_mgr - .with_card(|sd| { - sd.read_battery_save(rom_id).unwrap_or_else(|e| { - defmt::warn!("battery load failed: {:?}", defmt::Debug2Format(&e)); - None - }) - }) - .flatten() + sd_mgr.read_battery_save(rom_id).unwrap_or_else(|e| { + defmt::warn!("battery load failed: {:?}", defmt::Debug2Format(&e)); + None + }) }; match boot_load_saves(battery_data, save_state_blob) { None => {} @@ -527,7 +537,7 @@ async fn main(spawner: Spawner) { p.PIN_21, p.PIN_22, p.PIN_26, p.PIN_27, p.PIN_0, p.PIN_1, p.PIN_2, p.PIN_3, ); - let (mut sd_mgr, _sd_mode) = init_sd_card(p.PIN_17, p.SPI0, p.PIN_6, p.PIN_7, p.PIN_4, p.PIN_5); + let mut sd_mgr = init_sd_card(p.PIN_17, p.SPI0, p.PIN_6, p.PIN_7, p.PIN_4, p.PIN_5); // Check for unread crash records so menus can show the crash-report badge. // (check_and_commit already ran earlier — before the splash.) @@ -691,14 +701,10 @@ async fn main(spawner: Spawner) { wifi_periphs, }; // Runs after the watchdog is armed and before the main loop's first feed, so - // on a dead card its acquisition timeout would itself trip the watchdog. - // Skipped when the card did not respond: this runs after the watchdog is - // armed and before the main loop's first feed, so its acquisition timeout - // would itself trip the watchdog. `app.save_slot_available` is already - // false from construction. - if sd_mgr.card_present() { - refresh_save_slot_available(&mut app, &sd_mgr); - } + // on a dead card an acquisition timeout here would itself trip the watchdog. + // `save_state_exists` short-circuits to false on a card already known + // absent, so that timeout is only ever paid once, at the boot read above. + refresh_save_slot_available(&mut app, &sd_mgr); #[cfg(feature = "fps")] let mut tracker = perf::PerfTracker::new(); diff --git a/platform/pico2w/src/mpu.rs b/platform/pico2w/src/mpu.rs index f645bfa..41b1d9c 100644 --- a/platform/pico2w/src/mpu.rs +++ b/platform/pico2w/src/mpu.rs @@ -6,17 +6,6 @@ /// (MemManage → HardFault) on an illicit write, with the stacked PC /// identifying the exact corrupt-store instruction. -/// Configure Core 0's PMSAv8-M MPU with one privileged-read-only region: -/// - region 0: .data RAM code from `__sdata` to immediately before `_SEGGER_RTT` -/// -/// Core 0 never legitimately writes this range after startup. Any write -/// fires DACCVIOL (MemManage → HardFault) with the exact writer PC stacked. -/// -/// The upper bound is derived from `_SEGGER_RTT` so adding another RAM-resident -/// function cannot silently leave the end of `.data` writable. -#[cfg(target_arch = "arm")] -#[inline(never)] - /// Bounds of the RAM-resident `.data` thunk/code region, as an MPU (base, limit) /// pair — or `None` if the range is empty. /// @@ -61,6 +50,16 @@ unsafe fn arm_ro_exec_region(region: u32, base: u32, limit: u32) { MPU_RLAR.write_volatile(limit | 1); } +/// Configure Core 0's PMSAv8-M MPU with one privileged-read-only region: +/// - region 0: .data RAM code from `__sdata` to immediately before `_SEGGER_RTT` +/// +/// Core 0 never legitimately writes this range after startup. Any write +/// fires DACCVIOL (MemManage → HardFault) with the exact writer PC stacked. +/// +/// The upper bound is derived from `_SEGGER_RTT` so adding another RAM-resident +/// function cannot silently leave the end of `.data` writable. +#[cfg(target_arch = "arm")] +#[inline(never)] pub unsafe fn setup_core0_data_mpu() { unsafe extern "C" { static _SEGGER_RTT: u8; @@ -229,16 +228,30 @@ pub unsafe fn setup_core1_mpu() { // a stale hardcode (0x20066B60) sat ~1 KB below the real stack bottom, inside // the defmt RTT buffer (0x20066b3c–0x20066f3c); after the copy_dma_step .data // fix grew SRAM, legit core-1 RTT logging wrote into the covered range and - // crash-looped on MMFAR DACCVIOL. Aligning the base DOWN to the 32-byte MPU - // granule keeps it above the RTT buffer (which lives below _stack_end in - // .uninit) while still covering the entire core-0 stack. + // crash-looped on MMFAR DACCVIOL. + // + // Round the base UP to the 32-byte MPU granule, not down. `_stack_end` is the + // end of `.uninit` (`__euninit`) and is essentially never 32-byte aligned — + // it is 0x20066b08 as of this writing. Rounding DOWN puts the region base + // BELOW `__euninit`, which makes the tail of the last `.uninit` object + // privileged read-only for core 1. That object is `DMA_CRASH_SNAPSHOT`, and + // `capture_dma_snapshot()` writes all 18 of its words unconditionally from + // core 1's panic handler — which runs in thread mode with the MPU live, + // unlike the HardFault path (MPU_CTRL.HFNMIENA=0 disables the MPU there). + // The store took a DACCVIOL that escalated to HardFault, so a core-1 panic + // was recorded as a bogus HardFault and lost its file/line. + // + // Rounding up cannot do this: the base lands at or above `__euninit`, so no + // `.uninit` byte is ever inside the region. The cost is that up to 31 bytes + // at the very bottom of core 0's stack go unfenced by core 1 — the deepest + // point, reached only at near-overflow, which MSPLIM already traps. extern "C" { static _stack_end: u32; } let stack_bottom = core::ptr::addr_of!(_stack_end) as u32; - let base = stack_bottom & !0x1F; // 32-byte aligned MPU region base - // RBAR: BASE | SH=11(b4:3) | AP=10(b2:1) | XN=1(b0) = base | 0x1D - // RLAR: LIMIT=0x2007FFE0 | AttrIndx=0(b3:1) | EN=1(b0) = 0x2007FFE1 + let base = (stack_bottom + 0x1F) & !0x1F; // 32-byte aligned, at or above __euninit + // RBAR: BASE | SH=11(b4:3) | AP=10(b2:1) | XN=1(b0) = base | 0x1D + // RLAR: LIMIT=0x2007FFE0 | AttrIndx=0(b3:1) | EN=1(b0) = 0x2007FFE1 MPU_RNR.write_volatile(0); MPU_RBAR.write_volatile(base | 0x1D); MPU_RLAR.write_volatile(0x2007_FFE1); diff --git a/platform/pico2w/src/sd.rs b/platform/pico2w/src/sd.rs index c398154..179c7b5 100644 --- a/platform/pico2w/src/sd.rs +++ b/platform/pico2w/src/sd.rs @@ -43,13 +43,14 @@ impl SdError { /// (`OutOfMemory`). Treating all of those as "card absent" caused the boot /// path to skip the battery-save read on a perfectly healthy card, boot /// with blank cart RAM, and then overwrite a good `.sav` with it. + /// Only `DeviceError` qualifies. `NoSuchVolume` is NOT matched: + /// `open_raw_volume` returns it solely for `VolumeIdx >= 4` and this code + /// always opens volume 0, so that arm was unreachable. A card that answers + /// but is unformatted or has a bad MBR returns `FormatError`, which is + /// deliberately excluded — the card responded, so the contents are the + /// problem, not the transport. pub fn is_card_absent(&self) -> bool { - matches!( - self, - SdError::Sdmmc( - embedded_sdmmc::Error::DeviceError(_) | embedded_sdmmc::Error::NoSuchVolume - ) - ) + matches!(self, SdError::Sdmmc(embedded_sdmmc::Error::DeviceError(_))) } } @@ -109,20 +110,18 @@ where } } - /// False once any operation has reported the card absent. + /// True once any operation has reported the card absent. /// /// Latching, never cleared: this type has no insertion event to clear it /// on, and a re-seated card is not observable without a full re-acquire. - pub fn card_present(&self) -> bool { - !self.card_absent.get() - } - - /// Run `f` only while the card is believed present, else `None`. /// - /// Lets a caller skip work that would otherwise pay an acquisition - /// timeout for a result already known to be unavailable. - pub fn with_card(&self, f: impl FnOnce(&Self) -> R) -> Option { - self.card_present().then(|| f(self)) + /// Private on purpose. Callers should not have to know there is a card — + /// the read paths below consult this themselves and report "nothing there" + /// instead of paying another acquisition timeout to be told the same. A + /// public accessor would put that decision back on every call site, which + /// is what it replaced. + fn card_absent(&self) -> bool { + self.card_absent.get() } /// Open volume 0, latching `card_absent` on a transport failure. @@ -139,11 +138,6 @@ where }) } - /// Return up to `page_size` ROM filenames starting at `page_offset`, - /// sorted alphabetically, plus a flag indicating whether more entries follow. - /// - /// Iterates the full root directory each call (up to 100 entries) and - /// slices the requested page. /// List a page of ROMs, calling `on_progress` for every directory entry /// visited. /// @@ -262,6 +256,9 @@ where } pub fn read_battery_save(&self, rom_id: &RomId) -> Result>, SdError> { + if self.card_absent() { + return Ok(None); + } let Some(data) = self.with_existing_rom_save_dir(rom_id, |mgr, dir| { read_file_optional(mgr, dir, battery_save_filename()) })? @@ -293,6 +290,9 @@ where rom_id: &RomId, slot: SaveSlot, ) -> Result>, SdError> { + if self.card_absent() { + return Ok(None); + } let filename = save_state_filename(slot); let Some(data) = self.with_existing_rom_save_dir(rom_id, |mgr, dir| { read_file_optional(mgr, dir, filename.as_str()) @@ -312,6 +312,9 @@ where rom_id: &RomId, slot: SaveSlot, ) -> Result> { + if self.card_absent() { + return Ok(false); + } let filename = save_state_filename(slot); let Some(exists) = self.with_existing_rom_save_dir(rom_id, |mgr, dir| { file_exists(mgr, dir, filename.as_str()) diff --git a/platform/pico2w/src/state/loading.rs b/platform/pico2w/src/state/loading.rs index 7471d3a..2808fd5 100644 --- a/platform/pico2w/src/state/loading.rs +++ b/platform/pico2w/src/state/loading.rs @@ -170,9 +170,8 @@ async fn stage_rom_from_sd( )) .await; - // RP2350 watchdog maximum is 16,777,215 µs (~16.7 s). Use 16 s so the - // erase phase (which can take several seconds) does not starve the watchdog. - wdt::feed_for(Duration::from_millis(16_000)); + // The erase phase can take several seconds; re-arm the full window first. + wdt::feed(); let mut stager = RomStager::begin(flash, &mut reader, filename).map_err(|e| { defmt::error!("stager begin failed: {:?}", defmt::Debug2Format(&e)); })?; diff --git a/platform/pico2w/src/wdt.rs b/platform/pico2w/src/wdt.rs index e8d6b26..77a8981 100644 --- a/platform/pico2w/src/wdt.rs +++ b/platform/pico2w/src/wdt.rs @@ -43,6 +43,11 @@ pub fn init(mut watchdog: Watchdog, window: Duration) { } /// Re-arm with the configured window. The call every progress point wants. +/// +/// A no-op before `init`, because the peripheral is not published yet — note it +/// is the missing peripheral that makes this silent, NOT the zero window. A +/// zero window reaching `Watchdog::feed` would write `LOAD = 0` and reset the +/// chip on the next tick, which is the opposite of ignoring it. pub fn feed() { feed_for(Duration::from_millis(u64::from( WINDOW_MS.load(Ordering::Relaxed), @@ -63,16 +68,27 @@ pub fn feed_for(window: Duration) { }); } -/// Arm a short window and spin, so the watchdog resets the chip. +/// Reboot the chip. Used to boot into a newly staged ROM. /// -/// Used to reboot into a newly staged ROM. Diverges: the reset lands before -/// the loop can make progress. +/// Arms a short watchdog window and spins; the reset lands before the loop can +/// make progress. If the watchdog was never published — which the type system +/// cannot rule out now that the peripheral lives behind a global — falls back +/// to `SCB::sys_reset()` rather than spinning forever. Threading `&mut Watchdog` +/// made that case unrepresentable; a global has to handle it explicitly, or a +/// `-> !` function silently becomes a hang with the display frozen mid-switch. pub fn force_reset() -> ! { - critical_section::with(|cs| { + let armed = critical_section::with(|cs| { if let Some(w) = WATCHDOG.borrow(cs).borrow_mut().as_mut() { w.start(Duration::from_millis(100)); + true + } else { + false } }); + if !armed { + defmt::error!("wdt::force_reset before init — falling back to SCB::sys_reset"); + cortex_m::peripheral::SCB::sys_reset(); + } loop { cortex_m::asm::nop(); }