feat: box watchdogs and self-healing

README.md

@@ -13,6 +13,10 @@ The box is **client only**: it runs no Syncphony server. Point it at yours.
 - Raspberry Pi 4 or Pi 5, 2 GB RAM or more
 - 16 GB+ SD card
 - arm64 only. Pi 3, Pi Zero 2 W and armhf are not supported.
+- A case with a heatsink or a fan. A long party with the visualizer on
+  warms a Pi in a closed case past its soft temperature limit, and the
+  firmware silently halves the CPU first (the box logs temperature and
+  throttling — see [Self-healing](#self-healing-watchdogs) below).
 
 ## Building
 
@@ -164,6 +168,42 @@ sudo syncphony-box chroot            # shell on the real root, read-write
 Reboot afterwards. Changes to the pairing, config and anything else on
 `/data` need no chroot — that partition is writable at runtime.
 
+## Self-healing: watchdogs
+
+The box notices when it's stuck and fixes itself — nobody walks over to
+it. Three watchdogs, each owning one failure class
+([ADR 0008](docs/adr/0008-watchdogs-and-self-healing.md)):
+
+- **Hardware watchdog (kernel/systemd hangs).** systemd arms the Pi's
+  hardware watchdog at 15 s. A kernel or systemd hang stops the petting
+  and the hardware hard-resets the box; with the read-only root it comes
+  back clean, paired, playing.
+- **Kiosk watchdog in boxd (a wedged page).** The `/tv` page heartbeats
+  to boxd every 30 s. If the heartbeat stops for 90 s while the server
+  still answers `/healthz`, boxd restarts the kiosk — Chromium comes
+  back and the speaker resumes by itself. If that happens 3 times in 15
+  minutes, the box reboots instead. When the server is *unreachable*,
+  the watchdog stays quiet: that's the offline screen's job, not a
+  stuck page.
+- **Time.** `systemd-timesyncd` runs and the kiosk waits for
+  `time-sync.target`: the server's playback clock sync trusts the
+  browser's clock, and a Pi has no RTC battery by default.
+
+Two hygiene loops ride along:
+
+- **Thermals.** boxd logs `vcgencmd measure_temp` and the throttling
+  flags to the journal every minute (`journalctl -u boxd | grep
+  thermals`) and reports them in `/v1/info` for the room's Screens list
+  (stage 4). A persistent `throttled=0x50000`-style value means the
+  power supply is weak; a hot `temp=` means the case needs airflow.
+- **Nightly reload.** Once a day around 05:00 local, while the page is
+  up and nothing is playing, boxd restarts the kiosk so a long-lived
+  Chromium gives its memory back. A party that runs through the night
+  is never interrupted.
+
+The freeze and kernel-hang procedures that verify all this are in
+[docs/testing.md](docs/testing.md).
+
 ## Audio
 
 The box is the room's speaker: Chromium plays everything through the
@@ -273,6 +313,21 @@ has its own page: [docs/testing.md](docs/testing.md).
 - Pull the server's network cable: the offline screen appears within a
   couple of minutes (or at the next kiosk start) with the box's IP and a
   countdown; plugging it back returns to the room with no touch.
+- Freeze Chromium's renderer (`pkill -STOP -f 'type=renderer'`): the
+  room comes back by itself within about 2 minutes (the boxd kiosk
+  watchdog), and music resumes.
+- Kernel-hang reboot (destructive; see docs/testing.md):
+  `echo c > /proc/sysrq-trigger` — the hardware watchdog resets the box
+  and it comes back paired and playing.
+- `journalctl -u boxd | grep thermals` shows a temperature line every
+  minute; `curl 127.0.0.1:8099/v1/info` includes `temperatureC`. Check
+  on a Pi 4 and a Pi 5.
+- `timedatectl` shows the clock synchronized and
+  `systemctl status systemd-timesyncd` is active, on both a Pi 4 and a
+  Pi 5 (which has an RTC header but no battery by default).
+- A 12-hour playback soak with the visualizer on, on a 2 GB Pi 4 (the
+  procedure is in docs/testing.md): memory plateaus, and the 05:00
+  nightly reload happens only while idle.
 - An hour of real playback as the room's speaker — Navidrome (MP3 and
   FLAC) and Spotify (Ogg): no gaps between songs, no drift, no clipped
   first notes.

boxd/cmd/boxd/main.go

@@ -1,7 +1,8 @@
 // boxd is the Syncphony box's helper daemon (MAD-803, ADR 0005 in the box
 // repo; the browser-facing bridge contract is Syncphony ADR 0016). It owns
 // everything on the box that isn't Chromium: the syncphony.txt config, the
-// box bridge on loopback, and the local setup and offline screens.
+// box bridge on loopback, the local setup and offline screens, and the
+// kiosk watchdog and thermals (MAD-806, ADR 0008).
 package main
 
 import (

boxd/internal/daemon/bridge.go

@@ -109,12 +109,26 @@ func (d *Daemon) guard(next http.HandlerFunc) http.HandlerFunc {
 }
 
 func (d *Daemon) handleInfo(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(map[string]any{
+	d.mu.Lock()
+	th := d.therm
+	d.mu.Unlock()
+	// The ADR 0016 core; the thermal fields (MAD-806) ride along once a
+	// sample exists — stage 4's Screens list reads them ("52 °C",
+	// "running hot / throttled"), and everyone else ignores them.
+	info := map[string]any{
 		"version":      d.version,
 		"name":         d.displayName(),
 		"capabilities": []string{"reboot", "reload"}, // cec arrives in stage 3
-	})
+	}
+	if th.valid() {
+		// No sample (no vcgencmd here) means no fields: absent reads
+		// better than a misleading 0 °C.
+		for k, v := range th.infoFields() {
+			info[k] = v
+		}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(info)
 }
 
 var eventTypes = map[string]bool{
@@ -181,8 +195,6 @@ func (d *Daemon) handleReload(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(`{"ok":true}` + "\n"))
 	go func() {
 		time.Sleep(250 * time.Millisecond)
-		if err := d.run("systemctl", "restart", "syncphony-kiosk.service"); err != nil {
-			d.log.Printf("boxd: reload failed: %v", err)
-		}
+		d.restartKiosk("reload (remote care)", false)
 	}()
 }

boxd/internal/daemon/daemon.go

@@ -1,6 +1,8 @@
 // Package daemon is boxd's running half: the box bridge on loopback
-// (ADR 0016) and the local screens the kiosk shows when there's no server
-// to display — setup with a QR code, and the offline "can't reach" screen.
+// (ADR 0016), the local screens the kiosk shows when there's no server
+// to display — setup with a QR code, and the offline "can't reach"
+// screen — and the watchdogs and thermals that make the box heal itself
+// (MAD-806, ADR 0008).
 package daemon
 
 import (
@@ -24,16 +26,6 @@ const (
 	setupWindow  = 10 * time.Minute // after SIGUSR1 (stage 3 wires the OK long-press)
 	healthzCache = 2 * time.Second
 
-	// The offline screen takes over when the server has been unreachable
-	// for this long while the page is alive (heartbeats fresh): the kiosk is
-	// restarted through the local start page, which shows the offline
-	// screen and returns to /tv on its own once /healthz answers. After the
-	// restart there are no heartbeats (the page is local), so this cannot
-	// loop; the cooldown is a second guard.
-	offlineGrace      = 2 * time.Minute
-	heartbeatFresh    = 90 * time.Second
-	kioskRestartGuard = 10 * time.Minute
-
 	// Pairing durability (MAD-804, ADR 0006): Chromium writes its cookie
 	// store and localStorage lazily, so the writes that make this box stay
 	// paired sit in memory and the page cache for a while. After the
@@ -88,25 +80,35 @@ type Daemon struct {
 	// a func field so tests can count calls instead of syncing.
 	syncDisk func() error
 
-	setupUntil   time.Time // LAN setup window open until then (zero = closed)
-	lastBeat     time.Time // last /v1/heartbeat from the page
-	lastEvent    string
-	lastEventAt  time.Time
-	lastOK       bool      // last /healthz answer
-	lastAt       time.Time // when it was checked
-	offlineSince time.Time // server unreachable continuously since
-	lastRestart  time.Time // last kiosk restart triggered by the offline watch
-	lastFlush    time.Time // last pairing-durability sync burst
+	// vcgencmd samples the hardware thermals; a func field so tests can
+	// feed readings without a Pi.
+	vcgencmd func(args ...string) (string, error)
+
+	startedAt      time.Time   // boxd start; watchdog graces count from here
+	setupUntil     time.Time   // LAN setup window open until then (zero = closed)
+	lastBeat       time.Time   // last /v1/heartbeat from the page
+	lastEvent      string      // last /v1/events type (the nightly reload waits out "playing")
+	lastEventAt    time.Time   // when it arrived
+	lastOK         bool        // last /healthz answer
+	lastAt         time.Time   // when it was checked
+	offlineSince   time.Time   // server unreachable continuously since
+	lastRestart    time.Time   // last kiosk restart boxd performed (settle window anchor)
+	restarts       []time.Time // watchdog-caused kiosk restarts inside watchdogWindow
+	lastNightlyDay string      // local day the nightly reload last ran ("2006-01-02")
+	therm          thermals    // last thermal sample (zero = none yet)
+	lastFlush      time.Time   // last pairing-durability sync burst
 }
 
 // New builds a daemon around the current syncphony.txt (the SB_* env hooks
 // from the config package apply).
 func New(version string, log *log.Logger, run config.RunCommand) *Daemon {
 	d := &Daemon{
-		version: version,
-		log:     log,
-		run:     run,
-		paths:   config.PathsFromEnv(),
+		version:   version,
+		log:       log,
+		run:       run,
+		paths:     config.PathsFromEnv(),
+		startedAt: time.Now(),
+		vcgencmd:  realVcgencmd,
 		syncDisk: func() error {
 			syscall.Sync()
 			return nil
@@ -196,6 +198,7 @@ func (d *Daemon) Run() error {
 
 	d.log.Printf("boxd: %s bridge on http://%s (server: %s)", d.version, bridgeAddr, d.serverOrNone())
 	go d.watch()
+	go d.sampleThermalsLoop()
 
 	sig := make(chan os.Signal, 4)
 	signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1, syscall.SIGUSR2)
@@ -313,42 +316,6 @@ func (d *Daemon) flushDisk(event string) {
 	}()
 }
 
-// watch is the background loop: the offline screen trigger. While the page
-// is alive (fresh heartbeats) but the server stays unreachable past the
-// grace period, restart the kiosk so it re-enters through the local start
-// page — which shows the offline screen and returns on its own.
-func (d *Daemon) watch() {
-	tick := time.NewTicker(5 * time.Second)
-	defer tick.Stop()
-	for range tick.C {
-		if !d.configured() {
-			continue
-		}
-		ok := d.serverReachable()
-		d.mu.Lock()
-		now := time.Now()
-		if ok {
-			d.offlineSince = time.Time{}
-		} else if d.offlineSince.IsZero() {
-			d.offlineSince = now
-		}
-		restart := !d.offlineSince.IsZero() &&
-			now.Sub(d.offlineSince) >= offlineGrace &&
-			!d.lastBeat.IsZero() && now.Sub(d.lastBeat) < heartbeatFresh &&
-			(d.lastRestart.IsZero() || now.Sub(d.lastRestart) > kioskRestartGuard)
-		if restart {
-			d.lastRestart = now
-		}
-		d.mu.Unlock()
-		if restart {
-			d.log.Printf("boxd: server unreachable for %s with a live page; restarting the kiosk onto the offline screen", offlineGrace)
-			if err := d.run("systemctl", "restart", "syncphony-kiosk.service"); err != nil {
-				d.log.Printf("boxd: kiosk restart failed: %v", err)
-			}
-		}
-	}
-}
-
 // storageHealthy reports whether /data — the only partition that survives
 // the overlay (MAD-805, ADR 0007) — is mounted. Without it the box boots
 // anyway, but the pairing, the applied config and the logs are gone (they

boxd/internal/daemon/screens.go

@@ -12,8 +12,10 @@ import (
 // handleStatus reports everything the local screens (and tests) need.
 func (d *Daemon) handleStatus(w http.ResponseWriter, r *http.Request) {
 	server := d.serverURL()
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(map[string]any{
+	d.mu.Lock()
+	th := d.therm
+	d.mu.Unlock()
+	status := map[string]any{
 		"configured": server != "",
 		"server":     server,
 		"reachable":  d.serverReachable(),
@@ -24,7 +26,14 @@ func (d *Daemon) handleStatus(w http.ResponseWriter, r *http.Request) {
 		"ips":        boxIPs(),
 		"network":    networkState(),
 		"storage":    storageHealthy(),
-	})
+	}
+	if th.valid() {
+		for k, v := range th.infoFields() {
+			status[k] = v
+		}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(status)
 }
 
 func hostname() string {

boxd/internal/daemon/setup.go

@@ -94,9 +94,7 @@ func (d *Daemon) handleSetupSave(w http.ResponseWriter, r *http.Request) {
 		d.run("systemctl", "try-restart", "avahi-daemon.service")
 	}
 	d.closeSetupWindow()
-	if err := d.run("systemctl", "restart", "syncphony-kiosk.service"); err != nil {
-		d.log.Printf("boxd: setup: kiosk restart failed: %v", err)
-	}
+	d.restartKiosk("setup saved; entering the server through the start page", false)
 
 	writePage(w, savedPageHTML(serverURL))
 }