fix(net): survive edge connection-table flushes — DERP/disco resilience

.gitignore

@@ -17,6 +17,7 @@ host/machine_app_runner
 # host unit-test binaries
 host/test_clock_guard
 host/test_crypto_kat
+host/test_demote_veto
 # cppcheck
 *.ctu-info
 cppcheck-build/

components/microlink/components/wireguard_lwip/src/wireguard.h

@@ -182,6 +182,15 @@ struct wireguard_peer {
 	uint32_t last_tx;
 	uint32_t last_rx;
 
+	// last_rx of a data packet that arrived on the DIRECT UDP path (a real
+	// source address; DERP-relayed injections carry 0.0.0.0). Authenticated
+	// direct data is first-class liveness evidence for the direct path —
+	// consumed by wireguardif_peer_direct_rx_age() for demote-verification.
+	uint32_t last_direct_rx;
+	// Worst inter-frame gap across ANY path — the "received nothing" window
+	// that maps to a heartbeat-timeout disarm (2026-08-12 edge-flush metric).
+	uint32_t worst_rx_gap;
+
 	// We set this flag on RX/TX of packets if we think that we should initiate a new handshake
 	bool send_handshake;
 };

components/microlink/components/wireguard_lwip/src/wireguardif.c

@@ -79,6 +79,11 @@ volatile int wg_verbose_logging = 0;
 // network wedge from userspace. Count failures here so /state.json can surface
 // the symptom instead of leaving it invisible.
 volatile uint32_t wireguardif_pbuf_alloc_fails = 0;
+// TX failures by cause (run-20/21 ENOTCONN forensics — see wireguardif_output_to_peer):
+// keypair_expired = session hit REJECT_AFTER_TIME (rekey starved ~60 s prior);
+// no_valid_keys   = subsequent sends with no session at all (surface as errno 128).
+volatile uint32_t wireguardif_tx_keypair_expired = 0;
+volatile uint32_t wireguardif_tx_no_valid_keys = 0;
 
 #define WIREGUARDIF_TIMER_MSECS 400
 
@@ -348,11 +353,18 @@ static err_t wireguardif_output_to_peer(struct netif *netif, struct pbuf *q, con
 			}
 		} else {
 			// key has expired...
+			// Diag (run-20/21 ENOTCONN forensics): this exit means the session
+			// aged past REJECT_AFTER_TIME with no successful rekey — i.e. the
+			// handshake leg has been failing for ~REJECT-REKEY = 60 s already.
+			wireguardif_tx_keypair_expired++;
 			keypair_destroy(keypair);
 			result = ERR_CONN;
 		}
 	} else {
 		// No valid keys!
+		// Diag: every send in this state surfaces to the app as ENOTCONN
+		// (errno 128) — the "peer-present-but-no-keypair" signature.
+		wireguardif_tx_no_valid_keys++;
 		result = ERR_CONN;
 	}
 	return result;
@@ -509,8 +521,22 @@ static void wireguardif_process_data_message(struct wireguard_device *device, st
 					update_peer_addr(peer, addr, port);
 
 					now = wireguard_sys_now();
+					// Worst "received nothing" window across ANY path — the metric
+					// that maps to a disarm (a >2s gap == a heartbeat timeout). ~200ms
+					// steady-state, spikes to the stall duration during an edge flush.
+					// Path-agnostic on purpose: a relay-only gap misses the
+					// direct->relay handover (2026-08-12 edge-flush investigation).
+					if (peer->last_rx != 0) {
+						uint32_t gap = now - peer->last_rx;
+						if (gap > peer->worst_rx_gap) peer->worst_rx_gap = gap;
+					}
 					keypair->last_rx = now;
 					peer->last_rx = now;
+					// Direct-path liveness: only a real outer source refreshes it
+					// (DERP injections arrive as 0.0.0.0; see update_peer_addr above).
+					if (!ip_addr_isany(addr)) {
+						peer->last_direct_rx = now;
+					}
 
 					// Might need to shuffle next key --> current keypair
 					keypair_update(peer, keypair);
@@ -1023,6 +1049,71 @@ err_t wireguardif_peer_is_up(struct netif *netif, u8_t peer_index, ip_addr_t *cu
 	return result;
 }
 
+// Session-key freshness for this peer (run-20/21 ENOTCONN forensics):
+// *keypair_age_ms  = age of curr_keypair (0xFFFFFFFF when no valid keypair —
+//                    sends are failing ENOTCONN right now);
+// *init_tx_age_ms  = age of our last handshake-initiation TX (0xFFFFFFFF when
+//                    never sent). keypair_age > REKEY_AFTER_TIME (120 s) with
+//                    a recent init_tx = a rekey is IN FLIGHT and failing.
+err_t wireguardif_peer_handshake_age(struct netif *netif, u8_t peer_index, u32_t *keypair_age_ms, u32_t *init_tx_age_ms) {
+	struct wireguard_peer *peer;
+	err_t result = wireguardif_lookup_peer(netif, peer_index, &peer);
+	if (result == ERR_OK) {
+		uint32_t now = wireguard_sys_now();
+		if (keypair_age_ms) {
+			*keypair_age_ms = peer->curr_keypair.valid ? (now - peer->curr_keypair.keypair_millis) : 0xFFFFFFFFu;
+		}
+		if (init_tx_age_ms) {
+			*init_tx_age_ms = (peer->last_initiation_tx != 0) ? (now - peer->last_initiation_tx) : 0xFFFFFFFFu;
+		}
+	}
+	return result;
+}
+
+// Age (ms) of the last authenticated data packet received from this peer on
+// ANY path (direct or DERP-relayed). ERR_VAL when no data has ever arrived.
+// Proves the WG session itself is live — used to veto control-plane teardowns
+// of an actively-used safety session (hitless re-ingest).
+err_t wireguardif_peer_rx_age(struct netif *netif, u8_t peer_index, u32_t *age_ms) {
+	struct wireguard_peer *peer;
+	err_t result = wireguardif_lookup_peer(netif, peer_index, &peer);
+	if (result == ERR_OK) {
+		if (peer->last_rx == 0) {
+			result = ERR_VAL;
+		} else if (age_ms) {
+			*age_ms = wireguard_sys_now() - peer->last_rx;
+		}
+	}
+	return result;
+}
+
+// Age (ms) of the last authenticated data packet received from this peer on
+// the DIRECT UDP path. ERR_VAL when no direct data has ever arrived (fresh
+// peer, or all traffic DERP-relayed). Age is computed against the same
+// wireguard_sys_now() base that stamps it, so callers never mix time bases.
+err_t wireguardif_peer_direct_rx_age(struct netif *netif, u8_t peer_index, u32_t *age_ms) {
+	struct wireguard_peer *peer;
+	err_t result = wireguardif_lookup_peer(netif, peer_index, &peer);
+	if (result == ERR_OK) {
+		if (peer->last_direct_rx == 0) {
+			result = ERR_VAL;
+		} else if (age_ms) {
+			*age_ms = wireguard_sys_now() - peer->last_direct_rx;
+		}
+	}
+	return result;
+}
+
+// Worst inter-frame rx gap (any path) for this peer — the "received nothing"
+// window that maps to a heartbeat-timeout disarm. ~2000 during an edge flush
+// pinpoints a ~2s inbound stall on THIS node even if its DERP conn stayed up.
+err_t wireguardif_peer_worst_rx_gap(struct netif *netif, u8_t peer_index, u32_t *worst_gap_ms) {
+	struct wireguard_peer *peer;
+	err_t result = wireguardif_lookup_peer(netif, peer_index, &peer);
+	if (result == ERR_OK && worst_gap_ms) *worst_gap_ms = peer->worst_rx_gap;
+	return result;
+}
+
 err_t wireguardif_remove_peer(struct netif *netif, u8_t peer_index) {
 	struct wireguard_peer *peer;
 	err_t result = wireguardif_lookup_peer(netif, peer_index, &peer);

components/microlink/components/wireguard_lwip/src/wireguardif.h

@@ -134,6 +134,30 @@ err_t wireguardif_disconnect(struct netif *netif, u8_t peer_index);
 // Is the given peer "up"? A peer is up if it has a valid session key it can communicate with
 err_t wireguardif_peer_is_up(struct netif *netif, u8_t peer_index, ip_addr_t *current_ip, u16_t *current_port);
 
+// Age (ms) of the last authenticated data packet received from this peer on
+// the DIRECT UDP path (DERP-relayed rx never refreshes it). ERR_VAL when no
+// direct data has ever arrived. Used for direct-path demote-verification.
+err_t wireguardif_peer_direct_rx_age(struct netif *netif, u8_t peer_index, u32_t *age_ms);
+
+// Age (ms) of the last authenticated data packet received on ANY path.
+// ERR_VAL when none ever. Used to veto control-plane teardowns of an
+// actively-used safety session (hitless re-ingest).
+err_t wireguardif_peer_rx_age(struct netif *netif, u8_t peer_index, u32_t *age_ms);
+
+// Worst inter-frame rx gap (any path) — the "received nothing" window that
+// maps to a heartbeat-timeout disarm (edge-flush receive-stall metric).
+err_t wireguardif_peer_worst_rx_gap(struct netif *netif, u8_t peer_index, u32_t *worst_gap_ms);
+
+// Session-key freshness: curr-keypair age (0xFFFFFFFF = none valid, sends are
+// failing ENOTCONN) and last handshake-initiation TX age. Diag + the
+// negotiator's rekey-in-flight freeze.
+err_t wireguardif_peer_handshake_age(struct netif *netif, u8_t peer_index, u32_t *keypair_age_ms, u32_t *init_tx_age_ms);
+
+// TX failures by cause: keypair expired (rekey starved past REJECT_AFTER_TIME)
+// vs no-valid-keys (every such send surfaces as ENOTCONN/errno 128).
+extern volatile uint32_t wireguardif_tx_keypair_expired;
+extern volatile uint32_t wireguardif_tx_no_valid_keys;
+
 // Register a DERP relay output callback for peers without direct endpoints
 // This callback is invoked when a WireGuard packet needs to be sent to a peer
 // that has no direct IP endpoint (ip is 0.0.0.0 or port is 0)

components/microlink/include/microlink_internal.h

@@ -229,6 +229,29 @@ extern "C"
                                                 * safety impact (dual-path send
                                                 * keeps the uplink gapless anyway). */
 #define ML_DISCO_PRIORITY_REPROBE_MS 5000
+/* Demote-verification (see ml_demote_verdict.h): the PATH_DEAD_MS comment
+ * above assumes a spurious demotion is harmless ("re-probe restores direct
+ * in ~5s"). 2026-08-11 falsified that for a symmetric-NAT/USB-NCM safety
+ * peer: demotion tears down a direct path that may never re-establish
+ * without a reboot, onto a relay too jittery for the 2 s pstop timeout. So:
+ * hold the demote while authenticated WG data still arrives DIRECT within
+ * this window. 1000 ms = 5 missed 5 Hz heartbeats — a path that quiet is
+ * genuinely suspect and may demote; a working path refreshes every 200 ms. */
+#define ML_DEMOTE_DIRECT_RX_FRESH_MS 1000
+/* Cap on consecutive demote-veto ticks (~1 s apart): past this, the disco
+ * side-channel has been dead for ~10 s while data still flows — stop pinning
+ * and let the demote proceed (DERP fallback + re-probe is the safer state;
+ * see ml_peer_t.demote_veto_ticks). */
+#define ML_DEMOTE_VETO_MAX_TICKS 10
+/* Hitless re-ingest (run-20 green drop, 2026-08-11): a coord/netmap teardown
+ * (re-key retire, REMOVE) of a safety peer is VETOED while the WG session
+ * shows authenticated data rx (any path) within this window — an immediate
+ * wireguardif_remove_peer makes heartbeat sends fail ENOTCONN until a fresh
+ * handshake, and >2 s of that is a machine STOP. 2000 ms = the pstop
+ * heartbeat timeout: data within it means the safety bond is alive by
+ * definition. A genuine re-key/removal goes stale within seconds (the old
+ * key stops authenticating) and applies on the next update. */
+#define ML_TEARDOWN_RX_FRESH_MS 2000
 
 /* Relay-bound direct-path re-establishment (net/derp-direct-reestablish):
  * while a SAFETY peer's session is alive but riding DERP, periodically re-run
@@ -520,6 +543,26 @@ extern "C"
     uint64_t relay_retry_next_ms;
     uint8_t relay_retry_count;
 
+    /* Demote-verification veto streak (ml_demote_verdict.h): consecutive
+     * maintenance ticks the veto held this peer's direct path. Capped at
+     * ML_DEMOTE_VETO_MAX_TICKS — a veto that persists that long means the
+     * disco side-channel is durably broken even though data flows; fail
+     * toward the DERP demote rather than pin the path indefinitely (the
+     * audited a175361 hazard: RX-only evidence can pin a TX-dead path).
+     * Reset when no demote trigger fires or a demote executes. */
+    uint8_t demote_veto_ticks;
+
+    /* Learn-from-ping ring eviction (run-20/21 finding B-1): with the 8-slot
+     * endpoint table full of dead symmetric-NAT candidates, the append-only
+     * learn silently dropped the ONE live candidate a rebooted peer presents
+     * — a terminal, reboot-surviving wedge on the MACHINE side. When full,
+     * learned candidates now overwrite the last two slots round-robin. */
+    uint8_t learn_evict_next;
+
+    /* Disco-reset v2: per-peer rate limit (was a single global stamp that let
+     * two stuck peers starve each other). 0 = never fired. */
+    uint64_t disco_reset_next_ms;
+
     /* Chip<->chip CMM chain breaker (see ML_DISCO_CMM_MIN_INTERVAL_MS): ms of
      * the last CallMeMaybe SENT to this peer. 0 = never sent. */
     uint64_t last_cmm_sent_ms;

components/microlink/include/ml_demote_verdict.h

@@ -0,0 +1,77 @@
+// SPDX-FileCopyrightText: 2026 Polymath Robotics
+// SPDX-License-Identifier: Apache-2.0
+
+/* Demote-verification verdict — pure, host-testable (host/test_demote_veto.c).
+ *
+ * Direct-path demotion (ml_wg_mgr.c disco maintenance) is decided by the DISCO
+ * side-channel: a trust-lease expiry or the priority-peer pong watchdog. But
+ * disco rides its own UDP flow; on a jittery uplink (USB-NCM tether) its pings
+ * can go silent while the WireGuard DATA flow — the 5 Hz safety heartbeat —
+ * still arrives on the direct path. Demoting then TEARS DOWN A WORKING DIRECT
+ * PATH (wireguardif_connect_derp clears the endpoint) and dumps the safety
+ * heartbeat onto a relay measured too jittery to hold green (2026-08-11: DUT
+ * relay rtt spikes >1.2 s in BOTH regions vs the 2.0 s pstop timeout).
+ *
+ * Verdict: authenticated WG data received ON THE DIRECT PATH within fresh_ms
+ * is stronger liveness evidence than a missing disco pong — hold the path
+ * (VETO). A truly dead path stops producing direct rx, the age blows past
+ * fresh_ms within ~fresh_ms, and demotion proceeds unchanged (GO). Safety
+ * (priority/health-tracked) peers only: bulk tailnet peers keep the plain
+ * lease behavior.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+
+typedef enum
+{
+  ML_DEMOTE_NONE = 0, /* no demote trigger — leave the path alone       */
+  ML_DEMOTE_VETO, /* trigger fired, but direct WG rx proves liveness */
+  ML_DEMOTE_GO, /* trigger fired and nothing vouches for the path  */
+} ml_demote_verdict_t;
+
+static inline ml_demote_verdict_t ml_demote_verdict(
+  bool lease_expired,
+  bool pong_dead,
+  bool is_safety_peer,
+  bool direct_rx_age_valid, /* false = no direct data rx ever recorded */
+  uint32_t direct_rx_age_ms,
+  uint32_t fresh_ms)
+{
+  if (!lease_expired && !pong_dead) return ML_DEMOTE_NONE;
+  if (is_safety_peer && direct_rx_age_valid && direct_rx_age_ms <= fresh_ms) return ML_DEMOTE_VETO;
+  return ML_DEMOTE_GO;
+}
+
+/* Veto-streak cap (audited a175361 hazard): the veto's evidence is RX-only —
+ * it can pin a path whose TX leg is dead. A veto that persists for max_ticks
+ * consecutive ticks (~1 s apart) means disco has been durably silent while
+ * data flows; past the cap the verdict degrades to GO so the peer fails over
+ * to DERP + re-probe instead of being pinned indefinitely. */
+static inline ml_demote_verdict_t ml_demote_verdict_capped(
+  ml_demote_verdict_t verdict, uint32_t veto_streak_ticks, uint32_t max_ticks)
+{
+  if (verdict == ML_DEMOTE_VETO && veto_streak_ticks >= max_ticks) return ML_DEMOTE_GO;
+  return verdict;
+}
+
+/* Hitless re-ingest (2026-08-11 green drop, run-20): a coord/netmap event
+ * (re-sync, re-key retire, REMOVE, cap-evict) must never invalidate a WG
+ * session that is actively passing authenticated safety data — an immediate
+ * wireguardif_remove_peer makes the 5 Hz heartbeat sends fail ENOTCONN until
+ * a fresh handshake lands, and >2 s of that is a machine STOP. Same principle
+ * as the demote verdict above, one layer up: authenticated data rx within
+ * fresh_ms outranks a control-plane teardown signal. A GENUINE re-key or
+ * removal goes stale within seconds (the old key stops authenticating), so
+ * deferring costs one update cycle; vetoing a working session saves the bond.
+ * Safety peers only — bulk tailnet peers keep the plain teardown behavior. */
+static inline bool ml_teardown_veto(
+  bool is_safety_peer,
+  bool rx_age_valid, /* false = no authenticated data rx ever recorded */
+  uint32_t rx_age_ms, /* age of last authenticated data rx, ANY path */
+  uint32_t fresh_ms)
+{
+  return is_safety_peer && rx_age_valid && rx_age_ms <= fresh_ms;
+}

components/microlink/src/ml_config_httpd.c

@@ -1026,12 +1026,72 @@ static esp_err_t handler_monitor(httpd_req_t * req)
        *                    includes bulk peers; a climbing regains_safety
        *                    with flat direct_relay_bound = the safety path
        *                    itself is oscillating. */
-      extern void ml_wg_get_disco_obs_diag(uint32_t[3]);
-      uint32_t od[3] = {0};
+      extern void ml_wg_get_disco_obs_diag(uint32_t[4]);
+      uint32_t od[4] = {0};
       ml_wg_get_disco_obs_diag(od);
       cJSON_AddNumberToObject(json, "probe_tbl_hw", od[0]);
       cJSON_AddNumberToObject(json, "cmm_rx_count", od[1]);
       cJSON_AddNumberToObject(json, "regains_safety", od[2]);
+      /* demote_vetoes: direct demotes vetoed because authenticated WG data was
+       * still arriving on the direct path (disco-silent-but-data-alive). Climbs
+       * ~1/s while such an episode lasts — a nonzero value means the demote-
+       * verification saved a working direct path from a spurious teardown. */
+      cJSON_AddNumberToObject(json, "demote_vetoes", od[3]);
+
+      /* Hitless re-ingest (run-20 green-drop forensics): which WG session-
+       * teardown path fired, and whether the live-safety-session veto saved
+       * the bond. rekey_retires / peer_removes = teardowns APPLIED;
+       * *_vetoes = teardowns of an actively-authenticating safety session
+       * DEFERRED instead; evict_safety_skips = LRU eviction passed over a
+       * safety peer. */
+      extern void ml_wg_get_reingest_diag(uint32_t[6]);
+      uint32_t rg[6] = {0};
+      ml_wg_get_reingest_diag(rg);
+      cJSON_AddNumberToObject(json, "rekey_retires", rg[0]);
+      cJSON_AddNumberToObject(json, "rekey_retire_vetoes", rg[1]);
+      cJSON_AddNumberToObject(json, "peer_removes", rg[2]);
+      cJSON_AddNumberToObject(json, "remove_vetoes", rg[3]);
+      cJSON_AddNumberToObject(json, "evict_safety_skips", rg[4]);
+      cJSON_AddNumberToObject(json, "relay_disco_resets", rg[5]);
+
+      /* WG session health (run-20/21 ENOTCONN forensics). The failure
+       * signature to watch: wg_kp_age_max climbing past 120000 (rekey
+       * starving) -> wg_tx_keypair_expired increments (session died) ->
+       * wg_tx_no_valid_keys climbing (every send now ENOTCONN). Correlate
+       * with derp_route_fallbacks (handshake frames black-holed to a DERP
+       * conn the peer isn't on) and ep_learn_evictions (the symmetric-NAT
+       * table wedge being absorbed instead of going terminal). */
+      extern volatile uint32_t wireguardif_tx_keypair_expired;
+      extern volatile uint32_t wireguardif_tx_no_valid_keys;
+      cJSON_AddNumberToObject(json, "wg_tx_keypair_expired", wireguardif_tx_keypair_expired);
+      cJSON_AddNumberToObject(json, "wg_tx_no_valid_keys", wireguardif_tx_no_valid_keys);
+      extern uint32_t ml_derp_get_route_fallbacks(void);
+      cJSON_AddNumberToObject(json, "derp_route_fallbacks", ml_derp_get_route_fallbacks());
+      /* home_pumps: home-conn rx drains performed DURING an aux DERP connect,
+       * so the safety heartbeat relay keeps flowing through a ~1-2s aux TLS
+       * handshake (edge-flush fix; replaced the self-deadlocking defer-aux).
+       * Climbs while a standby (re)connects; green should hold across it. */
+      extern uint32_t ml_derp_get_home_pumps(void);
+      cJSON_AddNumberToObject(json, "derp_home_pumps", ml_derp_get_home_pumps());
+      /* Home-DERP reconnect speed (edge-flush fix): worst < ~1500ms confirms
+       * the relay is back inside the 2s pstop timeout so green rides it
+       * through a firewall connection-table flush. */
+      extern void ml_derp_get_reconnect_diag(uint32_t[3]);
+      uint32_t rc[3] = {0};
+      ml_derp_get_reconnect_diag(rc);
+      cJSON_AddNumberToObject(json, "derp_reconnects", rc[0]);
+      cJSON_AddNumberToObject(json, "derp_reconnect_last_ms", rc[1]);
+      cJSON_AddNumberToObject(json, "derp_reconnect_worst_ms", rc[2]);
+      extern void ml_wg_get_session_diag(microlink_t *, uint32_t[5]);
+      uint32_t sd[5] = {0};
+      ml_wg_get_session_diag(ml, sd);
+      cJSON_AddNumberToObject(json, "ep_learn_evictions", sd[0]);
+      cJSON_AddNumberToObject(json, "wg_kp_age_max", sd[1]);
+      cJSON_AddNumberToObject(json, "wg_init_age_max", sd[2]);
+      /* Receive-stall metric (2026-08-12 edge-flush): worst any-path inter-frame
+       * rx gap from a safety peer. ~2000 during a flush = a ~2s inbound stall on
+       * THIS node = the disarm cause, even with derp_reconnects=0. */
+      cJSON_AddNumberToObject(json, "rx_worst_gap_ms", sd[3]);
 
       /* Same-LAN direct-path diagnostics for the priority peer (the machine):
        * what LAN endpoint we advertise, which candidate endpoints we hold for