diff --git a/.gitignore b/.gitignore index e465116..28f771e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ host/machine_app_runner # host unit-test binaries host/test_clock_guard host/test_crypto_kat +host/test_demote_veto # cppcheck *.ctu-info cppcheck-build/ diff --git a/components/microlink/components/wireguard_lwip/src/wireguard.h b/components/microlink/components/wireguard_lwip/src/wireguard.h index 11d6af2..6ad29f2 100644 --- a/components/microlink/components/wireguard_lwip/src/wireguard.h +++ b/components/microlink/components/wireguard_lwip/src/wireguard.h @@ -182,6 +182,15 @@ struct wireguard_peer { uint32_t last_tx; uint32_t last_rx; + // last_rx of a data packet that arrived on the DIRECT UDP path (a real + // source address; DERP-relayed injections carry 0.0.0.0). Authenticated + // direct data is first-class liveness evidence for the direct path — + // consumed by wireguardif_peer_direct_rx_age() for demote-verification. + uint32_t last_direct_rx; + // Worst inter-frame gap across ANY path — the "received nothing" window + // that maps to a heartbeat-timeout disarm (2026-08-12 edge-flush metric). + uint32_t worst_rx_gap; + // We set this flag on RX/TX of packets if we think that we should initiate a new handshake bool send_handshake; }; diff --git a/components/microlink/components/wireguard_lwip/src/wireguardif.c b/components/microlink/components/wireguard_lwip/src/wireguardif.c index 0a91b64..c263b54 100644 --- a/components/microlink/components/wireguard_lwip/src/wireguardif.c +++ b/components/microlink/components/wireguard_lwip/src/wireguardif.c @@ -79,6 +79,11 @@ volatile int wg_verbose_logging = 0; // network wedge from userspace. Count failures here so /state.json can surface // the symptom instead of leaving it invisible. volatile uint32_t wireguardif_pbuf_alloc_fails = 0; +// TX failures by cause (run-20/21 ENOTCONN forensics — see wireguardif_output_to_peer): +// keypair_expired = session hit REJECT_AFTER_TIME (rekey starved ~60 s prior); +// no_valid_keys = subsequent sends with no session at all (surface as errno 128). +volatile uint32_t wireguardif_tx_keypair_expired = 0; +volatile uint32_t wireguardif_tx_no_valid_keys = 0; #define WIREGUARDIF_TIMER_MSECS 400 @@ -348,11 +353,18 @@ static err_t wireguardif_output_to_peer(struct netif *netif, struct pbuf *q, con } } else { // key has expired... + // Diag (run-20/21 ENOTCONN forensics): this exit means the session + // aged past REJECT_AFTER_TIME with no successful rekey — i.e. the + // handshake leg has been failing for ~REJECT-REKEY = 60 s already. + wireguardif_tx_keypair_expired++; keypair_destroy(keypair); result = ERR_CONN; } } else { // No valid keys! + // Diag: every send in this state surfaces to the app as ENOTCONN + // (errno 128) — the "peer-present-but-no-keypair" signature. + wireguardif_tx_no_valid_keys++; result = ERR_CONN; } return result; @@ -509,8 +521,22 @@ static void wireguardif_process_data_message(struct wireguard_device *device, st update_peer_addr(peer, addr, port); now = wireguard_sys_now(); + // Worst "received nothing" window across ANY path — the metric + // that maps to a disarm (a >2s gap == a heartbeat timeout). ~200ms + // steady-state, spikes to the stall duration during an edge flush. + // Path-agnostic on purpose: a relay-only gap misses the + // direct->relay handover (2026-08-12 edge-flush investigation). + if (peer->last_rx != 0) { + uint32_t gap = now - peer->last_rx; + if (gap > peer->worst_rx_gap) peer->worst_rx_gap = gap; + } keypair->last_rx = now; peer->last_rx = now; + // Direct-path liveness: only a real outer source refreshes it + // (DERP injections arrive as 0.0.0.0; see update_peer_addr above). + if (!ip_addr_isany(addr)) { + peer->last_direct_rx = now; + } // Might need to shuffle next key --> current keypair keypair_update(peer, keypair); @@ -1023,6 +1049,71 @@ err_t wireguardif_peer_is_up(struct netif *netif, u8_t peer_index, ip_addr_t *cu return result; } +// Session-key freshness for this peer (run-20/21 ENOTCONN forensics): +// *keypair_age_ms = age of curr_keypair (0xFFFFFFFF when no valid keypair — +// sends are failing ENOTCONN right now); +// *init_tx_age_ms = age of our last handshake-initiation TX (0xFFFFFFFF when +// never sent). keypair_age > REKEY_AFTER_TIME (120 s) with +// a recent init_tx = a rekey is IN FLIGHT and failing. +err_t wireguardif_peer_handshake_age(struct netif *netif, u8_t peer_index, u32_t *keypair_age_ms, u32_t *init_tx_age_ms) { + struct wireguard_peer *peer; + err_t result = wireguardif_lookup_peer(netif, peer_index, &peer); + if (result == ERR_OK) { + uint32_t now = wireguard_sys_now(); + if (keypair_age_ms) { + *keypair_age_ms = peer->curr_keypair.valid ? (now - peer->curr_keypair.keypair_millis) : 0xFFFFFFFFu; + } + if (init_tx_age_ms) { + *init_tx_age_ms = (peer->last_initiation_tx != 0) ? (now - peer->last_initiation_tx) : 0xFFFFFFFFu; + } + } + return result; +} + +// Age (ms) of the last authenticated data packet received from this peer on +// ANY path (direct or DERP-relayed). ERR_VAL when no data has ever arrived. +// Proves the WG session itself is live — used to veto control-plane teardowns +// of an actively-used safety session (hitless re-ingest). +err_t wireguardif_peer_rx_age(struct netif *netif, u8_t peer_index, u32_t *age_ms) { + struct wireguard_peer *peer; + err_t result = wireguardif_lookup_peer(netif, peer_index, &peer); + if (result == ERR_OK) { + if (peer->last_rx == 0) { + result = ERR_VAL; + } else if (age_ms) { + *age_ms = wireguard_sys_now() - peer->last_rx; + } + } + return result; +} + +// Age (ms) of the last authenticated data packet received from this peer on +// the DIRECT UDP path. ERR_VAL when no direct data has ever arrived (fresh +// peer, or all traffic DERP-relayed). Age is computed against the same +// wireguard_sys_now() base that stamps it, so callers never mix time bases. +err_t wireguardif_peer_direct_rx_age(struct netif *netif, u8_t peer_index, u32_t *age_ms) { + struct wireguard_peer *peer; + err_t result = wireguardif_lookup_peer(netif, peer_index, &peer); + if (result == ERR_OK) { + if (peer->last_direct_rx == 0) { + result = ERR_VAL; + } else if (age_ms) { + *age_ms = wireguard_sys_now() - peer->last_direct_rx; + } + } + return result; +} + +// Worst inter-frame rx gap (any path) for this peer — the "received nothing" +// window that maps to a heartbeat-timeout disarm. ~2000 during an edge flush +// pinpoints a ~2s inbound stall on THIS node even if its DERP conn stayed up. +err_t wireguardif_peer_worst_rx_gap(struct netif *netif, u8_t peer_index, u32_t *worst_gap_ms) { + struct wireguard_peer *peer; + err_t result = wireguardif_lookup_peer(netif, peer_index, &peer); + if (result == ERR_OK && worst_gap_ms) *worst_gap_ms = peer->worst_rx_gap; + return result; +} + err_t wireguardif_remove_peer(struct netif *netif, u8_t peer_index) { struct wireguard_peer *peer; err_t result = wireguardif_lookup_peer(netif, peer_index, &peer); diff --git a/components/microlink/components/wireguard_lwip/src/wireguardif.h b/components/microlink/components/wireguard_lwip/src/wireguardif.h index b665c7c..0e7af52 100644 --- a/components/microlink/components/wireguard_lwip/src/wireguardif.h +++ b/components/microlink/components/wireguard_lwip/src/wireguardif.h @@ -134,6 +134,30 @@ err_t wireguardif_disconnect(struct netif *netif, u8_t peer_index); // Is the given peer "up"? A peer is up if it has a valid session key it can communicate with err_t wireguardif_peer_is_up(struct netif *netif, u8_t peer_index, ip_addr_t *current_ip, u16_t *current_port); +// Age (ms) of the last authenticated data packet received from this peer on +// the DIRECT UDP path (DERP-relayed rx never refreshes it). ERR_VAL when no +// direct data has ever arrived. Used for direct-path demote-verification. +err_t wireguardif_peer_direct_rx_age(struct netif *netif, u8_t peer_index, u32_t *age_ms); + +// Age (ms) of the last authenticated data packet received on ANY path. +// ERR_VAL when none ever. Used to veto control-plane teardowns of an +// actively-used safety session (hitless re-ingest). +err_t wireguardif_peer_rx_age(struct netif *netif, u8_t peer_index, u32_t *age_ms); + +// Worst inter-frame rx gap (any path) — the "received nothing" window that +// maps to a heartbeat-timeout disarm (edge-flush receive-stall metric). +err_t wireguardif_peer_worst_rx_gap(struct netif *netif, u8_t peer_index, u32_t *worst_gap_ms); + +// Session-key freshness: curr-keypair age (0xFFFFFFFF = none valid, sends are +// failing ENOTCONN) and last handshake-initiation TX age. Diag + the +// negotiator's rekey-in-flight freeze. +err_t wireguardif_peer_handshake_age(struct netif *netif, u8_t peer_index, u32_t *keypair_age_ms, u32_t *init_tx_age_ms); + +// TX failures by cause: keypair expired (rekey starved past REJECT_AFTER_TIME) +// vs no-valid-keys (every such send surfaces as ENOTCONN/errno 128). +extern volatile uint32_t wireguardif_tx_keypair_expired; +extern volatile uint32_t wireguardif_tx_no_valid_keys; + // Register a DERP relay output callback for peers without direct endpoints // This callback is invoked when a WireGuard packet needs to be sent to a peer // that has no direct IP endpoint (ip is 0.0.0.0 or port is 0) diff --git a/components/microlink/include/microlink_internal.h b/components/microlink/include/microlink_internal.h index b2f4e23..0ca808d 100644 --- a/components/microlink/include/microlink_internal.h +++ b/components/microlink/include/microlink_internal.h @@ -229,6 +229,29 @@ extern "C" * safety impact (dual-path send * keeps the uplink gapless anyway). */ #define ML_DISCO_PRIORITY_REPROBE_MS 5000 +/* Demote-verification (see ml_demote_verdict.h): the PATH_DEAD_MS comment + * above assumes a spurious demotion is harmless ("re-probe restores direct + * in ~5s"). 2026-08-11 falsified that for a symmetric-NAT/USB-NCM safety + * peer: demotion tears down a direct path that may never re-establish + * without a reboot, onto a relay too jittery for the 2 s pstop timeout. So: + * hold the demote while authenticated WG data still arrives DIRECT within + * this window. 1000 ms = 5 missed 5 Hz heartbeats — a path that quiet is + * genuinely suspect and may demote; a working path refreshes every 200 ms. */ +#define ML_DEMOTE_DIRECT_RX_FRESH_MS 1000 +/* Cap on consecutive demote-veto ticks (~1 s apart): past this, the disco + * side-channel has been dead for ~10 s while data still flows — stop pinning + * and let the demote proceed (DERP fallback + re-probe is the safer state; + * see ml_peer_t.demote_veto_ticks). */ +#define ML_DEMOTE_VETO_MAX_TICKS 10 +/* Hitless re-ingest (run-20 green drop, 2026-08-11): a coord/netmap teardown + * (re-key retire, REMOVE) of a safety peer is VETOED while the WG session + * shows authenticated data rx (any path) within this window — an immediate + * wireguardif_remove_peer makes heartbeat sends fail ENOTCONN until a fresh + * handshake, and >2 s of that is a machine STOP. 2000 ms = the pstop + * heartbeat timeout: data within it means the safety bond is alive by + * definition. A genuine re-key/removal goes stale within seconds (the old + * key stops authenticating) and applies on the next update. */ +#define ML_TEARDOWN_RX_FRESH_MS 2000 /* Relay-bound direct-path re-establishment (net/derp-direct-reestablish): * while a SAFETY peer's session is alive but riding DERP, periodically re-run @@ -520,6 +543,26 @@ extern "C" uint64_t relay_retry_next_ms; uint8_t relay_retry_count; + /* Demote-verification veto streak (ml_demote_verdict.h): consecutive + * maintenance ticks the veto held this peer's direct path. Capped at + * ML_DEMOTE_VETO_MAX_TICKS — a veto that persists that long means the + * disco side-channel is durably broken even though data flows; fail + * toward the DERP demote rather than pin the path indefinitely (the + * audited a175361 hazard: RX-only evidence can pin a TX-dead path). + * Reset when no demote trigger fires or a demote executes. */ + uint8_t demote_veto_ticks; + + /* Learn-from-ping ring eviction (run-20/21 finding B-1): with the 8-slot + * endpoint table full of dead symmetric-NAT candidates, the append-only + * learn silently dropped the ONE live candidate a rebooted peer presents + * — a terminal, reboot-surviving wedge on the MACHINE side. When full, + * learned candidates now overwrite the last two slots round-robin. */ + uint8_t learn_evict_next; + + /* Disco-reset v2: per-peer rate limit (was a single global stamp that let + * two stuck peers starve each other). 0 = never fired. */ + uint64_t disco_reset_next_ms; + /* Chip<->chip CMM chain breaker (see ML_DISCO_CMM_MIN_INTERVAL_MS): ms of * the last CallMeMaybe SENT to this peer. 0 = never sent. */ uint64_t last_cmm_sent_ms; diff --git a/components/microlink/include/ml_demote_verdict.h b/components/microlink/include/ml_demote_verdict.h new file mode 100644 index 0000000..972694d --- /dev/null +++ b/components/microlink/include/ml_demote_verdict.h @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: 2026 Polymath Robotics +// SPDX-License-Identifier: Apache-2.0 + +/* Demote-verification verdict — pure, host-testable (host/test_demote_veto.c). + * + * Direct-path demotion (ml_wg_mgr.c disco maintenance) is decided by the DISCO + * side-channel: a trust-lease expiry or the priority-peer pong watchdog. But + * disco rides its own UDP flow; on a jittery uplink (USB-NCM tether) its pings + * can go silent while the WireGuard DATA flow — the 5 Hz safety heartbeat — + * still arrives on the direct path. Demoting then TEARS DOWN A WORKING DIRECT + * PATH (wireguardif_connect_derp clears the endpoint) and dumps the safety + * heartbeat onto a relay measured too jittery to hold green (2026-08-11: DUT + * relay rtt spikes >1.2 s in BOTH regions vs the 2.0 s pstop timeout). + * + * Verdict: authenticated WG data received ON THE DIRECT PATH within fresh_ms + * is stronger liveness evidence than a missing disco pong — hold the path + * (VETO). A truly dead path stops producing direct rx, the age blows past + * fresh_ms within ~fresh_ms, and demotion proceeds unchanged (GO). Safety + * (priority/health-tracked) peers only: bulk tailnet peers keep the plain + * lease behavior. + */ + +#pragma once + +#include +#include + +typedef enum +{ + ML_DEMOTE_NONE = 0, /* no demote trigger — leave the path alone */ + ML_DEMOTE_VETO, /* trigger fired, but direct WG rx proves liveness */ + ML_DEMOTE_GO, /* trigger fired and nothing vouches for the path */ +} ml_demote_verdict_t; + +static inline ml_demote_verdict_t ml_demote_verdict( + bool lease_expired, + bool pong_dead, + bool is_safety_peer, + bool direct_rx_age_valid, /* false = no direct data rx ever recorded */ + uint32_t direct_rx_age_ms, + uint32_t fresh_ms) +{ + if (!lease_expired && !pong_dead) return ML_DEMOTE_NONE; + if (is_safety_peer && direct_rx_age_valid && direct_rx_age_ms <= fresh_ms) return ML_DEMOTE_VETO; + return ML_DEMOTE_GO; +} + +/* Veto-streak cap (audited a175361 hazard): the veto's evidence is RX-only — + * it can pin a path whose TX leg is dead. A veto that persists for max_ticks + * consecutive ticks (~1 s apart) means disco has been durably silent while + * data flows; past the cap the verdict degrades to GO so the peer fails over + * to DERP + re-probe instead of being pinned indefinitely. */ +static inline ml_demote_verdict_t ml_demote_verdict_capped( + ml_demote_verdict_t verdict, uint32_t veto_streak_ticks, uint32_t max_ticks) +{ + if (verdict == ML_DEMOTE_VETO && veto_streak_ticks >= max_ticks) return ML_DEMOTE_GO; + return verdict; +} + +/* Hitless re-ingest (2026-08-11 green drop, run-20): a coord/netmap event + * (re-sync, re-key retire, REMOVE, cap-evict) must never invalidate a WG + * session that is actively passing authenticated safety data — an immediate + * wireguardif_remove_peer makes the 5 Hz heartbeat sends fail ENOTCONN until + * a fresh handshake lands, and >2 s of that is a machine STOP. Same principle + * as the demote verdict above, one layer up: authenticated data rx within + * fresh_ms outranks a control-plane teardown signal. A GENUINE re-key or + * removal goes stale within seconds (the old key stops authenticating), so + * deferring costs one update cycle; vetoing a working session saves the bond. + * Safety peers only — bulk tailnet peers keep the plain teardown behavior. */ +static inline bool ml_teardown_veto( + bool is_safety_peer, + bool rx_age_valid, /* false = no authenticated data rx ever recorded */ + uint32_t rx_age_ms, /* age of last authenticated data rx, ANY path */ + uint32_t fresh_ms) +{ + return is_safety_peer && rx_age_valid && rx_age_ms <= fresh_ms; +} diff --git a/components/microlink/src/ml_config_httpd.c b/components/microlink/src/ml_config_httpd.c index 93f9c5d..16003dd 100644 --- a/components/microlink/src/ml_config_httpd.c +++ b/components/microlink/src/ml_config_httpd.c @@ -1026,12 +1026,72 @@ static esp_err_t handler_monitor(httpd_req_t * req) * includes bulk peers; a climbing regains_safety * with flat direct_relay_bound = the safety path * itself is oscillating. */ - extern void ml_wg_get_disco_obs_diag(uint32_t[3]); - uint32_t od[3] = {0}; + extern void ml_wg_get_disco_obs_diag(uint32_t[4]); + uint32_t od[4] = {0}; ml_wg_get_disco_obs_diag(od); cJSON_AddNumberToObject(json, "probe_tbl_hw", od[0]); cJSON_AddNumberToObject(json, "cmm_rx_count", od[1]); cJSON_AddNumberToObject(json, "regains_safety", od[2]); + /* demote_vetoes: direct demotes vetoed because authenticated WG data was + * still arriving on the direct path (disco-silent-but-data-alive). Climbs + * ~1/s while such an episode lasts — a nonzero value means the demote- + * verification saved a working direct path from a spurious teardown. */ + cJSON_AddNumberToObject(json, "demote_vetoes", od[3]); + + /* Hitless re-ingest (run-20 green-drop forensics): which WG session- + * teardown path fired, and whether the live-safety-session veto saved + * the bond. rekey_retires / peer_removes = teardowns APPLIED; + * *_vetoes = teardowns of an actively-authenticating safety session + * DEFERRED instead; evict_safety_skips = LRU eviction passed over a + * safety peer. */ + extern void ml_wg_get_reingest_diag(uint32_t[6]); + uint32_t rg[6] = {0}; + ml_wg_get_reingest_diag(rg); + cJSON_AddNumberToObject(json, "rekey_retires", rg[0]); + cJSON_AddNumberToObject(json, "rekey_retire_vetoes", rg[1]); + cJSON_AddNumberToObject(json, "peer_removes", rg[2]); + cJSON_AddNumberToObject(json, "remove_vetoes", rg[3]); + cJSON_AddNumberToObject(json, "evict_safety_skips", rg[4]); + cJSON_AddNumberToObject(json, "relay_disco_resets", rg[5]); + + /* WG session health (run-20/21 ENOTCONN forensics). The failure + * signature to watch: wg_kp_age_max climbing past 120000 (rekey + * starving) -> wg_tx_keypair_expired increments (session died) -> + * wg_tx_no_valid_keys climbing (every send now ENOTCONN). Correlate + * with derp_route_fallbacks (handshake frames black-holed to a DERP + * conn the peer isn't on) and ep_learn_evictions (the symmetric-NAT + * table wedge being absorbed instead of going terminal). */ + extern volatile uint32_t wireguardif_tx_keypair_expired; + extern volatile uint32_t wireguardif_tx_no_valid_keys; + cJSON_AddNumberToObject(json, "wg_tx_keypair_expired", wireguardif_tx_keypair_expired); + cJSON_AddNumberToObject(json, "wg_tx_no_valid_keys", wireguardif_tx_no_valid_keys); + extern uint32_t ml_derp_get_route_fallbacks(void); + cJSON_AddNumberToObject(json, "derp_route_fallbacks", ml_derp_get_route_fallbacks()); + /* home_pumps: home-conn rx drains performed DURING an aux DERP connect, + * so the safety heartbeat relay keeps flowing through a ~1-2s aux TLS + * handshake (edge-flush fix; replaced the self-deadlocking defer-aux). + * Climbs while a standby (re)connects; green should hold across it. */ + extern uint32_t ml_derp_get_home_pumps(void); + cJSON_AddNumberToObject(json, "derp_home_pumps", ml_derp_get_home_pumps()); + /* Home-DERP reconnect speed (edge-flush fix): worst < ~1500ms confirms + * the relay is back inside the 2s pstop timeout so green rides it + * through a firewall connection-table flush. */ + extern void ml_derp_get_reconnect_diag(uint32_t[3]); + uint32_t rc[3] = {0}; + ml_derp_get_reconnect_diag(rc); + cJSON_AddNumberToObject(json, "derp_reconnects", rc[0]); + cJSON_AddNumberToObject(json, "derp_reconnect_last_ms", rc[1]); + cJSON_AddNumberToObject(json, "derp_reconnect_worst_ms", rc[2]); + extern void ml_wg_get_session_diag(microlink_t *, uint32_t[5]); + uint32_t sd[5] = {0}; + ml_wg_get_session_diag(ml, sd); + cJSON_AddNumberToObject(json, "ep_learn_evictions", sd[0]); + cJSON_AddNumberToObject(json, "wg_kp_age_max", sd[1]); + cJSON_AddNumberToObject(json, "wg_init_age_max", sd[2]); + /* Receive-stall metric (2026-08-12 edge-flush): worst any-path inter-frame + * rx gap from a safety peer. ~2000 during a flush = a ~2s inbound stall on + * THIS node = the disarm cause, even with derp_reconnects=0. */ + cJSON_AddNumberToObject(json, "rx_worst_gap_ms", sd[3]); /* Same-LAN direct-path diagnostics for the priority peer (the machine): * what LAN endpoint we advertise, which candidate endpoints we hold for diff --git a/components/microlink/src/ml_derp.c b/components/microlink/src/ml_derp.c index aee0f00..5f3d823 100644 --- a/components/microlink/src/ml_derp.c +++ b/components/microlink/src/ml_derp.c @@ -556,6 +556,60 @@ esp_err_t ml_derp_queue_send(microlink_t * ml, const uint8_t * dest_key, const u * safety peer's frame is NEVER silently dropped (the caller logs the * fallback). Returns NULL only when the chosen conn is down (nothing can * carry it this pass). */ +/* Frames stamped for a non-home region that had NO connected aux conn and + * fell back to the HOME conn. A DERP server only delivers to clients + * connected to IT — so unless the destination peer also sits on our home + * region, every one of these frames is silently eaten server-side ("does not + * know about peer"). This is the black-hole that starves WG rekeys after a + * region switch when a peer-region record is stale (run-20/21 forensics). */ +static uint32_t s_diag_route_home_fallbacks; + +uint32_t ml_derp_get_route_fallbacks(void) +{ + return s_diag_route_home_fallbacks; +} + +/* Drain up to a few pending frames from the HOME conn — called from inside a + * blocking AUX connect so the safety heartbeat relay riding home keeps flowing + * during the ~1-2s aux TLS handshake (2026-08-12 edge-flush fix; replaces the + * self-deadlocking defer-aux guard). Single-task, so re-entrant use of + * poll_derp_read on a DIFFERENT conn is safe. Stops on no-data (bounded 100ms + * per read) or error (the main loop handles a dead home conn after we return). */ +static uint32_t s_diag_home_pumps; /* home-rx drains performed during aux connects */ + +static void derp_pump_home_rx(microlink_t * ml) +{ + ml_derp_conn_t * home = &ml->derp[ml->derp_home_slot]; + if (!home->connected || home->sockfd < 0) return; + for (int i = 0; i < 4; i++) { + int r = poll_derp_read(ml, home); + if (r <= 0) break; /* 0 = no more data; <0 = home needs the main loop */ + home->last_recv_ms = ml_get_time_ms(); + s_diag_home_pumps++; + } +} + +uint32_t ml_derp_get_home_pumps(void) +{ + return s_diag_home_pumps; +} + +/* Home-DERP reconnect telemetry (2026-08-12 edge-flush fix). reconnects = + * count of successful RST/EOF-triggered home reconnects; last/worst = + * wall-clock ms from RST detection to reconnected. worst < ~1500 confirms the + * relay is back inside the 2 s pstop timeout window (green rides it through a + * flush); a worst climbing toward/over 2000 means recovery is still too slow. */ +static uint32_t s_diag_derp_reconnects; +static uint32_t s_diag_derp_last_reconnect_ms; +static uint32_t s_diag_derp_worst_reconnect_ms; + +void ml_derp_get_reconnect_diag(uint32_t out[3]) +{ + out[0] = s_diag_derp_reconnects; + out[1] = s_diag_derp_last_reconnect_ms; + out[2] = s_diag_derp_worst_reconnect_ms; +} + static ml_derp_conn_t * derp_route_conn(microlink_t * ml, uint16_t region_id, uint16_t eff_home) { int hs = ml->derp_home_slot; @@ -567,6 +621,7 @@ static ml_derp_conn_t * derp_route_conn(microlink_t * ml, uint16_t region_id, ui return &ml->derp[s]; } } + s_diag_route_home_fallbacks++; } return home->connected ? home : NULL; } @@ -655,6 +710,13 @@ static void derp_manage_aux(microlink_t * ml, uint16_t eff_home, int aux_burst[] * inbound path) for tens of seconds → nuisance STOP risk. We attempt one * region and return; the rest are picked up on the next task iteration, * AFTER home connect/drain/read have run again. (Adversarial review 2026-08-04.) */ + + /* Home-rx protection (2026-08-12): the one-connect-per-call cap above bounds + * N back-to-back connects, but a single aux TLS handshake still takes ~1-2s. + * Rather than DEFER the aux (the removed guard self-deadlocked a cross-region + * relay-bound safety peer — @claude PR review), ml_derp_connect now services + * the home conn's rx during an aux handshake (derp_pump_home_rx), so the + * safety heartbeat relay keeps flowing while the standby (re)connects. */ for (int w = 0; w < nwant; w++) { uint16_t rid = want[w]; bool served = false; @@ -1210,10 +1272,21 @@ void ml_derp_tx_task(void * arg) if (bits & ML_EVT_DERP_RECONNECT) { xEventGroupClearBits(ml->events, ML_EVT_DERP_RECONNECT); ESP_LOGW(TAG, "DERP reconnect requested (home was %s)", home->connected ? "connected" : "disconnected"); + uint64_t reconn_t0 = ml_get_time_ms(); ml_derp_disconnect(ml, home); verbose_phase = false; - /* Auto-reconnect after disconnect */ - vTaskDelay(pdMS_TO_TICKS(1000)); + /* IMMEDIATE first reconnect (run-20/21/22 root cause, 2026-08-12 + * DUT-host wire+journal evidence): an RST/EOF here is almost always a + * middlebox connection-table FLUSH (the office edge firewall RSTs + * long-lived DERP TCP across regions and rebinds NAT in the same + * instant), NOT a down server. The old unconditional 1000 ms pre-delay + * + TLS handshake pushed relay recovery past the 2 s pstop timeout — + * and because the same flush kills the direct hairpin simultaneously, + * the heartbeat had NO path for those seconds -> green drop. tailscaled + * recovers in <1 s (connGen++); match it: try NOW, back off (2 s) only + * on CONSECUTIVE failures (a genuinely-down server). A short yield lets + * ml_derp_disconnect's socket close settle without stalling recovery. */ + vTaskDelay(pdMS_TO_TICKS(20)); for (int attempt = 0; attempt < 3 && !home->connected; attempt++) { if (attempt > 0) { ESP_LOGW(TAG, "DERP reconnect retry %d/3 in 2s...", attempt + 1); @@ -1222,6 +1295,13 @@ void ml_derp_tx_task(void * arg) if (ml_derp_connect(ml, home, ml_effective_home_region(ml)) == ESP_OK) { connected_since_ms = ml_get_time_ms(); verbose_phase = true; + s_diag_derp_reconnects++; + s_diag_derp_last_reconnect_ms = (uint32_t)(ml_get_time_ms() - reconn_t0); + if (s_diag_derp_last_reconnect_ms > s_diag_derp_worst_reconnect_ms) { + s_diag_derp_worst_reconnect_ms = s_diag_derp_last_reconnect_ms; + } + ESP_LOGW( + TAG, "DERP home reconnected in %ums (attempt %d)", (unsigned)s_diag_derp_last_reconnect_ms, attempt + 1); break; } ESP_LOGW(TAG, "DERP reconnect attempt %d failed", attempt + 1); @@ -1539,10 +1619,25 @@ esp_err_t ml_derp_connect(microlink_t * ml, ml_derp_conn_t * c, uint16_t region_ c->sockfd = sock; mbedtls_ssl_set_bio(&c->ssl, &c->sockfd, ml_derp_bio_send, NULL, ml_derp_bio_recv_timeout); - /* TLS handshake - socket has 10s SO_RCVTIMEO from connect phase. */ + /* TLS handshake. For an AUX conn, shorten the read timeout to 100 ms so the + * BIO returns SSL_TIMEOUT between flights (mbedTLS resumes cleanly on a TLS + * stream) — we service the home conn's rx there so a ~1-2s aux handshake + * doesn't gap the safety heartbeat relay riding home (2026-08-12 edge-flush + * fix; replaces defer-aux). Home keeps the long timeout. Overall deadline is + * unchanged (DERP_CONNECT_TIMEOUT_MS). */ + if (!is_home) { + mbedtls_ssl_conf_read_timeout(&c->ssl_conf, 100); + } + uint64_t hs_deadline = ml_get_time_ms() + DERP_CONNECT_TIMEOUT_MS; int ret; while ((ret = mbedtls_ssl_handshake(&c->ssl)) != 0) { - if (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE) { + if (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE || ret == MBEDTLS_ERR_SSL_TIMEOUT) { + if (!is_home) derp_pump_home_rx(ml); /* keep the safety relay flowing */ + if (ml_get_time_ms() > hs_deadline) { + ESP_LOGE(TAG, "TLS handshake timeout (%s)", is_home ? "home" : "aux"); + DERP_CONNECT_FAIL_CLEANUP(); + return ESP_FAIL; + } continue; } char err_buf[128]; @@ -1551,6 +1646,11 @@ esp_err_t ml_derp_connect(microlink_t * ml, ml_derp_conn_t * c, uint16_t region_ DERP_CONNECT_FAIL_CLEANUP(); return ESP_FAIL; } + /* Restore the long read timeout for the HTTP-upgrade + DERP-handshake reads + * below (those are fast, few round-trips — no per-flight pumping needed). */ + if (!is_home) { + mbedtls_ssl_conf_read_timeout(&c->ssl_conf, DERP_CONNECT_TIMEOUT_MS); + } int64_t t_derp_tls = esp_timer_get_time(); ESP_LOGW(TAG, "[TIMING-DERP] TLS handshake: %lld ms", (t_derp_tls - t_derp_tcp) / 1000); diff --git a/components/microlink/src/ml_wg_mgr.c b/components/microlink/src/ml_wg_mgr.c index f669eee..478751e 100644 --- a/components/microlink/src/ml_wg_mgr.c +++ b/components/microlink/src/ml_wg_mgr.c @@ -33,6 +33,7 @@ #include "mbedtls/base64.h" #include "microlink_internal.h" #include "ml_config_httpd.h" +#include "ml_demote_verdict.h" #include "nacl_box.h" #include "wireguard.h" #include "wireguardif.h" @@ -836,6 +837,10 @@ static uint32_t s_diag_relay_retries; /* CMM + forced-sweep rounds fired */ static uint32_t s_diag_direct_regains; /* has_direct_path false -> true edges */ static uint64_t s_last_relay_refetch_ms; /* rate-limit coord re-fetch on relay-stuck symmetric-NAT safety peer */ static uint32_t s_diag_relay_refetch_reqs; /* coord re-fetch (reconnect) requests issued for endpoint refresh */ +static uint32_t s_diag_relay_disco_resets; /* per-peer from-scratch disco resets on a relay-stuck safety peer + * (v2: rate-limited per-peer via p->disco_reset_next_ms) */ +static uint32_t s_diag_ep_learn_evictions; /* learn-from-ping ring-evictions on a full endpoint table — + * nonzero = the B-1 wedge trigger occurred and was absorbed */ /* DISCO observability counters — the 2026-08-09 regains-oscillation root cause * rested on CALCULATED probe-table/CMM dynamics; these measure them instead. @@ -847,12 +852,92 @@ static uint32_t s_diag_regains_safety; /* direct regains on SAFETY peers only (p * or health-tracked) — s_diag_direct_regains * also counts bulk tailnet peers, which masked * whether the SAFETY path was the one flapping */ +static uint32_t s_diag_demote_vetoes; /* direct demotes vetoed by fresh direct WG data + * rx (ml_demote_verdict.h) — counts veto-ticks, + * so it climbs ~1/s for the duration of a + * disco-silent-but-data-alive episode */ +static uint64_t s_last_demote_veto_log_ms; /* rate-limit the veto log line */ + +/* Hitless re-ingest counters (run-20 green drop, 2026-08-11): the three WG + * session-teardown paths (re-key retire, coord REMOVE, cap-evict) were + * log-only, so the ENOTCONN green-drop forensics could not tell WHICH fired. + * Counters-not-logs, read via /admin/api/monitor. The *_vetoes counters mark + * teardowns of an ACTIVELY-USED safety session that were deferred instead + * (ml_teardown_veto): nonzero = the guard saved a live safety bond. */ +static uint32_t s_diag_rekey_retires; /* stale-entry retires on a key change */ +static uint32_t s_diag_rekey_retire_vetoes; /* ... vetoed: session still live */ +static uint32_t s_diag_peer_removes; /* coord ML_PEER_REMOVE teardowns applied */ +static uint32_t s_diag_remove_vetoes; /* ... vetoed: session still live */ +static uint32_t s_diag_evict_safety_skips; /* LRU evict skipped a safety peer */ + +void ml_wg_get_reingest_diag(uint32_t out[6]) +{ + out[0] = s_diag_rekey_retires; + out[1] = s_diag_rekey_retire_vetoes; + out[2] = s_diag_peer_removes; + out[3] = s_diag_remove_vetoes; + out[4] = s_diag_evict_safety_skips; + out[5] = s_diag_relay_disco_resets; /* from-scratch disco resets on a relay-stuck safety peer (@claude review) */ +} + +/* WG session-health diag (run-20/21 ENOTCONN forensics): learn-evictions + + * the WORST safety-peer keypair/init ages. wg_kp_age_max steadily ~<120000 + * (rekey cadence) = healthy; climbing past 120000 = a rekey is starving; a + * green drop then follows ~60 s later when REJECT_AFTER_TIME kills the key. */ +void ml_wg_get_session_diag(microlink_t * ml, uint32_t out[5]) +{ + out[0] = s_diag_ep_learn_evictions; + out[1] = 0; /* max safety keypair age ms (0xFFFFFFFF = a peer has NO valid key) */ + out[2] = 0; /* max safety handshake-init TX age ms */ + out[3] = 0; /* max safety worst any-path rx gap ms (edge-flush receive-stall metric) */ + out[4] = 0; /* reserved */ + if (ml == NULL || ml->wg_netif == NULL) return; + for (int i = 0; i < ml->peer_count; i++) { + ml_peer_t * p = &ml->peers[i]; + if (!p->active || p->wg_peer_index < 0) continue; + bool safety = + (ml->config.priority_peer_ip != 0 && p->vpn_ip == ml->config.priority_peer_ip) || is_health_tracked(p->vpn_ip); + if (!safety) continue; + u32_t kp_age = 0, init_age = 0; + if ( + wireguardif_peer_handshake_age((struct netif *)ml->wg_netif, (u8_t)p->wg_peer_index, &kp_age, &init_age) == + ERR_OK) + { + if (kp_age > out[1]) out[1] = kp_age; + if (init_age != 0xFFFFFFFFu && init_age > out[2]) out[2] = init_age; + } + u32_t rx_worst = 0; + if ( + wireguardif_peer_worst_rx_gap((struct netif *)ml->wg_netif, (u8_t)p->wg_peer_index, &rx_worst) == ERR_OK && + rx_worst > out[3]) + { + out[3] = rx_worst; + } + } +} + +/* True when tearing down this peer's WG session must be deferred: it is a + * SAFETY peer (priority/health-tracked) and its session shows authenticated + * data rx within ML_TEARDOWN_RX_FRESH_MS — i.e. the 5 Hz safety heartbeat is + * flowing RIGHT NOW and a wireguardif_remove_peer would fail those sends + * ENOTCONN until a fresh handshake (>2 s = machine STOP). See + * ml_demote_verdict.h for the invariant argument. */ +static bool teardown_vetoed(microlink_t * ml, uint32_t vpn_ip, int wg_peer_index) +{ + bool is_safety = + (ml->config.priority_peer_ip != 0 && vpn_ip == ml->config.priority_peer_ip) || is_health_tracked(vpn_ip); + u32_t age_ms = 0; + bool valid = ml->wg_netif && wg_peer_index >= 0 && + wireguardif_peer_rx_age((struct netif *)ml->wg_netif, (u8_t)wg_peer_index, &age_ms) == ERR_OK; + return ml_teardown_veto(is_safety, valid, age_ms, ML_TEARDOWN_RX_FRESH_MS); +} -void ml_wg_get_disco_obs_diag(uint32_t out[3]) +void ml_wg_get_disco_obs_diag(uint32_t out[4]) { out[0] = s_diag_probe_tbl_hw; out[1] = s_diag_cmm_rx; out[2] = s_diag_regains_safety; + out[3] = s_diag_demote_vetoes; } void ml_wg_get_direct_retry_diag(microlink_t * ml, uint32_t out[4]) @@ -1124,6 +1209,41 @@ static void neg_note_auto_apply(microlink_t * ml, uint16_t region, uint8_t src, * than MBB for boot/cold recovery; behavior identical to today. * live bond — §7 MBB only (I1: the bond is never dropped by a switch), * gated by every §8 damping guard. */ +/* True while any safety peer's WG session is missing its keypair or a rekey + * is overdue with initiations actively firing. A region switch in that window + * re-routes the DERP leg the handshake needs (silent home-conn fallback + + * old-home reap, audit §1c) — the second-transition trigger behind both + * run-20/21 green drops (dwell expiry at T+10min, keypair death at T+~14min). + * Freezing the negotiator here costs only switch latency, never safety. */ +static bool safety_rekey_inflight(microlink_t * ml) +{ + if (!ml->wg_netif) return false; + for (int i = 0; i < ml->peer_count; i++) { + ml_peer_t * p = &ml->peers[i]; + if (!p->active || p->wg_peer_index < 0) continue; + bool safety = + (ml->config.priority_peer_ip != 0 && p->vpn_ip == ml->config.priority_peer_ip) || is_health_tracked(p->vpn_ip); + if (!safety) continue; + u32_t kp_age = 0, init_age = 0; + if ( + wireguardif_peer_handshake_age((struct netif *)ml->wg_netif, (u8_t)p->wg_peer_index, &kp_age, &init_age) != + ERR_OK) + { + continue; + } + if (kp_age == 0xFFFFFFFFu) { + /* No valid keypair: freeze only if we're actively trying to handshake + * (a never-connected passive peer must not pin the negotiator). */ + if (init_age != 0xFFFFFFFFu && init_age < 30000u) return true; + continue; + } + /* Keypair older than REKEY_AFTER_TIME (120 s, wireguard.h) with a recent + * initiation = a rekey is in flight and not completing. */ + if (kp_age > 120000u && init_age < 30000u) return true; + } + return false; +} + static void neg_apply_target(microlink_t * ml, uint16_t target, uint8_t src) { uint64_t now = ml_get_time_ms(); @@ -1189,6 +1309,13 @@ static void neg_apply_target(microlink_t * ml, uint16_t target, uint8_t src) s_neg_damping_suppressed++; return; } + if (safety_rekey_inflight(ml)) { + /* Never start a region transition while a safety session's handshake is + * struggling — the switch would re-route the very DERP leg the rekey + * needs (run-20/21 trigger). Re-evaluated next tick; costs latency only. */ + s_neg_damping_suppressed++; + return; + } if (s_neg_last_commit_ms != 0 && (now - s_neg_last_commit_ms) < ML_NEG_MIN_DWELL_MS) { s_neg_damping_suppressed++; return; @@ -1493,6 +1620,18 @@ static int add_peer(microlink_t * ml, const ml_peer_update_t * update) * the dead one — so retire the stale entry and reuse its slot. */ idx = find_peer_by_ip(ml, update->vpn_ip); if (idx >= 0) { + /* Hitless re-ingest: if the "stale" entry is a SAFETY peer whose session + * is authenticating data RIGHT NOW, the old key is provably still in use + * — this update is transient/bogus (netmap churn), not a real re-key. + * Defer: skip the whole update. A genuine re-key stops authenticating + * within seconds and the retry lands on the next coord update. */ + if (teardown_vetoed(ml, update->vpn_ip, ml->peers[idx].wg_peer_index)) { + s_diag_rekey_retire_vetoes++; + ESP_LOGW( + TAG, "IGNORING re-key retire for %s: session still passing authenticated data (deferred)", update->hostname); + return -1; + } + s_diag_rekey_retires++; ESP_LOGW(TAG, "Peer %s re-keyed — retiring stale entry (idx=%d)", update->hostname, idx); if (ml->peers[idx].wg_peer_index >= 0 && ml->wg_netif) { wireguardif_remove_peer((struct netif *)ml->wg_netif, (u8_t)ml->peers[idx].wg_peer_index); @@ -1532,6 +1671,15 @@ static int add_peer(microlink_t * ml, const ml_peer_update_t * update) for (int i = 0; i < ML_MAX_PEERS; i++) { if (!ml->peers[i].active) continue; if (is_pinned_peer(ml, ml->peers[i].vpn_ip)) continue; + /* Never LRU-evict a safety peer, pinned or not — evicting the peer + * carrying the 5 Hz heartbeat is a guaranteed machine STOP. */ + if ( + (ml->config.priority_peer_ip != 0 && ml->peers[i].vpn_ip == ml->config.priority_peer_ip) || + is_health_tracked(ml->peers[i].vpn_ip)) + { + s_diag_evict_safety_skips++; + continue; + } uint64_t last_activity = ml->peers[i].last_send_ms; if (ml->peers[i].last_pong_recv_ms > last_activity) last_activity = ml->peers[i].last_pong_recv_ms; if (last_activity < oldest_ms) { @@ -1724,6 +1872,23 @@ static void remove_peer(microlink_t * ml, const ml_peer_update_t * update) int idx = find_peer_by_key(ml, update->public_key); if (idx < 0) return; + /* Hitless re-ingest: a coord REMOVE for a SAFETY peer whose session is + * authenticating data RIGHT NOW is presumptively a server glitch (netmap + * trim/re-sync churn) — applying it fails the 5 Hz heartbeat ENOTCONN and + * stops the machine. Defer: keep the live session; a genuinely-removed + * peer's session goes stale within seconds and a later REMOVE (or the + * peer's own disappearance) applies. Fail toward keeping a working safety + * bond, never toward a tidy peer table. */ + if (teardown_vetoed(ml, ml->peers[idx].vpn_ip, ml->peers[idx].wg_peer_index)) { + s_diag_remove_vetoes++; + ESP_LOGW( + TAG, + "IGNORING coord REMOVE for %s: session still passing authenticated data (deferred)", + ml->peers[idx].hostname); + return; + } + s_diag_peer_removes++; + /* Remove from wireguard-lwip */ if (ml->wg_netif && ml->peers[idx].wg_peer_index >= 0) { struct netif * netif = (struct netif *)ml->wg_netif; @@ -2158,12 +2323,36 @@ static void process_disco_ping( break; } } - if (!known && p->endpoint_count < ML_MAX_ENDPOINTS) { - p->endpoints[p->endpoint_count].ip = pkt->src_ip; - p->endpoints[p->endpoint_count].port = pkt->src_port; - p->endpoints[p->endpoint_count].is_ipv6 = false; - p->endpoint_count++; - ESP_LOGI(TAG, "Learned candidate endpoint for %s from inbound direct ping", p->hostname); + if (!known) { + if (p->endpoint_count < ML_MAX_ENDPOINTS) { + p->endpoints[p->endpoint_count].ip = pkt->src_ip; + p->endpoints[p->endpoint_count].port = pkt->src_port; + p->endpoints[p->endpoint_count].is_ipv6 = false; + p->endpoint_count++; + ESP_LOGI(TAG, "Learned candidate endpoint for %s from inbound direct ping", p->hostname); + } else { + /* Ring-evict (run-20/21 finding B-1): the append-only learn silently + * dropped the candidate when the 8-slot table was full — and a + * symmetric NAT mints a new dead candidate per rebind, so the table + * WILL fill, after which the ONE live mapping a rebooted peer + * presents (the source of this very ping) could never be stored. + * Terminal, reboot-surviving wedge on the machine side (only a + * machine reboot cleared it: NVS persists 2 endpoints). Overwrite + * the last two slots round-robin — coord-delivered endpoints live at + * the front and are replaced wholesale by coord updates anyway. */ + int slot = ML_MAX_ENDPOINTS - 2 + (p->learn_evict_next & 1u); + p->learn_evict_next++; + p->endpoints[slot].ip = pkt->src_ip; + p->endpoints[slot].port = pkt->src_port; + p->endpoints[slot].is_ipv6 = false; + s_diag_ep_learn_evictions++; + ESP_LOGW( + TAG, + "Endpoint table full for %s: ring-evicted slot %d for learned candidate (evictions=%u)", + p->hostname, + slot, + (unsigned)s_diag_ep_learn_evictions); + } } disco_send_ping_to_peer(ml, peer_idx, true); } @@ -3173,6 +3362,16 @@ static void disco_periodic_probes(microlink_t * ml) * safety-peer treatment: 3 s disco heartbeats (which also keep * the path RTT sample fresh on BOTH ends) + pong watchdog. */ + /* EVERY safety peer gets the DERP data+handshake mirror (dual_path) — it + * was priority-peer-only, so the MACHINE's TX leg to its remotes (data + * replies AND rekey handshakes, wireguardif_peer_output) was direct-only + * and could die silently; a starved rekey then surfaces as ENOTCONN + * 180 s later (run-20/21 audit §1b). Reconciled here (idempotent flag + * write) because health-tracking can register after peer add. */ + if (is_priority && ml->wg_netif && p->wg_peer_index >= 0) { + wireguardif_set_dual_path((struct netif *)ml->wg_netif, (u8_t)p->wg_peer_index, true); + } + /* Direct-path health. Two triggers: * - trust-lease expiry (all peers, the original behaviour), and * - pong-recency watchdog (priority peer only): liveness is what @@ -3183,7 +3382,45 @@ static void disco_periodic_probes(microlink_t * ml) bool lease_expired = p->has_direct_path && now > p->trust_until_ms; bool pong_dead = p->has_direct_path && is_priority && p->last_pong_recv_ms != 0 && now - p->last_pong_recv_ms > ML_DISCO_PRIORITY_PATH_DEAD_MS; - if (lease_expired || pong_dead) { + + /* Demote-VERIFICATION (ml_demote_verdict.h): both triggers above rest on + * the DISCO side-channel, whose pings can go silent on a jittery uplink + * while the WG DATA flow (the 5 Hz safety heartbeat) still arrives on the + * direct path. Demoting then tears down a WORKING direct path — for a + * symmetric-NAT peer possibly terminally (2026-08-11: only a reboot + * re-established it, and the relay fallback is too jittery for the 2 s + * pstop timeout in ANY region). Authenticated direct data rx within + * ML_DEMOTE_DIRECT_RX_FRESH_MS outranks a missing pong: renew the lease + * and hold the path. A truly dead path stops producing direct rx within + * the window, and demotion proceeds unchanged. Safety peers only. */ + u32_t direct_rx_age_ms = 0; + bool direct_rx_age_valid = + (lease_expired || pong_dead) && is_priority && ml->wg_netif && p->wg_peer_index >= 0 && + wireguardif_peer_direct_rx_age((struct netif *)ml->wg_netif, (u8_t)p->wg_peer_index, &direct_rx_age_ms) == ERR_OK; + ml_demote_verdict_t demote = ml_demote_verdict( + lease_expired, pong_dead, is_priority, direct_rx_age_valid, direct_rx_age_ms, ML_DEMOTE_DIRECT_RX_FRESH_MS); + /* Veto-streak cap (audited a175361 hazard): the veto's evidence is + * RX-only and can pin a TX-dead path. Past the cap, fail over to DERP. + * Streak resets whenever the trigger clears or a demote executes. */ + demote = ml_demote_verdict_capped(demote, p->demote_veto_ticks, ML_DEMOTE_VETO_MAX_TICKS); + if (demote == ML_DEMOTE_VETO) { + p->demote_veto_ticks++; + p->trust_until_ms = now + ML_DISCO_TRUST_DURATION_MS; + s_diag_demote_vetoes++; + if (now - s_last_demote_veto_log_ms > 10000) { + s_last_demote_veto_log_ms = now; + ESP_LOGW( + TAG, + "direct demote VETOED for %s: WG data rx %ums ago on direct (disco %s, streak %u)", + p->hostname, + (unsigned)direct_rx_age_ms, + pong_dead ? "pong-dead" : "lease-expired", + (unsigned)p->demote_veto_ticks); + } + } else { + p->demote_veto_ticks = 0; + } + if (demote == ML_DEMOTE_GO) { ESP_LOGI( TAG, "Direct path to %s %s, reverting to DERP", @@ -3326,6 +3563,40 @@ static void disco_periodic_probes(microlink_t * ml) (void)xQueueSend(ml->coord_cmd_queue, &rc, 0); ESP_LOGW(TAG, "relay-stuck safety peer %s: coord re-fetch (reconnect) for endpoint refresh", p->hostname); } + /* Escalation: the coord re-fetch above refreshes the PEER's endpoints, + * but a re-fetch re-ingests an EXISTING peer with disco state PRESERVED + * (the !existing guard in ml_wg_apply_peer_update), so we keep re-probing + * the stale best_ip that no longer reaches a symmetric-NAT peer whose + * mapping rebound — which is why only a reboot recovered direct. After a + * few failed rounds, do the disco half of a per-peer "reboot": clear THIS + * relay-stuck safety peer's disco session so the next sweep re-runs a + * from-scratch hole-punch (fresh CMM + unthrottled ping to the just- + * refreshed endpoints). The peer then pings the machine from its LIVE + * mapping and the machine's !has_direct_path learn-from-ping adopts it. + * best_ip/pong timers are disco candidates only — the WG data path and the + * DERP-carried heartbeat are untouched. Gated to a relay-bound safety peer + * (no direct path to disturb) and rate-limited. */ + /* v2 (audit of 38fca7d): rate limit is PER-PEER (the global stamp let + * two stuck peers starve each other), best_ip/best_port are KEPT (the + * last-proven endpoint is "typically the ONLY reachable candidate" — + * wiping it destroyed a recovery vector when the peer's mapping had + * NOT rebound), and the hard-NAT flap backoff is cleared so the + * re-punch can actually probe. Ping/pong/CMM timers still reset so + * the next sweep runs unthrottled from scratch. */ + if (p->relay_retry_count >= 3u && (p->disco_reset_next_ms == 0 || now >= p->disco_reset_next_ms)) { + p->disco_reset_next_ms = now + ML_RELAY_REFETCH_MIN_MS; + s_diag_relay_disco_resets++; + p->last_ping_sent_ms = 0; + p->last_pong_recv_ms = 0; + p->last_cmm_sent_ms = 0; + p->trust_until_ms = 0; + p->direct_backoff_until = 0; + p->relay_retry_next_ms = 1; /* fire the fresh CMM + ping next sweep tick */ + ESP_LOGW( + TAG, + "relay-stuck safety peer %s: disco-session reset v2 (timers+backoff cleared, best_ip kept)", + p->hostname); + } } } diff --git a/docs/DERP_REGION_FOLLOW_CONVERGENCE_GAP.md b/docs/DERP_REGION_FOLLOW_CONVERGENCE_GAP.md new file mode 100644 index 0000000..65c2f65 --- /dev/null +++ b/docs/DERP_REGION_FOLLOW_CONVERGENCE_GAP.md @@ -0,0 +1,63 @@ +# DERP region auto-negotiation — region-follow convergence gap (KNOWN issue) + +**Status:** Known, **green-safe** (never drops the safety loop), documented here; robust fix **deferred** (design below). Not a blocker for the "12 h without losing green" goal. +**First observed:** run-19 soak, 2026-08-11, build `5d25354` — but the underlying mechanism predates it and is independent of the relay-recovery work on this branch. + +--- + +## 1. Symptom + +In DERP region **auto-negotiation** (`derp_region_override = 0` → `derp_region_source = auto-primary`), a remote follows the **primary machine's** region. When the machine's region changes, a remote **occasionally fails to follow** and stays homed on the *old* region. + +Observed on the bench (host-side, `/admin/api/monitor`): + +- machine → region 9; DUT followed (`derp_home_region=9`, `mbb_commits`++), **PS did not** (`derp_home_region=2`, `mbb_state=0`, `mbb_commits=0`, `mbb_proofs_*=0`). +- PS never *attempted* the switch — no MBB candidate, no proof activity — i.e. its computed **target region never became 9**. +- **Green never dropped.** machn stayed `armed` (l0=1 l1=1); both remotes stayed `wg_direct=1`. PS@region2 ↔ machn@region9 held a direct bond throughout. +- Frequency: rare — ~1 dropped follow in ~12 forced switches (run-18 had 11/11 clean; run-19 dropped 1 on its first switch). + +Net effect: a **region split** (machn + one remote on the new region, the other on the old), which is *cosmetic/optimization-level* while the direct path is up, not a safety event. + +## 2. Root cause + +A peer's DERP region reaches a remote **only** via a coordination-server `PeersChangedPatch` carrying `DERPRegion`. Two facts combine into a permanent-until-next-change stale state: + +1. **The patch can be dropped.** It is delivered through the peer-update queue. `net(derp): never drop the region-bearing peer patch` (commit `1b068a3`) front-queues region-bearing patches (2000 ms) like pinned full-adds, which *reduces* the drop rate but does **not** eliminate it under a heavy peer-churn burst on a busy tailnet. + +2. **There is no recovery for a dropped region patch.** `neg_target_region()` (`ml_wg_mgr.c`) reads the **cached** `peers[idx].derp_region`. Steady-state coordination updates are `OmitPeers=true` (`ml_coord.c` — `do_send_endpoint_update`), so a remote **never re-pulls** a peer's current region; a full re-sync only happens on a coord reconnect / reboot. And crucially, the coordination server only pushes a `PeersChanged` on an **actual change** — verified on the bench: re-issuing the machine's *unchanged* region (`POST /admin/api/settings {"derp_region":9}` while already 9) generated **no** new patch to PS. So once the patch drops, the remote's cached machine-region stays stale **until the machine's region genuinely changes again** (i.e. the next switch), at which point a fresh patch is generated — which may itself drop. + +This is the same steady-state "`OmitPeers=true`, never re-pull peer state" gap that also underlies the endpoint-staleness / relay-stuck issue (see `reference_pstop_autoneg_split_green_drop` and the relay-recovery work on branch `derp/relay-recovery`); here it manifests on the **region** field instead of **endpoints**. + +## 3. Why it is green-safe + +The 5 Hz protective-stop heartbeat rides the **WireGuard** tunnel, not the DERP home region: + +- When bonded **direct** (`wg_direct=1`, the normal state), packets go peer-to-peer and never touch a DERP relay — the home region is irrelevant to a running bond. A region mismatch between two directly-bonded peers therefore does **not** gap the heartbeat. +- The DERP relay leg is a fallback used only if the direct path dies. A region split raises the *cost* of that fallback (a cross-region relay hop), but for a LAN remote (PS) the direct path is stable, so green holds indefinitely. + +Hence: region-follow lag = a convergence/optimization miss, **not** a green-status loss. It fails only the strict "every switch converges both remotes" bar, not the "12 h without losing green" bar. + +## 4. Mitigation design (deferred — not yet implemented) + +Ranked; **option A is recommended**. + +### A. Carry the machine's authoritative region in-band over the pstop bond (recommended) +The machine is region-authoritative (§4) and knows its own region; the **pstop safety bond is the one channel guaranteed up whenever auto-primary is active** (green ⇒ bond up). Have the machine include its current effective DERP region in its periodic pstop reply, surface it through the dcs primary-machine slot table (`dcs_primary_machine_info`), and have `neg_target_region()` **prefer that reported region** over the possibly-stale `peers[idx].derp_region`. +- **Pros:** eliminates the coord-patch dependence entirely; convergence becomes as reliable as the bond itself; recovers automatically because it is re-sent every heartbeat. +- **Cons/constraints:** a pstop message-format addition — **safety-path-adjacent**. Must be strictly additive/optional and gated so a missing/garbled field can **never** affect the STOP/OK decision (per the "no test/aux logic in the safety path" and "protective-stop, not E-stop" rules). Bump the pstop protocol version; both roles must tolerate the field's absence for mixed-firmware fleets. + +### B. Periodic reconciliation re-fetch (auto-primary only) +When auto-primary is active and the primary machine is bonded, issue a bounded-cadence (e.g. every 2–5 min) `Stream=false, OmitPeers=false` MapRequest to re-pull peer `HomeDERP`s, so a missed region patch self-corrects. +- **Pros:** self-contained in microlink; also refreshes endpoints (helps the relay-stuck case). +- **Cons:** control-plane traffic; **depends on the coord full peer entry actually including `HomeDERP`** — but full re-syncs frequently deliver `derp_region=0` (region is carried separately via patch; see the region-preserve guard in `ml_wg_apply_peer_update`), so this may **not** reliably deliver the region. Needs server-side verification before relying on it. + +### C. Machine periodic re-advertise +Have the machine periodically re-advertise its region even absent a change, so a dropped patch is covered by the next advertise. +- **Cons:** the coord only emits a `PeersChanged` on an actual **change** (bench-verified), so a plain re-advertise of the unchanged region produces no patch. Would require a server-side change to re-push, or a machine-side region "toggle," which is hacky. **Not recommended.** + +**Recommendation:** Option **A** (in-band region over the pstop bond) is the robust, coord-independent fix. It should be scoped carefully as a safety-path-adjacent change (additive field, version-bumped, absence-tolerant) when this is picked up. + +## 5. Interim posture + +- Accept region-follow lag as a **known, green-safe** convergence miss; it does not gate the green soak. +- The relay-recovery work on `derp/relay-recovery` (disco-session reset for a relay-stuck safety peer) addresses the *endpoint* half of the same steady-state gap; the *region* half is documented here for a later, separate change. diff --git a/docs/MULTI_REMOTE_SCALE_CEILING.md b/docs/MULTI_REMOTE_SCALE_CEILING.md new file mode 100644 index 0000000..1093838 --- /dev/null +++ b/docs/MULTI_REMOTE_SCALE_CEILING.md @@ -0,0 +1,45 @@ +# Multi-remote scale ceiling (machn) + +**Status:** documented limit, 2026-08-12. Raising it is future work (see *Raising the ceiling*). +**One-line:** a single machine (machn) reliably holds **~3 bonded remotes** (comfortable) / **4 (marginal)**; **5+ collapses** — machn's own processing latency, not any link, grows with remote count until the 5 Hz safety heartbeat exceeds the 2.0 s timeout. + +## Symptom +Adding remotes to one machine uniformly inflates the pstop heartbeat rtt of **every** bonded remote — including a same-subnet LAN-direct remote (PS) that is normally ~200 ms. Past the ceiling, rtt oscillates and machn disarms (fail-safe STOP) repeatedly, then persistently. Because the elevation is uniform across all remotes regardless of their path (LAN-direct, inter-subnet, hairpin, relay), the delay is **machn-side processing**, not per-link. + +## Measured curve +Bench: machn `192.168.107.192`, all remotes DERP-region-locked to 9, **no region switching**, build `5ef3baa`. Bisect: add one remote, watch 7 min, record disarms + peak rtt. Edge confirmed quiet by the DUT-host session throughout (external flush ruled out). + +| Remotes | Disarms / 7 min | Peak rtt | Verdict | +|--------:|:---------------:|:--------:|:--------| +| 2 | 0 | ~200 ms | stable, healthy margin | +| 3 | 0 | 668 ms | **stable — comfortable ceiling** | +| 4 | 0 | 1034 ms | stable but **marginal** (>½ the 2 s budget) | +| 5 | 15 (collapse after ~2 min, then persistent) | 1525 ms | **broken** | +| 6 | many, oscillating | 500–1300 ms | broken | + +Per added remote, peak rtt rises ~300–400 ms (roughly linear-to-superlinear). At 5 remotes the peak crosses the 2.0 s heartbeat timeout and machn cannot hold armed. + +## It is NOT the recent DERP fixes (build-independent) +Reverting machn to **`1874ba1`** — the commit *before* `f2f883e` (immediate-reconnect) and the defer-aux work, i.e. none of the recent DERP-resilience changes — **still broke at 6 remotes** with the identical signature. So the ceiling is a pre-existing property of machn's multi-remote handling, exposed by scaling up, not a regression from PR #94's changes. + +## Mechanism (most likely) +`components/microlink/include/microlink_internal.h` (~L137) already documents the class: + +> *"each DERP (re)connect is a full TLS handshake whose CPU burst starves lwIP/httpd (400–700 ms latency spikes; an httpd handler-budget exhaustion took down the admin API during the multi-remote validation). An undamped negotiator is a reconnect-storm generator by construction."* + +Under N-remote load, machn's single DERP I/O task does more work (per-remote disco/CMM, home-conn keepalive servicing, occasional reconnects). When the home DERP conn's read starves, machn tears it down and reconnects; each reconnect is a blocking TLS handshake (~1 s CPU burst) that stalls the task that also drives pstop rx/tx → all remotes' rtt spikes → feedback loop. The blocking-TLS-in-the-safety-loop pattern is the same root the defer-aux work targets, here amplified by remote count rather than an edge flush. + +## Recommendation (current) +- **Deploy ≤ 3 remotes per machine** for comfortable margin; **4 is a hard, marginal ceiling**; do not exceed 4. +- Multi-remote validation soaks should run at ≤ 4 remotes until the ceiling is raised. + +## Raising the ceiling (future — tracked in the scale-ceiling task) +Candidate levers, in rough priority: +1. **Non-blocking / async DERP TLS handshake** — drive the handshake across I/O-loop iterations so a (re)connect never blocks the pstop rx/tx path. This is the single highest-value change (also fixes the defer-aux self-deadlock and the edge-flush home-rx stall in one architecture change). +2. **Reduce home-conn reconnect triggers under load** — understand why the home conn read-starves at scale (keepalive cadence, read-watchdog thresholds) and throttle/avoid spurious teardowns. +3. **Offload TLS or move aux/standby connects off the safety I/O task** (separate task / core). +4. **Per-remote disco/CMM cost reduction** at scale. + +## Caveats +- Measured on the office bench with all remotes on region 9 (dfw, a far relay); a nearer region or true LAN-direct-only fleet may shift the numbers, but the machn-side scaling is the invariant. +- Separate, unrelated limits observed the same day: remote must pin its machine in the peer table at 128-node scale; the sustained-ENOTCONN kick doesn't always recover; `/api/pstop_peer?clear=1` is broken (use `ip=127.0.0.1&port=1` to unbond). diff --git a/firmware/components/dcs_support/src/dcs_admin_html.h b/firmware/components/dcs_support/src/dcs_admin_html.h index cb0feb2..4efa38a 100644 --- a/firmware/components/dcs_support/src/dcs_admin_html.h +++ b/firmware/components/dcs_support/src/dcs_admin_html.h @@ -134,6 +134,11 @@ static const char k_index_html[] = "" "" #endif + /* Master LED brightness: one scale over ALL ring output (persisted). */ + "
" + "
LED brightness (master ring dimming, 0-100% — live & persisted)
" + "

Brightness: -%

" /* All diagnostics/config (unchanged) collected into one fleet-style panel. */ "
" "
Diagnostics
" @@ -447,6 +452,10 @@ static const char k_index_html[] = "document.getElementById('txdbm').textContent=(j.tx_q/4).toFixed(2);" "const tx=document.getElementById('txq');" "if(!tx.dataset.dirty)tx.value=j.tx_q;" + "if(j.led_brightness!==undefined){" + "document.getElementById('lbrv').textContent=j.led_brightness;" + "const lbr=document.getElementById('lbr');" + "if(!lbr.dataset.dirty)lbr.value=j.led_brightness;}" "document.getElementById('rssi').textContent=j.rssi;" "document.getElementById('heap').textContent=Math.round(j.free_heap/1024);" "document.getElementById('lh').textContent=Math.round(j.largest/1024);" @@ -519,6 +528,16 @@ static const char k_index_html[] = "catch(e){}" "finally{delete tx.dataset.dirty;poll();}" "});" + "const lbr=document.getElementById('lbr');" + "lbr.addEventListener('input',()=>{" + "lbr.dataset.dirty=1;" + "document.getElementById('lbrv').textContent=lbr.value;" + "});" + "lbr.addEventListener('change',async()=>{" + "try{await fetch('/api/led_brightness?pct='+lbr.value,{method:'POST'});}" + "catch(e){}" + "finally{delete lbr.dataset.dirty;poll();}" + "});" "function tickGraph(){" "h0.shift();h0.push(lastLoad0);" "h1.shift();h1.push(lastLoad1);" diff --git a/firmware/components/dcs_support/src/dcs_admin_pages.c b/firmware/components/dcs_support/src/dcs_admin_pages.c index 0527d82..5a97afe 100644 --- a/firmware/components/dcs_support/src/dcs_admin_pages.c +++ b/firmware/components/dcs_support/src/dcs_admin_pages.c @@ -22,6 +22,7 @@ * POST /api/pstop_num?n=N Set USB "PSTOPxx" unit number (0 = auto) * POST /api/pstop_peers?slot.. Multi-machine peer table (set/clear slot) * POST /api/ring_offset?n=N Set + persist LED-ring rotation (physical LED 1) + * POST /api/led_brightness?pct=N Set + persist master LED brightness (0..100%) * POST /api/ring_led1?on=0|1 Locate mode: only LED 1 white (auto-expires) * POST /api/enter_download Enter USB download (flashing) mode * @@ -275,7 +276,7 @@ static esp_err_t page_state(httpd_req_t * req) "\"rst_hist\":%s,\"ota_state\":%d,\"pstop_num\":%d," "\"relay_fault_a\":%lu,\"relay_fault_b\":%lu,\"relay_stop\":%lu,\"relay_feedback_monitored\":%lu," "\"remote_stop_id\":%lu,\"restart_state\":%lu," - "\"ring_offset\":%d,\"ring_led1\":%d,\"pstop_machines\":", + "\"ring_offset\":%d,\"ring_led1\":%d,\"led_brightness\":%d,\"pstop_machines\":", (unsigned long)atomic_load(&g_dcs_core_tick[0]), (unsigned long)atomic_load(&g_dcs_core_tick[1]), (int)atomic_load(&g_dcs_core_verdict[0]), @@ -377,7 +378,8 @@ static esp_err_t page_state(httpd_req_t * req) (unsigned long)atomic_load(&g_dcs_machn_arm_owner), (unsigned long)atomic_load(&g_dcs_machn_restart_state), (int)dcs_pstop_ring_get_offset(), - dcs_pstop_ring_locate_active() ? 1 : 0); + dcs_pstop_ring_locate_active() ? 1 : 0, + (int)dcs_pstop_ring_get_brightness()); /* Clamp the running offset after EVERY append so `buf + n` and `cap - n` * can never run past the buffer. snprintf() returns the length it WOULD @@ -980,6 +982,45 @@ static esp_err_t api_ring_offset(httpd_req_t * req) return httpd_resp_send(req, buf, len); } +/* === POST /api/led_brightness?pct=0..100 ================================== * + * Master LED brightness: a single 0..100% scale applied to ALL ring output. + * Applies immediately (next repaint) and persists to NVS; survives reboots and + * firmware updates. Absent key -> DCS_LED_BRIGHTNESS_DEFAULT. */ +static esp_err_t api_led_brightness(httpd_req_t * req) +{ + char query[48], val[8]; + if ( + (httpd_req_get_url_query_str(req, query, sizeof(query)) != ESP_OK) || + (httpd_query_key_value(query, "pct", val, sizeof(val)) != ESP_OK)) + { + (void)httpd_resp_set_status(req, "400 Bad Request"); + (void)httpd_resp_set_type(req, "application/json"); + return httpd_resp_sendstr(req, "{\"ok\":false,\"error\":\"missing ?pct=0..100\"}"); + } + int pct = (int)strtol(val, NULL, 10); + if ((pct < 0) || (pct > 100)) { + (void)httpd_resp_set_status(req, "400 Bad Request"); + (void)httpd_resp_set_type(req, "application/json"); + return httpd_resp_sendstr(req, "{\"ok\":false,\"error\":\"pct must be 0..100\"}"); + } + esp_err_t r = dcs_nvs_write_led_brightness((uint8_t)pct); + if (r == ESP_OK) { + dcs_pstop_ring_set_brightness((uint8_t)pct); /* live, next repaint */ + } + char buf[80]; + int len = snprintf( + buf, + sizeof(buf), + "{\"ok\":%s,\"led_brightness\":%d}", + (r == ESP_OK) ? "true" : "false", + (int)dcs_pstop_ring_get_brightness()); + (void)httpd_resp_set_type(req, "application/json"); + if (r != ESP_OK) { + (void)httpd_resp_set_status(req, "500 Internal Server Error"); + } + return httpd_resp_send(req, buf, len); +} + /* === POST /api/ring_led1?on=0|1 =========================================== * * Locate mode: only LED 1 lit, white — shows where the current ring_offset * puts LED 1 so an installer can verify/adjust it. Overrides the state @@ -1122,6 +1163,7 @@ void dcs_admin_pages_register(ml_app_t * app) (void)ml_app_add_page(app, "/api/iface/usb", HTTP_POST, api_iface_usb); (void)ml_app_add_page(app, "/api/pstop_num", HTTP_POST, api_pstop_num); (void)ml_app_add_page(app, "/api/ring_offset", HTTP_POST, api_ring_offset); + (void)ml_app_add_page(app, "/api/led_brightness", HTTP_POST, api_led_brightness); (void)ml_app_add_page(app, "/api/ring_led1", HTTP_POST, api_ring_led1); (void)ml_app_add_page(app, "/api/enter_download", HTTP_POST, api_enter_download); (void)ml_app_add_page(app, "/api/operators", HTTP_GET, api_operators_get); diff --git a/firmware/components/dcs_support/src/dcs_internal.h b/firmware/components/dcs_support/src/dcs_internal.h index b33215e..72579bb 100644 --- a/firmware/components/dcs_support/src/dcs_internal.h +++ b/firmware/components/dcs_support/src/dcs_internal.h @@ -38,6 +38,9 @@ extern "C" #define DCS_NVS_KEY_PSTOP_PEERS "ps_peers" /* multi-machine peer table (blob, see dcs_nvs.c) */ #define DCS_NVS_KEY_OPERATORS "operators" /* operator allowlist (blob: count byte + u32 ids) */ #define DCS_NVS_KEY_WIFI_TXP "wifi_txp" /* WiFi max TX power, quarter-dBm (8..84); 0/absent = config default */ +#define DCS_NVS_KEY_LED_BRIGHT "led_bri" /* master LED brightness, 0..100%; absent = default */ + +#define DCS_LED_BRIGHTNESS_DEFAULT 50 /* master ring brightness when the NVS key is absent */ #define DCS_RST_HIST_LEN 16 @@ -303,6 +306,12 @@ extern "C" uint8_t dcs_nvs_read_wifi_tx_power(void); esp_err_t dcs_nvs_write_wifi_tx_power(uint8_t quarter_dbm); + /* Master LED brightness (0..100%) scaling ALL ring output. Read returns + * DCS_LED_BRIGHTNESS_DEFAULT when unset or the persisted value is corrupt + * (>100); applied live by dcs_pstop_ring_set_brightness and re-loaded at boot. */ + uint8_t dcs_nvs_read_led_brightness(void); + esp_err_t dcs_nvs_write_led_brightness(uint8_t pct); + /* Multi-machine peer table (ps_peers blob). One record per slot. Read * falls back to migrating the legacy ps_ip/ps_port pair into slot 0 when * the blob is absent (first boot on this firmware). */ @@ -379,6 +388,13 @@ extern "C" void dcs_pstop_ring_set_offset(uint8_t off); uint8_t dcs_pstop_ring_get_offset(void); + /* Master ring brightness (live value, mirrors NVS led_bri): a 0..100% scale + * applied to EVERY ring pixel at transmit time, so all state colours dim + * proportionally. set clamps to 0..100 and takes effect on the next repaint + * (<=250 ms); the caller persists to NVS separately (dcs_nvs_write_led_brightness). */ + void dcs_pstop_ring_set_brightness(uint8_t pct); + uint8_t dcs_pstop_ring_get_brightness(void); + /* Locate mode: paint ONLY logical LED 1 solid white so an installer can see / * verify the rotation offset. Overrides the state colours; auto-expires after * DCS_RING_LOCATE_TIMEOUT_MS so a forgotten locate can't mask STOP/OK forever. */ diff --git a/firmware/components/dcs_support/src/dcs_nvs.c b/firmware/components/dcs_support/src/dcs_nvs.c index e00d3e9..aba5043 100644 --- a/firmware/components/dcs_support/src/dcs_nvs.c +++ b/firmware/components/dcs_support/src/dcs_nvs.c @@ -16,6 +16,7 @@ * (absent -> migrate legacy ps_ip/ps_port into slot 0) * operators blob operator allowlist: count byte + u32 ids * wifi_txp u8 WiFi max TX power, quarter-dBm (8..84); 0/absent = config default + * led_bri u8 master LED brightness, 0..100%; absent/corrupt = default 50 */ #include @@ -226,6 +227,31 @@ esp_err_t dcs_nvs_write_wifi_tx_power(uint8_t quarter_dbm) return r; } +uint8_t dcs_nvs_read_led_brightness(void) +{ + nvs_handle_t h; + if (nvs_open(DCS_NVS_NS, NVS_READONLY, &h) != ESP_OK) return DCS_LED_BRIGHTNESS_DEFAULT; + uint8_t v = DCS_LED_BRIGHTNESS_DEFAULT; + (void)nvs_get_u8(h, DCS_NVS_KEY_LED_BRIGHT, &v); /* absent -> default */ + nvs_close(h); + if (v > 100u) return DCS_LED_BRIGHTNESS_DEFAULT; /* corrupt/older schema degrades to default */ + return v; +} + +esp_err_t dcs_nvs_write_led_brightness(uint8_t pct) +{ + if (pct > 100u) return ESP_ERR_INVALID_ARG; + nvs_handle_t h; + esp_err_t r = nvs_open(DCS_NVS_NS, NVS_READWRITE, &h); + if (r != ESP_OK) return r; + r = nvs_set_u8(h, DCS_NVS_KEY_LED_BRIGHT, pct); + if (r == ESP_OK) { + r = nvs_commit(h); + } + nvs_close(h); + return r; +} + /* ps_peers blob layout (byte-serialized, no struct padding on the wire): * [0] format version (1) * per slot, DCS_PSTOP_MAX_MACHINES records of 11 bytes: diff --git a/firmware/components/dcs_support/src/dcs_pstop_ring.c b/firmware/components/dcs_support/src/dcs_pstop_ring.c index 07443e1..5cc24de 100644 --- a/firmware/components/dcs_support/src/dcs_pstop_ring.c +++ b/firmware/components/dcs_support/src/dcs_pstop_ring.c @@ -117,6 +117,12 @@ static uint8_t s_grb[RING_LEDS * 3]; /* WS2812 wants GRB order, per pixel — LO * spins unrotated — it's a sign of life, orientation is irrelevant. */ static atomic_uint_fast32_t s_ring_offset; +/* Master brightness (0..100%): scales EVERY pixel at transmit time so all ring + * states dim proportionally. Seeded to the default so the boot sign-of-life + * spinner (paints before NVS is up) is visible; dcs_pstop_ring_start() replaces + * it with the persisted led_bri. */ +static atomic_uint_fast32_t s_ring_brightness_pct = DCS_LED_BRIGHTNESS_DEFAULT; + /* Locate mode: ms-uptime deadline until which ONLY logical LED 1 is painted * white (0 = off). Set via dcs_pstop_ring_locate(); auto-expires so a * forgotten locate can't mask the safety state colours indefinitely. */ @@ -133,9 +139,14 @@ static void ring_show(void) } static uint8_t tx_grb[RING_LEDS * 3]; uint32_t off = (uint32_t)atomic_load(&s_ring_offset); + /* Master-brightness scale, applied HERE (the single WS2812 write for every + * ring state) so all colours dim by the same factor. */ + uint32_t bri = (uint32_t)atomic_load(&s_ring_brightness_pct); for (int i = 0; i < RING_LEDS; i++) { int p = (int)(((uint32_t)i + off) % RING_LEDS); - (void)memcpy(&tx_grb[p * 3], &s_grb[i * 3], 3); + for (int c = 0; c < 3; c++) { + tx_grb[(p * 3) + c] = (uint8_t)(((uint32_t)s_grb[(i * 3) + c] * bri) / 100u); + } } rmt_transmit_config_t tx = {.loop_count = 0}; if (rmt_transmit(s_chan, s_enc, tx_grb, sizeof(tx_grb), &tx) == ESP_OK) { @@ -537,6 +548,7 @@ void dcs_pstop_ring_start(void) /* Load the persisted rotation HERE (caller's internal stack, NVS already * up) — the ring task lives on a PSRAM stack and must not touch NVS. */ atomic_store(&s_ring_offset, dcs_nvs_read_ring_offset()); + atomic_store(&s_ring_brightness_pct, dcs_nvs_read_led_brightness()); /* PSRAM stack: LED ring is non-safety, does no flash/NVS. */ (void)dcs_task_spawn_psram(ring_task, "pstop_ring", 4096, NULL, 2, tskNO_AFFINITY); } @@ -552,6 +564,17 @@ uint8_t dcs_pstop_ring_get_offset(void) return (uint8_t)atomic_load(&s_ring_offset); } +void dcs_pstop_ring_set_brightness(uint8_t pct) +{ + atomic_store(&s_ring_brightness_pct, (uint32_t)((pct > 100u) ? 100u : pct)); + /* Next repaint (<=250 ms) — or the next locate/comet frame — picks it up. */ +} + +uint8_t dcs_pstop_ring_get_brightness(void) +{ + return (uint8_t)atomic_load(&s_ring_brightness_pct); +} + void dcs_pstop_ring_locate(bool on) { if (on) { diff --git a/host/Makefile b/host/Makefile index 06bb828..ec6c3ff 100644 --- a/host/Makefile +++ b/host/Makefile @@ -46,10 +46,16 @@ CRYPTO_SRCS := \ test_crypto_kat: test_crypto_kat.c $(CRYPTO_SRCS) $(CC) -O3 -Wall -Wextra -Wno-unused-parameter -o $@ $^ +# Direct-path demote-verification verdict (ml_demote_verdict.h): pure +# header-only logic, no ESP-IDF/lwIP — see test_demote_veto.c for the contract. +test_demote_veto: test_demote_veto.c $(MICROLINK)/include/ml_demote_verdict.h + $(CC) $(CFLAGS) -I$(MICROLINK)/include -o $@ test_demote_veto.c + .PHONY: test clean -test: test_clock_guard test_crypto_kat +test: test_clock_guard test_crypto_kat test_demote_veto ./test_clock_guard ./test_crypto_kat + ./test_demote_veto clean: - rm -f machine_app_runner test_clock_guard test_crypto_kat + rm -f machine_app_runner test_clock_guard test_crypto_kat test_demote_veto diff --git a/host/test_demote_veto.c b/host/test_demote_veto.c new file mode 100644 index 0000000..1d6e030 --- /dev/null +++ b/host/test_demote_veto.c @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2026 Polymath Robotics +// SPDX-License-Identifier: Apache-2.0 + +/* Host unit tests for the direct-path demote-verification verdict + * (ml_demote_verdict.h). Fully host-runnable: the verdict is a pure function + * of the demote triggers and the direct-rx freshness evidence. Style follows + * test_clock_guard.c. + * + * Contract under test (2026-08-11 DUT direct-death design): + * - no trigger -> NONE, whatever the evidence says + * - trigger + safety peer + fresh authenticated direct WG rx -> VETO + * - trigger + safety peer + stale rx -> GO (a truly dead path must demote) + * - trigger + safety peer + no direct rx ever recorded -> GO + * - trigger + bulk (non-safety) peer -> GO even with fresh rx (behavior + * for the 16-peer tailnet load profile is unchanged) + * - freshness boundary is inclusive (age == fresh_ms still vetoes) + */ + +#include + +#include "ml_demote_verdict.h" + +static int g_checks = 0; +static int g_fails = 0; +#define CHECK(cond, what) \ + do { \ + g_checks++; \ + if (!(cond)) { \ + g_fails++; \ + printf(" FAIL: %s\n", (what)); \ + } \ + } while (0) + +int main(void) +{ + const uint32_t FRESH = 1000; + + /* No trigger: always NONE, even with terrible evidence. */ + CHECK(ml_demote_verdict(false, false, true, true, 0, FRESH) == ML_DEMOTE_NONE, "no trigger, fresh rx -> NONE"); + CHECK(ml_demote_verdict(false, false, true, false, 0, FRESH) == ML_DEMOTE_NONE, "no trigger, no rx -> NONE"); + CHECK(ml_demote_verdict(false, false, false, true, 999999, FRESH) == ML_DEMOTE_NONE, "no trigger, bulk -> NONE"); + + /* Safety peer, fresh direct rx: veto on either trigger (and both). */ + CHECK(ml_demote_verdict(true, false, true, true, 200, FRESH) == ML_DEMOTE_VETO, "lease + fresh rx -> VETO"); + CHECK(ml_demote_verdict(false, true, true, true, 200, FRESH) == ML_DEMOTE_VETO, "pong-dead + fresh rx -> VETO"); + CHECK(ml_demote_verdict(true, true, true, true, 0, FRESH) == ML_DEMOTE_VETO, "both triggers + rx now -> VETO"); + + /* Freshness boundary: inclusive at fresh_ms, GO just past it. */ + CHECK(ml_demote_verdict(true, false, true, true, FRESH, FRESH) == ML_DEMOTE_VETO, "age == fresh -> VETO"); + CHECK(ml_demote_verdict(true, false, true, true, FRESH + 1, FRESH) == ML_DEMOTE_GO, "age just stale -> GO"); + + /* A truly dead path must still demote: stale rx or none ever. */ + CHECK(ml_demote_verdict(false, true, true, true, 5000, FRESH) == ML_DEMOTE_GO, "pong-dead + stale rx -> GO"); + CHECK(ml_demote_verdict(true, false, true, false, 0, FRESH) == ML_DEMOTE_GO, "lease + no rx ever -> GO"); + CHECK(ml_demote_verdict(true, true, true, false, 0, FRESH) == ML_DEMOTE_GO, "both + no rx ever -> GO"); + + /* Bulk peers: verification never applies (original lease behavior). */ + CHECK(ml_demote_verdict(true, false, false, true, 0, FRESH) == ML_DEMOTE_GO, "bulk + fresh rx -> GO"); + CHECK(ml_demote_verdict(false, true, false, true, 100, FRESH) == ML_DEMOTE_GO, "bulk pong-dead + fresh rx -> GO"); + + /* Hitless re-ingest teardown veto (run-20 green drop): a control-plane + * teardown of a safety session is vetoed iff authenticated data (any path) + * is fresh. Bulk peers and stale/never-rx sessions tear down as before. */ + const uint32_t TFRESH = 2000; + CHECK(ml_teardown_veto(true, true, 150, TFRESH), "safety + fresh rx -> veto teardown"); + CHECK(ml_teardown_veto(true, true, TFRESH, TFRESH), "boundary age == fresh -> veto"); + CHECK(!ml_teardown_veto(true, true, TFRESH + 1, TFRESH), "stale rx -> teardown proceeds"); + CHECK(!ml_teardown_veto(true, false, 0, TFRESH), "no rx ever -> teardown proceeds"); + CHECK(!ml_teardown_veto(false, true, 0, TFRESH), "bulk peer -> teardown proceeds"); + + /* Veto-streak cap: RX-only evidence must not pin a path indefinitely. + * Below the cap the veto stands; at/past the cap it degrades to GO. + * NONE and GO verdicts pass through untouched at any streak. */ + const uint32_t CAP = 10; + CHECK(ml_demote_verdict_capped(ML_DEMOTE_VETO, 0, CAP) == ML_DEMOTE_VETO, "streak 0 -> veto stands"); + CHECK(ml_demote_verdict_capped(ML_DEMOTE_VETO, CAP - 1, CAP) == ML_DEMOTE_VETO, "streak cap-1 -> veto stands"); + CHECK(ml_demote_verdict_capped(ML_DEMOTE_VETO, CAP, CAP) == ML_DEMOTE_GO, "streak == cap -> GO"); + CHECK(ml_demote_verdict_capped(ML_DEMOTE_VETO, CAP + 5, CAP) == ML_DEMOTE_GO, "streak past cap -> GO"); + CHECK(ml_demote_verdict_capped(ML_DEMOTE_NONE, CAP + 5, CAP) == ML_DEMOTE_NONE, "NONE unaffected by streak"); + CHECK(ml_demote_verdict_capped(ML_DEMOTE_GO, 0, CAP) == ML_DEMOTE_GO, "GO unaffected by streak"); + + printf("test_demote_veto: %d checks, %d failures\n", g_checks, g_fails); + return g_fails == 0 ? 0 : 1; +} diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..156c883 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,28 @@ +# Repo-local Ruff config, mirroring polymath_code_standard v2.1.1 config/ruff.toml. +# +# The pre-commit `polymath-python` checker copies its own ruff.toml to a +# gitignored `.ruff.toml` at runtime and relies on ruff's ancestor-walk +# discovery. On CI runners that copy is not discovered from the hook's cwd, so +# ruff silently falls back to its built-in default (quote-style = "double") and +# rewrites the repo's single-quoted Python — failing every PR on unchanged +# files. This tracked, non-dot `ruff.toml` is discovered by ruff regardless of +# cwd, pinning the org standard (single quotes) in CI. When the checker's +# runtime `.ruff.toml` IS present it takes precedence and is identical, so the +# two never disagree. Keep in sync with polymath_code_standard. + +line-length = 120 +indent-width = 4 + +[format] +preview = true +quote-style = "single" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "lf" + +[lint] +select = ["E4", "E7", "E9", "F", "I"] +ignore = [] +fixable = ["ALL"] +unfixable = [] +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"