From a3bfcfb8085767c49ea2cc2b61a785a99875a23b Mon Sep 17 00:00:00 2001 From: ehila Date: Tue, 18 Aug 2026 16:12:35 +0200 Subject: [PATCH] OCPBUGS-111074: Taint TNF nodes on graceful membership loss STONITH-only alerts leave DaemonSet endpoints Ready after ACPI shutdown. Also taint when the peer is stably Offline so kube-controller-manager can drop those backends. Co-authored-by: Cursor Signed-off-by: ehila --- .../files/taint-fenced-node.yaml | 66 +++++++++++++++++++ .../files/tnf-taint-alert.yaml | 43 +++++++++--- .../units/taint-node@.service.yaml | 6 +- 3 files changed, 106 insertions(+), 9 deletions(-) diff --git a/templates/master/00-master/two-node-with-fencing/files/taint-fenced-node.yaml b/templates/master/00-master/two-node-with-fencing/files/taint-fenced-node.yaml index 066ea0fdc7..faddfb674b 100644 --- a/templates/master/00-master/two-node-with-fencing/files/taint-fenced-node.yaml +++ b/templates/master/00-master/two-node-with-fencing/files/taint-fenced-node.yaml @@ -17,6 +17,72 @@ contents: exit 1 fi + parse_online_nodes() { + pcs status nodes 2>/dev/null | awk ' + /^[[:space:]]+Online:/ { + for (i=2; i<=NF; i++) { gsub(/[][]/,"",$i); if ($i != "") printf "%s ",$i } + }' || true + } + + node_is_listed() { + local target="$1" + local list="$2" + local short="${target%%.*}" + [[ " $list " == *" $target "* ]] || [[ " $list " == *" $short "* ]] + } + + LOCAL_NAME=$(crm_node -n 2>/dev/null || hostname -s) + LOCAL_SHORT="${LOCAL_NAME%%.*}" + FENCED_SHORT="${FENCED_NODE%%.*}" + + if [[ "$FENCED_SHORT" == "$LOCAL_SHORT" ]]; then + logger -t taint-fenced-node "Refusing to taint local node ${FENCED_NODE}" + exit 0 + fi + + # Confirm we are the survivor and the target stays off. Node-lost alerts also + # fire on brief corosync blips; tainting a live peer would force-delete its pods. + STABLE_REQUIRED=3 + STABLE_INTERVAL=5 + MAX_CONFIRM_ATTEMPTS=12 + STABLE_COUNT=0 + LAST_ONLINE="" + for attempt in $(seq 1 $MAX_CONFIRM_ATTEMPTS); do + LAST_ONLINE=$(parse_online_nodes) + if [[ -z "$LAST_ONLINE" ]]; then + logger -t taint-fenced-node "pacemaker Online list empty (attempt ${attempt}/${MAX_CONFIRM_ATTEMPTS}); retrying" + STABLE_COUNT=0 + sleep $STABLE_INTERVAL + continue + fi + if ! node_is_listed "$LOCAL_NAME" "$LAST_ONLINE"; then + logger -t taint-fenced-node "Local node ${LOCAL_NAME} is not pacemaker-Online; skipping taint (this node may be shutting down)" + exit 0 + fi + if node_is_listed "$FENCED_NODE" "$LAST_ONLINE"; then + logger -t taint-fenced-node "${FENCED_NODE} is still pacemaker-Online (attempt ${attempt}/${MAX_CONFIRM_ATTEMPTS}); waiting to confirm loss" + STABLE_COUNT=0 + sleep $STABLE_INTERVAL + continue + fi + STABLE_COUNT=$((STABLE_COUNT + 1)) + logger -t taint-fenced-node "${FENCED_NODE} is not pacemaker-Online (${STABLE_COUNT}/${STABLE_REQUIRED})" + if [[ "$STABLE_COUNT" -ge "$STABLE_REQUIRED" ]]; then + break + fi + sleep $STABLE_INTERVAL + done + + LAST_ONLINE=$(parse_online_nodes) + if [[ -n "$LAST_ONLINE" ]] && node_is_listed "$FENCED_NODE" "$LAST_ONLINE"; then + logger -t taint-fenced-node "Skipping taint: ${FENCED_NODE} returned to pacemaker-Online (transient membership loss)" + exit 0 + fi + if [[ "$STABLE_COUNT" -lt "$STABLE_REQUIRED" ]]; then + logger -t taint-fenced-node "ERROR: Could not confirm ${FENCED_NODE} is stably Offline; will retry via systemd" + exit 1 + fi + logger -t taint-fenced-node "Applying out-of-service taint to fenced node ${FENCED_NODE}" PATCH=$(cat < when a TNF node is gone. + # + # Two pacemaker alerts invoke this script: + # tnf-taint-alert (select_fencing) - STONITH completed successfully + # tnf-taint-lost-alert (select_nodes) - peer left corosync membership + # + # Node-lost covers graceful ACPI shutdown, which does not emit a fencing + # event. taint-fenced-node.sh refuses to taint the local node and waits + # until the target is stably Offline so a transient membership blip is not + # treated as a fence. + start_taint_unit() { + local reason="$1" + local unit="taint-node@${CRM_alert_node}.service" + logger -t tnf-taint-alert "${reason} for ${CRM_alert_node}, triggering taint service" + if sudo systemctl start --no-block "$unit"; then + return 0 fi + # Fencing and node-lost can fire together; a oneshot already running is success. + local state + state=$(systemctl is-active "$unit" 2>/dev/null || true) + if [ "$state" = "activating" ] || [ "$state" = "active" ]; then + logger -t tnf-taint-alert "taint service ${unit} already ${state}" + return 0 + fi + logger -t tnf-taint-alert "ERROR: Failed to start ${unit} (state=${state})" + return 1 + } + + if [ -z "${CRM_alert_node:-}" ]; then + exit 0 + fi + + if [ "$CRM_alert_kind" = "fencing" ] && [ "$CRM_alert_rc" = "0" ]; then + start_taint_unit "Fencing succeeded" || exit 1 + elif [ "$CRM_alert_kind" = "node" ] && [ "$CRM_alert_desc" = "lost" ]; then + start_taint_unit "Node lost" || exit 1 fi exit 0 diff --git a/templates/master/00-master/two-node-with-fencing/units/taint-node@.service.yaml b/templates/master/00-master/two-node-with-fencing/units/taint-node@.service.yaml index fd4dc005f4..6cd01de162 100644 --- a/templates/master/00-master/two-node-with-fencing/units/taint-node@.service.yaml +++ b/templates/master/00-master/two-node-with-fencing/units/taint-node@.service.yaml @@ -3,8 +3,12 @@ contents: | [Unit] Description=Taint fenced node %i in Kubernetes After=network.target + StartLimitIntervalSec=2400 + StartLimitBurst=3 [Service] Type=oneshot ExecStart=/usr/local/bin/taint-fenced-node.sh %i - TimeoutStartSec=660 + TimeoutStartSec=720 + Restart=on-failure + RestartSec=10