From ed79dbc462e2b993d2352358681b9621ceb5bf3f Mon Sep 17 00:00:00 2001 From: jasper9 Date: Fri, 6 Nov 2026 10:29:05 -0700 Subject: [PATCH] feat: add optional nohz_full GRUB check Fail if nohz_full= is missing from GRUB_CMDLINE_LINUX_DEFAULT and warn if the running kernel does not match yet (reboot pending). The check is off by default and only runs when checksToRun.nohzFull is explicitly true, so existing local configs without the key are unaffected. should_run_check now takes an optional default for missing keys; existing checks keep defaulting to true. Co-Authored-By: Claude Opus 5.5 --- .claude/CLAUDE.md | 1 + README.md | 3 +++ config.json | 3 ++- health_check.sh | 61 ++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 32a59bf..988e69e 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -65,6 +65,7 @@ sudo ./health_check.sh -h # Help | NVMe Wear | `nvmeWear` | Monitors NVMe drive wear levels against threshold | | GRUB Cmdline | `grubCmdline` | Verifies required parameters (`audit=1`, `audit_backlog_limit=8192`, `transparent_hugepage=never`) exist in GRUB_CMDLINE_LINUX_DEFAULT | | Huge Pages | `hugepages` | Verifies `hugepagesz=`/`hugepages=` in GRUB match `systemChecks.memory.hugepageSize` (default `2M`) and `hugepageCount` (minimum, default `256`), and that the running kernel has allocated them; reports pages in use (and agave-validator's share) and warns if the pool is exhausted | +| nohz_full | `nohzFull` | **Off by default** (missing key = off). Fails if `nohz_full=` is missing from GRUB_CMDLINE_LINUX_DEFAULT; warns if the running kernel doesn't match yet (reboot pending). Enable per host in `local_config.json` | **NIC Exclusion**: Network interfaces can be excluded from ring buffer checks via config: ```json diff --git a/README.md b/README.md index 8f0833b..753430a 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The script verifies the following aspects of your system: - **CPU Driver**: Ensures that the p-state CPU scaling driver is being used - **SMT (Hyperthreading)**: Verifies SMT matches the configured state (default: disabled) - **Memory Management**: Confirms swap is disabled +- **Tickless CPUs (nohz_full)**: Optional (off by default) - verifies `nohz_full=` is set in GRUB on hosts that use tickless CPU cores - **Huge Pages**: Verifies `hugepagesz`/`hugepages` are set in GRUB and that the running kernel has allocated at least that many huge pages - **Security Services**: Checks that fail2ban is installed, enabled, and running - **System Updates**: Validates that there are no more than 5 package updates pending @@ -68,6 +69,7 @@ The configuration file is structured as follows: "smtStatus": true, "swapStatus": true, "hugepages": true, + "nohzFull": false, "packageUpdates": true, "ntpSync": true, "fail2ban": true, @@ -172,5 +174,6 @@ The script provides color-coded output for easy interpretation: - CPU boost check handles both Intel and AMD-specific boost mechanisms - SMT check reads `/sys/devices/system/cpu/smt/active`; `systemChecks.cpu.smt` accepts `disabled` (default), `enabled`, or `any` (report only). If the key is missing from the config, the check still runs and expects `disabled` - Huge pages check reads `hugepagesz=`/`hugepages=` from `GRUB_CMDLINE_LINUX_DEFAULT` and `/sys/kernel/mm/hugepages/hugepages-kB/nr_hugepages`; `systemChecks.memory.hugepageSize` (default `2M`) must match exactly and `hugepageCount` (default `256`) is a minimum. Pages in use are reported for information only (a stopped validator uses none); a warning is shown if every page is in use +- The nohz_full check is off unless `checksToRun.nohzFull` is explicitly `true` (a missing key counts as off, unlike other checks). It fails if `nohz_full=` is missing from `GRUB_CMDLINE_LINUX_DEFAULT` and warns if the running kernel (`/proc/cmdline`) does not match yet - NTP check supports multiple time synchronization methods (systemd-timesyncd, chronyd, ntpd, OpenNTPD) - Package update checker automatically detects apt, dnf, yum, pacman, or zypper package managers \ No newline at end of file diff --git a/config.json b/config.json index 600db91..01e15eb 100644 --- a/config.json +++ b/config.json @@ -22,7 +22,8 @@ "nvmeWear": true, "requiredPackages": true, "grubCmdline": true, - "hugepages": true + "hugepages": true, + "nohzFull": false }, "systemChecks": { "cpu": { diff --git a/health_check.sh b/health_check.sh index ac872fe..88a0a11 100755 --- a/health_check.sh +++ b/health_check.sh @@ -168,8 +168,10 @@ get_config() { # Function to check if a specific check should be run should_run_check() { local check_name="$1" + # Optional second argument: value to use when the key is missing from the config (default: true) + local default="${2:-true}" local result - result=$(get_config ".checksToRun.$check_name" "true") + result=$(get_config ".checksToRun.$check_name" "$default") # Convert to lowercase for standardization result=$(echo "$result" | tr '[:upper:]' '[:lower:]') @@ -2224,6 +2226,63 @@ else echo -e "${BLUE}Checking huge page configuration... ${YELLOW}[SKIPPED]${NC}" fi +# Function to check nohz_full is set in GRUB (only enabled on hosts that use tickless CPUs) +check_nohz_full() { + echo -e "\n${YELLOW}=== Tickless CPUs (nohz_full) ===${NC}" + echo -e "${BLUE}Checking nohz_full in GRUB command line...${NC}" + + local grub_file="/etc/default/grub" + + if [ ! -f "$grub_file" ]; then + echo -e " ${RED}FAIL: GRUB configuration file not found at $grub_file${NC}" + return 1 + fi + + local cmdline_default + cmdline_default=$(grep "^GRUB_CMDLINE_LINUX_DEFAULT=" "$grub_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"'"'") + + # Match whole tokens so other parameters containing "nohz_full" are not mistaken for it + local grub_value="" token + for token in $cmdline_default; do + case "$token" in + nohz_full=*) grub_value=${token#nohz_full=} ;; + esac + done + + if [ -z "$grub_value" ]; then + echo -e " ${RED}FAIL: 'nohz_full=' missing from GRUB_CMDLINE_LINUX_DEFAULT${NC}" + echo -e " ${YELLOW}Add 'nohz_full=' to GRUB_CMDLINE_LINUX_DEFAULT in $grub_file${NC}" + echo -e " ${YELLOW}Then run: sudo update-grub && sudo reboot${NC}" + return 1 + fi + + echo -e " ${GREEN}PASS: 'nohz_full=$grub_value' found in GRUB_CMDLINE_LINUX_DEFAULT${NC}" + + # nohz_full only takes effect at boot, so flag a GRUB change that has not been applied yet + local running_value="" + for token in $(cat /proc/cmdline 2>/dev/null); do + case "$token" in + nohz_full=*) running_value=${token#nohz_full=} ;; + esac + done + + if [ "$running_value" != "$grub_value" ]; then + echo -e " ${YELLOW}WARNING: Running kernel has nohz_full='${running_value:-}' - run 'sudo update-grub' and reboot to apply${NC}" + fi + return 0 +} + +# Check nohz_full if enabled in config (off unless a config explicitly enables it) +if should_run_check "nohzFull" "false"; then + if ! check_nohz_full; then + ((failures++)) + failed_checks+=("Tickless CPUs: nohz_full") + fi +else + echo -e "\n${YELLOW}=== Tickless CPUs (nohz_full) ===${NC}" + echo -e "${BLUE}Checking nohz_full in GRUB command line... ${YELLOW}[SKIPPED]${NC}" +fi + # Summary - Always show this part even in quiet mode if [ "$QUIET_MODE" = true ]; then # Restore stdout for the summary