diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 32a59bf..988e69e 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -65,6 +65,7 @@ sudo ./health_check.sh -h # Help | NVMe Wear | `nvmeWear` | Monitors NVMe drive wear levels against threshold | | GRUB Cmdline | `grubCmdline` | Verifies required parameters (`audit=1`, `audit_backlog_limit=8192`, `transparent_hugepage=never`) exist in GRUB_CMDLINE_LINUX_DEFAULT | | Huge Pages | `hugepages` | Verifies `hugepagesz=`/`hugepages=` in GRUB match `systemChecks.memory.hugepageSize` (default `2M`) and `hugepageCount` (minimum, default `256`), and that the running kernel has allocated them; reports pages in use (and agave-validator's share) and warns if the pool is exhausted | +| nohz_full | `nohzFull` | **Off by default** (missing key = off). Fails if `nohz_full=` is missing from GRUB_CMDLINE_LINUX_DEFAULT; warns if the running kernel doesn't match yet (reboot pending). Enable per host in `local_config.json` | **NIC Exclusion**: Network interfaces can be excluded from ring buffer checks via config: ```json diff --git a/README.md b/README.md index 8f0833b..753430a 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The script verifies the following aspects of your system: - **CPU Driver**: Ensures that the p-state CPU scaling driver is being used - **SMT (Hyperthreading)**: Verifies SMT matches the configured state (default: disabled) - **Memory Management**: Confirms swap is disabled +- **Tickless CPUs (nohz_full)**: Optional (off by default) - verifies `nohz_full=` is set in GRUB on hosts that use tickless CPU cores - **Huge Pages**: Verifies `hugepagesz`/`hugepages` are set in GRUB and that the running kernel has allocated at least that many huge pages - **Security Services**: Checks that fail2ban is installed, enabled, and running - **System Updates**: Validates that there are no more than 5 package updates pending @@ -68,6 +69,7 @@ The configuration file is structured as follows: "smtStatus": true, "swapStatus": true, "hugepages": true, + "nohzFull": false, "packageUpdates": true, "ntpSync": true, "fail2ban": true, @@ -172,5 +174,6 @@ The script provides color-coded output for easy interpretation: - CPU boost check handles both Intel and AMD-specific boost mechanisms - SMT check reads `/sys/devices/system/cpu/smt/active`; `systemChecks.cpu.smt` accepts `disabled` (default), `enabled`, or `any` (report only). If the key is missing from the config, the check still runs and expects `disabled` - Huge pages check reads `hugepagesz=`/`hugepages=` from `GRUB_CMDLINE_LINUX_DEFAULT` and `/sys/kernel/mm/hugepages/hugepages-kB/nr_hugepages`; `systemChecks.memory.hugepageSize` (default `2M`) must match exactly and `hugepageCount` (default `256`) is a minimum. Pages in use are reported for information only (a stopped validator uses none); a warning is shown if every page is in use +- The nohz_full check is off unless `checksToRun.nohzFull` is explicitly `true` (a missing key counts as off, unlike other checks). It fails if `nohz_full=` is missing from `GRUB_CMDLINE_LINUX_DEFAULT` and warns if the running kernel (`/proc/cmdline`) does not match yet - NTP check supports multiple time synchronization methods (systemd-timesyncd, chronyd, ntpd, OpenNTPD) - Package update checker automatically detects apt, dnf, yum, pacman, or zypper package managers \ No newline at end of file diff --git a/config.json b/config.json index 600db91..01e15eb 100644 --- a/config.json +++ b/config.json @@ -22,7 +22,8 @@ "nvmeWear": true, "requiredPackages": true, "grubCmdline": true, - "hugepages": true + "hugepages": true, + "nohzFull": false }, "systemChecks": { "cpu": { diff --git a/health_check.sh b/health_check.sh index ac872fe..88a0a11 100755 --- a/health_check.sh +++ b/health_check.sh @@ -168,8 +168,10 @@ get_config() { # Function to check if a specific check should be run should_run_check() { local check_name="$1" + # Optional second argument: value to use when the key is missing from the config (default: true) + local default="${2:-true}" local result - result=$(get_config ".checksToRun.$check_name" "true") + result=$(get_config ".checksToRun.$check_name" "$default") # Convert to lowercase for standardization result=$(echo "$result" | tr '[:upper:]' '[:lower:]') @@ -2224,6 +2226,63 @@ else echo -e "${BLUE}Checking huge page configuration... ${YELLOW}[SKIPPED]${NC}" fi +# Function to check nohz_full is set in GRUB (only enabled on hosts that use tickless CPUs) +check_nohz_full() { + echo -e "\n${YELLOW}=== Tickless CPUs (nohz_full) ===${NC}" + echo -e "${BLUE}Checking nohz_full in GRUB command line...${NC}" + + local grub_file="/etc/default/grub" + + if [ ! -f "$grub_file" ]; then + echo -e " ${RED}FAIL: GRUB configuration file not found at $grub_file${NC}" + return 1 + fi + + local cmdline_default + cmdline_default=$(grep "^GRUB_CMDLINE_LINUX_DEFAULT=" "$grub_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"'"'") + + # Match whole tokens so other parameters containing "nohz_full" are not mistaken for it + local grub_value="" token + for token in $cmdline_default; do + case "$token" in + nohz_full=*) grub_value=${token#nohz_full=} ;; + esac + done + + if [ -z "$grub_value" ]; then + echo -e " ${RED}FAIL: 'nohz_full=' missing from GRUB_CMDLINE_LINUX_DEFAULT${NC}" + echo -e " ${YELLOW}Add 'nohz_full=' to GRUB_CMDLINE_LINUX_DEFAULT in $grub_file${NC}" + echo -e " ${YELLOW}Then run: sudo update-grub && sudo reboot${NC}" + return 1 + fi + + echo -e " ${GREEN}PASS: 'nohz_full=$grub_value' found in GRUB_CMDLINE_LINUX_DEFAULT${NC}" + + # nohz_full only takes effect at boot, so flag a GRUB change that has not been applied yet + local running_value="" + for token in $(cat /proc/cmdline 2>/dev/null); do + case "$token" in + nohz_full=*) running_value=${token#nohz_full=} ;; + esac + done + + if [ "$running_value" != "$grub_value" ]; then + echo -e " ${YELLOW}WARNING: Running kernel has nohz_full='${running_value:-}' - run 'sudo update-grub' and reboot to apply${NC}" + fi + return 0 +} + +# Check nohz_full if enabled in config (off unless a config explicitly enables it) +if should_run_check "nohzFull" "false"; then + if ! check_nohz_full; then + ((failures++)) + failed_checks+=("Tickless CPUs: nohz_full") + fi +else + echo -e "\n${YELLOW}=== Tickless CPUs (nohz_full) ===${NC}" + echo -e "${BLUE}Checking nohz_full in GRUB command line... ${YELLOW}[SKIPPED]${NC}" +fi + # Summary - Always show this part even in quiet mode if [ "$QUIET_MODE" = true ]; then # Restore stdout for the summary