feat(ansible): BOXP-178 control-plane journal/watchdog verification

ansible/playbooks/verify-control-plane-resilience.yml

@@ -0,0 +1,257 @@
+---
+# Read-only post-apply verification for control-plane journal persistence and hardware watchdog.
+#
+# Run AFTER playbooks/control-plane.yml has been applied to a live node:
+#   ansible-playbook -i inventories/production playbooks/verify-control-plane-resilience.yml
+#
+# Checks the EFFECTIVE state (not just configuration files) and writes a machine-readable
+# JSON artifact to {{ verify_artifact_path }} on the controller.
+#
+# Conditions that cause this playbook to FAIL (not warn):
+#   - journald effective Storage is not "persistent" in the running daemon config
+#   - /var/log/journal directory is absent
+#   - RuntimeWatchdogUSec is 0 (watchdog configured but not activated by systemd)
+#   - /dev/watchdog is absent when watchdog is expected
+#   - armbian-ramlog (ramlog/zram) is backing /var/log
+#
+# This playbook makes NO changes to the target host.
+
+- name: Verify control-plane journal persistence and hardware watchdog
+  hosts: all
+  gather_facts: true
+  become: true
+  vars:
+    # Local path on the Ansible controller where the JSON artifact is written.
+    verify_artifact_dir: "{{ playbook_dir }}/../.verify-artifacts"
+    verify_artifact_path: >-
+      {{ verify_artifact_dir }}/control-plane-resilience-{{ inventory_hostname }}.json
+
+  tasks:
+    # ── journald Storage effective value ──────────────────────────────────────
+
+    - name: Check if /var/log/journal directory exists
+      ansible.builtin.stat:
+        path: /var/log/journal
+      register: journal_dir_stat
+
+    - name: Read journald drop-in config
+      ansible.builtin.slurp:
+        path: /etc/systemd/journald.conf.d/10-persistent-storage.conf
+      register: journald_dropin_slurp
+      failed_when: false
+
+    - name: Determine journald effective Storage via systemd-analyze cat-config
+      ansible.builtin.command: systemd-analyze cat-config systemd/journald.conf
+      register: journald_effective_config
+      changed_when: false
+      failed_when: false
+
+    - name: List journal boots to check for previous-boot records
+      ansible.builtin.command: journalctl --list-boots --no-pager
+      register: journalctl_boots
+      changed_when: false
+      failed_when: false
+
+    # ── hardware watchdog ─────────────────────────────────────────────────────
+
+    - name: Check if /dev/watchdog exists
+      ansible.builtin.stat:
+        path: /dev/watchdog
+      register: watchdog_dev_stat
+
+    - name: Read RuntimeWatchdogUSec from running systemd
+      ansible.builtin.command: systemctl show --property=RuntimeWatchdogUSec --value  # noqa: command-instead-of-module
+      register: runtime_watchdog_usec
+      changed_when: false
+      failed_when: false
+
+    - name: Read watchdog drop-in config
+      ansible.builtin.slurp:
+        path: /etc/systemd/system.conf.d/10-watchdog.conf
+      register: watchdog_dropin_slurp
+      failed_when: false
+
+    # ── armbian-ramlog state ──────────────────────────────────────────────────
+
+    - name: Check if armbian-ramlog config exists
+      ansible.builtin.stat:
+        path: /etc/default/armbian-ramlog
+      register: armbian_ramlog_stat
+
+    - name: Read armbian-ramlog ENABLED setting
+      ansible.builtin.command: grep -E '^ENABLED=' /etc/default/armbian-ramlog
+      register: armbian_ramlog_enabled
+      changed_when: false
+      failed_when: false
+      when: armbian_ramlog_stat.stat.exists
+
+    - name: Check whether /var/log is currently backed by ramlog or zram
+      ansible.builtin.command: findmnt --noheadings --output SOURCE /var/log
+      register: var_log_mount
+      changed_when: false
+      failed_when: false
+
+    # ── assertions (fail loudly on misconfiguration) ──────────────────────────
+
+    - name: Assert /var/log/journal directory exists
+      ansible.builtin.assert:
+        that:
+          - journal_dir_stat.stat.exists
+          - journal_dir_stat.stat.isdir
+        fail_msg: >-
+          /var/log/journal does not exist — journald persistent storage is NOT active.
+          Pre-reboot logs will be lost on the next boot.
+
+    - name: Assert journald drop-in sets Storage=persistent
+      ansible.builtin.assert:
+        that:
+          - journald_dropin_slurp.content is defined
+          - "'Storage=persistent' in (journald_dropin_slurp.content | b64decode)"
+        fail_msg: >-
+          /etc/systemd/journald.conf.d/10-persistent-storage.conf is absent or does not
+          contain Storage=persistent — journald may be running in volatile mode.
+
+    - name: Assert systemd-analyze cat-config succeeded (needed to verify effective Storage)
+      ansible.builtin.assert:
+        that:
+          - journald_effective_config.rc == 0
+        fail_msg: >-
+          systemd-analyze cat-config systemd/journald.conf failed (rc={{ journald_effective_config.rc }}).
+          Cannot verify effective journald Storage value. Output: {{ journald_effective_config.stderr }}
+
+    - name: Assert journald effective Storage is persistent (last Storage= wins)
+      vars:
+        _storage_lines: >-
+          {{ journald_effective_config.stdout_lines
+             | select('match', '^Storage=')
+             | list }}
+        _effective_storage: >-
+          {{ (_storage_lines | last) if _storage_lines | length > 0 else 'auto' }}
+      ansible.builtin.assert:
+        that:
+          - _effective_storage == 'Storage=persistent'
+        fail_msg: >-
+          journald effective Storage is "{{ _effective_storage }}", not "Storage=persistent".
+          A drop-in may have been overridden by another drop-in, or journald has not been
+          reloaded after the drop-in was written. Run: systemctl restart systemd-journald
+
+    - name: Assert journalctl --list-boots succeeded (needed to verify persistent journal)
+      ansible.builtin.assert:
+        that:
+          - journalctl_boots.rc == 0
+        fail_msg: >-
+          journalctl --list-boots failed (rc={{ journalctl_boots.rc }}).
+          Cannot verify persistent journal. Error: {{ journalctl_boots.stderr }}
+
+    - name: Assert previous boot journal records exist
+      ansible.builtin.assert:
+        that:
+          - journalctl_boots.stdout_lines | length > 1
+        fail_msg: >-
+          Only one boot entry found in journalctl --list-boots.
+          Either this is the first boot after setup, or the persistent journal is not
+          recording previous boots. Verify /var/log/journal is on persistent storage
+          and is not backed by armbian-ramlog (zram).
+
+    - name: Assert /dev/watchdog exists
+      ansible.builtin.assert:
+        that:
+          - watchdog_dev_stat.stat.exists
+        fail_msg: >-
+          /dev/watchdog does not exist — the hardware watchdog cannot activate even if
+          the drop-in is present. Ensure sunxi-wdt (H618) or an equivalent driver is loaded.
+
+    - name: Assert systemctl show RuntimeWatchdogUSec succeeded
+      ansible.builtin.assert:
+        that:
+          - runtime_watchdog_usec.rc == 0
+        fail_msg: >-
+          systemctl show --property=RuntimeWatchdogUSec failed (rc={{ runtime_watchdog_usec.rc }}).
+          Cannot verify watchdog activation state. Error: {{ runtime_watchdog_usec.stderr }}
+
+    - name: Assert RuntimeWatchdogUSec is non-zero (watchdog actually activated by systemd)
+      ansible.builtin.assert:
+        that:
+          - runtime_watchdog_usec.stdout | trim != '0'
+          - runtime_watchdog_usec.stdout | trim != ''
+        fail_msg: >-
+          RuntimeWatchdogUSec={{ runtime_watchdog_usec.stdout | trim }}.
+          The watchdog drop-in is present but systemd has NOT activated the watchdog.
+          Possible causes: /dev/watchdog absent, WDIOC_SETTIMEOUT EINVAL (timeout exceeds
+          sunxi-wdt max_timeout=16s), or systemd daemon-reexec has not been run yet.
+
+    - name: Assert armbian-ramlog is not backing /var/log
+      ansible.builtin.assert:
+        that:
+          - not (var_log_mount.stdout | trim | regex_search('(ramlog|zram)'))
+        fail_msg: >-
+          /var/log is still mounted by armbian-ramlog/zram (source: {{ var_log_mount.stdout | trim }}).
+          journald persistent storage will be written to zram and lost on hard hang.
+          Ensure armbian-ramlog has been stopped and disabled (node_resilience_disable_armbian_ramlog).
+
+    # ── collect results into machine-readable artifact ─────────────────────────
+
+    - name: Extract effective Storage value from cat-config output
+      ansible.builtin.set_fact:
+        _storage_lines_fact: >-
+          {{ journald_effective_config.stdout_lines
+             | select('match', '^Storage=')
+             | list }}
+
+    - name: Collect verification results
+      ansible.builtin.set_fact:
+        resilience_verify_result:
+          host: "{{ inventory_hostname }}"
+          journald:
+            var_log_journal_exists: "{{ journal_dir_stat.stat.exists }}"
+            dropin_storage_persistent: >-
+              {{
+                journald_dropin_slurp.content is defined and
+                'Storage=persistent' in (journald_dropin_slurp.content | b64decode)
+              }}
+            effective_storage: >-
+              {{ (_storage_lines_fact | last) if _storage_lines_fact | length > 0 else 'auto' }}
+            previous_boot_exists: "{{ journalctl_boots.stdout_lines | length > 1 }}"
+            boot_count: "{{ journalctl_boots.stdout_lines | length }}"
+          watchdog:
+            device_exists: "{{ watchdog_dev_stat.stat.exists }}"
+            device_path: /dev/watchdog
+            runtime_watchdog_usec: "{{ runtime_watchdog_usec.stdout | trim }}"
+            runtime_watchdog_active: >-
+              {{
+                runtime_watchdog_usec.stdout | trim != '' and
+                runtime_watchdog_usec.stdout | trim != '0'
+              }}
+            dropin_exists: "{{ watchdog_dropin_slurp.content is defined }}"
+          armbian_ramlog:
+            config_exists: "{{ armbian_ramlog_stat.stat.exists }}"
+            enabled: >-
+              {{
+                armbian_ramlog_stat.stat.exists and
+                armbian_ramlog_enabled is defined and
+                armbian_ramlog_enabled.rc == 0 and
+                'ENABLED=true' in armbian_ramlog_enabled.stdout
+              }}
+            var_log_backed_by_ramlog: >-
+              {{ var_log_mount.stdout | trim | regex_search('(ramlog|zram)') | bool }}
+
+    - name: Show verification summary
+      ansible.builtin.debug:
+        var: resilience_verify_result
+
+    - name: Ensure local artifact directory exists
+      ansible.builtin.file:
+        path: "{{ verify_artifact_dir }}"
+        state: directory
+        mode: '0755'
+      delegate_to: localhost
+      become: false
+      run_once: false
+
+    - name: Write machine-readable JSON artifact
+      ansible.builtin.copy:
+        content: "{{ resilience_verify_result | to_nice_json }}\n"
+        dest: "{{ verify_artifact_path }}"
+        mode: '0644'
+      delegate_to: localhost
+      become: false

ansible/roles/kubernetes_components/molecule/default/verify.yml

@@ -303,3 +303,15 @@
           hung task drop-in must set hung_task_panic=1 together with a non-zero kernel.panic,
           otherwise the panic never reboots the board
       when: node_resilience_hung_task_panic | bool
+
+    # Watchdog device check: in Docker the device is absent, so the assert task in
+    # node_resilience.yml is skipped (guarded by ansible_virtualization_type != "docker").
+    # Verify the drop-in was written correctly — the device-existence assertion is not
+    # exercised here because it only runs on real hardware.
+    - name: Confirm watchdog drop-in was written (device-check skip is expected in Docker)
+      ansible.builtin.assert:
+        that:
+          - watchdog_dropin.stat.exists
+        fail_msg: >-
+          watchdog drop-in was not written even though node_resilience_watchdog is true
+      when: node_resilience_watchdog | bool

ansible/roles/kubernetes_components/tasks/node_resilience.yml

@@ -40,6 +40,34 @@
   when: not (node_resilience_watchdog | bool)
   notify: Reexec systemd
 
+# /dev/watchdog が存在しない場合、drop-in が書かれても systemd は watchdog を有効化できない。
+# サイレントに成功扱いにせず、明示的に失敗させる。Docker/chroot は watchdog 不在が正常なので除外。
+- name: Check watchdog device exists on real hardware
+  ansible.builtin.stat:
+    path: /dev/watchdog
+  register: watchdog_device_stat
+  become: true
+  when:
+    - node_resilience_watchdog | bool
+    - ansible_virtualization_type | default('') != "docker"
+    - chroot_build is not defined or not chroot_build
+
+- name: Fail if watchdog device is absent (do not silently treat missing device as success)
+  ansible.builtin.assert:
+    that:
+      - watchdog_device_stat.stat.exists
+    fail_msg: >-
+      /dev/watchdog does not exist on this host.
+      The drop-in /etc/systemd/system.conf.d/10-watchdog.conf has been written
+      but the hardware watchdog CANNOT activate — a hung node will NOT auto-reset.
+      Ensure the sunxi-wdt (H618) driver is loaded and the device node is present
+      before setting node_resilience_watchdog: true.
+  when:
+    - node_resilience_watchdog | bool
+    - ansible_virtualization_type | default('') != "docker"
+    - chroot_build is not defined or not chroot_build
+    - watchdog_device_stat is defined
+
 # hung_task_panic は panic を経由して再起動するため kernel.panic が 0 だと固まったままになる。
 # Armbian の既定は 10 だが、依存せずここで明示する。
 - name: Configure hung task panic sysctl