From b8608a8b9df44001c15952d9337146a07f2002c5 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 25 Aug 2026 03:47:18 +0000
Subject: [PATCH 1/2] feat(ansible): BOXP-178 control-plane journal/watchdog
 verification

- node_resilience.yml: fail loudly when /dev/watchdog is absent on real
  hardware (watchdog configured but not activatable must not be silent)
- playbooks/verify-control-plane-resilience.yml: new read-only post-apply
  playbook that asserts effective journald Storage, previous-boot journal,
  RuntimeWatchdogUSec > 0, /dev/watchdog presence, and armbian-ramlog not
  mounting /var/log; outputs machine-readable JSON artifact per host
- molecule default verify.yml: document that watchdog device check is
  expected to be skipped in Docker; assert drop-in was written

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../verify-control-plane-resilience.yml       | 215 ++++++++++++++++++
 .../molecule/default/verify.yml               |  12 +
 .../tasks/node_resilience.yml                 |  28 +++
 3 files changed, 255 insertions(+)
 create mode 100644 ansible/playbooks/verify-control-plane-resilience.yml

diff --git a/ansible/playbooks/verify-control-plane-resilience.yml b/ansible/playbooks/verify-control-plane-resilience.yml
new file mode 100644
index 0000000000..0074cc1006
--- /dev/null
+++ b/ansible/playbooks/verify-control-plane-resilience.yml
@@ -0,0 +1,215 @@
+---
+# Read-only post-apply verification for control-plane journal persistence and hardware watchdog.
+#
+# Run AFTER playbooks/control-plane.yml has been applied to a live node:
+#   ansible-playbook -i inventories/production playbooks/verify-control-plane-resilience.yml
+#
+# Checks the EFFECTIVE state (not just configuration files) and writes a machine-readable
+# JSON artifact to {{ verify_artifact_path }} on the controller.
+#
+# Conditions that cause this playbook to FAIL (not warn):
+#   - journald Storage is not "persistent" in the running daemon
+#   - /var/log/journal directory is absent
+#   - RuntimeWatchdogUSec is 0 (watchdog configured but not activated by systemd)
+#   - /dev/watchdog is absent when watchdog is expected
+#
+# This playbook makes NO changes to the target host.
+
+- name: Verify control-plane journal persistence and hardware watchdog
+  hosts: all
+  gather_facts: true
+  become: true
+  vars:
+    # Local path on the Ansible controller where the JSON artifact is written.
+    verify_artifact_dir: "{{ playbook_dir }}/../.verify-artifacts"
+    verify_artifact_path: >-
+      {{ verify_artifact_dir }}/control-plane-resilience-{{ inventory_hostname }}.json
+
+  tasks:
+    # ── journald Storage effective value ──────────────────────────────────────
+
+    - name: Check if /var/log/journal directory exists
+      ansible.builtin.stat:
+        path: /var/log/journal
+      register: journal_dir_stat
+
+    - name: Read journald drop-in config
+      ansible.builtin.slurp:
+        path: /etc/systemd/journald.conf.d/10-persistent-storage.conf
+      register: journald_dropin_slurp
+      failed_when: false
+
+    - name: Determine journald Storage effective value via journalctl
+      ansible.builtin.command: journalctl --header --no-pager
+      register: journalctl_header
+      changed_when: false
+      failed_when: false
+
+    - name: List journal boots to check for previous-boot records
+      ansible.builtin.command: journalctl --list-boots --no-pager
+      register: journalctl_boots
+      changed_when: false
+      failed_when: false
+
+    # ── hardware watchdog ─────────────────────────────────────────────────────
+
+    - name: Check if /dev/watchdog exists
+      ansible.builtin.stat:
+        path: /dev/watchdog
+      register: watchdog_dev_stat
+
+    - name: Read RuntimeWatchdogUSec from running systemd
+      ansible.builtin.command: systemctl show --property=RuntimeWatchdogUSec --value  # noqa: command-instead-of-module
+      register: runtime_watchdog_usec
+      changed_when: false
+      failed_when: false
+
+    - name: Read watchdog drop-in config
+      ansible.builtin.slurp:
+        path: /etc/systemd/system.conf.d/10-watchdog.conf
+      register: watchdog_dropin_slurp
+      failed_when: false
+
+    # ── armbian-ramlog state ──────────────────────────────────────────────────
+
+    - name: Check if armbian-ramlog config exists
+      ansible.builtin.stat:
+        path: /etc/default/armbian-ramlog
+      register: armbian_ramlog_stat
+
+    - name: Read armbian-ramlog ENABLED setting
+      ansible.builtin.command: grep -E '^ENABLED=' /etc/default/armbian-ramlog
+      register: armbian_ramlog_enabled
+      changed_when: false
+      failed_when: false
+      when: armbian_ramlog_stat.stat.exists
+
+    - name: Check whether /var/log is currently backed by armbian-ramlog
+      ansible.builtin.command: findmnt --noheadings --output SOURCE /var/log
+      register: var_log_mount
+      changed_when: false
+      failed_when: false
+
+    # ── assertions (fail loudly on misconfiguration) ──────────────────────────
+
+    - name: Assert /var/log/journal directory exists
+      ansible.builtin.assert:
+        that:
+          - journal_dir_stat.stat.exists
+          - journal_dir_stat.stat.isdir
+        fail_msg: >-
+          /var/log/journal does not exist — journald persistent storage is NOT active.
+          Pre-reboot logs will be lost on the next boot.
+
+    - name: Assert journald drop-in sets Storage=persistent
+      ansible.builtin.assert:
+        that:
+          - journald_dropin_slurp.content is defined
+          - "'Storage=persistent' in (journald_dropin_slurp.content | b64decode)"
+        fail_msg: >-
+          /etc/systemd/journald.conf.d/10-persistent-storage.conf is absent or does not
+          contain Storage=persistent — journald may be running in volatile mode.
+
+    - name: Assert previous boot journal records exist
+      ansible.builtin.assert:
+        that:
+          - journalctl_boots.stdout_lines | length > 1
+        fail_msg: >-
+          Only one boot entry found in journalctl --list-boots.
+          Either this is the first boot after setup, or the persistent journal is not
+          recording previous boots. Verify /var/log/journal is on persistent storage
+          and is not backed by armbian-ramlog (zram).
+      when: journalctl_boots.rc == 0
+
+    - name: Assert /dev/watchdog exists
+      ansible.builtin.assert:
+        that:
+          - watchdog_dev_stat.stat.exists
+        fail_msg: >-
+          /dev/watchdog does not exist — the hardware watchdog cannot activate even if
+          the drop-in is present. Ensure sunxi-wdt (H618) or an equivalent driver is loaded.
+
+    - name: Assert RuntimeWatchdogUSec is non-zero (watchdog actually activated by systemd)
+      ansible.builtin.assert:
+        that:
+          - runtime_watchdog_usec.stdout | trim != '0'
+          - runtime_watchdog_usec.stdout | trim != ''
+        fail_msg: >-
+          RuntimeWatchdogUSec={{ runtime_watchdog_usec.stdout | trim }}.
+          The watchdog drop-in is present but systemd has NOT activated the watchdog.
+          Possible causes: /dev/watchdog absent, WDIOC_SETTIMEOUT EINVAL (timeout exceeds
+          sunxi-wdt max_timeout=16s), or systemd daemon-reexec has not been run yet.
+      when: runtime_watchdog_usec.rc == 0
+
+    - name: Assert armbian-ramlog is not backing /var/log
+      ansible.builtin.assert:
+        that:
+          - var_log_mount.rc != 0 or var_log_mount.stdout | trim == ''
+        fail_msg: >-
+          /var/log is still mounted by armbian-ramlog ({{ var_log_mount.stdout | trim }}).
+          journald persistent storage will be written to zram and lost on hard hang.
+          Ensure armbian-ramlog has been stopped and disabled (node_resilience_disable_armbian_ramlog).
+
+    # ── collect results into machine-readable artifact ─────────────────────────
+
+    - name: Compute previous_boot_exists
+      ansible.builtin.set_fact:
+        _previous_boot_exists: "{{ journalctl_boots.stdout_lines | length > 1 }}"
+      when: journalctl_boots.rc == 0
+
+    - name: Collect verification results
+      ansible.builtin.set_fact:
+        resilience_verify_result:
+          host: "{{ inventory_hostname }}"
+          journald:
+            var_log_journal_exists: "{{ journal_dir_stat.stat.exists }}"
+            dropin_storage_persistent: >-
+              {{
+                journald_dropin_slurp.content is defined and
+                'Storage=persistent' in (journald_dropin_slurp.content | b64decode)
+              }}
+            previous_boot_exists: "{{ _previous_boot_exists | default(false) }}"
+            boot_count: "{{ journalctl_boots.stdout_lines | length if journalctl_boots.rc == 0 else 0 }}"
+          watchdog:
+            device_exists: "{{ watchdog_dev_stat.stat.exists }}"
+            device_path: /dev/watchdog
+            runtime_watchdog_usec: "{{ runtime_watchdog_usec.stdout | trim if runtime_watchdog_usec.rc == 0 else 'unavailable' }}"
+            runtime_watchdog_active: >-
+              {{
+                runtime_watchdog_usec.rc == 0 and
+                runtime_watchdog_usec.stdout | trim != '' and
+                runtime_watchdog_usec.stdout | trim != '0'
+              }}
+            dropin_exists: "{{ watchdog_dropin_slurp.content is defined }}"
+          armbian_ramlog:
+            config_exists: "{{ armbian_ramlog_stat.stat.exists }}"
+            enabled: >-
+              {{
+                armbian_ramlog_stat.stat.exists and
+                armbian_ramlog_enabled is defined and
+                armbian_ramlog_enabled.rc == 0 and
+                'ENABLED=true' in armbian_ramlog_enabled.stdout
+              }}
+            var_log_mounted_by_ramlog: >-
+              {{ var_log_mount.rc == 0 and var_log_mount.stdout | trim != '' }}
+
+    - name: Show verification summary
+      ansible.builtin.debug:
+        var: resilience_verify_result
+
+    - name: Ensure local artifact directory exists
+      ansible.builtin.file:
+        path: "{{ verify_artifact_dir }}"
+        state: directory
+        mode: '0755'
+      delegate_to: localhost
+      become: false
+      run_once: false
+
+    - name: Write machine-readable JSON artifact
+      ansible.builtin.copy:
+        content: "{{ resilience_verify_result | to_nice_json }}\n"
+        dest: "{{ verify_artifact_path }}"
+        mode: '0644'
+      delegate_to: localhost
+      become: false
diff --git a/ansible/roles/kubernetes_components/molecule/default/verify.yml b/ansible/roles/kubernetes_components/molecule/default/verify.yml
index 86f12f3e37..70400688bb 100644
--- a/ansible/roles/kubernetes_components/molecule/default/verify.yml
+++ b/ansible/roles/kubernetes_components/molecule/default/verify.yml
@@ -303,3 +303,15 @@
           hung task drop-in must set hung_task_panic=1 together with a non-zero kernel.panic,
           otherwise the panic never reboots the board
       when: node_resilience_hung_task_panic | bool
+
+    # Watchdog device check: in Docker the device is absent, so the assert task in
+    # node_resilience.yml is skipped (guarded by ansible_virtualization_type != "docker").
+    # Verify the drop-in was written correctly — the device-existence assertion is not
+    # exercised here because it only runs on real hardware.
+    - name: Confirm watchdog drop-in was written (device-check skip is expected in Docker)
+      ansible.builtin.assert:
+        that:
+          - watchdog_dropin.stat.exists
+        fail_msg: >-
+          watchdog drop-in was not written even though node_resilience_watchdog is true
+      when: node_resilience_watchdog | bool
diff --git a/ansible/roles/kubernetes_components/tasks/node_resilience.yml b/ansible/roles/kubernetes_components/tasks/node_resilience.yml
index 9ed0826f0c..23535bd045 100644
--- a/ansible/roles/kubernetes_components/tasks/node_resilience.yml
+++ b/ansible/roles/kubernetes_components/tasks/node_resilience.yml
@@ -40,6 +40,34 @@
   when: not (node_resilience_watchdog | bool)
   notify: Reexec systemd
 
+# /dev/watchdog が存在しない場合、drop-in が書かれても systemd は watchdog を有効化できない。
+# サイレントに成功扱いにせず、明示的に失敗させる。Docker/chroot は watchdog 不在が正常なので除外。
+- name: Check watchdog device exists on real hardware
+  ansible.builtin.stat:
+    path: /dev/watchdog
+  register: watchdog_device_stat
+  become: true
+  when:
+    - node_resilience_watchdog | bool
+    - ansible_virtualization_type | default('') != "docker"
+    - chroot_build is not defined or not chroot_build
+
+- name: Fail if watchdog device is absent (do not silently treat missing device as success)
+  ansible.builtin.assert:
+    that:
+      - watchdog_device_stat.stat.exists
+    fail_msg: >-
+      /dev/watchdog does not exist on this host.
+      The drop-in /etc/systemd/system.conf.d/10-watchdog.conf has been written
+      but the hardware watchdog CANNOT activate — a hung node will NOT auto-reset.
+      Ensure the sunxi-wdt (H618) driver is loaded and the device node is present
+      before setting node_resilience_watchdog: true.
+  when:
+    - node_resilience_watchdog | bool
+    - ansible_virtualization_type | default('') != "docker"
+    - chroot_build is not defined or not chroot_build
+    - watchdog_device_stat is defined
+
 # hung_task_panic は panic を経由して再起動するため kernel.panic が 0 だと固まったままになる。
 # Armbian の既定は 10 だが、依存せずここで明示する。
 - name: Configure hung task panic sysctl

From adf2c9ba498aef126e621d817d853df7a57143d9 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 25 Aug 2026 04:10:21 +0000
Subject: [PATCH 2/2] fix(ansible): address codex-review findings in
 verify-control-plane-resilience

- Fix armbian-ramlog mount check: use regex_search('ramlog|zram') instead of
  checking for empty stdout, which was wrong when /var/log has no dedicated
  mount (findmnt returns root fs source, not empty)
- Remove 'when: rc == 0' guards that silently skipped assertions on command
  failure; add explicit assert tasks that fail loudly when journalctl
  --list-boots or systemctl show RuntimeWatchdogUSec fail
- Replace unused journalctl --header with systemd-analyze cat-config to
  verify the effective merged Storage= value (catches cases where a drop-in
  is overridden by another drop-in or where journald has not been reloaded)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../verify-control-plane-resilience.yml       | 78 ++++++++++++++-----
 1 file changed, 60 insertions(+), 18 deletions(-)

diff --git a/ansible/playbooks/verify-control-plane-resilience.yml b/ansible/playbooks/verify-control-plane-resilience.yml
index 0074cc1006..61bf6424cb 100644
--- a/ansible/playbooks/verify-control-plane-resilience.yml
+++ b/ansible/playbooks/verify-control-plane-resilience.yml
@@ -8,10 +8,11 @@
 # JSON artifact to {{ verify_artifact_path }} on the controller.
 #
 # Conditions that cause this playbook to FAIL (not warn):
-#   - journald Storage is not "persistent" in the running daemon
+#   - journald effective Storage is not "persistent" in the running daemon config
 #   - /var/log/journal directory is absent
 #   - RuntimeWatchdogUSec is 0 (watchdog configured but not activated by systemd)
 #   - /dev/watchdog is absent when watchdog is expected
+#   - armbian-ramlog (ramlog/zram) is backing /var/log
 #
 # This playbook makes NO changes to the target host.
 
@@ -39,9 +40,9 @@
       register: journald_dropin_slurp
       failed_when: false
 
-    - name: Determine journald Storage effective value via journalctl
-      ansible.builtin.command: journalctl --header --no-pager
-      register: journalctl_header
+    - name: Determine journald effective Storage via systemd-analyze cat-config
+      ansible.builtin.command: systemd-analyze cat-config systemd/journald.conf
+      register: journald_effective_config
       changed_when: false
       failed_when: false
 
@@ -84,7 +85,7 @@
       failed_when: false
       when: armbian_ramlog_stat.stat.exists
 
-    - name: Check whether /var/log is currently backed by armbian-ramlog
+    - name: Check whether /var/log is currently backed by ramlog or zram
       ansible.builtin.command: findmnt --noheadings --output SOURCE /var/log
       register: var_log_mount
       changed_when: false
@@ -110,6 +111,38 @@
           /etc/systemd/journald.conf.d/10-persistent-storage.conf is absent or does not
           contain Storage=persistent — journald may be running in volatile mode.
 
+    - name: Assert systemd-analyze cat-config succeeded (needed to verify effective Storage)
+      ansible.builtin.assert:
+        that:
+          - journald_effective_config.rc == 0
+        fail_msg: >-
+          systemd-analyze cat-config systemd/journald.conf failed (rc={{ journald_effective_config.rc }}).
+          Cannot verify effective journald Storage value. Output: {{ journald_effective_config.stderr }}
+
+    - name: Assert journald effective Storage is persistent (last Storage= wins)
+      vars:
+        _storage_lines: >-
+          {{ journald_effective_config.stdout_lines
+             | select('match', '^Storage=')
+             | list }}
+        _effective_storage: >-
+          {{ (_storage_lines | last) if _storage_lines | length > 0 else 'auto' }}
+      ansible.builtin.assert:
+        that:
+          - _effective_storage == 'Storage=persistent'
+        fail_msg: >-
+          journald effective Storage is "{{ _effective_storage }}", not "Storage=persistent".
+          A drop-in may have been overridden by another drop-in, or journald has not been
+          reloaded after the drop-in was written. Run: systemctl restart systemd-journald
+
+    - name: Assert journalctl --list-boots succeeded (needed to verify persistent journal)
+      ansible.builtin.assert:
+        that:
+          - journalctl_boots.rc == 0
+        fail_msg: >-
+          journalctl --list-boots failed (rc={{ journalctl_boots.rc }}).
+          Cannot verify persistent journal. Error: {{ journalctl_boots.stderr }}
+
     - name: Assert previous boot journal records exist
       ansible.builtin.assert:
         that:
@@ -119,7 +152,6 @@
           Either this is the first boot after setup, or the persistent journal is not
           recording previous boots. Verify /var/log/journal is on persistent storage
           and is not backed by armbian-ramlog (zram).
-      when: journalctl_boots.rc == 0
 
     - name: Assert /dev/watchdog exists
       ansible.builtin.assert:
@@ -129,6 +161,14 @@
           /dev/watchdog does not exist — the hardware watchdog cannot activate even if
           the drop-in is present. Ensure sunxi-wdt (H618) or an equivalent driver is loaded.
 
+    - name: Assert systemctl show RuntimeWatchdogUSec succeeded
+      ansible.builtin.assert:
+        that:
+          - runtime_watchdog_usec.rc == 0
+        fail_msg: >-
+          systemctl show --property=RuntimeWatchdogUSec failed (rc={{ runtime_watchdog_usec.rc }}).
+          Cannot verify watchdog activation state. Error: {{ runtime_watchdog_usec.stderr }}
+
     - name: Assert RuntimeWatchdogUSec is non-zero (watchdog actually activated by systemd)
       ansible.builtin.assert:
         that:
@@ -139,23 +179,24 @@
           The watchdog drop-in is present but systemd has NOT activated the watchdog.
           Possible causes: /dev/watchdog absent, WDIOC_SETTIMEOUT EINVAL (timeout exceeds
           sunxi-wdt max_timeout=16s), or systemd daemon-reexec has not been run yet.
-      when: runtime_watchdog_usec.rc == 0
 
     - name: Assert armbian-ramlog is not backing /var/log
       ansible.builtin.assert:
         that:
-          - var_log_mount.rc != 0 or var_log_mount.stdout | trim == ''
+          - not (var_log_mount.stdout | trim | regex_search('(ramlog|zram)'))
         fail_msg: >-
-          /var/log is still mounted by armbian-ramlog ({{ var_log_mount.stdout | trim }}).
+          /var/log is still mounted by armbian-ramlog/zram (source: {{ var_log_mount.stdout | trim }}).
           journald persistent storage will be written to zram and lost on hard hang.
           Ensure armbian-ramlog has been stopped and disabled (node_resilience_disable_armbian_ramlog).
 
     # ── collect results into machine-readable artifact ─────────────────────────
 
-    - name: Compute previous_boot_exists
+    - name: Extract effective Storage value from cat-config output
       ansible.builtin.set_fact:
-        _previous_boot_exists: "{{ journalctl_boots.stdout_lines | length > 1 }}"
-      when: journalctl_boots.rc == 0
+        _storage_lines_fact: >-
+          {{ journald_effective_config.stdout_lines
+             | select('match', '^Storage=')
+             | list }}
 
     - name: Collect verification results
       ansible.builtin.set_fact:
@@ -168,15 +209,16 @@
                 journald_dropin_slurp.content is defined and
                 'Storage=persistent' in (journald_dropin_slurp.content | b64decode)
               }}
-            previous_boot_exists: "{{ _previous_boot_exists | default(false) }}"
-            boot_count: "{{ journalctl_boots.stdout_lines | length if journalctl_boots.rc == 0 else 0 }}"
+            effective_storage: >-
+              {{ (_storage_lines_fact | last) if _storage_lines_fact | length > 0 else 'auto' }}
+            previous_boot_exists: "{{ journalctl_boots.stdout_lines | length > 1 }}"
+            boot_count: "{{ journalctl_boots.stdout_lines | length }}"
           watchdog:
             device_exists: "{{ watchdog_dev_stat.stat.exists }}"
             device_path: /dev/watchdog
-            runtime_watchdog_usec: "{{ runtime_watchdog_usec.stdout | trim if runtime_watchdog_usec.rc == 0 else 'unavailable' }}"
+            runtime_watchdog_usec: "{{ runtime_watchdog_usec.stdout | trim }}"
             runtime_watchdog_active: >-
               {{
-                runtime_watchdog_usec.rc == 0 and
                 runtime_watchdog_usec.stdout | trim != '' and
                 runtime_watchdog_usec.stdout | trim != '0'
               }}
@@ -190,8 +232,8 @@
                 armbian_ramlog_enabled.rc == 0 and
                 'ENABLED=true' in armbian_ramlog_enabled.stdout
               }}
-            var_log_mounted_by_ramlog: >-
-              {{ var_log_mount.rc == 0 and var_log_mount.stdout | trim != '' }}
+            var_log_backed_by_ramlog: >-
+              {{ var_log_mount.stdout | trim | regex_search('(ramlog|zram)') | bool }}
 
     - name: Show verification summary
       ansible.builtin.debug: