fix(ops): add logrotate installer and stack rotation validation (#100)

Taskfile.yml

@@ -245,7 +245,7 @@ tasks:
   rotate-logs:
     desc: Rotate the OTel-tailed log files once now (uses scripts/logrotate-localobserve.conf)
     cmds:
-      - logrotate -f scripts/logrotate-localobserve.conf || echo "logrotate not installed system-wide; see scripts/logrotate-localobserve.conf header for install steps"
+      - bash scripts/install-logrotate.sh --force || echo "logrotate failed; if permission denied run: sudo bash scripts/install-logrotate.sh --force"
 
   validate-otel-config:
     desc: Validate otel-collector-config.yaml with the pinned collector image

scripts/install-logrotate.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# install-logrotate.sh — Host logrotate installation and verification for LocalObserve
+#
+# Usage:
+#   sudo bash scripts/install-logrotate.sh           # Install /etc/logrotate.d/localobserve
+#   bash scripts/install-logrotate.sh --dry-run     # Validate syntax and test run without modifying files
+#   bash scripts/install-logrotate.sh --status      # Check log sizes and rotation status
+#   sudo bash scripts/install-logrotate.sh --force  # Force rotation immediately
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+SOURCE_CONF="${REPO_ROOT}/scripts/logrotate-localobserve.conf"
+TARGET_CONF="/etc/logrotate.d/localobserve"
+
+generate_config() {
+    local target_file="$1"
+    # Replace default path with actual repo root if different
+    sed "s|/home/john/LocalObserve|${REPO_ROOT}|g" "$SOURCE_CONF" > "$target_file"
+}
+
+case "${1:-}" in
+    --dry-run)
+        tmp_conf=$(mktemp)
+        generate_config "$tmp_conf"
+        echo "[*] Testing logrotate configuration syntax (dry-run)..."
+        logrotate -d "$tmp_conf"
+        rm -f "$tmp_conf"
+        echo "[+] Logrotate configuration is valid."
+        exit 0
+        ;;
+    --status)
+        echo "=== LocalObserve Log File Status ==="
+        for f in "${REPO_ROOT}"/.data/{osquery/osqueryd.results.log,falco/events.jsonl,clamav/scan.log,goflow2/flows.jsonl}; do
+            if [ -f "$f" ]; then
+                ls -lh "$f"
+            else
+                echo "  (not present): $f"
+            fi
+        done
+        if [ -f "$TARGET_CONF" ]; then
+            echo "[+] Installed at: $TARGET_CONF"
+        else
+            echo "[!] Not installed at: $TARGET_CONF"
+        fi
+        exit 0
+        ;;
+    --force)
+        tmp_conf=$(mktemp)
+        generate_config "$tmp_conf"
+        mkdir -p "${REPO_ROOT}/.data"
+        echo "[*] Forcing rotation of LocalObserve logs..."
+        logrotate -s "${REPO_ROOT}/.data/logrotate.status" -f "$tmp_conf"
+        rm -f "$tmp_conf"
+        echo "[+] Successfully forced rotation of LocalObserve logs."
+        exit 0
+        ;;
+    --install|"")
+        if [ "$(id -u)" -ne 0 ]; then
+            echo "Error: installation requires root privileges." >&2
+            echo "Usage: sudo bash $0" >&2
+            exit 1
+        fi
+        tmp_conf=$(mktemp)
+        generate_config "$tmp_conf"
+        install -m 0644 "$tmp_conf" "$TARGET_CONF"
+        rm -f "$tmp_conf"
+        echo "[+] Successfully installed $TARGET_CONF"
+        echo "[+] LocalObserve log files will now be rotated weekly or when exceeding maxsize (50M/100M)."
+        exit 0
+        ;;
+    *)
+        echo "Usage: $0 [--install|--dry-run|--status|--force]" >&2
+        exit 1
+        ;;
+esac

tests/test_stack_rotation.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+"""
+test_stack_rotation.py
+
+Validates log rotation configuration and mechanics:
+1. scripts/logrotate-localobserve.conf syntax and directives
+2. copytruncate usage across all tailed log paths
+3. scripts/install-logrotate.sh presence and execution
+4. Bounded rotation caps to prevent unbounded disk growth (Issue #100)
+"""
+
+import os
+import shutil
+import stat
+import subprocess
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+LOGROTATE_CONF = REPO_ROOT / "scripts" / "logrotate-localobserve.conf"
+INSTALL_SCRIPT = REPO_ROOT / "scripts" / "install-logrotate.sh"
+
+
+def test_logrotate_config_exists() -> None:
+    """logrotate configuration file must exist in scripts/."""
+    assert LOGROTATE_CONF.exists(), "scripts/logrotate-localobserve.conf missing"
+
+
+def test_install_logrotate_script_exists_and_executable() -> None:
+    """scripts/install-logrotate.sh must exist and be executable."""
+    assert INSTALL_SCRIPT.exists(), "scripts/install-logrotate.sh missing"
+    mode = INSTALL_SCRIPT.stat().st_mode
+    assert mode & stat.S_IXUSR, "scripts/install-logrotate.sh must be executable by owner"
+
+
+def test_logrotate_syntax_validation() -> None:
+    """scripts/logrotate-localobserve.conf must pass logrotate -d validation."""
+    if not shutil.which("logrotate"):
+        pytest.skip("logrotate binary not available on this host")
+
+    res = subprocess.run(
+        ["bash", str(INSTALL_SCRIPT), "--dry-run"],
+        cwd=REPO_ROOT,
+        capture_output=True,
+        text=True,
+    )
+    assert res.returncode == 0, f"logrotate validation failed:\n{res.stderr}"
+    assert "Logrotate configuration is valid" in res.stdout
+
+
+def test_all_log_sections_use_copytruncate() -> None:
+    """All logrotate sections must specify copytruncate.
+
+    copytruncate is required because the OpenTelemetry Collector's file_log
+    receiver maintains open file descriptors on the active log files.
+    Renaming files breaks tailing continuity unless copytruncate preserves
+    the inode and active file path.
+    """
+    content = LOGROTATE_CONF.read_text()
+    sections = [s.strip() for s in content.split("}") if "{" in s]
+    assert len(sections) >= 3, "Expected at least 3 logrotate sections"
+
+    for sec in sections:
+        header = sec.split("{")[0].strip()
+        assert "copytruncate" in sec, (
+            f"Section '{header}' is missing 'copytruncate' directive"
+        )
+
+
+def test_all_log_sections_specify_bounded_size() -> None:
+    """All log sections must specify maxsize or size caps to prevent unbounded SSD growth."""
+    content = LOGROTATE_CONF.read_text()
+    sections = [s.strip() for s in content.split("}") if "{" in s]
+
+    for sec in sections:
+        header = sec.split("{")[0].strip()
+        assert "maxsize" in sec or "size" in sec, (
+            f"Section '{header}' is missing 'maxsize' or 'size' cap"
+        )
+
+
+def test_osquery_results_log_is_capped_at_or_below_50m() -> None:
+    """osqueryd.results.log must be rotated at or below 50M to prevent unbounded growth (Issue #100)."""
+    content = LOGROTATE_CONF.read_text()
+    assert "osqueryd.results.log" in content
+
+    # Locate the osquery block
+    osquery_sec = None
+    for sec in content.split("}"):
+        if "osqueryd.results.log" in sec:
+            osquery_sec = sec
+            break
+
+    assert osquery_sec is not None, "osquery section not found in logrotate config"
+    assert "maxsize 50M" in osquery_sec or "size 50M" in osquery_sec, (
+        "osqueryd.results.log rotation must be bounded at 50M"
+    )
+
+
+def test_copytruncate_simulation_preserves_tailing_continuity(tmp_path: Path) -> None:
+    """Verify that copytruncate semantics allow continued append and read without inode changes."""
+    log_file = tmp_path / "test.log"
+    rot_file = tmp_path / "test.log.1"
+
+    # Step 1: Write initial lines
+    with open(log_file, "w") as f:
+        f.write('{"event": "start", "seq": 1}\n')
+        f.write('{"event": "auth", "seq": 2}\n')
+
+    initial_inode = os.stat(log_file).st_ino
+
+    # Step 2: Simulate copytruncate (copy active -> .1, then truncate active to 0)
+    shutil.copyfile(log_file, rot_file)
+    with open(log_file, "r+") as f:
+        f.truncate(0)
+
+    post_truncate_inode = os.stat(log_file).st_ino
+    assert initial_inode == post_truncate_inode, "Inode must remain unchanged across copytruncate"
+    assert os.path.getsize(log_file) == 0, "Log file size must be 0 after truncate"
+    assert os.path.getsize(rot_file) > 0, "Rotated file must contain preserved records"
+
+    # Step 3: Append new records to active file
+    with open(log_file, "a") as f:
+        f.write('{"event": "post_rotation", "seq": 3}\n')
+
+    with open(log_file, "r") as f:
+        lines = f.readlines()
+
+    assert len(lines) == 1
+    assert "post_rotation" in lines[0]