fix: alert before sqlite filesystem exhaustion

docs/operations.md

@@ -172,8 +172,18 @@ It checks:
 - blocked jobs;
 - pending jobs older than 300 seconds;
 - running jobs older than review/work thresholds;
+- free capacity on the filesystem containing the SQLite database (by default,
+  alert below either 5 GiB or 10% available);
 - configured SLO thresholds after a minimum sample count and sustained breach.
 
+The disk thresholds can be tuned in the private systemd environment. Set an
+individual threshold to `0` to disable that condition:
+
+```text
+GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES=5368709120
+GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_PERCENT=10
+```
+
 By default the monitor command also writes bounded observability rows to the
 bridge database:
 

src/github_agent_bridge/cli.py

@@ -26,7 +26,13 @@
 from .dispatch import FEEDBACK_LEARNING_RULES, GitHubClient, OpenClawDispatcher, RunMode, prompt_rule
 from .executor import ExecutorConfig, ExecutorPool
 from .models import Notification, utc_now
-from .monitor import MonitorThresholds, monitor, report_json
+from .monitor import (
+    DEFAULT_DATABASE_FREE_WARN_BYTES,
+    DEFAULT_DATABASE_FREE_WARN_PERCENT,
+    MonitorThresholds,
+    monitor,
+    report_json,
+)
 from .mcp import authenticate_token, create_token, list_tokens, revoke_token, serve_stdio
 from .observability import DEFAULT_PROCESS_SAMPLE_RETENTION_SECONDS, configure_sentry
 from .parser import decode_header_value, extract_body_text, is_github_notification_message, parse_auth_results
@@ -396,6 +402,8 @@ def cmd_monitor(args: argparse.Namespace) -> int:
         review_running_warn_seconds=args.review_running_warn_seconds,
         work_running_warn_seconds=args.work_running_warn_seconds,
         progress_warn_seconds=args.progress_warn_seconds,
+        database_free_warn_bytes=args.database_free_warn_bytes,
+        database_free_warn_percent=args.database_free_warn_percent,
         slo_window=args.slo_window,
         slo_min_samples=args.slo_min_samples,
         slo_breach_observations=args.slo_breach_observations,
@@ -664,6 +672,8 @@ def build_parser() -> argparse.ArgumentParser:
     s.add_argument("--review-running-warn-seconds", type=int, default=1200)
     s.add_argument("--work-running-warn-seconds", type=int, default=4200)
     s.add_argument("--progress-warn-seconds", type=int, default=600)
+    s.add_argument("--database-free-warn-bytes", type=int, default=DEFAULT_DATABASE_FREE_WARN_BYTES)
+    s.add_argument("--database-free-warn-percent", type=float, default=DEFAULT_DATABASE_FREE_WARN_PERCENT)
     s.add_argument("--slo-window", choices=("24h", "7d", "30d"), default="7d")
     s.add_argument("--slo-min-samples", type=int, default=20)
     s.add_argument("--slo-breach-observations", type=int, default=3)

src/github_agent_bridge/monitor.py

@@ -3,6 +3,7 @@
 import hashlib
 import json
 import os
+import shutil
 import subprocess
 import time
 import urllib.error
@@ -25,6 +26,8 @@
 ALERT_RELEASE_AVAILABLE = "monitor.release_available"
 ALERT_DATABASE_MISSING = "monitor.database_missing"
 ALERT_SCHEMA_MISSING = "monitor.schema_missing"
+ALERT_DATABASE_FILESYSTEM_LOW = "monitor.database_filesystem_low"
+ALERT_DATABASE_FILESYSTEM_UNAVAILABLE = "monitor.database_filesystem_unavailable"
 ALERT_BLOCKED_JOBS = "monitor.blocked_jobs"
 ALERT_PENDING_QUEUE_OLD = "monitor.pending_queue_old"
 ALERT_EXECUTOR_SERVICE = "monitor.executor_service"
@@ -45,6 +48,8 @@
 
 
 SLO_WINDOWS = {"24h", "7d", "30d"}
+DEFAULT_DATABASE_FREE_WARN_BYTES = 5 * 1024 * 1024 * 1024
+DEFAULT_DATABASE_FREE_WARN_PERCENT = 10.0
 SLO_THRESHOLD_FIELDS = (
     "slo_queued_to_claimed_p90_seconds",
     "slo_claimed_to_first_output_p90_seconds",
@@ -62,6 +67,8 @@ class MonitorThresholds:
     progress_warn_seconds: int = 600
     reader_recent_seconds: int = 180
     worker_heartbeat_warn_seconds: int = 20
+    database_free_warn_bytes: int = DEFAULT_DATABASE_FREE_WARN_BYTES
+    database_free_warn_percent: float = DEFAULT_DATABASE_FREE_WARN_PERCENT
     slo_window: str = "7d"
     slo_min_samples: int = 20
     slo_breach_observations: int = 3
@@ -74,6 +81,10 @@ class MonitorThresholds:
     slo_retry_rate_percent: float | None = None
 
     def __post_init__(self) -> None:
+        if self.database_free_warn_bytes < 0:
+            raise ValueError("database_free_warn_bytes must be non-negative")
+        if not 0 <= self.database_free_warn_percent <= 100:
+            raise ValueError("database_free_warn_percent must be between 0 and 100")
         if self.slo_window not in SLO_WINDOWS:
             raise ValueError(f"unsupported SLO window {self.slo_window!r}")
         if self.slo_min_samples < 1:
@@ -139,6 +150,13 @@ def text(self) -> str:
             f"oldest_pending={metrics.get('oldest_pending_age_seconds') if metrics.get('oldest_pending_age_seconds') is not None else '-'}",
             f"last_uid={metrics.get('last_uid', '-')}",
         ]
+        database_filesystem = metrics.get("database_filesystem") or {}
+        if database_filesystem:
+            compact.append(
+                "db_disk_free="
+                f"{_format_bytes(database_filesystem.get('available_bytes'))}/"
+                f"{database_filesystem.get('available_percent', 0):.1f}%"
+            )
         parts.append(" ".join(compact))
         if self.alerts:
             alert_codes = metrics.get("alert_codes") or []
@@ -229,6 +247,35 @@ def inspect_db(path: str | Path) -> dict[str, Any]:
     return DashboardQueries(path).status()
 
 
+def _database_filesystem_usage(path: str | Path) -> dict[str, Any]:
+    database_path = Path(path).expanduser().resolve(strict=False)
+    probe_path = database_path.parent
+    while not probe_path.exists() and probe_path != probe_path.parent:
+        probe_path = probe_path.parent
+    usage = shutil.disk_usage(probe_path)
+    available_percent = (usage.free / usage.total * 100.0) if usage.total else 0.0
+    return {
+        "path": str(probe_path.resolve()),
+        "total_bytes": int(usage.total),
+        "used_bytes": int(usage.used),
+        "available_bytes": int(usage.free),
+        "available_percent": available_percent,
+    }
+
+
+def _format_bytes(value: Any) -> str:
+    try:
+        size = float(value)
+    except (TypeError, ValueError):
+        return "-"
+    units = ("B", "KiB", "MiB", "GiB", "TiB")
+    for unit in units[:-1]:
+        if abs(size) < 1024:
+            return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
+        size /= 1024
+    return f"{size:.1f} {units[-1]}"
+
+
 def _package_version() -> str:
     try:
         return metadata.version("github-agent-bridge")
@@ -314,6 +361,42 @@ def monitor(
     thresholds = thresholds or MonitorThresholds()
     metrics = inspect_db(db)
     alerts: list[str] = []
+    try:
+        database_filesystem = _database_filesystem_usage(db)
+    except OSError as exc:
+        metrics["database_filesystem_error"] = str(exc)
+        _add_alert(
+            metrics,
+            alerts,
+            ALERT_DATABASE_FILESYSTEM_UNAVAILABLE,
+            f"database filesystem capacity check failed: {exc}",
+        )
+    else:
+        metrics["database_filesystem"] = database_filesystem
+        below_bytes = (
+            thresholds.database_free_warn_bytes > 0
+            and database_filesystem["available_bytes"] < thresholds.database_free_warn_bytes
+        )
+        below_percent = (
+            thresholds.database_free_warn_percent > 0
+            and database_filesystem["available_percent"] < thresholds.database_free_warn_percent
+        )
+        if below_bytes or below_percent:
+            _add_alert(
+                metrics,
+                alerts,
+                ALERT_DATABASE_FILESYSTEM_LOW,
+                "database filesystem low space: "
+                f"{_format_bytes(database_filesystem['available_bytes'])} available "
+                f"({database_filesystem['available_percent']:.1f}%) at {database_filesystem['path']}; "
+                f"requires at least {_format_bytes(thresholds.database_free_warn_bytes)} "
+                f"and {thresholds.database_free_warn_percent:.1f}%",
+                context={
+                    **database_filesystem,
+                    "warn_bytes": thresholds.database_free_warn_bytes,
+                    "warn_percent": thresholds.database_free_warn_percent,
+                },
+            )
     package_version = _package_version()
     release_repo = os.getenv("GITHUB_AGENT_BRIDGE_RELEASE_REPO", "").strip()
     metrics["package_version"] = package_version

src/github_agent_bridge/monitor_alert.py

@@ -12,7 +12,12 @@
 from dataclasses import dataclass
 from pathlib import Path
 
-from .monitor import ALERT_RUNNING_NO_EXECUTOR_CHILD, ALERT_RUNNING_PROCESS_MISMATCH
+from .monitor import (
+    ALERT_RUNNING_NO_EXECUTOR_CHILD,
+    ALERT_RUNNING_PROCESS_MISMATCH,
+    DEFAULT_DATABASE_FREE_WARN_BYTES,
+    DEFAULT_DATABASE_FREE_WARN_PERCENT,
+)
 from .observability import configure_sentry
 
 
@@ -40,6 +45,8 @@ class AlertConfig:
     kill_stale_children: bool
     terminate_grace_seconds: int
     proc_idle_seconds: int
+    database_free_warn_bytes: int = DEFAULT_DATABASE_FREE_WARN_BYTES
+    database_free_warn_percent: float = DEFAULT_DATABASE_FREE_WARN_PERCENT
     executor_unit: str = "github-agent-bridge.service"
     slo_window: str = "7d"
     slo_min_samples: int = 20
@@ -84,6 +91,12 @@ def from_env(cls, env: dict[str, str] | None = None) -> AlertConfig:
             kill_stale_children=_parse_bool(values.get("GITHUB_AGENT_BRIDGE_KILL_STALE_CHILDREN", "")),
             terminate_grace_seconds=int(values.get("GITHUB_AGENT_BRIDGE_TERMINATE_GRACE_SECONDS", "5")),
             proc_idle_seconds=int(values.get("GITHUB_AGENT_BRIDGE_PROC_IDLE_SECONDS", "240")),
+            database_free_warn_bytes=int(
+                values.get("GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES", str(DEFAULT_DATABASE_FREE_WARN_BYTES))
+            ),
+            database_free_warn_percent=float(
+                values.get("GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_PERCENT", str(DEFAULT_DATABASE_FREE_WARN_PERCENT))
+            ),
             executor_unit=values.get("GITHUB_AGENT_BRIDGE_EXECUTOR_UNIT", "github-agent-bridge.service"),
             slo_window=values.get("GITHUB_AGENT_BRIDGE_SLO_WINDOW", "7d"),
             slo_min_samples=int(values.get("GITHUB_AGENT_BRIDGE_SLO_MIN_SAMPLES", "20")),
@@ -154,6 +167,10 @@ def run_monitor(config: AlertConfig) -> subprocess.CompletedProcess[str]:
         str(config.work_running_warn_seconds),
         "--progress-warn-seconds",
         str(config.progress_warn_seconds),
+        "--database-free-warn-bytes",
+        str(config.database_free_warn_bytes),
+        "--database-free-warn-percent",
+        str(config.database_free_warn_percent),
         "--executor-unit",
         config.executor_unit,
         "--slo-window",

systemd/env.example

@@ -12,6 +12,10 @@ GITHUB_AGENT_BRIDGE_WORK_TIMEOUT=7200
 GITHUB_AGENT_BRIDGE_REVIEW_RUNNING_WARN_SECONDS=600
 GITHUB_AGENT_BRIDGE_WORK_RUNNING_WARN_SECONDS=900
 GITHUB_AGENT_BRIDGE_PROGRESS_WARN_SECONDS=600
+# Alert before SQLite writes fail because the filesystem containing the bridge
+# database is exhausted. Set either threshold to 0 to disable that condition.
+GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES=5368709120
+GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_PERCENT=10
 GITHUB_AGENT_BRIDGE_AUTO_UNLOCK_STALE_SECONDS=900
 # Optional: after a stalled-job alert, sample executor child process activity and
 # terminate stale child process groups before retrying the affected job ids.

tests/test_monitor.py

@@ -1,4 +1,5 @@
 import sqlite3
+from types import SimpleNamespace
 
 import pytest
 
@@ -55,6 +56,85 @@ def test_monitor_ok_on_empty_initialized_db(tmp_path):
     assert "pending=0" in report.text()
 
 
+def test_monitor_alerts_before_database_filesystem_is_exhausted(tmp_path, monkeypatch):
+    db = tmp_path / "bridge.sqlite3"
+    JobQueue(db)
+
+    def disk_usage(path):
+        assert path == db.parent.resolve()
+        return SimpleNamespace(total=1000, used=950, free=50)
+
+    monkeypatch.setattr(monitor_module.shutil, "disk_usage", disk_usage)
+
+    report = monitor(
+        db,
+        thresholds=MonitorThresholds(
+            database_free_warn_bytes=100,
+            database_free_warn_percent=10,
+        ),
+        check_systemd=False,
+    )
+
+    assert report.ok is False
+    assert report.metrics["database_filesystem"]["available_bytes"] == 50
+    assert report.metrics["database_filesystem"]["available_percent"] == 5
+    assert report.metrics["alert_codes"] == ["monitor.database_filesystem_low"]
+    assert report.metrics["alert_details"][0]["context"]["warn_bytes"] == 100
+    assert "database filesystem low space" in report.alerts[0]
+    assert "db_disk_free=50 B/5.0%" in report.text()
+
+
+def test_monitor_database_filesystem_thresholds_can_be_disabled(tmp_path, monkeypatch):
+    db = tmp_path / "bridge.sqlite3"
+    JobQueue(db)
+    monkeypatch.setattr(
+        monitor_module.shutil,
+        "disk_usage",
+        lambda path: SimpleNamespace(total=1000, used=999, free=1),
+    )
+
+    report = monitor(
+        db,
+        thresholds=MonitorThresholds(
+            database_free_warn_bytes=0,
+            database_free_warn_percent=0,
+        ),
+        check_systemd=False,
+    )
+
+    assert report.ok is True
+    assert report.metrics["database_filesystem"]["available_bytes"] == 1
+
+
+def test_monitor_fails_closed_when_database_filesystem_cannot_be_measured(tmp_path, monkeypatch):
+    db = tmp_path / "bridge.sqlite3"
+    JobQueue(db)
+
+    def fail_disk_usage(path):
+        raise OSError("statvfs failed")
+
+    monkeypatch.setattr(monitor_module.shutil, "disk_usage", fail_disk_usage)
+
+    report = monitor(db, check_systemd=False)
+
+    assert report.ok is False
+    assert report.metrics["database_filesystem_error"] == "statvfs failed"
+    assert report.metrics["alert_codes"] == ["monitor.database_filesystem_unavailable"]
+
+
+@pytest.mark.parametrize(
+    ("kwargs", "message"),
+    [
+        ({"database_free_warn_bytes": -1}, "database_free_warn_bytes must be non-negative"),
+        ({"database_free_warn_percent": -1}, "database_free_warn_percent must be between 0 and 100"),
+        ({"database_free_warn_percent": 101}, "database_free_warn_percent must be between 0 and 100"),
+    ],
+)
+def test_monitor_rejects_invalid_database_filesystem_thresholds(kwargs, message):
+    with pytest.raises(ValueError, match=message):
+        MonitorThresholds(**kwargs)
+
+
 def test_monitor_alerts_when_github_release_is_newer(tmp_path, monkeypatch):
     db = tmp_path / "bridge.sqlite3"
     JobQueue(db)

tests/test_monitor_alert.py

@@ -141,6 +141,26 @@ def test_alert_config_passes_slo_thresholds_from_environment(tmp_path, monkeypat
     assert args[args.index("--slo-retry-rate-percent") + 1] == "12.5"
 
 
+def test_alert_config_passes_database_filesystem_thresholds_from_environment(tmp_path, monkeypatch):
+    config = monitor_alert.AlertConfig.from_env({
+        "GITHUB_AGENT_BRIDGE_STATE_DIR": str(tmp_path),
+        "GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES": "1073741824",
+        "GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_PERCENT": "7.5",
+    })
+    calls = []
+    monkeypatch.setattr(
+        monitor_alert,
+        "_run",
+        lambda args, check=False: calls.append(args) or type("Proc", (), {"stdout": "", "returncode": 0})(),
+    )
+
+    monitor_alert.run_monitor(config)
+
+    args = calls[0]
+    assert args[args.index("--database-free-warn-bytes") + 1] == "1073741824"
+    assert args[args.index("--database-free-warn-percent") + 1] == "7.5"
+
+
 def test_load_state_accepts_legacy_shell_format(tmp_path):
     state_file = tmp_path / "monitor-alert.state"
     state_file.write_text("LAST_HASH='abc'\nLAST_TS=123\n", encoding="utf-8")