From 5f180354373ee7992732e4492040fae3b6701059 Mon Sep 17 00:00:00 2001 From: GISCE Bot Date: Wed, 11 Nov 2026 17:56:33 +0100 Subject: [PATCH 1/2] fix: alert before sqlite filesystem exhaustion --- docs/operations.md | 10 +++ src/github_agent_bridge/cli.py | 12 +++- src/github_agent_bridge/monitor.py | 83 ++++++++++++++++++++++++ src/github_agent_bridge/monitor_alert.py | 19 +++++- systemd/env.example | 4 ++ tests/test_monitor.py | 79 ++++++++++++++++++++++ tests/test_monitor_alert.py | 20 ++++++ 7 files changed, 225 insertions(+), 2 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 84e3b74..28a1173 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -172,8 +172,18 @@ It checks: - blocked jobs; - pending jobs older than 300 seconds; - running jobs older than review/work thresholds; +- free capacity on the filesystem containing the SQLite database (by default, + alert below either 5 GiB or 10% available); - configured SLO thresholds after a minimum sample count and sustained breach. +The disk thresholds can be tuned in the private systemd environment. Set an +individual threshold to `0` to disable that condition: + +```text +GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES=5368709120 +GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_PERCENT=10 +``` + By default the monitor command also writes bounded observability rows to the bridge database: diff --git a/src/github_agent_bridge/cli.py b/src/github_agent_bridge/cli.py index 71a1999..ff33500 100644 --- a/src/github_agent_bridge/cli.py +++ b/src/github_agent_bridge/cli.py @@ -26,7 +26,13 @@ from .dispatch import FEEDBACK_LEARNING_RULES, GitHubClient, OpenClawDispatcher, RunMode, prompt_rule from .executor import ExecutorConfig, ExecutorPool from .models import Notification, utc_now -from .monitor import MonitorThresholds, monitor, report_json +from .monitor import ( + DEFAULT_DATABASE_FREE_WARN_BYTES, + DEFAULT_DATABASE_FREE_WARN_PERCENT, + MonitorThresholds, + monitor, + report_json, +) from .mcp import authenticate_token, create_token, list_tokens, revoke_token, serve_stdio from .observability import DEFAULT_PROCESS_SAMPLE_RETENTION_SECONDS, configure_sentry from .parser import decode_header_value, extract_body_text, is_github_notification_message, parse_auth_results @@ -396,6 +402,8 @@ def cmd_monitor(args: argparse.Namespace) -> int: review_running_warn_seconds=args.review_running_warn_seconds, work_running_warn_seconds=args.work_running_warn_seconds, progress_warn_seconds=args.progress_warn_seconds, + database_free_warn_bytes=args.database_free_warn_bytes, + database_free_warn_percent=args.database_free_warn_percent, slo_window=args.slo_window, slo_min_samples=args.slo_min_samples, slo_breach_observations=args.slo_breach_observations, @@ -664,6 +672,8 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("--review-running-warn-seconds", type=int, default=1200) s.add_argument("--work-running-warn-seconds", type=int, default=4200) s.add_argument("--progress-warn-seconds", type=int, default=600) + s.add_argument("--database-free-warn-bytes", type=int, default=DEFAULT_DATABASE_FREE_WARN_BYTES) + s.add_argument("--database-free-warn-percent", type=float, default=DEFAULT_DATABASE_FREE_WARN_PERCENT) s.add_argument("--slo-window", choices=("24h", "7d", "30d"), default="7d") s.add_argument("--slo-min-samples", type=int, default=20) s.add_argument("--slo-breach-observations", type=int, default=3) diff --git a/src/github_agent_bridge/monitor.py b/src/github_agent_bridge/monitor.py index c6ab3e2..c445e80 100644 --- a/src/github_agent_bridge/monitor.py +++ b/src/github_agent_bridge/monitor.py @@ -3,6 +3,7 @@ import hashlib import json import os +import shutil import subprocess import time import urllib.error @@ -25,6 +26,8 @@ ALERT_RELEASE_AVAILABLE = "monitor.release_available" ALERT_DATABASE_MISSING = "monitor.database_missing" ALERT_SCHEMA_MISSING = "monitor.schema_missing" +ALERT_DATABASE_FILESYSTEM_LOW = "monitor.database_filesystem_low" +ALERT_DATABASE_FILESYSTEM_UNAVAILABLE = "monitor.database_filesystem_unavailable" ALERT_BLOCKED_JOBS = "monitor.blocked_jobs" ALERT_PENDING_QUEUE_OLD = "monitor.pending_queue_old" ALERT_EXECUTOR_SERVICE = "monitor.executor_service" @@ -45,6 +48,8 @@ SLO_WINDOWS = {"24h", "7d", "30d"} +DEFAULT_DATABASE_FREE_WARN_BYTES = 5 * 1024 * 1024 * 1024 +DEFAULT_DATABASE_FREE_WARN_PERCENT = 10.0 SLO_THRESHOLD_FIELDS = ( "slo_queued_to_claimed_p90_seconds", "slo_claimed_to_first_output_p90_seconds", @@ -62,6 +67,8 @@ class MonitorThresholds: progress_warn_seconds: int = 600 reader_recent_seconds: int = 180 worker_heartbeat_warn_seconds: int = 20 + database_free_warn_bytes: int = DEFAULT_DATABASE_FREE_WARN_BYTES + database_free_warn_percent: float = DEFAULT_DATABASE_FREE_WARN_PERCENT slo_window: str = "7d" slo_min_samples: int = 20 slo_breach_observations: int = 3 @@ -74,6 +81,10 @@ class MonitorThresholds: slo_retry_rate_percent: float | None = None def __post_init__(self) -> None: + if self.database_free_warn_bytes < 0: + raise ValueError("database_free_warn_bytes must be non-negative") + if not 0 <= self.database_free_warn_percent <= 100: + raise ValueError("database_free_warn_percent must be between 0 and 100") if self.slo_window not in SLO_WINDOWS: raise ValueError(f"unsupported SLO window {self.slo_window!r}") if self.slo_min_samples < 1: @@ -139,6 +150,13 @@ def text(self) -> str: f"oldest_pending={metrics.get('oldest_pending_age_seconds') if metrics.get('oldest_pending_age_seconds') is not None else '-'}", f"last_uid={metrics.get('last_uid', '-')}", ] + database_filesystem = metrics.get("database_filesystem") or {} + if database_filesystem: + compact.append( + "db_disk_free=" + f"{_format_bytes(database_filesystem.get('available_bytes'))}/" + f"{database_filesystem.get('available_percent', 0):.1f}%" + ) parts.append(" ".join(compact)) if self.alerts: alert_codes = metrics.get("alert_codes") or [] @@ -229,6 +247,35 @@ def inspect_db(path: str | Path) -> dict[str, Any]: return DashboardQueries(path).status() +def _database_filesystem_usage(path: str | Path) -> dict[str, Any]: + database_path = Path(path).expanduser().resolve(strict=False) + probe_path = database_path.parent + while not probe_path.exists() and probe_path != probe_path.parent: + probe_path = probe_path.parent + usage = shutil.disk_usage(probe_path) + available_percent = (usage.free / usage.total * 100.0) if usage.total else 0.0 + return { + "path": str(probe_path.resolve()), + "total_bytes": int(usage.total), + "used_bytes": int(usage.used), + "available_bytes": int(usage.free), + "available_percent": available_percent, + } + + +def _format_bytes(value: Any) -> str: + try: + size = float(value) + except (TypeError, ValueError): + return "-" + units = ("B", "KiB", "MiB", "GiB", "TiB") + for unit in units[:-1]: + if abs(size) < 1024: + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} {units[-1]}" + + def _package_version() -> str: try: return metadata.version("github-agent-bridge") @@ -314,6 +361,42 @@ def monitor( thresholds = thresholds or MonitorThresholds() metrics = inspect_db(db) alerts: list[str] = [] + try: + database_filesystem = _database_filesystem_usage(db) + except OSError as exc: + metrics["database_filesystem_error"] = str(exc) + _add_alert( + metrics, + alerts, + ALERT_DATABASE_FILESYSTEM_UNAVAILABLE, + f"database filesystem capacity check failed: {exc}", + ) + else: + metrics["database_filesystem"] = database_filesystem + below_bytes = ( + thresholds.database_free_warn_bytes > 0 + and database_filesystem["available_bytes"] < thresholds.database_free_warn_bytes + ) + below_percent = ( + thresholds.database_free_warn_percent > 0 + and database_filesystem["available_percent"] < thresholds.database_free_warn_percent + ) + if below_bytes or below_percent: + _add_alert( + metrics, + alerts, + ALERT_DATABASE_FILESYSTEM_LOW, + "database filesystem low space: " + f"{_format_bytes(database_filesystem['available_bytes'])} available " + f"({database_filesystem['available_percent']:.1f}%) at {database_filesystem['path']}; " + f"requires at least {_format_bytes(thresholds.database_free_warn_bytes)} " + f"and {thresholds.database_free_warn_percent:.1f}%", + context={ + **database_filesystem, + "warn_bytes": thresholds.database_free_warn_bytes, + "warn_percent": thresholds.database_free_warn_percent, + }, + ) package_version = _package_version() release_repo = os.getenv("GITHUB_AGENT_BRIDGE_RELEASE_REPO", "").strip() metrics["package_version"] = package_version diff --git a/src/github_agent_bridge/monitor_alert.py b/src/github_agent_bridge/monitor_alert.py index 50861d6..61399e3 100644 --- a/src/github_agent_bridge/monitor_alert.py +++ b/src/github_agent_bridge/monitor_alert.py @@ -12,7 +12,12 @@ from dataclasses import dataclass from pathlib import Path -from .monitor import ALERT_RUNNING_NO_EXECUTOR_CHILD, ALERT_RUNNING_PROCESS_MISMATCH +from .monitor import ( + ALERT_RUNNING_NO_EXECUTOR_CHILD, + ALERT_RUNNING_PROCESS_MISMATCH, + DEFAULT_DATABASE_FREE_WARN_BYTES, + DEFAULT_DATABASE_FREE_WARN_PERCENT, +) from .observability import configure_sentry @@ -40,6 +45,8 @@ class AlertConfig: kill_stale_children: bool terminate_grace_seconds: int proc_idle_seconds: int + database_free_warn_bytes: int = DEFAULT_DATABASE_FREE_WARN_BYTES + database_free_warn_percent: float = DEFAULT_DATABASE_FREE_WARN_PERCENT executor_unit: str = "github-agent-bridge.service" slo_window: str = "7d" slo_min_samples: int = 20 @@ -84,6 +91,12 @@ def from_env(cls, env: dict[str, str] | None = None) -> AlertConfig: kill_stale_children=_parse_bool(values.get("GITHUB_AGENT_BRIDGE_KILL_STALE_CHILDREN", "")), terminate_grace_seconds=int(values.get("GITHUB_AGENT_BRIDGE_TERMINATE_GRACE_SECONDS", "5")), proc_idle_seconds=int(values.get("GITHUB_AGENT_BRIDGE_PROC_IDLE_SECONDS", "240")), + database_free_warn_bytes=int( + values.get("GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES", str(DEFAULT_DATABASE_FREE_WARN_BYTES)) + ), + database_free_warn_percent=float( + values.get("GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_PERCENT", str(DEFAULT_DATABASE_FREE_WARN_PERCENT)) + ), executor_unit=values.get("GITHUB_AGENT_BRIDGE_EXECUTOR_UNIT", "github-agent-bridge.service"), slo_window=values.get("GITHUB_AGENT_BRIDGE_SLO_WINDOW", "7d"), slo_min_samples=int(values.get("GITHUB_AGENT_BRIDGE_SLO_MIN_SAMPLES", "20")), @@ -154,6 +167,10 @@ def run_monitor(config: AlertConfig) -> subprocess.CompletedProcess[str]: str(config.work_running_warn_seconds), "--progress-warn-seconds", str(config.progress_warn_seconds), + "--database-free-warn-bytes", + str(config.database_free_warn_bytes), + "--database-free-warn-percent", + str(config.database_free_warn_percent), "--executor-unit", config.executor_unit, "--slo-window", diff --git a/systemd/env.example b/systemd/env.example index 7f9c822..58d55b8 100644 --- a/systemd/env.example +++ b/systemd/env.example @@ -12,6 +12,10 @@ GITHUB_AGENT_BRIDGE_WORK_TIMEOUT=7200 GITHUB_AGENT_BRIDGE_REVIEW_RUNNING_WARN_SECONDS=600 GITHUB_AGENT_BRIDGE_WORK_RUNNING_WARN_SECONDS=900 GITHUB_AGENT_BRIDGE_PROGRESS_WARN_SECONDS=600 +# Alert before SQLite writes fail because the filesystem containing the bridge +# database is exhausted. Set either threshold to 0 to disable that condition. +GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES=5368709120 +GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_PERCENT=10 GITHUB_AGENT_BRIDGE_AUTO_UNLOCK_STALE_SECONDS=900 # Optional: after a stalled-job alert, sample executor child process activity and # terminate stale child process groups before retrying the affected job ids. diff --git a/tests/test_monitor.py b/tests/test_monitor.py index ea22469..9737177 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -1,4 +1,5 @@ import sqlite3 +from types import SimpleNamespace import pytest @@ -55,6 +56,84 @@ def test_monitor_ok_on_empty_initialized_db(tmp_path): assert "pending=0" in report.text() +def test_monitor_alerts_before_database_filesystem_is_exhausted(tmp_path, monkeypatch): + db = tmp_path / "bridge.sqlite3" + JobQueue(db) + monkeypatch.setattr( + monitor_module.shutil, + "disk_usage", + lambda path: SimpleNamespace(total=1000, used=950, free=50), + ) + + report = monitor( + db, + thresholds=MonitorThresholds( + database_free_warn_bytes=100, + database_free_warn_percent=10, + ), + check_systemd=False, + ) + + assert report.ok is False + assert report.metrics["database_filesystem"]["available_bytes"] == 50 + assert report.metrics["database_filesystem"]["available_percent"] == 5 + assert report.metrics["alert_codes"] == ["monitor.database_filesystem_low"] + assert report.metrics["alert_details"][0]["context"]["warn_bytes"] == 100 + assert "database filesystem low space" in report.alerts[0] + assert "db_disk_free=50 B/5.0%" in report.text() + + +def test_monitor_database_filesystem_thresholds_can_be_disabled(tmp_path, monkeypatch): + db = tmp_path / "bridge.sqlite3" + JobQueue(db) + monkeypatch.setattr( + monitor_module.shutil, + "disk_usage", + lambda path: SimpleNamespace(total=1000, used=999, free=1), + ) + + report = monitor( + db, + thresholds=MonitorThresholds( + database_free_warn_bytes=0, + database_free_warn_percent=0, + ), + check_systemd=False, + ) + + assert report.ok is True + assert report.metrics["database_filesystem"]["available_bytes"] == 1 + + +def test_monitor_fails_closed_when_database_filesystem_cannot_be_measured(tmp_path, monkeypatch): + db = tmp_path / "bridge.sqlite3" + JobQueue(db) + + def fail_disk_usage(path): + raise OSError("statvfs failed") + + monkeypatch.setattr(monitor_module.shutil, "disk_usage", fail_disk_usage) + + report = monitor(db, check_systemd=False) + + assert report.ok is False + assert report.metrics["database_filesystem_error"] == "statvfs failed" + assert report.metrics["alert_codes"] == ["monitor.database_filesystem_unavailable"] + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"database_free_warn_bytes": -1}, "database_free_warn_bytes must be non-negative"), + ({"database_free_warn_percent": -1}, "database_free_warn_percent must be between 0 and 100"), + ({"database_free_warn_percent": 101}, "database_free_warn_percent must be between 0 and 100"), + ], +) +def test_monitor_rejects_invalid_database_filesystem_thresholds(kwargs, message): + with pytest.raises(ValueError, match=message): + MonitorThresholds(**kwargs) + + def test_monitor_alerts_when_github_release_is_newer(tmp_path, monkeypatch): db = tmp_path / "bridge.sqlite3" JobQueue(db) diff --git a/tests/test_monitor_alert.py b/tests/test_monitor_alert.py index 1c3befc..b345e54 100644 --- a/tests/test_monitor_alert.py +++ b/tests/test_monitor_alert.py @@ -141,6 +141,26 @@ def test_alert_config_passes_slo_thresholds_from_environment(tmp_path, monkeypat assert args[args.index("--slo-retry-rate-percent") + 1] == "12.5" +def test_alert_config_passes_database_filesystem_thresholds_from_environment(tmp_path, monkeypatch): + config = monitor_alert.AlertConfig.from_env({ + "GITHUB_AGENT_BRIDGE_STATE_DIR": str(tmp_path), + "GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES": "1073741824", + "GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_PERCENT": "7.5", + }) + calls = [] + monkeypatch.setattr( + monitor_alert, + "_run", + lambda args, check=False: calls.append(args) or type("Proc", (), {"stdout": "", "returncode": 0})(), + ) + + monitor_alert.run_monitor(config) + + args = calls[0] + assert args[args.index("--database-free-warn-bytes") + 1] == "1073741824" + assert args[args.index("--database-free-warn-percent") + 1] == "7.5" + + def test_load_state_accepts_legacy_shell_format(tmp_path): state_file = tmp_path / "monitor-alert.state" state_file.write_text("LAST_HASH='abc'\nLAST_TS=123\n", encoding="utf-8") From 88a3ab5120fd017399b64f81e66d35150657b2f8 Mon Sep 17 00:00:00 2001 From: GISCE Bot Date: Wed, 11 Nov 2026 21:33:24 +0100 Subject: [PATCH 2/2] test: verify sqlite filesystem probe path --- tests/test_monitor.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 9737177..c5f3ee6 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -59,11 +59,12 @@ def test_monitor_ok_on_empty_initialized_db(tmp_path): def test_monitor_alerts_before_database_filesystem_is_exhausted(tmp_path, monkeypatch): db = tmp_path / "bridge.sqlite3" JobQueue(db) - monkeypatch.setattr( - monitor_module.shutil, - "disk_usage", - lambda path: SimpleNamespace(total=1000, used=950, free=50), - ) + + def disk_usage(path): + assert path == db.parent.resolve() + return SimpleNamespace(total=1000, used=950, free=50) + + monkeypatch.setattr(monitor_module.shutil, "disk_usage", disk_usage) report = monitor( db,