fix: alert before SQLite filesystem exhaustion

docs/operations.md

@@ -166,6 +166,7 @@ The monitor exits:
 
 It checks:
 
+- database filesystem free space (1 GiB warning threshold by default);
 - executor service active;
 - reader timer active;
 - last reader service result;
@@ -174,6 +175,12 @@ It checks:
 - running jobs older than review/work thresholds;
 - configured SLO thresholds after a minimum sample count and sustained breach.
 
+Override the storage threshold with
+`GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES` or
+`gab monitor --database-free-warn-bytes`; set it to `0` to disable the alert.
+When the threshold is breached, the monitor skips its own observability write so
+it does not consume the remaining SQLite filesystem capacity.
+
 By default the monitor command also writes bounded observability rows to the
 bridge database:
 

src/github_agent_bridge/cli.py

@@ -392,6 +392,7 @@ def cmd_backfill_trigger_actors(args: argparse.Namespace) -> int:
 
 def cmd_monitor(args: argparse.Namespace) -> int:
     thresholds = MonitorThresholds(
+        database_free_warn_bytes=args.database_free_warn_bytes,
         pending_warn_seconds=args.pending_warn_seconds,
         review_running_warn_seconds=args.review_running_warn_seconds,
         work_running_warn_seconds=args.work_running_warn_seconds,
@@ -660,6 +661,11 @@ def build_parser() -> argparse.ArgumentParser:
     s.add_argument("--executor-unit", default="github-agent-bridge.service")
     s.add_argument("--reader-timer-unit", default="github-agent-bridge-reader.timer")
     s.add_argument("--reader-service-unit", default="github-agent-bridge-reader.service")
+    s.add_argument(
+        "--database-free-warn-bytes",
+        type=int,
+        default=int(os.getenv("GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES", str(1024**3))),
+    )
     s.add_argument("--pending-warn-seconds", type=int, default=300)
     s.add_argument("--review-running-warn-seconds", type=int, default=1200)
     s.add_argument("--work-running-warn-seconds", type=int, default=4200)

src/github_agent_bridge/monitor.py

@@ -3,6 +3,7 @@
 import hashlib
 import json
 import os
+import shutil
 import subprocess
 import time
 import urllib.error
@@ -25,6 +26,7 @@
 ALERT_RELEASE_AVAILABLE = "monitor.release_available"
 ALERT_DATABASE_MISSING = "monitor.database_missing"
 ALERT_SCHEMA_MISSING = "monitor.schema_missing"
+ALERT_DATABASE_DISK_SPACE_LOW = "monitor.database_disk_space_low"
 ALERT_BLOCKED_JOBS = "monitor.blocked_jobs"
 ALERT_PENDING_QUEUE_OLD = "monitor.pending_queue_old"
 ALERT_EXECUTOR_SERVICE = "monitor.executor_service"
@@ -56,6 +58,7 @@
 
 @dataclass(frozen=True)
 class MonitorThresholds:
+    database_free_warn_bytes: int = 1024**3
     pending_warn_seconds: int = 300
     review_running_warn_seconds: int = 1200
     work_running_warn_seconds: int = 4200
@@ -74,6 +77,8 @@ class MonitorThresholds:
     slo_retry_rate_percent: float | None = None
 
     def __post_init__(self) -> None:
+        if self.database_free_warn_bytes < 0:
+            raise ValueError("database_free_warn_bytes must be non-negative")
         if self.slo_window not in SLO_WINDOWS:
             raise ValueError(f"unsupported SLO window {self.slo_window!r}")
         if self.slo_min_samples < 1:
@@ -229,6 +234,13 @@ def inspect_db(path: str | Path) -> dict[str, Any]:
     return DashboardQueries(path).status()
 
 
+def _database_filesystem_path(path: str | Path) -> Path:
+    candidate = Path(path).expanduser().parent
+    while not candidate.exists() and candidate != candidate.parent:
+        candidate = candidate.parent
+    return candidate
+
+
 def _package_version() -> str:
     try:
         return metadata.version("github-agent-bridge")
@@ -312,8 +324,27 @@ def monitor(
     process_sample_retention_seconds: int = DEFAULT_PROCESS_SAMPLE_RETENTION_SECONDS,
 ) -> MonitorReport:
     thresholds = thresholds or MonitorThresholds()
-    metrics = inspect_db(db)
+    disk_usage = shutil.disk_usage(_database_filesystem_path(db))
+    metrics = {
+        "database_filesystem_total_bytes": disk_usage.total,
+        "database_filesystem_used_bytes": disk_usage.used,
+        "database_filesystem_free_bytes": disk_usage.free,
+    }
+    metrics.update(inspect_db(db))
     alerts: list[str] = []
+    database_space_low = disk_usage.free < thresholds.database_free_warn_bytes
+    if database_space_low:
+        _add_alert(
+            metrics,
+            alerts,
+            ALERT_DATABASE_DISK_SPACE_LOW,
+            "database filesystem free space "
+            f"{disk_usage.free} bytes < {thresholds.database_free_warn_bytes} bytes",
+            context={
+                "free_bytes": disk_usage.free,
+                "warn_bytes": thresholds.database_free_warn_bytes,
+            },
+        )
     package_version = _package_version()
     release_repo = os.getenv("GITHUB_AGENT_BRIDGE_RELEASE_REPO", "").strip()
     metrics["package_version"] = package_version
@@ -523,14 +554,21 @@ def monitor(
             "metrics": {},
         }
 
-    if persist_observability and metrics.get("db_exists") and metrics.get("schema_ok", True):
+    if (
+        persist_observability
+        and metrics.get("db_exists")
+        and metrics.get("schema_ok", True)
+        and not database_space_low
+    ):
         record_monitor_observation(
             db,
             metrics,
             alerts,
             process_sample_retention_seconds=process_sample_retention_seconds,
             monitor_slo_state=slo_state_update,
         )
+    elif persist_observability and database_space_low:
+        metrics["observability_persistence_skipped"] = "low_database_disk_space"
 
     return MonitorReport(ok=not alerts, alerts=alerts, metrics=metrics)
 

src/github_agent_bridge/monitor_alert.py

@@ -40,6 +40,7 @@ class AlertConfig:
     kill_stale_children: bool
     terminate_grace_seconds: int
     proc_idle_seconds: int
+    database_free_warn_bytes: int = 1024**3
     executor_unit: str = "github-agent-bridge.service"
     slo_window: str = "7d"
     slo_min_samples: int = 20
@@ -84,6 +85,9 @@ def from_env(cls, env: dict[str, str] | None = None) -> AlertConfig:
             kill_stale_children=_parse_bool(values.get("GITHUB_AGENT_BRIDGE_KILL_STALE_CHILDREN", "")),
             terminate_grace_seconds=int(values.get("GITHUB_AGENT_BRIDGE_TERMINATE_GRACE_SECONDS", "5")),
             proc_idle_seconds=int(values.get("GITHUB_AGENT_BRIDGE_PROC_IDLE_SECONDS", "240")),
+            database_free_warn_bytes=int(
+                values.get("GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES", str(1024**3))
+            ),
             executor_unit=values.get("GITHUB_AGENT_BRIDGE_EXECUTOR_UNIT", "github-agent-bridge.service"),
             slo_window=values.get("GITHUB_AGENT_BRIDGE_SLO_WINDOW", "7d"),
             slo_min_samples=int(values.get("GITHUB_AGENT_BRIDGE_SLO_MIN_SAMPLES", "20")),
@@ -154,6 +158,8 @@ def run_monitor(config: AlertConfig) -> subprocess.CompletedProcess[str]:
         str(config.work_running_warn_seconds),
         "--progress-warn-seconds",
         str(config.progress_warn_seconds),
+        "--database-free-warn-bytes",
+        str(config.database_free_warn_bytes),
         "--executor-unit",
         config.executor_unit,
         "--slo-window",

systemd/env.example

@@ -12,6 +12,8 @@ GITHUB_AGENT_BRIDGE_WORK_TIMEOUT=7200
 GITHUB_AGENT_BRIDGE_REVIEW_RUNNING_WARN_SECONDS=600
 GITHUB_AGENT_BRIDGE_WORK_RUNNING_WARN_SECONDS=900
 GITHUB_AGENT_BRIDGE_PROGRESS_WARN_SECONDS=600
+# Alert before SQLite can exhaust the filesystem. Set 0 to disable.
+GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES=1073741824
 GITHUB_AGENT_BRIDGE_AUTO_UNLOCK_STALE_SECONDS=900
 # Optional: after a stalled-job alert, sample executor child process activity and
 # terminate stale child process groups before retrying the affected job ids.

tests/test_monitor.py

@@ -10,6 +10,19 @@
 from github_agent_bridge.queue import JobQueue
 
 
+@pytest.fixture(autouse=True)
+def healthy_database_filesystem(monkeypatch):
+    monkeypatch.setattr(
+        monitor_module.shutil,
+        "disk_usage",
+        lambda path: type(
+            "DiskUsage",
+            (),
+            {"total": 20 * 1024**3, "used": 10 * 1024**3, "free": 10 * 1024**3},
+        )(),
+    )
+
+
 def notif(uid=1, mid="<1@github.com>", body="@pilipilisbot https://github.com/gisce/erp/pull/1#issuecomment-10"):
     return Notification(uid=uid, message_id=mid, subject="Re: [gisce/erp] PR", from_addr="ecarreras <notifications@github.com>", body=body, auth={"spf": True, "dkim": True, "dmarc": True})
 
@@ -55,6 +68,38 @@ def test_monitor_ok_on_empty_initialized_db(tmp_path):
     assert "pending=0" in report.text()
 
 
+def test_monitor_alerts_before_database_filesystem_is_exhausted(tmp_path, monkeypatch):
+    db = tmp_path / "bridge.sqlite3"
+    JobQueue(db)
+    monkeypatch.setattr(
+        monitor_module.shutil,
+        "disk_usage",
+        lambda path: type(
+            "DiskUsage",
+            (),
+            {"total": 10 * 1024**3, "used": 10 * 1024**3 - 512, "free": 512},
+        )(),
+    )
+
+    report = monitor(
+        db,
+        thresholds=MonitorThresholds(database_free_warn_bytes=1024),
+        check_systemd=False,
+        persist_observability=True,
+    )
+
+    assert report.ok is False
+    assert report.metrics["database_filesystem_free_bytes"] == 512
+    assert report.metrics["observability_persistence_skipped"] == "low_database_disk_space"
+    assert report.metrics["alert_codes"] == ["monitor.database_disk_space_low"]
+    assert "database filesystem free space 512 bytes < 1024 bytes" in report.alerts
+
+
+def test_monitor_thresholds_reject_negative_database_free_space():
+    with pytest.raises(ValueError, match="database_free_warn_bytes must be non-negative"):
+        MonitorThresholds(database_free_warn_bytes=-1)
+
+
 def test_monitor_alerts_when_github_release_is_newer(tmp_path, monkeypatch):
     db = tmp_path / "bridge.sqlite3"
     JobQueue(db)

tests/test_monitor_alert.py

@@ -141,6 +141,25 @@ def test_alert_config_passes_slo_thresholds_from_environment(tmp_path, monkeypat
     assert args[args.index("--slo-retry-rate-percent") + 1] == "12.5"
 
 
+def test_alert_config_passes_database_free_space_threshold(tmp_path, monkeypatch):
+    config = monitor_alert.AlertConfig.from_env({
+        "GITHUB_AGENT_BRIDGE_STATE_DIR": str(tmp_path),
+        "GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES": "2147483648",
+    })
+    calls = []
+    monkeypatch.setattr(
+        monitor_alert,
+        "_run",
+        lambda args, check=False: calls.append(args) or type("Proc", (), {"stdout": "", "returncode": 0})(),
+    )
+
+    monitor_alert.run_monitor(config)
+
+    args = calls[0]
+    assert config.database_free_warn_bytes == 2147483648
+    assert args[args.index("--database-free-warn-bytes") + 1] == "2147483648"
+
+
 def test_load_state_accepts_legacy_shell_format(tmp_path):
     state_file = tmp_path / "monitor-alert.state"
     state_file.write_text("LAST_HASH='abc'\nLAST_TS=123\n", encoding="utf-8")