From 4c3481b4ac6ec202c8d5e37d4599d3390577a93c Mon Sep 17 00:00:00 2001 From: GISCE Bot Date: Wed, 11 Nov 2026 17:48:31 +0100 Subject: [PATCH] fix: alert before SQLite filesystem exhaustion --- docs/operations.md | 7 ++++ src/github_agent_bridge/cli.py | 6 ++++ src/github_agent_bridge/monitor.py | 42 ++++++++++++++++++++-- src/github_agent_bridge/monitor_alert.py | 6 ++++ systemd/env.example | 2 ++ tests/test_monitor.py | 45 ++++++++++++++++++++++++ tests/test_monitor_alert.py | 19 ++++++++++ 7 files changed, 125 insertions(+), 2 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 84e3b74..31264cb 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -166,6 +166,7 @@ The monitor exits: It checks: +- database filesystem free space (1 GiB warning threshold by default); - executor service active; - reader timer active; - last reader service result; @@ -174,6 +175,12 @@ It checks: - running jobs older than review/work thresholds; - configured SLO thresholds after a minimum sample count and sustained breach. +Override the storage threshold with +`GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES` or +`gab monitor --database-free-warn-bytes`; set it to `0` to disable the alert. +When the threshold is breached, the monitor skips its own observability write so +it does not consume the remaining SQLite filesystem capacity. + By default the monitor command also writes bounded observability rows to the bridge database: diff --git a/src/github_agent_bridge/cli.py b/src/github_agent_bridge/cli.py index 71a1999..d1f81de 100644 --- a/src/github_agent_bridge/cli.py +++ b/src/github_agent_bridge/cli.py @@ -392,6 +392,7 @@ def cmd_backfill_trigger_actors(args: argparse.Namespace) -> int: def cmd_monitor(args: argparse.Namespace) -> int: thresholds = MonitorThresholds( + database_free_warn_bytes=args.database_free_warn_bytes, pending_warn_seconds=args.pending_warn_seconds, review_running_warn_seconds=args.review_running_warn_seconds, work_running_warn_seconds=args.work_running_warn_seconds, @@ -660,6 +661,11 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("--executor-unit", default="github-agent-bridge.service") s.add_argument("--reader-timer-unit", default="github-agent-bridge-reader.timer") s.add_argument("--reader-service-unit", default="github-agent-bridge-reader.service") + s.add_argument( + "--database-free-warn-bytes", + type=int, + default=int(os.getenv("GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES", str(1024**3))), + ) s.add_argument("--pending-warn-seconds", type=int, default=300) s.add_argument("--review-running-warn-seconds", type=int, default=1200) s.add_argument("--work-running-warn-seconds", type=int, default=4200) diff --git a/src/github_agent_bridge/monitor.py b/src/github_agent_bridge/monitor.py index c6ab3e2..d627efb 100644 --- a/src/github_agent_bridge/monitor.py +++ b/src/github_agent_bridge/monitor.py @@ -3,6 +3,7 @@ import hashlib import json import os +import shutil import subprocess import time import urllib.error @@ -25,6 +26,7 @@ ALERT_RELEASE_AVAILABLE = "monitor.release_available" ALERT_DATABASE_MISSING = "monitor.database_missing" ALERT_SCHEMA_MISSING = "monitor.schema_missing" +ALERT_DATABASE_DISK_SPACE_LOW = "monitor.database_disk_space_low" ALERT_BLOCKED_JOBS = "monitor.blocked_jobs" ALERT_PENDING_QUEUE_OLD = "monitor.pending_queue_old" ALERT_EXECUTOR_SERVICE = "monitor.executor_service" @@ -56,6 +58,7 @@ @dataclass(frozen=True) class MonitorThresholds: + database_free_warn_bytes: int = 1024**3 pending_warn_seconds: int = 300 review_running_warn_seconds: int = 1200 work_running_warn_seconds: int = 4200 @@ -74,6 +77,8 @@ class MonitorThresholds: slo_retry_rate_percent: float | None = None def __post_init__(self) -> None: + if self.database_free_warn_bytes < 0: + raise ValueError("database_free_warn_bytes must be non-negative") if self.slo_window not in SLO_WINDOWS: raise ValueError(f"unsupported SLO window {self.slo_window!r}") if self.slo_min_samples < 1: @@ -229,6 +234,13 @@ def inspect_db(path: str | Path) -> dict[str, Any]: return DashboardQueries(path).status() +def _database_filesystem_path(path: str | Path) -> Path: + candidate = Path(path).expanduser().parent + while not candidate.exists() and candidate != candidate.parent: + candidate = candidate.parent + return candidate + + def _package_version() -> str: try: return metadata.version("github-agent-bridge") @@ -312,8 +324,27 @@ def monitor( process_sample_retention_seconds: int = DEFAULT_PROCESS_SAMPLE_RETENTION_SECONDS, ) -> MonitorReport: thresholds = thresholds or MonitorThresholds() - metrics = inspect_db(db) + disk_usage = shutil.disk_usage(_database_filesystem_path(db)) + metrics = { + "database_filesystem_total_bytes": disk_usage.total, + "database_filesystem_used_bytes": disk_usage.used, + "database_filesystem_free_bytes": disk_usage.free, + } + metrics.update(inspect_db(db)) alerts: list[str] = [] + database_space_low = disk_usage.free < thresholds.database_free_warn_bytes + if database_space_low: + _add_alert( + metrics, + alerts, + ALERT_DATABASE_DISK_SPACE_LOW, + "database filesystem free space " + f"{disk_usage.free} bytes < {thresholds.database_free_warn_bytes} bytes", + context={ + "free_bytes": disk_usage.free, + "warn_bytes": thresholds.database_free_warn_bytes, + }, + ) package_version = _package_version() release_repo = os.getenv("GITHUB_AGENT_BRIDGE_RELEASE_REPO", "").strip() metrics["package_version"] = package_version @@ -523,7 +554,12 @@ def monitor( "metrics": {}, } - if persist_observability and metrics.get("db_exists") and metrics.get("schema_ok", True): + if ( + persist_observability + and metrics.get("db_exists") + and metrics.get("schema_ok", True) + and not database_space_low + ): record_monitor_observation( db, metrics, @@ -531,6 +567,8 @@ def monitor( process_sample_retention_seconds=process_sample_retention_seconds, monitor_slo_state=slo_state_update, ) + elif persist_observability and database_space_low: + metrics["observability_persistence_skipped"] = "low_database_disk_space" return MonitorReport(ok=not alerts, alerts=alerts, metrics=metrics) diff --git a/src/github_agent_bridge/monitor_alert.py b/src/github_agent_bridge/monitor_alert.py index 50861d6..d9d42bb 100644 --- a/src/github_agent_bridge/monitor_alert.py +++ b/src/github_agent_bridge/monitor_alert.py @@ -40,6 +40,7 @@ class AlertConfig: kill_stale_children: bool terminate_grace_seconds: int proc_idle_seconds: int + database_free_warn_bytes: int = 1024**3 executor_unit: str = "github-agent-bridge.service" slo_window: str = "7d" slo_min_samples: int = 20 @@ -84,6 +85,9 @@ def from_env(cls, env: dict[str, str] | None = None) -> AlertConfig: kill_stale_children=_parse_bool(values.get("GITHUB_AGENT_BRIDGE_KILL_STALE_CHILDREN", "")), terminate_grace_seconds=int(values.get("GITHUB_AGENT_BRIDGE_TERMINATE_GRACE_SECONDS", "5")), proc_idle_seconds=int(values.get("GITHUB_AGENT_BRIDGE_PROC_IDLE_SECONDS", "240")), + database_free_warn_bytes=int( + values.get("GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES", str(1024**3)) + ), executor_unit=values.get("GITHUB_AGENT_BRIDGE_EXECUTOR_UNIT", "github-agent-bridge.service"), slo_window=values.get("GITHUB_AGENT_BRIDGE_SLO_WINDOW", "7d"), slo_min_samples=int(values.get("GITHUB_AGENT_BRIDGE_SLO_MIN_SAMPLES", "20")), @@ -154,6 +158,8 @@ def run_monitor(config: AlertConfig) -> subprocess.CompletedProcess[str]: str(config.work_running_warn_seconds), "--progress-warn-seconds", str(config.progress_warn_seconds), + "--database-free-warn-bytes", + str(config.database_free_warn_bytes), "--executor-unit", config.executor_unit, "--slo-window", diff --git a/systemd/env.example b/systemd/env.example index 7f9c822..8b21d79 100644 --- a/systemd/env.example +++ b/systemd/env.example @@ -12,6 +12,8 @@ GITHUB_AGENT_BRIDGE_WORK_TIMEOUT=7200 GITHUB_AGENT_BRIDGE_REVIEW_RUNNING_WARN_SECONDS=600 GITHUB_AGENT_BRIDGE_WORK_RUNNING_WARN_SECONDS=900 GITHUB_AGENT_BRIDGE_PROGRESS_WARN_SECONDS=600 +# Alert before SQLite can exhaust the filesystem. Set 0 to disable. +GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES=1073741824 GITHUB_AGENT_BRIDGE_AUTO_UNLOCK_STALE_SECONDS=900 # Optional: after a stalled-job alert, sample executor child process activity and # terminate stale child process groups before retrying the affected job ids. diff --git a/tests/test_monitor.py b/tests/test_monitor.py index ea22469..e056296 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -10,6 +10,19 @@ from github_agent_bridge.queue import JobQueue +@pytest.fixture(autouse=True) +def healthy_database_filesystem(monkeypatch): + monkeypatch.setattr( + monitor_module.shutil, + "disk_usage", + lambda path: type( + "DiskUsage", + (), + {"total": 20 * 1024**3, "used": 10 * 1024**3, "free": 10 * 1024**3}, + )(), + ) + + def notif(uid=1, mid="<1@github.com>", body="@pilipilisbot https://github.com/gisce/erp/pull/1#issuecomment-10"): return Notification(uid=uid, message_id=mid, subject="Re: [gisce/erp] PR", from_addr="ecarreras ", body=body, auth={"spf": True, "dkim": True, "dmarc": True}) @@ -55,6 +68,38 @@ def test_monitor_ok_on_empty_initialized_db(tmp_path): assert "pending=0" in report.text() +def test_monitor_alerts_before_database_filesystem_is_exhausted(tmp_path, monkeypatch): + db = tmp_path / "bridge.sqlite3" + JobQueue(db) + monkeypatch.setattr( + monitor_module.shutil, + "disk_usage", + lambda path: type( + "DiskUsage", + (), + {"total": 10 * 1024**3, "used": 10 * 1024**3 - 512, "free": 512}, + )(), + ) + + report = monitor( + db, + thresholds=MonitorThresholds(database_free_warn_bytes=1024), + check_systemd=False, + persist_observability=True, + ) + + assert report.ok is False + assert report.metrics["database_filesystem_free_bytes"] == 512 + assert report.metrics["observability_persistence_skipped"] == "low_database_disk_space" + assert report.metrics["alert_codes"] == ["monitor.database_disk_space_low"] + assert "database filesystem free space 512 bytes < 1024 bytes" in report.alerts + + +def test_monitor_thresholds_reject_negative_database_free_space(): + with pytest.raises(ValueError, match="database_free_warn_bytes must be non-negative"): + MonitorThresholds(database_free_warn_bytes=-1) + + def test_monitor_alerts_when_github_release_is_newer(tmp_path, monkeypatch): db = tmp_path / "bridge.sqlite3" JobQueue(db) diff --git a/tests/test_monitor_alert.py b/tests/test_monitor_alert.py index 1c3befc..cbef8f7 100644 --- a/tests/test_monitor_alert.py +++ b/tests/test_monitor_alert.py @@ -141,6 +141,25 @@ def test_alert_config_passes_slo_thresholds_from_environment(tmp_path, monkeypat assert args[args.index("--slo-retry-rate-percent") + 1] == "12.5" +def test_alert_config_passes_database_free_space_threshold(tmp_path, monkeypatch): + config = monitor_alert.AlertConfig.from_env({ + "GITHUB_AGENT_BRIDGE_STATE_DIR": str(tmp_path), + "GITHUB_AGENT_BRIDGE_DATABASE_FREE_WARN_BYTES": "2147483648", + }) + calls = [] + monkeypatch.setattr( + monitor_alert, + "_run", + lambda args, check=False: calls.append(args) or type("Proc", (), {"stdout": "", "returncode": 0})(), + ) + + monitor_alert.run_monitor(config) + + args = calls[0] + assert config.database_free_warn_bytes == 2147483648 + assert args[args.index("--database-free-warn-bytes") + 1] == "2147483648" + + def test_load_state_accepts_legacy_shell_format(tmp_path): state_file = tmp_path / "monitor-alert.state" state_file.write_text("LAST_HASH='abc'\nLAST_TS=123\n", encoding="utf-8")