diff --git a/collector/jq_collector/__main__.py b/collector/jq_collector/__main__.py index c921c0f..adf1d84 100644 --- a/collector/jq_collector/__main__.py +++ b/collector/jq_collector/__main__.py @@ -26,6 +26,7 @@ from . import gitlab as gl from . import localgit from .config import Config +from .forge import RateLimited from .metrics import FleetCollector from .state import RemoteRepo, SourceHealth, Store @@ -82,9 +83,20 @@ def _refresh_remote(cfg: Config, store: Store) -> None: template_tags: list[str] = [] def gather_github() -> None: - result, api, tags, dropped = gh.collect( - cfg, ref_cache, coverage_cache, fleet=by_forge.get("github", ()) - ) + try: + result, api, tags, dropped = gh.collect( + cfg, ref_cache, coverage_cache, fleet=by_forge.get("github", ()) + ) + except RateLimited as exc: + # The quota says nothing about the repos, so the last pass's view + # of them stands: slightly old data rather than No data on every + # panel until the window resets. Re-raised so the failure is still + # recorded against github and the staleness is visible. + share = set(by_forge.get("github", ())) + remote.update({n: r for n, r in snap.remote.items() if r.forge == "github"}) + excluded.update(n for n in snap.excluded if n in share) + store.update(rate_limit_remaining=0.0, rate_limit_reset=exc.reset) + raise try: remote.update(result) excluded.update(dropped) diff --git a/collector/jq_collector/forge.py b/collector/jq_collector/forge.py index 61f408b..f3aa891 100644 --- a/collector/jq_collector/forge.py +++ b/collector/jq_collector/forge.py @@ -34,6 +34,23 @@ INCONCLUSIVE_CONCLUSIONS = frozenset({"cancelled", "stale"}) +class RateLimited(Exception): + """The forge has stopped answering until its rate-limit window resets. + + Not one repo's problem but the whole pass's: every call after this one gets + the same refusal. Treating it like a 403 on one repo - logged, that repo + skipped - turned an exhausted quota into a "successful" pass with no repos + in it, and that empty snapshot replaced the last good one, so every panel + read No data until the window reset. Raising it instead fails the pass and + lets the caller keep what it had. + """ + + def __init__(self, path: str, reset: float) -> None: + self.path = path + self.reset = reset + super().__init__(f"rate limit exhausted at {path}; resets at {reset:.0f}") + + def ts(value: str | None) -> float: """An ISO timestamp as epoch seconds, or 0.0 when there is none to read. @@ -225,6 +242,10 @@ def fan_out( One bad repo must not sink the refresh: a repo whose build raises is logged on the caller's logger, so the line still names the forge, and left out. + + ``RateLimited`` is the exception: it is not one bad repo but every repo + after it, so leaving each one out would hand back a fleet missing however + many repos the quota did not stretch to. It sinks the refresh on purpose. """ result: dict[str, RemoteRepo] = {} with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: @@ -233,6 +254,11 @@ def fan_out( full_name = futures[future] try: result[full_name] = future.result() + except RateLimited: + # The repos still queued would each spend a call learning the + # same thing. + pool.shutdown(wait=False, cancel_futures=True) + raise except Exception as exc: # noqa: BLE001 - one bad repo must not sink the refresh log.warning("repo %s failed: %s", full_name, exc) return result diff --git a/collector/jq_collector/github.py b/collector/jq_collector/github.py index 962415d..3a54b7a 100644 --- a/collector/jq_collector/github.py +++ b/collector/jq_collector/github.py @@ -28,6 +28,7 @@ from .forge import ( GOOD_CONCLUSIONS, INCONCLUSIVE_CONCLUSIONS, + RateLimited, behind_count, cached_ref, ci_summary, @@ -84,19 +85,26 @@ def collect( """ coverage_cache = coverage_cache or {} api = GitHub(cfg) - tags = api.release_tags(cfg.template_repo) - - # Dropped repos are reported back so the local scan can skip their clones - # too - otherwise a checkout keeps a dead repo on the board after the GitHub - # half has correctly stopped reporting it. - listing = api.list_repos(fleet) - excluded = frozenset(r["full_name"] for r in listing if _dropped(cfg, r)) - repos = [r for r in listing if r["full_name"] not in excluded] - - def one(raw: dict[str, Any]) -> RemoteRepo: - return _remote_repo(api, cfg, raw, tags, ref_cache, coverage_cache) - - return fan_out(repos, "full_name", one, log, _MAX_WORKERS), api, tags, excluded + # A RateLimited out of here never hands the client back, so this is the + # last place that can close it. + try: + tags = api.release_tags(cfg.template_repo) + + # Dropped repos are reported back so the local scan can skip their clones + # too - otherwise a checkout keeps a dead repo on the board after the GitHub + # half has correctly stopped reporting it. + listing = api.list_repos(fleet) + excluded = frozenset(r["full_name"] for r in listing if _dropped(cfg, r)) + repos = [r for r in listing if r["full_name"] not in excluded] + + def one(raw: dict[str, Any]) -> RemoteRepo: + return _remote_repo(api, cfg, raw, tags, ref_cache, coverage_cache) + + result = fan_out(repos, "full_name", one, log, _MAX_WORKERS) + except RateLimited: + api.close() + raise + return result, api, tags, excluded def _dropped(cfg: Config, raw: dict[str, Any]) -> bool: diff --git a/collector/jq_collector/github_api.py b/collector/jq_collector/github_api.py index f70e2fd..d119e3b 100644 --- a/collector/jq_collector/github_api.py +++ b/collector/jq_collector/github_api.py @@ -22,7 +22,7 @@ from . import coverage_xml from .config import Config -from .forge import ts +from .forge import RateLimited, ts from .github_payloads import ( active_workflow_names, alert_counts, @@ -41,6 +41,19 @@ _ACCEPT = "application/vnd.github+json" +def _rate_limited(response: httpx.Response) -> bool: + """Whether a refusal is GitHub's rate limit rather than a permission. + + Both arrive as 403 (or 429), so the status alone cannot tell them apart. + The primary limit says so with ``x-ratelimit-remaining: 0``; the secondary, + abuse-detection limit with a ``retry-after``. A 403 carrying neither is a + real permission answer - Dependabot alerts switched off, say - and stays one. + """ + if response.status_code not in (403, 429): + return False + return response.headers.get("x-ratelimit-remaining") == "0" or "retry-after" in response.headers + + class GitHub: """A GitHub REST client for one refresh, tracking the rate limit off every response.""" @@ -64,9 +77,16 @@ def close(self) -> None: self._client.close() def _get(self, path: str, **params: str | int) -> httpx.Response: - """GET ``path``, noting the rate-limit headers whatever the status.""" + """GET ``path``, noting the rate-limit headers whatever the status. + + Raises ``RateLimited`` when GitHub refuses because the quota is spent, + so that refusal is never mistaken for one repo being unreadable. + """ response = self._client.get(path, params=params or None) self._note_rate_limit(response) + if _rate_limited(response): + log.warning("%s -> %s: rate limit exhausted", path, response.status_code) + raise RateLimited(path, self.rate_reset) return response def _note_rate_limit(self, response: httpx.Response) -> None: @@ -86,9 +106,10 @@ def _note_rate_limit(self, response: httpx.Response) -> None: def _json(self, path: str, **params: str | int) -> object | None: """GET returning parsed JSON, or None for the expected empty cases. - 404 (no such file), 403 (rate limited or forbidden) and 409 (empty - repository) are all states the fleet legitimately contains; they are - logged and skipped rather than failing the whole refresh. + 404 (no such file), 403 (forbidden) and 409 (empty repository) are + all states the fleet legitimately contains; they are logged and + skipped rather than failing the whole refresh. A 403 that is really + the rate limit never gets here - ``_get`` raises ``RateLimited``. """ response = self._get(path, **params) if response.status_code in (403, 404, 409, 451): diff --git a/collector/tests/test_github_http.py b/collector/tests/test_github_http.py index e73a598..f52caf8 100644 --- a/collector/tests/test_github_http.py +++ b/collector/tests/test_github_http.py @@ -21,8 +21,9 @@ from test_coverage import StubAPI from jq_collector.config import Config -from jq_collector.forge import behind_count, ts +from jq_collector.forge import RateLimited, behind_count, fan_out, ts from jq_collector.github import GitHub +from jq_collector.state import RemoteRepo def client(handler, **cfg_overrides) -> GitHub: @@ -135,6 +136,75 @@ def test_the_expected_empties_are_none_not_exceptions(status, caplog): assert f"-> {status}" in caplog.text +@pytest.mark.parametrize( + ("status", "headers"), + [ + (403, {"x-ratelimit-remaining": "0", "x-ratelimit-reset": "1794585785"}), + (429, {"x-ratelimit-remaining": "0", "x-ratelimit-reset": "1794585785"}), + (403, {"retry-after": "60"}), + (429, {"retry-after": "60"}), + ], +) +def test_a_spent_quota_raises_rather_than_reading_as_unreadable(status, headers): + """Read as an unreadable repo, an exhausted quota dropped every repo from a + pass that still counted as a success, and that empty snapshot blanked the + board until the window reset.""" + api = client(lambda _r: httpx.Response(status, json={"message": "limit"}, headers=headers)) + + with pytest.raises(RateLimited) as caught: + api._json("/repos/o/r") + with pytest.raises(RateLimited): + api._bytes("/artifact/zip") + + assert caught.value.path == "/repos/o/r" + + +def test_the_reset_time_rides_on_the_exception(): + api = client( + lambda _r: httpx.Response( + 403, headers={"x-ratelimit-remaining": "0", "x-ratelimit-reset": "1794585785"} + ) + ) + + with pytest.raises(RateLimited) as caught: + api._json("/repos/o/r") + + assert caught.value.reset == 1794585785.0 + + +def test_a_403_with_quota_left_is_still_a_permission_answer(): + """Dependabot alerts switched off answer 403 with plenty of quota left; + that is a fact about the repo, not a reason to abandon the pass.""" + api = client(lambda _r: httpx.Response(403, headers={"x-ratelimit-remaining": "4200"})) + + assert api._json("/repos/o/r/dependabot/alerts") is None + + +def test_a_rate_limit_stops_the_fan_out_instead_of_dropping_repos(): + """Every repo after the quota runs out would fail the same way; leaving + each one out would publish a fleet cut short at an arbitrary point.""" + + def build(raw): + if raw["n"] == "b": + raise RateLimited("/repos/o/b", 0.0) + return RemoteRepo(name=raw["n"]) + + with pytest.raises(RateLimited): + fan_out([{"n": "a"}, {"n": "b"}, {"n": "c"}], "n", build, logging.getLogger("t"), 1) + + +def test_one_bad_repo_is_still_left_out_of_the_fan_out(caplog): + def build(raw): + if raw["n"] == "b": + raise ValueError("odd payload") + return RemoteRepo(name=raw["n"]) + + result = fan_out([{"n": "a"}, {"n": "b"}], "n", build, logging.getLogger("t"), 1) + + assert set(result) == {"a"} + assert "repo b failed" in caplog.text + + def test_an_unexpected_status_fails_the_refresh(): """A 500 is not a fleet state; swallowing it would publish a snapshot that silently lost a repo, which Prometheus then records as fact.""" diff --git a/collector/tests/test_main.py b/collector/tests/test_main.py index 81f1a6c..0414cec 100644 --- a/collector/tests/test_main.py +++ b/collector/tests/test_main.py @@ -16,6 +16,7 @@ from jq_collector import __main__ as entry from jq_collector.config import Config +from jq_collector.forge import RateLimited from jq_collector.state import LocalRepo, RemoteRepo, SourceHealth, Store # -- _run_once: health bookkeeping either way ------------------------------ @@ -565,6 +566,72 @@ def exploding(*_a, **_k): assert store.health_for("github").errors == 0 +def _rate_limited(*_a, **_k): + raise RateLimited("/repos/o/gh", 1794585785.0) + + +def test_a_spent_github_quota_keeps_the_last_good_repos(monkeypatch): + """The quota says nothing about the repos. Publishing an empty map for it + put No data on every GitHub panel until the window reset.""" + store = Store() + store.update( + remote={ + "o/gh": RemoteRepo(name="gh", forge="github"), + "acme/web": RemoteRepo(name="web", forge="gitlab"), + }, + excluded=frozenset({"o/archived", "acme/old"}), + ) + monkeypatch.setattr(entry.gh, "collect", _rate_limited) + monkeypatch.setattr(entry.gl, "collect", _fake_gitlab({})) + + entry._refresh_remote( + _cfg_with( + {"acme/web": "gitlab", "acme/old": "gitlab"}, + ("o/gh", "o/archived", "acme/web", "acme/old"), + ), + store, + ) + + snap = store.snapshot() + assert set(snap.remote) == {"o/gh"}, "GitHub's repos carried over; GitLab's are its own pass's" + assert snap.excluded == frozenset({"o/archived"}) + + +def test_a_spent_github_quota_is_recorded_as_a_failure(monkeypatch): + """Kept data is stale data, and the board must be able to say so.""" + store = Store() + monkeypatch.setattr(entry.gh, "collect", _rate_limited) + + entry._refresh_remote(_cfg_with({}, ("o/gh",)), store) + + health = store.health_for("github") + assert health.errors == 1 + assert "rate limit exhausted" in health.last_error + assert store.snapshot().rate_limit_remaining == 0.0 + assert store.snapshot().rate_limit_reset == 1794585785.0 + + +def test_collect_closes_its_client_when_the_quota_runs_out(monkeypatch): + closed: list[bool] = [] + + class Spent: + def __init__(self, _cfg): + pass + + def release_tags(self, _repo): + raise RateLimited("/repos/o/t/releases", 0.0) + + def close(self): + closed.append(True) + + monkeypatch.setattr(entry.gh, "GitHub", Spent) + + with pytest.raises(RateLimited): + entry.gh.collect(Config(), {}) + + assert closed == [True] + + def test_both_forges_contribute_to_the_excluded_set(monkeypatch): """Reported back so the local scan skips their clones too - a checkout would otherwise keep a dropped repo on the board."""