Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions collector/jq_collector/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from . import gitlab as gl
from . import localgit
from .config import Config
from .forge import RateLimited
from .metrics import FleetCollector
from .state import RemoteRepo, SourceHealth, Store

Expand Down Expand Up @@ -82,9 +83,20 @@ def _refresh_remote(cfg: Config, store: Store) -> None:
template_tags: list[str] = []

def gather_github() -> None:
result, api, tags, dropped = gh.collect(
cfg, ref_cache, coverage_cache, fleet=by_forge.get("github", ())
)
try:
result, api, tags, dropped = gh.collect(
cfg, ref_cache, coverage_cache, fleet=by_forge.get("github", ())
)
except RateLimited as exc:
# The quota says nothing about the repos, so the last pass's view
# of them stands: slightly old data rather than No data on every
# panel until the window resets. Re-raised so the failure is still
# recorded against github and the staleness is visible.
share = set(by_forge.get("github", ()))
remote.update({n: r for n, r in snap.remote.items() if r.forge == "github"})
excluded.update(n for n in snap.excluded if n in share)
store.update(rate_limit_remaining=0.0, rate_limit_reset=exc.reset)
raise
try:
remote.update(result)
excluded.update(dropped)
Expand Down
26 changes: 26 additions & 0 deletions collector/jq_collector/forge.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,23 @@
INCONCLUSIVE_CONCLUSIONS = frozenset({"cancelled", "stale"})


class RateLimited(Exception):
"""The forge has stopped answering until its rate-limit window resets.

Not one repo's problem but the whole pass's: every call after this one gets
the same refusal. Treating it like a 403 on one repo - logged, that repo
skipped - turned an exhausted quota into a "successful" pass with no repos
in it, and that empty snapshot replaced the last good one, so every panel
read No data until the window reset. Raising it instead fails the pass and
lets the caller keep what it had.
"""

def __init__(self, path: str, reset: float) -> None:
self.path = path
self.reset = reset
super().__init__(f"rate limit exhausted at {path}; resets at {reset:.0f}")


def ts(value: str | None) -> float:
"""An ISO timestamp as epoch seconds, or 0.0 when there is none to read.

Expand Down Expand Up @@ -225,6 +242,10 @@ def fan_out(

One bad repo must not sink the refresh: a repo whose build raises is logged
on the caller's logger, so the line still names the forge, and left out.

``RateLimited`` is the exception: it is not one bad repo but every repo
after it, so leaving each one out would hand back a fleet missing however
many repos the quota did not stretch to. It sinks the refresh on purpose.
"""
result: dict[str, RemoteRepo] = {}
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
Expand All @@ -233,6 +254,11 @@ def fan_out(
full_name = futures[future]
try:
result[full_name] = future.result()
except RateLimited:
# The repos still queued would each spend a call learning the
# same thing.
pool.shutdown(wait=False, cancel_futures=True)
raise
except Exception as exc: # noqa: BLE001 - one bad repo must not sink the refresh
log.warning("repo %s failed: %s", full_name, exc)
return result
34 changes: 21 additions & 13 deletions collector/jq_collector/github.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
from .forge import (
GOOD_CONCLUSIONS,
INCONCLUSIVE_CONCLUSIONS,
RateLimited,
behind_count,
cached_ref,
ci_summary,
Expand Down Expand Up @@ -84,19 +85,26 @@ def collect(
"""
coverage_cache = coverage_cache or {}
api = GitHub(cfg)
tags = api.release_tags(cfg.template_repo)

# Dropped repos are reported back so the local scan can skip their clones
# too - otherwise a checkout keeps a dead repo on the board after the GitHub
# half has correctly stopped reporting it.
listing = api.list_repos(fleet)
excluded = frozenset(r["full_name"] for r in listing if _dropped(cfg, r))
repos = [r for r in listing if r["full_name"] not in excluded]

def one(raw: dict[str, Any]) -> RemoteRepo:
return _remote_repo(api, cfg, raw, tags, ref_cache, coverage_cache)

return fan_out(repos, "full_name", one, log, _MAX_WORKERS), api, tags, excluded
# A RateLimited out of here never hands the client back, so this is the
# last place that can close it.
try:
tags = api.release_tags(cfg.template_repo)

# Dropped repos are reported back so the local scan can skip their clones
# too - otherwise a checkout keeps a dead repo on the board after the GitHub
# half has correctly stopped reporting it.
listing = api.list_repos(fleet)
excluded = frozenset(r["full_name"] for r in listing if _dropped(cfg, r))
repos = [r for r in listing if r["full_name"] not in excluded]

def one(raw: dict[str, Any]) -> RemoteRepo:
return _remote_repo(api, cfg, raw, tags, ref_cache, coverage_cache)

result = fan_out(repos, "full_name", one, log, _MAX_WORKERS)
except RateLimited:
api.close()
raise
return result, api, tags, excluded


def _dropped(cfg: Config, raw: dict[str, Any]) -> bool:
Expand Down
31 changes: 26 additions & 5 deletions collector/jq_collector/github_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

from . import coverage_xml
from .config import Config
from .forge import ts
from .forge import RateLimited, ts
from .github_payloads import (
active_workflow_names,
alert_counts,
Expand All @@ -41,6 +41,19 @@
_ACCEPT = "application/vnd.github+json"


def _rate_limited(response: httpx.Response) -> bool:
"""Whether a refusal is GitHub's rate limit rather than a permission.

Both arrive as 403 (or 429), so the status alone cannot tell them apart.
The primary limit says so with ``x-ratelimit-remaining: 0``; the secondary,
abuse-detection limit with a ``retry-after``. A 403 carrying neither is a
real permission answer - Dependabot alerts switched off, say - and stays one.
"""
if response.status_code not in (403, 429):
return False
return response.headers.get("x-ratelimit-remaining") == "0" or "retry-after" in response.headers


class GitHub:
"""A GitHub REST client for one refresh, tracking the rate limit off every response."""

Expand All @@ -64,9 +77,16 @@ def close(self) -> None:
self._client.close()

def _get(self, path: str, **params: str | int) -> httpx.Response:
"""GET ``path``, noting the rate-limit headers whatever the status."""
"""GET ``path``, noting the rate-limit headers whatever the status.

Raises ``RateLimited`` when GitHub refuses because the quota is spent,
so that refusal is never mistaken for one repo being unreadable.
"""
response = self._client.get(path, params=params or None)
self._note_rate_limit(response)
if _rate_limited(response):
log.warning("%s -> %s: rate limit exhausted", path, response.status_code)
raise RateLimited(path, self.rate_reset)
return response

def _note_rate_limit(self, response: httpx.Response) -> None:
Expand All @@ -86,9 +106,10 @@ def _note_rate_limit(self, response: httpx.Response) -> None:
def _json(self, path: str, **params: str | int) -> object | None:
"""GET returning parsed JSON, or None for the expected empty cases.

404 (no such file), 403 (rate limited or forbidden) and 409 (empty
repository) are all states the fleet legitimately contains; they are
logged and skipped rather than failing the whole refresh.
404 (no such file), 403 (forbidden) and 409 (empty repository) are
all states the fleet legitimately contains; they are logged and
skipped rather than failing the whole refresh. A 403 that is really
the rate limit never gets here - ``_get`` raises ``RateLimited``.
"""
response = self._get(path, **params)
if response.status_code in (403, 404, 409, 451):
Expand Down
72 changes: 71 additions & 1 deletion collector/tests/test_github_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@
from test_coverage import StubAPI

from jq_collector.config import Config
from jq_collector.forge import behind_count, ts
from jq_collector.forge import RateLimited, behind_count, fan_out, ts
from jq_collector.github import GitHub
from jq_collector.state import RemoteRepo


def client(handler, **cfg_overrides) -> GitHub:
Expand Down Expand Up @@ -135,6 +136,75 @@ def test_the_expected_empties_are_none_not_exceptions(status, caplog):
assert f"-> {status}" in caplog.text


@pytest.mark.parametrize(
("status", "headers"),
[
(403, {"x-ratelimit-remaining": "0", "x-ratelimit-reset": "1794585785"}),
(429, {"x-ratelimit-remaining": "0", "x-ratelimit-reset": "1794585785"}),
(403, {"retry-after": "60"}),
(429, {"retry-after": "60"}),
],
)
def test_a_spent_quota_raises_rather_than_reading_as_unreadable(status, headers):
"""Read as an unreadable repo, an exhausted quota dropped every repo from a
pass that still counted as a success, and that empty snapshot blanked the
board until the window reset."""
api = client(lambda _r: httpx.Response(status, json={"message": "limit"}, headers=headers))

with pytest.raises(RateLimited) as caught:
api._json("/repos/o/r")
with pytest.raises(RateLimited):
api._bytes("/artifact/zip")

assert caught.value.path == "/repos/o/r"


def test_the_reset_time_rides_on_the_exception():
api = client(
lambda _r: httpx.Response(
403, headers={"x-ratelimit-remaining": "0", "x-ratelimit-reset": "1794585785"}
)
)

with pytest.raises(RateLimited) as caught:
api._json("/repos/o/r")

assert caught.value.reset == 1794585785.0


def test_a_403_with_quota_left_is_still_a_permission_answer():
"""Dependabot alerts switched off answer 403 with plenty of quota left;
that is a fact about the repo, not a reason to abandon the pass."""
api = client(lambda _r: httpx.Response(403, headers={"x-ratelimit-remaining": "4200"}))

assert api._json("/repos/o/r/dependabot/alerts") is None


def test_a_rate_limit_stops_the_fan_out_instead_of_dropping_repos():
"""Every repo after the quota runs out would fail the same way; leaving
each one out would publish a fleet cut short at an arbitrary point."""

def build(raw):
if raw["n"] == "b":
raise RateLimited("/repos/o/b", 0.0)
return RemoteRepo(name=raw["n"])

with pytest.raises(RateLimited):
fan_out([{"n": "a"}, {"n": "b"}, {"n": "c"}], "n", build, logging.getLogger("t"), 1)


def test_one_bad_repo_is_still_left_out_of_the_fan_out(caplog):
def build(raw):
if raw["n"] == "b":
raise ValueError("odd payload")
return RemoteRepo(name=raw["n"])

result = fan_out([{"n": "a"}, {"n": "b"}], "n", build, logging.getLogger("t"), 1)

assert set(result) == {"a"}
assert "repo b failed" in caplog.text


def test_an_unexpected_status_fails_the_refresh():
"""A 500 is not a fleet state; swallowing it would publish a snapshot that
silently lost a repo, which Prometheus then records as fact."""
Expand Down
67 changes: 67 additions & 0 deletions collector/tests/test_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

from jq_collector import __main__ as entry
from jq_collector.config import Config
from jq_collector.forge import RateLimited
from jq_collector.state import LocalRepo, RemoteRepo, SourceHealth, Store

# -- _run_once: health bookkeeping either way ------------------------------
Expand Down Expand Up @@ -565,6 +566,72 @@ def exploding(*_a, **_k):
assert store.health_for("github").errors == 0


def _rate_limited(*_a, **_k):
raise RateLimited("/repos/o/gh", 1794585785.0)


def test_a_spent_github_quota_keeps_the_last_good_repos(monkeypatch):
"""The quota says nothing about the repos. Publishing an empty map for it
put No data on every GitHub panel until the window reset."""
store = Store()
store.update(
remote={
"o/gh": RemoteRepo(name="gh", forge="github"),
"acme/web": RemoteRepo(name="web", forge="gitlab"),
},
excluded=frozenset({"o/archived", "acme/old"}),
)
monkeypatch.setattr(entry.gh, "collect", _rate_limited)
monkeypatch.setattr(entry.gl, "collect", _fake_gitlab({}))

entry._refresh_remote(
_cfg_with(
{"acme/web": "gitlab", "acme/old": "gitlab"},
("o/gh", "o/archived", "acme/web", "acme/old"),
),
store,
)

snap = store.snapshot()
assert set(snap.remote) == {"o/gh"}, "GitHub's repos carried over; GitLab's are its own pass's"
assert snap.excluded == frozenset({"o/archived"})


def test_a_spent_github_quota_is_recorded_as_a_failure(monkeypatch):
"""Kept data is stale data, and the board must be able to say so."""
store = Store()
monkeypatch.setattr(entry.gh, "collect", _rate_limited)

entry._refresh_remote(_cfg_with({}, ("o/gh",)), store)

health = store.health_for("github")
assert health.errors == 1
assert "rate limit exhausted" in health.last_error
assert store.snapshot().rate_limit_remaining == 0.0
assert store.snapshot().rate_limit_reset == 1794585785.0


def test_collect_closes_its_client_when_the_quota_runs_out(monkeypatch):
closed: list[bool] = []

class Spent:
def __init__(self, _cfg):
pass

def release_tags(self, _repo):
raise RateLimited("/repos/o/t/releases", 0.0)

def close(self):
closed.append(True)

monkeypatch.setattr(entry.gh, "GitHub", Spent)

with pytest.raises(RateLimited):
entry.gh.collect(Config(), {})

assert closed == [True]


def test_both_forges_contribute_to_the_excluded_set(monkeypatch):
"""Reported back so the local scan skips their clones too - a checkout would
otherwise keep a dropped repo on the board."""
Expand Down
Loading