Repository navigation
Expand file tree
/
Copy pathtrace_client.py
More file actions
359 lines (316 loc) · 16.5 KB
/
Copy pathtrace_client.py
File metadata and controls
359 lines (316 loc) · 16.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
"""trace-client 0.4.1 -- https://github.com/Stephenson-Software/trace-client-python
One call to report that a program was used. Copy this file into a project as
is, or vendor the package; either way there is nothing else to add. Standard
library only; Python 3.8+.
MIT licensed. Keep this header when vendoring so the file can be found again.
Vendored into patchwork unmodified apart from this note.
"""
from __future__ import annotations
import json
import logging
import os
import queue
import re
import threading
import urllib.error
import urllib.request
import uuid
from typing import Dict, Mapping, Optional, Union
__version__ = "0.4.1"
_LOG = logging.getLogger("trace")
#: Environment variables that turn reporting off for every program using a
#: trace client, checked before the program's own setting. Set
#: ``TRACE_USAGE_REPORTING=off`` (also ``false``, ``0``, ``no``; case does not
#: matter) or ``DO_NOT_TRACK=1`` (also ``true``, ``yes``; see
#: https://consoledonottrack.com).
ENV_TRACE_USAGE_REPORTING = "TRACE_USAGE_REPORTING"
ENV_DO_NOT_TRACK = "DO_NOT_TRACK"
_OFF_VALUES = frozenset(("off", "false", "0", "no"))
_DO_NOT_TRACK_VALUES = frozenset(("1", "true", "yes"))
#: The values :attr:`TraceClient.disabled_reason` can take. First match wins.
REASON_ENVIRONMENT = "environment"
REASON_CONFIG = "config"
REASON_NO_KEY = "no key"
#: The longest a program version may be, after trimming: the trace server's
#: limit on a tag value.
MAX_TAG_LENGTH = 255
#: The most tags one event carries: the trace server's limit. The ``install``
#: tag is only added while an event has fewer than this.
MAX_TAGS = 32
#: The tag every event carries the installation's ID as.
INSTALL_TAG = "install"
# What install_id_from_file accepts as an ID on a line of its file.
_INSTALL_ID_LINE = re.compile(r"^[A-Za-z0-9_.-]{1,%d}$" % MAX_TAG_LENGTH)
def environment_opts_out(environ: Optional[Mapping[str, str]] = None) -> bool:
"""Whether the environment asks for usage reporting to be off, via
``TRACE_USAGE_REPORTING=off`` or ``DO_NOT_TRACK=1``. Only the listed
values count; anything else (including an empty value) leaves the
program's own setting in charge."""
env = os.environ if environ is None else environ
if env.get(ENV_TRACE_USAGE_REPORTING, "").strip().lower() in _OFF_VALUES:
return True
if env.get(ENV_DO_NOT_TRACK, "").strip().lower() in _DO_NOT_TRACK_VALUES:
return True
return False
class TraceClient:
"""Reports usage events to a trace server, and never gets in the way of
the program doing the reporting.
Three properties hold for every call to :meth:`report`:
* **It returns immediately.** The HTTP call happens on a single daemon
thread owned by this client. A game loop can report from its main
thread without a frame ever waiting on the network.
* **It never raises.** A server that is down, slow, or rejecting the key
is a dropped report, not an exception in the host program. Failures
are logged at DEBUG on the ``trace`` logger and otherwise not at all.
* **It is bounded.** At most :attr:`QUEUE_CAPACITY` reports wait to be
sent; beyond that, new reports are dropped rather than accumulated.
A trace server that is unreachable for a week costs a few kilobytes,
not the host's memory.
Reporting is opt-out: ``enabled=False``, or no key, yields a client that
does nothing and costs nothing. So does the environment: the constructor
checks ``TRACE_USAGE_REPORTING=off`` and ``DO_NOT_TRACK=1`` before it
looks at ``enabled``, so a user can switch off every trace-reporting
program at once. :attr:`disabled_reason` says which of those applied
(``"environment"``, ``"config"`` or ``"no key"``; ``None`` when on) so
the program can say so in its notice. Programs that run on other
people's machines should expose that switch in their settings and say
so once, pointing at https://danielstephenson.dev/usage-reporting.
Every event carries the program's own version as the tag ``version`` --
the third argument, required, so a ``command`` event can be tied to a
release as well as a ``startup`` one. An event's own ``version`` tag wins
over it.
Every event also carries a random per-installation ID as the tag
``install``, so the trace server can count distinct installations rather
than raw events -- when the program supplies one: ``install_id=`` (an ID
it stores itself) or ``install_id_file=`` (a path the client loads the ID
from, or writes a new random one to; see :meth:`install_id_from_file`).
Without either, no ``install`` tag is sent and nothing is written
anywhere. Both are resolved only after the opt-outs, so a disabled client
never makes up an ID and never writes one. An event's own ``install`` tag
wins over it.
::
trace = TraceClient("https://trace.example.org", "roam", __version__,
key=settings.usage_key, enabled=settings.usage_reporting)
trace.report("startup")
...
trace.close() # on shutdown
"""
QUEUE_CAPACITY = 256
TIMEOUT_SECONDS = 5.0
def __init__(self, base_url: str, application: str, version: str, *, key: Optional[str] = None,
enabled: bool = True, install_id: Optional[str] = None,
install_id_file: Optional[Union[str, "os.PathLike[str]"]] = None) -> None:
"""A client for the program named ``application``, at ``version``,
reporting to the trace server at ``base_url``. The version is sent as
the tag ``version`` on every event; a blank one, or one longer than
:data:`MAX_TAG_LENGTH` characters, is a :class:`ValueError`.
``install_id`` is the installation's ID, sent as the tag ``install``
on every event. It should be random -- e.g. a :func:`uuid.uuid4` the
program stores in its own settings -- and never derived from a
person, account or address. Trimmed; ``None`` or blank means none;
longer than :data:`MAX_TAG_LENGTH` characters is a
:class:`ValueError`. ``install_id_file`` is a path to load it from
(or create it in) with :meth:`install_id_from_file`, only when the
client is enabled; an explicit ``install_id`` wins over it."""
if not base_url or not base_url.strip():
raise ValueError("base_url is required")
if not application or not application.strip():
raise ValueError("application is required")
if not version or not version.strip():
raise ValueError("version is required")
if len(version.strip()) > MAX_TAG_LENGTH:
raise ValueError("version is longer than %d characters" % MAX_TAG_LENGTH)
explicit_install_id = (install_id or "").strip() or None
if explicit_install_id is not None and len(explicit_install_id) > MAX_TAG_LENGTH:
raise ValueError("install_id is longer than %d characters" % MAX_TAG_LENGTH)
self._endpoint = base_url.strip().rstrip("/") + "/api/metrics"
self._application = application.strip()
self._version = version.strip()
self._key = (key or "").strip()
self._queue: Optional["queue.Queue[Optional[bytes]]"] = None
self._thread: Optional[threading.Thread] = None
#: Why this client reports nothing: ``"environment"`` (the
#: ``TRACE_USAGE_REPORTING`` / ``DO_NOT_TRACK`` variables),
#: ``"config"`` (``enabled=False``) or ``"no key"``; ``None`` when it
#: reports. Decided once, here, in that order of precedence.
self.disabled_reason: Optional[str] = None
if environment_opts_out():
self.disabled_reason = REASON_ENVIRONMENT
elif not enabled:
self.disabled_reason = REASON_CONFIG
elif not self._key:
self.disabled_reason = REASON_NO_KEY
self._install_id: Optional[str] = None
if self.disabled_reason is None:
# After the opt-outs, never before: a disabled client neither
# makes up an ID nor writes one to disk.
if explicit_install_id is not None:
self._install_id = explicit_install_id
elif install_id_file is not None:
self._install_id = TraceClient.install_id_from_file(install_id_file)
self._queue = queue.Queue(maxsize=self.QUEUE_CAPACITY)
self._thread = threading.Thread(target=self._drain, name="trace-client/" + self._application,
daemon=True)
self._thread.start()
@classmethod
def disabled(cls) -> "TraceClient":
"""A client that reports nothing. Useful as a default before settings are read."""
return cls("http://disabled.invalid", "disabled", "disabled", enabled=False)
@property
def install_id(self) -> Optional[str]:
"""The per-installation ID every event carries as the tag ``install``,
or ``None`` when the client is disabled or was given none (no
``install_id`` and no ``install_id_file``)."""
return self._install_id
@staticmethod
def install_id_from_file(path: Union[str, "os.PathLike[str]"]) -> str:
"""The installation's ID kept in the file at ``path``, which the
program chooses -- there is no default location. The first line that
is an ID (``[A-Za-z0-9_.-]``, at most :data:`MAX_TAG_LENGTH`
characters, surrounding whitespace ignored) is returned. If the file
does not exist or holds no such line, a new random :func:`uuid.uuid4`
is written to it (parent directories created) and returned. Delete
the file to get a new one.
Never raises: if the file exists but cannot be read, or cannot be
written, a fresh random ID is returned for this process only, and an
unreadable file is left as it is.
Called directly, this writes whatever the opt-outs say. Pass the path
as ``install_id_file=`` instead to keep the guarantee that a disabled
client writes nothing."""
fresh = str(uuid.uuid4())
try:
target = os.fspath(path)
if not target or not str(target).strip():
return fresh
try:
with open(target, "r", encoding="utf-8") as existing:
for line in existing:
candidate = line.strip()
if _INSTALL_ID_LINE.match(candidate):
return candidate
except FileNotFoundError:
pass
except Exception as failure: # noqa: BLE001 - unreadable: never overwrite it
_LOG.debug("[trace] could not read install ID file %s, using an in-memory one: %s",
target, failure)
return fresh
parent = os.path.dirname(os.path.abspath(target))
os.makedirs(parent, exist_ok=True)
with open(target, "w", encoding="utf-8") as written:
written.write(fresh + "\n")
return fresh
except Exception as failure: # noqa: BLE001 - an ID must never be the reason a program stops
_LOG.debug("[trace] could not write install ID file %s, using an in-memory one: %s",
path, failure)
return fresh
@property
def enabled(self) -> bool:
"""Whether :meth:`report` will actually send anything. ``False`` after
:meth:`close` too; :attr:`disabled_reason` keeps the reason it was
built off, if it was."""
return self._queue is not None
def report(self, name: str, value: Optional[float] = None,
tags: Optional[Mapping[str, str]] = None) -> None:
"""Report that ``name`` happened, with an optional numeric value and
optional string tags. Returns immediately; see the class docstring."""
if self._queue is None:
return
try:
if not name or not name.strip():
return
tags = _with_install(_with_version(tags, self._version), self._install_id)
body = _json(self._application, name, value, tags)
self._queue.put_nowait(body)
except queue.Full:
_LOG.debug("[trace] queue full, dropped %s", name)
except Exception as failure: # noqa: BLE001 - a report must never be the reason a program stops
_LOG.debug("[trace] could not queue %s: %s", name, failure)
def close(self, timeout: float = TIMEOUT_SECONDS) -> None:
"""Stop the sending thread, giving reports already queued up to
``timeout`` seconds in total to be sent first. A program that reports
and then exits within milliseconds -- a CLI, a short script -- would
otherwise lose its one event to the race between queueing it and the
sender thread picking it up. The bound still holds: an unreachable
server costs at most ``timeout``, never a hang. Safe to call more than
once, and on a disabled client."""
if self._queue is None or self._thread is None:
return
q, thread = self._queue, self._thread
self._queue = None # report() is a no-op from here on
try:
q.put_nowait(None) # sentinel behind whatever is queued
except queue.Full:
# A full queue means 256 reports are waiting; the sentinel would be
# the 257th. Drop the oldest to make room -- one lost report beats a
# thread that never stops.
try:
q.get_nowait()
q.put_nowait(None)
except (queue.Empty, queue.Full):
pass
thread.join(timeout)
# -- internals --------------------------------------------------------
def _drain(self) -> None:
q = self._queue
assert q is not None
while True:
body = q.get()
if body is None:
return
try:
self._send(body)
except Exception as failure: # noqa: BLE001 - a dead sender would leave the queue filling forever
_LOG.debug("[trace] sender failed on %s: %s", body, failure)
def _send(self, body: bytes) -> None:
try:
# Built inside the try: a base URL without a scheme fails here, and
# an uncaught error would kill the sender thread with a traceback.
request = urllib.request.Request(
self._endpoint, data=body, method="POST",
headers={
"Content-Type": "application/json; charset=utf-8",
"Authorization": "Bearer " + self._key,
"User-Agent": "trace-client-python/%s (%s)" % (__version__, self._application),
})
with urllib.request.urlopen(request, timeout=self.TIMEOUT_SECONDS) as response:
status = response.status
response.read()
except urllib.error.HTTPError as answered:
status = answered.code
try:
answered.read()
except Exception: # noqa: BLE001
pass
except Exception as failure: # noqa: BLE001 - see report()
_LOG.debug("[trace] could not deliver %s: %s", body, failure)
return
if status != 201:
_LOG.debug("[trace] trace server answered %s for %s", status, body)
def _with_version(tags: Optional[Mapping[str, str]], version: str) -> Dict[str, str]:
"""The event's own tags plus ``version``, unless the event already carries
one. A copy; the caller's mapping is never modified."""
merged: Dict[str, str] = {}
if tags:
for k, v in dict(tags).items():
if k is not None and v is not None:
merged[str(k)] = str(v)
if "version" not in merged:
merged["version"] = version
return merged
def _with_install(tags: Dict[str, str], install_id: Optional[str]) -> Dict[str, str]:
"""The tags plus ``install``, unless they already carry one, there is no
ID, or adding it would pass :data:`MAX_TAGS`. A copy when it adds."""
if install_id is None or INSTALL_TAG in tags or len(tags) >= MAX_TAGS:
return tags
merged = dict(tags)
merged[INSTALL_TAG] = install_id
return merged
def _json(application: str, name: str, value: Optional[float], tags: Optional[Mapping[str, str]]) -> bytes:
payload: Dict[str, object] = {"application": application, "name": name}
if value is not None and value == value and value not in (float("inf"), float("-inf")):
payload["value"] = value
if tags:
clean = {str(k): str(v) for k, v in tags.items() if k is not None and v is not None}
if clean:
payload["tags"] = clean
return json.dumps(payload, separators=(",", ":")).encode("utf-8")