From 917ed2511b59fb263c78a0066cf9d44ace31a4f5 Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Wed, 4 Nov 2026 21:59:05 -0700 Subject: [PATCH 1/2] Let a Logger report on a full queue, and close the client Shared::log held logMutex while the logger ran, so a logger whose report() found the queue full locked that mutex a second time on the same thread and hung, on the reporting thread or the sending one. A logger calling close() hung in forgetLogger the same way, or aborted the program: the destructor could find close() returning at once while thread_ was still joinable. Shared now records which thread is running the logger. A message raised on that thread is dropped, and close() called from it waits for nothing: it detaches the thread under sharedMutex_ and abandons what is queued, leaving log() to clear the logger once it has returned. Closes #24 Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- test/test_trace_client.cpp | 129 +++++++++++++++++++++++++++++++++++++ trace_client.hpp | 84 ++++++++++++++++++++---- 3 files changed, 204 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 5e61e81..afd5efb 100644 --- a/README.md +++ b/README.md @@ -101,9 +101,9 @@ client never generates one, never reads or writes the file, and sends nothing. | Property | Meaning | |---|---| | **Returns immediately** | The HTTP call runs on one thread the client owns. A game loop can report from its main thread and no frame waits on the network. | -| **Never throws** | Nothing in the client throws. A server that is down, slow, or rejecting the key — or a machine with no `curl` — is a dropped report, not an exception or a crash in your program. Drops are passed to the optional `Logger` (a `std::function`, last constructor argument), otherwise not mentioned at all. A failed delivery is mentioned on the client's own thread, a full queue on the thread that called `report`, so a `Logger` that touches the program's state must be safe to call from either. | +| **Never throws** | Nothing in the client throws. A server that is down, slow, or rejecting the key — or a machine with no `curl` — is a dropped report, not an exception or a crash in your program. Drops are passed to the optional `Logger` (a `std::function`, last constructor argument), otherwise not mentioned at all. A failed delivery is mentioned on the client's own thread, a full queue on the thread that called `report`, so a `Logger` that touches the program's state must be safe to call from either. A `Logger` may itself call `report`; a drop that call causes is not mentioned back to it. | | **Bounded** | At most 256 reports wait to be sent; past that, new ones are dropped. A trace server that is unreachable for a week costs a few kilobytes, not your memory. | -| **`close()` drains** | Reports already queued get up to the client timeout (5 s total) to be sent before the thread stops, so a CLI that reports and exits at once does not lose its event. Still bounded: an unreachable server delays exit by at most the timeout — a request still in flight then is abandoned and its `curl` process killed. `close(seconds)` takes another bound: a NaN or negative one waits for nothing, and `close(INFINITY)` waits until the queue is sent, each request still bounded by the timeout. Once `close()` returns, the `Logger` is never called again. | +| **`close()` drains** | Reports already queued get up to the client timeout (5 s total) to be sent before the thread stops, so a CLI that reports and exits at once does not lose its event. Still bounded: an unreachable server delays exit by at most the timeout — a request still in flight then is abandoned and its `curl` process killed. `close(seconds)` takes another bound: a NaN or negative one waits for nothing, and `close(INFINITY)` waits until the queue is sent, each request still bounded by the timeout. Called from inside the `Logger`, `close()` waits for nothing, whatever the bound, and what is queued is dropped. Once `close()` returns, the `Logger` is never called again; one that called `close()` itself finishes that call first. | | **Fits the server's limits** | Names and tag keys/values are cut to 255 bytes (never mid-character), at most 32 tags are sent, blank tag keys are dropped, invalid UTF-8 becomes U+FFFD, and NaN or infinite values are left out — so a report is never rejected as a whole for one bad field. A report whose name is empty or blank is not an event at all: it is ignored, without a mention to the `Logger`. | ## Turning it off diff --git a/test/test_trace_client.cpp b/test/test_trace_client.cpp index 2a54d95..a40f4b6 100644 --- a/test/test_trace_client.cpp +++ b/test/test_trace_client.cpp @@ -10,9 +10,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -505,6 +507,129 @@ static void aFullQueueIsLoggedOnTheThreadThatCalledReport() { client.close(1.0); } +// Runs body on a thread of its own, and says whether it finished in time. A +// hang then fails a check instead of the suite: the stuck thread is left +// behind with everything on its stack, so body should own what it uses. +static bool finishesWithin(double seconds, const std::function &body) { + struct State { + std::mutex mutex; + std::condition_variable changed; + bool done; + State() : done(false) {} + }; + std::shared_ptr state = std::make_shared(); + std::thread([state, body] { + body(); + std::lock_guard lock(state->mutex); + state->done = true; + state->changed.notify_all(); + }).detach(); + std::unique_lock lock(state->mutex); + return state->changed.wait_for(lock, std::chrono::milliseconds(static_cast(seconds * 1000)), + [&] { return state->done; }); +} + +static void aLoggerMayReportWhileTheQueueIsFull() { + // The logger hears of a full queue on the reporting thread, and its own + // report finds the queue just as full. It reports on every message, so a + // drop mentioned back to it would recurse rather than hear one per flood. + CHECK(finishesWithin(30, [] { + StubServer server; + server.hold(); // the sender waits on the first report + trace_client::TraceClient *self = NULL; + std::atomic heard(0); + trace_client::Logger reporting = [&](const std::string &) { + ++heard; + self->report("from-logger"); + }; + trace_client::TraceClient client(server.baseUrl(), "MyGame", "1.2.3", "k", true, reporting); + self = &client; + const std::size_t flood = trace_client::QUEUE_CAPACITY * 2; + for (std::size_t i = 0; i < flood; ++i) client.report("flood"); + CHECK(heard >= flood - trace_client::QUEUE_CAPACITY - 1); + CHECK(heard <= flood); + server.release(); + client.close(0.5); + })); +} + +static void aLoggerMayFillTheQueueFromTheSendingThread() { + // The same lock reached from the other thread the logger runs on, where a + // hang would also take close() with it. + CHECK(finishesWithin(30, [] { + StubServer server; + server.status = 500; // the first report is a drop, so it reaches the logger + trace_client::TraceClient *self = NULL; + std::atomic filled(false); + std::atomic returned(false); + std::atomic full(0); + trace_client::Logger filling = [&](const std::string &line) { + if (line.find("queue full") != std::string::npos) ++full; + if (filled.exchange(true)) return; + for (std::size_t i = 0; i < trace_client::QUEUE_CAPACITY + 2; ++i) self->report("flood"); + returned = true; + }; + trace_client::TraceClient client(server.baseUrl(), "MyGame", "1.2.3", "k", true, filling); + self = &client; + client.report("startup"); + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + while (!returned && secondsSince(start) < 10) std::this_thread::sleep_for(std::chrono::milliseconds(10)); + CHECK(returned); + client.close(0.5); + CHECK_EQ(0u, full.load()); + })); +} + +static void aLoggerMayCloseFromTheSendingThread() { + // close() cannot wait for the thread it is running on, so it does not: + // what was queued behind the failed report is dropped and nothing more + // reaches the logger, however long the bound it was given. + CHECK(finishesWithin(30, [] { + StubServer server; + server.status = 500; + trace_client::TraceClient *self = NULL; + std::atomic calls(0); + std::atomic returned(false); + trace_client::Logger closing = [&](const std::string &) { + if (++calls > 1) return; + self->close(std::numeric_limits::infinity()); + returned = true; + }; + trace_client::TraceClient client(server.baseUrl(), "MyGame", "1.2.3", "k", true, closing); + self = &client; + client.report("startup"); + client.report("second"); + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + while (!returned && secondsSince(start) < 10) std::this_thread::sleep_for(std::chrono::milliseconds(10)); + CHECK(returned); + CHECK(!client.isEnabled()); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + CHECK_EQ(1, calls.load()); + })); +} + +static void aLoggerMayCloseFromTheReportingThread() { + // Here the sender is mid-request: abandoned and killed, it has a failed + // delivery to mention, which the closed client's logger must not hear. + CHECK(finishesWithin(30, [] { + StubServer server; + server.hold(); + trace_client::TraceClient *self = NULL; + std::atomic calls(0); + trace_client::Logger closing = [&](const std::string &) { + ++calls; + self->close(std::numeric_limits::infinity()); + }; + trace_client::TraceClient client(server.baseUrl(), "MyGame", "1.2.3", "k", true, closing); + self = &client; + for (std::size_t i = 0; i < trace_client::QUEUE_CAPACITY + 2; ++i) client.report("flood"); + CHECK(!client.isEnabled()); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + CHECK_EQ(1, calls.load()); + server.release(); + })); +} + static void missingCurlIsAnUnreachableServer() { #if defined(TRACE_CLIENT_POSIX) StubServer server; @@ -1483,6 +1608,10 @@ int main() { {"theLoggerIsNeverCalledOnceCloseReturns", theLoggerIsNeverCalledOnceCloseReturns}, {"aFailedDeliveryIsLoggedOnTheClientsOwnThread", aFailedDeliveryIsLoggedOnTheClientsOwnThread}, {"aFullQueueIsLoggedOnTheThreadThatCalledReport", aFullQueueIsLoggedOnTheThreadThatCalledReport}, + {"aLoggerMayReportWhileTheQueueIsFull", aLoggerMayReportWhileTheQueueIsFull}, + {"aLoggerMayFillTheQueueFromTheSendingThread", aLoggerMayFillTheQueueFromTheSendingThread}, + {"aLoggerMayCloseFromTheSendingThread", aLoggerMayCloseFromTheSendingThread}, + {"aLoggerMayCloseFromTheReportingThread", aLoggerMayCloseFromTheReportingThread}, {"missingCurlIsAnUnreachableServer", missingCurlIsAnUnreachableServer}, {"disabledClientSendsNothing", disabledClientSendsNothing}, {"disabledReasonIsEmptyWhenTheClientReports", disabledReasonIsEmptyWhenTheClientReports}, diff --git a/trace_client.hpp b/trace_client.hpp index e7a89b5..732c2cb 100644 --- a/trace_client.hpp +++ b/trace_client.hpp @@ -92,7 +92,9 @@ typedef std::map Tags; /** * Where dropped reports are mentioned. Called on the client's own thread for - * failed deliveries and on the reporting thread for a full queue. + * failed deliveries and on the reporting thread for a full queue. It may call + * report(), and a drop that causes is not mentioned to it; see close() for + * calling that. */ typedef std::function Logger; @@ -418,13 +420,14 @@ struct Shared { bool closing; // close() was called: send what is queued, then stop bool abandoned; // close() gave up waiting: stop now, drop the rest bool done; // the sending thread has stopped + bool forgotten; // the logger closed the client: clear it when it returns #if defined(TRACE_CLIENT_POSIX) pid_t child; // the curl process in flight, or 0 #elif defined(_WIN32) && !defined(TRACE_CLIENT_USE_LIBCURL) HANDLE child; // the curl process in flight, or NULL #endif - Shared() : closing(false), abandoned(false), done(false) + Shared() : closing(false), abandoned(false), done(false), forgotten(false) #if defined(TRACE_CLIENT_POSIX) , child(0) #elif defined(_WIN32) && !defined(TRACE_CLIENT_USE_LIBCURL) @@ -432,22 +435,60 @@ struct Shared { #endif {} - // Guards logger alone, so a logger may itself call report(). close() + // Guards logger and forgotten, and is held while the logger runs. close() // clears the logger under it: once close() returns, the program's logger // is never called again, even by a thread that close() gave up on. std::mutex logMutex; + std::thread::id loggerThread; // running the logger now, if any; guarded by mutex + + void setLoggerThread(std::thread::id thread) { + std::lock_guard lock(mutex); + loggerThread = thread; + } + + /** + * Whether the logger is what called this. That thread already holds + * logMutex, and locking it again would hang: whatever a logger calls has + * to check here first. + */ + bool inLogger() { + try { + std::lock_guard lock(mutex); + return loggerThread == std::this_thread::get_id(); + } catch (...) { + return false; + } + } void log(const std::string &message) { + // A report from inside the logger that finds the queue full is + // dropped unmentioned: mentioning it would call the logger from + // inside itself, once per drop if the logger always reports. + if (inLogger()) return; try { std::lock_guard lock(logMutex); - if (logger) logger("[trace] " + message); + if (!logger) return; + setLoggerThread(std::this_thread::get_id()); + try { + logger("[trace] " + message); + } catch (...) { + // a logger that throws must not stop the sending thread + } + setLoggerThread(std::thread::id()); + if (forgotten) logger = Logger(); } catch (...) { - // a logger that throws must not stop the sending thread } } void forgetLogger() { try { + // The logger closed the client. It cannot be destroyed while it + // runs, so log() clears it on the way out; logMutex is this + // thread's already. + if (inLogger()) { + forgotten = true; + return; + } std::lock_guard lock(logMutex); logger = Logger(); } catch (...) { @@ -462,6 +503,14 @@ struct Shared { if (child != NULL) ::TerminateProcess(child, 1); #endif } + + /** Called by close() with the mutex held: stop now, drop the rest. */ + void abandon() { + abandoned = true; + queue.clear(); + killChild(); + changed.notify_all(); + } }; inline long timeoutWholeSeconds() { @@ -1034,19 +1083,35 @@ class TraceClient { * timeoutSeconds in total to be sent first; whatever is still in flight * then is abandoned. A NaN or negative timeoutSeconds waits for nothing; * an infinite one waits until the queue is sent. Safe to call more than - * once, and on a disabled client. + * once, and on a disabled client. Called from inside the Logger it waits + * for nothing, whatever timeoutSeconds says, and abandons what is queued. */ void close(double timeoutSeconds = TIMEOUT_SECONDS) { std::shared_ptr shared; + bool insideLogger = false; try { std::lock_guard lock(sharedMutex_); shared.swap(shared_); + insideLogger = shared && shared->inLogger(); + // Under the lock: a second close() finds shared_ empty and returns + // at once, and the destructor it may be part of must not find + // thread_ still joinable. + if (insideLogger) thread_.detach(); } catch (...) { return; } if (!shared) return; try { std::unique_lock lock(shared->mutex); + if (insideLogger) { + // The sending thread cannot finish while the logger runs: it + // is either running it or waiting its turn to. Nothing here + // touches this object, which the program may now be destroying. + shared->abandon(); + lock.unlock(); + shared->forgetLogger(); + return; + } shared->closing = true; shared->changed.notify_all(); // NaN and negative bounds wait for nothing. Past about three years @@ -1065,10 +1130,7 @@ class TraceClient { lock.unlock(); thread_.join(); } else { - shared->abandoned = true; - shared->queue.clear(); - shared->killChild(); - shared->changed.notify_all(); + shared->abandon(); lock.unlock(); shared->forgetLogger(); // The thread holds its own reference to the shared state, so @@ -1076,7 +1138,7 @@ class TraceClient { thread_.detach(); } } catch (...) { - if (thread_.joinable()) thread_.detach(); + if (!insideLogger && thread_.joinable()) thread_.detach(); } } From 318611d4cc5f5ec7687c649bb09066ffb553570a Mon Sep 17 00:00:00 2001 From: Daniel McCoy Stephenson Date: Wed, 4 Nov 2026 22:01:16 -0700 Subject: [PATCH 2/2] Say why close() does not wait inside the Logger On the reporting thread the sending thread is not always stuck, only possibly so; "cannot finish" overstated it. Co-Authored-By: Claude Opus 5.5 --- trace_client.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trace_client.hpp b/trace_client.hpp index 732c2cb..f397c73 100644 --- a/trace_client.hpp +++ b/trace_client.hpp @@ -1104,9 +1104,9 @@ class TraceClient { try { std::unique_lock lock(shared->mutex); if (insideLogger) { - // The sending thread cannot finish while the logger runs: it - // is either running it or waiting its turn to. Nothing here - // touches this object, which the program may now be destroying. + // Waiting could never end: the sending thread may be running + // the logger, or waiting its turn to. Nothing here touches + // this object, which the program may now be destroying. shared->abandon(); lock.unlock(); shared->forgetLogger();