From f2dc11a388b98742392adcc28963d35a01a0c81a Mon Sep 17 00:00:00 2001 From: yluoc Date: Sat, 17 Oct 2026 21:08:04 -0400 Subject: [PATCH 01/11] feat(ws): add configurable heartbeats, idle reconnect and sports WS URL --- cmake/PolymarketTests.cmake | 5 + include/polymarket/environment.hpp | 1 + include/polymarket/types.hpp | 1 + include/polymarket/websocket_client.hpp | 10 ++ src/environment.cpp | 3 + src/websocket_client_resilience.cpp | 59 +++++-- src/websocket_client_state.hpp | 4 + src/websocket_client_transport.cpp | 9 ++ src/websocket_resilience.cpp | 16 ++ src/websocket_resilience.hpp | 4 + tests/test_environment.cpp | 8 +- tests/test_websocket_heartbeat.cpp | 199 ++++++++++++++++++++++++ 12 files changed, 302 insertions(+), 17 deletions(-) create mode 100644 tests/test_websocket_heartbeat.cpp diff --git a/cmake/PolymarketTests.cmake b/cmake/PolymarketTests.cmake index 9fb361a..81814c5 100644 --- a/cmake/PolymarketTests.cmake +++ b/cmake/PolymarketTests.cmake @@ -265,6 +265,11 @@ if(POLYMARKET_CLIENT_BUILD_TESTS) target_link_libraries(test_websocket_resilience PRIVATE polymarket::client) add_test(NAME test_websocket_resilience COMMAND test_websocket_resilience) + add_executable(test_websocket_heartbeat tests/test_websocket_heartbeat.cpp) + target_include_directories(test_websocket_heartbeat PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + target_link_libraries(test_websocket_heartbeat PRIVATE polymarket::client) + add_test(NAME test_websocket_heartbeat COMMAND test_websocket_heartbeat) + add_executable(test_network_routing tests/test_network_routing.cpp) target_link_libraries(test_network_routing PRIVATE polymarket::client) add_test(NAME test_network_routing COMMAND test_network_routing) diff --git a/include/polymarket/environment.hpp b/include/polymarket/environment.hpp index 894fcd5..056154c 100644 --- a/include/polymarket/environment.hpp +++ b/include/polymarket/environment.hpp @@ -26,6 +26,7 @@ namespace polymarket std::string data_url; std::string relayer_url; std::string rtds_ws_url; + std::string sports_ws_url; static Environment production(); static Environment preproduction(); diff --git a/include/polymarket/types.hpp b/include/polymarket/types.hpp index 36852a9..b4bdfca 100644 --- a/include/polymarket/types.hpp +++ b/include/polymarket/types.hpp @@ -264,6 +264,7 @@ namespace polymarket std::string clob_ws_url = "wss://ws-subscriptions-clob.polymarket.com/ws/market"; std::string gamma_api_url = "https://gamma-api.polymarket.com"; std::string rtds_ws_url = "wss://ws-live-data.polymarket.com"; + std::string sports_ws_url = "wss://sports-api.polymarket.com/ws"; std::string clob_user_ws_url = "wss://ws-subscriptions-clob.polymarket.com/ws/user"; // Trading parameters diff --git a/include/polymarket/websocket_client.hpp b/include/polymarket/websocket_client.hpp index 2f9ff7d..48996cf 100644 --- a/include/polymarket/websocket_client.hpp +++ b/include/polymarket/websocket_client.hpp @@ -38,6 +38,15 @@ namespace polymarket uint32_t min_backoff_ms{250}; uint32_t max_backoff_ms{10000}; int ping_interval_ms{10000}; + // Text sent every ping_interval_ms; empty sends no client pings. + std::string ping_message{"PING"}; + // A text frame equal to server_ping_message is answered with + // server_ping_reply and not delivered. Both set, or both empty. + std::string server_ping_message; + std::string server_ping_reply; + // Reconnects, reporting a stream gap, when no frame arrives for this + // long while connected; 0 disables. + uint32_t idle_timeout_ms{0}; std::size_t message_queue_limit{1024}; // Route for this connection; empty fields fall back to the default // route in . A set route sends the connection @@ -55,6 +64,7 @@ namespace polymarket uint64_t parse_errors{0}; uint64_t last_message_time_ns{0}; uint64_t callback_errors{0}; + uint64_t idle_timeouts{0}; }; struct TypedWebSocketMessage diff --git a/src/environment.cpp b/src/environment.cpp index a545d43..739e798 100644 --- a/src/environment.cpp +++ b/src/environment.cpp @@ -38,6 +38,7 @@ namespace polymarket env.data_url = "https://data-api.polymarket.com"; env.relayer_url = "https://relayer-v2.polymarket.com"; env.rtds_ws_url = "wss://ws-live-data.polymarket.com"; + env.sports_ws_url = "wss://sports-api.polymarket.com/ws"; return env; } @@ -75,6 +76,7 @@ namespace polymarket {"clob_market_ws_url", &clob_market_ws_url}, {"clob_user_ws_url", &clob_user_ws_url}, {"rtds_ws_url", &rtds_ws_url}, + {"sports_ws_url", &sports_ws_url}, }; for (const auto &[field, url] : ws_urls) require_url(field, *url, "wss://", "ws://"); @@ -91,6 +93,7 @@ namespace polymarket config.clob_user_ws_url = environment.clob_user_ws_url; config.gamma_api_url = environment.gamma_url; config.rtds_ws_url = environment.rtds_ws_url; + config.sports_ws_url = environment.sports_ws_url; return config; } diff --git a/src/websocket_client_resilience.cpp b/src/websocket_client_resilience.cpp index 18e9201..1f542a3 100644 --- a/src/websocket_client_resilience.cpp +++ b/src/websocket_client_resilience.cpp @@ -3,6 +3,7 @@ #include "websocket_resilience.hpp" #include +#include #include #include @@ -29,6 +30,9 @@ namespace polymarket::detail throw std::invalid_argument("WebSocket ping interval must be positive"); if (options.message_queue_limit == 0) throw std::invalid_argument("WebSocket message queue limit must be positive"); + if (options.server_ping_message.empty() != options.server_ping_reply.empty()) + throw std::invalid_argument( + "WebSocket server ping message and reply must be set together"); } void WebSocketClientState::require_inactive_locked() const @@ -69,8 +73,8 @@ namespace polymarket::detail { const int ping_seconds = static_cast( (static_cast(options_.ping_interval_ms) + 999) / 1000); - ws_.setPingMessage("PING", ix::SendMessageKind::Text); - ws_.setPingInterval(ping_seconds); + ws_.setPingMessage(options_.ping_message, ix::SendMessageKind::Text); + ws_.setPingInterval(options_.ping_message.empty() ? -1 : ping_seconds); ws_.setMinWaitBetweenReconnectionRetries(options_.min_backoff_ms); ws_.setMaxWaitBetweenReconnectionRetries(options_.max_backoff_ms); if (options_.reconnect_enabled) @@ -134,13 +138,9 @@ namespace polymarket::detail WebSocketStats WebSocketClientState::stats() const { - return {messages_received_.load(), - bytes_received_.load(), - reconnects_.load(), - message_queue_->dropped(), - parse_errors_.load(), - last_message_time_ns_.load(), - callback_errors_.load()}; + return {messages_received_.load(), bytes_received_.load(), reconnects_.load(), + message_queue_->dropped(), parse_errors_.load(), last_message_time_ns_.load(), + callback_errors_.load(), idle_timeouts_.load()}; } void WebSocketClientState::start_message_worker() @@ -157,14 +157,41 @@ namespace polymarket::detail worker_running_.store(true); auto self = shared_from_this(); - message_worker_ = std::thread([self] - { - while (self->worker_running_.load()) + message_worker_ = std::thread( + [self] { - auto message = self->message_queue_->pop(); - if (!message) break; - self->dispatch_message(message->payload, message->generation); - } }); + const auto idle_timeout = std::chrono::milliseconds(self->options_.idle_timeout_ms); + const auto check_every = std::clamp(idle_timeout / 4, std::chrono::milliseconds(10), + std::chrono::milliseconds(1000)); + while (self->worker_running_.load()) + { + auto message = idle_timeout.count() > 0 + ? self->message_queue_->pop_for(check_every) + : self->message_queue_->pop(); + if (message) + { + self->dispatch_message(message->payload, message->generation); + continue; + } + if (self->message_queue_->closed()) break; + self->check_idle(idle_timeout); + } + }); + } + + void WebSocketClientState::check_idle(std::chrono::milliseconds idle_timeout) + { + if (state_.load() != WsState::CONNECTED) return; + const auto last_activity = last_activity_ns_.load(); + const auto now = now_ns(); + if (now < last_activity || + now - last_activity < + static_cast( + std::chrono::duration_cast(idle_timeout).count())) + return; + last_activity_ns_.store(now_ns()); + idle_timeouts_++; + request_resnapshot("no WebSocket frame within the idle timeout; reconnecting"); } void WebSocketClientState::request_message_worker_stop() diff --git a/src/websocket_client_state.hpp b/src/websocket_client_state.hpp index 17af311..6b1cf94 100644 --- a/src/websocket_client_state.hpp +++ b/src/websocket_client_state.hpp @@ -3,6 +3,7 @@ #include "polymarket/websocket_client.hpp" #include +#include #include #include #include @@ -100,6 +101,7 @@ namespace polymarket::detail void dispatch_message(const std::string &message, uint64_t generation); void mark_stream_gap(); bool restore_subscriptions(); + void check_idle(std::chrono::milliseconds idle_timeout); void report_callback_failure(const char *name, const char *reason) noexcept; bool invoke_error_callback(const std::string &message) noexcept; @@ -182,6 +184,8 @@ namespace polymarket::detail std::atomic parse_errors_{0}; std::atomic callback_errors_{0}; std::atomic last_message_time_ns_{0}; + std::atomic last_activity_ns_{0}; + std::atomic idle_timeouts_{0}; std::atomic stream_generation_{0}; }; } diff --git a/src/websocket_client_transport.cpp b/src/websocket_client_transport.cpp index 9220df4..65b7d72 100644 --- a/src/websocket_client_transport.cpp +++ b/src/websocket_client_transport.cpp @@ -158,6 +158,13 @@ namespace polymarket::detail handle_error(message); break; case ix::WebSocketMessageType::Message: + last_activity_ns_.store(now_ns()); + if (!options_.server_ping_message.empty() && + message->str == options_.server_ping_message) + { + ws_.send(options_.server_ping_reply); + break; + } messages_received_++; bytes_received_ += message->str.size(); last_message_time_ns_.store(now_ns()); @@ -166,12 +173,14 @@ namespace polymarket::detail case ix::WebSocketMessageType::Ping: case ix::WebSocketMessageType::Pong: case ix::WebSocketMessageType::Fragment: + last_activity_ns_.store(now_ns()); break; } } void WebSocketClientState::handle_open() { + last_activity_ns_.store(now_ns()); mark_stream_gap(); { std::lock_guard lock(lifecycle_mutex_); diff --git a/src/websocket_resilience.cpp b/src/websocket_resilience.cpp index af9146e..2f1c82c 100644 --- a/src/websocket_resilience.cpp +++ b/src/websocket_resilience.cpp @@ -44,6 +44,22 @@ namespace polymarket::detail return message; } + std::optional BoundedMessageQueue::pop_for(std::chrono::milliseconds timeout) + { + std::unique_lock lock(mutex_); + cv_.wait_for(lock, timeout, [this]() { return closed_ || !messages_.empty(); }); + if (messages_.empty()) return std::nullopt; + auto message = std::move(messages_.front()); + messages_.pop_front(); + return message; + } + + bool BoundedMessageQueue::closed() const + { + std::lock_guard lock(mutex_); + return closed_; + } + std::size_t BoundedMessageQueue::discard_pending() { std::lock_guard lock(mutex_); diff --git a/src/websocket_resilience.hpp b/src/websocket_resilience.hpp index 3e1eb09..802f571 100644 --- a/src/websocket_resilience.hpp +++ b/src/websocket_resilience.hpp @@ -2,6 +2,7 @@ #include "polymarket/websocket_client.hpp" #include "polymarket/types.hpp" +#include #include #include #include @@ -52,6 +53,9 @@ namespace polymarket::detail QueuePushResult push(QueuedMessage message); std::optional pop(); + // Empty when `timeout` passes first or the queue is closed. + std::optional pop_for(std::chrono::milliseconds timeout); + bool closed() const; std::size_t discard_pending(); void record_drop(); void close(); diff --git a/tests/test_environment.cpp b/tests/test_environment.cpp index 28b6eb2..f0cdaaa 100644 --- a/tests/test_environment.cpp +++ b/tests/test_environment.cpp @@ -37,6 +37,7 @@ namespace expect_equal("production data", env.data_url, "https://data-api.polymarket.com"); expect_equal("production relayer", env.relayer_url, "https://relayer-v2.polymarket.com"); expect_equal("production rtds", env.rtds_ws_url, "wss://ws-live-data.polymarket.com"); + expect_equal("production sports", env.sports_ws_url, "wss://sports-api.polymarket.com/ws"); env.validate(); } @@ -68,7 +69,8 @@ namespace restored.contracts.collateral_token == prod.contracts.collateral_token && restored.clob_market_ws_url == prod.clob_market_ws_url && restored.clob_user_ws_url == prod.clob_user_ws_url && - restored.rtds_ws_url == prod.rtds_ws_url, + restored.rtds_ws_url == prod.rtds_ws_url && + restored.sports_ws_url == prod.sports_ws_url, "preproduction must match production outside the four REST hosts"); env.validate(); } @@ -111,6 +113,7 @@ namespace mutated([](Environment &e) { e.data_url = "wss://x"; })); expect_invalid("http scheme for ws", mutated([](Environment &e) { e.clob_user_ws_url = "https://x"; })); + expect_invalid("empty sports ws", mutated([](Environment &e) { e.sports_ws_url.clear(); })); expect_invalid("missing scheme", mutated([](Environment &e) { e.rpc_url = "polygon.drpc.org"; })); expect_invalid("bad contract", @@ -130,6 +133,7 @@ namespace expect_equal("default user ws", config.clob_user_ws_url, from_preset.clob_user_ws_url); expect_equal("default gamma", config.gamma_api_url, from_preset.gamma_api_url); expect_equal("default rtds", config.rtds_ws_url, from_preset.rtds_ws_url); + expect_equal("default sports", config.sports_ws_url, from_preset.sports_ws_url); const PositionClientConfig position; expect_equal("default position gamma", position.gamma_api_url, production.gamma_url); @@ -147,6 +151,7 @@ namespace env.clob_market_ws_url = "ws://127.0.0.1:9001/ws/market"; env.clob_user_ws_url = "ws://127.0.0.1:9001/ws/user"; env.rtds_ws_url = "ws://127.0.0.1:9002"; + env.sports_ws_url = "ws://127.0.0.1:9003/ws"; const auto config = Config::for_environment(env); const Config defaults; expect_equal("config clob rest", config.clob_rest_url, env.clob_url); @@ -154,6 +159,7 @@ namespace expect_equal("config user ws", config.clob_user_ws_url, env.clob_user_ws_url); expect_equal("config gamma", config.gamma_api_url, env.gamma_url); expect_equal("config rtds", config.rtds_ws_url, env.rtds_ws_url); + expect_equal("config sports", config.sports_ws_url, env.sports_ws_url); check(config.http_timeout_ms == defaults.http_timeout_ms && config.ws_ping_interval_ms == defaults.ws_ping_interval_ms && config.max_markets == defaults.max_markets && diff --git a/tests/test_websocket_heartbeat.cpp b/tests/test_websocket_heartbeat.cpp new file mode 100644 index 0000000..2420bd2 --- /dev/null +++ b/tests/test_websocket_heartbeat.cpp @@ -0,0 +1,199 @@ +#include "polymarket/websocket_client.hpp" +#include "websocket_resilience.hpp" +#include "websocket_test_server.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + using namespace std::chrono_literals; + using websocket_test::LocalWebSocketServer; + + bool check(bool condition, const std::string &message) + { + if (!condition) std::cerr << "failed: " << message << '\n'; + return condition; + } + + template + bool wait_until(Predicate predicate, std::chrono::milliseconds timeout) + { + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) + { + if (predicate()) return true; + std::this_thread::sleep_for(2ms); + } + return predicate(); + } + + polymarket::WebSocketOptions quiet_options() + { + polymarket::WebSocketOptions options; + options.ping_message.clear(); + options.min_backoff_ms = 10; + options.max_backoff_ms = 50; + return options; + } + + bool test_timed_pop() + { + polymarket::detail::BoundedMessageQueue queue(4); + const auto empty = queue.pop_for(20ms); + queue.push({"one", 1}); + const auto first = queue.pop_for(20ms); + queue.close(); + const auto closed = queue.pop_for(1000ms); + return check(!empty && first && first->payload == "one" && !closed && queue.closed(), + "a timed pop must time out, deliver and stop on close"); + } + + bool test_option_validation() + { + polymarket::WebSocketClient client; + auto options = quiet_options(); + options.server_ping_message = "ping"; + bool rejected = false; + try + { + client.configure(options); + } + catch (const std::invalid_argument &) + { + rejected = true; + } + options.server_ping_message.clear(); + options.server_ping_reply = "pong"; + bool reply_rejected = false; + try + { + client.configure(options); + } + catch (const std::invalid_argument &) + { + reply_rejected = true; + } + return check(rejected && reply_rejected, + "a server ping needs both the message and the reply"); + } + + bool test_server_ping_reply() + { + LocalWebSocketServer server; + polymarket::WebSocketClient client; + auto options = quiet_options(); + options.server_ping_message = "ping"; + options.server_ping_reply = "pong"; + client.configure(options); + client.set_url(server.url()); + std::mutex mutex; + std::vector delivered; + client.on_message( + [&](const std::string &message) + { + std::lock_guard lock(mutex); + delivered.push_back(message); + }); + client.connect(); + const bool connected = client.wait_until_connected(2s); + server.send_to_clients("ping"); + server.send_to_clients("{\"gameId\":1}"); + const bool ponged = server.wait_for_message_count("pong", 1, 2s); + const bool received = wait_until( + [&] + { + std::lock_guard lock(mutex); + return !delivered.empty(); + }, + 2s); + const bool no_client_ping = !server.wait_for_message_count("PING", 1, 1500ms); + client.disconnect(); + std::lock_guard lock(mutex); + return check(connected && ponged, "a server ping must be answered with the reply") && + check(received && delivered.size() == 1 && delivered[0] == "{\"gameId\":1}", + "a server ping must not be delivered as a message") && + check(no_client_ping, "an empty ping message must send no client pings") && + check(client.stats().messages_received == 1, + "a server ping must not count as a message"); + } + + bool test_custom_ping_message() + { + LocalWebSocketServer server; + polymarket::WebSocketClient client; + auto options = quiet_options(); + options.ping_message = "hb"; + options.ping_interval_ms = 1000; + client.configure(options); + client.set_url(server.url()); + client.connect(); + const bool connected = client.wait_until_connected(2s); + const bool pinged = server.wait_for_message_count("hb", 1, 3s); + client.disconnect(); + return check(connected && pinged, "the client must send its configured ping text"); + } + + bool test_idle_timeout_reconnects() + { + LocalWebSocketServer server; + polymarket::WebSocketClient client; + auto options = quiet_options(); + options.idle_timeout_ms = 200; + client.configure(options); + client.set_url(server.url()); + std::atomic gaps{0}; + client.on_stream_gap([&](uint64_t) { gaps++; }); + client.track_subscription("{\"action\":\"subscribe\"}"); + client.connect(); + const bool reconnected = server.wait_for_connections(2, 3s); + const bool restored = server.wait_for_message_count("{\"action\":\"subscribe\"}", 2, 3s); + const bool connected_again = client.wait_until_connected(2s); + const auto stats = client.stats(); + client.disconnect(); + return check(reconnected && restored && connected_again, + "an idle connection must reconnect and restore subscriptions") && + check(stats.idle_timeouts >= 1 && stats.reconnects >= 1 && gaps.load() >= 2, + "an idle timeout must be counted and reported as a gap"); + } + + bool test_activity_keeps_connection() + { + LocalWebSocketServer server; + polymarket::WebSocketClient client; + auto options = quiet_options(); + options.idle_timeout_ms = 400; + options.server_ping_message = "ping"; + options.server_ping_reply = "pong"; + client.configure(options); + client.set_url(server.url()); + client.connect(); + const bool connected = client.wait_until_connected(2s); + for (int i = 0; i < 12; ++i) + { + server.send_to_clients("ping"); + std::this_thread::sleep_for(100ms); + } + const bool still_one = !server.wait_for_connections(2, 200ms); + const auto stats = client.stats(); + client.disconnect(); + return check(connected && still_one && stats.idle_timeouts == 0, + "server pings must keep an idle timeout from firing"); + } +} // namespace + +int main() +{ + const bool passed = test_timed_pop() && test_option_validation() && test_server_ping_reply() && + test_custom_ping_message() && test_idle_timeout_reconnects() && + test_activity_keeps_connection(); + if (!passed) return 1; + std::cout << "test_websocket_heartbeat passed\n"; + return 0; +} From f189c5b7213eed0d06f0893dbb12e82298f20f12 Mon Sep 17 00:00:00 2001 From: yluoc Date: Sat, 17 Oct 2026 21:18:39 -0400 Subject: [PATCH 02/11] feat(sports): add Sports WebSocket stream --- CMakeLists.txt | 3 + cmake/PolymarketTests.cmake | 9 + include/polymarket/sports_stream.hpp | 85 +++++++++ include/polymarket/types.hpp | 4 +- src/public_stream.cpp | 174 ++++++++++++++++++ src/public_stream.hpp | 96 ++++++++++ src/sports_parsing.cpp | 117 ++++++++++++ src/sports_parsing.hpp | 21 +++ src/sports_stream.cpp | 166 +++++++++++++++++ .../corpus/websocket_messages/sports_result | 1 + tests/fuzz/fuzz_websocket_messages.cpp | 8 +- tests/sports_fixtures.hpp | 17 ++ tests/test_sports_parsing.cpp | 111 +++++++++++ tests/test_sports_stream.cpp | 158 ++++++++++++++++ 14 files changed, 967 insertions(+), 3 deletions(-) create mode 100644 include/polymarket/sports_stream.hpp create mode 100644 src/public_stream.cpp create mode 100644 src/public_stream.hpp create mode 100644 src/sports_parsing.cpp create mode 100644 src/sports_parsing.hpp create mode 100644 src/sports_stream.cpp create mode 100644 tests/fuzz/corpus/websocket_messages/sports_result create mode 100644 tests/sports_fixtures.hpp create mode 100644 tests/test_sports_parsing.cpp create mode 100644 tests/test_sports_stream.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9cd8cc6..ee4b29c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,9 @@ set(POLYMARKET_CLIENT_SOURCES src/user_stream_runtime.cpp src/user_stream_runtime_events.cpp src/user_stream_subscription.cpp + src/public_stream.cpp + src/sports_parsing.cpp + src/sports_stream.cpp src/order_amounts.cpp src/clob_order_execution.cpp src/clob_order_submission.cpp diff --git a/cmake/PolymarketTests.cmake b/cmake/PolymarketTests.cmake index 81814c5..be582a0 100644 --- a/cmake/PolymarketTests.cmake +++ b/cmake/PolymarketTests.cmake @@ -270,6 +270,15 @@ if(POLYMARKET_CLIENT_BUILD_TESTS) target_link_libraries(test_websocket_heartbeat PRIVATE polymarket::client) add_test(NAME test_websocket_heartbeat COMMAND test_websocket_heartbeat) + add_executable(test_sports_parsing tests/test_sports_parsing.cpp) + target_include_directories(test_sports_parsing PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + target_link_libraries(test_sports_parsing PRIVATE polymarket::client) + add_test(NAME test_sports_parsing COMMAND test_sports_parsing) + + add_executable(test_sports_stream tests/test_sports_stream.cpp) + target_link_libraries(test_sports_stream PRIVATE polymarket::client) + add_test(NAME test_sports_stream COMMAND test_sports_stream) + add_executable(test_network_routing tests/test_network_routing.cpp) target_link_libraries(test_network_routing PRIVATE polymarket::client) add_test(NAME test_network_routing COMMAND test_network_routing) diff --git a/include/polymarket/sports_stream.hpp b/include/polymarket/sports_stream.hpp new file mode 100644 index 0000000..a3cbeac --- /dev/null +++ b/include/polymarket/sports_stream.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include "polymarket/types.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace polymarket +{ + namespace detail + { + class SportsStreamRuntime; + } + + // One live game update from the Sports WebSocket, following the official + // py-sdk model. Absent or empty text stays empty; `raw` holds the whole + // object as received. + struct SportsGameResult + { + std::int64_t game_id{0}; + std::string sportradar_game_id; + std::string slug; + std::string league_abbreviation; + std::string home_team; + std::string away_team; + std::string status; + bool live{false}; + bool ended{false}; + std::string score; + std::string period; + std::string elapsed; + std::string turn; + std::optional finished_at_ms; // Unix milliseconds + nlohmann::json raw; + }; + + using SportsResultCallback = std::function; + // Fired whenever updates may have been missed: every connect, disconnect, + // queue overflow or idle reconnect. The server does not replay them. + using SportsStreamGapCallback = std::function; + // Fired once the connection is back after a gap; refresh game state from + // GammaClient here if it matters. + using SportsStreamRecoveredCallback = std::function; + using SportsStreamErrorCallback = std::function; + + // Public Sports stream (Config::sports_ws_url). Every connection receives + // every game update; there are no filters. The stream answers the + // server's heartbeat and reconnects when no frame arrives for 30 seconds. + // Callbacks run on the stream's own threads; a malformed update is + // counted in dropped_events() and skipped. + class SportsStream + { + public: + explicit SportsStream(const Config &config = Config{}); + ~SportsStream(); + + SportsStream(const SportsStream &) = delete; + SportsStream &operator=(const SportsStream &) = delete; + + void on_result(SportsResultCallback callback); + void on_stream_gap(SportsStreamGapCallback callback); + void on_stream_recovered(SportsStreamRecoveredCallback callback); + void on_error(SportsStreamErrorCallback callback); + + // Waits up to Config::ws_connect_timeout_ms for the connection. + bool connect(); + void disconnect(); + bool is_connected() const; + // Blocks until stop(). + void run(); + void stop(); + + std::uint64_t result_events() const; + std::uint64_t dropped_events() const; + + private: + std::shared_ptr runtime_; + }; + +} // namespace polymarket diff --git a/include/polymarket/types.hpp b/include/polymarket/types.hpp index b4bdfca..9e44827 100644 --- a/include/polymarket/types.hpp +++ b/include/polymarket/types.hpp @@ -284,8 +284,8 @@ namespace polymarket "btc", "eth", "xrp", "sol", "doge", "bnb", "ada", "avax", "matic", "link", "dot", "ltc"}; - // CLOB, Gamma and RTDS endpoints from `environment`, which is validated - // first; every other field keeps its default. + // CLOB, Gamma, RTDS and Sports endpoints from `environment`, which is + // validated first; every other field keeps its default. static Config for_environment(const Environment &environment); }; diff --git a/src/public_stream.cpp b/src/public_stream.cpp new file mode 100644 index 0000000..c9673e8 --- /dev/null +++ b/src/public_stream.cpp @@ -0,0 +1,174 @@ +#include "public_stream.hpp" + +#include +#include + +namespace polymarket::detail +{ + std::shared_ptr PublicStream::create(const std::string &url, + const WebSocketOptions &options, + std::chrono::milliseconds connect_timeout, + MessageHandler handler) + { + auto stream = std::shared_ptr( + new PublicStream(url, options, connect_timeout, std::move(handler))); + stream->bind_websocket_callbacks(); + return stream; + } + + PublicStream::PublicStream(const std::string &url, const WebSocketOptions &options, + std::chrono::milliseconds connect_timeout, MessageHandler handler) + : connect_timeout_(connect_timeout), handler_(std::move(handler)) + { + if (url.empty()) throw std::invalid_argument("stream URL must not be empty"); + if (connect_timeout.count() <= 0) + throw std::invalid_argument("ws_connect_timeout_ms must be positive"); + websocket_.configure(options); + websocket_.set_url(url); + } + + PublicStream::~PublicStream() + { + shutdown(); + } + + void PublicStream::bind_websocket_callbacks() + { + std::weak_ptr weak = shared_from_this(); + websocket_.on_sequenced_message( + [weak](const std::string &message, std::uint64_t generation) + { + if (auto stream = weak.lock()) stream->handle_message(message, generation); + }); + websocket_.on_stream_gap( + [weak](std::uint64_t generation) + { + if (auto stream = weak.lock()) stream->handle_gap(generation); + }); + websocket_.on_connect( + [weak] + { + if (auto stream = weak.lock()) stream->handle_connect(); + }); + websocket_.on_error( + [weak](const std::string &error) + { + if (auto stream = weak.lock()) stream->handle_error(error); + }); + } + + void PublicStream::shutdown() + { + if (shutdown_started_.exchange(true)) return; + owner_active_.store(false, std::memory_order_release); + stream_active_.store(false, std::memory_order_release); + { + std::lock_guard lock(callback_update_mutex_); + std::shared_ptr empty = + std::make_shared(); + std::atomic_store_explicit(&callbacks_, std::move(empty), std::memory_order_release); + } + websocket_.stop(); + } + + void PublicStream::on_stream_gap(std::function callback) + { + update_callbacks([callback = std::move(callback)](auto &callbacks) mutable + { callbacks.gap = std::move(callback); }); + } + + void PublicStream::on_stream_recovered(std::function callback) + { + update_callbacks([callback = std::move(callback)](auto &callbacks) mutable + { callbacks.recovered = std::move(callback); }); + } + + void PublicStream::on_error(std::function callback) + { + update_callbacks([callback = std::move(callback)](auto &callbacks) mutable + { callbacks.error = std::move(callback); }); + } + + bool PublicStream::active() const + { + return owner_active_.load(std::memory_order_acquire) && + stream_active_.load(std::memory_order_acquire); + } + + void PublicStream::handle_message(const std::string &message, std::uint64_t generation) + { + if (!active() || generation != websocket_.stream_generation()) return; + handler_(message); + } + + void PublicStream::handle_gap(std::uint64_t generation) + { + if (!active()) return; + recovery_generation_.store(generation, std::memory_order_release); + const auto callbacks = callbacks_snapshot(); + if (callbacks->gap) callbacks->gap(); + } + + void PublicStream::handle_connect() + { + if (!active()) return; + const auto pending = recovery_generation_.exchange(0, std::memory_order_acq_rel); + if (pending == 0 || pending != websocket_.stream_generation()) return; + const auto callbacks = callbacks_snapshot(); + if (callbacks->recovered) callbacks->recovered(); + } + + void PublicStream::handle_error(const std::string &error) + { + if (!active()) return; + const auto callbacks = callbacks_snapshot(); + if (callbacks->error) callbacks->error(error); + } + + bool PublicStream::connect() + { + if (!owner_active_.load(std::memory_order_acquire)) return false; + stream_active_.store(true, std::memory_order_release); + const bool connected = + websocket_.connect() && websocket_.wait_until_connected(connect_timeout_); + if (!connected) disconnect(); + return connected; + } + + void PublicStream::disconnect() + { + stream_active_.store(false, std::memory_order_release); + websocket_.disconnect(); + } + + bool PublicStream::is_connected() const + { + return owner_active_.load(std::memory_order_acquire) && websocket_.is_connected(); + } + + void PublicStream::run() + { + if (owner_active_.load(std::memory_order_acquire)) websocket_.run(); + } + + void PublicStream::stop() + { + stream_active_.store(false, std::memory_order_release); + websocket_.stop(); + } + + void PublicStream::replace_subscriptions(std::vector messages) + { + websocket_.replace_subscriptions(std::move(messages)); + } + + bool PublicStream::send(const std::string &message) + { + return websocket_.send(message); + } + + void PublicStream::request_resnapshot(const std::string &reason) + { + websocket_.request_resnapshot(reason); + } +} // namespace polymarket::detail diff --git a/src/public_stream.hpp b/src/public_stream.hpp new file mode 100644 index 0000000..5107bcc --- /dev/null +++ b/src/public_stream.hpp @@ -0,0 +1,96 @@ +#pragma once + +#include "polymarket/websocket_client.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +// A public WebSocket feed shared by SportsStream and RtdsStream: it owns the +// connection, reports gaps and recovery, and hands each current message to +// the owner's handler on the WebSocket message thread. +namespace polymarket::detail +{ + struct PublicStreamCallbacks + { + std::function gap; + std::function recovered; + std::function error; + }; + + class PublicStream final : public std::enable_shared_from_this + { + public: + using MessageHandler = std::function; + + static std::shared_ptr create(const std::string &url, + const WebSocketOptions &options, + std::chrono::milliseconds connect_timeout, + MessageHandler handler); + ~PublicStream(); + + PublicStream(const PublicStream &) = delete; + PublicStream &operator=(const PublicStream &) = delete; + + void shutdown(); + void on_stream_gap(std::function callback); + void on_stream_recovered(std::function callback); + void on_error(std::function callback); + + bool connect(); + void disconnect(); + bool is_connected() const; + void run(); + void stop(); + + // Subscription frames sent again after every reconnect. + void replace_subscriptions(std::vector messages); + bool send(const std::string &message); + void request_resnapshot(const std::string &reason); + + private: + PublicStream(const std::string &url, const WebSocketOptions &options, + std::chrono::milliseconds connect_timeout, MessageHandler handler); + void bind_websocket_callbacks(); + void handle_message(const std::string &message, std::uint64_t generation); + void handle_gap(std::uint64_t generation); + void handle_connect(); + void handle_error(const std::string &error); + bool active() const; + + template void update_callbacks(Update &&update) + { + std::lock_guard lock(callback_update_mutex_); + if (!owner_active_.load(std::memory_order_acquire)) return; + auto next = std::make_shared(*callbacks_snapshot()); + update(*next); + std::shared_ptr immutable = std::move(next); + std::atomic_store_explicit(&callbacks_, std::move(immutable), + std::memory_order_release); + } + + std::shared_ptr callbacks_snapshot() const + { + return std::atomic_load_explicit(&callbacks_, std::memory_order_acquire); + } + + WebSocketClient websocket_; + std::chrono::milliseconds connect_timeout_; + MessageHandler handler_; + + mutable std::mutex callback_update_mutex_; + std::shared_ptr callbacks_{ + std::make_shared()}; + + std::atomic owner_active_{true}; + std::atomic stream_active_{false}; + std::atomic shutdown_started_{false}; + // WebSocket generation of the gap awaiting recovery; 0 when none. + std::atomic recovery_generation_{0}; + }; +} // namespace polymarket::detail diff --git a/src/sports_parsing.cpp b/src/sports_parsing.cpp new file mode 100644 index 0000000..b5ff20d --- /dev/null +++ b/src/sports_parsing.cpp @@ -0,0 +1,117 @@ +#include "sports_parsing.hpp" + +#include "market_time.hpp" + +#include +#include +#include + +namespace polymarket::detail +{ + namespace + { + constexpr std::int64_t max_epoch_ms = 253402300799999; // 9999-12-31T23:59:59.999Z + + bool all_digits(const std::string &text) + { + return !text.empty() && text.find_first_not_of("0123456789") == std::string::npos; + } + + std::int64_t checked_ms(const char *key, const std::string &digits) + { + if (digits.size() > 15) throw_bad_field(key, "Unix milliseconds"); + const auto ms = std::stoll(digits); + if (ms > max_epoch_ms) throw_bad_field(key, "Unix milliseconds"); + return ms; + } + + std::int64_t iso_ms(const char *key, const std::string &text) + { + std::string utc = text; + std::int64_t offset_minutes = 0; + const auto size = text.size(); + if (size > 6 && (text[size - 6] == '+' || text[size - 6] == '-') && + text[size - 3] == ':') + { + const auto hours = text.substr(size - 5, 2); + const auto minutes = text.substr(size - 2, 2); + if (!all_digits(hours) || !all_digits(minutes)) + throw_bad_field(key, "an ISO 8601 time"); + offset_minutes = std::stoll(hours) * 60 + std::stoll(minutes); + if (text[size - 6] == '-') offset_minutes = -offset_minutes; + utc = text.substr(0, size - 6) + "Z"; + } + else if (text.back() != 'Z') + { + utc = text + "Z"; + } + const auto ms = parse_iso8601_ms(utc); + if (ms == 0) throw_bad_field(key, "an ISO 8601 time"); + return static_cast(ms) - offset_minutes * 60000; + } + + std::int64_t required_integer(const json &item, const char *key) + { + const auto *value = find_field(item, key); + if (value != nullptr && value->is_string()) + { + const auto &text = value->get_ref(); + if (!all_digits(text) || text.size() > 18) throw_bad_field(key, "an integer"); + return std::stoll(text); + } + const auto number = field_integer(item, key); + if (!number) throw_bad_field(key, "an integer"); + return *number; + } + + bool required_bool(const json &item, const char *key) + { + const auto value = field_bool(item, key); + if (!value) throw_bad_field(key, "a boolean"); + return *value; + } + } // namespace + + std::optional stream_epoch_ms_or_iso(const json &object, const char *key) + { + const auto *value = find_field(object, key); + if (value == nullptr) return std::nullopt; + if (value->is_string()) + { + const auto &text = value->get_ref(); + if (text.empty()) return std::nullopt; + return all_digits(text) ? checked_ms(key, text) : iso_ms(key, text); + } + const auto ms = field_integer(object, key); + if (!ms || *ms < 0 || *ms > max_epoch_ms) throw_bad_field(key, "Unix milliseconds"); + return ms; + } + + SportsGameResult parse_sports_result(const json &item) + { + SportsGameResult result; + result.game_id = required_integer(item, "gameId"); + result.sportradar_game_id = field_string(item, "sportradarGameId"); + result.slug = field_string(item, "slug"); + result.league_abbreviation = field_required_string(item, "leagueAbbreviation"); + result.home_team = field_string(item, "homeTeam"); + result.away_team = field_string(item, "awayTeam"); + result.status = field_required_string(item, "status"); + result.live = required_bool(item, "live"); + result.ended = required_bool(item, "ended"); + result.score = field_required_string(item, "score"); + result.period = field_string(item, "period"); + result.elapsed = field_string(item, "elapsed"); + result.turn = field_string(item, "turn"); + result.finished_at_ms = stream_epoch_ms_or_iso(item, "finishedTimestamp"); + if (!result.finished_at_ms) + result.finished_at_ms = stream_epoch_ms_or_iso(item, "finished_timestamp"); + result.raw = item; + return result; + } + + SportsGameResult parse_sports_message(const std::string &message) + { + return parse_sports_result(json::parse(message)); + } +} // namespace polymarket::detail diff --git a/src/sports_parsing.hpp b/src/sports_parsing.hpp new file mode 100644 index 0000000..03aa4f9 --- /dev/null +++ b/src/sports_parsing.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include "json_fields.hpp" +#include "polymarket/sports_stream.hpp" + +#include +#include +#include + +// Sports WebSocket parsing, following py-sdk models/sports_events.py. +// Malformed input throws std::exception. +namespace polymarket::detail +{ + // Unix milliseconds from an integer, a digit string, or an ISO 8601 time + // with Z, a +HH:MM/-HH:MM offset, or no zone (read as UTC); unset for + // null or "". + std::optional stream_epoch_ms_or_iso(const json &object, const char *key); + + SportsGameResult parse_sports_result(const json &item); + SportsGameResult parse_sports_message(const std::string &message); +} // namespace polymarket::detail diff --git a/src/sports_stream.cpp b/src/sports_stream.cpp new file mode 100644 index 0000000..c8b18b4 --- /dev/null +++ b/src/sports_stream.cpp @@ -0,0 +1,166 @@ +#include "polymarket/sports_stream.hpp" + +#include "public_stream.hpp" +#include "sports_parsing.hpp" + +#include +#include +#include +#include + +namespace polymarket +{ + namespace detail + { + namespace + { + constexpr std::uint32_t sports_idle_timeout_ms = 30000; + + WebSocketOptions sports_options() + { + WebSocketOptions options; + options.ping_message.clear(); + options.server_ping_message = "ping"; + options.server_ping_reply = "pong"; + options.idle_timeout_ms = sports_idle_timeout_ms; + return options; + } + } // namespace + + class SportsStreamRuntime final : public std::enable_shared_from_this + { + public: + static std::shared_ptr create(const Config &config) + { + auto runtime = std::shared_ptr(new SportsStreamRuntime()); + std::weak_ptr weak = runtime; + runtime->stream_ = + PublicStream::create(config.sports_ws_url, sports_options(), + std::chrono::milliseconds(config.ws_connect_timeout_ms), + [weak](const std::string &message) + { + if (auto self = weak.lock()) + self->handle_message(message); + }); + return runtime; + } + + void shutdown() + { + { + std::lock_guard lock(callback_mutex_); + result_callback_ = std::make_shared(); + } + stream_->shutdown(); + } + + void on_result(SportsResultCallback callback) + { + std::lock_guard lock(callback_mutex_); + result_callback_ = + std::make_shared(std::move(callback)); + } + + PublicStream &stream() { return *stream_; } + std::uint64_t result_events() const { return result_events_.load(); } + std::uint64_t dropped_events() const { return dropped_events_.load(); } + + private: + SportsStreamRuntime() = default; + + void handle_message(const std::string &message) + { + SportsGameResult result; + try + { + result = parse_sports_message(message); + } + catch (const std::exception &) + { + dropped_events_++; + return; + } + result_events_++; + std::shared_ptr callback; + { + std::lock_guard lock(callback_mutex_); + callback = result_callback_; + } + if (*callback) (*callback)(result); + } + + std::shared_ptr stream_; + std::mutex callback_mutex_; + std::shared_ptr result_callback_{ + std::make_shared()}; + std::atomic result_events_{0}; + std::atomic dropped_events_{0}; + }; + } // namespace detail + + SportsStream::SportsStream(const Config &config) + : runtime_(detail::SportsStreamRuntime::create(config)) + { + } + + SportsStream::~SportsStream() + { + auto runtime = std::move(runtime_); + if (runtime) runtime->shutdown(); + } + + void SportsStream::on_result(SportsResultCallback callback) + { + runtime_->on_result(std::move(callback)); + } + + void SportsStream::on_stream_gap(SportsStreamGapCallback callback) + { + runtime_->stream().on_stream_gap(std::move(callback)); + } + + void SportsStream::on_stream_recovered(SportsStreamRecoveredCallback callback) + { + runtime_->stream().on_stream_recovered(std::move(callback)); + } + + void SportsStream::on_error(SportsStreamErrorCallback callback) + { + runtime_->stream().on_error(std::move(callback)); + } + + bool SportsStream::connect() + { + return runtime_->stream().connect(); + } + + void SportsStream::disconnect() + { + runtime_->stream().disconnect(); + } + + bool SportsStream::is_connected() const + { + return runtime_->stream().is_connected(); + } + + void SportsStream::run() + { + runtime_->stream().run(); + } + + void SportsStream::stop() + { + runtime_->stream().stop(); + } + + std::uint64_t SportsStream::result_events() const + { + return runtime_->result_events(); + } + + std::uint64_t SportsStream::dropped_events() const + { + return runtime_->dropped_events(); + } +} // namespace polymarket diff --git a/tests/fuzz/corpus/websocket_messages/sports_result b/tests/fuzz/corpus/websocket_messages/sports_result new file mode 100644 index 0000000..b18b6a1 --- /dev/null +++ b/tests/fuzz/corpus/websocket_messages/sports_result @@ -0,0 +1 @@ +{"gameId":70918912,"sportradarGameId":"568982f7-6051-4a30-8e7c-d9d6235f9472","leagueAbbreviation":"cfb","turn":"nd","homeTeam":"BYU","awayTeam":"ND","turnProviderId":"d7f19bff-74e7-4b71-a221-12dfeb144fd8","status":"inprogress","score":"10-21","elapsed":"00:33","period":"Q2","live":true,"ended":false} \ No newline at end of file diff --git a/tests/fuzz/fuzz_websocket_messages.cpp b/tests/fuzz/fuzz_websocket_messages.cpp index 5bf0555..210b540 100644 --- a/tests/fuzz/fuzz_websocket_messages.cpp +++ b/tests/fuzz/fuzz_websocket_messages.cpp @@ -1,3 +1,4 @@ +#include "sports_parsing.hpp" #include "user_stream_protocol.hpp" #include "websocket_resilience.hpp" @@ -10,7 +11,7 @@ #include #include -// Feeds one raw WebSocket frame to every market and user stream parser. +// Feeds one raw WebSocket frame to every market, user and sports stream parser. // Rejected input must raise std::exception; accepted book updates must leave // each book with valid, ordered levels. namespace @@ -70,5 +71,10 @@ extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size require_valid_book(book); } }); + parse_or_reject( + [&] + { + if (!detail::parse_sports_message(message).raw.is_object()) std::abort(); + }); return 0; } diff --git a/tests/sports_fixtures.hpp b/tests/sports_fixtures.hpp new file mode 100644 index 0000000..0a73ae1 --- /dev/null +++ b/tests/sports_fixtures.hpp @@ -0,0 +1,17 @@ +#pragma once + +// Frames recorded from wss://sports-api.polymarket.com/ws on 2026-10-18. They +// carry public game data only. sports_unscored_result (a UFC bout without a +// score) and sports_partial_result (a cricket update without gameId or status) +// are rejected, as both official SDKs reject them. +namespace sports_fixtures +{ + inline constexpr const char *sports_live_result = + R"json({"gameId":70918912,"sportradarGameId":"568982f7-6051-4a30-8e7c-d9d6235f9472","leagueAbbreviation":"cfb","turn":"nd","homeTeam":"BYU","awayTeam":"ND","turnProviderId":"d7f19bff-74e7-4b71-a221-12dfeb144fd8","status":"inprogress","score":"10-21","elapsed":"00:33","period":"Q2","live":true,"ended":false})json"; + inline constexpr const char *sports_camel_status_result = + R"json({"gameId":90104527,"leagueAbbreviation":"mls","homeTeam":"Atlanta United FC","awayTeam":"Inter Miami CF","status":"InProgress","score":"0-3","elapsed":"69","period":"2H","live":true,"ended":false})json"; + inline constexpr const char *sports_unscored_result = + R"json({"gameId":10699,"leagueAbbreviation":"ufc","homeTeam":"Kyle Daukaus","awayTeam":"Marc-Andre Barriault","status":"In Progress","elapsed":"02:25","period":"R1","live":true,"ended":false})json"; + inline constexpr const char *sports_partial_result = + R"json({"metadataGameId":"id2704651975152240","leagueAbbreviation":"cricket","score":"70-199","period":"Live","live":true,"ended":false})json"; +} // namespace sports_fixtures diff --git a/tests/test_sports_parsing.cpp b/tests/test_sports_parsing.cpp new file mode 100644 index 0000000..afda7d8 --- /dev/null +++ b/tests/test_sports_parsing.cpp @@ -0,0 +1,111 @@ +#include "check_support.hpp" +#include "sports_fixtures.hpp" +#include "sports_parsing.hpp" + +#include +#include + +namespace +{ + using check_support::check; + namespace detail = polymarket::detail; + + void expect_rejected(const std::string &name, const std::function &parse) + { + try + { + parse(); + } + catch (const std::exception &) + { + return; + } + check(false, name + " must be rejected"); + } + + nlohmann::json live_row() + { + return nlohmann::json::parse(sports_fixtures::sports_live_result); + } + + std::optional finished(const nlohmann::json &value) + { + auto row = live_row(); + row["finishedTimestamp"] = value; + return detail::parse_sports_result(row).finished_at_ms; + } + + void test_recorded_results() + { + const auto live = detail::parse_sports_message(sports_fixtures::sports_live_result); + check(live.game_id == 70918912 && live.league_abbreviation == "cfb" && + live.home_team == "BYU" && live.away_team == "ND" && + live.status == "inprogress" && live.score == "10-21" && live.period == "Q2" && + live.elapsed == "00:33" && live.turn == "nd" && live.live && !live.ended && + !live.finished_at_ms && + live.sportradar_game_id == "568982f7-6051-4a30-8e7c-d9d6235f9472" && + live.raw.contains("turnProviderId"), + "a recorded live game must parse"); + + const auto camel = + detail::parse_sports_message(sports_fixtures::sports_camel_status_result); + check(camel.status == "InProgress" && camel.league_abbreviation == "mls" && + camel.score == "0-3" && camel.sportradar_game_id.empty() && camel.turn.empty(), + "a status is kept as sent"); + } + + void test_finished_timestamps() + { + check(finished(1792290000123) == 1792290000123, "epoch milliseconds must be kept"); + check(finished("1792290000123") == 1792290000123, "a digit string is milliseconds"); + check(finished("2026-10-18T03:40:00Z") == 1792294800000, "a UTC time must parse"); + check(finished("2026-10-18T03:40:00.5Z") == 1792294800500, "fractions must parse"); + check(finished("2026-10-18T05:40:00+02:00") == 1792294800000, "an offset must apply"); + check(finished("2026-10-17T22:40:00-05:00") == 1792294800000, + "a negative offset must apply"); + check(finished("2026-10-18T03:40:00") == 1792294800000, "a zoneless time is UTC"); + check(!finished("") && !finished(nullptr), "an empty time is unset"); + + auto snake = live_row(); + snake["finishedTimestamp"] = nullptr; + snake["finished_timestamp"] = 1792290000000; + check(detail::parse_sports_result(snake).finished_at_ms == 1792290000000, + "finished_timestamp must fill a missing finishedTimestamp"); + + auto text_id = live_row(); + text_id["gameId"] = "77"; + check(detail::parse_sports_result(text_id).game_id == 77, "a digit-string game ID parses"); + } + + void test_malformed_input() + { + expect_rejected( + "a partial cricket update", + [] { (void)detail::parse_sports_message(sports_fixtures::sports_partial_result); }); + expect_rejected( + "an update without a score", + [] { (void)detail::parse_sports_message(sports_fixtures::sports_unscored_result); }); + expect_rejected("non-JSON text", [] { (void)detail::parse_sports_message("ping"); }); + expect_rejected("a list", [] { (void)detail::parse_sports_message("[]"); }); + const auto rejected_field = [](const char *name, const char *key, nlohmann::json value) + { + auto row = live_row(); + row[key] = std::move(value); + expect_rejected(name, [&] { (void)detail::parse_sports_result(row); }); + }; + rejected_field("a string live flag", "live", "true"); + rejected_field("a missing score", "score", nullptr); + rejected_field("a fractional game ID", "gameId", 1.5); + rejected_field("a malformed finish time", "finishedTimestamp", "18 Oct 2026"); + rejected_field("a boolean finish time", "finishedTimestamp", true); + rejected_field("an oversized finish time", "finishedTimestamp", "9999999999999999"); + } +} // namespace + +int main() +{ + test_recorded_results(); + test_finished_timestamps(); + test_malformed_input(); + return check_support::finish("test_sports_parsing"); +} diff --git a/tests/test_sports_stream.cpp b/tests/test_sports_stream.cpp new file mode 100644 index 0000000..6a0cb10 --- /dev/null +++ b/tests/test_sports_stream.cpp @@ -0,0 +1,158 @@ +#include "polymarket/sports_stream.hpp" +#include "sports_fixtures.hpp" +#include "websocket_test_server.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + using namespace std::chrono_literals; + using websocket_test::LocalWebSocketServer; + + bool check(bool condition, const std::string &message) + { + if (!condition) std::cerr << "failed: " << message << '\n'; + return condition; + } + + template + bool wait_until(Predicate predicate, std::chrono::milliseconds timeout) + { + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) + { + if (predicate()) return true; + std::this_thread::sleep_for(2ms); + } + return predicate(); + } + + polymarket::Config local_config(const std::string &url) + { + polymarket::Config config; + config.sports_ws_url = url; + return config; + } + + bool test_stream_delivery() + { + LocalWebSocketServer server; + polymarket::SportsStream stream(local_config(server.url())); + std::mutex mutex; + std::vector results; + std::atomic gaps{0}; + std::atomic recoveries{0}; + stream.on_result( + [&](const polymarket::SportsGameResult &result) + { + std::lock_guard lock(mutex); + results.push_back(result); + }); + stream.on_stream_gap([&] { gaps++; }); + stream.on_stream_recovered([&] { recoveries++; }); + + const bool connected = stream.connect(); + const bool first_recovery = wait_until([&] { return recoveries.load() == 1; }, 2s); + server.send_to_clients("ping"); + server.send_to_clients(sports_fixtures::sports_partial_result); + server.send_to_clients(sports_fixtures::sports_live_result); + const bool ponged = server.wait_for_message_count("pong", 1, 2s); + const bool delivered = wait_until( + [&] + { + std::lock_guard lock(mutex); + return results.size() == 1; + }, + 2s); + + server.close_clients(); + const bool reconnected = server.wait_for_connections(2, 5s); + const bool second_recovery = wait_until([&] { return recoveries.load() == 2; }, 5s); + server.send_to_clients(sports_fixtures::sports_camel_status_result); + const bool after_reconnect = wait_until( + [&] + { + std::lock_guard lock(mutex); + return results.size() == 2; + }, + 2s); + const bool no_client_ping = !server.wait_for_message_count("PING", 1, 100ms); + stream.disconnect(); + + std::lock_guard lock(mutex); + return check(connected && first_recovery, "a connect must report recovery") && + check(ponged, "the stream must answer the server heartbeat") && + check(delivered && results[0].game_id == 70918912 && results[0].score == "10-21", + "a game update must be delivered") && + check(stream.dropped_events() == 1 && stream.result_events() == 2, + "a malformed update must be counted and skipped") && + check(reconnected && second_recovery && gaps.load() >= 3, + "a dropped connection must report a gap and recover") && + check(after_reconnect && results[1].league_abbreviation == "mls", + "updates must resume after a reconnect") && + check(no_client_ping, "the stream must not send client pings"); + } + + bool test_disconnect_stops_delivery() + { + LocalWebSocketServer server; + polymarket::SportsStream stream(local_config(server.url())); + std::atomic results{0}; + stream.on_result([&](const polymarket::SportsGameResult &) { results++; }); + const bool connected = stream.connect(); + stream.disconnect(); + const bool disconnected = !stream.is_connected(); + const bool reconnected = stream.connect(); + server.send_to_clients(sports_fixtures::sports_live_result); + const bool delivered = wait_until([&] { return results.load() == 1; }, 2s); + stream.disconnect(); + return check(connected && disconnected && reconnected && delivered, + "a stream must connect again after disconnect"); + } + + bool test_invalid_config() + { + polymarket::Config empty; + empty.sports_ws_url.clear(); + polymarket::Config zero_timeout; + zero_timeout.ws_connect_timeout_ms = 0; + int rejected = 0; + for (const auto &config : {empty, zero_timeout}) + { + try + { + polymarket::SportsStream stream(config); + } + catch (const std::invalid_argument &) + { + rejected++; + } + } + return check(rejected == 2, "an empty URL or zero timeout must be rejected"); + } + + bool test_unreachable_server() + { + polymarket::Config config = local_config("ws://127.0.0.1:1"); + config.ws_connect_timeout_ms = 300; + polymarket::SportsStream stream(config); + return check(!stream.connect() && !stream.is_connected(), + "an unreachable server must fail to connect"); + } +} // namespace + +int main() +{ + const bool passed = test_stream_delivery() && test_disconnect_stops_delivery() && + test_invalid_config() && test_unreachable_server(); + if (!passed) return 1; + std::cout << "test_sports_stream passed\n"; + return 0; +} From 780f88289e8a144b898af35595f05dc1ae1ac6ec Mon Sep 17 00:00:00 2001 From: yluoc Date: Sat, 17 Oct 2026 21:33:09 -0400 Subject: [PATCH 03/11] fix(gamma): accept integer-string comment position sizes --- src/gamma_social_parsing.cpp | 44 ++++++++++++++++++++++++------------ tests/test_gamma_social.cpp | 15 ++++++++++++ 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/gamma_social_parsing.cpp b/src/gamma_social_parsing.cpp index bc30d95..348626c 100644 --- a/src/gamma_social_parsing.cpp +++ b/src/gamma_social_parsing.cpp @@ -1,5 +1,7 @@ #include "gamma_parsing.hpp" +#include +#include #include #include #include @@ -15,6 +17,18 @@ namespace polymarket::detail return field_string(item, "proxyWallet"); } + std::optional integer_or_text(const json &item, const char *key) + { + const auto *value = find_field(item, key); + if (value == nullptr || !value->is_string()) return field_integer(item, key); + const auto &text = value->get_ref(); + const auto digits = text.starts_with('-') ? text.substr(1) : text; + if (digits.empty() || digits.size() > 18 || + digits.find_first_not_of("0123456789") != std::string::npos) + throw_bad_field(key, "an integer"); + return std::stoll(text); + } + std::optional comment_profile(const json &item, const char *key) { if (!item.contains(key) || item.at(key).is_null()) return std::nullopt; @@ -31,23 +45,10 @@ namespace polymarket::detail profile.is_creator = field_bool(value, "isCreator"); for (const auto &position : field_objects(value, "positions")) profile.positions.push_back( - {field_id(position, "tokenId"), field_integer(position, "positionSize")}); + {field_id(position, "tokenId"), integer_or_text(position, "positionSize")}); return profile; } - GammaReaction reaction(const json &item) - { - GammaReaction result; - result.id = field_required_id(item, "id"); - result.comment_id = field_integer(item, "commentID"); - result.reaction_type = field_string(item, "reactionType"); - result.icon = field_string(item, "icon"); - result.user_address = field_string(item, "userAddress"); - result.created_at = field_string(item, "createdAt"); - result.profile = comment_profile(item, "profile"); - return result; - } - GammaSearchProfile search_profile(const json &item) { GammaSearchProfile profile; @@ -76,6 +77,19 @@ namespace polymarket::detail } } // namespace + GammaReaction parse_gamma_reaction(const json &item) + { + GammaReaction result; + result.id = field_required_id(item, "id"); + result.comment_id = field_integer(item, "commentID"); + result.reaction_type = field_string(item, "reactionType"); + result.icon = field_string(item, "icon"); + result.user_address = field_string(item, "userAddress"); + result.created_at = field_string(item, "createdAt"); + result.profile = comment_profile(item, "profile"); + return result; + } + GammaComment parse_gamma_comment(const json &item) { GammaComment comment; @@ -90,7 +104,7 @@ namespace polymarket::detail comment.updated_at = field_string(item, "updatedAt"); comment.profile = comment_profile(item, "profile"); for (const auto &value : field_objects(item, "reactions")) - comment.reactions.push_back(reaction(value)); + comment.reactions.push_back(parse_gamma_reaction(value)); comment.report_count = field_integer(item, "reportCount"); comment.reaction_count = field_integer(item, "reactionCount"); comment.trade_asset = field_string(item, "tradeAsset"); diff --git a/tests/test_gamma_social.cpp b/tests/test_gamma_social.cpp index 76fdc8f..0cfb15b 100644 --- a/tests/test_gamma_social.cpp +++ b/tests/test_gamma_social.cpp @@ -97,6 +97,21 @@ namespace clob_test "non-array search results must be rejected") && check(rejected([] { (void)detail::parse_gamma_comments(R"([{"body":"x"}])"); }), "a comment without id must be rejected") && + check( + detail::parse_gamma_comments( + R"([{"id":"1","profile":{"positions":[{"tokenId":"7","positionSize":"47612999494"}]}}])") + .at(0) + .profile->positions.at(0) + .position_size == 47612999494, + "a digit-string position size must parse") && + check( + rejected( + [] + { + (void)detail::parse_gamma_comments( + R"([{"id":"1","profile":{"positions":[{"positionSize":"4.5"}]}}])"); + }), + "a fractional position size must be rejected") && check(rejected([] { (void)detail::parse_gamma_public_profile(R"([])"); }), "a non-object profile must be rejected"); } From 1c37ba91f1420e278e9b6da5e31a9aaf9c08cfa7 Mon Sep 17 00:00:00 2001 From: yluoc Date: Sat, 17 Oct 2026 21:33:31 -0400 Subject: [PATCH 04/11] feat(rtds): add RTDS stream with comment subscriptions --- CMakeLists.txt | 3 + cmake/PolymarketTests.cmake | 9 + include/polymarket/rtds_stream.hpp | 94 +++++++ src/gamma_parsing.hpp | 1 + src/rtds_parsing.cpp | 115 ++++++++ src/rtds_parsing.hpp | 43 +++ src/rtds_stream.cpp | 257 ++++++++++++++++++ src/sports_parsing.cpp | 54 +--- src/sports_parsing.hpp | 7 - src/stream_time.cpp | 66 +++++ src/stream_time.hpp | 16 ++ .../websocket_messages/rtds_comment_created | 1 + tests/fuzz/fuzz_websocket_messages.cpp | 10 +- tests/rtds_fixtures.hpp | 15 + tests/test_rtds_parsing.cpp | 164 +++++++++++ tests/test_rtds_stream.cpp | 187 +++++++++++++ 16 files changed, 981 insertions(+), 61 deletions(-) create mode 100644 include/polymarket/rtds_stream.hpp create mode 100644 src/rtds_parsing.cpp create mode 100644 src/rtds_parsing.hpp create mode 100644 src/rtds_stream.cpp create mode 100644 src/stream_time.cpp create mode 100644 src/stream_time.hpp create mode 100644 tests/fuzz/corpus/websocket_messages/rtds_comment_created create mode 100644 tests/rtds_fixtures.hpp create mode 100644 tests/test_rtds_parsing.cpp create mode 100644 tests/test_rtds_stream.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ee4b29c..e277b99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -170,8 +170,11 @@ set(POLYMARKET_CLIENT_SOURCES src/user_stream_runtime_events.cpp src/user_stream_subscription.cpp src/public_stream.cpp + src/rtds_parsing.cpp + src/rtds_stream.cpp src/sports_parsing.cpp src/sports_stream.cpp + src/stream_time.cpp src/order_amounts.cpp src/clob_order_execution.cpp src/clob_order_submission.cpp diff --git a/cmake/PolymarketTests.cmake b/cmake/PolymarketTests.cmake index be582a0..331d667 100644 --- a/cmake/PolymarketTests.cmake +++ b/cmake/PolymarketTests.cmake @@ -279,6 +279,15 @@ if(POLYMARKET_CLIENT_BUILD_TESTS) target_link_libraries(test_sports_stream PRIVATE polymarket::client) add_test(NAME test_sports_stream COMMAND test_sports_stream) + add_executable(test_rtds_parsing tests/test_rtds_parsing.cpp) + target_include_directories(test_rtds_parsing PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + target_link_libraries(test_rtds_parsing PRIVATE polymarket::client) + add_test(NAME test_rtds_parsing COMMAND test_rtds_parsing) + + add_executable(test_rtds_stream tests/test_rtds_stream.cpp) + target_link_libraries(test_rtds_stream PRIVATE polymarket::client) + add_test(NAME test_rtds_stream COMMAND test_rtds_stream) + add_executable(test_network_routing tests/test_network_routing.cpp) target_link_libraries(test_network_routing PRIVATE polymarket::client) add_test(NAME test_network_routing COMMAND test_network_routing) diff --git a/include/polymarket/rtds_stream.hpp b/include/polymarket/rtds_stream.hpp new file mode 100644 index 0000000..dc15f84 --- /dev/null +++ b/include/polymarket/rtds_stream.hpp @@ -0,0 +1,94 @@ +#pragma once + +#include "polymarket/gamma_social_types.hpp" +#include "polymarket/types.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace polymarket +{ + namespace detail + { + class RtdsStreamRuntime; + } + + // Comment events to receive. `types` holds comment_created (the default + // when empty), comment_removed, reaction_created or reaction_removed. + // A parent filter keeps comments on that event or market and drops + // reactions, which carry no parent. + struct RtdsCommentFilter + { + std::vector types; + std::optional parent_entity_id; + std::string parent_entity_type; // Event or Market + }; + + // One comment event, following the official py-sdk models. `comment` is + // set for comment_created and comment_removed, `reaction` for the + // reaction events. `raw` holds the whole frame as received. + struct RtdsCommentEvent + { + std::string type; + std::optional timestamp_ms; // Unix milliseconds + std::optional comment; + std::optional reaction; + nlohmann::json raw; + }; + + using RtdsSubscriptionId = std::uint64_t; + using RtdsCommentCallback = std::function; + // Fired whenever events may have been missed: every connect, disconnect, + // queue overflow or idle reconnect. RTDS does not replay them. + using RtdsStreamGapCallback = std::function; + // Fired once the connection and its subscriptions are back after a gap. + using RtdsStreamRecoveredCallback = std::function; + using RtdsStreamErrorCallback = std::function; + + // Public real-time data stream (Config::rtds_ws_url). Subscriptions can + // be added and removed before or after connect(); the stream sends the + // combined server subscription and restores it after every reconnect. + // It pings every 5 seconds and reconnects when no frame arrives for 10 + // minutes. Callbacks run on the stream's own threads; a malformed or + // unknown frame is counted in dropped_events() and skipped. + class RtdsStream + { + public: + explicit RtdsStream(const Config &config = Config{}); + ~RtdsStream(); + + RtdsStream(const RtdsStream &) = delete; + RtdsStream &operator=(const RtdsStream &) = delete; + + // Throws std::invalid_argument for an unknown type or parent type. + RtdsSubscriptionId subscribe_comments(const RtdsCommentFilter &filter, + RtdsCommentCallback callback); + // Unknown IDs are ignored. + void unsubscribe(RtdsSubscriptionId id); + + void on_stream_gap(RtdsStreamGapCallback callback); + void on_stream_recovered(RtdsStreamRecoveredCallback callback); + void on_error(RtdsStreamErrorCallback callback); + + // Waits up to Config::ws_connect_timeout_ms for the connection. + bool connect(); + void disconnect(); + bool is_connected() const; + // Blocks until stop(). + void run(); + void stop(); + + std::uint64_t events() const; + std::uint64_t dropped_events() const; + + private: + std::shared_ptr runtime_; + }; + +} // namespace polymarket diff --git a/src/gamma_parsing.hpp b/src/gamma_parsing.hpp index ef81aca..84ceb15 100644 --- a/src/gamma_parsing.hpp +++ b/src/gamma_parsing.hpp @@ -40,6 +40,7 @@ namespace polymarket::detail std::vector parse_gamma_teams(const std::string &body); GammaComment parse_gamma_comment(const json &item); + GammaReaction parse_gamma_reaction(const json &item); std::vector parse_gamma_comments(const std::string &body); Page parse_gamma_comment_keyset_page(const std::string &body); // An offset page counted by its top-level comments. diff --git a/src/rtds_parsing.cpp b/src/rtds_parsing.cpp new file mode 100644 index 0000000..c7dd88e --- /dev/null +++ b/src/rtds_parsing.cpp @@ -0,0 +1,115 @@ +#include "rtds_parsing.hpp" + +#include "gamma_parsing.hpp" +#include "stream_time.hpp" + +#include +#include +#include + +namespace polymarket::detail +{ + namespace + { + constexpr const char *comments_topic = "comments"; + + bool is_comment_type(const std::string &type) + { + return type == "comment_created" || type == "comment_removed" || + type == "reaction_created" || type == "reaction_removed"; + } + + const json &payload_object(const json &frame) + { + const auto *payload = find_field(frame, "payload"); + if (payload == nullptr || !payload->is_object()) + throw_bad_field("payload", "an object"); + return *payload; + } + } // namespace + + RtdsCommentEvent parse_rtds_comment_event(const json &frame) + { + RtdsCommentEvent event; + event.type = field_required_string(frame, "type"); + if (!is_comment_type(event.type)) throw_bad_field("type", "a comment event type"); + event.timestamp_ms = stream_epoch_ms_or_iso(frame, "timestamp"); + const auto &payload = payload_object(frame); + if (event.type.starts_with("comment_")) + event.comment = parse_gamma_comment(payload); + else + event.reaction = parse_gamma_reaction(payload); + event.raw = frame; + return event; + } + + std::optional parse_rtds_message(const std::string &message) + { + if (message.empty()) return std::nullopt; + const auto frame = json::parse(message); + RtdsParsedEvent event; + event.topic = field_required_string(frame, "topic"); + event.type = field_required_string(frame, "type"); + if (event.topic == comments_topic) + event.comment = parse_rtds_comment_event(frame); + else + throw std::invalid_argument("unknown RTDS topic " + event.topic); + return event; + } + + std::string rtds_subscription_frame(const char *action, const std::vector &keys) + { + json subscriptions = json::array(); + for (const auto &key : keys) + { + json subscription = {{"topic", key.topic}, {"type", key.type}}; + if (!key.filters.empty()) subscription["filters"] = key.filters; + subscriptions.push_back(std::move(subscription)); + } + return json{{"action", action}, {"subscriptions", std::move(subscriptions)}}.dump(); + } + + RtdsCommentFilter normalized_comment_filter(const RtdsCommentFilter &filter) + { + RtdsCommentFilter normalized; + for (const auto &type : filter.types) + { + if (!is_comment_type(type)) + throw std::invalid_argument("invalid comments event type: " + type); + if (std::find(normalized.types.begin(), normalized.types.end(), type) == + normalized.types.end()) + normalized.types.push_back(type); + } + if (normalized.types.empty()) normalized.types.push_back("comment_created"); + if (!filter.parent_entity_type.empty() && filter.parent_entity_type != "Event" && + filter.parent_entity_type != "Market") + throw std::invalid_argument("parent_entity_type must be Event or Market"); + normalized.parent_entity_id = filter.parent_entity_id; + normalized.parent_entity_type = filter.parent_entity_type; + return normalized; + } + + std::vector rtds_comment_keys(const RtdsCommentFilter &normalized) + { + std::vector keys; + for (const auto &type : normalized.types) + keys.push_back({comments_topic, type, ""}); + return keys; + } + + bool rtds_comment_matches(const RtdsCommentFilter &normalized, const RtdsCommentEvent &event) + { + if (std::find(normalized.types.begin(), normalized.types.end(), event.type) == + normalized.types.end()) + return false; + const bool parent_filter = + normalized.parent_entity_id.has_value() || !normalized.parent_entity_type.empty(); + if (!parent_filter) return true; + if (!event.comment) return false; + if (normalized.parent_entity_id && + event.comment->parent_entity_id != std::to_string(*normalized.parent_entity_id)) + return false; + return normalized.parent_entity_type.empty() || + event.comment->parent_entity_type == normalized.parent_entity_type; + } +} // namespace polymarket::detail diff --git a/src/rtds_parsing.hpp b/src/rtds_parsing.hpp new file mode 100644 index 0000000..46b5819 --- /dev/null +++ b/src/rtds_parsing.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include "json_fields.hpp" +#include "polymarket/rtds_stream.hpp" + +#include +#include +#include + +// RTDS frames, following py-sdk _internal/streams/rtds/protocol.py and +// models/rtds_events.py. Malformed input throws std::exception. +namespace polymarket::detail +{ + // One server subscription; `filters` is sent only when non-empty. + struct RtdsServerKey + { + std::string topic; // wire topic + std::string type; + std::string filters; + + bool operator==(const RtdsServerKey &) const = default; + }; + + struct RtdsParsedEvent + { + std::string topic; // wire topic + std::string type; + std::optional comment; + }; + + // Unset for the empty frame RTDS sends after connecting. + std::optional parse_rtds_message(const std::string &message); + RtdsCommentEvent parse_rtds_comment_event(const json &frame); + + // {"action": action, "subscriptions": [...]} for `keys`. + std::string rtds_subscription_frame(const char *action, const std::vector &keys); + + // Checked and deduplicated, with the default type when empty; throws + // std::invalid_argument. + RtdsCommentFilter normalized_comment_filter(const RtdsCommentFilter &filter); + std::vector rtds_comment_keys(const RtdsCommentFilter &normalized); + bool rtds_comment_matches(const RtdsCommentFilter &normalized, const RtdsCommentEvent &event); +} // namespace polymarket::detail diff --git a/src/rtds_stream.cpp b/src/rtds_stream.cpp new file mode 100644 index 0000000..1805c1c --- /dev/null +++ b/src/rtds_stream.cpp @@ -0,0 +1,257 @@ +#include "polymarket/rtds_stream.hpp" + +#include "public_stream.hpp" +#include "rtds_parsing.hpp" + +#include +#include +#include +#include +#include +#include + +namespace polymarket +{ + namespace detail + { + namespace + { + constexpr int rtds_ping_interval_ms = 5000; + constexpr std::uint32_t rtds_idle_timeout_ms = 10 * 60 * 1000; + + WebSocketOptions rtds_options() + { + WebSocketOptions options; + options.ping_message = "PING"; + options.ping_interval_ms = rtds_ping_interval_ms; + options.idle_timeout_ms = rtds_idle_timeout_ms; + return options; + } + + bool contains(const std::vector &keys, const RtdsServerKey &key) + { + return std::find(keys.begin(), keys.end(), key) != keys.end(); + } + } // namespace + + class RtdsStreamRuntime final : public std::enable_shared_from_this + { + public: + using Deliver = std::function; + + static std::shared_ptr create(const Config &config) + { + auto runtime = std::shared_ptr(new RtdsStreamRuntime()); + std::weak_ptr weak = runtime; + runtime->stream_ = + PublicStream::create(config.rtds_ws_url, rtds_options(), + std::chrono::milliseconds(config.ws_connect_timeout_ms), + [weak](const std::string &message) + { + if (auto self = weak.lock()) + self->handle_message(message); + }); + return runtime; + } + + void shutdown() + { + { + std::lock_guard lock(mutex_); + subscriptions_.clear(); + } + stream_->shutdown(); + } + + RtdsSubscriptionId add(std::vector keys, Deliver deliver) + { + bool resync = false; + RtdsSubscriptionId id = 0; + { + std::lock_guard lock(mutex_); + const auto before = server_keys(); + id = ++next_id_; + subscriptions_.emplace(id, Subscription{std::move(keys), std::move(deliver)}); + resync = !sync(before, server_keys()); + } + if (resync) stream_->request_resnapshot("RTDS subscribe failed; reconnecting"); + return id; + } + + void remove(RtdsSubscriptionId id) + { + bool resync = false; + { + std::lock_guard lock(mutex_); + const auto before = server_keys(); + if (subscriptions_.erase(id) == 0) return; + resync = !sync(before, server_keys()); + } + if (resync) stream_->request_resnapshot("RTDS unsubscribe failed; reconnecting"); + } + + PublicStream &stream() { return *stream_; } + std::uint64_t events() const { return events_.load(); } + std::uint64_t dropped_events() const { return dropped_events_.load(); } + + private: + struct Subscription + { + std::vector keys; + Deliver deliver; + }; + + RtdsStreamRuntime() = default; + + std::vector server_keys() const + { + std::vector keys; + for (const auto &[id, subscription] : subscriptions_) + { + for (const auto &key : subscription.keys) + { + if (!contains(keys, key)) keys.push_back(key); + } + } + return keys; + } + + // False when a frame could not be sent on a live connection. + bool sync(const std::vector &before, + const std::vector &after) + { + stream_->replace_subscriptions( + after.empty() + ? std::vector{} + : std::vector{rtds_subscription_frame("subscribe", after)}); + if (!stream_->is_connected()) return true; + std::vector added; + std::vector removed; + for (const auto &key : after) + { + if (!contains(before, key)) added.push_back(key); + } + for (const auto &key : before) + { + if (!contains(after, key)) removed.push_back(key); + } + if (!added.empty() && !stream_->send(rtds_subscription_frame("subscribe", added))) + return false; + return removed.empty() || + stream_->send(rtds_subscription_frame("unsubscribe", removed)); + } + + void handle_message(const std::string &message) + { + std::optional event; + try + { + event = parse_rtds_message(message); + } + catch (const std::exception &) + { + dropped_events_++; + return; + } + if (!event) return; + events_++; + std::vector targets; + { + std::lock_guard lock(mutex_); + for (const auto &[id, subscription] : subscriptions_) + targets.push_back(subscription.deliver); + } + for (const auto &deliver : targets) + deliver(*event); + } + + std::shared_ptr stream_; + mutable std::mutex mutex_; + std::map subscriptions_; + RtdsSubscriptionId next_id_{0}; + std::atomic events_{0}; + std::atomic dropped_events_{0}; + }; + } // namespace detail + + RtdsStream::RtdsStream(const Config &config) + : runtime_(detail::RtdsStreamRuntime::create(config)) + { + } + + RtdsStream::~RtdsStream() + { + auto runtime = std::move(runtime_); + if (runtime) runtime->shutdown(); + } + + RtdsSubscriptionId RtdsStream::subscribe_comments(const RtdsCommentFilter &filter, + RtdsCommentCallback callback) + { + auto normalized = detail::normalized_comment_filter(filter); + auto keys = detail::rtds_comment_keys(normalized); + return runtime_->add(std::move(keys), + [normalized = std::move(normalized), + callback = std::move(callback)](const detail::RtdsParsedEvent &event) + { + if (callback && event.comment && + detail::rtds_comment_matches(normalized, *event.comment)) + callback(*event.comment); + }); + } + + void RtdsStream::unsubscribe(RtdsSubscriptionId id) + { + runtime_->remove(id); + } + + void RtdsStream::on_stream_gap(RtdsStreamGapCallback callback) + { + runtime_->stream().on_stream_gap(std::move(callback)); + } + + void RtdsStream::on_stream_recovered(RtdsStreamRecoveredCallback callback) + { + runtime_->stream().on_stream_recovered(std::move(callback)); + } + + void RtdsStream::on_error(RtdsStreamErrorCallback callback) + { + runtime_->stream().on_error(std::move(callback)); + } + + bool RtdsStream::connect() + { + return runtime_->stream().connect(); + } + + void RtdsStream::disconnect() + { + runtime_->stream().disconnect(); + } + + bool RtdsStream::is_connected() const + { + return runtime_->stream().is_connected(); + } + + void RtdsStream::run() + { + runtime_->stream().run(); + } + + void RtdsStream::stop() + { + runtime_->stream().stop(); + } + + std::uint64_t RtdsStream::events() const + { + return runtime_->events(); + } + + std::uint64_t RtdsStream::dropped_events() const + { + return runtime_->dropped_events(); + } +} // namespace polymarket diff --git a/src/sports_parsing.cpp b/src/sports_parsing.cpp index b5ff20d..19ef763 100644 --- a/src/sports_parsing.cpp +++ b/src/sports_parsing.cpp @@ -1,55 +1,18 @@ #include "sports_parsing.hpp" -#include "market_time.hpp" +#include "stream_time.hpp" -#include -#include #include namespace polymarket::detail { namespace { - constexpr std::int64_t max_epoch_ms = 253402300799999; // 9999-12-31T23:59:59.999Z - bool all_digits(const std::string &text) { return !text.empty() && text.find_first_not_of("0123456789") == std::string::npos; } - std::int64_t checked_ms(const char *key, const std::string &digits) - { - if (digits.size() > 15) throw_bad_field(key, "Unix milliseconds"); - const auto ms = std::stoll(digits); - if (ms > max_epoch_ms) throw_bad_field(key, "Unix milliseconds"); - return ms; - } - - std::int64_t iso_ms(const char *key, const std::string &text) - { - std::string utc = text; - std::int64_t offset_minutes = 0; - const auto size = text.size(); - if (size > 6 && (text[size - 6] == '+' || text[size - 6] == '-') && - text[size - 3] == ':') - { - const auto hours = text.substr(size - 5, 2); - const auto minutes = text.substr(size - 2, 2); - if (!all_digits(hours) || !all_digits(minutes)) - throw_bad_field(key, "an ISO 8601 time"); - offset_minutes = std::stoll(hours) * 60 + std::stoll(minutes); - if (text[size - 6] == '-') offset_minutes = -offset_minutes; - utc = text.substr(0, size - 6) + "Z"; - } - else if (text.back() != 'Z') - { - utc = text + "Z"; - } - const auto ms = parse_iso8601_ms(utc); - if (ms == 0) throw_bad_field(key, "an ISO 8601 time"); - return static_cast(ms) - offset_minutes * 60000; - } - std::int64_t required_integer(const json &item, const char *key) { const auto *value = find_field(item, key); @@ -72,21 +35,6 @@ namespace polymarket::detail } } // namespace - std::optional stream_epoch_ms_or_iso(const json &object, const char *key) - { - const auto *value = find_field(object, key); - if (value == nullptr) return std::nullopt; - if (value->is_string()) - { - const auto &text = value->get_ref(); - if (text.empty()) return std::nullopt; - return all_digits(text) ? checked_ms(key, text) : iso_ms(key, text); - } - const auto ms = field_integer(object, key); - if (!ms || *ms < 0 || *ms > max_epoch_ms) throw_bad_field(key, "Unix milliseconds"); - return ms; - } - SportsGameResult parse_sports_result(const json &item) { SportsGameResult result; diff --git a/src/sports_parsing.hpp b/src/sports_parsing.hpp index 03aa4f9..a5efe49 100644 --- a/src/sports_parsing.hpp +++ b/src/sports_parsing.hpp @@ -3,19 +3,12 @@ #include "json_fields.hpp" #include "polymarket/sports_stream.hpp" -#include -#include #include // Sports WebSocket parsing, following py-sdk models/sports_events.py. // Malformed input throws std::exception. namespace polymarket::detail { - // Unix milliseconds from an integer, a digit string, or an ISO 8601 time - // with Z, a +HH:MM/-HH:MM offset, or no zone (read as UTC); unset for - // null or "". - std::optional stream_epoch_ms_or_iso(const json &object, const char *key); - SportsGameResult parse_sports_result(const json &item); SportsGameResult parse_sports_message(const std::string &message); } // namespace polymarket::detail diff --git a/src/stream_time.cpp b/src/stream_time.cpp new file mode 100644 index 0000000..96b5ea0 --- /dev/null +++ b/src/stream_time.cpp @@ -0,0 +1,66 @@ +#include "stream_time.hpp" + +#include "market_time.hpp" + +#include + +namespace polymarket::detail +{ + namespace + { + constexpr std::int64_t max_epoch_ms = 253402300799999; // 9999-12-31T23:59:59.999Z + + bool all_digits(const std::string &text) + { + return !text.empty() && text.find_first_not_of("0123456789") == std::string::npos; + } + + std::int64_t checked_ms(const char *key, const std::string &digits) + { + if (digits.size() > 15) throw_bad_field(key, "Unix milliseconds"); + const auto ms = std::stoll(digits); + if (ms > max_epoch_ms) throw_bad_field(key, "Unix milliseconds"); + return ms; + } + + std::int64_t iso_ms(const char *key, const std::string &text) + { + std::string utc = text; + std::int64_t offset_minutes = 0; + const auto size = text.size(); + if (size > 6 && (text[size - 6] == '+' || text[size - 6] == '-') && + text[size - 3] == ':') + { + const auto hours = text.substr(size - 5, 2); + const auto minutes = text.substr(size - 2, 2); + if (!all_digits(hours) || !all_digits(minutes)) + throw_bad_field(key, "an ISO 8601 time"); + offset_minutes = std::stoll(hours) * 60 + std::stoll(minutes); + if (text[size - 6] == '-') offset_minutes = -offset_minutes; + utc = text.substr(0, size - 6) + "Z"; + } + else if (text.back() != 'Z') + { + utc = text + "Z"; + } + const auto ms = parse_iso8601_ms(utc); + if (ms == 0) throw_bad_field(key, "an ISO 8601 time"); + return static_cast(ms) - offset_minutes * 60000; + } + } // namespace + + std::optional stream_epoch_ms_or_iso(const json &object, const char *key) + { + const auto *value = find_field(object, key); + if (value == nullptr) return std::nullopt; + if (value->is_string()) + { + const auto &text = value->get_ref(); + if (text.empty()) return std::nullopt; + return all_digits(text) ? checked_ms(key, text) : iso_ms(key, text); + } + const auto ms = field_integer(object, key); + if (!ms || *ms < 0 || *ms > max_epoch_ms) throw_bad_field(key, "Unix milliseconds"); + return ms; + } +} // namespace polymarket::detail diff --git a/src/stream_time.hpp b/src/stream_time.hpp new file mode 100644 index 0000000..cd18686 --- /dev/null +++ b/src/stream_time.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include "json_fields.hpp" + +#include +#include + +// Timestamps shared by the public stream parsers. Malformed input throws +// std::invalid_argument naming the field. +namespace polymarket::detail +{ + // Unix milliseconds from an integer, a digit string, or an ISO 8601 time + // with Z, a +HH:MM/-HH:MM offset, or no zone (read as UTC); unset for + // null or "". + std::optional stream_epoch_ms_or_iso(const json &object, const char *key); +} // namespace polymarket::detail diff --git a/tests/fuzz/corpus/websocket_messages/rtds_comment_created b/tests/fuzz/corpus/websocket_messages/rtds_comment_created new file mode 100644 index 0000000..c90373e --- /dev/null +++ b/tests/fuzz/corpus/websocket_messages/rtds_comment_created @@ -0,0 +1 @@ +{"connection_id":"example-connection","payload":{"body":"Example comment body.","createdAt":"2026-10-18T01:20:29.190359Z","id":"3297168","parentEntityID":45915,"parentEntityType":"Event","profile":{"baseAddress":"0x2222222222222222222222222222222222222222","displayUsernamePublic":true,"name":"Example Commenter","proxyWallet":"0x1111111111111111111111111111111111111111","pseudonym":"Example-Pseudonym"},"reactionCount":0,"reportCount":0,"userAddress":"0x2222222222222222222222222222222222222222"},"timestamp":1792286429215,"topic":"comments","type":"comment_created"} \ No newline at end of file diff --git a/tests/fuzz/fuzz_websocket_messages.cpp b/tests/fuzz/fuzz_websocket_messages.cpp index 210b540..6a2eede 100644 --- a/tests/fuzz/fuzz_websocket_messages.cpp +++ b/tests/fuzz/fuzz_websocket_messages.cpp @@ -1,3 +1,4 @@ +#include "rtds_parsing.hpp" #include "sports_parsing.hpp" #include "user_stream_protocol.hpp" #include "websocket_resilience.hpp" @@ -11,7 +12,7 @@ #include #include -// Feeds one raw WebSocket frame to every market, user and sports stream parser. +// Feeds one raw WebSocket frame to every market, user, sports and RTDS parser. // Rejected input must raise std::exception; accepted book updates must leave // each book with valid, ordered levels. namespace @@ -76,5 +77,12 @@ extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size { if (!detail::parse_sports_message(message).raw.is_object()) std::abort(); }); + parse_or_reject( + [&] + { + const auto event = detail::parse_rtds_message(message); + if (event && event->comment && !event->comment->comment && !event->comment->reaction) + std::abort(); + }); return 0; } diff --git a/tests/rtds_fixtures.hpp b/tests/rtds_fixtures.hpp new file mode 100644 index 0000000..9644313 --- /dev/null +++ b/tests/rtds_fixtures.hpp @@ -0,0 +1,15 @@ +#pragma once + +// Frames recorded from wss://ws-live-data.polymarket.com on 2026-10-18 (comments +// topic). Comment bodies, names, pseudonyms, wallets, addresses, avatars and the +// connection ID are replaced with synthetic values. rtds_comment_removed is the +// created frame relabelled, with an ISO timestamp, since none arrived live. +namespace rtds_fixtures +{ + inline constexpr const char *rtds_comment_created = + R"json({"connection_id":"example-connection","payload":{"body":"Example comment body.","createdAt":"2026-10-18T01:20:29.190359Z","id":"3297168","parentEntityID":45915,"parentEntityType":"Event","profile":{"baseAddress":"0x2222222222222222222222222222222222222222","displayUsernamePublic":true,"name":"Example Commenter","proxyWallet":"0x1111111111111111111111111111111111111111","pseudonym":"Example-Pseudonym"},"reactionCount":0,"reportCount":0,"userAddress":"0x2222222222222222222222222222222222222222"},"timestamp":1792286429215,"topic":"comments","type":"comment_created"})json"; + inline constexpr const char *rtds_reaction_created = + R"json({"connection_id":"example-connection","payload":{"commentID":3297136,"createdAt":"2026-10-18T01:19:50.897177Z","id":"3157076","profile":{"baseAddress":"0x2222222222222222222222222222222222222222","displayUsernamePublic":true,"name":"Example Commenter","proxyWallet":"0x1111111111111111111111111111111111111111","pseudonym":"Example-Pseudonym"},"reactionType":"HEART","userAddress":"0x2222222222222222222222222222222222222222"},"timestamp":1792286390914,"topic":"comments","type":"reaction_created"})json"; + inline constexpr const char *rtds_comment_removed = + R"json({"connection_id":"example-connection","payload":{"body":"Example comment body.","createdAt":"2026-10-18T01:20:29.190359Z","id":"3297168","parentEntityID":45915,"parentEntityType":"Event","profile":{"baseAddress":"0x2222222222222222222222222222222222222222","displayUsernamePublic":true,"name":"Example Commenter","proxyWallet":"0x1111111111111111111111111111111111111111","pseudonym":"Example-Pseudonym"},"reactionCount":0,"reportCount":0,"userAddress":"0x2222222222222222222222222222222222222222"},"timestamp":"2026-10-18T03:40:00Z","topic":"comments","type":"comment_removed"})json"; +} // namespace rtds_fixtures diff --git a/tests/test_rtds_parsing.cpp b/tests/test_rtds_parsing.cpp new file mode 100644 index 0000000..9704735 --- /dev/null +++ b/tests/test_rtds_parsing.cpp @@ -0,0 +1,164 @@ +#include "check_support.hpp" +#include "rtds_fixtures.hpp" +#include "rtds_parsing.hpp" + +#include +#include +#include + +namespace +{ + using check_support::check; + using check_support::expect_equal; + namespace detail = polymarket::detail; + + void expect_rejected(const std::string &name, const std::function &parse) + { + try + { + parse(); + } + catch (const std::exception &) + { + return; + } + check(false, name + " must be rejected"); + } + + polymarket::RtdsCommentEvent comment_event(const char *frame) + { + return *detail::parse_rtds_message(frame)->comment; + } + + void test_recorded_frames() + { + const auto created = comment_event(rtds_fixtures::rtds_comment_created); + check(created.type == "comment_created" && created.timestamp_ms == 1792286429215 && + created.comment && !created.reaction && created.comment->id == "3297168" && + created.comment->parent_entity_id == "45915" && + created.comment->parent_entity_type == "Event" && + created.comment->body == "Example comment body." && created.comment->profile && + created.comment->profile->wallet == + "0x1111111111111111111111111111111111111111" && + created.raw.contains("connection_id"), + "a recorded comment must parse"); + + const auto reaction = comment_event(rtds_fixtures::rtds_reaction_created); + check(reaction.type == "reaction_created" && reaction.reaction && !reaction.comment && + reaction.reaction->id == "3157076" && reaction.reaction->comment_id == 3297136 && + reaction.reaction->reaction_type == "HEART", + "a recorded reaction must parse"); + + const auto removed = comment_event(rtds_fixtures::rtds_comment_removed); + check(removed.type == "comment_removed" && removed.comment && + removed.timestamp_ms == 1792294800000, + "a removed comment must parse an ISO timestamp"); + + check(!detail::parse_rtds_message(""), "the empty frame after connecting is skipped"); + + auto with_position = nlohmann::json::parse(rtds_fixtures::rtds_comment_created); + with_position["payload"]["profile"]["positions"] = { + {{"positionSize", "47612999494"}, {"tokenId", "99"}}}; + const auto positioned = comment_event(with_position.dump().c_str()); + check(positioned.comment->profile->positions.at(0).position_size == 47612999494, + "a live comment with a text position size must parse"); + } + + void test_matching() + { + const auto created = comment_event(rtds_fixtures::rtds_comment_created); + const auto reaction = comment_event(rtds_fixtures::rtds_reaction_created); + const auto defaults = detail::normalized_comment_filter({}); + check(defaults.types == std::vector{"comment_created"}, + "comment_created is the default type"); + check(detail::rtds_comment_matches(defaults, created) && + !detail::rtds_comment_matches(defaults, reaction), + "the default filter keeps only created comments"); + + polymarket::RtdsCommentFilter both; + both.types = {"reaction_created", "comment_created", "reaction_created"}; + const auto normalized = detail::normalized_comment_filter(both); + check(normalized.types.size() == 2 && detail::rtds_comment_matches(normalized, reaction), + "types must be deduplicated and matched"); + + auto parent = both; + parent.parent_entity_id = 45915; + parent.parent_entity_type = "Event"; + const auto by_parent = detail::normalized_comment_filter(parent); + check(detail::rtds_comment_matches(by_parent, created) && + !detail::rtds_comment_matches(by_parent, reaction), + "a parent filter keeps its comments and drops reactions"); + parent.parent_entity_id = 1; + check(!detail::rtds_comment_matches(detail::normalized_comment_filter(parent), created), + "another parent must not match"); + parent.parent_entity_id.reset(); + parent.parent_entity_type = "Market"; + check(!detail::rtds_comment_matches(detail::normalized_comment_filter(parent), created), + "another parent type must not match"); + + const auto keys = detail::rtds_comment_keys(normalized); + expect_equal( + "subscribe frame", detail::rtds_subscription_frame("subscribe", keys), + R"({"action":"subscribe","subscriptions":[{"topic":"comments","type":"reaction_created"},)" + R"({"topic":"comments","type":"comment_created"}]})"); + expect_equal( + "filtered frame", + detail::rtds_subscription_frame("unsubscribe", {{"t", "x", "{\"a\":1}"}}), + R"({"action":"unsubscribe","subscriptions":[{"filters":"{\"a\":1}","topic":"t","type":"x"}]})"); + } + + void test_rejections() + { + polymarket::RtdsCommentFilter bad_type; + bad_type.types = {"comment_edited"}; + polymarket::RtdsCommentFilter bad_parent; + bad_parent.parent_entity_type = "Series"; + expect_rejected("an unknown comment type", + [&] { (void)detail::normalized_comment_filter(bad_type); }); + expect_rejected("an unknown parent type", + [&] { (void)detail::normalized_comment_filter(bad_parent); }); + + expect_rejected("non-JSON text", [] { (void)detail::parse_rtds_message("PONG"); }); + expect_rejected("an unknown topic", + [] + { + (void)detail::parse_rtds_message( + R"({"topic":"activity","type":"trades","payload":{}})"); + }); + expect_rejected( + "an unknown comment type", + [] + { + (void)detail::parse_rtds_message( + R"({"topic":"comments","type":"comment_edited","payload":{"id":"1"}})"); + }); + expect_rejected("a missing payload", + [] + { + (void)detail::parse_rtds_message( + R"({"topic":"comments","type":"comment_created"})"); + }); + expect_rejected( + "a comment without an ID", + [] + { + (void)detail::parse_rtds_message( + R"({"topic":"comments","type":"comment_created","payload":{"body":"x"}})"); + }); + expect_rejected( + "a boolean timestamp", + [] + { + (void)detail::parse_rtds_message( + R"({"topic":"comments","type":"reaction_created","timestamp":true,"payload":{"id":"1"}})"); + }); + } +} // namespace + +int main() +{ + test_recorded_frames(); + test_matching(); + test_rejections(); + return check_support::finish("test_rtds_parsing"); +} diff --git a/tests/test_rtds_stream.cpp b/tests/test_rtds_stream.cpp new file mode 100644 index 0000000..bf80326 --- /dev/null +++ b/tests/test_rtds_stream.cpp @@ -0,0 +1,187 @@ +#include "polymarket/rtds_stream.hpp" +#include "rtds_fixtures.hpp" +#include "websocket_test_server.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + using namespace std::chrono_literals; + using websocket_test::LocalWebSocketServer; + + constexpr const char *created_frame = + R"({"action":"subscribe","subscriptions":[{"topic":"comments","type":"comment_created"}]})"; + constexpr const char *both_frame = + R"({"action":"subscribe","subscriptions":[{"topic":"comments","type":"comment_created"},)" + R"({"topic":"comments","type":"reaction_created"}]})"; + constexpr const char *reaction_frame = + R"({"action":"subscribe","subscriptions":[{"topic":"comments","type":"reaction_created"}]})"; + constexpr const char *reaction_unsubscribe = + R"({"action":"unsubscribe","subscriptions":[{"topic":"comments","type":"reaction_created"}]})"; + + bool check(bool condition, const std::string &message) + { + if (!condition) std::cerr << "failed: " << message << '\n'; + return condition; + } + + template + bool wait_until(Predicate predicate, std::chrono::milliseconds timeout) + { + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) + { + if (predicate()) return true; + std::this_thread::sleep_for(2ms); + } + return predicate(); + } + + polymarket::Config local_config(const std::string &url) + { + polymarket::Config config; + config.rtds_ws_url = url; + return config; + } + + struct Recorder + { + std::mutex mutex; + std::vector types; + + polymarket::RtdsCommentCallback callback() + { + return [this](const polymarket::RtdsCommentEvent &event) + { + std::lock_guard lock(mutex); + types.push_back(event.type); + }; + } + + std::size_t size() + { + std::lock_guard lock(mutex); + return types.size(); + } + }; + + bool test_subscriptions_and_delivery() + { + LocalWebSocketServer server; + polymarket::RtdsStream stream(local_config(server.url())); + Recorder all_comments; + Recorder on_event; + Recorder reactions; + stream.subscribe_comments({}, all_comments.callback()); + polymarket::RtdsCommentFilter parent; + parent.parent_entity_id = 45915; + parent.types = {"comment_created", "reaction_created"}; + stream.subscribe_comments(parent, on_event.callback()); + + const bool connected = stream.connect(); + const bool replayed = server.wait_for_message_count(both_frame, 1, 2s); + + polymarket::RtdsCommentFilter reaction_filter; + reaction_filter.types = {"reaction_created"}; + const auto reaction_id = stream.subscribe_comments(reaction_filter, reactions.callback()); + const bool no_new_frame = !server.wait_for_message_count(reaction_frame, 1, 200ms); + + server.send_to_clients(""); + server.send_to_clients(rtds_fixtures::rtds_comment_created); + server.send_to_clients(rtds_fixtures::rtds_reaction_created); + server.send_to_clients(R"({"topic":"activity","type":"trades","payload":{}})"); + const bool delivered = wait_until( + [&] + { return all_comments.size() == 1 && on_event.size() == 1 && reactions.size() == 1; }, + 2s); + + stream.unsubscribe(reaction_id); + stream.unsubscribe(reaction_id); + const bool kept = !server.wait_for_message_count(reaction_unsubscribe, 1, 200ms); + stream.disconnect(); + return check(connected && replayed, "subscriptions added before connect must be sent") && + check(no_new_frame, "an already subscribed key must not be sent again") && + check(delivered, "each subscription must receive its matching events") && + check(stream.events() == 2 && stream.dropped_events() == 1, + "unknown topics must be dropped and the empty frame skipped") && + check(kept, "a key still used by another subscription must stay subscribed"); + } + + bool test_diffs_and_replay() + { + LocalWebSocketServer server; + polymarket::RtdsStream stream(local_config(server.url())); + std::atomic recoveries{0}; + stream.on_stream_recovered([&] { recoveries++; }); + const bool connected = stream.connect(); + Recorder reactions; + polymarket::RtdsCommentFilter reaction_filter; + reaction_filter.types = {"reaction_created"}; + const auto id = stream.subscribe_comments(reaction_filter, reactions.callback()); + const bool subscribed = server.wait_for_message_count(reaction_frame, 1, 2s); + Recorder comments; + stream.subscribe_comments({}, comments.callback()); + const bool added = server.wait_for_message_count(created_frame, 1, 2s); + stream.unsubscribe(id); + const bool removed = server.wait_for_message_count(reaction_unsubscribe, 1, 2s); + + server.close_clients(); + const bool reconnected = server.wait_for_connections(2, 5s); + const bool restored = server.wait_for_message_count(created_frame, 2, 5s); + const bool recovered = wait_until([&] { return recoveries.load() == 2; }, 5s); + server.send_to_clients(rtds_fixtures::rtds_reaction_created); + server.send_to_clients(rtds_fixtures::rtds_comment_created); + const bool resumed = wait_until([&] { return comments.size() == 1; }, 2s); + stream.disconnect(); + return check(connected && subscribed && added && removed, + "changes on a live connection must send subscribe and unsubscribe diffs") && + check(reconnected && restored && recovered, + "a reconnect must restore the combined subscription") && + check(resumed && reactions.size() == 0, + "a removed subscription must stop receiving events"); + } + + bool test_invalid_arguments() + { + polymarket::RtdsStream stream; + polymarket::RtdsCommentFilter bad; + bad.types = {"comment_edited"}; + bool rejected = false; + try + { + stream.subscribe_comments(bad, {}); + } + catch (const std::invalid_argument &) + { + rejected = true; + } + polymarket::Config empty; + empty.rtds_ws_url.clear(); + bool empty_rejected = false; + try + { + polymarket::RtdsStream invalid(empty); + } + catch (const std::invalid_argument &) + { + empty_rejected = true; + } + return check(rejected && empty_rejected, "invalid filters and URLs must be rejected"); + } +} // namespace + +int main() +{ + const bool passed = + test_subscriptions_and_delivery() && test_diffs_and_replay() && test_invalid_arguments(); + if (!passed) return 1; + std::cout << "test_rtds_stream passed\n"; + return 0; +} From bf2095766664fc8c2fc4d7a7b3a07fc3eb02affc Mon Sep 17 00:00:00 2001 From: yluoc Date: Sat, 17 Oct 2026 21:47:44 -0400 Subject: [PATCH 05/11] feat(rtds): add RTDS crypto and equity price subscriptions --- CMakeLists.txt | 1 + cmake/PolymarketTests.cmake | 5 + include/polymarket/rtds_stream.hpp | 65 ++++++ src/rtds_parsing.cpp | 2 +- src/rtds_parsing.hpp | 21 ++ src/rtds_price_parsing.cpp | 221 ++++++++++++++++++ src/rtds_stream.cpp | 35 +++ .../websocket_messages/rtds_chainlink_update | 1 + .../websocket_messages/rtds_equity_history | 1 + .../websocket_messages/rtds_twap_sixty_update | 1 + tests/fuzz/fuzz_websocket_messages.cpp | 3 + tests/rtds_price_fixtures.hpp | 20 ++ tests/test_rtds_price_parsing.cpp | 199 ++++++++++++++++ tests/test_rtds_stream.cpp | 69 +++++- 14 files changed, 640 insertions(+), 4 deletions(-) create mode 100644 src/rtds_price_parsing.cpp create mode 100644 tests/fuzz/corpus/websocket_messages/rtds_chainlink_update create mode 100644 tests/fuzz/corpus/websocket_messages/rtds_equity_history create mode 100644 tests/fuzz/corpus/websocket_messages/rtds_twap_sixty_update create mode 100644 tests/rtds_price_fixtures.hpp create mode 100644 tests/test_rtds_price_parsing.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e277b99..2448435 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -171,6 +171,7 @@ set(POLYMARKET_CLIENT_SOURCES src/user_stream_subscription.cpp src/public_stream.cpp src/rtds_parsing.cpp + src/rtds_price_parsing.cpp src/rtds_stream.cpp src/sports_parsing.cpp src/sports_stream.cpp diff --git a/cmake/PolymarketTests.cmake b/cmake/PolymarketTests.cmake index 331d667..c517fc2 100644 --- a/cmake/PolymarketTests.cmake +++ b/cmake/PolymarketTests.cmake @@ -288,6 +288,11 @@ if(POLYMARKET_CLIENT_BUILD_TESTS) target_link_libraries(test_rtds_stream PRIVATE polymarket::client) add_test(NAME test_rtds_stream COMMAND test_rtds_stream) + add_executable(test_rtds_price_parsing tests/test_rtds_price_parsing.cpp) + target_include_directories(test_rtds_price_parsing PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + target_link_libraries(test_rtds_price_parsing PRIVATE polymarket::client) + add_test(NAME test_rtds_price_parsing COMMAND test_rtds_price_parsing) + add_executable(test_network_routing tests/test_network_routing.cpp) target_link_libraries(test_network_routing PRIVATE polymarket::client) add_test(NAME test_network_routing COMMAND test_network_routing) diff --git a/include/polymarket/rtds_stream.hpp b/include/polymarket/rtds_stream.hpp index dc15f84..19d6b99 100644 --- a/include/polymarket/rtds_stream.hpp +++ b/include/polymarket/rtds_stream.hpp @@ -42,8 +42,62 @@ namespace polymarket nlohmann::json raw; }; + // Live crypto prices from one source: binance (USDT pairs such as btcusdt) + // or chainlink (pairs such as btc/usd). Every symbol when empty. + struct RtdsCryptoPriceFilter + { + std::string source; + std::vector symbols; + }; + + // Chainlink TWAPs over a 30- or 60-second window. Every symbol when empty. + struct RtdsCryptoTwapFilter + { + int window_seconds{60}; + std::vector symbols; + }; + + // Pyth prices for one symbol, matched case-insensitively. `types` holds + // update and/or subscribe (history snapshots); both when empty. + struct RtdsEquityPriceFilter + { + std::string symbol; + std::vector types; + }; + + // `exact_value` is the most precise decimal the frame carries + // (full_accuracy_value, with Chainlink's 18-decimal integers scaled); + // `value` is the same number as a double. + struct RtdsPricePoint + { + std::int64_t timestamp_ms{0}; // Unix milliseconds + double value{0.0}; + std::string exact_value; + }; + + // One price event, following the official py-sdk models. `topic` is + // prices.crypto.binance, prices.crypto.chainlink, + // prices.crypto.chainlink.twap or prices.equity.pyth. An update carries + // `price`; an equity subscribe event carries `history` instead. Equity + // prices use the instrument's quote currency; crypto prices are USD, or + // USDT for Binance. `raw` holds the whole frame as received. + struct RtdsPriceEvent + { + std::string topic; + std::string type; // update or subscribe + std::optional timestamp_ms; + std::string symbol; + RtdsPricePoint price; + std::optional window_seconds; // TWAP + std::optional received_at; // equity, as sent + std::optional is_carried_forward; // equity + std::vector history; // equity subscribe + nlohmann::json raw; + }; + using RtdsSubscriptionId = std::uint64_t; using RtdsCommentCallback = std::function; + using RtdsPriceCallback = std::function; // Fired whenever events may have been missed: every connect, disconnect, // queue overflow or idle reconnect. RTDS does not replay them. using RtdsStreamGapCallback = std::function; @@ -69,6 +123,17 @@ namespace polymarket // Throws std::invalid_argument for an unknown type or parent type. RtdsSubscriptionId subscribe_comments(const RtdsCommentFilter &filter, RtdsCommentCallback callback); + // Price topics are subscribed whole and narrowed here, as the official + // SDKs do, because RTDS keeps one filter per topic and connection. + // The official SDKs deprecate these topics in favour of the + // authenticated price service. Throw std::invalid_argument for an + // unknown source, window or type, or an empty symbol. + RtdsSubscriptionId subscribe_crypto_prices(const RtdsCryptoPriceFilter &filter, + RtdsPriceCallback callback); + RtdsSubscriptionId subscribe_crypto_twap(const RtdsCryptoTwapFilter &filter, + RtdsPriceCallback callback); + RtdsSubscriptionId subscribe_equity_prices(const RtdsEquityPriceFilter &filter, + RtdsPriceCallback callback); // Unknown IDs are ignored. void unsubscribe(RtdsSubscriptionId id); diff --git a/src/rtds_parsing.cpp b/src/rtds_parsing.cpp index c7dd88e..7cdec5a 100644 --- a/src/rtds_parsing.cpp +++ b/src/rtds_parsing.cpp @@ -53,7 +53,7 @@ namespace polymarket::detail if (event.topic == comments_topic) event.comment = parse_rtds_comment_event(frame); else - throw std::invalid_argument("unknown RTDS topic " + event.topic); + event.price = parse_rtds_price_event(frame, event.topic); return event; } diff --git a/src/rtds_parsing.hpp b/src/rtds_parsing.hpp index 46b5819..ede89ff 100644 --- a/src/rtds_parsing.hpp +++ b/src/rtds_parsing.hpp @@ -26,6 +26,16 @@ namespace polymarket::detail std::string topic; // wire topic std::string type; std::optional comment; + std::optional price; + }; + + // A price subscription with its source checked and symbols deduplicated. + struct RtdsPriceMatch + { + std::string topic; // API topic + std::vector symbols; + std::optional window_seconds; + std::vector types; }; // Unset for the empty frame RTDS sends after connecting. @@ -40,4 +50,15 @@ namespace polymarket::detail RtdsCommentFilter normalized_comment_filter(const RtdsCommentFilter &filter); std::vector rtds_comment_keys(const RtdsCommentFilter &normalized); bool rtds_comment_matches(const RtdsCommentFilter &normalized, const RtdsCommentEvent &event); + + // "-12345" with 18 implied decimals becomes "-0.000000000000012345". + std::string scaled_e18_decimal(const std::string &integer); + RtdsPriceEvent parse_rtds_price_event(const json &frame, const std::string &wire_topic); + + // Checked filters as matches; throw std::invalid_argument. + RtdsPriceMatch rtds_crypto_price_match(const RtdsCryptoPriceFilter &filter); + RtdsPriceMatch rtds_crypto_twap_match(const RtdsCryptoTwapFilter &filter); + RtdsPriceMatch rtds_equity_price_match(const RtdsEquityPriceFilter &filter); + std::vector rtds_price_keys(const RtdsPriceMatch &match); + bool rtds_price_matches(const RtdsPriceMatch &match, const RtdsPriceEvent &event); } // namespace polymarket::detail diff --git a/src/rtds_price_parsing.cpp b/src/rtds_price_parsing.cpp new file mode 100644 index 0000000..be0e0ca --- /dev/null +++ b/src/rtds_price_parsing.cpp @@ -0,0 +1,221 @@ +#include "rest_numeric.hpp" +#include "rtds_parsing.hpp" +#include "stream_time.hpp" + +#include +#include +#include +#include + +// Price topics, following py-sdk models/rtds_events.py and +// _internal/streams/rtds/protocol.py. +namespace polymarket::detail +{ + namespace + { + constexpr const char *binance_topic = "prices.crypto.binance"; + constexpr const char *chainlink_topic = "prices.crypto.chainlink"; + constexpr const char *twap_topic = "prices.crypto.chainlink.twap"; + constexpr const char *equity_topic = "prices.equity.pyth"; + + bool signed_integer(const std::string &text) + { + const auto digits = text.starts_with('-') ? text.substr(1) : text; + return !digits.empty() && digits.find_first_not_of("0123456789") == std::string::npos; + } + + std::string lowercase(std::string text) + { + std::transform(text.begin(), text.end(), text.begin(), + [](unsigned char c) { return std::tolower(c); }); + return text; + } + + std::string required_decimal(const json &item, const char *key) + { + auto text = field_decimal_text(item, key); + if (text.empty()) throw_bad_field(key, "a decimal number"); + return text; + } + + std::int64_t required_integer(const json &item, const char *key) + { + const auto value = field_integer(item, key); + if (!value) throw_bad_field(key, "an integer"); + return *value; + } + + RtdsPricePoint point(std::int64_t timestamp_ms, std::string exact) + { + const auto value = decimal_string_to_double(exact); + return {timestamp_ms, value, std::move(exact)}; + } + + // The full-accuracy text when it is valid for the source, else `value`. + std::string crypto_exact(const json &payload, bool chainlink) + { + const auto value = required_decimal(payload, "value"); + const auto *full = find_field(payload, "full_accuracy_value"); + if (full == nullptr || !full->is_string()) return value; + const auto &text = full->get_ref(); + if (chainlink) return signed_integer(text) ? scaled_e18_decimal(text) : value; + return is_decimal_number(text) ? text : value; + } + + const json &payload_object(const json &frame) + { + const auto *payload = find_field(frame, "payload"); + if (payload == nullptr || !payload->is_object()) + throw_bad_field("payload", "an object"); + return *payload; + } + + std::vector distinct_symbols(const std::vector &symbols) + { + std::vector unique; + for (const auto &symbol : symbols) + { + if (symbol.empty()) throw std::invalid_argument("symbol must be non-empty"); + if (std::find(unique.begin(), unique.end(), symbol) == unique.end()) + unique.push_back(symbol); + } + return unique; + } + } // namespace + + std::string scaled_e18_decimal(const std::string &integer) + { + if (!signed_integer(integer)) + throw std::invalid_argument("full_accuracy_value must be a signed integer string"); + const bool negative = integer.starts_with('-'); + auto digits = negative ? integer.substr(1) : integer; + digits.erase(0, std::min(digits.find_first_not_of('0'), digits.size() - 1)); + if (digits.size() <= 18) digits.insert(0, 19 - digits.size(), '0'); + auto whole = digits.substr(0, digits.size() - 18); + auto fraction = digits.substr(digits.size() - 18); + fraction.erase(fraction.find_last_not_of('0') + 1); + auto text = whole + (fraction.empty() ? "" : "." + fraction); + return negative && text != "0" ? "-" + text : text; + } + + RtdsPriceEvent parse_rtds_price_event(const json &frame, const std::string &wire_topic) + { + RtdsPriceEvent event; + event.type = field_required_string(frame, "type"); + event.timestamp_ms = stream_epoch_ms_or_iso(frame, "timestamp"); + const auto &payload = payload_object(frame); + event.symbol = field_required_string(payload, "symbol"); + if (wire_topic == "equity_prices") + { + event.topic = equity_topic; + if (event.type == "subscribe") + { + for (const auto &entry : field_objects(payload, "data")) + event.history.push_back(point(required_integer(entry, "timestamp"), + required_decimal(entry, "value"))); + if (find_field(payload, "data") == nullptr) throw_bad_field("data", "an array"); + } + else if (event.type == "update") + { + const auto *full = find_field(payload, "full_accuracy_value"); + event.price = point( + required_integer(payload, "timestamp"), + required_decimal(payload, full != nullptr ? "full_accuracy_value" : "value")); + event.received_at = field_integer(payload, "received_at"); + event.is_carried_forward = field_bool(payload, "is_carried_forward"); + } + else + { + throw_bad_field("type", "update or subscribe"); + } + event.raw = frame; + return event; + } + + if (event.type != "update") throw_bad_field("type", "update"); + const auto timestamp = required_integer(payload, "timestamp"); + if (wire_topic == "crypto_prices_twap_thirty" || wire_topic == "crypto_prices_twap_sixty") + { + event.topic = twap_topic; + const int expected = wire_topic == "crypto_prices_twap_thirty" ? 30 : 60; + const auto window = field_integer(payload, "window_s"); + if (!window || *window != expected) + throw std::invalid_argument("RTDS TWAP topic " + wire_topic + + " requires window_s=" + std::to_string(expected)); + event.window_seconds = expected; + (void)required_decimal(payload, "value"); + const auto *full = find_field(payload, "full_accuracy_value"); + if (full == nullptr || !full->is_string()) + throw_bad_field("full_accuracy_value", "a signed integer string"); + event.price = point(timestamp, scaled_e18_decimal(full->get())); + } + else if (wire_topic == "crypto_prices" || wire_topic == "crypto_prices_chainlink") + { + const bool chainlink = wire_topic == "crypto_prices_chainlink"; + event.topic = chainlink ? chainlink_topic : binance_topic; + event.price = point(timestamp, crypto_exact(payload, chainlink)); + } + else + { + throw std::invalid_argument("unknown RTDS topic " + wire_topic); + } + event.raw = frame; + return event; + } + + RtdsPriceMatch rtds_crypto_price_match(const RtdsCryptoPriceFilter &filter) + { + if (filter.source != "binance" && filter.source != "chainlink") + throw std::invalid_argument("source must be binance or chainlink"); + return {filter.source == "binance" ? binance_topic : chainlink_topic, + distinct_symbols(filter.symbols), + std::nullopt, + {}}; + } + + RtdsPriceMatch rtds_crypto_twap_match(const RtdsCryptoTwapFilter &filter) + { + if (filter.window_seconds != 30 && filter.window_seconds != 60) + throw std::invalid_argument("window_seconds must be 30 or 60"); + return {twap_topic, distinct_symbols(filter.symbols), filter.window_seconds, {}}; + } + + RtdsPriceMatch rtds_equity_price_match(const RtdsEquityPriceFilter &filter) + { + if (filter.symbol.empty()) throw std::invalid_argument("symbol must be a non-empty string"); + RtdsPriceMatch match{equity_topic, {lowercase(filter.symbol)}, std::nullopt, {}}; + for (const auto &type : filter.types) + { + if (type != "update" && type != "subscribe") + throw std::invalid_argument("invalid equity event type: " + type); + if (std::find(match.types.begin(), match.types.end(), type) == match.types.end()) + match.types.push_back(type); + } + return match; + } + + std::vector rtds_price_keys(const RtdsPriceMatch &match) + { + if (match.topic == binance_topic) return {{"crypto_prices", "update", ""}}; + if (match.topic == chainlink_topic) return {{"crypto_prices_chainlink", "update", ""}}; + if (match.topic == twap_topic) + return {{match.window_seconds == 30 ? "crypto_prices_twap_thirty" + : "crypto_prices_twap_sixty", + "update", ""}}; + return {{"equity_prices", "update", ""}}; + } + + bool rtds_price_matches(const RtdsPriceMatch &match, const RtdsPriceEvent &event) + { + if (event.topic != match.topic) return false; + if (match.topic == equity_topic) + { + return lowercase(event.symbol) == match.symbols.front() && + (match.types.empty() || std::find(match.types.begin(), match.types.end(), + event.type) != match.types.end()); + } + if (match.window_seconds && event.window_seconds != match.window_seconds) return false; + return match.symbols.empty() || std::find(match.symbols.begin(), match.symbols.end(), + event.symbol) != match.symbols.end(); + } +} // namespace polymarket::detail diff --git a/src/rtds_stream.cpp b/src/rtds_stream.cpp index 1805c1c..40730cc 100644 --- a/src/rtds_stream.cpp +++ b/src/rtds_stream.cpp @@ -200,6 +200,41 @@ namespace polymarket }); } + namespace + { + RtdsSubscriptionId add_price(detail::RtdsStreamRuntime &runtime, + detail::RtdsPriceMatch match, RtdsPriceCallback callback) + { + auto keys = detail::rtds_price_keys(match); + return runtime.add( + std::move(keys), + [match = std::move(match), + callback = std::move(callback)](const detail::RtdsParsedEvent &event) + { + if (callback && event.price && detail::rtds_price_matches(match, *event.price)) + callback(*event.price); + }); + } + } // namespace + + RtdsSubscriptionId RtdsStream::subscribe_crypto_prices(const RtdsCryptoPriceFilter &filter, + RtdsPriceCallback callback) + { + return add_price(*runtime_, detail::rtds_crypto_price_match(filter), std::move(callback)); + } + + RtdsSubscriptionId RtdsStream::subscribe_crypto_twap(const RtdsCryptoTwapFilter &filter, + RtdsPriceCallback callback) + { + return add_price(*runtime_, detail::rtds_crypto_twap_match(filter), std::move(callback)); + } + + RtdsSubscriptionId RtdsStream::subscribe_equity_prices(const RtdsEquityPriceFilter &filter, + RtdsPriceCallback callback) + { + return add_price(*runtime_, detail::rtds_equity_price_match(filter), std::move(callback)); + } + void RtdsStream::unsubscribe(RtdsSubscriptionId id) { runtime_->remove(id); diff --git a/tests/fuzz/corpus/websocket_messages/rtds_chainlink_update b/tests/fuzz/corpus/websocket_messages/rtds_chainlink_update new file mode 100644 index 0000000..c4d6e93 --- /dev/null +++ b/tests/fuzz/corpus/websocket_messages/rtds_chainlink_update @@ -0,0 +1 @@ +{"connection_id":"example-connection","payload":{"full_accuracy_value":"81529465877486595000000","symbol":"btc/usd","timestamp":1792287296000,"value":81529.4658774866},"timestamp":1792287296856,"topic":"crypto_prices_chainlink","type":"update"} \ No newline at end of file diff --git a/tests/fuzz/corpus/websocket_messages/rtds_equity_history b/tests/fuzz/corpus/websocket_messages/rtds_equity_history new file mode 100644 index 0000000..8227226 --- /dev/null +++ b/tests/fuzz/corpus/websocket_messages/rtds_equity_history @@ -0,0 +1 @@ +{"connection_id":"example-connection","payload":{"symbol":"aapl","data":[{"timestamp":1792330140000,"value":231.3},{"timestamp":1792330170000,"value":"231.35"}]},"timestamp":1792330200100,"topic":"equity_prices","type":"subscribe"} \ No newline at end of file diff --git a/tests/fuzz/corpus/websocket_messages/rtds_twap_sixty_update b/tests/fuzz/corpus/websocket_messages/rtds_twap_sixty_update new file mode 100644 index 0000000..cba4ca1 --- /dev/null +++ b/tests/fuzz/corpus/websocket_messages/rtds_twap_sixty_update @@ -0,0 +1 @@ +{"connection_id":"example-connection","payload":{"full_accuracy_value":"81527437376226865446912","symbol":"btc/usd","timestamp":1792287328000,"value":81527.43737622687,"window_s":60},"timestamp":1792287329255,"topic":"crypto_prices_twap_sixty","type":"update"} \ No newline at end of file diff --git a/tests/fuzz/fuzz_websocket_messages.cpp b/tests/fuzz/fuzz_websocket_messages.cpp index 6a2eede..092a9ce 100644 --- a/tests/fuzz/fuzz_websocket_messages.cpp +++ b/tests/fuzz/fuzz_websocket_messages.cpp @@ -83,6 +83,9 @@ extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size const auto event = detail::parse_rtds_message(message); if (event && event->comment && !event->comment->comment && !event->comment->reaction) std::abort(); + if (event && event->price && event->price->type == "update" && + event->price->price.exact_value.empty()) + std::abort(); }); return 0; } diff --git a/tests/rtds_price_fixtures.hpp b/tests/rtds_price_fixtures.hpp new file mode 100644 index 0000000..34f7c7a --- /dev/null +++ b/tests/rtds_price_fixtures.hpp @@ -0,0 +1,20 @@ +#pragma once + +// Crypto frames recorded from wss://ws-live-data.polymarket.com on 2026-10-18, with +// the connection ID replaced. The equity frames are synthetic, built from the py-sdk +// models, because equity_prices was silent while markets were closed. +namespace rtds_fixtures +{ + inline constexpr const char *rtds_binance_update = + R"json({"connection_id":"example-connection","payload":{"full_accuracy_value":"81633.21000000","symbol":"btcusdt","timestamp":1792287281000,"value":81633.21},"timestamp":1792287281124,"topic":"crypto_prices","type":"update"})json"; + inline constexpr const char *rtds_chainlink_update = + R"json({"connection_id":"example-connection","payload":{"full_accuracy_value":"81529465877486595000000","symbol":"btc/usd","timestamp":1792287296000,"value":81529.4658774866},"timestamp":1792287296856,"topic":"crypto_prices_chainlink","type":"update"})json"; + inline constexpr const char *rtds_twap_thirty_update = + R"json({"connection_id":"example-connection","payload":{"full_accuracy_value":"81529202885321459499008","symbol":"btc/usd","timestamp":1792287312000,"value":81529.20288532146,"window_s":30},"timestamp":1792287313142,"topic":"crypto_prices_twap_thirty","type":"update"})json"; + inline constexpr const char *rtds_twap_sixty_update = + R"json({"connection_id":"example-connection","payload":{"full_accuracy_value":"81527437376226865446912","symbol":"btc/usd","timestamp":1792287328000,"value":81527.43737622687,"window_s":60},"timestamp":1792287329255,"topic":"crypto_prices_twap_sixty","type":"update"})json"; + inline constexpr const char *rtds_equity_update = + R"json({"connection_id":"example-connection","payload":{"symbol":"AAPL","timestamp":1792330200000,"value":231.42,"full_accuracy_value":"231.4215","received_at":1792330200150,"is_carried_forward":false},"timestamp":1792330200180,"topic":"equity_prices","type":"update"})json"; + inline constexpr const char *rtds_equity_history = + R"json({"connection_id":"example-connection","payload":{"symbol":"aapl","data":[{"timestamp":1792330140000,"value":231.3},{"timestamp":1792330170000,"value":"231.35"}]},"timestamp":1792330200100,"topic":"equity_prices","type":"subscribe"})json"; +} // namespace rtds_fixtures diff --git a/tests/test_rtds_price_parsing.cpp b/tests/test_rtds_price_parsing.cpp new file mode 100644 index 0000000..72fb734 --- /dev/null +++ b/tests/test_rtds_price_parsing.cpp @@ -0,0 +1,199 @@ +#include "check_support.hpp" +#include "rtds_parsing.hpp" +#include "rtds_price_fixtures.hpp" + +#include +#include +#include +#include + +namespace +{ + using check_support::check; + using check_support::expect_equal; + namespace detail = polymarket::detail; + + void expect_rejected(const std::string &name, const std::function &parse) + { + try + { + parse(); + } + catch (const std::exception &) + { + return; + } + check(false, name + " must be rejected"); + } + + polymarket::RtdsPriceEvent price(const std::string &frame) + { + return *detail::parse_rtds_message(frame)->price; + } + + std::string with_payload(const char *frame, const char *key, const nlohmann::json &value) + { + auto parsed = nlohmann::json::parse(frame); + parsed["payload"][key] = value; + return parsed.dump(); + } + + void test_scaling() + { + expect_equal("whole", detail::scaled_e18_decimal("81529465877486595000000"), + "81529.465877486595"); + expect_equal("integer", detail::scaled_e18_decimal("2000000000000000000"), "2"); + expect_equal("fraction", detail::scaled_e18_decimal("12345"), "0.000000000000012345"); + expect_equal("negative", detail::scaled_e18_decimal("-1500000000000000000"), "-1.5"); + expect_equal("zero", detail::scaled_e18_decimal("-000"), "0"); + expect_rejected("a decimal input", [] { (void)detail::scaled_e18_decimal("1.5"); }); + expect_rejected("an empty input", [] { (void)detail::scaled_e18_decimal("-"); }); + } + + void test_recorded_crypto() + { + const auto binance = price(rtds_fixtures::rtds_binance_update); + check(binance.topic == "prices.crypto.binance" && binance.type == "update" && + binance.symbol == "btcusdt" && binance.price.timestamp_ms == 1792287281000 && + binance.price.exact_value == "81633.21000000" && + binance.price.value == 81633.21 && binance.timestamp_ms == 1792287281124 && + !binance.window_seconds, + "a recorded Binance price must parse"); + + const auto chainlink = price(rtds_fixtures::rtds_chainlink_update); + check(chainlink.topic == "prices.crypto.chainlink" && chainlink.symbol == "btc/usd" && + chainlink.price.exact_value == "81529.465877486595" && + std::abs(chainlink.price.value - 81529.4658774866) < 1e-6, + "a Chainlink price must scale its full-accuracy value"); + + const auto thirty = price(rtds_fixtures::rtds_twap_thirty_update); + const auto sixty = price(rtds_fixtures::rtds_twap_sixty_update); + check(thirty.topic == "prices.crypto.chainlink.twap" && thirty.window_seconds == 30 && + thirty.price.exact_value == "81529.202885321459499008" && + sixty.window_seconds == 60 && + sixty.price.exact_value == "81527.437376226865446912", + "TWAPs must carry their window and scaled value"); + + const auto fallback = + price(with_payload(rtds_fixtures::rtds_chainlink_update, "full_accuracy_value", "n/a")); + check(fallback.price.exact_value == "81529.4658774866", + "an unusable full-accuracy value must fall back to value"); + } + + void test_equity() + { + const auto update = price(rtds_fixtures::rtds_equity_update); + check(update.topic == "prices.equity.pyth" && update.type == "update" && + update.symbol == "AAPL" && update.price.exact_value == "231.4215" && + update.received_at == 1792330200150 && update.is_carried_forward == false && + update.history.empty(), + "an equity update must prefer its full-accuracy value"); + const auto history = price(rtds_fixtures::rtds_equity_history); + check(history.type == "subscribe" && history.history.size() == 2 && + history.history[0].timestamp_ms == 1792330140000 && + history.history[1].exact_value == "231.35" && history.price.exact_value.empty(), + "an equity snapshot must carry its history"); + } + + void test_matching() + { + const auto binance = price(rtds_fixtures::rtds_binance_update); + const auto chainlink = price(rtds_fixtures::rtds_chainlink_update); + const auto thirty = price(rtds_fixtures::rtds_twap_thirty_update); + const auto equity = price(rtds_fixtures::rtds_equity_update); + const auto history = price(rtds_fixtures::rtds_equity_history); + + const auto all_binance = detail::rtds_crypto_price_match({"binance", {}}); + const auto eth_binance = detail::rtds_crypto_price_match({"binance", {"ethusdt"}}); + check(detail::rtds_price_matches(all_binance, binance) && + !detail::rtds_price_matches(all_binance, chainlink) && + !detail::rtds_price_matches(eth_binance, binance), + "crypto matches must check the source and symbols"); + + const auto twap_thirty = detail::rtds_crypto_twap_match({30, {"btc/usd"}}); + const auto twap_sixty = detail::rtds_crypto_twap_match({60, {}}); + check(detail::rtds_price_matches(twap_thirty, thirty) && + !detail::rtds_price_matches(twap_sixty, thirty) && + !detail::rtds_price_matches(twap_thirty, chainlink), + "TWAP matches must check the window"); + + const auto aapl = detail::rtds_equity_price_match({"aapl", {}}); + const auto aapl_updates = detail::rtds_equity_price_match({"Aapl", {"update", "update"}}); + check( + detail::rtds_price_matches(aapl, equity) && detail::rtds_price_matches(aapl, history) && + detail::rtds_price_matches(aapl_updates, equity) && + !detail::rtds_price_matches(aapl_updates, history) && + aapl_updates.types.size() == 1 && + !detail::rtds_price_matches(detail::rtds_equity_price_match({"msft", {}}), equity), + "equity matches must compare symbols case-insensitively and check types"); + + const auto keys = detail::rtds_price_keys(twap_thirty); + check(keys.size() == 1 && keys[0].topic == "crypto_prices_twap_thirty" && + keys[0].type == "update" && keys[0].filters.empty() && + detail::rtds_price_keys(aapl)[0].topic == "equity_prices", + "price keys must subscribe whole topics"); + } + + void test_rejections() + { + expect_rejected("an unknown source", + [] { (void)detail::rtds_crypto_price_match({"coinbase", {}}); }); + expect_rejected("an empty symbol", + [] { (void)detail::rtds_crypto_price_match({"binance", {""}}); }); + expect_rejected("a 45-second window", + [] { (void)detail::rtds_crypto_twap_match({45, {}}); }); + expect_rejected("an empty equity symbol", + [] { (void)detail::rtds_equity_price_match({"", {}}); }); + expect_rejected("an unknown equity type", + [] { (void)detail::rtds_equity_price_match({"aapl", {"snapshot"}}); }); + + expect_rejected( + "a TWAP with the wrong window", [] + { (void)price(with_payload(rtds_fixtures::rtds_twap_thirty_update, "window_s", 60)); }); + expect_rejected("a TWAP without full accuracy", + [] + { + (void)price(with_payload(rtds_fixtures::rtds_twap_sixty_update, + "full_accuracy_value", nullptr)); + }); + expect_rejected("a TWAP with decimal full accuracy", + [] + { + (void)price(with_payload(rtds_fixtures::rtds_twap_sixty_update, + "full_accuracy_value", "1.5")); + }); + expect_rejected( + "a price without a value", [] + { (void)price(with_payload(rtds_fixtures::rtds_binance_update, "value", nullptr)); }); + expect_rejected( + "a fractional timestamp", [] + { (void)price(with_payload(rtds_fixtures::rtds_binance_update, "timestamp", 1.5)); }); + expect_rejected("a crypto snapshot type", + [] + { + auto frame = nlohmann::json::parse(rtds_fixtures::rtds_binance_update); + frame["type"] = "subscribe"; + (void)price(frame.dump()); + }); + expect_rejected( + "an equity snapshot without data", + [] + { + (void)price( + R"({"topic":"equity_prices","type":"subscribe","payload":{"symbol":"aapl"}})"); + }); + expect_rejected( + "an unknown topic", [] + { (void)price(R"({"topic":"activity","type":"trades","payload":{"symbol":"x"}})"); }); + } +} // namespace + +int main() +{ + test_scaling(); + test_recorded_crypto(); + test_equity(); + test_matching(); + test_rejections(); + return check_support::finish("test_rtds_price_parsing"); +} diff --git a/tests/test_rtds_stream.cpp b/tests/test_rtds_stream.cpp index bf80326..4a6550c 100644 --- a/tests/test_rtds_stream.cpp +++ b/tests/test_rtds_stream.cpp @@ -1,5 +1,6 @@ #include "polymarket/rtds_stream.hpp" #include "rtds_fixtures.hpp" +#include "rtds_price_fixtures.hpp" #include "websocket_test_server.hpp" #include @@ -148,6 +149,58 @@ namespace "a removed subscription must stop receiving events"); } + bool test_price_subscriptions() + { + constexpr const char *binance_frame = + R"({"action":"subscribe","subscriptions":[{"topic":"crypto_prices","type":"update"}]})"; + constexpr const char *twap_frame = + R"({"action":"subscribe","subscriptions":[{"topic":"crypto_prices_twap_sixty","type":"update"}]})"; + constexpr const char *binance_unsubscribe = + R"({"action":"unsubscribe","subscriptions":[{"topic":"crypto_prices","type":"update"}]})"; + LocalWebSocketServer server; + polymarket::RtdsStream stream(local_config(server.url())); + const bool connected = stream.connect(); + std::mutex mutex; + std::vector received; + const auto record = [&](const char *name) + { + return [&, name](const polymarket::RtdsPriceEvent &event) + { + std::lock_guard lock(mutex); + received.push_back(std::string(name) + ":" + event.symbol); + }; + }; + const auto btc = stream.subscribe_crypto_prices({"binance", {"btcusdt"}}, record("btc")); + const auto eth = stream.subscribe_crypto_prices({"binance", {"ethusdt"}}, record("eth")); + stream.subscribe_crypto_twap({60, {}}, record("twap")); + const bool subscribed = server.wait_for_message_count(binance_frame, 1, 2s) && + server.wait_for_message_count(twap_frame, 1, 2s); + const bool shared = !server.wait_for_message_count(binance_frame, 2, 200ms); + + server.send_to_clients(rtds_fixtures::rtds_binance_update); + server.send_to_clients(rtds_fixtures::rtds_twap_thirty_update); + server.send_to_clients(rtds_fixtures::rtds_twap_sixty_update); + const bool delivered = wait_until( + [&] + { + std::lock_guard lock(mutex); + return received.size() == 2; + }, + 2s); + stream.unsubscribe(btc); + const bool still_shared = !server.wait_for_message_count(binance_unsubscribe, 1, 200ms); + stream.unsubscribe(eth); + const bool released = server.wait_for_message_count(binance_unsubscribe, 1, 2s); + stream.disconnect(); + std::lock_guard lock(mutex); + return check(connected && subscribed && shared, + "price subscriptions must share one whole-topic subscription") && + check(delivered && received[0] == "btc:btcusdt" && received[1] == "twap:btc/usd", + "prices must be narrowed by symbol and window") && + check(still_shared && released, + "a topic must be unsubscribed only when its last subscription goes"); + } + bool test_invalid_arguments() { polymarket::RtdsStream stream; @@ -173,14 +226,24 @@ namespace { empty_rejected = true; } - return check(rejected && empty_rejected, "invalid filters and URLs must be rejected"); + bool price_rejected = false; + try + { + stream.subscribe_crypto_twap({45, {}}, {}); + } + catch (const std::invalid_argument &) + { + price_rejected = true; + } + return check(rejected && empty_rejected && price_rejected, + "invalid filters and URLs must be rejected"); } } // namespace int main() { - const bool passed = - test_subscriptions_and_delivery() && test_diffs_and_replay() && test_invalid_arguments(); + const bool passed = test_subscriptions_and_delivery() && test_diffs_and_replay() && + test_price_subscriptions() && test_invalid_arguments(); if (!passed) return 1; std::cout << "test_rtds_stream passed\n"; return 0; From 0a287343ee91a1831055800fbd4b90d262836e86 Mon Sep 17 00:00:00 2001 From: yluoc Date: Sat, 17 Oct 2026 22:19:07 -0400 Subject: [PATCH 06/11] style(rtds): satisfy clang-tidy performance checks --- src/rtds_parsing.cpp | 1 + src/rtds_price_parsing.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rtds_parsing.cpp b/src/rtds_parsing.cpp index 7cdec5a..c1ef4e1 100644 --- a/src/rtds_parsing.cpp +++ b/src/rtds_parsing.cpp @@ -92,6 +92,7 @@ namespace polymarket::detail std::vector rtds_comment_keys(const RtdsCommentFilter &normalized) { std::vector keys; + keys.reserve(normalized.types.size()); for (const auto &type : normalized.types) keys.push_back({comments_topic, type, ""}); return keys; diff --git a/src/rtds_price_parsing.cpp b/src/rtds_price_parsing.cpp index be0e0ca..34470e6 100644 --- a/src/rtds_price_parsing.cpp +++ b/src/rtds_price_parsing.cpp @@ -54,7 +54,7 @@ namespace polymarket::detail // The full-accuracy text when it is valid for the source, else `value`. std::string crypto_exact(const json &payload, bool chainlink) { - const auto value = required_decimal(payload, "value"); + auto value = required_decimal(payload, "value"); const auto *full = find_field(payload, "full_accuracy_value"); if (full == nullptr || !full->is_string()) return value; const auto &text = full->get_ref(); From 4f3ef7f4b1d2e1b74ebc36a01a979e97570ce1e5 Mon Sep 17 00:00:00 2001 From: yluoc Date: Sat, 17 Oct 2026 22:47:36 -0400 Subject: [PATCH 07/11] fix: avoid GCC 11 leaks when building structs from throwing parsers --- src/gamma_market_parsing.cpp | 14 ++++++++++---- src/gamma_social_parsing.cpp | 8 ++++++-- src/rtds_price_parsing.cpp | 17 ++++++++++++----- tests/test_gamma_social.cpp | 2 +- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/gamma_market_parsing.cpp b/src/gamma_market_parsing.cpp index aa325ba..491ab76 100644 --- a/src/gamma_market_parsing.cpp +++ b/src/gamma_market_parsing.cpp @@ -30,14 +30,20 @@ namespace polymarket::detail GammaEventReference event_reference(const json &item) { - return {field_required_id(item, "id"), field_string(item, "slug"), - field_string(item, "title")}; + GammaEventReference reference; + reference.id = field_required_id(item, "id"); + reference.slug = field_string(item, "slug"); + reference.title = field_string(item, "title"); + return reference; } GammaSeriesReference series_reference(const json &item) { - return {field_required_id(item, "id"), field_string(item, "slug"), - field_string(item, "title")}; + GammaSeriesReference reference; + reference.id = field_required_id(item, "id"); + reference.slug = field_string(item, "slug"); + reference.title = field_string(item, "title"); + return reference; } } // namespace diff --git a/src/gamma_social_parsing.cpp b/src/gamma_social_parsing.cpp index 348626c..9af1869 100644 --- a/src/gamma_social_parsing.cpp +++ b/src/gamma_social_parsing.cpp @@ -44,8 +44,12 @@ namespace polymarket::detail profile.is_mod = field_bool(value, "isMod"); profile.is_creator = field_bool(value, "isCreator"); for (const auto &position : field_objects(value, "positions")) - profile.positions.push_back( - {field_id(position, "tokenId"), integer_or_text(position, "positionSize")}); + { + GammaCommentPosition entry; + entry.position_size = integer_or_text(position, "positionSize"); + entry.token_id = field_id(position, "tokenId"); + profile.positions.push_back(std::move(entry)); + } return profile; } diff --git a/src/rtds_price_parsing.cpp b/src/rtds_price_parsing.cpp index 34470e6..18993cb 100644 --- a/src/rtds_price_parsing.cpp +++ b/src/rtds_price_parsing.cpp @@ -6,6 +6,7 @@ #include #include #include +#include // Price topics, following py-sdk models/rtds_events.py and // _internal/streams/rtds/protocol.py. @@ -167,17 +168,23 @@ namespace polymarket::detail { if (filter.source != "binance" && filter.source != "chainlink") throw std::invalid_argument("source must be binance or chainlink"); - return {filter.source == "binance" ? binance_topic : chainlink_topic, - distinct_symbols(filter.symbols), - std::nullopt, - {}}; + auto symbols = distinct_symbols(filter.symbols); + RtdsPriceMatch match; + match.topic = filter.source == "binance" ? binance_topic : chainlink_topic; + match.symbols = std::move(symbols); + return match; } RtdsPriceMatch rtds_crypto_twap_match(const RtdsCryptoTwapFilter &filter) { if (filter.window_seconds != 30 && filter.window_seconds != 60) throw std::invalid_argument("window_seconds must be 30 or 60"); - return {twap_topic, distinct_symbols(filter.symbols), filter.window_seconds, {}}; + auto symbols = distinct_symbols(filter.symbols); + RtdsPriceMatch match; + match.topic = twap_topic; + match.symbols = std::move(symbols); + match.window_seconds = filter.window_seconds; + return match; } RtdsPriceMatch rtds_equity_price_match(const RtdsEquityPriceFilter &filter) diff --git a/tests/test_gamma_social.cpp b/tests/test_gamma_social.cpp index 0cfb15b..738aa73 100644 --- a/tests/test_gamma_social.cpp +++ b/tests/test_gamma_social.cpp @@ -109,7 +109,7 @@ namespace clob_test [] { (void)detail::parse_gamma_comments( - R"([{"id":"1","profile":{"positions":[{"positionSize":"4.5"}]}}])"); + R"([{"id":"1","profile":{"positions":[{"tokenId":"99390970186095476158604553843389938788064583141526825109668507201084266675707","positionSize":"4.5"}]}}])"); }), "a fractional position size must be rejected") && check(rejected([] { (void)detail::parse_gamma_public_profile(R"([])"); }), From 43095854dcab39a7b93c1bbc1c107a83a5b90049 Mon Sep 17 00:00:00 2001 From: yluoc Date: Sat, 17 Oct 2026 22:48:02 -0400 Subject: [PATCH 08/11] test: widen heartbeat timing margin and report counts on failure --- tests/test_websocket_heartbeat.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_websocket_heartbeat.cpp b/tests/test_websocket_heartbeat.cpp index 2420bd2..e49719a 100644 --- a/tests/test_websocket_heartbeat.cpp +++ b/tests/test_websocket_heartbeat.cpp @@ -168,23 +168,30 @@ namespace LocalWebSocketServer server; polymarket::WebSocketClient client; auto options = quiet_options(); - options.idle_timeout_ms = 400; + options.idle_timeout_ms = 2000; options.server_ping_message = "ping"; options.server_ping_reply = "pong"; client.configure(options); client.set_url(server.url()); client.connect(); const bool connected = client.wait_until_connected(2s); - for (int i = 0; i < 12; ++i) + const auto until = std::chrono::steady_clock::now() + 3s; + int pings = 0; + while (std::chrono::steady_clock::now() < until) { server.send_to_clients("ping"); + pings++; std::this_thread::sleep_for(100ms); } + const bool ponged = server.wait_for_message_count("pong", 1, 2s); const bool still_one = !server.wait_for_connections(2, 200ms); const auto stats = client.stats(); client.disconnect(); - return check(connected && still_one && stats.idle_timeouts == 0, - "server pings must keep an idle timeout from firing"); + return check(connected && ponged && still_one && stats.idle_timeouts == 0, + "server pings must keep an idle timeout from firing (pings " + + std::to_string(pings) + ", reconnects " + + std::to_string(stats.reconnects) + ", idle timeouts " + + std::to_string(stats.idle_timeouts) + ")"); } } // namespace From bc3c073feb782f50b6446f3acefea681af2c4d96 Mon Sep 17 00:00:00 2001 From: yluoc Date: Sat, 17 Oct 2026 22:48:18 -0400 Subject: [PATCH 09/11] test(evm): allow more time for the dedup replay check under sanitizers --- tests/test_evm_event_indexer.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_evm_event_indexer.cpp b/tests/test_evm_event_indexer.cpp index 85407f5..ef684bc 100644 --- a/tests/test_evm_event_indexer.cpp +++ b/tests/test_evm_event_indexer.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include namespace @@ -221,9 +222,12 @@ int main() const auto before_replay = handed_off_count.load(); handoff_transport->emit(make_indexer_log(108, 0, 0, "0xlater")); handoff_transport->emit(make_indexer_log(175, 0, 0, "0x175")); - require(wait_for_indexer([&]() - { return handed_off_count.load() == before_replay + 1; }), - "dedup cache should prune durable history but retain its recent window"); + require(wait_for_indexer([&]() { return handed_off_count.load() == before_replay + 1; }, + std::chrono::seconds(10)), + ("dedup cache should prune durable history but retain its recent window (saw " + + std::to_string(handed_off_count.load()) + " deliveries, expected " + + std::to_string(before_replay + 1) + ")") + .c_str()); } require(handed_off[0] == std::make_pair(std::string("0xconfirmed"), false), "confirmed catch-up log should be first"); From 92d2866a0b521188c18a88c1ac1e2d661d01df44 Mon Sep 17 00:00:00 2001 From: yluoc Date: Sun, 18 Oct 2026 12:21:16 -0400 Subject: [PATCH 10/11] fix(rtds): stop delivering to subscribers once the stream stops --- src/public_stream.cpp | 9 +++++++-- src/public_stream.hpp | 5 ++++- src/rtds_stream.cpp | 20 +++++++++++--------- src/sports_stream.cpp | 2 +- tests/test_rtds_stream.cpp | 32 +++++++++++++++++++++++++++++++- 5 files changed, 54 insertions(+), 14 deletions(-) diff --git a/src/public_stream.cpp b/src/public_stream.cpp index c9673e8..f7d49e0 100644 --- a/src/public_stream.cpp +++ b/src/public_stream.cpp @@ -97,8 +97,13 @@ namespace polymarket::detail void PublicStream::handle_message(const std::string &message, std::uint64_t generation) { - if (!active() || generation != websocket_.stream_generation()) return; - handler_(message); + if (!is_current(generation)) return; + handler_(message, generation); + } + + bool PublicStream::is_current(std::uint64_t generation) const + { + return active() && generation == websocket_.stream_generation(); } void PublicStream::handle_gap(std::uint64_t generation) diff --git a/src/public_stream.hpp b/src/public_stream.hpp index 5107bcc..2a8bed5 100644 --- a/src/public_stream.hpp +++ b/src/public_stream.hpp @@ -26,7 +26,8 @@ namespace polymarket::detail class PublicStream final : public std::enable_shared_from_this { public: - using MessageHandler = std::function; + using MessageHandler = + std::function; static std::shared_ptr create(const std::string &url, const WebSocketOptions &options, @@ -52,6 +53,8 @@ namespace polymarket::detail void replace_subscriptions(std::vector messages); bool send(const std::string &message); void request_resnapshot(const std::string &reason); + // False once the stream stops or the connection that sent `generation` ends. + bool is_current(std::uint64_t generation) const; private: PublicStream(const std::string &url, const WebSocketOptions &options, diff --git a/src/rtds_stream.cpp b/src/rtds_stream.cpp index 40730cc..5739053 100644 --- a/src/rtds_stream.cpp +++ b/src/rtds_stream.cpp @@ -43,14 +43,13 @@ namespace polymarket { auto runtime = std::shared_ptr(new RtdsStreamRuntime()); std::weak_ptr weak = runtime; - runtime->stream_ = - PublicStream::create(config.rtds_ws_url, rtds_options(), - std::chrono::milliseconds(config.ws_connect_timeout_ms), - [weak](const std::string &message) - { - if (auto self = weak.lock()) - self->handle_message(message); - }); + runtime->stream_ = PublicStream::create( + config.rtds_ws_url, rtds_options(), + std::chrono::milliseconds(config.ws_connect_timeout_ms), + [weak](const std::string &message, std::uint64_t generation) + { + if (auto self = weak.lock()) self->handle_message(message, generation); + }); return runtime; } @@ -141,7 +140,7 @@ namespace polymarket stream_->send(rtds_subscription_frame("unsubscribe", removed)); } - void handle_message(const std::string &message) + void handle_message(const std::string &message, std::uint64_t generation) { std::optional event; try @@ -162,7 +161,10 @@ namespace polymarket targets.push_back(subscription.deliver); } for (const auto &deliver : targets) + { + if (!stream_->is_current(generation)) return; deliver(*event); + } } std::shared_ptr stream_; diff --git a/src/sports_stream.cpp b/src/sports_stream.cpp index c8b18b4..83116af 100644 --- a/src/sports_stream.cpp +++ b/src/sports_stream.cpp @@ -37,7 +37,7 @@ namespace polymarket runtime->stream_ = PublicStream::create(config.sports_ws_url, sports_options(), std::chrono::milliseconds(config.ws_connect_timeout_ms), - [weak](const std::string &message) + [weak](const std::string &message, std::uint64_t) { if (auto self = weak.lock()) self->handle_message(message); diff --git a/tests/test_rtds_stream.cpp b/tests/test_rtds_stream.cpp index 4a6550c..a846755 100644 --- a/tests/test_rtds_stream.cpp +++ b/tests/test_rtds_stream.cpp @@ -201,6 +201,35 @@ namespace "a topic must be unsubscribed only when its last subscription goes"); } + bool test_stop_from_callback() + { + LocalWebSocketServer server; + polymarket::RtdsStream stream(local_config(server.url())); + const bool connected = stream.connect(); + std::atomic first_calls{0}; + std::atomic second_calls{0}; + stream.subscribe_comments({}, + [&](const polymarket::RtdsCommentEvent &) + { + first_calls++; + stream.stop(); + }); + stream.subscribe_comments({}, + [&](const polymarket::RtdsCommentEvent &) { second_calls++; }); + const bool subscribed = server.wait_for_message_count(created_frame, 1, 2s); + + server.send_to_clients(rtds_fixtures::rtds_comment_created); + const bool stopped = wait_until([&] { return first_calls.load() == 1; }, 2s); + std::this_thread::sleep_for(200ms); + stream.disconnect(); + return check(connected && subscribed && stopped, + "the first subscriber must receive the event") && + check(first_calls.load() == 1 && second_calls.load() == 0, + "no subscriber may run after stop() from a callback, got first=" + + std::to_string(first_calls.load()) + + " second=" + std::to_string(second_calls.load())); + } + bool test_invalid_arguments() { polymarket::RtdsStream stream; @@ -243,7 +272,8 @@ namespace int main() { const bool passed = test_subscriptions_and_delivery() && test_diffs_and_replay() && - test_price_subscriptions() && test_invalid_arguments(); + test_price_subscriptions() && test_stop_from_callback() && + test_invalid_arguments(); if (!passed) return 1; std::cout << "test_rtds_stream passed\n"; return 0; From 0c89c82b04d31979a87647e537a09c0906f49b4c Mon Sep 17 00:00:00 2001 From: yluoc Date: Sun, 18 Oct 2026 12:44:25 -0400 Subject: [PATCH 11/11] fix(streams): report malformed Sports and RTDS events as gaps --- include/polymarket/rtds_stream.hpp | 9 +++-- include/polymarket/sports_stream.hpp | 8 +++-- src/public_stream.cpp | 8 +++++ src/public_stream.hpp | 2 ++ src/rtds_parsing.cpp | 10 ++++++ src/rtds_parsing.hpp | 3 ++ src/rtds_price_parsing.cpp | 7 ++++ src/rtds_stream.cpp | 5 ++- src/sports_stream.cpp | 21 +++++------ tests/test_rtds_stream.cpp | 54 +++++++++++++++++++++++++++- tests/test_sports_stream.cpp | 53 ++++++++++++++++++++++++--- 11 files changed, 157 insertions(+), 23 deletions(-) diff --git a/include/polymarket/rtds_stream.hpp b/include/polymarket/rtds_stream.hpp index 19d6b99..ee7ae0b 100644 --- a/include/polymarket/rtds_stream.hpp +++ b/include/polymarket/rtds_stream.hpp @@ -99,7 +99,8 @@ namespace polymarket using RtdsCommentCallback = std::function; using RtdsPriceCallback = std::function; // Fired whenever events may have been missed: every connect, disconnect, - // queue overflow or idle reconnect. RTDS does not replay them. + // queue overflow, idle reconnect or malformed event. RTDS does not replay + // them. using RtdsStreamGapCallback = std::function; // Fired once the connection and its subscriptions are back after a gap. using RtdsStreamRecoveredCallback = std::function; @@ -109,8 +110,10 @@ namespace polymarket // be added and removed before or after connect(); the stream sends the // combined server subscription and restores it after every reconnect. // It pings every 5 seconds and reconnects when no frame arrives for 10 - // minutes. Callbacks run on the stream's own threads; a malformed or - // unknown frame is counted in dropped_events() and skipped. + // minutes. Callbacks run on the stream's own threads. Malformed and + // unknown frames are counted in dropped_events(). A malformed frame on a + // comment or price topic is also reported to on_error() with its cause + // and followed by a reconnect; other frames are skipped. class RtdsStream { public: diff --git a/include/polymarket/sports_stream.hpp b/include/polymarket/sports_stream.hpp index a3cbeac..5c21ff2 100644 --- a/include/polymarket/sports_stream.hpp +++ b/include/polymarket/sports_stream.hpp @@ -41,7 +41,8 @@ namespace polymarket using SportsResultCallback = std::function; // Fired whenever updates may have been missed: every connect, disconnect, - // queue overflow or idle reconnect. The server does not replay them. + // queue overflow, idle reconnect or malformed update. The server does not + // replay them. using SportsStreamGapCallback = std::function; // Fired once the connection is back after a gap; refresh game state from // GammaClient here if it matters. @@ -51,8 +52,9 @@ namespace polymarket // Public Sports stream (Config::sports_ws_url). Every connection receives // every game update; there are no filters. The stream answers the // server's heartbeat and reconnects when no frame arrives for 30 seconds. - // Callbacks run on the stream's own threads; a malformed update is - // counted in dropped_events() and skipped. + // Callbacks run on the stream's own threads. A malformed update is + // counted in dropped_events(), reported to on_error() with its cause, and + // followed by a reconnect. class SportsStream { public: diff --git a/src/public_stream.cpp b/src/public_stream.cpp index f7d49e0..b079881 100644 --- a/src/public_stream.cpp +++ b/src/public_stream.cpp @@ -106,6 +106,14 @@ namespace polymarket::detail return active() && generation == websocket_.stream_generation(); } + void PublicStream::report_invalid_message(const std::string &error, std::uint64_t generation) + { + if (!is_current(generation)) return; + handle_error(error); + if (!is_current(generation)) return; + websocket_.request_resnapshot(error + "; reconnecting"); + } + void PublicStream::handle_gap(std::uint64_t generation) { if (!active()) return; diff --git a/src/public_stream.hpp b/src/public_stream.hpp index 2a8bed5..7a7b9f3 100644 --- a/src/public_stream.hpp +++ b/src/public_stream.hpp @@ -55,6 +55,8 @@ namespace polymarket::detail void request_resnapshot(const std::string &reason); // False once the stream stops or the connection that sent `generation` ends. bool is_current(std::uint64_t generation) const; + // Reports `error` and reconnects, firing the gap and recovery callbacks. + void report_invalid_message(const std::string &error, std::uint64_t generation); private: PublicStream(const std::string &url, const WebSocketOptions &options, diff --git a/src/rtds_parsing.cpp b/src/rtds_parsing.cpp index c1ef4e1..8dded79 100644 --- a/src/rtds_parsing.cpp +++ b/src/rtds_parsing.cpp @@ -57,6 +57,16 @@ namespace polymarket::detail return event; } + bool rtds_supported_frame(const std::string &message) + { + const auto frame = json::parse(message, nullptr, false); + if (!frame.is_object()) return false; + const auto *topic = find_field(frame, "topic"); + if (topic == nullptr || !topic->is_string()) return false; + const auto &name = topic->get_ref(); + return name == comments_topic || is_rtds_price_wire_topic(name); + } + std::string rtds_subscription_frame(const char *action, const std::vector &keys) { json subscriptions = json::array(); diff --git a/src/rtds_parsing.hpp b/src/rtds_parsing.hpp index ede89ff..8cdd772 100644 --- a/src/rtds_parsing.hpp +++ b/src/rtds_parsing.hpp @@ -40,6 +40,9 @@ namespace polymarket::detail // Unset for the empty frame RTDS sends after connecting. std::optional parse_rtds_message(const std::string &message); + // True for a JSON frame whose topic is one this client parses. + bool rtds_supported_frame(const std::string &message); + bool is_rtds_price_wire_topic(const std::string &topic); RtdsCommentEvent parse_rtds_comment_event(const json &frame); // {"action": action, "subscriptions": [...]} for `keys`. diff --git a/src/rtds_price_parsing.cpp b/src/rtds_price_parsing.cpp index 18993cb..445041d 100644 --- a/src/rtds_price_parsing.cpp +++ b/src/rtds_price_parsing.cpp @@ -201,6 +201,13 @@ namespace polymarket::detail return match; } + bool is_rtds_price_wire_topic(const std::string &topic) + { + return topic == "crypto_prices" || topic == "crypto_prices_chainlink" || + topic == "crypto_prices_twap_thirty" || topic == "crypto_prices_twap_sixty" || + topic == "equity_prices"; + } + std::vector rtds_price_keys(const RtdsPriceMatch &match) { if (match.topic == binance_topic) return {{"crypto_prices", "update", ""}}; diff --git a/src/rtds_stream.cpp b/src/rtds_stream.cpp index 5739053..e1ef886 100644 --- a/src/rtds_stream.cpp +++ b/src/rtds_stream.cpp @@ -147,9 +147,12 @@ namespace polymarket { event = parse_rtds_message(message); } - catch (const std::exception &) + catch (const std::exception &error) { dropped_events_++; + if (rtds_supported_frame(message)) + stream_->report_invalid_message( + std::string("invalid RTDS event: ") + error.what(), generation); return; } if (!event) return; diff --git a/src/sports_stream.cpp b/src/sports_stream.cpp index 83116af..f759641 100644 --- a/src/sports_stream.cpp +++ b/src/sports_stream.cpp @@ -34,14 +34,13 @@ namespace polymarket { auto runtime = std::shared_ptr(new SportsStreamRuntime()); std::weak_ptr weak = runtime; - runtime->stream_ = - PublicStream::create(config.sports_ws_url, sports_options(), - std::chrono::milliseconds(config.ws_connect_timeout_ms), - [weak](const std::string &message, std::uint64_t) - { - if (auto self = weak.lock()) - self->handle_message(message); - }); + runtime->stream_ = PublicStream::create( + config.sports_ws_url, sports_options(), + std::chrono::milliseconds(config.ws_connect_timeout_ms), + [weak](const std::string &message, std::uint64_t generation) + { + if (auto self = weak.lock()) self->handle_message(message, generation); + }); return runtime; } @@ -68,16 +67,18 @@ namespace polymarket private: SportsStreamRuntime() = default; - void handle_message(const std::string &message) + void handle_message(const std::string &message, std::uint64_t generation) { SportsGameResult result; try { result = parse_sports_message(message); } - catch (const std::exception &) + catch (const std::exception &error) { dropped_events_++; + stream_->report_invalid_message( + std::string("invalid Sports update: ") + error.what(), generation); return; } result_events_++; diff --git a/tests/test_rtds_stream.cpp b/tests/test_rtds_stream.cpp index a846755..debece2 100644 --- a/tests/test_rtds_stream.cpp +++ b/tests/test_rtds_stream.cpp @@ -230,6 +230,58 @@ namespace " second=" + std::to_string(second_calls.load())); } + bool test_malformed_event_reports_gap() + { + constexpr const char *malformed_comment = + R"({"topic":"comments","type":"comment_created","timestamp":1792286429215})"; + LocalWebSocketServer server; + polymarket::RtdsStream stream(local_config(server.url())); + std::mutex mutex; + std::vector errors; + std::atomic gaps{0}; + std::atomic recoveries{0}; + Recorder comments; + stream.subscribe_comments({}, comments.callback()); + stream.on_error( + [&](const std::string &error) + { + std::lock_guard lock(mutex); + errors.push_back(error); + }); + stream.on_stream_gap([&] { gaps++; }); + stream.on_stream_recovered([&] { recoveries++; }); + + const bool connected = stream.connect(); + const bool subscribed = server.wait_for_message_count(created_frame, 1, 2s); + const bool first_recovery = wait_until([&] { return recoveries.load() == 1; }, 2s); + const int gaps_before = gaps.load(); + server.send_to_clients(R"({"topic":"activity","type":"trades","payload":{}})"); + const bool unknown_skipped = !server.wait_for_connections(2, 200ms); + const int gaps_after_unknown = gaps.load(); + server.send_to_clients(malformed_comment); + const bool reconnected = server.wait_for_connections(2, 5s); + const bool restored = server.wait_for_message_count(created_frame, 2, 5s); + const bool recovered = wait_until([&] { return recoveries.load() == 2; }, 5s); + server.send_to_clients(rtds_fixtures::rtds_comment_created); + const bool resumed = wait_until([&] { return comments.size() == 1; }, 2s); + stream.disconnect(); + + std::lock_guard lock(mutex); + const bool reported = errors.size() == 1 && + errors[0].find("invalid RTDS event") != std::string::npos && + errors[0].find("payload") != std::string::npos; + return check(connected && subscribed && first_recovery, "the stream must connect") && + check(unknown_skipped && gaps_after_unknown == gaps_before, + "an unknown topic must be skipped without a gap") && + check(stream.dropped_events() == 2, "both bad frames must be counted") && + check(reported, "a malformed event must be reported once with its cause") && + check(gaps.load() > gaps_before && reconnected && restored && recovered, + "a malformed event must report a gap and recover, got gaps=" + + std::to_string(gaps.load() - gaps_before) + + " recoveries=" + std::to_string(recoveries.load())) && + check(resumed, "events must resume after recovery"); + } + bool test_invalid_arguments() { polymarket::RtdsStream stream; @@ -273,7 +325,7 @@ int main() { const bool passed = test_subscriptions_and_delivery() && test_diffs_and_replay() && test_price_subscriptions() && test_stop_from_callback() && - test_invalid_arguments(); + test_malformed_event_reports_gap() && test_invalid_arguments(); if (!passed) return 1; std::cout << "test_rtds_stream passed\n"; return 0; diff --git a/tests/test_sports_stream.cpp b/tests/test_sports_stream.cpp index 6a0cb10..e7476e4 100644 --- a/tests/test_sports_stream.cpp +++ b/tests/test_sports_stream.cpp @@ -61,7 +61,6 @@ namespace const bool connected = stream.connect(); const bool first_recovery = wait_until([&] { return recoveries.load() == 1; }, 2s); server.send_to_clients("ping"); - server.send_to_clients(sports_fixtures::sports_partial_result); server.send_to_clients(sports_fixtures::sports_live_result); const bool ponged = server.wait_for_message_count("pong", 1, 2s); const bool delivered = wait_until( @@ -91,8 +90,8 @@ namespace check(ponged, "the stream must answer the server heartbeat") && check(delivered && results[0].game_id == 70918912 && results[0].score == "10-21", "a game update must be delivered") && - check(stream.dropped_events() == 1 && stream.result_events() == 2, - "a malformed update must be counted and skipped") && + check(stream.dropped_events() == 0 && stream.result_events() == 2, + "valid updates must not be dropped") && check(reconnected && second_recovery && gaps.load() >= 3, "a dropped connection must report a gap and recover") && check(after_reconnect && results[1].league_abbreviation == "mls", @@ -100,6 +99,49 @@ namespace check(no_client_ping, "the stream must not send client pings"); } + bool test_malformed_update_reports_gap() + { + LocalWebSocketServer server; + polymarket::SportsStream stream(local_config(server.url())); + std::mutex mutex; + std::vector errors; + std::atomic results{0}; + std::atomic gaps{0}; + std::atomic recoveries{0}; + stream.on_result([&](const polymarket::SportsGameResult &) { results++; }); + stream.on_error( + [&](const std::string &error) + { + std::lock_guard lock(mutex); + errors.push_back(error); + }); + stream.on_stream_gap([&] { gaps++; }); + stream.on_stream_recovered([&] { recoveries++; }); + + const bool connected = stream.connect(); + const bool first_recovery = wait_until([&] { return recoveries.load() == 1; }, 2s); + const int gaps_before = gaps.load(); + server.send_to_clients(sports_fixtures::sports_partial_result); + const bool reconnected = server.wait_for_connections(2, 5s); + const bool recovered = wait_until([&] { return recoveries.load() == 2; }, 5s); + server.send_to_clients(sports_fixtures::sports_live_result); + const bool resumed = wait_until([&] { return results.load() == 1; }, 2s); + stream.disconnect(); + + std::lock_guard lock(mutex); + const bool reported = !errors.empty() && + errors[0].find("invalid Sports update") != std::string::npos && + errors[0].find("gameId") != std::string::npos; + return check(connected && first_recovery, "the stream must connect") && + check(stream.dropped_events() == 1, "a malformed update must be counted") && + check(reported, "a malformed update must be reported with its cause") && + check(gaps.load() > gaps_before && reconnected && recovered, + "a malformed update must report a gap and recover, got gaps=" + + std::to_string(gaps.load() - gaps_before) + + " recoveries=" + std::to_string(recoveries.load())) && + check(resumed, "updates must resume after recovery"); + } + bool test_disconnect_stops_delivery() { LocalWebSocketServer server; @@ -150,8 +192,9 @@ namespace int main() { - const bool passed = test_stream_delivery() && test_disconnect_stops_delivery() && - test_invalid_config() && test_unreachable_server(); + const bool passed = test_stream_delivery() && test_malformed_update_reports_gap() && + test_disconnect_stops_delivery() && test_invalid_config() && + test_unreachable_server(); if (!passed) return 1; std::cout << "test_sports_stream passed\n"; return 0;