diff --git a/CHANGELOG.md b/CHANGELOG.md index c17dbed..f4a76ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ - Added the health check feature. The SDK now sends a health check request once after `start()`. Like any request, it updates the user's last seen time on the server. - Added the SDK name and version (`sdk_name` and `sdk_version`) to every request, including the remote config and SDK Behavior Settings fetches. Before, only the begin session request carried them. - Added `multiplyCustomUserDetail` for multiplying the numeric value of a custom user property on the server. +- Added `changeServerUrl` to change the server URL after init. +- Added `changeAppKey` to change the app key after init. +- Added `flushQueues` to clear the request and event queues. +- Added `replaceAllAppKeysInQueueWithCurrentAppKey` to give the queued requests the current app key. +- Added `removeDifferentAppKeysFromQueue` to remove the queued requests of other app keys. +- Added `attemptToSendStoredRequests` to send the stored requests now. - Added `disableHealthCheck()` for turning off the health check request. - Added `status_code` and `body` to `HTTPResponse`. A custom HTTP client can fill them so that the health check reports the last request the server refused; a client that leaves them empty works as before. - Calls made before `start()`, a second `start()`, beginning a session that already runs and ending one that does not are now logged at warning level. Failed and non-JSON server answers, and session calls ignored because the SDK Behavior Settings turned session tracking off, are now logged at debug level. diff --git a/CMakeLists.txt b/CMakeLists.txt index 397566f..222462c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,7 +173,8 @@ if(COUNTLY_BUILD_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/tests/behavior_settings.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests/remote_config.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests/crash_reporting.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/tests/location.cpp) + ${CMAKE_CURRENT_SOURCE_DIR}/tests/location.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/server_url_app_key.cpp) target_compile_options(countly-tests PRIVATE $<$>:-g>) target_compile_definitions(countly-tests PRIVATE COUNTLY_BUILD_TESTS) diff --git a/include/countly.hpp b/include/countly.hpp index 41c534e..7daa0c1 100644 --- a/include/countly.hpp +++ b/include/countly.hpp @@ -274,6 +274,24 @@ class Countly : public cly::CountlyDelegates { void stop(); + /** + * Changes the server URL, in memory only. Host and port work as in start(): port 0 is the scheme's default and + * trailing slashes are removed. A host that is not an absolute http:// or https:// URL is ignored and logged. Queued + * requests go to the new server; a request already on its way finishes at the old one. The SDK then sends the consent + * state (when consent is required), fetches the SDK behavior settings and, with enableRemoteConfig(), downloads + * remote config. A custom HTTP client gets only the path, so it does not follow the change. Call after start(). + */ + void changeServerUrl(const std::string &host, int port = 0); + + /** + * Changes the app key, in memory only. Running views end, and their end events, the queued events, waiting user + * details and the end of the running session go with the old key; a session that was running then begins with the + * new key. Queued requests keep their key (see replaceAllAppKeysInQueueWithCurrentAppKey() and + * removeDifferentAppKeysFromQueue()). The old app's remote config values are cleared and the new app is caught up as + * in changeServerUrl(). An empty key, "YOUR_APP_KEY" and the current key are ignored and logged. Call after start(). + */ + void changeAppKey(const std::string &appKey); + void setUpdateInterval(size_t milliseconds); void addEvent(const cly::Event &event) override; @@ -315,6 +333,28 @@ class Countly : public cly::CountlyDelegates { */ int checkRQSize(); + /** + * Clears the request queue and the event queue. Running views, timed events, user details still waiting for the + * first session and the health check counters stay. A request already on its way is still delivered. + */ + void flushQueues(); + + /** + * Gives every queued request the current app key, so that requests queued before changeAppKey() go to the new app. + */ + void replaceAllAppKeysInQueueWithCurrentAppKey(); + + /** + * Removes the queued requests whose app key is not the current one, such as those queued before changeAppKey(). + */ + void removeDifferentAppKeysFromQueue(); + + /** + * Moves the queued events into a request and sends the request queue now, on the calling thread, like each update: + * until the queue is empty, a request fails or a batch is sent. Returns at once if the SDK's own thread is sending. + */ + void attemptToSendStoredRequests(); + /** * Checks and returns the size of the event queue in persistent storage. */ @@ -817,6 +857,25 @@ class Countly : public cly::CountlyDelegates { */ bool endSessionInternal(bool automatic); + /** + * Whether the public call 'caller' may run: never inside the log callback, and not before start(), which is logged + * with w101. + */ + bool isCallAllowed(const char *caller); + + /** + * Queues the end_session request of the running session and marks the session ended, counted in the health check as + * a call of the SDK with 'automatic'. The caller holds the instance mutex. + */ + void queueEndSession(bool automatic); + + /** + * After a server URL or app key change, fetches the SDK behavior settings and, with enableRemoteConfig(), remote + * config, unless 'remoteConfigRequested' says a session start downloads it already. The caller does not hold the + * instance mutex. + */ + void fetchAfterServerChange(bool remoteConfigRequested); + /** * Sends the health check once, as an immediate request to /i, unless it is disabled, the SDK behavior settings * turn networking off, or it was sent already. It carries the base request parameters, the app version as diff --git a/include/countly/remote_config_store.hpp b/include/countly/remote_config_store.hpp index 18407ed..04dec18 100644 --- a/include/countly/remote_config_store.hpp +++ b/include/countly/remote_config_store.hpp @@ -27,7 +27,10 @@ struct RemoteConfigStore { std::mutex mutex; nlohmann::json values = nlohmann::json::object(); - // True from the moment a fetch is started until its thread body returns. + // The app key the values belong to. Only a fetch for this key stores its result or clears fetch_running. + std::string app_key; + + // True from the moment a fetch is started until its thread body returns, or until the app key changes. std::atomic fetch_running{false}; // Bumped when the device ID changes, so that a download that started for an earlier user is discarded. diff --git a/include/countly/request_module.hpp b/include/countly/request_module.hpp index 1a358c0..f04368a 100644 --- a/include/countly/request_module.hpp +++ b/include/countly/request_module.hpp @@ -40,13 +40,29 @@ class RequestModule { void addRequestToQueue(const std::map &data); /** - * Clear request queue. - * Warning: This method is for debugging purposes, and it is going to be removed in the future. - * You should not be using this method. - * @return a vector object containing requests. + * Clears the request queue. A request that processQueue() is sending meanwhile is not removed again once it is + * delivered. The caller holds the mutex it passes to processQueue(). */ void clearRequestQueue(); + /** + * Replaces the server URL and the port that the following requests go to, the way start() sets them: a URL without + * a scheme gets "http://" in front, trailing slashes are removed, and a port of 0 becomes the scheme's default port. + */ + void setServerUrl(const std::string &url, unsigned int port); + + /** + * Gives every queued request the app key. The caller holds the mutex it passes to processQueue(). + */ + void replaceAppKeysInQueue(const std::string &appKey); + + /** + * Removes the queued requests whose app key is not the given one. The caller holds the mutex it passes to + * processQueue(). + * @return the number of removed requests + */ + long long removeOtherAppKeysFromQueue(const std::string &appKey); + long long RQSize(); void setConfigurationProvider(std::weak_ptr provider); // try injecting diff --git a/include/countly/storage_module_base.hpp b/include/countly/storage_module_base.hpp index 82c0937..ba8127b 100644 --- a/include/countly/storage_module_base.hpp +++ b/include/countly/storage_module_base.hpp @@ -95,6 +95,22 @@ class StorageModuleBase { */ virtual void RQInsertAtEnd(const std::string &request) = 0; + /** + * Replace the content of a stored request, keeping its id and its place in the queue. + * @param id: id of the request + * @param request: new content of the request + */ + virtual void RQUpdate(long long id, const std::string &request) { + (void)id; + (void)request; + } + + /** + * Remove the stored request with the id, wherever it is in the queue. + * @param id: id of the request + */ + virtual void RQRemove(long long id) { (void)id; } + virtual void storeSDKBehaviorSettings(const std::string &sdk_behavior_settings) = 0; virtual std::string getSDKBehaviorSettings() = 0; diff --git a/include/countly/storage_module_db.hpp b/include/countly/storage_module_db.hpp index 8c1e20f..b214477 100644 --- a/include/countly/storage_module_db.hpp +++ b/include/countly/storage_module_db.hpp @@ -24,6 +24,8 @@ class StorageModuleDB : public StorageModuleBase { std::vector> RQPeekAll() override; void RQRemoveFront(std::shared_ptr request) override; void RQInsertAtEnd(const std::string &request) override; + void RQUpdate(long long id, const std::string &request) override; + void RQRemove(long long id) override; void storeSDKBehaviorSettings(const std::string &sdk_behavior_settings) override; std::string getSDKBehaviorSettings() override; void storeHealthCheckState(const std::string &state) override; diff --git a/include/countly/storage_module_memory.hpp b/include/countly/storage_module_memory.hpp index 116d40c..9faa366 100644 --- a/include/countly/storage_module_memory.hpp +++ b/include/countly/storage_module_memory.hpp @@ -36,6 +36,8 @@ class StorageModuleMemory : public StorageModuleBase { std::vector> RQPeekAll() override; void RQRemoveFront(std::shared_ptr request) override; void RQInsertAtEnd(const std::string &request) override; + void RQUpdate(long long id, const std::string &request) override; + void RQRemove(long long id) override; void storeSDKBehaviorSettings(const std::string &sdk_behavior_settings) override; std::string getSDKBehaviorSettings() override; }; diff --git a/src/countly.cpp b/src/countly.cpp index 0a353ff..85bd710 100644 --- a/src/countly.cpp +++ b/src/countly.cpp @@ -175,6 +175,21 @@ void appendStoredEvents(const std::deque &queue, nlohmann::json &ev * Whether the list of features contains the feature. */ bool containsFeature(const std::vector &features, const char *feature) { return std::find(features.begin(), features.end(), feature) != features.end(); } + +/** + * Whether the URL is absolute, with the http or https scheme and a host. + */ +bool isValidServerUrl(const std::string &url) { + std::string::size_type hostStart; + if (url.compare(0, 7, "http://") == 0) { + hostStart = 7; + } else if (url.compare(0, 8, "https://") == 0) { + hostStart = 8; + } else { + return false; + } + return hostStart < url.size() && std::string(":/?#").find(url[hostStart]) == std::string::npos; +} } // namespace @@ -227,6 +242,8 @@ struct RemoteConfigFetch { std::map data; // true merges the response into the stored values, false replaces them. bool merge = false; + // The app key of the request. + std::string app_key; }; /** @@ -250,6 +267,10 @@ void runRemoteConfigFetch(std::shared_ptr fetch) { const HTTPResponse response = fetch->requestModule->sendImmediate("/o/sdk", fetch->requestBuilder->buildRequest(fetch->data)); std::lock_guard lk(fetch->store->mutex); + if (fetch->store->app_key != fetch->app_key) { + // after an app key change the values, the flag and the pending download belong to the fetches for the new key + return; + } if (!response.success || !response.data.is_object()) { fetch->logger->log(LogLevel::DEBUG, "[Countly] runRemoteConfigFetch, the remote config download failed, the stored values stay."); } else if (fetch->store->user_generation != generation) { @@ -272,8 +293,10 @@ void runRemoteConfigFetch(std::shared_ptr fetch) { } } catch (const std::exception &e) { std::lock_guard lk(fetch->store->mutex); - fetch->store->has_pending = false; - fetch->store->fetch_running.store(false); + if (fetch->store->app_key == fetch->app_key) { + fetch->store->has_pending = false; + fetch->store->fetch_running.store(false); + } fetch->logger->log(LogLevel::DEBUG, std::string("[Countly] runRemoteConfigFetch, the remote config download failed: ") + e.what()); } } @@ -1368,6 +1391,156 @@ void Countly::onMergeDelivered() { } #pragma endregion Device Id +bool Countly::isCallAllowed(const char *caller) { + if (LoggerModule::isInsideCallback()) { + return false; + } + if (!is_sdk_initialized) { + log(LogLevel::WARNING, std::string("[Countly] ") + caller + ", SDK is not initialized."); + logger->recordLogCode("w101"); + return false; + } + return true; +} + +void Countly::changeServerUrl(const std::string &host, int port) { + if (!isCallAllowed("changeServerUrl")) { + return; + } + if (!isValidServerUrl(host)) { + log(LogLevel::ERROR, "[Countly] changeServerUrl, The server URL [" + host + "] is not an absolute http:// or https:// URL, the current server URL stays."); + logger->recordLogCode("e102"); + return; + } + if (port < 0 || port > 65535) { + log(LogLevel::WARNING, "[Countly] changeServerUrl, Port number is out of valid boundaries. Setting it to 0."); + logger->recordLogCode("w121"); + port = 0; + } + + std::lock_guard change(consent_change_mutex); + if (is_being_disposed) { + return; + } + log(LogLevel::INFO, "[Countly] changeServerUrl, Changing the server URL to [" + host + "], port = [" + std::to_string(port) + "]"); + requestModule->setServerUrl(host, static_cast(port)); + if (consent_module->isRequired()) { + queueConsentState(); + } + fetchAfterServerChange(false); +} + +void Countly::changeAppKey(const std::string &appKey) { + if (!isCallAllowed("changeAppKey")) { + return; + } + if (appKey.empty() || appKey == "YOUR_APP_KEY") { + log(LogLevel::ERROR, "[Countly] changeAppKey, The app key can't be empty or the placeholder \"YOUR_APP_KEY\", the current app key stays."); + logger->recordLogCode("e103"); + return; + } + + std::lock_guard change(consent_change_mutex); + if (is_being_disposed) { + return; + } + { + std::lock_guard lk(*mutex); + if (configuration->appKey == appKey) { + log(LogLevel::DEBUG, "[Countly] changeAppKey, The new app key is the current one, the call is ignored."); + return; + } + } + log(LogLevel::INFO, "[Countly] changeAppKey, Changing the app key to [" + appKey + "]"); + + views_module->stopRunningViews(); + packEvents(); + bool sessionWasRunning = false; + { + // One section, so that the update loop can't begin a session with the old key after its session ended + std::lock_guard lk(*mutex); + if (session_params.contains("user_details")) { + queueUserDetails(); + } + sessionWasRunning = began_session; + if (began_session) { + queueEndSession(true); + } + configuration->appKey = appKey; + session_params["app_key"] = appKey; + // In the same section, so that a fetch built with the new key always finds that key in the store + std::lock_guard store_lock(remote_config_store->mutex); + remote_config_store->app_key = appKey; + remote_config_store->values = nlohmann::json::object(); + remote_config_store->fetch_running.store(false); + remote_config_store->has_pending = false; + } + if (sessionWasRunning) { + views_module->resetFirstView(); + health_check->saveState(); + } + + if (consent_module->isRequired()) { + queueConsentState(); + } + if (sessionWasRunning) { + beginSessionInternal(true); + } + fetchAfterServerChange(false); +} + +void Countly::fetchAfterServerChange(bool remoteConfigRequested) { + configurationModule->fetchConfigFromServer(nullptr); + if (!remoteConfigRequested) { + downloadRemoteConfigAutomatically(); + } +} + +void Countly::flushQueues() { + if (!isCallAllowed("flushQueues")) { + return; + } + + std::lock_guard lk(*mutex); + log(LogLevel::INFO, "[Countly] flushQueues, Clearing the request queue and the event queue."); + requestModule->clearRequestQueue(); + if (use_sqlite_storage) { + clearPersistentEQ(); + } else { + event_queue.clear(); + } +} + +void Countly::replaceAllAppKeysInQueueWithCurrentAppKey() { + if (!isCallAllowed("replaceAllAppKeysInQueueWithCurrentAppKey")) { + return; + } + + std::lock_guard lk(*mutex); + log(LogLevel::INFO, "[Countly] replaceAllAppKeysInQueueWithCurrentAppKey, Giving every queued request the current app key."); + requestModule->replaceAppKeysInQueue(configuration->appKey); +} + +void Countly::removeDifferentAppKeysFromQueue() { + if (!isCallAllowed("removeDifferentAppKeysFromQueue")) { + return; + } + + std::lock_guard lk(*mutex); + const long long removed = requestModule->removeOtherAppKeysFromQueue(configuration->appKey); + log(LogLevel::INFO, "[Countly] removeDifferentAppKeysFromQueue, Removed [" + std::to_string(removed) + "] queued requests with another app key."); +} + +void Countly::attemptToSendStoredRequests() { + if (!isCallAllowed("attemptToSendStoredRequests")) { + return; + } + + log(LogLevel::INFO, "[Countly] attemptToSendStoredRequests, Sending the stored requests now."); + packEvents(); + requestModule->processQueue(mutex); +} + void Countly::start(const std::string &app_key, const std::string &host, int port, bool start_thread) { // unique_lock so the mutex is released on scope exit (incl. allocation/json // throws while constructing modules); unlock/re-acquire around the self-locking @@ -1452,6 +1625,10 @@ void Countly::start(const std::string &app_key, const std::string &host, int por configuration->serverUrl = server_url; session_params["app_key"] = app_key; + { + std::lock_guard store_lock(remote_config_store->mutex); + remote_config_store->app_key = app_key; + } if (use_sqlite_storage) { storageModule.reset(new StorageModuleDB(configuration, logger)); @@ -1538,6 +1715,7 @@ bool Countly::startRemoteConfigFetch(const std::map &d return false; } + std::string app_key; { std::lock_guard lk(remote_config_store->mutex); if (remote_config_store->fetch_running.load()) { @@ -1552,6 +1730,8 @@ bool Countly::startRemoteConfigFetch(const std::map &d return false; } remote_config_store->fetch_running.store(true); + // taken with the flag, so that the fetch owns the flag exactly while its app key is the current one + app_key = remote_config_store->app_key; } std::shared_ptr fetch(new RemoteConfigFetch()); @@ -1561,6 +1741,7 @@ bool Countly::startRemoteConfigFetch(const std::map &d fetch->store = remote_config_store; fetch->data = data; fetch->merge = merge; + fetch->app_key = app_key; inflight_remote_config_fetches.fetch_add(1); try { @@ -2294,8 +2475,6 @@ bool Countly::endSessionInternal(bool automatic) { if (sessionRunning && !is_being_disposed) { packEvents(); } - const auto timestamp = std::chrono::duration_cast(Countly::getTimestamp().time_since_epoch()); - const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); // lock_guard so the mutex is released on scope exit, including the early // returns below and any exception (e.g. a json type_error from a session_params @@ -2316,27 +2495,32 @@ bool Countly::endSessionInternal(bool automatic) { return true; } - // getSessionDuration() takes this same mutex, so compute it inline here. - const auto duration = std::chrono::duration_cast(now - last_sent_session_request); - - std::map data = {{"app_key", session_params["app_key"].get()}, {"device_id", session_params["device_id"].get()}, {"session_duration", std::to_string(duration.count())}, {"timestamp", std::to_string(timestamp.count())}, {"end_session", "1"}}; - if (is_being_disposed) { // if SDK is being destroyed, don't attempt to send the end-session request. return false; } + queueEndSession(automatic); + lk.unlock(); + views_module->resetFirstView(); + health_check->saveState(); + return true; +} + +void Countly::queueEndSession(bool automatic) { + const auto timestamp = std::chrono::duration_cast(Countly::getTimestamp().time_since_epoch()); + const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); + // getSessionDuration() takes this same mutex, so compute it inline here. + const auto duration = std::chrono::duration_cast(now - last_sent_session_request); + + std::map data = {{"app_key", session_params["app_key"].get()}, {"device_id", session_params["device_id"].get()}, {"session_duration", std::to_string(duration.count())}, {"timestamp", std::to_string(timestamp.count())}, {"end_session", "1"}}; requestModule->addRequestToQueue(data); last_sent_session_request = now; began_session = false; - lk.unlock(); - views_module->resetFirstView(); if (automatic) { logger->recordUsage(HEALTH_CHECK_AREA_SESSIONS, "end:a"); } - health_check->saveState(); - return true; } std::chrono::system_clock::time_point Countly::getTimestamp() { return std::chrono::system_clock::now(); } @@ -2805,8 +2989,14 @@ void Countly::sendHealthCheck() { data["metrics"] = nlohmann::json({{"_app_version", appVersion->get()}}).dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); } + std::string request; + { + // buildRequest reads the app key and the device ID, which other threads change under the instance mutex + std::lock_guard lk(*mutex); + request = requestBuilder->buildRequest(data); + } log(LogLevel::DEBUG, "[Countly] sendHealthCheck, sending the health check"); - const HTTPResponse response = requestModule->sendImmediate("/i", requestBuilder->buildRequest(data)); + const HTTPResponse response = requestModule->sendImmediate("/i", request); if (response.success && response.data.is_object() && response.data.contains("result")) { log(LogLevel::DEBUG, "[Countly] sendHealthCheck, the server accepted the health check"); health_check->clearSentValues(snapshot); diff --git a/src/request_module.cpp b/src/request_module.cpp index 7bad6cc..e344ccd 100644 --- a/src/request_module.cpp +++ b/src/request_module.cpp @@ -36,8 +36,13 @@ namespace cly { class RequestModule::RequestModuleImpl { private: public: - bool use_https = true; bool is_queue_being_processed = false; + // Guards the server URL and the port in the configuration, which a server URL change replaces while requests are sent. + std::mutex server_mutex; + // The id of the request processQueue() is sending, -1 while it sends none, and whether a queue operation removed + // that request meanwhile. Guarded by the mutex processQueue() takes. + long long sending_id = -1; + bool sending_removed = false; // Both guarded by the instance mutex: no queued request is sent before them. std::chrono::steady_clock::time_point next_attempt_after_failure; std::chrono::steady_clock::time_point backoff_ends; @@ -48,23 +53,30 @@ class RequestModule::RequestModuleImpl { std::shared_ptr _storageModule; RequestModuleImpl(std::shared_ptr config, std::shared_ptr logger, std::shared_ptr requestBuilder, std::shared_ptr storageModule) : _configuration(config), _logger(logger), _requestBuilder(requestBuilder), _storageModule(storageModule) { - if (_configuration->serverUrl.find("http://") == 0) { - use_https = false; - } else if (_configuration->serverUrl.find("https://") == 0) { - use_https = true; - } else { - use_https = false; - _configuration->serverUrl.insert(0, "http://"); + setServerUrl(_configuration->serverUrl, _configuration->port); + } + + /** + * Normalizes and stores the server URL and the port, see RequestModule::setServerUrl(). + */ + void setServerUrl(std::string url, unsigned int port) { + const bool use_https = url.find("https://") == 0; + if (!use_https && url.find("http://") != 0) { + url.insert(0, "http://"); } const size_t scheme_length = use_https ? (sizeof("https://") - 1) : (sizeof("http://") - 1); - while (_configuration->serverUrl.size() > scheme_length && _configuration->serverUrl.back() == '/') { - _configuration->serverUrl.pop_back(); + while (url.size() > scheme_length && url.back() == '/') { + url.pop_back(); } - if (_configuration->port <= 0) { - _configuration->port = use_https ? 443 : 80; + if (port == 0) { + port = use_https ? 443 : 80; } + + std::lock_guard lk(server_mutex); + _configuration->serverUrl = url; + _configuration->port = port; } ~RequestModuleImpl() { _logger.reset(); } @@ -192,6 +204,21 @@ CurlGlobal &curlGlobal() { * -- the actual count is 1 by construction and proves nothing on its own. */ std::atomic networking_init_requests(0); + +/** + * Finds the app_key value of a stored request: sets 'start' and 'length' and returns true, or returns false when the + * request carries none. Stored values are URL-encoded, so a "&" always starts the next parameter. + */ +bool findAppKey(const std::string &request, std::string::size_type &start, std::string::size_type &length) { + start = request.compare(0, 8, "app_key=") == 0 ? 0 : request.find("&app_key="); + if (start == std::string::npos) { + return false; + } + start = request.find('=', start) + 1; + const std::string::size_type end = request.find('&', start); + length = (end == std::string::npos ? request.size() : end) - start; + return true; +} } // namespace void RequestModule::initGlobalNetworking() { @@ -267,7 +294,42 @@ void RequestModule::addRequestToQueue(const std::map & impl->_storageModule->RQInsertAtEnd(request); } -void RequestModule::clearRequestQueue() { impl->_storageModule->RQClearAll(); } +void RequestModule::clearRequestQueue() { + impl->_storageModule->RQClearAll(); + impl->sending_removed = true; +} + +void RequestModule::setServerUrl(const std::string &url, unsigned int port) { impl->setServerUrl(url, port); } + +void RequestModule::replaceAppKeysInQueue(const std::string &appKey) { + const std::string encodedKey = RequestBuilder::encodeURL(appKey); + for (const std::shared_ptr &request : impl->_storageModule->RQPeekAll()) { + std::string data = request->getData(); + std::string::size_type start, length; + if (findAppKey(data, start, length) && data.compare(start, length, encodedKey) != 0) { + data.replace(start, length, encodedKey); + impl->_storageModule->RQUpdate(request->getId(), data); + } + } +} + +long long RequestModule::removeOtherAppKeysFromQueue(const std::string &appKey) { + const std::string encodedKey = RequestBuilder::encodeURL(appKey); + long long removed = 0; + for (const std::shared_ptr &request : impl->_storageModule->RQPeekAll()) { + const std::string &data = request->getData(); + std::string::size_type start, length; + if (findAppKey(data, start, length) && data.compare(start, length, encodedKey) == 0) { + continue; + } + impl->_storageModule->RQRemove(request->getId()); + if (request->getId() == impl->sending_id) { + impl->sending_removed = true; + } + removed++; + } + return removed; +} void RequestModule::processQueue(std::shared_ptr mutex) { std::shared_ptr config; @@ -335,6 +397,8 @@ void RequestModule::processQueue(std::shared_ptr mutex) { impl->_logger->recordLogCode("w330"); continue; } + impl->sending_id = data->getId(); + impl->sending_removed = false; behind = count - 1; } @@ -345,6 +409,7 @@ void RequestModule::processQueue(std::shared_ptr mutex) { { std::lock_guard lk(*mutex); + impl->sending_id = -1; std::shared_ptr healthCheck = impl->_logger->healthCheck(); if (!isDelivered(response)) { impl->_logger->log(LogLevel::DEBUG, "[Countly] [RequestModule] processQueue: Failed to deliver to server, will try again after the cool-down of [" + std::to_string(impl->_configuration->failedRequestCooldownSeconds) + "] seconds."); @@ -358,7 +423,10 @@ void RequestModule::processQueue(std::shared_ptr mutex) { // we pop the front only if it is still the same request // the queue might have changed while we were sending the request - impl->_storageModule->RQRemoveFront(data); + // a removed request's id can belong to a newer request by now, so nothing is removed then + if (!impl->sending_removed) { + impl->_storageModule->RQRemoveFront(data); + } processedRequestsCounter++; mergeDelivered = data->getData().find("old_device_id=") != std::string::npos; @@ -431,7 +499,16 @@ HTTPResponse RequestModule::sendHTTP(std::string path, std::string data) { if (impl->_configuration->http_client_function) { return impl->_configuration->http_client_function(use_post, path, data); } + + std::string server_url; + unsigned int port; + { + std::lock_guard lk(impl->server_mutex); + server_url = impl->_configuration->serverUrl; + port = impl->_configuration->port; + } #ifdef _WIN32 + const bool use_https = server_url.find("https://") == 0; HINTERNET hSession = nullptr; HINTERNET hConnect = nullptr; HINTERNET hRequest = nullptr; @@ -445,12 +522,12 @@ HTTPResponse RequestModule::sendHTTP(std::string path, std::string data) { 10000, // nSendTimeout: 10sec (default 30sec). 10000); // nReceiveTimeout: 10sec (default 30sec). - size_t scheme_offset = impl->use_https ? (sizeof("https://") - 1) : (sizeof("http://") - 1); - size_t buffer_size = MultiByteToWideChar(CP_ACP, 0, impl->_configuration->serverUrl.c_str() + scheme_offset, -1, nullptr, 0); + size_t scheme_offset = use_https ? (sizeof("https://") - 1) : (sizeof("http://") - 1); + size_t buffer_size = MultiByteToWideChar(CP_ACP, 0, server_url.c_str() + scheme_offset, -1, nullptr, 0); wchar_t *wide_hostname = new wchar_t[buffer_size]; - MultiByteToWideChar(CP_ACP, 0, impl->_configuration->serverUrl.c_str() + scheme_offset, -1, wide_hostname, static_cast(buffer_size)); + MultiByteToWideChar(CP_ACP, 0, server_url.c_str() + scheme_offset, -1, wide_hostname, static_cast(buffer_size)); - hConnect = WinHttpConnect(hSession, wide_hostname, impl->_configuration->port, 0); + hConnect = WinHttpConnect(hSession, wide_hostname, port, 0); delete[] wide_hostname; } @@ -460,7 +537,7 @@ HTTPResponse RequestModule::sendHTTP(std::string path, std::string data) { wchar_t *wide_path = new wchar_t[buffer_size]; MultiByteToWideChar(CP_ACP, 0, path.c_str(), -1, wide_path, static_cast(buffer_size)); - hRequest = WinHttpOpenRequest(hConnect, L"POST", wide_path, NULL, WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES, impl->use_https ? WINHTTP_FLAG_SECURE : 0); + hRequest = WinHttpOpenRequest(hConnect, L"POST", wide_path, NULL, WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES, use_https ? WINHTTP_FLAG_SECURE : 0); delete[] wide_path; } @@ -533,7 +610,7 @@ HTTPResponse RequestModule::sendHTTP(std::string path, std::string data) { curl = curl_easy_init(); if (curl) { std::ostringstream full_url_stream; - full_url_stream << impl->_configuration->serverUrl << ':' << std::dec << impl->_configuration->port << path; + full_url_stream << server_url << ':' << std::dec << port << path; curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.c_str()); diff --git a/src/storage_module_db.cpp b/src/storage_module_db.cpp index c1d4938..1ab1b1f 100644 --- a/src/storage_module_db.cpp +++ b/src/storage_module_db.cpp @@ -381,6 +381,78 @@ void StorageModuleDB::RQInsertAtEnd(const std::string &request) { } } +void StorageModuleDB::RQUpdate(long long id, const std::string &request) { + try { + if (!_is_initialized) { + _logger->log(LogLevel::ERROR, "[Countly] [StorageModuleDB] RQUpdate, Module is not initialized"); + return; + } + + _logger->log(LogLevel::DEBUG, "[Countly] [StorageModuleDB] RQUpdate, RequestID = " + std::to_string(id) + ", request = [" + request + "]"); + +#ifdef COUNTLY_USE_SQLITE + sqlite3 *database; + int return_value = cly::utils::openDatabase(_configuration->databasePath, &database); + if (return_value == SQLITE_OK) { + std::ostringstream sql_statement_stream; + sql_statement_stream << "UPDATE " << REQUESTS_TABLE_NAME << " SET " << REQUESTS_TABLE_REQUEST_DATA << " = ? WHERE " << REQUESTS_TABLE_REQUEST_ID << " = ?;"; + std::string sql_statement = sql_statement_stream.str(); + + sqlite3_stmt *statement = nullptr; + return_value = sqlite3_prepare_v2(database, sql_statement.c_str(), -1, &statement, nullptr); + if (return_value == SQLITE_OK) { + sqlite3_bind_text(statement, 1, request.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 2, id); + return_value = sqlite3_step(statement); + } + if (return_value != SQLITE_DONE) { + _logger->log(LogLevel::ERROR, "[Countly] [StorageModuleDB] RQUpdate, error = " + std::string(sqlite3_errmsg(database))); + } + sqlite3_finalize(statement); + } + sqlite3_close(database); +#endif + } catch (const std::system_error &e) { + std::ostringstream log_message; + log_message << "[Countly] [StorageModuleDB] RQUpdate, error: " << e.what(); + _logger->log(LogLevel::FATAL, log_message.str()); + } +} + +void StorageModuleDB::RQRemove(long long id) { + try { + if (!_is_initialized) { + _logger->log(LogLevel::ERROR, "[Countly] [StorageModuleDB] RQRemove, Module is not initialized"); + return; + } + + _logger->log(LogLevel::DEBUG, "[Countly] [StorageModuleDB] RQRemove, RequestID = " + std::to_string(id)); + +#ifdef COUNTLY_USE_SQLITE + sqlite3 *database; + char *error_message; + int return_value = cly::utils::openDatabase(_configuration->databasePath, &database); + if (return_value == SQLITE_OK) { + std::ostringstream sql_statement_stream; + sql_statement_stream << "DELETE FROM " << REQUESTS_TABLE_NAME << " WHERE " << REQUESTS_TABLE_REQUEST_ID << " = " << id << ';'; + std::string sql_statement = sql_statement_stream.str(); + + return_value = sqlite3_exec(database, sql_statement.c_str(), nullptr, nullptr, &error_message); + if (return_value != SQLITE_OK) { + std::string error(error_message); + _logger->log(LogLevel::ERROR, "[Countly] [StorageModuleDB] RQRemove, error = " + error); + sqlite3_free(error_message); + } + } + sqlite3_close(database); +#endif + } catch (const std::system_error &e) { + std::ostringstream log_message; + log_message << "[Countly] [StorageModuleDB] RQRemove, error: " << e.what(); + _logger->log(LogLevel::FATAL, log_message.str()); + } +} + void StorageModuleDB::RQClearAll() { try { if (!_is_initialized) { diff --git a/src/storage_module_memory.cpp b/src/storage_module_memory.cpp index 5b60afb..71387a0 100644 --- a/src/storage_module_memory.cpp +++ b/src/storage_module_memory.cpp @@ -98,6 +98,38 @@ void StorageModuleMemory::RQInsertAtEnd(const std::string &request) { } } +void StorageModuleMemory::RQUpdate(long long id, const std::string &request) { + if (!_is_initialized) { + _logger->log(LogLevel::ERROR, "[Countly] [StorageModuleMemory] RQUpdate, Module is not initialized"); + return; + } + + _logger->log(LogLevel::DEBUG, "[Countly] [StorageModuleMemory] RQUpdate, id = [" + std::to_string(id) + "], request = [" + request + "]"); + std::lock_guard lk(_mutex); + for (std::shared_ptr &entry : request_queue) { + if (entry->getId() == id) { + entry = std::make_shared(id, request); + return; + } + } +} + +void StorageModuleMemory::RQRemove(long long id) { + if (!_is_initialized) { + _logger->log(LogLevel::ERROR, "[Countly] [StorageModuleMemory] RQRemove, Module is not initialized"); + return; + } + + _logger->log(LogLevel::DEBUG, "[Countly] [StorageModuleMemory] RQRemove, id = [" + std::to_string(id) + "]"); + std::lock_guard lk(_mutex); + for (auto entry = request_queue.begin(); entry != request_queue.end(); ++entry) { + if ((*entry)->getId() == id) { + request_queue.erase(entry); + return; + } + } +} + std::vector> StorageModuleMemory::RQPeekAll() { if (!_is_initialized) { _logger->log(LogLevel::ERROR, "[Countly] [StorageModuleMemory] RQPeekAll, Module is not initialized"); diff --git a/tests/request.cpp b/tests/request.cpp index 64ad8d8..922b8a3 100644 --- a/tests/request.cpp +++ b/tests/request.cpp @@ -693,4 +693,71 @@ TEST_CASE("Built-in HTTP client") { CHECK(open_durations[0] <= request_timeout + std::chrono::seconds(5)); } } + +/** + * Whether one of the request bodies a server received contains the text. + */ +static bool receivedRequestWith(const LocalHTTPServer &server, const std::string &text) { + for (const std::string &request_body : server.requestBodies()) { + if (request_body.find(text) != std::string::npos) { + return true; + } + } + return false; +} + +TEST_CASE("Changing the server URL with the built-in HTTP client") { + const std::string database_path = "test-countly-http.db"; + remove(database_path.c_str()); + LocalHTTPServer first_server(true); + LocalHTTPServer second_server(true); + std::shared_ptr countly = std::make_shared(); + countly->setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly->SetPath(database_path); + // Only processRQDebug() sends: the update loop sleeps through the test, and immediate stop still ends it at once + countly->enableImmediateRequestOnStop(); + countly->setUpdateInterval(60000); + countly->start(COUNTLY_TEST_APP_KEY, "http://127.0.0.1", first_server.port(), false); + countly->addEvent(cly::Event("recorded_before_the_change", 1)); + countly->flushEvents(); + + countly->changeServerUrl("ftp://127.0.0.1", second_server.port()); + countly->changeServerUrl("127.0.0.1", second_server.port()); + countly->changeServerUrl("http://", second_server.port()); + const std::string url_before_the_change = countly->getConfiguration().serverUrl; + const unsigned int port_before_the_change = countly->getConfiguration().port; + countly->changeServerUrl("http://127.0.0.1///", second_server.port()); + const std::string changed_url = countly->getConfiguration().serverUrl; + + const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (std::chrono::steady_clock::now() < deadline) { + countly->processRQDebug(); + if (receivedRequestWith(second_server, "begin_session=1") && receivedRequestWith(second_server, "recorded_before_the_change") && receivedRequestWith(second_server, "method=sc") && countly->checkRQSize() == 0) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + const int queued_requests = countly->checkRQSize(); + + countly.reset(); + first_server.stop(); + second_server.stop(); + remove(database_path.c_str()); + + CHECK(url_before_the_change == "http://127.0.0.1"); + CHECK(port_before_the_change == static_cast(first_server.port())); + CHECK(changed_url == "http://127.0.0.1"); + CHECK(queued_requests == 0); + // The requests queued before the change, and the SDK behavior settings fetch that follows it, reach the new server + CHECK(receivedRequestWith(second_server, "begin_session=1")); + CHECK(receivedRequestWith(second_server, "recorded_before_the_change")); + CHECK(receivedRequestWith(second_server, "method=sc")); + for (const std::string &request_line : second_server.requestLines()) { + CHECK((request_line.rfind("POST /i ", 0) == 0 || request_line.rfind("POST /o/sdk ", 0) == 0)); + } + // The first server got at most what the SDK sent before the change: the SDK behavior settings fetch and the health check + for (const std::string &request_body : first_server.requestBodies()) { + CHECK((request_body.find("method=sc") != std::string::npos || request_body.find("hc=") != std::string::npos)); + } +} #endif diff --git a/tests/server_url_app_key.cpp b/tests/server_url_app_key.cpp new file mode 100644 index 0000000..b11bbec --- /dev/null +++ b/tests/server_url_app_key.cpp @@ -0,0 +1,561 @@ +#include "test_utils.hpp" + +#include "doctest.h" +#include +#include +#include +#include +#include +#include +#include + +using namespace cly; +using namespace test_utils; + +namespace { +const std::string FIRST_APP_KEY = COUNTLY_TEST_APP_KEY; +const std::string SECOND_APP_KEY = "b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4"; +const std::string THIRD_APP_KEY = "c4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8b7c6d5"; + +/** + * The requests the fake HTTP client received that carry every given parameter, in the order they were sent. An empty + * value matches any value. + */ +std::vector callsWith(const std::map ¶meters) { + std::vector calls; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + bool matches = true; + for (const auto ¶meter : parameters) { + const auto found = call.data.find(parameter.first); + matches = matches && found != call.data.end() && (parameter.second.empty() || found->second == parameter.second); + } + if (matches) { + calls.push_back(call); + } + } + return calls; +} + +/** + * Waits until the condition holds, at most five seconds. + */ +template bool waitUntil(Condition condition) { + const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!condition()) { + if (std::chrono::steady_clock::now() >= deadline) { + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + return true; +} + +/** + * Waits up to five seconds until the fake HTTP client received 'count' requests carrying every given parameter. + */ +bool waitForRequest(const std::map ¶meters, size_t count = 1) { + return waitUntil([&]() { return callsWith(parameters).size() >= count; }); +} + +/** + * The requests sent to /i, each as what it is for and the app key it carries, such as "begin_session ". + */ +std::vector queuedRequestsSent() { + std::vector sent; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + if (call.url != "/i") { + continue; + } + std::string kind = "other"; + // end_session comes before events because the end of a session carries no events, and begin_session before user_details + for (const char *parameter : {"begin_session", "end_session", "events", "user_details", "consent", "location"}) { + if (call.data.count(parameter) > 0) { + kind = parameter; + break; + } + } + sent.push_back(kind + " " + call.data["app_key"]); + } + return sent; +} + +/** + * The keys of the events a request carries, in their order. + */ +std::vector eventKeys(const HTTPCall &call) { + std::vector keys; + for (const nlohmann::json &event : nlohmann::json::parse(call.data.at("events"))) { + keys.push_back(event["key"].get()); + } + return keys; +} + +/** + * How often the health check of the instance counted the log code. + */ +long long logCodeCount(Countly &countly, const std::string &code) { + const nlohmann::json codes = countly.debugHealthCheck()->snapshot().at("lc"); + return codes.contains(code) ? codes.at(code).get() : 0; +} + +/** + * Gets the default instance ready for start(): the HTTP client, the test device ID and database, a captured log, and + * an update loop that sleeps through the test, so that only the test sends the request queue. + */ +Countly &prepareCountly(HTTPClientFunction client = fakeSendHTTP) { + clearSDK(); + clearCapturedLogs(); + Countly &countly = Countly::getInstance(); + countly.setLogger(captureLog); + countly.setHTTPClient(client); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(60000); + return countly; +} + +/** + * Starts the instance, sends what start() queued and forgets every request sent so far. + */ +void startAndForget(Countly &countly) { + countly.start(FIRST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + REQUIRE(waitForRequest({{"method", "sc"}})); + // The fetches of start() run on threads of their own, a remote config download with the first session included + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + countly.processRQDebug(); + http_call_queue.clear(); +} + +/** + * Holds back the first request whose data contains a marker until the test releases it, so that the test can act + * while that request is on its way. + */ +class RequestHold { +public: + /** + * Holds back the next request whose data contains 'marker'. + */ + void holdNext(const std::string &marker) { + std::lock_guard lock(_mutex); + _marker = marker; + _held = false; + _released = false; + } + + /** + * Waits up to five seconds until a request is held back. + */ + bool waitUntilHeld() { + std::unique_lock lock(_mutex); + return _changed.wait_for(lock, std::chrono::seconds(5), [this]() { return _held; }); + } + + /** + * Lets the held request and every later one through. + */ + void release() { + { + std::lock_guard lock(_mutex); + _released = true; + } + _changed.notify_all(); + } + + /** + * HTTP client that holds back the marked request, then answers every request as fakeSendHTTP does. + */ + HTTPResponse send(bool use_post, const std::string &url, const std::string &data) { + { + std::unique_lock lock(_mutex); + if (!_released && !_held && data.find(_marker) != std::string::npos) { + _held = true; + _changed.notify_all(); + _changed.wait(lock, [this]() { return _released; }); + } + } + return fakeSendHTTP(use_post, url, data); + } + +private: + std::mutex _mutex; + std::condition_variable _changed; + std::string _marker; + bool _held = false; + bool _released = true; +}; + +RequestHold request_hold; + +HTTPResponse sendThroughHold(bool use_post, const std::string &url, const std::string &data) { return request_hold.send(use_post, url, data); } + +/** + * Lets the held request through and waits for the thread that sends it, at the latest when the test ends, also when + * it ends early, so that no thread stays blocked. + */ +struct HoldScope { + std::thread sender; + + void release() { + request_hold.release(); + if (sender.joinable()) { + sender.join(); + } + } + + ~HoldScope() { release(); } +}; + +/** + * HTTP client that holds back the marked request like sendThroughHold, and answers a download of every remote config + * value for the second app key with that app's value. + */ +HTTPResponse sendThroughHoldWithSecondAppConfig(bool use_post, const std::string &url, const std::string &data) { + HTTPResponse response = request_hold.send(use_post, url, data); + if (url == "/o/sdk" && data.find("method=rc") != std::string::npos && data.find("app_key=" + SECOND_APP_KEY) != std::string::npos && data.find("keys=") == std::string::npos) { + response.success = true; + response.data = {{"color", "#00FF00"}}; + } + return response; +} + +std::mutex raw_requests_mutex; +std::vector raw_requests; + +/** + * HTTP client that keeps the data of every queued request as it was sent, then answers as fakeSendHTTP does. + */ +HTTPResponse recordRawRequest(bool use_post, const std::string &url, const std::string &data) { + if (url == "/i" && data.find("hc=") == std::string::npos) { + std::lock_guard lock(raw_requests_mutex); + raw_requests.push_back(data); + } + return fakeSendHTTP(use_post, url, data); +} +} // namespace + +TEST_CASE("Changing the server URL") { + SUBCASE("a valid URL is taken with the port the way start() takes them, and the app key stays") { + Countly &countly = prepareCountly(); + startAndForget(countly); + const long long out_of_range_ports = logCodeCount(countly, "w121"); + + countly.changeServerUrl("https://changed.countly.notarealdomain//"); + CHECK(countly.getConfiguration().serverUrl == "https://changed.countly.notarealdomain"); + CHECK(countly.getConfiguration().port == 443); + + countly.changeServerUrl("http://other.countly.notarealdomain/", 9000); + CHECK(countly.getConfiguration().serverUrl == "http://other.countly.notarealdomain"); + CHECK(countly.getConfiguration().port == 9000); + + countly.changeServerUrl("http://other.countly.notarealdomain", 70000); + CHECK(countly.getConfiguration().port == 80); + CHECK(logCodeCount(countly, "w121") == out_of_range_ports + 1); + + CHECK(countly.getConfiguration().appKey == FIRST_APP_KEY); + CHECK(capturedLogContains("changeServerUrl, Changing the server URL to [http://other.countly.notarealdomain/], port = [9000]")); + } + + SUBCASE("an invalid URL is ignored and logged, and the current one stays") { + Countly &countly = prepareCountly(); + startAndForget(countly); + const long long invalid_urls = logCodeCount(countly, "e102"); + + const std::vector urls = {"", "not a url", "ftp://x", "http://", "http:///", "https://:443", "changed.countly.notarealdomain"}; + for (const std::string &url : urls) { + countly.changeServerUrl(url); + CHECK(countly.getConfiguration().serverUrl == COUNTLY_TEST_HOST); + CHECK(countly.getConfiguration().port == COUNTLY_TEST_PORT); + } + CHECK(logCodeCount(countly, "e102") == invalid_urls + static_cast(urls.size())); + CHECK(capturedLogContains("changeServerUrl, The server URL [ftp://x] is not an absolute http:// or https:// URL, the current server URL stays.")); + + // Nothing is queued or fetched for an ignored URL + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + countly.processRQDebug(); + CHECK(http_call_queue.empty()); + } + + SUBCASE("a change is followed by the consent state, the SDK behavior settings fetch and the remote config download, with the current app key") { + Countly &countly = prepareCountly(); + countly.setRequiresConsent(true); + countly.setConsentEnabled({consent::SESSIONS, consent::REMOTE_CONFIG}); + countly.enableRemoteConfig(); + startAndForget(countly); + + countly.changeServerUrl("https://changed.countly.notarealdomain"); + REQUIRE(waitForRequest({{"method", "sc"}})); + REQUIRE(waitForRequest({{"method", "rc"}})); + countly.processRQDebug(); + + CHECK(queuedRequestsSent() == std::vector{"consent " + FIRST_APP_KEY}); + const nlohmann::json state = nlohmann::json::parse(callsWith({{"consent", ""}}).at(0).data["consent"]); + CHECK(state["sessions"] == true); + CHECK(state["remote-config"] == true); + CHECK(state["events"] == false); + for (const char *method : {"sc", "rc"}) { + const HTTPCall fetch = callsWith({{"method", method}}).at(0); + CHECK(fetch.data.at("app_key") == FIRST_APP_KEY); + CHECK(fetch.data.at("device_id") == COUNTLY_TEST_DEVICE_ID); + } + } + + SUBCASE("a call before start is ignored") { + Countly &countly = prepareCountly(); + countly.changeServerUrl("https://changed.countly.notarealdomain"); + CHECK(countly.getConfiguration().serverUrl.empty()); + CHECK(logCodeCount(countly, "w101") == 1); + CHECK(capturedLogContains("changeServerUrl, SDK is not initialized.")); + } +} + +TEST_CASE("Changing the app key") { + SUBCASE("the old app gets its views, events and session end, the new app a new session, and queued requests keep their key") { + Countly &countly = prepareCountly(); + countly.setRequiresConsent(true); + countly.setConsentEnabled(consent::allFeatures()); + startAndForget(countly); + + countly.addEvent(cly::Event("queued_before", 1)); + countly.flushEvents(); + countly.views().openView("Home"); + countly.addEvent(cly::Event("recorded_before", 1)); + countly.changeAppKey(SECOND_APP_KEY); + countly.addEvent(cly::Event("recorded_after", 1)); + countly.flushEvents(); + REQUIRE(waitForRequest({{"method", "sc"}})); + countly.processRQDebug(); + + CHECK(queuedRequestsSent() == std::vector{"events " + FIRST_APP_KEY, "events " + FIRST_APP_KEY, "end_session " + FIRST_APP_KEY, "consent " + SECOND_APP_KEY, "begin_session " + SECOND_APP_KEY, "events " + SECOND_APP_KEY}); + const std::vector events = callsWith({{"events", ""}}); + REQUIRE(events.size() == 3); + CHECK(eventKeys(events[0]) == std::vector{"queued_before"}); + CHECK(eventKeys(events[1]) == std::vector{"[CLY]_view", "recorded_before", "[CLY]_view"}); + CHECK(eventKeys(events[2]) == std::vector{"recorded_after"}); + CHECK(callsWith({{"method", "sc"}}).at(0).data["app_key"] == SECOND_APP_KEY); + CHECK(callsWith({{"device_id", COUNTLY_TEST_DEVICE_ID}}).size() == http_call_queue.size()); + CHECK(countly.getConfiguration().appKey == SECOND_APP_KEY); + } + + SUBCASE("an invalid key and the current key are ignored") { + Countly &countly = prepareCountly(); + startAndForget(countly); + const long long invalid_keys = logCodeCount(countly, "e103"); + + countly.changeAppKey(""); + countly.changeAppKey("YOUR_APP_KEY"); + countly.changeAppKey(FIRST_APP_KEY); + + CHECK(logCodeCount(countly, "e103") == invalid_keys + 2); + CHECK(capturedLogContains("changeAppKey, The app key can't be empty or the placeholder \"YOUR_APP_KEY\", the current app key stays.")); + CHECK(capturedLogContains("changeAppKey, The new app key is the current one, the call is ignored.")); + CHECK(countly.getConfiguration().appKey == FIRST_APP_KEY); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + countly.processRQDebug(); + CHECK(http_call_queue.empty()); + } + + SUBCASE("with manual sessions, user details still waiting go to the old app, and only a running session begins again") { + Countly &countly = prepareCountly(); + countly.enableManualSessionControl(); + countly.setUserDetails({{"name", "Waiting User"}}); + startAndForget(countly); + + countly.changeAppKey(SECOND_APP_KEY); + countly.beginSession(); + countly.changeAppKey(THIRD_APP_KEY); + REQUIRE(waitForRequest({{"method", "sc"}, {"app_key", THIRD_APP_KEY}})); + countly.processRQDebug(); + + CHECK(queuedRequestsSent() == std::vector{"user_details " + FIRST_APP_KEY, "begin_session " + SECOND_APP_KEY, "end_session " + SECOND_APP_KEY, "begin_session " + THIRD_APP_KEY}); + CHECK(nlohmann::json::parse(callsWith({{"user_details", ""}}).at(0).data["user_details"]) == nlohmann::json({{"name", "Waiting User"}})); + } + + SUBCASE("the remote config values of the old app are cleared and the new app's are downloaded once") { + Countly &countly = prepareCountly(); + countly.enableRemoteConfig(); + startAndForget(countly); + std::string keys[] = {"color"}; + countly.updateRemoteConfigFor(keys, 1); + REQUIRE(waitForRequest({{"keys", "[\"color\"]"}})); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + REQUIRE(countly.getRemoteConfigValue("color") == "#FF9900"); + + countly.changeAppKey(SECOND_APP_KEY); + CHECK(countly.getRemoteConfigValue("color").is_null()); + CHECK(waitForRequest({{"method", "rc"}, {"app_key", SECOND_APP_KEY}})); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + CHECK(callsWith({{"method", "rc"}, {"app_key", SECOND_APP_KEY}}).size() == 1); + } + + SUBCASE("a call before start is ignored") { + Countly &countly = prepareCountly(); + countly.changeAppKey(SECOND_APP_KEY); + CHECK(countly.getConfiguration().appKey.empty()); + CHECK(logCodeCount(countly, "w101") == 1); + CHECK(capturedLogContains("changeAppKey, SDK is not initialized.")); + } +} + +TEST_CASE("Queue operations") { + SUBCASE("flushQueues clears both queues, and what is not queued yet stays") { + Countly &countly = prepareCountly(); + countly.enableManualSessionControl(); + countly.setUserDetails({{"name", "Waiting User"}}); + startAndForget(countly); + countly.addEvent(cly::Event("queued_event", 1)); + countly.flushEvents(); + countly.views().openView("Home"); + countly.addEvent(cly::Event("flushed_event", 1)); + countly.setLocation("TR", "Izmir", "38.42,27.14", ""); + REQUIRE(countly.checkRQSize() == 1); + REQUIRE(countly.checkEQSize() == 2); + + countly.flushQueues(); + CHECK(countly.checkRQSize() == 0); + CHECK(countly.checkEQSize() == 0); + + countly.views().closeViewWithName("Home"); + countly.beginSession(); + countly.attemptToSendStoredRequests(); + CHECK(queuedRequestsSent() == std::vector{"begin_session " + FIRST_APP_KEY, "events " + FIRST_APP_KEY}); + const HTTPCall begin = callsWith({{"begin_session", ""}}).at(0); + CHECK(nlohmann::json::parse(begin.data.at("user_details")) == nlohmann::json({{"name", "Waiting User"}})); + CHECK(begin.data.at("city") == "Izmir"); + CHECK(eventKeys(callsWith({{"events", ""}}).at(0)) == std::vector{"[CLY]_view"}); + } + + SUBCASE("replaceAllAppKeysInQueueWithCurrentAppKey sends the queued requests to the new app with a valid checksum") { + Countly &countly = prepareCountly(recordRawRequest); + countly.setSalt("test-salt"); + startAndForget(countly); + { + std::lock_guard lock(raw_requests_mutex); + raw_requests.clear(); + } + + countly.addEvent(cly::Event("queued_before", 1)); + countly.flushEvents(); + countly.changeAppKey(SECOND_APP_KEY); + countly.replaceAllAppKeysInQueueWithCurrentAppKey(); + countly.processRQDebug(); + + CHECK(queuedRequestsSent() == std::vector{"events " + SECOND_APP_KEY, "end_session " + SECOND_APP_KEY, "begin_session " + SECOND_APP_KEY}); + CHECK(capturedLogContains("replaceAllAppKeysInQueueWithCurrentAppKey, Giving every queued request the current app key.")); + std::lock_guard lock(raw_requests_mutex); + REQUIRE(raw_requests.size() == 3); + for (const std::string &raw_request : raw_requests) { + const std::string::size_type checksum_start = raw_request.find("&checksum256="); + REQUIRE(checksum_start != std::string::npos); + CHECK(raw_request.find("app_key=" + SECOND_APP_KEY) != std::string::npos); + CHECK(countly.calculateChecksum("test-salt", raw_request.substr(0, checksum_start)) == raw_request.substr(checksum_start + 13)); + } + } + + SUBCASE("removeDifferentAppKeysFromQueue drops the queued requests of the old app") { + Countly &countly = prepareCountly(); + startAndForget(countly); + countly.addEvent(cly::Event("queued_before", 1)); + countly.flushEvents(); + countly.changeAppKey(SECOND_APP_KEY); + countly.removeDifferentAppKeysFromQueue(); + countly.processRQDebug(); + + CHECK(queuedRequestsSent() == std::vector{"begin_session " + SECOND_APP_KEY}); + CHECK(capturedLogContains("removeDifferentAppKeysFromQueue, Removed [2] queued requests with another app key.")); + } + + SUBCASE("attemptToSendStoredRequests sends the queued events and requests at once") { + Countly &countly = prepareCountly(); + startAndForget(countly); + countly.setLocation("TR", "Izmir", "38.42,27.14", ""); + countly.addEvent(cly::Event("sent_at_once", 1)); + countly.attemptToSendStoredRequests(); + + CHECK(queuedRequestsSent() == std::vector{"location " + FIRST_APP_KEY, "events " + FIRST_APP_KEY}); + CHECK(eventKeys(callsWith({{"events", ""}}).at(0)) == std::vector{"sent_at_once"}); + CHECK(countly.checkRQSize() == 0); + CHECK(countly.checkEQSize() == 0); + } + + SUBCASE("a call before start is ignored") { + Countly &countly = prepareCountly(); + countly.flushQueues(); + countly.replaceAllAppKeysInQueueWithCurrentAppKey(); + countly.removeDifferentAppKeysFromQueue(); + countly.attemptToSendStoredRequests(); + CHECK(logCodeCount(countly, "w101") == 4); + CHECK(capturedLogContains("flushQueues, SDK is not initialized.")); + CHECK(capturedLogContains("attemptToSendStoredRequests, SDK is not initialized.")); + } +} + +TEST_CASE("Queue operations while a request is on its way") { + HoldScope hold; + request_hold.holdNext("events="); + Countly &countly = prepareCountly(sendThroughHold); + countly.enableManualSessionControl(); + startAndForget(countly); + countly.addEvent(cly::Event("on_its_way", 1)); + hold.sender = std::thread([&countly]() { countly.attemptToSendStoredRequests(); }); + CHECK(request_hold.waitUntilHeld()); + + SUBCASE("flushQueues: the request on its way is delivered, and the request queued next is not lost") { + countly.flushQueues(); + // The queue is empty now, so this request takes the id of the one on its way + countly.beginSession(); + hold.release(); + + CHECK(queuedRequestsSent() == std::vector{"events " + FIRST_APP_KEY, "begin_session " + FIRST_APP_KEY}); + CHECK(countly.checkRQSize() == 0); + } + + SUBCASE("removeDifferentAppKeysFromQueue: the request queued next is not lost") { + countly.changeAppKey(SECOND_APP_KEY); + countly.removeDifferentAppKeysFromQueue(); + countly.beginSession(); + hold.release(); + + CHECK(queuedRequestsSent() == std::vector{"events " + FIRST_APP_KEY, "begin_session " + SECOND_APP_KEY}); + CHECK(capturedLogContains("removeDifferentAppKeysFromQueue, Removed [1] queued requests with another app key.")); + CHECK(countly.checkRQSize() == 0); + } + + SUBCASE("replaceAllAppKeysInQueueWithCurrentAppKey: the request on its way is delivered once, with the old key") { + countly.changeAppKey(SECOND_APP_KEY); + countly.replaceAllAppKeysInQueueWithCurrentAppKey(); + hold.release(); + + CHECK(queuedRequestsSent() == std::vector{"events " + FIRST_APP_KEY}); + CHECK(countly.checkRQSize() == 0); + } +} + +TEST_CASE("After an app key change the new app's remote config is downloaded, and a download for the old key stores nothing") { + HoldScope hold; + // the download at the end of start() goes through, so the held one is the download for some keys + request_hold.holdNext("keys="); + Countly &countly = prepareCountly(sendThroughHoldWithSecondAppConfig); + countly.enableManualSessionControl(); + countly.enableRemoteConfig(); + startAndForget(countly); + std::string keys[] = {"color"}; + countly.updateRemoteConfigFor(keys, 1); + REQUIRE(request_hold.waitUntilHeld()); + + countly.changeAppKey(SECOND_APP_KEY); + // The download for the new key does not wait for the one still on its way for the old key + CHECK(waitForRequest({{"method", "rc"}, {"app_key", SECOND_APP_KEY}})); + CHECK(waitUntil([&countly]() { return countly.getRemoteConfigValue("color") == "#00FF00"; })); + + hold.release(); + REQUIRE(waitForRequest({{"keys", "[\"color\"]"}})); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + CHECK(countly.getRemoteConfigValue("color") == "#00FF00"); + + // Neither download keeps the next one from starting + countly.updateRemoteConfig(); + CHECK(waitForRequest({{"method", "rc"}, {"app_key", SECOND_APP_KEY}}, 2)); +} diff --git a/tests/storage.cpp b/tests/storage.cpp index a6f50c7..d12b150 100644 --- a/tests/storage.cpp +++ b/tests/storage.cpp @@ -495,6 +495,33 @@ void RQClearAll_WithNonEmptyQueue(std::shared_ptr storageModu CHECK(storageModule->RQPeekAll().size() == 0); } +/** + * Validate methods 'RQUpdate' and 'RQRemove'. + * Result: the updated request keeps its id and place, the removed one is gone, unknown ids change nothing. + * @param *storageModule: a pointer to the storage module. + */ +void RQUpdateAndRemove(std::shared_ptr storageModule) { + storageModule->RQInsertAtEnd("request 1"); + storageModule->RQInsertAtEnd("request 2"); + storageModule->RQInsertAtEnd("request 3"); + std::vector> before = storageModule->RQPeekAll(); + REQUIRE(before.size() == 3); + + storageModule->RQUpdate(before[0]->getId(), "request 1 updated"); + storageModule->RQRemove(before[1]->getId()); + storageModule->RQUpdate(12345, "unknown request"); + storageModule->RQRemove(12345); + + std::vector> after = storageModule->RQPeekAll(); + REQUIRE(after.size() == 2); + CHECK(after[0]->getId() == before[0]->getId()); + CHECK(after[0]->getData() == "request 1 updated"); + CHECK(after[1]->getId() == before[2]->getId()); + CHECK(after[1]->getData() == "request 3"); + CHECK(storageModule->RQPeekFront()->getData() == "request 1 updated"); + validateSizes(storageModule, 2); +} + TEST_CASE("Test Memory Storage Module") { test_utils::clearSDK(); shared_ptr logger; @@ -531,6 +558,7 @@ TEST_CASE("Test Memory Storage Module") { SUBCASE("Validate method 'RQClearAll' when the request queue is empty.") { RQClearAll_WithEmptyQueue(storageModule); } SUBCASE("Validate method 'RQClearAll' when the request queue is not empty.") { RQClearAll_WithNonEmptyQueue(storageModule); } + SUBCASE("Validate methods 'RQUpdate' and 'RQRemove'.") { RQUpdateAndRemove(storageModule); } } TEST_CASE("Test Memory Storage Module without calling 'init'") { @@ -582,6 +610,7 @@ TEST_CASE("Test Sqlite Storage Module") { SUBCASE("Validate method 'RQClearAll' when the request queue is empty.") { RQClearAll_WithEmptyQueue(storageModule); } SUBCASE("Validate method 'RQClearAll' when the request queue is not empty.") { RQClearAll_WithNonEmptyQueue(storageModule); } + SUBCASE("Validate methods 'RQUpdate' and 'RQRemove'.") { RQUpdateAndRemove(storageModule); } } TEST_CASE("Test SQlite Storage Module without calling 'init'") {