From 01667a67cfc72df6c082a5c28ec6f5b9393fa939 Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:10:23 +0900 Subject: [PATCH] Fix differences from the SDK development guide Brings the C++ SDK in line with the updated SDK development guide: crash and data-loss fixes, wrong behaviours and small missing features. Crashes and data loss: start() without a device ID no longer throws (a UUID is generated, SQLite builds store it); views() and crash() before start() are safe; a throwing log listener is contained; SBS segmentation filters no longer strip the SDK's own view data; st=false no longer blocks for 15 s and deletes waiting events; only a 2xx JSON answer with "result" counts as delivered; invalid event counts, empty keys, device IDs and app keys are refused; stop() sends waiting events and ends the session; crash reports carry the OS and app version; limits below their minimum are ignored. Behaviour: every request is POST; start() refuses invalid server URLs and app keys; events are sent on the session update timer; session calls work only with manual session control; remote config uses method=rc and enrolls in A/B tests only on request; a stored device ID wins at start() unless enableClearStoredDeviceID() is called; SBS, views, crashes and location follow the guide. Added: getDeviceID, getDeviceIDType, enableClearStoredDeviceID, t/av/rr on every request, OS metrics, event IDs, failed request cool-down, request drop age, backoff, gradual queue trimming, crash filter and crash segmentation, clearBreadcrumbs, stopAllViews, disableLocation, enrollABOnRCDownload. Build: CMAKE_CXX_STANDARD is honoured, with C++14 as the default and the minimum, so the cpp17 and cpp20 CI jobs compile at C++17 and C++20. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Wc7z4S3SpREqhYT28iCtkY --- CHANGELOG.md | 46 + CMakeLists.txt | 24 +- examples/example_integration.cpp | 9 +- examples/qt_demo/README.md | 1 - examples/qt_demo/main.cpp | 11 - include/countly.hpp | 233 +++- include/countly/configuration_module.hpp | 11 + include/countly/configuration_provider.hpp | 18 + include/countly/constants.hpp | 70 ++ include/countly/countly_configuration.hpp | 76 ++ include/countly/crash_module.hpp | 19 +- include/countly/event.hpp | 11 + include/countly/internal_limits.hpp | 137 +++ include/countly/remote_config_store.hpp | 10 + include/countly/request_module.hpp | 17 + include/countly/storage_module_base.hpp | 12 + include/countly/storage_module_db.hpp | 2 + include/countly/views_module.hpp | 40 +- src/configuration_module.cpp | 218 ++-- src/countly.cpp | 1146 +++++++++++++++----- src/crash_module.cpp | 162 ++- src/event.cpp | 7 +- src/logger_module.cpp | 10 +- src/platform_info.cpp | 88 ++ src/platform_info.hpp | 22 + src/request_builder.cpp | 6 + src/request_module.cpp | 177 ++- src/storage_module_db.cpp | 97 +- src/views_module.cpp | 219 +++- tests/behavior_settings.cpp | 226 ++++ tests/config.cpp | 15 +- tests/consent.cpp | 54 +- tests/crash.cpp | 18 +- tests/crash_reporting.cpp | 164 +++ tests/event_queue.cpp | 75 +- tests/event_recording.cpp | 270 +++++ tests/init.cpp | 487 +++++++++ tests/internal_limits_integration.cpp | 2 +- tests/location.cpp | 164 +++ tests/multi_instance.cpp | 4 +- tests/mutex_exception_safety.cpp | 35 +- tests/remote_config.cpp | 207 ++++ tests/request.cpp | 93 +- tests/request_pipeline.cpp | 554 ++++++++++ tests/sbs.cpp | 66 +- tests/session.cpp | 35 +- tests/session_lifecycle.cpp | 270 +++++ tests/storage_db.cpp | 10 +- tests/test_utils.hpp | 3 +- tests/user_details.cpp | 1 - tests/views.cpp | 4 +- 51 files changed, 4970 insertions(+), 686 deletions(-) create mode 100644 src/platform_info.cpp create mode 100644 src/platform_info.hpp create mode 100644 tests/behavior_settings.cpp create mode 100644 tests/crash_reporting.cpp create mode 100644 tests/event_recording.cpp create mode 100644 tests/init.cpp create mode 100644 tests/location.cpp create mode 100644 tests/remote_config.cpp create mode 100644 tests/request_pipeline.cpp create mode 100644 tests/session_lifecycle.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index ec32e51..c17dbed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## XX.XX.XX - ! Minor breaking change ! The "Consent required" option of the SDK Behavior Settings now applies. While it is on, the SDK collects nothing until the app gives consent; while it is off, the SDK works as if every consent were given, also in apps that require consent. +- ! Minor breaking change ! Every request is now sent as POST, with its data in the request body. A client set with `setHTTPClient` gets `usePost` true for every request and must support POST. `alwaysUsePost` is deprecated and has no effect. +- ! Minor breaking change ! `start()` now refuses a server URL without `http://` or `https://` or without a host, an empty app key and the placeholder `YOUR_APP_KEY`: it logs an error and the SDK stays off. Before, a URL without a scheme was used over plain HTTP. +- ! Minor breaking change ! Recorded events are now sent on the session update interval (60 seconds by default) or when the event queue threshold is reached, instead of every 3 seconds. `flushEvents()` moves them into a request right away. +- ! Minor breaking change ! `beginSession`, `updateSession` and `endSession` now work only with `enableManualSessionControl()`; with automatic sessions they are ignored with a warning. A manual `updateSession()` now always reports the session duration, also when called sooner than the session update interval. +- ! Minor breaking change ! Remote config downloads now use the current remote config request and no longer enroll the user in A/B tests; call `enrollABOnRCDownload()` before `start()` to keep enrolling. With `enableRemoteConfig()`, values are downloaded at the end of `start()`, after a device ID change and when the consent is given, instead of at every session start. +- ! Minor breaking change ! Config values below their minimum are now ignored with a warning instead of being raised: 0 for the internal limits, `setMaxRequestQueueSize` and `setAutomaticSessionUpdateInterval`, and values below 1 for `setEventsToRQThreshold`. `setEventsToRQThreshold` also keeps values above 10000; the event queue still sends at 1000 events. +- ! Minor breaking change ! `setLocation` no longer uses values that don't fit their form, such as a lower case country code, coordinates that are not decimal degrees or an invalid IP address; it logs a warning for each. +- ! Minor breaking change ! SQLite builds now keep the device ID in use between starts, and at `start()` a stored device ID wins: a different device ID given with `setDeviceID` before `start()` is ignored with a warning. To use the given device ID, call `enableClearStoredDeviceID()` before `start()`, or change the device ID with `setDeviceID` after `start()`. - Added user consent. With `setRequiresConsent(true)` a feature works only after its consent is given, at start with `setConsentEnabled` or later with `giveConsent`, `giveConsentAll` or `setConsent`, and stops when the consent is removed with `removeConsent`, `removeConsentAll` or `setConsent`. The SDK sends every change to the server. The feature names are in `cly::consent`, and `cly::consent::allFeatures()` lists them all. `createFeatureGroup` and `setConsentFeatureGroup` handle groups of features; `getConsent` and `checkAllConsent` show the current state. - While consent is required, changing the device ID without merge removes all consent, so the app gives it again for the new user. @@ -9,10 +17,48 @@ - Added `disableHealthCheck()` for turning off the health check request. - Added `status_code` and `body` to `HTTPResponse`. A custom HTTP client can fill them so that the health check reports the last request the server refused; a client that leaves them empty works as before. - Calls made before `start()`, a second `start()`, beginning a session that already runs and ending one that does not are now logged at warning level. Failed and non-JSON server answers, and session calls ignored because the SDK Behavior Settings turned session tracking off, are now logged at debug level. +- Added device ID generation: when no device ID is stored and the app gives none before `start()`, the SDK generates one. `getDeviceID()` and `getDeviceIDType()` return the device ID in use and where it came from. Every request carries the device ID type (`t`). +- Added `enableClearStoredDeviceID()` for replacing the stored device ID at `start()` with the given or a newly generated one. The events the stored user left are first sent under the stored device ID. It is meant for testing and forced user resets: left on, every start creates a new user. +- Added the app version (`av`) and the number of waiting requests (`rr`) to every request. +- Added the operating system name and version (`_os`, `_os_version`) to the device metrics the SDK sends; values given with `setMetrics` still win. +- Added event IDs: every event carries its own ID and the current view (`cvid`), custom events the previous custom event (`peid`), view events the previous view (`pvid`) and the platform (`segment`), and crash reports the current view (`_cvid`). +- Added `setFailedRequestCooldown`: a failed request is tried again after this cool-down, 60 seconds by default, instead of on every update. +- Added `setRequestDropAgeHours` and the server's "dort" setting for dropping stored requests older than the given age. +- Added the backoff mechanism: when the server answers slowly, sending pauses for a while. `disableBackoffMechanism()` turns it off, and the SDK Behavior Settings can configure it. +- Added gradual trimming of a request queue far over its size limit, at most 100 requests at a time; `disableGradualRequestCleaner()` drops the whole overflow at once. +- Added `setCrashFilterCallback` for changing or dropping crash reports before they are queued, and `setCustomCrashSegmentation` for segmentation added to every crash report. +- Added `crash().clearBreadcrumbs()`, `views().stopAllViews()`, and segmentation for `closeViewWithName` and `closeViewWithID`. +- Added `disableLocation()`. A location given with `setLocation` before `start()` is now used from `start()` on. - Mitigated an issue where reaching the event queue threshold logged a warning that events were dropped, although they were sent. - Mitigated an issue where a view that was open during a device ID change without merge was ended under the new device ID. - Mitigated an issue where session starts let the server work out the user's location from the IP address while the SDK Behavior Settings turned location tracking off. +- Mitigated an issue where `start()` threw an exception when the app had set no device ID. +- Mitigated an issue where calling `views()` or `crash()` before `start()` crashed the app. +- Mitigated an issue where a log callback that threw an exception broke SDK calls or ended the app. +- Mitigated an issue where a segmentation whitelist or blacklist in the SDK Behavior Settings removed the names of views, so the views were missing from the dashboard. +- Mitigated an issue where, while the SDK Behavior Settings turned session tracking off, setting user details or changing the device ID blocked for 15 seconds and then deleted the waiting events, and a running session never ended. +- Mitigated an issue where any 2xx answer, such as a proxy's HTML page, counted as delivered, so the request was deleted. +- Mitigated an issue where a negative event count was sent as a very large number. Counts below 1 now become 1, and events with an empty key are ignored, with a warning. +- Mitigated an issue where an empty device ID was accepted and the server rejected every request. +- Mitigated an issue where `stop()` left the session open; it now sends the waiting events and ends the session before the SDK stops. +- Mitigated an issue where sessions were not updated and events were not sent until 100 were waiting when `start()` was called without its last parameter. +- Mitigated an issue where crash reports lacked the operating system and app version, so the server rejected them. A crash without a stack trace is now ignored with a warning. +- Mitigated an issue where breadcrumbs stayed after a crash and appeared in the next crash report. +- Mitigated an issue where an event timed with `startTimer` and `stopTimer` was recorded at its start instead of its end. +- Mitigated an issue where only the first view after `start()` was marked as the first view of a session, also without a running session. +- Mitigated an issue where custom view segmentation could replace the view's own keys, and `closeViewWithName` closed any one of several views with that name instead of the last one started. +- Mitigated an issue where a setting removed from the SDK Behavior Settings kept its last server value until the next start. +- Mitigated an issue where an empty blacklist in the SDK Behavior Settings switched off the matching whitelist. +- Mitigated an issue where events with a `[CLY]_` key the SDK does not know skipped the custom event switch, the event filters and the size limits. +- Mitigated an issue where `setSDKBehaviorSettings` ignored settings given as a copied server answer, and kept the given settings like a server answer for later starts. +- Mitigated an issue where `disableSDKBehaviorSettingsUpdates()` still let one settings request go out at start. +- Mitigated an issue where the periodic settings fetch used the device ID from `start()`. The settings are now also fetched after a device ID change. +- Mitigated an issue where remote config values downloaded for the previous user stayed after a device ID change. +- Mitigated an issue where a separate location request carried the IP address. +- Mitigated an issue where values cut to the internal limits were not logged. +- Mitigated an issue where `setUserDetails` and `setCustomUserDetails` called before `start()` logged that the call was ignored, although the details are sent with the first session. +- Mitigated an issue where a C++ standard set with `CMAKE_CXX_STANDARD`, for example by a Conan profile, was ignored and the SDK was always compiled as C++14. C++14 stays the default and the minimum. ## 26.8.1 - ! Minor breaking change ! Builds now stop with a clear error when the SDK headers find a nlohmann/json older than 3.11 first on the include path, instead of mixing two versions of the library. diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f87040..397566f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,11 @@ string(REGEX REPLACE "#define COUNTLY_SDK_VERSION \"([^\"]+)\"" "\\1" COUNTLY_SD project(countly VERSION ${COUNTLY_SDK_VERSION} LANGUAGES CXX) +# C++14 by default; a newer standard set with CMAKE_CXX_STANDARD is used, an older one is raised to C++14 +if(NOT DEFINED CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 14) +endif() + # setup configuration options and their default values option(BUILD_SHARED_LIBS "Build shared libraries" ON) option(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "Create a module definition (.def) on Windows." ON) @@ -59,7 +64,8 @@ add_library(countly ${CMAKE_CURRENT_SOURCE_DIR}/src/event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/configuration_module.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/consent_module.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/health_check_module.cpp) + ${CMAKE_CURRENT_SOURCE_DIR}/src/health_check_module.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/platform_info.cpp) add_library(countly::countly ALIAS countly) target_include_directories(countly @@ -68,9 +74,9 @@ target_include_directories(countly set_target_properties(countly PROPERTIES PUBLIC_HEADER "${COUNTLY_PUBLIC_HEADERS}" - CXX_STANDARD 14 CXX_STANDARD_REQUIRED YES CXX_EXTENSIONS NO) +target_compile_features(countly PRIVATE cxx_std_14) find_package(Threads) target_link_libraries(countly Threads::Threads) @@ -159,7 +165,15 @@ if(COUNTLY_BUILD_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/tests/multi_instance.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests/user_details.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests/health_check.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/tests/consent.cpp) + ${CMAKE_CURRENT_SOURCE_DIR}/tests/consent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/init.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/request_pipeline.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/event_recording.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/session_lifecycle.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/behavior_settings.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/remote_config.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/crash_reporting.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/location.cpp) target_compile_options(countly-tests PRIVATE $<$>:-g>) target_compile_definitions(countly-tests PRIVATE COUNTLY_BUILD_TESTS) @@ -181,9 +195,9 @@ if(COUNTLY_BUILD_TESTS) target_include_directories(countly-tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/vendor/doctest/doctest) target_link_libraries(countly-tests countly) set_target_properties(countly-tests PROPERTIES - CXX_STANDARD 14 CXX_STANDARD_REQUIRED YES CXX_EXTENSIONS NO) + target_compile_features(countly-tests PRIVATE cxx_std_14) endif() if(COUNTLY_BUILD_SAMPLE) @@ -201,9 +215,9 @@ endif() target_include_directories(countly-sample PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/vendor/doctest/doctest) target_link_libraries(countly-sample countly) set_target_properties(countly-sample PROPERTIES - CXX_STANDARD 14 CXX_STANDARD_REQUIRED YES CXX_EXTENSIONS NO) + target_compile_features(countly-sample PRIVATE cxx_std_14) endif() install(TARGETS countly diff --git a/examples/example_integration.cpp b/examples/example_integration.cpp index 3366b62..ac2cea8 100644 --- a/examples/example_integration.cpp +++ b/examples/example_integration.cpp @@ -183,13 +183,12 @@ static void configureAndStart(Countly &instance, const string &appKey, const str // Custom HTTP client // HTTPClientFunction clientPtr = customClient; // instance.setHTTPClient(clientPtr); - // instance.alwaysUsePost(true); instance.setLogger(printLog); instance.SetPath(dbPath); // this will be only built into account if the correct configurations are set instance.setDeviceID(deviceId); // instance.setSalt("test-salt"); // OS, OS_version, device, resolution, carrier, app_version); - instance.SetMetrics("Windows 10", "10.22", "Mac", "800x600", "Carrier", "1.0"); + instance.SetMetrics("Windows", "10.0", "Mac", "800x600", "Carrier", "1.0"); instance.setAutomaticSessionUpdateInterval(5); // The value is set so low just for internal validation. Has to be set before start. instance.setMaxRQProcessingBatchSize(2); // in most cases not needed to be set. The value is set so low just for internal validation @@ -257,7 +256,7 @@ static void stressBothInstances(int threadsPerInstance, int iterations) { } if (i % 5 == 0) { - target->updateSession(); + target->flushEvents(); } recorded.fetch_add(2); @@ -308,7 +307,7 @@ int main() { cout << "2) Event with count and sum" << endl; cout << "3) Event with count, sum, duration" << endl; cout << "4) Event with sum, count, duration and segmentation" << endl; - cout << "5) Update Session" << endl; + cout << "5) Send waiting events now" << endl; cout << "6) Download remote config" << endl; cout << "7) Send user detail to server" << endl; cout << "8) Change device id with server merge" << endl; @@ -347,7 +346,7 @@ int main() { break; } case 5: - ct.updateSession(); + ct.flushEvents(); break; case 6: { ct.updateRemoteConfig(); diff --git a/examples/qt_demo/README.md b/examples/qt_demo/README.md index 590ed88..406bf76 100644 --- a/examples/qt_demo/README.md +++ b/examples/qt_demo/README.md @@ -86,7 +86,6 @@ struct DevConfig { std::string sbsJson = ""; bool manualSession = false; bool disableSBSUpdates = false; - bool alwaysPost = false; bool enableRemoteConfig = false; bool disableAutoEventsOnUP = false; }; diff --git a/examples/qt_demo/main.cpp b/examples/qt_demo/main.cpp index f56e749..b943600 100644 --- a/examples/qt_demo/main.cpp +++ b/examples/qt_demo/main.cpp @@ -527,7 +527,6 @@ public slots: QTextEdit *sbsJsonEdit = nullptr; QCheckBox *manualSessionCheck = nullptr; QCheckBox *disableSBSCheck = nullptr; - QCheckBox *alwaysPostCheck = nullptr; QCheckBox *enableRemoteConfigCheck = nullptr; QCheckBox *disableAutoEventsOnUPCheck = nullptr; QLineEdit *saltEdit = nullptr; @@ -758,9 +757,6 @@ public slots: disableSBSCheck = new QCheckBox("Disable SBS Updates"); optLayout->addWidget(disableSBSCheck); - alwaysPostCheck = new QCheckBox("Always Use POST"); - optLayout->addWidget(alwaysPostCheck); - enableRemoteConfigCheck = new QCheckBox("Enable Remote Config"); optLayout->addWidget(enableRemoteConfigCheck); @@ -1478,7 +1474,6 @@ public slots: sbsJsonEdit->setText(QString::fromStdString(dc.sbsJson)); manualSessionCheck->setChecked(dc.manualSession); disableSBSCheck->setChecked(dc.disableSBSUpdates); - alwaysPostCheck->setChecked(dc.alwaysPost); enableRemoteConfigCheck->setChecked(dc.enableRemoteConfig); disableAutoEventsOnUPCheck->setChecked(dc.disableAutoEventsOnUP); logMsg("[App] Dev config loaded."); @@ -1497,7 +1492,6 @@ public slots: std::string sbsJson = sbsJsonEdit->toPlainText().trimmed().toStdString(); bool manualSession = manualSessionCheck->isChecked(); bool disableSBS = disableSBSCheck->isChecked(); - bool usePost = alwaysPostCheck->isChecked(); if (serverUrl.empty() || appKey.empty()) { logMsg("[App] Server URL and App Key are required."); @@ -1550,11 +1544,6 @@ public slots: logMsg("[App] SBS JSON provided: " + sbsJson); } - if (usePost) { - countly.alwaysUsePost(true); - logMsg("[App] Always POST: enabled"); - } - if (enableRemoteConfigCheck->isChecked()) { countly.enableRemoteConfig(); logMsg("[App] Remote config: enabled"); diff --git a/include/countly.hpp b/include/countly.hpp index 3133c6d..41c534e 100644 --- a/include/countly.hpp +++ b/include/countly.hpp @@ -110,10 +110,40 @@ class Countly : public cly::CountlyDelegates { Countly(Countly &&) = delete; Countly &operator=(Countly &&) = delete; + /** + * @deprecated Every request is sent as POST, so this has no effect. + */ void alwaysUsePost(bool value); void setMaxRequestQueueSize(unsigned int requestQueueSize); + /** + * Sets how many seconds the SDK waits after a failed request before it tries the request again. At least 1, 60 by + * default. Requests queued in the meantime don't make it try sooner. Call it before start(). + */ + void setFailedRequestCooldown(int seconds); + + /** + * Sets after how many hours a stored request is dropped without being sent, for example because the device was + * offline for a long time. 0, the default, keeps requests however old they are. The server can override it with + * its SDK behavior settings. Call it before start(). + */ + void setRequestDropAgeHours(int hours); + + /** + * Turns off the backoff mechanism: by default, when the server answers a request slowly, the SDK pauses sending + * for a while so that the server can recover. The server's SDK behavior settings can turn it on again. Call it + * before start(). + */ + void disableBackoffMechanism(); + + /** + * By default, a request queue far over its size limit, for example after the server lowered the limit, drops at + * most 100 of its oldest requests with each new request. With this, it drops all of them at once. Call it before + * start(). + */ + void disableGradualRequestCleaner(); + void setMaxRQProcessingBatchSize(unsigned int requestQueueProcessingSize); void setSalt(const std::string &value); @@ -178,8 +208,66 @@ class Countly : public cly::CountlyDelegates { */ void setLocation(const std::string &countryCode, const std::string &city, const std::string &gpsCoordinates, const std::string &ipAddress); + /** + * Before start(): the device ID to use when no device ID is stored from an earlier start (SQLite builds keep the + * device ID in use between starts). A stored device ID wins, and a different ID given here is ignored with a + * warning, unless enableClearStoredDeviceID() is called too. Without a device ID, start() generates one. + * After start(): changes the device ID. With same_user the server merges the old user's data into the new device + * ID; without it the new device ID is a new user: the waiting events, and with automatic sessions the session, end + * under the old device ID first. + */ void setDeviceID(const std::string &value, bool same_user = false); + /** + * Makes start() forget the device ID stored by an earlier start. The events that user left in the database are + * first sent under the stored device ID; then start() uses the device ID given with setDeviceID, or generates a new + * one. Meant for testing and for forced user resets: left on in a released app, every start creates a new user. + * Call it before start(); after start() it is ignored with a warning. + */ + void enableClearStoredDeviceID(); + + /** + * Sets segmentation that every crash report gets under "_custom". The segmentation a recordException call gives + * is merged in and wins for keys in both. Call it before start(). + */ + void setCustomCrashSegmentation(const std::map &segmentation); + + /** + * Sets a callback that sees every crash report before it is queued. It may change the report, for example to + * remove personal data, and returns true to drop it. Which fields it changed is reported to the server as "_ob". + * A callback that throws drops the crash. Call it before start(). + */ + void setCrashFilterCallback(CrashFilterCallback callback); + + /** + * Turns location tracking off until setLocation is called with a value: the stored location values are cleared, + * an erase request is sent, and every session start carries the empty location, so that the server works out no + * location from the IP address. Called before start(), the SDK starts with location off. While the server's SDK + * behavior settings contain "Allow Location Tracking", that setting wins. + */ + void disableLocation(); + + /** + * The device ID in use: the one stored by an earlier start, the one given with setDeviceID, or the one the SDK + * generated. Before start() it returns an empty string and logs a warning. + */ + std::string getDeviceID(); + + /** + * Where the device ID in use came from: given by the app, or generated by the SDK. Before start() it returns + * DEVELOPER_SUPPLIED and logs a warning. + */ + DeviceIdType getDeviceIDType(); + + /** + * Starts the SDK. With automatic session tracking it begins a session, updates it on the update timer and ends it + * at stop(). + * + * @param app_key: the app key of your Countly app + * @param host: the server URL, an absolute http or https URL such as "https://your.server.ly" + * @param port: the server port, 0 for the scheme's default + * @param start_thread: has no effect, the SDK always starts its update thread + */ void start(const std::string &app_key, const std::string &host, int port = 0, bool start_thread = false); void startOnCloud(const std::string &app_key); @@ -188,7 +276,34 @@ class Countly : public cly::CountlyDelegates { void setUpdateInterval(size_t milliseconds); - void addEvent(const cly::Event &event); + void addEvent(const cly::Event &event) override; + + /** + * Whether a session is running. + */ + bool IsSessionRunning() override; + + /** + * The platform name, the same value as the "_os" metric. + */ + std::string PlatformName() override; + + /** + * The device metrics: the operating system and its version the SDK detects, replaced by the values the app gave + * with setMetrics. + */ + nlohmann::json DeviceMetrics() override; + + /** + * The ID of the view started last. Empty before the first view. + */ + std::string CurrentViewId() override; + + /** + * Applies a change of the server's "Allow Location Tracking" setting: turning it off clears the stored location + * and erases the location on the server. + */ + void LocationSettingChanged() override; /* * Checks and returns the size of the event queue in memory or persistent storage. @@ -266,6 +381,12 @@ class Countly : public cly::CountlyDelegates { void setMaxStackTraceLinesPerThread(unsigned int value); void setMaxStackTraceLineLength(unsigned int value); + /** + * Moves the waiting events into a request right away, instead of at the next tick of the update timer. The + * request is sent with the next pass of the update loop. + * + * @param timeout: has no effect, the call returns at once + */ void flushEvents(std::chrono::seconds timeout = std::chrono::seconds(30)); bool beginSession(); @@ -276,6 +397,12 @@ class Countly : public cly::CountlyDelegates { void enableRemoteConfig(); + /** + * Makes every remote config download also enroll the user in the A/B tests of the keys it returns. Off by default: + * without it, downloads enroll nobody. Call it before start(). + */ + void enrollABOnRCDownload(); + void updateRemoteConfig(); nlohmann::json getRemoteConfigValue(const std::string &key); @@ -357,6 +484,11 @@ class Countly : public cly::CountlyDelegates { log(LogLevel::WARNING, "[Countly]setAutomaticSessionUpdateInterval, You can not set the session duration after SDK initialization."); return; } + if (updateInterval < 1) { + log(LogLevel::WARNING, "[Countly] setAutomaticSessionUpdateInterval, The interval must be at least 1 second, so the value [0] is ignored."); + logger->recordLogCode("w121"); + return; + } configuration->sessionDuration = updateInterval; } @@ -385,7 +517,7 @@ class Countly : public cly::CountlyDelegates { /** * Provide SDK behavior settings in JSON format string. */ - void setSDKBehaviorSettings(std::string &settings_json) { + void setSDKBehaviorSettings(const std::string &settings_json) { if (is_sdk_initialized) { log(LogLevel::WARNING, "[Countly] setSDKBehaviorSettings, You can not provide SDK behavior settings after SDK initialization."); return; @@ -555,11 +687,36 @@ class Countly : public cly::CountlyDelegates { * @param merge: true merges the response into the stored values, false replaces * them wholesale * @param caller: public method name, used in log messages - * @return true if a fetch was started + * @param queueIfRunning: true makes a call during a running fetch follow it, instead of being dropped + * @return true if a fetch was started or queued + */ + bool startRemoteConfigFetch(const std::map &data, bool merge, const char *caller, bool queueIfRunning = false); + + /** + * The parameters of a remote config download: the rc method, the device metrics with the "metrics" consent, and + * the A/B test enrollment when the app asked for it. */ - bool startRemoteConfigFetch(const std::map &data, bool merge, const char *caller); + std::map remoteConfigRequest(); + + /** + * Downloads every remote config value when the app enabled automatic downloads and the "remote-config" consent is + * given. A download already running is followed by this one. + */ + void downloadRemoteConfigAutomatically(); + + /** + * Deletes the stored remote config values, which belonged to the previous user, and makes a download that is + * still running for that user discard its answer. + */ + void clearRemoteConfigForNewUser(); void _deleteThread(); + + /** + * The device metrics: the operating system and its version the SDK detects, replaced by the values the app gave + * with setMetrics. + */ + nlohmann::json deviceMetrics(); void _sendIndependantLocationRequest(); void log(LogLevel level, const std::string &message); bool createEventTableSchema(); @@ -568,6 +725,45 @@ class Countly : public cly::CountlyDelegates { void _changeDeviceIdWithoutMerge(const std::string &value); + /** + * Runs after a merge request was delivered: what needs the merged user on the server, such as the SDK behavior + * settings fetch, may go out now. + */ + void onMergeDelivered(); + + /** + * Decides the device ID at start(): the one stored by an earlier start, otherwise the given one, otherwise a newly + * generated UUID. Returns true when enableClearStoredDeviceID asks to replace a stored device ID: the stored one + * stays in use until replaceStoredDeviceId() has queued its user's events. The caller must hold the instance mutex + * and the storage must be initialized. + */ + bool resolveDeviceId(const std::string &given_device_id); + + /** + * Queues the events the stored user left in the database under the stored device ID, then uses the given device ID, + * or a newly generated one, and stores it. The caller must hold the instance mutex and the modules must be wired. + */ + void replaceStoredDeviceId(const std::string &given_device_id); + + /** + * Uses the given device ID, or a newly generated UUID when it is empty, and stores it. The caller must hold the + * instance mutex. + */ + void useNewDeviceId(const std::string &given_device_id); + + /** + * Reads every event stored in the database into the array, and the IDs of the rows read, as "(1,2,3)", into + * event_ids; empty when there is no row. A row that is not valid JSON is skipped with a warning, its ID is still + * listed, so that removing the read rows drops it. The caller must hold the instance mutex. + */ + void readStoredEvents(nlohmann::json &events, std::string &event_ids); + + /** + * Stores the device ID in use and its type, so that a later start without a given ID keeps it. Memory storage + * keeps nothing. The caller must hold the instance mutex. + */ + void storeDeviceId(); + /** * Queues a request with the stored user details for the current device ID and drops the stored copy, * so a later session does not send them again. The caller must hold the instance mutex. @@ -584,6 +780,21 @@ class Countly : public cly::CountlyDelegates { std::chrono::steady_clock::duration getSessionDuration(); void updateLoop(); + + /** + * One pass of the update loop: once an update interval passed, updates the automatic session and moves the waiting + * events into a request; then sends the request queue and stores the health check counters when due. + * + * @param last_update: when the session was last updated by the timer, moved forward when it is updated now + */ + void updateTick(std::chrono::steady_clock::time_point &last_update); + + /** + * Whether a session call of the app has to be ignored because the SDK tracks sessions automatically. Logs the + * warning and counts the call and log code w740. + */ + bool rejectSessionCallInAutomaticMode(const char *caller, const char *usage); + void packEvents(); /** @@ -681,12 +892,23 @@ class Countly : public cly::CountlyDelegates { */ void clearStoredLocation(); + /** + * Queues the empty location, which erases the user's location on the server. + */ + void sendLocationErase(); + /** * Applies the server's stored "Consent required" setting, and while consent is required, queues the consent * state and, when location is off and no session start will be sent, a location erase. Called at the end of start(). */ void activateConsent(); bool began_session = false; + // Set by disableLocation(), cleared by setLocation() with a value; the server's "lt" setting wins over it + std::atomic location_disabled_by_app{false}; + // Set at the end of start(): before that, a change of the server's location setting only updates the state + std::atomic start_finished{false}; + // The ID of the custom event recorded last, sent as "peid" with the next one. Guarded by the instance mutex. + std::string previous_event_id; std::atomic is_being_disposed{false}; std::atomic is_sdk_initialized{false}; @@ -729,7 +951,8 @@ class Countly : public cly::CountlyDelegates { std::shared_ptr mutex = std::make_shared(); bool is_queue_being_processed = false; - std::atomic enable_automatic_session{false}; + // Set by stop(): the update loop begins and updates no session from then on + std::atomic stopping{false}; std::atomic stop_thread{false}; std::atomic running{false}; std::condition_variable stop_cv; // Wakes updateLoop immediately on stop diff --git a/include/countly/configuration_module.hpp b/include/countly/configuration_module.hpp index ee8143d..f9f69a8 100644 --- a/include/countly/configuration_module.hpp +++ b/include/countly/configuration_module.hpp @@ -35,6 +35,11 @@ class ConfigurationModule : public ConfigurationProvider { bool isNetworkingEnabled() const override; bool isLocationTrackingEnabled(); + + /** + * The server's "lt" setting: -1 while the settings don't contain it, otherwise 0 or 1. + */ + int getServerLocationTracking() const; bool isViewTrackingEnabled() const override; bool isSessionTrackingEnabled(); bool isCustomEventTrackingEnabled(); @@ -46,6 +51,12 @@ class ConfigurationModule : public ConfigurationProvider { int getConsentRequired() const; unsigned int getRequestQueueSizeLimit() const override; + unsigned int getRequestDropAgeHours() const override; + bool isBackoffEnabled() const override; + unsigned int getBackoffAcceptedTimeout() const override; + double getBackoffRequestQueuePercentage() const override; + unsigned int getBackoffRequestAge() const override; + unsigned int getBackoffDuration() const override; SDKLimits getLimits() const override; unsigned int getEventQueueSizeLimit(); unsigned int getSessionUpdateInterval(); diff --git a/include/countly/configuration_provider.hpp b/include/countly/configuration_provider.hpp index 943f280..86da0d8 100644 --- a/include/countly/configuration_provider.hpp +++ b/include/countly/configuration_provider.hpp @@ -21,6 +21,24 @@ class ConfigurationProvider { virtual bool isViewTrackingEnabled() const = 0; virtual unsigned int getRequestQueueSizeLimit() const = 0; virtual SDKLimits getLimits() const = 0; + + /** Hours after which a stored request is dropped without being sent, 0 to keep requests however old they are. */ + virtual unsigned int getRequestDropAgeHours() const { return 0; } + + /** Whether the backoff mechanism is on. */ + virtual bool isBackoffEnabled() const { return true; } + + /** Seconds of response time from which an answer counts as slow. */ + virtual unsigned int getBackoffAcceptedTimeout() const { return 10; } + + /** Share of the request queue size, above 0 and at most 1, up to which a slow answer starts a backoff. */ + virtual double getBackoffRequestQueuePercentage() const { return 0.5; } + + /** Hours of request age up to which a slow answer starts a backoff. */ + virtual unsigned int getBackoffRequestAge() const { return 24; } + + /** Seconds a backoff pauses sending. */ + virtual unsigned int getBackoffDuration() const { return 60; } }; } // namespace cly #endif \ No newline at end of file diff --git a/include/countly/constants.hpp b/include/countly/constants.hpp index fd1723e..6841a81 100644 --- a/include/countly/constants.hpp +++ b/include/countly/constants.hpp @@ -23,6 +23,7 @@ static_assert(NLOHMANN_JSON_VERSION_MAJOR == 3 && NLOHMANN_JSON_VERSION_MINOR >= #define COUNTLY_POST_THRESHOLD 2000 #define COUNTLY_KEEPALIVE_INTERVAL 3000 #define COUNTLY_MAX_EVENTS_DEFAULT 200 +#define COUNTLY_MAX_EVENT_QUEUE_SIZE 1000 #define COUNTLY_MAX_KEY_LENGTH_DEFAULT 128 #define COUNTLY_MAX_VALUE_SIZE_DEFAULT 256 #define COUNTLY_MAX_SEGMENTATION_VALUES_DEFAULT 100 @@ -148,10 +149,79 @@ inline std::string generateEventID() { return r.str(); } +/** + * Generates a random version 4 UUID in lower case, for example "1b9d6bcd-bbfd-4b2d-9b5d-ab8dfbbd4bed". + * + * The engine is seeded from std::random_device together with the clocks and the thread, because some standard + * libraries implement std::random_device with a fixed sequence. + * + * @return the UUID text + */ +inline std::string generateUUID() { + static thread_local std::mt19937_64 engine([] { + std::random_device device; + std::seed_seq seed{static_cast(device()), static_cast(device()), static_cast(std::chrono::system_clock::now().time_since_epoch().count()), + static_cast(std::chrono::steady_clock::now().time_since_epoch().count()), static_cast(std::hash()(std::this_thread::get_id()))}; + return std::mt19937_64(seed); + }()); + std::uniform_int_distribution byte(0, 255); + + unsigned char bytes[16]; + for (unsigned char &value : bytes) { + value = static_cast(byte(engine)); + } + bytes[6] = static_cast((bytes[6] & 0x0F) | 0x40); + bytes[8] = static_cast((bytes[8] & 0x3F) | 0x80); + + static const char digits[] = "0123456789abcdef"; + std::string uuid; + uuid.reserve(36); + for (int index = 0; index < 16; index++) { + if (index == 4 || index == 6 || index == 8 || index == 10) { + uuid += '-'; + } + uuid += digits[bytes[index] >> 4]; + uuid += digits[bytes[index] & 0x0F]; + } + return uuid; +} + } // namespace utils +class Event; + class CountlyDelegates { public: + /** + * Records an event, with the event-level fields the SDK set on it, such as the ID of a view. + */ + virtual void addEvent(const cly::Event &event) = 0; + + /** + * Whether a session is running, so that the first view of a session can be marked. + */ + virtual bool IsSessionRunning() { return false; } + + /** + * The platform name, the same value as the "_os" metric, sent as the "segment" of view events. + */ + virtual std::string PlatformName() { return std::string(); } + + /** + * The device metrics, such as "_os" and "_app_version", for crash reports. + */ + virtual nlohmann::json DeviceMetrics() { return nlohmann::json::object(); } + + /** + * The ID of the view started last, for the "_cvid" of crash reports. Empty before the first view. + */ + virtual std::string CurrentViewId() { return std::string(); } + + /** + * Called by the SDK behavior settings when the server's "Allow Location Tracking" setting changes. + */ + virtual void LocationSettingChanged() {} + virtual void RecordEvent(const std::string &key, int count) = 0; virtual void RecordEvent(const std::string &key, int count, double sum) = 0; diff --git a/include/countly/countly_configuration.hpp b/include/countly/countly_configuration.hpp index 916ed49..816dd0b 100644 --- a/include/countly/countly_configuration.hpp +++ b/include/countly/countly_configuration.hpp @@ -1,9 +1,40 @@ #ifndef COUNTLY_CONFIGURATION_HPP_ #define COUNTLY_CONFIGURATION_HPP_ #include "countly/constants.hpp" +#include +#include #include +#include namespace cly { +/** + * Where the device ID came from. Sent with every request as the "t" parameter. + */ +enum class DeviceIdType { DEVELOPER_SUPPLIED = 0, SDK_GENERATED = 1 }; + +/** + * A crash report as the crash filter sees it. The filter may change any field before the report is queued. + */ +struct CrashData { + /** The crash title, sent as "_name". */ + std::string description; + /** The stack trace, sent as "_error". */ + std::string stackTrace; + /** The custom segmentation, sent as "_custom". */ + std::map crashSegmentation; + /** The breadcrumbs, oldest first, sent joined as "_logs". */ + std::vector breadcrumbs; + /** The device metrics, such as "_os" and "_app_version". */ + std::map crashMetrics; + /** Whether the crash ended the app; sent as the opposite, "_nonfatal". */ + bool fatal = false; +}; + +/** + * Sees every crash report before it is queued and may change it. Returns true to drop the report. + */ +using CrashFilterCallback = std::function; + struct CountlyConfiguration { /** * URL of the Countly server to submit data to. @@ -22,6 +53,11 @@ struct CountlyConfiguration { */ std::string deviceId; + /** + * Where the device ID came from. + */ + DeviceIdType deviceIdType = DeviceIdType::DEVELOPER_SUPPLIED; + /** * Set to prevent parameter tampering. */ @@ -52,6 +88,46 @@ struct CountlyConfiguration { */ unsigned int maxProcessingBatchSize = 100; + /** + * Seconds to wait after a failed request before it is tried again. + */ + unsigned int failedRequestCooldownSeconds = 60; + + /** + * Hours after which a stored request is dropped without being sent, 0 to keep requests however old they are. + */ + unsigned int requestDropAgeHours = 0; + + /** + * Whether sending pauses for a while after the server answered slowly. + */ + bool backoffMechanismEnabled = true; + + /** + * Whether a request queue far over its limit is trimmed by at most 100 requests at a time. + */ + bool gradualRequestCleanerEnabled = true; + + /** + * Whether remote config downloads also enroll the user in the A/B tests of the keys they return. + */ + bool enrollABOnRCDownload = false; + + /** + * Whether start() replaces a device ID stored by an earlier start with the given or a newly generated one. + */ + bool clearStoredDeviceID = false; + + /** + * Segmentation added to every crash report. + */ + std::map crashSegmentation; + + /** + * Sees every crash report before it is queued, null when the app set none. + */ + CrashFilterCallback crashFilter = nullptr; + /** * Set the maximum amount of breadcrumbs. */ diff --git a/include/countly/crash_module.hpp b/include/countly/crash_module.hpp index d0f09a8..dd06a26 100644 --- a/include/countly/crash_module.hpp +++ b/include/countly/crash_module.hpp @@ -36,21 +36,36 @@ class CrashModule { */ void recordException(const std::string &title, const std::string &stackTrace, const bool fatal, const std::map &crashMetrics, const std::map &segmentation = {}); + /** + * Removes every breadcrumb recorded so far, so that the next crash report starts with an empty trail. + */ + void clearBreadcrumbs(); + private: friend class Countly; class CrashModuleImpl; std::unique_ptr impl; void setConfigurationProvider(std::weak_ptr provider); // try injecting + /** + * Sets the request queue that crash reports go to. Set by start(); until then crash calls are ignored. + */ + void setRequestModule(std::shared_ptr requestModule); + + /** + * Sets the SDK instance the module asks for the device metrics and the current view. + */ + void setDelegates(cly::CountlyDelegates *cly); + /** * Sets the consent state the crash calls check for the "crashes" consent. */ void setConsentModule(std::shared_ptr consent); /** - * Forgets every breadcrumb. + * Forgets every breadcrumb, without counting it as a call of the app. */ - void clearBreadcrumbs(); + void forgetBreadcrumbs(); }; } // namespace cly #endif diff --git a/include/countly/event.hpp b/include/countly/event.hpp index 7495c74..35e858d 100644 --- a/include/countly/event.hpp +++ b/include/countly/event.hpp @@ -5,6 +5,8 @@ #include namespace cly { +class Countly; +class ViewsModule; class Event { public: @@ -39,6 +41,15 @@ class Event { void applyLimits(unsigned int maxKeyLength, unsigned int maxValueSize, unsigned int maxSegmentationValues); private: + // The SDK adds the event-level fields an app can't set itself, such as the event ID. + friend class Countly; + friend class ViewsModule; + + /** + * Sets the timestamp, hour and day of week of the event to the current time. + */ + void setRecordingTime(); + nlohmann::json object; bool timer_running; std::chrono::system_clock::time_point timestamp; diff --git a/include/countly/internal_limits.hpp b/include/countly/internal_limits.hpp index 22931e2..68b79b3 100644 --- a/include/countly/internal_limits.hpp +++ b/include/countly/internal_limits.hpp @@ -2,6 +2,8 @@ #define COUNTLY_INTERNAL_LIMITS_HPP_ #include "countly/configuration_provider.hpp" // for cly::SDKLimits +#include "countly/logger_module.hpp" +#include "nlohmann/json.hpp" #include #include @@ -60,6 +62,141 @@ inline std::string truncateStackTrace(const std::string &trace, unsigned int max return result; } +/** + * The most characters a stack trace keeps, whatever its line limits allow. + */ +const size_t MAX_STACK_TRACE_LENGTH = 20000; + +/** + * Cuts a key to the key length limit. A cut key is logged as a warning and counted as log code w201. + * + * @param key: the key + * @param maxLength: the key length limit + * @param logger: logs the cut, may be null + * @param where: the module and method, for example "[ViewsModule] openView" + * @return the key, cut when it was too long + */ +inline std::string limitKey(const std::string &key, unsigned int maxLength, LoggerModule *logger, const std::string &where) { + const std::string limited = truncateString(key, maxLength); + if (logger != nullptr && limited.size() != key.size()) { + logger->log(LogLevel::WARNING, "[Countly] " + where + ", The key [" + limited + "...] is longer than the key length limit of [" + std::to_string(maxLength) + "] and is cut."); + logger->recordLogCode("w201"); + } + return limited; +} + +/** + * Cuts a value to its size limit. A cut value is logged as a warning and counted as log code w202. + * + * @param value: the value + * @param maxLength: the size limit + * @param logger: logs the cut, may be null + * @param where: the module and method + * @param name: what the value belongs to, for example its segmentation key + * @return the value, cut when it was too long + */ +inline std::string limitValue(const std::string &value, unsigned int maxLength, LoggerModule *logger, const std::string &where, const std::string &name) { + const std::string limited = truncateString(value, maxLength); + if (logger != nullptr && limited.size() != value.size()) { + logger->log(LogLevel::WARNING, "[Countly] " + where + ", The value of [" + name + "] is longer than the size limit of [" + std::to_string(maxLength) + "] and is cut."); + logger->recordLogCode("w202"); + } + return limited; +} + +/** + * Logs the segmentation entries dropped because of the entry count limit, as a warning counted as log code w203. + */ +inline void logDroppedEntries(const std::string &dropped, unsigned int maxEntries, LoggerModule *logger, const std::string &where) { + if (logger != nullptr && !dropped.empty()) { + logger->log(LogLevel::WARNING, "[Countly] " + where + ", The segmentation has more than [" + std::to_string(maxEntries) + "] entries, so these are dropped: [" + dropped + "]"); + logger->recordLogCode("w203"); + } +} + +/** + * Applies the key length, value size and entry count limits to a segmentation map, logging what it cuts or drops. + */ +inline std::map limitSegmentation(const std::map &in, const SDKLimits &lim, LoggerModule *logger, const std::string &where) { + std::map out; + std::string dropped; + unsigned int kept = 0; + for (const auto &kv : in) { + if (kept >= lim.maxSegmentationValues) { + dropped += (dropped.empty() ? "" : ", ") + kv.first; + continue; + } + out[limitKey(kv.first, lim.maxKeyLength, logger, where)] = limitValue(kv.second, lim.maxValueSize, logger, where, kv.first); + ++kept; + } + logDroppedEntries(dropped, lim.maxSegmentationValues, logger, where); + return out; +} + +/** + * Applies the key length, value size and entry count limits to a segmentation object, logging what it cuts or + * drops. Values that are not strings are kept as they are. + */ +inline void limitSegmentation(nlohmann::json &segmentation, const SDKLimits &lim, LoggerModule *logger, const std::string &where) { + if (!segmentation.is_object()) { + return; + } + nlohmann::json limited = nlohmann::json::object(); + std::string dropped; + unsigned int kept = 0; + for (nlohmann::json::const_iterator it = segmentation.begin(); it != segmentation.end(); ++it) { + if (kept >= lim.maxSegmentationValues) { + dropped += (dropped.empty() ? "" : ", ") + it.key(); + continue; + } + const std::string key = limitKey(it.key(), lim.maxKeyLength, logger, where); + limited[key] = it.value().is_string() ? nlohmann::json(limitValue(it.value().get(), lim.maxValueSize, logger, where, it.key())) : it.value(); + ++kept; + } + logDroppedEntries(dropped, lim.maxSegmentationValues, logger, where); + segmentation = limited; +} + +/** + * Cuts an error text, such as a crash title, to the stack trace line length limit. A cut text is logged as a warning + * and counted as log code w205. + */ +inline std::string limitErrorText(const std::string &text, unsigned int maxLength, LoggerModule *logger, const std::string &where) { + const std::string limited = truncateString(text, maxLength); + if (logger != nullptr && limited.size() != text.size()) { + logger->log(LogLevel::WARNING, "[Countly] " + where + ", The error text is longer than the limit of [" + std::to_string(maxLength) + "] characters and is cut."); + logger->recordLogCode("w205"); + } + return limited; +} + +/** + * Applies the line count and line length limits to a stack trace and caps it at MAX_STACK_TRACE_LENGTH characters. + * A cut stack trace is logged as a warning and counted as log code w205. + */ +inline std::string limitStackTrace(const std::string &trace, unsigned int maxLines, unsigned int maxLineLength, LoggerModule *logger, const std::string &where) { + std::string limited = truncateStackTrace(trace, maxLines, maxLineLength); + bool cut = false; + std::istringstream stream(trace); + std::string line; + unsigned int count = 0; + while (std::getline(stream, line)) { + if (++count > maxLines || line.size() > maxLineLength) { + cut = true; + break; + } + } + if (limited.size() > MAX_STACK_TRACE_LENGTH) { + limited = truncateString(limited, static_cast(MAX_STACK_TRACE_LENGTH)); + cut = true; + } + if (logger != nullptr && cut) { + logger->log(LogLevel::WARNING, "[Countly] " + where + ", The stack trace is longer than its limits of [" + std::to_string(maxLines) + "] lines of [" + std::to_string(maxLineLength) + "] characters and is cut."); + logger->recordLogCode("w205"); + } + return limited; +} + } // namespace limits } // namespace cly diff --git a/include/countly/remote_config_store.hpp b/include/countly/remote_config_store.hpp index b6966a6..18407ed 100644 --- a/include/countly/remote_config_store.hpp +++ b/include/countly/remote_config_store.hpp @@ -3,7 +3,9 @@ #include "nlohmann/json.hpp" #include +#include #include +#include namespace cly { @@ -27,6 +29,14 @@ struct RemoteConfigStore { // True from the moment a fetch is started until its thread body returns. std::atomic fetch_running{false}; + + // Bumped when the device ID changes, so that a download that started for an earlier user is discarded. + unsigned long long user_generation = 0; + + // A download the SDK asked for while another one ran, made right after it. + bool has_pending = false; + std::map pending_data; + bool pending_merge = false; }; } // namespace cly diff --git a/include/countly/request_module.hpp b/include/countly/request_module.hpp index 77d3e0a..1a358c0 100644 --- a/include/countly/request_module.hpp +++ b/include/countly/request_module.hpp @@ -1,5 +1,6 @@ #ifndef REQUEST_MODULE_HPP_ #define REQUEST_MODULE_HPP_ +#include #include #include #include @@ -20,6 +21,16 @@ class RequestModule { HTTPResponse sendHTTP(std::string path, std::string data); + /** + * Sends a request that does not go through the request queue, such as a remote config download or the health + * check, with the number of queued requests as "rr". + * + * @param path: endpoint path, for example "/o/sdk" + * @param data: the request built with the base request parameters + * @return the server's answer + */ + HTTPResponse sendImmediate(const std::string &path, const std::string &data); + /** * SDK central execution call for processing requests in the request queue. * Only one sender is active at a time. Requests are processed in order. @@ -39,6 +50,12 @@ class RequestModule { long long RQSize(); void setConfigurationProvider(std::weak_ptr provider); // try injecting + /** + * Sets what runs after a queued merge request (one with "old_device_id") was delivered, on the thread that sent + * it and without the instance mutex held. + */ + void setOnMergeDelivered(std::function callback); + /** * Process-wide network initialisation. Idempotent and harmless to call at any * time; a no-op unless the SDK is built against libcurl. diff --git a/include/countly/storage_module_base.hpp b/include/countly/storage_module_base.hpp index 23b4c35..82c0937 100644 --- a/include/countly/storage_module_base.hpp +++ b/include/countly/storage_module_base.hpp @@ -110,6 +110,18 @@ class StorageModuleBase { * @return the counters as JSON text, empty when none are stored */ virtual std::string getHealthCheckState() { return std::string(); } + + /** + * Store the device ID in use and its type, for a later start that is given no device ID. + * @param state: the device ID and its type as JSON text + */ + virtual void storeDeviceIdState(const std::string &state) { (void)state; } + + /** + * Read the stored device ID and its type. + * @return the device ID and its type as JSON text, empty when none are stored + */ + virtual std::string getDeviceIdState() { return std::string(); } }; } // namespace cly diff --git a/include/countly/storage_module_db.hpp b/include/countly/storage_module_db.hpp index 540b303..8c1e20f 100644 --- a/include/countly/storage_module_db.hpp +++ b/include/countly/storage_module_db.hpp @@ -28,6 +28,8 @@ class StorageModuleDB : public StorageModuleBase { std::string getSDKBehaviorSettings() override; void storeHealthCheckState(const std::string &state) override; std::string getHealthCheckState() override; + void storeDeviceIdState(const std::string &state) override; + std::string getDeviceIdState() override; }; } // namespace cly #endif \ No newline at end of file diff --git a/include/countly/views_module.hpp b/include/countly/views_module.hpp index 090a76d..5959967 100644 --- a/include/countly/views_module.hpp +++ b/include/countly/views_module.hpp @@ -10,6 +10,7 @@ namespace cly { class ConsentModule; +class Event; class ViewsModule { @@ -21,15 +22,17 @@ class ViewsModule { * Close view with id. * * @param viewId: id of the view. + * @param segmentation: custom data added to the event that ends the view. */ - void closeViewWithID(const std::string &viewId); + void closeViewWithID(const std::string &viewId, const std::map &segmentation = {}); /** - * Close view with name. + * Close view with name. When several running views have that name, the one started last is closed. * * @param name: name of the view. + * @param segmentation: custom data added to the event that ends the view. */ - void closeViewWithName(const std::string &name); + void closeViewWithName(const std::string &name, const std::map &segmentation = {}); /** * Open view with name. @@ -39,10 +42,15 @@ class ViewsModule { */ std::string openView(const std::string &name, const std::map &segmentation = {}); + /** + * Closes every running view, each with its own duration. + * + * @param segmentation: custom data added to the event that ends each view. + */ + void stopAllViews(const std::map &segmentation = {}); private: friend class Countly; - void _recordView(std::string eventID); class ViewModuleImpl; std::unique_ptr impl; void setConfigurationProvider(std::weak_ptr provider); // try injecting @@ -54,9 +62,29 @@ class ViewsModule { /** * Ends every running view and records its end event, while the SDK behavior settings allow view tracking; - * otherwise the views are dropped without an event. + * otherwise the views are dropped without an event. Not counted as a call of the app. + */ + void stopRunningViews(); + + /** + * Lets the next view started while a session runs be the first view of that session. + */ + void resetFirstView(); + + /** + * Forgets the current and the previous view, after the running views were stopped for another user. + */ + void forgetViews(); + + /** + * The ID of the view started last, sent as "cvid" with other events. Empty before the first view. + */ + std::string currentViewId(); + + /** + * Sets an event-level field of a view event, such as the view ID. */ - void stopAllViews(); + static void setEventField(cly::Event &event, const std::string &key, const std::string &value); }; } // namespace cly #endif diff --git a/src/configuration_module.cpp b/src/configuration_module.cpp index c0b04af..74fe035 100644 --- a/src/configuration_module.cpp +++ b/src/configuration_module.cpp @@ -1,5 +1,7 @@ #include "countly/configuration_module.hpp" +#include #include +#include #include #include @@ -94,8 +96,19 @@ class ConfigurationModule::ConfigurationModuleImpl { std::atomic maxStackTraceLinesPerThread{COUNTLY_MAX_STACK_TRACE_LINES_PER_THREAD_DEFAULT}; std::atomic maxStackTraceLineLength{COUNTLY_MAX_STACK_TRACE_LINE_LENGTH_DEFAULT}; + std::atomic requestDropAgeHours{0}; + std::atomic backoffEnabled{true}; + std::atomic backoffAcceptedTimeout{10}; + std::atomic backoffRequestQueuePercentage{0.5}; + std::atomic backoffRequestAge{24}; + std::atomic backoffDuration{60}; + mutable std::mutex sbsMutex; std::thread configFetchThread; + std::atomic fetchRunning{false}; + std::atomic fetchAgain{false}; + // -1 while the settings don't contain "lt", otherwise 0 or 1 + std::atomic serverLocationTracking{-1}; mutable std::mutex filterMutex; FilterList> eventFilter; @@ -134,38 +147,60 @@ class ConfigurationModule::ConfigurationModuleImpl { // Helper to populate a list filter from blacklist/whitelist keys, reducing duplication template void populateListFilter(FilterList &filter, const char *blacklistKey, const char *whitelistKey, ParseFunc parseFunc) { - if (sdk_behavior_settings.contains(blacklistKey)) { + T blacklist = sdk_behavior_settings.contains(blacklistKey) ? parseFunc(sdk_behavior_settings[blacklistKey]) : T(); + T whitelist = sdk_behavior_settings.contains(whitelistKey) ? parseFunc(sdk_behavior_settings[whitelistKey]) : T(); + if (!blacklist.empty()) { filter.isWhitelist = false; - filter.filterList = parseFunc(sdk_behavior_settings[blacklistKey]); - } else if (sdk_behavior_settings.contains(whitelistKey)) { + filter.filterList = blacklist; + } else if (!whitelist.empty()) { filter.isWhitelist = true; - filter.filterList = parseFunc(sdk_behavior_settings[whitelistKey]); + filter.filterList = whitelist; } else { filter.filterList.clear(); filter.isWhitelist = false; } } - void _fetchConfigFromServerHTTP(const std::map &data, const nlohmann::json &session_params) { + /** + * Fetches the settings for the device ID in use, and again while another fetch was asked for during it. + */ + void _fetchLoop() { + do { + fetchAgain = false; + _fetchConfigFromServerHTTP(); + } while (fetchAgain.load()); + fetchRunning = false; + } + + /** + * Whether the answer is a valid settings answer: a JSON object with a numeric "v", a numeric "t" and an object "c". + */ + static bool isValidAnswer(const nlohmann::json &answer) { + return answer.is_object() && answer.contains(KEY_VERSION) && answer[KEY_VERSION].is_number() && answer.contains(KEY_TIMESTAMP) && answer[KEY_TIMESTAMP].is_number() && answer.contains(KEY_CONFIG) && answer[KEY_CONFIG].is_object(); + } + + void _fetchConfigFromServerHTTP() { try { - HTTPResponse response = _requestModule->sendHTTP("/o/sdk", _requestBuilder->buildFetchRequest(data)); - if (response.success && response.data.is_object() && response.data.contains(KEY_CONFIG)) { + std::map data; + { + std::lock_guard mutexLock(*_mutex); + data = {{"method", "sc"}, {"app_key", _configuration->appKey}, {"device_id", _configuration->deviceId}}; + } + HTTPResponse response = _requestModule->sendImmediate("/o/sdk", _requestBuilder->buildRequest(data)); + if (response.success && isValidAnswer(response.data)) { nlohmann::json changedSettings; { std::lock_guard lock(sbsMutex); - sanitizeConfig(response.data[KEY_CONFIG]); - sdk_behavior_settings = response.data[KEY_CONFIG]; + nlohmann::json settings = response.data[KEY_CONFIG]; + sanitizeConfig(settings); + sdk_behavior_settings = settings; _storageModule->storeSDKBehaviorSettings(sdk_behavior_settings.dump()); _logger->log(LogLevel::INFO, "[Countly] [ConfigurationModule] _fetchConfigFromServerHTTP, SDK config:\n" + sdk_behavior_settings.dump(2)); changedSettings = _populateConfigValues(); } - _onSBSChanged(changedSettings, session_params); + _onSBSChanged(changedSettings); } else { - _logger->log(LogLevel::WARNING, - "[Countly] [ConfigurationModule] _fetchConfigFromServerHTTP, failed to fetch." - " success=" + - std::string(response.success ? "true" : "false") + ", is_object=" + std::string(response.data.is_object() ? "true" : "false") + - ", has_config=" + std::string((response.data.is_object() && response.data.contains(KEY_CONFIG)) ? "true" : "false") + ", response=" + response.data.dump()); + _logger->log(LogLevel::DEBUG, "[Countly] [ConfigurationModule] _fetchConfigFromServerHTTP, the settings could not be fetched, the ones in use stay. success=" + std::string(response.success ? "true" : "false") + ", response=" + response.data.dump()); } } catch (const std::exception &e) { _logger->log(LogLevel::ERROR, "[Countly] [ConfigurationModule] _fetchConfigFromServerHTTP, exception: [" + std::string(e.what()) + "]"); @@ -181,9 +216,13 @@ class ConfigurationModule::ConfigurationModuleImpl { _logger->log(LogLevel::INFO, "[Countly] [ConfigurationModule] _initializeSBSFromStorage, initialized SDK behavior settings from storage."); return changed; } else if (!_configuration->sdkBehaviorSettings.empty()) { - nlohmann::json changed = _processSDKBehaviorSettings(_configuration->sdkBehaviorSettings); - // Persist the provided SBS so it's available on future re-inits - _storageModule->storeSDKBehaviorSettings(sdk_behavior_settings.dump()); + const nlohmann::json provided = nlohmann::json::parse(_configuration->sdkBehaviorSettings, nullptr, false); + if (provided.is_discarded() || !provided.is_object()) { + _logger->log(LogLevel::WARNING, "[Countly] [ConfigurationModule] _initializeSBSFromStorage, The provided SDK behavior settings are not a JSON object and are ignored."); + return nlohmann::json{}; + } + const nlohmann::json settings = provided.contains(KEY_CONFIG) && provided[KEY_CONFIG].is_object() ? provided[KEY_CONFIG] : provided; + nlohmann::json changed = _processSDKBehaviorSettings(settings.dump()); _logger->log(LogLevel::INFO, "[Countly] [ConfigurationModule] _initializeSBSFromStorage, initialized SDK behavior settings from configuration."); return changed; } @@ -200,6 +239,9 @@ class ConfigurationModule::ConfigurationModuleImpl { maxBreadcrumbCount.store(_configuration->breadcrumbsThreshold, std::memory_order_release); maxStackTraceLinesPerThread.store(_configuration->maxStackTraceLinesPerThread, std::memory_order_release); maxStackTraceLineLength.store(_configuration->maxStackTraceLineLength, std::memory_order_release); + + requestDropAgeHours.store(_configuration->requestDropAgeHours, std::memory_order_release); + backoffEnabled.store(_configuration->backoffMechanismEnabled, std::memory_order_release); } nlohmann::json _processSDKBehaviorSettings(const std::string &settings) { @@ -228,23 +270,25 @@ class ConfigurationModule::ConfigurationModuleImpl { _cly->ConsentRequirementChanged(); } - if (changedSettings.contains(KEY_LOCATION_TRACKING) && changedSettings[KEY_LOCATION_TRACKING] == false) { - // disable location - safe because _onSBSChanged is called outside of _mutex - _cly->RecordLocation("", "", "", ""); + if (changedSettings.contains(KEY_LOCATION_TRACKING)) { + // safe because _onSBSChanged is called outside of _mutex + _cly->LocationSettingChanged(); } } nlohmann::json _populateConfigValues() { - bool trackingEnabledVal = trackingEnabled.load(std::memory_order_acquire); - bool networkingEnabledVal = networkingEnabled.load(std::memory_order_acquire); - bool sessionTrackingEnabledVal = sessionTrackingEnabled.load(std::memory_order_acquire); - bool viewTrackingEnabledVal = viewTrackingEnabled.load(std::memory_order_acquire); - bool locationTrackingEnabledVal = locationTrackingEnabled.load(std::memory_order_acquire); - bool customEventTrackingEnabledVal = customEventTrackingEnabled.load(std::memory_order_acquire); - bool crashReportingEnabledVal = crashReportingEnabled.load(std::memory_order_acquire); unsigned int serverConfigUpdateIntervalVal = serverConfigUpdateInterval.load(std::memory_order_acquire); - bool locationTrackingCurrent = getBool(KEY_LOCATION_TRACKING, locationTrackingEnabledVal); + bool locationTrackingCurrent = getBool(KEY_LOCATION_TRACKING, true); + const int previousServerLocation = serverLocationTracking.load(std::memory_order_acquire); + int serverLocation = -1; + if (sdk_behavior_settings.is_object()) { + nlohmann::json::const_iterator locationSetting = sdk_behavior_settings.find(KEY_LOCATION_TRACKING); + if (locationSetting != sdk_behavior_settings.end() && locationSetting->is_boolean()) { + serverLocation = locationSetting->get() ? 1 : 0; + } + } + serverLocationTracking.store(serverLocation, std::memory_order_release); unsigned int serverConfigUpdateIntervalCurrent = getUInt(KEY_SERVER_CONFIG_UPDATE_INTERVAL, serverConfigUpdateIntervalVal); int consentRequiredVal = consentRequired.load(std::memory_order_acquire); int consentRequiredCurrent = -1; @@ -256,13 +300,13 @@ class ConfigurationModule::ConfigurationModuleImpl { } consentRequired.store(consentRequiredCurrent, std::memory_order_release); - trackingEnabled.store(getBool(KEY_TRACKING, trackingEnabledVal), std::memory_order_release); - networkingEnabled.store(getBool(KEY_NETWORKING, networkingEnabledVal), std::memory_order_release); - sessionTrackingEnabled.store(getBool(KEY_SESSION_TRACKING, sessionTrackingEnabledVal), std::memory_order_release); - viewTrackingEnabled.store(getBool(KEY_VIEW_TRACKING, viewTrackingEnabledVal), std::memory_order_release); + trackingEnabled.store(getBool(KEY_TRACKING, true), std::memory_order_release); + networkingEnabled.store(getBool(KEY_NETWORKING, true), std::memory_order_release); + sessionTrackingEnabled.store(getBool(KEY_SESSION_TRACKING, true), std::memory_order_release); + viewTrackingEnabled.store(getBool(KEY_VIEW_TRACKING, true), std::memory_order_release); locationTrackingEnabled.store(locationTrackingCurrent, std::memory_order_release); - customEventTrackingEnabled.store(getBool(KEY_CUSTOM_EVENT_TRACKING, customEventTrackingEnabledVal), std::memory_order_release); - crashReportingEnabled.store(getBool(KEY_CRASH_REPORTING, crashReportingEnabledVal), std::memory_order_release); + customEventTrackingEnabled.store(getBool(KEY_CUSTOM_EVENT_TRACKING, true), std::memory_order_release); + crashReportingEnabled.store(getBool(KEY_CRASH_REPORTING, true), std::memory_order_release); eventQueueThreshold.store(getUInt(KEY_EVENT_QUEUE_SIZE, 0), std::memory_order_release); unsigned int rqs = getUInt(KEY_REQ_QUEUE_SIZE, _configuration->requestQueueThreshold); @@ -280,8 +324,6 @@ class ConfigurationModule::ConfigurationModuleImpl { unsigned int scui = getUInt(KEY_SERVER_CONFIG_UPDATE_INTERVAL, 4); if (scui < 1) { scui = 4; - } else if (scui > 720) { - scui = 720; // cap at 30 days } serverConfigUpdateInterval.store(scui, std::memory_order_release); @@ -307,9 +349,23 @@ class ConfigurationModule::ConfigurationModuleImpl { maxStackTraceLinesPerThread.store(readLimit(KEY_LIMIT_TRACE_LINE, _configuration->maxStackTraceLinesPerThread), std::memory_order_release); maxStackTraceLineLength.store(readLimit(KEY_LIMIT_TRACE_LENGTH, _configuration->maxStackTraceLineLength), std::memory_order_release); + requestDropAgeHours.store(getUInt(KEY_DROP_OLD_REQUEST_TIME, _configuration->requestDropAgeHours), std::memory_order_release); + backoffEnabled.store(getBool(KEY_BACKOFF_MECHANISM, _configuration->backoffMechanismEnabled), std::memory_order_release); + backoffAcceptedTimeout.store(readLimit(KEY_BOM_ACCEPTED_TIMEOUT, 10), std::memory_order_release); + backoffRequestAge.store(readLimit(KEY_BOM_REQUEST_AGE, 24), std::memory_order_release); + backoffDuration.store(readLimit(KEY_BOM_DURATION, 60), std::memory_order_release); + double queuePercentage = 0.5; + if (sdk_behavior_settings.is_object()) { + nlohmann::json::const_iterator setting = sdk_behavior_settings.find(KEY_BOM_RQ_PERCENTAGE); + if (setting != sdk_behavior_settings.end() && setting->is_number()) { + queuePercentage = setting->get(); + } + } + backoffRequestQueuePercentage.store(queuePercentage, std::memory_order_release); + nlohmann::json changedSettings; - if (locationTrackingCurrent != locationTrackingEnabledVal) { - changedSettings[KEY_LOCATION_TRACKING] = locationTrackingCurrent; + if (serverLocation != previousServerLocation) { + changedSettings[KEY_LOCATION_TRACKING] = serverLocation; } if (serverConfigUpdateIntervalCurrent != serverConfigUpdateIntervalVal) { changedSettings[KEY_SERVER_CONFIG_UPDATE_INTERVAL] = serverConfigUpdateIntervalCurrent; @@ -329,32 +385,36 @@ class ConfigurationModule::ConfigurationModuleImpl { for (auto it = c.begin(); it != c.end();) { std::string key = it.key(); auto value = it.value(); + bool known = true; + bool valid = true; if (key == KEY_REQ_QUEUE_SIZE || key == KEY_EVENT_QUEUE_SIZE || key == KEY_SESSION_UPDATE_INTERVAL || key == KEY_LIMIT_KEY_LENGTH || key == KEY_LIMIT_VALUE_SIZE || key == KEY_LIMIT_SEG_VALUES || key == KEY_LIMIT_BREADCRUMB || key == KEY_LIMIT_TRACE_LINE || key == KEY_LIMIT_TRACE_LENGTH || - key == KEY_USER_PROPERTY_CACHE_LIMIT || key == KEY_DROP_OLD_REQUEST_TIME || key == KEY_SERVER_CONFIG_UPDATE_INTERVAL) { - if (!value.is_number_unsigned()) { - it = c.erase(it); - continue; - } - } else if (key == KEY_TRACKING || key == KEY_NETWORKING || key == KEY_LOGGING || key == KEY_SESSION_TRACKING || key == KEY_VIEW_TRACKING || key == KEY_LOCATION_TRACKING || key == KEY_CUSTOM_EVENT_TRACKING || key == KEY_CONSENT_REQUIRED || key == KEY_CRASH_REPORTING) { - if (!value.is_boolean()) { - it = c.erase(it); - continue; - } + key == KEY_USER_PROPERTY_CACHE_LIMIT || key == KEY_SERVER_CONFIG_UPDATE_INTERVAL || key == KEY_BOM_ACCEPTED_TIMEOUT || key == KEY_BOM_REQUEST_AGE || key == KEY_BOM_DURATION) { + valid = value.is_number_unsigned() && value.get() >= 1 && value.get() <= UINT_MAX; + } else if (key == KEY_DROP_OLD_REQUEST_TIME) { + valid = value.is_number_unsigned() && value.get() <= UINT_MAX; + } else if (key == KEY_BOM_RQ_PERCENTAGE) { + valid = value.is_number() && value.get() > 0 && value.get() <= 1; + } else if (key == KEY_TRACKING || key == KEY_NETWORKING || key == KEY_LOGGING || key == KEY_SESSION_TRACKING || key == KEY_VIEW_TRACKING || key == KEY_LOCATION_TRACKING || key == KEY_CUSTOM_EVENT_TRACKING || key == KEY_CONSENT_REQUIRED || key == KEY_CRASH_REPORTING || key == KEY_BACKOFF_MECHANISM) { + valid = value.is_boolean(); } else if (key == KEY_EVENT_BLACKLIST || key == KEY_USER_PROPERTY_BLACKLIST || key == KEY_SEGMENTATION_BLACKLIST || key == KEY_EVENT_WHITELIST || key == KEY_USER_PROPERTY_WHITELIST || key == KEY_SEGMENTATION_WHITELIST) { - if (!value.is_array()) { - it = c.erase(it); - continue; - } + valid = value.is_array(); } else if (key == KEY_EVENT_SEGMENTATION_BLACKLIST || key == KEY_EVENT_SEGMENTATION_WHITELIST) { - if (!value.is_object()) { - it = c.erase(it); - continue; - } + valid = value.is_object(); } else { + known = false; + } + + if (!known) { _logger->log(LogLevel::DEBUG, "[Countly] [ConfigurationModule] sanitizeConfig, removing unknown key: [" + key + "]"); it = c.erase(it); continue; } + if (!valid) { + _logger->log(LogLevel::WARNING, "[Countly] [ConfigurationModule] sanitizeConfig, The SDK behavior setting [" + key + "] has an invalid value [" + value.dump() + "] and is ignored."); + _logger->recordLogCode("w121"); + it = c.erase(it); + continue; + } ++it; } } @@ -372,7 +432,8 @@ class ConfigurationModule::ConfigurationModuleImpl { interval = 4; } - auto deadline = std::chrono::steady_clock::now() + std::chrono::hours(interval); + // about 11 years at most, so that a huge interval can't overflow the clock + auto deadline = std::chrono::steady_clock::now() + std::chrono::hours(std::min(interval, 100000u)); bool stopped = configUpdateCv.wait_until(lock, deadline, [&] { return stopConfigThread.load(std::memory_order_acquire); }); if (stopped) { return; @@ -383,12 +444,7 @@ class ConfigurationModule::ConfigurationModuleImpl { } lock.unlock(); - std::map data; - { - std::lock_guard mutexLock(*_mutex); - data = {{"method", "sc"}, {"app_key", session_params["app_key"].get()}, {"device_id", session_params["device_id"].get()}}; - } - _fetchConfigFromServerHTTP(data, session_params); + _fetchConfigFromServerHTTP(); lock.lock(); } catch (const std::exception &e) { _logger->log(LogLevel::ERROR, "[Countly] [ConfigurationModule] _updateConfigPeriodically, exception: [" + std::string(e.what()) + "]"); @@ -480,18 +536,20 @@ ConfigurationModule::ConfigurationModule(cly::CountlyDelegates *cly, std::shared ConfigurationModule::~ConfigurationModule() { impl.reset(); } void ConfigurationModule::fetchConfigFromServer(nlohmann::json session_params) { - // Join any previous fetch thread before starting a new one + (void)session_params; + if (impl->_configuration->sdkBehaviorSettingsUpdatesDisabled) { + impl->_logger->log(LogLevel::DEBUG, "[Countly] [ConfigurationModule] fetchConfigFromServer, SDK behavior settings updates are disabled."); + return; + } + if (impl->fetchRunning.exchange(true)) { + impl->fetchAgain = true; + return; + } + // the previous fetch thread has finished, so this returns at once if (impl->configFetchThread.joinable()) { impl->configFetchThread.join(); } - - std::map data; - { - std::lock_guard lock(*impl->_mutex); - data = {{"method", "sc"}, {"app_key", session_params["app_key"].get()}, {"device_id", session_params["device_id"].get()}}; - } - - impl->configFetchThread = std::thread(&ConfigurationModule::ConfigurationModuleImpl::_fetchConfigFromServerHTTP, impl.get(), data, session_params); + impl->configFetchThread = std::thread(&ConfigurationModule::ConfigurationModuleImpl::_fetchLoop, impl.get()); } void ConfigurationModule::fetchConfigFromStorage() { @@ -517,6 +575,8 @@ bool ConfigurationModule::isNetworkingEnabled() const { return impl->networkingE bool ConfigurationModule::isLocationTrackingEnabled() { return impl->locationTrackingEnabled.load(std::memory_order_acquire); } +int ConfigurationModule::getServerLocationTracking() const { return impl->serverLocationTracking.load(std::memory_order_acquire); } + bool ConfigurationModule::isViewTrackingEnabled() const { return impl->viewTrackingEnabled.load(std::memory_order_acquire); } bool ConfigurationModule::isSessionTrackingEnabled() { return impl->sessionTrackingEnabled.load(std::memory_order_acquire); } @@ -529,6 +589,18 @@ int ConfigurationModule::getConsentRequired() const { return impl->consentRequir unsigned int ConfigurationModule::getRequestQueueSizeLimit() const { return impl->requestQueueSizeLimit.load(std::memory_order_acquire); } +unsigned int ConfigurationModule::getRequestDropAgeHours() const { return impl->requestDropAgeHours.load(std::memory_order_acquire); } + +bool ConfigurationModule::isBackoffEnabled() const { return impl->backoffEnabled.load(std::memory_order_acquire); } + +unsigned int ConfigurationModule::getBackoffAcceptedTimeout() const { return impl->backoffAcceptedTimeout.load(std::memory_order_acquire); } + +double ConfigurationModule::getBackoffRequestQueuePercentage() const { return impl->backoffRequestQueuePercentage.load(std::memory_order_acquire); } + +unsigned int ConfigurationModule::getBackoffRequestAge() const { return impl->backoffRequestAge.load(std::memory_order_acquire); } + +unsigned int ConfigurationModule::getBackoffDuration() const { return impl->backoffDuration.load(std::memory_order_acquire); } + SDKLimits ConfigurationModule::getLimits() const { SDKLimits lim; lim.maxKeyLength = impl->maxKeyLength.load(std::memory_order_acquire); diff --git a/src/countly.cpp b/src/countly.cpp index 7f50699..0a353ff 100644 --- a/src/countly.cpp +++ b/src/countly.cpp @@ -3,6 +3,9 @@ #include "countly/storage_module_db.hpp" #include "countly/storage_module_memory.hpp" #include +#include +#include +#include #include #include #include @@ -19,6 +22,7 @@ #endif #include "countly.hpp" +#include "platform_info.hpp" #ifdef COUNTLY_USE_SQLITE #include "countly/sqlite_utils.hpp" @@ -30,6 +34,143 @@ namespace { const char *const HEALTH_CHECK_AREA_SESSIONS = "sessions"; const char *const HEALTH_CHECK_AREA_EVENTS = "events"; +/** + * Checks that the server URL is an absolute http or https URL with a host, and gives it back with a lower case scheme + * and without trailing '/' characters. + * + * @param url: the URL the app gave + * @param normalized: receives the URL to use + * @param trimmed: set to true when trailing '/' characters were removed + * @return false when the URL can't be used + */ +bool normalizeServerUrl(const std::string &url, std::string &normalized, bool &trimmed) { + const size_t separator = url.find("://"); + if (separator == std::string::npos) { + return false; + } + std::string scheme = url.substr(0, separator); + std::transform(scheme.begin(), scheme.end(), scheme.begin(), [](unsigned char character) { return static_cast(std::tolower(character)); }); + if (scheme != "http" && scheme != "https") { + return false; + } + std::string rest = url.substr(separator + 3); + const size_t length = rest.size(); + while (!rest.empty() && rest.back() == '/') { + rest.pop_back(); + } + trimmed = rest.size() != length; + if (rest.empty() || rest.front() == '/' || rest.find_first_of(" \t\r\n") != std::string::npos) { + return false; + } + normalized = scheme + "://" + rest; + return true; +} + +/** + * Whether the text is an ISO 3166-1 alpha-2 country code in upper case, such as "US". + */ +bool isCountryCode(const std::string &text) { return text.size() == 2 && std::isupper(static_cast(text[0])) && std::isupper(static_cast(text[1])); } + +/** + * Whether the text is a decimal number within the range, without anything else around it. + */ +bool isDecimalWithin(const std::string &text, double minimum, double maximum) { + if (text.empty() || text.find_first_not_of("+-.0123456789") != std::string::npos) { + return false; + } + char *end = nullptr; + const double value = std::strtod(text.c_str(), &end); + return end == text.c_str() + text.size() && value >= minimum && value <= maximum; +} + +/** + * Whether the text is a latitude and a longitude in decimal degrees separated by a comma, such as "52.52001,13.40495". + */ +bool isGpsCoordinates(const std::string &text) { + const std::string::size_type comma = text.find(','); + if (comma == std::string::npos) { + return false; + } + std::string latitude = text.substr(0, comma); + std::string longitude = text.substr(comma + 1); + latitude.erase(std::remove(latitude.begin(), latitude.end(), ' '), latitude.end()); + longitude.erase(std::remove(longitude.begin(), longitude.end(), ' '), longitude.end()); + return isDecimalWithin(latitude, -90, 90) && isDecimalWithin(longitude, -180, 180); +} + +/** + * Whether the text is an IPv4 address such as "203.0.113.7", or an IPv6 address such as "2001:db8::1". + */ +bool isIpAddress(const std::string &text) { + if (text.find(':') != std::string::npos) { + return text.size() <= 45 && text.find_first_not_of("0123456789abcdefABCDEF:.") == std::string::npos; + } + std::istringstream parts(text); + std::string part; + int count = 0; + while (std::getline(parts, part, '.')) { + if (part.empty() || part.size() > 3 || part.find_first_not_of("0123456789") != std::string::npos || std::stoi(part) > 255) { + return false; + } + count++; + } + return count == 4 && text.back() != '.'; +} + +/** + * Whether the key is one of the internal events the SDK knows. Other "[CLY]_" keys are custom events. + */ +bool isInternalEventKey(const std::string &key) { + static const char *const internal_keys[] = {"[CLY]_view", "[CLY]_nps", "[CLY]_survey", "[CLY]_star_rating", "[CLY]_orientation", "[CLY]_push_action", "[CLY]_action", "[CLY]_content_shown", "[CLY]_content_interacted"}; + for (const char *internal_key : internal_keys) { + if (key == internal_key) { + return true; + } + } + return false; +} + +/** + * Replaces a count that is not a whole number from 1 to INT_MAX with 1 and removes a negative duration, logging + * each as a warning counted as log code w750. + */ +void validateEventValues(nlohmann::json &object, const std::string &key, LoggerModule &logger) { + nlohmann::json::iterator count = object.find("count"); + bool validCount = false; + if (count != object.end() && count->is_number_unsigned()) { + validCount = count->get() >= 1 && count->get() <= static_cast(INT_MAX); + } else if (count != object.end() && count->is_number_integer()) { + validCount = count->get() >= 1 && count->get() <= INT_MAX; + } + if (!validCount) { + logger.log(LogLevel::WARNING, "[Countly] addEvent, The count of the event [" + key + "] must be a whole number from 1 to " + std::to_string(INT_MAX) + ", so the default 1 is used."); + logger.recordLogCode("w750"); + object["count"] = 1; + } + + nlohmann::json::iterator duration = object.find("dur"); + if (duration != object.end() && (!duration->is_number() || duration->get() < 0)) { + logger.log(LogLevel::WARNING, "[Countly] addEvent, The duration of the event [" + key + "] can't be negative, so the default 0 is used."); + logger.recordLogCode("w750"); + object.erase(duration); + } +} + +/** + * Adds the stored events of the in-memory event queue to the array. An entry that is not valid JSON is dropped with + * a warning instead of throwing out of the call that sends the events. + */ +void appendStoredEvents(const std::deque &queue, nlohmann::json &events, LoggerModule &logger) { + for (const std::string &event_json : queue) { + nlohmann::json event = nlohmann::json::parse(event_json, nullptr, false); + if (event.is_discarded()) { + logger.log(LogLevel::WARNING, "[Countly] appendStoredEvents, A stored event is not valid JSON and is dropped."); + continue; + } + events.push_back(event); + } +} + /** * Whether the list of features contains the feature. */ @@ -93,27 +234,47 @@ struct RemoteConfigFetch { * member precisely so it cannot reach the instance. */ void runRemoteConfigFetch(std::shared_ptr fetch) { - // Clear the in-flight flag and the process counter however this exits. + // Lower the process counter however this exits. struct Guard { - std::shared_ptr fetch; - ~Guard() { - fetch->store->fetch_running.store(false); - inflight_remote_config_fetches.fetch_sub(1); - } - } guard{fetch}; + ~Guard() { inflight_remote_config_fetches.fetch_sub(1); } + } guard; - const HTTPResponse response = fetch->requestModule->sendHTTP("/o/sdk", fetch->requestBuilder->buildFetchRequest(fetch->data)); - if (!response.success) { - return; - } + try { + while (true) { + unsigned long long generation = 0; + { + std::lock_guard lk(fetch->store->mutex); + generation = fetch->store->user_generation; + } + + const HTTPResponse response = fetch->requestModule->sendImmediate("/o/sdk", fetch->requestBuilder->buildRequest(fetch->data)); + + std::lock_guard lk(fetch->store->mutex); + if (!response.success || !response.data.is_object()) { + fetch->logger->log(LogLevel::DEBUG, "[Countly] runRemoteConfigFetch, the remote config download failed, the stored values stay."); + } else if (fetch->store->user_generation != generation) { + fetch->logger->log(LogLevel::DEBUG, "[Countly] runRemoteConfigFetch, the device ID changed during the download, so its values are discarded."); + } else if (fetch->merge) { + for (auto it = response.data.begin(); it != response.data.end(); ++it) { + fetch->store->values[it.key()] = it.value(); + } + } else { + fetch->store->values = response.data; + } - std::lock_guard lk(fetch->store->mutex); - if (fetch->merge) { - for (auto it = response.data.begin(); it != response.data.end(); ++it) { - fetch->store->values[it.key()] = it.value(); + if (!fetch->store->has_pending) { + fetch->store->fetch_running.store(false); + return; + } + fetch->data = fetch->store->pending_data; + fetch->merge = fetch->store->pending_merge; + fetch->store->has_pending = false; } - } else { - fetch->store->values = response.data; + } catch (const std::exception &e) { + std::lock_guard lk(fetch->store->mutex); + fetch->store->has_pending = false; + fetch->store->fetch_running.store(false); + fetch->logger->log(LogLevel::DEBUG, std::string("[Countly] runRemoteConfigFetch, the remote config download failed: ") + e.what()); } } @@ -137,12 +298,15 @@ Countly::Countly() { std::lock_guard lk(networking_lifecycle_mutex); live_instance_count.fetch_add(1); } - crash_module = nullptr; - views_module = nullptr; logger.reset(new cly::LoggerModule()); health_check->setLogger(logger); logger->setHealthCheck(health_check); configuration.reset(new cly::CountlyConfiguration("", "")); + // Built with the instance, so views() and crash() are usable before start(); their calls are ignored until then. + crash_module.reset(new cly::CrashModule(configuration, logger, nullptr, mutex)); + views_module.reset(new cly::ViewsModule(this, logger)); + views_module->setConsentModule(consent_module); + crash_module->setConsentModule(consent_module); } Countly::~Countly() { @@ -433,6 +597,11 @@ void Countly::setMaxRequestQueueSize(unsigned int requestQueueSize) { log(LogLevel::WARNING, "[Countly] setMaxRequestQueueSize, This method can't be called after SDK initialization. Returning."); return; } + if (requestQueueSize < 1) { + log(LogLevel::WARNING, "[Countly] setMaxRequestQueueSize, The request queue size must be at least 1, so the value [0] is ignored."); + logger->recordLogCode("w121"); + return; + } std::lock_guard lk(*mutex); configuration->requestQueueThreshold = requestQueueSize; @@ -449,8 +618,9 @@ void Countly::setMaxRQProcessingBatchSize(unsigned int batchSize) { } void Countly::alwaysUsePost(bool value) { + log(LogLevel::WARNING, "[Countly] alwaysUsePost, 'alwaysUsePost' is deprecated: every request is sent as POST, so the call has no effect."); + logger->recordLogCode("w120"); if (is_sdk_initialized) { - log(LogLevel::WARNING, "[Countly] alwaysUsePost, This method can't be called after SDK initialization. Returning."); return; } @@ -458,6 +628,56 @@ void Countly::alwaysUsePost(bool value) { configuration->forcePost = value; } +void Countly::setFailedRequestCooldown(int seconds) { + if (is_sdk_initialized) { + log(LogLevel::WARNING, "[Countly] setFailedRequestCooldown, This method can't be called after SDK initialization. Returning."); + return; + } + if (seconds < 1) { + log(LogLevel::WARNING, "[Countly] setFailedRequestCooldown, The cool-down must be at least 1 second, so the value [" + std::to_string(seconds) + "] is ignored."); + logger->recordLogCode("w121"); + return; + } + + std::lock_guard lk(*mutex); + configuration->failedRequestCooldownSeconds = static_cast(seconds); +} + +void Countly::setRequestDropAgeHours(int hours) { + if (is_sdk_initialized) { + log(LogLevel::WARNING, "[Countly] setRequestDropAgeHours, This method can't be called after SDK initialization. Returning."); + return; + } + if (hours < 0) { + log(LogLevel::WARNING, "[Countly] setRequestDropAgeHours, The age can't be negative, so the value [" + std::to_string(hours) + "] is ignored."); + logger->recordLogCode("w121"); + return; + } + + std::lock_guard lk(*mutex); + configuration->requestDropAgeHours = static_cast(hours); +} + +void Countly::disableBackoffMechanism() { + if (is_sdk_initialized) { + log(LogLevel::WARNING, "[Countly] disableBackoffMechanism, This method can't be called after SDK initialization. Returning."); + return; + } + + std::lock_guard lk(*mutex); + configuration->backoffMechanismEnabled = false; +} + +void Countly::disableGradualRequestCleaner() { + if (is_sdk_initialized) { + log(LogLevel::WARNING, "[Countly] disableGradualRequestCleaner, This method can't be called after SDK initialization. Returning."); + return; + } + + std::lock_guard lk(*mutex); + configuration->gradualRequestCleanerEnabled = false; +} + void Countly::setSalt(const std::string &value) { if (is_sdk_initialized) { log(LogLevel::WARNING, "[Countly] setSalt, This method can't be called after SDK initialization. Returning."); @@ -539,6 +759,11 @@ void Countly::setMaxKeyLength(unsigned int value) { log(LogLevel::WARNING, "[Countly] setMaxKeyLength, This method can't be called after SDK initialization. Returning."); return; } + if (value < 1) { + log(LogLevel::WARNING, "[Countly] setMaxKeyLength, The limit must be at least 1, so the value [0] is ignored."); + logger->recordLogCode("w121"); + return; + } std::lock_guard lk(*mutex); configuration->maxKeyLength = value; } @@ -548,6 +773,11 @@ void Countly::setMaxValueSize(unsigned int value) { log(LogLevel::WARNING, "[Countly] setMaxValueSize, This method can't be called after SDK initialization. Returning."); return; } + if (value < 1) { + log(LogLevel::WARNING, "[Countly] setMaxValueSize, The limit must be at least 1, so the value [0] is ignored."); + logger->recordLogCode("w121"); + return; + } std::lock_guard lk(*mutex); configuration->maxValueSize = value; } @@ -557,6 +787,11 @@ void Countly::setMaxSegmentationValues(unsigned int value) { log(LogLevel::WARNING, "[Countly] setMaxSegmentationValues, This method can't be called after SDK initialization. Returning."); return; } + if (value < 1) { + log(LogLevel::WARNING, "[Countly] setMaxSegmentationValues, The limit must be at least 1, so the value [0] is ignored."); + logger->recordLogCode("w121"); + return; + } std::lock_guard lk(*mutex); configuration->maxSegmentationValues = value; } @@ -566,6 +801,11 @@ void Countly::setMaxBreadcrumbCount(unsigned int value) { log(LogLevel::WARNING, "[Countly] setMaxBreadcrumbCount, This method can't be called after SDK initialization. Returning."); return; } + if (value < 1) { + log(LogLevel::WARNING, "[Countly] setMaxBreadcrumbCount, The limit must be at least 1, so the value [0] is ignored."); + logger->recordLogCode("w121"); + return; + } std::lock_guard lk(*mutex); configuration->breadcrumbsThreshold = value; } @@ -575,6 +815,11 @@ void Countly::setMaxStackTraceLinesPerThread(unsigned int value) { log(LogLevel::WARNING, "[Countly] setMaxStackTraceLinesPerThread, This method can't be called after SDK initialization. Returning."); return; } + if (value < 1) { + log(LogLevel::WARNING, "[Countly] setMaxStackTraceLinesPerThread, The limit must be at least 1, so the value [0] is ignored."); + logger->recordLogCode("w121"); + return; + } std::lock_guard lk(*mutex); configuration->maxStackTraceLinesPerThread = value; } @@ -584,6 +829,11 @@ void Countly::setMaxStackTraceLineLength(unsigned int value) { log(LogLevel::WARNING, "[Countly] setMaxStackTraceLineLength, This method can't be called after SDK initialization. Returning."); return; } + if (value < 1) { + log(LogLevel::WARNING, "[Countly] setMaxStackTraceLineLength, The limit must be at least 1, so the value [0] is ignored."); + logger->recordLogCode("w121"); + return; + } std::lock_guard lk(*mutex); configuration->maxStackTraceLineLength = value; } @@ -631,13 +881,12 @@ void Countly::setUserDetails(const std::map &value) { std::map limitedDetails; for (const auto &kv : value) { unsigned int cap = (kv.first == "picture") ? COUNTLY_MAX_VALUE_SIZE_PICTURE : lim.maxValueSize; - limitedDetails[kv.first] = cly::limits::truncateString(kv.second, cap); + limitedDetails[kv.first] = cly::limits::limitValue(kv.second, cap, logger.get(), "setUserDetails", kv.first); } session_params["user_details"] = limitedDetails; if (!is_sdk_initialized) { - log(LogLevel::WARNING, "[Countly] setUserDetails, This method can't be called before SDK initialization. Returning."); - logger->recordLogCode("w101"); + log(LogLevel::INFO, "[Countly] setUserDetails, The SDK is not started yet, the details are sent with the first session."); return; } @@ -682,13 +931,12 @@ void Countly::setCustomUserDetails(const std::map &val // properties (that would be the out-of-scope 'upcl'). std::map limitedCustom; for (const auto &kv : customValue) { - limitedCustom[cly::limits::truncateString(kv.first, lim.maxKeyLength)] = cly::limits::truncateString(kv.second, lim.maxValueSize); + limitedCustom[cly::limits::limitKey(kv.first, lim.maxKeyLength, logger.get(), "setCustomUserDetails")] = cly::limits::limitValue(kv.second, lim.maxValueSize, logger.get(), "setCustomUserDetails", kv.first); } session_params["user_details"]["custom"] = limitedCustom; if (!is_sdk_initialized) { - log(LogLevel::WARNING, "[Countly] setCustomUserDetails, This method can't be called before SDK initialization. Returning."); - logger->recordLogCode("w101"); + log(LogLevel::INFO, "[Countly] setCustomUserDetails, The SDK is not started yet, the properties are sent with the first session."); return; } @@ -716,7 +964,7 @@ void Countly::multiplyCustomUserDetail(const std::string &key, double value) { logger->recordLogCode("w700"); return; } - const std::string truncatedKey = cly::limits::truncateString(key, configurationModule->getLimits().maxKeyLength); + const std::string truncatedKey = cly::limits::limitKey(key, configurationModule->getLimits().maxKeyLength, logger.get(), "multiplyCustomUserDetail"); if (truncatedKey.empty()) { log(LogLevel::WARNING, "[Countly] multiplyCustomUserDetail, The parameter 'key' can't be empty. Returning."); logger->recordLogCode("w700"); @@ -777,38 +1025,65 @@ void Countly::setLocation(double lattitude, double longitude) { } void Countly::setLocation(const std::string &countryCode, const std::string &city, const std::string &gpsCoordinates, const std::string &ipAddress) { + log(LogLevel::INFO, "[Countly] setLocation, Setting location: countryCode = [" + countryCode + "], city = [" + city + "], gpsCoordinates = [" + gpsCoordinates + "], ipAddress = [" + ipAddress + "]"); + + std::string validCountryCode = countryCode; + if (!countryCode.empty() && !isCountryCode(countryCode)) { + log(LogLevel::WARNING, "[Countly] setLocation, The country code [" + countryCode + "] is not an upper case ISO 3166-1 alpha-2 code such as 'US', so it is not used."); + logger->recordLogCode("w700"); + validCountryCode.clear(); + } + std::string validCoordinates = gpsCoordinates; + if (!gpsCoordinates.empty() && !isGpsCoordinates(gpsCoordinates)) { + log(LogLevel::WARNING, "[Countly] setLocation, The coordinates [" + gpsCoordinates + "] are not a latitude and a longitude in decimal degrees such as '52.52001,13.40495', so they are not used."); + logger->recordLogCode("w700"); + validCoordinates.clear(); + } + std::string validIpAddress = ipAddress; + if (!ipAddress.empty() && !isIpAddress(ipAddress)) { + log(LogLevel::WARNING, "[Countly] setLocation, The IP address [" + ipAddress + "] is not an IPv4 or IPv6 address, so it is not used."); + logger->recordLogCode("w700"); + validIpAddress.clear(); + } + if (validCountryCode.empty() != city.empty()) { + log(LogLevel::WARNING, "[Countly] setLocation, It's required that both 'country_code' and 'city' should be set together"); + } + const bool anyValue = !validCountryCode.empty() || !city.empty() || !validCoordinates.empty() || !validIpAddress.empty(); + if (!is_sdk_initialized) { - log(LogLevel::WARNING, "[Countly] setLocation, SDK is not initialized."); - logger->recordLogCode("w101"); + std::lock_guard lk(*mutex); + session_params["city"] = city; + session_params["ip_address"] = validIpAddress; + session_params["location"] = validCoordinates; + session_params["country_code"] = validCountryCode; + log(LogLevel::DEBUG, "[Countly] setLocation, The SDK is not started yet, it uses the location from start() on."); return; } if (!hasConsent(consent::LOCATION)) { log(LogLevel::DEBUG, "[Countly] setLocation, the 'location' consent is not given, the call is ignored."); return; } - bool isClearingLocation = countryCode.empty() && city.empty() && gpsCoordinates.empty() && ipAddress.empty(); - // unique_lock so the mutex is released on scope exit (incl. throwing json writes); - // explicit unlock before the self-locking _sendIndependantLocationRequest() below. - std::unique_lock lk(*mutex); - if (!isClearingLocation && configurationModule->isLocationTrackingEnabled() == false) { - log(LogLevel::ERROR, "[Countly] setLocation, Location tracking is disabled in server configuration, can not set location."); + if (configurationModule->getServerLocationTracking() == 0) { + log(LogLevel::DEBUG, "[Countly] setLocation, The server's SDK behavior settings turned location tracking off, the call is ignored."); return; } - log(LogLevel::INFO, "[Countly] setLocation, Setting location: countryCode = [" + countryCode + "], city = [" + city + "], gpsCoordinates = [" + gpsCoordinates + "], ipAddress = [" + ipAddress + "]"); - if ((!countryCode.empty() && city.empty()) || (!city.empty() && countryCode.empty())) { - log(LogLevel::WARNING, "[Countly] setLocation, It's required that both 'country_code' and 'city' should be set together"); - logger->recordLogCode("w700"); + bool sendNow = false; + { + // unique_lock so the mutex is released on scope exit (incl. throwing json writes) + std::unique_lock lk(*mutex); + session_params["city"] = city; + session_params["ip_address"] = validIpAddress; + session_params["location"] = validCoordinates; + session_params["country_code"] = validCountryCode; + if (anyValue) { + location_disabled_by_app = false; + } + // without a session start to come, the values go in a separate request right away + sendNow = anyValue && (began_session || !hasConsent(consent::SESSIONS) || !configurationModule->isSessionTrackingEnabled()); } - session_params["city"] = city; - session_params["ip_address"] = ipAddress; - session_params["location"] = gpsCoordinates; - session_params["country_code"] = countryCode; - - lk.unlock(); - - if (is_sdk_initialized) { + if (sendNow) { _sendIndependantLocationRequest(); } } @@ -817,56 +1092,49 @@ void Countly::_sendIndependantLocationRequest() { std::lock_guard lk(*mutex); log(LogLevel::DEBUG, "[Countly] _sendIndependantLocationRequest, Start"); - /* - * Empty country code, city and IP address can not be sent. - */ - + // the server uses an IP address only with a session start, so it goes with the next begin_session std::map data; - - if (session_params.contains("city") && session_params["city"].is_string() && !session_params["city"].get().empty()) { - data["city"] = session_params["city"].get(); - } - - if (session_params.contains("location") && session_params["location"].is_string() && !session_params["location"].get().empty()) { - data["location"] = session_params["location"].get(); - } - - if (session_params.contains("country_code") && session_params["country_code"].is_string() && !session_params["country_code"].get().empty()) { - data["country_code"] = session_params["country_code"].get(); + for (const char *key : {"city", "location", "country_code"}) { + if (session_params.contains(key) && session_params[key].is_string() && !session_params[key].get().empty()) { + data[key] = session_params[key].get(); + } } - - if (session_params.contains("ip_address") && session_params["ip_address"].is_string() && !session_params["ip_address"].get().empty()) { - data["ip_address"] = session_params["ip_address"].get(); + if (data.empty()) { + return; } const std::chrono::system_clock::time_point now = Countly::getTimestamp(); const auto timestamp = std::chrono::duration_cast(now.time_since_epoch()); - data["app_key"] = session_params["app_key"].get(); data["device_id"] = session_params["device_id"].get(); data["timestamp"] = std::to_string(timestamp.count()); - - if (data.size() == 3) { - // No location fields were added — send empty location to clear server-side location - data["location"] = ""; - } - requestModule->addRequestToQueue(data); } +void Countly::sendLocationErase() { + std::lock_guard lk(*mutex); + log(LogLevel::DEBUG, "[Countly] sendLocationErase, erasing the location on the server"); + requestModule->addRequestToQueue({{"location", ""}}); +} + #pragma endregion User location #pragma region Device Id void Countly::setDeviceID(const std::string &value, bool same_user) { + if (value.empty()) { + log(LogLevel::WARNING, "[Countly] setDeviceID, The device ID can't be empty, the call is ignored."); + logger->recordLogCode("w601"); + return; + } // unique_lock so the mutex is released on scope exit (incl. throwing json writes); // explicit unlock before the self-locking _changeDeviceId* helpers below. std::unique_lock lk(*mutex); log(LogLevel::INFO, "[Countly] setDeviceID, Device ID change requested, new value = [" + value + "]"); - if (!session_params.contains("device_id")) { + if (!is_sdk_initialized) { session_params["device_id"] = value; configuration->deviceId = value; - log(LogLevel::DEBUG, "[Countly] setDeviceID, No previous device id, assigning initial device id"); + log(LogLevel::DEBUG, "[Countly] setDeviceID, The SDK is not started yet, it uses this device ID from start() on."); return; } @@ -876,12 +1144,6 @@ void Countly::setDeviceID(const std::string &value, bool same_user) { } lk.unlock(); - if (!is_sdk_initialized) { - log(LogLevel::WARNING, "[Countly] setDeviceID, Device id can't be changed while the SDK has not been initialized."); - logger->recordLogCode("w101"); - return; - } - if (same_user) { _changeDeviceIdWithMerge(value); } else { @@ -889,6 +1151,143 @@ void Countly::setDeviceID(const std::string &value, bool same_user) { } } +bool Countly::resolveDeviceId(const std::string &given_device_id) { + const nlohmann::json stored = nlohmann::json::parse(storageModule->getDeviceIdState(), nullptr, false); + if (!stored.is_object() || !stored.contains("id") || !stored["id"].is_string() || stored["id"].get().empty() || !stored.contains("type") || !stored["type"].is_number_integer()) { + useNewDeviceId(given_device_id); + return false; + } + + configuration->deviceId = stored["id"].get(); + configuration->deviceIdType = stored["type"].get() == static_cast(DeviceIdType::SDK_GENERATED) ? DeviceIdType::SDK_GENERATED : DeviceIdType::DEVELOPER_SUPPLIED; + session_params["device_id"] = configuration->deviceId; + if (configuration->clearStoredDeviceID) { + log(LogLevel::INFO, "[Countly] start, The device ID stored by an earlier start is replaced, as asked with enableClearStoredDeviceID: [" + configuration->deviceId + "]"); + return true; + } + if (!given_device_id.empty() && given_device_id != configuration->deviceId) { + log(LogLevel::WARNING, "[Countly] start, The device ID stored by an earlier start is used: [" + configuration->deviceId + "], and the device ID given with setDeviceID is ignored: [" + given_device_id + + "]. To use it, call enableClearStoredDeviceID() before start(), or change the device ID with setDeviceID after start()."); + } else { + log(LogLevel::DEBUG, "[Countly] start, Using the device ID stored by an earlier start: [" + configuration->deviceId + "]"); + } + return false; +} + +void Countly::replaceStoredDeviceId(const std::string &given_device_id) { + if (use_sqlite_storage) { + nlohmann::json events = nlohmann::json::array(); + std::string event_ids; + readStoredEvents(events, event_ids); + sendEventsToRQ(events); + if (!event_ids.empty()) { + removeEventWithId(event_ids); + } + } + useNewDeviceId(given_device_id); +} + +void Countly::useNewDeviceId(const std::string &given_device_id) { + if (!given_device_id.empty()) { + configuration->deviceId = given_device_id; + configuration->deviceIdType = DeviceIdType::DEVELOPER_SUPPLIED; + } else { + configuration->deviceId = cly::utils::generateUUID(); + configuration->deviceIdType = DeviceIdType::SDK_GENERATED; + log(LogLevel::INFO, "[Countly] start, No device ID was given, so the SDK generated one: [" + configuration->deviceId + "]"); + } + session_params["device_id"] = configuration->deviceId; + storeDeviceId(); +} + +void Countly::enableClearStoredDeviceID() { + if (is_sdk_initialized) { + log(LogLevel::WARNING, "[Countly] enableClearStoredDeviceID, This method can't be called after SDK initialization. Returning."); + return; + } + std::lock_guard lk(*mutex); + configuration->clearStoredDeviceID = true; +} + +void Countly::storeDeviceId() { + if (storageModule) { + storageModule->storeDeviceIdState(nlohmann::json({{"id", configuration->deviceId}, {"type", static_cast(configuration->deviceIdType)}}).dump()); + } +} + +void Countly::setCustomCrashSegmentation(const std::map &segmentation) { + if (is_sdk_initialized) { + log(LogLevel::WARNING, "[Countly] setCustomCrashSegmentation, This method can't be called after SDK initialization. Returning."); + return; + } + + std::lock_guard lk(*mutex); + configuration->crashSegmentation = segmentation; +} + +void Countly::setCrashFilterCallback(CrashFilterCallback callback) { + if (is_sdk_initialized) { + log(LogLevel::WARNING, "[Countly] setCrashFilterCallback, This method can't be called after SDK initialization. Returning."); + return; + } + + std::lock_guard lk(*mutex); + configuration->crashFilter = callback; +} + +void Countly::disableLocation() { + log(LogLevel::INFO, "[Countly] disableLocation, turning location tracking off"); + location_disabled_by_app = true; + clearStoredLocation(); + if (!is_sdk_initialized) { + return; + } + if (configurationModule->getServerLocationTracking() == 1) { + log(LogLevel::DEBUG, "[Countly] disableLocation, the server's SDK behavior settings keep location tracking on, so no erase request is sent."); + return; + } + sendLocationErase(); +} + +nlohmann::json Countly::DeviceMetrics() { return deviceMetrics(); } + +std::string Countly::CurrentViewId() { return views_module->currentViewId(); } + +void Countly::LocationSettingChanged() { + if (!start_finished) { + return; + } + const int server = configurationModule->getServerLocationTracking(); + if (server == 0) { + clearStoredLocation(); + if (hasConsent(consent::LOCATION)) { + sendLocationErase(); + } + } else if (server < 0 && location_disabled_by_app && hasConsent(consent::LOCATION)) { + sendLocationErase(); + } +} + +std::string Countly::getDeviceID() { + if (!is_sdk_initialized) { + log(LogLevel::WARNING, "[Countly] getDeviceID, SDK is not initialized."); + logger->recordLogCode("w101"); + return std::string(); + } + std::lock_guard lk(*mutex); + return configuration->deviceId; +} + +DeviceIdType Countly::getDeviceIDType() { + if (!is_sdk_initialized) { + log(LogLevel::WARNING, "[Countly] getDeviceIDType, SDK is not initialized."); + logger->recordLogCode("w101"); + return DeviceIdType::DEVELOPER_SUPPLIED; + } + std::lock_guard lk(*mutex); + return configuration->deviceIdType; +} + /* Change device ID with merge after SDK has been initialized.*/ void Countly::_changeDeviceIdWithMerge(const std::string &value) { std::lock_guard lk(*mutex); @@ -897,6 +1296,9 @@ void Countly::_changeDeviceIdWithMerge(const std::string &value) { session_params["old_device_id"] = session_params["device_id"]; session_params["device_id"] = value; configuration->deviceId = value; + configuration->deviceIdType = DeviceIdType::DEVELOPER_SUPPLIED; + storeDeviceId(); + clearRemoteConfigForNewUser(); const std::chrono::system_clock::time_point now = Countly::getTimestamp(); const auto timestamp = std::chrono::duration_cast(now.time_since_epoch()); @@ -916,7 +1318,7 @@ void Countly::_changeDeviceIdWithoutMerge(const std::string &value) { std::lock_guard change(consent_change_mutex); log(LogLevel::DEBUG, "[Countly] _changeDeviceIdWithoutMerge, deviceId = [" + value + "]"); - views_module->stopAllViews(); + views_module->stopRunningViews(); // send all event to server and end current session of old user flushEvents(); if (configuration->manualSessionControl == false) { @@ -941,12 +1343,28 @@ void Countly::_changeDeviceIdWithoutMerge(const std::string &value) { std::lock_guard lk(*mutex); session_params["device_id"] = value; configuration->deviceId = value; + configuration->deviceIdType = DeviceIdType::DEVELOPER_SUPPLIED; + storeDeviceId(); + previous_event_id.clear(); } + views_module->forgetViews(); + clearRemoteConfigForNewUser(); // start a new session for new user if (configuration->manualSessionControl == false) { beginSessionInternal(true); } + + configurationModule->fetchConfigFromServer(nullptr); + downloadRemoteConfigAutomatically(); +} + +void Countly::onMergeDelivered() { + if (is_being_disposed || stopping) { + return; + } + configurationModule->fetchConfigFromServer(nullptr); + downloadRemoteConfigAutomatically(); } #pragma endregion Device Id @@ -961,6 +1379,21 @@ void Countly::start(const std::string &app_key, const std::string &host, int por return; } + if (app_key.empty() || app_key == "YOUR_APP_KEY") { + log(LogLevel::ERROR, "[Countly] start, The app key must be the key of your Countly app, not empty and not the placeholder 'YOUR_APP_KEY'. SDK will not be initialized."); + return; + } + + std::string server_url; + bool trimmed_server_url = false; + if (!normalizeServerUrl(host, server_url, trimmed_server_url)) { + log(LogLevel::ERROR, "[Countly] start, The server URL [" + host + "] is not an absolute http or https URL, for example 'https://your.server.ly'. SDK will not be initialized."); + return; + } + if (trimmed_server_url) { + log(LogLevel::WARNING, "[Countly] start, The trailing '/' characters of the server URL are removed: [" + server_url + "]"); + } + #ifdef COUNTLY_USE_SQLITE if (configuration->databasePath == "" || configuration->databasePath == " ") { log(LogLevel::WARNING, "[Countly] start, No database path is set, so events and requests are kept in memory and are lost when the app closes. Call setDatabasePath() before start() to keep them."); @@ -1006,8 +1439,7 @@ void Countly::start(const std::string &app_key, const std::string &host, int por log(LogLevel::INFO, "[Countly] start, '_WIN32' is not defined"); #endif - enable_automatic_session = start_thread; - start_thread = true; + (void)start_thread; if (port < 0 || port > 65535) { log(LogLevel::WARNING, "[Countly] start, Port number is out of valid boundaries. Setting it to 0."); @@ -1017,7 +1449,7 @@ void Countly::start(const std::string &app_key, const std::string &host, int por configuration->port = port; configuration->appKey = app_key; - configuration->serverUrl = host; + configuration->serverUrl = server_url; session_params["app_key"] = app_key; @@ -1034,19 +1466,23 @@ void Countly::start(const std::string &app_key, const std::string &host, int por storageModule.reset(new StorageModuleMemory(configuration, logger)); storageModule->init(); } + const std::string given_device_id = session_params.contains("device_id") ? session_params["device_id"].get() : std::string(); + const bool replace_stored_device_id = resolveDeviceId(given_device_id); requestBuilder.reset(new RequestBuilder(configuration, logger)); requestModule.reset(new RequestModule(configuration, logger, requestBuilder, storageModule)); configurationModule.reset(new cly::ConfigurationModule(this, configuration, logger, requestBuilder, storageModule, requestModule, mutex)); - crash_module.reset(new cly::CrashModule(configuration, logger, requestModule, mutex)); - views_module.reset(new cly::ViewsModule(this, logger)); + crash_module->setRequestModule(requestModule); + crash_module->setDelegates(this); requestModule->setConfigurationProvider(configurationModule); + requestModule->setOnMergeDelivered([this]() { onMergeDelivered(); }); views_module->setConfigurationProvider(configurationModule); crash_module->setConfigurationProvider(configurationModule); - views_module->setConsentModule(consent_module); - crash_module->setConsentModule(consent_module); health_check->setStorage(storageModule); + if (replace_stored_device_id) { + replaceStoredDeviceId(given_device_id); + } is_sdk_initialized = true; // after this point SDK is initialized. @@ -1065,18 +1501,24 @@ void Countly::start(const std::string &app_key, const std::string &host, int por lk.lock(); } - if (start_thread) { - stop_thread = false; + stop_thread = false; - try { - thread.reset(new std::thread(&Countly::updateLoop, this)); - } catch (const std::system_error &e) { - std::ostringstream log_message; - log_message << "[Countly] start, Could not create thread: " << e.what(); - log(LogLevel::FATAL, log_message.str()); - } + try { + thread.reset(new std::thread(&Countly::updateLoop, this)); + } catch (const std::system_error &e) { + std::ostringstream log_message; + log_message << "[Countly] start, Could not create thread: " << e.what(); + log(LogLevel::FATAL, log_message.str()); } } + + lk.unlock(); + // without a session start to carry the empty location, a separate request keeps the location off + if (!isLocationOn() && (!hasConsent(consent::SESSIONS) || !configurationModule->isSessionTrackingEnabled())) { + sendLocationErase(); + } + start_finished = true; + downloadRemoteConfigAutomatically(); } /** @@ -1088,19 +1530,28 @@ void Countly::startOnCloud(const std::string &app_key) { this->start(app_key, "https://cloud.count.ly", 443); } -bool Countly::startRemoteConfigFetch(const std::map &data, bool merge, const char *caller) { +bool Countly::startRemoteConfigFetch(const std::map &data, bool merge, const char *caller, bool queueIfRunning) { // Checked here rather than on the fetch thread: see RemoteConfigFetch on why it // must not hold the configuration module. if (configurationModule->isNetworkingEnabled() == false) { - log(LogLevel::ERROR, std::string("[Countly] ") + caller + ", Error fetching remote config, networking is disabled in SBS"); + log(LogLevel::DEBUG, std::string("[Countly] ") + caller + ", the SDK behavior settings turned networking off, so remote config is not downloaded."); return false; } - // exchange, not load-then-store: two threads calling updateRemoteConfig at the - // same moment must not both get through. - if (remote_config_store->fetch_running.exchange(true)) { - log(LogLevel::WARNING, std::string("[Countly] ") + caller + ", a remote config fetch is already in flight; ignoring this call."); - return false; + { + std::lock_guard lk(remote_config_store->mutex); + if (remote_config_store->fetch_running.load()) { + if (queueIfRunning) { + remote_config_store->has_pending = true; + remote_config_store->pending_data = data; + remote_config_store->pending_merge = merge; + log(LogLevel::DEBUG, std::string("[Countly] ") + caller + ", a remote config fetch is in flight; this one follows it."); + return true; + } + log(LogLevel::WARNING, std::string("[Countly] ") + caller + ", a remote config fetch is already in flight; ignoring this call."); + return false; + } + remote_config_store->fetch_running.store(true); } std::shared_ptr fetch(new RemoteConfigFetch()); @@ -1118,23 +1569,60 @@ bool Countly::startRemoteConfigFetch(const std::map &d std::thread(runRemoteConfigFetch, fetch).detach(); } catch (const std::system_error &e) { inflight_remote_config_fetches.fetch_sub(1); - remote_config_store->fetch_running.store(false); + { + std::lock_guard lk(remote_config_store->mutex); + remote_config_store->fetch_running.store(false); + } log(LogLevel::ERROR, std::string("[Countly] ") + caller + ", could not create remote config thread: " + e.what()); return false; } return true; } +std::map Countly::remoteConfigRequest() { + std::map data = {{"method", "rc"}}; + if (hasConsent(consent::METRICS)) { + data["metrics"] = deviceMetrics().dump(); + } + if (configuration->enrollABOnRCDownload) { + data["oi"] = "1"; + } + return data; +} + +void Countly::downloadRemoteConfigAutomatically() { + bool automatic = false; + { + std::lock_guard lk(*mutex); + automatic = remote_config_enabled; + } + if (!automatic || !hasConsent(consent::REMOTE_CONFIG)) { + return; + } + startRemoteConfigFetch(remoteConfigRequest(), false, "downloadRemoteConfigAutomatically", true); +} + +void Countly::clearRemoteConfigForNewUser() { + std::lock_guard lk(remote_config_store->mutex); + remote_config_store->values = nlohmann::json::object(); + remote_config_store->user_generation++; +} + void Countly::stop() { - // Deliberately unchanged from before multi-instance support. Joining the SDK - // Behavior Settings timer or an in-flight remote-config fetch here would make - // stop() block where it previously returned at once, and integrators call it - // on the UI thread at shutdown (see examples/qt_demo). Both joins belong in - // ~Countly, which is where the lifetime problem they solve actually arises. - _deleteThread(); - if (configuration->manualSessionControl == false) { - endSessionInternal(true); + // Joining the SDK Behavior Settings timer or an in-flight remote-config fetch + // here would make stop() block where it previously returned at once, and + // integrators call it on the UI thread at shutdown (see examples/qt_demo). Both + // joins belong in ~Countly, which is where the lifetime problem they solve + // actually arises. + stopping = true; + if (is_sdk_initialized && !is_being_disposed) { + // queued while the update loop still runs, so that its last pass can send them + packEvents(); + if (configuration->manualSessionControl == false) { + endSessionInternal(true); + } } + _deleteThread(); health_check->saveState(); } @@ -1174,18 +1662,30 @@ void Countly::addEvent(const cly::Event &event) { return; } + if (eventKey.empty()) { + log(LogLevel::WARNING, "[Countly] addEvent, The event key can't be empty, the event is not recorded."); + logger->recordLogCode("w701"); + return; + } + if (!hasEventConsent(eventKey)) { log(LogLevel::DEBUG, "[Countly] addEvent, the consent for the event [" + eventKey + "] is not given, it is not recorded."); return; } - bool isInternalEvent = eventKey.find("[CLY]_") == 0; + const bool isInternalEvent = isInternalEventKey(eventKey); if (!isInternalEvent) { logger->recordUsage(HEALTH_CHECK_AREA_EVENTS, "record"); } + if (!configurationModule->isTrackingEnabled()) { + log(LogLevel::WARNING, "[Countly] addEvent, The SDK behavior settings turned tracking off, so the event [" + eventKey + "] is not recorded."); + logger->recordLogCode("w505"); + return; + } + // Check custom event tracking (only blocks custom events) - if (!configurationModule->isCustomEventTrackingEnabled() && !isInternalEvent) { + if (!isInternalEvent && !configurationModule->isCustomEventTrackingEnabled()) { log(LogLevel::DEBUG, "[Countly] addEvent, custom event tracking is disabled in server configuration, can not add event with key: [" + eventKey + "]"); return; } @@ -1198,9 +1698,10 @@ void Countly::addEvent(const cly::Event &event) { // Copy the event so we can apply segmentation filters without modifying the caller's object cly::Event filteredEvent = event; + validateEventValues(filteredEvent.object, eventKey, *logger); - // Apply segmentation filters - if (filteredEvent.hasSegmentation()) { + // Apply segmentation filters (only for custom events) + if (!isInternalEvent && filteredEvent.hasSegmentation()) { try { auto applySegFilter = [&filteredEvent](const std::set &filterKeys, bool isWhitelist) { if (filterKeys.empty()) { @@ -1237,14 +1738,42 @@ void Countly::addEvent(const cly::Event &event) { } } - // Apply SDK internal limits to developer-supplied events only. Internal - // [CLY]_* events are limited at their own module boundary (e.g. views). + // Custom events follow every limit. The SDK's own keys and values in internal events are not cut, but the values + // an app gives in them follow the value size limit. + SDKLimits limits = configurationModule->getLimits(); + nlohmann::json &object = filteredEvent.object; + nlohmann::json::iterator segmentation = object.find("segmentation"); if (!isInternalEvent) { - SDKLimits limits = configurationModule->getLimits(); - filteredEvent.applyLimits(limits.maxKeyLength, limits.maxValueSize, limits.maxSegmentationValues); + object["key"] = cly::limits::limitKey(eventKey, limits.maxKeyLength, logger.get(), "addEvent"); + if (segmentation != object.end() && segmentation->is_object()) { + cly::limits::limitSegmentation(*segmentation, limits, logger.get(), "addEvent"); + } + } else if (segmentation != object.end() && segmentation->is_object()) { + for (nlohmann::json::iterator it = segmentation->begin(); it != segmentation->end(); ++it) { + const bool viewField = eventKey == "[CLY]_view" && (it.key() == "name" || it.key() == "_idv" || it.key() == "segment"); + if (it.value().is_string() && !viewField) { + it.value() = cly::limits::limitValue(it.value().get(), limits.maxValueSize, logger.get(), "addEvent", it.key()); + } + } } + const std::string currentViewId = views_module->currentViewId(); std::unique_lock lk(*mutex); + if (!object.contains("id")) { + object["id"] = cly::utils::generateEventID(); + } + if (eventKey == "[CLY]_view") { + if (!object.contains("pvid")) { + object["pvid"] = currentViewId; + } + } else { + object["cvid"] = currentViewId; + if (!isInternalEvent) { + object["peid"] = previous_event_id; + previous_event_id = object["id"].get(); + } + } + if (use_sqlite_storage) { addEventToSqlite(filteredEvent); } else { @@ -1254,6 +1783,32 @@ void Countly::addEvent(const cly::Event &event) { checkAndSendEventToRQ(); } +bool Countly::IsSessionRunning() { + std::lock_guard lk(*mutex); + return began_session; +} + +std::string Countly::PlatformName() { + nlohmann::json metrics = deviceMetrics(); + nlohmann::json::const_iterator os = metrics.find("_os"); + return os != metrics.end() && os->is_string() ? os->get() : std::string(); +} + +nlohmann::json Countly::deviceMetrics() { + nlohmann::json metrics = nlohmann::json::object(); + metrics["_os"] = cly::platform::osName(); + const std::string osVersion = cly::platform::osVersion(); + if (!osVersion.empty()) { + metrics["_os_version"] = osVersion; + } + if (configuration->metrics.is_object()) { + for (nlohmann::json::const_iterator it = configuration->metrics.begin(); it != configuration->metrics.end(); ++it) { + metrics[it.key()] = it.value(); + } + } + return metrics; +} + void Countly::checkAndSendEventToRQ() { nlohmann::json events = nlohmann::json::array(); int queueSize = checkEQSize(); @@ -1261,12 +1816,13 @@ void Countly::checkAndSendEventToRQ() { if (queueSize < 0) { return; } - // unique_lock so the mutex is released on scope exit, including when json::parse - // on a queued event or sendEventsToRQ throws; unlock/re-acquire around the - // self-locking fillEventsIntoJson(). + // unique_lock so the mutex is released on scope exit, including when + // sendEventsToRQ throws; unlock/re-acquire around the self-locking + // fillEventsIntoJson(). std::unique_lock lk(*mutex); + const int threshold = static_cast(std::min(configurationModule->getEventQueueSizeLimit(), static_cast(COUNTLY_MAX_EVENT_QUEUE_SIZE))); if (use_sqlite_storage) { - if (queueSize >= configurationModule->getEventQueueSizeLimit()) { + if (queueSize >= threshold) { log(LogLevel::DEBUG, "[Countly] checkAndSendEventToRQ, Event queue threshold is reached"); std::string event_ids; @@ -1280,11 +1836,9 @@ void Countly::checkAndSendEventToRQ() { removeEventWithId(event_ids); } } else { - if (queueSize >= configurationModule->getEventQueueSizeLimit()) { + if (queueSize >= threshold) { log(LogLevel::DEBUG, "[Countly] checkAndSendEventToRQ, Event queue threshold is reached, sending the events to the request queue"); - for (const auto &event_json : event_queue) { - events.push_back(nlohmann::json::parse(event_json)); - } + appendStoredEvents(event_queue, events, *logger); sendEventsToRQ(events); event_queue.clear(); } @@ -1301,13 +1855,9 @@ void Countly::setEventsToRQThreshold(int value) { log(LogLevel::DEBUG, "[Countly] setEventsToRQThreshold, Given threshold:[" + std::to_string(value) + "]"); std::unique_lock lk(*mutex); if (value < 1) { - log(LogLevel::WARNING, "[Countly] setEventsToRQThreshold, Threshold can not be less than 1. Setting it to 1 instead of:[" + std::to_string(value) + "]"); + log(LogLevel::WARNING, "[Countly] setEventsToRQThreshold, The threshold must be at least 1, so the value [" + std::to_string(value) + "] is ignored."); logger->recordLogCode("w121"); - value = 1; - } else if (value > 10000) { - log(LogLevel::WARNING, "[Countly] setEventsToRQThreshold, Threshold can not be greater than 10000. Setting it to 10000 instead of:[" + std::to_string(value) + "]"); - logger->recordLogCode("w121"); - value = 10000; + return; } // set the value @@ -1318,41 +1868,14 @@ void Countly::setEventsToRQThreshold(int value) { } void Countly::flushEvents(std::chrono::seconds timeout) { - log(LogLevel::DEBUG, "[Countly] flushEvents, timeout: [" + std::to_string(timeout.count()) + "] seconds"); - - try { - auto wait_duration = std::chrono::seconds(1); - bool update_failed; - - // Try to update session - while (timeout.count() != 0) { - - // try to update session if event queue is not empty - update_failed = attemptSessionUpdateEQ(); - - // if update is successful or EQ is empty, break the loop - if (!update_failed) { - break; - } - - // wait for a while - std::this_thread::sleep_for(wait_duration); - // increase wait/retry duration (exponential backoff: 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, ...) - wait_duration *= 2; - // if wait/retry time is bigger than timeout stop trying, else decrease timeout - timeout = (wait_duration > timeout) ? std::chrono::seconds(0) : (timeout - wait_duration); - } - - // Clear the event queue - clearEQInternal(); // TODO: Check if this is necessary - - // TODO: Check if we capture anything other than a system_error - } catch (const std::system_error &e) { - std::ostringstream log_message; - log_message << "[Countly] flushEvents, error: " << e.what(); - log(LogLevel::FATAL, log_message.str()); - logger->recordLogCode("e901"); + (void)timeout; + if (!is_sdk_initialized) { + log(LogLevel::WARNING, "[Countly] flushEvents, SDK is not initialized."); + logger->recordLogCode("w101"); + return; } + log(LogLevel::DEBUG, "[Countly] flushEvents, moving the waiting events into a request"); + packEvents(); } bool Countly::attemptSessionUpdateEQ() { @@ -1446,7 +1969,24 @@ void Countly::debugInjectRawEvent(const std::string &raw) { } #endif -bool Countly::beginSession() { return beginSessionInternal(false); } +bool Countly::beginSession() { + if (rejectSessionCallInAutomaticMode("beginSession", "begin")) { + return false; + } + return beginSessionInternal(false); +} + +bool Countly::rejectSessionCallInAutomaticMode(const char *caller, const char *usage) { + if (!is_sdk_initialized || configuration->manualSessionControl) { + return false; + } + if (hasConsent(consent::SESSIONS)) { + logger->recordUsage(HEALTH_CHECK_AREA_SESSIONS, usage); + logger->recordLogCode("w740"); + } + log(LogLevel::WARNING, std::string("[Countly] ") + caller + ", The SDK tracks sessions automatically, so the call is ignored. To make session calls, call enableManualSessionControl() before start()."); + return true; +} bool Countly::beginSessionInternal(bool automatic) { if (!is_sdk_initialized) { @@ -1472,6 +2012,10 @@ bool Countly::beginSessionInternal(bool automatic) { log(LogLevel::DEBUG, "[Countly] beginSession, Session tracking is disabled in server configuration, can not begin session."); return false; } + if (automatic && stopping) { + log(LogLevel::DEBUG, "[Countly] beginSession, The SDK is stopping, no session begins."); + return false; + } if (began_session == true) { if (automatic) { log(LogLevel::DEBUG, "[Countly] beginSession, Session is already active."); @@ -1516,8 +2060,8 @@ bool Countly::beginSessionInternal(bool automatic) { session_params.erase("user_details"); } - if (configuration->metrics.size() > 0 && hasConsent(consent::METRICS)) { - data["metrics"] = configuration->metrics.dump(); + if (hasConsent(consent::METRICS)) { + data["metrics"] = deviceMetrics().dump(); } requestModule->addRequestToQueue(data); @@ -1527,20 +2071,19 @@ bool Countly::beginSessionInternal(bool automatic) { if (automatic) { logger->recordUsage(HEALTH_CHECK_AREA_SESSIONS, "begin:a"); } - // snapshot guarded state before releasing the lock - bool shouldUpdateRemoteConfig = remote_config_enabled; lk.unlock(); - - if (shouldUpdateRemoteConfig) { - updateRemoteConfig(); - } return true; } /** * @brief Update session */ -bool Countly::updateSession() { return updateSessionInternal(false); } +bool Countly::updateSession() { + if (rejectSessionCallInAutomaticMode("updateSession", "update")) { + return false; + } + return updateSessionInternal(false); +} bool Countly::updateSessionInternal(bool automatic) { if (!is_sdk_initialized) { @@ -1568,20 +2111,25 @@ bool Countly::updateSessionInternal(bool automatic) { // Check if there was a session, if not try to start one lk.lock(); if (configurationModule->isSessionTrackingEnabled() == false) { - log(LogLevel::DEBUG, "[Countly] updateSession, Session tracking is disabled in server configuration, can not update session."); lk.unlock(); + if (automatic) { + // sessions are off, but the update timer still sends the events + packEvents(); + return true; + } + log(LogLevel::DEBUG, "[Countly] updateSession, Session tracking is disabled in server configuration, can not update session."); return false; } if (began_session == false) { lk.unlock(); - if (configuration->manualSessionControl == true) { + if (!automatic) { log(LogLevel::WARNING, "[Countly] updateSession, SDK is in manual session control mode and there is no active session. Please start a session first."); logger->recordLogCode("w760"); return false; } if (!beginSessionInternal(true)) { log(LogLevel::DEBUG, "[Countly] updateSession, Failed to begin session."); - // if beginSession fails, we should not try to update session + packEvents(); return false; } lk.lock(); @@ -1601,9 +2149,7 @@ bool Countly::updateSessionInternal(bool automatic) { fillEventsIntoJson(events, event_ids); lk.lock(); } else { - for (const auto &event_json : event_queue) { - events.push_back(nlohmann::json::parse(event_json)); - } + appendStoredEvents(event_queue, events, *logger); } } else { log(LogLevel::DEBUG, "[Countly] updateSession, EQ empty."); @@ -1612,8 +2158,8 @@ bool Countly::updateSessionInternal(bool automatic) { auto duration = std::chrono::duration_cast(getSessionDuration()); lk.lock(); - // report session duration if it is greater than the configured session duration value - if (duration.count() >= configurationModule->getSessionUpdateInterval()) { + // the update timer reports the session once an update interval passed; the app's own call always does + if (!automatic || duration.count() >= configurationModule->getSessionUpdateInterval()) { log(LogLevel::DEBUG, "[Countly] updateSession, Sending session update."); std::map data = {{"app_key", session_params["app_key"].get()}, {"device_id", session_params["device_id"].get()}, {"session_duration", std::to_string(duration.count())}}; requestModule->addRequestToQueue(data); @@ -1639,7 +2185,7 @@ bool Countly::updateSessionInternal(bool automatic) { } else { event_queue.clear(); } - } catch (const std::system_error &e) { + } catch (const std::exception &e) { std::ostringstream log_message; log_message << "[Countly] updateSession, error: " << e.what(); log(LogLevel::FATAL, log_message.str()); @@ -1668,9 +2214,7 @@ void Countly::packEvents() { fillEventsIntoJson(events, event_ids); lk.lock(); } else { - for (const auto &event_json : event_queue) { - events.push_back(nlohmann::json::parse(event_json)); - } + appendStoredEvents(event_queue, events, *logger); } } else { log(LogLevel::DEBUG, "[Countly] packEvents, EQ empty."); @@ -1690,10 +2234,11 @@ void Countly::packEvents() { } else { event_queue.clear(); } - } catch (const std::system_error &e) { + } catch (const std::exception &e) { std::ostringstream log_message; log_message << "[Countly] packEvents, error: " << e.what(); log(LogLevel::FATAL, log_message.str()); + logger->recordLogCode("e901"); } // lk releases the mutex on scope exit if it is still held. } @@ -1713,7 +2258,12 @@ void Countly::sendEventsToRQ(const nlohmann::json &events) { requestModule->addRequestToQueue(data); } -bool Countly::endSession() { return endSessionInternal(false); } +bool Countly::endSession() { + if (rejectSessionCallInAutomaticMode("endSession", "end")) { + return false; + } + return endSessionInternal(false); +} bool Countly::endSessionInternal(bool automatic) { if (!is_sdk_initialized && !is_being_disposed) { @@ -1731,10 +2281,19 @@ bool Countly::endSessionInternal(bool automatic) { logger->recordUsage(HEALTH_CHECK_AREA_SESSIONS, "end"); } log(LogLevel::INFO, "[Countly] endSession, Ending session"); - if (is_being_disposed == false && configurationModule->isSessionTrackingEnabled() == false) { - log(LogLevel::DEBUG, "[Countly] endSession, Session tracking is disabled in server configuration, can not end session."); + bool sessionRunning = false; + { + std::lock_guard lk(*mutex); + sessionRunning = began_session; + } + if (!sessionRunning && !is_being_disposed && configurationModule->isSessionTrackingEnabled() == false) { + log(LogLevel::DEBUG, "[Countly] endSession, Session tracking is disabled in server configuration, there is no session to end."); return false; } + // the events recorded during the session reach the server before its end + if (sessionRunning && !is_being_disposed) { + packEvents(); + } const auto timestamp = std::chrono::duration_cast(Countly::getTimestamp().time_since_epoch()); const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); @@ -1772,6 +2331,7 @@ bool Countly::endSessionInternal(bool automatic) { last_sent_session_request = now; began_session = false; lk.unlock(); + views_module->resetFirstView(); if (automatic) { logger->recordUsage(HEALTH_CHECK_AREA_SESSIONS, "end:a"); } @@ -1871,13 +2431,16 @@ void Countly::removeEventWithId(const std::string &event_ids) { } void Countly::fillEventsIntoJson(nlohmann::json &events, std::string &event_ids) { - // lock_guard so the mutex is released on every exit path, including the early - // return below and any exception (e.g. nlohmann::json::parse on a corrupt - // stored row at the loop below) thrown while it is held. + // lock_guard so the mutex is released on every exit path, including an + // exception thrown while it is held. std::lock_guard lk(*mutex); + readStoredEvents(events, event_ids); +} + +void Countly::readStoredEvents(nlohmann::json &events, std::string &event_ids) { + event_ids = ""; if (database_path.empty()) { log(LogLevel::FATAL, "[Countly] fillEventsIntoJson, SQLite database path is not set."); - event_ids = ""; return; } @@ -1905,16 +2468,24 @@ void Countly::fillEventsIntoJson(nlohmann::json &events, std::string &event_ids) for (int event_index = 1; event_index < row_count + 1; event_index++) { event_id_stream << table[event_index * column_count] << ','; // add event to the events array - events.push_back(nlohmann::json::parse(table[(event_index * column_count) + 1])); + const char *stored_event = table[(event_index * column_count) + 1]; + nlohmann::json event = nlohmann::json::parse(stored_event ? stored_event : "", nullptr, false); + if (event.is_discarded()) { + log(LogLevel::WARNING, "[Countly] fillEventsIntoJson, A stored event is not valid JSON and is dropped."); + continue; + } + events.push_back(event); } log(LogLevel::DEBUG, "[Countly] fillEventsIntoJson, Events count = [" + std::to_string(events.size()) + "]"); - event_id_stream.seekp(-1, event_id_stream.cur); - event_id_stream << ')'; + if (row_count > 0) { + event_id_stream.seekp(-1, event_id_stream.cur); + event_id_stream << ')'; - // write event ids to a string stream (or more like copy out that stream here) to be used in the delete statement - event_ids = event_id_stream.str(); + // write event ids to a string stream (or more like copy out that stream here) to be used in the delete statement + event_ids = event_id_stream.str(); + } } else { log(LogLevel::ERROR, "[Countly] fillEventsIntoJson, SQLite error: " + std::string(error_message)); sqlite3_free(error_message); @@ -2083,6 +2654,11 @@ void Countly::fillEventsIntoJson(nlohmann::json &events, std::string &event_ids) (void)event_ids; } +void Countly::readStoredEvents(nlohmann::json &events, std::string &event_ids) { + (void)events; + (void)event_ids; +} + void Countly::addEventToSqlite(const cly::Event &event) { (void)event; } bool Countly::createEventTableSchema() { return false; } @@ -2142,29 +2718,19 @@ void Countly::updateLoop() { } try { sendHealthCheck(); - if (configuration->immediateRequestOnStop) { - while (true) { - { - std::unique_lock lk(*mutex); - stop_cv.wait_for(lk, std::chrono::milliseconds(wait_milliseconds), [this] { - return stop_thread.load(); - }); - if (stop_thread) { - stop_thread = false; - running = false; - return; - } - } - if (enable_automatic_session == true && configuration->manualSessionControl == false) { - updateSessionInternal(true); - } else if (configuration->manualSessionControl == true) { - packEvents(); + std::chrono::steady_clock::time_point last_update = std::chrono::steady_clock::now(); + while (true) { + if (configuration->immediateRequestOnStop) { + std::unique_lock lk(*mutex); + stop_cv.wait_for(lk, std::chrono::milliseconds(wait_milliseconds), [this] { + return stop_thread.load(); + }); + if (stop_thread) { + stop_thread = false; + running = false; + return; } - requestModule->processQueue(mutex); - saveHealthCheckOnTimer(); - } - } else { - while (true) { + } else { size_t last_wait_milliseconds; { std::lock_guard lk(*mutex); @@ -2175,17 +2741,11 @@ void Countly::updateLoop() { last_wait_milliseconds = wait_milliseconds; } std::this_thread::sleep_for(std::chrono::milliseconds(last_wait_milliseconds)); - if (enable_automatic_session == true && configuration->manualSessionControl == false) { - updateSessionInternal(true); - } else if (configuration->manualSessionControl == true) { - packEvents(); - } - requestModule->processQueue(mutex); - saveHealthCheckOnTimer(); } - std::lock_guard lk(*mutex); - running = false; + updateTick(last_update); } + std::lock_guard lk(*mutex); + running = false; } catch (const std::exception &e) { running = false; log(LogLevel::ERROR, std::string("[Countly][updateLoop] exception in update loop: ") + e.what()); @@ -2195,6 +2755,22 @@ void Countly::updateLoop() { } } +void Countly::updateTick(std::chrono::steady_clock::time_point &last_update) { + if (!stopping) { + const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); + if (now - last_update >= std::chrono::seconds(configurationModule->getSessionUpdateInterval())) { + last_update = now; + if (configuration->manualSessionControl == false) { + updateSessionInternal(true); + } else { + packEvents(); + } + } + } + requestModule->processQueue(mutex); + saveHealthCheckOnTimer(); +} + void Countly::disableHealthCheck() { if (is_sdk_initialized) { log(LogLevel::WARNING, "[Countly] disableHealthCheck, You can not disable the health check after SDK initialization."); @@ -2230,7 +2806,7 @@ void Countly::sendHealthCheck() { } log(LogLevel::DEBUG, "[Countly] sendHealthCheck, sending the health check"); - const HTTPResponse response = requestModule->sendHTTP("/i", requestBuilder->buildRequest(data)); + const HTTPResponse response = requestModule->sendImmediate("/i", requestBuilder->buildRequest(data)); if (response.success && response.data.is_object() && response.data.contains("result")) { log(LogLevel::DEBUG, "[Countly] sendHealthCheck, the server accepted the health check"); health_check->clearSentValues(snapshot); @@ -2255,6 +2831,16 @@ void Countly::enableRemoteConfig() { remote_config_enabled = true; } +void Countly::enrollABOnRCDownload() { + if (is_sdk_initialized) { + log(LogLevel::WARNING, "[Countly] enrollABOnRCDownload, This method can't be called after SDK initialization. Returning."); + return; + } + + std::lock_guard lk(*mutex); + configuration->enrollABOnRCDownload = true; +} + void Countly::updateRemoteConfig() { if (!is_sdk_initialized) { log(LogLevel::WARNING, "[Countly] updateRemoteConfig, SDK is not initialized."); @@ -2265,19 +2851,10 @@ void Countly::updateRemoteConfig() { log(LogLevel::DEBUG, "[Countly] updateRemoteConfig, the 'remote-config' consent is not given, nothing is downloaded."); return; } - std::unique_lock lk(*mutex); - if (!session_params["app_key"].is_string() || !session_params["device_id"].is_string()) { - - log(LogLevel::ERROR, "[Countly] updateRemoteConfig, Error updating remote config, app key or device id is missing"); - return; - } - std::map data = {{"method", "fetch_remote_config"}, {"app_key", session_params["app_key"].get()}, {"device_id", session_params["device_id"].get()}}; - - lk.unlock(); // Asynchronous and detached: neither this call nor destruction waits for it. // Replaces the stored values wholesale, since this fetches all of them. - startRemoteConfigFetch(data, false, "updateRemoteConfig"); + startRemoteConfigFetch(remoteConfigRequest(), false, "updateRemoteConfig"); } nlohmann::json Countly::getRemoteConfigValue(const std::string &key) { @@ -2298,17 +2875,22 @@ void Countly::updateRemoteConfigFor(std::string *keys, size_t key_count) { log(LogLevel::DEBUG, "[Countly] updateRemoteConfigFor, the 'remote-config' consent is not given, nothing is downloaded."); return; } - std::unique_lock lk(*mutex); - std::map data = {{"method", "fetch_remote_config"}, {"app_key", session_params["app_key"].get()}, {"device_id", session_params["device_id"].get()}}; - - { - nlohmann::json keys_json = nlohmann::json::array(); - for (size_t key_index = 0; key_index < key_count; key_index++) { - keys_json.push_back(keys[key_index]); + std::map data = remoteConfigRequest(); + nlohmann::json keys_json = nlohmann::json::array(); + for (size_t key_index = 0; keys != nullptr && key_index < key_count; key_index++) { + if (keys[key_index].empty()) { + log(LogLevel::WARNING, "[Countly] updateRemoteConfigFor, An empty key is ignored."); + logger->recordLogCode("w700"); + continue; } - data["keys"] = keys_json.dump(); + keys_json.push_back(keys[key_index]); } - lk.unlock(); + if (keys_json.empty()) { + // without a valid key this is a download of every value + startRemoteConfigFetch(data, false, "updateRemoteConfigFor"); + return; + } + data["keys"] = keys_json.dump(); // Asynchronous and detached: neither this call nor destruction waits for it. // Merges into the stored values, since this fetches only some of them. @@ -2325,17 +2907,22 @@ void Countly::updateRemoteConfigExcept(std::string *keys, size_t key_count) { log(LogLevel::DEBUG, "[Countly] updateRemoteConfigExcept, the 'remote-config' consent is not given, nothing is downloaded."); return; } - std::unique_lock lk(*mutex); - std::map data = {{"method", "fetch_remote_config"}, {"app_key", session_params["app_key"].get()}, {"device_id", session_params["device_id"].get()}}; - - { - nlohmann::json keys_json = nlohmann::json::array(); - for (size_t key_index = 0; key_index < key_count; key_index++) { - keys_json.push_back(keys[key_index]); + std::map data = remoteConfigRequest(); + nlohmann::json keys_json = nlohmann::json::array(); + for (size_t key_index = 0; keys != nullptr && key_index < key_count; key_index++) { + if (keys[key_index].empty()) { + log(LogLevel::WARNING, "[Countly] updateRemoteConfigExcept, An empty key is ignored."); + logger->recordLogCode("w700"); + continue; } - data["omit_keys"] = keys_json.dump(); + keys_json.push_back(keys[key_index]); } - lk.unlock(); + if (keys_json.empty()) { + // without a valid key this is a download of every value + startRemoteConfigFetch(data, false, "updateRemoteConfigExcept"); + return; + } + data["omit_keys"] = keys_json.dump(); // Asynchronous and detached: neither this call nor destruction waits for it. // Merges into the stored values, since this fetches only some of them. @@ -2505,7 +3092,13 @@ bool Countly::hasEventConsent(const std::string &key) const { return hasConsent(consent::EVENTS); } -bool Countly::isLocationOn() { return hasConsent(consent::LOCATION) && configurationModule->isLocationTrackingEnabled(); } +bool Countly::isLocationOn() { + if (!hasConsent(consent::LOCATION)) { + return false; + } + const int server = configurationModule->getServerLocationTracking(); + return server >= 0 ? server == 1 : !location_disabled_by_app; +} void Countly::changeConsent(const std::vector &names, bool given, const char *caller) { if (LoggerModule::isInsideCallback()) { @@ -2560,7 +3153,7 @@ void Countly::runConsentChange(const std::vector &features, bool gi void Countly::consentWillBeRemoved(const std::vector &features) { if (containsFeature(features, consent::VIEWS)) { - views_module->stopAllViews(); + views_module->stopRunningViews(); } if (containsFeature(features, consent::SESSIONS)) { endSessionInternal(true); @@ -2570,7 +3163,7 @@ void Countly::consentWillBeRemoved(const std::vector &features) { void Countly::consentChanged(const std::vector &features, bool given, ConsentChangeSource source) { if (!given) { if (containsFeature(features, consent::CRASHES)) { - crash_module->clearBreadcrumbs(); + crash_module->forgetBreadcrumbs(); } if (containsFeature(features, consent::USERS)) { std::lock_guard lk(*mutex); @@ -2579,7 +3172,7 @@ void Countly::consentChanged(const std::vector &features, bool give if (containsFeature(features, consent::LOCATION)) { clearStoredLocation(); if (source != ConsentChangeSource::DeviceIdChangedWithoutMerge) { - _sendIndependantLocationRequest(); + sendLocationErase(); } } return; @@ -2594,16 +3187,19 @@ void Countly::consentChanged(const std::vector &features, bool give } sessionBegan = !sessionRunning && beginSessionInternal(true); } - if (containsFeature(features, consent::REMOTE_CONFIG)) { - bool automaticDownload = false; + if (containsFeature(features, consent::LOCATION) && !sessionBegan && isLocationOn()) { + bool sendNow = false; { std::lock_guard lk(*mutex); - automaticDownload = remote_config_enabled; + sendNow = began_session || !hasConsent(consent::SESSIONS) || !configurationModule->isSessionTrackingEnabled(); } - if (automaticDownload && !sessionBegan) { - updateRemoteConfig(); + if (sendNow) { + _sendIndependantLocationRequest(); } } + if (containsFeature(features, consent::REMOTE_CONFIG)) { + downloadRemoteConfigAutomatically(); + } } void Countly::queueConsentState() { @@ -2630,9 +3226,5 @@ void Countly::activateConsent() { } queueConsentState(); - if (!hasConsent(consent::LOCATION) && (!hasConsent(consent::SESSIONS) || !configurationModule->isSessionTrackingEnabled())) { - clearStoredLocation(); - _sendIndependantLocationRequest(); - } } } // namespace cly diff --git a/src/crash_module.cpp b/src/crash_module.cpp index bececd2..be64846 100644 --- a/src/crash_module.cpp +++ b/src/crash_module.cpp @@ -20,11 +20,29 @@ class CrashModule::CrashModuleImpl { std::shared_ptr _mutex; std::weak_ptr _configProvider; std::shared_ptr _consent; + cly::CountlyDelegates *_cly = nullptr; CrashModuleImpl(std::shared_ptr config, std::shared_ptr logger, std::shared_ptr requestModule, std::shared_ptr mutex) : _configuration(config), _logger(logger), _requestModule(requestModule), _mutex(mutex) {} // destructor to reset logger ~CrashModuleImpl() { _logger.reset(); } + /** + * Whether start() has wired the module up, logged as a warning and counted as w101 when it has not. + */ + bool isInitialized(const char *caller) { + bool initialized = false; + { + std::lock_guard lk(*_mutex); + initialized = !_configProvider.expired() && _requestModule != nullptr; + } + if (initialized) { + return true; + } + _logger->log(LogLevel::WARNING, std::string("[Countly] [CrashModule] ") + caller + ", SDK is not initialized."); + _logger->recordLogCode("w101"); + return false; + } + /** * Whether the "crashes" consent allows the call, logged at debug level when it does not. */ @@ -48,7 +66,7 @@ CrashModule::CrashModule(std::shared_ptr config, std::shar // function to add breadcrumb void CrashModule::addBreadcrumb(const std::string &value) { - if (!impl->hasConsent("addBreadcrumb")) { + if (!impl->isInitialized("addBreadcrumb") || !impl->hasConsent("addBreadcrumb")) { return; } SDKLimits lim{COUNTLY_MAX_KEY_LENGTH_DEFAULT, COUNTLY_MAX_VALUE_SIZE_DEFAULT, COUNTLY_MAX_SEGMENTATION_VALUES_DEFAULT, COUNTLY_MAX_BREADCRUMB_COUNT_DEFAULT, COUNTLY_MAX_STACK_TRACE_LINES_PER_THREAD_DEFAULT, COUNTLY_MAX_STACK_TRACE_LINE_LENGTH_DEFAULT}; @@ -56,7 +74,7 @@ void CrashModule::addBreadcrumb(const std::string &value) { lim = config->getLimits(); } - std::string limited = cly::limits::truncateString(value, lim.maxValueSize); + std::string limited = cly::limits::limitValue(value, lim.maxValueSize, impl->_logger.get(), "[CrashModule] addBreadcrumb", "breadcrumb"); impl->_logger->log(LogLevel::INFO, "[Countly] [CrashModule] addBreadcrumb, value = [" + limited + "]"); impl->_logger->recordUsage("crashes", "log"); @@ -69,11 +87,28 @@ void CrashModule::addBreadcrumb(const std::string &value) { impl->_breadCrumbs.push_back(limited); } -// function to record exception +namespace { +/** + * Applies the internal limits to a crash report: the title and the stack trace follow the stack trace limits, the + * segmentation the key, value and entry limits, the breadcrumbs the value size and count limits. + */ +void limitCrash(CrashData &crash, const SDKLimits &lim, LoggerModule *logger) { + crash.description = cly::limits::limitErrorText(crash.description, lim.maxStackTraceLineLength, logger, "[CrashModule] recordException"); + crash.stackTrace = cly::limits::limitStackTrace(crash.stackTrace, lim.maxStackTraceLinesPerThread, lim.maxStackTraceLineLength, logger, "[CrashModule] recordException"); + crash.crashSegmentation = cly::limits::limitSegmentation(crash.crashSegmentation, lim, logger, "[CrashModule] recordException"); + if (crash.breadcrumbs.size() > lim.maxBreadcrumbCount) { + crash.breadcrumbs.erase(crash.breadcrumbs.begin(), crash.breadcrumbs.end() - lim.maxBreadcrumbCount); + } + for (std::string &breadcrumb : crash.breadcrumbs) { + breadcrumb = cly::limits::limitValue(breadcrumb, lim.maxValueSize, logger, "[CrashModule] recordException", "breadcrumb"); + } +} +} // namespace + void CrashModule::recordException(const std::string &title, const std::string &stackTrace, const bool fatal, const std::map &crashMetrics, const std::map &segmentation) { impl->_logger->log(LogLevel::INFO, cly::utils::format_string("[Countly] [CrashModule] recordException, title = [%s], stackTrace = [%s]", title.c_str(), stackTrace.c_str())); - if (!impl->hasConsent("recordException")) { + if (!impl->isInitialized("recordException") || !impl->hasConsent("recordException")) { return; } impl->_logger->recordUsage("crashes", "record"); @@ -90,54 +125,88 @@ void CrashModule::recordException(const std::string &title, const std::string &s return; } - std::string limitedTitle = cly::limits::truncateString(title, lim.maxStackTraceLineLength); - std::string limitedTrace = cly::limits::truncateStackTrace(stackTrace, lim.maxStackTraceLinesPerThread, lim.maxStackTraceLineLength); - std::map limitedSegmentation = cly::limits::applySegmentationLimits(segmentation, lim); - - if (title.empty()) { - impl->_logger->log(LogLevel::WARNING, "[Countly] [CrashModule] recordException, The parameter 'title' can't be empty"); + if (stackTrace.empty()) { + impl->_logger->log(LogLevel::WARNING, "[Countly] [CrashModule] recordException, The parameter 'stackTrace' can't be empty, the crash is not recorded."); impl->_logger->recordLogCode("w700"); + return; } - if (stackTrace.empty()) { - impl->_logger->log(LogLevel::WARNING, "[Countly] [CrashModule] recordException, The parameter 'stackTrace' can't be empty"); - impl->_logger->recordLogCode("w700"); + CrashData crash; + crash.description = title; + crash.stackTrace = stackTrace; + crash.fatal = fatal; + const nlohmann::json deviceMetrics = impl->_cly != nullptr ? impl->_cly->DeviceMetrics() : nlohmann::json::object(); + for (const char *key : {"_os", "_os_version", "_app_version", "_device", "_resolution"}) { + nlohmann::json::const_iterator metric = deviceMetrics.find(key); + if (metric != deviceMetrics.end() && metric->is_string() && !metric->get().empty()) { + crash.crashMetrics[key] = metric->get(); + } + } + for (const auto &metric : crashMetrics) { + crash.crashMetrics[metric.first] = metric.second; + } + crash.crashSegmentation = impl->_configuration->crashSegmentation; + for (const auto &entry : segmentation) { + crash.crashSegmentation[entry.first] = entry.second; + } + { + // a processed crash takes the breadcrumbs, so the next crash report starts with a fresh trail + std::lock_guard lk(*impl->_mutex); + crash.breadcrumbs.assign(impl->_breadCrumbs.begin(), impl->_breadCrumbs.end()); + impl->_breadCrumbs.clear(); + } + limitCrash(crash, lim, impl->_logger.get()); + + int changedFields = 0; + if (impl->_configuration->crashFilter) { + const CrashData original = crash; + bool drop = false; + try { + drop = impl->_configuration->crashFilter(crash); + } catch (...) { + impl->_logger->log(LogLevel::ERROR, "[Countly] [CrashModule] recordException, The crash filter threw an exception, so the crash is dropped."); + return; + } + if (drop) { + impl->_logger->log(LogLevel::DEBUG, "[Countly] [CrashModule] recordException, The crash filter dropped the crash."); + return; + } + changedFields |= crash.description != original.description ? 64 : 0; + changedFields |= crash.stackTrace != original.stackTrace ? 16 : 0; + changedFields |= crash.crashSegmentation != original.crashSegmentation ? 8 : 0; + changedFields |= crash.breadcrumbs != original.breadcrumbs ? 4 : 0; + changedFields |= crash.crashMetrics != original.crashMetrics ? 2 : 0; + changedFields |= crash.fatal != original.fatal ? 1 : 0; + limitCrash(crash, lim, impl->_logger.get()); } - // check if the crash metric '_os' exists and is not empty - auto it = crashMetrics.find("_os"); - if (it == crashMetrics.end() || it->second.empty()) { - impl->_logger->log(LogLevel::WARNING, "[Countly] [CrashModule] recordException, The crash metric '_os' can't be empty"); - impl->_logger->recordLogCode("w700"); + for (const char *required : {"_os", "_app_version"}) { + std::map::const_iterator metric = crash.crashMetrics.find(required); + if (metric == crash.crashMetrics.end() || metric->second.empty()) { + impl->_logger->log(LogLevel::WARNING, std::string("[Countly] [CrashModule] recordException, The crash metric '") + required + "' is missing, so the server will not accept the crash report. Give the app version with setMetrics()."); + impl->_logger->recordLogCode("w700"); + } } - // check if the crash metric '_app_version' exists and is not empty - it = crashMetrics.find("_app_version"); - if (it == crashMetrics.end() || it->second.empty()) { - impl->_logger->log(LogLevel::WARNING, "[Countly] [CrashModule] recordException, The crash metric '_app_version' can't be empty"); - impl->_logger->recordLogCode("w700"); + std::ostringstream outstream; + std::copy(crash.breadcrumbs.begin(), crash.breadcrumbs.end(), std::ostream_iterator(outstream, "\n")); + + nlohmann::json report(crash.crashMetrics); + report["_name"] = crash.description; + report["_error"] = crash.stackTrace; + report["_logs"] = outstream.str(); + report["_custom"] = nlohmann::json(crash.crashSegmentation); + report["_nonfatal"] = !crash.fatal; + report["_ob"] = changedFields; + const std::string currentViewId = impl->_cly != nullptr ? impl->_cly->CurrentViewId() : std::string(); + if (!currentViewId.empty()) { + report["_cvid"] = currentViewId; } // lock mutex to avoid concurrent access; lock_guard releases on scope exit, // including when json construction / crash.dump() / addRequestToQueue throws. std::lock_guard lk(*impl->_mutex); - // convert breadcrumbs vector to a string and add to json object - std::ostringstream outstream; - std::copy(impl->_breadCrumbs.begin(), impl->_breadCrumbs.end(), std::ostream_iterator(outstream, "\n")); - - // create json objects for crash metrics and segmentation - nlohmann::json crash(crashMetrics); - nlohmann::json segments(limitedSegmentation); - - // add relevant fields to the crash json object - crash["_name"] = limitedTitle; - crash["_error"] = limitedTrace; - crash["_logs"] = outstream.str(); - crash["_custom"] = segments; - crash["_nonfatal"] = !fatal; - - // create a map with the crash json object as value and "crash" as key, and add the map to the request queue - std::map data = {{"crash", crash.dump()}}; + std::map data = {{"crash", report.dump()}}; impl->_requestModule->addRequestToQueue(data); } @@ -145,7 +214,20 @@ void CrashModule::setConfigurationProvider(std::weak_ptr void CrashModule::setConsentModule(std::shared_ptr consent) { impl->_consent = std::move(consent); } +void CrashModule::setRequestModule(std::shared_ptr requestModule) { impl->_requestModule = std::move(requestModule); } + +void CrashModule::setDelegates(cly::CountlyDelegates *cly) { impl->_cly = cly; } + void CrashModule::clearBreadcrumbs() { + impl->_logger->log(LogLevel::INFO, "[Countly] [CrashModule] clearBreadcrumbs"); + if (!impl->isInitialized("clearBreadcrumbs") || !impl->hasConsent("clearBreadcrumbs")) { + return; + } + impl->_logger->recordUsage("crashes", "clear"); + forgetBreadcrumbs(); +} + +void CrashModule::forgetBreadcrumbs() { std::lock_guard lk(*impl->_mutex); impl->_breadCrumbs.clear(); } diff --git a/src/event.cpp b/src/event.cpp index c9e6c6e..f88b6e7 100644 --- a/src/event.cpp +++ b/src/event.cpp @@ -27,8 +27,12 @@ Event::Event(const std::string &key, size_t count, double sum, double duration) } void Event::setTimestamp() { - timestamp = std::chrono::system_clock::now(); + setRecordingTime(); timer_start = std::chrono::steady_clock::now(); +} + +void Event::setRecordingTime() { + timestamp = std::chrono::system_clock::now(); object["timestamp"] = std::chrono::duration_cast(timestamp.time_since_epoch()).count(); std::time_t time = std::chrono::system_clock::to_time_t(timestamp); @@ -49,6 +53,7 @@ void Event::stopTimer() { auto now = std::chrono::steady_clock::now(); object["dur"] = std::chrono::duration_cast(now - timer_start).count(); timer_running = false; + setRecordingTime(); } } diff --git a/src/logger_module.cpp b/src/logger_module.cpp index 3ba708c..b5e97f1 100644 --- a/src/logger_module.cpp +++ b/src/logger_module.cpp @@ -28,6 +28,8 @@ class LoggerModule::LoggerModuleImpl { // Set once, before the instance logs from other threads, and never cleared. std::shared_ptr health_check; std::atomic has_health_check{false}; + // Set once the listener threw, so that the console gets one line about it, not one per log. + std::atomic reported_listener_failure{false}; }; LoggerModule::LoggerModule() { impl = std::make_unique(); } @@ -85,7 +87,13 @@ void LoggerModule::log(LogLevel level, const std::string &message) { ~FlagGuard() { inside_log_callback = false; } } guard; inside_log_callback = true; - callback(level, message); + try { + callback(level, message); + } catch (...) { + if (!impl->reported_listener_failure.exchange(true)) { + std::cerr << "[Countly] [LoggerModule] log, The log listener threw an exception. Its exceptions are ignored, so that the SDK keeps working." << std::endl; + } + } } } diff --git a/src/platform_info.cpp b/src/platform_info.cpp new file mode 100644 index 0000000..610af4d --- /dev/null +++ b/src/platform_info.cpp @@ -0,0 +1,88 @@ +#include "platform_info.hpp" + +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#elif defined(__APPLE__) +#include +#include +#elif defined(__ANDROID__) +#include +#else +#include +#endif + +namespace cly { +namespace platform { +std::string osName() { +#if defined(_WIN32) + return "Windows"; +#elif defined(__APPLE__) +#if defined(TARGET_OS_VISION) && TARGET_OS_VISION + return "visionOS"; +#elif TARGET_OS_WATCH + return "watchOS"; +#elif TARGET_OS_TV + return "tvOS"; +#elif TARGET_OS_IPHONE + return "iOS"; +#else + return "macOS"; +#endif +#elif defined(__ANDROID__) + return "Android"; +#elif defined(__linux__) + return "Linux"; +#elif defined(__unix__) + return "Unix"; +#else + return "Unknown"; +#endif +} + +std::string osVersion() { +#if defined(_WIN32) + // GetVersionEx reports the version the app is manifested for; RtlGetVersion reports the real one + typedef LONG(WINAPI * RtlGetVersionFunction)(PRTL_OSVERSIONINFOW); + HMODULE ntdll = GetModuleHandleW(L"ntdll.dll"); + if (ntdll == nullptr) { + return std::string(); + } + RtlGetVersionFunction getVersion = reinterpret_cast(reinterpret_cast(GetProcAddress(ntdll, "RtlGetVersion"))); + if (getVersion == nullptr) { + return std::string(); + } + RTL_OSVERSIONINFOW info = {}; + info.dwOSVersionInfoSize = sizeof(info); + if (getVersion(&info) != 0) { + return std::string(); + } + return std::to_string(info.dwMajorVersion) + "." + std::to_string(info.dwMinorVersion) + "." + std::to_string(info.dwBuildNumber); +#elif defined(__APPLE__) + char version[64] = {0}; + size_t size = sizeof(version); + if (sysctlbyname("kern.osproductversion", version, &size, nullptr, 0) != 0) { + return std::string(); + } + return std::string(version); +#elif defined(__ANDROID__) + char version[PROP_VALUE_MAX] = {0}; + if (__system_property_get("ro.build.version.release", version) <= 0) { + return std::string(); + } + return std::string(version); +#else + struct utsname name; + if (uname(&name) != 0) { + return std::string(); + } + return std::string(name.release); +#endif +} +} // namespace platform +} // namespace cly diff --git a/src/platform_info.hpp b/src/platform_info.hpp new file mode 100644 index 0000000..ff08bd3 --- /dev/null +++ b/src/platform_info.hpp @@ -0,0 +1,22 @@ +#ifndef COUNTLY_PLATFORM_INFO_HPP_ +#define COUNTLY_PLATFORM_INFO_HPP_ + +#include + +namespace cly { +namespace platform { +/** + * The operating system the SDK was built for, named as the server expects it in the "_os" metric, for example + * "Windows", "macOS", "Linux" or "Android". + */ +std::string osName(); + +/** + * The version of the running operating system without its name, for example "10.0.19045" on Windows or "14.4" on + * macOS. On Linux and other Unix systems it is the kernel release. Empty when it can't be read. + */ +std::string osVersion(); +} // namespace platform +} // namespace cly + +#endif diff --git a/src/request_builder.cpp b/src/request_builder.cpp index b13c2c7..ad83af3 100644 --- a/src/request_builder.cpp +++ b/src/request_builder.cpp @@ -46,11 +46,17 @@ std::string RequestBuilder::buildRequest(const std::map request = {{"app_key", _configuration->appKey}, {"device_id", _configuration->deviceId}, + {"t", std::to_string(static_cast(_configuration->deviceIdType))}, {"timestamp", std::to_string(timestamp.count())}, {"dow", std::to_string(local_tm.tm_wday)}, {"hour", std::to_string(local_tm.tm_hour)}, {"tz", std::to_string(tz_offset_minutes)}}; + nlohmann::json::const_iterator appVersion = _configuration->metrics.find("_app_version"); + if (appVersion != _configuration->metrics.end() && appVersion->is_string() && !appVersion->get().empty()) { + request["av"] = appVersion->get(); + } + request.insert(data.begin(), data.end()); return buildFetchRequest(request); } diff --git a/src/request_module.cpp b/src/request_module.cpp index ad91110..7bad6cc 100644 --- a/src/request_module.cpp +++ b/src/request_module.cpp @@ -2,6 +2,7 @@ #include "countly/health_check_module.hpp" #include "countly/request_builder.hpp" +#include #include #include #include @@ -13,6 +14,9 @@ #ifndef COUNTLY_USE_CUSTOM_HTTP #ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif #include "Windows.h" #include "WinHTTP.h" #undef ERROR @@ -34,6 +38,10 @@ class RequestModule::RequestModuleImpl { public: bool use_https = true; bool is_queue_being_processed = false; + // Both guarded by the instance mutex: no queued request is sent before them. + std::chrono::steady_clock::time_point next_attempt_after_failure; + std::chrono::steady_clock::time_point backoff_ends; + std::function on_merge_delivered; std::shared_ptr _configuration; std::shared_ptr _logger; std::shared_ptr _requestBuilder; @@ -89,6 +97,48 @@ class RequestModule::RequestModuleImpl { }; namespace { +/** + * The "timestamp" parameter of a stored request in milliseconds, or -1 when the request has none. + */ +long long requestTimestamp(const std::string &request) { + std::string::size_type position = 0; + if (request.compare(0, 10, "timestamp=") != 0) { + position = request.find("×tamp="); + if (position == std::string::npos) { + return -1; + } + position += 1; + } + position += 10; + const std::string::size_type end = request.find('&', position); + const std::string value = request.substr(position, end == std::string::npos ? std::string::npos : end - position); + if (value.empty() || value.size() > 18 || value.find_first_not_of("0123456789") != std::string::npos) { + return -1; + } + return std::stoll(value); +} + +/** + * The current time in milliseconds since the epoch, the unit of the "timestamp" parameter. + */ +long long nowMilliseconds() { return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); } + +/** + * Whether the request is older than the drop age. A request without a timestamp is never too old. + */ +bool isOlderThan(const std::string &request, unsigned int dropAgeHours) { + if (dropAgeHours == 0) { + return false; + } + const long long timestamp = requestTimestamp(request); + return timestamp >= 0 && nowMilliseconds() - timestamp > static_cast(dropAgeHours) * 3600LL * 1000LL; +} + +/** + * Whether the answer to a queued request means it was delivered: a 2xx status and a JSON object with a "result". + */ +bool isDelivered(const HTTPResponse &response) { return response.success && response.data.is_object() && response.data.contains("result"); } + /** * Process-wide libcurl initialisation. * @@ -183,14 +233,34 @@ void RequestModule::addRequestToQueue(const std::map & } if (config->isTrackingEnabled() == false) { - impl->_logger->log(LogLevel::DEBUG, "[Countly] [RequestModule] addRequestToQueue: Tracking is disabled. Not adding request to queue."); + impl->_logger->log(LogLevel::WARNING, "[Countly] [RequestModule] addRequestToQueue: The SDK behavior settings turned tracking off, so the request is not recorded."); + impl->_logger->recordLogCode("w505"); return; } - if (config->getRequestQueueSizeLimit() <= impl->_storageModule->RQCount()) { - impl->_logger->log(LogLevel::WARNING, cly::utils::format_string("[Countly] [RequestModule] addRequestToQueue: Request Queue is full. Dropping the oldest request.")); - impl->_logger->recordLogCode("w310"); - impl->_storageModule->RQRemoveFront(); + const long long limit = config->getRequestQueueSizeLimit(); + long long count = impl->_storageModule->RQCount(); + if (count >= limit) { + const unsigned int dropAgeHours = config->getRequestDropAgeHours(); + while (count > 0) { + std::shared_ptr front = impl->_storageModule->RQPeekFront(); + if (front->getId() < 0 || !isOlderThan(front->getData(), dropAgeHours)) { + break; + } + impl->_storageModule->RQRemoveFront(front); + impl->_logger->log(LogLevel::WARNING, "[Countly] [RequestModule] addRequestToQueue: A request older than the drop age of [" + std::to_string(dropAgeHours) + "] hours is removed without being sent."); + impl->_logger->recordLogCode("w330"); + count--; + } + } + if (count >= limit) { + const long long overflow = count - limit; + const long long toRemove = (impl->_configuration->gradualRequestCleanerEnabled ? std::min(overflow, 100LL) : overflow) + 1; + impl->_logger->log(LogLevel::WARNING, "[Countly] [RequestModule] addRequestToQueue: Request Queue is full. Dropping the oldest [" + std::to_string(toRemove) + "] request(s)."); + for (long long index = 0; index < toRemove; index++) { + impl->_storageModule->RQRemoveFront(); + impl->_logger->recordLogCode("w310"); + } } const std::string request = impl->_requestBuilder->buildRequest(data); @@ -200,30 +270,37 @@ void RequestModule::addRequestToQueue(const std::map & void RequestModule::clearRequestQueue() { impl->_storageModule->RQClearAll(); } void RequestModule::processQueue(std::shared_ptr mutex) { + std::shared_ptr config; { // lock_guard so the mutex is released on every exit path of this block, // including early returns and any exception thrown while it is held. std::lock_guard lk(*mutex); - if (std::shared_ptr config = _configProvider.lock()) { - if (config->isTrackingEnabled() == false) { - impl->_logger->log(LogLevel::DEBUG, "[Countly] [RequestModule] processQueue: Tracking is disabled. Not processing request queue."); - return; - } - if (config->isNetworkingEnabled() == false) { - impl->_logger->log(LogLevel::DEBUG, "[Countly] [RequestModule] processQueue: Networking is disabled. Not processing request queue."); - return; - } - } else { + config = _configProvider.lock(); + if (!config) { impl->_logger->log(LogLevel::WARNING, "[Countly] [RequestModule] processQueue: ConfigurationProvider unavailable, skipping queue processing."); return; } + if (config->isNetworkingEnabled() == false) { + impl->_logger->log(LogLevel::DEBUG, "[Countly] [RequestModule] processQueue: Networking is disabled. Not processing request queue."); + return; + } // making sure that no other thread is processing the queue if (impl->is_queue_being_processed) { return; } + const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); + if (now < impl->next_attempt_after_failure) { + impl->_logger->log(LogLevel::DEBUG, "[Countly] [RequestModule] processQueue: Waiting for the cool-down after a failed request."); + return; + } + if (now < impl->backoff_ends) { + impl->_logger->log(LogLevel::DEBUG, "[Countly] [RequestModule] processQueue: Waiting for the backoff to end."); + return; + } + // if this is the only thread, mark that processing is happening impl->is_queue_being_processed = true; } @@ -239,10 +316,12 @@ void RequestModule::processQueue(std::shared_ptr mutex) { try { while (true) { std::shared_ptr data; + long long behind = 0; { std::lock_guard lk(*mutex); impl->_logger->log(LogLevel::DEBUG, cly::utils::format_string("[Countly] [RequestModule] processQueue: Processing the request queue.")); - if (impl->_storageModule->RQCount() == 0) { + const long long count = impl->_storageModule->RQCount(); + if (count <= 0) { impl->_logger->log(LogLevel::DEBUG, cly::utils::format_string("[Countly] [RequestModule] processQueue: Queue is empty.")); // stop sending requests once the queue is empty @@ -250,17 +329,29 @@ void RequestModule::processQueue(std::shared_ptr mutex) { } data = impl->_storageModule->RQPeekFront(); + if (isOlderThan(data->getData(), config->getRequestDropAgeHours())) { + impl->_storageModule->RQRemoveFront(data); + impl->_logger->log(LogLevel::WARNING, "[Countly] [RequestModule] processQueue: A request older than the drop age of [" + std::to_string(config->getRequestDropAgeHours()) + "] hours is removed without being sent."); + impl->_logger->recordLogCode("w330"); + continue; + } + behind = count - 1; } - HTTPResponse response = sendHTTP("/i", data->getData()); + + bool mergeDelivered = false; + const std::chrono::steady_clock::time_point sent = std::chrono::steady_clock::now(); + HTTPResponse response = sendHTTP("/i", data->getData() + "&rr=" + std::to_string(behind)); + const std::chrono::steady_clock::duration responseTime = std::chrono::steady_clock::now() - sent; { std::lock_guard lk(*mutex); - if (!response.success) { - impl->_logger->log(LogLevel::DEBUG, cly::utils::format_string("[Countly] [RequestModule] processQueue: Failed to deliver to server, will try again later.")); - std::shared_ptr healthCheck = impl->_logger->healthCheck(); + std::shared_ptr healthCheck = impl->_logger->healthCheck(); + if (!isDelivered(response)) { + impl->_logger->log(LogLevel::DEBUG, "[Countly] [RequestModule] processQueue: Failed to deliver to server, will try again after the cool-down of [" + std::to_string(impl->_configuration->failedRequestCooldownSeconds) + "] seconds."); if (healthCheck && response.status_code > 0) { healthCheck->logFailedNetworkRequest(response.status_code, !response.body.empty() || response.data.is_null() ? response.body : response.data.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace)); } + impl->next_attempt_after_failure = std::chrono::steady_clock::now() + std::chrono::seconds(impl->_configuration->failedRequestCooldownSeconds); // if the request was not a success, abort sending and try again in the future break; } @@ -269,12 +360,34 @@ void RequestModule::processQueue(std::shared_ptr mutex) { // the queue might have changed while we were sending the request impl->_storageModule->RQRemoveFront(data); processedRequestsCounter++; + mergeDelivered = data->getData().find("old_device_id=") != std::string::npos; + + if (config->isBackoffEnabled()) { + const long long timestamp = requestTimestamp(data->getData()); + const bool slow = responseTime >= std::chrono::seconds(config->getBackoffAcceptedTimeout()); + const bool queueNotFull = impl->_storageModule->RQCount() <= config->getBackoffRequestQueuePercentage() * config->getRequestQueueSizeLimit(); + const bool recent = timestamp >= 0 && nowMilliseconds() - timestamp <= static_cast(config->getBackoffRequestAge()) * 3600LL * 1000LL; + if (slow && queueNotFull && recent) { + impl->backoff_ends = std::chrono::steady_clock::now() + std::chrono::seconds(config->getBackoffDuration()); + impl->_logger->log(LogLevel::WARNING, "[Countly] [RequestModule] processQueue: The server answered slowly, so sending pauses for [" + std::to_string(config->getBackoffDuration()) + "] seconds."); + if (healthCheck) { + healthCheck->logBackoffRequest(); + } + break; + } + if (healthCheck) { + healthCheck->logConsecutiveBackoffRequest(); + } + } if (processedRequestsCounter > impl->_configuration->maxProcessingBatchSize) { impl->_logger->log(LogLevel::DEBUG, cly::utils::format_string("[Countly] [RequestModule] processQueue: Batch limit has been reached, will do next batch later.")); break; } } + if (mergeDelivered && impl->on_merge_delivered) { + impl->on_merge_delivered(); + } } } catch (...) { std::lock_guard lk(*mutex); @@ -289,8 +402,10 @@ void RequestModule::processQueue(std::shared_ptr mutex) { impl->is_queue_being_processed = false; } +HTTPResponse RequestModule::sendImmediate(const std::string &path, const std::string &data) { return sendHTTP(path, data + "&rr=" + std::to_string(std::max(0LL, RQSize()))); } + HTTPResponse RequestModule::sendHTTP(std::string path, std::string data) { - bool use_post = impl->_configuration->forcePost || (data.size() > COUNTLY_POST_THRESHOLD); + const bool use_post = true; impl->_logger->log(LogLevel::DEBUG, "[Countly] [RequestModule] sendHTTP, data: [" + data + "]"); if (!impl->_configuration->salt.empty()) { std::string checksum = impl->calculateChecksum(impl->_configuration->salt, data); @@ -341,22 +456,17 @@ HTTPResponse RequestModule::sendHTTP(std::string path, std::string data) { } if (hConnect) { - if (!use_post) { - path += '?'; - path += data; - } - size_t buffer_size = MultiByteToWideChar(CP_ACP, 0, path.c_str(), -1, nullptr, 0); wchar_t *wide_path = new wchar_t[buffer_size]; MultiByteToWideChar(CP_ACP, 0, path.c_str(), -1, wide_path, static_cast(buffer_size)); - hRequest = WinHttpOpenRequest(hConnect, use_post ? L"POST" : L"GET", wide_path, NULL, WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES, impl->use_https ? WINHTTP_FLAG_SECURE : 0); + hRequest = WinHttpOpenRequest(hConnect, L"POST", wide_path, NULL, WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES, impl->use_https ? WINHTTP_FLAG_SECURE : 0); delete[] wide_path; } if (hRequest) { - LPCWSTR headers = use_post ? L"content-type:application/x-www-form-urlencoded" : WINHTTP_NO_ADDITIONAL_HEADERS; - bool ok = WinHttpSendRequest(hRequest, headers, 0, use_post ? (LPVOID)data.data() : WINHTTP_NO_REQUEST_DATA, use_post ? data.size() : 0, use_post ? data.size() : 0, 0) != 0; + LPCWSTR headers = L"content-type:application/x-www-form-urlencoded"; + bool ok = WinHttpSendRequest(hRequest, headers, 0, (LPVOID)data.data(), static_cast(data.size()), static_cast(data.size()), 0) != 0; if (ok) { ok = WinHttpReceiveResponse(hRequest, NULL); if (ok) { @@ -425,12 +535,7 @@ HTTPResponse RequestModule::sendHTTP(std::string path, std::string data) { std::ostringstream full_url_stream; full_url_stream << impl->_configuration->serverUrl << ':' << std::dec << impl->_configuration->port << path; - if (!use_post) { - full_url_stream << '?' << data; - curl_easy_setopt(curl, CURLOPT_HTTPGET, 1); - } else { - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.c_str()); - } + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.c_str()); impl->_logger->log(LogLevel::DEBUG, "[Countly] [RequestModule] sendHTTP, request: [" + full_url_stream.str() + "]"); @@ -473,4 +578,6 @@ long long RequestModule::RQSize() { return impl->_storageModule->RQCount(); } void RequestModule::setConfigurationProvider(std::weak_ptr provider) { _configProvider = std::move(provider); } +void RequestModule::setOnMergeDelivered(std::function callback) { impl->on_merge_delivered = std::move(callback); } + } // namespace cly diff --git a/src/storage_module_db.cpp b/src/storage_module_db.cpp index 3f11938..c1d4938 100644 --- a/src/storage_module_db.cpp +++ b/src/storage_module_db.cpp @@ -19,6 +19,10 @@ const char REQUESTS_TABLE_REQUEST_DATA[] = "RequestData"; #define HEALTH_CHECK_KEY_COLUMN_NAME "Key" #define HEALTH_CHECK_DATA_COLUMN_NAME "StateData" #define HEALTH_CHECK_KEY_VALUE 1 +#define DEVICE_ID_TABLE_NAME "DeviceID" +#define DEVICE_ID_KEY_COLUMN_NAME "Key" +#define DEVICE_ID_DATA_COLUMN_NAME "StateData" +#define DEVICE_ID_KEY_VALUE 1 namespace cly { StorageModuleDB::StorageModuleDB(std::shared_ptr config, std::shared_ptr logger) : StorageModuleBase(config, logger) {} @@ -37,9 +41,9 @@ void StorageModuleDB::init() { } #endif - // Create schema for the requests table, the SDK behavior settings table and the health check table + // Create schema for the requests table, the SDK behavior settings table, the health check table and the device ID table _is_initialized = createSchema(REQUESTS_TABLE_NAME, REQUESTS_TABLE_REQUEST_ID, REQUESTS_TABLE_REQUEST_DATA) && createSchema(SDK_BEHAVIOR_SETTINGS_TABLE_NAME, SDK_BEHAVIOR_SETTINGS_KEY_COLUMN_NAME, SDK_BEHAVIOR_SETTINGS_DATA_COLUMN_NAME) && - createSchema(HEALTH_CHECK_TABLE_NAME, HEALTH_CHECK_KEY_COLUMN_NAME, HEALTH_CHECK_DATA_COLUMN_NAME); + createSchema(HEALTH_CHECK_TABLE_NAME, HEALTH_CHECK_KEY_COLUMN_NAME, HEALTH_CHECK_DATA_COLUMN_NAME) && createSchema(DEVICE_ID_TABLE_NAME, DEVICE_ID_KEY_COLUMN_NAME, DEVICE_ID_DATA_COLUMN_NAME); if (_is_initialized) { vacuumDatabase(); @@ -657,4 +661,93 @@ std::string StorageModuleDB::getHealthCheckState() { } } +void StorageModuleDB::storeDeviceIdState(const std::string &state) { + try { + if (!_is_initialized) { + _logger->log(LogLevel::DEBUG, "[Countly] [StorageModuleDB] storeDeviceIdState, Module is not initialized"); + return; + } + +#ifdef COUNTLY_USE_SQLITE + sqlite3 *db = nullptr; + sqlite3_stmt *stmt = nullptr; + + if (cly::utils::openDatabase(_configuration->databasePath, &db) != SQLITE_OK) { + _logger->log(LogLevel::ERROR, "[Countly] [StorageModuleDB] storeDeviceIdState, Failed to open database"); + _logger->recordLogCode("e501"); + sqlite3_close(db); + return; + } + + const char *sql = "INSERT OR REPLACE INTO " DEVICE_ID_TABLE_NAME " (" DEVICE_ID_KEY_COLUMN_NAME ", " DEVICE_ID_DATA_COLUMN_NAME ") VALUES (?, ?);"; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr) != SQLITE_OK) { + _logger->log(LogLevel::ERROR, "[Countly] [StorageModuleDB] storeDeviceIdState, Failed to prepare statement"); + _logger->recordLogCode("e501"); + sqlite3_close(db); + return; + } + + sqlite3_bind_int(stmt, 1, DEVICE_ID_KEY_VALUE); + sqlite3_bind_text(stmt, 2, state.c_str(), -1, SQLITE_TRANSIENT); + if (sqlite3_step(stmt) != SQLITE_DONE) { + _logger->log(LogLevel::ERROR, std::string("[Countly] [StorageModuleDB] storeDeviceIdState, failed: [") + sqlite3_errmsg(db) + "]"); + _logger->recordLogCode("e501"); + } + + sqlite3_finalize(stmt); + sqlite3_close(db); +#else + (void)state; +#endif + } catch (const std::exception &e) { + _logger->log(LogLevel::ERROR, std::string("[Countly] [StorageModuleDB] storeDeviceIdState, exception: [") + e.what() + "]"); + _logger->recordLogCode("e501"); + } +} + +std::string StorageModuleDB::getDeviceIdState() { + try { + if (!_is_initialized) { + _logger->log(LogLevel::DEBUG, "[Countly] [StorageModuleDB] getDeviceIdState, Module is not initialized"); + return ""; + } + +#ifdef COUNTLY_USE_SQLITE + sqlite3 *db = nullptr; + sqlite3_stmt *stmt = nullptr; + std::string result; + + if (cly::utils::openDatabase(_configuration->databasePath, &db) != SQLITE_OK) { + _logger->log(LogLevel::ERROR, "[Countly] [StorageModuleDB] getDeviceIdState, Failed to open database"); + sqlite3_close(db); + return ""; + } + + const char *sql = "SELECT " DEVICE_ID_DATA_COLUMN_NAME " FROM " DEVICE_ID_TABLE_NAME " WHERE " DEVICE_ID_KEY_COLUMN_NAME " = ?;"; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr) == SQLITE_OK) { + sqlite3_bind_int(stmt, 1, DEVICE_ID_KEY_VALUE); + if (sqlite3_step(stmt) == SQLITE_ROW) { + const unsigned char *text = sqlite3_column_text(stmt, 0); + if (text) { + result = reinterpret_cast(text); + } + } + } else { + _logger->log(LogLevel::ERROR, "[Countly] [StorageModuleDB] getDeviceIdState, Failed to prepare statement"); + } + + if (stmt) { + sqlite3_finalize(stmt); + } + sqlite3_close(db); + return result; +#else + return ""; +#endif + } catch (const std::exception &e) { + _logger->log(LogLevel::ERROR, std::string("[Countly] [StorageModuleDB] getDeviceIdState, exception: [") + e.what() + "]"); + return ""; + } +} + }; // namespace cly \ No newline at end of file diff --git a/src/views_module.cpp b/src/views_module.cpp index 30bec9f..9ff21bd 100644 --- a/src/views_module.cpp +++ b/src/views_module.cpp @@ -1,43 +1,60 @@ #include "countly/views_module.hpp" #include "countly/consent_module.hpp" +#include "countly/event.hpp" #include "countly/internal_limits.hpp" #include #include +#include #include #define CLY_VIEW_KEY "[CLY]_view" namespace cly { +namespace { +/** + * Segmentation keys of view events that custom segmentation can't replace. + */ +const std::set RESERVED_VIEW_KEYS = {"name", "visit", "start", "segment", "bounce", "exit", "view", "domain", "dur", "_idv"}; +} // namespace + class ViewsModule::ViewModuleImpl { struct ViewInfo { public: std::string name; std::string viewId; + // The view that was current when this one started, sent as "pvid" with this view's events + std::string previousViewId; + // Start order, so that the last started view is the one closed by name + unsigned long long sequence = 0; // steady_clock, so a system clock change can't distort the view duration std::chrono::steady_clock::time_point startTime; }; private: - // Guards _isFirstView and _viewsStartTime. Views are opened and closed from - // whichever thread the integrator uses, and nothing else in the SDK - // serialises those calls, so the module has to do it itself. + // Guards every member below. Views are opened and closed from whichever + // thread the integrator uses, and nothing else in the SDK serialises those + // calls, so the module has to do it itself. std::mutex _views_mutex; bool _isFirstView = true; + std::string _currentViewId; + unsigned long long _sequence = 0; std::map> _viewsStartTime; + std::weak_ptr _configProvider; cly::CountlyDelegates *_cly; // Call with _views_mutex held. std::shared_ptr findViewByName(const std::string &name) { + std::shared_ptr found; for (auto &x : _viewsStartTime) { - if (x.second->name == name) { - return x.second; + if (x.second->name == name && (found == nullptr || x.second->sequence > found->sequence)) { + found = x.second; } } - return nullptr; + return found; } /** @@ -47,11 +64,12 @@ class ViewsModule::ViewModuleImpl { * lock across it would risk a deadlock, so the shared state is read and * updated before this runs and the outcome is passed in. * + * @param segmentation: custom segmentation, already limited and without reserved keys * @param isFirstView: only meaningful when isOpenView is true */ void _recordView(std::shared_ptr v, const std::map &segmentation, bool isOpenView, bool isFirstView = false) { double duration = 0; - std::map viewSegments; + std::map viewSegments = segmentation; if (isOpenView) { viewSegments["visit"] = "1"; @@ -59,15 +77,6 @@ class ViewsModule::ViewModuleImpl { if (isFirstView) { viewSegments["start"] = "1"; } - - for (auto key_value : segmentation) { - auto itr = viewSegments.find(key_value.first); - if (itr != viewSegments.end()) { - (*itr).second = key_value.second; - } else { - viewSegments[key_value.first] = key_value.second; - } - } } else { const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); @@ -77,16 +86,53 @@ class ViewsModule::ViewModuleImpl { viewSegments["_idv"] = v->viewId; viewSegments["name"] = v->name; + const std::string platform = _cly->PlatformName(); + if (!platform.empty()) { + viewSegments["segment"] = platform; + } - _cly->RecordEvent(CLY_VIEW_KEY, viewSegments, 1, 0, duration); + cly::Event event(CLY_VIEW_KEY, 1, 0, duration); + for (const auto &key_value : viewSegments) { + event.addSegmentation(key_value.first, key_value.second); + } + ViewsModule::setEventField(event, "id", v->viewId); + ViewsModule::setEventField(event, "pvid", v->previousViewId); + _cly->addEvent(event); } public: std::shared_ptr _logger; - std::weak_ptr _configProvider; std::shared_ptr _consent; ViewModuleImpl(cly::CountlyDelegates *cly, std::shared_ptr logger) : _cly(cly), _logger(logger) {} + /** + * The SDK behavior settings in use, empty before start() wires the module up. + */ + std::shared_ptr provider() { + std::lock_guard lk(_views_mutex); + return _configProvider.lock(); + } + + /** + * Sets the SDK behavior settings in use. + */ + void setProvider(std::weak_ptr configProvider) { + std::lock_guard lk(_views_mutex); + _configProvider = std::move(configProvider); + } + + /** + * Whether start() has wired the module up, logged as a warning and counted as w101 when it has not. + */ + bool isInitialized(const char *caller) { + if (provider()) { + return true; + } + _logger->log(LogLevel::WARNING, std::string("[Countly] [ViewsModule] ") + caller + ", SDK is not initialized."); + _logger->recordLogCode("w101"); + return false; + } + ~ViewModuleImpl() { _logger.reset(); } /** @@ -100,10 +146,27 @@ class ViewsModule::ViewModuleImpl { return false; } + /** + * Custom segmentation without the reserved view keys, which are logged as a warning and ignored, and with the + * internal limits applied. + */ + std::map customSegmentation(const std::map &segmentation, const SDKLimits &lim, const std::string &caller) { + std::map custom; + for (const auto &key_value : segmentation) { + if (RESERVED_VIEW_KEYS.count(key_value.first)) { + _logger->log(LogLevel::WARNING, "[Countly] [ViewsModule] " + caller + ", [" + key_value.first + "] is a reserved view segmentation key and is ignored."); + _logger->recordLogCode("w700"); + continue; + } + custom[key_value.first] = key_value.second; + } + return cly::limits::limitSegmentation(custom, lim, _logger.get(), "[ViewsModule] " + caller); + } + /** * Takes every running view out of the map and records its end event, or drops them while view tracking is off. */ - void _stopAllViews() { + void _stopAllViews(const std::map &segmentation, const std::string &caller) { std::vector> running; { std::lock_guard lk(_views_mutex); @@ -113,19 +176,20 @@ class ViewsModule::ViewModuleImpl { _viewsStartTime.clear(); } - std::shared_ptr config = _configProvider.lock(); + std::shared_ptr config = provider(); if (!config || config->isViewTrackingEnabled() == false) { _logger->log(LogLevel::DEBUG, "[Countly] [ViewsModule] _stopAllViews, View tracking is disabled, the running views are dropped."); return; } + const std::map custom = customSegmentation(segmentation, config->getLimits(), caller); for (const std::shared_ptr &view : running) { - _recordView(view, {}, false); + _recordView(view, custom, false); } } std::string _openView(const std::string &name, const std::map &segmentation) { SDKLimits lim{COUNTLY_MAX_KEY_LENGTH_DEFAULT, COUNTLY_MAX_VALUE_SIZE_DEFAULT, COUNTLY_MAX_SEGMENTATION_VALUES_DEFAULT, COUNTLY_MAX_BREADCRUMB_COUNT_DEFAULT, COUNTLY_MAX_STACK_TRACE_LINES_PER_THREAD_DEFAULT, COUNTLY_MAX_STACK_TRACE_LINE_LENGTH_DEFAULT}; - if (std::shared_ptr config = _configProvider.lock()) { + if (std::shared_ptr config = provider()) { if (config->isViewTrackingEnabled() == false) { _logger->log(LogLevel::DEBUG, "[Countly] [ViewsModule] _openView, View tracking is disabled. Not opening view."); return ""; @@ -138,40 +202,45 @@ class ViewsModule::ViewModuleImpl { // A view name is a "key" per the guide -> maxKeyLength. Developer segmentation // is limited before internal keys (visit/start/_idv/name) are merged. - std::string limitedName = cly::limits::truncateString(name, lim.maxKeyLength); - std::map limitedSeg = cly::limits::applySegmentationLimits(segmentation, lim); + const std::string limitedName = cly::limits::limitKey(name, lim.maxKeyLength, _logger.get(), "[ViewsModule] openView"); + const std::map limitedSeg = customSegmentation(segmentation, lim, "openView"); + // Asked before taking the view lock: the answer takes the instance mutex + const bool sessionRunning = _cly->IsSessionRunning(); - ViewModuleImpl::ViewInfo *v = new ViewModuleImpl::ViewInfo(); - v->name = limitedName; - v->viewId = cly::utils::generateEventID(); - v->startTime = std::chrono::steady_clock::now(); - - std::shared_ptr ptr(v); + std::shared_ptr ptr = std::make_shared(); + ptr->name = limitedName; + ptr->viewId = cly::utils::generateEventID(); + ptr->startTime = std::chrono::steady_clock::now(); bool isFirstView = false; { std::lock_guard lk(_views_mutex); + ptr->previousViewId = _currentViewId; + ptr->sequence = ++_sequence; + _currentViewId = ptr->viewId; _viewsStartTime[ptr->viewId] = ptr; // Claimed under the lock, so 'start' goes on exactly one view even when // two threads open their first view at the same time. - isFirstView = _isFirstView; - _isFirstView = false; + isFirstView = _isFirstView && sessionRunning; + if (isFirstView) { + _isFirstView = false; + } } _recordView(ptr, limitedSeg, true, isFirstView); return ptr->viewId; } - void _closeViewWithName(const std::string &name) { - if (std::shared_ptr config = _configProvider.lock()) { - if (config->isViewTrackingEnabled() == false) { - _logger->log(LogLevel::DEBUG, "[Countly] [ViewsModule] _closeViewWithName, View tracking is disabled. Not closing view."); - return; - } - } else { + void _closeViewWithName(const std::string &name, const std::map &segmentation) { + std::shared_ptr config = provider(); + if (!config) { _logger->log(LogLevel::WARNING, "[Countly] [ViewsModule] _closeViewWithName, ConfigurationProvider unavailable."); return; } + if (config->isViewTrackingEnabled() == false) { + _logger->log(LogLevel::DEBUG, "[Countly] [ViewsModule] _closeViewWithName, View tracking is disabled. Not closing view."); + return; + } // The view is taken out of the map under the lock, so two threads closing // the same view cannot both record it. std::shared_ptr v; @@ -190,19 +259,19 @@ class ViewsModule::ViewModuleImpl { _logger->recordLogCode("w730"); return; } - _recordView(v, {}, false); + _recordView(v, customSegmentation(segmentation, config->getLimits(), "closeViewWithName"), false); } - void _closeViewWithID(const std::string &viewId) { - if (std::shared_ptr config = _configProvider.lock()) { - if (config->isViewTrackingEnabled() == false) { - _logger->log(LogLevel::DEBUG, "[Countly] [ViewsModule] _closeViewWithID, View tracking is disabled. Not closing view."); - return; - } - } else { + void _closeViewWithID(const std::string &viewId, const std::map &segmentation) { + std::shared_ptr config = provider(); + if (!config) { _logger->log(LogLevel::WARNING, "[Countly] [ViewsModule] _closeViewWithID, ConfigurationProvider unavailable."); return; } + if (config->isViewTrackingEnabled() == false) { + _logger->log(LogLevel::DEBUG, "[Countly] [ViewsModule] _closeViewWithID, View tracking is disabled. Not closing view."); + return; + } std::shared_ptr v; { @@ -222,7 +291,23 @@ class ViewsModule::ViewModuleImpl { return; } - _recordView(v, {}, false); + _recordView(v, customSegmentation(segmentation, config->getLimits(), "closeViewWithID"), false); + } + + void resetFirstView() { + std::lock_guard lk(_views_mutex); + _isFirstView = true; + } + + void forgetViews() { + std::lock_guard lk(_views_mutex); + _currentViewId.clear(); + _isFirstView = true; + } + + std::string currentViewId() { + std::lock_guard lk(_views_mutex); + return _currentViewId; } }; @@ -238,7 +323,7 @@ std::string ViewsModule::openView(const std::string &name, const std::map_logger->log(cly::LogLevel::INFO, cly::utils::format_string("[Countly] [ViewsModule] openView, name = [%s], segmentation = [%s]", name.c_str(), utils::mapToString(segmentation).c_str())); - if (!impl->hasConsent("openView")) { + if (!impl->isInitialized("openView") || !impl->hasConsent("openView")) { return {}; } impl->_logger->recordUsage("views", "start"); @@ -251,10 +336,10 @@ std::string ViewsModule::openView(const std::string &name, const std::map_openView(name, segmentation); } -void ViewsModule::closeViewWithName(const std::string &name) { +void ViewsModule::closeViewWithName(const std::string &name, const std::map &segmentation) { impl->_logger->log(cly::LogLevel::INFO, cly::utils::format_string("[Countly] [ViewsModule] closeViewWithName, name = [%s]", name.c_str())); - if (!impl->hasConsent("closeViewWithName")) { + if (!impl->isInitialized("closeViewWithName") || !impl->hasConsent("closeViewWithName")) { return; } impl->_logger->recordUsage("views", "stop"); @@ -263,13 +348,13 @@ void ViewsModule::closeViewWithName(const std::string &name) { impl->_logger->recordLogCode("w700"); return; } - impl->_closeViewWithName(name); + impl->_closeViewWithName(name, segmentation); } -void ViewsModule::closeViewWithID(const std::string &viewId) { +void ViewsModule::closeViewWithID(const std::string &viewId, const std::map &segmentation) { impl->_logger->log(cly::LogLevel::INFO, cly::utils::format_string("[Countly] [ViewsModule] closeViewWithID, viewId = [%s]", viewId.c_str())); - if (!impl->hasConsent("closeViewWithID")) { + if (!impl->isInitialized("closeViewWithID") || !impl->hasConsent("closeViewWithID")) { return; } impl->_logger->recordUsage("views", "stop"); @@ -279,13 +364,31 @@ void ViewsModule::closeViewWithID(const std::string &viewId) { return; } - impl->_closeViewWithID(viewId); + impl->_closeViewWithID(viewId, segmentation); +} + +void ViewsModule::stopAllViews(const std::map &segmentation) { + impl->_logger->log(cly::LogLevel::INFO, "[Countly] [ViewsModule] stopAllViews, segmentation = [" + utils::mapToString(segmentation) + "]"); + + if (!impl->isInitialized("stopAllViews") || !impl->hasConsent("stopAllViews")) { + return; + } + impl->_logger->recordUsage("views", "stopAll"); + impl->_stopAllViews(segmentation, "stopAllViews"); } -void ViewsModule::setConfigurationProvider(std::weak_ptr provider) { impl->_configProvider = std::move(provider); } +void ViewsModule::setConfigurationProvider(std::weak_ptr provider) { impl->setProvider(std::move(provider)); } void ViewsModule::setConsentModule(std::shared_ptr consent) { impl->_consent = std::move(consent); } -void ViewsModule::stopAllViews() { impl->_stopAllViews(); } +void ViewsModule::stopRunningViews() { impl->_stopAllViews({}, "stopRunningViews"); } + +void ViewsModule::resetFirstView() { impl->resetFirstView(); } + +void ViewsModule::forgetViews() { impl->forgetViews(); } + +std::string ViewsModule::currentViewId() { return impl->currentViewId(); } + +void ViewsModule::setEventField(cly::Event &event, const std::string &key, const std::string &value) { event.object[key] = value; } -} // namespace cly \ No newline at end of file +} // namespace cly diff --git a/tests/behavior_settings.cpp b/tests/behavior_settings.cpp new file mode 100644 index 0000000..1202d28 --- /dev/null +++ b/tests/behavior_settings.cpp @@ -0,0 +1,226 @@ +#include +#include +#include +#include +#include +#include + +#include "countly.hpp" +#include "doctest.h" +#include "nlohmann/json.hpp" +#include "test_utils.hpp" + +using namespace cly; +using namespace test_utils; +using json = nlohmann::json; + +namespace { +/** + * Answers the behavior settings fetches with the queued answers, one per fetch, and fails a fetch when none is left. + */ +struct SettingsAnswers { + std::mutex mutex; + std::vector answers; + std::vector fetchedFor; + + void reset(std::vector next) { + std::lock_guard lock(mutex); + answers = std::move(next); + fetchedFor.clear(); + } + + size_t fetches() { + std::lock_guard lock(mutex); + return fetchedFor.size(); + } + + std::vector devices() { + std::lock_guard lock(mutex); + return fetchedFor; + } +}; + +SettingsAnswers settings_answers; + +HTTPResponse settingsClient(bool use_post, const std::string &url, const std::string &data) { + HTTPResponse response = fakeSendHTTP(use_post, url, data); + if (url != "/o/sdk" || data.find("method=sc") == std::string::npos) { + return response; + } + std::string device_id; + const std::string::size_type start = data.find("device_id="); + if (start != std::string::npos) { + device_id = data.substr(start + 10, data.find('&', start) - start - 10); + decodeURL(device_id); + } + std::lock_guard lock(settings_answers.mutex); + settings_answers.fetchedFor.push_back(device_id); + if (settings_answers.answers.empty()) { + return response; + } + json answer = settings_answers.answers.front(); + settings_answers.answers.erase(settings_answers.answers.begin()); + return HTTPResponse{true, answer, 200, answer.dump()}; +} + +bool waitFor(std::function condition) { + const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (std::chrono::steady_clock::now() < deadline) { + if (condition()) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + return condition(); +} + +long long settingsLogCode(Countly &countly, const std::string &code) { + json hc = countly.debugHealthCheck()->snapshot(); + return hc["lc"].contains(code) ? hc["lc"][code].get() : 0; +} + +Countly &prepareSettings(const std::string &provided = "", bool fetchUpdates = true) { + clearSDK(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(settingsClient); + countly.disableHealthCheck(); + countly.enableManualSessionControl(); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + if (!provided.empty()) { + countly.setSDKBehaviorSettings(provided); + } + if (!fetchUpdates) { + countly.disableSDKBehaviorSettingsUpdates(); + } + return countly; +} + +void startSettings(Countly &countly) { countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT); } + +/** + * Records the event, sends it if it was recorded, and tells whether the server received it. + */ +bool recorded(Countly &countly, const std::string &key) { + countly.RecordEvent(key, 1); + countly.flushEvents(); + countly.processRQDebug(); + return httpQueueContainsEvent(key); +} +} // namespace + +TEST_CASE("A setting missing from a new answer goes back to its default") { + settings_answers.reset({json({{"v", 2}, {"t", 1}, {"c", {{"tracking", false}}}}), json({{"v", 2}, {"t", 2}, {"c", json::object()}})}); + Countly &countly = prepareSettings(); + startSettings(countly); + REQUIRE(waitFor([&countly] { + countly.RecordEvent("probe", 1); + return settingsLogCode(countly, "w505") > 0; + })); + + // the device ID change fetches the settings again, and that answer has no "tracking" + countly.setDeviceID("second-user", false); + REQUIRE(waitFor([] { return settings_answers.fetches() == 2; })); + CHECK(settings_answers.devices() == std::vector{COUNTLY_TEST_DEVICE_ID, "second-user"}); + CHECK(waitFor([&countly] { return recorded(countly, "after_reset"); })); +} + +TEST_CASE("After a device ID change with merge the settings are fetched once the merge request was sent") { + settings_answers.reset({}); + Countly &countly = prepareSettings(); + startSettings(countly); + REQUIRE(waitFor([] { return settings_answers.fetches() == 1; })); + + countly.setDeviceID("merged-user", true); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + CHECK(settings_answers.fetches() == 1); + + countly.processRQDebug(); + REQUIRE(waitFor([] { return settings_answers.fetches() == 2; })); + CHECK(settings_answers.devices()[1] == "merged-user"); + bool mergeBeforeFetch = false; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + if (call.data.count("old_device_id")) { + mergeBeforeFetch = true; + } + if (call.data["method"] == "sc" && call.data["device_id"] == "merged-user") { + break; + } + } + CHECK(mergeBeforeFetch); +} + +TEST_CASE("An empty filter list is ignored") { + SUBCASE("an empty blacklist lets the whitelist apply") { + Countly &countly = prepareSettings("{\"eb\":[],\"ew\":[\"allowed\"]}", false); + startSettings(countly); + CHECK(recorded(countly, "allowed")); + CHECK_FALSE(recorded(countly, "not_listed")); + } + + SUBCASE("a blacklist wins when both lists have entries") { + Countly &countly = prepareSettings("{\"eb\":[\"blocked\"],\"ew\":[\"allowed\"]}", false); + startSettings(countly); + CHECK_FALSE(recorded(countly, "blocked")); + CHECK(recorded(countly, "not_listed")); + } +} + +TEST_CASE("Provided settings") { + SUBCASE("are accepted in the format of a server answer") { + Countly &countly = prepareSettings("{\"v\":2,\"t\":1742459739383,\"c\":{\"cet\":false}}", false); + startSettings(countly); + CHECK_FALSE(recorded(countly, "custom_event")); + } + + SUBCASE("are accepted as the settings object alone") { + Countly &countly = prepareSettings("{\"cet\":false}", false); + startSettings(countly); + CHECK_FALSE(recorded(countly, "custom_event")); + } + + SUBCASE("that are not JSON are ignored with a warning") { + Countly &countly = prepareSettings("not json", false); + startSettings(countly); + CHECK(recorded(countly, "custom_event")); + } + +#ifdef COUNTLY_USE_SQLITE + SUBCASE("are not kept for a later start") { + Countly *countly = &prepareSettings("{\"cet\":false}", false); + startSettings(*countly); + CHECK_FALSE(recorded(*countly, "first_run")); + + Countly::halt(); + http_call_queue.clear(); + countly = &Countly::getInstance(); + countly->setHTTPClient(settingsClient); + countly->disableHealthCheck(); + countly->disableSDKBehaviorSettingsUpdates(); + countly->enableManualSessionControl(); + countly->setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly->SetPath(TEST_DATABASE_NAME); + startSettings(*countly); + CHECK(recorded(*countly, "second_run")); + } +#endif +} + +TEST_CASE("disableSDKBehaviorSettingsUpdates sends no settings fetch at all") { + settings_answers.reset({}); + Countly &countly = prepareSettings("", false); + startSettings(countly); + countly.setDeviceID("second-user", false); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + CHECK(settings_answers.fetches() == 0); +} + +TEST_CASE("Server settings with an invalid value are ignored with a warning") { + Countly &countly = prepareSettings("{\"rqs\":0,\"eqs\":\"ten\",\"bom_rqp\":2,\"tracking\":\"yes\",\"lkl\":-5}", false); + startSettings(countly); + CHECK(settingsLogCode(countly, "w121") == 5); + CHECK(recorded(countly, "still_tracked")); +} diff --git a/tests/config.cpp b/tests/config.cpp index d8e648d..cd3a7c5 100644 --- a/tests/config.cpp +++ b/tests/config.cpp @@ -80,12 +80,12 @@ TEST_CASE("Validate setting configuration values") { ct.setMaxRQProcessingBatchSize(10); ct.enableManualSessionControl(); ct.enableImmediateRequestOnStop(); - ct.start("YOUR_APP_KEY", "https://try.count.ly", -1, false); + ct.start(COUNTLY_TEST_APP_KEY, "https://try.count.ly", -1, false); // Get configuration values using Countly getters const CountlyConfiguration config = ct.getConfiguration(); CHECK(config.serverUrl == "https://try.count.ly"); - CHECK(config.appKey == "YOUR_APP_KEY"); + CHECK(config.appKey == COUNTLY_TEST_APP_KEY); CHECK(config.deviceId == "test-device-id"); CHECK(config.salt == "salt"); #ifdef COUNTLY_USE_SQLITE @@ -140,13 +140,13 @@ TEST_CASE("Validate setting configuration values") { ct.SetPath(TEST_DATABASE_NAME); // Server and port - ct.start("YOUR_APP_KEY", "https://try.count.ly", 443, false); + ct.start(COUNTLY_TEST_APP_KEY, "https://try.count.ly", 443, false); CountlyConfiguration config = ct.getConfiguration(); // Validate configuration values CHECK(config.serverUrl == "https://try.count.ly"); - CHECK(config.appKey == "YOUR_APP_KEY"); + CHECK(config.appKey == COUNTLY_TEST_APP_KEY); CHECK(config.deviceId == "test-device-id"); CHECK(config.salt == "salt"); #ifdef COUNTLY_USE_SQLITE @@ -190,7 +190,7 @@ TEST_CASE("Validate setting configuration values") { // get SDK configuration again and make sure that they haven't changed config = ct.getConfiguration(); CHECK(config.serverUrl == "https://try.count.ly"); - CHECK(config.appKey == "YOUR_APP_KEY"); + CHECK(config.appKey == COUNTLY_TEST_APP_KEY); CHECK(config.deviceId == "test-device-id"); CHECK(config.salt == "salt"); #ifdef COUNTLY_USE_SQLITE @@ -224,7 +224,6 @@ TEST_CASE("Validate setting configuration values") { const std::vector> server_urls = { {"https://try.count.ly/", "https://try.count.ly"}, {"https://try.count.ly//", "https://try.count.ly"}, - {"try.count.ly/", "http://try.count.ly"}, {"https://try.count.ly", "https://try.count.ly"}, }; @@ -234,7 +233,7 @@ TEST_CASE("Validate setting configuration values") { ct.setHTTPClient(customClient); ct.setDeviceID("test-device-id"); ct.SetPath(TEST_DATABASE_NAME); - ct.start("YOUR_APP_KEY", server_url.first, 443, false); + ct.start(COUNTLY_TEST_APP_KEY, server_url.first, 443, false); CHECK(ct.getConfiguration().serverUrl == server_url.second); } @@ -253,7 +252,7 @@ TEST_CASE("setDatabasePath is accepted and ignored in builds without SQLite") { CHECK(capturedLogContains("does not include SQLite storage")); ct.setDeviceID("test-device-id"); - ct.start("YOUR_APP_KEY", "https://try.count.ly", 443, false); + ct.start(COUNTLY_TEST_APP_KEY, "https://try.count.ly", 443, false); CHECK(ct.checkEQSize() == 0); CHECK(ct.getConfiguration().databasePath.empty()); ct.setLogger(nullptr); diff --git a/tests/consent.cpp b/tests/consent.cpp index 1cbdb32..4bc639d 100644 --- a/tests/consent.cpp +++ b/tests/consent.cpp @@ -88,7 +88,7 @@ struct ReleaseSettingsOnExit { */ HTTPResponse consentSendHTTP(bool use_post, const std::string &url, const std::string &data) { HTTPResponse response = fakeSendHTTP(use_post, url, data); - if (url == "/o/sdk" && data.find("method=fetch_remote_config") != std::string::npos && data.find("keys=") == std::string::npos) { + if (url == "/o/sdk" && data.find("method=rc") != std::string::npos && data.find("keys=") == std::string::npos) { return HTTPResponse{true, json({{"color", "#FF9900"}, {"playerQueueTimeout", 32}}), 200, ""}; } if (url != "/o/sdk" || data.find("method=sc") == std::string::npos) { @@ -118,13 +118,16 @@ template bool waitUntil(Condition condition) { /** * Sets the instance up for a consent test: the fake client, the test device ID and database, no - * periodic settings fetch, and an update loop that stays out of the test's way and stops at once. + * settings fetch unless the test serves settings, and an update loop that stays out of the test's way + * and stops at once. */ -void prepare(Countly &countly) { +void prepare(Countly &countly, bool fetchSettings = false) { countly.setHTTPClient(consentSendHTTP); countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); countly.SetPath(TEST_DATABASE_NAME); - countly.disableSDKBehaviorSettingsUpdates(); + if (!fetchSettings) { + countly.disableSDKBehaviorSettingsUpdates(); + } countly.enableImmediateRequestOnStop(); countly.setUpdateInterval(60000); } @@ -138,7 +141,7 @@ Countly &freshInstance(const json &settingsAnswer = nullptr, bool holdSettings = clearSDK(); settings_server.reset(settingsAnswer, holdSettings); Countly &countly = Countly::getInstance(); - prepare(countly); + prepare(countly, !settingsAnswer.is_null()); return countly; } @@ -205,7 +208,7 @@ size_t remoteConfigFetches() { size_t count = 0; for (size_t index = 0; index < http_call_queue.size(); index++) { HTTPCall call = http_call_queue.at(index); - if (call.url == "/o/sdk" && valueOf(call, "method") == "fetch_remote_config") { + if (call.url == "/o/sdk" && valueOf(call, "method") == "rc") { count++; } } @@ -343,18 +346,17 @@ TEST_CASE("consent: removing consent stops the feature and runs its clean-up") { countly.addEvent(Event("before", 1)); countly.removeConsent({cly::consent::SESSIONS}); requests = takeRequests(countly); - REQUIRE(requests.size() == 2); - CHECK(valueOf(requests[0], "end_session") == "1"); - CHECK(consentOf(requests[1]) == stateWith(allExcept("sessions"))); + // the events recorded in the session go out before the session ends + REQUIRE(requests.size() == 3); + std::vector events = eventsIn({requests[0]}); + REQUIRE(events.size() == 1); + CHECK(events[0]["key"] == "before"); + CHECK(valueOf(requests[1], "end_session") == "1"); + CHECK(consentOf(requests[2]) == stateWith(allExcept("sessions"))); CHECK_FALSE(countly.updateSession()); countly.flushEvents(std::chrono::seconds(1)); - requests = takeRequests(countly); - REQUIRE(requests.size() == 1); - CHECK(!has(requests[0], "session_duration")); - std::vector events = eventsIn(requests); - REQUIRE(events.size() == 1); - CHECK(events[0]["key"] == "before"); + CHECK(takeRequests(countly).empty()); clearSDK(); } @@ -364,6 +366,8 @@ TEST_CASE("consent: removing consent stops the feature and runs its clean-up") { countly.setRequiresConsent(true); countly.setConsentEnabled(cly::consent::allFeatures()); start(countly); + // with a running session the location goes in a separate request right away + REQUIRE(countly.beginSession()); takeRequests(countly); countly.setLocation("TR", "Istanbul", "41.0082,28.9784", ""); @@ -380,6 +384,8 @@ TEST_CASE("consent: removing consent stops the feature and runs its clean-up") { CHECK(!has(requests[1], "country_code")); CHECK(!has(requests[1], "city")); + REQUIRE(countly.endSession()); + takeRequests(countly); REQUIRE(countly.beginSession()); requests = takeRequests(countly); REQUIRE(requests.size() == 1); @@ -561,19 +567,21 @@ TEST_CASE("consent: the server's consent requirement (cr)") { CHECK(countly.getConsent(cly::consent::EVENTS)); settings_server.release(); - REQUIRE(waitUntil([&countly] { return countly.checkRQSize() == 3; })); + REQUIRE(waitUntil([&countly] { return countly.checkRQSize() == 4; })); requests = takeRequests(countly); - REQUIRE(requests.size() == 3); - CHECK(valueOf(requests[0], "end_session") == "1"); - CHECK(consentOf(requests[1]) == stateWith({})); - CHECK(valueOf(requests[2], "location") == ""); + REQUIRE(requests.size() == 4); + // the events recorded in the session go out before the session ends + std::vector events = eventsIn({requests[0]}); + REQUIRE(events.size() == 1); + CHECK(events[0]["key"] == "before"); + CHECK(valueOf(requests[1], "end_session") == "1"); + CHECK(consentOf(requests[2]) == stateWith({})); + CHECK(valueOf(requests[3], "location") == ""); CHECK_FALSE(countly.getConsent(cly::consent::EVENTS)); countly.addEvent(Event("after", 1)); countly.flushEvents(std::chrono::seconds(1)); - std::vector events = eventsIn(takeRequests(countly)); - REQUIRE(events.size() == 1); - CHECK(events[0]["key"] == "before"); + CHECK(eventsIn(takeRequests(countly)).empty()); clearSDK(); } diff --git a/tests/crash.cpp b/tests/crash.cpp index ffe382a..ebabad1 100644 --- a/tests/crash.cpp +++ b/tests/crash.cpp @@ -75,18 +75,20 @@ TEST_CASE("crash unit tests") { validateCrashParams("Divided By Zero", "stackTrack", true, "", crashMetrics, {}); } - SUBCASE("record crash without title, stackTrace, metrics and bread crumbs") { + SUBCASE("a crash without a stack trace is not recorded, one without title and metrics is") { // clear the request queue, it contains session begin request countly.clearRequestQueue(); - // record fatal exception with empty title and stack trace + // the server needs the stack trace, so a crash without one is ignored countly.crash().recordException("", "", true, {}); - - // start processing request queue countly.processRQDebug(); + CHECK(http_call_queue.size() == 1); - // validate crash request - validateCrashParams("", "", true, "", {}, {}); + // without a title the server uses the first line of the stack trace; the SDK adds the device metrics + countly.crash().recordException("", "stackTrack", true, {}); + countly.processRQDebug(); + validateCrashParams("", "stackTrack", true, "", {}, {}); + CHECK(nlohmann::json::parse(http_call_queue.at(1).data["crash"]).contains("_os")); } SUBCASE("record crash with bread crumbs") { @@ -123,7 +125,7 @@ TEST_CASE("crash unit tests") { // start processing request queue countly.processRQDebug(); - // validate crash request - validateCrashParams("Divided By Zero", "stackTrack", true, "first\nsecond\n", crashMetrics, segmentation, 1); + // validate crash request: the first crash took the first breadcrumb with it + validateCrashParams("Divided By Zero", "stackTrack", true, "second\n", crashMetrics, segmentation, 1); } } diff --git a/tests/crash_reporting.cpp b/tests/crash_reporting.cpp new file mode 100644 index 0000000..a0bd8fe --- /dev/null +++ b/tests/crash_reporting.cpp @@ -0,0 +1,164 @@ +#include +#include +#include + +#include "countly.hpp" +#include "doctest.h" +#include "nlohmann/json.hpp" +#include "test_utils.hpp" + +using namespace cly; +using namespace test_utils; +using json = nlohmann::json; + +namespace { +Countly &prepareCrashes() { + clearSDK(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(fakeSendHTTP); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.enableManualSessionControl(); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + return countly; +} + +void startCrashes(Countly &countly) { countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT); } + +/** + * The crash reports the fake server received, in order. + */ +std::vector sentCrashes(Countly &countly) { + countly.processRQDebug(); + std::vector crashes; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + if (call.data.count("crash")) { + crashes.push_back(json::parse(call.data["crash"])); + } + } + return crashes; +} + +long long crashLogCode(Countly &countly, const std::string &code) { + json hc = countly.debugHealthCheck()->snapshot(); + return hc["lc"].contains(code) ? hc["lc"][code].get() : 0; +} +} // namespace + +TEST_CASE("A crash takes the breadcrumbs recorded before it, and the next crash starts a new trail") { + Countly &countly = prepareCrashes(); + startCrashes(countly); + countly.crash().addBreadcrumb("opened settings"); + countly.crash().recordException("First", "frame 1", false, {{"_os", "Linux"}, {"_app_version", "1.0"}}); + countly.crash().addBreadcrumb("pressed save"); + countly.crash().recordException("Second", "frame 1", false, {{"_os", "Linux"}, {"_app_version", "1.0"}}); + + std::vector crashes = sentCrashes(countly); + REQUIRE(crashes.size() == 2); + CHECK(crashes[0]["_logs"] == "opened settings\n"); + CHECK(crashes[1]["_logs"] == "pressed save\n"); +} + +TEST_CASE("clearBreadcrumbs empties the trail") { + Countly &countly = prepareCrashes(); + startCrashes(countly); + countly.crash().addBreadcrumb("old step"); + countly.crash().clearBreadcrumbs(); + countly.crash().recordException("Crash", "frame 1", false, {}); + std::vector crashes = sentCrashes(countly); + REQUIRE(crashes.size() == 1); + CHECK(crashes[0]["_logs"] == ""); + CHECK(countly.debugHealthCheck()->snapshot()["fu"]["crashes"]["clear"] == 1); +} + +TEST_CASE("Crash reports get the device metrics the server needs") { + Countly &countly = prepareCrashes(); + countly.setMetrics("", "", "Test PC", "1920x1080", "", "3.1"); + startCrashes(countly); + countly.crash().recordException("No metrics given", "frame 1", true, {}); + countly.crash().recordException("Own metrics", "frame 1", true, {{"_app_version", "9.9"}}); + + std::vector crashes = sentCrashes(countly); + REQUIRE(crashes.size() == 2); + CHECK(crashes[0]["_os"] == countly.PlatformName()); + CHECK_FALSE(crashes[0]["_os"].get().empty()); + CHECK(crashes[0]["_app_version"] == "3.1"); + CHECK(crashes[0]["_device"] == "Test PC"); + CHECK(crashes[0]["_resolution"] == "1920x1080"); + CHECK(crashes[1]["_app_version"] == "9.9"); +} + +TEST_CASE("A crash without a stack trace is ignored with a warning") { + Countly &countly = prepareCrashes(); + startCrashes(countly); + countly.crash().recordException("No stack", "", false, {}); + CHECK(sentCrashes(countly).empty()); + CHECK(crashLogCode(countly, "w700") == 1); +} + +TEST_CASE("Crash reports carry the current view") { + Countly &countly = prepareCrashes(); + startCrashes(countly); + const std::string viewId = countly.views().openView("Checkout"); + countly.crash().recordException("In checkout", "frame 1", false, {}); + std::vector crashes = sentCrashes(countly); + REQUIRE(crashes.size() == 1); + CHECK(crashes[0]["_cvid"] == viewId); +} + +TEST_CASE("Global crash segmentation is added to every crash") { + Countly &countly = prepareCrashes(); + countly.setCustomCrashSegmentation({{"build", "nightly"}, {"region", "eu"}}); + startCrashes(countly); + countly.crash().recordException("Crash", "frame 1", false, {}, {{"region", "us"}, {"screen", "home"}}); + std::vector crashes = sentCrashes(countly); + REQUIRE(crashes.size() == 1); + CHECK(crashes[0]["_custom"] == json({{"build", "nightly"}, {"region", "us"}, {"screen", "home"}})); +} + +TEST_CASE("The crash filter sees every crash before it is queued") { + SUBCASE("it can change the report, and the changed fields are reported in _ob") { + Countly &countly = prepareCrashes(); + countly.setCrashFilterCallback([](CrashData &crash) { + crash.stackTrace = "redacted"; + crash.crashSegmentation["email"] = "hidden"; + crash.fatal = !crash.fatal; + return false; + }); + startCrashes(countly); + countly.crash().addBreadcrumb("step"); + countly.crash().recordException("Crash", "frame with jane@example.com", true, {}, {{"email", "jane@example.com"}}); + std::vector crashes = sentCrashes(countly); + REQUIRE(crashes.size() == 1); + CHECK(crashes[0]["_error"] == "redacted"); + CHECK(crashes[0]["_custom"]["email"] == "hidden"); + CHECK(crashes[0]["_nonfatal"] == true); + CHECK(crashes[0]["_ob"] == 25); + } + + SUBCASE("returning true drops the crash, and its breadcrumbs go with it") { + Countly &countly = prepareCrashes(); + countly.setCrashFilterCallback([](CrashData &crash) { return crash.description == "Drop me"; }); + startCrashes(countly); + countly.crash().addBreadcrumb("before the dropped crash"); + countly.crash().recordException("Drop me", "frame 1", false, {}); + countly.crash().recordException("Keep me", "frame 1", false, {}); + std::vector crashes = sentCrashes(countly); + REQUIRE(crashes.size() == 1); + CHECK(crashes[0]["_name"] == "Keep me"); + CHECK(crashes[0]["_logs"] == ""); + CHECK(crashes[0]["_ob"] == 0); + } + + SUBCASE("a filter that throws drops the crash") { + Countly &countly = prepareCrashes(); + countly.setCrashFilterCallback([](CrashData &) -> bool { throw std::runtime_error("filter failed"); }); + startCrashes(countly); + countly.crash().recordException("Crash", "frame 1", false, {}); + CHECK(sentCrashes(countly).empty()); + } +} diff --git a/tests/event_queue.cpp b/tests/event_queue.cpp index 6ddaed0..1e74f4a 100644 --- a/tests/event_queue.cpp +++ b/tests/event_queue.cpp @@ -80,30 +80,19 @@ TEST_CASE("Tests setting 'setEventsToRQThreshold' before we start the SDK") { test_utils::checkTopRequestEventSize(90, countly); // RQ should have the 90 events } - SUBCASE("Internal constraints (1) should be used instead of the negative custom value") { + SUBCASE("A negative threshold is ignored and the default stays") { countly.setEventsToRQThreshold(-6); // before start test_utils::initCountlyWithFakeNetworking(true, countly); // generate 3 events test_utils::generateEvents(3, countly); - // threshold is now set to 1 so we should have 0 events still in the EQ - CHECK(countly.checkEQSize() == 0); - - // queue should have 3 requests + // the default threshold of 100 is still in use, so the 3 events wait in the EQ + CHECK(countly.checkEQSize() == 3); countly.processRQDebug(); - CHECK(!http_call_queue.empty()); - int count = 3; - - while (count--) { - CHECK(http_call_queue.size() == count + 1); - HTTPCall oldest_call1 = http_call_queue.front(); - nlohmann::json events1 = nlohmann::json::parse(oldest_call1.data["events"]); - CHECK(events1.size() == 1); - http_call_queue.pop_front(); - } + CHECK(http_call_queue.empty()); } - SUBCASE("Internal constraints (10000) should be used instead of the positive large custom value") { + SUBCASE("A large threshold is kept, and the event queue still sends at 1000 events") { #ifdef COUNTLY_USE_SQLITE // Use 205 instead of 10005 so we can observe the clamp at a scale SQLite handles countly.setEventsToRQThreshold(205); // before start — clamped to 205 (within [1, 10000]) @@ -118,7 +107,8 @@ TEST_CASE("Tests setting 'setEventsToRQThreshold' before we start the SDK") { test_utils::generateEvents(10003, countly); CHECK(countly.checkEQSize() == 3); - test_utils::checkTopRequestEventSize(10000, countly); + // the event queue never holds more than 1000 events, whatever the threshold + test_utils::checkTopRequestEventSize(1000, countly); #endif } } @@ -137,30 +127,19 @@ TEST_CASE("Tests setting 'setEventsToRQThreshold' after we start the SDK") { test_utils::checkTopRequestEventSize(90, countly); // RQ should have the 90 events } - SUBCASE("Internal constraints (1) should be used instead of the negative custom value") { + SUBCASE("A negative threshold is ignored and the default stays") { test_utils::initCountlyWithFakeNetworking(true, countly); countly.setEventsToRQThreshold(-6); // generate 3 events test_utils::generateEvents(3, countly); - // threshold is now set to 1 so we should have 0 events still in the EQ - CHECK(countly.checkEQSize() == 0); - - // queue should have 3 requests + // the default threshold of 100 is still in use, so the 3 events wait in the EQ + CHECK(countly.checkEQSize() == 3); countly.processRQDebug(); - CHECK(!http_call_queue.empty()); - - int count = 3; - while (count--) { - CHECK(http_call_queue.size() == count + 1); - HTTPCall oldest_call1 = http_call_queue.front(); - nlohmann::json events1 = nlohmann::json::parse(oldest_call1.data["events"]); - CHECK(events1.size() == 1); - http_call_queue.pop_front(); - } + CHECK(http_call_queue.empty()); } - SUBCASE("Internal constraints (10000) should be used instead of the positive large custom value") { + SUBCASE("A large threshold is kept, and the event queue still sends at 1000 events") { test_utils::initCountlyWithFakeNetworking(true, countly); #ifdef COUNTLY_USE_SQLITE countly.setEventsToRQThreshold(205); @@ -173,7 +152,8 @@ TEST_CASE("Tests setting 'setEventsToRQThreshold' after we start the SDK") { test_utils::generateEvents(10003, countly); CHECK(countly.checkEQSize() == 3); - test_utils::checkTopRequestEventSize(10000, countly); + // the event queue never holds more than 1000 events, whatever the threshold + test_utils::checkTopRequestEventSize(1000, countly); #endif } } @@ -218,11 +198,10 @@ TEST_CASE("Tests that sets 'setEventsToRQThreshold' before and after SDK starts" SUBCASE("Only the latest set threshold must be used, negative last") { countly.setEventsToRQThreshold(90); // before start test_utils::initCountlyWithFakeNetworking(true, countly); - countly.setEventsToRQThreshold(-5); // after start + countly.setEventsToRQThreshold(-5); // after start, ignored test_utils::generateEvents(1, countly); - CHECK(countly.checkEQSize() == 0); - test_utils::checkTopRequestEventSize(1, countly); + CHECK(countly.checkEQSize() == 1); } // Setting threshold size both before and after start, with update session in between @@ -249,19 +228,23 @@ TEST_CASE("Tests that sets 'setEventsToRQThreshold' before and after SDK starts" test_utils::generateEvents(4, countly); CHECK(countly.checkEQSize() == 1); - // RQ should have the 3 events // trigger RQ to send requests to http_call_queue countly.processRQDebug(); - // queue should have 3 requests - CHECK(!http_call_queue.empty()); - CHECK(http_call_queue.size() == 3); - http_call_queue.pop_front(); // begin session - http_call_queue.pop_front(); // change device ID - HTTPCall oldest_call = http_call_queue.front(); - CHECK(http_call_queue.size() == 1); + // the queued requests: the old ID's session end, the new ID's session start and the 3 events; the settings + // fetch for the new ID goes to /o/sdk + std::vector queued; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + if (call.url == "/i") { + queued.push_back(call); + } + } + REQUIRE(queued.size() == 3); + CHECK(queued[0].data.count("end_session") == 1); + CHECK(queued[1].data.count("begin_session") == 1); // last call should have 3 events - nlohmann::json events = nlohmann::json::parse(oldest_call.data["events"]); + nlohmann::json events = nlohmann::json::parse(queued[2].data["events"]); CHECK(events.size() == 3); } } diff --git a/tests/event_recording.cpp b/tests/event_recording.cpp new file mode 100644 index 0000000..3c0f51e --- /dev/null +++ b/tests/event_recording.cpp @@ -0,0 +1,270 @@ +#include +#include +#include +#include + +#include "countly.hpp" +#include "doctest.h" +#include "nlohmann/json.hpp" +#include "test_utils.hpp" + +using namespace cly; +using namespace test_utils; +using json = nlohmann::json; + +namespace { +/** + * Prepares the default instance with the fake HTTP client, manual sessions and no health check request. + */ +Countly &prepareEvents(const std::string &settings = "") { + clearSDK(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(fakeSendHTTP); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.enableManualSessionControl(); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + if (!settings.empty()) { + countly.setSDKBehaviorSettings(settings); + } + return countly; +} + +void startEvents(Countly &countly) { countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); } + +long long eventLogCode(Countly &countly, const std::string &code) { + json hc = countly.debugHealthCheck()->snapshot(); + return hc["lc"].contains(code) ? hc["lc"][code].get() : 0; +} + +/** + * Moves the waiting events into a request, sends the queue and returns every event the fake server received, in order. + */ +std::vector sentEvents(Countly &countly) { + countly.flushEvents(); + countly.processRQDebug(); + std::vector events; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + if (call.data.count("events")) { + for (const json &event : json::parse(call.data["events"])) { + events.push_back(event); + } + } + } + return events; +} + +/** + * The first sent event with the key, or null. + */ +json eventWithKey(const std::vector &events, const std::string &key) { + for (const json &event : events) { + if (event["key"] == key) { + return event; + } + } + return json(); +} +} // namespace + +TEST_CASE("Invalid event values are ignored or replaced by their defaults") { + Countly &countly = prepareEvents(); + startEvents(countly); + + countly.RecordEvent("", 1); + countly.RecordEvent("negative_count", -1); + countly.RecordEvent("zero_count", 0); + countly.addEvent(cly::Event("huge_count", static_cast(-1))); + countly.RecordEvent("negative_duration", {}, 1, 0, -5); + countly.RecordEvent("valid", {}, 3, -2.5, 4); + + std::vector events = sentEvents(countly); + REQUIRE(events.size() == 5); + CHECK(eventWithKey(events, "negative_count")["count"] == 1); + CHECK(eventWithKey(events, "zero_count")["count"] == 1); + CHECK(eventWithKey(events, "huge_count")["count"] == 1); + CHECK_FALSE(eventWithKey(events, "negative_duration").contains("dur")); + json valid = eventWithKey(events, "valid"); + CHECK(valid["count"] == 3); + CHECK(valid["sum"] == -2.5); + CHECK(valid["dur"] == 4); + CHECK(eventLogCode(countly, "w701") == 1); + CHECK(eventLogCode(countly, "w750") == 4); +} + +TEST_CASE("A [CLY]_ key that is no internal event is a custom event") { + SUBCASE("the custom event switch and the event filter apply to it") { + Countly &countly = prepareEvents("{\"cet\":false}"); + startEvents(countly); + countly.RecordEvent("[CLY]_my_event", 1); + countly.RecordEvent("[CLY]_view", {{"name", "Home"}, {"visit", "1"}}, 1); + std::vector events = sentEvents(countly); + CHECK(eventWithKey(events, "[CLY]_my_event").is_null()); + CHECK_FALSE(eventWithKey(events, "[CLY]_view").is_null()); + } + + SUBCASE("the event blacklist applies to it") { + Countly &countly = prepareEvents("{\"eb\":[\"[CLY]_my_event\"]}"); + startEvents(countly); + countly.RecordEvent("[CLY]_my_event", 1); + CHECK(sentEvents(countly).empty()); + } + + SUBCASE("the size limits apply to it") { + Countly &countly = prepareEvents("{\"lkl\":20,\"lvs\":10}"); + startEvents(countly); + countly.RecordEvent("[CLY]_" + std::string(40, 'k'), {{std::string(30, 's'), std::string(30, 'v')}}, 1); + std::vector events = sentEvents(countly); + REQUIRE(events.size() == 1); + CHECK(events[0]["key"] == "[CLY]_" + std::string(14, 'k')); + CHECK(events[0]["segmentation"] == json({{std::string(20, 's'), std::string(10, 'v')}})); + } +} + +TEST_CASE("The values an app gives in an internal event follow the value size limit") { + Countly &countly = prepareEvents("{\"lvs\":10}"); + startEvents(countly); + countly.RecordEvent("[CLY]_nps", {{"widget_id", "123"}, {"comment", std::string(30, 'c')}}, 1); + std::vector events = sentEvents(countly); + REQUIRE(events.size() == 1); + CHECK(events[0]["key"] == "[CLY]_nps"); + CHECK(events[0]["segmentation"]["comment"] == std::string(10, 'c')); + CHECK(events[0]["segmentation"]["widget_id"] == "123"); + CHECK(eventLogCode(countly, "w202") == 1); +} + +TEST_CASE("Segmentation filters change only custom events") { + SUBCASE("a segmentation whitelist keeps the view fields") { + Countly &countly = prepareEvents("{\"sw\":[\"level\"]}"); + startEvents(countly); + countly.beginSession(); + countly.views().openView("Home"); + countly.RecordEvent("level_done", {{"level", "3"}, {"score", "50"}}, 1); + std::vector events = sentEvents(countly); + json view = eventWithKey(events, "[CLY]_view"); + REQUIRE_FALSE(view.is_null()); + CHECK(view["segmentation"]["name"] == "Home"); + CHECK(view["segmentation"]["visit"] == "1"); + CHECK(view["segmentation"].contains("_idv")); + CHECK(eventWithKey(events, "level_done")["segmentation"] == json({{"level", "3"}})); + } + + SUBCASE("a segmentation blacklist keeps the view name") { + Countly &countly = prepareEvents("{\"sb\":[\"name\"]}"); + startEvents(countly); + countly.views().openView("Home"); + countly.RecordEvent("signup", {{"name", "Alice"}, {"plan", "pro"}}, 1); + std::vector events = sentEvents(countly); + CHECK(eventWithKey(events, "[CLY]_view")["segmentation"]["name"] == "Home"); + CHECK(eventWithKey(events, "signup")["segmentation"] == json({{"plan", "pro"}})); + } +} + +TEST_CASE("Cut and dropped values are logged as warnings") { + Countly &countly = prepareEvents("{\"lkl\":10,\"lvs\":10,\"lsv\":2}"); + startEvents(countly); + countly.RecordEvent("a_long_event_key", {{"a_long_segmentation_key", "short"}, {"b", "a long value"}, {"c", "dropped"}}, 1); + std::vector events = sentEvents(countly); + REQUIRE(events.size() == 1); + CHECK(events[0]["key"] == "a_long_eve"); + CHECK(events[0]["segmentation"] == json({{"a_long_seg", "short"}, {"b", "a long val"}})); + CHECK(eventLogCode(countly, "w201") == 2); + CHECK(eventLogCode(countly, "w202") == 1); + CHECK(eventLogCode(countly, "w203") == 1); +} + +TEST_CASE("Events carry their own ID and the IDs that link them") { + Countly &countly = prepareEvents(); + startEvents(countly); + countly.beginSession(); + + countly.RecordEvent("before_view", 1); + const std::string viewId = countly.views().openView("Home"); + countly.RecordEvent("in_home", 1); + const std::string secondViewId = countly.views().openView("Settings"); + countly.RecordEvent("in_settings", 1); + countly.views().closeViewWithID(viewId); + + std::vector events = sentEvents(countly); + REQUIRE(events.size() == 6); + + std::set ids; + for (const json &event : events) { + REQUIRE(event.contains("id")); + ids.insert(event["id"].get()); + } + CHECK(ids.size() == 5); // the start and the end event of a view share its ID + + json beforeView = eventWithKey(events, "before_view"); + json inHome = eventWithKey(events, "in_home"); + json inSettings = eventWithKey(events, "in_settings"); + CHECK(beforeView["peid"] == ""); + CHECK(beforeView["cvid"] == ""); + CHECK(inHome["peid"] == beforeView["id"]); + CHECK(inHome["cvid"] == viewId); + CHECK(inSettings["peid"] == inHome["id"]); + CHECK(inSettings["cvid"] == secondViewId); + + std::vector views; + for (const json &event : events) { + if (event["key"] == "[CLY]_view") { + CHECK_FALSE(event.contains("peid")); + CHECK_FALSE(event.contains("cvid")); + views.push_back(event); + } + } + REQUIRE(views.size() == 3); + CHECK(views[0]["id"] == viewId); + CHECK(views[0]["pvid"] == ""); + CHECK(views[1]["id"] == secondViewId); + CHECK(views[1]["pvid"] == viewId); + CHECK(views[2]["id"] == viewId); + CHECK(views[2]["pvid"] == ""); +} + +TEST_CASE("A timed event is recorded at the time it ends") { + Countly &countly = prepareEvents(); + startEvents(countly); + cly::Event event("timed_action", 1); + event.startTimer(); + std::this_thread::sleep_for(std::chrono::milliseconds(1200)); + const long long stopped = getUnixTimestamp(); + event.stopTimer(); + countly.addEvent(event); + + std::vector events = sentEvents(countly); + REQUIRE(events.size() == 1); + CHECK(events[0]["timestamp"].get() >= stopped); + CHECK(events[0]["dur"] == 1); +} + +TEST_CASE("The event queue moves its events into a request at 1000 events") { + Countly &countly = prepareEvents(); + startEvents(countly); + countly.setEventsToRQThreshold(5000); + for (int index = 0; index < 1000; index++) { + countly.RecordEvent("tap", 1); + } + CHECK(countly.checkEQSize() == 0); + countly.processRQDebug(); + std::vector batches; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + if (call.data.count("events")) { + batches.push_back(json::parse(call.data["events"]).size()); + } + } + CHECK(batches == std::vector{1000}); +} + +TEST_CASE("Events are not recorded while the server turns tracking off") { + Countly &countly = prepareEvents("{\"tracking\":false}"); + startEvents(countly); + countly.RecordEvent("ignored", 1); + CHECK(countly.checkEQSize() == 0); + CHECK(eventLogCode(countly, "w505") == 1); +} diff --git a/tests/init.cpp b/tests/init.cpp new file mode 100644 index 0000000..86bbcf0 --- /dev/null +++ b/tests/init.cpp @@ -0,0 +1,487 @@ +#include +#include +#include + +#include "countly.hpp" +#include "doctest.h" +#include "nlohmann/json.hpp" +#include "test_utils.hpp" + +#ifdef COUNTLY_USE_SQLITE +#include "sqlite3.h" +#endif + +using namespace cly; +using namespace test_utils; +using json = nlohmann::json; + +namespace { +/** + * Whether the text is a version 4 UUID in lower case. + */ +bool isUuidV4(const std::string &text) { + static const std::regex pattern("^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"); + return std::regex_match(text, pattern); +} + +/** + * The count of a log code in the health check counters, 0 when it was not recorded. + */ +long long logCodeCount(Countly &countly, const std::string &code) { + json hc = countly.debugHealthCheck()->snapshot(); + return hc["lc"].contains(code) ? hc["lc"][code].get() : 0; +} + +/** + * Prepares the default instance with the fake HTTP client and no health check request. + */ +Countly &prepareInstance() { + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(fakeSendHTTP); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.SetPath(TEST_DATABASE_NAME); + return countly; +} + +/** + * The device_id and t parameters of the first begin_session request the fake client received. + */ +std::pair beginSessionIdentity() { + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + if (call.data.count("begin_session")) { + return {call.data["device_id"], call.data.count("t") ? call.data["t"] : std::string("")}; + } + } + return {"", ""}; +} + +void throwingLogger(LogLevel, const std::string &) { throw std::runtime_error("listener failed"); } + +#ifdef COUNTLY_USE_SQLITE +const char *const GIVEN_ID_IGNORED = "the device ID given with setDeviceID is ignored"; + +/** + * Whether any request the fake client received was sent under the device ID. + */ +bool anyRequestFor(const std::string &device_id) { + for (size_t index = 0; index < http_call_queue.size(); index++) { + if (http_call_queue.at(index).data["device_id"] == device_id) { + return true; + } + } + return false; +} + +/** + * Writes an event into the events table of the test database, as an app that ended without stop() leaves it. + */ +void storeEventInDatabase(const std::string &event) { + sqlite3 *database = nullptr; + REQUIRE(sqlite3_open(TEST_DATABASE_NAME, &database) == SQLITE_OK); + sqlite3_stmt *statement = nullptr; + REQUIRE(sqlite3_prepare_v2(database, "INSERT INTO events (event) VALUES(?);", -1, &statement, nullptr) == SQLITE_OK); + sqlite3_bind_text(statement, 1, event.c_str(), -1, SQLITE_TRANSIENT); + CHECK(sqlite3_step(statement) == SQLITE_DONE); + sqlite3_finalize(statement); + sqlite3_close(database); +} +#endif +} // namespace + +TEST_CASE("Device ID when the app gives none") { + SUBCASE("start() generates a UUID, sends it with type 1 and getDeviceID returns it") { + clearSDK(); + Countly &countly = prepareInstance(); + CHECK_NOTHROW(countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false)); + REQUIRE(countly.checkEQSize() == 0); + countly.processRQDebug(); + + std::pair identity = beginSessionIdentity(); + CHECK(isUuidV4(identity.first)); + CHECK(identity.second == "1"); + CHECK(countly.getDeviceID() == identity.first); + CHECK(countly.getDeviceIDType() == DeviceIdType::SDK_GENERATED); + } + + SUBCASE("an ID given with setDeviceID is sent with type 0") { + clearSDK(); + Countly &countly = prepareInstance(); + countly.setDeviceID("developer-id"); + countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + countly.processRQDebug(); + + std::pair identity = beginSessionIdentity(); + CHECK(identity.first == "developer-id"); + CHECK(identity.second == "0"); + CHECK(countly.getDeviceIDType() == DeviceIdType::DEVELOPER_SUPPLIED); + } + +#ifdef COUNTLY_USE_SQLITE + SUBCASE("a generated ID is kept for the next starts, also when an ID is given later") { + clearSDK(); + Countly *countly = &prepareInstance(); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + const std::string generated = countly->getDeviceID(); + REQUIRE(isUuidV4(generated)); + + Countly::halt(); + http_call_queue.clear(); + countly = &prepareInstance(); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + CHECK(countly->getDeviceID() == generated); + CHECK(countly->getDeviceIDType() == DeviceIdType::SDK_GENERATED); + + Countly::halt(); + clearCapturedLogs(); + countly = &prepareInstance(); + countly->setLogger(captureLog); + countly->setDeviceID("developer-id"); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + countly->processRQDebug(); + CHECK(countly->getDeviceID() == generated); + CHECK(countly->getDeviceIDType() == DeviceIdType::SDK_GENERATED); + CHECK_FALSE(anyRequestFor("developer-id")); + CHECK(capturedLogContains(GIVEN_ID_IGNORED)); + } + + SUBCASE("an ID given at the first start is kept: given again it is used quietly, a different one is ignored") { + clearSDK(); + Countly *countly = &prepareInstance(); + countly->setDeviceID("first-user"); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + + Countly::halt(); + clearCapturedLogs(); + countly = &prepareInstance(); + countly->setLogger(captureLog); + countly->setDeviceID("first-user"); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + CHECK(countly->getDeviceID() == "first-user"); + CHECK_FALSE(capturedLogContains(GIVEN_ID_IGNORED)); + + Countly::halt(); + clearCapturedLogs(); + http_call_queue.clear(); + countly = &prepareInstance(); + countly->setLogger(captureLog); + countly->setDeviceID("second-user"); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + countly->processRQDebug(); + CHECK(countly->getDeviceID() == "first-user"); + CHECK(countly->getDeviceIDType() == DeviceIdType::DEVELOPER_SUPPLIED); + CHECK_FALSE(anyRequestFor("second-user")); + CHECK(capturedLogContains(GIVEN_ID_IGNORED)); + } + + SUBCASE("enableClearStoredDeviceID makes start() use the given ID, which the next start keeps") { + clearSDK(); + Countly *countly = &prepareInstance(); + countly->setDeviceID("first-user"); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + + Countly::halt(); + countly = &prepareInstance(); + countly->enableClearStoredDeviceID(); + countly->setDeviceID("second-user"); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + CHECK(countly->getDeviceID() == "second-user"); + CHECK(countly->getDeviceIDType() == DeviceIdType::DEVELOPER_SUPPLIED); + + Countly::halt(); + countly = &prepareInstance(); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + CHECK(countly->getDeviceID() == "second-user"); + } + + SUBCASE("enableClearStoredDeviceID without a given ID generates a new ID") { + clearSDK(); + Countly *countly = &prepareInstance(); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + const std::string first = countly->getDeviceID(); + REQUIRE(isUuidV4(first)); + + Countly::halt(); + countly = &prepareInstance(); + countly->enableClearStoredDeviceID(); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + CHECK(isUuidV4(countly->getDeviceID())); + CHECK(countly->getDeviceID() != first); + CHECK(countly->getDeviceIDType() == DeviceIdType::SDK_GENERATED); + } + + SUBCASE("enableClearStoredDeviceID first sends the events the stored user left behind under the stored ID") { + clearSDK(); + Countly *countly = &prepareInstance(); + countly->setDeviceID("first-user"); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + Countly::halt(); + storeEventInDatabase("{\"key\":\"left_behind\",\"count\":1,\"timestamp\":1700000000000,\"hour\":10,\"dow\":2}"); + http_call_queue.clear(); + + countly = &prepareInstance(); + countly->enableClearStoredDeviceID(); + countly->setDeviceID("second-user"); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + CHECK(countly->checkEQSize() == 0); + countly->processRQDebug(); + + std::vector left_behind; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + if (call.data.count("events") && call.data["events"].find("left_behind") != std::string::npos) { + left_behind.push_back(call); + } + } + REQUIRE(left_behind.size() == 1); + CHECK(left_behind[0].data["device_id"] == "first-user"); + CHECK(left_behind[0].data["t"] == "0"); + CHECK(countly->getDeviceID() == "second-user"); + } + + SUBCASE("an ID changed after start is kept for the next start") { + clearSDK(); + Countly *countly = &prepareInstance(); + countly->setDeviceID("first-user"); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + countly->setDeviceID("second-user", false); + + Countly::halt(); + countly = &prepareInstance(); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + CHECK(countly->getDeviceID() == "second-user"); + } +#else + SUBCASE("without SQLite a new ID is generated at every start") { + clearSDK(); + Countly *countly = &prepareInstance(); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + const std::string first = countly->getDeviceID(); + + Countly::halt(); + countly = &prepareInstance(); + countly->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + CHECK(isUuidV4(countly->getDeviceID())); + CHECK(countly->getDeviceID() != first); + } +#endif + + SUBCASE("enableClearStoredDeviceID after start() is ignored with a warning") { + clearSDK(); + clearCapturedLogs(); + Countly &countly = prepareInstance(); + countly.setLogger(captureLog); + countly.setDeviceID("first-user"); + countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + countly.enableClearStoredDeviceID(); + CHECK(capturedLogContains("enableClearStoredDeviceID, This method can't be called after SDK initialization")); + CHECK(countly.getDeviceID() == "first-user"); + } +} + +TEST_CASE("An empty device ID is ignored") { + SUBCASE("before start, the SDK generates an ID instead") { + clearSDK(); + Countly &countly = prepareInstance(); + countly.setDeviceID(""); + CHECK(logCodeCount(countly, "w601") == 1); + countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + CHECK(isUuidV4(countly.getDeviceID())); + } + + SUBCASE("after start, the device ID stays") { + clearSDK(); + Countly &countly = prepareInstance(); + countly.setDeviceID("developer-id"); + countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + countly.processRQDebug(); + http_call_queue.clear(); + + countly.setDeviceID("", false); + countly.setDeviceID("", true); + countly.RecordEvent("after_empty_id", 1); + countly.flushEvents(); + countly.processRQDebug(); + + CHECK(countly.getDeviceID() == "developer-id"); + CHECK(logCodeCount(countly, "w601") == 2); + CHECK_FALSE(httpQueueContains("device_id", "")); + CHECK_FALSE(httpQueueContains("end_session", "1")); + CHECK(httpQueueContainsEvent("after_empty_id")); + } +} + +TEST_CASE("start() refuses a missing or placeholder app key") { + const std::vector app_keys = {"", "YOUR_APP_KEY"}; + for (const std::string &app_key : app_keys) { + CAPTURE(app_key); + clearSDK(); + clearCapturedLogs(); + Countly &countly = prepareInstance(); + countly.setLogger(captureLog); + countly.setDeviceID("developer-id"); + + CHECK_NOTHROW(countly.start(app_key, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false)); + CHECK(countly.checkEQSize() == -1); + CHECK(capturedLogContains("app key")); + countly.RecordEvent("ignored", 1); + CHECK(http_call_queue.empty()); + + // the app can fix the key and start again + countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + CHECK(countly.checkEQSize() == 0); + countly.setLogger(nullptr); + } +} + +TEST_CASE("start() refuses a server URL that is not an absolute http or https URL") { + const std::vector urls = {"", "try.count.ly", "ftp://try.count.ly", "https://", "http:///i"}; + for (const std::string &url : urls) { + CAPTURE(url); + clearSDK(); + clearCapturedLogs(); + Countly &countly = prepareInstance(); + countly.setLogger(captureLog); + countly.setDeviceID("developer-id"); + + CHECK_NOTHROW(countly.start(COUNTLY_TEST_APP_KEY, url, COUNTLY_TEST_PORT, false)); + CHECK(countly.checkEQSize() == -1); + CHECK(capturedLogContains("server URL")); + countly.setLogger(nullptr); + } + + SUBCASE("a valid URL loses its trailing slashes with a warning") { + clearSDK(); + clearCapturedLogs(); + Countly &countly = prepareInstance(); + countly.setLogger(captureLog); + countly.setDeviceID("developer-id"); + countly.start(COUNTLY_TEST_APP_KEY, "HTTPS://try.count.ly//", COUNTLY_TEST_PORT, false); + + CHECK(countly.checkEQSize() == 0); + CHECK(countly.getConfiguration().serverUrl == "https://try.count.ly"); + CHECK(capturedLogContains("trailing")); + countly.setLogger(nullptr); + } +} + +TEST_CASE("views() and crash() can be used before start()") { + clearSDK(); + Countly &countly = prepareInstance(); + countly.setDeviceID("developer-id"); + + CHECK(countly.views().openView("BeforeStart").empty()); + countly.views().closeViewWithName("BeforeStart"); + countly.crash().addBreadcrumb("before start"); + countly.crash().recordException("Before start", "frame", true, {{"_os", "Linux"}, {"_app_version", "1.0"}}); + CHECK(logCodeCount(countly, "w101") == 4); + + countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + countly.crash().recordException("After start", "frame", false, {{"_os", "Linux"}, {"_app_version", "1.0"}}); + countly.processRQDebug(); + + bool crashSeen = false; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + if (call.data.count("crash")) { + json crash = json::parse(call.data["crash"]); + CHECK(crash["_name"] == "After start"); + CHECK(crash["_logs"] == ""); + crashSeen = true; + } + } + CHECK(crashSeen); + CHECK_FALSE(httpQueueContainsEvent("[CLY]_view")); +} + +TEST_CASE("A log listener that throws does not break the SDK") { + clearSDK(); + Countly &countly = prepareInstance(); + countly.setLogger(throwingLogger); + countly.setDeviceID("developer-id"); + + CHECK_NOTHROW(countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false)); + CHECK_NOTHROW(countly.RecordEvent("still_recorded", 1)); + CHECK_NOTHROW(countly.flushEvents()); + CHECK_NOTHROW(countly.processRQDebug()); + CHECK(httpQueueContainsEvent("still_recorded")); + countly.setLogger(nullptr); +} + +TEST_CASE("Config values below their minimum are ignored") { + clearSDK(); + Countly &countly = prepareInstance(); + countly.setMaxKeyLength(0); + countly.setMaxValueSize(0); + countly.setMaxSegmentationValues(0); + countly.setMaxBreadcrumbCount(0); + countly.setMaxStackTraceLinesPerThread(0); + countly.setMaxStackTraceLineLength(0); + countly.setMaxRequestQueueSize(0); + countly.setAutomaticSessionUpdateInterval(0); + countly.setEventsToRQThreshold(0); + CHECK(logCodeCount(countly, "w121") == 9); + + const CountlyConfiguration &config = countly.getConfiguration(); + CHECK(config.maxKeyLength == COUNTLY_MAX_KEY_LENGTH_DEFAULT); + CHECK(config.maxValueSize == COUNTLY_MAX_VALUE_SIZE_DEFAULT); + CHECK(config.maxSegmentationValues == COUNTLY_MAX_SEGMENTATION_VALUES_DEFAULT); + CHECK(config.breadcrumbsThreshold == COUNTLY_MAX_BREADCRUMB_COUNT_DEFAULT); + CHECK(config.maxStackTraceLinesPerThread == COUNTLY_MAX_STACK_TRACE_LINES_PER_THREAD_DEFAULT); + CHECK(config.maxStackTraceLineLength == COUNTLY_MAX_STACK_TRACE_LINE_LENGTH_DEFAULT); + CHECK(config.requestQueueThreshold == 1000); + CHECK(config.sessionDuration == 60); + CHECK(config.eventQueueThreshold == 100); + + countly.setDeviceID("developer-id"); + countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + countly.RecordEvent("purchase", {{"color", "red"}}, 1); + countly.flushEvents(); + countly.processRQDebug(); + CHECK(httpQueueContainsEvent("purchase")); + + SUBCASE("a large event queue threshold is kept as given") { + countly.setEventsToRQThreshold(20000); + CHECK(countly.getConfiguration().eventQueueThreshold == 20000); + } +} + +TEST_CASE("User details given before start() are kept for the first session without a warning") { + clearSDK(); + Countly &countly = prepareInstance(); + countly.setUserDetails({{"name", "Alice"}}); + countly.setCustomUserDetails({{"plan", "gold"}}); + CHECK(logCodeCount(countly, "w101") == 0); + + countly.setDeviceID("developer-id"); + countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + countly.processRQDebug(); + bool sent = false; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + if (call.data.count("begin_session")) { + CHECK(json::parse(call.data["user_details"]) == json({{"name", "Alice"}, {"custom", {{"plan", "gold"}}}})); + sent = true; + } + } + CHECK(sent); +} + +TEST_CASE("Cut user detail values are logged as warnings") { + clearSDK(); + Countly &countly = prepareInstance(); + countly.setSDKBehaviorSettings("{\"lvs\":5,\"lkl\":4}"); + countly.setDeviceID("developer-id"); + countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + countly.processRQDebug(); + http_call_queue.clear(); + + countly.setUserDetails({{"name", "Alexander"}}); + countly.setCustomUserDetails({{"favourite", "chocolate"}}); + countly.processRQDebug(); + CHECK(httpQueueContains("user_details", "{\"name\":\"Alexa\"}")); + CHECK(httpQueueContains("user_details", "{\"custom\":{\"favo\":\"choco\"}}")); + CHECK(logCodeCount(countly, "w202") == 2); + CHECK(logCodeCount(countly, "w201") == 1); +} diff --git a/tests/internal_limits_integration.cpp b/tests/internal_limits_integration.cpp index ca7ec27..551c7a8 100644 --- a/tests/internal_limits_integration.cpp +++ b/tests/internal_limits_integration.cpp @@ -151,7 +151,7 @@ TEST_CASE("Internal Limits Views") { int devCount = 0; for (auto it = seg.begin(); it != seg.end(); ++it) { const std::string &k = it.key(); - if (k == "visit" || k == "start" || k == "_idv" || k == "name") { + if (k == "visit" || k == "start" || k == "_idv" || k == "name" || k == "segment") { continue; } devCount++; diff --git a/tests/location.cpp b/tests/location.cpp new file mode 100644 index 0000000..2bc7b56 --- /dev/null +++ b/tests/location.cpp @@ -0,0 +1,164 @@ +#include +#include + +#include "countly.hpp" +#include "doctest.h" +#include "nlohmann/json.hpp" +#include "test_utils.hpp" + +using namespace cly; +using namespace test_utils; +using json = nlohmann::json; + +namespace { +Countly &prepareLocation(bool manualSessions = false) { + clearSDK(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(fakeSendHTTP); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + if (manualSessions) { + countly.enableManualSessionControl(); + } + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + return countly; +} + +void startLocation(Countly &countly) { countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT); } + +/** + * The location parameters of each request the fake server received, with "begin" for a session start. Requests + * without a location parameter are left out. + */ +std::vector> locationRequests(Countly &countly) { + countly.processRQDebug(); + std::vector> requests; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + std::map location; + for (const char *key : {"country_code", "city", "location", "ip_address", "begin_session"}) { + if (call.data.count(key)) { + location[key] = call.data[key]; + } + } + if (location.size() > (location.count("begin_session") ? 1u : 0u) || location.count("begin_session")) { + requests.push_back(location); + } + } + http_call_queue.clear(); + return requests; +} + +long long locationLogCode(Countly &countly, const std::string &code) { + json hc = countly.debugHealthCheck()->snapshot(); + return hc["lc"].contains(code) ? hc["lc"][code].get() : 0; +} +} // namespace + +TEST_CASE("A location given before start() goes with the first session") { + Countly &countly = prepareLocation(); + countly.setLocation("DE", "Berlin", "52.52001,13.40495", "203.0.113.7"); + startLocation(countly); + std::vector> requests = locationRequests(countly); + REQUIRE(requests.size() == 1); + CHECK(requests[0] == std::map{{"begin_session", "1"}, {"country_code", "DE"}, {"city", "Berlin"}, {"location", "52.52001,13.40495"}, {"ip_address", "203.0.113.7"}}); +} + +TEST_CASE("A separate location request leaves out the IP address") { + Countly &countly = prepareLocation(); + startLocation(countly); + locationRequests(countly); + countly.setLocation("DE", "Berlin", "52.52001,13.40495", "203.0.113.7"); + std::vector> requests = locationRequests(countly); + REQUIRE(requests.size() == 1); + CHECK(requests[0] == std::map{{"country_code", "DE"}, {"city", "Berlin"}, {"location", "52.52001,13.40495"}}); +} + +TEST_CASE("Location values that don't fit their form are not used") { + Countly &countly = prepareLocation(); + startLocation(countly); + locationRequests(countly); + countly.setLocation("us", "Boston", "north,south", "not-an-ip"); + std::vector> requests = locationRequests(countly); + REQUIRE(requests.size() == 1); + CHECK(requests[0] == std::map{{"city", "Boston"}}); + CHECK(locationLogCode(countly, "w700") == 3); +} + +TEST_CASE("A setLocation call without any value only clears the stored values") { + Countly &countly = prepareLocation(true); + countly.setLocation("DE", "Berlin", "", ""); + startLocation(countly); + countly.setLocation("", "", "", ""); + CHECK(locationRequests(countly).empty()); + countly.beginSession(); + std::vector> requests = locationRequests(countly); + REQUIRE(requests.size() == 1); + CHECK(requests[0] == std::map{{"begin_session", "1"}}); +} + +TEST_CASE("disableLocation turns location off") { + SUBCASE("after start: the values are cleared, an erase request goes out and sessions carry the empty location") { + Countly &countly = prepareLocation(true); + startLocation(countly); + countly.setLocation("DE", "Berlin", "", ""); + countly.disableLocation(); + countly.beginSession(); + std::vector> requests = locationRequests(countly); + REQUIRE(requests.size() == 2); + CHECK(requests[0] == std::map{{"location", ""}}); + CHECK(requests[1] == std::map{{"begin_session", "1"}, {"location", ""}}); + + // a location given while no session runs waits for the next session start, and turns location back on + countly.endSession(); + countly.setLocation("FR", "Paris", "", ""); + countly.beginSession(); + requests = locationRequests(countly); + REQUIRE(requests.size() == 1); + CHECK(requests[0] == std::map{{"begin_session", "1"}, {"country_code", "FR"}, {"city", "Paris"}}); + } + + SUBCASE("before start: the location given before start is not used") { + Countly &countly = prepareLocation(); + countly.setLocation("DE", "Berlin", "", ""); + countly.disableLocation(); + startLocation(countly); + std::vector> requests = locationRequests(countly); + REQUIRE(requests.size() == 1); + CHECK(requests[0] == std::map{{"begin_session", "1"}, {"location", ""}}); + } + + SUBCASE("the server's location setting wins over the app's choice") { + Countly &countly = prepareLocation(); + countly.setSDKBehaviorSettings("{\"lt\":true}"); + countly.disableLocation(); + startLocation(countly); + std::vector> requests = locationRequests(countly); + REQUIRE(requests.size() == 1); + CHECK(requests[0] == std::map{{"begin_session", "1"}}); + } +} + +TEST_CASE("While the server turns location off, setLocation is ignored") { + Countly &countly = prepareLocation(); + countly.setSDKBehaviorSettings("{\"lt\":false}"); + startLocation(countly); + locationRequests(countly); + countly.setLocation("DE", "Berlin", "", ""); + CHECK(locationRequests(countly).empty()); + json hc = countly.debugHealthCheck()->snapshot(); + CHECK(hc["el"] == 0); +} + +TEST_CASE("At the end of start(), an erase request goes out when location is off and no session starts") { + Countly &countly = prepareLocation(); + countly.setSDKBehaviorSettings("{\"st\":false}"); + countly.disableLocation(); + startLocation(countly); + std::vector> requests = locationRequests(countly); + REQUIRE(requests.size() == 1); + CHECK(requests[0] == std::map{{"location", ""}}); +} diff --git a/tests/multi_instance.cpp b/tests/multi_instance.cpp index 5fe08c4..10edd67 100644 --- a/tests/multi_instance.cpp +++ b/tests/multi_instance.cpp @@ -210,7 +210,7 @@ TEST_CASE("a second remote config fetch is dropped rather than blocking the call cly::HTTPResponse response; response.success = true; response.data = nlohmann::json::object(); - if (data.find("fetch_remote_config") != std::string::npos) { + if (data.find("method=rc") != std::string::npos) { fetches_started.fetch_add(1); while (!release_fetch.load()) { std::this_thread::sleep_for(std::chrono::milliseconds(5)); @@ -338,7 +338,7 @@ TEST_CASE("destruction does not wait for an in-flight remote config fetch") { response.data = nlohmann::json::object(); // Only the remote-config fetch is held. The SDK Behavior Settings fetch also // targets /o/sdk (with method=sc) and must not be blocked. - if (data.find("fetch_remote_config") != std::string::npos) { + if (data.find("method=rc") != std::string::npos) { fetch_entered.store(true); while (!release_fetch.load()) { std::this_thread::sleep_for(std::chrono::milliseconds(5)); diff --git a/tests/mutex_exception_safety.cpp b/tests/mutex_exception_safety.cpp index e4fcda2..37d0d18 100644 --- a/tests/mutex_exception_safety.cpp +++ b/tests/mutex_exception_safety.cpp @@ -45,14 +45,12 @@ TEST_CASE("mutex exception safety - malformed event in updateSession does not de // Let the loop run a couple of cycles so the session has begun. std::this_thread::sleep_for(std::chrono::milliseconds(200)); - // Inject a string that is NOT valid JSON. On the next cycle, updateSession() - // iterates the event queue and calls nlohmann::json::parse on it WHILE holding - // the mutex, which throws. With RAII the unique_lock releases on unwind; the - // exception reaches updateLoop's catch, the loop ends, and the thread returns - // WITHOUT owning the mutex. + // Inject a string that is NOT valid JSON. Moving the event queue into a request + // parses it while holding the mutex; the entry is dropped with a warning, and + // nothing may leave the mutex owned. ct.debugInjectRawEvent("{not valid json"); - // Give the loop time to pick up the bad event and throw. + // Give the loop time to pick up the bad event. std::this_thread::sleep_for(std::chrono::milliseconds(300)); // stop() must return. Run it on another thread so a deadlock fails the test @@ -82,22 +80,27 @@ TEST_CASE("mutex exception safety - throwing user-API path does not leak the mut ct.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); // Seed a malformed event, then add a valid one. addEvent -> checkAndSendEventToRQ - // iterates the queue (size >= threshold) and json::parse throws while the mutex is held. + // parses the queue (size >= threshold) while the mutex is held; the malformed entry + // is dropped with a warning instead of throwing out of addEvent. ct.debugInjectRawEvent("{not valid json"); bool threw = false; try { cly::Event ev("trigger", 1); ct.addEvent(ev); } catch (...) { - threw = true; // exception propagating out is expected; the point is the mutex is released + threw = true; } + CHECK_FALSE(threw); // If the mutex had leaked, this locking call would block forever. auto fut = std::async(std::launch::async, [&]() { return ct.checkEQSize(); }); bool returned = fut.wait_for(std::chrono::seconds(5)) == std::future_status::ready; REQUIRE(returned); fut.get(); - (void)threw; + + // the valid event still reaches the server + ct.processRQDebug(); + CHECK(httpQueueContainsEvent("trigger")); } #endif @@ -143,26 +146,28 @@ TEST_CASE("the SDK can be called from the log callback") { static std::atomic callback_invocations(0); static std::atomic rq_non_negative(0); static std::atomic eq_refusals(0); - static Countly *target = nullptr; + // The update loop calls the callback from its own thread, so the test may clear this while a call is running. + static std::atomic target(nullptr); callback_invocations.store(0); rq_non_negative.store(0); eq_refusals.store(0); Countly &ct = Countly::getInstance(); - target = &ct; + target.store(&ct); // No recursion guard of its own on purpose: the SDK has to be the one that // stops the recursion. ct.setLogger([](LogLevel level, const std::string &message) { (void)level; (void)message; - if (target == nullptr) { + Countly *instance = target.load(); + if (instance == nullptr) { return; } callback_invocations.fetch_add(1); - if (target->checkRQSize() >= 0) { + if (instance->checkRQSize() >= 0) { rq_non_negative.fetch_add(1); } - if (target->checkEQSize() < 0) { + if (instance->checkEQSize() < 0) { eq_refusals.fetch_add(1); } }); @@ -186,6 +191,6 @@ TEST_CASE("the SDK can be called from the log callback") { // checkEQSize cannot, and reports "cannot determine" rather than deadlocking. CHECK(eq_refusals.load() > 0); - target = nullptr; + target.store(nullptr); ct.setLogger(nullptr); } diff --git a/tests/remote_config.cpp b/tests/remote_config.cpp new file mode 100644 index 0000000..ec4f849 --- /dev/null +++ b/tests/remote_config.cpp @@ -0,0 +1,207 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "countly.hpp" +#include "doctest.h" +#include "nlohmann/json.hpp" +#include "test_utils.hpp" + +using namespace cly; +using namespace test_utils; +using json = nlohmann::json; + +namespace { +std::mutex rc_mutex; +std::vector> rc_requests; +std::atomic hold_rc_answers{false}; +std::atomic rc_answer_number{0}; + +/** + * Records the remote config downloads and answers each with a value telling which download it was. Holds the + * answers back while hold_rc_answers is set. + */ +HTTPResponse remoteConfigClient(bool use_post, const std::string &url, const std::string &data) { + HTTPResponse response = fakeSendHTTP(use_post, url, data); + if (url != "/o/sdk" || data.find("method=rc") == std::string::npos) { + return response; + } + std::map params; + std::string::size_type start = 0; + while (start < data.size()) { + std::string::size_type end = data.find('&', start); + if (end == std::string::npos) { + end = data.size(); + } + const std::string pair = data.substr(start, end - start); + const std::string::size_type equals = pair.find('='); + std::string value = equals == std::string::npos ? std::string() : pair.substr(equals + 1); + decodeURL(value); + params[pair.substr(0, equals)] = value; + start = end + 1; + } + { + std::lock_guard lock(rc_mutex); + rc_requests.push_back(params); + } + const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (hold_rc_answers.load() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + const int number = ++rc_answer_number; + return HTTPResponse{true, json({{"color", "red"}, {"download", number}}), 200, ""}; +} + +std::vector> rcRequests() { + std::lock_guard lock(rc_mutex); + return rc_requests; +} + +bool waitUntilTrue(std::function condition) { + const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (std::chrono::steady_clock::now() < deadline) { + if (condition()) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + return condition(); +} + +Countly &prepareRemoteConfig() { + clearSDK(); + { + std::lock_guard lock(rc_mutex); + rc_requests.clear(); + } + hold_rc_answers = false; + rc_answer_number = 0; + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(remoteConfigClient); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.enableManualSessionControl(); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + countly.setMetrics("", "", "", "", "", "2.0"); + return countly; +} + +void startRemoteConfig(Countly &countly) { countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT); } +} // namespace + +TEST_CASE("Remote config downloads use the rc method") { + SUBCASE("with the device metrics, enrolling nobody in A/B tests by default") { + Countly &countly = prepareRemoteConfig(); + startRemoteConfig(countly); + countly.updateRemoteConfig(); + REQUIRE(waitUntilTrue([] { return rcRequests().size() == 1; })); + std::map request = rcRequests()[0]; + CHECK(request["method"] == "rc"); + CHECK_FALSE(request.count("oi")); + CHECK_FALSE(request.count("keys")); + json metrics = json::parse(request["metrics"]); + CHECK(metrics["_app_version"] == "2.0"); + CHECK(metrics.contains("_os")); + CHECK(request["t"] == "0"); + } + + SUBCASE("enrolling in A/B tests when the app asks for it") { + Countly &countly = prepareRemoteConfig(); + countly.enrollABOnRCDownload(); + startRemoteConfig(countly); + countly.updateRemoteConfig(); + REQUIRE(waitUntilTrue([] { return rcRequests().size() == 1; })); + CHECK(rcRequests()[0]["oi"] == "1"); + } + + SUBCASE("asking for some keys, or for all but some") { + Countly &countly = prepareRemoteConfig(); + startRemoteConfig(countly); + std::string keys[] = {"color"}; + countly.updateRemoteConfigFor(keys, 1); + REQUIRE(waitUntilTrue([] { return rcRequests().size() == 1; })); + countly.updateRemoteConfigExcept(keys, 1); + REQUIRE(waitUntilTrue([] { return rcRequests().size() == 2; })); + CHECK(rcRequests()[0]["keys"] == "[\"color\"]"); + CHECK(rcRequests()[1]["omit_keys"] == "[\"color\"]"); + } +} + +TEST_CASE("Remote config values belong to the user they were downloaded for") { + SUBCASE("a device ID change without merge clears them") { + Countly &countly = prepareRemoteConfig(); + startRemoteConfig(countly); + countly.updateRemoteConfig(); + REQUIRE(waitUntilTrue([&countly] { return countly.getRemoteConfigValue("color") == "red"; })); + countly.setDeviceID("second-user", false); + CHECK(countly.getRemoteConfigValue("color").is_null()); + } + + SUBCASE("a device ID change with merge clears them") { + Countly &countly = prepareRemoteConfig(); + startRemoteConfig(countly); + countly.updateRemoteConfig(); + REQUIRE(waitUntilTrue([&countly] { return countly.getRemoteConfigValue("color") == "red"; })); + countly.setDeviceID("merged-user", true); + CHECK(countly.getRemoteConfigValue("color").is_null()); + } + + SUBCASE("an answer that arrives after a device ID change is discarded") { + Countly &countly = prepareRemoteConfig(); + startRemoteConfig(countly); + hold_rc_answers = true; + countly.updateRemoteConfig(); + REQUIRE(waitUntilTrue([] { return rcRequests().size() == 1; })); + countly.setDeviceID("second-user", false); + hold_rc_answers = false; + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + CHECK(countly.getRemoteConfigValue("color").is_null()); + } +} + +TEST_CASE("Automatic remote config downloads") { + SUBCASE("happen at the end of start(), not at every session start") { + Countly &countly = prepareRemoteConfig(); + countly.enableRemoteConfig(); + startRemoteConfig(countly); + REQUIRE(waitUntilTrue([] { return rcRequests().size() == 1; })); + countly.beginSession(); + countly.endSession(); + countly.beginSession(); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + CHECK(rcRequests().size() == 1); + } + + SUBCASE("follow a device ID change without merge, also when a download for the previous user is running") { + Countly &countly = prepareRemoteConfig(); + countly.enableRemoteConfig(); + hold_rc_answers = true; + startRemoteConfig(countly); + REQUIRE(waitUntilTrue([] { return rcRequests().size() == 1; })); + countly.setDeviceID("second-user", false); + hold_rc_answers = false; + REQUIRE(waitUntilTrue([] { return rcRequests().size() == 2; })); + CHECK(rcRequests()[1]["device_id"] == "second-user"); + CHECK(waitUntilTrue([&countly] { return countly.getRemoteConfigValue("download") == 2; })); + } + + SUBCASE("follow a device ID change with merge once the merge request was sent") { + Countly &countly = prepareRemoteConfig(); + countly.enableRemoteConfig(); + startRemoteConfig(countly); + REQUIRE(waitUntilTrue([] { return rcRequests().size() == 1; })); + countly.setDeviceID("merged-user", true); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + CHECK(rcRequests().size() == 1); + countly.processRQDebug(); + REQUIRE(waitUntilTrue([] { return rcRequests().size() == 2; })); + CHECK(rcRequests()[1]["device_id"] == "merged-user"); + } +} diff --git a/tests/request.cpp b/tests/request.cpp index 2ffeb38..64ad8d8 100644 --- a/tests/request.cpp +++ b/tests/request.cpp @@ -230,7 +230,7 @@ TEST_CASE("Requests contain the SDK name and version") { for (const char *clock_key : {"timestamp", "dow", "hour", "tz"}) { CHECK(parsed.erase(clock_key) == 1); } - CHECK(parsed == std::map{{"app_key", "test_app_key"}, {"device_id", "test_device_id"}, {"sdk_name", EXPECTED_SDK_NAME}, {"sdk_version", COUNTLY_SDK_VERSION}}); + CHECK(parsed == std::map{{"app_key", "test_app_key"}, {"device_id", "test_device_id"}, {"t", "0"}, {"sdk_name", EXPECTED_SDK_NAME}, {"sdk_version", COUNTLY_SDK_VERSION}}); } SUBCASE("a fetch request carries them and gains no other parameter") { @@ -294,8 +294,8 @@ TEST_CASE("Every request carries the SDK name and version") { Countly &countly = Countly::getInstance(); countly.setHTTPClient(test_utils::fakeSendHTTP); countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); - // 0 makes updateSession() report the session duration now rather than after a minute - countly.setAutomaticSessionUpdateInterval(0); + // the session calls below work only with manual session control + countly.enableManualSessionControl(); countly.SetPath(TEST_DATABASE_NAME); // Only processRQDebug() sends: the update loop sleeps through the test, and immediate stop still ends it at once countly.enableImmediateRequestOnStop(); @@ -304,8 +304,9 @@ TEST_CASE("Every request carries the SDK name and version") { // Both fetches run on threads of their own, so each is awaited to keep the order fixed REQUIRE(waitForRequestWith("method", "sc")); countly.updateRemoteConfig(); - REQUIRE(waitForRequestWith("method", "fetch_remote_config")); + REQUIRE(waitForRequestWith("method", "rc")); + countly.beginSession(); countly.addEvent(cly::Event("click", 1)); countly.updateSession(); countly.setUserDetails({{"name", "Jane Doe"}}); @@ -314,15 +315,35 @@ TEST_CASE("Every request carries the SDK name and version") { countly.setDeviceID("merged-device-id", true); countly.endSession(); countly.processRQDebug(); + // once the merge request was delivered, the settings are fetched for the merged user + const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (test_utils::http_call_queue.size() < 11 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } - std::vector kinds; + std::vector queued; + std::vector fetches; + size_t merge_index = 0; + size_t last_fetch_index = 0; for (size_t index = 0; index < test_utils::http_call_queue.size(); index++) { test_utils::HTTPCall call = test_utils::http_call_queue.at(index); - kinds.push_back(requestKind(call)); + const std::string kind = requestKind(call); + if (call.url == "/o/sdk") { + fetches.push_back(kind); + last_fetch_index = index; + } else { + queued.push_back(kind); + } + if (kind == "old_device_id") { + merge_index = index; + } CHECK(call.data["sdk_name"] == EXPECTED_SDK_NAME); CHECK(call.data["sdk_version"] == COUNTLY_SDK_VERSION); } - CHECK(kinds == std::vector{"sc", "fetch_remote_config", "begin_session", "session_duration", "events", "user_details", "location", "crash", "old_device_id", "end_session"}); + CHECK(queued == std::vector{"begin_session", "session_duration", "events", "user_details", "location", "crash", "old_device_id", "end_session"}); + // the fetch for the merged user runs on its own thread, so only its place after the merge request is fixed + CHECK(fetches == std::vector{"sc", "rc", "sc"}); + CHECK(last_fetch_index > merge_index); } /** @@ -347,6 +368,7 @@ TEST_CASE("A request the server failed is sent again with the SDK name and versi countly.setHTTPClient(sendHTTPToFlakyServer); countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); countly.SetPath(TEST_DATABASE_NAME); + countly.setFailedRequestCooldown(1); // Only processRQDebug() sends: the update loop sleeps through the test, and immediate stop still ends it at once countly.enableImmediateRequestOnStop(); countly.setUpdateInterval(60000); @@ -356,10 +378,14 @@ TEST_CASE("A request the server failed is sent again with the SDK name and versi test_utils::http_call_queue.clear(); countly.addEvent(cly::Event("click", 1)); - countly.updateSession(); + countly.flushEvents(); server_fails_requests.store(true); countly.processRQDebug(); server_fails_requests.store(false); + // a failed request is tried again only after the cool-down + countly.processRQDebug(); + CHECK(test_utils::http_call_queue.size() == 1); + std::this_thread::sleep_for(std::chrono::milliseconds(1100)); countly.processRQDebug(); REQUIRE(test_utils::http_call_queue.size() == 2); @@ -377,8 +403,8 @@ namespace { /** * HTTP server on 127.0.0.1 for exercising the SDK's built-in libcurl client. * - * An answering server replies to every request with 200 and "{}" and records its - * request line. A silent server accepts connections and never replies; it records + * An answering server replies to every request with 200 and {"result":"Success"} + * and records its request line and body. A silent server accepts connections and never replies; it records * how long each connection stayed open until the client closed it. */ class LocalHTTPServer { @@ -421,13 +447,21 @@ class LocalHTTPServer { int port() const { return _port; } /** - * Request lines received so far, such as "GET /i?app_key=... HTTP/1.1". + * Request lines received so far, such as "POST /i HTTP/1.1". */ std::vector requestLines() const { std::lock_guard lock(_mutex); return _request_lines; } + /** + * Request bodies received so far, in the order of requestLines(). + */ + std::vector requestBodies() const { + std::lock_guard lock(_mutex); + return _request_bodies; + } + /** * For each connection the client closed, how long it had stayed open. */ @@ -522,9 +556,10 @@ class LocalHTTPServer { { std::lock_guard lock(_mutex); _request_lines.push_back(request.substr(0, request.find("\r\n"))); + _request_bodies.push_back(request.substr(header_end + 4)); } - const std::string response = "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}"; + const std::string response = "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 20\r\nConnection: close\r\n\r\n{\"result\":\"Success\"}"; #ifdef SO_NOSIGPIPE int no_sigpipe = 1; ::setsockopt(client, SOL_SOCKET, SO_NOSIGPIPE, &no_sigpipe, sizeof(no_sigpipe)); @@ -544,6 +579,7 @@ class LocalHTTPServer { std::atomic _stopping{false}; mutable std::mutex _mutex; std::vector _request_lines; + std::vector _request_bodies; std::vector _closed_durations; }; } // namespace @@ -564,8 +600,10 @@ TEST_CASE("Built-in HTTP client") { const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); while (std::chrono::steady_clock::now() < deadline) { countly->processRQDebug(); - for (const std::string &request_line : server.requestLines()) { - if (request_line.rfind("GET /i?", 0) == 0 && request_line.find("begin_session=1") != std::string::npos) { + const std::vector request_lines = server.requestLines(); + const std::vector request_bodies = server.requestBodies(); + for (size_t index = 0; index < request_lines.size() && index < request_bodies.size(); index++) { + if (request_lines[index].rfind("POST /i ", 0) == 0 && request_bodies[index].find("begin_session=1") != std::string::npos) { begin_session_received = true; } } @@ -592,13 +630,15 @@ TEST_CASE("Built-in HTTP client") { countly->enableRemoteConfig(); countly->start(COUNTLY_TEST_APP_KEY, "http://127.0.0.1", server.port(), false); countly->addEvent(cly::Event("click", 1)); - countly->updateSession(); + countly->flushEvents(); std::vector request_lines; + std::vector request_bodies; const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); while (std::chrono::steady_clock::now() < deadline) { countly->processRQDebug(); request_lines = server.requestLines(); + request_bodies = server.requestBodies(); if (request_lines.size() >= 5 && countly->checkRQSize() == 0) { break; } @@ -610,21 +650,22 @@ TEST_CASE("Built-in HTTP client") { remove(database_path.c_str()); std::multiset kinds; - for (const std::string &request_line : request_lines) { - // These requests are short enough to be sent as GET, so the parameters are in the request line: "GET /o/sdk?method=sc&app_key=... HTTP/1.1" - REQUIRE(request_line.rfind("GET /", 0) == 0); - const std::string::size_type path_start = 4; - const std::string::size_type query_start = request_line.find('?'); - const std::string::size_type query_end = request_line.rfind(' '); - REQUIRE(query_start != std::string::npos); - REQUIRE(query_end > query_start); - - test_utils::HTTPCall call{false, request_line.substr(path_start, query_start - path_start), parseRequest(request_line.substr(query_start + 1, query_end - query_start - 1))}; + REQUIRE(request_bodies.size() == request_lines.size()); + for (size_t index = 0; index < request_lines.size(); index++) { + // Every request is a POST with its parameters in the body, and only the path in the request line: "POST /o/sdk HTTP/1.1" + const std::string &request_line = request_lines[index]; + REQUIRE(request_line.rfind("POST /", 0) == 0); + const std::string::size_type path_start = 5; + const std::string::size_type path_end = request_line.rfind(' '); + REQUIRE(path_end > path_start); + CHECK(request_line.find('?') == std::string::npos); + + test_utils::HTTPCall call{true, request_line.substr(path_start, path_end - path_start), parseRequest(request_bodies[index])}; kinds.insert(requestKind(call)); CHECK(call.data["sdk_name"] == EXPECTED_SDK_NAME); CHECK(call.data["sdk_version"] == COUNTLY_SDK_VERSION); } - CHECK(kinds == std::multiset{"sc", "fetch_remote_config", "begin_session", "events", "hc"}); + CHECK(kinds == std::multiset{"sc", "rc", "begin_session", "events", "hc"}); } SUBCASE("a request to a server that never answers is given up after the request timeout") { diff --git a/tests/request_pipeline.cpp b/tests/request_pipeline.cpp new file mode 100644 index 0000000..36f9dae --- /dev/null +++ b/tests/request_pipeline.cpp @@ -0,0 +1,554 @@ +#include +#include +#include +#include +#include +#include + +#include "countly.hpp" +#include "doctest.h" +#include "nlohmann/json.hpp" +#include "test_utils.hpp" + +#ifdef COUNTLY_USE_SQLITE +#include "sqlite3.h" +#endif + +using namespace cly; +using namespace test_utils; +using json = nlohmann::json; + +namespace { +/** + * A request the scripted server received: whether it came as POST, its path, its parameters and its raw body. + */ +struct ReceivedRequest { + bool post; + std::string path; + std::map params; + std::string raw; +}; + +/** + * A fake Countly server whose answers a test can script. By default it answers every request like the server does. + */ +struct ScriptedServer { + std::mutex mtx; + std::vector received; + // Called for every request; returns the answer. Replaced by tests that need another answer. + std::function answer; + + ScriptedServer() { reset(); } + + void reset() { + std::lock_guard lock(mtx); + received.clear(); + answer = [](const ReceivedRequest &request) { + HTTPResponse response{true, json({{"result", "Success"}}), 200}; + if (request.path == "/o/sdk") { + response.success = false; + response.status_code = 0; + } + return response; + }; + } + + /** + * Splits a request body into its decoded parameters. + */ + static std::map parse(const std::string &data) { + std::map params; + std::string::size_type start = 0; + while (start < data.size()) { + std::string::size_type end = data.find('&', start); + if (end == std::string::npos) { + end = data.size(); + } + const std::string pair = data.substr(start, end - start); + const std::string::size_type equals = pair.find('='); + std::string value = equals == std::string::npos ? std::string() : pair.substr(equals + 1); + decodeURL(value); + params[pair.substr(0, equals)] = value; + start = end + 1; + } + return params; + } + + HTTPResponse handle(bool post, const std::string &path, const std::string &data) { + ReceivedRequest request{post, path, parse(data), data}; + std::function respond; + { + std::lock_guard lock(mtx); + received.push_back(request); + respond = answer; + } + return respond(request); + } + + std::vector queued() { + std::lock_guard lock(mtx); + std::vector result; + for (const ReceivedRequest &request : received) { + if (request.path == "/i" && !request.params.count("hc")) { + result.push_back(request); + } + } + return result; + } + + void setAnswer(std::function respond) { + std::lock_guard lock(mtx); + answer = std::move(respond); + } +}; + +ScriptedServer server; + +HTTPResponse scriptedClient(bool post, const std::string &path, const std::string &data) { return server.handle(post, path, data); } + +std::string fakeSha(const std::string &data) { return "sha:" + std::to_string(std::hash()(data)); } + +/** + * Prepares the default instance with the scripted server, manual sessions and no health check request. + */ +Countly &prepare() { + clearSDK(); + server.reset(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(scriptedClient); + countly.setSha256(fakeSha); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.enableManualSessionControl(); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + // The update loop would send the queue every 3 seconds; the tests send it themselves + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + return countly; +} + +void start(Countly &countly) { countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); } + +long long logCode(Countly &countly, const std::string &code) { + json hc = countly.debugHealthCheck()->snapshot(); + return hc["lc"].contains(code) ? hc["lc"][code].get() : 0; +} + +/** + * Polls until the condition holds or the timeout passes, calling the step between checks. + */ +bool eventually(std::function condition, std::chrono::milliseconds timeout, std::function step = [] {}) { + const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) { + step(); + if (condition()) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + return condition(); +} + +HTTPResponse failingAnswer(const ReceivedRequest &) { return HTTPResponse{false, json(), 503, "Service Unavailable"}; } + +#ifdef COUNTLY_USE_SQLITE +/** + * Stores a request with the given timestamp in the database, the way an earlier run would have left it. + */ +void storeRequest(const std::string &body) { + sqlite3 *database = nullptr; + REQUIRE(sqlite3_open(TEST_DATABASE_NAME, &database) == SQLITE_OK); + REQUIRE(sqlite3_exec(database, "CREATE TABLE IF NOT EXISTS Requests (RequestID INTEGER PRIMARY KEY, RequestData TEXT)", nullptr, nullptr, nullptr) == SQLITE_OK); + sqlite3_stmt *statement = nullptr; + REQUIRE(sqlite3_prepare_v2(database, "INSERT INTO Requests (RequestData) VALUES(?);", -1, &statement, nullptr) == SQLITE_OK); + sqlite3_bind_text(statement, 1, body.c_str(), -1, SQLITE_TRANSIENT); + CHECK(sqlite3_step(statement) == SQLITE_DONE); + sqlite3_finalize(statement); + sqlite3_close(database); +} +#endif +} // namespace + +TEST_CASE("Every request is sent as POST with its data in the body") { + Countly &countly = prepare(); + countly.enableRemoteConfig(); + start(countly); + countly.beginSession(); + countly.RecordEvent("purchase", 1); + countly.flushEvents(); + countly.updateRemoteConfig(); + countly.processRQDebug(); + + REQUIRE(eventually([] { return server.queued().size() >= 2; }, std::chrono::seconds(5))); + REQUIRE(eventually( + [] { + std::lock_guard lock(server.mtx); + for (const ReceivedRequest &request : server.received) { + if (request.path == "/o/sdk") { + return true; + } + } + return false; + }, + std::chrono::seconds(5))); + std::lock_guard lock(server.mtx); + for (const ReceivedRequest &request : server.received) { + CAPTURE(request.raw); + CHECK(request.post); + } +} + +TEST_CASE("A queued request counts as delivered only with a JSON answer that has a result") { + Countly &countly = prepare(); + countly.setFailedRequestCooldown(1); + start(countly); + server.setAnswer([](const ReceivedRequest &) { return HTTPResponse{true, json(), 200, "proxy login"}; }); + countly.setUserDetails({{"name", "Alice"}}); + countly.processRQDebug(); + + CHECK(server.queued().size() == 1); + CHECK(countly.checkRQSize() == 1); + json hc = countly.debugHealthCheck()->snapshot(); + CHECK(hc["sc"] == 200); + CHECK(hc["em"] == "proxy login"); + + SUBCASE("a JSON object without result is a failure too") { + server.setAnswer([](const ReceivedRequest &) { return HTTPResponse{true, json({{"status", "ok"}}), 200, "{\"status\":\"ok\"}"}; }); + CHECK(eventually([&countly] { return server.queued().size() == 2; }, std::chrono::seconds(3), [&countly] { countly.processRQDebug(); })); + CHECK(countly.checkRQSize() == 1); + } + + SUBCASE("the request is sent again and removed once the server accepts it") { + server.reset(); + CHECK(eventually([&countly] { return countly.checkRQSize() == 0; }, std::chrono::seconds(3), [&countly] { countly.processRQDebug(); })); + REQUIRE(server.queued().size() == 1); + CHECK(server.queued()[0].params["user_details"] == "{\"name\":\"Alice\"}"); + } +} + +TEST_CASE("A failed request is tried again only after the cool-down") { + SUBCASE("the default cool-down is 60 seconds") { + Countly &countly = prepare(); + start(countly); + server.setAnswer(failingAnswer); + countly.setUserDetails({{"name", "Alice"}}); + countly.processRQDebug(); + countly.processRQDebug(); + countly.processRQDebug(); + CHECK(server.queued().size() == 1); + CHECK(countly.checkRQSize() == 1); + } + + SUBCASE("a shorter cool-down lets the request through after it") { + Countly &countly = prepare(); + countly.setFailedRequestCooldown(1); + start(countly); + server.setAnswer(failingAnswer); + countly.setUserDetails({{"name", "Alice"}}); + countly.processRQDebug(); + countly.processRQDebug(); + CHECK(server.queued().size() == 1); + + server.reset(); + std::this_thread::sleep_for(std::chrono::milliseconds(1100)); + countly.processRQDebug(); + CHECK(server.queued().size() == 1); + CHECK(countly.checkRQSize() == 0); + } + + SUBCASE("a cool-down below 1 second is ignored") { + Countly &countly = prepare(); + countly.setFailedRequestCooldown(0); + CHECK(logCode(countly, "w121") == 1); + CHECK(countly.getConfiguration().failedRequestCooldownSeconds == 60); + } +} + +TEST_CASE("Requests carry the number of requests waiting behind them") { + Countly &countly = prepare(); + start(countly); + countly.setUserDetails({{"name", "first"}}); + countly.setUserDetails({{"name", "second"}}); + countly.setUserDetails({{"name", "third"}}); + + SUBCASE("queued requests") { + countly.processRQDebug(); + std::vector queued = server.queued(); + REQUIRE(queued.size() == 3); + CHECK(queued[0].params["rr"] == "2"); + CHECK(queued[1].params["rr"] == "1"); + CHECK(queued[2].params["rr"] == "0"); + } + + SUBCASE("an immediate request") { + countly.updateRemoteConfig(); + REQUIRE(eventually( + [] { + std::lock_guard lock(server.mtx); + for (const ReceivedRequest &request : server.received) { + if (request.path == "/o/sdk" && request.params.at("method") != "sc") { + return true; + } + } + return false; + }, + std::chrono::seconds(5))); + std::lock_guard lock(server.mtx); + bool remoteConfigSeen = false; + for (const ReceivedRequest &request : server.received) { + if (request.path == "/o/sdk" && request.params.at("method") != "sc") { + CHECK(request.params.at("rr") == "3"); + remoteConfigSeen = true; + } + } + CHECK(remoteConfigSeen); + } + + SUBCASE("the checksum covers the rr parameter") { + Countly::halt(); + Countly &salted = prepare(); + salted.setSalt("test-salt"); + start(salted); + salted.setUserDetails({{"name", "salted"}}); + salted.processRQDebug(); + std::vector queued = server.queued(); + REQUIRE(queued.size() == 1); + const std::string &raw = queued[0].raw; + const std::string::size_type checksum = raw.rfind("&checksum256="); + REQUIRE(checksum != std::string::npos); + CHECK(raw.substr(0, checksum).find("&rr=0") != std::string::npos); + CHECK(raw.substr(checksum + 13) == salted.calculateChecksum("test-salt", raw.substr(0, checksum))); + } +} + +TEST_CASE("A full request queue drops its oldest requests") { + Countly &countly = prepare(); + countly.setMaxRequestQueueSize(3); + start(countly); + server.setAnswer(failingAnswer); + for (int index = 0; index < 5; index++) { + countly.setUserDetails({{"name", "user " + std::to_string(index)}}); + } + CHECK(countly.checkRQSize() == 3); + CHECK(logCode(countly, "w310") == 2); + + server.reset(); + countly.processRQDebug(); + std::vector queued = server.queued(); + REQUIRE(queued.size() == 3); + CHECK(queued[0].params["user_details"] == "{\"name\":\"user 2\"}"); +} + +#ifdef COUNTLY_USE_SQLITE +TEST_CASE("A queue far over its limit shrinks step by step") { + // An earlier run left 250 requests; this run allows 10 + clearSDK(); + for (int index = 0; index < 250; index++) { + storeRequest("app_key=" COUNTLY_TEST_APP_KEY "&device_id=old×tamp=" + std::to_string(getUnixTimestamp()) + "&user_details=%7B%7D"); + } + + SUBCASE("by default at most 100 of the overflow plus one go per new request") { + server.reset(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(scriptedClient); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.enableManualSessionControl(); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + countly.setMaxRequestQueueSize(10); + start(countly); + countly.setUserDetails({{"name", "new"}}); + CHECK(countly.checkRQSize() == 150); + CHECK(logCode(countly, "w310") == 101); + } + + SUBCASE("without the gradual cleaner the whole overflow goes at once") { + server.reset(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(scriptedClient); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.enableManualSessionControl(); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + countly.setMaxRequestQueueSize(10); + countly.disableGradualRequestCleaner(); + start(countly); + countly.setUserDetails({{"name", "new"}}); + CHECK(countly.checkRQSize() == 10); + CHECK(logCode(countly, "w310") == 241); + } +} + +TEST_CASE("Requests older than the drop age are removed without being sent") { + const long long now = getUnixTimestamp(); + const std::string old_request = "app_key=" COUNTLY_TEST_APP_KEY "&device_id=old×tamp=" + std::to_string(now - 3 * 3600 * 1000LL) + "&user_details=%7B%22name%22%3A%22old%22%7D"; + const std::string fresh_request = "app_key=" COUNTLY_TEST_APP_KEY "&device_id=old×tamp=" + std::to_string(now - 1000) + "&user_details=%7B%22name%22%3A%22fresh%22%7D"; + const std::string untimed_request = "app_key=" COUNTLY_TEST_APP_KEY "&device_id=old&user_details=%7B%22name%22%3A%22untimed%22%7D"; + + SUBCASE("set by the app") { + clearSDK(); + storeRequest(old_request); + storeRequest(fresh_request); + storeRequest(untimed_request); + server.reset(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(scriptedClient); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.enableManualSessionControl(); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + countly.setRequestDropAgeHours(2); + start(countly); + countly.processRQDebug(); + + std::vector queued = server.queued(); + REQUIRE(queued.size() == 2); + CHECK(queued[0].params["user_details"] == "{\"name\":\"fresh\"}"); + CHECK(queued[1].params["user_details"] == "{\"name\":\"untimed\"}"); + CHECK(logCode(countly, "w330") == 1); + CHECK(countly.checkRQSize() == 0); + } + + SUBCASE("set by the server") { + clearSDK(); + storeRequest(old_request); + storeRequest(fresh_request); + server.reset(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(scriptedClient); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.enableManualSessionControl(); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + countly.setSDKBehaviorSettings("{\"dort\":2}"); + start(countly); + countly.processRQDebug(); + + std::vector queued = server.queued(); + REQUIRE(queued.size() == 1); + CHECK(queued[0].params["user_details"] == "{\"name\":\"fresh\"}"); + } + + SUBCASE("without a drop age old requests are sent") { + clearSDK(); + storeRequest(old_request); + server.reset(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(scriptedClient); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.enableManualSessionControl(); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + start(countly); + countly.processRQDebug(); + CHECK(server.queued().size() == 1); + } +} + +TEST_CASE("Turning tracking off stops recording but not sending") { + clearSDK(); + storeRequest("app_key=" COUNTLY_TEST_APP_KEY "&device_id=old×tamp=" + std::to_string(getUnixTimestamp()) + "&user_details=%7B%22name%22%3A%22stored%22%7D"); + server.reset(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(scriptedClient); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.enableManualSessionControl(); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + countly.setSDKBehaviorSettings("{\"tracking\":false}"); + start(countly); + countly.setUserDetails({{"name", "ignored"}}); + countly.processRQDebug(); + + std::vector queued = server.queued(); + REQUIRE(queued.size() == 1); + CHECK(queued[0].params["user_details"] == "{\"name\":\"stored\"}"); + CHECK(logCode(countly, "w505") == 1); +} +#endif + +TEST_CASE("A slow answer pauses sending for the backoff duration") { + SUBCASE("the server's settings set the backoff") { + Countly &countly = prepare(); + countly.setSDKBehaviorSettings("{\"bom_at\":1,\"bom_d\":2}"); + start(countly); + countly.setUserDetails({{"name", "first"}}); + countly.setUserDetails({{"name", "second"}}); + server.setAnswer([](const ReceivedRequest &) { + std::this_thread::sleep_for(std::chrono::milliseconds(1100)); + return HTTPResponse{true, json({{"result", "Success"}}), 200}; + }); + countly.processRQDebug(); + CHECK(server.queued().size() == 1); + countly.processRQDebug(); + CHECK(server.queued().size() == 1); + + json hc = countly.debugHealthCheck()->snapshot(); + CHECK(hc["bom"] == 1); + CHECK(hc["cbom"] == 1); + CHECK(logCode(countly, "w320") == 1); + + server.reset(); + std::this_thread::sleep_for(std::chrono::milliseconds(2100)); + countly.processRQDebug(); + CHECK(server.queued().size() == 1); + CHECK(countly.checkRQSize() == 0); + } + + SUBCASE("disableBackoffMechanism keeps sending") { + Countly &countly = prepare(); + countly.disableBackoffMechanism(); + countly.setSDKBehaviorSettings("{\"bom_at\":1,\"bom_d\":2}"); + start(countly); + countly.setUserDetails({{"name", "first"}}); + countly.setUserDetails({{"name", "second"}}); + server.setAnswer([](const ReceivedRequest &) { + std::this_thread::sleep_for(std::chrono::milliseconds(1100)); + return HTTPResponse{true, json({{"result", "Success"}}), 200}; + }); + countly.processRQDebug(); + CHECK(server.queued().size() == 2); + CHECK(countly.debugHealthCheck()->snapshot()["bom"] == 0); + } + + SUBCASE("the server's bom turns it on even when the app turned it off") { + Countly &countly = prepare(); + countly.disableBackoffMechanism(); + countly.setSDKBehaviorSettings("{\"bom\":true,\"bom_at\":1,\"bom_d\":2}"); + start(countly); + countly.setUserDetails({{"name", "first"}}); + countly.setUserDetails({{"name", "second"}}); + server.setAnswer([](const ReceivedRequest &) { + std::this_thread::sleep_for(std::chrono::milliseconds(1100)); + return HTTPResponse{true, json({{"result", "Success"}}), 200}; + }); + countly.processRQDebug(); + CHECK(server.queued().size() == 1); + } + + SUBCASE("a fast answer keeps sending") { + Countly &countly = prepare(); + countly.setSDKBehaviorSettings("{\"bom_at\":1,\"bom_d\":2}"); + start(countly); + countly.setUserDetails({{"name", "first"}}); + countly.setUserDetails({{"name", "second"}}); + countly.processRQDebug(); + CHECK(server.queued().size() == 2); + CHECK(countly.debugHealthCheck()->snapshot()["bom"] == 0); + } +} diff --git a/tests/sbs.cpp b/tests/sbs.cpp index 36a6092..d7d2a36 100644 --- a/tests/sbs.cpp +++ b/tests/sbs.cpp @@ -36,6 +36,44 @@ static void initWithSBSConfig(const json &sbsConfig, Countly &countly) { http_call_queue.clear(); } +/** + * The settings the fake server answers a settings fetch with, for initWithServerAnswer. + */ +static json server_settings_answer; + +/** + * Fake HTTP client that answers the settings fetch with server_settings_answer, like the server does. + */ +static HTTPResponse sendHTTPWithServerSettings(bool use_post, const std::string &url, const std::string &data) { + HTTPResponse response = test_utils::fakeSendHTTP(use_post, url, data); + if (url == "/o/sdk" && data.find("method=sc") != std::string::npos) { + return HTTPResponse{true, json({{"v", 2}, {"t", 1}, {"c", server_settings_answer}}), 200, ""}; + } + return response; +} + +/** + * Helper to initialize the SDK with settings that the server answers and the SDK stores, as after a real fetch. + * Uses manual session control and clears the HTTP call queue after setup. + */ +static void initWithServerAnswer(const json &sbsConfig, Countly &countly) { + server_settings_answer = sbsConfig; + countly.setHTTPClient(sendHTTPWithServerSettings); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + countly.enableManualSessionControl(); + countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!httpQueueContains("method", "sc") && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + // the answer is applied right after the fake client returns it + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + countly.processRQDebug(); + countly.clearRequestQueue(); + http_call_queue.clear(); +} + /** * Helper to initialize the SDK WITHOUT providing SBS config. * Uses whatever SBS is already stored in the database (or defaults). @@ -969,6 +1007,8 @@ TEST_CASE("SBS Feature Flags") { Countly &countly = Countly::getInstance(); json sbs = {{"lt", true}}; initWithSBSConfig(sbs, countly); + // with a running session the location goes in a separate request right away + countly.beginSession(); countly.setLocation("US", "New York", "40.7128,-74.0060", "192.168.1.1"); @@ -1557,6 +1597,8 @@ TEST_CASE("SBS Scenarios") { Countly &countly1 = Countly::getInstance(); json sbs1 = json::object(); initWithSBSConfig(sbs1, countly1); + // with a running session the location goes in a separate request right away + countly1.beginSession(); // setLocation should succeed countly1.setLocation("US", "New York", "40.7128,-74.0060", "192.168.1.1"); @@ -1852,11 +1894,11 @@ TEST_CASE("SBS Storage Behavior") { http_call_queue.clear(); SUBCASE("Stored SBS is loaded on re-init without provided SBS") { - // Step 1: Init SDK with provided SBS that disables custom event tracking + // Step 1: Init SDK with a server answer that disables custom event tracking; the SDK stores it clearSDK(); Countly &countly1 = Countly::getInstance(); json sbs = {{"cet", false}, {"eqs", 1}}; - initWithSBSConfig(sbs, countly1); + initWithServerAnswer(sbs, countly1); // Verify that custom events are blocked in this session cly::Event e1("custom_event", 1); @@ -1882,11 +1924,11 @@ TEST_CASE("SBS Storage Behavior") { } SUBCASE("Stored SBS takes precedence over provided SBS") { - // Step 1: Init SDK with SBS that disables session tracking (st=false) + // Step 1: Init SDK with a server answer that disables session tracking (st=false); the SDK stores it clearSDK(); Countly &countly1 = Countly::getInstance(); json sbs1 = {{"st", false}}; - initWithSBSConfig(sbs1, countly1); + initWithServerAnswer(sbs1, countly1); // Verify session tracking is disabled CHECK(countly1.beginSession() == false); @@ -1979,11 +2021,11 @@ TEST_CASE("SBS Storage Behavior") { } SUBCASE("Stored SBS feature flag persists across multiple re-inits") { - // Step 1: Init with SBS that disables crash reporting and sets eqs=2 + // Step 1: Init with a server answer that disables crash reporting and sets eqs=2; the SDK stores it clearSDK(); Countly &countly1 = Countly::getInstance(); json sbs = {{"crt", false}, {"eqs", 2}}; - initWithSBSConfig(sbs, countly1); + initWithServerAnswer(sbs, countly1); // Verify crash reporting is disabled countly1.crash().recordException("crash1", "trace1", false, {{"_os", "TestOS"}}, {}); @@ -2047,7 +2089,7 @@ TEST_CASE("SBS Location Clearing") { clearSDK(); http_call_queue.clear(); - SUBCASE("setLocation clearing (all empty) is allowed when lt=false") { + SUBCASE("setLocation is ignored when lt=false, also a call that only clears") { clearSDK(); Countly &countly = Countly::getInstance(); json sbs = {{"lt", false}}; @@ -2065,7 +2107,7 @@ TEST_CASE("SBS Location Clearing") { } CHECK_FALSE(hasLocationSet); - // But clearing location (all empty) should be allowed even when lt=false + // a call without values only clears the stored values; the server's setting already keeps location off countly.setLocation("", "", "", ""); countly.processRQDebug(); bool hasClearRequest = false; @@ -2075,7 +2117,7 @@ TEST_CASE("SBS Location Clearing") { hasClearRequest = true; } } - CHECK(hasClearRequest); + CHECK_FALSE(hasClearRequest); } } @@ -2149,7 +2191,7 @@ TEST_CASE("SBS Session Update Interval Override") { CHECK(hasSessionUpdate); } - SUBCASE("Large sui prevents premature session updates") { + SUBCASE("A large sui does not hold back the app's own session update") { clearSDK(); Countly &countly = Countly::getInstance(); json sbs = {{"sui", 300}}; @@ -2165,7 +2207,7 @@ TEST_CASE("SBS Session Update Interval Override") { countly.updateSession(); countly.processRQDebug(); - // No session update should be sent (duration < sui) + // sui paces the SDK's update timer; a manual updateSession() always reports the duration bool hasSessionUpdate = false; while (!http_call_queue.empty()) { HTTPCall call = popCall(); @@ -2173,7 +2215,7 @@ TEST_CASE("SBS Session Update Interval Override") { hasSessionUpdate = true; } } - CHECK_FALSE(hasSessionUpdate); + CHECK(hasSessionUpdate); } } diff --git a/tests/session.cpp b/tests/session.cpp index 5d3c915..7f24aca 100644 --- a/tests/session.cpp +++ b/tests/session.cpp @@ -54,9 +54,12 @@ TEST_CASE("sessions unit tests") { countly.clearRequestQueue(); // request queue contains session begin request http_call_queue.clear(); // clear local HTTP request queue. - std::this_thread::sleep_for(3s); - countly.updateSession(); - countly.processRQDebug(); + // the update timer reports the automatic session once the 2 second update interval passed + const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + 10s; + while (http_call_queue.empty() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(100ms); + countly.processRQDebug(); + } HTTPCall http_call = popHTTPCall(); CHECK(http_call.data["app_key"] == COUNTLY_TEST_APP_KEY); @@ -69,12 +72,13 @@ TEST_CASE("sessions unit tests") { countly.clearRequestQueue(); // request queue contains session begin request http_call_queue.clear(); // clear local HTTP request queue. - countly.endSession(); + // automatic sessions end when the SDK stops + const long long stopped = getUnixTimestamp(); + countly.stop(); countly.processRQDebug(); HTTPCall http_call = popHTTPCall(); - long long timestamp = getUnixTimestamp(); - long long timestampDiff = timestamp - stoll(http_call.data["timestamp"]); + long long timestampDiff = stoll(http_call.data["timestamp"]) - stopped; CHECK(http_call.data["app_key"] == COUNTLY_TEST_APP_KEY); CHECK(http_call.data["device_id"] == COUNTLY_TEST_DEVICE_ID); CHECK(!http_call.data["session_duration"].empty()); @@ -105,12 +109,12 @@ TEST_CASE("event request unit tests") { event.addSegmentation("points", 100); countly.addEvent(event); - countly.updateSession(); + countly.flushEvents(); countly.processRQDebug(); HTTPCall http_call = popHTTPCall(); - CHECK(!http_call.use_post); + CHECK(http_call.use_post); CHECK(http_call.data["app_key"] == COUNTLY_TEST_APP_KEY); CHECK(http_call.data["device_id"] == COUNTLY_TEST_DEVICE_ID); @@ -130,11 +134,11 @@ TEST_CASE("event request unit tests") { countly.addEvent(event1); countly.addEvent(event2); - countly.updateSession(); + countly.flushEvents(); countly.processRQDebug(); HTTPCall http_call = popHTTPCall(); - CHECK(!http_call.use_post); + CHECK(http_call.use_post); CHECK(http_call.data["app_key"] == COUNTLY_TEST_APP_KEY); CHECK(http_call.data["device_id"] == COUNTLY_TEST_DEVICE_ID); @@ -155,11 +159,11 @@ TEST_CASE("event request unit tests") { event.addSegmentation("points", 2000); countly.addEvent(event); - countly.updateSession(); + countly.flushEvents(); countly.processRQDebug(); HTTPCall http_call = popHTTPCall(); - CHECK(!http_call.use_post); + CHECK(http_call.use_post); CHECK(http_call.data["app_key"] == COUNTLY_TEST_APP_KEY); CHECK(http_call.data["device_id"] == COUNTLY_TEST_DEVICE_ID); @@ -180,7 +184,7 @@ TEST_CASE("event request unit tests") { countly.addEvent(event); } - countly.updateSession(); + countly.flushEvents(); countly.processRQDebug(); HTTPCall http_call = popHTTPCall(); @@ -199,7 +203,7 @@ TEST_CASE("event request unit tests") { countly.addEvent(event); REQUIRE(countly.checkEQSize() == 1); - countly.updateSession(); + countly.flushEvents(); countly.processRQDebug(); HTTPCall http_call = popHTTPCall(); @@ -231,7 +235,10 @@ TEST_CASE("concurrent endSession calls end the session only once") { }); countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); countly.SetPath(TEST_DATABASE_NAME); + // the app's session calls work only with manual session control + countly.enableManualSessionControl(); countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); + REQUIRE(countly.beginSession()); const int rq_before = countly.checkRQSize(); // holds the begin_session request REQUIRE(rq_before >= 0); diff --git a/tests/session_lifecycle.cpp b/tests/session_lifecycle.cpp new file mode 100644 index 0000000..e254691 --- /dev/null +++ b/tests/session_lifecycle.cpp @@ -0,0 +1,270 @@ +#include +#include +#include +#include +#include +#include + +#include "countly.hpp" +#include "doctest.h" +#include "nlohmann/json.hpp" +#include "test_utils.hpp" + +using namespace cly; +using namespace test_utils; +using json = nlohmann::json; + +namespace { +/** + * What a request did, in a word: begin, update, end, events or user_details. Empty for other requests. + */ +std::string requestKind(const HTTPCall &call) { + if (call.data.count("begin_session")) { + return "begin"; + } + if (call.data.count("end_session")) { + return "end"; + } + if (call.data.count("session_duration")) { + return "update"; + } + if (call.data.count("events")) { + return "events"; + } + if (call.data.count("user_details")) { + return "user_details"; + } + return ""; +} + +/** + * The kinds of the queued requests the fake server received, in order. + */ +std::vector receivedKinds() { + std::vector kinds; + for (size_t index = 0; index < http_call_queue.size(); index++) { + const std::string kind = requestKind(http_call_queue.at(index)); + if (!kind.empty()) { + kinds.push_back(kind); + } + } + return kinds; +} + +bool received(const std::string &kind) { + for (const std::string &each : receivedKinds()) { + if (each == kind) { + return true; + } + } + return false; +} + +/** + * Polls until the condition holds or the timeout passes. + */ +bool eventually(std::function condition, std::chrono::milliseconds timeout) { + const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) { + if (condition()) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + return condition(); +} + +long long sessionLogCode(Countly &countly, const std::string &code) { + json hc = countly.debugHealthCheck()->snapshot(); + return hc["lc"].contains(code) ? hc["lc"][code].get() : 0; +} + +Countly &prepareSessions() { + clearSDK(); + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(fakeSendHTTP); + countly.disableHealthCheck(); + countly.disableSDKBehaviorSettingsUpdates(); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + return countly; +} + +void startSessions(Countly &countly) { countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT); } + +std::atomic release_settings{false}; + +/** + * Answers the behavior settings fetch with session tracking off once the test releases it. + */ +HTTPResponse delayedSessionTrackingOff(bool use_post, const std::string &url, const std::string &data) { + if (url == "/o/sdk" && data.find("method=sc") != std::string::npos) { + const std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!release_settings.load() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + return HTTPResponse{true, json::parse(R"({"v":2,"t":1,"c":{"st":false}})"), 200}; + } + return fakeSendHTTP(use_post, url, data); +} +} // namespace + +TEST_CASE("Automatic sessions are updated and events are sent on the update timer") { + SUBCASE("also when start() is called without its last parameter") { + Countly &countly = prepareSessions(); + countly.setAutomaticSessionUpdateInterval(1); + countly.setUpdateInterval(100); + startSessions(countly); + countly.RecordEvent("tap", 1); + + CHECK(eventually([] { return received("update") && received("events"); }, std::chrono::seconds(4))); + CHECK(countly.checkEQSize() == 0); + } + + SUBCASE("events wait for the update timer instead of going out on every tick") { + Countly &countly = prepareSessions(); + countly.setAutomaticSessionUpdateInterval(2); + countly.setUpdateInterval(100); + startSessions(countly); + countly.RecordEvent("tap", 1); + + std::this_thread::sleep_for(std::chrono::milliseconds(800)); + CHECK_FALSE(received("events")); + CHECK(countly.checkEQSize() == 1); + CHECK(eventually([] { return received("events"); }, std::chrono::seconds(4))); + } + + SUBCASE("with manual sessions the update timer still sends the events") { + Countly &countly = prepareSessions(); + countly.enableManualSessionControl(); + countly.setAutomaticSessionUpdateInterval(1); + countly.setUpdateInterval(100); + startSessions(countly); + countly.RecordEvent("tap", 1); + + CHECK(eventually([] { return received("events"); }, std::chrono::seconds(4))); + CHECK_FALSE(received("update")); + } +} + +TEST_CASE("Session calls of the app are ignored while sessions are automatic") { + Countly &countly = prepareSessions(); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + startSessions(countly); + countly.processRQDebug(); + http_call_queue.clear(); + + CHECK_FALSE(countly.beginSession()); + CHECK_FALSE(countly.updateSession()); + CHECK_FALSE(countly.endSession()); + countly.processRQDebug(); + + CHECK(receivedKinds().empty()); + CHECK(sessionLogCode(countly, "w740") == 3); + CHECK(countly.IsSessionRunning()); +} + +TEST_CASE("Ending a session sends the waiting events first") { + Countly &countly = prepareSessions(); + countly.enableManualSessionControl(); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + startSessions(countly); + countly.beginSession(); + countly.RecordEvent("inside_the_session", 1); + countly.endSession(); + countly.processRQDebug(); + + CHECK(receivedKinds() == std::vector{"begin", "events", "end"}); +} + +TEST_CASE("stop() ends the session with its events before it stops") { + Countly &countly = prepareSessions(); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + startSessions(countly); + countly.RecordEvent("last_tap", 1); + countly.stop(); + countly.processRQDebug(); + + CHECK(receivedKinds() == std::vector{"begin", "events", "end"}); +} + +TEST_CASE("While the server turns session tracking off") { + SUBCASE("saving user details neither blocks nor drops the waiting events") { + Countly &countly = prepareSessions(); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + countly.setSDKBehaviorSettings("{\"st\":false}"); + startSessions(countly); + countly.RecordEvent("before_details", 1); + + const std::chrono::steady_clock::time_point before = std::chrono::steady_clock::now(); + countly.setUserDetails({{"name", "Alice"}}); + CHECK(std::chrono::steady_clock::now() - before < std::chrono::seconds(1)); + + countly.processRQDebug(); + CHECK(receivedKinds() == std::vector{"events", "user_details"}); + } + + SUBCASE("a running session still ends") { + clearSDK(); + release_settings = false; + Countly &countly = Countly::getInstance(); + countly.setHTTPClient(delayedSessionTrackingOff); + countly.disableHealthCheck(); + countly.setDeviceID(COUNTLY_TEST_DEVICE_ID); + countly.SetPath(TEST_DATABASE_NAME); + countly.enableManualSessionControl(); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + startSessions(countly); + countly.beginSession(); + release_settings = true; + + // once the settings arrived, a new session can't begin + REQUIRE(eventually( + [&countly] { + countly.endSession(); + countly.beginSession(); + return !countly.IsSessionRunning(); + }, + std::chrono::seconds(5))); + countly.processRQDebug(); + std::vector kinds = receivedKinds(); + REQUIRE(!kinds.empty()); + CHECK(kinds.back() == "end"); + } +} + +TEST_CASE("The first view of every session is marked") { + Countly &countly = prepareSessions(); + countly.enableManualSessionControl(); + countly.enableImmediateRequestOnStop(); + countly.setUpdateInterval(600000); + startSessions(countly); + + countly.views().openView("BeforeAnySession"); + countly.beginSession(); + countly.views().openView("FirstOfFirstSession"); + countly.views().openView("SecondOfFirstSession"); + countly.endSession(); + countly.beginSession(); + countly.views().openView("FirstOfSecondSession"); + countly.flushEvents(); + countly.processRQDebug(); + + std::map marked; + for (size_t index = 0; index < http_call_queue.size(); index++) { + HTTPCall call = http_call_queue.at(index); + if (call.data.count("events")) { + for (const json &event : json::parse(call.data["events"])) { + if (event["key"] == "[CLY]_view" && event["segmentation"].contains("visit")) { + marked[event["segmentation"]["name"].get()] = event["segmentation"].contains("start"); + } + } + } + } + CHECK(marked == std::map{{"BeforeAnySession", false}, {"FirstOfFirstSession", true}, {"SecondOfFirstSession", false}, {"FirstOfSecondSession", true}}); +} diff --git a/tests/storage_db.cpp b/tests/storage_db.cpp index 4a55b82..c6e9a5b 100644 --- a/tests/storage_db.cpp +++ b/tests/storage_db.cpp @@ -91,7 +91,7 @@ TEST_CASE("SQLite builds keep events and requests in memory when there is no usa countly.addEvent(cly::Event("event_without_database_path", 1)); CHECK(countly.checkEQSize() == 1); - CHECK(eventually([&countly]() { countly.updateSession(); countly.processRQDebug(); }, []() { return test_utils::httpQueueContainsEvent("event_without_database_path"); })); + CHECK(eventually([&countly]() { countly.flushEvents(); countly.processRQDebug(); }, []() { return test_utils::httpQueueContainsEvent("event_without_database_path"); })); } SUBCASE("the database cannot be opened") { @@ -101,7 +101,7 @@ TEST_CASE("SQLite builds keep events and requests in memory when there is no usa countly.addEvent(cly::Event("event_after_database_failure", 1)); CHECK(countly.checkEQSize() == 1); - CHECK(eventually([&countly]() { countly.updateSession(); countly.processRQDebug(); }, []() { return test_utils::httpQueueContainsEvent("event_after_database_failure"); })); + CHECK(eventually([&countly]() { countly.flushEvents(); countly.processRQDebug(); }, []() { return test_utils::httpQueueContainsEvent("event_after_database_failure"); })); } countly.setLogger(nullptr); @@ -116,8 +116,8 @@ TEST_CASE("SQLite-build instances without database paths run side by side") { first.sdk->addEvent(cly::Event("first_app_event", 1)); second.sdk->addEvent(cly::Event("second_app_event", 1)); - CHECK(eventually([&first]() { first.sdk->updateSession(); first.flush(); }, [&first]() { return first.sawEvent("first_app_event"); })); - CHECK(eventually([&second]() { second.sdk->updateSession(); second.flush(); }, [&second]() { return second.sawEvent("second_app_event"); })); + CHECK(eventually([&first]() { first.sdk->flushEvents(); first.flush(); }, [&first]() { return first.sawEvent("first_app_event"); })); + CHECK(eventually([&second]() { second.sdk->flushEvents(); second.flush(); }, [&second]() { return second.sawEvent("second_app_event"); })); CHECK_FALSE(first.sawEvent("second_app_event")); CHECK_FALSE(second.sawEvent("first_app_event")); } @@ -148,7 +148,7 @@ TEST_CASE("SQLite builds with a database path keep queued events across a restar after_restart->setDatabasePath(database_path); after_restart->start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); REQUIRE(after_restart->checkEQSize() >= 0); - CHECK(eventually([&after_restart]() { after_restart->updateSession(); after_restart->processRQDebug(); }, []() { return test_utils::httpQueueContainsEvent("event_before_restart"); })); + CHECK(eventually([&after_restart]() { after_restart->flushEvents(); after_restart->processRQDebug(); }, []() { return test_utils::httpQueueContainsEvent("event_before_restart"); })); after_restart.reset(); remove(database_path.c_str()); diff --git a/tests/test_utils.hpp b/tests/test_utils.hpp index 300a18b..384074e 100644 --- a/tests/test_utils.hpp +++ b/tests/test_utils.hpp @@ -187,7 +187,8 @@ static HTTPResponse fakeSendHTTPInto(ThreadSafeHTTPCallQueue &queue, bool use_po if (http_call.url == "/i") { response.success = true; - } else if ((http_call.url == "/o/sdk") && (http_call.data["method"] == "fetch_remote_config")) { + response.data = {{"result", "Success"}}; + } else if ((http_call.url == "/o/sdk") && (http_call.data["method"] == "rc")) { nlohmann::json remote_config = {{"color", "#FF9900"}, {"playerQueueTimeout", 32}, {"isChristmas", true}}; response.success = true; diff --git a/tests/user_details.cpp b/tests/user_details.cpp index 61447dc..345f02d 100644 --- a/tests/user_details.cpp +++ b/tests/user_details.cpp @@ -41,7 +41,6 @@ static void startForUserDetails(Countly &countly, const json &sbs = json::object countly.SetPath(TEST_DATABASE_NAME); countly.enableManualSessionControl(); countly.start(COUNTLY_TEST_APP_KEY, COUNTLY_TEST_HOST, COUNTLY_TEST_PORT, false); - waitForConfigFetch(); countly.clearRequestQueue(); http_call_queue.clear(); } diff --git a/tests/views.cpp b/tests/views.cpp index 6aef52d..3dd6f29 100644 --- a/tests/views.cpp +++ b/tests/views.cpp @@ -49,7 +49,7 @@ TEST_CASE("recording views") { Countly &ct = Countly::getInstance(); ct.SetPath(TEST_DATABASE_NAME); ct.setDeviceID("test-device-id"); - ct.start("YOUR_APP_KEY", "https://try.count.ly", 443, false); + ct.start(COUNTLY_TEST_APP_KEY, "https://try.count.ly", 443, false); /* * It validates the views recorded without segmentation. @@ -228,7 +228,7 @@ TEST_CASE("views opened and closed from several threads at once") { ct.setHTTPClient(test_utils::fakeSendHTTP); ct.SetPath(TEST_DATABASE_NAME); ct.setDeviceID("test-device-id"); - ct.start("YOUR_APP_KEY", "https://try.count.ly", 443, false); + ct.start(COUNTLY_TEST_APP_KEY, "https://try.count.ly", 443, false); // Keep every view event in the event queue so all of them can be counted. ct.setEventsToRQThreshold(10000); REQUIRE(ct.debugReturnStateOfEQ().size() == 0);