From bf07b2065f250ad39f02c68a4ed13f330668af66 Mon Sep 17 00:00:00 2001 From: Shijing Xian Date: Fri, 23 Oct 2026 15:22:52 -0700 Subject: [PATCH 1/6] Add proportional jitter to the reconnect delays The first entry of defaultRetryDelaysInMs is 0 and jitter only applied from the third attempt onwards, so every client disconnected by the same event reconnected at the same instant, then again in lockstep 300ms later. Only from the third attempt did they start to spread. The jitter that was applied was also additive (a flat 0-1000ms), which is disproportionate at both ends: it swamps a 300ms delay and barely moves a 7000ms one. Extract the delay into reconnectDelayInMs and scale each delay by a random factor in [0.5, 1.5). The first retry has no base delay to scale, so it draws from a fixed 0-500ms window; a zero at any later index still passes through as an immediate retry. Port of livekit/client-sdk-js#2124. Co-Authored-By: Claude Opus 5 (1M context) --- lib/src/core/engine.dart | 26 +++++++++-- test/core/reconnect_delay_test.dart | 71 +++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 test/core/reconnect_delay_test.dart diff --git a/lib/src/core/engine.dart b/lib/src/core/engine.dart index 7ac40969f..83687a7a9 100644 --- a/lib/src/core/engine.dart +++ b/lib/src/core/engine.dart @@ -66,6 +66,26 @@ const defaultRetryDelaysInMs = [ maxRetryDelay, ]; +/// Upper bound of the random delay used for the first retry, whose base delay is 0. It keeps a +/// brief network blip recovering quickly while spreading out clients that disconnect together. +const maxFirstRetryJitterInMs = 500; + +/// The delay to wait before reconnect attempt [retryCount], with jitter applied +/// so that clients disconnected by the same event do not retry in lockstep. +@internal +int reconnectDelayInMs(int retryCount) { + final retryDelay = defaultRetryDelaysInMs[retryCount]; + + // The first retry has no base delay to scale, so it gets a fixed window instead. A zero at a + // later index is an explicit request for an immediate retry and passes through the formula as zero. + if (retryCount == 0 && retryDelay == 0) { + return (math.Random().nextDouble() * maxFirstRetryJitterInMs).round(); + } + + // Jitter scales with the delay (+/-50%) so clients spread out further on later retries. + return (retryDelay * (0.5 + math.Random().nextDouble())).round(); +} + class Engine extends Disposable with EventsEmittable { static const _lossyDCLabel = '_lossy'; static const _reliableDCLabel = '_reliable'; @@ -1118,11 +1138,7 @@ class Engine extends Disposable with EventsEmittable { return; } - var delay = defaultRetryDelaysInMs[_reconnectAttempts]; - // Add random jitter to prevent thundering herd on reconnect - if (_reconnectAttempts > 1) { - delay += math.Random().nextInt(1000); - } + final delay = reconnectDelayInMs(_reconnectAttempts); events.emit( EngineAttemptReconnectEvent( diff --git a/test/core/reconnect_delay_test.dart b/test/core/reconnect_delay_test.dart new file mode 100644 index 000000000..ad0fd0da5 --- /dev/null +++ b/test/core/reconnect_delay_test.dart @@ -0,0 +1,71 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:livekit_client/src/core/engine.dart'; + +void main() { + group('reconnect delay', () { + test('spreads out clients that disconnect at the same moment', () { + // The first retry's base delay is 0, so without jitter every client + // disconnected by the same event reconnects at the same instant. + final observed = List.generate(100, (_) => reconnectDelayInMs(0)).toSet(); + + expect( + observed.length, + greaterThan(1), + reason: 'every caller got the same first retry delay: $observed', + ); + }); + + test('keeps the first retry inside a bounded window', () { + for (var i = 0; i < 100; i++) { + final delay = reconnectDelayInMs(0); + + expect(delay, greaterThanOrEqualTo(0)); + expect(delay, lessThan(500)); + } + }); + + test('scales jitter with the base delay', () { + for (var retryCount = 1; + retryCount < defaultRetryDelaysInMs.length; + retryCount++) { + final base = defaultRetryDelaysInMs[retryCount]; + + for (var i = 0; i < 100; i++) { + final delay = reconnectDelayInMs(retryCount); + + expect( + delay, + greaterThanOrEqualTo((base * 0.5).round()), + reason: 'retry $retryCount fell below half of $base', + ); + expect( + delay, + lessThanOrEqualTo((base * 1.5).round()), + reason: 'retry $retryCount exceeded 1.5x of $base', + ); + } + } + }); + + test('later retries also vary between callers', () { + final observed = List.generate(100, (_) => reconnectDelayInMs(1)).toSet(); + + expect(observed.length, greaterThan(1)); + }); + }); +} From 6f887b8e585bbf10757ec047e2a8886a67cb23aa Mon Sep 17 00:00:00 2001 From: Shijing Xian Date: Fri, 23 Oct 2026 16:52:18 -0700 Subject: [PATCH 2/6] Apply dart format Co-Authored-By: Claude Opus 5 (1M context) --- test/core/reconnect_delay_test.dart | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/core/reconnect_delay_test.dart b/test/core/reconnect_delay_test.dart index ad0fd0da5..eba1e7757 100644 --- a/test/core/reconnect_delay_test.dart +++ b/test/core/reconnect_delay_test.dart @@ -40,9 +40,7 @@ void main() { }); test('scales jitter with the base delay', () { - for (var retryCount = 1; - retryCount < defaultRetryDelaysInMs.length; - retryCount++) { + for (var retryCount = 1; retryCount < defaultRetryDelaysInMs.length; retryCount++) { final base = defaultRetryDelaysInMs[retryCount]; for (var i = 0; i < 100; i++) { From dbf62f09dcfb129557520de454885df25f59dfed Mon Sep 17 00:00:00 2001 From: Shijing Xian Date: Fri, 23 Oct 2026 16:53:09 -0700 Subject: [PATCH 3/6] Fix a flaky bound in the first-retry jitter test The delay is rounded to whole milliseconds, so a draw near the top of the window rounds up to the bound itself and the exclusive assertion failed roughly one run in a thousand. Assert the inclusive bound, and raise the iteration count so the edge is actually exercised. Co-Authored-By: Claude Opus 5 (1M context) --- test/core/reconnect_delay_test.dart | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/core/reconnect_delay_test.dart b/test/core/reconnect_delay_test.dart index eba1e7757..abd2fa36f 100644 --- a/test/core/reconnect_delay_test.dart +++ b/test/core/reconnect_delay_test.dart @@ -31,11 +31,13 @@ void main() { }); test('keeps the first retry inside a bounded window', () { - for (var i = 0; i < 100; i++) { + for (var i = 0; i < 1000; i++) { final delay = reconnectDelayInMs(0); expect(delay, greaterThanOrEqualTo(0)); - expect(delay, lessThan(500)); + // Inclusive: the window is rounded to whole milliseconds, so the top of + // the range rounds up to the bound itself. + expect(delay, lessThanOrEqualTo(maxFirstRetryJitterInMs)); } }); From ae1560fb1d39fbe318cd5b4ed5941a579b4b2480 Mon Sep 17 00:00:00 2001 From: Shijing Xian Date: Fri, 23 Oct 2026 16:58:45 -0700 Subject: [PATCH 4/6] Shorten the first-retry jitter window and add the release entry 500ms is longer than the 300ms that follows it in this schedule, which delays recovery from a brief blip for no added benefit. The window only has to pull a herd of clients apart, so 150ms is enough. Co-Authored-By: Claude Opus 5 (1M context) --- .changes/reconnect-delay-jitter | 1 + lib/src/core/engine.dart | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 .changes/reconnect-delay-jitter diff --git a/.changes/reconnect-delay-jitter b/.changes/reconnect-delay-jitter new file mode 100644 index 000000000..04f98c97f --- /dev/null +++ b/.changes/reconnect-delay-jitter @@ -0,0 +1 @@ +patch type="fixed" "Reconnect delays now carry proportional jitter, so clients disconnected by the same event no longer retry in lockstep. The first retry draws from a small random window instead of firing immediately, and later retries are scaled by a random factor between 0.5 and 1.5" diff --git a/lib/src/core/engine.dart b/lib/src/core/engine.dart index 83687a7a9..51609ac6d 100644 --- a/lib/src/core/engine.dart +++ b/lib/src/core/engine.dart @@ -66,9 +66,13 @@ const defaultRetryDelaysInMs = [ maxRetryDelay, ]; -/// Upper bound of the random delay used for the first retry, whose base delay is 0. It keeps a -/// brief network blip recovering quickly while spreading out clients that disconnect together. -const maxFirstRetryJitterInMs = 500; +/// Upper bound of the random delay used for the first retry, whose base delay is 0. +/// +/// Kept well under the 300ms of the second retry: the first attempt exists to recover from a +/// brief blip straight away, so the window only has to be wide enough to pull a herd of clients +/// apart, not to back off. The JS SDK uses 500ms here, which is longer than the next step in +/// this schedule. +const maxFirstRetryJitterInMs = 150; /// The delay to wait before reconnect attempt [retryCount], with jitter applied /// so that clients disconnected by the same event do not retry in lockstep. From 104686c68a5ec6d9f8a63dc304a77f702cfe6884 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:13:26 +0800 Subject: [PATCH 5/6] Jitter any zero base delay in the reconnect schedule defaultRetryDelaysInMs is a fixed const with no policy hook, so the "explicit immediate retry at a later index" case carried over from the JS SDK cannot happen here. Key the fixed window on the zero base delay alone, which is the actual reason it is needed. --- lib/src/core/engine.dart | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/src/core/engine.dart b/lib/src/core/engine.dart index 51609ac6d..cb85b99b4 100644 --- a/lib/src/core/engine.dart +++ b/lib/src/core/engine.dart @@ -80,9 +80,8 @@ const maxFirstRetryJitterInMs = 150; int reconnectDelayInMs(int retryCount) { final retryDelay = defaultRetryDelaysInMs[retryCount]; - // The first retry has no base delay to scale, so it gets a fixed window instead. A zero at a - // later index is an explicit request for an immediate retry and passes through the formula as zero. - if (retryCount == 0 && retryDelay == 0) { + // A zero base delay has nothing to scale, so it draws from a fixed window instead. + if (retryDelay == 0) { return (math.Random().nextDouble() * maxFirstRetryJitterInMs).round(); } From 58105f03e5efd2bea0071e0e6e8ceae7210e5d2c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:13:31 +0800 Subject: [PATCH 6/6] Drop the JS SDK value from the first-retry jitter doc The other SDK's constant can change independently and the comment would go stale. The comparison belongs in the PR discussion, not the source. --- lib/src/core/engine.dart | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/src/core/engine.dart b/lib/src/core/engine.dart index cb85b99b4..0f46a92f8 100644 --- a/lib/src/core/engine.dart +++ b/lib/src/core/engine.dart @@ -70,8 +70,7 @@ const defaultRetryDelaysInMs = [ /// /// Kept well under the 300ms of the second retry: the first attempt exists to recover from a /// brief blip straight away, so the window only has to be wide enough to pull a herd of clients -/// apart, not to back off. The JS SDK uses 500ms here, which is longer than the next step in -/// this schedule. +/// apart, not to back off. const maxFirstRetryJitterInMs = 150; /// The delay to wait before reconnect attempt [retryCount], with jitter applied