diff --git a/.changes/reconnect-delay-jitter b/.changes/reconnect-delay-jitter new file mode 100644 index 000000000..04f98c97f --- /dev/null +++ b/.changes/reconnect-delay-jitter @@ -0,0 +1 @@ +patch type="fixed" "Reconnect delays now carry proportional jitter, so clients disconnected by the same event no longer retry in lockstep. The first retry draws from a small random window instead of firing immediately, and later retries are scaled by a random factor between 0.5 and 1.5" diff --git a/lib/src/core/engine.dart b/lib/src/core/engine.dart index 7ac40969f..0f46a92f8 100644 --- a/lib/src/core/engine.dart +++ b/lib/src/core/engine.dart @@ -66,6 +66,28 @@ const defaultRetryDelaysInMs = [ maxRetryDelay, ]; +/// Upper bound of the random delay used for the first retry, whose base delay is 0. +/// +/// Kept well under the 300ms of the second retry: the first attempt exists to recover from a +/// brief blip straight away, so the window only has to be wide enough to pull a herd of clients +/// apart, not to back off. +const maxFirstRetryJitterInMs = 150; + +/// The delay to wait before reconnect attempt [retryCount], with jitter applied +/// so that clients disconnected by the same event do not retry in lockstep. +@internal +int reconnectDelayInMs(int retryCount) { + final retryDelay = defaultRetryDelaysInMs[retryCount]; + + // A zero base delay has nothing to scale, so it draws from a fixed window instead. + if (retryDelay == 0) { + return (math.Random().nextDouble() * maxFirstRetryJitterInMs).round(); + } + + // Jitter scales with the delay (+/-50%) so clients spread out further on later retries. + return (retryDelay * (0.5 + math.Random().nextDouble())).round(); +} + class Engine extends Disposable with EventsEmittable { static const _lossyDCLabel = '_lossy'; static const _reliableDCLabel = '_reliable'; @@ -1118,11 +1140,7 @@ class Engine extends Disposable with EventsEmittable { return; } - var delay = defaultRetryDelaysInMs[_reconnectAttempts]; - // Add random jitter to prevent thundering herd on reconnect - if (_reconnectAttempts > 1) { - delay += math.Random().nextInt(1000); - } + final delay = reconnectDelayInMs(_reconnectAttempts); events.emit( EngineAttemptReconnectEvent( diff --git a/test/core/reconnect_delay_test.dart b/test/core/reconnect_delay_test.dart new file mode 100644 index 000000000..abd2fa36f --- /dev/null +++ b/test/core/reconnect_delay_test.dart @@ -0,0 +1,71 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:livekit_client/src/core/engine.dart'; + +void main() { + group('reconnect delay', () { + test('spreads out clients that disconnect at the same moment', () { + // The first retry's base delay is 0, so without jitter every client + // disconnected by the same event reconnects at the same instant. + final observed = List.generate(100, (_) => reconnectDelayInMs(0)).toSet(); + + expect( + observed.length, + greaterThan(1), + reason: 'every caller got the same first retry delay: $observed', + ); + }); + + test('keeps the first retry inside a bounded window', () { + for (var i = 0; i < 1000; i++) { + final delay = reconnectDelayInMs(0); + + expect(delay, greaterThanOrEqualTo(0)); + // Inclusive: the window is rounded to whole milliseconds, so the top of + // the range rounds up to the bound itself. + expect(delay, lessThanOrEqualTo(maxFirstRetryJitterInMs)); + } + }); + + test('scales jitter with the base delay', () { + for (var retryCount = 1; retryCount < defaultRetryDelaysInMs.length; retryCount++) { + final base = defaultRetryDelaysInMs[retryCount]; + + for (var i = 0; i < 100; i++) { + final delay = reconnectDelayInMs(retryCount); + + expect( + delay, + greaterThanOrEqualTo((base * 0.5).round()), + reason: 'retry $retryCount fell below half of $base', + ); + expect( + delay, + lessThanOrEqualTo((base * 1.5).round()), + reason: 'retry $retryCount exceeded 1.5x of $base', + ); + } + } + }); + + test('later retries also vary between callers', () { + final observed = List.generate(100, (_) => reconnectDelayInMs(1)).toSet(); + + expect(observed.length, greaterThan(1)); + }); + }); +}