Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .changes/reconnect-delay-jitter
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
patch type="fixed" "Reconnect delays now carry proportional jitter, so clients disconnected by the same event no longer retry in lockstep. The first retry draws from a small random window instead of firing immediately, and later retries are scaled by a random factor between 0.5 and 1.5"
28 changes: 23 additions & 5 deletions lib/src/core/engine.dart
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,28 @@ const defaultRetryDelaysInMs = [
maxRetryDelay,
];

/// Upper bound of the random delay used for the first retry, whose base delay is 0.
///
/// Kept well under the 300ms of the second retry: the first attempt exists to recover from a
/// brief blip straight away, so the window only has to be wide enough to pull a herd of clients
/// apart, not to back off.
const maxFirstRetryJitterInMs = 150;

/// The delay to wait before reconnect attempt [retryCount], with jitter applied
/// so that clients disconnected by the same event do not retry in lockstep.
@internal
int reconnectDelayInMs(int retryCount) {
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
final retryDelay = defaultRetryDelaysInMs[retryCount];

// A zero base delay has nothing to scale, so it draws from a fixed window instead.
if (retryDelay == 0) {
return (math.Random().nextDouble() * maxFirstRetryJitterInMs).round();
Comment thread
xianshijing-lk marked this conversation as resolved.
}

// Jitter scales with the delay (+/-50%) so clients spread out further on later retries.
return (retryDelay * (0.5 + math.Random().nextDouble())).round();
}

class Engine extends Disposable with EventsEmittable<EngineEvent> {
static const _lossyDCLabel = '_lossy';
static const _reliableDCLabel = '_reliable';
Expand Down Expand Up @@ -1118,11 +1140,7 @@ class Engine extends Disposable with EventsEmittable<EngineEvent> {
return;
}

var delay = defaultRetryDelaysInMs[_reconnectAttempts];
// Add random jitter to prevent thundering herd on reconnect
if (_reconnectAttempts > 1) {
delay += math.Random().nextInt(1000);
}
final delay = reconnectDelayInMs(_reconnectAttempts);

events.emit(
EngineAttemptReconnectEvent(
Expand Down
71 changes: 71 additions & 0 deletions test/core/reconnect_delay_test.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
// Copyright 2026 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import 'package:flutter_test/flutter_test.dart';

import 'package:livekit_client/src/core/engine.dart';

void main() {
group('reconnect delay', () {
test('spreads out clients that disconnect at the same moment', () {
// The first retry's base delay is 0, so without jitter every client
// disconnected by the same event reconnects at the same instant.
final observed = List.generate(100, (_) => reconnectDelayInMs(0)).toSet();

expect(
observed.length,
greaterThan(1),
reason: 'every caller got the same first retry delay: $observed',
);
});

test('keeps the first retry inside a bounded window', () {
for (var i = 0; i < 1000; i++) {
final delay = reconnectDelayInMs(0);

expect(delay, greaterThanOrEqualTo(0));
// Inclusive: the window is rounded to whole milliseconds, so the top of
// the range rounds up to the bound itself.
expect(delay, lessThanOrEqualTo(maxFirstRetryJitterInMs));
}
});

test('scales jitter with the base delay', () {
for (var retryCount = 1; retryCount < defaultRetryDelaysInMs.length; retryCount++) {
final base = defaultRetryDelaysInMs[retryCount];

for (var i = 0; i < 100; i++) {
final delay = reconnectDelayInMs(retryCount);

expect(
delay,
greaterThanOrEqualTo((base * 0.5).round()),
reason: 'retry $retryCount fell below half of $base',
);
expect(
delay,
lessThanOrEqualTo((base * 1.5).round()),
reason: 'retry $retryCount exceeded 1.5x of $base',
);
}
}
});

test('later retries also vary between callers', () {
final observed = List.generate(100, (_) => reconnectDelayInMs(1)).toSet();

expect(observed.length, greaterThan(1));
});
});
}
Loading