diff --git a/.claude/skills/create-implementation-plan-for-redis-api-change/SKILL.md b/.claude/skills/create-implementation-plan-for-redis-api-change/SKILL.md index 4c32691ee..a13cadc9d 100644 --- a/.claude/skills/create-implementation-plan-for-redis-api-change/SKILL.md +++ b/.claude/skills/create-implementation-plan-for-redis-api-change/SKILL.md @@ -234,8 +234,8 @@ present (write "none" rather than omitting one): malformed), the `RoundTripUnitTests` cases (exact bytes), the live tests written-not-run with `Create(require: RedisFeatures.vX_Y_Z)`, the HLD section 8 scenarios copied and marked `expected`, and the exact commands: - `dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter "FullyQualifiedName~"` - and, when the analyzer changed, `dotnet test tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj`. + `dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter-class "**"` + and, when the analyzer changed, `dotnet test --project tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj`. 7. **Docs / changelog / public-API files** - `docs/.md` + `docs/index.md` yes/no, `docs/exp/SERxxx.md` yes/no, the `PublicAPI` files, and the note that `docs/ReleaseNotes.md` is frozen. diff --git a/.claude/skills/implement-resp-command/SKILL.md b/.claude/skills/implement-resp-command/SKILL.md index dff1b62f5..9223d8cad 100644 --- a/.claude/skills/implement-resp-command/SKILL.md +++ b/.claude/skills/implement-resp-command/SKILL.md @@ -145,6 +145,6 @@ The in-process managed server (`toys/StackExchange.Redis.Server`) may also need ## Before finishing - `dotnet build Build.csproj -c Release /p:CI=true` — analyzers + `TreatWarningsAsErrors` must pass (this catches a missing `PublicAPI.Unshipped.txt` entry). -- `dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter "FullyQualifiedName~MyCommand"` — runs your new unit tests without any server. -- `dotnet test tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj` — if you touched `TransactionAnalyzer`. Also needs no server, and takes seconds. +- `dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter-class "*MyCommand*"` — runs your new unit tests without any server (Microsoft.Testing.Platform filters take fully qualified names, `*` at either end). +- `dotnet test --project tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj` — if you touched `TransactionAnalyzer`. Also needs no server, and takes seconds. - Double-check no shipped signature changed (back-compat). diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 733c57a65..616674d66 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -71,42 +71,46 @@ runs: - name: StackExchange.Redis.Tests shell: bash + # Microsoft.Testing.Platform `dotnet test` (selected in global.json); the GitHub Actions reporter is a platform + # extension that is off by default, and the TRX report feeds dorny/test-reporter below (the platform's + # reporter, not xunit's --report-xunit-trx, which truncated the file in 4.0.2) run: >- - dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj + dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj --no-build -c Release -f net10.0 - --logger trx - --logger "GitHubActions;summary-include-passed=false;summary-include-skipped=false" --results-directory ./test-results/ - --blame-hang-timeout 10m - --blame-hang-dump-type none - /p:CI=true + --report-trx + --report-github + --report-github-summary-include-passed false + --report-github-summary-include-skipped false # Neither of these needs a server, and between them they took about four seconds - but nothing ran # them until now, so a namespace rename silently broke all 18 SER309 tests and nobody found out. - name: RESPite.Tests shell: bash run: >- - dotnet test tests/RESPite.Tests/RESPite.Tests.csproj + dotnet test --project tests/RESPite.Tests/RESPite.Tests.csproj --no-build -c Release -f net10.0 - --logger trx - --logger "GitHubActions;summary-include-passed=false;summary-include-skipped=false" --results-directory ./test-results/ - /p:CI=true + --report-trx + --report-github + --report-github-summary-include-passed false + --report-github-summary-include-skipped false - name: StackExchange.Redis.Build.Tests shell: bash run: >- - dotnet test tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj + dotnet test --project tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj --no-build -c Release - --logger trx - --logger "GitHubActions;summary-include-passed=false;summary-include-skipped=false" --results-directory ./test-results/ - /p:CI=true + --report-trx + --report-github + --report-github-summary-include-passed false + --report-github-summary-include-skipped false - uses: dorny/test-reporter@v3 continue-on-error: true diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f98cbcb10..7d8a9dfe1 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -274,14 +274,17 @@ jobs: exit 1 } - name: StackExchange.Redis.Tests - # A hung test otherwise holds the job until GitHub's six-hour limit, and says nothing about which - # test it was. --blame-hang names it (in the run's Sequence file and the log) and kills it, so the - # rest still report; the step timeout is the backstop if even that does not return. + # A hung test otherwise holds the job until GitHub's six-hour limit; the step timeout is the backstop. + # (VSTest's --blame-hang named the hung test and killed it; the Microsoft.Testing.Platform runner has + # no equivalent without its HangDump extension, so for now the timeout is all there is.) timeout-minutes: 30 run: | $exitCode = 0 foreach ($tfm in @("net10.0", "net481")) { - dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj --no-build -c Release -f $tfm --logger trx --logger GitHubActions --results-directory ./test-results/ --blame-hang-timeout 10m --blame-hang-dump-type none /p:CI=true + # Microsoft.Testing.Platform `dotnet test` (selected in global.json); the GitHub Actions reporter is a platform + # extension that is off by default, and the TRX report feeds dorny/test-reporter below (the platform's + # reporter, not xunit's --report-xunit-trx, which truncated the file in 4.0.2) + dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj --no-build -c Release -f $tfm --results-directory ./test-results/ --report-trx --report-github --report-github-summary-include-passed false --report-github-summary-include-skipped false if ($LASTEXITCODE -ne 0) { $exitCode = $LASTEXITCODE } diff --git a/AGENTS.md b/AGENTS.md index 2f07ec7b5..5f484d71e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ StackExchange.Redis — a high-performance .NET client for RESP servers (Redis, - `toys/` — runnable samples and an in-process RESP server (`StackExchange.Redis.Server`, used by tests as a managed fake server), Kestrel-hosted server, console tools. - `docs/` — published documentation site source (markdown). `docs/ReleaseNotes.md` is the changelog (frozen at 3.0 — from v3 onward release notes live in GitHub Releases). -`Build.csproj` is a `Microsoft.Build.Traversal` project that references everything under `eng/src/tests/toys` — build/test/pack it to act on the whole repo. `StackExchange.Redis.slnx` is the IDE solution (XML SLNX format, not classic `.sln`). +`Build.csproj` is a `Microsoft.Build.Traversal` project that references everything under `eng/src/tests/toys` — build/pack it to act on the whole repo. `StackExchange.Redis.slnx` is the IDE solution (XML SLNX format, not classic `.sln`); it is also what `dotnet test --solution` runs, because tests use the Microsoft.Testing.Platform runner (selected in `global.json`), which cannot see a traversal project on the .NET 10 SDK. ## Build, test, pack @@ -30,10 +30,14 @@ pwsh ./build.ps1 -StartServers docker compose --file tests/RedisConfigs/docker-compose.yml up -d --wait # Run the main test suite against one target framework (fastest inner loop) -dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -c Release -f net10.0 +dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -c Release -f net10.0 -# Run a single test class / method (xUnit v3 + Microsoft.Testing.Platform) -dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter "FullyQualifiedName~ClassName.MethodName" +# Run a single test class / method (xUnit v3 on Microsoft.Testing.Platform; names are fully qualified, '*' allowed at either end) +dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter-class StackExchange.Redis.Tests.ClassName +dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter-method StackExchange.Redis.Tests.ClassName.MethodName + +# Every test project at once (what build.ps1 does) +dotnet test --solution StackExchange.Redis.slnx -c Release --no-build # Pack the library dotnet pack src/StackExchange.Redis/StackExchange.Redis.csproj --no-build -c Release /p:Packing=true diff --git a/Directory.Packages.props b/Directory.Packages.props index 71874e2d5..06388f21b 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,15 +1,15 @@ - - - - + + + - + + - + @@ -26,35 +26,36 @@ 3.3.3 - a version that cannot see descriptors declared outside the analyzer type, as ours are in Diagnostics.cs. Pinned separately because it is a build-time analyzer we never ship: unlike the reference above, its version has nothing to do with the host floor, so upgrade at will. --> - + - + - - + + + + - + - - - + + + + - + - - - + + + - - - - + + \ No newline at end of file diff --git a/StackExchange.Redis.slnx b/StackExchange.Redis.slnx index f3a6ac32f..55d863161 100644 --- a/StackExchange.Redis.slnx +++ b/StackExchange.Redis.slnx @@ -40,6 +40,7 @@ + diff --git a/build.ps1 b/build.ps1 index 3ace75a06..09a2a264c 100644 --- a/build.ps1 +++ b/build.ps1 @@ -29,11 +29,13 @@ if ($RunTests) { & .\RedisConfigs\start-all.cmd Write-Host "Servers Started." -ForegroundColor "Green" } - Write-Host "Running tests: Build.csproj traversal (all frameworks)" -ForegroundColor "Magenta" + Write-Host "Running tests: StackExchange.Redis.slnx (all test projects)" -ForegroundColor "Magenta" if ($NetCoreOnlyTests) { - dotnet test ".\Build.csproj" -c Release -f net8.0 --no-build --logger trx + # net10.0, not net8.0: the analyzer and fault-injector test projects only target net10.0, and the + # Microsoft.Testing.Platform runner fails a project that lacks the requested framework rather than skipping it + dotnet test --solution ".\StackExchange.Redis.slnx" -c Release -f net10.0 --no-build --report-trx } else { - dotnet test ".\Build.csproj" -c Release --no-build --logger trx + dotnet test --solution ".\StackExchange.Redis.slnx" -c Release --no-build --report-trx } if ($LastExitCode -ne 0) { Write-Host "Error with tests, aborting build." -Foreground "Red" diff --git a/docs/Configuration.md b/docs/Configuration.md index ffb75df88..63f3e2f6f 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -36,6 +36,18 @@ pointing to that primary server that will automatically be updated if the primar var conn = ConnectionMultiplexer.Connect("localhost,serviceName=myprimary"); ``` +`abortConnect` applies to sentinel connections the same way it applies to everything else. With `abortConnect=true` (the default), `Connect` +throws if the sentinels cannot be reached or do not know the service. With `abortConnect=false`, `Connect` returns a multiplexer even then: +it is not connected, operations fail with a connection exception that names the problem, and it connects itself to the primary once the +sentinels become reachable and name one. If you want startup to fail when the sentinels are unavailable, keep `abortConnect=true`. + +A sentinel-managed connection tracks failovers through several signals: the `+switch-master` announcement, a reconnect poll whenever the +connection to the primary is lost, and - because the announcement is delivered over pub/sub and can be missed - a periodic check that +notices when the node last named as primary is still connected but now reports itself as a replica (a failover where nothing disconnected, +such as a primary demoted in place). In that last case the client re-asks sentinel for the current primary and follows its answer, even to +a node at an address it has never seen; while sentinel still names the demoted node (sentinel itself has not yet failed over), the client +keeps asking without tearing down or reconfiguring the healthy connections. + An overview of mapping between the `string` and `ConfigurationOptions` representation is shown below, but you can switch between them trivially: ```csharp @@ -412,6 +424,8 @@ The library determines whether to use RESP3 by: - Otherwise: RESP3 is attempted if `defaultVersion` (6.0 unless overridden) is v6 or above Note that `HELLO` is issued either way: `HELLO 2` when staying on RESP2. The reply tells us the server version, replication role, mode (standalone/sentinel/cluster) and connection -identifier, none of which we would otherwise know without `INFO` or `CONFIG GET` - both of which are in the `@dangerous` ACL category, and are commonly restricted. `HELLO` itself is +identifier, none of which we would otherwise know without `INFO` or `CONFIG GET` - both of which are in the `@dangerous` ACL category, and are commonly restricted. The role and mode +are only taken from the interactive connection (the subscription connection does no discovery), and a reported mode of `cluster` only leads to cluster *routing* once a slot map has +been obtained from `CLUSTER SLOTS`/`CLUSTER NODES`; a proxy that announces a cluster but refuses those commands is routed as a single standalone server. `HELLO` itself is in the `@connection` category. If you need to prevent it (for example when talking to a proxy that does not understand it), disable it at the command-map level with `$hello=` in the configuration string - or specify a `defaultVersion` below 6.0, since `HELLO` did not exist before then. diff --git a/docs/Failover.md b/docs/Failover.md index 15eb0841a..c91fb29af 100644 --- a/docs/Failover.md +++ b/docs/Failover.md @@ -83,6 +83,7 @@ Because the policies are immutable, there is no question of whether a change "ta | `RetryPolicy` | `MultiGroupOptions.RetryPolicy` | *(none; retry is applied per-database via `WithRetry`)* | | `HealthCheckInterval` | `MultiGroupOptions.HealthCheckInterval` | *(none; it is the group's re-evaluation cadence)* | | `FailbackDelay` | `MultiGroupOptions.FailbackDelay` | `ConnectionGroupMember.FailbackDelay` | +| `FailbackHealthCheck` | `MultiGroupOptions.FailbackHealthCheck` | `ConnectionGroupMember.FailbackHealthCheck` | | `Weight` | *(none)* | `ConnectionGroupMember.Weight` | | `SkipInitialHealthCheck` | *(none)* | `ConnectionGroupMember.SkipInitialHealthCheck` | @@ -384,7 +385,7 @@ ConnectionGroupMember[] members = [ ]; ``` -To disable periodic checking for the whole group, set `MultiGroupOptions.HealthCheckInterval` to `TimeSpan.MaxValue`; the group is then only re-evaluated in response to connection events (such as a tripped circuit-breaker). +To disable periodic checking for the whole group, set `MultiGroupOptions.HealthCheckInterval` to `TimeSpan.MaxValue`; the group is then only re-evaluated in response to connection events (a tripped circuit-breaker, or a member answering `LOADING`). ### Built-in Probes @@ -435,7 +436,7 @@ The probe policy determines how multiple probe results are evaluated to determin #### HealthCheckProbePolicy.AnySuccess -The health check passes if **any** probe succeeds. This provides the most lenient evaluation: +The health check passes if **any** probe succeeds, and fails only if at least one probe failed and none succeeded. This provides the most lenient evaluation. A run in which every probe was `Inconclusive` reports `Inconclusive`, not `Unhealthy`: ```csharp HealthCheck healthCheck = new HealthCheck.Builder @@ -448,7 +449,7 @@ HealthCheck healthCheck = new HealthCheck.Builder #### HealthCheckProbePolicy.AllSuccess (Default) -The health check passes only if **all** probes succeed. This provides the strictest evaluation: +The health check passes only if **no** probe fails and at least one succeeds. This provides the strictest evaluation. A run in which every probe was `Inconclusive` (the probe could not tell) reports `Inconclusive`, not `Healthy`: ```csharp HealthCheck healthCheck = new HealthCheck.Builder @@ -499,6 +500,8 @@ Where health checks *actively* probe each member on a timer, a **circuit breaker When a circuit breaker trips, it shuts the underlying physical connection down with `ConnectionFailureType.CircuitBreaker`. The connection then reconnects as usual, and - in a connection group - the health check and member-selection logic route traffic to other members until the affected member recovers. A circuit breaker is evaluated **per connection**, so its state is scoped to exactly the connection whose health it is measuring; a reconnect starts from a clean slate - but breaking the connection is sufficient for the group to notice non-availability. +A member that answers `-LOADING` (a server still loading its dataset after a restart) is treated the same way by a connection group: the connection is dropped with `ConnectionFailureType.Loading`, the group re-probes and re-selects immediately, and traffic moves to another member rather than queuing against one whose every reconnect will fail with the same error until the load completes. The health check brings the member back once it passes. + ### Configuring a Circuit Breaker for a Group Circuit breakers are configured for all members via `MultiGroupOptions`, alongside the health check. The setting flows into every member connection: @@ -760,7 +763,88 @@ ConnectionGroupMember[] members = [ This guards against **flapping**: a member that is intermittently failing will keep pushing its "last failure" time forward, so it never satisfies the delay and stays out of rotation until it is genuinely stable. -> Implementation note: the failback check is pure tick math on the wall clock — the last-failure time and the cutoff (`UtcNow - FailbackDelay`) are both compared as raw `long` UTC ticks, so `DateTimeKind` never enters into it. (`DateTime.Ticks` and `TimeSpan.Ticks` share the same 100 ns unit.) +> Implementation note: the `FailbackDelay` comparison is pure tick math on the wall clock — the last-failure time and the cutoff (`UtcNow - FailbackDelay`) are both compared as raw `long` UTC ticks, so `DateTimeKind` never enters into it. (`DateTime.Ticks` and `TimeSpan.Ticks` share the same 100 ns unit.) + +### Failback health checks + +`HealthCheck` answers "is this member alive". Some deployments need a second question answered before traffic +moves *to* a member: "has it caught up". In an Active-Active deployment, a region that has just come back can be +reachable but stale, and failing back onto it reads (and builds on) data that is behind. +`ConnectionGroupMember.FailbackHealthCheck` (or the group default, `MultiGroupOptions.FailbackHealthCheck`) is +that second question: + +```csharp +ConnectionGroupMember member = new("us-west.redis.example.com:6379", name: "US West") +{ + Weight = 10, + // CaughtUpProbe stands for a probe of your own; see "Custom Health Check Probes" + FailbackHealthCheck = new HealthCheck.Builder { Probe = new CaughtUpProbe(), ProbeCount = 1 }, +}; +``` + +The rules: + +- **It never removes the active member.** It is consulted only for members that are not active, and only + decides whether they may *become* active. Liveness stays with `HealthCheck`. +- **Only `Healthy` makes a member eligible.** `Inconclusive` (for example, the probe could not reach whatever + it asks) leaves the member ineligible, as does `Unhealthy`; neither marks it unhealthy. +- **A verdict is not kept while it cannot be acted on.** A member that is active, or not connected, is checked + afresh before it is next eligible. +- **If no connected member is eligible, selection falls back to liveness alone.** In a partition between + regions every member can be behind at once, and a group that refused to serve from any of them would turn a + consistency precaution into an outage. When the active member is lost, the best eligible survivor is + preferred, then the best live one. +- **A member chosen that way is provisional.** It is left for an eligible member as soon as one is + available, whatever their weights, and becomes an ordinary active member once it passes its own failback + check. Other clients never select a member that fails its lag check at all; this is the nearest + equivalent that still serves when every member is behind. +- **An explicit `TryFailoverTo(member)` overrides it**, as it does `IsUnhealthy`. + +A probe that asks about the database as a whole, rather than one endpoint, can set +`HealthCheckProbe.Scope` to `HealthCheckProbeScope.Member`, so that it runs once per member per pass rather than +once per endpoint. + +#### Redis Enterprise: lag-aware availability + +For Redis Enterprise Active-Active deployments, `HealthCheck.LagAware(...)` asks the cluster's own +[database availability API](https://redis.io/docs/latest/operate/rs/monitoring/db-availability/) whether a +member is available and caught up. The only thing it needs is the cluster's management credentials: + +```csharp +ConnectionGroupMember member = new("redis-14460.us-west.example.com:14460,password=...", name: "US West") +{ + Weight = 10, + FailbackHealthCheck = HealthCheck.LagAware(new LagAwareOptions + { + Credentials = _ => new(new NetworkCredential("probe@example.com", secrets.Get("us-west-mgmt"))), + }), +}; +``` + +What it does by default, and why: + +- **The REST endpoint is derived** from the member's first configured endpoint: a database named + `redis-{port}.{cluster}` (the form Redis Enterprise assigns) is asked via `https://{cluster}:9443/`. The + cluster name matters, because only the cluster's master answers every route the check uses; other nodes + redirect to its internal address, which is usually unreachable from outside the cluster's network. + Redirects are never followed. Set `RestEndpoint` if the database is reached through a name of your own. +- **The database is found** by matching that endpoint against the cluster's databases, using only filtered + requests (unfiltered database listings include database passwords, even for read-only roles); set + `DatabaseId` to skip this. +- **The lag question is only asked of Active-Active databases** (`LagCheck = LagCheckMode.Auto`). Asked of + any other database, the server reports it unavailable regardless of tolerance. +- **No tolerance is sent**, so the cluster's own `availability_lag_tolerance_ms` applies; set `LagTolerance` + to override it. (Other clients default to 5000 ms; the server's default is 100 ms.) +- **"Could not ask" is `Inconclusive`, not `Unhealthy`.** A management-plane problem (REST API unreachable, + credentials rejected) never fails anything over; as a failback check it simply leaves the member + ineligible. A role with `management: db_viewer` is sufficient. +- **The management certificate is usually self-signed by the cluster**, so it will not validate against the + platform trust store; `options.TrustIssuer(path)` with the cluster's own certificate (or its issuer) works, + as it does for the data plane, but it is a separate setting from the database's TLS trust. + +One limitation to plan around: measured on Redis Enterprise 8.0, when the link between regions breaks, the +lag check keeps reporting "caught up" for roughly a minute before it notices. A failback in that window is not +protected by it; a `FailbackDelay` of at least that long closes the gap. ### Anti-flap tiebreak in selection diff --git a/global.json b/global.json index 230c467f3..fd014f0e6 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,8 @@ { "sdk": { "allowPrerelease": true + }, + "test": { + "runner": "Microsoft.Testing.Platform" } } diff --git a/notes/lag-aware/api-sketch.md b/notes/lag-aware/api-sketch.md new file mode 100644 index 000000000..169039ce0 --- /dev/null +++ b/notes/lag-aware/api-sketch.md @@ -0,0 +1,260 @@ +# Lag-aware availability: API sketch + +Status: **sections 1 to 3 implemented** (2026-10-08): the failback check, member-scoped probes, and the +lag-aware probe (`HealthCheck.LagAware`, `HealthCheckProbe.LagAware`, `LagAwareOptions`, `LagCheckMode`), +with stubbed-transport tests replaying the measured response shapes. The fault-injector scenario test passes +against a live Active-Active pair. Not yet: client-certificate support for the REST call. +Evidence is in [`findings.md`](findings.md) (section 10 especially); sequence in [`plan.md`](plan.md). +*Correction:* `SER007` was retired when geo-redundant failover went GA (#3247), so the shipped +`Availability/` API is not experimental, and nothing here is either. + +Aim: parity with Lettuce, Jedis, redis-py and go-redis where it serves users, and deliberate, +documented deviations where the measurements say the others get it wrong. + +## The shape in one example + +```csharp +var east = new ConnectionGroupMember("redis-14460.c1.example.com:14460,password=...", "east") +{ + // new: consulted only before moving traffic *to* this member (see 1) + FailbackHealthCheck = HealthCheck.LagAware(new LagAwareOptions + { + Credentials = _ => new(new NetworkCredential("probe@example.com", secrets.Get("re-mgmt"))), + }), +}; +``` + +That is the whole opt-in. The REST endpoint, database uid, whether the database is Active-Active, and +the tolerance are all derived or left to the cluster (see 3). Liveness stays with the member's +existing `HealthCheck` (PING by default). + +## 1. Where the lag verdict goes: a failback-only check, not a new result value + +**Proposal:** a second, optional health check per member, consulted only when deciding whether a +member may *become* active. + +```csharp +public sealed class ConnectionGroupMember +{ + public HealthCheck? HealthCheck { get; set; } // existing: liveness, may evict + public HealthCheck? FailbackHealthCheck { get; set; } // new: eligibility, never evicts +} + +public sealed class MultiGroupOptions.Builder +{ + public HealthCheck? FailbackHealthCheck { get; set; } // new: group default, null = none +} +``` + +Rules, applied by `SelectPreferredGroup` / `UpdateState`: + +- **The active member is kept while its liveness check passes**, whatever its failback check says. +- **A member becomes eligible for selection** when its liveness check passes *and* its failback + check (if any) answers `Healthy`. `Inconclusive` from the failback check means **not eligible**: + "could not ask" must not let the group fail back onto a member that may be stale. +- **If no member is eligible, select on liveness alone.** Every member lagging (a partition between + regions, say) must degrade to "serve from somewhere", never to "serve from nowhere". Measured under a + partition (findings section 10): both members' lag checks pass for about a minute, then both fail at + 100 ms. So a region outage that lasts more than a minute leaves *no* member passing the lag check; + a strict rule (go-redis's) would allow failover in the first minute and refuse it afterwards. In + short: the failback check gates moving *away from a live active member*; it never blocks replacing a + dead one. +- **The first minute after a link breaks is blind** (measured): the lag check reports no lag for + ~60 s after sync stops. So a failback decision in that window is not protected by the lag check; + `FailbackDelay` covers some of it, and the docs must say so. +- The failback check runs on the same `HealthCheckInterval` poll as liveness, but only for members + that are not active (the active member's answer is never used). + +**Why not the new `HealthCheckResult` value** (accepted in principle, but this is better, I think): + +- One check would have to answer two different questions: "is it alive" (PING) and "is it caught up" + (REST). A new value means one probe that does both, so every lag-aware user also takes over + liveness. +- Every consumer of `HealthCheckResult` would have to learn the new value: `CollateAsync`, + `UpdateState`, all three `HealthCheckProbePolicy` implementations, and any user-written probe or + policy. Measured: they currently treat everything except `Unhealthy` as healthy, so a value they do + not know would silently mean "fine". +- It matches go-redis's `FailbackOnly()`, the only client that addresses this at all. + +**Parity note:** Lettuce, Jedis and redis-py put the lag check in the single health-check slot, so a +stale reading evicts the active member. Measured: only the member *behind* flips, which is never the +one being written to, so for a single-writer deployment that eviction does not happen, and for a +multi-writer one it would be wrong. This is a deliberate deviation. + +**As built.** One refinement, decided 2026-10-08 by "do what the other clients do": a member chosen by +the liveness fallback is **provisional**. It is left for an eligible member as soon as one is available +(whatever the weights), and its own failback check keeps running so that it becomes an ordinary active +member once it passes. Rationale: Lettuce, Jedis and redis-py treat a lagging database as unhealthy, and +go-redis never selects a lagging candidate, so none of them can end up on an unverified member while a +verified one exists; this is the nearest equivalent that still serves when every member is behind. +Tested in `FailbackHealthCheckTests`; the fixture holds all members down until every connection is up, so +that a start-up race does not pick a provisional member before the test means it to. + +Implementing it also surfaced an existing bug: `HealthCheckProbePolicy.AllSuccess` (the default) reported +`Healthy` for a run of nothing but `Inconclusive` probes, contrary to its own comment. Harmless for +liveness (both count as connected) but fatal for a failback check, where "could not ask" would have read as +"caught up". Fixed: such a run is now `Inconclusive`. The mirror-image bug in `AnySuccess` (all +`Inconclusive` reads as `Unhealthy`, which *does* affect liveness) is left alone for a separate decision. + +## 2. Running once per member, not per endpoint + +`HealthCheck.CheckHealthAsync(IConnectionMultiplexer)` calls the probe once per server. A REST probe +answers per database, so an OSS-cluster member with six endpoints would make six identical calls. + +```csharp +public enum HealthCheckProbeScope { Endpoint = 0, Member = 1 } + +public abstract class HealthCheckProbe +{ + public virtual HealthCheckProbeScope Scope => HealthCheckProbeScope.Endpoint; // new +} +``` + +A `Member`-scoped probe is called once per pass, with `HealthCheckContext.Server` set to a connected +server of that member (needed for the configured endpoint and its `RawConfig`). Adding a virtual +member is binary-safe for existing subclasses. + +## 3. The probe + +```csharp +public sealed partial class HealthCheck // existing type; new static factories +{ + public static HealthCheck LagAware(LagAwareOptions options); // extend_check=lag (if Active-Active) + public static HealthCheck DatabaseAvailability(LagAwareOptions options); // plain availability only +} + +public sealed class LagAwareOptions +{ + // required: the one thing that cannot be derived + public Func> Credentials { get; set; } + + // optional; null = derive or defer + public Uri? RestEndpoint { get; set; } // null: https://:9443 + public int? DatabaseId { get; set; } // null: discover by host and port + public LagCheckMode LagCheck { get; set; } // default Auto + public TimeSpan? LagTolerance { get; set; } // null: send none, cluster setting applies + public RemoteCertificateValidationCallback? CertificateValidation { get; set; } + public void TrustIssuer(string path); // same helper as ConfigurationOptions + public void TrustIssuer(X509Certificate2 issuer); +} + +public enum LagCheckMode { Auto = 0, Enabled = 1, Disabled = 2 } +``` + +The `HealthCheck` returned is an ordinary `HealthCheck` (so `ProbeCount`, `ProbeTimeout` and +`ProbeInterval` apply, via `new HealthCheck.Builder(HealthCheck.LagAware(...)) { ... }`), whose +probe is `Member`-scoped. + +### What it does, per pass + +1. **Endpoint:** `RestEndpoint`, else derived from the member's first configured endpoint: the cluster + FQDN when the host is `redis-.`, else the host itself. *As built:* the cluster + FQDN, not the database host, because non-master nodes redirect management routes to the master's + internal address (findings section 10); redirects are never followed. Configured endpoint, not + discovered nodes: an OSS-cluster database advertises IP addresses. +2. **uid** (once, cached; cleared on 404, as Lettuce does): `DatabaseId`, else + `GET /v1/bdbs?fields=uid,endpoints`, matched on `dns_name` and port. +3. **Active-Active** (once, cached), for `LagCheck = Auto`: `GET /v1/bdbs/{uid}?fields=uid,crdt`. +4. **Check:** `GET /v1/bdbs/{uid}/availability`, adding `extend_check=lag` when the database is + Active-Active (`Auto`) or `Enabled`, and `availability_lag_tolerance_ms` only if `LagTolerance` is set. + +**Never an unfiltered `/v1/bdbs` or `/v1/crdbs`.** Measured: those return database passwords even to +`db_viewer` when `mask_bdb_credentials` is off. + +### Result mapping + +| response | result | note | +| --- | --- | --- | +| 200 | `Healthy` | body is empty | +| 503 (any `error_code`) | `Unhealthy` | lag and "unavailable" share `bdb_unavailable` | +| 404 | `Unhealthy` | and clear the cached uid | +| 401, 403, other status | `Inconclusive` | management-plane problem; logged | +| connect / TLS failure, timeout | `Inconclusive` | as above | + +The last two rows are the deviation from every other client (all map REST failure to unhealthy). As a +failback check, `Inconclusive` already means "not eligible" (1), so the deviation only changes the +outcome where it should: a management-plane outage can no longer evict anything. + +### Defaults compared + +| | Lettuce | Jedis | redis-py | go-redis (branch) | **this** | +| --- | --- | --- | --- | --- | --- | +| activation | explicit | explicit | explicit | explicit | explicit | +| REST endpoint | supplied | supplied | supplied | derived from db host | **derived**, overridable | +| uid | discovered | discovered | discovered (unfiltered) | discovered | **discovered (filtered)**, overridable | +| lag check | on | on | on | on | **only if Active-Active** (`Auto`) | +| tolerance | 5000 ms | 5000 ms | 5000 ms | 5000 ms | **unset: cluster's own** (100 ms measured) | +| REST failure | unhealthy | unhealthy | unhealthy | unhealthy | **inconclusive** | +| evicts active member | yes | yes | yes | no (failback only) | **no (failback only)** | +| credentials | rotating supplier | supplier | static | static | **rotating callback** | +| TLS | own truststore | own truststore | own CA / verify | own `tls.Config` | **same callback type as Redis**, separate setting | + +## 4. Activation + +Explicit, as in every other client: setting `FailbackHealthCheck`. Management credentials are a +separate secret from the Redis password, so no options provider can supply them, and nothing else +needs configuring. What a provider *can* do, later and separately: `RedisEnterpriseOptionsProvider` +could carry a group-wide default once credentials have a home in configuration. Not proposed now. + +## 5. Platform + +| TFM | | +| --- | --- | +| `net8.0`, `net10.0` | everything | +| `net472`, `netstandard2.0` | everything; JSON (steps 2 and 3, and `error_code` for logging) via a `System.Text.Json` package reference on these targets only - in-box on `net8.0`+. *Decided 2026-10-07:* NRedisStack, which builds on SE.Redis, already depends on it | +| `net461` | works with ambient trust; setting `CertificateValidation`/`TrustIssuer` throws `PlatformNotSupportedException` at creation, on the understanding that `ServerCertificateCustomValidationCallback` needs .NET Framework 4.7.1 (*unverified*; the build will say) | + +Test seam: an internal `Func` on `LagAwareOptions`, reached via +`InternalsVisibleTo`, so tests replay captured responses without a listener. Kept internal so no +`System.Net.Http` type enters the public API; Lettuce's public `HttpClient` constructor shows there is +demand, and it can be made public later. + +## Open questions, compared with the other clients + +Read from source 2026-10-07: Lettuce `114a3ef`, Jedis `bb5f01d`, redis-py `c3bb0e6`, go-redis +`feature/multidb-integration` `9377b5b`. (Correction to the earlier survey: redis-py's sync client +imports the same `LagAwareHealthCheck`, so it is not asyncio-only.) + +1. ~~**JSON on older targets**~~ *Decided:* `System.Text.Json`, see 5. + +2. **Names.** + + | | the check | general abstraction | returning to the preferred database | failback-only check | + | --- | --- | --- | --- | --- | + | Lettuce | `LagAwareStrategy` (`databaseAvailability` / `lagAware` / `lagAwareWithTolerance`) | `HealthCheckStrategy` | `failbackSupported`, `failbackCheckInterval`, `gracePeriod` | none | + | Jedis | `LagAwareStrategy`, same factories | `HealthCheckStrategy` | same three as Lettuce | none | + | redis-py | `LagAwareHealthCheck` | `HealthCheck` | `auto_fallback_interval`, `grace_period` | none | + | go-redis | `LagAwareHealthCheck`, `NewLagAwareHealthCheck(WithLagAware...)` | `MultiDBHealthCheck` | `AutoFallbackInterval`, `GracePeriod` | `FailbackOnly() bool`, a marker on the check | + + *Recommendation:* `LagAware` is universal, keep it, with `DatabaseAvailability` alongside as + Lettuce/Jedis have. Use "failback", not "fallback": it is our existing word (`FailbackDelay`) and + the Java clients'. `FailbackHealthCheck` as a separate slot rather than go-redis's marker, because + our member has one `HealthCheck` rather than a list, and a slot keeps liveness independent. + +3. **Credentials.** + + | | type | rotates | client certificate to REST | + | --- | --- | --- | --- | + | Lettuce | `Supplier`, read per request | yes | via `SslOptions` keystore (*inferred*) | + | Jedis | `Supplier`, read per call | yes | via `SslOptions` keystore (*inferred*) | + | redis-py | `auth_basic: Tuple[str, str]` | no | `client_cert_file` / `client_key_file` / `client_key_password` | + | go-redis | `WithLagAwareBasicAuth(user, password)` | no | `WithLagAwareClientCert(PEM)` / `...FromFiles` | + + *Recommendation:* keep the callback returning `NetworkCredential` (rotation parity with the Java + clients; in-box everywhere), and add client-certificate support as its own option, mirroring + `ConfigurationOptions`' existing `CertificateSelection` callback and PEM/PFX helpers, as every client + supports it. No dedicated credentials type needed. + +4. **When every member fails.** + + | | active member's own lag check fails | everything unhealthy | "all lagging" special-cased | + | --- | --- | --- | --- | + | Lettuce | evicted | `RedisNoHealthyDatabaseException`, `AllDatabasesUnhealthyEvent`, retries forever | no | + | Jedis | evicted | `JedisTemporarilyNotAvailableException`, then `...PermanentlyNotAvailableException` | no | + | redis-py | evicted | `NoValidDatabaseException` inside `TemporaryUnavailableException` | no | + | go-redis | **kept** (PING judges it) | stays on a live active; otherwise `ErrTemporarilyNotAvailable`, then `ErrPermanentlyNotAvailable` | no; lagging candidates stay unselectable | + + *Recommendation:* no client falls back to liveness alone, so the proposal in 1 has no precedent. + Keep it anyway. *Measured 2026-10-08* by partitioning the clusters: after ~60 s every member fails + the lag check, so go-redis's rule would refuse a failover during any region outage longer than a + minute (its behaviour there is *inferred* from its code, not run). diff --git a/notes/lag-aware/findings.md b/notes/lag-aware/findings.md new file mode 100644 index 000000000..26ccdb84a --- /dev/null +++ b/notes/lag-aware/findings.md @@ -0,0 +1,759 @@ +# Lag-aware availability checks + +Investigation, September 2026. Does Redis Enterprise's lag-aware database-availability API belong in +our health checks for geo-redundant failover, and if so, where? The sequence that follows from this +is in [`plan.md`](plan.md). + +Short answer: **yes, it is the same feature every other Redis client has already shipped, and our +abstractions already fit it — but it needs HTTP and JSON, which `src/` has neither of, and it does +not need to live in `StackExchange.Redis` at all.** + +## 1. What the feature is + +Redis Enterprise (Redis Software) exposes a +[database availability API](https://redis.io/docs/latest/operate/rs/monitoring/db-availability/) for +load balancers and monitoring: + +```text +GET /v1/bdbs/{uid}/availability # whole database +GET /v1/local/bdbs/{uid}/endpoint/availability # this node's local endpoint, no redirect +``` + +`200 OK` means available. An endpoint counts as available only if the database's primary shards are +reachable *and* the endpoint's listener port is bound. With the OSS Cluster API enabled, the +database form verifies **all** endpoints; otherwise at least one. + +The **lag-aware** extension is what makes this interesting for failover, as opposed to a liveness +probe: + +```text +GET /v1/bdbs/{uid}/availability?extend_check=lag +GET /v1/bdbs/{uid}/availability?extend_check=lag&availability_lag_tolerance_ms=100 +``` + +`extend_check=lag` additionally asks whether a replica is *sufficiently synchronised with the +primary* to be a safe failover/failback target. The threshold is cluster-wide, default **100 ms**, +changed with: + +```text +PUT /v1/cluster { "availability_lag_tolerance_ms": 100 } +``` + +and overridable per request via the query parameter. The REST docs say "Recommended value: 100 +milliseconds". + +The purpose is explicitly disaster recovery: *"reduce the risk of data inconsistencies during +disaster recovery by … ensuring failover-failback flows only occur when databases are accessible and +sufficiently synchronized."* This is the check that stops you failing **back** onto a region that is +reachable but stale. + +> **Corrected by measurement (§10).** On RS 8.0.22 the lag is Active-Active (CRDB) sync lag, not +> intra-cluster replica lag. On any database that is not an Active-Active member, `extend_check=lag` +> answers `503 bdb_unavailable` unconditionally - with or without `replication`, and whatever +> `availability_lag_tolerance_ms` says - while the plain check answers 200. The "replica" wording +> above is the documentation's, and it is misleading. + +### Response detail + +| status | meaning | +| --- | --- | +| 200 | available | +| 400 | invalid schema | +| 404 | database not found | +| 503 | unavailable, or no quorum | + +Non-200 returns a JSON body with `error_code` and `description`: + +| form | `error_code` values | +| --- | --- | +| database | `no_quorum`, `db_not_found`, `bdb_unavailable` — suffixed `_shard_unreachable` and/or `_port_unbound`, e.g. `bdb_unavailable_shard_unreachable_port_unbound` | +| endpoint | `no_quorum`, `db_not_found`, `bdb_endpoint_unavailable` | + +Authentication is HTTP Basic against the **cluster REST API** (default port 9443), requiring the +`view_bdb_info` permission — roles `admin`, `cluster_member`, `cluster_viewer`, `db_member`, +`db_viewer`, `user_manager`. Note these are *cluster management* credentials, unrelated to the Redis +credentials in `ConfigurationOptions`. + +Documented known issue: **RS155734** — endpoint availability metrics are miscalculated. + +## 2. The reference implementation + +[`LagAwareStrategy`](https://github.com/redis/lettuce/blob/main/src/main/java/io/lettuce/core/failover/health/LagAwareStrategy.java) +in Lettuce implements `HealthCheckStrategy`, whose surface is `getInterval()`, `getTimeout()`, +`getNumProbes()`, `getPolicy()`, `getDelayInBetweenProbes()`, `doHealthCheck(RedisURI)` and +`close()`. + +`doHealthCheck` is short: + +1. If no cached bdb id: `GET /v1/bdbs?fields=uid,endpoints`, find the first bdb whose endpoints match + the connection's host, cache its `uid`. No match → log and throw. +2. `GET /v1/bdbs/{uid}/availability`, with `extend_check=lag` and + `availability_lag_tolerance_ms` when extended checking is on. +3. 200 → `HEALTHY`, anything else → `UNHEALTHY`. +4. On REST failure → **invalidate the cached bdb id** and return `UNHEALTHY`. + +Config (`LagAwareStrategy.Config`): `restEndpoint` (URI), `credentialsSupplier` +(`Supplier` — so credentials can rotate), `sslOptions`, `availabilityLagTolerance`, +`extendedCheckEnabled`. Convenience factories `databaseAvailability(...)` (plain), +`lagAware(...)`, `lagAwareWithTolerance(...)`. + +Two defaults worth noting: `EXTENDED_CHECK_DEFAULT = true` (lag-aware is on by default), and +`AVAILABILITY_LAG_TOLERANCE_DEFAULT = Duration.ofMillis(5000)`. + +## 3. Every client has this, and the defaults disagree + +This is not a Java curiosity — it is the client-side geographic failover feature family that Redis +has been rolling out across clients: + +| client | type | lag tolerance default | +| --- | --- | --- | +| Lettuce | `LagAwareStrategy` | **5000 ms** | +| Jedis | `LagAwareStrategy` | **5000 ms** | +| redis-py | `LagAwareHealthCheck` | **5000 ms** (*corrected*: its docstring says 100) | +| Redis Enterprise cluster | `availability_lag_tolerance_ms` | **100 ms** | +| REST API docs | "Recommended value" | **100 ms** | + +> **Corrected 2026-10-06, read from source rather than docs** (heads as of that date; `gh` search of +> issues/PRs was blocked by the org's SSO, so discussion that did not reach code is not covered): +> +> - Lettuce `LagAwareStrategy.java:163-165`: `EXTENDED_CHECK_DEFAULT = true`, tolerance 5000 ms. +> Unchanged since it was introduced (#3576, 2026-02-06); #3914 (2026-09-15) promoted it to GA without +> touching either value. +> - Jedis `mcf/LagAwareStrategy.java:96-97`: the same `true` / 5000. +> - redis-py `asyncio/multidb/healthcheck.py:73`: `DEFAULT_LAG_AWARE_TOLERANCE = 5000`, while line 502's +> docstring says "(default: 100)" - both from the same commit, `f3806fad`, 2025-10-07. It always sends +> `extend_check=lag` with the tolerance; there is no plain mode. +> - None checks `crdt`, so given §10, *inferred*: all three report every non-Active-Active database as +> permanently unhealthy under default settings. +> - All three map any REST failure to unhealthy; none has a third state (§4). +> - TLS is truststore/CA based in all three; none supports pinning, so with the cluster's self-signed 9443 +> certificate a user must import that leaf as a trust anchor. +> +> So the server and docs say 100 ms and every client says 5000 ms. Our plan sidesteps it: send no +> tolerance unless configured, and let the cluster's own setting apply. + +**Lettuce is fifty times more permissive than everyone else, including the server's own default and +its own documentation's recommendation.** That is either a deliberate call about real-world WAN +replication lag or an oversight; either way we should not copy a number without deciding it, and it +is worth asking Redis which is intended. 100 ms across a geo-replicated WAN link looks optimistic; +5 s looks like it would permit a failback that loses seconds of writes. + +redis-py's other defaults are useful reference points: `rest_api_port` 9443, `verify_tls` True, with +`auth_basic`, `ca_file`, `client_cert_file` and `client_key_file` — i.e. mTLS is supported, not just +Basic. + +## 4. It fits what we already have — almost exactly + +`src/StackExchange.Redis/Availability/` already implements this shape, gated behind +`[Experimental(Experiments.GeoRedundantFailover)]` (`SER007`). The correspondence with redis-py's +model is close to one-to-one: + +| concept | StackExchange.Redis | redis-py | Lettuce | +| --- | --- | --- | --- | +| probe abstraction | `HealthCheckProbe.CheckHealthAsync(HealthCheckContext)` | `AbstractHealthCheck.check_health()` | `HealthCheckStrategy.doHealthCheck(RedisURI)` | +| default probe | `HealthCheckProbe.Ping` | `PingHealthCheck` | ping strategy | +| aggregation policy | `HealthCheckProbePolicy.AllSuccess` / `AnySuccess` / `MajoritySuccess` | `HEALTHY_ALL` / `HEALTHY_ANY` / `HEALTHY_MAJORITY` | `ProbingPolicy` | +| probe count / timeout / interval | `HealthCheck.ProbeCount` / `.ProbeTimeout` / `.ProbeInterval` | `MultiDbConfig` | `getNumProbes()` / `getTimeout()` / `getInterval()` | +| failure latch | `CircuitBreaker` | circuit breaker OPEN | — | +| failover unit | `ConnectionGroupMember` | database in multi-db config | — | +| failback damping | `MultiGroupOptions.FailbackDelay` | weighted selection | — | + +Two details that make a lag-aware probe land cleanly: + +- **`HealthCheckProbe` is externally subclassable today.** It is a `public abstract partial class` + with one `public abstract` method and *no* internal or `private protected` abstract members — + checked. Anyone can implement a probe outside this assembly. The only in-assembly conveniences are + the memoised `protected internal` result tasks, which an external probe can trivially do without. +- **Health checks can already be configured per member.** `ConnectionGroupMember` carries nullable + per-member overrides of the group-wide `MultiGroupOptions`, health check included + (`ResolveHealthCheck(options) => HealthCheck ?? options.HealthCheck`). That matters, because each + geo member is a *different* Redis Enterprise cluster with its own REST endpoint, its own + credentials and its own bdb uid — so per-member probe configuration is exactly the shape needed, + and it already exists. + +### Where we are better placed than Lettuce + +`HealthCheckResult` has three states — `Healthy`, `Unhealthy`, **`Inconclusive`** — where Lettuce's +`HealthStatus` has two. + +That difference is more than cosmetic here. Lettuce maps *"the REST call failed"* to `UNHEALTHY`, +which means a Redis Enterprise **management-plane** outage (REST API down, credentials expired, +9443 firewalled) is indistinguishable from the **data plane** being unavailable — and can therefore +trigger a failover of a database that is serving traffic perfectly well. With `Inconclusive` we can +say "I could not determine this" and let the policy decide, which is the honest answer and the safer +one. `KeyWriteHealthCheckProbe` already uses `Inconclusive` this way for replicas. + +This is the single most valuable thing we could do differently, and it should be a deliberate, +documented deviation rather than an accident. + +## 5. The awkward part: HTTP and JSON + +`src/` contains **no HTTP client and no JSON parser** — checked across `StackExchange.Redis` and +`RESPite`; the only matches are in `obj/` transitive restore graphs. A lag-aware probe needs both: + +- **HTTP.** `System.Net.Http` is in-box on `net8.0`/`net10.0` and available on `net472` / + `netstandard2.0`. Manageable, but it is a new dependency class for this library, and it drags in + TLS configuration, proxy behaviour, timeouts and connection pooling as things we would own. +- **JSON.** Needed only for `GET /v1/bdbs?fields=uid,endpoints` discovery and for parsing + `error_code` out of failure bodies. `System.Text.Json` would be a new package reference on + `net472`/`netstandard2.0`. + +**JSON is avoidable on the hot path.** The availability call itself is a *status code* check — 200 +versus not-200 — with no body parsing required for the healthy case. If the bdb uid is supplied by +configuration rather than discovered, the required path needs no JSON at all, and `error_code` can be +surfaced as an opaque string for diagnostics. Discovery-by-host (Lettuce's step 1) is the only part +that genuinely needs a parser, and it is optional convenience: anyone configuring a REST endpoint, +credentials and a tolerance already knows their database. + +That suggests an explicit-uid-first design, with discovery as a later, optional extra. + +## 6. Options + +**A — in `StackExchange.Redis`.** Ships a `HealthCheckProbe.LagAware(...)` alongside `Ping` and +`StringSet`. Most discoverable; matches what Lettuce/Jedis/redis-py do (all in-box). Costs the core +package an `HttpClient` dependency, plus TLS/proxy/credential surface, for a feature that only +applies to one commercial deployment target. + +**B — a separate package.** `StackExchange.Redis.Enterprise` (or similar) subclassing the *already +public* `HealthCheckProbe`. **This requires no change to `StackExchange.Redis` whatsoever** — the +extension point exists and is sufficient. Keeps HTTP, JSON, TLS and cluster credentials out of the +core package and lets the Enterprise-specific bits version on their own cadence. + +**C — do nothing, and document.** The extension point is public; users targeting Redis Enterprise +can write a ~50-line probe. Cheapest, and the least good discovery story for the people who most +need it. + +B looks strongest on the evidence: the feature is deployment-specific, the dependency is real, and — +unlike the OpenTelemetry case, where a second assembly would have needed internals it could not have +— the hook here is *already public and already sufficient*. Worth confirming that nothing in +`HealthCheckContext` is missing for a real implementation before committing to it. + +## 7. Testing: the toy server can spoof this, and already has the HTTP half + +`toys/KestrelRedisServer` **already listens on HTTP**. From `Program.cs`, one Kestrel host serves +both planes over the same `RedisServer` singleton: + +```csharp +options.ListenLocalhost(5000); // "HTTP 5000 (test/debug API only)" +options.ListenLocalhost(ip.Port, b => b.UseConnectionHandler()); // RESP 6379 +``` + +with a single catch-all route today (`app.Run(ctx => ctx.Response.WriteAsync(server.GetStats()))`). +Adding `/v1/bdbs/{uid}/availability` beside that is a handful of lines, and — because the route +closes over the *same* `RedisServer` instance the client is talking RESP to — the fake management +plane can be made to lie **coherently with** the data plane rather than independently of it. + +Spoofing infrastructure for test purposes is also already the established intent of this toy; the +file carries a commented-out block headed *"demonstrate cluster spoofing"* that flips +`ServerType` to `Cluster`, adds an empty node and migrates a slot. A fake Redis Enterprise +management plane is the same idea applied to a different API. + +What that buys is the ability to script the scenarios that decide whether the failover logic is +correct, none of which can be produced on demand against a real cluster: + +- healthy (200) +- `503 bdb_unavailable_shard_unreachable`, `503 no_quorum`, `404 db_not_found` +- **200 for the plain check, 503 for `extend_check=lag`** — reachable but stale. *This asymmetry is + the entire point of the feature.* (See the fault injector below: this one **can** also be produced + against a real cluster, contrary to what one would assume.) +- slow responses, to exercise `HealthCheck.ProbeTimeout` +- 401/403, to exercise credential rotation and — importantly — to check we return `Inconclusive` + rather than `Unhealthy` when the management plane is the thing that is broken (§4) + +### Two levels, and a design consequence + +The existing test harness points at the right answer for the cheap level — but the two planes are +**not** symmetric, and this is the thing to get right. + +`InProcessTestServer` derives from `MemoryCacheRedisServer` and sets `Tunnel = new InProcTunnel(this)`. +It **does not run as a network server at all**: there is no listener, no port, no socket. The client +reaches it because `ConfigurationOptions.Tunnel` replaces the transport wholesale, so "connecting" is +a method call. That is what makes those tests fast and deterministic. + +This is not a hack bolted on for tests — it is a first-class, public, documented extension point. +`StackExchange.Redis.Configuration.Tunnel` is a `public abstract class` whose +`ConnectTransportAsync(...)` returns a `RESPite.Transports.DuplexTransport`, i.e. an implementation +**hijacks the transport layer completely**; the shipped `Tunnel.HttpProxy(...)` does the same thing +for CONNECT proxying. `InProcTunnel` is simply another implementation of that same public seam. + +There is no such seam on the HTTP side. `HttpClient` resolves a URL and opens a socket, so **you +cannot point a lag-aware probe at the in-process server — there is nothing there to point at.** + +So the model to copy is the one the library already has: hijack the HTTP transport completely, the +same way `Tunnel` hijacks the RESP one. In .NET that means an injectable `HttpMessageHandler` (or our +own abstraction over it), which is the exact analogue — `HttpMessageHandler` *is* the HTTP transport, +and substituting it answers requests in-process with no listener, no port and no TLS. + +Hence the sharper version of the design consequence below: a configurable *base URL* is not enough. +If the probe's only seam is a `Uri`, every test needs a real listener on a real port, and the cheap +in-process level is simply unavailable. Lettuce evidently hit the same wall — its second constructor, +`LagAwareStrategy(Config, HttpClient)`, exists so the transport can be substituted. + +`toys/KestrelRedisServer` then covers the other level: a real, out-of-process, full-fidelity fake with +actual sockets on both planes, for manual exercise and end-to-end runs. Note that this is also the +only level where the *whole* thing is exercised, because it is the only place both planes are real. + +The combination is what makes this worth doing: **in-process RESP fakes for several members, plus a +stubbed management plane per member, is a complete geo-redundant failover test with no external +infrastructure at all** — and geo-redundant failover is otherwise close to untestable, which is +presumably why it is still behind `SER007`. + +### Real servers are available, and they answer different questions + +mgravell notes we have access to real servers that will support this. That does not replace the +fakes; it does something the fakes cannot, and it should come **first**. + +Everything in §1 of these notes is read off *documentation*, and documentation for a recent feature +is exactly where reality and prose diverge. A short session against a real cluster settles a list of +things currently taken on trust: + +- does the deployed version support `extend_check=lag` at all, and does the + `availability_lag_tolerance_ms` query parameter genuinely override the cluster default? +- what does a failure body actually look like — are the composed codes + (`bdb_unavailable_shard_unreachable_port_unbound`) real, and is `error_code` reliably present? +- is a `Host: cnm.cluster.fqdn` header actually required, as the docs' header table implies? + (`ClusterRestClient` sets no such header and works, so: apparently not — but it only issues + `GET /v1/bdbs`, so worth re-checking on the availability route.) +- **what lag do real geo-replicated links actually show?** This is the empirical way to settle + 100 ms versus Lettuce's 5000 ms (§3), and it is a much better argument than reading either + default off a page. + +The two then compose rather than compete, and the synthesis is the useful bit: **use the real +cluster to capture responses, and the fakes to replay them.** A recorded set of genuine 200/503/404 +bodies baked into the stub means the deterministic tests stop guessing at the wire format. + +### The fault injector — and we already have one + +I had assumed a real cluster could not be made to show "reachable but stale" on demand. **That is +wrong twice over**: Redis ships a fault injection service for exactly this, *and we already have a +test tier built on it.* + +**`tests/StackExchange.Redis.FaultInjector.Tests`** exists on `marc/maint-optin-server`, open as +**[#3191](https://github.com/StackExchange/StackExchange.Redis/pull/3191)** (107 files, +13,160, +unmerged at the time of writing). Per its README it drives *"a real Redis Enterprise deployment +through the fault injector, and watch[es] how SE.Redis reacts … the tier that can observe what no +in-process fake can: real DNS, real TLS identity, real timing."* Gating is +`SER_FI_CONFIG_DIR` plus an explicit `E2E_SCENARIO_TESTS=true` opt-in, with `FAULT_INJECTION_API_URL` +defaulting to `http://127.0.0.1:20324`. + +Two pieces of it matter directly here. + +**`FaultInjector/FaultInjectorClient.cs`** — `POST /action` returning an id, `GET /action/{id}` to +poll, with its own note that this is *"the same shape go-redis, redis-py and node-redis wrap … +they integrated independently and converged, so the contract is stable"*. It is richer than Lettuce's: +alongside `StartActionAsync` / `RunActionAsync` / `WaitForActionAsync` there is +`GetValidTriggersAsync(scenario, effect, clusterIndex)`, so **we can ask the injector what +lag-inducing effects the deployment actually supports rather than guessing at an action name.** +Lettuce's own Active-Active failover test uses `network_latency` with `bdb_id`, `delay_ms` and +`duration`, which is the obvious candidate to look for. + +**`Environment/ClusterRestClient.cs`** — *already a Redis Enterprise REST client*, described as "the +cluster's own REST API on port 9443, for the few facts the fault injector does not expose". It +already does `GET /v1/bdbs?fields=uid,name`, with HTTP Basic auth and `System.Text.Json`. Adding +`GET /v1/bdbs/{uid}/availability?extend_check=lag` beside it is a method, not a project. + +That client also **answers several questions in §9 empirically**, which is worth more than the +documentation they were drawn from: + +- **TLS on 9443 is self-signed per environment.** The client pins to a CA from the config directory + via `ServerCertificateCustomValidationCallback` with `X509ChainTrustMode.CustomRootTrust`, with the + comment *"this channel carries credentials"*. So a lag-aware probe **does** need CA/cert + configuration in the redis-py mould; relying on the ambient trust store will not do. + *Corrected (§10):* the 9443 certificate is self-signed by the cluster itself (issuer = subject = + cluster FQDN, minted at bootstrap), and the CA in the config directory signs only the RESP-plane + proxy certificates - so that chain build could never succeed. `ClusterRestClient` now pins the leaf + on first use instead. The conclusion stands and sharpens: a probe needs a pinned certificate or + fingerprint option, not merely a custom CA. +- **Auth is HTTP Basic**, confirmed against a real cluster rather than inferred from Lettuce. +- **`/v1/bdbs?fields=…` returns an array of objects carrying the requested fields**, so Lettuce's + `fields=uid,endpoints` discovery is straightforward. That weakens the §5 argument a little: skipping + discovery still keeps JSON out of **`src/`**, but "discovery is hard" is not the reason to skip it. + +The fakes are demoted, not removed: they still cover machines with no Enterprise cluster, they are +far faster (injector actions budget minutes, plus a stabilisation wait), and they reach what the +injector does not obviously reach — specific `error_code` bodies, 401/403 credential expiry, and +slow-but-successful responses for probe-timeout behaviour. + +**Sequencing consequence:** an end-to-end lag-aware test is downstream of #3191. That is a real +dependency to plan around, not a detail. + +And #3191's own write-up is the strongest possible support for validating §1 before writing code: +*"the specifications are prose, the payloads were never published, and nearly every assumption I +started with was wrong in some way that mattered."* Same vendor, same class of API, same trap. + +Gating is a solved problem here: `tests/.../Helpers/Skip.cs` already has `IfNoServer(host, port)`, +`IfNoCluster()`, `IfNoFailoverPair()` and `UnlessLongRunning()`, and `TestConfig` carries per-role +host/port settings, so an Enterprise-backed test suite follows an established pattern and skips as +inconclusive wherever the cluster is absent — which is everywhere except the machines that have one. + +The design consequence: **whatever we build must be able to hijack the HTTP transport completely, +from day one** — an injectable `HttpMessageHandler`-shaped seam, not merely a configurable base URL. +That is a requirement on the API shape, it falls out of testability rather than taste, and it is the +same decision the library already made for RESP with `Tunnel`. It also argues mildly against option +C — "document the extension point and let users write it" leaves us with no test coverage of a +failover path we ship. + +## 8. If `Tunnel` gains an API for this + +mgravell has offered to add one. It is a good fit — but the *shape* decides whether it helps or +quietly undoes §6. + +### Two functional arguments for `Tunnel` specifically + +1. **The proxy case is already right.** If Redis traffic goes through a CONNECT proxy + (`Tunnel.HttpProxy(...)`), the Redis Enterprise REST API is almost certainly behind the same + corporate proxy. Hanging the management-plane transport off the same object means that case works + by construction instead of needing a second, parallel proxy setting that users must remember to + keep in sync. +2. **One assignment configures both planes in tests.** `InProcessTestServer` already sets + `Tunnel = new InProcTunnel(this)`. If the HTTP seam rides on the same object, the harness gets + management-plane spoofing everywhere it already gets data-plane spoofing, with no new wiring. + +### The shape that would be a mistake + +```csharp +// don't +public virtual HttpMessageHandler? CreateHttpMessageHandler(Uri restEndpoint); +``` + +That puts `System.Net.Http` types into **StackExchange.Redis's public API**, giving the core package +a hard dependency for the sake of one commercial deployment target — exactly what option B exists to +avoid. It would make the core-vs-package question moot by deciding it the expensive way. + +### The shape that works + +Stay transport-neutral. `Tunnel` yields a `Stream` (or `DuplexTransport`) for an endpoint, and the +Enterprise-side package composes that into an HTTP stack itself: + +```csharp +// core: no System.Net.Http anywhere +public virtual ValueTask ConnectManagementStreamAsync( + EndPoint endpoint, CancellationToken cancellationToken) => default; + +// Enterprise package: hand it to SocketsHttpHandler +new SocketsHttpHandler +{ + ConnectCallback = async (ctx, ct) => + await tunnel.ConnectManagementStreamAsync(Resolve(ctx.DnsEndPoint), ct) + ?? await DefaultConnectAsync(ctx, ct), +}; +``` + +`SocketsHttpHandler.ConnectCallback` is `Func>` — a `Stream` is precisely the currency needed, so a stream-shaped `Tunnel` member +composes into a complete HTTP hijack with no HTTP types in core at all. `Tunnel` already speaks this +language: `BeforeAuthenticateAsync` returns `ValueTask` and `ConnectTransportAsync` returns +`ValueTask` (itself public, gated `SER009`). + +Adding a **virtual** member to a public abstract class is source- and binary-safe for existing +subclasses, so this is additive in the sense AGENTS.md requires. + +### Caveats worth stating + +- **`ConnectCallback` is net5.0+.** There is no equivalent on `HttpClientHandler`/`WinHttpHandler`, + so a down-level implementation would need direct handler injection instead. An Enterprise package + targeting `net8.0`+ sidesteps this entirely, and is defensible for a new deployment-specific + feature. +- **Signature mismatch.** Existing `Tunnel` members are RESP-connection-shaped (`EndPoint` plus + `ConnectionType`). HTTP wants scheme, host, port and TLS — i.e. a `Uri`. Either shape is workable; + neither is free of a little awkwardness. +- **It widens `Tunnel`'s remit** from "the Redis connection" to "transports this library initiates", + which the existing subclasses then have to have an opinion about. `HttpProxyTunnel` almost + certainly *should* apply to the management plane; `LoggingTunnel` probably should *not* start + logging REST traffic by default. Both need deciding rather than falling out. + +### One question this already answers + +Open question 5 below asked whether `HealthCheckContext` carries enough for a real probe. It does: +`IConnectionMultiplexer.RawConfig` and `ConfigurationOptions.Tunnel` are **both public**, so an +external probe can already reach `context.Server.Multiplexer.RawConfig.Tunnel` with no change to +`HealthCheckContext` at all. + +*Corrected 2026-10-08:* `RawConfig` is **not** public. The declaration found was on the internal +`IInternalConnectionMultiplexer` in the same file; the public `IConnectionMultiplexer` does not expose it. +So an external probe cannot reach the member's configured endpoints, which the lag-aware probe needs to +derive the REST endpoint and discover the database - one more reason it lives in this assembly. + +## 9. Open questions + +1. **Which tolerance default?** 100 ms (server, docs, redis-py) or 5000 ms (Lettuce)? Worth asking + Redis directly; the divergence looks unintentional. +2. **`Unhealthy` or `Inconclusive` when the REST call fails?** See §4. Recommend `Inconclusive`, but + note that this makes us behave differently from every other client, so it needs to be a stated + choice. +3. **Which endpoint — `/v1/bdbs/{uid}/availability` or `/v1/local/bdbs/{uid}/endpoint/availability`?** + Lettuce uses the former. The latter does not redirect to the primary node, which may be the more + accurate signal when probing a specific endpoint, and is what the docs recommend for load + balancers under the `all-nodes` proxy policy. Note known issue RS155734 against the endpoint form. + *Partly answered (§10):* the local form answers for whichever node served the REST call, so through + the cluster FQDN it is effectively random under `proxy_policy: single`. Not usable from a client + unless it addresses a specific node. +4. **Explicit bdb uid, discovery, or both?** Explicit avoids JSON entirely (§5). +5. ~~**Is `HealthCheckContext` sufficient?**~~ Answered in §8 — `RawConfig` and + `ConfigurationOptions.Tunnel` are both public, so a probe can reach the tunnel from the context it + already gets. What remains: if a single probe instance is ever shared across members it still + needs to map endpoint → REST configuration, which per-member health checks make unnecessary but do + not forbid. +6. **Credentials and rotation.** Lettuce takes a `Supplier` so credentials can + rotate; redis-py supports Basic plus mTLS; our own `ClusterRestClient` pins a CA because the + management certificate is self-signed per environment (§7). So the probe needs *at least* Basic + plus CA pinning, and should not bake in a static password. +7. ~~**How is this tested?**~~ Largely answered in §7: the tier exists, in #3191. What is genuinely + open is **sequencing and scope** — an end-to-end lag-aware test is downstream of an unmerged + 107-file PR, so either this waits for #3191 or it starts at the stubbed tier and the scenario test + follows. Also open: whether the deployments we have access to expose a lag-inducing effect at all + (ask `GetValidTriggersAsync` rather than assuming `network_latency`). +8. **Does any of this need `src/` changes at all?** On the evidence so far, possibly none: + `HealthCheckProbe` is externally subclassable, `RawConfig`/`Tunnel` are public, and the REST + plumbing precedent lives in the test tier. The `Tunnel` addition in §8 is the one thing that would + have to land in the core package — and only if we want the management plane to honour tunnels. + +## 10. Measured against real deployments (2026-09-18 to 2026-09-30) + +Everything here was observed, not read. Three AWS-template environments, all RS 8.0.22, three nodes per +cluster; the last one (`marcgravell-test-dc1342d5`) has two clusters, `c1` and `c2`, carrying two +Active-Active databases between them. Where a line is an inference rather than an observation it says so. + +### `extend_check=lag` is an Active-Active check + +Same cluster (c1), same session: + +| database | `crdt` | `replication` | plain | `extend_check=lag` at 100 ms / 600000 ms / 0 ms | +| --- | --- | --- | --- | --- | +| `re-active-active` | true | false | 200 | 200 / 200 / 200 | +| `re-active-active-oss-cluster` | true | false | 200 | 200 / 200 / 200 | +| control, freshly created | false | false | 200 | 503 / 503 / 503 | +| control, freshly created | false | true | 200 | 503 / 503 / 503 | + +Both Active-Active databases answer the same on c2. The control 503 is always +`{"error_code":"bdb_unavailable","description":"BDB is not available"}`, held for the whole 30 s +observed from creation, and was reproduced on two earlier environments over 60 s. + +- **Measured:** off Active-Active, the lag check fails regardless of replication and tolerance - the + tolerance parameter has no observable effect at all there. +- **Inferred:** the server has no sync lag to measure on such a database and reports "unavailable" + rather than "not applicable". +- **Consequence for the probe:** lag-awareness on a non-Active-Active member turns a healthy database + into a permanently unhealthy one. It must be opt-in per member, and that specific 503 should not be + read as a data-plane verdict - `Inconclusive` at most. This is a stronger version of the §4 argument. +- **Inferred about Lettuce, untested:** `EXTENDED_CHECK_DEFAULT = true` should make `LagAwareStrategy` + report `UNHEALTHY` forever against any database that is not Active-Active. Worth putting to Redis with + the 100 ms versus 5000 ms question (§3). +- **Unknown:** whether a 0 ms tolerance is honoured (and an idle link simply reports zero lag) or + ignored. The idle Active-Active result cannot tell those apart; it needs real lag. + +### Under real lag: the check flips, and only on the stale member (2026-10-05) + +Environment `marcgravell-test-9db7bbf0`, two clusters, `re-active-active` (uid 1 on both, paired by +`crdt_guid`). Sync *into* c2's member paused with `PUT /v1/bdbs/1 {"crdt_sync":"paused"}` on c2 for 60 s +while c1 took an `INCR` every ~5 s and c2 was read back; then `{"crdt_sync":"enabled"}`. + +| phase | c1 member | c2 member | data | +| --- | --- | --- | --- | +| baseline | plain 200, lag 200 (100 / 600000 / 0 ms) | plain 200, lag 200 (all three) | c2 tracks c1 | +| c2 sync paused | unchanged | plain 200, **lag 503 at all three**, first sample ~2 s in | c1 3 to 14, c2 stuck at 2 | +| resumed | unchanged | lag 503 ~5 s more, then 200 | c2 reaches 14 | + +- **Measured:** the plain check never noticed; the lag check on the stale member flipped within seconds + and cleared once the data had caught up. This is the "reachable but stale" case the feature exists for. +- **Measured:** only the member that is *behind* flips. The member being written to stays green - which + is the right polarity for failback, where the question is "is the target caught up". +- **Measured:** with sync paused, 600000 ms still answers 503 although the true lag was seconds. A paused + link is unavailable regardless of tolerance. +- **Measured: the tolerance does govern a running link.** Sync enabled; a server-side Lua loop on c1 + wrote ~50 MB per call (500 x 100 KB over a 200-key space) for 30 s, ~2.75 GB in all, while c2 was + sampled every ~0.5 s. In 66 samples the 100 ms check failed twice, each time with c2 visibly a few + batches behind, and recovered on the next sample; the 600000 ms check never failed. So 100 ms and + 600000 ms disagree exactly when the link is running but behind - the parameter is honoured. +- **Caveat on the numbers:** both clusters are in `us-east-1`. Under that load sync stayed under + 100 ms almost all the time, but that says nothing about a cross-region link, which is the case the + 100 ms versus 5000 ms question (§3) is actually about. +- **Measured:** the failure body is the same as for a non-Active-Active database - + `{"error_code":"bdb_unavailable","description":"BDB 1 is not available"}`. No lag-specific code, so a + probe cannot tell "lagging" from "not applicable" by the response; Active-Active membership has to be + configuration, not inference. +- **Inferred:** at baseline c2 was observed one write behind (read 1 after c1 wrote 2) while the 0 ms + check passed - either lag is sampled coarsely or 0 is not honoured. +- **Tooling:** `crdt_sync` `paused`/`enabled` is a narrow, reliably reversible way to produce lag - one + member, no network change, `status` returns to `active` within a few seconds. Far better suited to a + scenario test than `network_latency` or `network_failure` (which drops all inbound bar SSH on every + hosting node and kills `dmcproxy`). It is a direct REST mutation, not an injector action. + +### Under a partition: blind for about a minute, then tolerance-governed, and symmetric (2026-10-08) + +Environment `marcgravell-test-41f58c28`. A network partition between the clusters, nothing else: on each +c1 node, iptables rules tagged `lagprobe` dropped traffic to and from c2's six node addresses (private and +public), with a `systemd-run` timer on each node as a self-heal backstop. SSH, DNS, 9443 and the client +path were untouched. 60 s, then removed; all three nodes verified back at their baseline rule sets. +Writes to c1 every ~5 s; both members sampled (database and local forms). + +| observer time | c2 data | c2 lag check (100 / 600000 / 0 ms) | c1 lag check (100 / 600000 / 0 ms) | +| --- | --- | --- | --- | +| t+0 to 27 s, before | in step | 200 / 200 / 200 | 200 / 200 / 200 | +| t+28 s | **partition applied** | | | +| t+31 to 84 s | **stuck at 8**, c1 reaching 20 | **200 / 200 / 200** | 200 / 200 / 200 | +| t+88 s | stuck | **503 / 200 / 503** | 200 / 200 / **503** | +| t+94 s | stuck | 503 / 200 / 503 | **503** / 200 / 503 | +| t+92 s | **partition removed** | | | +| t+101 s on | caught up | 200 / 200 / 200 | 200 / 200 / 200 | + +Plain checks 200 throughout, both forms, both members; the local form tracked the database form. + +- **Measured: the lag check is blind to a dead link for about 60 s.** Sync stopped at once, but both + members reported no lag for ~57 s. Unlike paused sync, which fails immediately at any tolerance. +- **Measured: then it compares a growing lag against the tolerance.** 0 ms failed first, then 100 ms; + 600000 ms never failed. **A tolerance of 0 is honoured** (open since 2026-09-30). +- **Measured: both members fail.** Each is missing the other's stream, so "every member lagging" is the + ordinary state of a partition, not an edge case. +- **Inferred:** a dead peer region looks the same from the survivor as this partition does, so the + survivor's lag check passes for about a minute after the region dies and fails after that (at 100 ms). + The ~60 s presumably comes from the syncer's own failure detection; not investigated. + +### Other answers + +- **Cluster default** `availability_lag_tolerance_ms` is 100, on every environment - the server agrees + with the docs and redis-py, not Lettuce. +- **`Host: cnm.cluster.fqdn` is not required.** A bogus `Host` on the availability route still gets 200. +- **Both error-code families are real.** Database form: `bdb_unavailable`. Local form: + `bdb_endpoint_unavailable`, description `"Local endpoint of BDB on node is not available"`. + The composed suffixes (`_shard_unreachable`, `_port_unbound`) have not been observed yet - nothing so + far has broken a shard. +- **Healthy is an empty 200.** No body to parse; JSON appears only on failure, as §5 hoped. +- **`fields=` filters on `/v1/bdbs` but not on `/v1/cluster`**, which returns the whole object. +- **The local form answers for the node that served the request.** Through the cluster FQDN that was + always node 1, which usually does not host the endpoint under `proxy_policy: single` - so 503 for three + of four Active-Active members that were serving traffic perfectly well. *Corrected 2026-10-06:* reached + through the **database's own hostname** (`https://redis-.:9443`, as go-redis does), the + local form answered 200, plain and lag, for all four members, from the node actually serving the + endpoint; via the cluster FQDN the same calls gave 503 for three of them. Every node presents the same + cluster certificate (pinned on each), and its `*.` SAN covers the database hostname, so the + `TrustIssuer` pin should hold there too (*inferred*; only the fingerprint was checked on that path). So + the local form is usable, and arguably the more truthful one, provided it is routed by database host. + *Measured 2026-10-07 (`marcgravell-test-823d3f1e`):* under paused sync the local form, routed by + database host, behaves exactly like the database form - plain 200 throughout; lag 503 on the stale + member only, at 100 ms and 600000 ms alike, from the first sample (~2 s); cleared on the same sample as + the database form once data caught up. Its lag failure is `bdb_endpoint_unavailable`, which is also + what it answers on a node not serving the endpoint, so it is no more distinguishable by body than the + database form. For lag, the two forms give the same verdict; the difference is call shape (one per + database versus one per endpoint, and the local form only works routed by database host). + *Qualified 2026-10-08 (`marcgravell-test-41f58c28`):* routing by database host is fine for the + availability routes, but **not for the management routes**. On any node other than the master, + `/v1/bdbs?fields=...`, `/v1/bdbs/{uid}?fields=...` and `/v1/cluster` answer **307, redirecting to the + master's internal address** (`https://10.0.101.x:9443/...`), while `/availability` is answered directly. + Following that redirect from outside the VPC hangs. The cluster FQDN resolves to the master, which + answers everything. Found by running the real probe against the live clusters: three of four members + timed out until the probe derived the cluster name (strip `redis-{port}.`) instead and stopped following + redirects. *Inferred:* go-redis's discovery, routed via the database host, would meet the same redirect. +- **`/v1/bdbs` returns database passwords** (`authentication_redis_pass`, `authentication_admin_pass`) + when `mask_bdb_credentials` is false, as on these clusters, and `/v1/crdbs` returns the same for every + instance. Captured bodies cannot be committed as fixtures without scrubbing. Relevant to credential + scope too: whether `cluster_viewer`/`db_viewer` see the same was not checked (we used admin). + +### What a probe can discover, and with what (2026-10-06, `marcgravell-test-60790432`) + +**Active-Active membership is not visible over RESP.** Compared on c1: both Active-Active members and a +freshly created plain database (since deleted). + +- `HELLO` is identical in shape (`mode: cluster`, `role: master` on all three, including the plain one). +- `CRDT.INFO` is an unknown command everywhere; `INFO crdt` / `INFO crdb` are empty everywhere; no INFO + field mentions crdt. +- The only INFO differences are incidental: `cmdstat_replconf` on the members (the syncer's own traffic, + so absent until a peer has connected), a different module list (the members load only ReJSON and + search), and `rdb_changes_since_last_save`. None is a signal to build on. +- So the probe has to learn it from REST: `GET /v1/bdbs/{uid}?fields=uid,crdt` answers + `{"crdt":true,"uid":1}`, or it is configuration. + +**`db_viewer` is enough, and the least privileged role still sees database passwords.** Temporary +roles (`management: db_viewer`, then `cluster_viewer`) and users, deleted afterwards. RS 8 rejects the +legacy `role` field (`role_not_exist`); roles are objects referenced by `role_uids`, and a fresh cluster +has only `Admin`. + +| route, as `db_viewer` (`cluster_viewer` identical) | result | +| --- | --- | +| `/v1/bdbs/{uid}/availability`, with and without `extend_check=lag` | 200 | +| `/v1/bdbs/{uid}?fields=uid,crdt` | 200, no password fields | +| `/v1/bdbs?fields=uid,endpoints` | 200, 599 bytes, no password fields | +| `/v1/bdbs/{uid}` (unfiltered) | 200, **`authentication_redis_pass` and `authentication_admin_pass` populated** | +| `/v1/crdbs` | 200, every instance's passwords populated | + +This is with `mask_bdb_credentials: false`, the cluster's setting here; masked clusters were not +tested. Consequence: the probe must only ever issue `fields=`-filtered requests, and the docs should +recommend `db_viewer`. Possibly worth raising with the RS team that the viewer roles see secrets. + +**The REST endpoint is derivable from the database hostname.** Every database's `dns_name`, plain, +Active-Active and OSS-cluster alike, is `redis-.`, so the management endpoint is +`https://:9443`. But the OSS-cluster database advertises **IP addresses** through +`CLUSTER SLOTS`/`CLUSTER SHARDS` (`oss_cluster_api_preferred_endpoint_type: ip`), so the derivation must +start from the member's *configured* endpoint, not from discovered nodes. A customer CNAME or a bare IP +defeats it, so it can only be a default with an override. uid discovery by matching `dns_name` and port +against `/v1/bdbs?fields=uid,endpoints` is safe on credentials (filtered, no passwords). + +### Certificate trust reuses the existing TLS callback (2026-10-06) + +The library's own `ConfigurationOptions.TrustIssuer(...)` callback, obtained through the public +`TlsOptions.CertificateValidationCallback` and adapted to `HttpClientHandler` with a one-line lambda, +validates the 9443 call. Against c1, `GET /v1/cluster`: + +| trust | result | +| --- | --- | +| platform default | rejected, `UntrustedRoot` | +| `TrustIssuer(c1's own 9443 certificate)` | **200** | +| `TrustIssuer(config-dir ca.crt)` - the CA that signs the RESP proxy certs | rejected | +| `TrustIssuer(c2's 9443 certificate)` | rejected | + +The only policy error ever presented was `RemoteCertificateChainErrors`: the certificate carries SANs for +`` and `*.` and server-auth EKU, so host-name checks pass, which is the one condition +`TrustIssuer` requires. `TrustIssuer(leaf)` therefore *is* a pin, because the self-signed leaf is its own +chain. No new trust API is needed. + +What does **not** carry over is the trust *configuration*: the data-plane and management-plane +certificates have different issuers (proxy certs from the deployment CA, 9443 self-signed per cluster), +so a member's Redis `TrustIssuer` cannot be reused for its REST calls; it is the same mechanism with a +separate setting. The adapter is the same on every TFM (`ServerCertificateCustomValidationCallback`), and +on `net5`+ `SocketsHttpHandler.SslOptions.RemoteCertificateValidationCallback` takes the delegate +unchanged, which matters if the HTTP path rides on `Tunnel` (§8). A production cluster with a CA-signed +9443 certificate needs no callback at all. + +### Who else implements it, and what turns it on (2026-10-06) + +Read from anonymous shallow clones, heads of 2026-09-26 to 2026-10-06 (`gh` search is blocked by SSO). + +| client | support | activation | REST endpoint | uid | lag / tolerance | REST TLS | +| --- | --- | --- | --- | --- | --- | --- | +| Lettuce | `failover/health/LagAwareStrategy.java` | explicit strategy supplier; default `PingStrategy` | user-supplied URI | discovered from `/v1/bdbs` by host | on / 5000 | own `SslOptions` | +| Jedis | `mcf/LagAwareStrategy.java` | explicit strategy supplier; default `PingStrategy` | user-supplied | `/v1/bdbs?fields=uid,endpoints` by host | on / 5000 | own `SslOptions` | +| redis-py (asyncio only) | `asyncio/multidb/healthcheck.py` | explicit `health_checks`; default `PingHealthCheck` | user-supplied `health_check_url` + port 9443 | **unfiltered** `/v1/bdbs`, by `dns_name`/`addr` | always on / 5000 | own CA / `verify_tls` / mTLS | +| go-redis | **feature branch only** (`feature/multidb-integration`), `multidb/healthcheck_lag_aware.go` | explicit `HealthChecks`; default PING | **derived: `https://:9443`**, override available | `/v1/bdbs?fields=uid,endpoints`, host + resolved IPs + port | on / 5000 | own `tls.Config` incl. `InsecureSkipVerify` | +| node-redis, NRedisStack, redis-rs, rueidis, predis, phpredis | not found | | | | | | + +- **Nobody activates it automatically** - no domain detection, no provider. It is an opt-in health + check everywhere, and PING is the default everywhere. +- **No client checks `crdt`**, so (*inferred*, from §10) all fail permanently on non-Active-Active. +- **No client reuses the Redis connection's TLS settings** for REST, and none pins. +- redis-py's unfiltered `/v1/bdbs` receives database passwords (per the role test above). +- **go-redis has the two design ideas worth taking:** + 1. it uses the *local* form routed via each member's own database host, once per master in cluster + mode - the routing that makes the local form truthful (see the correction above); + 2. `FailbackOnly() == true`: the lag verdict only gates moving traffic *to* a member and never evicts + the active one, while PING keeps judging the active member's liveness. That is the `Inconclusive` + failback problem in `plan.md` step 3, solved by giving the check a scope rather than a new result. + +### Multi-cluster shape + +- `env_output.json` nests clusters under `.clusters.value[N]`, each with its own FQDN and credentials. + `FaultInjectorEnvironment` reads only `[0]`. +- Each cluster has its own self-signed 9443 certificate, so pins are per host + (`cluster-cert..sha256`). +- Member uids happened to match across c1 and c2; the platform does not promise it. `/v1/crdbs` on one + cluster reports the other's instance with `db_uid=None`, so pairing members means reading each + cluster's `/v1/bdbs` and matching on `crdt_guid`. The probe's configuration should therefore be + "cluster REST endpoint plus uid" per member, not a uid shared across the group. + +### Fault injection + +- **`network_latency` is not safe to use for this.** It applies `netem` to the node's primary interface, + not to the database, and its scheduled removal fails ("No existing session" - the cleanup thread reuses + an SSH session that has been torn down) while the action still reports success. On 2026-09-18 that + left a 5 s delay on two nodes, which took down the cluster's own DNS and 9443 until removed by hand + over SSH. Reported to `redis-developer/cae-client-testing`. +- **Holding back sync:** done with `crdt_sync=paused` on the stale member (above), not with the injector. diff --git a/notes/lag-aware/plan.md b/notes/lag-aware/plan.md new file mode 100644 index 000000000..9faef4b75 --- /dev/null +++ b/notes/lag-aware/plan.md @@ -0,0 +1,204 @@ +# Lag-aware availability checks: the plan + +What we intend to do about [`findings.md`](findings.md). That file is the evidence; this one is the +sequence. + +Status: **proposed. Step 0 is cleared; step 1 is answered (findings §10): the lag check flips under +real Active-Active lag, on the stale member only, and the tolerance is honoured on a running link. +Cross-region lag magnitudes remain unmeasured (both test clusters share a region).** The core change +(failback health check, member-scoped probes, selection rules) and the REST probe are implemented, with +stubbed-transport tests (step 5, tier 1) and a fault-injector scenario test (tier 3, +`LagAwareFailbackScenarioTests`, passing against a live Active-Active pair on 2026-10-08). Tier 2 (the +Kestrel toy) is not done and arguably not needed now that tiers 1 and 3 exist. +See [`api-sketch.md`](api-sketch.md). + +> **This branch is not only a planning branch.** `marc/lag-aware-availability` carries these notes +> now, and is the feature branch from here — the implementation lands on top of the same branch and +> the same PR, rather than the notes being merged separately and the work starting again elsewhere. +> +> It has been rebased onto `main` post-#3191 (`git rebase origin/main`), which is what step 0 asked +> for; do not merge `main` in. #3191 squash-merged, so merging it back would move the merge base +> without giving us its ancestry, and the commit list here would start carrying phantoms. + +> **Where this can be worked on.** Step 1 needs a real Redis Enterprise deployment and the fault +> injector, which live on a machine holding the environment directory (`~/aws` on the box that has +> it — the AWS multi-cluster template, so `env_output.json` nests its outputs under +> `.clusters.value[0]`, which `FaultInjectorEnvironment` already handles). Nothing in step 1 can be +> done or verified without it, so the work continues there rather than on whichever box happens to +> have the repo. Steps 2 onward are ordinary code and are not tied to a machine. + +## Step 0 — wait for #3191 ✅ cleared + +**Done.** #3191 merged as `dc915bbc` on 2026-09-18, and this branch has been rebased onto it, so +`tests/StackExchange.Redis.FaultInjector.Tests` is present here now. The rest of this section records +why the wait was worth it. + +[#3191](https://github.com/StackExchange/StackExchange.Redis/pull/3191) ("Server-native maintenance +notifications") is open, 107 files, +13,160. It carries +`tests/StackExchange.Redis.FaultInjector.Tests`, and with it the three things this work would +otherwise have to build from scratch: + +| what | why it matters here | +| --- | --- | +| `FaultInjector/FaultInjectorClient.cs` | `POST /action` + `GET /action/{id}`, plus `GetValidTriggersAsync(scenario, effect, …)` — lets us ask a deployment which lag-inducing effects it supports instead of guessing | +| `Environment/ClusterRestClient.cs` | already a Redis Enterprise REST client on 9443: Basic auth, CA pinning, `System.Text.Json`, `GET /v1/bdbs?fields=…` | +| the tier itself | environment fixtures, provisioning, gating (`SER_FI_CONFIG_DIR`, `E2E_SCENARIO_TESTS`), and the README describing how to run it | + +Building any of that again on this branch would mean writing a second copy of a REST client against +the same cluster, and then reconciling them at merge. Not worth it for a feature whose entire +implementation is smaller than the client that tests it. + +It has landed, so nothing here is gated on it any more. Step 1 is the next thing, and its only +prerequisite is a cluster to point at. + +## Step 1 — validate the API against a real deployment + +Before any code. Everything in findings §1 is read off documentation, and #3191's own summary is the +warning: *"the specifications are prose, the payloads were never published, and nearly every +assumption I started with was wrong in some way that mattered."* Same vendor, same class of API. + +The checklist, each item currently an assumption: + +- [x] does the deployed version support `extend_check=lag` at all? **Yes (RS 8.0.22), but only + meaningfully on Active-Active members; everything else gets `503 bdb_unavailable` + unconditionally.** Findings §10. +- [ ] does `availability_lag_tolerance_ms` as a query parameter genuinely override the cluster + default, and what happens if it is absurd (0, negative, enormous)? **Honoured on a running + link:** under a write burst 100 ms failed while 600000 ms passed. No effect off Active-Active or + with sync paused (both always fail). Negative not tried. +- [ ] what is actually in a failure body — is `error_code` reliably present, and are the composed + forms (`bdb_unavailable_shard_unreachable_port_unbound`) real? *Partly:* `error_code` and + `description` present on every failure seen; lag failure is plain `bdb_unavailable`, the same + as "not Active-Active"; composed suffixes not yet observed. +- [x] does the lag check flip while the plain check stays green? **Yes, on the stale member only, + within seconds, clearing when data catches up** (findings §10, `crdt_sync=paused`). +- [x] is `Host: cnm.cluster.fqdn` required on the availability route? **No.** +- [x] `/v1/bdbs/{uid}/availability` versus `/v1/local/bdbs/{uid}/endpoint/availability`? **The local + form reports for whichever node answered the REST call; unusable through the cluster FQDN.** Use + the database form. RS155734 not assessed. +- [ ] what does `GET /v1/bdbs?fields=uid,endpoints` return, in the shape Lettuce matches hosts + against? Not yet asked with those fields; full `/v1/bdbs` bodies captured (contain passwords, + not committed). +- [ ] **what lag do real geo-replicated links show under load?** The empirical way to settle 100 ms + (server default, docs, redis-py) versus 5000 ms (Lettuce) — see findings §3. Needs a two-cluster + environment, which now exists. +- [ ] which fault-injector effects can drive lag past the tolerance — ask `GetValidTriggersAsync`, + do not assume `network_latency`. **`network_latency` is ruled out** (node-wide, and its cleanup + fails; findings §10). The `/action` enum also lists `network_failure`, `execute_rladmin_command` + and `update_cluster_config`; which of these can hold back Active-Active sync safely is the open + question. + +Output: the answers written into `findings.md`, and **captured response bodies saved as fixtures**, so +the stubbed tests in step 5 replay real payloads instead of guessing at the wire format. + +## Step 2 — decide where it lives + +Findings §6 sets out three options; step 1 informs the choice, so this is deliberately not decided +here. The honest current state: + +- **The dependency argument is weaker than it first looked.** `System.Net.Http` needs no package + reference on any target we ship — it is in-box on .NET Framework, in `netstandard2.0`, and in the + shared framework on `net8.0`/`net10.0`. The genuinely new dependency would be `System.Text.Json` on + `net472`/`netstandard2.0`, and findings §5 shows JSON is avoidable if the bdb uid is configured + rather than discovered. +- **`src/` may need no changes at all** *(no longer true - see the `Inconclusive` point in step 3)*. `HealthCheckProbe` is externally subclassable (no internal + abstract members), and `IConnectionMultiplexer.RawConfig` and `ConfigurationOptions.Tunnel` are both + public, so a probe can reach everything it needs from the context it already gets. +- So the question is not really "can we" but "should a general-purpose Redis client carry a + management-plane HTTP client for one commercial deployment target" — which is a judgement call, not + a technical constraint. + +Leaning: **a separate package** (option B), `net8.0`+ only. It keeps Enterprise-specific surface out +of core, versions independently, and `SocketsHttpHandler.ConnectCallback` — needed for step 4 — is +net5+ anyway, so a down-level build would be degraded regardless. But a new shipping package is a real +ownership cost for a small feature, and that is mgravell's call, not this document's. + +## Step 3 — the probe + +```csharp +// shape only; names and home package per step 2 +public sealed class LagAwareHealthCheckProbe : HealthCheckProbe +{ + public override Task CheckHealthAsync(HealthCheckContext context); +} +``` + +Behaviour, with the deliberate deviations called out: + +- `GET /v1/bdbs/{uid}/availability`, adding `extend_check=lag` and `availability_lag_tolerance_ms` + when lag-awareness is on. +- 200 → `Healthy`. A definite negative (503 with a recognised `error_code`) → `Unhealthy`. +- **Lag-awareness is per member and only for Active-Active members.** Off Active-Active the server + answers `503 bdb_unavailable` to every lag check (findings §10), so enabling it there would fail a + healthy database forever. Either refuse the combination at configuration time, or check `crdt` once + via `/v1/bdbs/{uid}` and fall back to the plain check - the latter needs JSON, the former does not. +- Addressing is REST endpoint **plus** uid per member; uids are not stable across the clusters of one + Active-Active database. +- **The probe is called per endpoint, but the answer is per database.** `HealthCheck.CheckHealthAsync + (IConnectionMultiplexer)` fans out one probe call per server (`HealthCheck.Execute.cs:25-40`), so an + OSS-cluster member with six endpoints would make six identical REST calls per pass. Either cache + within a pass, or hook in at member level instead. +- **`Inconclusive` is right for failover and wrong for failback.** `ConnectionGroupMember.UpdateState` + (`MultiGroupMultiplexer.cs:315`) treats anything but `Unhealthy` as connected, and once + `FailbackDelay` has passed that makes the member eligible again. So "the REST API could not be + asked" would let the group fail back onto a member that may be stale - the exact case the lag check + exists to prevent. Keeping an active member while refusing to *return* to an unverified one needs a + distinction `UpdateState` does not currently make, which means a small `src/` change after all (step 2 + assumed none). +- **A failed REST call → `Inconclusive`, not `Unhealthy`.** This is the deviation from every other + client (findings §4): Lettuce's two-state `HealthStatus` cannot distinguish "the database is down" + from "I could not ask", so a management-plane outage can fail over a database that is serving + traffic perfectly. We have `Inconclusive`; we should use it. Document it as a difference. +- Configuration: REST endpoint, credentials **via a callback so they can rotate** (Lettuce takes a + `Supplier`), certificate options (step 1 confirmed the management certificate is self-signed by the + cluster itself, so neither ambient trust nor a separate CA works; the option has to accept a pinned + certificate or fingerprint), bdb uid, tolerance, and a lag-aware on/off switch. +- Defaults deferred to step 1's measurements rather than copied from either existing client. + +Explicit uid first; host-based discovery is a later, optional extra (findings §5). + +## Step 4 — the transport seam, if we want it + +Only if we want the management plane to honour tunnels. mgravell has offered a `Tunnel` addition; +findings §8 has the shape — a **stream-returning** member, composed into +`SocketsHttpHandler.ConnectCallback` on the consuming side, so no `System.Net.Http` type enters +StackExchange.Redis's public API. + +Two reasons it is worth doing beyond testing: a CONNECT proxy configured for Redis almost certainly +applies to the Enterprise REST API too, and `InProcessTestServer` already assigns `Tunnel`, so one +assignment would spoof both planes. Two things it forces: `HttpProxyTunnel` and `LoggingTunnel` each +need an opinion about whether they apply to the management plane. + +Separable from step 3 — the probe works without it, just without tunnel support. + +## Step 5 — tests + +Three tiers, cheapest first: + +1. **Stubbed transport.** Hijack the HTTP transport completely, the way `Tunnel` does for RESP, and + replay the step-1 fixtures: 200, each 503 `error_code`, 404, 401/403, malformed bodies, and + slow-but-successful responses for probe-timeout behaviour. No listener, no port, runs in CI. +2. **`toys/KestrelRedisServer`.** It already serves HTTP on 5000 over the same `RedisServer` + singleton that serves RESP, so a fake `/v1/bdbs/{uid}/availability` can lie *coherently with* the + data plane. Real sockets on both planes; the level where the whole thing is exercised. +3. **Fault-injector scenario test**, in the tier from step 0: drive real lag past the tolerance and + assert the lag-aware check flips while the plain check stays green — then that the group fails + over, and fails back only once lag recovers. Produce the lag with `crdt_sync=paused` on the target + member via its cluster's REST API (findings §10), and resume in a `finally`; not with + `network_latency`. + +Tier 3 is the authoritative one and the slowest; tier 1 is what runs on every push. + +## Step 6 — documentation + +`docs/` needs the probe, the tolerance decision and its reasoning, the credential/CA requirements, +and — prominently — the `Inconclusive` deviation, because anyone comparing us against Lettuce or +redis-py will otherwise read it as a bug. + +## Not doing + +- **Not writing a second Redis Enterprise REST client.** Step 0 exists so we extend + `ClusterRestClient` instead. +- **Not copying a tolerance default.** 100 ms and 5000 ms cannot both be right; step 1 measures. +- **Not implementing host-based discovery first.** Explicit uid keeps JSON out of the required path. +- **Not changing the profiling or health-check abstractions.** They already fit; that is the finding. diff --git a/notes/readme.md b/notes/readme.md new file mode 100644 index 000000000..b51960340 --- /dev/null +++ b/notes/readme.md @@ -0,0 +1,24 @@ +# Working notes + +This directory holds working material for whoever picks a topic up next: investigations, the +reasoning behind a decision, and snapshots that were expensive to produce and would be expensive +to produce again. + +The split to keep: + +- `docs/` — documentation for people *using* StackExchange.Redis. It is a Jekyll site + (`docs/_config.yml`, published at ), so anything placed there is built into + the site *and* listed in `sitemap.xml`. +- `notes/` — internal handover material. Not built, not published, not indexed. + +Notes are grouped by topic in a subdirectory, and the files inside are named for their content +rather than repeating the topic — `notes/lag-aware/findings.md`, not +`notes/lag-aware/lag-aware-findings.md`. + +When something in here graduates into advice for users, write it up in `docs/` and leave the note +as the record of *why*. + +## Topics + +- [`lag-aware/`](lag-aware/) — Redis Enterprise lag-aware availability checks, and whether they + belong in the geo-redundant failover health checks. diff --git a/src/RESPite.Benchmark/BenchmarkBase.cs b/src/RESPite.Benchmark/BenchmarkBase.cs index 620bbd6fe..3f4b1e9f7 100644 --- a/src/RESPite.Benchmark/BenchmarkBase.cs +++ b/src/RESPite.Benchmark/BenchmarkBase.cs @@ -437,19 +437,13 @@ private static string GetNameCore(Delegate underlyingAction, out string descript { string name = underlyingAction.Method.Name; - if (underlyingAction.Method.GetCustomAttribute(typeof(DisplayNameAttribute)) is DisplayNameAttribute - { - DisplayName: { Length: > 0 } - } dna) + if (underlyingAction.Method.GetCustomAttribute(typeof(DisplayNameAttribute)) is DisplayNameAttribute { DisplayName: { Length: > 0 } } dna) { name = dna.DisplayName; } description = ""; - if (underlyingAction.Method.GetCustomAttribute(typeof(DescriptionAttribute)) is DescriptionAttribute - { - Description: { Length: > 0 } - } da) + if (underlyingAction.Method.GetCustomAttribute(typeof(DescriptionAttribute)) is DescriptionAttribute { Description: { Length: > 0 } } da) { description = da.Description; } @@ -565,7 +559,7 @@ private async Task RunAsyncCore( $"{TotalOperations(divisor):###,###,##0} requests completed in {seconds:0.00} seconds, {rate:###,###,##0} ops/sec"); } - if (!Quiet & typeof(T) != typeof(DBNull)) + if (!Quiet & (typeof(T) != typeof(DBNull))) { const string format = "Typical result: {0}"; diff --git a/src/RESPite/Buffers/CycleBuffer.cs b/src/RESPite/Buffers/CycleBuffer.cs index 30436546a..a56c0a4b2 100644 --- a/src/RESPite/Buffers/CycleBuffer.cs +++ b/src/RESPite/Buffers/CycleBuffer.cs @@ -105,7 +105,7 @@ private void CopyDueToDiscardDuringWrite(int count) .CopyTo(full.Slice(targetOffset, count)); } } - public bool CommittedIsEmpty => ReferenceEquals(startSegment, endSegment) & endSegmentCommitted == 0; + public bool CommittedIsEmpty => ReferenceEquals(startSegment, endSegment) & (endSegmentCommitted == 0); /// /// Marks committed data as fully consumed; it will no longer appear in later calls to . @@ -117,8 +117,8 @@ public void DiscardCommitted(int count) // optimize for most common case, where we consume everything if (ReferenceEquals(startSegment, endSegment) - & count == endSegmentCommitted - & count > 0) + & (count == endSegmentCommitted) + & (count > 0)) { /* we are consuming all the data in the single segment; we can @@ -144,8 +144,8 @@ public void DiscardCommitted(long count) // optimize for most common case, where we consume everything if (ReferenceEquals(startSegment, endSegment) - & count == endSegmentCommitted - & count > 0) // checks sign *and* non-trimmed + & (count == endSegmentCommitted) + & (count > 0)) // checks sign *and* non-trimmed { // see for logic endSegmentCommitted = 0; @@ -339,7 +339,7 @@ public bool TryGetFirstCommittedMemory(int minBytes, out ReadOnlyMemory me // partial segment (and we know it isn't empty) memory = memory.Slice(start: 0, length: available); - return available >= minBytes & minBytes > 0; // last check here applies the -ve logic + return (available >= minBytes) & (minBytes > 0); // last check here applies the -ve logic } // multi-page; hand out the first page (which is, by definition: full) @@ -611,7 +611,7 @@ public int Untrim(bool expandBackwards = false) { _flags &= ~(Flags.StartTrim | Flags.EndTrim); Memory = fullMemory; - if (expandBackwards & RunningIndex >= delta) + if (expandBackwards & (RunningIndex >= delta)) { // push our origin earlier; only valid if // we're the first segment, otherwise diff --git a/src/RESPite/Internal/BlockBuffer.cs b/src/RESPite/Internal/BlockBuffer.cs index 752d74c8d..efdcfaa77 100644 --- a/src/RESPite/Internal/BlockBuffer.cs +++ b/src/RESPite/Internal/BlockBuffer.cs @@ -114,8 +114,8 @@ public static BlockBuffer GetBuffer(BlockBufferSerializer parent, int sizeHint) private int AvailableWithResetIfUseful() { if (_finalizedOffset != 0 // at least some chunks have been finalized - && Volatile.Read(ref _refCount) == 1 // all finalized chunks returned - & _writeOffset == _finalizedOffset) // we're not in the middle of serializing something new + && (Volatile.Read(ref _refCount) == 1) // all finalized chunks returned + & (_writeOffset == _finalizedOffset)) // we're not in the middle of serializing something new { _writeOffset = _finalizedOffset = 0; // swipe left } @@ -207,8 +207,8 @@ private void CopyFrom(Span source) private bool TryResizeFor(int extraBytes) { - if (_finalizedOffset == 0 & // we can only do this if there are no other messages in the buffer - Volatile.Read(ref _refCount) == 1) // and no-one else is looking (we already tried reset) + if ((_finalizedOffset == 0) & // we can only do this if there are no other messages in the buffer + (Volatile.Read(ref _refCount) == 1)) // and no-one else is looking (we already tried reset) { // we're already on the boundary - don't scrimp; just do the math from the end of the buffer byte[] newArray = _arrayPool.Rent(_array.Length + extraBytes); diff --git a/src/RESPite/Messages/RespFrameScanner.cs b/src/RESPite/Messages/RespFrameScanner.cs index 28cd92de7..ac974aee5 100644 --- a/src/RESPite/Messages/RespFrameScanner.cs +++ b/src/RESPite/Messages/RespFrameScanner.cs @@ -98,7 +98,7 @@ private static OperationStatus TryFastRead(ReadOnlySpan data, ref RespScan lo = ((uint)UnsafeCpuUInt16(data, 4)) << 16; break; case 7: - lo = ((uint)UnsafeCpuUInt16(data, 4)) << 16 | ((uint)UnsafeCpuByte(data, 6)) << 8; + lo = (((uint)UnsafeCpuUInt16(data, 4)) << 16) | (((uint)UnsafeCpuByte(data, 6)) << 8); break; default: lo = UnsafeCpuUInt32(data, 4); @@ -123,7 +123,7 @@ private static OperationStatus TryFastRead(ReadOnlySpan data, ref RespScan /// public OperationStatus TryRead(ref RespScanState state, in ReadOnlySequence data) { - if (!_pubsub & state.TotalBytes == 0 & data.IsSingleSegment) + if (!_pubsub & (state.TotalBytes == 0) & data.IsSingleSegment) { var status = TryFastRead(data.FirstSpan, ref state); if (status != UseReader) return status; @@ -148,7 +148,7 @@ static OperationStatus TryReadViaReader(ref RespScanState state, in ReadOnlySequ /// public OperationStatus TryRead(ref RespScanState state, ReadOnlySpan data) { - if (!_pubsub & state.TotalBytes == 0) + if (!_pubsub & (state.TotalBytes == 0)) { #if NET var status = TryFastRead(data, ref state); diff --git a/src/RESPite/Messages/RespReader.Utils.cs b/src/RESPite/Messages/RespReader.Utils.cs index 4a504ec1f..076b3ba28 100644 --- a/src/RESPite/Messages/RespReader.Utils.cs +++ b/src/RESPite/Messages/RespReader.Utils.cs @@ -349,7 +349,7 @@ public readonly byte[] Serialize() target = target.Slice(span.Length); if (!reader.TryMoveToNextSegment()) break; } - if (remaining != 0 | !target.IsEmpty) ThrowEof(); + if ((remaining != 0) | !target.IsEmpty) ThrowEof(); return arr; } } diff --git a/src/RESPite/Messages/RespReader.cs b/src/RESPite/Messages/RespReader.cs index 4bdf6112c..bde467503 100644 --- a/src/RESPite/Messages/RespReader.cs +++ b/src/RESPite/Messages/RespReader.cs @@ -240,7 +240,8 @@ private readonly bool ScalarLengthIsSlow(int expected) /// the API, using the API to update the outer reader. public readonly int AggregateLength() => (_flags & (RespFlags.IsAggregate | RespFlags.IsStreaming)) == RespFlags.IsAggregate - ? _length : AggregateLengthSlow(); + ? _length + : AggregateLengthSlow(); /// /// Indicates whether the number of child elements associated with an aggregate is exactly the specified value. @@ -250,7 +251,8 @@ public readonly int AggregateLength() => /// i.e. a map of the form %2\r\n... will report 4 as the length. public readonly bool AggregateLengthIs(int count) => (_flags & (RespFlags.IsAggregate | RespFlags.IsStreaming)) == RespFlags.IsAggregate - ? _length == count : AggregateLengthIsSlow(count); + ? _length == count + : AggregateLengthIsSlow(count); /// /// Indicates whether every child of the current aggregate is itself an aggregate of exactly two @@ -439,7 +441,7 @@ internal readonly bool IsNonNullAggregate internal int Delta() => (_flags & (RespFlags.IsScalar | RespFlags.IsAggregate | RespFlags.IsStreaming | RespFlags.IsAttribute)) switch { - RespFlags.IsScalar | RespFlags.IsAggregate=> -1, // null has this + RespFlags.IsScalar | RespFlags.IsAggregate => -1, // null has this RespFlags.IsScalar => -1, RespFlags.IsAggregate => _length - 1, RespFlags.IsAggregate | RespFlags.IsAttribute => _length, @@ -477,7 +479,7 @@ private bool TryReadNextSkipAttributes(bool skipStreamTerminator) { SkipChildren(); } - else if (skipStreamTerminator & Prefix is RespPrefix.StreamTerminator) + else if (skipStreamTerminator & (Prefix is RespPrefix.StreamTerminator)) { // skip terminator } @@ -500,7 +502,7 @@ private bool TryReadNextProcessAttributes(RespAttributeReader respAttribut { respAttributeReader.Read(ref this, ref attributes); } - else if (skipStreamTerminator & Prefix is RespPrefix.StreamTerminator) + else if (skipStreamTerminator & (Prefix is RespPrefix.StreamTerminator)) { // skip terminator } @@ -1740,7 +1742,7 @@ internal readonly bool IsOK() // go mad with this, because it is used so often if (TryGetSpan(out var span) && span.Length == 2) { var u16 = Unsafe.ReadUnaligned(ref UnsafeCurrent); - return u16 == RespConstants.OKUInt16 | u16 == RespConstants.OKUInt16_LC; + return (u16 == RespConstants.OKUInt16) | (u16 == RespConstants.OKUInt16_LC); } return IsSlow(RespConstants.OKBytes, RespConstants.OKBytes_LC); @@ -2143,7 +2145,7 @@ public readonly bool TryReadInt64(out long value) var span = Buffer(stackalloc byte[RespConstants.MaxRawBytesInt64 + 1]); if (span.Length <= RespConstants.MaxRawBytesInt64) { - return Utf8Parser.TryParse(span, out value, out int bytes) & bytes == span.Length; + return Utf8Parser.TryParse(span, out value, out int bytes) & (bytes == span.Length); } value = 0; @@ -2177,7 +2179,7 @@ public readonly bool TryReadInt32(out int value) var span = Buffer(stackalloc byte[RespConstants.MaxRawBytesInt32 + 1]); if (span.Length <= RespConstants.MaxRawBytesInt32) { - return Utf8Parser.TryParse(span, out value, out int bytes) & bytes == span.Length; + return Utf8Parser.TryParse(span, out value, out int bytes) & (bytes == span.Length); } value = 0; @@ -2265,7 +2267,7 @@ internal readonly bool UnsafeTryReadShortAscii(out string value, int maxLength = foreach (var b in span) { // allow [SPACE]-thru-[DEL], plus CR/LF - if (!(b < 127 & (b >= 32 | (b is 12 or 13)))) + if (!((b < 127) & ((b >= 32) | (b is 12 or 13)))) { return false; } diff --git a/src/RESPite/Messages/RespScanState.cs b/src/RESPite/Messages/RespScanState.cs index 37cd3f8b6..ef5588ee6 100644 --- a/src/RESPite/Messages/RespScanState.cs +++ b/src/RESPite/Messages/RespScanState.cs @@ -127,7 +127,7 @@ private long ReadCore(ref RespReader reader, long startOffset = 0) #if DEBUG _elementCount++; #endif - if (!reader.IsAttribute & _prefix == RespPrefix.None) + if (!reader.IsAttribute & (_prefix == RespPrefix.None)) { _prefix = reader.Prefix; } diff --git a/src/RESPite/PublicAPI/PublicAPI.Unshipped.txt b/src/RESPite/PublicAPI/PublicAPI.Unshipped.txt index a68e8f01c..c7a0beb41 100644 --- a/src/RESPite/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/RESPite/PublicAPI/PublicAPI.Unshipped.txt @@ -53,3 +53,5 @@ virtual RESPite.Messages.RespReader.PairProjection.Invoke(ref T [SER012]static RESPite.Messages.RespValue.TryCaptureNext(object? owner, scoped ref RESPite.Messages.RespReader reader, out RESPite.Messages.RespValue value) -> bool [SER012]RESPite.Messages.RespValue.TryGetSpan(out System.ReadOnlySpan payload) -> bool [SER012]static RESPite.Messages.RespValue.explicit operator string?(RESPite.Messages.RespValue value) -> string? +*REMOVED*[SER004]override RESPite.AsciiHash.Equals(object? other) -> bool +[SER004]override RESPite.AsciiHash.Equals(object? obj) -> bool diff --git a/src/RESPite/Shared/AsciiHash.Comparers.cs b/src/RESPite/Shared/AsciiHash.Comparers.cs index 7b69a15a4..bc3bd5ff7 100644 --- a/src/RESPite/Shared/AsciiHash.Comparers.cs +++ b/src/RESPite/Shared/AsciiHash.Comparers.cs @@ -13,7 +13,7 @@ private CaseSensitiveComparer() { } public bool Equals(AsciiHash x, AsciiHash y) { var len = x.Length; - return (len == y.Length & x._hashCS == y._hashCS) + return ((len == y.Length) & (x._hashCS == y._hashCS)) && (len <= MaxBytesHashed || x.Span.SequenceEqual(y.Span)); } @@ -28,7 +28,7 @@ private CaseInsensitiveComparer() { } public bool Equals(AsciiHash x, AsciiHash y) { var len = x.Length; - return (len == y.Length & x._hashUC == y._hashUC) + return ((len == y.Length) & (x._hashUC == y._hashUC)) && (len <= MaxBytesHashed || SequenceEqualsCI(x.Span, y.Span)); } diff --git a/src/RESPite/Shared/AsciiHash.Instance.cs b/src/RESPite/Shared/AsciiHash.Instance.cs index 93b8258ca..94a774f81 100644 --- a/src/RESPite/Shared/AsciiHash.Instance.cs +++ b/src/RESPite/Shared/AsciiHash.Instance.cs @@ -37,13 +37,13 @@ public AsciiHash(string? value) : this(value is null ? [] : Encoding.ASCII.GetBy public override string ToString() => _length == 0 ? "" : Encoding.ASCII.GetString(_arr, _index, _length); /// - public override bool Equals(object? other) => other is AsciiHash hash && Equals(hash); + public override bool Equals(object? obj) => obj is AsciiHash hash && Equals(hash); /// [CLSCompliant(false)] public bool Equals(in AsciiHash other) { - return (_length == other.Length & _hashCS == other._hashCS) + return ((_length == other.Length) & (_hashCS == other._hashCS)) && (_length <= MaxBytesHashed || Span.SequenceEqual(other.Span)); } @@ -65,7 +65,7 @@ public bool IsCS(ReadOnlySpan value) { var cs = HashCS(value); var len = _length; - if (cs != _hashCS | value.Length != len) return false; + if ((cs != _hashCS) | (value.Length != len)) return false; return len <= MaxBytesHashed || Span.SequenceEqual(value); } @@ -73,7 +73,7 @@ public bool IsCI(ReadOnlySpan value) { var uc = HashUC(value); var len = _length; - if (uc != _hashUC | value.Length != len) return false; + if ((uc != _hashUC) | (value.Length != len)) return false; return len <= MaxBytesHashed || SequenceEqualsCI(Span, value); } diff --git a/src/RESPite/Shared/AsciiHash.cs b/src/RESPite/Shared/AsciiHash.cs index 15164c4fa..3fa732343 100644 --- a/src/RESPite/Shared/AsciiHash.cs +++ b/src/RESPite/Shared/AsciiHash.cs @@ -106,7 +106,7 @@ public static unsafe bool SequenceEqualsCI(ReadOnlySpan first, ReadOnlySpa { byte x = firstPtr[i]; var xCI = x & CS_MASK; - if (xCI >= 'A' & xCI <= 'Z') + if ((xCI >= 'A') & (xCI <= 'Z')) { // alpha mismatch if (xCI != (secondPtr[i] & CS_MASK)) return false; @@ -171,7 +171,7 @@ public static unsafe bool SequenceEqualsCI(ReadOnlySpan first, ReadOnlySpa { int x = (byte)firstPtr[i]; var xCI = x & CS_MASK; - if (xCI >= 'A' & xCI <= 'Z') + if ((xCI >= 'A') & (xCI <= 'Z')) { // alpha mismatch if (xCI != (secondPtr[i] & CS_MASK)) return false; @@ -206,7 +206,7 @@ public static unsafe bool SequenceEqualsCI(ReadOnlySpan first, ReadOnlySpa { int x = (byte)firstPtr[i]; var xCI = x & CS_MASK; - if (xCI >= 'A' & xCI <= 'Z') + if ((xCI >= 'A') & (xCI <= 'Z')) { // alpha mismatch if (xCI != (secondPtr[i] & CS_MASK)) return false; diff --git a/src/StackExchange.Redis/ArrayGrepRequest.cs b/src/StackExchange.Redis/ArrayGrepRequest.cs index d8e4273da..07b557363 100644 --- a/src/StackExchange.Redis/ArrayGrepRequest.cs +++ b/src/StackExchange.Redis/ArrayGrepRequest.cs @@ -60,7 +60,10 @@ private void SetFlag(LocalFlags flag, bool value) /// public RedisArrayIndex? Start { - get => GetFlag(LocalFlags.StartSpecified) ? _start : null; + get + { + return GetFlag(LocalFlags.StartSpecified) ? _start : null; + } set { if (value.HasValue) @@ -85,7 +88,10 @@ public RedisArrayIndex? Start /// public RedisArrayIndex? End { - get => GetFlag(LocalFlags.EndSpecified) ? _end : null; + get + { + return GetFlag(LocalFlags.EndSpecified) ? _end : null; + } set { if (value.HasValue) @@ -111,7 +117,10 @@ public RedisArrayIndex? End /// Corresponds to the LIMIT parameter. public long? Limit { - get => GetFlag(LocalFlags.LimitSpecified) ? _limit : null; + get + { + return GetFlag(LocalFlags.LimitSpecified) ? _limit : null; + } set { if (value.HasValue) diff --git a/src/StackExchange.Redis/Availability/CircuitBreaker.cs b/src/StackExchange.Redis/Availability/CircuitBreaker.cs index bf10bf834..e04c6a946 100644 --- a/src/StackExchange.Redis/Availability/CircuitBreaker.cs +++ b/src/StackExchange.Redis/Availability/CircuitBreaker.cs @@ -78,11 +78,11 @@ public CircuitBreaker Create() if (MetricsWindowSize <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(MetricsWindowSize), MetricsWindowSize, "A positive window is required."); if (MetricsWindowSize.TotalSeconds > int.MaxValue) throw new ArgumentOutOfRangeException(nameof(MetricsWindowSize), MetricsWindowSize, "The window is too large."); - if ((FailureRateThreshold is DefaultFailureRateThreshold + if (((FailureRateThreshold is DefaultFailureRateThreshold) #if NET8_0_OR_GREATER - & TimeProvider is null + & (TimeProvider is null) #endif - & MinimumNumberOfFailures is DefaultMinimumNumberOfFailures) + & (MinimumNumberOfFailures is DefaultMinimumNumberOfFailures)) && MetricsWindowSize == DefaultMetricsWindowSize) return DefaultInstance; @@ -182,7 +182,7 @@ private sealed class NulAccumulator : Accumulator { public static readonly NulAccumulator AccumulatorInstance = new(); private NulAccumulator() { } - public override void ObserveResult(in FaultContext context) { } + public override void ObserveResult(in FaultContext fault) { } public override bool IsHealthy() => true; public override void Reset() { } } @@ -299,7 +299,7 @@ public void Reset() } } - public override void ObserveResult(in FaultContext result) + public override void ObserveResult(in FaultContext fault) { // which time-slice are we in, and where does it live in the ring? long epoch = breaker.GetEpoch(); @@ -312,7 +312,7 @@ public override void ObserveResult(in FaultContext result) // only getting results one at a time, so we shouldn't be over-stomping much *anyway* Span buckets = _buckets; // *not* a payload copy; this is in-place over the data ref Bucket bucket = ref buckets[index]; - bucket.Count(epoch, success: !result.IsFault); + bucket.Count(epoch, success: !fault.IsFault); } public override bool IsHealthy() => Evaluate(breaker.GetEpoch()); @@ -337,7 +337,7 @@ private bool Evaluate(long epoch) } // don't act until we've seen enough failures to be statistically meaningful - if (total is 0 | failures < breaker._minimumNumberOfFailures) + if ((total is 0) | (failures < breaker._minimumNumberOfFailures)) { return true; } diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs index e7b61e383..d147a8d9e 100644 --- a/src/StackExchange.Redis/Availability/FaultContext.cs +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -53,7 +53,7 @@ public FaultContext(Exception fault) NotApplied = IsKnownNotApplied(kind, status); - if (kind is not RedisErrorKind.None & _connectionFailureType is ConnectionFailureType.None) + if ((kind is not RedisErrorKind.None) & (_connectionFailureType is ConnectionFailureType.None)) { // fill in some blanks switch (kind) diff --git a/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs index 1ee5dda8a..13ca8694d 100644 --- a/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs @@ -11,16 +11,26 @@ public sealed partial class HealthCheck /// Evaluate the health of the specified multiplexer, by evaluating all endpoints. /// public Task CheckHealthAsync(IConnectionMultiplexer multiplexer) - => !IsEnabled ? HealthCheckProbe.InconclusiveTask - : multiplexer.IsConnected ? CheckHealthCoreAsync(multiplexer) - : HealthCheckProbe.UnhealthyTask; + => !IsEnabled + ? HealthCheckProbe.InconclusiveTask + : multiplexer.IsConnected + ? CheckHealthCoreAsync(multiplexer) + : HealthCheckProbe.UnhealthyTask; private async Task CheckHealthCoreAsync(IConnectionMultiplexer multiplexer) { try { Task[] pending; - if (multiplexer is IInternalConnectionMultiplexer internalMultiplexer) + if (Probe.Scope is HealthCheckProbeScope.Member) + { + // the answer is about the database, not an endpoint: ask once, via any connected server + var server = FirstConnectedServer(multiplexer); + if (server is null) return HealthCheckResult.Unhealthy; + pending = GetReusablePending(ref _reusablePending, 1); + pending[0] = CheckHealthAsync(server); + } + else if (multiplexer is IInternalConnectionMultiplexer internalMultiplexer) { var snapshot = internalMultiplexer.GetServerSnapshot(); pending = GetReusablePending(ref _reusablePending, snapshot.Length); @@ -51,6 +61,26 @@ private async Task CheckHealthCoreAsync(IConnectionMultiplexe } } + private static IServer? FirstConnectedServer(IConnectionMultiplexer multiplexer) + { + if (multiplexer is IInternalConnectionMultiplexer internalMultiplexer) + { + foreach (var endpoint in internalMultiplexer.GetServerSnapshot()) + { + if (endpoint.IsConnected) return endpoint.GetRedisServer(null); + } + + return null; + } + + foreach (var server in multiplexer.GetServers()) + { + if (server.IsConnected) return server; + } + + return null; + } + internal int TotalTimeoutMillis() { bool valid = TryComputeTotalTimeoutMillis(ProbeCount, ProbeTimeout, ProbeInterval, out int millis); @@ -143,9 +173,11 @@ internal static void PutReusablePending(ref Task[]? field, re /// Evaluate the health of an endpoint. /// public Task CheckHealthAsync(IServer server) - => !IsEnabled ? HealthCheckProbe.InconclusiveTask - : server.IsConnected ? CheckHealthCoreAsync(server) - : HealthCheckProbe.UnhealthyTask; + => !IsEnabled + ? HealthCheckProbe.InconclusiveTask + : server.IsConnected + ? CheckHealthCoreAsync(server) + : HealthCheckProbe.UnhealthyTask; private async Task CheckHealthCoreAsync(IServer server) { diff --git a/src/StackExchange.Redis/Availability/HealthCheck.LagAware.cs b/src/StackExchange.Redis/Availability/HealthCheck.LagAware.cs new file mode 100644 index 000000000..5ccbad622 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheck.LagAware.cs @@ -0,0 +1,16 @@ +namespace StackExchange.Redis.Availability; + +public sealed partial class HealthCheck +{ + /// + /// A health check that asks the Redis Enterprise database availability API, once per member and once per + /// pass, whether the database is available and (for an Active-Active member) caught up; intended as a + /// . See . + /// + /// + /// For different probe counts or timeouts, use + /// new HealthCheck.Builder { Probe = HealthCheckProbe.LagAware(options), ... }. + /// + public static HealthCheck LagAware(LagAwareOptions options) + => new Builder { Probe = HealthCheckProbe.LagAware(options), ProbeCount = 1 }.Create(); +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.LagAware.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.LagAware.cs new file mode 100644 index 000000000..2546df4f6 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.LagAware.cs @@ -0,0 +1,249 @@ +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +public abstract partial class HealthCheckProbe +{ + /// + /// Ask the Redis Enterprise database availability API whether the member's database is available and, + /// for an Active-Active member, caught up; see . Runs once per member. + /// + /// + /// Healthy on HTTP 200; unhealthy on 503 (unavailable or lagging) or 404 (no such database); + /// when the question could not be asked (the REST API + /// unreachable, credentials rejected, the database not found by discovery), so that a management-plane + /// problem is never mistaken for the database being down. Intended as a + /// . + /// + public static HealthCheckProbe LagAware(LagAwareOptions options) => new LagAwareProbe(options); + + internal sealed class LagAwareProbe : HealthCheckProbe + { + private const int DefaultRestPort = 9443; + + private readonly Func> _credentials; + private readonly Uri? _restEndpoint; + private readonly int? _databaseId; + private readonly LagCheckMode _mode; + private readonly TimeSpan? _tolerance; + private readonly HttpClient _http; + + // per (REST endpoint, database endpoint): one probe can serve every member of a group + private readonly ConcurrentDictionary _discovered = new(StringComparer.OrdinalIgnoreCase); + + private sealed class Discovered + { + public int? Uid; + public bool? ActiveActive; + } + + public LagAwareProbe(LagAwareOptions options) + { + if (options is null) throw new ArgumentNullException(nameof(options)); + _credentials = options.Credentials ?? throw new ArgumentException("Credentials are required.", nameof(options)); + if (options.LagTolerance is { } tolerance && tolerance < TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(options), tolerance, "LagTolerance must not be negative."); + } + + _restEndpoint = options.RestEndpoint; + _databaseId = options.DatabaseId; + _mode = options.LagCheck; + _tolerance = options.LagTolerance; + var handler = options.HttpMessageHandlerFactory?.Invoke() ?? CreateHandler(options.CertificateValidation, options.ClientCertificate); + _http = new HttpClient(handler) { Timeout = Timeout.InfiniteTimeSpan }; // bounded per call instead + } + + public override HealthCheckProbeScope Scope => HealthCheckProbeScope.Member; + + internal static HttpMessageHandler CreateHandler(RemoteCertificateValidationCallback? validation, X509Certificate2? clientCertificate) + { +#if NET + // redirects point at a node's internal address, so following one only ever hangs; report it instead + var handler = new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(2), AllowAutoRedirect = false }; + if (validation is not null) handler.SslOptions.RemoteCertificateValidationCallback = validation; + if (clientCertificate is not null) handler.SslOptions.ClientCertificates = [clientCertificate]; + return handler; +#elif NET461 + if (validation is not null || clientCertificate is not null) + { + throw new PlatformNotSupportedException("Custom certificate validation and client certificates for the REST API require .NET Framework 4.7.1 or later."); + } + return new HttpClientHandler { AllowAutoRedirect = false }; +#else + var handler = new HttpClientHandler { AllowAutoRedirect = false }; + if (validation is not null) + { + handler.ServerCertificateCustomValidationCallback = (request, certificate, chain, errors) => validation(request, certificate, chain, errors); + } + if (clientCertificate is not null) + { + handler.ClientCertificateOptions = ClientCertificateOption.Manual; + handler.ClientCertificates.Add(clientCertificate); + } + return handler; +#endif + } + + public override async Task CheckHealthAsync(HealthCheckContext context) + { + var multiplexer = context.Server.Multiplexer; + var endpoints = (multiplexer as IInternalConnectionMultiplexer)?.RawConfig.EndPoints; + var configured = endpoints is { Count: > 0 } ? endpoints[0] : null; + if (!Format.TryGetHostPort(configured, out var host, out var port)) + { + return Inconclusive(multiplexer, "(none)", "the member has no configured endpoint"); + } + + var rest = _restEndpoint ?? new UriBuilder(Uri.UriSchemeHttps, ClusterHost(host, port.Value), DefaultRestPort).Uri; + var key = $"{rest}|{host}:{port}"; + var known = _discovered.GetOrAdd(key, static _ => new Discovered()); + + // finish inside the probe timeout, so that "could not ask" is reported as such rather than as the + // Unhealthy a timed-out probe is otherwise given + using var cts = new CancellationTokenSource(Budget(context.ProbeTimeout)); + try + { + var credential = await _credentials(cts.Token).ConfigureAwait(false); + var auth = new AuthenticationHeaderValue( + "Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{credential.UserName}:{credential.Password}"))); + + int? uid = _databaseId ?? known.Uid; + if (uid is null) + { + // filtered: an unfiltered /v1/bdbs carries database passwords, even for db_viewer + using var list = await GetAsync(rest, "v1/bdbs?fields=uid,endpoints", auth, cts.Token).ForAwait(); + if (list.StatusCode != HttpStatusCode.OK) return Inconclusive(multiplexer, rest, Describe("database discovery", list)); + uid = FindUid(await list.Content.ReadAsStringAsync().ForAwait(), host, port!.Value); + if (uid is null) return Inconclusive(multiplexer, rest, $"no database on this cluster has the endpoint {host}:{port}"); + known.Uid = uid; + } + + bool lag = _mode switch + { + LagCheckMode.Enabled => true, + LagCheckMode.Disabled => false, + _ => known.ActiveActive ?? false, + }; + if (_mode is LagCheckMode.Auto && known.ActiveActive is null) + { + using var bdb = await GetAsync(rest, $"v1/bdbs/{uid}?fields=uid,crdt", auth, cts.Token).ForAwait(); + if (bdb.StatusCode == HttpStatusCode.NotFound) return Forget(key); + if (bdb.StatusCode != HttpStatusCode.OK) return Inconclusive(multiplexer, rest, Describe("database lookup", bdb)); + lag = IsActiveActive(await bdb.Content.ReadAsStringAsync().ForAwait()); + known.ActiveActive = lag; + } + + var path = $"v1/bdbs/{uid}/availability"; + if (lag) + { + path += "?extend_check=lag"; + if (_tolerance is { } tolerance) path += $"&availability_lag_tolerance_ms={(long)tolerance.TotalMilliseconds}"; + } + + using var response = await GetAsync(rest, path, auth, cts.Token).ForAwait(); + return response.StatusCode switch + { + HttpStatusCode.OK => HealthCheckResult.Healthy, + HttpStatusCode.ServiceUnavailable => HealthCheckResult.Unhealthy, // unavailable, or lagging + HttpStatusCode.NotFound => Forget(key), + _ => Inconclusive(multiplexer, rest, Describe("availability", response)), + }; + } + catch (Exception ex) when (ex is HttpRequestException or OperationCanceledException or JsonException or IOException) + { + return Inconclusive(multiplexer, rest, ex is OperationCanceledException ? "timed out" : ex.Message); + } + } + + // Database endpoints are named "redis-{port}.{cluster fqdn}"; ask the cluster name, not the database's. Any + // node answers the availability routes itself, but the others (discovery, the crdt lookup) are answered + // only by the master: other nodes redirect to its *internal* address, unreachable from outside the + // cluster's network. The cluster name resolves to the master. A host that does not follow the pattern + // is used as given. + internal static string ClusterHost(string host, int port) + { + var prefix = $"redis-{port}."; + return host.Length > prefix.Length && host.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + ? host.Substring(prefix.Length) + : host; + } + + // the database has gone (or the configured uid is wrong): unhealthy, and rediscover next time + private HealthCheckResult Forget(string key) + { + _discovered.TryRemove(key, out _); + return HealthCheckResult.Unhealthy; + } + + private static string Describe(string what, HttpResponseMessage response) + => response.Headers.Location is { } location + ? $"{what} answered {(int)response.StatusCode}, redirecting to {location} (set RestEndpoint to an address that answers directly)" + : $"{what} answered {(int)response.StatusCode}"; + + private static TimeSpan Budget(TimeSpan probeTimeout) + { + var budget = TimeSpan.FromTicks(probeTimeout.Ticks / 10 * 9); + return budget > TimeSpan.Zero ? budget : TimeSpan.FromMilliseconds(1); + } + + private async Task GetAsync(Uri rest, string path, AuthenticationHeaderValue auth, CancellationToken cancellationToken) + { + using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(rest, path)); + request.Headers.Authorization = auth; + return await _http.SendAsync(request, cancellationToken).ForAwait(); + } + + private static HealthCheckResult Inconclusive(IConnectionMultiplexer multiplexer, object rest, string reason) + { + (multiplexer as ConnectionMultiplexer)?.Logger?.LogWarningLagAwareInconclusive(rest.ToString() ?? "", reason); + return HealthCheckResult.Inconclusive; + } + + // GET /v1/bdbs?fields=uid,endpoints => [{"uid":1,"endpoints":[{"dns_name":"redis-1.example","port":1,"addr":["10.0.0.1"]}]}] + internal static int? FindUid(string json, string host, int port) + { + using var document = JsonDocument.Parse(json); + if (document.RootElement.ValueKind != JsonValueKind.Array) return null; + foreach (var bdb in document.RootElement.EnumerateArray()) + { + if (!bdb.TryGetProperty("uid", out var uid) || !uid.TryGetInt32(out var id)) continue; + if (!bdb.TryGetProperty("endpoints", out var endpoints) || endpoints.ValueKind != JsonValueKind.Array) continue; + foreach (var endpoint in endpoints.EnumerateArray()) + { + if (!endpoint.TryGetProperty("port", out var p) || !p.TryGetInt32(out var endpointPort) || endpointPort != port) continue; + if (endpoint.TryGetProperty("dns_name", out var dns) && string.Equals(dns.GetString(), host, StringComparison.OrdinalIgnoreCase)) return id; + if (endpoint.TryGetProperty("addr", out var addrs) && addrs.ValueKind == JsonValueKind.Array) + { + foreach (var addr in addrs.EnumerateArray()) + { + if (string.Equals(addr.GetString(), host, StringComparison.OrdinalIgnoreCase)) return id; + } + } + } + } + + return null; + } + + // GET /v1/bdbs/{uid}?fields=uid,crdt => {"crdt":true,"uid":1} + internal static bool IsActiveActive(string json) + { + using var document = JsonDocument.Parse(json); + return document.RootElement.ValueKind == JsonValueKind.Object + && document.RootElement.TryGetProperty("crdt", out var crdt) + && crdt.ValueKind == JsonValueKind.True; + } + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.cs index 6b449e034..6edb08208 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckProbe.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.cs @@ -15,6 +15,11 @@ public abstract partial class HealthCheckProbe /// public abstract Task CheckHealthAsync(HealthCheckContext context); + /// + /// Whether this probe is invoked per endpoint (the default) or once per member. + /// + public virtual HealthCheckProbeScope Scope => HealthCheckProbeScope.Endpoint; + private static Task? _inconclusive, _healthy, _unhealthy; /// diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs b/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs index b09848814..19e3869d3 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs @@ -52,8 +52,10 @@ public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) // Fail as soon as we have any failure if (context.Failure > 0) return HealthCheckResult.Unhealthy; - // Succeed only when all probes have succeeded (no remaining) - if (context.Remaining == 0) return HealthCheckResult.Healthy; + // With no failures and nothing remaining, succeed - but only if something actually succeeded: a run + // of nothing but inconclusive probes ("could not tell") is not a pass, and a failback check + // depends on the difference + if (context.Remaining == 0) return context.Success > 0 ? HealthCheckResult.Healthy : HealthCheckResult.Inconclusive; // Can't determine yet return HealthCheckResult.Inconclusive; @@ -70,8 +72,10 @@ public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) // Succeed as soon as we have any success if (context.Success > 0) return HealthCheckResult.Healthy; - // Fail only when all probes have failed (no remaining) - if (context.Remaining == 0) return HealthCheckResult.Unhealthy; + // With nothing remaining and no success, fail - but only if something actually failed: a run of + // nothing but inconclusive probes ("could not tell") is not a failure, and reporting it as one marks + // a member unhealthy that may be perfectly well (a key-write probe on a replica, for example) + if (context.Remaining == 0) return context.Failure > 0 ? HealthCheckResult.Unhealthy : HealthCheckResult.Inconclusive; // Can't determine yet return HealthCheckResult.Inconclusive; diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbeScope.cs b/src/StackExchange.Redis/Availability/HealthCheckProbeScope.cs new file mode 100644 index 000000000..77ce912d1 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbeScope.cs @@ -0,0 +1,19 @@ +namespace StackExchange.Redis.Availability; + +/// +/// How often a is invoked when a whole member is checked. +/// +public enum HealthCheckProbeScope +{ + /// + /// Once per server endpoint of the member; the default, and right for probes that test the connection + /// itself (such as ). + /// + Endpoint = 0, + + /// + /// Once per member, against one connected endpoint; for probes whose answer is about the database as a + /// whole rather than any single endpoint, such as a call to a management API. + /// + Member = 1, +} diff --git a/src/StackExchange.Redis/Availability/LagAwareOptions.cs b/src/StackExchange.Redis/Availability/LagAwareOptions.cs new file mode 100644 index 000000000..e74b4f08d --- /dev/null +++ b/src/StackExchange.Redis/Availability/LagAwareOptions.cs @@ -0,0 +1,135 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +/// +/// Configures a health check against the Redis Enterprise database availability API, optionally asking +/// whether the database has caught up with its Active-Active peers; see . +/// +/// +/// Only is required. The REST endpoint, the database id and whether the database is +/// an Active-Active member are discovered, and no lag tolerance is sent unless one is set, so the cluster's +/// own setting applies. The options are read when the probe is created; later changes have no effect on it. +/// +public sealed class LagAwareOptions +{ + /// + /// Supplies the cluster management credentials, called for every request so that they can rotate. + /// + /// + /// These are the cluster's REST API credentials, not the database's. A role with + /// management: db_viewer is sufficient. + /// + public Func>? Credentials { get; set; } + + /// + /// The cluster REST API to ask; when (the default), derived from the member's first + /// configured endpoint: https://{cluster}:9443/ for a database named redis-{port}.{cluster} (the + /// form Redis Enterprise assigns), else https://{host}:9443/. + /// + /// + /// The cluster name matters: only the cluster's master answers every route the probe uses, and other nodes + /// redirect to its internal address. Redirects are never followed; set this explicitly if the database is + /// reached through a name that does not follow the pattern. + /// + public Uri? RestEndpoint { get; set; } + + /// + /// The database id (uid) on that cluster; when (the default), it is found by + /// matching the member's first configured endpoint against the cluster's databases. + /// + public int? DatabaseId { get; set; } + + /// + /// Whether to ask the lag question; (the default) asks it only of + /// Active-Active databases, since the server reports any other database as unavailable when asked. + /// + public LagCheckMode LagCheck { get; set; } + + /// + /// The lag tolerance to request; when (the default), none is sent and the cluster's + /// own availability_lag_tolerance_ms applies. + /// + public TimeSpan? LagTolerance { get; set; } + + /// + /// Validates the REST API's certificate; when , the platform's default validation + /// applies. + /// + /// + /// The management certificate of a Redis Enterprise cluster is self-signed by the cluster unless one has + /// been installed, so this is usually needed; see . Not supported on .NET + /// Framework 4.6.1. + /// + public RemoteCertificateValidationCallback? CertificateValidation { get; set; } + + /// + /// Trust certificates issued by the given certificate (which may be the cluster's own self-signed + /// certificate), as does for the data plane. + /// + public void TrustIssuer(string issuerCertificatePath) + => CertificateValidation = ConfigurationOptions.TrustIssuerCallback(issuerCertificatePath); + + /// + /// Trust certificates issued by the given certificate (which may be the cluster's own self-signed + /// certificate), as does for the data plane. + /// + public void TrustIssuer(X509Certificate2 issuer) + => CertificateValidation = ConfigurationOptions.TrustIssuerCallback(issuer); + + /// + /// A client certificate to present to the REST API, for clusters that require mutual TLS on it; when + /// (the default), none. Not supported on .NET Framework 4.6.1. + /// + public X509Certificate2? ClientCertificate { get; set; } + +#if NET + /// + /// Present a client certificate loaded from a PEM file pair, as + /// does for the data plane. + /// + /// The path for the user certificate (commonly a .crt file). + /// The path for the user key (commonly a .key file). + public void SetUserPemCertificate(string userCertificatePath, string? userKeyPath = null) + => ClientCertificate = ConfigurationOptions.LoadPemUserCertificate(userCertificatePath, userKeyPath); +#endif + + /// + /// Present a client certificate loaded from a PFX file, as + /// does for the data plane. + /// + /// The path for the user certificate (commonly a .pfx file). + /// The password for the certificate file. + public void SetUserPfxCertificate(string userCertificatePath, string? password = null) + => ClientCertificate = ConfigurationOptions.LoadPfxUserCertificate(userCertificatePath, password); + + // test seam: replaces the HTTP transport entirely, so tests can answer requests without a listener + internal Func? HttpMessageHandlerFactory { get; set; } +} + +/// +/// Whether a check asks the lag question. +/// +public enum LagCheckMode +{ + /// + /// Ask only if the database is an Active-Active member; otherwise check plain availability. + /// + Auto = 0, + + /// + /// Always ask. A database that is not an Active-Active member is then always reported unavailable. + /// + Enabled = 1, + + /// + /// Never ask; check plain availability only. + /// + Disabled = 2, +} diff --git a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs index acd4876a7..8a4156822 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs @@ -89,6 +89,7 @@ private enum MemberFlags ExplicitOverrideFlag = 1 << 2, // manual failover target (see ExplicitOverride) SkipInitialHealthCheck = 1 << 3, // see SkipInitialHealthCheck Unhealthy = 1 << 4, // disabled by a health-check or circuit-breaker + FailbackVerified = 1 << 5, // most recent failback health-check passed (see FailbackHealthCheck) } private int _flags; @@ -124,6 +125,8 @@ internal void SetUnhealthy() // UTC ticks (DateTime.Ticks and TimeSpan.Ticks share the same 100ns unit); long for atomicity private long _lastUnhealthyTicks; + internal long LastUnhealthyTicks => Volatile.Read(ref _lastUnhealthyTicks); + /// /// Clear the flag against this endpoint, allowing it to be reconsidered. /// @@ -231,6 +234,21 @@ public bool SkipInitialHealthCheck /// public TimeSpan? FailbackDelay { get; set; } + /// + /// A health-check consulted only before traffic moves to this member; when + /// , is used, and when that + /// is also , there is none. + /// + /// + /// Unlike , this never removes the active member: it decides whether a member + /// that is not active may become so, typically by asking whether it has caught up with the others. + /// A member is only eligible once this reports ; an + /// answer (for example, the question could not be asked) + /// leaves it ineligible. If no connected member is eligible, selection falls back to connectivity and + /// alone, so the group always serves from somewhere when it can. + /// + public HealthCheck? FailbackHealthCheck { get; set; } + // the breaker to hand to this member's connections: member override, else its own config, else the group internal CircuitBreaker? ResolveCircuitBreaker(MultiGroupOptions options) => CircuitBreaker ?? Configuration.CircuitBreaker ?? options.CircuitBreaker; @@ -239,6 +257,15 @@ public bool SkipInitialHealthCheck internal TimeSpan ResolveFailbackDelay(MultiGroupOptions options) => FailbackDelay ?? options.FailbackDelay; + internal HealthCheck? ResolveFailbackHealthCheck(MultiGroupOptions options) => FailbackHealthCheck ?? options.FailbackHealthCheck; + + // maintained by the group on each pass; only meaningful when a failback health-check resolves + internal bool FailbackVerified + { + get => GetFlag(MemberFlags.FailbackVerified); + set => SetFlag(MemberFlags.FailbackVerified, value); + } + /// /// The relative weight of this group member; higher is preferred. /// @@ -330,7 +357,10 @@ internal GroupConnectionChangedEventArgs.ChangeType UpdateState(HealthCheckResul if (isConnected) { - if (Volatile.Read(ref _lastUnhealthyTicks) < failbackFailureCutoffTicks) ResetIsUnhealthy(); + // at or before the cutoff: with a zero FailbackDelay the cutoff is "now", and on a coarse clock + // (DateTime.UtcNow advances every ~15.6ms on .NET Framework) a failure and the pass that clears + // it can share a tick - a strict comparison would then hold the member out for another interval + if (Volatile.Read(ref _lastUnhealthyTicks) <= failbackFailureCutoffTicks) ResetIsUnhealthy(); } else { @@ -340,9 +370,11 @@ internal GroupConnectionChangedEventArgs.ChangeType UpdateState(HealthCheckResul var oldConnected = IsConnected; IsConnected = isConnected; - return isConnected == oldConnected ? GroupConnectionChangedEventArgs.ChangeType.Unknown - : isConnected ? GroupConnectionChangedEventArgs.ChangeType.Reconnected - : GroupConnectionChangedEventArgs.ChangeType.Disconnected; + return isConnected == oldConnected + ? GroupConnectionChangedEventArgs.ChangeType.Unknown + : isConnected + ? GroupConnectionChangedEventArgs.ChangeType.Reconnected + : GroupConnectionChangedEventArgs.ChangeType.Disconnected; } internal void UpdateLatency() @@ -718,7 +750,7 @@ static bool TryGetHealthCheck(object? target, out TimeSpan interval) // note the interval is a group-level concern (how often we re-evaluate the active // member), not a property of any individual health-check interval = typed._options.HealthCheckInterval; - return interval > TimeSpan.Zero & interval != TimeSpan.MaxValue; + return (interval > TimeSpan.Zero) & (interval != TimeSpan.MaxValue); } interval = TimeSpan.Zero; @@ -772,8 +804,82 @@ internal async Task RunHealthCheckAsync() } HealthCheck.PutReusablePending(ref _reusableHealthCheckBuffer, ref pending); + await RunFailbackChecksAsync(members).ForAwait(); + } + + // Failback checks decide eligibility, never health: they run only for members that are up and not + // active (the active member's answer is never used), after liveness has been updated, and a verdict + // is only kept while it can still be acted on - so a member that stops being up, or becomes active, + // must be checked afresh before it is next eligible. Anything short of Healthy, including a + // timeout, leaves the member ineligible; none of it marks a member unhealthy. + private async Task RunFailbackChecksAsync(ConnectionGroupMember[] members) + { + // a provisional active member (chosen by the liveness fallback) is still checked, so that it can + // stop being provisional once it has caught up; a normal active member is not + var active = _activeStub.Active; + var provisional = Volatile.Read(ref _provisional); + List? checks = null; + int totalTimeoutMillis = 0; + foreach (var member in members) + { + var check = member.ResolveFailbackHealthCheck(_options); + if (check is null) continue; + + if (!member.ConsiderActive || (ReferenceEquals(member.Multiplexer, active) && !ReferenceEquals(active, provisional))) + { + member.FailbackVerified = false; + continue; + } + + totalTimeoutMillis = Math.Max(totalTimeoutMillis, check.TotalTimeoutMillis()); + (checks ??= []).Add(new(member, check.CheckHealthAsync(member.Multiplexer))); + } + + if (checks is null) return; + + var all = new Task[checks.Count]; + for (int i = 0; i < all.Length; i++) all[i] = checks[i].Pending; + await Task.WhenAll(all).TimeoutAfter(totalTimeoutMillis).ForAwait(); + + foreach (var item in checks) + { + var member = item.Member; + var pending = item.Pending; + if (pending.IsCompletedSuccessfully) + { + member.FailbackVerified = await pending.ForAwait() is HealthCheckResult.Healthy; + } + else + { + _ = pending.ObserveErrors(); + member.FailbackVerified = false; + } + } + } + + // a named pair rather than a tuple: the library must not reference System.ValueTuple (see SanityCheckTests) + private readonly struct PendingFailbackCheck(ConnectionGroupMember member, Task pending) + { + public ConnectionGroupMember Member { get; } = member; + + public Task Pending { get; } = pending; } + // Eligible to become (or stay) active: an explicit failover target always is, as is the active member + // - unless it is provisional, having been chosen by the liveness fallback without ever passing its + // failback check; otherwise a member with a failback check must have passed it on the latest pass. + private bool IsFailbackEligible(ConnectionGroupMember member, ConnectionMultiplexer? active) + => member.ExplicitOverride + || (ReferenceEquals(member.Multiplexer, active) && !ReferenceEquals(active, Volatile.Read(ref _provisional))) + || member.ResolveFailbackHealthCheck(_options) is null + || member.FailbackVerified; + + // The active member when it was chosen by the liveness fallback (no live member was eligible), else null. + // Other clients never select a member that fails its lag check at all; this is the nearest equivalent + // that still serves when every member is behind: such a member is left for an eligible one as soon as + // one is available, and becomes an ordinary active member once it passes its own failback check. + private ConnectionMultiplexer? _provisional; + private long GetFailbackFailureCutoff(ConnectionGroupMember member) { // the minimum last-observed unhealthy time (as UTC ticks) that we'll allow for reconnect; @@ -790,7 +896,7 @@ internal void SelectPreferredGroup() { if (_disposed) return; var existingActive = _activeStub.Active; - ConnectionGroupMember? preferredMember = null, previousMember = null; + ConnectionGroupMember? preferredMember = null, fallbackMember = null, previousMember = null; var members = _members; foreach (var member in members) { @@ -804,10 +910,28 @@ internal void SelectPreferredGroup() member.UpdateLatency(); // this can change passively // (note that when in doubt, we prefer the active muxer, to prevent flapping) - preferredMember = ConnectionGroupMember.Select(preferredMember, member, existingActive); + fallbackMember = ConnectionGroupMember.Select(fallbackMember, member, existingActive); + if (IsFailbackEligible(member, existingActive)) + { + preferredMember = ConnectionGroupMember.Select(preferredMember, member, existingActive); + } } } + // The active member is eligible unless provisional, so this only applies when it is gone, there is + // none, or it is itself provisional: if no live member has passed its failback check - every one + // lagging, say, which is the normal state of a partition between regions - serve from the best live + // member rather than from none, marking it provisional. + if (preferredMember is null) + { + preferredMember = fallbackMember; + Volatile.Write(ref _provisional, fallbackMember?.Multiplexer); + } + else + { + Volatile.Write(ref _provisional, null); + } + SetActive(preferredMember?.Multiplexer); if (preferredMember is not null && !ReferenceEquals(preferredMember, previousMember)) @@ -976,13 +1100,17 @@ private static bool RemoveHandler(ref T? field, T? value) where T : Delegate } } - private int _circuitBreakerDebounce = 0; + private int _rerouteDebounce = 0; private void OnMemberConnectionFailed(object? sender, ConnectionFailedEventArgs e) { - // deliberately scoped to this one failure type; re-probe and re-select promptly rather - // than waiting for the next poll tick, so traffic routes away from the dropped member - if (e.FailureType is ConnectionFailureType.CircuitBreaker - && Interlocked.CompareExchange(ref _circuitBreakerDebounce, 1, 0) is 0) + // deliberately scoped to these failure types; re-probe and re-select promptly rather than + // waiting for the next poll tick, so traffic routes away from the dropped member. A breaker + // trip is the member's own traffic telling us it is unstable; LOADING is the server telling + // us outright that it cannot serve yet (and its handshake will keep failing the same way + // until it can, so nothing routed there in the meantime gets a reply - the breaker only ever + // sees replies, so it cannot be what moves us). Ordinary socket drops are left to reconnect + if (e.FailureType is ConnectionFailureType.CircuitBreaker or ConnectionFailureType.Loading + && Interlocked.CompareExchange(ref _rerouteDebounce, 1, 0) is 0) { GetMember(sender as ConnectionMultiplexer)?.SetUnhealthy(); _ = Task.Run(async () => @@ -997,7 +1125,7 @@ private void OnMemberConnectionFailed(object? sender, ConnectionFailedEventArgs } finally { - Volatile.Write(ref _circuitBreakerDebounce, 0); + Volatile.Write(ref _rerouteDebounce, 0); } }); } @@ -1228,7 +1356,7 @@ public ISubscriber GetSubscriber(object? asyncState = null) public IDatabase GetDatabase(int db = -1, object? asyncState = null) { - if (asyncState is null & db >= -1 & db <= ConnectionMultiplexer.MaxCachedDatabaseInstance) + if ((asyncState is null) & (db >= -1) & (db <= ConnectionMultiplexer.MaxCachedDatabaseInstance)) { return _databases[db + 1] ??= new MultiGroupDatabase(this, db, null); } @@ -1436,6 +1564,12 @@ public async Task AddAsync(ConnectionGroupMember member, TextWriter? log = null) { var health = await member.ResolveHealthCheck(_options).CheckHealthAsync(muxer).ConfigureAwait(false); member.UpdateState(health, GetFailbackFailureCutoff(member)); + + // a newly added member is never the active one, so its failback check applies at once + if (member.ResolveFailbackHealthCheck(_options) is { } failback && member.ConsiderActive) + { + member.FailbackVerified = await failback.CheckHealthAsync(muxer).ConfigureAwait(false) is HealthCheckResult.Healthy; + } } // apply any shared hooks diff --git a/src/StackExchange.Redis/Availability/MultiGroupOptions.cs b/src/StackExchange.Redis/Availability/MultiGroupOptions.cs index c80ee8cf0..2cf4d1ccd 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupOptions.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupOptions.cs @@ -18,7 +18,7 @@ public sealed class MultiGroupOptions internal static readonly TimeSpan DefaultFailbackDelay = TimeSpan.Zero; private static readonly MultiGroupOptions DefaultInstance = new( - HealthCheck.Default, CircuitBreaker.Default, RetryPolicy.Default, DefaultHealthCheckInterval, DefaultFailbackDelay); + HealthCheck.Default, CircuitBreaker.Default, RetryPolicy.Default, DefaultHealthCheckInterval, DefaultFailbackDelay, null); /// /// Default shared options. @@ -30,13 +30,15 @@ private MultiGroupOptions( CircuitBreaker circuitBreaker, RetryPolicy retryPolicy, TimeSpan healthCheckInterval, - TimeSpan failbackDelay) + TimeSpan failbackDelay, + HealthCheck? failbackHealthCheck) { HealthCheck = healthCheck; CircuitBreaker = circuitBreaker; RetryPolicy = retryPolicy; HealthCheckInterval = healthCheckInterval; FailbackDelay = failbackDelay; + FailbackHealthCheck = failbackHealthCheck; } /// @@ -74,6 +76,12 @@ private MultiGroupOptions( /// public TimeSpan FailbackDelay { get; } + /// + /// The failback health check to use for members of the group when no per-member one is specified; + /// (the default) means none. See . + /// + public HealthCheck? FailbackHealthCheck { get; } + /// /// Allows configuration of . /// @@ -96,6 +104,7 @@ public Builder(MultiGroupOptions options) RetryPolicy = options.RetryPolicy; HealthCheckInterval = options.HealthCheckInterval; FailbackDelay = options.FailbackDelay; + FailbackHealthCheck = options.FailbackHealthCheck; } /// @@ -126,6 +135,12 @@ public Builder(MultiGroupOptions options) /// public TimeSpan FailbackDelay { get; set; } = DefaultFailbackDelay; + /// + /// The failback health check to use for members of the group when no per-member one is specified; + /// (the default) means none. See . + /// + public HealthCheck? FailbackHealthCheck { get; set; } + /// /// Create a new options instance. /// @@ -142,12 +157,13 @@ public MultiGroupOptions Create() && ReferenceEquals(CircuitBreaker, CircuitBreaker.Default) && ReferenceEquals(RetryPolicy, RetryPolicy.Default) && HealthCheckInterval == DefaultHealthCheckInterval - && FailbackDelay == DefaultFailbackDelay) + && FailbackDelay == DefaultFailbackDelay + && FailbackHealthCheck is null) { return DefaultInstance; } - return new MultiGroupOptions(HealthCheck, CircuitBreaker, RetryPolicy, HealthCheckInterval, FailbackDelay); + return new MultiGroupOptions(HealthCheck, CircuitBreaker, RetryPolicy, HealthCheckInterval, FailbackDelay, FailbackHealthCheck); } /// diff --git a/src/StackExchange.Redis/Availability/RetryController.cs b/src/StackExchange.Redis/Availability/RetryController.cs index 824b675d3..97fe6eae2 100644 --- a/src/StackExchange.Redis/Availability/RetryController.cs +++ b/src/StackExchange.Redis/Availability/RetryController.cs @@ -62,7 +62,7 @@ public Task WatchConflictDelayAsync() /// Whether it is ever worth capturing the next-failover token: only when there is more than one /// attempt and the failover threshold sits below the attempt cap. /// - public bool TracksFailover => _maxAttempts > 1 & _maxBeforeFailover < _maxAttempts; + public bool TracksFailover => (_maxAttempts > 1) & (_maxBeforeFailover < _maxAttempts); /// /// Whether could ever answer for a command carrying @@ -172,7 +172,7 @@ public bool CanRetry( // by count, we should really switch over to the failover now; is failover available *and* are we allowed? delay = failover; failover = CancellationToken.None; // only failover once - return delay.CanBeCanceled & (policy & RetryResult.FailoverServer) != 0; + return delay.CanBeCanceled & ((policy & RetryResult.FailoverServer) != 0); } // can we pause and retry on the same server? diff --git a/src/StackExchange.Redis/BufferReader.cs b/src/StackExchange.Redis/BufferReader.cs index 22b36ccb6..8b4d2b4b5 100644 --- a/src/StackExchange.Redis/BufferReader.cs +++ b/src/StackExchange.Redis/BufferReader.cs @@ -78,7 +78,8 @@ public ConsumeResult TryConsumeCRLF() default: var offset = OffsetThisSpan; var result = _current[offset++] == (byte)'\r' && _current[offset] == (byte)'\n' - ? ConsumeResult.Success : ConsumeResult.Failure; + ? ConsumeResult.Success + : ConsumeResult.Failure; Consume(2); return result; } diff --git a/src/StackExchange.Redis/Caching/CacheOptions.cs b/src/StackExchange.Redis/Caching/CacheOptions.cs index ce3b1df5f..f585a6800 100644 --- a/src/StackExchange.Redis/Caching/CacheOptions.cs +++ b/src/StackExchange.Redis/Caching/CacheOptions.cs @@ -103,7 +103,11 @@ public sealed class CacheOptions /// public int? MaxPayloadBytes { - get => _maxPayloadBytes; + get + { + return _maxPayloadBytes; + } + init => _maxPayloadBytes = value is null or > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value), "The maximum payload size must be positive, or null for no limit."); @@ -137,7 +141,11 @@ public int? MaxPayloadBytes /// public long? MaxBytes { - get => _maxBytes; + get + { + return _maxBytes; + } + init => _maxBytes = value is null or > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value), "The memory budget must be positive, or null for no limit."); @@ -155,7 +163,11 @@ public long? MaxBytes /// public int? MaxEntries { - get => _maxEntries; + get + { + return _maxEntries; + } + init => _maxEntries = value is null or > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value), "The entry limit must be positive, or null for no limit."); @@ -190,7 +202,11 @@ public int? MaxEntries /// public int EvictionSampleSize { - get => _evictionSampleSize; + get + { + return _evictionSampleSize; + } + init => _evictionSampleSize = value > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value), "The eviction sample size must be positive."); @@ -268,7 +284,11 @@ public int EvictionSampleSize /// public IReadOnlyList Prefixes { - get => _prefixes; + get + { + return _prefixes; + } + init { _prefixes = value ?? throw new ArgumentNullException(nameof(value)); diff --git a/src/StackExchange.Redis/Caching/RespClientCache.Slabs.cs b/src/StackExchange.Redis/Caching/RespClientCache.Slabs.cs index fac3d396c..3b6caf9a2 100644 --- a/src/StackExchange.Redis/Caching/RespClientCache.Slabs.cs +++ b/src/StackExchange.Redis/Caching/RespClientCache.Slabs.cs @@ -6,9 +6,9 @@ namespace StackExchange.Redis.Caching { - /// + /// /// Where cached replies live: carved, in fill order, out of large arrays ("slabs") that the GC owns. - /// + /// /// /// /// Why not one array per reply. That made a hit free of shared writes - the GC is the reference count, so diff --git a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs index 1ab612bea..31177de71 100644 --- a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs @@ -59,7 +59,7 @@ private bool IsHostInDomains(string hostName, string[] domains) } /// - public override Task AfterConnectAsync(ConnectionMultiplexer muxer, Action log) + public override Task AfterConnectAsync(ConnectionMultiplexer multiplexer, Action log) => Task.CompletedTask; /// diff --git a/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs b/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs index ee577eb59..0f9810f1c 100644 --- a/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs @@ -61,8 +61,8 @@ private bool IsHostInDomains(string hostName, string[] domains) } /// - public override Task AfterConnectAsync(ConnectionMultiplexer muxer, Action log) - => AzureMaintenanceEvent.AddListenerAsync(muxer, log); + public override Task AfterConnectAsync(ConnectionMultiplexer multiplexer, Action log) + => AzureMaintenanceEvent.AddListenerAsync(multiplexer, log); /// public override bool GetDefaultSsl(EndPointCollection endPoints) diff --git a/src/StackExchange.Redis/Configuration/LoggingTunnel.cs b/src/StackExchange.Redis/Configuration/LoggingTunnel.cs index ad0cffffe..8375a346e 100644 --- a/src/StackExchange.Redis/Configuration/LoggingTunnel.cs +++ b/src/StackExchange.Redis/Configuration/LoggingTunnel.cs @@ -390,14 +390,16 @@ private void CreateLogFiles(EndPoint endpoint, ConnectionType connectionType, ou /// public override ValueTask BeforeSocketConnectAsync(EndPoint endPoint, ConnectionType connectionType, Socket? socket, CancellationToken cancellationToken) { - return _tail is null ? base.BeforeSocketConnectAsync(endPoint, connectionType, socket, cancellationToken) + return _tail is null + ? base.BeforeSocketConnectAsync(endPoint, connectionType, socket, cancellationToken) : _tail.BeforeSocketConnectAsync(endPoint, connectionType, socket, cancellationToken); } /// public override ValueTask GetSocketConnectEndpointAsync(EndPoint endpoint, CancellationToken cancellationToken) { - return _tail is null ? base.GetSocketConnectEndpointAsync(endpoint, cancellationToken) + return _tail is null + ? base.GetSocketConnectEndpointAsync(endpoint, cancellationToken) : _tail.GetSocketConnectEndpointAsync(endpoint, cancellationToken); } diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index 907994cbe..490046690 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -407,7 +407,10 @@ private void SetWithValue(OptionFlags hasValue, ref T storage, T value) where /// public DefaultOptionsProvider Defaults { - get => defaultOptions ??= DefaultOptionsProvider.GetProvider(EndPoints); + get + { + return defaultOptions ??= DefaultOptionsProvider.GetProvider(EndPoints); + } set { defaultOptions = value; @@ -563,25 +566,32 @@ public void SetUserPfxCertificate(string userCertificatePath, string? password = #if NET internal static LocalCertificateSelectionCallback CreatePemUserCertificateCallback(string userCertificatePath, string? userKeyPath) + { + var pfx = LoadPemUserCertificate(userCertificatePath, userKeyPath); + return (sender, targetHost, localCertificates, remoteCertificate, acceptableIssuers) => pfx; + } + + internal static X509Certificate2 LoadPemUserCertificate(string userCertificatePath, string? userKeyPath) { // PEM handshakes not universally supported and causes a runtime error about ephemeral certificates; to avoid, export as PFX using var pem = X509Certificate2.CreateFromPemFile(userCertificatePath, userKeyPath); #pragma warning disable SYSLIB0057 // X509 loading - var pfx = new X509Certificate2(pem.Export(X509ContentType.Pfx)); + return new X509Certificate2(pem.Export(X509ContentType.Pfx)); #pragma warning restore SYSLIB0057 // X509 loading - - return (sender, targetHost, localCertificates, remoteCertificate, acceptableIssuers) => pfx; } #endif internal static LocalCertificateSelectionCallback CreatePfxUserCertificateCallback(string userCertificatePath, string? password, X509KeyStorageFlags storageFlags = X509KeyStorageFlags.DefaultKeySet) { -#pragma warning disable SYSLIB0057 // X509 loading - var pfx = new X509Certificate2(userCertificatePath, password ?? "", storageFlags); -#pragma warning restore SYSLIB0057 // X509 loading + var pfx = LoadPfxUserCertificate(userCertificatePath, password, storageFlags); return (sender, targetHost, localCertificates, remoteCertificate, acceptableIssuers) => pfx; } + internal static X509Certificate2 LoadPfxUserCertificate(string userCertificatePath, string? password, X509KeyStorageFlags storageFlags = X509KeyStorageFlags.DefaultKeySet) +#pragma warning disable SYSLIB0057 // X509 loading + => new X509Certificate2(userCertificatePath, password ?? "", storageFlags); +#pragma warning restore SYSLIB0057 // X509 loading + /// /// Create a certificate validation check that checks against the supplied issuer even when not known by the machine. /// @@ -593,7 +603,7 @@ internal static RemoteCertificateValidationCallback TrustIssuerCallback(string i => TrustIssuerCallback(new X509Certificate2(issuerCertificatePath)); #pragma warning restore SYSLIB0057 // X509 loading - private static RemoteCertificateValidationCallback TrustIssuerCallback(X509Certificate2 issuer) + internal static RemoteCertificateValidationCallback TrustIssuerCallback(X509Certificate2 issuer) { if (issuer == null) throw new ArgumentNullException(nameof(issuer)); @@ -705,7 +715,10 @@ public CommandMap CommandMap Proxy.Envoyproxy => CommandMap.Envoyproxy, _ => CommandMap.Default, }; - set => commandMap = value ?? throw new ArgumentNullException(nameof(value)); + set + { + commandMap = value ?? throw new ArgumentNullException(nameof(value)); + } } /// @@ -741,7 +754,10 @@ public int ConnectTimeout var defaultTimeout = Defaults.ConnectTimeout; return defaultTimeout.HasValue ? (int)defaultTimeout.GetValueOrDefault().TotalMilliseconds : Math.Max(5000, SyncTimeout); } - set => SetWithValue(OptionFlags.ConnectTimeoutHasValue, ref connectTimeout, value); + set + { + SetWithValue(OptionFlags.ConnectTimeoutHasValue, ref connectTimeout, value); + } } /// @@ -1079,9 +1095,10 @@ internal EventHandler? ConnectionAttemptCom /// default), or 0 to never do so. /// /// - /// Sends an INFO replication on each *established* interactive connection. That is how a - /// primary/replica change is noticed on a deployment that does not announce one, and it doubles as the - /// keep-alive for those sockets, which is why the interval is short. + /// Re-reads the role of each *established* interactive connection's server (ROLE; the shipped core + /// sent INFO replication). That is how a primary/replica change is noticed on a deployment that + /// does not announce one - a primary demoted in place under sentinel, say - and it doubles as a keep-alive + /// for those sockets, which is why the interval is short. /// /// Despite the name, this is not a topology re-read. It asks a server we are already talking to what it /// says about itself, so it reveals nothing about servers we cannot reach, about endpoints that have @@ -1113,7 +1130,7 @@ public int ConfigCheckSeconds /// /// /// Distinct from , which despite its name does not re-read topology - - /// it sends an INFO replication on an established connection to check a server's role. + /// it asks an established connection's server for its role. /// /// public int TopologyRefreshSeconds @@ -1304,7 +1321,8 @@ public string ToString(bool includePassword) commandMap?.AppendDeltas(sb); return sb.ToString(); - static string FormatProtocol(RedisProtocol protocol) => protocol switch { + static string FormatProtocol(RedisProtocol protocol) => protocol switch + { RedisProtocol.Resp2 => "resp2", RedisProtocol.Resp3 => "resp3", _ => protocol.ToString(), @@ -1701,7 +1719,10 @@ public MaintenanceEndpointType MaintenanceMovingEndpointType get => HasValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue) ? _maintenanceMovingEndpointType : Defaults.MaintenanceMovingEndpointType; - set => SetWithValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue, ref _maintenanceMovingEndpointType, value); + set + { + SetWithValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue, ref _maintenanceMovingEndpointType, value); + } } /// @@ -1735,7 +1756,10 @@ public TimeSpan MaintenanceRelaxedWindowMax get => HasValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue) ? _maintenanceRelaxedWindowMax : Defaults.MaintenanceRelaxedWindowMax ?? Multiply(MaintenanceRelaxedTimeout, 3); - set => SetWithValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue, ref _maintenanceRelaxedWindowMax, value); + set + { + SetWithValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue, ref _maintenanceRelaxedWindowMax, value); + } } /// @@ -1754,7 +1778,10 @@ public TimeSpan MaintenancePostEventRelaxedDuration get => HasValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue) ? _maintenancePostEventRelaxedDuration : Defaults.MaintenancePostEventRelaxedDuration ?? Multiply(MaintenanceRelaxedTimeout, 2); - set => SetWithValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue, ref _maintenancePostEventRelaxedDuration, value); + set + { + SetWithValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue, ref _maintenancePostEventRelaxedDuration, value); + } } // TimeSpan * int is netstandard2.1+, so this is ticks arithmetic to keep the down-level TFMs building diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs b/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs index 8effd0288..6229aefc7 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs @@ -176,10 +176,10 @@ public static bool GetFeatureFlag(string flag) /// there is no separate subscription connection, so asking about answers /// about the shared one. /// - bool? IInternalConnectionMultiplexer.IsSyncReader(EndPoint endpoint, ConnectionType connectionType) - => ConnectionsIfCreated?.IsDedicatedThread(endpoint, connectionType, writer: false); + bool? IInternalConnectionMultiplexer.IsSyncReader(EndPoint endPoint, ConnectionType type) + => ConnectionsIfCreated?.IsDedicatedThread(endPoint, type, writer: false); /// As IsSyncReader, for the writer. - bool? IInternalConnectionMultiplexer.IsSyncWriter(EndPoint endpoint, ConnectionType connectionType) - => ConnectionsIfCreated?.IsDedicatedThread(endpoint, connectionType, writer: true); + bool? IInternalConnectionMultiplexer.IsSyncWriter(EndPoint endPoint, ConnectionType type) + => ConnectionsIfCreated?.IsDedicatedThread(endPoint, type, writer: true); } diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs index 164aad430..50aad0a37 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs @@ -11,18 +11,28 @@ namespace StackExchange.Redis; public partial class ConnectionMultiplexer { - internal EndPoint? currentSentinelPrimaryEndPoint; + // volatile: written by SwitchPrimary on worker/handler threads and read by the heartbeat + // (IsSentinelPrimaryDemoted / RecoverDemotedPrimary) - the demotion check must see the latest switch + internal volatile EndPoint? currentSentinelPrimaryEndPoint; internal Timer? sentinelPrimaryReconnectTimer; internal readonly Dictionary sentinelConnectionChildren = new(); internal ConnectionMultiplexer? sentinelConnection; + private int _sentinelInitialized; + private int _demotedPrimaryRecoveryActive; /// /// Initializes the connection as a Sentinel connection and adds the necessary event handlers to track changes to the managed primaries. /// + /// + /// Runs once. It is reached from the connect paths, and also from the reconfigure loop when a connection + /// that soft-failed its initial connect ( is false) later + /// reaches a sentinel: without that, a connection that started while the sentinels were down would never subscribe + /// to +switch-master, and would track failovers only through the fallback polling. + /// /// The to log to, if any. internal void InitializeSentinel(ILogger? log) { - if (!_isSentinel) + if (!_isSentinel || Interlocked.Exchange(ref _sentinelInitialized, 1) != 0) { return; } @@ -55,9 +65,7 @@ internal void InitializeSentinel(ILogger? log) // Is the connection still valid? if (child.IsDisposed) { - child.ConnectionFailed -= OnManagedConnectionFailed; - child.ConnectionRestored -= OnManagedConnectionRestored; - sentinelConnectionChildren.Remove(messageParts[0]); + ForgetDisposedChild(messageParts[0], child); } else { @@ -253,8 +261,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co } if (connection is not null) try { connection.Dispose(); } catch { } - endpoints = config.EndPoints.Clone(); - connection = ConnectImpl(config, log, endpoints: endpoints); + connection = SeedConnection(failure); break; } @@ -330,11 +337,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co throw failure; } - if (connection is null) - { - endpoints = config.EndPoints.Clone(); - connection = ConnectImpl(config, log, endpoints: endpoints); - } + connection ??= SeedConnection(failure); connection.LastException = failure; connection.Logger?.LogErrorSyncConnectTimeout(failure, failure.Message); @@ -385,6 +388,20 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co } throw; } + + // AbortOnConnectFail=false with no primary discovered: hand back a multiplexer seeded with whatever endpoints + // the caller gave us (the sentinel addresses, via Connect), for SwitchPrimary to rebuild once discovery works. + // A config with no endpoints of its own has nothing to seed with, and that is the discovery failure the caller + // should see - not the "No nodes to consider" that ConnectImpl would throw at an empty endpoint list. + ConnectionMultiplexer SeedConnection(RedisConnectionException discoveryFailure) + { + if (config.EndPoints.Count == 0) + { + throw discoveryFailure; + } + endpoints = config.EndPoints.Clone(); + return ConnectImpl(config, log, endpoints: endpoints); + } } [System.Diagnostics.CodeAnalysis.SuppressMessage("Roslynator", "RCS1075:Avoid empty catch clause that catches System.Exception.", Justification = "We don't care.")] @@ -423,6 +440,11 @@ internal void OnManagedConnectionRestored(object? sender, ConnectionFailedEventA // we are not aware of, so retry SwitchPrimary(e.EndPoint, connection); } + + // A soft-failed connect (AbortOnConnectFail=false) records the discovery failure as LastException, and + // the exception factory folds that into every "no connection available" message. We are connected to a + // sentinel-approved primary again, so stop reporting it. + connection.LastException = null; } catch (Exception) { @@ -498,6 +520,167 @@ private void ScheduleSentinelPrimaryReconnect(ConnectionMultiplexer connection, } } + /// + /// Unhooks and forgets a disposed managed connection. Must be called under the + /// lock. + /// + private void ForgetDisposedChild(string serviceName, ConnectionMultiplexer child) + { + child.ConnectionFailed -= OnManagedConnectionFailed; + child.ConnectionRestored -= OnManagedConnectionRestored; + sentinelConnectionChildren.Remove(serviceName); + } + + /// + /// Re-resolves the primary through sentinel for every managed connection whose primary has been demoted + /// while its connection stayed up. + /// + /// + /// + /// Every other recovery path needs a signal that this case does not produce: +switch-master can be + /// missed (pub/sub delivery is not guaranteed, and the sentinel-clients guidance says not to rely on it), + /// and the reconnect timer and the restore handler only run once a connection fails. A primary that is + /// demoted in place, with nothing announced and nothing disconnected, is still noticed by the managed + /// connection - its periodic INFO replication sees the role change - but then only re-elected + /// among the nodes it already knows. When sentinel promoted a node at an address it has never seen, + /// there is no primary among them, and nothing asked sentinel again. + /// + /// + /// Runs on this sentinel connection's heartbeat. The check reads only local state, so sentinel is asked + /// only when the managed connection itself reports the demotion, and at most one recovery per managed + /// connection is in flight. A recovery that does not take - sentinel not yet updated, or a reconfigure + /// already running - is simply retried on a later heartbeat, and the check clears itself once the + /// sentinel-reported primary is connected as a primary. + /// + /// + private void CheckManagedPrimariesDemoted() + { + List? demoted = null; + List? disposed = null; + lock (sentinelConnectionChildren) + { + foreach (var pair in sentinelConnectionChildren) + { + if (pair.Value.IsDisposed) + { + (disposed ??= new List()).Add(pair.Key); + } + else if (IsSentinelPrimaryDemoted(pair.Value)) + { + (demoted ??= new List()).Add(pair.Value); + } + } + + if (disposed is not null) + { + foreach (var serviceName in disposed) + { + ForgetDisposedChild(serviceName, sentinelConnectionChildren[serviceName]); + } + } + } + + if (demoted is null) return; + foreach (var child in demoted) + { + if (Interlocked.CompareExchange(ref child._demotedPrimaryRecoveryActive, 1, 0) != 0) continue; + + // asking sentinel and SwitchPrimary both block; keep that off the heartbeat + ThreadPool.QueueUserWorkItem(_ => + { + try + { + RecoverDemotedPrimary(child); + } + catch (Exception ex) + { + // sentinel unreachable, or either connection disposed meanwhile: a later heartbeat looks again + Trace("Demoted primary recovery failed: " + ex.Message, "Sentinel"); + } + finally + { + Volatile.Write(ref child._demotedPrimaryRecoveryActive, 0); + } + }); + } + } + + /// + /// One recovery attempt for a managed connection whose sentinel-reported primary was seen demoted. + /// + /// + /// Sentinel is asked first, because what to do depends on its answer, and a repeat of an unchanged answer + /// must stay cheap - the attempt runs on every heartbeat until the view clears: + /// + /// no answer: nothing to act on; the next heartbeat asks again. + /// a node we hold as a connected replica: either sentinel has not failed over yet, or our + /// view of that node's role is stale. Re-reading its role settles both, where a switch would reconfigure every + /// node to elect a primary that may not exist - once per second for as long as sentinel lags. Where that read + /// cannot be sent (INFO unavailable), fall through to the switch rather than ask sentinel forever. + /// anything else: switch, exactly as a +switch-master announcement would have. + /// Sentinel's answer is handed to so the switch acts on the answer these guards + /// evaluated, rather than asking again and possibly acting on a different one. + /// + /// A reconfigure already running is left alone: claims the reconfigure gate + /// before touching anything, so a lost race leaves no trace and a later heartbeat retries. + /// + private void RecoverDemotedPrimary(ConnectionMultiplexer connection) + { + if (IsDisposed || connection.IsDisposed || connection.RawConfig.ServiceName is not string serviceName) return; + + // cheap early out before the blocking sentinel query; SwitchPrimary's claim is the authoritative check + if (Volatile.Read(ref connection.activeConfigCause) is not null) return; + + if (GetConfiguredPrimaryForService(serviceName) is not EndPoint reported) return; + if (connection.TryResolveServerEndPoint(reported) is { IsConnected: true, IsReplica: true } replica + && connection.ConnectionsIfCreated?.TryRefreshRole(replica.EndPoint) == true) + { + return; + } + + // the view may have been put right while sentinel was answering + if (!IsSentinelPrimaryDemoted(connection)) return; + + if (SwitchPrimary(connection.currentSentinelPrimaryEndPoint, connection, reportedPrimary: reported) + && connection.TryResolveServerEndPoint(reported) is { IsConnected: true, IsReplica: false }) + { + // As OnManagedConnectionRestored does: we are connected to a sentinel-approved primary again, so a + // soft-failed connect's discovery failure should stop being folded into every timeout message. + connection.LastException = null; + } + } + + /// + /// Whether the primary sentinel last reported for is connected and reporting + /// itself as a replica, or was never added (a switch to it whose reconfigure did not run). A disconnected + /// primary is not this case: that is the reconnect timer's job. + /// + private static bool IsSentinelPrimaryDemoted(ConnectionMultiplexer connection) + { + if (connection.IsDisposed + || Volatile.Read(ref connection.sentinelPrimaryReconnectTimer) is not null + || connection.currentSentinelPrimaryEndPoint is not EndPoint primary) + { + return false; + } + + // a lookup, not GetServerEndPoint: that would create a server for an endpoint the connection does not have + if (connection.TryResolveServerEndPoint(primary) is { } server) + { + return server.IsConnected && server.IsReplica; + } + + // never added: a switch to it that was refused (another reconfigure held the gate) or threw. A missing server + // alone is not enough evidence - the endpoint sentinel gave may simply be spelled differently from the + // server's own (a DNS name against an address) - so only while nothing else is serving as the primary, + // which keeps a healthy connection from switching on every heartbeat. + foreach (var other in connection.GetServerSnapshot()) + { + if (other.ServerType != ServerType.Sentinel && other.IsConnected && !other.IsReplica) return false; + } + return true; + } + internal EndPoint? GetConfiguredPrimaryForService(string serviceName) => _serverSnapshot // same as GetServerSnapshot, but without forcing span .Where(static s => s.ServerType == ServerType.Sentinel) @@ -526,78 +709,108 @@ private void ScheduleSentinelPrimaryReconnect(ConnectionMultiplexer connection, /// The endpoint responsible for the switch. /// The connection that should be switched over to a new primary endpoint. /// The writer to log to, if any. - internal void SwitchPrimary(EndPoint? switchBlame, ConnectionMultiplexer connection, TextWriter? writer = null) + /// The primary sentinel already reported, when the caller has just asked it; + /// otherwise sentinel is queried here. + /// true when a switch ran (the endpoint set and/or roles were reconfigured); false when + /// the view was already correct, or another reconfigure holds the gate - the callers that retry (the reconnect + /// timer, the demotion heartbeat) get a later attempt either way. + internal bool SwitchPrimary(EndPoint? switchBlame, ConnectionMultiplexer connection, TextWriter? writer = null, EndPoint? reportedPrimary = null) { var logger = Logger.With(writer); if (connection.RawConfig.ServiceName is not string serviceName) { logger?.LogInformationServiceNameNotDefined(); - return; + return false; } - // Get new primary - try twice - EndPoint newPrimaryEndPoint = GetConfiguredPrimaryForService(serviceName) + // Get new primary - try twice (unless the caller already holds sentinel's answer) + EndPoint newPrimaryEndPoint = reportedPrimary + ?? GetConfiguredPrimaryForService(serviceName) ?? GetConfiguredPrimaryForService(serviceName) ?? throw new RedisConnectionException(ConnectionFailureType.UnableToConnect, CommandFlags.None, $"Sentinel: Failed connecting to switch primary for service: {serviceName}"); connection.currentSentinelPrimaryEndPoint = newPrimaryEndPoint; - // Trigger a blocking reconfigure for the switch, then refresh the sentinel address list. - // reconfigureAll:true forces every already-connected node to re-read its role before the - // primary is re-elected (see IsStalePrimaryView); reconfigureAll:false matches the legacy - // behavior for the "unknown endpoint" rebuild path. - void TriggerReconfigure(bool reconfigureAll) + // Claim the reconfigure gate BEFORE reshaping the connection. ReconfigureAsync refuses to run while + // another reconfigure holds this gate, so rewriting the endpoint list and retiring servers first would + // leave the connection reshaped but unconfigured whenever we lose that race - and two concurrent + // switches could interleave their endpoint rewrites. Claim first and a lost race touches nothing. + string cause = $"Primary switch {serviceName}"; + if (!connection.TryClaimReconfigure(cause)) { - Trace($"Switching primary to {newPrimaryEndPoint}"); - connection.ReconfigureAsync( - first: false, - reconfigureAll: reconfigureAll, - log: logger, - blame: switchBlame, - cause: $"Primary switch {serviceName}", - publishReconfigure: false, - publishReconfigureFlags: CommandFlags.PreferMaster).Wait(); - - UpdateSentinelAddressList(serviceName); + Trace($"Primary switch for {serviceName} skipped; reconfiguration already in progress"); + return false; } - if (!connection.servers.Contains(newPrimaryEndPoint)) + bool switched = false; + try { - // The sentinel-reported primary is an endpoint we have never seen (e.g. a - // recreated node that came back on a brand-new IP). Rebuild the endpoint set - // from scratch around it and reconfigure. - EndPoint[]? replicaEndPoints = GetReplicasForService(serviceName) - ?? GetReplicasForService(serviceName); + // Trigger a blocking reconfigure for the switch, then refresh the sentinel address list. + // reconfigureAll:true forces every already-connected node to re-read its role before the + // primary is re-elected (see IsStalePrimaryView); reconfigureAll:false matches the legacy + // behavior for the "unknown endpoint" rebuild path. + void TriggerReconfigure(bool reconfigureAll) + { + Trace($"Switching primary to {newPrimaryEndPoint}"); + switched = true; // from here the claim is ReconfigureAsync's to release, however it exits + connection.ReconfigureAsync( + first: false, + reconfigureAll: reconfigureAll, + log: logger, + blame: switchBlame, + cause: cause, + publishReconfigure: false, + publishReconfigureFlags: CommandFlags.PreferMaster, + claimed: true).Wait(); + + UpdateSentinelAddressList(serviceName); + } - connection.EndPoints.Clear(); - connection.EndPoints.TryAdd(newPrimaryEndPoint); - if (replicaEndPoints is not null) + if (!connection.servers.Contains(newPrimaryEndPoint)) { - foreach (var replicaEndPoint in replicaEndPoints) + // The sentinel-reported primary is an endpoint we have never seen (e.g. a + // recreated node that came back on a brand-new IP). Rebuild the endpoint set + // from scratch around it and reconfigure. + EndPoint[]? replicaEndPoints = GetReplicasForService(serviceName) + ?? GetReplicasForService(serviceName); + + connection.EndPoints.Clear(); + connection.EndPoints.TryAdd(newPrimaryEndPoint); + if (replicaEndPoints is not null) + { + foreach (var replicaEndPoint in replicaEndPoints) + { + connection.EndPoints.TryAdd(replicaEndPoint); + } + } + + // Only once sentinel gave us the whole picture: a null replica list is a failed query, not a + // deployment with no replicas, and retiring on one would drop healthy nodes over a hiccup. + if (replicaEndPoints is not null) { - connection.EndPoints.TryAdd(replicaEndPoint); + RetireServersAbsentFromEndPoints(connection, logger); } - } - // Only once sentinel gave us the whole picture: a null replica list is a failed query, not a - // deployment with no replicas, and retiring on one would drop healthy nodes over a hiccup. - if (replicaEndPoints is not null) + TriggerReconfigure(reconfigureAll: false); + } + else if (IsStalePrimaryView(connection, newPrimaryEndPoint)) { - RetireServersAbsentFromEndPoints(connection, logger); + // The sentinel-reported primary IS an endpoint we already know about, but our + // cached view of the topology is stale (failover between known nodes: the old + // primary was demoted to a replica in-place and a former replica was promoted). + // A plain reconfigure with reconfigureAll:false would NOT re-read the role of an + // already-connected node, so the primary election would re-pick the stale demoted + // node. Force a full reconfigure so every connected node's role is refreshed before + // the primary is re-elected. + TriggerReconfigure(reconfigureAll: true); } - TriggerReconfigure(reconfigureAll: false); + return switched; } - else if (IsStalePrimaryView(connection, newPrimaryEndPoint)) + finally { - // The sentinel-reported primary IS an endpoint we already know about, but our - // cached view of the topology is stale (failover between known nodes: the old - // primary was demoted to a replica in-place and a former replica was promoted). - // A plain reconfigure with reconfigureAll:false would NOT re-read the role of an - // already-connected node, so the primary election would re-pick the stale demoted - // node. Force a full reconfigure so every connected node's role is refreshed before - // the primary is re-elected. - TriggerReconfigure(reconfigureAll: true); + // not handed over (view already correct, or we threw before the reconfigure): release it ourselves + if (!switched) connection.ReleaseReconfigureClaim(); } } diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 7266ce48b..16c194ede 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -132,8 +132,8 @@ public sealed partial class ConnectionMultiplexer : IInternalConnectionMultiplex private int _activeHeartbeatErrors, lastHeartbeatTicks; internal long LastHeartbeatSecondsAgo => pulse is null - ? -1 - : unchecked(Environment.TickCount - Volatile.Read(ref lastHeartbeatTicks)) / 1000; + ? -1 + : unchecked(Environment.TickCount - Volatile.Read(ref lastHeartbeatTicks)) / 1000; private static int lastGlobalHeartbeatTicks = Environment.TickCount; internal static long LastGlobalHeartbeatSecondsAgo => @@ -1040,7 +1040,8 @@ ServerEndPoint IInternalConnectionMultiplexer.GetServerEndPoint(EndPoint endpoin // a retired server may still be referenced by an alias for a moment; never hand one back, or the // caller receives something whose connections are gone return _serverIdentities.TryGetValue(endpoint, out var byIdentity) && !byIdentity.IsDisposed - ? byIdentity : null; + ? byIdentity + : null; } // bumped once per applied cluster topology; absence is measured in these rather than in time, so a @@ -1365,6 +1366,7 @@ internal void OnHeartbeat() ClientCache?.SweepIfDue(); CheckTopologyRefreshDue(now); + if (_isSentinel) CheckManagedPrimariesDemoted(); // the core's endpoints, for the same reason the snapshot below is pulsed: a late reply // announces itself to nobody, so noticing one is work that happens on a clock. @@ -1439,7 +1441,8 @@ public IDatabase GetDatabase(int db = -1, object? asyncState = null) // if there's no async-state, and the DB is suitable, we can hand out a re-used instance return (asyncState == null && db <= MaxCachedDatabaseInstance) - ? GetCachedDatabaseInstance(db) : Surface(db, asyncState); + ? GetCachedDatabaseInstance(db) + : Surface(db, asyncState); } /// A database over the new core's connections. @@ -1517,7 +1520,7 @@ internal RespConnectionManager Connections for (int i = 0; i < len; i++) { var server = tmp[(int)(((uint)i + startOffset) % len)]; - if (server != null && server.ServerType == serverType && server.IsSelectable(command, allowDisconnected)) + if (server != null && server.ServerType.IsRoutableAs(serverType) && server.IsSelectable(command, allowDisconnected)) { if (server.IsReplica) { @@ -1564,8 +1567,8 @@ public IServer GetServer(string host, int port, object? asyncState = null) => /// The async state to pass into the resulting . public IServer GetServer(string hostAndPort, object? asyncState = null) => Format.TryParseEndPoint(hostAndPort, out var ep) - ? GetServer(ep, asyncState) - : throw new ArgumentException($"The specified host and port could not be parsed: {hostAndPort}", nameof(hostAndPort)); + ? GetServer(ep, asyncState) + : throw new ArgumentException($"The specified host and port could not be parsed: {hostAndPort}", nameof(hostAndPort)); /// /// Obtain a configuration API for an individual server. @@ -1772,6 +1775,20 @@ internal static void ActivateServer(ServerEndPoint server, ILogger? log) } } + /// + /// Claims the reconfigure gate without running a reconfigure, so that work that must not interleave with + /// one (such as rewriting the endpoint list) can make its pre-checks and its + /// side effects atomic with respect to competing reconfigures. A successful claim MUST be handed to + /// with + /// claimed: true (which releases it however it exits), or released via + /// - a leaked claim refuses every later reconfigure. + /// + internal bool TryClaimReconfigure(string cause) => + Interlocked.CompareExchange(ref activeConfigCause, cause, null) == null; + + /// Releases a claim taken by that was never handed to a reconfigure. + internal void ReleaseReconfigureClaim() => Interlocked.Exchange(ref activeConfigCause, null); + internal bool ReconfigureIfNeeded(EndPoint? blame, bool fromBroadcast, string cause, bool publishReconfigure = false, CommandFlags flags = CommandFlags.None) { if (fromBroadcast) @@ -1800,16 +1817,23 @@ internal bool ReconfigureIfNeeded(EndPoint? blame, bool fromBroadcast, string ca public Task ReconfigureAsync(string reason) => ReconfigureAsync(first: false, reconfigureAll: false, log: Logger, blame: null, cause: reason); - internal async Task ReconfigureAsync(bool first, bool reconfigureAll, ILogger? log, EndPoint? blame, string cause, bool publishReconfigure = false, CommandFlags publishReconfigureFlags = CommandFlags.None) + internal async Task ReconfigureAsync(bool first, bool reconfigureAll, ILogger? log, EndPoint? blame, string cause, bool publishReconfigure = false, CommandFlags publishReconfigureFlags = CommandFlags.None, bool claimed = false) { - if (_isDisposed) throw new ObjectDisposedException(ToString()); + if (_isDisposed) + { + // a caller-held claim must not leak: nothing else would ever release it + if (claimed) ReleaseReconfigureClaim(); + throw new ObjectDisposedException(ToString()); + } bool showStats = log is not null; bool ranThisCall = false; try { - // Note that we *always* exchange the reason (first one counts) to prevent duplicate runs - ranThisCall = Interlocked.CompareExchange(ref activeConfigCause, cause, null) == null; + // Note that we *always* exchange the reason (first one counts) to prevent duplicate runs; + // claimed means the caller already owns the gate (TryClaimReconfigure) and we run under - and + // release - that claim + ranThisCall = claimed || Interlocked.CompareExchange(ref activeConfigCause, cause, null) == null; if (!ranThisCall) { @@ -2079,7 +2103,19 @@ internal async Task ReconfigureAsync(bool first, bool reconfigureAll, ILog } } - if (clusterCount == 0) + // A server *saying* it is a cluster is not enough to route as one: that needs a slot map, and + // proxied deployments can announce "cluster" while refusing every CLUSTER subcommand (AMR with + // the Enterprise clustering policy, #3284). With no map there is nothing to route by and the + // only thing cluster mode would add is rejecting cross-slot commands client-side, which the + // proxy would have accepted. The map is never cleared once populated, so an established + // cluster whose refresh is refused (#3275) keeps routing as one. + bool noSlotMap = ServerSelectionStrategy.IsSlotMapEmpty; + if (clusterCount > 0 && noSlotMap) + { + log?.LogInformationClusterWithoutSlotMap(clusterCount); + } + + if (clusterCount == 0 || noSlotMap) { // Set the serverSelectionStrategy if (RawConfig.Proxy == Proxy.Twemproxy) @@ -2093,8 +2129,15 @@ internal async Task ReconfigureAsync(bool first, bool reconfigureAll, ILog else if (standaloneCount == 0 && sentinelCount > 0) { ServerSelectionStrategy.ServerType = ServerType.Sentinel; + if (!first) + { + // A sentinel connection whose initial connect soft-failed (AbortOnConnectFail=false) is + // only now reaching a sentinel; the connect path that would have wired up the sentinel + // handlers has long since returned. (The first connect leaves that to ConnectImpl.) + InitializeSentinel(log); + } } - else if (standaloneCount > 0) + else if (standaloneCount > 0 || clusterCount > 0) { ServerSelectionStrategy.ServerType = ServerType.Standalone; } @@ -2123,8 +2166,14 @@ internal async Task ReconfigureAsync(bool first, bool reconfigureAll, ILog else { ServerSelectionStrategy.ServerType = ServerType.Cluster; - long coveredSlots = ServerSelectionStrategy.CountCoveredSlots(); - log?.LogInformationClusterSlotsCovered(coveredSlots, ServerSelectionStrategy.TotalSlots); + log?.LogInformationClusterSlotsCovered(ServerSelectionStrategy.CountCoveredSlots(), ServerSelectionStrategy.TotalSlots); + + // a cluster has many primaries by design; if we routed as standalone before the map + // arrived, the nomination above may have marked all but one of them redundant + foreach (var primary in primaries) + { + primary.ClearUnselectable(UnselectableFlags.RedundantPrimary); + } } if (!first) { @@ -2219,6 +2268,58 @@ public EndPoint[] GetEndPoints(bool configuredOnly = false) => return slots; } + /// Ask one node for both views of the cluster, tolerating a server that refuses either. + /// + /// A refusal is a posture, not a fault: proxied deployments commonly answer NODES while blocking + /// SLOTS (Redis Enterprise exposes SLOTS only under the OSS cluster API; see #3274), so a + /// refused SLOTS falls back to the NODES-only path exactly as a pre-4.0 server does, and a + /// refused NODES means this node exposes no topology at all, however its HELLO announced + /// itself - the caller gets no configuration. Ported by intent from the shipped core (main #3275), where + /// it lived in the Message path. Both reads ride the same connection, so the one not being awaited + /// when the other throws must have its fault observed rather than left to UnobservedTaskException. + /// + private static async Task ReadClusterViewsAsync(ServerEndPoint server, ILogger? log) + { + var slotsRead = ReadClusterSlotsAsync(server).ObserveErrors(); + var nodesRead = new RedisServer(server, null).ClusterNodesAsync().ObserveErrors(); + + var topology = await AwaitSlotsViewAsync(slotsRead, server, log).ForAwait(); + try + { + return new(await nodesRead.ForAwait(), topology); + } + catch (RedisServerException ex) + { + log?.LogInformationClusterNodesRefused(new(server), ex.Message); + return new(null, topology); + } + } + + /// The SLOTS view alone, or null when the server refused it (logged, not thrown). + private static Task ReadSlotsViewAsync(ServerEndPoint server, ILogger? log) + => AwaitSlotsViewAsync(ReadClusterSlotsAsync(server), server, log); + + private static async Task AwaitSlotsViewAsync(Task slotsRead, ServerEndPoint server, ILogger? log) + { + try + { + return ClusterTopology.From(await slotsRead.ForAwait()); + } + catch (RedisServerException ex) + { + log?.LogInformationClusterSlotsRefused(new(server), ex.Message); + return null; + } + } + + /// Both views of a cluster from one node; either may be missing (see ). + /// A struct rather than a tuple: the library must not reference System.ValueTuple (SanityChecks.ValueTupleNotReferenced). + private readonly struct ClusterViews(ClusterConfiguration? nodes, ClusterTopology? topology) + { + public ClusterConfiguration? Nodes { get; } = nodes; + public ClusterTopology? Topology { get; } = topology; + } + private async Task GetEndpointsFromClusterNodes(ServerEndPoint server, ILogger? log) { try @@ -2275,18 +2376,26 @@ public EndPoint[] GetEndPoints(bool configuredOnly = false) => // rather than returning null and reporting a deployment with no endpoints if (clusterConfig is null) { - var slotsRead = ReadClusterSlotsAsync(server); - clusterConfig = await new RedisServer(server, null).ClusterNodesAsync().ForAwait(); - topology ??= ClusterTopology.From(await slotsRead.ForAwait()); + var asked = await ReadClusterViewsAsync(server, log).ForAwait(); + clusterConfig = asked.Nodes; + topology ??= asked.Topology; + } + else if (topology is null) + { + // NODES is known but nobody holds a SLOTS view: ask for one, so that a server which keeps + // refusing it says so in the log on every pass (main #3275) rather than leaving "why is + // there no slot map?" unanswered, and so that a cluster whose refusal was transient gets + // its map back without waiting for a redirect + topology = await ReadSlotsViewAsync(server, log).ForAwait(); } } else { // not connected on the new core yet, or nothing cached: ask, as a pair - SLOTS says who serves // what and under which names, NODES lists every node including those serving nothing - var slotsRead = ReadClusterSlotsAsync(server); - clusterConfig = await new RedisServer(server, null).ClusterNodesAsync().ForAwait(); - topology = ClusterTopology.From(await slotsRead.ForAwait()); + var asked = await ReadClusterViewsAsync(server, log).ForAwait(); + clusterConfig = asked.Nodes; + topology = asked.Topology; } if (clusterConfig is null) @@ -2794,12 +2903,12 @@ private void DisposeAndClearServers() } } - long? IInternalConnectionMultiplexer.GetConnectionId(EndPoint endpoint, ConnectionType type) + long? IInternalConnectionMultiplexer.GetConnectionId(EndPoint endPoint, ConnectionType type) // the core's connection, which is the one actually carrying the commands and the only one the // server can name. Asking must not dial: while both cores existed this asked a v3 bridge first, // and creating one here connected it - a whole v3 handshake on a third socket just to answer a // question - => ConnectionsIfCreated?.ConnectionId(endpoint, type); + => ConnectionsIfCreated?.ConnectionId(endPoint, type); internal uint UpdateLatency() { diff --git a/src/StackExchange.Redis/Contexts/RespConnectionManager.cs b/src/StackExchange.Redis/Contexts/RespConnectionManager.cs index 386dcc37f..df04e0100 100644 --- a/src/StackExchange.Redis/Contexts/RespConnectionManager.cs +++ b/src/StackExchange.Redis/Contexts/RespConnectionManager.cs @@ -111,7 +111,7 @@ internal RespConnectionManager(ConnectionMultiplexer multiplexer) RoleLearned = PublishRoleToSelector, }; _features = new ObservedFeatureProbe(this); - _select = new SelectPreamble(new RespContext(multiplexer.RawConfig.CommandMap)); + _select = new SelectPreamble(new RespContext(multiplexer.CommandMap)); _defaultDatabase = multiplexer.RawConfig.DefaultDatabase.GetValueOrDefault(); _router = Rebind(multiplexer.RawConfig.DefaultDatabase.GetValueOrDefault()); SeedRoles(); @@ -137,9 +137,11 @@ internal RespConnectionManager(ConnectionMultiplexer multiplexer) /// /// private static RespClusterState SeedTopology(ConnectionMultiplexer multiplexer) - => multiplexer.ServerSelectionStrategy.ServerType == ServerType.Cluster ? RespClusterState.Yes - : multiplexer.IsConnected ? RespClusterState.No - : RespClusterState.Unknown; + => multiplexer.ServerSelectionStrategy.ServerType == ServerType.Cluster + ? RespClusterState.Yes + : multiplexer.IsConnected + ? RespClusterState.No + : RespClusterState.Unknown; internal bool RoutesBySlotForTest => _topology.RoutesBySlot; @@ -186,7 +188,7 @@ internal int ConnectedEndpointCountForTest internal RespDatabaseContext GetDatabase(int database = 0) { return new(new RespContext( - _multiplexer.RawConfig.CommandMap, + _multiplexer.CommandMap, database: database, serverType: _multiplexer.ServerSelectionStrategy.ServerType) .WithTopology(_topology) @@ -469,6 +471,15 @@ private static bool Faulted( if (wantsReplica && PickReplica(_topology.Replicas) is { } replica) return replica; + // A primary-only command sent with PreferReplica goes to the primary, which the shipped core + // arranged by rewriting the flags before the selector saw them (Message.SetPrimaryOnly; main #3291, + // #2821): the selector honours the preference as written, and the pre-send check would then refuse + // the command on the replica it chose. DemandReplica is left alone so that it still fails the same way. + if (command.IsPrimaryOnly() && CommandFlagsInternal.GetPrimaryReplicaFlags(flags) == CommandFlags.PreferReplica) + { + flags = CommandFlagsInternal.SetPrimaryReplicaFlags(flags, CommandFlags.DemandMaster); + } + // ...and the selector for the keyless choice, for the same reason and with the same measurement: // it knows every configured endpoint's role and selectability without this core having dialled // any of them. The role-aware choice below is what will replace it. @@ -758,7 +769,7 @@ internal RespEndpointExecutor SubscriptionEndpoint(EndPoint endpoint) /// internal RespServerContext ServerContext(EndPoint endpoint) => new(new RespContext( - _multiplexer.RawConfig.CommandMap, + _multiplexer.CommandMap, database: -1, serverType: _multiplexer.ServerSelectionStrategy.ServerType) .WithScriptCache(_multiplexer.ScriptCache) @@ -776,7 +787,7 @@ internal RespServerContext ServerContext(EndPoint endpoint) /// prefix, because that IS the channel as far as the wire is concerned. /// internal RespContext SubscriptionContext(EndPoint endpoint) - => new RespContext(_multiplexer.RawConfig.CommandMap) + => new RespContext(_multiplexer.CommandMap) .AppendChannelPrefix(_multiplexer.RawConfig.ChannelPrefix) .WithServices(_features) .WithExecutor(SubscriptionEndpoint(endpoint)); @@ -1209,7 +1220,7 @@ await RespHandshake.DiscoverServerConfigAsync( && result.Protocol < RedisProtocol.Resp3 && config.TryResp3() && _multiplexer.RawConfig.SharedSubscriptionConnection - && _multiplexer.RawConfig.CommandMap.IsAvailable(RedisCommand.SUBSCRIBE) + && _multiplexer.CommandMap.IsAvailable(RedisCommand.SUBSCRIBE) && (_multiplexer.AnySubscribed() || _multiplexer.ConfigurationChangedChannel is not null)) { @@ -1331,7 +1342,7 @@ await SubscribeToConfigurationChannelAsync(context, cancellationToken) internal void DialSubscriptionSocketForConfigurationChannel(EndPoint endpoint) { if (_multiplexer.ConfigurationChangedChannel is null - || !_multiplexer.RawConfig.CommandMap.IsAvailable(RedisCommand.SUBSCRIBE)) + || !_multiplexer.CommandMap.IsAvailable(RedisCommand.SUBSCRIBE)) { return; } @@ -2121,7 +2132,7 @@ internal void AddCounters(EndPoint endpoint, ConnectionType connectionType, Conn internal void SetLibraryName(string libraryName) { if (string.IsNullOrWhiteSpace(libraryName)) return; - if (!_multiplexer.RawConfig.CommandMap.IsAvailable(RedisCommand.CLIENT)) return; + if (!_multiplexer.CommandMap.IsAvailable(RedisCommand.CLIENT)) return; foreach (var pair in _endpoints) Announce(pair.Key, pair.Value); foreach (var pair in _subscriptions) Announce(pair.Key, pair.Value); @@ -2131,7 +2142,7 @@ void Announce(EndPoint endpoint, RespEndpointExecutor executor) if (!executor.IsConnectedNow) return; try { - var context = new RespContext(_multiplexer.RawConfig.CommandMap, database: -1) + var context = new RespContext(_multiplexer.CommandMap, database: -1) .WithExecutor(executor.WithDatabase(-1)); _ = context.SendAsync( $"{RedisCommand.CLIENT}{RedisLiterals.SETINFO}{RedisLiterals.lib_name}{(RedisValue)libraryName}", @@ -2170,6 +2181,7 @@ private void Publish(ServerEndPoint server, in RespHandshakeResult result) { if (result.Version is { } version) server.Version = version; if (result.RoleFromHello is { } isReplica) server.IsReplica = isReplica; + server.OnRoleChecked(); // the handshake read the role; the heartbeat's re-read starts counting from here // The SLOTS view first, and this ordering is load-bearing: SetClusterConfiguration below reads // ServerEndPoint.ClusterTopology to decide which reply drives the selector's slot map, and falls @@ -2236,6 +2248,54 @@ private void Publish(ServerEndPoint server, in RespHandshakeResult result) internal void OnRole(EndPoint endpoint, bool isReplica) => _topology.OnRole(endpoint, isReplica ? RespEndpointRole.Replica : RespEndpointRole.Primary); + /// + /// Re-read one endpoint's role in the background, so a possibly stale view of it is corrected without + /// reconfiguring anything; returns whether the read could be sent at all. + /// + /// The server whose role is in doubt. + /// Where a failed read is noted; it is not an error. + /// + /// The sentinel recovery path's question (RecoverDemotedPrimary): sentinel still names a node this + /// core holds as a replica, which is either sentinel lagging or our role for it being stale, and re-reading + /// the role settles both where a switch would reconfigure every node once per heartbeat. The shipped core + /// sent INFO replication fire-and-forget through its auto-configure processor + /// (ServerEndPoint.CheckInfoReplication, main #3281); here it is the same ROLE read the + /// topology sweep uses, applied through so it reaches + /// ServerEndPoint.IsReplica by the one route every learned role takes. False when ROLE is + /// unavailable or the endpoint is not connected, which the caller reads as "cannot settle it this way" + /// and falls through to the switch. + /// + internal bool TryRefreshRole(EndPoint endpoint, ILogger? log = null) + { + if (!_multiplexer.CommandMap.IsAvailable(RedisCommand.ROLE) + || !_endpoints.TryGetValue(endpoint, out var executor) + || !executor.IsConnectedNow) + { + return false; + } + + _ = RefreshAsync(); + return true; + + async Task RefreshAsync() + { + try + { + var context = new RespDatabaseContext(new RespContext(_multiplexer.CommandMap, database: -1).WithExecutor(executor)); + var role = await context.SendAsync($"{RedisCommand.ROLE}", handler: RespHandshake.RoleHandler.Instance).ConfigureAwait(false); + _topology.OnRole(endpoint, role.Role); + if (role.Peers is not null) + { + foreach (var peer in role.Peers) _topology.OnRole(peer, role.PeerRole); + } + } + catch (Exception ex) + { + log?.LogInformationRoleRefreshFailed(ex, new(endpoint), ex.Message); + } + } + } + /// Tell the selector a role this core discovered, which is the reverse of . /// The endpoint whose role changed. /// What it turned out to be. @@ -2309,9 +2369,11 @@ private void PublishRoleToSelector(EndPoint endpoint, bool isReplica) /// internal async Task RefreshTopologyAsync(ILogger? log = null) { - if (_topology.RoutesBySlot) + // NeedsSlots rather than RoutesBySlot: a cluster that has refused SLOTS so far routes as standalone + // (RoutesBySlot wants a map), and this sweep is exactly how it would ever get one + if (_topology.NeedsSlots) { - if (!_multiplexer.RawConfig.CommandMap.IsAvailable(RedisCommand.CLUSTER)) return; + if (!_multiplexer.CommandMap.IsAvailable(RedisCommand.CLUSTER)) return; foreach (var pair in _endpoints) { @@ -2339,7 +2401,7 @@ internal async Task RefreshTopologyAsync(ILogger? log = null) return; } - if (!_topology.WantsRoles || !_multiplexer.RawConfig.CommandMap.IsAvailable(RedisCommand.ROLE)) return; + if (!_topology.WantsRoles || !_multiplexer.CommandMap.IsAvailable(RedisCommand.ROLE)) return; foreach (var pair in _endpoints) { @@ -2363,7 +2425,7 @@ internal async Task RefreshTopologyAsync(ILogger? log = null) } RespDatabaseContext Context(RespEndpointExecutor executor) - => new(new RespContext(_multiplexer.RawConfig.CommandMap, database: -1).WithExecutor(executor)); + => new(new RespContext(_multiplexer.CommandMap, database: -1).WithExecutor(executor)); // this core's map AND the selector's, for the same reason the handshake does both: the selector // still answers for IServer and for the routing fallbacks, and a refresh that corrected only one @@ -2509,6 +2571,24 @@ internal void OnHeartbeat() var timeout = _multiplexer.AsyncTimeoutMilliseconds; foreach (var pair in _endpoints) pair.Value.OnHeartbeat(timeout); foreach (var pair in _subscriptions) pair.Value.OnHeartbeat(timeout); + + // The periodic role re-read the shipped bridge made on its heartbeat (CheckInfoReplication, paced by + // ConfigCheckSeconds with the exponential ramp a suspected failover forces). A node demoted in place + // reports its new role only to somebody who asks, and nothing else here would: the sentinel recovery + // path (main #3281) acts on ServerEndPoint.IsReplica, which this is what keeps honest. Interactive + // endpoints only, as before, and never a sentinel, which has no role worth asking for. + foreach (var pair in _endpoints) + { + if (!pair.Value.IsConnectedNow) continue; + if (_multiplexer.TryResolveServerEndPoint(pair.Key) is not { } server + || server.ServerType == ServerType.Sentinel + || !server.RoleCheckDue) + { + continue; + } + + if (TryRefreshRole(pair.Key, _multiplexer.Logger)) server.OnRoleChecked(); + } } /// Release a connection-local claim on this core's interactive connection to an endpoint. diff --git a/src/StackExchange.Redis/Contexts/RespEndpointExecutor.cs b/src/StackExchange.Redis/Contexts/RespEndpointExecutor.cs index a1dd90659..3a84ad4ae 100644 --- a/src/StackExchange.Redis/Contexts/RespEndpointExecutor.cs +++ b/src/StackExchange.Redis/Contexts/RespEndpointExecutor.cs @@ -1337,6 +1337,45 @@ private bool Enqueue(RespPayloadOperation operation) /// Strictly healthy -> tripped -> actuated, then healthy again with fresh counters. private int _circuitBreakerState; + /// Whether the current LOADING episode has been reported; cleared by a success, and per connection. + private int _loadingReported; + + /// The LOADING reply being reported, for . + private RedisServerException? _loadingFault; + + private static readonly WaitCallback s_LoadingObserved = static state => + { + var executor = (RespEndpointExecutor)state!; + try + { + if (Volatile.Read(ref executor._loadingFault) is { } loading) executor.ActuateLoading(loading); + } + catch + { + // a report that cannot be made is not a reason to bring the process down with it + } + }; + + /// Report a LOADING reply through ConnectionFailed, as a breaker trip is reported. + /// + /// From the pool rather than the completion path, as is, so that the thread that + /// just finished somebody's command does not run the group's handlers. reconfigure: false: a + /// loading server has not changed the topology, and a group's failover is driven by the event itself. + /// + private void ActuateLoading(RedisServerException loading) + { + if (_endpoint is { } reportAt && Server is { } server && !server.Multiplexer.IsDisposed) + { + server.Multiplexer.OnConnectionFailed( + reportAt, + IsSubscriptionEndpoint ? ConnectionType.Subscription : ConnectionType.Interactive, + ConnectionFailureType.Loading, + loading, + reconfigure: false, + physicalName: null); + } + } + private const int CircuitHealthy = 0, CircuitTripped = 1, CircuitActuated = 2; /// @@ -1349,6 +1388,30 @@ private bool Enqueue(RespPayloadOperation operation) /// void IRespOutcomeObserver.ObserveOutcome(Exception? fault) { + // A server that is loading its dataset is a server that cannot serve. The shipped core treated the + // first LOADING reply as a connection failure (RecordConnectionFailed(ConnectionFailureType.Loading), + // main #3289): ConnectionFailed is the event a connection group listens for to steer away, and the + // breaker cannot be what moves it, since a loading server answers everything - its handshake included + // - with the same error. The event is kept; the drop is not. The shipped core tore the connection down + // and dialled again into a handshake that failed the same way, and a retrying caller then found the + // member mid-dial and was moved off it (RetryEndToEndTests.WithRetry_WhenFailoverNeverArrives...). + // A LOADING reply is prompt, and a prompt refusal is a better answer than a reconnect loop. Said once + // per episode: the burst that follows the first reply is the same fact repeated, and a success in + // between re-arms it, since a replica re-syncing with replica-serve-stale-data off can go LOADING + // again on the same long-lived connection. + if (fault is RedisServerException { Kind: RedisErrorKind.Loading } loading) + { + if (Interlocked.CompareExchange(ref _loadingReported, 1, 0) == 0) + { + Volatile.Write(ref _loadingFault, loading); // carried in a field: the library does not reference ValueTuple + ThreadPool.QueueUserWorkItem(s_LoadingObserved, this); + } + } + else if (fault is null && Volatile.Read(ref _loadingReported) != 0) + { + Volatile.Write(ref _loadingReported, 0); + } + var accumulator = Volatile.Read(ref _circuitBreaker); if (accumulator is null || !accumulator.Trip(fault)) return; @@ -1399,6 +1462,7 @@ private void ActuateTrip() Volatile.Write(ref _circuitBreaker, _circuitBreakerFactory?.Invoke()); Volatile.Write(ref _circuitBreakerState, CircuitHealthy); + Volatile.Write(ref _loadingReported, 0); } /// Bring this endpoint up now, and wait for it. @@ -1753,7 +1817,7 @@ private async Task PingAsync() { var started = DateTime.UtcNow; await new RespDatabaseContext( - new RespContext(Server!.Multiplexer.RawConfig.CommandMap, database: -1).WithExecutor(this)) + new RespContext(Server!.Multiplexer.CommandMap, database: -1).WithExecutor(this)) .PingAsync(CommandFlags.NoRedirect) .ConfigureAwait(false); @@ -1900,7 +1964,7 @@ private void Validate(in RespRequest request, int database) // core that renders its own frames has to make it somewhere; found by porting SUBSCRIBE, which // is what ConfigTests.ConnectWithSubscribeDisabled asks about. if (request.Command is not (RedisCommand.NONE or RedisCommand.UNKNOWN) - && !config.CommandMap.IsAvailable(request.Command)) + && !server.Multiplexer.CommandMap.IsAvailable(request.Command)) { throw ExceptionFactory.CommandDisabled(request.Command); } diff --git a/src/StackExchange.Redis/Contexts/RespHandshake.cs b/src/StackExchange.Redis/Contexts/RespHandshake.cs index 7c11a5049..f0b3e35c9 100644 --- a/src/StackExchange.Redis/Contexts/RespHandshake.cs +++ b/src/StackExchange.Redis/Contexts/RespHandshake.cs @@ -306,6 +306,19 @@ internal static async Task PerformAsync( version = hello.Version; roleFromHello = hello.IsReplica; + // what HELLO said the product is, when it named a vendor (main #3279): "server" for a server that + // is not Redis, or a "{vendor}_version" field beside a Redis-compatible "version" + if (server is not null) + { + if (hello.Variant is { } variant && hello.ProductVersion is { Length: > 0 } productVersion) + { + server.SetProductVariant(variant, productVersion); + log?.LogInformationAutoConfiguredHelloProductVariant(new(server), variant, productVersion); + } + + if (hello.UnknownServer is { } unknown) log?.LogInformationAutoConfiguredHelloUnknownServer(new(server), unknown); + } + if (log is not null) { log.LogInformationAutoConfiguredHelloProtocol(new(server!), protocol); @@ -402,9 +415,11 @@ internal static async Task PerformAsync( // slots exist, so this is a cluster whatever CLUSTER INFO did or did not say if (slots is { Assignments.Count: > 0 }) serverType = ServerType.Cluster; } - catch (RedisServerException) + catch (RedisServerException ex) { - // no map from this server; the selector still answers + // no map from this server; the selector still answers. Said in the log (main #3275), because + // the consequence - a cluster routed as standalone until a map arrives - is otherwise silent + if (server is not null) log?.LogInformationClusterSlotsRefused(new(server), ex.Message); } } @@ -416,8 +431,16 @@ internal static async Task PerformAsync( // BEFORE the connection is handed back, and that ordering is the point: the endpoint executor // publishes the connection and drains its backlog the moment this returns, and a backlog - // draining against an unset topology is exactly the window that loses per-slot ordering - topology?.OnServerType(serverType); + // draining against an unset topology is exactly the window that loses per-slot ordering. + // + // ...and only from the connection that does discovery (main #3285, #3284). The subscription + // connection's handshake is HELLO alone, with no INFO or CLUSTER follow-up, so a proxy whose HELLO + // disagrees with its INFO (AMR says "cluster" to one and "standalone" to the other) would otherwise + // have the last word every time pub/sub reconnects on its own. "mode" and "role" describe the + // server, and the interactive connection is the one that establishes them; a reply that carried + // slots is proof of a cluster whichever socket asked, and counts. + var discovers = connectionType == ConnectionType.Interactive || slots is { Assignments.Count: > 0 }; + if (discovers) topology?.OnServerType(serverType); if (slots is not null && topology is not null) { @@ -573,7 +596,8 @@ internal static async Task PerformAsync( LogResponse(log, server, connectionType, "SELECT " + database.ToString(CultureInfo.InvariantCulture), "OK"); } - return new RespHandshakeResult(protocol, serverType, version, knowServerType, connectionId, roleFromHello, clusterNodes, slots); + // the server type and role are published only by the connection that discovers them (see `discovers`) + return new RespHandshakeResult(protocol, serverType, version, knowServerType && discovers, connectionId, discovers ? roleFromHello : null, clusterNodes, slots); } /// Read the server-wide settings the client models, for a server nothing has described yet. @@ -1057,9 +1081,12 @@ private static async Task DiscoverTieBreakerAsync(RespDatabaseContext context, S /// never learns the product applies the wrong rule to a real deployment. /// /// - /// From INFO, not from HELLO, which is deliberate and matches the v3 core. - /// HELLO does carry a server field, and reading it would be free - but it names the - /// product without its product version, and the version is half of what is being established. The + /// From INFO, unless HELLO already named the product. HELLO carries a + /// server field and, for some products, a {vendor}_version beside it; since main #3279 the + /// handshake reads those (see HelloHandler), and a server so described skips this. What is left + /// for INFO is a server whose HELLO said nothing - RESP2-only, or plain "redis" with no + /// vendor field - and, separately, the Redis API version of a Valkey, whose HELLO "version" is its + /// own (that is the version fallback above, not this). The /// variant is decided across the whole reply rather than per line, because both /// redis_version and valkey_version can be present in either order; that was the /// v3 processor's rule and this reuses its field table rather than growing a second copy. @@ -1373,16 +1400,36 @@ await context.SendAsync($"{command}{RedisLiterals.TRACKING}{RedisLiterals.ON}{Re /// The version it reported, if it did. /// What it said this server's role is, when it said. /// What it said this connection's id is, when it said. + /// The product the reply named, when it named one other than plain Redis. + /// That product's own version, when is set. + /// A "server" value this library does not know, for logging; null otherwise. /// /// A struct rather than a tuple: the library must not reference System.ValueTuple, which /// would add a facade dependency on the down-level targets - asserted by /// SanityCheckTests.ValueTupleNotReferenced, which is how this was caught. /// - private readonly struct HelloReply(int proto, ServerType? mode, Version? version, bool? isReplica, long? connectionId = null) + private readonly struct HelloReply( + int proto, + ServerType? mode, + Version? version, + bool? isReplica, + long? connectionId = null, + ProductVariant? variant = null, + string? productVersion = null, + string? unknownServer = null) { /// What the reply said this connection's id is, when it said. internal long? ConnectionId { get; } = connectionId; + /// The product the reply named, when it named one other than plain Redis. + internal ProductVariant? Variant { get; } = variant; + + /// That product's own version, when is set. + internal string? ProductVersion { get; } = productVersion; + + /// A "server" value this library does not know, so that it can be logged; null otherwise. + internal string? UnknownServer { get; } = unknownServer; + internal int Proto { get; } = proto; internal ServerType? Mode { get; } = mode; @@ -1444,35 +1491,66 @@ public HelloReply Parse(ref RespReader reader) { var proto = 2; // a reply we could not read is not a reason to claim RESP3 ServerType? mode = null; - Version? version = null; bool? isReplica = null; long? connectionId = null; + // "version" means different things depending on "server" (main #3279, #3278): Redis - and Garnet, + // which presents as "redis" and adds "garnet_version" - report the Redis API version, but Valkey + // reports ITS OWN ("server" => "valkey", "version" => "9.1.2", while INFO still says + // redis_version:7.2.4). So what "version" is can only be decided once the whole map has been read. + var serverIsRedis = true; // absent means redis: older servers + ProductVariant? helloVariant = null, vendorVariant = null; + string? versionText = null, vendorVersion = null, unknownServer = null; + var count = reader.AggregateLength(); for (var i = 0; i < count; i++) { if (!reader.TryMoveNext()) break; - var isProto = reader.Is("proto"u8); - var isMode = !isProto && reader.Is("mode"u8); - var isVersion = !isProto && !isMode && reader.Is("version"u8); - var isRole = !isProto && !isMode && !isVersion && reader.Is("role"u8); - var isId = !isProto && !isMode && !isVersion && !isRole && reader.Is("id"u8); + var field = reader.IsScalar && reader.TryGetSpan(out var name) && HelloFieldMetadata.TryParse(name, out var parsed) + ? parsed + : HelloField.Unknown; if (!reader.TryMoveNext()) break; i++; - if (isProto && reader.IsScalar && reader.TryReadInt64(out var value)) + if (field == HelloField.Proto && reader.IsScalar && reader.TryReadInt64(out var value)) { proto = (int)value; } - else if (isId && reader.IsScalar && reader.TryReadInt64(out var id)) + else if (field == HelloField.Id && reader.IsScalar && reader.TryReadInt64(out var id)) { connectionId = id; } - else if (isVersion && reader.IsScalar) + else if (field == HelloField.Server && reader.IsScalar) { - _ = Format.TryParseVersion(reader.ReadString(), out version); + if (reader.TryGetSpan(out var span) && ProductVariantMetadata.TryParse(span, out var named)) + { + helloVariant = named; + serverIsRedis = named == ProductVariant.Redis; + } + else + { + serverIsRedis = false; + unknownServer = reader.ReadString() ?? ""; + } + } + else if (field == HelloField.Version && reader.IsScalar) + { + versionText = reader.ReadString(); } - else if (isRole && reader.IsScalar) + else if (field is HelloField.GarnetVersion or HelloField.ValkeyVersion or HelloField.DragonflyVersion + or HelloField.MemuraiVersion or HelloField.RedictVersion && reader.IsScalar) + { + vendorVariant = field switch + { + HelloField.GarnetVersion => ProductVariant.Garnet, + HelloField.ValkeyVersion => ProductVariant.Valkey, + HelloField.DragonflyVersion => ProductVariant.Dragonfly, + HelloField.MemuraiVersion => ProductVariant.Memurai, + _ => ProductVariant.Redict, + }; + vendorVersion = reader.ReadString(); + } + else if (field == HelloField.Role && reader.IsScalar) { // FREE, and it removes the need to ask: the v3 handshake tracked exactly this // as RoleKnownFromHello, and skipped its SET probe when it was set. #2968 is about @@ -1483,12 +1561,14 @@ public HelloReply Parse(ref RespReader reader) isReplica = replica; } } - else if (isMode && reader.IsScalar) + else if (field == HelloField.Mode && reader.IsScalar) { // "standalone" | "sentinel" | "cluster"; only the last changes routing - mode = reader.Is("cluster"u8) ? ServerType.Cluster - : reader.Is("sentinel"u8) ? ServerType.Sentinel - : ServerType.Standalone; + mode = reader.Is("cluster"u8) + ? ServerType.Cluster + : reader.Is("sentinel"u8) + ? ServerType.Sentinel + : ServerType.Standalone; } else { @@ -1496,7 +1576,28 @@ public HelloReply Parse(ref RespReader reader) } } - return new HelloReply(proto, mode, version, isReplica, connectionId); + Version? version = null; + ProductVariant? variant = null; + string? productVersion = null; + if (serverIsRedis) + { + // "version" is the Redis API version; a "{vendor}_version" field, if any, names the product + if (versionText is not null) _ = Format.TryParseVersion(versionText, out version); + if (vendorVariant is { } fromField && !string.IsNullOrWhiteSpace(vendorVersion)) + { + variant = fromField; + productVersion = vendorVersion; + } + } + else if (helloVariant is { } fromServer) + { + // a known non-Redis server: "version" is the vendor's own, and the Redis API version has to + // come from INFO (redis_version), or stay at the configured default - never from here + variant = fromServer; + productVersion = !string.IsNullOrWhiteSpace(vendorVersion) ? vendorVersion : versionText; + } + + return new HelloReply(proto, mode, version, isReplica, connectionId, variant, productVersion, unknownServer); } } diff --git a/src/StackExchange.Redis/Contexts/RespPayloadOperation.cs b/src/StackExchange.Redis/Contexts/RespPayloadOperation.cs index 9dc880783..c007c3290 100644 --- a/src/StackExchange.Redis/Contexts/RespPayloadOperation.cs +++ b/src/StackExchange.Redis/Contexts/RespPayloadOperation.cs @@ -288,7 +288,11 @@ internal string CommandAndKey /// internal string? UnroutableRedirectMessage { - get => _redirectUnroutable ? _redirectMessage : null; + get + { + return _redirectUnroutable ? _redirectMessage : null; + } + set { _redirectMessage = value; @@ -319,7 +323,11 @@ internal string? UnroutableRedirectMessage /// internal string? DeclinedRedirectMessage { - get => _redirectUnroutable ? null : _redirectMessage; + get + { + return _redirectUnroutable ? null : _redirectMessage; + } + set { _redirectMessage = value; diff --git a/src/StackExchange.Redis/Contexts/RespTopology.cs b/src/StackExchange.Redis/Contexts/RespTopology.cs index 55314988e..40990fbba 100644 --- a/src/StackExchange.Redis/Contexts/RespTopology.cs +++ b/src/StackExchange.Redis/Contexts/RespTopology.cs @@ -162,7 +162,18 @@ internal void OnServerType(ServerType serverType) /// Whether the slot decides which endpoint serves the command. /// False while unknown: routing on a guess is wrong, not merely wasteful. - internal bool RoutesBySlot => (RespClusterState)Volatile.Read(ref _state) == RespClusterState.Yes; + /// Whether commands are routed by slot: a cluster, with a slot map. + /// + /// A server saying it is a cluster is not enough (main #3285, #3284): proxied deployments announce + /// "cluster" in HELLO while refusing every CLUSTER subcommand, and with no map the only + /// thing cluster routing would add is refusing cross-slot commands client-side that the proxy would have + /// accepted. So until a map exists - from the handshake, a refresh, or a -MOVED - routing falls to + /// the selector as it does for a standalone server; the map is never cleared once populated, so an + /// established cluster whose refresh is refused keeps routing as one. is the + /// other question - "might this be a cluster?" - and stays true here so slots are still computed and + /// asked for. + /// + internal bool RoutesBySlot => (RespClusterState)Volatile.Read(ref _state) == RespClusterState.Yes && Volatile.Read(ref _slots) is not null; /// The server type, as far as anyone knows. internal ServerType ServerType => RoutesBySlot ? ServerType.Cluster : ServerType.Standalone; diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Arrays.cs b/src/StackExchange.Redis/Database/RedisDatabase.Arrays.cs index 61f8b1adf..b39c1ba7d 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Arrays.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Arrays.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The array commands, where they have moved to the RESP context surface. - /// + /// /// /// The whole family, ARGREP included: now renders its predicates /// through either writer, so the decision that was owed about that type is made. Arrays become spans diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Bitmaps.cs b/src/StackExchange.Redis/Database/RedisDatabase.Bitmaps.cs index 5d845bba3..260fc5f1d 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Bitmaps.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Bitmaps.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The bitmap commands, where they have moved to the RESP context surface. - /// + /// /// /// /// The old surface calls these StringBitXxx and the new one groups them under diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Execute.cs b/src/StackExchange.Redis/Database/RedisDatabase.Execute.cs index e2e77055a..8200c0466 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Execute.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Execute.cs @@ -5,9 +5,9 @@ namespace StackExchange.Redis { - /// + /// /// The ad-hoc escape hatch, where it has moved to the RESP context surface. - /// + /// /// /// /// Its own file rather than a command group, because it is not one: this is the route for commands diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Geospatial.cs b/src/StackExchange.Redis/Database/RedisDatabase.Geospatial.cs index 062072cd2..ffffa98ef 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Geospatial.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Geospatial.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The geospatial commands, where they have moved to the RESP context surface. - /// + /// /// /// /// GeoRadius is the interesting adapter: the group has no method for it, because diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Hashes.cs b/src/StackExchange.Redis/Database/RedisDatabase.Hashes.cs index 3c48fb7ac..d2a30b315 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Hashes.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Hashes.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The hash commands, where they have moved to the RESP context surface. - /// + /// /// /// /// The clearest illustration so far of what the new surface costs the old one: nothing. Six diff --git a/src/StackExchange.Redis/Database/RedisDatabase.HyperLogLog.cs b/src/StackExchange.Redis/Database/RedisDatabase.HyperLogLog.cs index a86e67e9a..52d039734 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.HyperLogLog.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.HyperLogLog.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The HyperLogLog commands, where they have moved to the RESP context surface. - /// + /// /// /// The old surface spells every one of these twice - a fixed-arity form and an array form - and the /// group collapses both onto the span, so the pairs meet again here. HyperLogLogMerge is the diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Keys.cs b/src/StackExchange.Redis/Database/RedisDatabase.Keys.cs index 330ec187c..9430e902b 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Keys.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Keys.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The key commands, where they have moved to the RESP context surface. - /// + /// /// /// /// The spelling of RespSurface.Keys.cs; see diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Lists.cs b/src/StackExchange.Redis/Database/RedisDatabase.Lists.cs index 8b51c4d8c..d445e5170 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Lists.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Lists.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The list commands, where they have moved to the RESP context surface. - /// + /// /// /// /// ListRightPopLeftPush is the interesting adapter here: the group has no method of its own for diff --git a/src/StackExchange.Redis/Database/RedisDatabase.PubSub.cs b/src/StackExchange.Redis/Database/RedisDatabase.PubSub.cs index 6d1e02b1e..5f936f086 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.PubSub.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.PubSub.cs @@ -2,9 +2,9 @@ namespace StackExchange.Redis { - /// + /// /// PUBLISH, where it has moved to the RESP context surface. - /// + /// /// /// /// The last two members SER352 was counting, and the ones that took the longest to be sure about. diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Scans.cs b/src/StackExchange.Redis/Database/RedisDatabase.Scans.cs index 4a572681a..eb6e7d880 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Scans.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Scans.cs @@ -5,9 +5,9 @@ namespace StackExchange.Redis; -/// +/// /// The cursor scans, over the context surface. -/// +/// /// /// /// These are hand-written because [AutoDatabase] skips them, and it skips them for a real diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Scripts.cs b/src/StackExchange.Redis/Database/RedisDatabase.Scripts.cs index 0e0bbd5fc..b051603c2 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Scripts.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Scripts.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis; -/// +/// /// The scripting commands, over the context surface. -/// +/// /// /// /// These keep the shipped wire behaviour, which means they DO pair. This paragraph previously said diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Sets.cs b/src/StackExchange.Redis/Database/RedisDatabase.Sets.cs index eef4e88dd..23f42f546 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Sets.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Sets.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The set commands, where they have moved to the RESP context surface. - /// + /// /// /// The group where the old surface's fixed-arity twins disappear entirely: SetCombine and /// SetCombineAndStore each have a (first, second) overload that existed only because diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Sort.cs b/src/StackExchange.Redis/Database/RedisDatabase.Sort.cs index 6c5d7e41e..003bc57db 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Sort.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Sort.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// SORT, where it has moved to the RESP context surface. - /// + /// /// /// Four members and one renderer. The group is Keys rather than a group of its own, because /// SORT takes a list, a set or a sorted set and Redis files it under the generic commands. diff --git a/src/StackExchange.Redis/Database/RedisDatabase.SortedSets.cs b/src/StackExchange.Redis/Database/RedisDatabase.SortedSets.cs index 81c2ba0d8..f683c1436 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.SortedSets.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.SortedSets.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The sorted-set commands, where they have moved to the RESP context surface. - /// + /// /// /// /// The biggest adapter file, and mostly for one reason: SortedSetAdd alone has six overloads diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Streams.cs b/src/StackExchange.Redis/Database/RedisDatabase.Streams.cs index 11280dc58..722637c8f 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Streams.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Streams.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The stream commands, where they have moved to the RESP context surface. - /// + /// /// /// The scalar half only; the reads return composites holding arrays and have not moved yet. Note the /// array parameters here: the old signatures promise RedisValue[], so this is where an array diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Strings.cs b/src/StackExchange.Redis/Database/RedisDatabase.Strings.cs index c2b5f19c6..502af10c6 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Strings.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Strings.cs @@ -5,9 +5,9 @@ namespace StackExchange.Redis { - /// + /// /// The string commands, where they have moved to the RESP context surface. - /// + /// /// /// /// One file per command group, named to match the group it adapts - RespSurface.Strings.cs diff --git a/src/StackExchange.Redis/Database/RedisDatabase.VectorSets.cs b/src/StackExchange.Redis/Database/RedisDatabase.VectorSets.cs index 3dd0a758d..be3fd9880 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.VectorSets.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.VectorSets.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The vector-set commands, where they have moved to the RESP context surface. - /// + /// /// /// The thinnest adapter of the lot, because the old surface was already lease-shaped: every member is /// a rename plus a writable-lease sibling, and there is no array anywhere except diff --git a/src/StackExchange.Redis/Enums/ProductVariant.cs b/src/StackExchange.Redis/Enums/ProductVariant.cs index fd3a17086..ca85ee8c3 100644 --- a/src/StackExchange.Redis/Enums/ProductVariant.cs +++ b/src/StackExchange.Redis/Enums/ProductVariant.cs @@ -1,48 +1,71 @@ -// ReSharper disable once CheckNamespace +using System; +using RESPite; + +// ReSharper disable once CheckNamespace namespace StackExchange.Redis; /// /// Indicates the flavor of RESP-server being used, if known. Unknown variants will be reported as . /// Inclusion (or omission) in this list does not imply support for any given variant; nor does it indicate any specific /// relationship with the vendor or rights to use the name. It is provided solely for informational purposes. Identification -/// is not guaranteed, and is based on the server's self-reporting (typically via `INFO`), which may be incomplete or misleading. +/// is not guaranteed, and is based on the server's self-reporting (via `INFO` and `HELLO`), which may be incomplete or misleading. /// public enum ProductVariant { /// /// The original Redis server. This is also the default value if the variant is unknown. /// + [AsciiHash("redis")] Redis, /// /// Valkey is a fork of open-source Redis associated with AWS. /// + [AsciiHash("valkey")] Valkey, /// /// Garnet is a Redis-compatible server from Microsoft. /// + [AsciiHash("garnet")] Garnet, /// /// Dragonfly /// + [AsciiHash("dragonfly")] Dragonfly, /// /// Memurai /// + [AsciiHash("memurai")] Memurai, /// /// Redict /// + [AsciiHash("redict")] Redict, /// /// KeyDB /// + [AsciiHash("keydb")] KeyDB, // if you want to add another variant here, please open an issue with the details (variant name, INFO output, etc.) } + +/// +/// Metadata and parsing methods for . +/// +internal static partial class ProductVariantMetadata +{ + /// + /// Parses the server value reported by HELLO (redis, valkey, etc); this is the vendor's + /// name rather than a protocol token, so the match is case-insensitive. + /// + [AsciiHash(CaseSensitive = false)] + internal static partial bool TryParse(ReadOnlySpan value, out ProductVariant variant); +} diff --git a/src/StackExchange.Redis/Enums/ServerType.cs b/src/StackExchange.Redis/Enums/ServerType.cs index a2710ae95..121f61add 100644 --- a/src/StackExchange.Redis/Enums/ServerType.cs +++ b/src/StackExchange.Redis/Enums/ServerType.cs @@ -78,5 +78,16 @@ internal static class ServerTypeExtensions ServerType.Envoyproxy => false, _ => true, }; + + /// + /// Whether a server of this type can be sent commands when the multiplexer is routing as . + /// + /// + /// Usually the types match, and the check keeps data traffic away from sentinels; but a server that calls itself + /// a cluster while giving us no slot map is routed as a single standalone server (see #3284), so it must remain a + /// target in that mode. + /// + internal static bool IsRoutableAs(this ServerType server, ServerType routing) + => server == routing || (routing == ServerType.Standalone && server == ServerType.Cluster); } } diff --git a/src/StackExchange.Redis/Format.cs b/src/StackExchange.Redis/Format.cs index a57bea0cd..0c4e33272 100644 --- a/src/StackExchange.Redis/Format.cs +++ b/src/StackExchange.Redis/Format.cs @@ -233,10 +233,10 @@ internal static bool TryParseUInt64(string s, out ulong value) => ulong.TryParse(s, NumberStyles.Integer, NumberFormatInfo.InvariantInfo, out value); internal static bool TryParseUInt64(ReadOnlySpan s, out ulong value) => - Utf8Parser.TryParse(s, out value, out int bytes, standardFormat: 'D') & bytes == s.Length; + Utf8Parser.TryParse(s, out value, out int bytes, standardFormat: 'D') & (bytes == s.Length); internal static bool TryParseInt64(ReadOnlySpan s, out long value) => - Utf8Parser.TryParse(s, out value, out int bytes, standardFormat: 'D') & bytes == s.Length; + Utf8Parser.TryParse(s, out value, out int bytes, standardFormat: 'D') & (bytes == s.Length); internal static bool CouldBeInteger(string s) { @@ -245,18 +245,18 @@ internal static bool CouldBeInteger(string s) for (int i = isSigned ? 1 : 0; i < s.Length; i++) { char c = s[i]; - if (c < '0' | c > '9') return false; + if ((c < '0') | (c > '9')) return false; } return true; } internal static bool CouldBeInteger(ReadOnlySpan s) { - if (s.IsEmpty | s.Length > Format.MaxInt64TextLen) return false; + if (s.IsEmpty | (s.Length > Format.MaxInt64TextLen)) return false; bool isSigned = s[0] == '-'; for (int i = isSigned ? 1 : 0; i < s.Length; i++) { byte c = s[i]; - if (c < (byte)'0' | c > (byte)'9') return false; + if ((c < (byte)'0') | (c > (byte)'9')) return false; } return true; } @@ -281,7 +281,7 @@ internal static bool TryParseDouble(ReadOnlySpan s, out double value) case 4 when s[0] == '-' && TryParseInfNaN(s.Slice(1), false, out value): return true; } - return Utf8Parser.TryParse(s, out value, out int bytes) & bytes == s.Length; + return Utf8Parser.TryParse(s, out value, out int bytes) & (bytes == s.Length); static bool TryParseInfNaN(ReadOnlySpan s, bool positive, out double value) { @@ -623,6 +623,51 @@ internal static bool TryParseVersion(ReadOnlySpan input, [NotNullWhen(true #endif } + /// + /// Parses a UTF-8 version (7.4.3, say) directly from the wire. This must give exactly the same result as + /// the char overload for the same text (including 's -1 for unspecified parts, and + /// its tolerance of whitespace and signs), because callers used to go via that overload: net10.0 fast-paths + /// the built-in UTF-8 Version.TryParse; everything else transcodes and defers to the char overload + /// (which, on .NET Framework, allocates a string: that's fine). + /// + internal static bool TryParseVersion(ReadOnlySpan input, [NotNullWhen(true)] out Version? version) + { +#if NET10_0_OR_GREATER + if (Version.TryParse(input, out version)) return true; + // anything else (including our major-only extension) defers to the char overload, so the edge cases + // (whitespace, signs, etc) resolve identically +#endif + if (input.IsEmpty) + { + version = null; + return false; + } + char[]? lease = null; + int maxChars = Encoding.UTF8.GetMaxCharCount(input.Length); + Span chars = maxChars <= 64 ? stackalloc char[64] : (lease = ArrayPool.Shared.Rent(maxChars)); + try + { + int len; +#if NET + len = Encoding.UTF8.GetChars(input, chars); +#else + unsafe + { + fixed (byte* bPtr = &MemoryMarshal.GetReference(input)) + fixed (char* cPtr = &MemoryMarshal.GetReference(chars)) + { + len = Encoding.UTF8.GetChars(bPtr, input.Length, cPtr, chars.Length); + } + } +#endif + return TryParseVersion(chars.Slice(0, len), out version); + } + finally + { + if (lease is not null) ArrayPool.Shared.Return(lease); + } + } + internal static bool TryParseVersion(string? input, [NotNullWhen(true)] out Version? version) { if (input is not null) diff --git a/src/StackExchange.Redis/Groups/Arrays.Methods.cs b/src/StackExchange.Redis/Groups/Arrays.Methods.cs index 79971d680..56a5b8d2a 100644 --- a/src/StackExchange.Redis/Groups/Arrays.Methods.cs +++ b/src/StackExchange.Redis/Groups/Arrays.Methods.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// The arrays commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Bitmaps.Methods.cs b/src/StackExchange.Redis/Groups/Bitmaps.Methods.cs index f1dfe0708..432562b80 100644 --- a/src/StackExchange.Redis/Groups/Bitmaps.Methods.cs +++ b/src/StackExchange.Redis/Groups/Bitmaps.Methods.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// The bitmaps commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Config.Methods.cs b/src/StackExchange.Redis/Groups/Config.Methods.cs index 9f0cd01d9..03d43d978 100644 --- a/src/StackExchange.Redis/Groups/Config.Methods.cs +++ b/src/StackExchange.Redis/Groups/Config.Methods.cs @@ -6,9 +6,9 @@ namespace StackExchange.Redis; -/// +/// /// The CONFIG commands. -/// +/// /// public static partial class Config { diff --git a/src/StackExchange.Redis/Groups/Diagnostics.Methods.cs b/src/StackExchange.Redis/Groups/Diagnostics.Methods.cs index feef97ad7..cc496ed5f 100644 --- a/src/StackExchange.Redis/Groups/Diagnostics.Methods.cs +++ b/src/StackExchange.Redis/Groups/Diagnostics.Methods.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// The commands that ask a server about itself. -/// +/// /// public static partial class Diagnostics { @@ -616,7 +616,8 @@ LatencyHistoryEntry[] IRespHandler.Parse(ref RespReader r if (!reader.IsAggregate) throw new RespException("Unexpected LATENCY HISTORY reply."); return reader.ReadPastArray( static (ref RespReader r) => LatencyHistoryEntry.TryParseEntry(ref r, out var parsed) - ? parsed : throw new RespException("Unexpected LATENCY HISTORY element."), + ? parsed + : throw new RespException("Unexpected LATENCY HISTORY element."), scalar: false) ?? []; } @@ -625,7 +626,8 @@ LatencyLatestEntry[] IRespHandler.Parse(ref RespReader rea if (!reader.IsAggregate) throw new RespException("Unexpected LATENCY LATEST reply."); return reader.ReadPastArray( static (ref RespReader r) => LatencyLatestEntry.TryParseEntry(ref r, out var parsed) - ? parsed : throw new RespException("Unexpected LATENCY LATEST element."), + ? parsed + : throw new RespException("Unexpected LATENCY LATEST element."), scalar: false) ?? []; } } diff --git a/src/StackExchange.Redis/Groups/Geospatial.Methods.cs b/src/StackExchange.Redis/Groups/Geospatial.Methods.cs index a9bb3838e..8b57a559b 100644 --- a/src/StackExchange.Redis/Groups/Geospatial.Methods.cs +++ b/src/StackExchange.Redis/Groups/Geospatial.Methods.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// The geospatial commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Hashes.Methods.cs b/src/StackExchange.Redis/Groups/Hashes.Methods.cs index 6a9255fec..41dfedc95 100644 --- a/src/StackExchange.Redis/Groups/Hashes.Methods.cs +++ b/src/StackExchange.Redis/Groups/Hashes.Methods.cs @@ -11,9 +11,9 @@ namespace StackExchange.Redis; -/// +/// /// The hashes commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/HyperLogLog.Methods.cs b/src/StackExchange.Redis/Groups/HyperLogLog.Methods.cs index 6f2258df3..67694cc08 100644 --- a/src/StackExchange.Redis/Groups/HyperLogLog.Methods.cs +++ b/src/StackExchange.Redis/Groups/HyperLogLog.Methods.cs @@ -8,9 +8,9 @@ namespace StackExchange.Redis; -/// +/// /// The hyperloglog commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. @@ -85,9 +85,8 @@ private static CommandFlags CountFlags(RespContext context, in RedisKey key, Com // only a KNOWN version demotes this. The old surface has the same rule for the same reason - // it demotes only when a server was actually selected - so a context that cannot ask keeps the // caller's routing, and the worst case is the one the caller asked for. - return context.TryGetFeatures(RedisCommand.PFCOUNT, in key, flags, out var features) - && !features.HyperLogLogCountReplicaSafe - ? CommandFlagsInternal.DemandPrimary(flags, RedisCommand.PFCOUNT) - : flags; + var replicaUnsafe = context.TryGetFeatures(RedisCommand.PFCOUNT, in key, flags, out var features) + && !features.HyperLogLogCountReplicaSafe; + return replicaUnsafe ? CommandFlagsInternal.DemandPrimary(flags, RedisCommand.PFCOUNT) : flags; } } diff --git a/src/StackExchange.Redis/Groups/Keys.Methods.cs b/src/StackExchange.Redis/Groups/Keys.Methods.cs index 1c02668ef..db3c812d0 100644 --- a/src/StackExchange.Redis/Groups/Keys.Methods.cs +++ b/src/StackExchange.Redis/Groups/Keys.Methods.cs @@ -10,9 +10,9 @@ namespace StackExchange.Redis; -/// +/// /// The keys commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Keys.Sort.cs b/src/StackExchange.Redis/Groups/Keys.Sort.cs index fffc60696..5828e2b70 100644 --- a/src/StackExchange.Redis/Groups/Keys.Sort.cs +++ b/src/StackExchange.Redis/Groups/Keys.Sort.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// SORT and SORT_RO, which hang off the key group rather than one of their own. -/// +/// /// /// A group file without a group type, and the reason the layout is per group rather than /// per file: SORT takes a key and belongs to , so it lives in that partial. It diff --git a/src/StackExchange.Redis/Groups/Lists.Methods.cs b/src/StackExchange.Redis/Groups/Lists.Methods.cs index 6de56575d..01bb67261 100644 --- a/src/StackExchange.Redis/Groups/Lists.Methods.cs +++ b/src/StackExchange.Redis/Groups/Lists.Methods.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// The lists commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/PubSub.Methods.cs b/src/StackExchange.Redis/Groups/PubSub.Methods.cs index 59e1bc476..cbe51929e 100644 --- a/src/StackExchange.Redis/Groups/PubSub.Methods.cs +++ b/src/StackExchange.Redis/Groups/PubSub.Methods.cs @@ -7,9 +7,9 @@ namespace StackExchange.Redis; -/// +/// /// The publish/subscribe commands. -/// +/// /// public static partial class PubSub { diff --git a/src/StackExchange.Redis/Groups/Scripts.Methods.cs b/src/StackExchange.Redis/Groups/Scripts.Methods.cs index d8f88cbbd..3da80a420 100644 --- a/src/StackExchange.Redis/Groups/Scripts.Methods.cs +++ b/src/StackExchange.Redis/Groups/Scripts.Methods.cs @@ -11,9 +11,9 @@ namespace StackExchange.Redis; -/// +/// /// The scripts commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Sets.Methods.cs b/src/StackExchange.Redis/Groups/Sets.Methods.cs index 88dd09c4e..a10bf9c42 100644 --- a/src/StackExchange.Redis/Groups/Sets.Methods.cs +++ b/src/StackExchange.Redis/Groups/Sets.Methods.cs @@ -10,9 +10,9 @@ namespace StackExchange.Redis; -/// +/// /// The sets commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/SortedSets.Methods.cs b/src/StackExchange.Redis/Groups/SortedSets.Methods.cs index 4bf0bb061..6dd56d716 100644 --- a/src/StackExchange.Redis/Groups/SortedSets.Methods.cs +++ b/src/StackExchange.Redis/Groups/SortedSets.Methods.cs @@ -11,9 +11,9 @@ namespace StackExchange.Redis; -/// +/// /// The sortedsets commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Streams.Methods.cs b/src/StackExchange.Redis/Groups/Streams.Methods.cs index 8b4abe8c4..c725d8dbc 100644 --- a/src/StackExchange.Redis/Groups/Streams.Methods.cs +++ b/src/StackExchange.Redis/Groups/Streams.Methods.cs @@ -8,9 +8,9 @@ namespace StackExchange.Redis; -/// +/// /// The stream commands: target.Streams.LengthAsync(...). -/// +/// /// /// /// Here rather than in a single surface-wide class, so that a group is one place: its type, its @@ -207,11 +207,13 @@ StreamPendingMessageInfo[] IRespHandler.Parse(ref Re StreamAutoClaimResult IRespHandler.Parse(ref RespReader reader) => RespParsers.TryParseStreamAutoClaim(ref reader, allowJaggedFields: true, out var value) - ? value : throw Unexpected("XAUTOCLAIM", ref reader); + ? value + : throw Unexpected("XAUTOCLAIM", ref reader); StreamAutoClaimIdsOnlyResult IRespHandler.Parse(ref RespReader reader) => RespParsers.TryParseStreamAutoClaimIdsOnly(ref reader, out var value) - ? value : throw Unexpected("XAUTOCLAIM", ref reader); + ? value + : throw Unexpected("XAUTOCLAIM", ref reader); RedisStream[] IRespHandler.Parse(ref RespReader reader) => RespParsers.ParseRedisStreams(ref reader, reader.Prefix == RespPrefix.Map, allowJaggedFields: true); @@ -799,10 +801,11 @@ internal static ValueTask ReadGroupArray( /// repeat. Raised here because only here are the positions visible; v3 did the same in its message. /// private static CommandFlags ReadGroupFlags(CommandFlags flags, RedisValue? position, TimeSpan? claimMinIdleTime) - => claimMinIdleTime is null && position is { } explicitPosition - && StreamPosition.Resolve(explicitPosition, RedisCommand.XREADGROUP) != StreamConstants.UndeliveredMessages - ? flags.WithRetryCategory(CommandFlags.CommandRetryReadOnly) - : flags; + { + var readOnly = claimMinIdleTime is null && position is { } explicitPosition + && StreamPosition.Resolve(explicitPosition, RedisCommand.XREADGROUP) != StreamConstants.UndeliveredMessages; + return readOnly ? flags.WithRetryCategory(CommandFlags.CommandRetryReadOnly) : flags; + } /// /// For several streams, a single > anywhere consumes, so all of them must be explicit. diff --git a/src/StackExchange.Redis/Groups/Streams.Types.cs b/src/StackExchange.Redis/Groups/Streams.Types.cs index 9ff0d4e50..484cb1f1a 100644 --- a/src/StackExchange.Redis/Groups/Streams.Types.cs +++ b/src/StackExchange.Redis/Groups/Streams.Types.cs @@ -6,9 +6,9 @@ namespace StackExchange.Redis; -/// +/// /// The stream reply shapes, as windows over the reply buffer. -/// +/// /// /// /// Nested in the group, because these types are the group's. A stream entry means nothing outside @@ -355,7 +355,8 @@ public StreamAutoClaimResult ToStreamAutoClaimResult() var reader = GetReader(); reader.MoveNext(); return RespParsers.TryParseStreamAutoClaim(ref reader, allowJaggedFields: true, out var value) - ? value : StreamAutoClaimResult.Null; + ? value + : StreamAutoClaimResult.Null; } /// @@ -427,7 +428,8 @@ public StreamAutoClaimIdsOnlyResult ToStreamAutoClaimIdsOnlyResult() var reader = GetReader(); reader.MoveNext(); return RespParsers.TryParseStreamAutoClaimIdsOnly(ref reader, out var value) - ? value : StreamAutoClaimIdsOnlyResult.Null; + ? value + : StreamAutoClaimIdsOnlyResult.Null; } /// diff --git a/src/StackExchange.Redis/Groups/Strings.Methods.cs b/src/StackExchange.Redis/Groups/Strings.Methods.cs index 5d18255ec..a20fd062d 100644 --- a/src/StackExchange.Redis/Groups/Strings.Methods.cs +++ b/src/StackExchange.Redis/Groups/Strings.Methods.cs @@ -10,9 +10,9 @@ namespace StackExchange.Redis; -/// +/// /// The strings commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/VectorSets.Methods.cs b/src/StackExchange.Redis/Groups/VectorSets.Methods.cs index 8c7e367c5..502894ed9 100644 --- a/src/StackExchange.Redis/Groups/VectorSets.Methods.cs +++ b/src/StackExchange.Redis/Groups/VectorSets.Methods.cs @@ -10,9 +10,9 @@ namespace StackExchange.Redis; -/// +/// /// The vectorsets commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/HelloField.cs b/src/StackExchange.Redis/HelloField.cs index e5d730ce4..a7a5c37e8 100644 --- a/src/StackExchange.Redis/HelloField.cs +++ b/src/StackExchange.Redis/HelloField.cs @@ -1,4 +1,4 @@ -using System; +using System; using RESPite; namespace StackExchange.Redis; @@ -15,11 +15,48 @@ internal enum HelloField Unknown = 0, /// - /// Redis server version. + /// The server product: redis for Redis and for servers that present as Redis (Garnet); Valkey reports valkey. + /// + [AsciiHash("server")] + Server, + + /// + /// Server version. This is only the Redis API version when is redis; Valkey reports its own + /// version here (9.1.2, say, while INFO still gives redis_version:7.2.4). /// [AsciiHash("version")] Version, + /// + /// Garnet's own version; Garnet reports server as redis and as the Redis API version. + /// + [AsciiHash("garnet_version")] + GarnetVersion, + + /// + /// Valkey's own version, if reported alongside . + /// + [AsciiHash("valkey_version")] + ValkeyVersion, + + /// + /// Dragonfly's own version, if reported. + /// + [AsciiHash("dragonfly_version")] + DragonflyVersion, + + /// + /// Memurai's own version, if reported. + /// + [AsciiHash("memurai_version")] + MemuraiVersion, + + /// + /// Redict's own version, if reported. + /// + [AsciiHash("redict_version")] + RedictVersion, + /// /// Protocol version (2 or 3). /// diff --git a/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs b/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs index 28f45851f..66bebc55b 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs @@ -4,9 +4,9 @@ // ReSharper disable once CheckNamespace namespace StackExchange.Redis; -/// -/// Describes functionality that is common to both standalone redis servers and redis clusters. -/// +/// +/// Vector set operations. +/// public partial interface IDatabase { // Vector Set operations diff --git a/src/StackExchange.Redis/Interfaces/IDatabase.cs b/src/StackExchange.Redis/Interfaces/IDatabase.cs index a252bf972..d29b809cd 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabase.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabase.cs @@ -3782,7 +3782,8 @@ IEnumerable SortedSetScan( /// The expiry to set. /// The condition to enforce. /// The flags to use for this operation. - /// See . + /// if the string was set, otherwise. + /// See . #pragma warning disable RS0027 // Public API with optional parameter(s) should have the most parameters amongst its public overloads bool StringSet(RedisKey key, RedisValue value, Expiration expiry = default, ValueCondition when = default, CommandFlags flags = CommandFlags.None); #pragma warning restore RS0027 diff --git a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs index 64e8e3878..ffb04f8eb 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs @@ -5,9 +5,9 @@ // ReSharper disable once CheckNamespace namespace StackExchange.Redis; -/// -/// Describes functionality that is common to both standalone redis servers and redis clusters. -/// +/// +/// Vector set operations. +/// public partial interface IDatabaseAsync { // Vector Set operations diff --git a/src/StackExchange.Redis/KeyNotification.cs b/src/StackExchange.Redis/KeyNotification.cs index 5550fe6e0..76bf87766 100644 --- a/src/StackExchange.Redis/KeyNotification.cs +++ b/src/StackExchange.Redis/KeyNotification.cs @@ -254,8 +254,9 @@ public int Database if (end <= 0) return -1; span = span.Slice(0, end); - return Utf8Parser.TryParse(span, out int database, out var bytes) - && bytes == end ? database : -1; + return Utf8Parser.TryParse(span, out int database, out var bytes) && bytes == end + ? database + : -1; } } @@ -276,7 +277,7 @@ public RedisKey GetKey() case KeyNotificationKind.KeyEvent: // Channel contains the event-type, payload contains the key byte[]? blob = _value; - if (_keyOffset != 0 & blob is not null) + if ((_keyOffset != 0) & (blob is not null)) { return blob.AsSpan(_keyOffset).ToArray(); } diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs index 380cac26f..3daa10b6d 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs @@ -587,8 +587,8 @@ public Task SortedSetAddAsync(RedisKey key, SortedSetEntry[] values, Comma public Task SortedSetAddAsync(RedisKey key, SortedSetEntry[] values, When when = When.Always, CommandFlags flags = CommandFlags.None) => Inner.SortedSetAddAsync(ToInner(key), values, when, flags); - public Task SortedSetAddAsync(RedisKey key, SortedSetEntry[] values, SortedSetWhen updateWhen = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => - Inner.SortedSetAddAsync(ToInner(key), values, updateWhen, flags); + public Task SortedSetAddAsync(RedisKey key, SortedSetEntry[] values, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => + Inner.SortedSetAddAsync(ToInner(key), values, when, flags); public Task SortedSetAddAsync(RedisKey key, RedisValue member, double score, CommandFlags flags) => Inner.SortedSetAddAsync(ToInner(key), member, score, flags); @@ -596,8 +596,8 @@ public Task SortedSetAddAsync(RedisKey key, RedisValue member, double scor public Task SortedSetAddAsync(RedisKey key, RedisValue member, double score, When when = When.Always, CommandFlags flags = CommandFlags.None) => Inner.SortedSetAddAsync(ToInner(key), member, score, when, flags); - public Task SortedSetAddAsync(RedisKey key, RedisValue member, double score, SortedSetWhen updateWhen = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => - Inner.SortedSetAddAsync(ToInner(key), member, score, updateWhen, flags); + public Task SortedSetAddAsync(RedisKey key, RedisValue member, double score, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => + Inner.SortedSetAddAsync(ToInner(key), member, score, when, flags); public Task SortedSetCombineAsync(SetOperation operation, RedisKey[] keys, double[]? weights = null, Aggregate aggregate = Aggregate.Sum, CommandFlags flags = CommandFlags.None) => Inner.SortedSetCombineAsync(operation, ToInner(keys), weights, aggregate, flags); @@ -695,11 +695,11 @@ public Task SortedSetRemoveRangeByValueAsync(RedisKey key, RedisValue min, public IAsyncEnumerable SortedSetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => Inner.SortedSetScanAsync(ToInner(key), pattern, pageSize, cursor, pageOffset, flags); - public Task SortedSetUpdateAsync(RedisKey key, SortedSetEntry[] values, SortedSetWhen updateWhen = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => - Inner.SortedSetUpdateAsync(ToInner(key), values, updateWhen, flags); + public Task SortedSetUpdateAsync(RedisKey key, SortedSetEntry[] values, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => + Inner.SortedSetUpdateAsync(ToInner(key), values, when, flags); - public Task SortedSetUpdateAsync(RedisKey key, RedisValue member, double score, SortedSetWhen updateWhen = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => - Inner.SortedSetUpdateAsync(ToInner(key), member, score, updateWhen, flags); + public Task SortedSetUpdateAsync(RedisKey key, RedisValue member, double score, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => + Inner.SortedSetUpdateAsync(ToInner(key), member, score, when, flags); public Task SortedSetPopAsync(RedisKey key, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => Inner.SortedSetPopAsync(ToInner(key), order, flags); @@ -734,17 +734,17 @@ public Task StreamAddAsync(RedisKey key, RedisValue streamField, Red public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId, int? maxLength, bool useApproximateMaxLength, CommandFlags flags) => Inner.StreamAddAsync(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, flags); - public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAddAsync(ToInner(key), streamField, streamValue, messageId, maxLength, useApproximateMaxLength, limit, mode, flags); + public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAddAsync(ToInner(key), streamField, streamValue, messageId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAddAsync(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, limit, mode, flags); + public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAddAsync(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAddAsync(ToInner(key), streamField, streamValue, idempotentId, maxLength, useApproximateMaxLength, limit, mode, flags); + public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAddAsync(ToInner(key), streamField, streamValue, idempotentId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAddAsync(ToInner(key), streamPairs, idempotentId, maxLength, useApproximateMaxLength, limit, mode, flags); + public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAddAsync(ToInner(key), streamPairs, idempotentId, maxLength, useApproximateMaxLength, limit, trimMode, flags); public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamAddOptions options, CommandFlags flags = CommandFlags.None) => Inner.StreamAddAsync(ToInner(key), streamField, streamValue, options, flags); diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs index 3353c0d6c..8ca3bca48 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs @@ -583,8 +583,8 @@ public RedisValue[] SortedSetRangeByRank(RedisKey key, long start = 0, long stop Inner.SortedSetRangeByRank(ToInner(key), start, stop, order, flags); public long SortedSetRangeAndStore( - RedisKey destinationKey, RedisKey sourceKey, + RedisKey destinationKey, RedisValue start, RedisValue stop, SortedSetOrder sortedSetOrder = SortedSetOrder.ByRank, @@ -673,17 +673,17 @@ public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue str public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId, int? maxLength, bool useApproximateMaxLength, CommandFlags flags) => Inner.StreamAdd(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, flags); - public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAdd(ToInner(key), streamField, streamValue, messageId, maxLength, useApproximateMaxLength, limit, mode, flags); + public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAdd(ToInner(key), streamField, streamValue, messageId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAdd(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, limit, mode, flags); + public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAdd(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAdd(ToInner(key), streamField, streamValue, idempotentId, maxLength, useApproximateMaxLength, limit, mode, flags); + public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAdd(ToInner(key), streamField, streamValue, idempotentId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAdd(ToInner(key), streamPairs, idempotentId, maxLength, useApproximateMaxLength, limit, mode, flags); + public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAdd(ToInner(key), streamPairs, idempotentId, maxLength, useApproximateMaxLength, limit, trimMode, flags); public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamAddOptions options, CommandFlags flags = CommandFlags.None) => Inner.StreamAdd(ToInner(key), streamField, streamValue, options, flags); diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index 1d2f8f4e3..cc999dd66 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -280,6 +280,12 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) Message = "Cluster: {CoveredSlots} of {TotalSlots} slots covered")] internal static partial void LogInformationClusterSlotsCovered(this ILogger logger, long coveredSlots, int totalSlots); + [LoggerMessage( + Level = LogLevel.Information, + EventId = 133, + Message = "{ClusterCount} server(s) identify as a cluster but no slot map is available; routing as standalone")] + internal static partial void LogInformationClusterWithoutSlotMap(this ILogger logger, int clusterCount); + [LoggerMessage( Level = LogLevel.Information, EventId = 35, @@ -328,6 +334,18 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) Message = "Encountered error while updating cluster config: {ErrorMessage}")] internal static partial void LogErrorEncounteredErrorWhileUpdatingClusterConfig(this ILogger logger, Exception exception, string errorMessage); + [LoggerMessage( + Level = LogLevel.Information, + EventId = 128, + Message = "{Server}: CLUSTER SLOTS was refused ({ErrorMessage}); continuing with CLUSTER NODES alone")] + internal static partial void LogInformationClusterSlotsRefused(this ILogger logger, ServerEndPointLogValue server, string errorMessage); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 129, + Message = "{Server}: CLUSTER NODES was refused ({ErrorMessage}); no cluster topology is available")] + internal static partial void LogInformationClusterNodesRefused(this ILogger logger, ServerEndPointLogValue server, string errorMessage); + [LoggerMessage( Level = LogLevel.Information, EventId = 111, @@ -630,6 +648,18 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) Message = "{Server}: Auto-configured (HELLO) role: {Role}")] internal static partial void LogInformationAutoConfiguredHelloRole(this ILogger logger, ServerEndPointLogValue server, string role); + [LoggerMessage( + Level = LogLevel.Information, + EventId = 131, + Message = "{Server}: Auto-configured (HELLO) product: {Variant} {Version}")] + internal static partial void LogInformationAutoConfiguredHelloProductVariant(this ILogger logger, ServerEndPointLogValue server, ProductVariant variant, string version); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 132, + Message = "{Server}: Auto-configured (HELLO) unrecognized server '{ServerName}'; ignoring its version")] + internal static partial void LogInformationAutoConfiguredHelloUnknownServer(this ILogger logger, ServerEndPointLogValue server, string serverName); + // PhysicalBridge logging methods [LoggerMessage( Level = LogLevel.Information, @@ -830,19 +860,21 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) [LoggerMessage( Level = LogLevel.Information, - EventId = 128, + // 134-137 rather than 128-131: main took those while this core was being built (merged 2026-10-23), and + // event ids are the one thing a merge cannot renumber for us. Main's ids are kept exactly as main has them. + EventId = 134, Message = "Refreshed the slot map from {EndPoint}: {RangeCount} range(s)")] internal static partial void LogInformationSlotMapRefreshed(this ILogger logger, EndPointLogValue endPoint, int rangeCount); [LoggerMessage( Level = LogLevel.Information, - EventId = 129, + EventId = 135, Message = "{EndPoint} could not refresh the slot map: {ErrorMessage}")] internal static partial void LogInformationSlotMapRefreshFailed(this ILogger logger, Exception exception, EndPointLogValue endPoint, string errorMessage); [LoggerMessage( Level = LogLevel.Information, - EventId = 130, + EventId = 136, Message = "{EndPoint} could not refresh its role: {ErrorMessage}")] internal static partial void LogInformationRoleRefreshFailed(this ILogger logger, Exception exception, EndPointLogValue endPoint, string errorMessage); @@ -850,7 +882,13 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) // case that is not - a subscription the server is delivering to that the registry cannot match. [LoggerMessage( Level = LogLevel.Debug, - EventId = 131, + EventId = 137, Message = "A pub/sub delivery for {Channel} (subscription {Subscription}) matched no subscription and was dropped")] internal static partial void LogDebugDeliveryUnmatched(this ILogger logger, string? channel, string? subscription); + + [LoggerMessage( + Level = LogLevel.Warning, + EventId = 130, + Message = "Lag-aware health check could not ask {RestEndpoint}: {Reason}")] + internal static partial void LogWarningLagAwareInconclusive(this ILogger logger, string restEndpoint, string reason); } diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs index e7bb2044e..84c69d943 100644 --- a/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs +++ b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs @@ -4,10 +4,10 @@ namespace StackExchange.Redis; -/// +/// /// Carries maintenance context on the faults a disruption can cause, so that a timeout during an announced /// migration is distinguishable from an ordinary one. -/// +/// /// /// This follows the established pattern on these types - Commandstatus and Flags on /// , FailureType on - rather diff --git a/src/StackExchange.Redis/Profiling/ProfiledCommand.cs b/src/StackExchange.Redis/Profiling/ProfiledCommand.cs index b546b382d..738a8cb49 100644 --- a/src/StackExchange.Redis/Profiling/ProfiledCommand.cs +++ b/src/StackExchange.Redis/Profiling/ProfiledCommand.cs @@ -131,11 +131,16 @@ public void SetCompleted() } } - public override string ToString() => -$@"{Command} (DB: {Db}, Flags: {Flags}) + public override string ToString() + { + var retransmission = RetransmissionOf != null + ? @" + RetransmissionOf = " + RetransmissionOf + : ""; + return $@"{Command} (DB: {Db}, Flags: {Flags}) EndPoint = {EndPoint} ({ConnectionType}) Created = {CommandCreated:HH:mm:ss.ffff} - ElapsedTime = {ElapsedTime.TotalMilliseconds} ms (CreationToEnqueued: {CreationToEnqueued.TotalMilliseconds} ms, EnqueuedToSending: {EnqueuedToSending.TotalMilliseconds} ms, SentToResponse: {SentToResponse.TotalMilliseconds} ms, ResponseToCompletion = {ResponseToCompletion.TotalMilliseconds} ms){(RetransmissionOf != null ? @" - RetransmissionOf = " + RetransmissionOf : "")}"; + ElapsedTime = {ElapsedTime.TotalMilliseconds} ms (CreationToEnqueued: {CreationToEnqueued.TotalMilliseconds} ms, EnqueuedToSending: {EnqueuedToSending.TotalMilliseconds} ms, SentToResponse: {SentToResponse.TotalMilliseconds} ms, ResponseToCompletion = {ResponseToCompletion.TotalMilliseconds} ms){retransmission}"; + } } } diff --git a/src/StackExchange.Redis/Protocol/RespWire.cs b/src/StackExchange.Redis/Protocol/RespWire.cs index ed21aeadd..5053b38d4 100644 --- a/src/StackExchange.Redis/Protocol/RespWire.cs +++ b/src/StackExchange.Redis/Protocol/RespWire.cs @@ -153,7 +153,7 @@ internal static void WriteMultiBulkHeader(IBufferWriter writer, long count internal static void WriteMultiBulkHeader(IBufferWriter writer, long count, RespPrefix prefix) { - if ((prefix is RespPrefix.Map or RespPrefix.Attribute) & count > 0) + if ((prefix is RespPrefix.Map or RespPrefix.Attribute) & (count > 0)) { if ((count & 1) != 0) Throw(prefix, count); count >>= 1; diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 972745ffa..775b20796 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -807,3 +807,41 @@ static StackExchange.Redis.DatabaseTaskExtensions.AsTask(this StackExchange.Redi static StackExchange.Redis.DatabaseTaskExtensions.AsTask(this StackExchange.Redis.IDatabaseAsync! database, System.Threading.Tasks.ValueTask pending, StackExchange.Redis.CommandFlags flags) -> System.Threading.Tasks.Task! static StackExchange.Redis.DatabaseTaskExtensions.AsTask(this StackExchange.Redis.IDatabaseAsync! database, System.Threading.Tasks.ValueTask pending) -> System.Threading.Tasks.Task! static StackExchange.Redis.DatabaseTaskExtensions.AsTask(this StackExchange.Redis.IDatabaseAsync! database, System.Threading.Tasks.ValueTask pending, StackExchange.Redis.CommandFlags flags) -> System.Threading.Tasks.Task! +StackExchange.Redis.Availability.ConnectionGroupMember.FailbackHealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? +StackExchange.Redis.Availability.ConnectionGroupMember.FailbackHealthCheck.set -> void +StackExchange.Redis.Availability.HealthCheckProbeScope +StackExchange.Redis.Availability.HealthCheckProbeScope.Endpoint = 0 -> StackExchange.Redis.Availability.HealthCheckProbeScope +StackExchange.Redis.Availability.HealthCheckProbeScope.Member = 1 -> StackExchange.Redis.Availability.HealthCheckProbeScope +StackExchange.Redis.Availability.MultiGroupOptions.Builder.FailbackHealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? +StackExchange.Redis.Availability.MultiGroupOptions.Builder.FailbackHealthCheck.set -> void +StackExchange.Redis.Availability.MultiGroupOptions.FailbackHealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? +virtual StackExchange.Redis.Availability.HealthCheckProbe.Scope.get -> StackExchange.Redis.Availability.HealthCheckProbeScope +StackExchange.Redis.Availability.LagAwareOptions +StackExchange.Redis.Availability.LagAwareOptions.CertificateValidation.get -> System.Net.Security.RemoteCertificateValidationCallback? +StackExchange.Redis.Availability.LagAwareOptions.CertificateValidation.set -> void +StackExchange.Redis.Availability.LagAwareOptions.Credentials.get -> System.Func>? +StackExchange.Redis.Availability.LagAwareOptions.Credentials.set -> void +StackExchange.Redis.Availability.LagAwareOptions.DatabaseId.get -> int? +StackExchange.Redis.Availability.LagAwareOptions.DatabaseId.set -> void +StackExchange.Redis.Availability.LagAwareOptions.LagAwareOptions() -> void +StackExchange.Redis.Availability.LagAwareOptions.LagCheck.get -> StackExchange.Redis.Availability.LagCheckMode +StackExchange.Redis.Availability.LagAwareOptions.LagCheck.set -> void +StackExchange.Redis.Availability.LagAwareOptions.LagTolerance.get -> System.TimeSpan? +StackExchange.Redis.Availability.LagAwareOptions.LagTolerance.set -> void +StackExchange.Redis.Availability.LagAwareOptions.RestEndpoint.get -> System.Uri? +StackExchange.Redis.Availability.LagAwareOptions.RestEndpoint.set -> void +StackExchange.Redis.Availability.LagAwareOptions.TrustIssuer(string! issuerCertificatePath) -> void +StackExchange.Redis.Availability.LagAwareOptions.TrustIssuer(System.Security.Cryptography.X509Certificates.X509Certificate2! issuer) -> void +StackExchange.Redis.Availability.LagCheckMode +StackExchange.Redis.Availability.LagCheckMode.Auto = 0 -> StackExchange.Redis.Availability.LagCheckMode +StackExchange.Redis.Availability.LagCheckMode.Disabled = 2 -> StackExchange.Redis.Availability.LagCheckMode +StackExchange.Redis.Availability.LagCheckMode.Enabled = 1 -> StackExchange.Redis.Availability.LagCheckMode +static StackExchange.Redis.Availability.HealthCheck.LagAware(StackExchange.Redis.Availability.LagAwareOptions! options) -> StackExchange.Redis.Availability.HealthCheck! +static StackExchange.Redis.Availability.HealthCheckProbe.LagAware(StackExchange.Redis.Availability.LagAwareOptions! options) -> StackExchange.Redis.Availability.HealthCheckProbe! +StackExchange.Redis.Availability.LagAwareOptions.ClientCertificate.get -> System.Security.Cryptography.X509Certificates.X509Certificate2? +StackExchange.Redis.Availability.LagAwareOptions.ClientCertificate.set -> void +StackExchange.Redis.Availability.LagAwareOptions.SetUserPfxCertificate(string! userCertificatePath, string? password = null) -> void +*REMOVED*override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.AfterConnectAsync(StackExchange.Redis.ConnectionMultiplexer! muxer, System.Action! log) -> System.Threading.Tasks.Task! +override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.AfterConnectAsync(StackExchange.Redis.ConnectionMultiplexer! multiplexer, System.Action! log) -> System.Threading.Tasks.Task! +*REMOVED*override StackExchange.Redis.Configuration.AzureOptionsProvider.AfterConnectAsync(StackExchange.Redis.ConnectionMultiplexer! muxer, System.Action! log) -> System.Threading.Tasks.Task! +override StackExchange.Redis.Configuration.AzureOptionsProvider.AfterConnectAsync(StackExchange.Redis.ConnectionMultiplexer! multiplexer, System.Action! log) -> System.Threading.Tasks.Task! diff --git a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt index ab058de62..04376438f 100644 --- a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt @@ -1 +1,2 @@ #nullable enable +StackExchange.Redis.Availability.LagAwareOptions.SetUserPemCertificate(string! userCertificatePath, string? userKeyPath = null) -> void diff --git a/src/StackExchange.Redis/RedisBase.cs b/src/StackExchange.Redis/RedisBase.cs index 73361bbd7..14b588c0a 100644 --- a/src/StackExchange.Redis/RedisBase.cs +++ b/src/StackExchange.Redis/RedisBase.cs @@ -75,6 +75,14 @@ internal RedisBase(ConnectionMultiplexer multiplexer, object? asyncState) internal virtual RedisFeatures GetFeatures(in RedisKey key, CommandFlags flags, RedisCommand command, out ServerEndPoint? server) { + // the server picked here is the one the command is sent to, so a primary-only command has to pick the primary + // here as well, the same way the send path does (CommandFlagsInternal.DemandPrimary); otherwise PreferReplica + // picks a replica that then rejects it (DemandReplica is left alone so that it still fails the same way). + // Main #3291, ported: the rule lives in CommandFlagsInternal here, Message having gone with the old core. + if (command.IsPrimaryOnly() && CommandFlagsInternal.GetPrimaryReplicaFlags(flags) != CommandFlags.DemandReplica) + { + flags = CommandFlagsInternal.SetPrimaryReplicaFlags(flags, CommandFlags.DemandMaster); + } server = multiplexer.SelectServer(command, flags, key); var version = server == null ? multiplexer.RawConfig.DefaultVersion : server.Version; return new RedisFeatures(version); diff --git a/src/StackExchange.Redis/RedisChannel.cs b/src/StackExchange.Redis/RedisChannel.cs index 819178428..8287b5873 100644 --- a/src/StackExchange.Redis/RedisChannel.cs +++ b/src/StackExchange.Redis/RedisChannel.cs @@ -478,7 +478,7 @@ private static Span AppendAndAdvance(Span target, scoped ReadOnlySpa private static RedisChannel BuildKeySpaceChannel(in RedisKey key, int? database, RedisChannelOptions options, ReadOnlySpan suffix, bool appendStar, bool allowKeyPatterns, bool subkey) { int fullKeyLength = key.TotalLength() + suffix.Length + (appendStar ? 1 : 0); - if (appendStar & (options & RedisChannelOptions.Pattern) == 0) throw new ArgumentNullException(nameof(key)); + if (appendStar & ((options & RedisChannelOptions.Pattern) == 0)) throw new ArgumentNullException(nameof(key)); if (fullKeyLength == 0) throw new ArgumentOutOfRangeException(nameof(key)); var db = AppendDatabase(stackalloc byte[DatabaseScratchBufferSize], database, options); diff --git a/src/StackExchange.Redis/RedisErrorKind.cs b/src/StackExchange.Redis/RedisErrorKind.cs index e4403778b..ccee997f0 100644 --- a/src/StackExchange.Redis/RedisErrorKind.cs +++ b/src/StackExchange.Redis/RedisErrorKind.cs @@ -201,7 +201,7 @@ internal static bool TryParse(ReadOnlySpan value, out RedisErrorKind kind) if (TryParseFirstTokenCI(firstToken, out kind)) { // check for more specific "ERR ..." scenarios - if (kind is RedisErrorKind.UnknownError & space > 0) + if ((kind is RedisErrorKind.UnknownError) & (space > 0)) { // get the message text after "ERR " value = value.Slice(space + 1); diff --git a/src/StackExchange.Redis/RedisKey.cs b/src/StackExchange.Redis/RedisKey.cs index 85f9ced59..31abd7c9a 100644 --- a/src/StackExchange.Redis/RedisKey.cs +++ b/src/StackExchange.Redis/RedisKey.cs @@ -342,9 +342,11 @@ internal static RedisKey WithPrefix(byte[]? prefix, RedisKey value) } int aLen = a?.Length ?? 0, - bLen = b == null ? 0 : (b is string bString - ? Encoding.UTF8.GetByteCount(bString) - : ((byte[])b).Length), + bLen = b == null + ? 0 + : b is string bString + ? Encoding.UTF8.GetByteCount(bString) + : ((byte[])b).Length, cLen = c?.Length ?? 0; var result = new byte[aLen + bLen + cLen]; diff --git a/src/StackExchange.Redis/RedisResult.cs b/src/StackExchange.Redis/RedisResult.cs index a20dedf4c..3595a1c89 100644 --- a/src/StackExchange.Redis/RedisResult.cs +++ b/src/StackExchange.Redis/RedisResult.cs @@ -44,8 +44,9 @@ public static RedisResult Create(RedisValue[] values) /// The explicit data type. /// new . public static RedisResult Create(RedisValue[] values, ResultType resultType) => - values == null ? NullArray : values.Length == 0 ? EmptyArray(resultType) : - new ArrayRedisResult(Array.ConvertAll(values, value => new SingleRedisResult(value, null)), resultType); + values == null ? NullArray : + values.Length == 0 ? EmptyArray(resultType) : + new ArrayRedisResult(Array.ConvertAll(values, value => new SingleRedisResult(value, null)), resultType); /// /// Create a new RedisResult representing an array of values. @@ -426,10 +427,11 @@ internal override bool AsBoolean() } internal override byte[][]? AsByteArrayArray() - => IsNull ? null - : _value!.Length == 0 - ? Array.Empty() - : Array.ConvertAll(_value, x => x.AsByteArray()!); + => IsNull + ? null + : _value!.Length == 0 + ? Array.Empty() + : Array.ConvertAll(_value, x => x.AsByteArray()!); private bool IsSingleton => _value?.Length == 1; private bool IsEmpty => _value?.Length == 0; @@ -440,9 +442,11 @@ internal override double AsDouble() } internal override double[]? AsDoubleArray() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsDouble()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsDouble()); internal override int AsInt32() { @@ -451,9 +455,11 @@ internal override int AsInt32() } internal override int[]? AsInt32Array() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsInt32()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsInt32()); internal override long AsInt64() { @@ -467,14 +473,18 @@ internal override ulong AsUInt64() } internal override long[]? AsInt64Array() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsInt64()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsInt64()); internal override ulong[]? AsUInt64Array() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsUInt64()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsUInt64()); internal override bool? AsNullableBoolean() { @@ -512,9 +522,11 @@ internal override RedisKey AsRedisKey() } internal override RedisKey[]? AsRedisKeyArray() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsRedisKey()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsRedisKey()); internal override RedisResult[]? AsRedisResultArray() => _value; @@ -525,9 +537,11 @@ internal override RedisValue AsRedisValue() } internal override RedisValue[]? AsRedisValueArray() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsRedisValue()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsRedisValue()); internal override string? AsString() { @@ -536,9 +550,11 @@ internal override RedisValue AsRedisValue() } internal override string?[]? AsStringArray() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsString()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsString()); } /// diff --git a/src/StackExchange.Redis/RedisSubscriber.cs b/src/StackExchange.Redis/RedisSubscriber.cs index f53bd77d6..130b6100c 100644 --- a/src/StackExchange.Redis/RedisSubscriber.cs +++ b/src/StackExchange.Redis/RedisSubscriber.cs @@ -348,7 +348,7 @@ private Task PingConnectionAsync(CommandFlags flags) // cores, handing back null sent the ping down the v3 path, which built a bridge (and dialled // it) just to answer. The round trip that remains meaningful is the one on the ordinary // connection to that server. - var direct = new RespContext(multiplexer.RawConfig.CommandMap, database: -1) + var direct = new RespContext(multiplexer.CommandMap, database: -1) .WithExecutor(multiplexer.Connections.ServerExecutor(server.EndPoint)); return Measured(new RespDatabaseContext(direct).PingMeasureAsync(flags)); } @@ -574,7 +574,7 @@ private bool UnregisterSubscription(in RedisChannel channel, Action value) /// Creates a new from an . /// /// The to convert to a . - public static implicit operator RedisValue(bool? value) => value == null ? Null + public static implicit operator RedisValue(bool? value) => value == null + ? Null : new(value.GetValueOrDefault() ? 1 : 0); /// diff --git a/src/StackExchange.Redis/RenderedArgs.cs b/src/StackExchange.Redis/RenderedArgs.cs index dfc71ec15..22a128627 100644 --- a/src/StackExchange.Redis/RenderedArgs.cs +++ b/src/StackExchange.Redis/RenderedArgs.cs @@ -124,7 +124,8 @@ public static RenderedArgs Create(ReadOnlySpan keys, ReadOnlySpan? pool) => new RenderedArgs { - _buffer = length == 0 ? Array.Empty() + _buffer = length == 0 + ? Array.Empty() : pool is null ? ArrayPool.Shared.Rent(length) : pool.Rent(length), _count = count, _length = length, diff --git a/src/StackExchange.Redis/RespReaderExtensions.cs b/src/StackExchange.Redis/RespReaderExtensions.cs index d3a525848..33e6f02af 100644 --- a/src/StackExchange.Redis/RespReaderExtensions.cs +++ b/src/StackExchange.Redis/RespReaderExtensions.cs @@ -126,8 +126,7 @@ public static RedisValue ReadRedisValue(this in RespReader reader) // discontiguous payload. if (length <= RedisValue.MaxInlineBytes) { - return RedisValue.FromRaw(reader.TryGetSpan(out var buffer) ? - buffer : reader.Buffer(stackalloc byte[RedisValue.MaxInlineBytes])); + return RedisValue.FromRaw(reader.TryGetSpan(out var buffer) ? buffer : reader.Buffer(stackalloc byte[RedisValue.MaxInlineBytes])); } // Longer payloads: prefer a compact numeric storage kind when the text is the *canonical* @@ -228,7 +227,7 @@ static bool Failure(out RedisValue value) } // integer: canonical exactly when the round-trip text length matches (rules out leading zeros, a // leading '+', "-0", trailing junk, etc.) - so no need to re-emit and compare bytes - if (span.IsEmpty | span.Length > MaxCanonicalLength) return Failure(out value); + if (span.IsEmpty | (span.Length > MaxCanonicalLength)) return Failure(out value); // restrict to *just* basic number tokens, tracking the two facts that let us pick a single parse: // whether a '-' appeared (so we know whether the integer is signed), and whether a '.'/'e'/'E' diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 104ad14b8..4dc563da5 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -104,8 +104,8 @@ public ServerEndPoint(ConnectionMultiplexer multiplexer, EndPoint endpoint, Serv private RedisServer? _defaultServer; public RedisServer GetRedisServer(object? asyncState) => asyncState is null - ? (_defaultServer ??= new RedisServer(this, null)) // reuse and memoize - : new RedisServer(this, asyncState); + ? (_defaultServer ??= new RedisServer(this, null)) // reuse and memoize + : new RedisServer(this, asyncState); public EndPoint EndPoint { get; } @@ -219,7 +219,11 @@ public ServerType ServerType public bool IsReplica { - get => isReplica; + get + { + return isReplica; + } + set { var changed = isReplica != value; @@ -241,7 +245,10 @@ public bool ReplicaReadOnly public bool AllowReplicaWrites { - get => allowReplicaWrites; + get + { + return allowReplicaWrites; + } set { allowReplicaWrites = value; @@ -607,7 +614,10 @@ internal void FlushScriptCache() private string? runId; internal string? RunId { - get => runId; + get + { + return runId; + } set { // We only care about changes @@ -806,6 +816,31 @@ public EndPoint? PrimaryEndPoint [ThreadStatic] private static Random? r; + private int lastRoleCheckTicks; + + /// Seconds since this server's role was last read, by the handshake or by the heartbeat. + internal int LastRoleCheckSecondsAgo => unchecked(Environment.TickCount - Volatile.Read(ref lastRoleCheckTicks)) / 1000; + + /// + /// Whether the heartbeat should re-read this server's role now: have passed + /// since the last read. Zero disables the check, as it did for the shipped core's INFO replication. + /// + internal bool RoleCheckDue => ConfigCheckSeconds > 0 && LastRoleCheckSecondsAgo >= ConfigCheckSeconds; + + /// + /// Note that the role was just read, and ease the pacing back towards the configured interval. + /// + /// + /// What CheckInfoReplication recorded in the shipped core before sending its INFO replication; + /// here the read itself is RespConnectionManager.TryRefreshRole, driven from the core's heartbeat, + /// and the handshake counts as a read too, since it learns the role as it connects. + /// + internal void OnRoleChecked() + { + Volatile.Write(ref lastRoleCheckTicks, Environment.TickCount); + ResetExponentiallyReplicationCheck(); + } + /// /// Forces frequent replication check starting from 1 second up to max ConfigCheckSeconds with an exponential increment. /// diff --git a/src/StackExchange.Redis/ServerSelectionStrategy.cs b/src/StackExchange.Redis/ServerSelectionStrategy.cs index 0fb17bac7..9d6bad26b 100644 --- a/src/StackExchange.Redis/ServerSelectionStrategy.cs +++ b/src/StackExchange.Redis/ServerSelectionStrategy.cs @@ -113,7 +113,8 @@ public int HashSlot(in RedisChannel channel) ReadOnlySpan routingSpan = channel.RoutingSpan; byte[] prefix; return channel.IgnoreChannelPrefix || (prefix = ChannelPrefix).Length == 0 - ? GetClusterSlot(routingSpan) : GetClusterSlotWithPrefix(prefix, routingSpan); + ? GetClusterSlot(routingSpan) + : GetClusterSlotWithPrefix(prefix, routingSpan); static int GetClusterSlotWithPrefix(byte[] prefixRaw, ReadOnlySpan routingSpan) { @@ -219,6 +220,18 @@ internal bool OwnsAnySlot(ServerEndPoint server) return false; } + /// + /// Whether no slot has an owner, i.e. we hold no cluster topology to route by. + /// + /// + /// Derived from the map rather than tracked separately, which is sound because the map is allocated only by + /// , immediately before a write; the writers never store null and refuse + /// empty ranges (see ); and nothing ever clears an entry - + /// retirement declines to drop a server the map still references (). So a non-null + /// map always covers at least one slot. O(1); is the scan, for logging. + /// + internal bool IsSlotMapEmpty => map is null; + internal int CountCoveredSlots() { var arr = map; @@ -231,6 +244,11 @@ internal int CountCoveredSlots() internal void UpdateClusterRange(int fromInclusive, int toInclusive, ServerEndPoint server) { + // never allocate without writing (see IsSlotMapEmpty): SlotRange does not validate ordering, and the + // reply-driven paths build ranges straight from what the server sent - an inverted or out-of-range + // range is garbage, so ignore it rather than guess at what was meant (or index out of bounds) + if (fromInclusive < 0 || toInclusive >= RedisClusterSlotCount || fromInclusive > toInclusive) return; + var arr = MapForMutation(); for (int i = fromInclusive; i <= toInclusive; i++) { diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index c1e511b1b..4be6b05fa 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -64,6 +64,10 @@ + + + + diff --git a/src/StackExchange.Redis/Subscription.cs b/src/StackExchange.Redis/Subscription.cs index 94f72d638..6d9c91a95 100644 --- a/src/StackExchange.Redis/Subscription.cs +++ b/src/StackExchange.Redis/Subscription.cs @@ -451,7 +451,7 @@ public bool Remove(Action? handler, ChannelMessageQueu ChannelMessageQueue.Remove(ref _queues, queue); } - return _handlers == null & _queues == null; + return (_handlers == null) & (_queues == null); } public ICompletable? ForInvoke(in RedisChannel channel, in RedisValue message, out ChannelMessageQueue? queues) @@ -751,7 +751,7 @@ internal override int EnsureSubscribedToServer( { // exclude sentinel, and only use replicas if we're explicitly asking for them bool useReplica = (Flags & CommandFlags.DemandReplica) != 0; - bool shouldBeConnected = server.ServerType != ServerType.Sentinel & server.IsReplica == useReplica; + bool shouldBeConnected = (server.ServerType != ServerType.Sentinel) & (server.IsReplica == useReplica); if (shouldBeConnected == IsConnectedTo(server.EndPoint)) { return null; diff --git a/src/StackExchange.Redis/ValueCondition.cs b/src/StackExchange.Redis/ValueCondition.cs index 96bd36a2f..24ac5f5db 100644 --- a/src/StackExchange.Redis/ValueCondition.cs +++ b/src/StackExchange.Redis/ValueCondition.cs @@ -233,9 +233,9 @@ public static ValueCondition ParseDigest(ReadOnlySpan digest) private static byte ParseNibble(int b) { - if (b >= '0' & b <= '9') return (byte)(b - '0'); - if (b >= 'a' & b <= 'f') return (byte)(b - 'a' + 10); - if (b >= 'A' & b <= 'F') return (byte)(b - 'A' + 10); + if ((b >= '0') & (b <= '9')) return (byte)(b - '0'); + if ((b >= 'a') & (b <= 'f')) return (byte)(b - 'a' + 10); + if ((b >= 'A') & (b <= 'F')) return (byte)(b - 'A' + 10); return ThrowInvalidBytes(); static byte ThrowInvalidBytes() => throw new ArgumentException("Invalid digest bytes"); @@ -266,9 +266,9 @@ public static ValueCondition ParseDigest(ReadOnlySpan digest) static byte ToNibble(int b) { - if (b >= '0' & b <= '9') return (byte)(b - '0'); - if (b >= 'a' & b <= 'f') return (byte)(b - 'a' + 10); - if (b >= 'A' & b <= 'F') return (byte)(b - 'A' + 10); + if ((b >= '0') & (b <= '9')) return (byte)(b - '0'); + if ((b >= 'a') & (b <= 'f')) return (byte)(b - 'a' + 10); + if ((b >= 'A') & (b <= 'F')) return (byte)(b - 'A' + 10); return ThrowInvalidBytes(); } diff --git a/src/StackExchange.Redis/VectorSetSimilaritySearchRequest.cs b/src/StackExchange.Redis/VectorSetSimilaritySearchRequest.cs index 4827da23b..a0f6da4b7 100644 --- a/src/StackExchange.Redis/VectorSetSimilaritySearchRequest.cs +++ b/src/StackExchange.Redis/VectorSetSimilaritySearchRequest.cs @@ -69,7 +69,10 @@ private void SetFlag(VsimFlags flag, bool value) /// public int? Count { - get => HasFlag(VsimFlags.Count) ? _count : null; + get + { + return HasFlag(VsimFlags.Count) ? _count : null; + } set { if (value.HasValue) @@ -107,7 +110,10 @@ public bool WithAttributes /// public double? Epsilon { - get => HasFlag(VsimFlags.Epsilon) ? _epsilon : null; + get + { + return HasFlag(VsimFlags.Epsilon) ? _epsilon : null; + } set { if (value.HasValue) @@ -127,7 +133,10 @@ public double? Epsilon /// public int? SearchExplorationFactor { - get => HasFlag(VsimFlags.SearchExplorationFactor) ? _searchExplorationFactor : null; + get + { + return HasFlag(VsimFlags.SearchExplorationFactor) ? _searchExplorationFactor : null; + } set { if (value.HasValue) @@ -147,7 +156,10 @@ public int? SearchExplorationFactor /// public int? MaxFilteringEffort { - get => HasFlag(VsimFlags.MaxFilteringEffort) ? _maxFilteringEffort : null; + get + { + return HasFlag(VsimFlags.MaxFilteringEffort) ? _maxFilteringEffort : null; + } set { if (value.HasValue) @@ -169,7 +181,10 @@ public int? MaxFilteringEffort /// public string? FilterExpression { - get => _filterExpression; + get + { + return _filterExpression; + } set { _filterExpression = value; diff --git a/tests/RESPite.Tests/RESPite.Tests.csproj b/tests/RESPite.Tests/RESPite.Tests.csproj index 7c73cb3c8..b3cd2b4b2 100644 --- a/tests/RESPite.Tests/RESPite.Tests.csproj +++ b/tests/RESPite.Tests/RESPite.Tests.csproj @@ -6,16 +6,16 @@ false true Exe + + true - - - - - + + diff --git a/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj b/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj index 63ee892ca..6bb6e92a7 100644 --- a/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj +++ b/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj @@ -2,6 +2,9 @@ net10.0 Exe + + true enable @@ -24,10 +27,10 @@ - - + - + + diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs b/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs index 40188bd5b..170669fc7 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs @@ -1,4 +1,5 @@ -using Xunit; +using Xunit.Sdk; +using Xunit.v3; // One cluster, one injector, and scenarios that mutate *cluster* state - node exclusions, maintenance mode, // endpoint policies. Running classes in parallel therefore breaks two ways at once: they interfere semantically @@ -8,4 +9,4 @@ // // So this tier is strictly serial. It costs wall-clock - the scenarios are minutes each - and buys results that // mean something. -[assembly: CollectionBehavior(DisableTestParallelization = true)] +[assembly: Parallelization(Mode = ParallelMode.None)] diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs index 8cddd2841..e07061166 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs @@ -1,7 +1,12 @@ using System; using System.Collections.Generic; +using System.IO; using System.Net.Http; using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Net.Security; +using System.Security.Authentication; +using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using System.Text; using System.Text.Json; @@ -13,38 +18,75 @@ namespace StackExchange.Redis.FaultInjector.Tests; /// The cluster's own REST API on port 9443, for the few facts the fault injector does not expose. /// /// -/// Reads only. Anything that *changes* state goes through the injector so it is recorded as a job with an -/// action id - which is also how the console works, and it means a scenario can be reconstructed afterwards -/// from the injector's history rather than from somebody's memory. +/// Reads, plus one deliberate exception: , because the injector has no action for +/// it and it is the narrowest reliable way to make an Active-Active member fall behind. Everything else that +/// *changes* state goes through the injector so it is recorded as a job with an action id - which is also how +/// the console works, and it means a scenario can be reconstructed afterwards from the injector's history rather +/// than from somebody's memory. /// public sealed class ClusterRestClient : IDisposable { private readonly HttpClient _http; - public ClusterRestClient(FaultInjectorEnvironment.ClusterCredentials credentials, string? certificateAuthorityPath) + public ClusterRestClient(FaultInjectorEnvironment.ClusterCredentials credentials, DirectoryInfo configDirectory) { - var handler = new HttpClientHandler(); - - if (certificateAuthorityPath is not null) + var validate = PinnedCertificateValidation(credentials, configDirectory); + var handler = new HttpClientHandler { - // The cluster's management certificate is self-signed per environment, like the proxy ones. Pin to - // the CA in the config directory rather than accepting anything: this channel carries credentials. - var issuer = X509CertificateLoader.LoadCertificateFromFile(certificateAuthorityPath); - handler.ServerCertificateCustomValidationCallback = (_, certificate, chain, _) => - { - if (certificate is null || chain is null) return false; - chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; - chain.ChainPolicy.CustomTrustStore.Add(issuer); - chain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; - return chain.Build(certificate); - }; - } + ServerCertificateCustomValidationCallback = (request, certificate, chain, errors) => validate(request, certificate, chain, errors), + }; _http = new HttpClient(handler) { BaseAddress = credentials.RestUrl, Timeout = TimeSpan.FromSeconds(30) }; var basic = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{credentials.Username}:{credentials.Password}")); _http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", basic); } + /// + /// Validates a cluster's management certificate by pinning it on first use. + /// + /// + /// The cluster's management (cnm) certificate is minted at bootstrap and is self-signed - its issuer is + /// itself, not any CA shipped in the config directory. ca.crt there signs the RESP-plane proxy + /// certificates (a different, coincidentally-named CA), so chain-building the cnm cert against it fails even + /// when it is the right environment. There is nowhere to read the cnm cert's expected fingerprint from ahead + /// of time (env_output.json does not carry it), so this pins on first use and persists the fingerprint next + /// to the rest of the environment's files - the same trust model as an SSH known_hosts entry. A later run + /// against the same environment must present the identical certificate; a changed fingerprint fails loudly + /// rather than silently accepting a new identity for a channel that carries credentials. Keyed by cluster: + /// multi-cluster templates put several clusters behind one directory, and every node of one cluster presents + /// the same certificate. Shared with the lag-aware probe under test, so both trust the same anchor. + /// + public static RemoteCertificateValidationCallback PinnedCertificateValidation( + FaultInjectorEnvironment.ClusterCredentials credentials, DirectoryInfo configDirectory) + { + var pinPath = Path.Combine(configDirectory.FullName, $"cluster-cert.{credentials.ClusterName}.sha256"); + return (_, certificate, _, _) => + { + if (certificate is null) return false; + var actual = Convert.ToHexString(SHA256.HashData(certificate.GetRawCertData())); + + if (File.Exists(pinPath)) + { + var expected = File.ReadAllText(pinPath).Trim(); + if (!string.Equals(expected, actual, StringComparison.OrdinalIgnoreCase)) + { + throw new AuthenticationException( + $"cluster management certificate at {credentials.RestUrl} does not match the pinned " + + $"fingerprint in {pinPath} (expected {expected}, got {actual}). If this environment " + + "was re-provisioned, delete that file and re-run to pin the new certificate."); + } + + return true; + } + + // trust on first use: nothing in the config directory names the expected fingerprint ahead of + // time, so the only available anchor is "whatever this environment presents the first time we + // ask", pinned from here on. + File.WriteAllText(pinPath, actual); + return true; + }; + } + public void Dispose() => _http.Dispose(); /// @@ -69,4 +111,49 @@ public ClusterRestClient(FaultInjectorEnvironment.ClusterCredentials credentials return results; } + /// + /// The uid of the named database on this cluster; uids are not shared across the clusters of an + /// Active-Active database, so always look up per cluster. + /// + public async Task FindDatabaseUidAsync(string name) + { + foreach (var (uid, candidate) in await ListDatabasesAsync()) + { + if (string.Equals(candidate, name, StringComparison.Ordinal)) return uid; + } + + throw new InvalidOperationException($"no database named '{name}' on {_http.BaseAddress}"); + } + + /// + /// Pauses or resumes sync into this cluster's member of an Active-Active database + /// (crdt_sync: paused or enabled), waiting until the database reports it and is active. + /// + /// + /// Measured as narrow and reliably undone: one member, no network change, active again within seconds. + /// Callers must resume in a finally. + /// + public async Task SetCrdtSyncAsync(int uid, string value) + { + using (var put = await _http.PutAsJsonAsync($"/v1/bdbs/{uid}", new Dictionary { ["crdt_sync"] = value })) + { + put.EnsureSuccessStatusCode(); + } + + for (int i = 0; i < 60; i++) + { + using var response = await _http.GetAsync($"/v1/bdbs/{uid}?fields=crdt_sync,status"); + response.EnsureSuccessStatusCode(); + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + if (document.RootElement.GetProperty("crdt_sync").GetString() == value + && document.RootElement.GetProperty("status").GetString() == "active") + { + return; + } + + await Task.Delay(1000); + } + + throw new TimeoutException($"bdb {uid} on {_http.BaseAddress} did not reach crdt_sync={value}"); + } } diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs index 6b1b18b5a..e246ed6cf 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs @@ -66,7 +66,13 @@ private FaultInjectorEnvironment(DirectoryInfo configDirectory, Uri injectorUrl) /// advertised endpoint type - which matter because they decide which notification family a database can /// even emit. Tests that provision their own database know what they asked for and need none of this. /// - public ClusterCredentials? Cluster { get; private set; } + public ClusterCredentials? Cluster => Clusters.Count > 0 ? Clusters[0] : null; + + /// + /// Credentials for every cluster in the environment, in env_output.json order (which is the + /// injector's cluster_index); one entry for single-cluster templates, two for Active-Active. + /// + public IReadOnlyList Clusters { get; private set; } = []; private static FaultInjectorEnvironment? Discover() { @@ -94,7 +100,7 @@ private FaultInjectorEnvironment(DirectoryInfo configDirectory, Uri injectorUrl) { CertificateAuthorityPath = FindCertificateAuthority(directory), }; - result.Cluster = ReadClusterCredentials(directory); + result.Clusters = ReadClusterCredentials(directory); return result; } @@ -124,10 +130,10 @@ private FaultInjectorEnvironment(DirectoryInfo configDirectory, Uri injectorUrl) return null; } - private static ClusterCredentials? ReadClusterCredentials(DirectoryInfo directory) + private static IReadOnlyList ReadClusterCredentials(DirectoryInfo directory) { var path = Path.Combine(directory.FullName, "env_output.json"); - if (!File.Exists(path)) return null; + if (!File.Exists(path)) return []; try { @@ -136,26 +142,35 @@ private FaultInjectorEnvironment(DirectoryInfo configDirectory, Uri injectorUrl) // two schemas in the wild: single-cluster templates put outputs at the top level, and the AWS // multi-cluster template nests them under .clusters.value[N]. Both are handled because which one // you get is a property of the template somebody chose, not of anything a test can control. - var root = document.RootElement; - if (root.TryGetProperty("clusters", out var clusters) + var roots = new List(); + if (document.RootElement.TryGetProperty("clusters", out var clusters) && clusters.TryGetProperty("value", out var values) - && values.ValueKind == JsonValueKind.Array - && values.GetArrayLength() > 0) + && values.ValueKind == JsonValueKind.Array) + { + foreach (var value in values.EnumerateArray()) roots.Add(value); + } + else { - root = values[0]; + roots.Add(document.RootElement); } - var name = ReadValue(root, "cluster_name") ?? ReadValue(root, "name"); - var user = ReadValue(root, "username") ?? ReadValue(root, "cluster_username"); - var password = ReadValue(root, "password") ?? ReadValue(root, "cluster_password"); + var result = new List(); + foreach (var root in roots) + { + var name = ReadValue(root, "cluster_name") ?? ReadValue(root, "name"); + var user = ReadValue(root, "username") ?? ReadValue(root, "cluster_username"); + var password = ReadValue(root, "password") ?? ReadValue(root, "cluster_password"); + if (name is null || user is null || password is null) break; // keep indexes aligned with cluster_index + result.Add(new ClusterCredentials(name, user, password)); + } - return name is null || user is null || password is null ? null : new ClusterCredentials(name, user, password); + return result; } catch (Exception) { // a malformed env_output.json is not fatal here: only the tests that need REST enrichment care, // and they report it themselves rather than failing every test in the suite at discovery time - return null; + return []; } } diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs index 8ea143f8e..77e9443f8 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs @@ -199,7 +199,7 @@ private async Task SweepOrphansAsync() // actual name is an assumption worth not making when the consequence is deleting the wrong thing. var template = ExistingDatabase.ReadAll(Environment).Values.Select(d => d.BdbId).ToHashSet(); - using var rest = new ClusterRestClient(cluster, Environment.CertificateAuthorityPath); + using var rest = new ClusterRestClient(cluster, Environment.ConfigDirectory); foreach (var (bdbId, name) in await rest.ListDatabasesAsync()) { if (template.Contains(bdbId)) continue; diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/LagAwareFailbackScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/LagAwareFailbackScenarioTests.cs new file mode 100644 index 000000000..d14e9daf1 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/LagAwareFailbackScenarioTests.cs @@ -0,0 +1,131 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// +/// The lag-aware failback check against a real Redis Enterprise Active-Active deployment: a group must not fail +/// back onto a member that is alive but behind, and must fail back once it has caught up. +/// +/// +/// No network fault is injected. The member is made to fall behind by pausing sync into it +/// (crdt_sync: paused through its cluster's REST API) while writes go to the other member - measured as +/// narrow and reliably undone, unlike network_latency, whose cleanup can fail and leave a cluster's DNS +/// and REST API unreachable (see notes/lag-aware/findings.md, section 10). +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "lag-aware")] +public class LagAwareFailbackScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper output) + : IClassFixture +{ + private const string DatabaseName = "re-active-active"; + private static readonly TimeSpan HoldOff = TimeSpan.FromSeconds(15); + + [Fact] + public async Task RefusesToFailBackOntoAStaleMemberAndReturnsOnceCaughtUp() + { + var database = fixture.Require(DatabaseName); + Assert.SkipWhen(database.MemberCount < 2, $"'{database.Key}' lists {database.MemberCount} endpoint(s); this needs two member databases"); + var environment = fixture.Environment; + Assert.SkipWhen(environment.Clusters.Count < 2, $"env_output.json carries credentials for {environment.Clusters.Count} cluster(s); this needs both"); + + // each member's own cluster: the one whose name its database host ends with ("redis-.") + FaultInjectorEnvironment.ClusterCredentials ClusterOf(DatabaseMember member) + => environment.Clusters.Single(c => member.Host.EndsWith("." + c.ClusterName, StringComparison.OrdinalIgnoreCase)); + var clusters = database.Members.Select(ClusterOf).ToArray(); + output.WriteLine($"member0={database.Members[0]} ({clusters[0]}), member1={database.Members[1]} ({clusters[1]})"); + + ConnectionGroupMember Member(int index, double weight) => new( + database.GetClientConfig(environment, MaintenanceNotificationMode.Disabled, index), $"member{index}") + { + Weight = weight, + FailbackHealthCheck = HealthCheck.LagAware(new LagAwareOptions + { + Credentials = _ => new(new NetworkCredential(clusters[index].Username, clusters[index].Password)), + CertificateValidation = ClusterRestClient.PinnedCertificateValidation(clusters[index], environment.ConfigDirectory), + }), + }; + + // member1 is preferred, so "not returning to it" is the observable refusal + var member0 = Member(0, weight: 1); + var member1 = Member(1, weight: 9); + MultiGroupOptions options = new MultiGroupOptions.Builder + { + HealthCheckInterval = TimeSpan.FromSeconds(1), + FailbackDelay = TimeSpan.Zero, // only the lag check stands between member1 and a failback + }; + + using var rest = new ClusterRestClient(clusters[1], environment.ConfigDirectory); + var uid = await rest.FindDatabaseUidAsync(DatabaseName); + + using var writer = new TestOutputWriter(output); + await using var group = await ConnectionMultiplexer.ConnectGroupAsync([member0, member1], options, writer); + await WaitForActiveAsync(group, member1, TimeSpan.FromSeconds(30), "the preferred, caught-up member should be selected first"); + + var db = group.GetDatabase(); + RedisKey key = "fi-lag-aware-counter"; + await db.KeyDeleteAsync(key); + + bool paused = false; + try + { + await rest.SetCrdtSyncAsync(uid, "paused"); + paused = true; + output.WriteLine($"{Now()} sync into member1 (bdb {uid} on {clusters[1]}) paused"); + + // leave member1 for member0, then release the override: member1 is the higher weight, alive and + // connected, so the only thing that can keep the group off it is its failback check + Assert.True(group.TryFailoverTo(member0)); + Assert.True(group.TryFailoverTo(null)); + + var watch = Stopwatch.StartNew(); + long written = 0; + while (watch.Elapsed < HoldOff) + { + written = await db.StringIncrementAsync(key); // on member0; member1 cannot see these + Assert.True(member1.IsConnected, "member1 should stay connected: the refusal must be lag, not liveness"); + Assert.True( + group.ActiveMember == member0, + $"{Now()} the group moved to {group.ActiveMember} while member1 was behind (failback verified: {member1.FailbackVerified})"); + await Task.Delay(500, TestContext.Current.CancellationToken); + } + + output.WriteLine($"{Now()} held on member0 for {HoldOff.TotalSeconds:0}s, {written} writes member1 has not seen"); + } + finally + { + if (paused) + { + await rest.SetCrdtSyncAsync(uid, "enabled"); + output.WriteLine($"{Now()} sync into member1 resumed"); + } + } + + await WaitForActiveAsync(group, member1, TimeSpan.FromSeconds(60), "the group should fail back once member1 has caught up"); + var seen = (long)await member1.Multiplexer.GetDatabase().StringGetAsync(key); + output.WriteLine($"{Now()} failed back to member1, which now has the counter at {seen}"); + Assert.True(seen > 0, "member1 should have received member0's writes by the time it was selected"); + await db.KeyDeleteAsync(key); + } + + private async Task WaitForActiveAsync(IConnectionGroup group, ConnectionGroupMember expected, TimeSpan timeout, string because) + { + var watch = Stopwatch.StartNew(); + while (group.ActiveMember != expected && watch.Elapsed < timeout) + { + await Task.Delay(250, TestContext.Current.CancellationToken); + } + + output.WriteLine($"{Now()} active: {group.ActiveMember} after {watch.Elapsed.TotalSeconds:0.0}s (wanted {expected})"); + Assert.True(group.ActiveMember == expected, $"{because}; active is {group.ActiveMember} after {timeout.TotalSeconds:0}s"); + } + + private static string Now() => DateTime.UtcNow.ToString("HH:mm:ss.f"); +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/README.md b/tests/StackExchange.Redis.FaultInjector.Tests/README.md index 0be35486c..0ae6ff8f2 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/README.md +++ b/tests/StackExchange.Redis.FaultInjector.Tests/README.md @@ -12,7 +12,7 @@ docker compose up -d # 10-15 minutes for the cluster to come export SER_FI_CONFIG_DIR=$PWD # or FI_CONSOLE_CONFIG_DIR, which is also honoured export E2E_SCENARIO_TESTS=true # explicit opt-in: these create and delete databases -dotnet test tests/StackExchange.Redis.FaultInjector.Tests +dotnet test --project tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj ``` **Point it at the *current* environment.** A stale directory passes every check we make: the credentials are @@ -125,6 +125,23 @@ is blackholed is the injector's choice - it evidently picks the first, which is every member's `host:port` and says so in the final assertion message, because a wrong-cluster injection otherwise looks exactly like a failback that never happened. +## Lag-aware failback + +`LagAwareFailbackScenarioTests` checks the lag-aware failback check (`HealthCheck.LagAware`) against the +same `re-active-active` pair: the group must not fail back onto a member that is alive but behind, and must +return once it has caught up. It needs credentials for **both** clusters in `env_output.json` (the AWS +multi-cluster template has them) and skips otherwise. + +No fault is injected. The preferred member is made to fall behind by pausing sync *into* it - `crdt_sync: paused` +through its cluster's REST API, via `ClusterRestClient.SetCrdtSyncAsync` - while writes go to the other member, +and sync is resumed in a `finally`. That is the one state change this tier makes outside the injector, because +the injector has no action for it, and because the obvious alternative is worse: `network_latency` shapes the +node's whole interface and its cleanup can fail, which on one environment left the cluster's DNS and REST API +unreachable until fixed by hand. + +Measured shape of a passing run: on the preferred member first; held off it for the 15 s it was behind, while +alive and connected; failed back about 4 s after sync resumed, by which time it had all of the writes. + ## Three states, deliberately distinct | state | behaviour | diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj b/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj index 052d85e11..308b3aa29 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj +++ b/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj @@ -11,15 +11,16 @@ full enable true + + true - - - + diff --git a/tests/StackExchange.Redis.Tests/BacklogTests.cs b/tests/StackExchange.Redis.Tests/BacklogTests.cs index f4ba0f607..5aa8e155f 100644 --- a/tests/StackExchange.Redis.Tests/BacklogTests.cs +++ b/tests/StackExchange.Redis.Tests/BacklogTests.cs @@ -463,6 +463,12 @@ public async Task TotalOutstandingIncludesBacklogQueue() Log("Test: Checking reconnected"); Assert.True(conn.IsConnected); + // lastPing completing proves the backlog drained, not that every connection is idle: the reconnect's own + // handshake traffic (and the subscription connection's) can still be awaiting replies at this instant, + // and TotalOutstanding counts those too (seen on CI as 7 and 1); so wait for quiescence rather than assert it + Log("Test: awaiting quiescence"); + await UntilConditionAsync(TimeSpan.FromSeconds(5), () => conn.GetCounters().TotalOutstanding == 0).ForAwait(); + Log("Test: asserting reconnected counters"); var reconnectedServerCounters = server.GetCounters(); var reconnectedConnCounters = conn.GetCounters(); diff --git a/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs b/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs new file mode 100644 index 000000000..303714560 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs @@ -0,0 +1,93 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Proxied deployments can refuse some or all CLUSTER subcommands while still identifying as a cluster +/// (Redis Enterprise exposes CLUSTER SLOTS only under the OSS cluster API, and its HELLO/INFO +/// can say "cluster" while CLUSTER stays blocked entirely). A refusal is a posture, not a fault: the +/// topology pass must fall back to whatever view it can get, quietly - not log a server error on every +/// reconfigure. See #3274. +/// +[RunPerProtocol] +public class ClusterCommandsRefusedUnitTests(ITestOutputHelper log) +{ + private sealed class RefusingServer(ITestOutputHelper log, bool refuseNodes) : InProcessTestServer(log) + { + protected override TypedRedisValue ClusterSlots(RedisClient client, in RedisRequest request) + => TypedRedisValue.Error("ERR command is not allowed"); + + protected override TypedRedisValue ClusterNodes(RedisClient client, in RedisRequest request) + => refuseNodes ? TypedRedisValue.Error("ERR command is not allowed") : base.ClusterNodes(client, request); + + // a proxy that hides the shards accepts cross-slot commands itself + public override bool CheckCrossSlot => false; + } + + [Fact] + public async Task SlotsRefusedFallsBackToNodes() + { + using var server = new RefusingServer(log, refuseNodes: false) { ServerType = ServerType.Cluster }; + var a = server.DefaultEndPoint; + var b = server.AddEmptyNode(); + + var config = server.GetClientConfig(defaultOnly: true); // only connect to a initially + var logger = new CapturingLogger(); + config.LoggerFactory = logger; + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + // discovery still happened, from the NODES view alone + var endpoints = conn.GetEndPoints(); + Assert.Contains(a, endpoints); + Assert.Contains(b, endpoints); + + log.WriteLine(logger.All); + Assert.Empty(logger.Matching("Encountered error while updating cluster config")); + Assert.NotEmpty(logger.Matching("CLUSTER SLOTS was refused")); + + await conn.GetDatabase().PingAsync(); + } + + [Fact] + public async Task AllClusterCommandsRefusedIsQuiet() + { + // the server still *identifies* as a cluster (HELLO "mode", INFO "redis_mode"), so the client + // interrogates it; both commands being refused must leave a working connection, not a logged error + using var server = new RefusingServer(log, refuseNodes: true) { ServerType = ServerType.Cluster }; + + var config = server.GetClientConfig(defaultOnly: true); + var logger = new CapturingLogger(); + config.LoggerFactory = logger; + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + try + { + Assert.Empty(logger.Matching("Encountered error while updating cluster config")); + Assert.NotEmpty(logger.Matching("CLUSTER NODES was refused")); + + await conn.GetDatabase().PingAsync(); + + // ...and a later explicit reconfigure stays quiet too; the connect's own follow-on reconfigures + // (connection restored, autoconfigure changes) may still be in flight, and ReconfigureAsync declines + // while one is, so insist on a pass that actually ran + await Poll.ReconfigureAsync(conn, "test"); + Assert.Empty(logger.Matching("Encountered error while updating cluster config")); + + // with no slot map there is nothing to route by, so we must not *route* as a cluster - in particular, + // cross-slot commands go to the proxy rather than being refused client-side (#3284) + Assert.NotEmpty(logger.Matching("no slot map is available; routing as standalone")); + Assert.Equal(ServerType.Standalone, conn.ServerSelectionStrategy.ServerType); + await conn.GetDatabase().StringGetAsync(["a", "b"]); + } + finally + { + // after the assertions, so a failure shows what the multiplexer actually logged + log.WriteLine(logger.All); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs b/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs index 48ae8b973..0bc6beb2b 100644 --- a/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs +++ b/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs @@ -145,34 +145,4 @@ public async Task TheReadIsRateLimitedRatherThanOncePerFailure() Assert.True(refreshes <= permitted, $"expected at most {permitted} rate-limited reads, but saw {refreshes}"); } } - - private sealed class CapturingLogger : ILoggerFactory, ILogger - { - private readonly List _messages = []; - - public ILogger CreateLogger(string categoryName) => this; - - public void AddProvider(ILoggerProvider provider) { } - - public IDisposable? BeginScope(TState state) where TState : notnull => null; - - public bool IsEnabled(LogLevel logLevel) => true; - - public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) - { - lock (_messages) _messages.Add(formatter(state, exception)); - } - - public List Matching(string fragment) - { - lock (_messages) return _messages.FindAll(x => x.Contains(fragment, StringComparison.Ordinal)); - } - - public string All - { - get { lock (_messages) return string.Join("\n", _messages); } - } - - public void Dispose() { } - } } diff --git a/tests/StackExchange.Redis.Tests/ConnectionAttemptUnitTests.cs b/tests/StackExchange.Redis.Tests/ConnectionAttemptUnitTests.cs index 018d60666..62c20afd2 100644 --- a/tests/StackExchange.Redis.Tests/ConnectionAttemptUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/ConnectionAttemptUnitTests.cs @@ -129,6 +129,51 @@ public async Task TimedOutAttemptCarriesTheTimeout() } } + [Fact] + public async Task AttemptIsNotAbandonedBeforeConnectTimeout() + { + // never answers the TLS handshake, so every attempt lives until the library gives up on it. The retry policy + // here says "retry after 50ms", but an attempt must still get its full ConnectTimeout: the default policy's + // 2.5s first interval was killing attempts whose name resolution alone took longer than that (#2894) + var listener = new TcpListener(IPAddress.Loopback, 0); // not IDisposable on net481 + listener.Start(); + try + { + const int connectTimeout = 1500; + var options = new ConfigurationOptions + { + EndPoints = { (IPEndPoint)listener.LocalEndpoint }, + Ssl = true, + AbortOnConnectFail = false, + ConnectTimeout = connectTimeout, + HeartbeatInterval = TimeSpan.FromMilliseconds(100), + ReconnectRetryPolicy = new LinearRetry(50), + }; + var attempts = Observe(options); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(options); + + // an abandoned attempt is followed at once by the next, so the gap between consecutive timeouts on one + // connection is how long an attempt was allowed to live + static bool IsInteractiveTimeout(ConnectionAttemptCompletedEventArgs a) + => a.ConnectionType == ConnectionType.Interactive && a.FailureType == ConnectionFailureType.UnableToConnect; + await WaitForAsync(() => attempts.Count(IsInteractiveTimeout) >= 3, timeoutMilliseconds: 20000); + var timedOut = attempts.Where(IsInteractiveTimeout).OrderBy(a => a.SequenceNumber).Take(3).ToArray(); + for (int i = 1; i < timedOut.Length; i++) + { + var lifetime = timedOut[i].CompletedTimeUtc - timedOut[i - 1].CompletedTimeUtc; + output.WriteLine($"attempt lifetime: {lifetime.TotalMilliseconds:0}ms"); + Assert.True( + lifetime >= TimeSpan.FromMilliseconds(connectTimeout - 250), + $"attempt abandoned after {lifetime.TotalMilliseconds:0}ms, inside the {connectTimeout}ms connect timeout"); + } + } + finally + { + listener.Stop(); + } + } + [Fact] public async Task AbandonedAttemptIsNotReportedAsAFailureToConnect() { diff --git a/tests/StackExchange.Redis.Tests/ConnectionDroppedExceptionTests.cs b/tests/StackExchange.Redis.Tests/ConnectionDroppedExceptionTests.cs new file mode 100644 index 000000000..1d4cf9a71 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ConnectionDroppedExceptionTests.cs @@ -0,0 +1,76 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// A command that fails because its connection is gone must fail with a : +/// never the raw transport exception, and never a when no caller token was +/// involved (#3283, main #3287) - and a batch whose connection is gone must complete every one of its commands +/// that way rather than leaving any of them pending (#3286). The shipped core's tests drove its bridge and +/// physical connection directly; this core is driven from the public surface, where the same facts are +/// observable. +/// +public class ConnectionDroppedExceptionTests(ITestOutputHelper log) +{ + private static async Task AwaitFailure(Task task) + { + var completed = await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(10))); + Assert.Same(task, completed); + return await Assert.ThrowsAnyAsync(() => task); + } + + private async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection)> ConnectThenLoseTheServerAsync() + { + var server = new InProcessTestServer(log); + var config = server.GetClientConfig(withPubSub: false, defaultOnly: true); + config.BacklogPolicy = BacklogPolicy.FailFast; + config.AbortOnConnectFail = false; + config.AllowSimulateConnectionFailure = true; + var conn = await ConnectionMultiplexer.ConnectAsync(config); + + // no connection, and no way to make one: what a command meets when the server has gone away + conn.AllowConnect = false; + conn.GetServerSnapshot()[0].SimulateConnectionFailure(SimulatedFailureType.All); + Assert.False(conn.IsConnected); + return (server, conn); + } + + [Fact] + public async Task ACommandWithNoConnectionFailsWithRedisConnectionExceptionNotCancellation() + { + var (server, conn) = await ConnectThenLoseTheServerAsync(); + using var _ = server; + await using var __ = conn; + + var task = conn.GetDatabase().PingAsync(); + var ex = await AwaitFailure(task); + log.WriteLine(ex.ToString()); + Assert.IsType(ex); + Assert.False(task.IsCanceled); + } + + [Fact] + public async Task ABatchWithNoConnectionCompletesEveryCommandWithRedisConnectionException() + { + var (server, conn) = await ConnectThenLoseTheServerAsync(); + using var _ = server; + await using var __ = conn; + + var batch = conn.GetDatabase().CreateBatch(); + var tasks = Enumerable.Range(0, 20).Select(i => batch.StringGetAsync("k" + i)).ToArray(); + batch.Execute(); + + // every command, not most: a batch's tail used to be left pending when its connection was poisoned + var all = Task.WhenAll(tasks); + var completed = await Task.WhenAny(all, Task.Delay(TimeSpan.FromSeconds(10))); + Assert.Same(all, completed); + foreach (var task in tasks) + { + Assert.True(task.IsFaulted, "every command must fault, not cancel or hang"); + Assert.IsType(task.Exception!.InnerException); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/FormatTests.cs b/tests/StackExchange.Redis.Tests/FormatTests.cs index c36e42f34..733756780 100644 --- a/tests/StackExchange.Redis.Tests/FormatTests.cs +++ b/tests/StackExchange.Redis.Tests/FormatTests.cs @@ -8,6 +8,76 @@ namespace StackExchange.Redis.Tests; public class FormatTests(ITestOutputHelper output) : TestBase(output) { + // System.Version is awkward: unspecified parts are -1, so "7.4" is (7, 4, -1, -1) and "7.4.0" is (7, 4, 0, -1), + // and the inbuilt operators order them differently; the UTF-8 overload (what the HELLO handler uses) must + // produce exactly the same raw parts as the char/string overloads, and the version gates must then treat the + // unspecified parts as zero (see VersionExtensions.IsAtLeast). + [Theory] + [InlineData("7", 7, 0, -1, -1)] // major-only is our extension; System.Version rejects it + [InlineData("7.4", 7, 4, -1, -1)] + [InlineData("7.4.0", 7, 4, 0, -1)] + [InlineData("7.4.3", 7, 4, 3, -1)] + [InlineData("7.4.3.0", 7, 4, 3, 0)] + [InlineData("9.1.2", 9, 1, 2, -1)] + [InlineData("7.4.3.1", 7, 4, 3, 1)] + [InlineData("255.255.65535.65535", 255, 255, 65535, 65535)] + [InlineData("0.0", 0, 0, -1, -1)] + [InlineData("007.040", 7, 40, -1, -1)] + // System.Version uses int.TryParse per part, which tolerates whitespace and a sign; we inherit that, not fix it + [InlineData(" 7.4", 7, 4, -1, -1)] + [InlineData("7.4 ", 7, 4, -1, -1)] + [InlineData("7. 4", 7, 4, -1, -1)] + [InlineData("+7.4", 7, 4, -1, -1)] + [InlineData("7.+4", 7, 4, -1, -1)] + [InlineData(" 7 ", 7, 0, -1, -1)] + [InlineData("", null, null, null, null)] + [InlineData(".", null, null, null, null)] + [InlineData("7.", null, null, null, null)] + [InlineData(".7", null, null, null, null)] + [InlineData("7..4", null, null, null, null)] + [InlineData("1.2.3.4.5", null, null, null, null)] + [InlineData("9.1.2-rc1", null, null, null, null)] + [InlineData("-1.2", null, null, null, null)] + [InlineData("a.b", null, null, null, null)] + [InlineData("1.2.3-preview4", null, null, null, null)] + public void TryParseVersionUtf8MatchesChars(string input, int? major, int? minor, int? build, int? revision) + { + bool success = Format.TryParseVersion(Encoding.UTF8.GetBytes(input), out var fromBytes); + Assert.Equal(major is not null, success); + Assert.Equal(success, Format.TryParseVersion(input.AsSpan(), out var fromChars)); + Assert.Equal(success, Format.TryParseVersion(input, out var fromString)); + + if (success) + { + Assert.NotNull(fromBytes); + Assert.Equal((major, minor, build, revision), (fromBytes.Major, fromBytes.Minor, fromBytes.Build, fromBytes.Revision)); + Assert.Equal(fromBytes, fromChars); // Version.Equals compares all four raw parts, -1 included + Assert.Equal(fromBytes, fromString); + } + else + { + Assert.Null(fromBytes); + Assert.Null(fromChars); + Assert.Null(fromString); + } + } + + [Theory] + [InlineData("7.4", "7.4.0", true, true)] // the unspecified parts count as zero for the gates + [InlineData("7.4.0", "7.4", true, true)] + [InlineData("7.4", "7.4.0.0", true, true)] + [InlineData("7.4.3", "7.4", true, false)] + [InlineData("7.4", "7.4.3", false, false)] + [InlineData("8", "7.4.3", true, false)] + [InlineData("7.4.3", "8", false, false)] + public void ParsedVersionGatesIgnoreUnspecifiedParts(string server, string required, bool atLeast, bool equal) + { + Assert.True(Format.TryParseVersion(Encoding.UTF8.GetBytes(server), out var x)); + Assert.True(Format.TryParseVersion(Encoding.UTF8.GetBytes(required), out var y)); + Assert.Equal(atLeast, x.IsAtLeast(y)); + Assert.Equal(equal, x.IsEqual(y)); + } + public static IEnumerable EndpointData() { // note: the 3rd arg is for formatting; null means "expect the original string" diff --git a/tests/StackExchange.Redis.Tests/HashFieldTests.cs b/tests/StackExchange.Redis.Tests/HashFieldTests.cs index 0b2604fe4..4964723c9 100644 --- a/tests/StackExchange.Redis.Tests/HashFieldTests.cs +++ b/tests/StackExchange.Redis.Tests/HashFieldTests.cs @@ -11,6 +11,11 @@ namespace StackExchange.Redis.Tests; [RunPerProtocol] public class HashFieldTests(ITestOutputHelper output, SharedConnectionFixture fixture) : TestBase(output, fixture) { + // absolute expiries (the DateTime overloads) are computed against the client clock and read back against the + // server's; on the Windows CI runner the server lives in WSL, whose clock can sit a few ms off the host's, and + // the TTL then lands just above the nominal 2h (seen: 7200001ms). Relative (TimeSpan) expiries use one clock only + private const double ClockSkewToleranceMs = 1000; + private readonly DateTime nextCentury = new DateTime(2101, 1, 1, 0, 0, 0, DateTimeKind.Utc); private readonly TimeSpan oneYearInMs = TimeSpan.FromMilliseconds(31536000000); @@ -319,7 +324,7 @@ public void HashFieldGetAndSetExpiry() fieldResult = db.HashFieldGetAndSetExpiry(hashKey, "f1", DateTime.Now.AddMinutes(120)); Assert.Equal(1, fieldResult); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing persist fieldResult = db.HashFieldGetAndSetExpiry(hashKey, "f1", persist: true); @@ -340,8 +345,8 @@ public void HashFieldGetAndSetExpiry() fieldResults = db.HashFieldGetAndSetExpiry(hashKey, fields, DateTime.Now.AddMinutes(120)); Assert.Equal(values, fieldResults); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with persist fieldResults = db.HashFieldGetAndSetExpiry(hashKey, fields, persist: true); @@ -369,7 +374,7 @@ public async Task HashFieldGetAndSetExpiryAsync() fieldResult = await db.HashFieldGetAndSetExpiryAsync(hashKey, "f1", DateTime.Now.AddMinutes(120)); Assert.Equal(1, fieldResult); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing persist fieldResult = await db.HashFieldGetAndSetExpiryAsync(hashKey, "f1", persist: true); @@ -390,8 +395,8 @@ public async Task HashFieldGetAndSetExpiryAsync() fieldResults = await db.HashFieldGetAndSetExpiryAsync(hashKey, fields, DateTime.Now.AddMinutes(120)); Assert.Equal(values, fieldResults); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with persist fieldResults = await db.HashFieldGetAndSetExpiryAsync(hashKey, fields, persist: true); @@ -417,13 +422,13 @@ public void HashFieldSetAndSetExpiry() result = db.HashFieldSetAndSetExpiry(hashKey, "f1", 1, DateTime.Now.AddMinutes(120)); Assert.Equal(1, result); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing with keepttl result = db.HashFieldSetAndSetExpiry(hashKey, "f1", 1, keepTtl: true); Assert.Equal(1, result); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with timespan result = db.HashFieldSetAndSetExpiry(hashKey, entries, TimeSpan.FromHours(1)); @@ -436,15 +441,15 @@ public void HashFieldSetAndSetExpiry() result = db.HashFieldSetAndSetExpiry(hashKey, entries, DateTime.Now.AddMinutes(120)); Assert.Equal(1, result); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with keepttl result = db.HashFieldSetAndSetExpiry(hashKey, entries, keepTtl: true); Assert.Equal(1, result); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing with ExpireWhen.Exists db.KeyDelete(hashKey); @@ -481,13 +486,13 @@ public async Task HashFieldSetAndSetExpiryAsync() result = await db.HashFieldSetAndSetExpiryAsync(hashKey, "f1", 1, DateTime.Now.AddMinutes(120)); Assert.Equal(1, result); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing with keepttl result = await db.HashFieldSetAndSetExpiryAsync(hashKey, "f1", 1, keepTtl: true); Assert.Equal(1, result); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with timespan result = await db.HashFieldSetAndSetExpiryAsync(hashKey, entries, TimeSpan.FromHours(1)); @@ -500,15 +505,15 @@ public async Task HashFieldSetAndSetExpiryAsync() result = await db.HashFieldSetAndSetExpiryAsync(hashKey, entries, DateTime.Now.AddMinutes(120)); Assert.Equal(1, result); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with keepttl result = await db.HashFieldSetAndSetExpiryAsync(hashKey, entries, keepTtl: true); Assert.Equal(1, result); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing with ExpireWhen.Exists db.KeyDelete(hashKey); diff --git a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs index b2bb7d8b3..79494374a 100644 --- a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs @@ -38,6 +38,7 @@ public void ProbeTrafficIsNotCallerFacingButIsRoutedNormally() [InlineData(0, 5, 0, HealthCheckResult.Unhealthy)] // All failures [InlineData(1, 0, 2, HealthCheckResult.Healthy)] // Early success [InlineData(0, 1, 2, HealthCheckResult.Inconclusive)] // Early failure but more probes remain + [InlineData(0, 0, 0, HealthCheckResult.Inconclusive)] // Every probe inconclusive: not a failure public void AnySuccess_EvaluatesCorrectly(int success, int failure, int remaining, HealthCheckResult expected) { var policy = HealthCheckProbePolicy.AnySuccess; @@ -59,6 +60,7 @@ public void AnySuccess_EvaluatesCorrectly(int success, int failure, int remainin [InlineData(1, 0, 2, HealthCheckResult.Inconclusive)] // Success but more probes remain [InlineData(0, 1, 2, HealthCheckResult.Unhealthy)] // Early failure [InlineData(4, 0, 1, HealthCheckResult.Inconclusive)] // Multiple successes but still waiting + [InlineData(0, 0, 0, HealthCheckResult.Inconclusive)] // Every probe inconclusive: not a pass public void AllSuccess_EvaluatesCorrectly(int success, int failure, int remaining, HealthCheckResult expected) { var policy = HealthCheckProbePolicy.AllSuccess; diff --git a/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs b/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs new file mode 100644 index 000000000..150fabe8f --- /dev/null +++ b/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs @@ -0,0 +1,98 @@ +using System; +using System.Threading.Tasks; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// A proxied deployment can answer HELLO with mode: cluster while INFO says +/// redis_mode:standalone and every CLUSTER subcommand is refused (Azure Managed Redis with the +/// Enterprise clustering policy; see #3284). Such a server gives the client no slot map, so it must not be +/// treated as a cluster - in particular, cross-slot commands must keep working, including after a reconnect +/// of the subscription connection alone, whose handshake is HELLO without the INFO that follows +/// on the interactive connection. +/// +public class HelloClusterModeUnitTests(ITestOutputHelper log) +{ + private sealed class HelloSaysClusterServer(ITestOutputHelper log) : InProcessTestServer(log) + { + protected override string HelloModeValue => "cluster"; + + protected override TypedRedisValue ClusterSlots(RedisClient client, in RedisRequest request) + => TypedRedisValue.Error("ERR command is not allowed"); + + protected override TypedRedisValue ClusterNodes(RedisClient client, in RedisRequest request) + => TypedRedisValue.Error("ERR command is not allowed"); + } + + [Theory] + [InlineData(RedisProtocol.Resp2)] + [InlineData(RedisProtocol.Resp3)] + public async Task CrossSlotSurvivesSubscriptionReconnect(RedisProtocol protocol) + { + using var server = new HelloSaysClusterServer(log); // ServerType stays Standalone: INFO says so + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = protocol; + config.AllowSimulateConnectionFailure = true; + config.AbortOnConnectFail = false; + var logger = new CapturingLogger(); + config.LoggerFactory = logger; + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var db = conn.GetDatabase(); + + RedisKey[] crossSlot = ["a", "b"]; // different hash slots + await db.StringGetAsync(crossSlot); // sanity: fine on a fresh connection + Assert.Equal(ServerType.Standalone, conn.ServerSelectionStrategy.ServerType); + + // the new core opens the subscription connection on first use rather than at connect, so give it a + // reason to exist: without one there is nothing to drop below, and the test would time out waiting + // for a reconnect that never has to happen + await conn.GetSubscriber().SubscribeAsync(RedisChannel.Literal("hello-mode"), static (_, _) => { }); + var endpoint = conn.GetServerSnapshot()[0]; + await WaitUntilAsync(() => endpoint.IsSubscriberConnected, "subscriber connected"); + + // drop *only* the subscription connection, as the server did in #3284; the reconnect is too quick + // to observe by polling, so watch for the restored event instead + var restored = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + conn.ConnectionRestored += (_, e) => + { + if (e.ConnectionType == ConnectionType.Subscription) restored.TrySetResult(true); + }; + log.WriteLine("dropping the subscription connection"); + endpoint.SimulateConnectionFailure(SimulatedFailureType.AllSubscription); + await restored.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await WaitUntilAsync(() => endpoint.IsSubscriberConnected, "subscriber reconnected"); + + // the restored connection triggers a fire-and-forget reconfigure (and so can a server-type change); + // ReconfigureAsync declines while one is in flight, so insist on one that actually ran + await Poll.ReconfigureAsync(conn, "after subscription reconnect"); + + try + { + // the user-visible symptom from #3284: cross-slot commands refused client-side + for (int i = 0; i < 5; i++) + { + await db.StringGetAsync(crossSlot); + await Task.Delay(100); + } + Assert.Equal(ServerType.Standalone, endpoint.ServerType); + Assert.Equal(ServerType.Standalone, conn.ServerSelectionStrategy.ServerType); + } + finally + { + log.WriteLine(logger.All); + } + } + + private static async Task WaitUntilAsync(Func condition, string what) + { + var deadline = DateTime.UtcNow.AddSeconds(10); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new TimeoutException("Timed out waiting for: " + what); + await Task.Delay(50); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/Helpers/Attributes.cs b/tests/StackExchange.Redis.Tests/Helpers/Attributes.cs index 275101569..4ef9b633f 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/Attributes.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/Attributes.cs @@ -57,8 +57,8 @@ public override ValueTask> Discover(ITestFra public class TheoryDiscoverer : Xunit.v3.TheoryDiscoverer { - protected override ValueTask> CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, IXunitTestMethod testMethod, ITheoryAttribute theoryAttribute, ITheoryDataRow dataRow, object?[] testMethodArguments) - => base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments).ExpandAsync(); + protected override ValueTask> CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, IXunitTestMethod testMethod, ITheoryAttribute theoryAttribute, ITheoryDataRow dataRow, object?[] testMethodArguments, string? index) + => base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments, index).ExpandAsync(); protected override ValueTask> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, IXunitTestMethod testMethod, ITheoryAttribute theoryAttribute) => base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute).ExpandAsync(); diff --git a/tests/StackExchange.Redis.Tests/Helpers/CapturingLogger.cs b/tests/StackExchange.Redis.Tests/Helpers/CapturingLogger.cs new file mode 100644 index 000000000..eec70187e --- /dev/null +++ b/tests/StackExchange.Redis.Tests/Helpers/CapturingLogger.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; + +namespace StackExchange.Redis.Tests; + +/// +/// An that records every formatted message, so a test can assert on what +/// was (or was not) logged. +/// +internal sealed class CapturingLogger : ILoggerFactory, ILogger +{ + private readonly List _messages = []; + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + lock (_messages) _messages.Add(formatter(state, exception)); + } + + public List Matching(string fragment) + { + lock (_messages) return _messages.FindAll(x => x.Contains(fragment, StringComparison.Ordinal)); + } + + public string All + { + get { lock (_messages) return string.Join("\n", _messages); } + } + + public void Dispose() { } +} diff --git a/tests/StackExchange.Redis.Tests/Helpers/Poll.cs b/tests/StackExchange.Redis.Tests/Helpers/Poll.cs index a5a0dc4ff..dd55d1d08 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/Poll.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/Poll.cs @@ -37,4 +37,24 @@ public static async Task UntilAsync(Func predicate, int timeoutMilli return false; } + + /// + /// Runs a reconfigure that is guaranteed to have actually executed, rather than been declined. + /// + /// + /// A freshly connected multiplexer fires its own fire-and-forget reconfigures (connection-restored for each + /// physical connection, autoconfigure field changes), and + /// returns false while one is in flight. A test that asserts on that return value races those, and + /// loses on a slow runner; this keeps asking until a pass runs under the caller's cause. + /// + public static async Task ReconfigureAsync(ConnectionMultiplexer conn, string cause, int timeoutMilliseconds = 10000, int pollMilliseconds = 50) + { + var cancellationToken = TestContext.Current.CancellationToken; + for (int i = 0; i <= timeoutMilliseconds / pollMilliseconds; i++) + { + if (await conn.ReconfigureAsync(cause).ConfigureAwait(false)) return; + await Task.Delay(pollMilliseconds, cancellationToken).ConfigureAwait(false); + } + throw new TimeoutException("Timed out waiting for a reconfigure to run: " + cause); + } } diff --git a/tests/StackExchange.Redis.Tests/Helpers/TcpForwarder.cs b/tests/StackExchange.Redis.Tests/Helpers/TcpForwarder.cs index ea2b92665..7f8ea83a7 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/TcpForwarder.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/TcpForwarder.cs @@ -69,9 +69,11 @@ private async Task PumpAsync(TcpClient client) // the buffer size is spelled out because the two-argument (stream, token) overload does not // exist on .NET Framework, which the test project still targets const int BufferSize = 4096; - await Task.WhenAny( - near.CopyToAsync(far, BufferSize, _cts.Token), - far.CopyToAsync(near, BufferSize, _cts.Token)).ForAwait(); + var inbound = near.CopyToAsync(far, BufferSize, _cts.Token); + var outbound = far.CopyToAsync(near, BufferSize, _cts.Token); + await Task.WhenAny(inbound, outbound).ForAwait(); + // the other direction fails when the sockets go away below; nobody awaits it, so observe it here + _ = Task.WhenAll(inbound, outbound).ContinueWith(static t => _ = t.Exception, TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously); } catch { diff --git a/tests/StackExchange.Redis.Tests/InProcessSentinelServer.cs b/tests/StackExchange.Redis.Tests/InProcessSentinelServer.cs new file mode 100644 index 000000000..8fecb98e6 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/InProcessSentinelServer.cs @@ -0,0 +1,147 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using RESPite.Messages; +using StackExchange.Redis.Configuration; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// An in-process sentinel that monitors one service on an : it answers the +/// SENTINEL queries a client makes from the monitored server's topology, and does nothing on its own - +/// no failover, no +switch-master. Which node is the primary is whatever the test says it is, which is +/// what lets a test put sentinel and the client's cached view out of step. +/// +public sealed class InProcessSentinelServer : InProcessTestServer +{ + private readonly InProcessTestServer _monitored; + private EndPoint _primary; + + public InProcessSentinelServer(InProcessTestServer monitored, string serviceName, EndPoint primary, EndPoint endpoint, ITestOutputHelper? log = null) + : base(log, endpoint) + { + _monitored = monitored ?? throw new ArgumentNullException(nameof(monitored)); + ServiceName = serviceName ?? throw new ArgumentNullException(nameof(serviceName)); + _primary = primary ?? throw new ArgumentNullException(nameof(primary)); + ServerType = ServerType.Sentinel; + } + + public string ServiceName { get; } + + /// + /// The primary this sentinel reports for ; changing it announces nothing. + /// + public EndPoint Primary + { + get => Volatile.Read(ref _primary); + set => Volatile.Write(ref _primary, value ?? throw new ArgumentNullException(nameof(value))); + } + + /// + /// A tunnel that routes each endpoint to whichever of these servers owns it, so that one client configuration + /// can reach the sentinel and the nodes it monitors. + /// + public static Tunnel CreateTunnel(params InProcessTestServer[] servers) => new RoutingTunnel(servers); + + private int _primaryQueries; + + /// How many times a client has asked this sentinel for the primary. + public int PrimaryQueries => Volatile.Read(ref _primaryQueries); + + [RedisCommand(3, nameof(RedisCommand.SENTINEL), "get-master-addr-by-name", LockFree = true)] + private TypedRedisValue SentinelGetMasterAddrByName(RedisClient client, in RedisRequest request) + { + Interlocked.Increment(ref _primaryQueries); + if (!IsService(request)) return TypedRedisValue.NullArray(RespPrefix.Array); + var host = GetHost(Primary, out var port); + var reply = TypedRedisValue.Rent(2, out var span, RespPrefix.Array); + span[0] = TypedRedisValue.BulkString(host); + span[1] = TypedRedisValue.BulkString(port.ToString()); + return reply; + } + + [RedisCommand(3, nameof(RedisCommand.SENTINEL), "replicas", LockFree = true)] + private TypedRedisValue SentinelReplicas(RedisClient client, in RedisRequest request) => Replicas(request); + + [RedisCommand(3, nameof(RedisCommand.SENTINEL), "slaves", LockFree = true)] + private TypedRedisValue SentinelSlaves(RedisClient client, in RedisRequest request) => Replicas(request); + + // only this sentinel exists + [RedisCommand(3, nameof(RedisCommand.SENTINEL), "sentinels", LockFree = true)] + private TypedRedisValue SentinelSentinels(RedisClient client, in RedisRequest request) + => IsService(request) ? TypedRedisValue.EmptyArray(RespPrefix.Array) : TypedRedisValue.Error("ERR No such master with that name"); + + [RedisCommand(2, nameof(RedisCommand.SENTINEL), "masters", LockFree = true)] + private TypedRedisValue SentinelMasters(RedisClient client, in RedisRequest request) + { + var reply = TypedRedisValue.Rent(1, out var span, RespPrefix.Array); + span[0] = Describe(Primary, "master"); + return reply; + } + + private bool IsService(in RedisRequest request) => string.Equals(request.GetString(2), ServiceName, StringComparison.Ordinal); + + private TypedRedisValue Replicas(in RedisRequest request) + { + if (!IsService(request)) return TypedRedisValue.Error("ERR No such master with that name"); + + // sentinel learns the replicas from the primary, so report whatever replicates it right now + var replicas = new List(); + if (_monitored.TryGetNode(Primary, out var primaryNode)) + { + foreach (var endpoint in _monitored.GetEndPoints()) + { + if (_monitored.TryGetNode(endpoint, out var node) && node.PrimaryId == primaryNode.Id) + { + replicas.Add(endpoint); + } + } + } + + var reply = TypedRedisValue.Rent(replicas.Count, out var span, RespPrefix.Array); + for (int i = 0; i < replicas.Count; i++) + { + span[i] = Describe(replicas[i], "slave"); + } + return reply; + } + + private TypedRedisValue Describe(EndPoint endpoint, string flags) + { + var host = GetHost(endpoint, out var port); + var item = TypedRedisValue.Rent(8, out var span, RespPrefix.Array); + span[0] = TypedRedisValue.BulkString("name"); + span[1] = TypedRedisValue.BulkString(flags == "master" ? ServiceName : $"{host}:{port}"); + span[2] = TypedRedisValue.BulkString("ip"); + span[3] = TypedRedisValue.BulkString(host); + span[4] = TypedRedisValue.BulkString("port"); + span[5] = TypedRedisValue.BulkString(port.ToString()); + span[6] = TypedRedisValue.BulkString("flags"); + span[7] = TypedRedisValue.BulkString(flags); + return item; + } + + private sealed class RoutingTunnel(InProcessTestServer[] servers) : Tunnel + { + private Tunnel? Route(EndPoint endpoint) + { + foreach (var server in servers) + { + if (server.TryGetNode(endpoint, out _)) return server.Tunnel; + } + return null; + } + + public override ValueTask GetSocketConnectEndpointAsync(EndPoint endpoint, CancellationToken cancellationToken) + => Route(endpoint) is { } tunnel ? tunnel.GetSocketConnectEndpointAsync(endpoint, cancellationToken) : base.GetSocketConnectEndpointAsync(endpoint, cancellationToken); + + public override ValueTask BeforeAuthenticateAsync(EndPoint endpoint, ConnectionType connectionType, Socket? socket, CancellationToken cancellationToken) + => Route(endpoint) is { } tunnel ? tunnel.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken) : base.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken); + } +} diff --git a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs index 047b6d04d..726eb0bbb 100644 --- a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs +++ b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs @@ -181,7 +181,14 @@ public ConfigurationOptions GetClientConfig(bool withPubSub = true, bool default public override void Log(string message) { - _log?.WriteLine(message); + // the server's own background work (write loops, delayed closes after MOVING) can outlive the test that + // created it; xunit v3 throws on output with no active test, and from a fire-and-forget task that only + // surfaces as an unobserved exception at the end of the run + try + { + _log?.WriteLine(message); + } + catch (InvalidOperationException) { } base.Log(message); } diff --git a/tests/StackExchange.Redis.Tests/KeyTests.cs b/tests/StackExchange.Redis.Tests/KeyTests.cs index 1988fd44e..871b64ce8 100644 --- a/tests/StackExchange.Redis.Tests/KeyTests.cs +++ b/tests/StackExchange.Redis.Tests/KeyTests.cs @@ -80,6 +80,37 @@ public async Task KeyTypeOfMissingKeyIsNone() // see #3156 Assert.Equal(RedisType.None, await db.KeyTypeAsync(key)); } + [Fact] + public async Task PrimaryOnlyCommandsWithPreferReplicaRunOnThePrimary() // see #2821 + { + await using var conn = Create(configuration: TestConfig.Current.PrimaryServerAndPort + "," + TestConfig.Current.ReplicaServerAndPort); + Assert.SkipUnless( + conn.GetEndPoints().Any(ep => conn.GetServer(ep).IsReplica), + "No replica in this configuration"); + + var db = conn.GetDatabase(); + var key = Me(); + RedisValue token = Guid.NewGuid().ToString(); + + // these look at the server's features before building the message, which used to pick the replica; + // the fractional-second expiry forces the PEXPIRE path, which is the one that checks features + db.StringSet(key, "abc"); + Assert.True(db.KeyExpire(key, TimeSpan.FromMilliseconds(100_500), CommandFlags.PreferReplica)); + Assert.True(db.KeyDelete(key, CommandFlags.PreferReplica)); + + db.StringSet(key, "abc"); + Assert.True(await db.KeyExpireAsync(key, TimeSpan.FromMilliseconds(100_500), CommandFlags.PreferReplica)); + Assert.True(await db.KeyDeleteAsync(key, CommandFlags.PreferReplica)); + + Assert.True(db.LockTake(key, token, TimeSpan.FromMinutes(1))); + Assert.True(db.LockExtend(key, token, TimeSpan.FromMinutes(1), CommandFlags.PreferReplica)); + Assert.True(await db.LockExtendAsync(key, token, TimeSpan.FromMinutes(1), CommandFlags.PreferReplica)); + Assert.True(db.LockRelease(key, token, CommandFlags.PreferReplica)); + + Assert.True(db.LockTake(key, token, TimeSpan.FromMinutes(1))); + Assert.True(await db.LockReleaseAsync(key, token, CommandFlags.PreferReplica)); + } + [Fact] public async Task Zeros() { diff --git a/tests/StackExchange.Redis.Tests/LagAwareProbeTests.cs b/tests/StackExchange.Redis.Tests/LagAwareProbeTests.cs new file mode 100644 index 000000000..af95e3135 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/LagAwareProbeTests.cs @@ -0,0 +1,401 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Tests.MultiGroupTests; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// The lag-aware probe against a stubbed REST transport. Response shapes are the ones captured from real +/// Redis Enterprise 8.0.22 deployments (see notes/lag-aware/findings.md, section 10), minus credentials. +/// +public class LagAwareProbeTests(ITestOutputHelper log) +{ + private const string Host = "redis-14460.c1.example.com"; + private const int Port = 14460; + + private const string BdbList = """ + [{"endpoints":[{"addr":["54.243.16.166"],"addr_type":"external","dns_name":"redis-14460.c1.example.com","oss_cluster_api_preferred_endpoint_type":"ip","oss_cluster_api_preferred_ip_type":"internal","port":14460,"proxy_policy":"single","uid":"1:1"}],"uid":1}, + {"endpoints":[{"addr":["100.53.190.50"],"addr_type":"external","dns_name":"redis-12706.c1.example.com","oss_cluster_api_preferred_endpoint_type":"ip","oss_cluster_api_preferred_ip_type":"external","port":12706,"proxy_policy":"all-master-shards","uid":"2:1"}],"uid":2}] + """; + + private const string ActiveActive = """{"crdt":true,"uid":1}"""; + private const string NotActiveActive = """{"crdt":false,"uid":1}"""; + private const string Unavailable = """{"description":"BDB 1 is not available","error_code":"bdb_unavailable"}"""; + + // answers by path-and-query; records every request + private sealed class StubHandler(Func> respond) : HttpMessageHandler + { + public StubHandler(Func answer) + : this((request, _) => + { + var (status, body) = answer(request.RequestUri!.PathAndQuery); + var response = new HttpResponseMessage(status); + if (body is not null) response.Content = new StringContent(body, Encoding.UTF8, "application/json"); + return Task.FromResult(response); + }) + { + } + + public ConcurrentQueue Requests { get; } = new(); + + public IEnumerable Paths => Requests.Select(r => r.RequestUri!.PathAndQuery); + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Enqueue(request); + return respond(request, cancellationToken); + } + } + + // the cluster as measured: discovery, crdt lookup, then the availability answer + private static Func Cluster(bool activeActive, HttpStatusCode availability, string? availabilityBody = null) + => path => path switch + { + "/v1/bdbs?fields=uid,endpoints" => (HttpStatusCode.OK, BdbList), + "/v1/bdbs/1?fields=uid,crdt" => (HttpStatusCode.OK, activeActive ? ActiveActive : NotActiveActive), + _ when path.StartsWith("/v1/bdbs/1/availability", StringComparison.Ordinal) => (availability, availabilityBody), + _ => (HttpStatusCode.NotFound, null), + }; + + private int _credentialCalls; + + private LagAwareOptions Options(StubHandler handler) => new() + { + Credentials = _ => + { + Interlocked.Increment(ref _credentialCalls); + return new(new NetworkCredential("probe@example.com", "s3cret")); + }, + HttpMessageHandlerFactory = () => handler, + }; + + private async Task<(HealthCheckResult Result, StubHandler Handler)> CheckOnceAsync( + StubHandler handler, LagAwareOptions? options = null, TimeSpan? timeout = null, HealthCheckProbe? probe = null) + { + using var server = new InProcessTestServer(log, endpoint: new DnsEndPoint(Host, Port)); + await using var conn = await server.ConnectAsync(); + probe ??= HealthCheckProbe.LagAware(options ?? Options(handler)); + var context = new HealthCheckContext(conn.GetServer(new DnsEndPoint(Host, Port)), timeout ?? TimeSpan.FromSeconds(3)); + return (await probe.CheckHealthAsync(context), handler); + } + + [Fact] + public void ProbeIsMemberScoped() + => Assert.Equal(HealthCheckProbeScope.Member, HealthCheckProbe.LagAware(Options(new StubHandler(Cluster(true, HttpStatusCode.OK)))).Scope); + + [Fact] + public async Task DiscoversTheDatabaseThenAsksTheLagQuestionOfAnActiveActiveMember() + { + var (result, handler) = await CheckOnceAsync(new StubHandler(Cluster(activeActive: true, HttpStatusCode.OK))); + + Assert.Equal(HealthCheckResult.Healthy, result); + Assert.Equal( + ["/v1/bdbs?fields=uid,endpoints", "/v1/bdbs/1?fields=uid,crdt", "/v1/bdbs/1/availability?extend_check=lag"], + handler.Paths); + + // the cluster name, derived from the configured database host; Basic auth on every request + Assert.All(handler.Requests, r => + { + Assert.Equal("https://c1.example.com:9443", r.RequestUri!.GetLeftPart(UriPartial.Authority)); + Assert.Equal("Basic", r.Headers.Authorization?.Scheme); + Assert.Equal(Convert.ToBase64String(Encoding.UTF8.GetBytes("probe@example.com:s3cret")), r.Headers.Authorization?.Parameter); + }); + } + + [Fact] + public async Task NeverIssuesAnUnfilteredDatabaseRead() + { + // measured: unfiltered /v1/bdbs and /v1/crdbs return database passwords, even to db_viewer + var (_, handler) = await CheckOnceAsync(new StubHandler(Cluster(activeActive: true, HttpStatusCode.OK))); + Assert.All(handler.Paths, path => + { + Assert.DoesNotContain("/v1/crdbs", path); + if (!path.Contains("/availability")) Assert.Contains("fields=", path); + }); + } + + [Fact] + public async Task ChecksPlainAvailabilityOfADatabaseThatIsNotActiveActive() + { + // measured: the lag check reports every non-Active-Active database unavailable, whatever the tolerance + var (result, handler) = await CheckOnceAsync(new StubHandler(Cluster(activeActive: false, HttpStatusCode.OK))); + Assert.Equal(HealthCheckResult.Healthy, result); + Assert.Equal("/v1/bdbs/1/availability", handler.Paths.Last()); + } + + [Fact] + public async Task DiscoveryIsCachedAcrossPassesAndCredentialsAreReadEachTime() + { + var handler = new StubHandler(Cluster(activeActive: true, HttpStatusCode.OK)); + var probe = HealthCheckProbe.LagAware(Options(handler)); + await CheckOnceAsync(handler, probe: probe); + await CheckOnceAsync(handler, probe: probe); + + Assert.Equal(4, handler.Requests.Count); // three, then just the availability call + Assert.Equal("/v1/bdbs/1/availability?extend_check=lag", handler.Paths.Last()); + Assert.Equal(2, _credentialCalls); // rotation: asked once per check + } + + [Theory] + [InlineData(HttpStatusCode.OK, HealthCheckResult.Healthy)] + [InlineData(HttpStatusCode.ServiceUnavailable, HealthCheckResult.Unhealthy)] // unavailable, or lagging + [InlineData(HttpStatusCode.NotFound, HealthCheckResult.Unhealthy)] + [InlineData(HttpStatusCode.Unauthorized, HealthCheckResult.Inconclusive)] // management plane, not the database + [InlineData(HttpStatusCode.Forbidden, HealthCheckResult.Inconclusive)] + [InlineData(HttpStatusCode.InternalServerError, HealthCheckResult.Inconclusive)] + public async Task MapsTheAvailabilityAnswer(HttpStatusCode status, HealthCheckResult expected) + { + var (result, _) = await CheckOnceAsync(new StubHandler(Cluster(activeActive: true, status, status == HttpStatusCode.ServiceUnavailable ? Unavailable : null))); + Assert.Equal(expected, result); + } + + [Fact] + public async Task NotFoundForgetsTheDiscoveredDatabase() + { + var gone = false; + var handler = new StubHandler(path => gone && path.Contains("/availability") ? (HttpStatusCode.NotFound, null) : Cluster(true, HttpStatusCode.OK)(path)); + var probe = HealthCheckProbe.LagAware(Options(handler)); + Assert.Equal(HealthCheckResult.Healthy, (await CheckOnceAsync(handler, probe: probe)).Result); + + gone = true; + Assert.Equal(HealthCheckResult.Unhealthy, (await CheckOnceAsync(handler, probe: probe)).Result); + + gone = false; + await CheckOnceAsync(handler, probe: probe); + Assert.Equal(2, handler.Paths.Count(p => p == "/v1/bdbs?fields=uid,endpoints")); // rediscovered + } + + [Fact] + public async Task TransportFailureIsInconclusive() + { + var (result, _) = await CheckOnceAsync(new StubHandler((_, _) => throw new HttpRequestException("connection refused"))); + Assert.Equal(HealthCheckResult.Inconclusive, result); + } + + [Fact] + public async Task ASlowAnswerIsInconclusiveWithinTheProbeTimeout() + { + var handler = new StubHandler(async (_, cancellationToken) => + { + await Task.Delay(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK); + }); + var watch = System.Diagnostics.Stopwatch.StartNew(); + var (result, _) = await CheckOnceAsync(handler, timeout: TimeSpan.FromMilliseconds(300)); + Assert.Equal(HealthCheckResult.Inconclusive, result); + Assert.True(watch.Elapsed < TimeSpan.FromSeconds(10), $"took {watch.Elapsed}"); + } + + [Fact] + public async Task ADatabaseThatCannotBeFoundIsInconclusive() + { + var (result, handler) = await CheckOnceAsync(new StubHandler(path => (HttpStatusCode.OK, path.Contains("fields=uid,endpoints") ? "[]" : null))); + Assert.Equal(HealthCheckResult.Inconclusive, result); + Assert.Single(handler.Requests); + } + + [Fact] + public async Task ExplicitOptionsSkipDiscoveryAndSendTheTolerance() + { + var handler = new StubHandler(_ => (HttpStatusCode.OK, null)); + var options = Options(handler); + options.RestEndpoint = new Uri("https://cluster.example.com:9443/"); + options.DatabaseId = 7; + options.LagCheck = LagCheckMode.Enabled; + options.LagTolerance = TimeSpan.FromMilliseconds(250); + + var (result, _) = await CheckOnceAsync(handler, options); + Assert.Equal(HealthCheckResult.Healthy, result); + var request = Assert.Single(handler.Requests); + Assert.Equal("https://cluster.example.com:9443/v1/bdbs/7/availability?extend_check=lag&availability_lag_tolerance_ms=250", request.RequestUri!.ToString()); + } + + [Fact] + public async Task DisabledNeverAsksTheLagQuestion() + { + var handler = new StubHandler(Cluster(activeActive: true, HttpStatusCode.OK)); + var options = Options(handler); + options.LagCheck = LagCheckMode.Disabled; + var (_, _) = await CheckOnceAsync(handler, options); + Assert.DoesNotContain(handler.Paths, p => p.Contains("extend_check") || p.Contains("crdt")); + } + + [Theory] + [InlineData("redis-14460.c1.example.com", 14460, "c1.example.com")] // the form Redis Enterprise assigns + [InlineData("REDIS-14460.c1.example.com", 14460, "c1.example.com")] + [InlineData("redis-14460.c1.example.com", 6379, "redis-14460.c1.example.com")] // port does not match + [InlineData("cache.example.com", 14460, "cache.example.com")] // a name of the operator's own + [InlineData("redis-14460.", 14460, "redis-14460.")] + public void DerivesTheClusterNameFromTheDatabaseHost(string host, int port, string expected) + => Assert.Equal(expected, HealthCheckProbe.LagAwareProbe.ClusterHost(host, port)); + + [Fact] + public async Task ARedirectIsReportedNotFollowed() + { + // measured: non-master nodes redirect management routes to the master's internal address + var handler = new StubHandler((request, _) => + { + var response = new HttpResponseMessage(HttpStatusCode.TemporaryRedirect); + response.Headers.Location = new Uri("https://10.0.101.184:9443" + request.RequestUri!.PathAndQuery); + return Task.FromResult(response); + }); + var (result, _) = await CheckOnceAsync(handler); + Assert.Equal(HealthCheckResult.Inconclusive, result); + Assert.Single(handler.Requests); + } + + [Fact] + public void FindsTheDatabaseByAddressAsWellAsByName() + => Assert.Equal(2, HealthCheckProbe.LagAwareProbe.FindUid(BdbList, "100.53.190.50", 12706)); + + // public-only and self-signed (CN=lag-aware-probe-test, private key discarded at generation): enough to show + // the certificate reaching the transport, where generating one is not possible (Mono has no CertificateRequest) + private const string PublicOnlyCertificate = "MIIDITCCAgmgAwIBAgIUbLJxUJKq4SkB9FnIFFLUTKBAaSkwDQYJKoZIhvcNAQELBQAwHzEdMBsGA1UEAwwUbGFnLWF3YXJlLXByb2JlLXRlc3QwIBcNMjYxMDA4MTUzNjU0WhgPMjEyNjA5MTQxNTM2NTRaMB8xHTAbBgNVBAMMFGxhZy1hd2FyZS1wcm9iZS10ZXN0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxXfFEzpB82vcUAwfk75Blw1BYG57k9XXIBqa0dx7KxivuF0iSZsSPGViBs6Pwjx7Yvq6+FRZHusFEZ0s3qRbbSf5p7dZhAJ4Oeb5TjqYj7rNQaoCjz2+W4eWbSpVpd8If+g3qy/U+C3gTdIPOUu9biExNj9iN/dUt4o/0S3vgp0iwUiP/E2TcnFRk0SHLgo0Irt1U12FWSpdJA+p65OoG4h/xbsO70qqn/ifyLX8B+eYBOaqQi8myls3MPef2b/pmQlMYUaTGR5kT2EY3pRusTEhdpC1xDrmYxUnxzptgkcAval7HjXEU0sB7c7atGgTaQlUSv8CsrOk/wtQ4Lp54QIDAQABo1MwUTAdBgNVHQ4EFgQUbo8ar3s44TIDl8yYq4Hc8TsTs9UwHwYDVR0jBBgwFoAUbo8ar3s44TIDl8yYq4Hc8TsTs9UwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAVMrCof8H0vBm2UBSxSQreCblVlz41Yinkp1Q2B+1s1CkvTXGhTPS3iJv2FYViJ5RsC36nG5e35vdW3L4r0hjn1lZA32lX2XAYUsrjx7YaQBqvm6qYiK9HqFs2uJUNN7EtDZsWPMN+SRoRAkcQsTZlMvf2qKIkcnulU36wQ+vEdEmLRRsUmIbdPqhT09kJGx0X7jVUmD6PnuAyk8/OJPAGQM+uCvS43Gfdm+ywQ9+rQIDs7F3lU9g7D0o8OS3yY7WY87vMbiwh7eUlDmn74ypCXSTpLTEy4OtP4Th+eXrZGV5+ExmVwzbeWucC5nCMQJ4+ZIpkjW6h0tb/4WTnfSE3w=="; + +#if NET + private static System.Security.Cryptography.X509Certificates.X509Certificate2 CreateClientCertificate() + { + using var rsa = System.Security.Cryptography.RSA.Create(2048); + var request = new System.Security.Cryptography.X509Certificates.CertificateRequest( + "CN=lag-aware-probe-test", rsa, System.Security.Cryptography.HashAlgorithmName.SHA256, System.Security.Cryptography.RSASignaturePadding.Pkcs1); + return request.CreateSelfSigned(DateTimeOffset.UtcNow.AddMinutes(-5), DateTimeOffset.UtcNow.AddHours(1)); + } + + [Fact] + public void LoadsAClientCertificateFromPfx() + { + using var certificate = CreateClientCertificate(); + var path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"lagaware-{Guid.NewGuid():n}.pfx"); + try + { + System.IO.File.WriteAllBytes(path, certificate.Export(System.Security.Cryptography.X509Certificates.X509ContentType.Pfx, "pw")); + var options = new LagAwareOptions(); + options.SetUserPfxCertificate(path, "pw"); + Assert.Equal(certificate.Thumbprint, options.ClientCertificate?.Thumbprint); + Assert.True(options.ClientCertificate!.HasPrivateKey); + } + finally + { + System.IO.File.Delete(path); + } + } + + [Fact] + public void LoadsAClientCertificateFromPem() + { + using var certificate = CreateClientCertificate(); + var dir = System.IO.Directory.CreateTempSubdirectory("lagaware-").FullName; + try + { + var crt = System.IO.Path.Combine(dir, "client.crt"); + var key = System.IO.Path.Combine(dir, "client.key"); + System.IO.File.WriteAllText(crt, certificate.ExportCertificatePem()); + System.IO.File.WriteAllText(key, System.Security.Cryptography.X509Certificates.RSACertificateExtensions.GetRSAPrivateKey(certificate)!.ExportPkcs8PrivateKeyPem()); + var options = new LagAwareOptions(); + options.SetUserPemCertificate(crt, key); + Assert.Equal(certificate.Thumbprint, options.ClientCertificate?.Thumbprint); + Assert.True(options.ClientCertificate!.HasPrivateKey); + } + finally + { + System.IO.Directory.Delete(dir, recursive: true); + } + } +#endif + + [Fact] + public void TheClientCertificateReachesTheTransport() + { +#pragma warning disable SYSLIB0057 // X509 loading; X509CertificateLoader is not available on every test TFM + using var certificate = new System.Security.Cryptography.X509Certificates.X509Certificate2(Convert.FromBase64String(PublicOnlyCertificate)); +#pragma warning restore SYSLIB0057 + using var handler = HealthCheckProbe.LagAwareProbe.CreateHandler(validation: null, certificate); +#if NET + var sockets = Assert.IsType(handler); + Assert.Contains(certificate, sockets.SslOptions.ClientCertificates!.Cast()); + Assert.False(sockets.AllowAutoRedirect); +#else + var client = Assert.IsType(handler); + Assert.Equal(ClientCertificateOption.Manual, client.ClientCertificateOptions); + Assert.Contains(certificate, client.ClientCertificates.Cast()); + Assert.False(client.AllowAutoRedirect); +#endif + } + + [Fact] + public void RejectsInvalidOptions() + { + Assert.Throws(() => HealthCheckProbe.LagAware(null!)); + Assert.Throws(() => HealthCheckProbe.LagAware(new LagAwareOptions())); + Assert.Throws(() => HealthCheckProbe.LagAware(new LagAwareOptions + { + Credentials = _ => new(new NetworkCredential()), + LagTolerance = TimeSpan.FromMilliseconds(-1), + })); + } + + [Fact] + public async Task AsAFailbackCheckItKeepsTrafficOffALaggingMember() + { + // end to end at the stub level: beta has the higher weight, but its cluster reports it lagging + var alphaEp = new DnsEndPoint("redis-14460.c1.example.com", 14460); + var betaEp = new DnsEndPoint("redis-14460.c2.example.com", 14460); + using var alpha = new InProcessTestServer(log, endpoint: alphaEp); + using var beta = new InProcessTestServer(log, endpoint: betaEp); + + var handler = new StubHandler((request, _) => + { + var path = request.RequestUri!.PathAndQuery; + var isC2 = request.RequestUri.Host == "c2.example.com"; // the cluster names, derived from the members' hosts + (HttpStatusCode Status, string? Body) answer = path switch + { + "/v1/bdbs?fields=uid,endpoints" => (HttpStatusCode.OK, isC2 ? BdbList.Replace(".c1.", ".c2.") : BdbList), + "/v1/bdbs/1?fields=uid,crdt" => (HttpStatusCode.OK, ActiveActive), + _ => isC2 ? (HttpStatusCode.ServiceUnavailable, Unavailable) : (HttpStatusCode.OK, null), + }; + var response = new HttpResponseMessage(answer.Status); + if (answer.Body is not null) response.Content = new StringContent(answer.Body); + return Task.FromResult(response); + }); + + // liveness held down until both members are connected, so the first real selection sees the whole group + var gate = new GateProbe(); + MultiGroupOptions groupOptions = new MultiGroupOptions.Builder + { + HealthCheck = new HealthCheck.Builder { Probe = gate, ProbeCount = 1 }, + FailbackHealthCheck = HealthCheck.LagAware(Options(handler)), + HealthCheckInterval = TimeSpan.MaxValue, + }; + ConnectionGroupMember[] members = [new(alpha.GetClientConfig(), "alpha") { Weight = 1 }, new(beta.GetClientConfig(), "beta") { Weight = 9 }]; + await using var group = await ConnectionMultiplexer.ConnectGroupAsync(members, groupOptions); + + var watch = System.Diagnostics.Stopwatch.StartNew(); + while (!Array.TrueForAll(members, m => m.Multiplexer.IsConnected) && watch.Elapsed < TimeSpan.FromSeconds(10)) + { + await Task.Delay(25, TestContext.Current.CancellationToken); + } + gate.Open = true; + await MultiGroupMultiplexer.TryHealthCheckAndSelectPreferredGroupAsync(group); + await GroupWait.AssertConnectedAsync(group); + + Assert.Equal("alpha", group.ActiveMember?.Name); + Assert.False(members[1].FailbackVerified); + Assert.Contains(handler.Requests, r => r.RequestUri!.Host == "c2.example.com" && r.RequestUri.AbsolutePath == "/v1/bdbs/1/availability"); + } + + private sealed class GateProbe : HealthCheckProbe + { + public volatile bool Open; + + public override Task CheckHealthAsync(HealthCheckContext context) => Open ? HealthyTask : UnhealthyTask; + } +} diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/FailbackHealthCheckTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/FailbackHealthCheckTests.cs new file mode 100644 index 000000000..9de4768c5 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/FailbackHealthCheckTests.cs @@ -0,0 +1,333 @@ +using System; +using System.Collections.Concurrent; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests.MultiGroupTests; + +/// +/// The failback health check: consulted only before traffic moves to a member, never evicting the active one, +/// with selection falling back to liveness when no live member is eligible. +/// +public class FailbackHealthCheckTests(ITestOutputHelper log) +{ + private static readonly DnsEndPoint Alpha = new("alpha", 6379), Beta = new("beta", 6379), Gamma = new("gamma", 6379); + + // answers per endpoint as scripted by the test (default Healthy), counting calls + private sealed class ScriptedProbe(HealthCheckProbeScope scope = HealthCheckProbeScope.Endpoint) : HealthCheckProbe + { + private readonly ConcurrentDictionary _answers = new(); + + public ConcurrentDictionary Calls { get; } = new(); + + public override HealthCheckProbeScope Scope => scope; + + public void Set(EndPoint endpoint, HealthCheckResult result) => _answers[endpoint] = result; + + public override Task CheckHealthAsync(HealthCheckContext context) + { + var endpoint = context.Server.EndPoint; + Calls.AddOrUpdate(endpoint, 1, (_, n) => n + 1); + return Task.FromResult(_answers.TryGetValue(endpoint, out var result) ? result : HealthCheckResult.Healthy); + } + } + + private static HealthCheck Single(HealthCheckProbe probe) => new HealthCheck.Builder { Probe = probe, ProbeCount = 1 }; + + private sealed class Fixture : IAsyncDisposable + { + private readonly InProcessTestServer[] _servers; + private readonly DnsEndPoint[] _endpoints; + + public Fixture(ITestOutputHelper log, params (DnsEndPoint Endpoint, double Weight)[] members) + { + _servers = new InProcessTestServer[members.Length]; + _endpoints = Array.ConvertAll(members, m => m.Endpoint); + Members = new ConnectionGroupMember[members.Length]; + for (int i = 0; i < members.Length; i++) + { + _servers[i] = new InProcessTestServer(log, endpoint: members[i].Endpoint); + Members[i] = new(_servers[i].GetClientConfig(), members[i].Endpoint.Host) { Weight = members[i].Weight }; + } + } + + public ConnectionGroupMember[] Members { get; } + + public ScriptedProbe Liveness { get; } = new(); + + public ScriptedProbe Failback { get; } = new(); + + public IConnectionGroup Group { get; private set; } = null!; + + public async Task ConnectAsync() + { + MultiGroupOptions options = new MultiGroupOptions.Builder + { + HealthCheck = Single(Liveness), + FailbackHealthCheck = Single(Failback), + HealthCheckInterval = TimeSpan.MaxValue, // passes are driven by the test + }; + // Hold every member down until all of their connections are up, so the first real selection + // sees the whole group: otherwise whichever member connects first can be selected by the liveness + // fallback (provisionally) before the others are up - a race, not the rule under test. + foreach (var endpoint in _endpoints) Liveness.Set(endpoint, HealthCheckResult.Unhealthy); + Group = await ConnectionMultiplexer.ConnectGroupAsync(Members, options); + + var watch = System.Diagnostics.Stopwatch.StartNew(); + while (!Array.TrueForAll(Members, m => m.Multiplexer.IsConnected) && watch.Elapsed < TimeSpan.FromSeconds(10)) + { + await Task.Delay(25, TestContext.Current.CancellationToken); + } + Assert.True(Array.TrueForAll(Members, m => m.Multiplexer.IsConnected), "members did not connect"); + + foreach (var endpoint in _endpoints) Liveness.Set(endpoint, HealthCheckResult.Healthy); + await PassAsync(); + await GroupWait.AssertConnectedAsync(Group); + } + + public void MarkDown(EndPoint endpoint) => Liveness.Set(endpoint, HealthCheckResult.Unhealthy); + + public Task PassAsync() => MultiGroupMultiplexer.TryHealthCheckAndSelectPreferredGroupAsync(Group); + + public string? Active => Group.ActiveMember?.Name; + + public async ValueTask DisposeAsync() + { + if (Group is not null) await Group.DisposeAsync(); + foreach (var server in _servers) server.Dispose(); + } + } + + [Fact] + public async Task IneligibleMemberIsNotSelectedUntilItPasses() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); // beta: higher weight, but behind + await f.ConnectAsync(); + Assert.Equal("alpha", f.Active); + + f.Failback.Set(Beta, HealthCheckResult.Healthy); // caught up + await f.PassAsync(); + Assert.Equal("beta", f.Active); + Assert.False(f.Members[1].IsUnhealthy); // eligibility is not health + } + + [Fact] + public async Task InconclusiveIsNotEligible() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + f.Failback.Set(Beta, HealthCheckResult.Inconclusive); // "could not ask" + await f.ConnectAsync(); + Assert.Equal("alpha", f.Active); + } + + [Fact] + public async Task ActiveMemberIsNeverEvictedByItsFailbackCheck() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); + + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); + await f.PassAsync(); + Assert.Equal("beta", f.Active); + + // checked once, on the first real pass when nothing was active yet; never while active + Assert.True(f.Failback.Calls.TryGetValue(Beta, out var calls)); + Assert.Equal(1, calls); + } + + [Fact] + public async Task NoEligibleMemberFallsBackToLiveness() + { + // every member lagging - the normal state of a partition between regions - must still serve + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + f.Failback.Set(Alpha, HealthCheckResult.Unhealthy); + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); + } + + [Fact] + public async Task AFallbackChoiceIsLeftForAnEligibleMember() + { + // other clients never select a member that fails its lag check; the nearest equivalent that still + // serves when every member is behind is to leave a fallback choice as soon as anything is eligible + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + f.Failback.Set(Alpha, HealthCheckResult.Unhealthy); + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); // provisional + + f.Failback.Set(Alpha, HealthCheckResult.Healthy); + await f.PassAsync(); + Assert.Equal("alpha", f.Active); // eligible beats provisional, whatever the weights + + f.Failback.Set(Beta, HealthCheckResult.Healthy); + await f.PassAsync(); + Assert.Equal("beta", f.Active); // and ordinary failback by weight resumes + } + + [Fact] + public async Task AProvisionalMemberThatCatchesUpBecomesOrdinary() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + f.Failback.Set(Alpha, HealthCheckResult.Unhealthy); + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); // provisional + + f.Failback.Set(Beta, HealthCheckResult.Healthy); // still checked while provisional + await f.PassAsync(); + Assert.Equal("beta", f.Active); // verified: no longer provisional + + // now an ordinary active member: never evicted by its own failback check + f.Failback.Set(Alpha, HealthCheckResult.Healthy); + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); + await f.PassAsync(); + Assert.Equal("beta", f.Active); + } + + [Fact] + public async Task DeadActiveIsReplacedEvenWhenNoSurvivorIsEligible() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); + + f.MarkDown(Beta); // the active region is gone... + f.Failback.Set(Alpha, HealthCheckResult.Unhealthy); // ...and the survivor reports lag because of it + await f.PassAsync(); + Assert.Equal("alpha", f.Active); + } + + [Fact] + public async Task DeadActivePrefersAnEligibleSurvivor() + { + await using var f = new Fixture(log, (Alpha, 1), (Gamma, 5), (Beta, 9)); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); + + f.MarkDown(Beta); + f.Failback.Set(Gamma, HealthCheckResult.Unhealthy); // higher weight, but behind + await f.PassAsync(); + Assert.Equal("alpha", f.Active); + } + + [Fact] + public async Task ExplicitFailoverOverridesIneligibility() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + await f.ConnectAsync(); + f.Failback.Set(Alpha, HealthCheckResult.Unhealthy); + await f.PassAsync(); + Assert.Equal("beta", f.Active); + + Assert.True(f.Group.TryFailoverTo(f.Members[0])); + Assert.Equal("alpha", f.Active); + } + + [Fact] + public async Task FormerActiveMustBeCheckedAfreshBeforeReturning() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + await f.ConnectAsync(); + await f.PassAsync(); // beta active: its verdict is not kept while active + Assert.Equal("beta", f.Active); + + Assert.True(f.Group.TryFailoverTo(f.Members[0])); + Assert.Equal("alpha", f.Active); + + // removing the override reselects at once, but beta has no current verdict, so it is not eligible yet + Assert.True(f.Group.TryFailoverTo(null)); + Assert.Equal("alpha", f.Active); + + await f.PassAsync(); // now checked, and healthy + Assert.Equal("beta", f.Active); + } + + [Fact] + public async Task MemberScopedProbeRunsOncePerMember() + { + using var server = new InProcessTestServer(log) { ServerType = ServerType.Cluster }; + GetHost(server.DefaultEndPoint, out var port); + var other = server.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, port + 1)); + server.Migrate((RedisKey)"scope-key", other); + await using var conn = await server.ConnectAsync(); + Assert.Equal(2, conn.GetServers().Length); + + // an endpoint that is not yet connected is reported unhealthy without being probed, so wait for both + var watch = System.Diagnostics.Stopwatch.StartNew(); + while (!Array.TrueForAll(conn.GetServers(), s => s.IsConnected) && watch.Elapsed < TimeSpan.FromSeconds(10)) + { + await Task.Delay(25, TestContext.Current.CancellationToken); + } + Assert.True(Array.TrueForAll(conn.GetServers(), s => s.IsConnected)); + + var perEndpoint = new ScriptedProbe(HealthCheckProbeScope.Endpoint); + var perMember = new ScriptedProbe(HealthCheckProbeScope.Member); + Assert.Equal(HealthCheckResult.Healthy, await Single(perEndpoint).CheckHealthAsync(conn)); + Assert.Equal(HealthCheckResult.Healthy, await Single(perMember).CheckHealthAsync(conn)); + + Assert.Equal(2, perEndpoint.Calls.Count); + Assert.Single(perMember.Calls); + Assert.All(perMember.Calls.Values, n => Assert.Equal(1, n)); + } + + [Theory] + [InlineData(nameof(HealthCheckProbePolicy.AllSuccess))] + [InlineData(nameof(HealthCheckProbePolicy.AnySuccess))] + [InlineData(nameof(HealthCheckProbePolicy.MajoritySuccess))] + public async Task ARunOfInconclusiveProbesIsInconclusiveUnderEveryPolicy(string policy) + { + // "could not tell" is neither a pass nor a failure, whichever policy collates it + using var server = new InProcessTestServer(log, endpoint: Alpha); + await using var conn = await server.ConnectAsync(); + var probe = new ScriptedProbe(); + probe.Set(Alpha, HealthCheckResult.Inconclusive); + HealthCheck check = new HealthCheck.Builder + { + Probe = probe, + ProbeCount = 3, + ProbeInterval = TimeSpan.Zero, + ProbePolicy = policy switch + { + nameof(HealthCheckProbePolicy.AllSuccess) => HealthCheckProbePolicy.AllSuccess, + nameof(HealthCheckProbePolicy.AnySuccess) => HealthCheckProbePolicy.AnySuccess, + _ => HealthCheckProbePolicy.MajoritySuccess, + }, + }; + + Assert.Equal(HealthCheckResult.Inconclusive, await check.CheckHealthAsync(conn)); + } + + [Fact] + public async Task AFailureInTheSameClockTickAsTheCutoffStillFailsBack() + { + // FailbackDelay zero means immediate: a passing check clears a failure recorded at the cutoff itself, + // which on .NET Framework's ~15.6ms clock is the ordinary case for two passes in quick succession + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + await f.ConnectAsync(); + var member = f.Members[0]; + + member.SetUnhealthy(); + Assert.True(member.IsUnhealthy); + member.UpdateState(HealthCheckResult.Healthy, failbackFailureCutoffTicks: member.LastUnhealthyTicks); + Assert.False(member.IsUnhealthy); + } + + [Fact] + public void OptionsCarryTheFailbackHealthCheck() + { + Assert.Null(MultiGroupOptions.Default.FailbackHealthCheck); + Assert.Same(MultiGroupOptions.Default, new MultiGroupOptions.Builder().Create()); + + var check = Single(new ScriptedProbe()); + MultiGroupOptions options = new MultiGroupOptions.Builder { FailbackHealthCheck = check }; + Assert.Same(check, options.FailbackHealthCheck); + Assert.Same(check, new MultiGroupOptions.Builder(options).Create().FailbackHealthCheck); + } +} diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/LoadingRerouteTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/LoadingRerouteTests.cs new file mode 100644 index 000000000..e03a26d03 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/LoadingRerouteTests.cs @@ -0,0 +1,78 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests.MultiGroupTests; + +[RunPerProtocol] +public class LoadingRerouteTests(ITestOutputHelper log) : TestBase(log) +{ + // A member answering LOADING must steer the group away promptly, via the same ConnectionFailed fast-path + // as a circuit-breaker trip. The breaker cannot be what moves us here: once the server is loading, every + // reconnect fails during the handshake, so later commands die in the backlog without a reply, and the + // breaker only ever observes replies. As with the breaker test, the poll interval is enormous so that the + // fast-path is the only thing that can reroute within the test window, and the probe holds A down so the + // reroute is deterministic. + [Fact] + public async Task LoadingMember_ReroutesAwayPromptly() + { + EndPoint alpha = new DnsEndPoint("alpha", 6379); + EndPoint bravo = new DnsEndPoint("bravo", 6379); + + using var serverA = new InProcessTestServer(Output, endpoint: alpha); + using var serverB = new InProcessTestServer(Output, endpoint: bravo); + + var probe = new ControllableProbe(); + + ConnectionGroupMember[] members = + [ + new(serverA.GetClientConfig(), "A") { Weight = 9 }, // highest weight -> initially active + new(serverB.GetClientConfig(), "B") { Weight = 1 }, // failover target + ]; + + MultiGroupOptions options = new MultiGroupOptions.Builder + { + HealthCheckInterval = TimeSpan.FromMinutes(30), // enormous: the fast-path is what reroutes us + HealthCheck = new HealthCheck.Builder + { + Probe = probe, + ProbeCount = 1, + ProbeTimeout = TimeSpan.FromSeconds(5), + }, + }; + + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + var typed = Assert.IsType(conn); + + var loadingEvents = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + typed.ConnectionFailed += (_, e) => + { + Log($"ConnectionFailed: {e.FailureType} @ {e.EndPoint}"); + if (e.FailureType == ConnectionFailureType.Loading) + { + loadingEvents.TrySetResult(true); + } + }; + + await GroupWait.AssertConnectedAsync(conn); + Assert.Same(members[0], conn.ActiveMember); + + // knock A into LOADING and hold it down; the next command to A is what surfaces the failure + probe.MarkDown(alpha); + serverA.IsLoading = true; + var fault = await Assert.ThrowsAnyAsync(() => conn.GetDatabase().PingAsync()); + Log($"observed fault: {fault.GetType().Name}: {fault.Message}"); + + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (DateTime.UtcNow < deadline && ReferenceEquals(conn.ActiveMember, members[0])) + { + await Task.Delay(50, TestContext.Current.CancellationToken); + } + Assert.Same(members[1], conn.ActiveMember); + + var completed = await Task.WhenAny(loadingEvents.Task, Task.Delay(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.True(ReferenceEquals(completed, loadingEvents.Task), "expected a ConnectionFailed event with FailureType == Loading"); + } +} diff --git a/tests/StackExchange.Redis.Tests/MultiMessageWriteTimeTests.cs b/tests/StackExchange.Redis.Tests/MultiMessageWriteTimeTests.cs new file mode 100644 index 000000000..4a1f0c408 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MultiMessageWriteTimeTests.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// The sub-commands of a transaction (and the SELECT/READONLY/ASKING prefixes written ahead of a command) are +/// added to the awaiting-response queue without going through the normal enqueue path. If they carry no write +/// time, the heartbeat sees them as overdue by the process uptime, and on a connection that has been idle for +/// a while that is enough to trip the dead-socket heuristic while the reply is still in flight; see #3292. +/// +[Collection(NonParallelCollection.Name)] +public class MultiMessageWriteTimeTests(ITestOutputHelper output) : TestBase(output) +{ + [Fact] + public async Task HeartbeatDuringTransactionOnIdleConnectionDoesNotTearItDown() + { + const int Timeout = 600; + var logs = new CapturingLoggerFactory(); + using var server = new InProcessTestServer(Output); + var config = server.GetClientConfig(defaultOnly: true); + config.SyncTimeout = config.AsyncTimeout = Timeout; + config.HeartbeatInterval = TimeSpan.FromHours(1); // only the heartbeats this test issues + config.KeepAlive = 600; // no pings resetting the last-read clock during the idle period + config.LoggerFactory = logs; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var db = conn.GetDatabase(1); // a non-default database, so a SELECT prefix is written ahead of the transaction + var key = Me(); + await db.StringSetAsync(key, "before"); + + // the heuristic needs the connection silent for more than 4x the timeout (whole seconds) before the write + await Task.Delay((4 * Timeout) + 1_000); + + // replies now take longer than the gap to the heartbeat, but well inside the timeout + server.SetLatency(TimeSpan.FromMilliseconds(250)); + var tran = db.CreateTransaction(); + tran.AddCondition(Condition.StringEqual(key, "before")); // WATCH + GET, written by the transaction itself + _ = tran.StringSetAsync(key, "after"); + var exec = Task.Run(() => tran.ExecuteAsync()); + + await Task.Delay(50); // the sub-commands are on the wire, their replies are not back yet + conn.OnHeartbeat(); + + var committed = await exec.ContinueWith(t => t.IsCompletedSuccessfully && t.Result, TaskScheduler.Default); + Assert.DoesNotContain(logs.Lines, l => l.Contains("Dead socket", StringComparison.Ordinal)); + Assert.True(committed, "the transaction should have committed"); + Assert.Equal("after", await db.StringGetAsync(key)); + } + + private sealed class CapturingLoggerFactory : ILoggerFactory, ILogger + { + private readonly List _lines = []; + + public IReadOnlyList Lines + { + get { lock (_lines) return [.. _lines]; } + } + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + lock (_lines) _lines.Add(formatter(state, exception)); + } + + public void Dispose() { } + } +} diff --git a/tests/StackExchange.Redis.Tests/ProductVariantUnitTests.cs b/tests/StackExchange.Redis.Tests/ProductVariantUnitTests.cs index 828df3435..bd4a6e54d 100644 --- a/tests/StackExchange.Redis.Tests/ProductVariantUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/ProductVariantUnitTests.cs @@ -1,10 +1,13 @@ -using System.Linq; +using System; +using System.Collections.Generic; +using System.Linq; using System.Text; using System.Threading.Tasks; using Xunit; namespace StackExchange.Redis.Tests; +[RunPerProtocol] public class ProductVariantUnitTests(ITestOutputHelper log) { [Theory] @@ -14,19 +17,76 @@ public class ProductVariantUnitTests(ITestOutputHelper log) public async Task DetectProductVariant(ProductVariant variant) { using var serverObj = new ProductServer(variant, log); - using var conn = await serverObj.ConnectAsync(withPubSub: false); + // with pub/sub, both connections issue HELLO, in either order relative to the interactive INFO + using var conn = await serverObj.ConnectAsync(withPubSub: true); var serverApi = conn.GetServer(conn.GetEndPoints().First()); serverApi.Ping(); var reportedProduct = serverApi.GetProductVariant(out var reportedVersion); Assert.Equal(variant, reportedProduct); - log.WriteLine($"Detected {reportedProduct} version: {reportedVersion}"); + log.WriteLine($"Detected {reportedProduct} version: {reportedVersion}; redis API version: {serverApi.Version}"); if (variant == ProductVariant.Redis) { Assert.Equal(serverObj.VersionString, reportedVersion); } else { - Assert.Equal("1.2.3-preview4", reportedVersion); + Assert.Equal(ProductServer.VendorVersion, reportedVersion); + } + // the Redis API version must not be confused with the vendor's own version (#3278) + Assert.Equal(serverObj.RedisVersion, serverApi.Version); + } + + [Theory] + [InlineData(ProductVariant.Redis)] + [InlineData(ProductVariant.Valkey)] + [InlineData(ProductVariant.Garnet)] + public async Task DetectProductVariantFromHelloAlone(ProductVariant variant) + { + // with INFO disabled, HELLO is the only source of version data, so this pins down what we take from it: + // "version" is the Redis API version only when "server" is redis (Redis, Garnet); Valkey repurposes it (#3278) + using var serverObj = new ProductServer(variant, log); + var config = serverObj.GetClientConfig(withPubSub: false); + var commands = serverObj.GetCommands(); + commands.Remove(nameof(RedisCommand.INFO)); + config.CommandMap = CommandMap.Create(commands); + var fallbackVersion = new Version(6, 0, 0); // HELLO exists from 6.0, and the fake server reports 7.0.0 + config.DefaultVersion = fallbackVersion; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var serverApi = conn.GetServer(conn.GetEndPoints().First()); + await serverApi.PingAsync(); + Assert.Equal(variant, serverApi.GetProductVariant(out var reportedVersion)); + log.WriteLine($"Detected {variant} version: {reportedVersion}; redis API version: {serverApi.Version}"); + switch (variant) + { + case ProductVariant.Redis: + Assert.Equal(serverObj.RedisVersion, serverApi.Version); + break; + case ProductVariant.Garnet: + Assert.Equal(serverObj.RedisVersion, serverApi.Version); + Assert.Equal(ProductServer.VendorVersion, reportedVersion); + break; + case ProductVariant.Valkey: + Assert.Equal(fallbackVersion, serverApi.Version); // HELLO told us nothing about the Redis API version + Assert.Equal(ProductServer.VendorVersion, reportedVersion); + break; + } + } + + [Fact] + public async Task ValkeyVersionIsStableAcrossConnections() + { + // #3278: Valkey reports its own version as HELLO "version", so the detected Redis API version used to + // depend on whether the (subscription) HELLO or the (interactive) INFO reply was processed last + using var serverObj = new ProductServer(ProductVariant.Valkey, log); + for (int i = 0; i < 10; i++) + { + await using var conn = await serverObj.ConnectAsync(withPubSub: true); + var serverApi = conn.GetServer(conn.GetEndPoints().First()); + await serverApi.PingAsync(); + Assert.Equal(serverObj.RedisVersion, serverApi.Version); + Assert.Equal(ProductVariant.Valkey, serverApi.GetProductVariant(out var reportedVersion)); + Assert.Equal(ProductServer.VendorVersion, reportedVersion); } } @@ -73,6 +133,8 @@ public async Task MultiDbSupportMatchesProductVariantAndServerType(ProductVarian private sealed class ProductServer : InProcessTestServer { + public const string VendorVersion = "9.1.2"; // must parse as a Version, or the HELLO-version confusion goes unnoticed + private readonly ProductVariant _variant; public ProductServer(ProductVariant variant, ITestOutputHelper log, ServerType serverType = ServerType.Standalone) @@ -90,16 +152,38 @@ protected override void Info(StringBuilder sb, string section, Server.RedisClien switch (_variant) { case ProductVariant.Garnet: - sb.AppendLine("garnet_version:1.2.3-preview4"); + sb.AppendLine("garnet_version:" + VendorVersion) + .AppendLine("server_name:garnet"); break; case ProductVariant.Valkey: - sb.AppendLine("valkey_version:1.2.3-preview4") + sb.AppendLine("valkey_version:" + VendorVersion) .AppendLine("server_name:valkey"); break; } } } + // HELLO shapes as measured against real servers (#3278): + // Garnet 2.2.2: server => redis, version => 7.4.3 (the Redis API version), garnet_version => 2.2.2 + // Valkey 9.1.2: server => valkey, version => 9.1.2 (Valkey's own version; INFO has redis_version:7.2.4) + protected override string HelloServerName => _variant switch + { + ProductVariant.Valkey => "valkey", + _ => "redis", + }; + + protected override string HelloVersionString => _variant switch + { + ProductVariant.Valkey => VendorVersion, + _ => VersionString, + }; + + protected override IReadOnlyList> HelloExtraFields => _variant switch + { + ProductVariant.Garnet => [new("garnet_version", VendorVersion)], + _ => [], + }; + protected override bool SupportMultiDb(out string err) { switch (_variant) diff --git a/tests/StackExchange.Redis.Tests/RespMultiplexerExecutorTests.cs b/tests/StackExchange.Redis.Tests/RespMultiplexerExecutorTests.cs index ff746c0f6..b90c926f1 100644 --- a/tests/StackExchange.Redis.Tests/RespMultiplexerExecutorTests.cs +++ b/tests/StackExchange.Redis.Tests/RespMultiplexerExecutorTests.cs @@ -49,10 +49,26 @@ private sealed class Router } } + /// + /// What discovery tells the topology when it finds a cluster: the server type and a slot map. + /// + /// + /// Routing by slot needs both (, main #3285): a server that merely + /// says "cluster" while giving no map is routed as standalone, so these tests, whose + /// answers the slot question itself, give the topology a map covering every slot to stand for "SLOTS was + /// answered". The owner recorded there is never consulted - the router is. + /// + private static void LearnCluster(RespTopology topology) + { + topology.OnServerType(ServerType.Cluster); + topology.SetSlotRange(0, ServerSelectionStrategy.TotalSlots - 1, new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 6379)); + } + private static (RespDatabaseContext Context, RespTopology Topology, Router Router) Build( RespClusterState known = RespClusterState.No) { var topology = new RespTopology(known); + if (known == RespClusterState.Yes) LearnCluster(topology); // a known cluster comes with its map var router = new Router { Default = new Endpoint("default", "$2\r\nok\r\n") }; var executor = new RespMultiplexerExecutor(topology, router.ForSlot, router.Any); var raw = new RespContext(serverType: ServerType.Standalone).WithTopology(topology).WithExecutor(executor); @@ -102,7 +118,7 @@ public async Task LearningItIsAClusterChangesRoutingWithoutRebuildingAnything() Assert.True(router.Default!.HasSent); // went to the only endpoint we knew about Assert.False(owner.HasSent); - topology.OnServerType(ServerType.Cluster); // the discovery + LearnCluster(topology); // the discovery Assert.Equal("mine", (string?)await context.Strings.GetAsync("user:1")); Assert.True(owner.HasSent); // and now it is routed by slot @@ -140,7 +156,7 @@ public async Task ARequestRenderedBeforeTheAnswerArrivedIsStillRoutable() using var request = context.Raw.Render($"{RedisCommand.GET}{(RedisKey)"user:1"}").Detach(); Assert.Equal(slot, request.Slot); // rendered while nobody knew, and carries its slot - topology.OnServerType(ServerType.Cluster); // ...and now we know + LearnCluster(topology); // ...and now we know var executor = context.Raw.Executor!; using var payload = await executor.SendAsync(request); @@ -217,7 +233,7 @@ public void RoutingQuestionsFollowTheSameTopology() Assert.True(db.IsConnected("user:1")); Assert.Equal(0, router.SlotLookups); // standalone: no hashing even for the routing question - topology.OnServerType(ServerType.Cluster); + LearnCluster(topology); Assert.True(db.IsConnected("user:1")); Assert.Equal(1, router.SlotLookups); // cluster: now it asks who owns the slot } diff --git a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs index aa9614027..0d49de0c8 100644 --- a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -225,8 +225,9 @@ public async Task WithRetry_FailsOverBetweenGroupsOnLoading() } // The unhappy path of the failover threshold: the attempt count says "wait for a failover now", but no - // failover ever comes (the member stays nominally healthy - a couple of LOADING replies are not enough - // to trip the default breaker). We wait out FailoverDelay and then carry on retrying the original + // failover ever comes (the member stays nominally healthy: a couple of LOADING replies are not enough + // to trip the default breaker, and the re-probe they trigger finds the member healthy, so the group + // keeps it). We wait out FailoverDelay and then carry on retrying the original // member rather than giving up, and the group never moves. [Fact] public async Task WithRetry_WhenFailoverNeverArrives_KeepsRetryingSameMember() diff --git a/tests/StackExchange.Redis.Tests/SentinelConfigTests.cs b/tests/StackExchange.Redis.Tests/SentinelConfigTests.cs index 814f99719..d1e58b37a 100644 --- a/tests/StackExchange.Redis.Tests/SentinelConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/SentinelConfigTests.cs @@ -93,6 +93,27 @@ public void Connect_UnreachableSentinel_AbortEnabled_Throws() Assert.Throws(() => ConnectionMultiplexer.Connect(options)); } + [Fact] + public void GetSentinelMasterConnection_NothingToSeedWith_AbortDisabled_ThrowsTheDiscoveryFailure() + { + // the direct API: a long-lived SentinelConnect, then a per-service config that carries only the service name + // (the data endpoints are sentinel's to supply). With nothing discovered and nothing to seed a multiplexer + // with, the caller should see the discovery failure as a RedisConnectionException - not the + // InvalidOperationException("No nodes to consider") that connecting an empty endpoint list produces. + using var sentinel = ConnectionMultiplexer.SentinelConnect(GetUnreachableSentinelOptions(abortOnConnectFail: false)); + var config = new ConfigurationOptions + { + AbortOnConnectFail = false, + ConnectRetry = 0, + ConnectTimeout = 100, + ServiceName = "unreachable-primary", + }; + + var exception = Assert.Throws(() => sentinel.GetSentinelMasterConnection(config)); + Assert.Equal(ConnectionFailureType.UnableToConnect, exception.FailureType); + Assert.Equal("Sentinel: Failed connecting to configured primary for service: unreachable-primary", exception.Message); + } + private static ConfigurationOptions GetUnreachableSentinelOptions(bool abortOnConnectFail) { var options = new ConfigurationOptions diff --git a/tests/StackExchange.Redis.Tests/SentinelPrimaryDemotionUnitTests.cs b/tests/StackExchange.Redis.Tests/SentinelPrimaryDemotionUnitTests.cs new file mode 100644 index 000000000..2a6a879c0 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/SentinelPrimaryDemotionUnitTests.cs @@ -0,0 +1,127 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// A sentinel-managed connection whose primary is demoted while every connection stays up. Sentinel's own +/// recovery levers - +switch-master, and killing the demoted node's clients - are exactly what is +/// missing here, so the only signal the client gets is the demoted node reporting itself as a replica. +/// +public class SentinelPrimaryDemotionUnitTests(ITestOutputHelper log) +{ + private const string ServiceName = "mymaster"; + + // documentation-range addresses (RFC 5737): a node that is not in-process can never be reached by accident + private static readonly EndPoint PrimaryA = new IPEndPoint(IPAddress.Parse("192.0.2.1"), 6379); + private static readonly EndPoint ReplicaB = new IPEndPoint(IPAddress.Parse("192.0.2.2"), 6379); + private static readonly EndPoint NewNodeC = new IPEndPoint(IPAddress.Parse("192.0.2.3"), 6379); + private static readonly EndPoint SentinelEndPoint = new IPEndPoint(IPAddress.Parse("192.0.2.10"), 26379); + + [Fact] + public async Task DemotedPrimaryFollowsSentinelToPrimaryAtUnknownAddress() + { + using var data = new InProcessTestServer(log, PrimaryA); + data.AddReplicaNode(ReplicaB, PrimaryA); + using var sentinel = new InProcessSentinelServer(data, ServiceName, PrimaryA, SentinelEndPoint, log); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(CreateConfig(data, sentinel)); + Assert.Equal([PrimaryA], ConnectedPrimaries(conn)); + + // the primary moves to a node the client has never been told about - a pod that came back on a new + // address - and the client misses the announcement; no connection is dropped + FailOverToNewNode(data); + sentinel.Primary = NewNodeC; + + await AssertPrimaryBecomesAsync(conn, NewNodeC); + } + + [Fact] + public async Task DemotedPrimaryDoesNotReconfigureWhileSentinelStillNamesIt() + { + using var data = new InProcessTestServer(log, PrimaryA); + data.AddReplicaNode(ReplicaB, PrimaryA); + using var sentinel = new InProcessSentinelServer(data, ServiceName, PrimaryA, SentinelEndPoint, log); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(CreateConfig(data, sentinel)); + int configurationChanges = 0; + conn.ConfigurationChanged += (_, _) => Interlocked.Increment(ref configurationChanges); + + // sentinel has not caught up: it still names the node that is now a replica + FailOverToNewNode(data); + await UntilAsync(() => ConnectedPrimaries(conn).Length == 0, "the demotion was not noticed"); + + // The property under test: the client keeps asking sentinel, but does not reconfigure every node to + // elect a primary that does not exist. Anchor on the sentinel polls themselves rather than wall-clock + // quiet: require a stretch of several polls with zero configuration changes. A straggler reconfigure + // from the demotion itself re-baselines the stretch; the per-heartbeat reconfigure storm this guards + // against would raise a change on every poll, so no clean stretch would ever complete. + int changesBaseline = Volatile.Read(ref configurationChanges); + int queriesBaseline = sentinel.PrimaryQueries; + await UntilAsync( + () => + { + int changesNow = Volatile.Read(ref configurationChanges); + if (changesNow != changesBaseline) + { + changesBaseline = changesNow; + queriesBaseline = sentinel.PrimaryQueries; + return false; + } + return sentinel.PrimaryQueries >= queriesBaseline + 3; + }, + "no stretch of sentinel polls without a reconfigure"); + log.WriteLine($"clean stretch: {sentinel.PrimaryQueries - queriesBaseline} sentinel polls, configuration changes still {changesBaseline}"); + + // and once sentinel does catch up, the connection follows it + sentinel.Primary = NewNodeC; + await AssertPrimaryBecomesAsync(conn, NewNodeC); + } + + private static ConfigurationOptions CreateConfig(InProcessTestServer data, InProcessSentinelServer sentinel) + { + var config = data.GetClientConfig(); + config.EndPoints.Clear(); + config.EndPoints.Add(SentinelEndPoint); + config.ServiceName = ServiceName; + config.Tunnel = InProcessSentinelServer.CreateTunnel(data, sentinel); + config.ConfigCheckSeconds = 1; // how soon the demoted node's role is re-read + return config; + } + + // C is promoted, and A and B follow it; nothing is announced and no connection drops + private static void FailOverToNewNode(InProcessTestServer data) + { + data.AddEmptyNode(NewNodeC); + data.SetReplicaOf(PrimaryA, NewNodeC); + data.SetReplicaOf(ReplicaB, NewNodeC); + } + + private async Task AssertPrimaryBecomesAsync(IConnectionMultiplexer conn, EndPoint expected) + { + var timer = Stopwatch.StartNew(); + await UntilAsync(() => ConnectedPrimaries(conn).SequenceEqual([expected]), $"primary did not become {Format.ToString(expected)}"); + log.WriteLine($"primary is {Format.ToString(expected)} after {timer.ElapsedMilliseconds}ms"); + + Assert.Equal([expected], ConnectedPrimaries(conn)); + Assert.Equal(expected, conn.GetDatabase().IdentifyEndpoint("demotion-key", CommandFlags.DemandMaster)); + } + + private static async Task UntilAsync(Func condition, string failure) + { + var timer = Stopwatch.StartNew(); + while (!condition()) + { + if (timer.Elapsed > TimeSpan.FromSeconds(20)) Assert.Fail($"{failure} within {timer.Elapsed.TotalSeconds:0}s"); + await Task.Delay(100); + } + } + + private static EndPoint[] ConnectedPrimaries(IConnectionMultiplexer conn) + => conn.GetServers().Where(static s => s.IsConnected && !s.IsReplica).Select(static s => s.EndPoint).ToArray(); +} diff --git a/tests/StackExchange.Redis.Tests/SentinelTests.cs b/tests/StackExchange.Redis.Tests/SentinelTests.cs index d99e47db7..ffe6d3705 100644 --- a/tests/StackExchange.Redis.Tests/SentinelTests.cs +++ b/tests/StackExchange.Redis.Tests/SentinelTests.cs @@ -565,6 +565,18 @@ await UntilConditionAsync( () => !conn.GetServerSnapshot().ToArray().Any(x => Equals(x.EndPoint, seedEndPoint))); Assert.DoesNotContain(seedEndPoint, conn.GetEndPoints()); + // the sentinel connection behind it soft-failed too, so the connect path never wired up its handlers; it + // must do that itself on reaching a sentinel, or this connection would never hear +switch-master + var sentinelConn = conn.sentinelConnection!; + var switchMaster = RedisChannel.Literal("+switch-master"); + await UntilConditionAsync( + TimeSpan.FromSeconds(10), + () => sentinelConn.GetSubscriber().SubscribedEndpoint(switchMaster) is not null && conn.LastException is null); + Assert.NotNull(sentinelConn.GetSubscriber().SubscribedEndpoint(switchMaster)); + + // and the discovery failure recorded at connect time is no longer reported now that we have a primary + Assert.Null(conn.LastException); + static bool TryPing(IDatabase db) { try diff --git a/tests/StackExchange.Redis.Tests/SlotMapUnitTests.cs b/tests/StackExchange.Redis.Tests/SlotMapUnitTests.cs new file mode 100644 index 000000000..3ab329e0d --- /dev/null +++ b/tests/StackExchange.Redis.Tests/SlotMapUnitTests.cs @@ -0,0 +1,33 @@ +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// is derived from whether the map has been allocated, which is +/// only sound if nothing can allocate it without writing an owner into it (see #3284/#3285). +/// +public class SlotMapUnitTests(ITestOutputHelper log) +{ + [Fact] + public async Task RejectedRangesDoNotAllocateTheMap() + { + using var server = new InProcessTestServer(log); + await using var conn = await ConnectionMultiplexer.ConnectAsync(server.GetClientConfig(withPubSub: false, defaultOnly: true)); + var strategy = conn.ServerSelectionStrategy; + var endpoint = conn.GetServerSnapshot()[0]; + + Assert.True(strategy.IsSlotMapEmpty); // standalone: nothing has been applied + Assert.Equal(0, strategy.CountCoveredSlots()); + + strategy.UpdateClusterRange(5, 3, endpoint); // inverted + strategy.UpdateClusterRange(-1, 3, endpoint); // below range + strategy.UpdateClusterRange(16380, 16384, endpoint); // above range + Assert.True(strategy.IsSlotMapEmpty); + Assert.Equal(0, strategy.CountCoveredSlots()); + + strategy.UpdateClusterRange(3, 5, endpoint); + Assert.False(strategy.IsSlotMapEmpty); + Assert.Equal(3, strategy.CountCoveredSlots()); + } +} diff --git a/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj b/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj index f20a2646a..a20caaa0f 100644 --- a/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj +++ b/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj @@ -9,9 +9,17 @@ full enable true + + true $(DefineConstants);BUILD_CURRENT + + + $(NoWarn);xUnit1069 @@ -29,15 +37,17 @@ - + + + + - - + diff --git a/tests/StackExchange.Redis.Tests/TransportCloseTests.cs b/tests/StackExchange.Redis.Tests/TransportCloseTests.cs new file mode 100644 index 000000000..8c70869a4 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/TransportCloseTests.cs @@ -0,0 +1,153 @@ +using System; +using System.IO; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using RESPite.Transports; +using StackExchange.Redis.Configuration; +using Xunit; + +namespace StackExchange.Redis.Tests; + +[RunPerProtocol] +public class TransportCloseTests(ITestOutputHelper output) +{ + [Fact] + public async Task PeerCloseFailsOutstandingCommandsAndRaisesConnectionFailed() + { + using var server = new InProcessTestServer(output); + var tunnel = new StreamTransportTunnel(server.Tunnel); + var config = server.GetClientConfig(withPubSub: false, defaultOnly: true); + config.Tunnel = tunnel; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var failed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + conn.ConnectionFailed += (_, e) => failed.TrySetResult(e); + + var db = conn.GetDatabase(); + await db.PingAsync(); + + // The next command is written to the transport but never reaches the server, so it is still + // awaiting its reply when the server closes the connection. + var transport = tunnel.Interactive!; + transport.DropWrites(); + var pending = db.PingAsync(); + await transport.Dropped.WaitAsync(TimeSpan.FromSeconds(5)); + transport.CloseAsIfByPeer(); + + var ex = await Assert.ThrowsAsync(() => pending); + Assert.Equal(ConnectionFailureType.SocketClosed, ex.FailureType); + + var args = await failed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(ConnectionFailureType.SocketClosed, args.FailureType); + } + + /// Connects through the in-process server's tunnel, but hands the client a + /// over that stream instead of the stream itself. + private sealed class StreamTransportTunnel(Tunnel inner) : Tunnel + { + public StreamTransport? Interactive { get; private set; } + + public override async ValueTask ConnectTransportAsync( + EndPoint endpoint, ConnectionType connectionType, TlsOptions tls, CancellationToken cancellationToken) + { + var stream = await inner.BeforeAuthenticateAsync(endpoint, connectionType, null, cancellationToken) + ?? throw new InvalidOperationException($"No in-process node at {endpoint}"); + var transport = new StreamTransport(stream); + if (connectionType == ConnectionType.Interactive) Interactive = transport; + return transport; + } + } + + /// A transport over a stream, with two levers for the test: drop what is written, and close the way + /// a transport does when its peer goes away - with no fault. + private sealed class StreamTransport(Stream stream) : DuplexTransport + { + private readonly CancellationTokenSource _peerClosed = new(); + private readonly TaskCompletionSource _dropped = new(TaskCreationOptions.RunContinuationsAsynchronously); + private byte[] _staged = new byte[1024]; + private int _stagedCount, _closed; + private volatile bool _dropWrites; + private TransportReceiver? _receiver; + + public Task Dropped => _dropped.Task; + + public void DropWrites() => _dropWrites = true; + + public void CloseAsIfByPeer() => _peerClosed.Cancel(); + + public override Memory GetMemory(int sizeHint = 0) => Reserve(sizeHint).AsMemory(_stagedCount); + + public override void Advance(int count) => _stagedCount += count; + + public override bool Flush() + { + if (Volatile.Read(ref _closed) != 0) + { + _stagedCount = 0; + return false; + } + + if (_stagedCount != 0) + { + if (_dropWrites) + { + _dropped.TrySetResult(true); + } + else + { + stream.Write(_staged, 0, _stagedCount); + stream.Flush(); + } + _stagedCount = 0; + } + return true; + } + + public override void Start(TransportReceiver receiver) + { + _receiver = receiver; + _ = Task.Run(ReceiveAsync); + } + + public override ValueTask DisposeAsync() + { + _peerClosed.Cancel(); + return default; + } + + private async Task ReceiveAsync() + { + var buffer = new byte[4096]; + Exception? fault = null; + try + { + while (true) + { + int read = await stream.ReadAsync(buffer, 0, buffer.Length, _peerClosed.Token); + if (read <= 0 || !_receiver!.OnReceived(new ReadOnlySpan(buffer, 0, read))) break; + _receiver.OnBatchEnd(); + } + } + catch (OperationCanceledException) when (_peerClosed.IsCancellationRequested) + { + // a clean close, as when the server ends the connection + } + catch (Exception ex) + { + fault = ex; + } + + Volatile.Write(ref _closed, 1); + stream.Dispose(); + _receiver!.OnClosed(fault); + } + + private byte[] Reserve(int sizeHint) + { + int needed = _stagedCount + Math.Max(sizeHint, 1); + if (needed > _staged.Length) Array.Resize(ref _staged, Math.Max(needed, _staged.Length * 2)); + return _staged; + } + } +} diff --git a/tests/StackExchange.Redis.Tests/ValkeyUnitTests.cs b/tests/StackExchange.Redis.Tests/ValkeyUnitTests.cs index d9a704a04..0ec3fc0c4 100644 --- a/tests/StackExchange.Redis.Tests/ValkeyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/ValkeyUnitTests.cs @@ -4,6 +4,7 @@ namespace StackExchange.Redis.Tests; +[RunPerProtocol] public class ValkeyUnitTests(ITestOutputHelper log) { [Theory] @@ -16,7 +17,8 @@ public async Task IdentifyValkeyCluster(ServerType type) var serverApi = client.GetServer(server.DefaultEndPoint); Assert.Equal(type, serverApi.ServerType); Assert.Equal(ProductVariant.Valkey, serverApi.GetProductVariant(out var version)); - Assert.Equal("8.1", version); + Assert.Equal("8.1", version); + Assert.Equal(server.RedisVersion, serverApi.Version); // not the valkey version from HELLO (#3278) } private sealed class InProcessValkeyLikeServer(ITestOutputHelper log) : InProcessTestServer(log) @@ -24,6 +26,10 @@ private sealed class InProcessValkeyLikeServer(ITestOutputHelper log) : InProces // see https://github.com/StackExchange/StackExchange.Redis/pull/3050 protected override string ServerModeKey => "server_mode"; + // Valkey reports its own version as HELLO "version" (see #3278); the name match is case-insensitive + protected override string HelloServerName => "Valkey"; + protected override string HelloVersionString => "8.1"; + protected override void Info(StringBuilder sb, string section, Server.RedisClient? client = null) { base.Info(sb, section, client); diff --git a/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs b/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs index 87bc95b21..0b49ac50f 100644 --- a/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs +++ b/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs @@ -252,8 +252,13 @@ public async Task VectorSetInfo(VectorSetQuantization quantization) Assert.Equal(quantization, v.Quantization); Assert.Null(v.QuantizationRaw); // Should be null for known quant types - Assert.NotEqual(0, v.VectorSetUid); - Assert.NotEqual(0, v.HnswMaxNodeUid); + // vset-uid is a zero-based per-process counter: the first vector set created after the server starts + // has uid 0, which is exactly what happens on a fresh CI server when this is the first such test to run. + // So check the parse against the raw reply rather than against zero. + var raw = (await db.ExecuteAsync("VINFO", key)).ToDictionary(); + Assert.Equal((long)raw["vset-uid"], v.VectorSetUid); + Assert.Equal((long)raw["hnsw-max-node-uid"], v.HnswMaxNodeUid); + Assert.NotEqual(0, v.HnswMaxNodeUid); // node uids start at 1, and there is one node } [Fact] diff --git a/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs b/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs index b70d215ee..772ed6823 100644 --- a/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs +++ b/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs @@ -131,4 +131,27 @@ public async Task ConditionTest() i = (int)raw.StringGet(prefix + "i"); Assert.Equal(1, i); } + + [Fact] + public async Task SortedSetRangeAndStoreUsesTheRightKeys() + { + // the sync prefixed overload declared its first two parameters as (destination, source) against the + // interface's (source, destination) and then forwarded them by name, so it read from the destination + // and wrote into (and so wiped) the source + await using var conn = Create(require: RedisFeatures.v6_2_0); + + var raw = conn.GetDatabase(); + var prefix = Me() + ":"; + var prefixed = raw.WithKeyPrefix(prefix); + raw.KeyDelete([prefix + "source", prefix + "destination"], CommandFlags.FireAndForget); + prefixed.SortedSetAdd("source", [new SortedSetEntry("a", 1), new SortedSetEntry("b", 2)], CommandFlags.FireAndForget); + + Assert.Equal(2, prefixed.SortedSetRangeAndStore("source", "destination", 0, -1)); + Assert.Equal(2, raw.SortedSetLength(prefix + "destination")); + Assert.Equal(2, raw.SortedSetLength(prefix + "source")); + + raw.KeyDelete(prefix + "destination", CommandFlags.FireAndForget); + Assert.Equal(2, await prefixed.SortedSetRangeAndStoreAsync("source", "destination", 0, -1)); + Assert.Equal(2, raw.SortedSetLength(prefix + "destination")); + } } diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index cbdce97f4..26d86a3bd 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -149,6 +149,39 @@ public void Failover(EndPoint replica) Log($"failover: {Format.ToString(replica)} promoted, {demoted.Host}:{demoted.Port} demoted"); } + /// + /// Makes replicate - or a primary again, when + /// is null - as REPLICAOF does on a standalone server. Nothing is announced and no connection is dropped; + /// a client finds out from ROLE or INFO replication, or not at all. Both nodes are left serving + /// the whole keyspace, as every standalone node does - including one added by , + /// which otherwise owns no slots and would redirect every key. + /// + public void SetReplicaOf(EndPoint node, EndPoint primary) + { + if (ServerType == ServerType.Cluster) throw new InvalidOperationException($"Server mode is {ServerType}; use {nameof(Failover)}"); + if (!TryGetNode(node ?? throw new ArgumentNullException(nameof(node)), out var target)) throw new KeyNotFoundException($"Node not found: {Format.ToString(node)}"); + Node primaryNode = null; + if (primary is not null) + { + if (!TryGetNode(primary, out primaryNode)) throw new KeyNotFoundException($"Node not found: {Format.ToString(primary)}"); + if (ReferenceEquals(primaryNode, target)) throw new ArgumentException("A node cannot replicate itself", nameof(primary)); + } + + target.UpdateSlots(null); + if (primaryNode is null) + { + target.PrimaryId = null; + target.Flags &= ~NodeFlags.Replica; + } + else + { + primaryNode.UpdateSlots(null); + target.PrimaryId = primaryNode.Id; + target.Flags |= NodeFlags.Replica; + } + Log($"replicaof: {Format.ToString(node)} now replicates {(primary is null ? "no one" : Format.ToString(primary))}"); + } + public bool Migrate(Span key, EndPoint to) => Migrate(ServerSelectionStrategy.GetClusterSlot(key), to); public bool Migrate(in RedisKey key, EndPoint to) => Migrate(GetHashSlot(key), to); @@ -637,24 +670,53 @@ protected virtual TypedRedisValue Hello(RedisClient client, in RedisRequest requ client.IsAuthenticated = isAuthed; client.Name = name; - var reply = TypedRedisValue.Rent(14, out var span, RespPrefix.Map); + var extras = HelloExtraFields; + var reply = TypedRedisValue.Rent(14 + (2 * extras.Count), out var span, RespPrefix.Map); span[0] = TypedRedisValue.BulkString("server"); - span[1] = TypedRedisValue.BulkString("redis"); + span[1] = TypedRedisValue.BulkString(HelloServerName); span[2] = TypedRedisValue.BulkString("version"); - span[3] = TypedRedisValue.BulkString(VersionString); - span[4] = TypedRedisValue.BulkString("proto"); - span[5] = TypedRedisValue.Integer(proto32); - span[6] = TypedRedisValue.BulkString("id"); - span[7] = TypedRedisValue.Integer(client.Id); - span[8] = TypedRedisValue.BulkString("mode"); - span[9] = TypedRedisValue.BulkString(ServerModeValue); - span[10] = TypedRedisValue.BulkString("role"); - span[11] = TypedRedisValue.BulkString(GetPrimaryOf(client?.Node) is null ? "master" : "replica"); - span[12] = TypedRedisValue.BulkString("modules"); - span[13] = TypedRedisValue.EmptyArray(RespPrefix.Array); + span[3] = TypedRedisValue.BulkString(HelloVersionString); + int index = 4; + foreach (var pair in extras) + { + span[index++] = TypedRedisValue.BulkString(pair.Key); + span[index++] = TypedRedisValue.BulkString(pair.Value); + } + span[index++] = TypedRedisValue.BulkString("proto"); + span[index++] = TypedRedisValue.Integer(proto32); + span[index++] = TypedRedisValue.BulkString("id"); + span[index++] = TypedRedisValue.Integer(client.Id); + span[index++] = TypedRedisValue.BulkString("mode"); + span[index++] = TypedRedisValue.BulkString(HelloModeValue); + span[index++] = TypedRedisValue.BulkString("role"); + span[index++] = TypedRedisValue.BulkString(GetPrimaryOf(client?.Node) is null ? "master" : "replica"); + span[index++] = TypedRedisValue.BulkString("modules"); + span[index] = TypedRedisValue.EmptyArray(RespPrefix.Array); return reply; } + /// + /// The server reported by HELLO; Redis (and Garnet) report redis, Valkey reports valkey. + /// + protected virtual string HelloServerName => "redis"; + + /// + /// The version reported by HELLO; this is the Redis API version when + /// is redis, but Valkey reports its own version here. + /// + protected virtual string HelloVersionString => VersionString; + + /// + /// Additional fields reported by HELLO (Garnet adds garnet_version, for example). + /// + protected virtual IReadOnlyList> HelloExtraFields => []; + + /// + /// The mode reported by HELLO; by default the same as INFO's redis_mode, but a + /// proxy can disagree with itself (see #3284), so tests can override this independently. + /// + protected virtual string HelloModeValue => ServerModeValue; + [RedisCommand(-3)] protected virtual TypedRedisValue Sadd(RedisClient client, in RedisRequest request) {