diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs b/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs index a5dff19984..c9472b6b94 100644 --- a/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs +++ b/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs @@ -71,6 +71,7 @@ public async Task Should_ingest_without_an_endpoint_and_without_the_single_owner "HeartbeatMonitoringHostedService", // warms the endpoint monitor, does not check heartbeats "InternalCustomChecksHostedService", // reports this node's ingestion health to the database "MetricsReporterHostedService", + "HealthCheckPublisherHostedService", // inert, no IHealthCheckPublisher is registered "ExternalIntegrationRequestsDataStore" // its drain is inert here, nothing calls Subscribe ]; @@ -138,6 +139,22 @@ public async Task Should_ingest_a_failed_message_into_the_shared_database() await WaitFor(host, async dbContext => await dbContext.EventLogItems.AsNoTracking() .AnyAsync(item => item.EventType == "MessageFailed" && item.Description == "Simulated failure"), "an event log entry for the failure"); + + using var client = host.GetTestClient(); + + var liveness = await client.GetAsync("/health"); + var readiness = await client.GetAsync("/health/ready"); + + using (Assert.EnterMultipleScope()) + { + // The container health check binary insists on non-empty JSON. + Assert.That(liveness.IsSuccessStatusCode, Is.True); + Assert.That(liveness.Content.Headers.ContentType?.MediaType, Is.EqualTo("application/json")); + Assert.That(await liveness.Content.ReadAsStringAsync(), Does.Contain("Healthy")); + + Assert.That(readiness.IsSuccessStatusCode, Is.True); + Assert.That(await readiness.Content.ReadAsStringAsync(), Does.Contain("error-ingestion")); + } } finally { diff --git a/src/ServiceControl.AcceptanceTests/WebApi/When_requesting_health.cs b/src/ServiceControl.AcceptanceTests/WebApi/When_requesting_health.cs new file mode 100644 index 0000000000..1ff58576be --- /dev/null +++ b/src/ServiceControl.AcceptanceTests/WebApi/When_requesting_health.cs @@ -0,0 +1,36 @@ +namespace ServiceControl.AcceptanceTests.WebApi +{ + using System.Threading.Tasks; + using AcceptanceTesting; + using NServiceBus.AcceptanceTesting; + using NUnit.Framework; + + class When_requesting_health : AcceptanceTest + { + [Test] + public async Task Should_report_liveness_and_readiness_as_json() + { + await Define() + .Done(async c => + { + var liveness = await this.GetRaw("/health"); + var readiness = await this.GetRaw("/health/ready"); + + using (Assert.EnterMultipleScope()) + { + // The container health check binary rejects anything that is not non-empty + // JSON, and the Dockerfile probes /health in every mode. + Assert.That(liveness.IsSuccessStatusCode, Is.True); + Assert.That(liveness.Content.Headers.ContentType?.MediaType, Is.EqualTo("application/json")); + Assert.That(await liveness.Content.ReadAsStringAsync(), Does.Contain("Healthy")); + + Assert.That(readiness.IsSuccessStatusCode, Is.True); + Assert.That(await readiness.Content.ReadAsStringAsync(), Does.Contain("error-ingestion")); + } + + return true; + }) + .Run(); + } + } +} diff --git a/src/ServiceControl/Dockerfile b/src/ServiceControl/Dockerfile index c50ffb8490..777bbc4224 100644 --- a/src/ServiceControl/Dockerfile +++ b/src/ServiceControl/Dockerfile @@ -23,7 +23,7 @@ ENV PersistenceType=RavenDB \ ForwardErrorMessages=false \ ErrorRetentionPeriod=15 -HEALTHCHECK --start-period=10s CMD ["/healthcheck/healthcheck", "http://localhost:33333/api/configuration"] +HEALTHCHECK --start-period=10s CMD ["/healthcheck/healthcheck", "http://localhost:33333/health"] USER $APP_UID ENTRYPOINT ["/app/ServiceControl"] \ No newline at end of file diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs index 732a60d181..d8790c9151 100644 --- a/src/ServiceControl/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs @@ -10,6 +10,7 @@ namespace Particular.ServiceControl using global::ServiceControl.Infrastructure.Auth; using global::ServiceControl.Infrastructure.BackgroundTasks; using global::ServiceControl.Infrastructure.DomainEvents; + using global::ServiceControl.Infrastructure.Health; using global::ServiceControl.Infrastructure.Metrics; using global::ServiceControl.Infrastructure.WebApi; using global::ServiceControl.Notifications.Email; @@ -92,6 +93,7 @@ public static void AddServiceControl(this IHostApplicationBuilder hostBuilder, S services.AddPersistence(settings); services.AddMetrics(settings.PrintMetrics); + services.AddServiceControlHealthChecks(); if (settings.ErrorIngestionOnly) { diff --git a/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs b/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs index 42599249f8..1fc1c68e4e 100644 --- a/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs @@ -11,6 +11,7 @@ namespace ServiceControl.Hosting.Commands using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.EventLog; using ServiceControl.ExternalIntegrations; + using ServiceControl.Infrastructure.Health; using ServiceControl.Monitoring; using ServiceControl.Persistence; using ServiceControl.Recoverability; @@ -46,7 +47,11 @@ internal static WebApplication BuildHost(Settings settings, Action + /// Reports the state the ingestion watchdog publishes. A batch that keeps failing, including + /// because the database is unreachable, trips the fault policy's circuit breaker, which raises a + /// critical error, which the watchdog records here. So this covers rather more than the receiver + /// failing to start. + /// + class ErrorIngestionHealthCheck(ErrorIngestionCustomCheck.State ingestionState, Settings settings) : IHealthCheck + { + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + if (!settings.IngestErrorMessages) + { + return Task.FromResult(HealthCheckResult.Healthy("Error ingestion is disabled")); + } + + var failure = ingestionState.GetLastFailure(); + + return Task.FromResult(failure == null + ? HealthCheckResult.Healthy("Ingesting error messages") + : HealthCheckResult.Unhealthy(failure)); + } + } +} diff --git a/src/ServiceControl/Infrastructure/Health/HealthCheckExtensions.cs b/src/ServiceControl/Infrastructure/Health/HealthCheckExtensions.cs new file mode 100644 index 0000000000..64fd04f30e --- /dev/null +++ b/src/ServiceControl/Infrastructure/Health/HealthCheckExtensions.cs @@ -0,0 +1,79 @@ +namespace ServiceControl.Infrastructure.Health +{ + using System.Linq; + using System.Text.Json; + using System.Threading.Tasks; + using Microsoft.AspNetCore.Builder; + using Microsoft.AspNetCore.Diagnostics.HealthChecks; + using Microsoft.AspNetCore.Http; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Diagnostics.HealthChecks; + + static class HealthCheckExtensions + { + public const string LivenessPath = "/health"; + public const string ReadinessPath = "/health/ready"; + + const string ReadyTag = "ready"; + + public static void AddServiceControlHealthChecks(this IServiceCollection services) => + services.AddHealthChecks() + .AddCheck("error-ingestion", tags: [ReadyTag]); + + /// + /// Liveness answers "is this process still serving", and is what a container health check + /// should restart on. Readiness additionally reports whether the work this host exists to do + /// is actually happening, which is a poor reason to kill a container but the right thing for + /// an operator or a load balancer to look at. + /// + public static void MapServiceControlHealthChecks(this WebApplication app) + { + app.MapHealthChecks(LivenessPath, new HealthCheckOptions + { + Predicate = _ => false, + ResponseWriter = WriteResponse + }).AllowAnonymous(); + + app.MapHealthChecks(ReadinessPath, new HealthCheckOptions + { + Predicate = registration => registration.Tags.Contains(ReadyTag), + ResponseWriter = WriteResponse + }).AllowAnonymous(); + } + + // The container health check binary rejects anything that is not non-empty JSON, so the + // default plain text writer cannot be used here. +#pragma warning disable PS0018 // The signature is fixed by HealthCheckOptions.ResponseWriter. + static Task WriteResponse(HttpContext context, HealthReport report) +#pragma warning restore PS0018 + { + context.Response.ContentType = "application/json"; + + return context.Response.WriteAsync(JsonSerializer.Serialize(new HealthResponse + { + Status = report.Status.ToString(), + Checks = [.. report.Entries.Select(entry => new HealthResponse.Check + { + Name = entry.Key, + Status = entry.Value.Status.ToString(), + Description = entry.Value.Description ?? entry.Value.Exception?.Message + })] + }, SerializerOptions), context.RequestAborted); + } + + static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + class HealthResponse + { + public required string Status { get; init; } + public required Check[] Checks { get; init; } + + public class Check + { + public required string Name { get; init; } + public required string Status { get; init; } + public string Description { get; init; } + } + } + } +} diff --git a/src/ServiceControl/WebApplicationExtensions.cs b/src/ServiceControl/WebApplicationExtensions.cs index d6a961a973..888759811f 100644 --- a/src/ServiceControl/WebApplicationExtensions.cs +++ b/src/ServiceControl/WebApplicationExtensions.cs @@ -6,6 +6,7 @@ namespace ServiceControl; using ServiceControl.Hosting.Https; using ServiceControl.Hosting.RequestId; using ServiceControl.Infrastructure; +using ServiceControl.Infrastructure.Health; public static class WebApplicationExtensions { @@ -19,5 +20,6 @@ public static void UseServiceControl(this WebApplication app, ForwardedHeadersSe app.UseHttpLogging(); app.UseCors(); app.MapControllers(); + app.MapServiceControlHealthChecks(); } } \ No newline at end of file