Scale out error ingestion for EF (3/3): liveness and readiness endpoints

src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs

@@ -71,6 +71,7 @@ public async Task Should_ingest_without_an_endpoint_and_without_the_single_owner
             "HeartbeatMonitoringHostedService",     // warms the endpoint monitor, does not check heartbeats
             "InternalCustomChecksHostedService",    // reports this node's ingestion health to the database
             "MetricsReporterHostedService",
+            "HealthCheckPublisherHostedService",  // inert, no IHealthCheckPublisher is registered
             "ExternalIntegrationRequestsDataStore"  // its drain is inert here, nothing calls Subscribe
         ];
 
@@ -138,6 +139,22 @@ public async Task Should_ingest_a_failed_message_into_the_shared_database()
                 await WaitFor(host, async dbContext => await dbContext.EventLogItems.AsNoTracking()
                         .AnyAsync(item => item.EventType == "MessageFailed" && item.Description == "Simulated failure"),
                     "an event log entry for the failure");
+
+                using var client = host.GetTestClient();
+
+                var liveness = await client.GetAsync("/health");
+                var readiness = await client.GetAsync("/health/ready");
+
+                using (Assert.EnterMultipleScope())
+                {
+                    // The container health check binary insists on non-empty JSON.
+                    Assert.That(liveness.IsSuccessStatusCode, Is.True);
+                    Assert.That(liveness.Content.Headers.ContentType?.MediaType, Is.EqualTo("application/json"));
+                    Assert.That(await liveness.Content.ReadAsStringAsync(), Does.Contain("Healthy"));
+
+                    Assert.That(readiness.IsSuccessStatusCode, Is.True);
+                    Assert.That(await readiness.Content.ReadAsStringAsync(), Does.Contain("error-ingestion"));
+                }
             }
             finally
             {

src/ServiceControl.AcceptanceTests/WebApi/When_requesting_health.cs

@@ -0,0 +1,36 @@
+namespace ServiceControl.AcceptanceTests.WebApi
+{
+    using System.Threading.Tasks;
+    using AcceptanceTesting;
+    using NServiceBus.AcceptanceTesting;
+    using NUnit.Framework;
+
+    class When_requesting_health : AcceptanceTest
+    {
+        [Test]
+        public async Task Should_report_liveness_and_readiness_as_json()
+        {
+            await Define<ScenarioContext>()
+                .Done(async c =>
+                {
+                    var liveness = await this.GetRaw("/health");
+                    var readiness = await this.GetRaw("/health/ready");
+
+                    using (Assert.EnterMultipleScope())
+                    {
+                        // The container health check binary rejects anything that is not non-empty
+                        // JSON, and the Dockerfile probes /health in every mode.
+                        Assert.That(liveness.IsSuccessStatusCode, Is.True);
+                        Assert.That(liveness.Content.Headers.ContentType?.MediaType, Is.EqualTo("application/json"));
+                        Assert.That(await liveness.Content.ReadAsStringAsync(), Does.Contain("Healthy"));
+
+                        Assert.That(readiness.IsSuccessStatusCode, Is.True);
+                        Assert.That(await readiness.Content.ReadAsStringAsync(), Does.Contain("error-ingestion"));
+                    }
+
+                    return true;
+                })
+                .Run();
+        }
+    }
+}

src/ServiceControl/Dockerfile

@@ -23,7 +23,7 @@ ENV PersistenceType=RavenDB \
     ForwardErrorMessages=false \
     ErrorRetentionPeriod=15
 
-HEALTHCHECK --start-period=10s CMD ["/healthcheck/healthcheck", "http://localhost:33333/api/configuration"]
+HEALTHCHECK --start-period=10s CMD ["/healthcheck/healthcheck", "http://localhost:33333/health"]
 
 USER $APP_UID
 ENTRYPOINT ["/app/ServiceControl"]

src/ServiceControl/HostApplicationBuilderExtensions.cs

@@ -10,6 +10,7 @@ namespace Particular.ServiceControl
     using global::ServiceControl.Infrastructure.Auth;
     using global::ServiceControl.Infrastructure.BackgroundTasks;
     using global::ServiceControl.Infrastructure.DomainEvents;
+    using global::ServiceControl.Infrastructure.Health;
     using global::ServiceControl.Infrastructure.Metrics;
     using global::ServiceControl.Infrastructure.WebApi;
     using global::ServiceControl.Notifications.Email;
@@ -92,6 +93,7 @@ public static void AddServiceControl(this IHostApplicationBuilder hostBuilder, S
 
             services.AddPersistence(settings);
             services.AddMetrics(settings.PrintMetrics);
+            services.AddServiceControlHealthChecks();
 
             if (settings.ErrorIngestionOnly)
             {

src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs

@@ -11,6 +11,7 @@ namespace ServiceControl.Hosting.Commands
     using ServiceBus.Management.Infrastructure.Settings;
     using ServiceControl.EventLog;
     using ServiceControl.ExternalIntegrations;
+    using ServiceControl.Infrastructure.Health;
     using ServiceControl.Monitoring;
     using ServiceControl.Persistence;
     using ServiceControl.Recoverability;
@@ -46,7 +47,11 @@ internal static WebApplication BuildHost(Settings settings, Action<WebApplicatio
 
             customize?.Invoke(hostBuilder);
 
-            return hostBuilder.Build();
+            var app = hostBuilder.Build();
+
+            app.MapServiceControlHealthChecks();
+
+            return app;
         }
 
         static void EnsureStorageCanScaleOut(Settings settings)

src/ServiceControl/Infrastructure/Health/ErrorIngestionHealthCheck.cs

@@ -0,0 +1,31 @@
+namespace ServiceControl.Infrastructure.Health
+{
+    using System.Threading;
+    using System.Threading.Tasks;
+    using Microsoft.Extensions.Diagnostics.HealthChecks;
+    using ServiceBus.Management.Infrastructure.Settings;
+    using ServiceControl.Operations;
+
+    /// <summary>
+    /// Reports the state the ingestion watchdog publishes. A batch that keeps failing, including
+    /// because the database is unreachable, trips the fault policy's circuit breaker, which raises a
+    /// critical error, which the watchdog records here. So this covers rather more than the receiver
+    /// failing to start.
+    /// </summary>
+    class ErrorIngestionHealthCheck(ErrorIngestionCustomCheck.State ingestionState, Settings settings) : IHealthCheck
+    {
+        public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
+        {
+            if (!settings.IngestErrorMessages)
+            {
+                return Task.FromResult(HealthCheckResult.Healthy("Error ingestion is disabled"));
+            }
+
+            var failure = ingestionState.GetLastFailure();
+
+            return Task.FromResult(failure == null
+                ? HealthCheckResult.Healthy("Ingesting error messages")
+                : HealthCheckResult.Unhealthy(failure));
+        }
+    }
+}

src/ServiceControl/Infrastructure/Health/HealthCheckExtensions.cs

@@ -0,0 +1,79 @@
+namespace ServiceControl.Infrastructure.Health
+{
+    using System.Linq;
+    using System.Text.Json;
+    using System.Threading.Tasks;
+    using Microsoft.AspNetCore.Builder;
+    using Microsoft.AspNetCore.Diagnostics.HealthChecks;
+    using Microsoft.AspNetCore.Http;
+    using Microsoft.Extensions.DependencyInjection;
+    using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+    static class HealthCheckExtensions
+    {
+        public const string LivenessPath = "/health";
+        public const string ReadinessPath = "/health/ready";
+
+        const string ReadyTag = "ready";
+
+        public static void AddServiceControlHealthChecks(this IServiceCollection services) =>
+            services.AddHealthChecks()
+                .AddCheck<ErrorIngestionHealthCheck>("error-ingestion", tags: [ReadyTag]);
+
+        /// <summary>
+        /// Liveness answers "is this process still serving", and is what a container health check
+        /// should restart on. Readiness additionally reports whether the work this host exists to do
+        /// is actually happening, which is a poor reason to kill a container but the right thing for
+        /// an operator or a load balancer to look at.
+        /// </summary>
+        public static void MapServiceControlHealthChecks(this WebApplication app)
+        {
+            app.MapHealthChecks(LivenessPath, new HealthCheckOptions
+            {
+                Predicate = _ => false,
+                ResponseWriter = WriteResponse
+            }).AllowAnonymous();
+
+            app.MapHealthChecks(ReadinessPath, new HealthCheckOptions
+            {
+                Predicate = registration => registration.Tags.Contains(ReadyTag),
+                ResponseWriter = WriteResponse
+            }).AllowAnonymous();
+        }
+
+        // The container health check binary rejects anything that is not non-empty JSON, so the
+        // default plain text writer cannot be used here.
+#pragma warning disable PS0018 // The signature is fixed by HealthCheckOptions.ResponseWriter.
+        static Task WriteResponse(HttpContext context, HealthReport report)
+#pragma warning restore PS0018
+        {
+            context.Response.ContentType = "application/json";
+
+            return context.Response.WriteAsync(JsonSerializer.Serialize(new HealthResponse
+            {
+                Status = report.Status.ToString(),
+                Checks = [.. report.Entries.Select(entry => new HealthResponse.Check
+                {
+                    Name = entry.Key,
+                    Status = entry.Value.Status.ToString(),
+                    Description = entry.Value.Description ?? entry.Value.Exception?.Message
+                })]
+            }, SerializerOptions), context.RequestAborted);
+        }
+
+        static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
+
+        class HealthResponse
+        {
+            public required string Status { get; init; }
+            public required Check[] Checks { get; init; }
+
+            public class Check
+            {
+                public required string Name { get; init; }
+                public required string Status { get; init; }
+                public string Description { get; init; }
+            }
+        }
+    }
+}

src/ServiceControl/WebApplicationExtensions.cs

@@ -6,6 +6,7 @@ namespace ServiceControl;
 using ServiceControl.Hosting.Https;
 using ServiceControl.Hosting.RequestId;
 using ServiceControl.Infrastructure;
+using ServiceControl.Infrastructure.Health;
 
 public static class WebApplicationExtensions
 {
@@ -19,5 +20,6 @@ public static void UseServiceControl(this WebApplication app, ForwardedHeadersSe
         app.UseHttpLogging();
         app.UseCors();
         app.MapControllers();
+        app.MapServiceControlHealthChecks();
     }
 }