diff --git a/Endpoints/EngineEndpoints.cs b/Endpoints/EngineEndpoints.cs index c771173..d44a9dd 100644 --- a/Endpoints/EngineEndpoints.cs +++ b/Endpoints/EngineEndpoints.cs @@ -23,6 +23,10 @@ public static void MapEngineEndpoints(this WebApplication app) app.MapPost("/api/memory/purge", async (VramOrchestrator orchestrator, IGpuTelemetryProvider telemetryProvider, ISettingsService settingsService, CancellationToken ct) => { var eviction = await orchestrator.PurgeAllCachesAsync(stopIdleEngines: true, ct); + System.Runtime.GCSettings.LargeObjectHeapCompactionMode = System.Runtime.GCLargeObjectHeapCompactionMode.CompactOnce; + GC.Collect(2, GCCollectionMode.Aggressive, blocking: true, compacting: true); + GC.WaitForPendingFinalizers(); + GC.Collect(2, GCCollectionMode.Aggressive, blocking: true, compacting: true); return Results.Ok(new { status = "purged", diff --git a/LocalLLMServerManager.Tests/HardwareEndpointsTests.cs b/LocalLLMServerManager.Tests/HardwareEndpointsTests.cs index 5156d54..ffc22a4 100644 --- a/LocalLLMServerManager.Tests/HardwareEndpointsTests.cs +++ b/LocalLLMServerManager.Tests/HardwareEndpointsTests.cs @@ -306,4 +306,35 @@ public void GpuTelemetryProvider_GetSystemMemoryInfo_ReturnsValidHostMetrics() Assert.True(snapshot.ProcessWorkingSetBytes > 0); Assert.True(snapshot.NonPagedPoolBytes >= 0); } + + [Fact] + public async Task GpuTelemetryProvider_GetTelemetryAsync_ReturnsValidTelemetryAndCachesWithinTtl() + { + var provider = new LocalLLMServerManager.Services.GpuTelemetryProvider(TimeSpan.FromSeconds(3)); + var first = await provider.GetTelemetryAsync(); + var second = await provider.GetTelemetryAsync(); + + Assert.NotNull(first); + Assert.False(string.IsNullOrWhiteSpace(first.GpuName)); + Assert.True(first.TotalVramMb > 0); + Assert.True(first.UsedVramMb >= 0); + Assert.True(first.FreeVramMb >= 0); + Assert.InRange(first.UtilizationPercent, 0.0, 100.0); + Assert.Same(first, second); + } + + [Fact] + public async Task PostMemoryPurge_CompactsHeapAndReturnsPurgedStatusWithMemoryTelemetry() + { + var response = await _client.PostAsync("/api/memory/purge", null); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + using var doc = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync()); + var root = doc.RootElement; + Assert.Equal("purged", root.GetProperty("status").GetString()); + Assert.True(root.TryGetProperty("eviction", out _)); + Assert.True(root.TryGetProperty("memory", out var memory)); + Assert.True(memory.GetProperty("totalRamBytes").GetInt64() > 0); + Assert.True(memory.GetProperty("processWorkingSetMb").GetDouble() > 0); + } } diff --git a/LocalLLMServerManager.csproj b/LocalLLMServerManager.csproj index 11a6aae..dbb1f1d 100644 --- a/LocalLLMServerManager.csproj +++ b/LocalLLMServerManager.csproj @@ -16,6 +16,8 @@ 4.1.0.0 Assets\app-icon.ico true + false + 1 diff --git a/Services/GpuTelemetryProvider.cs b/Services/GpuTelemetryProvider.cs index 0c997aa..d62edcf 100644 --- a/Services/GpuTelemetryProvider.cs +++ b/Services/GpuTelemetryProvider.cs @@ -2,18 +2,37 @@ using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; namespace LocalLLMServerManager.Services; public class GpuTelemetryProvider : IGpuTelemetryProvider { private readonly IAiEngineManager? _engineManager; + private readonly TimeSpan _cacheTtl = TimeSpan.FromSeconds(3); + private readonly object _cacheLock = new(); + private (string GpuName, long TotalVramBytes, long UsedVramBytes)? _cachedGpuInfo; + private GpuTelemetryResult? _cachedTelemetry; + private DateTimeOffset _cacheExpiresAtUtc = DateTimeOffset.MinValue; + [ActivatorUtilitiesConstructor] public GpuTelemetryProvider(IAiEngineManager? engineManager = null) { _engineManager = engineManager; } + public GpuTelemetryProvider(IAiEngineManager? engineManager, TimeSpan cacheTtl) + { + _engineManager = engineManager; + _cacheTtl = cacheTtl; + } + + public GpuTelemetryProvider(TimeSpan cacheTtl) + : this(null, cacheTtl) + { + } + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)] private struct MEMORYSTATUSEX { @@ -160,7 +179,7 @@ private static long GetSafeProcessWorkingSet(Process? proc) { if (proc.HasExited) return 0; proc.Refresh(); - return proc.WorkingSet64; + return Math.Max(0, proc.WorkingSet64); } catch { @@ -168,7 +187,62 @@ private static long GetSafeProcessWorkingSet(Process? proc) } } + public Task GetTelemetryAsync() + { + var now = DateTimeOffset.UtcNow; + lock (_cacheLock) + { + if (_cachedTelemetry != null && now < _cacheExpiresAtUtc) + { + return Task.FromResult(_cachedTelemetry); + } + } + + var (gpuName, totalBytes, usedBytes) = GetGpuInfo(); + lock (_cacheLock) + { + if (_cachedTelemetry != null) + { + return Task.FromResult(_cachedTelemetry); + } + + var totalMb = totalBytes / (1024 * 1024); + var usedMb = usedBytes / (1024 * 1024); + var freeMb = Math.Max(0, (totalBytes - usedBytes) / (1024 * 1024)); + var utilPercent = totalBytes > 0 ? Math.Round((double)usedBytes / totalBytes * 100.0, 1) : 0.0; + _cachedTelemetry = new GpuTelemetryResult(gpuName, totalMb, usedMb, freeMb, utilPercent); + return Task.FromResult(_cachedTelemetry); + } + } + public (string GpuName, long TotalVramBytes, long UsedVramBytes) GetGpuInfo() + { + var now = DateTimeOffset.UtcNow; + lock (_cacheLock) + { + if (_cachedGpuInfo.HasValue && now < _cacheExpiresAtUtc) + { + return _cachedGpuInfo.Value; + } + } + + var info = QueryGpuInfoCore(); + + lock (_cacheLock) + { + _cachedGpuInfo = info; + var totalMb = info.TotalVramBytes / (1024 * 1024); + var usedMb = info.UsedVramBytes / (1024 * 1024); + var freeMb = Math.Max(0, (info.TotalVramBytes - info.UsedVramBytes) / (1024 * 1024)); + var utilPercent = info.TotalVramBytes > 0 ? Math.Round((double)info.UsedVramBytes / info.TotalVramBytes * 100.0, 1) : 0.0; + _cachedTelemetry = new GpuTelemetryResult(info.GpuName, totalMb, usedMb, freeMb, utilPercent); + _cacheExpiresAtUtc = DateTimeOffset.UtcNow.Add(_cacheTtl); + } + + return info; + } + + private (string GpuName, long TotalVramBytes, long UsedVramBytes) QueryGpuInfoCore() { try { diff --git a/Services/MemoryLifecycleWatchdog.cs b/Services/MemoryLifecycleWatchdog.cs index e169898..ea877c6 100644 --- a/Services/MemoryLifecycleWatchdog.cs +++ b/Services/MemoryLifecycleWatchdog.cs @@ -1,6 +1,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -10,17 +11,30 @@ public sealed class MemoryLifecycleWatchdog : BackgroundService { private readonly VramOrchestrator _orchestrator; private readonly ILogger _logger; + private readonly IGpuTelemetryProvider? _telemetryProvider; private readonly TimeSpan _checkInterval; private readonly TimeProvider _timeProvider; public MemoryLifecycleWatchdog( VramOrchestrator orchestrator, ILogger logger, + TimeSpan? checkInterval, + TimeProvider? timeProvider = null) + : this(orchestrator, logger, null, checkInterval, timeProvider) + { + } + + [ActivatorUtilitiesConstructor] + public MemoryLifecycleWatchdog( + VramOrchestrator orchestrator, + ILogger logger, + IGpuTelemetryProvider? telemetryProvider = null, TimeSpan? checkInterval = null, TimeProvider? timeProvider = null) { _orchestrator = orchestrator; _logger = logger; + _telemetryProvider = telemetryProvider; _checkInterval = checkInterval ?? TimeSpan.FromSeconds(30); _timeProvider = timeProvider ?? TimeProvider.System; } @@ -35,6 +49,16 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) try { await _orchestrator.EvaluateCacheLifetimesAsync(stoppingToken); + + var memInfo = (_telemetryProvider ?? new GpuTelemetryProvider()).GetSystemMemoryInfo(); + if (memInfo.ProcessWorkingSetBytes > 512L * 1024 * 1024 || + GC.GetTotalMemory(false) > 256L * 1024 * 1024 || + memInfo.RamUsedPercent >= 75.0) + { + System.Runtime.GCSettings.LargeObjectHeapCompactionMode = System.Runtime.GCLargeObjectHeapCompactionMode.CompactOnce; + GC.Collect(2, GCCollectionMode.Aggressive, blocking: true, compacting: true); + GC.WaitForPendingFinalizers(); + } } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) {