From b64dcb1e3d5cc6431dc5bc6edfb8d4853549d550 Mon Sep 17 00:00:00 2001 From: Steven Pelech Date: Tue, 27 Oct 2026 13:37:24 -0500 Subject: [PATCH 01/10] docs: add spec and implementation plan for memory cache lifetimes and RAM telemetry (#97, #98) --- ...-cache-lifetimes-and-ram-telemetry-plan.md | 546 ++++++++++++++++++ ...ache-lifetimes-and-ram-telemetry-design.md | 146 +++++ 2 files changed, 692 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-27-memory-cache-lifetimes-and-ram-telemetry-plan.md create mode 100644 docs/superpowers/specs/2026-10-27-memory-cache-lifetimes-and-ram-telemetry-design.md diff --git a/docs/superpowers/plans/2026-10-27-memory-cache-lifetimes-and-ram-telemetry-plan.md b/docs/superpowers/plans/2026-10-27-memory-cache-lifetimes-and-ram-telemetry-plan.md new file mode 100644 index 0000000..5288e45 --- /dev/null +++ b/docs/superpowers/plans/2026-10-27-memory-cache-lifetimes-and-ram-telemetry-plan.md @@ -0,0 +1,546 @@ +# Memory & Cache Lifetime Management and Host RAM Telemetry Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Implement two-tier idle cache eviction (`15m` model weight unload, `30m` child engine process hibernation, `75%` host RAM pressure relief) and expose real-time Host RAM, Process Working Set, and Windows Non-Paged Kernel Pool telemetry in a Unified Hardware Pill in the `34px` titlebar ribbon. + +**Architecture:** Extend `AppSettings` and `SettingsViewModel` with cache lifetime parameters (`ModelCacheIdleTimeoutMinutes = 15`, `EngineHibernationTimeoutMinutes = 30`, `RamPressureThresholdPercent = 75`). Add OS-level physical RAM, working set, and non-paged pool telemetry to `IGpuTelemetryProvider` and `ITelemetryService`. Upgrade `VramOrchestrator` with activity timestamps, two-tier eviction, and a `MemoryLifecycleWatchdog` background `IHostedService`. Update `TelemetryRibbonControl` with a compact dual-bar Unified Hardware Pill (`VRAM` + `RAM`) and `SettingsTabControl` with a Memory & Cache Lifetimes card. + +**Tech Stack:** .NET 10, C# 13, ASP.NET Core Minimal APIs, Avalonia UI 11, CommunityToolkit.Mvvm, xUnit, Playwright. + +## Global Constraints + +- Default Tier 1 Model Weight Idle Unload TTL: `ModelCacheIdleTimeoutMinutes = 15` (`0` = disabled). +- Default Tier 2 Child Engine Hibernation TTL: `EngineHibernationTimeoutMinutes = 30` (`0` = disabled). +- Default Host RAM Pressure Threshold: `RamPressureThresholdPercent = 75`. +- Preserve all backward-compatible JSON properties on `GET /api/gpu/vram` and `AppSettings`. +- Run `npm run lint` and `npx tsc --noEmit` after code changes. +- Verify UI changes with Playwright across both Chromium and Firefox (`npx playwright test`), including `page.reload()`, and visually inspect screenshots. + +--- + +### Task 1: AppSettings & Host System Memory Telemetry (`IGpuTelemetryProvider` & `GpuTelemetryProvider`) + +**Files:** +- Modify: `LocalLLMServerManager.Shared/Models/AppSettings.cs` +- Modify: `Services/IGpuTelemetryProvider.cs` +- Modify: `Services/GpuTelemetryProvider.cs` +- Test: `LocalLLMServerManager.Tests/HardwareEndpointsTests.cs` + +**Interfaces:** +- Consumes: `IAiEngineManager` (optional process handles) +- Produces: + - `AppSettings.ModelCacheIdleTimeoutMinutes` (`int`, default `15`) + - `AppSettings.EngineHibernationTimeoutMinutes` (`int`, default `30`) + - `AppSettings.RamPressureThresholdPercent` (`int`, default `75`) + - `SystemMemorySnapshot` record (`TotalRamBytes`, `UsedRamBytes`, `AvailableRamBytes`, `RamUsedPercent`, `ProcessWorkingSetBytes`, `NonPagedPoolBytes`) + - `IGpuTelemetryProvider.GetSystemMemoryInfo()` + +- [ ] **Step 1: Write the failing test in `LocalLLMServerManager.Tests/HardwareEndpointsTests.cs`** + +Add the following test methods to `LocalLLMServerManager.Tests/HardwareEndpointsTests.cs`: + +```csharp +[Fact] +public void AppSettings_DefaultMemoryCacheSettings_AreConfigured() +{ + var settings = new AppSettings(); + Assert.Equal(15, settings.ModelCacheIdleTimeoutMinutes); + Assert.Equal(30, settings.EngineHibernationTimeoutMinutes); + Assert.Equal(75, settings.RamPressureThresholdPercent); +} + +[Fact] +public void GpuTelemetryProvider_GetSystemMemoryInfo_ReturnsValidHostMetrics() +{ + var provider = new LocalLLMServerManager.Services.GpuTelemetryProvider(); + var snapshot = provider.GetSystemMemoryInfo(); + + Assert.True(snapshot.TotalRamBytes > 0); + Assert.True(snapshot.UsedRamBytes > 0); + Assert.True(snapshot.AvailableRamBytes >= 0); + Assert.InRange(snapshot.RamUsedPercent, 1, 100); + Assert.True(snapshot.ProcessWorkingSetBytes > 0); + Assert.True(snapshot.NonPagedPoolBytes >= 0); +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test LocalLLMServerManager.Tests/LocalLLMServerManager.Tests.csproj --filter "FullyQualifiedName~GetSystemMemoryInfo_ReturnsValidHostMetrics|FullyQualifiedName~AppSettings_DefaultMemoryCacheSettings_AreConfigured"` +Expected: FAIL with compilation error (`ModelCacheIdleTimeoutMinutes` and `GetSystemMemoryInfo` not defined). + +- [ ] **Step 3: Implement `AppSettings`, `SystemMemorySnapshot`, and `GpuTelemetryProvider.GetSystemMemoryInfo()`** + +Update `LocalLLMServerManager.Shared/Models/AppSettings.cs`: + +```csharp +namespace LocalLLMServerManager; + +using System.Collections.Generic; +using LocalLLMServerManager.Shared.Models; + +/// +/// Application settings persisted to settings.json next to the executable. +/// +public record AppSettings( + string ForgeModelsPath = "", + string ComfyUiUrl = "http://127.0.0.1:8188", + string ThreeDModelsPath = "", + string WorkflowsPath = "", + string PreferredImageEngine = "comfy", + string ComfyUiExecutablePath = "", + string ForgeExecutablePath = "", + string OllamaExecutablePath = "ollama", + string ServiceName = "LocalLLMServerManager", + string PublishOutputPath = "C:\\LocalLLMServerManager", + string ComfyModelsPath = "", + string AudioPath = "", + string LanAccessUrl = "http://127.0.0.1:5246", + string SelectedThemeStyle = "semi", + string AudioEngineExecutablePath = "", + string AudioEngineUrl = "http://127.0.0.1:8880", + string PreferredAudioVoice = "af_heart", + string VideoModelsPath = "", + string VideoOutputPath = "", + List? CustomPresets = null, + bool AiAssistantEnabled = false, + string AiAssistantEndpoint = "http://127.0.0.1:4000/v1", + string AiAssistantApiKey = "", + string AiAssistantModel = "google/gemini-2.5-flash", + string AiAssistantCustomSystemPrompt = "", + string AiAssistantPromptsDirectory = "", + int ModelCacheIdleTimeoutMinutes = 15, + int EngineHibernationTimeoutMinutes = 30, + int RamPressureThresholdPercent = 75 +); +``` + +Update `Services/IGpuTelemetryProvider.cs` to add `SystemMemorySnapshot` and `GetSystemMemoryInfo()`: + +```csharp +using System; +using System.Threading.Tasks; + +namespace LocalLLMServerManager.Services; + +public record GpuTelemetryResult( + string GpuName, + long TotalVramMb, + long UsedVramMb, + long FreeVramMb, + double UtilizationPercent +); + +public record SystemMemorySnapshot( + long TotalRamBytes, + long UsedRamBytes, + long AvailableRamBytes, + int RamUsedPercent, + long ProcessWorkingSetBytes, + long NonPagedPoolBytes +); + +public interface IGpuTelemetryProvider +{ + (string GpuName, long TotalVramBytes, long UsedVramBytes) GetGpuInfo(); + (string GpuName, long TotalVramBytes, long UsedVramBytes)? GetLinuxMemoryInfo(); + (string GpuName, long TotalVramBytes, long UsedVramBytes)? ParseNvidiaSmiOutput(string output); + (string GpuName, long TotalVramBytes, long UsedVramBytes) GetGpuInfoFromRegistry(); + + SystemMemorySnapshot GetSystemMemoryInfo() + { + var gcInfo = GC.GetGCMemoryInfo(); + long totalBytes = gcInfo.TotalAvailableMemoryBytes > 0 ? gcInfo.TotalAvailableMemoryBytes : 32L * 1024 * 1024 * 1024; + long usedBytes = gcInfo.MemoryLoadBytes > 0 ? gcInfo.MemoryLoadBytes : totalBytes / 2; + long availBytes = Math.Max(0, totalBytes - usedBytes); + int pct = totalBytes > 0 ? (int)Math.Clamp(Math.Round((double)usedBytes / totalBytes * 100.0), 0, 100) : 0; + return new SystemMemorySnapshot(totalBytes, usedBytes, availBytes, pct, Environment.WorkingSet, 0); + } + + Task GetTelemetryAsync() + { + var (gpuName, totalBytes, usedBytes) = GetGpuInfo(); + var totalMb = totalBytes / (1024 * 1024); + var usedMb = usedBytes / (1024 * 1024); + var freeMb = Math.Max(0, (totalBytes - usedBytes) / (1024 * 1024)); + var utilPercent = totalBytes > 0 ? Math.Round((double)usedBytes / totalBytes * 100.0, 1) : 0.0; + return Task.FromResult(new GpuTelemetryResult(gpuName, totalMb, usedMb, freeMb, utilPercent)); + } +} +``` + +Update `Services/GpuTelemetryProvider.cs` to implement `GetSystemMemoryInfo()` using `GlobalMemoryStatusEx` and `GetPerformanceInfo` on Windows, `/proc/meminfo` on Linux, and `GC.GetGCMemoryInfo()` fallback, plus summing active child process working sets from `IAiEngineManager`: + +```csharp +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace LocalLLMServerManager.Services; + +public class GpuTelemetryProvider : IGpuTelemetryProvider +{ + private readonly IAiEngineManager? _engineManager; + + public GpuTelemetryProvider(IAiEngineManager? engineManager = null) + { + _engineManager = engineManager; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)] + private struct MEMORYSTATUSEX + { + public uint dwLength; + public uint dwMemoryLoad; + public ulong ullTotalPhys; + public ulong ullAvailPhys; + public ulong ullTotalPageFile; + public ulong ullAvailPageFile; + public ulong ullTotalVirtual; + public ulong ullAvailVirtual; + public ulong ullAvailExtendedVirtual; + } + + [StructLayout(LayoutKind.Sequential)] + private struct PERFORMANCE_INFORMATION + { + public uint cb; + public nuint CommitTotal; + public nuint CommitLimit; + public nuint CommitPeak; + public nuint PhysicalTotal; + public nuint PhysicalAvailable; + public nuint SystemCache; + public nuint KernelTotal; + public nuint KernelPaged; + public nuint KernelNonpaged; + public nuint PageSize; + public uint HandleCount; + public uint ProcessCount; + public uint ThreadCount; + } + + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GlobalMemoryStatusEx(ref MEMORYSTATUSEX lpBuffer); + + [DllImport("psapi.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GetPerformanceInfo(out PERFORMANCE_INFORMATION pPerformanceInformation, uint cb); + + public SystemMemorySnapshot GetSystemMemoryInfo() + { + long totalBytes = 0; + long availBytes = 0; + long nonPagedPoolBytes = 0; + + if (OperatingSystem.IsWindows()) + { + try + { + var memStatus = new MEMORYSTATUSEX { dwLength = (uint)Marshal.SizeOf() }; + if (GlobalMemoryStatusEx(ref memStatus) && memStatus.ullTotalPhys > 0) + { + totalBytes = (long)memStatus.ullTotalPhys; + availBytes = (long)memStatus.ullAvailPhys; + } + } + catch { } + + try + { + if (GetPerformanceInfo(out var perfInfo, (uint)Marshal.SizeOf())) + { + nonPagedPoolBytes = (long)((ulong)perfInfo.KernelNonpaged * (ulong)perfInfo.PageSize); + } + } + catch { } + } + else if (OperatingSystem.IsLinux()) + { + try + { + if (File.Exists("/proc/meminfo")) + { + long totalKb = 0; + long availKb = 0; + long sunreclaimKb = 0; + foreach (var line in File.ReadLines("/proc/meminfo")) + { + if (line.StartsWith("MemTotal:")) + { + var val = line.Split(':', StringSplitOptions.TrimEntries)[1].Split(' ', StringSplitOptions.RemoveEmptyEntries)[0]; + long.TryParse(val, out totalKb); + } + else if (line.StartsWith("MemAvailable:")) + { + var val = line.Split(':', StringSplitOptions.TrimEntries)[1].Split(' ', StringSplitOptions.RemoveEmptyEntries)[0]; + long.TryParse(val, out availKb); + } + else if (line.StartsWith("SUnreclaim:")) + { + var val = line.Split(':', StringSplitOptions.TrimEntries)[1].Split(' ', StringSplitOptions.RemoveEmptyEntries)[0]; + long.TryParse(val, out sunreclaimKb); + } + } + if (totalKb > 0) + { + totalBytes = totalKb * 1024; + availBytes = availKb * 1024; + nonPagedPoolBytes = sunreclaimKb * 1024; + } + } + } + catch { } + } + + if (totalBytes <= 0) + { + var gcInfo = GC.GetGCMemoryInfo(); + totalBytes = gcInfo.TotalAvailableMemoryBytes > 0 ? gcInfo.TotalAvailableMemoryBytes : 32L * 1024 * 1024 * 1024; + long gcUsed = gcInfo.MemoryLoadBytes > 0 ? gcInfo.MemoryLoadBytes : totalBytes / 2; + availBytes = Math.Max(0, totalBytes - gcUsed); + } + + long usedBytes = Math.Max(0, totalBytes - availBytes); + int ramUsedPct = totalBytes > 0 + ? (int)Math.Clamp(Math.Round((double)usedBytes / totalBytes * 100.0), 0, 100) + : 0; + + long workingSetBytes = Environment.WorkingSet; + if (_engineManager != null) + { + workingSetBytes += GetSafeProcessWorkingSet(_engineManager.ComfyProcess); + workingSetBytes += GetSafeProcessWorkingSet(_engineManager.ForgeProcess); + workingSetBytes += GetSafeProcessWorkingSet(_engineManager.AudioProcess); + workingSetBytes += GetSafeProcessWorkingSet(_engineManager.OllamaProcess); + } + + return new SystemMemorySnapshot( + TotalRamBytes: totalBytes, + UsedRamBytes: usedBytes, + AvailableRamBytes: availBytes, + RamUsedPercent: ramUsedPct, + ProcessWorkingSetBytes: workingSetBytes, + NonPagedPoolBytes: nonPagedPoolBytes + ); + } + + private static long GetSafeProcessWorkingSet(Process? proc) + { + if (proc == null) return 0; + try + { + if (proc.HasExited) return 0; + proc.Refresh(); + return proc.WorkingSet64; + } + catch + { + return 0; + } + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test LocalLLMServerManager.Tests/LocalLLMServerManager.Tests.csproj --filter "FullyQualifiedName~GetSystemMemoryInfo_ReturnsValidHostMetrics|FullyQualifiedName~AppSettings_DefaultMemoryCacheSettings_AreConfigured"` +Expected: PASS + +--- + +### Task 2: Two-Tier Idle Cache Eviction, RAM Pressure Relief & `MemoryLifecycleWatchdog` (`VramOrchestrator`) + +**Files:** +- Modify: `Services/VramOrchestrator.cs` +- Create: `Services/MemoryLifecycleWatchdog.cs` +- Modify: `Endpoints/EngineEndpoints.cs` +- Modify: `Program.cs` +- Test: `LocalLLMServerManager.Tests/ServicesAndEngineManagerTests.cs` + +**Interfaces:** +- Consumes: `ISettingsService`, `IGpuTelemetryProvider`, `IAiEngineManager`, `TimeProvider` +- Produces: + - `VramOrchestrator.RecordActivity(string engine)` + - `VramOrchestrator.GetLastActivityUtc(string engine)` + - `VramOrchestrator.EvaluateCacheLifetimesAsync(CancellationToken)` + - `VramOrchestrator.PurgeAllCachesAsync(bool stopIdleEngines = false, CancellationToken cancellationToken = default)` + - `MemoryLifecycleWatchdog` (`BackgroundService`) + - `POST /api/memory/purge` and `GET /api/hardware/memory` endpoints + +- [ ] **Step 1: Write the failing tests for Tier 1, Tier 2, RAM Pressure, and Purge in `LocalLLMServerManager.Tests/ServicesAndEngineManagerTests.cs`** + +Add tests verifying: +1. `RecordActivity` updates last activity timestamp for normalized engine names (`"ollama"`, `"comfy"`, `"forge"`, `"audio"`). +2. `EvaluateCacheLifetimesAsync` triggers Tier 1 model weight unload (`keep_alive = 0` to Ollama and `/free` to ComfyUI) after `15` minutes of inactivity. +3. `EvaluateCacheLifetimesAsync` triggers Tier 2 engine process hibernation (`StopComfyUiAsync`, `StopForgeAsync`, `StopAudioEngineAsync`) after `30` minutes of inactivity. +4. `EvaluateCacheLifetimesAsync` triggers immediate Tier 1 + Tier 2 eviction when `RamUsedPercent >= 75%` (`RamPressureThresholdPercent`). +5. `POST /api/memory/purge` triggers `PurgeAllCachesAsync` and returns `200 OK` with memory telemetry. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test LocalLLMServerManager.Tests/LocalLLMServerManager.Tests.csproj --filter "FullyQualifiedName~VramOrchestrator_EvaluateCacheLifetimes"` +Expected: FAIL with compilation error. + +- [ ] **Step 3: Implement `VramOrchestrator` cache lifetime methods, `MemoryLifecycleWatchdog`, and API endpoints** + +Update `Services/VramOrchestrator.cs`: +- Add optional constructor dependencies (`ISettingsService? settingsService = null`, `IGpuTelemetryProvider? telemetryProvider = null`, `IAiEngineManager? engineManager = null`, `TimeProvider? timeProvider = null`) while preserving the existing 2-parameter constructor `VramOrchestrator(HttpClient httpClient, ILogger logger)` for backward compatibility with existing tests. +- Track `_lastActivityUtc` (`ConcurrentDictionary`) and `_tier1Evicted` (`ConcurrentDictionary`). +- Implement `RecordActivity(string engine)`, `GetLastActivityUtc(string engine)`, `EvaluateCacheLifetimesAsync(CancellationToken)`, and `PurgeAllCachesAsync(bool stopEngines = false, CancellationToken cancellationToken = default)`. + +Create `Services/MemoryLifecycleWatchdog.cs`: +```csharp +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace LocalLLMServerManager.Services; + +public sealed class MemoryLifecycleWatchdog : BackgroundService +{ + private readonly VramOrchestrator _orchestrator; + private readonly ILogger _logger; + private readonly TimeSpan _checkInterval; + + public MemoryLifecycleWatchdog( + VramOrchestrator orchestrator, + ILogger logger, + TimeSpan? checkInterval = null) + { + _orchestrator = orchestrator; + _logger = logger; + _checkInterval = checkInterval ?? TimeSpan.FromSeconds(30); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(_checkInterval); + try + { + while (await timer.WaitForNextTickAsync(stoppingToken)) + { + await _orchestrator.EvaluateCacheLifetimesAsync(stoppingToken); + } + } + catch (OperationCanceledException) + { + // Normal shutdown + } + catch (Exception ex) + { + _logger.LogError(ex, "Unhandled error in MemoryLifecycleWatchdog loop."); + } + } +} +``` + +Update `Endpoints/EngineEndpoints.cs`: +- Extend `GET /api/gpu/vram` to include `totalRamBytes`, `usedRamBytes`, `availableRamBytes`, `totalRamGb`, `usedRamGb`, `ramUsedPercent`, `processWorkingSetMb`, `nonPagedPoolMb`, `modelCacheIdleTimeoutMinutes`, `engineHibernationTimeoutMinutes`, `ramPressureThresholdPercent`. +- Add `GET /api/hardware/memory` returning the full GPU + System RAM + Cache Lifetime status. +- Add `POST /api/memory/purge` invoking `orchestrator.PurgeAllCachesAsync(stopEngines: true)` and returning updated memory metrics. + +Update `Program.cs`: +- Register `builder.Services.AddHostedService();`. +- In the orchestration middleware (`app.Use`), stamp `orchestrator.RecordActivity(...)` when requests hit Ollama (`/api/generate`, `/api/chat`, `/v1/chat`), Forge (`/sdapi`, `/v1/images`), ComfyUI (`/comfyapi`, `/api/comfy/prompt`), or Audio (`/v1/audio`). + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `dotnet test LocalLLMServerManager.Tests/LocalLLMServerManager.Tests.csproj --filter "FullyQualifiedName~VramOrchestrator|FullyQualifiedName~EngineEndpoints"` +Expected: PASS + +--- + +### Task 3: Shared Telemetry Service, Unified Hardware Pill UI & Settings Cache Lifetime Controls + +**Files:** +- Modify: `LocalLLMServerManager.Shared/Interfaces/ITelemetryService.cs` +- Modify: `LocalLLMServerManager.Shared/Services/TelemetryService.cs` +- Modify: `LocalLLMServerManager.Shared/Models/EngineStatusCardModel.cs` +- Modify: `LocalLLMServerManager.Shared/ViewModels/TelemetryViewModel.cs` +- Modify: `LocalLLMServerManager.Shared/Views/Controls/TelemetryRibbonControl.axaml` +- Modify: `LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs` +- Modify: `LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml` +- Test: `LocalLLMServerManager.Tests/SettingsViewModelTests.cs` +- Test: `LocalLLMServerManager.Tests/MainViewModelTests.cs` + +**Interfaces:** +- Consumes: `GET /api/gpu/vram`, `POST /api/memory/purge`, `GET /api/settings`, `POST /api/settings` +- Produces: + - `GpuTelemetryInfo` optional/defaulted properties: `UsedRamGb`, `TotalRamGb`, `RamPercent`, `ProcessWorkingSetMb`, `NonPagedPoolMb` + - `TelemetryViewModel`: `RamUsedGb`, `RamTotalGb`, `RamPercentage`, `RamStatusText`, `RamCompactText`, `VramCompactText`, `ProcessWorkingSetMb`, `NonPagedPoolMb`, `RamBarColor`, `HardwareTooltipText`, `PurgeMemoryCommand` + - `SettingsViewModel`: `ModelCacheIdleTimeoutMinutes`, `EngineHibernationTimeoutMinutes`, `RamPressureThresholdPercent` + +- [ ] **Step 1: Write failing unit tests for `TelemetryViewModel` RAM properties & `SettingsViewModel` cache lifetime persistence** + +Add unit tests verifying: +1. `TelemetryViewModel` updates `RamUsedGb`, `RamTotalGb`, `RamPercentage`, `RamCompactText`, `VramCompactText`, `HardwareTooltipText`, and shifts `RamBarColor` to Amber (`#F59E0B`) at `>= 75%` and Red (`#EF4444`) at `>= 90%`. +2. `TelemetryViewModel.PurgeMemoryCommand` posts to `/api/memory/purge` and refreshes telemetry. +3. `SettingsViewModel` loads and saves `ModelCacheIdleTimeoutMinutes` (`15`), `EngineHibernationTimeoutMinutes` (`30`), and `RamPressureThresholdPercent` (`75`). + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test LocalLLMServerManager.Tests/LocalLLMServerManager.Tests.csproj --filter "FullyQualifiedName~SettingsViewModelTests|FullyQualifiedName~TelemetryViewModel"` +Expected: FAIL with compilation error. + +- [ ] **Step 3: Implement `TelemetryService`, `TelemetryViewModel`, `TelemetryRibbonControl.axaml`, `SettingsViewModel`, and `SettingsTabControl.axaml`** + +1. Update `GpuTelemetryInfo` in `LocalLLMServerManager.Shared/Interfaces/ITelemetryService.cs` with optional default parameters: +```csharp +public record GpuTelemetryInfo( + string GpuName, + double UsedVramGb, + double TotalVramGb, + double FreeVramGb, + int Percent, + double UsedRamGb = 0.0, + double TotalRamGb = 32.0, + int RamPercent = 0, + double ProcessWorkingSetMb = 0.0, + double NonPagedPoolMb = 0.0 +); +``` +2. Update `TelemetryService.QueryGpuVramAsync` to parse both camelCase and PascalCase fields (`usedVramGb` / `UsedVramBytes`, `totalVramGb` / `VramBytes` / `VramGB`, `usedRamGb`, `totalRamGb`, `ramUsedPercent`, `processWorkingSetMb`, `nonPagedPoolMb`). +3. Update `TelemetryViewModel.cs` with `RamUsedGb`, `RamTotalGb`, `RamPercentage`, `RamStatusText`, `RamCompactText`, `VramCompactText`, `ProcessWorkingSetMb`, `NonPagedPoolMb`, `RamBarColor`, `HardwareTooltipText`, and `PurgeMemoryCommand`. +4. Update `TelemetryRibbonControl.axaml` center pill (`Grid.Column="1"`) to display the **Unified Hardware Pill** with dual compact progress bars for `VRAM` and `RAM` plus the `PurgeMemoryCommand` icon button in `Grid.Column="2"`. +5. Update `SettingsViewModel.cs` and `SettingsTabControl.axaml` (under the Engines & Paths pane) to include the **Memory & Cache Lifetimes** configuration card with `ModelCacheIdleTimeoutMinutes`, `EngineHibernationTimeoutMinutes`, and `RamPressureThresholdPercent`. + +- [ ] **Step 4: Run unit tests to verify they pass** + +Run: `dotnet test LocalLLMServerManager.Tests/LocalLLMServerManager.Tests.csproj` +Expected: PASS + +--- + +### Task 4: Linting, Typechecking, Playwright Browser E2E & Visual Screenshot Inspection + +**Files:** +- Modify: `e2e` / Playwright tests if applicable +- Verify: Full test suite, `npm run lint`, `npx tsc --noEmit`, `npx playwright test`, and visual inspection + +- [ ] **Step 1: Run linting and typechecking** + +Run: +```powershell +npm run lint +npx tsc --noEmit +``` +Expected: 0 errors. + +- [ ] **Step 2: Run full .NET test suite** + +Run: +```powershell +dotnet test LocalLLMServerManager.Tests/LocalLLMServerManager.Tests.csproj +``` +Expected: All tests pass. + +- [ ] **Step 3: Run Playwright browser tests across Chromium and Firefox & inspect screenshots** + +Run: +```powershell +npx playwright test +``` +Verify hard reload (`page.reload()`) and visually inspect captured screenshots via `read_file` to confirm the Unified Hardware Pill in the 34px titlebar ribbon and the Memory & Cache Lifetimes card in Settings have clean alignment, no badge clipping, and proper layout. diff --git a/docs/superpowers/specs/2026-10-27-memory-cache-lifetimes-and-ram-telemetry-design.md b/docs/superpowers/specs/2026-10-27-memory-cache-lifetimes-and-ram-telemetry-design.md new file mode 100644 index 0000000..e1ec86a --- /dev/null +++ b/docs/superpowers/specs/2026-10-27-memory-cache-lifetimes-and-ram-telemetry-design.md @@ -0,0 +1,146 @@ +# Memory & Cache Lifetime Management and Host RAM Telemetry Design + +**Date:** 2026-10-27 +**Related Issues:** [#97](https://github.com/spelech/LocalLLMServerManager/issues/97), [#98](https://github.com/spelech/LocalLLMServerManager/issues/98) + +--- + +## 1. Problem Statement & Motivation + +During extended operation of `LocalLLMServerManager`, backend AI engines (`Ollama`, `ComfyUI`, `SD Forge`, `Kokoro TTS`) retain multi-gigabyte model weights in memory-mapped files (`mmap`), shared IPC sections, and native CUDA/DirectML host-pinned buffers (`Non-Paged Pool`). + +Live system diagnostics demonstrated: +- **25.00 GB / 31.72 GB (78.8%)** host RAM in use while `LocalLLMServerManager` reported only **~1.85 GB** in its private working set. +- Terminating `LocalLLMServerManager` immediately reclaimed **12.29 GB** of physical RAM and **~870 MB** of Windows Non-Paged Kernel Pool memory. +- The UI telemetry ribbon only monitored GPU VRAM via `nvidia-smi`, providing zero visibility into host system RAM, process working sets, or kernel driver pool consumption. +- `VramOrchestrator` only unloaded models reactively when switching between conflicting GPU workloads, lacking time-based idle cache eviction or host RAM pressure relief. + +--- + +## 2. Goals & Non-Goals + +### Goals +1. **Two-Tier Idle Cache Eviction ([#97](https://github.com/spelech/LocalLLMServerManager/issues/97)):** + - **Tier 1 (Weight Unload — Default `15` minutes):** Automatically unload idle model weights from VRAM and host RAM (`keep_alive: 0` for Ollama, `POST /free` with `unload_models: true, free_memory: true` for ComfyUI) after `ModelCacheIdleTimeoutMinutes` of inactivity. + - **Tier 2 (Engine Hibernation — Default `30` minutes):** Automatically stop dormant child engine processes (`ComfyUI`, `SD Forge`, `Kokoro TTS`) after `EngineHibernationTimeoutMinutes` of inactivity. + - **Host RAM Pressure Relief (Default `75%` threshold):** When host physical RAM usage reaches or exceeds `RamPressureThresholdPercent` (`75%`), immediately trigger Tier 1 weight unloads and Tier 2 hibernation of idle background engines (`> 2` minutes since last activity). +2. **Host RAM, Process Working Set & Kernel Pool Telemetry ([#98](https://github.com/spelech/LocalLLMServerManager/issues/98)):** + - Extend `IGpuTelemetryProvider` and `ITelemetryService` to query host physical RAM (`TotalRamBytes`, `UsedRamBytes`, `AvailableRamBytes`, `RamUsedPercent`), application + child engine working set (`ProcessWorkingSetBytes`), and Windows Non-Paged Kernel Pool (`NonPagedPoolBytes`). + - Expose these metrics via `GET /api/gpu/vram` (backward-compatible) and `GET /api/hardware/memory`, plus a manual `POST /api/memory/purge` endpoint. +3. **Unified Hardware Pill & Settings UI:** + - Upgrade the center pill in `TelemetryRibbonControl` (`34px` titlebar ribbon) into a compact **Unified Hardware Pill** displaying dual progress bars (`VRAM` and `RAM`), color-coded pressure states, a rich breakdown tooltip, and a one-click **Purge Memory Cache** button. + - Add a **Memory & Cache Lifetimes** configuration section in `SettingsTabControl` bound to `AppSettings`. + +### Non-Goals +- Modifying internal memory allocators inside third-party Python binaries (`ComfyUI`, `Forge`, `Kokoro`). +- Re-architecting UI bitmap converters ([#99](https://github.com/spelech/LocalLLMServerManager/issues/99)), which is deferred unless needed later. + +--- + +## 3. Architecture & Components + +### 3.1 Configuration (`AppSettings.cs` & `SettingsViewModel.cs`) +Extend `AppSettings` record with backward-compatible defaults: +```csharp +int ModelCacheIdleTimeoutMinutes = 15, +int EngineHibernationTimeoutMinutes = 30, +int RamPressureThresholdPercent = 75 +``` +- `0` disables automatic timeout for either tier (`Never`). +- `SettingsViewModel` exposes observable properties (`ModelCacheIdleTimeoutMinutes`, `EngineHibernationTimeoutMinutes`, `RamPressureThresholdPercent`) and preset options for the Settings UI. + +### 3.2 Two-Tier Eviction & Activity Tracking (`VramOrchestrator.cs` & `MemoryLifecycleWatchdog.cs`) +1. **Activity Tracking in `VramOrchestrator`:** + - Maintain a thread-safe `ConcurrentDictionary` mapping normalized engine keys (`"ollama"`, `"comfy"`, `"forge"`, `"audio"`) to their last active UTC timestamp, plus a `ConcurrentDictionary` tracking whether Tier 1 weight eviction has already been performed since the last activity. + - Inject `TimeProvider` (defaulting to `TimeProvider.System`) so unit tests can deterministically advance time without `Task.Delay`. + - Expose `RecordActivity(string engine)` and `GetEngineIdleStates()`. + - Record activity automatically in: + - Reverse-proxy and orchestration middleware in `Program.cs` (`/api/generate`, `/api/chat`, `/v1/chat`, `/sdapi/*`, `/v1/images/*`, `/comfyapi/*`, `/api/comfy/prompt`, `/v1/audio/*`). + - MCP tool invocations in `LocalLlmMcpTools.cs`. +2. **Evaluation Method (`EvaluateCacheLifetimesAsync`):** + - Queries current settings via `ISettingsService` and host memory telemetry via `IGpuTelemetryProvider`. + - **RAM Pressure Check:** If `RamUsedPercent >= settings.RamPressureThresholdPercent` (default `75%`): + - Immediately unloads Ollama models (`UnloadOllamaModelsAsync("Host RAM pressure threshold exceeded")`) and frees ComfyUI memory (`FreeComfyUiVramAsync`). + - Stops any running managed child engine (`ComfyUI`, `Forge`, `Audio`) that has been idle for at least 2 minutes. + - **Tier 1 Check (`ModelCacheIdleTimeoutMinutes > 0`, default `15m`):** + - If `"ollama"` idle duration $\ge$ `ModelCacheIdleTimeoutMinutes`, call `UnloadOllamaModelsAsync("Idle cache TTL expired")` and mark Tier 1 evicted. + - If `"comfy"` idle duration $\ge$ `ModelCacheIdleTimeoutMinutes`, call `FreeComfyUiVramAsync(settings.ComfyUiUrl)` and mark Tier 1 evicted. + - **Tier 2 Check (`EngineHibernationTimeoutMinutes > 0`, default `30m`):** + - For `"comfy"`, `"forge"`, and `"audio"`, if the process is running and idle duration $\ge$ `EngineHibernationTimeoutMinutes`, stop the engine via `IAiEngineManager` (`StopComfyUiAsync`, `StopForgeAsync`, `StopAudioEngineAsync`). +3. **Manual Purge (`PurgeAllCachesAsync` & `POST /api/memory/purge`):** + - Immediately unloads all Ollama models, calls ComfyUI `/free`, and optionally stops idle engines if `stopEngines = true`, returning a JSON summary of reclaimed state. +4. **Background Hosted Service (`MemoryLifecycleWatchdog`):** + - Implements `BackgroundService` (`IHostedService`) registered in `Program.CreateWebApplication`. + - Runs a `PeriodicTimer(TimeSpan.FromSeconds(30))` invoking `VramOrchestrator.EvaluateCacheLifetimesAsync(stoppingToken)`. + +### 3.3 Host RAM, Process Working Set & Kernel Pool Telemetry (`GpuTelemetryProvider.cs` & `TelemetryService.cs`) +1. **`SystemMemoryInfo` Model (`LocalLLMServerManager.Shared.Interfaces`):** + ```csharp + public record SystemMemoryInfo( + long TotalRamBytes, + long UsedRamBytes, + long AvailableRamBytes, + int RamUsedPercent, + long ProcessWorkingSetBytes, + long NonPagedPoolBytes); + ``` +2. **OS Collection in `GpuTelemetryProvider`:** + - **Windows:** + - P/Invoke `GlobalMemoryStatusEx` (`kernel32.dll`) for accurate physical `ullTotalPhys` and `ullAvailPhys`. + - P/Invoke `GetPerformanceInfo` (`psapi.dll`) for `KernelNonpaged` pages $\times$ `PageSize`. + - **Linux:** + - Parse `/proc/meminfo` for `MemTotal`, `MemAvailable`, and `SUnreclaim` (kernel slab unreclaimable / non-paged equivalent). + - **Fallback:** + - `GC.GetGCMemoryInfo().TotalAvailableMemoryBytes` and `Environment.WorkingSet`. + - **Process Working Set:** + - Sum `Environment.WorkingSet` of the current process plus any active child processes tracked by `IAiEngineManager` (`ComfyProcess`, `ForgeProcess`, `AudioProcess`, `OllamaProcess`). +3. **API Contract (`GET /api/gpu/vram` & `GET /api/hardware/memory`):** + - `GET /api/gpu/vram` preserves existing fields (`gpuName`, `vramBytes`, `usedVramBytes`, `vramGB`) and adds: + - `totalRamBytes`, `usedRamBytes`, `availableRamBytes`, `totalRamGb`, `usedRamGb`, `ramUsedPercent`, `processWorkingSetMb`, `nonPagedPoolMb`, `modelCacheIdleTimeoutMinutes`, `engineHibernationTimeoutMinutes`, `ramPressureThresholdPercent`. + - `GET /api/hardware/memory` returns the same rich memory & cache lifetime snapshot. + - `POST /api/memory/purge` triggers `VramOrchestrator.PurgeAllCachesAsync` and returns updated memory telemetry. + +### 3.4 UI: Unified Hardware Pill (`TelemetryRibbonControl.axaml` & `TelemetryViewModel.cs`) +1. **`TelemetryViewModel` Additions:** + - Observable properties: + - `RamUsedGb`, `RamTotalGb`, `RamPercentage`, `RamStatusText` (e.g., `"12.7 / 31.7 GB (40%)"`), `RamCompactText` (e.g., `"12.7/31.7G"`), `VramCompactText` (e.g., `"4.2/16.0G"`). + - `ProcessWorkingSetMb`, `NonPagedPoolMb`. + - `RamBarColor`: + - `< 75%`: `#38BDF8` / `{StaticResource PrimaryBrush}` + - `75% – 89%` (at or above pressure threshold): `#F59E0B` (Amber warning) + - `>= 90%`: `#EF4444` (Red critical) + - `HardwareTooltipText`: Multi-line summary showing GPU Name, VRAM (`Used / Total`), System RAM (`Used / Total`), App + Engine Working Set (`MB`), Kernel Non-Paged Pool (`MB`), and Cache Lifetime policy (`15m Unload / 30m Hibernate / 75% Pressure`). + - `PurgeMemoryCommand` (`[RelayCommand]`): Calls `POST /api/memory/purge`, refreshes telemetry, and shows a toast notification. +2. **`TelemetryRibbonControl.axaml` Layout:** + - In `Grid.Column="1"`, render a single `matte-pill` containing: + - GPU icon + truncated GPU name (`MaxWidth="110"`). + - **VRAM micro-gauge:** label `"VRAM"`, `36px` `ProgressBar`, and `VramCompactText`. + - Subtle vertical divider (`1px` `GlassBorderBrush`). + - **RAM micro-gauge:** label `"RAM"`, `36px` `ProgressBar` (bound to `RamBarColor`), and `RamCompactText`. + - **Purge Memory button** (`ribbon-icon-btn` with broom/sparkle icon) bound to `PurgeMemoryCommand` right next to Refresh in `Grid.Column="2"`. + +### 3.5 Settings UI (`SettingsTabControl.axaml`) +Add a **Memory & Cache Lifetimes** section card with three controls: +- **Model Weight Idle Unload TTL (minutes):** `NumericUpDown` / preset selector (default `15`). +- **Engine Process Hibernation TTL (minutes):** `NumericUpDown` / preset selector (default `30`). +- **System RAM Pressure Threshold (%):** `Slider` / `NumericUpDown` (`50%–95%`, default `75%`). + +--- + +## 4. Error Handling & Edge Cases +- All native P/Invoke calls (`GlobalMemoryStatusEx`, `GetPerformanceInfo`) are guarded by `OperatingSystem.IsWindows()` and `try/catch` blocks with managed `GC.GetGCMemoryInfo()` fallbacks. +- Child process `WorkingSet64` access handles `InvalidOperationException` (process exited mid-query) without throwing. +- Setting `ModelCacheIdleTimeoutMinutes = 0` or `EngineHibernationTimeoutMinutes = 0` cleanly disables time-based eviction for that tier while still allowing manual purge. + +--- + +## 5. Verification Strategy +1. **Unit & Integration Tests (`LocalLLMServerManager.Tests`):** + - `VramOrchestratorCacheLifetimeTests`: Verify Tier 1 weight unload at 15 minutes, Tier 2 engine hibernation at 30 minutes, immediate eviction when `RamUsedPercent >= 75%`, and manual `POST /api/memory/purge`. + - `GpuTelemetryProviderMemoryTests`: Verify `GetSystemMemoryInfo()` returns valid non-zero physical RAM, working set, and non-paged pool metrics on Windows/Linux. + - `TelemetryViewModelTests`: Verify `RamPercentage`, `RamBarColor` transitions (`<75%`, `>=75%`, `>=90%`), `HardwareTooltipText`, and `PurgeMemoryCommand`. +2. **Linting & Typechecking:** + - Run `npm run lint` and `npx tsc --noEmit`. +3. **Browser E2E & Visual Verification:** + - Run Playwright tests across both Chromium and Firefox (`npx playwright test`), including `page.reload()`. + - Capture and visually inspect screenshots of the updated titlebar Unified Hardware Pill and Settings Memory & Cache Lifetimes card. From 0683532596b08519b015281661767a58287a7aa9 Mon Sep 17 00:00:00 2001 From: Steven Pelech Date: Tue, 27 Oct 2026 13:56:20 -0500 Subject: [PATCH 02/10] feat(telemetry): add memory cache settings and host RAM telemetry (#97, #98) --- .../Models/AppSettings.cs | 7 +- .../HardwareEndpointsTests.cs | 23 ++- Services/GpuTelemetryProvider.cs | 164 ++++++++++++++++++ Services/IGpuTelemetryProvider.cs | 19 ++ 4 files changed, 209 insertions(+), 4 deletions(-) diff --git a/LocalLLMServerManager.Shared/Models/AppSettings.cs b/LocalLLMServerManager.Shared/Models/AppSettings.cs index 191a4fc..cba174f 100644 --- a/LocalLLMServerManager.Shared/Models/AppSettings.cs +++ b/LocalLLMServerManager.Shared/Models/AppSettings.cs @@ -32,7 +32,8 @@ public record AppSettings( string AiAssistantApiKey = "", string AiAssistantModel = "google/gemini-2.5-flash", string AiAssistantCustomSystemPrompt = "", - string AiAssistantPromptsDirectory = "" + string AiAssistantPromptsDirectory = "", + int ModelCacheIdleTimeoutMinutes = 15, + int EngineHibernationTimeoutMinutes = 30, + int RamPressureThresholdPercent = 75 ); - - diff --git a/LocalLLMServerManager.Tests/HardwareEndpointsTests.cs b/LocalLLMServerManager.Tests/HardwareEndpointsTests.cs index b198bb6..5156d54 100644 --- a/LocalLLMServerManager.Tests/HardwareEndpointsTests.cs +++ b/LocalLLMServerManager.Tests/HardwareEndpointsTests.cs @@ -284,5 +284,26 @@ public async Task PostHardwareEvaluate_MixedCasingAndAlternativeNames_Returns200 Assert.NotNull(result); Assert.Equal(48, result.TotalLayers); } -} + [Fact] + public void AppSettings_DefaultMemoryCacheSettings_AreConfigured() + { + var settings = new AppSettings(); + Assert.Equal(15, settings.ModelCacheIdleTimeoutMinutes); + Assert.Equal(30, settings.EngineHibernationTimeoutMinutes); + Assert.Equal(75, settings.RamPressureThresholdPercent); + } + [Fact] + public void GpuTelemetryProvider_GetSystemMemoryInfo_ReturnsValidHostMetrics() + { + var provider = new LocalLLMServerManager.Services.GpuTelemetryProvider(); + var snapshot = provider.GetSystemMemoryInfo(); + + Assert.True(snapshot.TotalRamBytes > 0); + Assert.True(snapshot.UsedRamBytes > 0); + Assert.True(snapshot.AvailableRamBytes >= 0); + Assert.InRange(snapshot.RamUsedPercent, 1, 100); + Assert.True(snapshot.ProcessWorkingSetBytes > 0); + Assert.True(snapshot.NonPagedPoolBytes >= 0); + } +} diff --git a/Services/GpuTelemetryProvider.cs b/Services/GpuTelemetryProvider.cs index b852644..0c997aa 100644 --- a/Services/GpuTelemetryProvider.cs +++ b/Services/GpuTelemetryProvider.cs @@ -1,9 +1,173 @@ +using System; using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; namespace LocalLLMServerManager.Services; public class GpuTelemetryProvider : IGpuTelemetryProvider { + private readonly IAiEngineManager? _engineManager; + + public GpuTelemetryProvider(IAiEngineManager? engineManager = null) + { + _engineManager = engineManager; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)] + private struct MEMORYSTATUSEX + { + public uint dwLength; + public uint dwMemoryLoad; + public ulong ullTotalPhys; + public ulong ullAvailPhys; + public ulong ullTotalPageFile; + public ulong ullAvailPageFile; + public ulong ullTotalVirtual; + public ulong ullAvailVirtual; + public ulong ullAvailExtendedVirtual; + } + + [StructLayout(LayoutKind.Sequential)] + private struct PERFORMANCE_INFORMATION + { + public uint cb; + public nuint CommitTotal; + public nuint CommitLimit; + public nuint CommitPeak; + public nuint PhysicalTotal; + public nuint PhysicalAvailable; + public nuint SystemCache; + public nuint KernelTotal; + public nuint KernelPaged; + public nuint KernelNonpaged; + public nuint PageSize; + public uint HandleCount; + public uint ProcessCount; + public uint ThreadCount; + } + + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GlobalMemoryStatusEx(ref MEMORYSTATUSEX lpBuffer); + + [DllImport("psapi.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GetPerformanceInfo(out PERFORMANCE_INFORMATION pPerformanceInformation, uint cb); + + public SystemMemorySnapshot GetSystemMemoryInfo() + { + long totalBytes = 0; + long availBytes = 0; + long nonPagedPoolBytes = 0; + + if (OperatingSystem.IsWindows()) + { + try + { + var memStatus = new MEMORYSTATUSEX { dwLength = (uint)Marshal.SizeOf() }; + if (GlobalMemoryStatusEx(ref memStatus) && memStatus.ullTotalPhys > 0) + { + totalBytes = (long)memStatus.ullTotalPhys; + availBytes = (long)memStatus.ullAvailPhys; + } + } + catch { } + + try + { + if (GetPerformanceInfo(out var perfInfo, (uint)Marshal.SizeOf())) + { + nonPagedPoolBytes = (long)((ulong)perfInfo.KernelNonpaged * (ulong)perfInfo.PageSize); + } + } + catch { } + } + else if (OperatingSystem.IsLinux()) + { + try + { + if (File.Exists("/proc/meminfo")) + { + long totalKb = 0; + long availKb = 0; + long sunreclaimKb = 0; + foreach (var line in File.ReadLines("/proc/meminfo")) + { + if (line.StartsWith("MemTotal:")) + { + var val = line.Split(':', StringSplitOptions.TrimEntries)[1].Split(' ', StringSplitOptions.RemoveEmptyEntries)[0]; + long.TryParse(val, out totalKb); + } + else if (line.StartsWith("MemAvailable:")) + { + var val = line.Split(':', StringSplitOptions.TrimEntries)[1].Split(' ', StringSplitOptions.RemoveEmptyEntries)[0]; + long.TryParse(val, out availKb); + } + else if (line.StartsWith("SUnreclaim:")) + { + var val = line.Split(':', StringSplitOptions.TrimEntries)[1].Split(' ', StringSplitOptions.RemoveEmptyEntries)[0]; + long.TryParse(val, out sunreclaimKb); + } + } + if (totalKb > 0) + { + totalBytes = totalKb * 1024; + availBytes = availKb * 1024; + nonPagedPoolBytes = sunreclaimKb * 1024; + } + } + } + catch { } + } + + if (totalBytes <= 0) + { + var gcInfo = GC.GetGCMemoryInfo(); + totalBytes = gcInfo.TotalAvailableMemoryBytes > 0 ? gcInfo.TotalAvailableMemoryBytes : 32L * 1024 * 1024 * 1024; + long gcUsed = gcInfo.MemoryLoadBytes > 0 ? gcInfo.MemoryLoadBytes : totalBytes / 2; + availBytes = Math.Max(0, totalBytes - gcUsed); + } + + long usedBytes = Math.Max(0, totalBytes - availBytes); + int ramUsedPct = totalBytes > 0 + ? (int)Math.Clamp(Math.Round((double)usedBytes / totalBytes * 100.0), 0, 100) + : 0; + + long workingSetBytes = Environment.WorkingSet; + if (_engineManager != null) + { + workingSetBytes += GetSafeProcessWorkingSet(_engineManager.ComfyProcess); + workingSetBytes += GetSafeProcessWorkingSet(_engineManager.ForgeProcess); + workingSetBytes += GetSafeProcessWorkingSet(_engineManager.AudioProcess); + workingSetBytes += GetSafeProcessWorkingSet(_engineManager.OllamaProcess); + } + + return new SystemMemorySnapshot( + TotalRamBytes: totalBytes, + UsedRamBytes: usedBytes, + AvailableRamBytes: availBytes, + RamUsedPercent: ramUsedPct, + ProcessWorkingSetBytes: workingSetBytes, + NonPagedPoolBytes: nonPagedPoolBytes + ); + } + + private static long GetSafeProcessWorkingSet(Process? proc) + { + if (proc == null) return 0; + try + { + if (proc.HasExited) return 0; + proc.Refresh(); + return proc.WorkingSet64; + } + catch + { + return 0; + } + } + public (string GpuName, long TotalVramBytes, long UsedVramBytes) GetGpuInfo() { try diff --git a/Services/IGpuTelemetryProvider.cs b/Services/IGpuTelemetryProvider.cs index bcb9810..98e933e 100644 --- a/Services/IGpuTelemetryProvider.cs +++ b/Services/IGpuTelemetryProvider.cs @@ -11,6 +11,15 @@ public record GpuTelemetryResult( double UtilizationPercent ); +public record SystemMemorySnapshot( + long TotalRamBytes, + long UsedRamBytes, + long AvailableRamBytes, + int RamUsedPercent, + long ProcessWorkingSetBytes, + long NonPagedPoolBytes +); + public interface IGpuTelemetryProvider { (string GpuName, long TotalVramBytes, long UsedVramBytes) GetGpuInfo(); @@ -18,6 +27,16 @@ public interface IGpuTelemetryProvider (string GpuName, long TotalVramBytes, long UsedVramBytes)? ParseNvidiaSmiOutput(string output); (string GpuName, long TotalVramBytes, long UsedVramBytes) GetGpuInfoFromRegistry(); + SystemMemorySnapshot GetSystemMemoryInfo() + { + var gcInfo = GC.GetGCMemoryInfo(); + long totalBytes = gcInfo.TotalAvailableMemoryBytes > 0 ? gcInfo.TotalAvailableMemoryBytes : 32L * 1024 * 1024 * 1024; + long usedBytes = gcInfo.MemoryLoadBytes > 0 ? gcInfo.MemoryLoadBytes : totalBytes / 2; + long availBytes = Math.Max(0, totalBytes - usedBytes); + int pct = totalBytes > 0 ? (int)Math.Clamp(Math.Round((double)usedBytes / totalBytes * 100.0), 0, 100) : 0; + return new SystemMemorySnapshot(totalBytes, usedBytes, availBytes, pct, Environment.WorkingSet, 0); + } + Task GetTelemetryAsync() { var (gpuName, totalBytes, usedBytes) = GetGpuInfo(); From 8912470200f511ac508ca35ed9d9707858c9879d Mon Sep 17 00:00:00 2001 From: Steven Pelech Date: Tue, 27 Oct 2026 14:18:54 -0500 Subject: [PATCH 03/10] feat(orchestration): implement two-tier idle cache eviction, RAM pressure relief & watchdog (#97, #98) --- Endpoints/EngineEndpoints.cs | 76 ++++- .../ServicesAndEngineManagerTests.cs | 312 +++++++++++++++++- Program.cs | 17 + Services/MemoryLifecycleWatchdog.cs | 55 +++ Services/VramOrchestrator.cs | 267 ++++++++++++++- 5 files changed, 705 insertions(+), 22 deletions(-) create mode 100644 Services/MemoryLifecycleWatchdog.cs diff --git a/Endpoints/EngineEndpoints.cs b/Endpoints/EngineEndpoints.cs index 9956dc3..c771173 100644 --- a/Endpoints/EngineEndpoints.cs +++ b/Endpoints/EngineEndpoints.cs @@ -1,4 +1,5 @@ using LocalLLMServerManager.Services; +using LocalLLMServerManager.Shared.Interfaces; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; @@ -9,15 +10,24 @@ public static class EngineEndpoints { public static void MapEngineEndpoints(this WebApplication app) { - app.MapGet("/api/gpu/vram", (IGpuTelemetryProvider telemetryProvider) => + app.MapGet("/api/gpu/vram", (IGpuTelemetryProvider telemetryProvider, ISettingsService settingsService) => { - var (gpuName, vramBytes, usedVramBytes) = telemetryProvider.GetGpuInfo(); + return Results.Ok(BuildMemoryTelemetryResponse(telemetryProvider, settingsService)); + }); + + app.MapGet("/api/hardware/memory", (IGpuTelemetryProvider telemetryProvider, ISettingsService settingsService) => + { + return Results.Ok(BuildMemoryTelemetryResponse(telemetryProvider, settingsService)); + }); + + app.MapPost("/api/memory/purge", async (VramOrchestrator orchestrator, IGpuTelemetryProvider telemetryProvider, ISettingsService settingsService, CancellationToken ct) => + { + var eviction = await orchestrator.PurgeAllCachesAsync(stopIdleEngines: true, ct); return Results.Ok(new { - GpuName = gpuName, - VramBytes = vramBytes, - UsedVramBytes = usedVramBytes, - VramGB = Math.Round((double)vramBytes / (1024 * 1024 * 1024), 2) + status = "purged", + eviction, + memory = BuildMemoryTelemetryResponse(telemetryProvider, settingsService) }); }); @@ -43,6 +53,7 @@ public static void MapEngineEndpoints(this WebApplication app) return Results.BadRequest(new { message = $"Invalid or missing ComfyUI executable path: {execPath}" }); } + orchestrator.RecordActivity("comfy"); await orchestrator.EnsureVramForComfyUiAsync(); var success = await engineManager.StartComfyUiAsync(execPath, logger); @@ -60,7 +71,7 @@ public static void MapEngineEndpoints(this WebApplication app) return Results.Ok(new { message = "ComfyUI Stopped" }); }); - app.MapPost("/api/forge/start", async (IAiEngineManager engineManager, ISettingsService settingsService, ILoggerFactory loggerFactory) => + app.MapPost("/api/forge/start", async (VramOrchestrator orchestrator, IAiEngineManager engineManager, ISettingsService settingsService, ILoggerFactory loggerFactory) => { var logger = loggerFactory.CreateLogger("EngineEndpoints"); var settings = settingsService.LoadSettings(); @@ -71,6 +82,7 @@ public static void MapEngineEndpoints(this WebApplication app) return Results.BadRequest(new { message = $"Invalid or missing SD Forge executable path: {execPath}" }); } + orchestrator.RecordActivity("forge"); var success = await engineManager.StartForgeAsync(execPath, logger); if (success) { @@ -88,11 +100,12 @@ public static void MapEngineEndpoints(this WebApplication app) app.MapPost("/api/comfy/free", async (VramOrchestrator orchestrator) => { + orchestrator.RecordActivity("comfy"); await orchestrator.FreeComfyUiVramAsync(); return Results.Ok(new { message = "ComfyUI VRAM freed" }); }); - app.MapPost("/api/audio/start", async (IAiEngineManager engineManager, ISettingsService settingsService, ILoggerFactory loggerFactory) => + app.MapPost("/api/audio/start", async (VramOrchestrator orchestrator, IAiEngineManager engineManager, ISettingsService settingsService, ILoggerFactory loggerFactory) => { var logger = loggerFactory.CreateLogger("EngineEndpoints"); var settings = settingsService.LoadSettings(); @@ -106,6 +119,7 @@ public static void MapEngineEndpoints(this WebApplication app) } } + orchestrator.RecordActivity("audio"); var success = await engineManager.StartAudioEngineAsync(execPath, logger); if (success) { @@ -123,12 +137,13 @@ public static void MapEngineEndpoints(this WebApplication app) app.MapGet("/api/audio/voices", HandleGetVoicesAsync); - app.MapPost("/api/ollama/start", async (IAiEngineManager engineManager, ISettingsService settingsService, ILoggerFactory loggerFactory) => + app.MapPost("/api/ollama/start", async (VramOrchestrator orchestrator, IAiEngineManager engineManager, ISettingsService settingsService, ILoggerFactory loggerFactory) => { var logger = loggerFactory.CreateLogger("EngineEndpoints"); var settings = settingsService.LoadSettings(); var execPath = AiEngineManager.ResolveOllamaExecutable(settings.OllamaExecutablePath); + orchestrator.RecordActivity("ollama"); var success = await engineManager.StartOllamaAsync(execPath, logger); if (success) { @@ -144,8 +159,9 @@ public static void MapEngineEndpoints(this WebApplication app) return Results.Ok(new { message = "Ollama Stopped" }); }); - app.MapPost("/api/engine/start", async (EngineToggleRequest req, IAiEngineManager engineManager) => + app.MapPost("/api/engine/start", async (EngineToggleRequest req, VramOrchestrator orchestrator, IAiEngineManager engineManager) => { + orchestrator.RecordActivity(req.Engine ?? ""); var result = await engineManager.StartEngineAsync(req.Engine ?? ""); if (result.Success) { @@ -165,6 +181,44 @@ public static void MapEngineEndpoints(this WebApplication app) }); } + public static object BuildMemoryTelemetryResponse(IGpuTelemetryProvider telemetryProvider, ISettingsService settingsService) + { + var (gpuName, vramBytes, usedVramBytes) = telemetryProvider.GetGpuInfo(); + var sysMem = telemetryProvider.GetSystemMemoryInfo(); + var settings = settingsService.LoadSettings(); + double totalVramGb = Math.Round((double)vramBytes / (1024.0 * 1024.0 * 1024.0), 2); + double usedVramGb = Math.Round((double)usedVramBytes / (1024.0 * 1024.0 * 1024.0), 2); + double freeVramGb = Math.Max(0.0, Math.Round(totalVramGb - usedVramGb, 2)); + int usedVramPct = vramBytes > 0 ? (int)Math.Clamp(Math.Round((double)usedVramBytes / vramBytes * 100.0), 0, 100) : 0; + double totalRamGb = Math.Round((double)sysMem.TotalRamBytes / (1024.0 * 1024.0 * 1024.0), 2); + double usedRamGb = Math.Round((double)sysMem.UsedRamBytes / (1024.0 * 1024.0 * 1024.0), 2); + double processWsMb = Math.Round((double)sysMem.ProcessWorkingSetBytes / (1024.0 * 1024.0), 1); + double nonPagedMb = Math.Round((double)sysMem.NonPagedPoolBytes / (1024.0 * 1024.0), 1); + + return new + { + GpuName = gpuName, + VramBytes = vramBytes, + UsedVramBytes = usedVramBytes, + VramGB = totalVramGb, + TotalVramGb = totalVramGb, + UsedVramGb = usedVramGb, + FreeVramGb = freeVramGb, + UsedPercent = usedVramPct, + TotalRamBytes = sysMem.TotalRamBytes, + UsedRamBytes = sysMem.UsedRamBytes, + AvailableRamBytes = sysMem.AvailableRamBytes, + TotalRamGb = totalRamGb, + UsedRamGb = usedRamGb, + RamUsedPercent = sysMem.RamUsedPercent, + ProcessWorkingSetMb = processWsMb, + NonPagedPoolMb = nonPagedMb, + ModelCacheIdleTimeoutMinutes = settings.ModelCacheIdleTimeoutMinutes, + EngineHibernationTimeoutMinutes = settings.EngineHibernationTimeoutMinutes, + RamPressureThresholdPercent = settings.RamPressureThresholdPercent + }; + } + public static async Task HandleGetVoicesAsync(ISettingsService settingsService, System.Net.Http.IHttpClientFactory clientFactory) { var settings = settingsService.LoadSettings(); @@ -197,5 +251,3 @@ public static async Task HandleGetVoicesAsync(ISettingsService settings return Results.Ok(new { voices = defaultVoices, preferred = settings.PreferredAudioVoice }); } } - - diff --git a/LocalLLMServerManager.Tests/ServicesAndEngineManagerTests.cs b/LocalLLMServerManager.Tests/ServicesAndEngineManagerTests.cs index 7d661bb..02f9246 100644 --- a/LocalLLMServerManager.Tests/ServicesAndEngineManagerTests.cs +++ b/LocalLLMServerManager.Tests/ServicesAndEngineManagerTests.cs @@ -1,14 +1,31 @@ using System; +using System.Collections.Generic; +using System.Diagnostics; using System.IO; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.Json.Nodes; +using System.Threading; using System.Threading.Tasks; using LocalLLMServerManager.Services; +using LocalLLMServerManager.Shared.Interfaces; +using LocalLLMServerManager.Shared.Models; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Xunit; namespace LocalLLMServerManager.Tests; -public class ServicesAndEngineManagerTests +public class ServicesAndEngineManagerTests : IClassFixture { + private readonly HttpClient _client; + + public ServicesAndEngineManagerTests(AppTestServerFixture fixture) + { + _client = fixture.CreateClient(); + } + [Fact] public void GpuTelemetryProvider_ParsesVariousTelemetrySources() { @@ -32,6 +49,7 @@ public void GpuTelemetryProvider_ParsesVariousTelemetrySources() Assert.True(parsed.Value.TotalVramBytes > 0); Assert.True(parsed.Value.UsedVramBytes > 0); } + [Fact] public void GitUpdateService_IsValidBranchName_ValidatesCorrectly() { @@ -164,4 +182,296 @@ public async Task AiEngineManager_StopEngineAsync_DispatchesExpectedEngine(strin Assert.Contains("Unsupported", result.Message); } } + + [Fact] + public async Task VramOrchestrator_RecordActivity_And_Tier1_Tier2_Eviction_WorksWithFakeTime() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 10, 27, 12, 0, 0, TimeSpan.Zero)); + var handler = new TestHttpMessageHandler(); + var httpClient = new HttpClient(handler); + var settings = new AppSettings( + ModelCacheIdleTimeoutMinutes: 15, + EngineHibernationTimeoutMinutes: 30, + RamPressureThresholdPercent: 75 + ); + var settingsService = new FakeSettingsService(settings); + var telemetryProvider = new FakeGpuTelemetryProvider(ramUsedPercent: 45); + var engineManager = new FakeAiEngineManager(); + + var orchestrator = new VramOrchestrator( + httpClient, + NullLogger.Instance, + settingsService, + telemetryProvider, + engineManager, + timeProvider + ); + + orchestrator.RecordActivity("ollama"); + orchestrator.RecordActivity("comfyui"); + orchestrator.RecordActivity("forge"); + orchestrator.RecordActivity("kokoro"); + + // Advance 10m -> neither tier triggers + timeProvider.Advance(TimeSpan.FromMinutes(10)); + var res10 = await orchestrator.EvaluateCacheLifetimesAsync(); + Assert.False(res10.OllamaUnloaded); + Assert.False(res10.ComfyVramFreed); + Assert.Empty(res10.HibernatedEngines); + Assert.Empty(engineManager.StoppedEngines); + + // Advance another 6m (total 16m >= 15m) -> Tier 1 triggers (Ollama unload + ComfyUI /free) + timeProvider.Advance(TimeSpan.FromMinutes(6)); + var res16 = await orchestrator.EvaluateCacheLifetimesAsync(); + Assert.True(res16.OllamaUnloaded); + Assert.True(res16.ComfyVramFreed); + Assert.Empty(res16.HibernatedEngines); + Assert.Empty(engineManager.StoppedEngines); + Assert.Contains(handler.Requests, r => r.RequestUri!.ToString().Contains("11434/api/generate")); + Assert.Contains(handler.Requests, r => r.RequestUri!.ToString().EndsWith("/free")); + + // Advance another 15m (total 31m >= 30m) -> Tier 2 triggers (comfy, forge, audio hibernated) + timeProvider.Advance(TimeSpan.FromMinutes(15)); + var res31 = await orchestrator.EvaluateCacheLifetimesAsync(); + Assert.Contains("comfy", res31.HibernatedEngines); + Assert.Contains("forge", res31.HibernatedEngines); + Assert.Contains("audio", res31.HibernatedEngines); + Assert.Contains("comfy", engineManager.StoppedEngines); + Assert.Contains("forge", engineManager.StoppedEngines); + Assert.Contains("audio", engineManager.StoppedEngines); + } + + [Fact] + public async Task VramOrchestrator_RamPressureAboveThreshold_EvictsImmediately() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 10, 27, 12, 0, 0, TimeSpan.Zero)); + var handler = new TestHttpMessageHandler(); + var httpClient = new HttpClient(handler); + var settings = new AppSettings( + ModelCacheIdleTimeoutMinutes: 15, + EngineHibernationTimeoutMinutes: 30, + RamPressureThresholdPercent: 75 + ); + var settingsService = new FakeSettingsService(settings); + var telemetryProvider = new FakeGpuTelemetryProvider(ramUsedPercent: 78); // >= 75% + var engineManager = new FakeAiEngineManager(); + + var orchestrator = new VramOrchestrator( + httpClient, + NullLogger.Instance, + settingsService, + telemetryProvider, + engineManager, + timeProvider + ); + + orchestrator.RecordActivity("ollama"); + orchestrator.RecordActivity("comfyui"); + orchestrator.RecordActivity("forge"); + orchestrator.RecordActivity("kokoro"); + + // Advance 3m (> 2m idle) -> immediately evicts Tier 1 + Tier 2 due to RAM pressure + timeProvider.Advance(TimeSpan.FromMinutes(3)); + var result = await orchestrator.EvaluateCacheLifetimesAsync(); + + Assert.True(result.OllamaUnloaded); + Assert.True(result.ComfyVramFreed); + Assert.Contains("comfy", result.HibernatedEngines); + Assert.Contains("forge", result.HibernatedEngines); + Assert.Contains("audio", result.HibernatedEngines); + Assert.Contains("RAM pressure", result.Reason); + } + + [Fact] + public async Task VramOrchestrator_PurgeAllCachesAsync_UnloadsAndStopsEngines() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 10, 27, 12, 0, 0, TimeSpan.Zero)); + var handler = new TestHttpMessageHandler(); + var httpClient = new HttpClient(handler); + var settings = new AppSettings(); + var settingsService = new FakeSettingsService(settings); + var telemetryProvider = new FakeGpuTelemetryProvider(ramUsedPercent: 40); + var engineManager = new FakeAiEngineManager(); + + var orchestrator = new VramOrchestrator( + httpClient, + NullLogger.Instance, + settingsService, + telemetryProvider, + engineManager, + timeProvider + ); + + var result = await orchestrator.PurgeAllCachesAsync(stopIdleEngines: true); + + Assert.True(result.OllamaUnloaded); + Assert.True(result.ComfyVramFreed); + Assert.Contains("comfy", result.HibernatedEngines); + Assert.Contains("forge", result.HibernatedEngines); + Assert.Contains("audio", result.HibernatedEngines); + Assert.Contains("comfy", engineManager.StoppedEngines); + Assert.Contains("forge", engineManager.StoppedEngines); + Assert.Contains("audio", engineManager.StoppedEngines); + Assert.Contains(handler.Requests, r => r.RequestUri!.ToString().Contains("11434/api/generate")); + Assert.Contains(handler.Requests, r => r.RequestUri!.ToString().EndsWith("/free")); + } + + [Fact] + public async Task MemoryEndpoints_GetMemory_And_PostPurge_ReturnExpectedPayloads() + { + var vramResponse = await _client.GetAsync("/api/gpu/vram"); + Assert.Equal(HttpStatusCode.OK, vramResponse.StatusCode); + var vramJson = await vramResponse.Content.ReadAsStringAsync(); + var vramNode = JsonNode.Parse(vramJson); + Assert.NotNull(vramNode); + Assert.NotNull(vramNode["totalRamBytes"]); + Assert.NotNull(vramNode["ramUsedPercent"]); + Assert.Equal(15, vramNode["modelCacheIdleTimeoutMinutes"]?.GetValue()); + Assert.Equal(30, vramNode["engineHibernationTimeoutMinutes"]?.GetValue()); + Assert.Equal(75, vramNode["ramPressureThresholdPercent"]?.GetValue()); + + var memResponse = await _client.GetAsync("/api/hardware/memory"); + Assert.Equal(HttpStatusCode.OK, memResponse.StatusCode); + var memJson = await memResponse.Content.ReadAsStringAsync(); + var memNode = JsonNode.Parse(memJson); + Assert.NotNull(memNode); + Assert.NotNull(memNode["totalRamBytes"]); + Assert.NotNull(memNode["usedRamBytes"]); + + var purgeResponse = await _client.PostAsync("/api/memory/purge", null); + Assert.Equal(HttpStatusCode.OK, purgeResponse.StatusCode); + var purgeJson = await purgeResponse.Content.ReadAsStringAsync(); + var purgeNode = JsonNode.Parse(purgeJson); + Assert.NotNull(purgeNode); + Assert.Equal("purged", purgeNode["status"]?.GetValue()); + Assert.NotNull(purgeNode["eviction"]); + Assert.NotNull(purgeNode["memory"]); + } + + private class FakeTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + public FakeTimeProvider(DateTimeOffset startUtc) => _utcNow = startUtc; + public override DateTimeOffset GetUtcNow() => _utcNow; + public void Advance(TimeSpan duration) => _utcNow = _utcNow.Add(duration); + } + + private class TestHttpMessageHandler : HttpMessageHandler + { + public List Requests { get; } = new(); + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Add(request); + var uri = request.RequestUri?.ToString() ?? ""; + + if (uri.Contains("11434/api/ps")) + { + var json = "{\"models\":[{\"name\":\"llama3:latest\"}]}"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }); + } + if (uri.Contains("11434/api/generate")) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"response\":\"\"}", Encoding.UTF8, "application/json") + }); + } + if (uri.EndsWith("/free")) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"status\":\"freed\"}", Encoding.UTF8, "application/json") + }); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)); + } + } + + private class FakeSettingsService : ISettingsService + { + private readonly AppSettings _settings; + public FakeSettingsService(AppSettings settings) => _settings = settings; + public AppSettings LoadSettings() => _settings; + public Task LoadSettingsAsync(CancellationToken cancellationToken = default) => Task.FromResult(_settings); + public void SaveSettings(AppSettings settings) { } + public Task SaveSettingsAsync(AppSettings settings, CancellationToken cancellationToken = default) => Task.CompletedTask; + public string SettingsFilePath() => "settings.json"; + } + + private class FakeGpuTelemetryProvider : IGpuTelemetryProvider + { + private readonly int _ramUsedPercent; + public FakeGpuTelemetryProvider(int ramUsedPercent = 45) => _ramUsedPercent = ramUsedPercent; + + public (string GpuName, long TotalVramBytes, long UsedVramBytes) GetGpuInfo() => + ("NVIDIA GeForce RTX 4070 Ti SUPER", 16L * 1024 * 1024 * 1024, 4L * 1024 * 1024 * 1024); + + public (string GpuName, long TotalVramBytes, long UsedVramBytes)? GetLinuxMemoryInfo() => null; + public (string GpuName, long TotalVramBytes, long UsedVramBytes)? ParseNvidiaSmiOutput(string output) => null; + public (string GpuName, long TotalVramBytes, long UsedVramBytes) GetGpuInfoFromRegistry() => GetGpuInfo(); + + public SystemMemorySnapshot GetSystemMemoryInfo() => + new SystemMemorySnapshot( + TotalRamBytes: 32L * 1024 * 1024 * 1024, + UsedRamBytes: (long)(32L * 1024 * 1024 * 1024 * (_ramUsedPercent / 100.0)), + AvailableRamBytes: (long)(32L * 1024 * 1024 * 1024 * ((100 - _ramUsedPercent) / 100.0)), + RamUsedPercent: _ramUsedPercent, + ProcessWorkingSetBytes: 500L * 1024 * 1024, + NonPagedPoolBytes: 300L * 1024 * 1024 + ); + } + + private class FakeAiEngineManager : IAiEngineManager + { + public List StoppedEngines { get; } = new(); + + public Process? ComfyProcess => null; + public Process? ForgeProcess => null; + public Process? AudioProcess => null; + public Process? OllamaProcess => null; + + public bool IsProcessRunning(string name) => false; + public Task StartComfyUiAsync(string path, ILogger logger) => Task.FromResult(true); + public Task StartForgeAsync(string path, ILogger logger) => Task.FromResult(true); + public Task StartAudioEngineAsync(string path, ILogger logger) => Task.FromResult(true); + public Task StartOllamaAsync(string path, ILogger logger) => Task.FromResult(true); + + public Task StopComfyUiAsync(ILogger logger) + { + StoppedEngines.Add("comfy"); + return Task.FromResult(true); + } + + public Task StopForgeAsync(ILogger logger) + { + StoppedEngines.Add("forge"); + return Task.FromResult(true); + } + + public Task StopAudioEngineAsync(ILogger logger) + { + StoppedEngines.Add("audio"); + return Task.FromResult(true); + } + + public Task StopOllamaAsync(ILogger logger) + { + StoppedEngines.Add("ollama"); + return Task.FromResult(true); + } + + public Task StartEngineAsync(string engine) => + Task.FromResult(new EngineOperationResult(true, engine, $"{engine} started")); + + public Task StopEngineAsync(string engine) + { + StoppedEngines.Add(engine); + return Task.FromResult(new EngineOperationResult(true, engine, $"{engine} stopped")); + } + } } diff --git a/Program.cs b/Program.cs index 51436f7..adc707b 100644 --- a/Program.cs +++ b/Program.cs @@ -168,7 +168,9 @@ public static WebApplication CreateWebApplication(string[] args, bool isServiceM builder.Logging.AddFilter("System.Net.Http.HttpClient", LogLevel.Warning); builder.Services.AddHttpClient(); + builder.Services.AddSingleton(TimeProvider.System); builder.Services.AddSingleton(); + builder.Services.AddHostedService(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -255,12 +257,18 @@ public static WebApplication CreateWebApplication(string[] args, bool isServiceM path.StartsWith("/v1/images", StringComparison.OrdinalIgnoreCase); var isComfyRequest = path.StartsWith("/comfyapi", StringComparison.OrdinalIgnoreCase) || path.StartsWith("/api/comfy/prompt", StringComparison.OrdinalIgnoreCase); + var isOllamaRequest = path.StartsWith("/api/generate", StringComparison.OrdinalIgnoreCase) || + path.StartsWith("/api/chat", StringComparison.OrdinalIgnoreCase) || + path.StartsWith("/v1/chat", StringComparison.OrdinalIgnoreCase); + var isAudioRequest = path.StartsWith("/v1/audio", StringComparison.OrdinalIgnoreCase) || + path.StartsWith("/api/audio", StringComparison.OrdinalIgnoreCase); var orchestrator = context.RequestServices.GetRequiredService(); var settingsService = context.RequestServices.GetRequiredService(); if (isForgeRequest) { + orchestrator.RecordActivity("forge"); await orchestrator.EnsureVramForImageGenerationAsync(); if (!await orchestrator.IsForgeHealthyAsync()) @@ -276,6 +284,7 @@ public static WebApplication CreateWebApplication(string[] args, bool isServiceM } else if (isComfyRequest) { + orchestrator.RecordActivity("comfy"); await orchestrator.EnsureVramForComfyUiAsync(); var settings = settingsService.LoadSettings(); @@ -291,6 +300,14 @@ public static WebApplication CreateWebApplication(string[] args, bool isServiceM } } } + else if (isOllamaRequest) + { + orchestrator.RecordActivity("ollama"); + } + else if (isAudioRequest) + { + orchestrator.RecordActivity("audio"); + } await next(); }); diff --git a/Services/MemoryLifecycleWatchdog.cs b/Services/MemoryLifecycleWatchdog.cs new file mode 100644 index 0000000..2e4bfe2 --- /dev/null +++ b/Services/MemoryLifecycleWatchdog.cs @@ -0,0 +1,55 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace LocalLLMServerManager.Services; + +public sealed class MemoryLifecycleWatchdog : BackgroundService +{ + private readonly VramOrchestrator _orchestrator; + private readonly ILogger _logger; + private readonly TimeSpan _checkInterval; + + public MemoryLifecycleWatchdog( + VramOrchestrator orchestrator, + ILogger logger, + TimeSpan? checkInterval = null) + { + _orchestrator = orchestrator; + _logger = logger; + _checkInterval = checkInterval ?? TimeSpan.FromSeconds(30); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(_checkInterval); + try + { + while (await timer.WaitForNextTickAsync(stoppingToken)) + { + try + { + await _orchestrator.EvaluateCacheLifetimesAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Unhandled error during memory cache evaluation in watchdog loop."); + } + } + } + catch (OperationCanceledException) + { + // Normal shutdown + } + catch (Exception ex) + { + _logger.LogError(ex, "Unhandled error in MemoryLifecycleWatchdog loop."); + } + } +} diff --git a/Services/VramOrchestrator.cs b/Services/VramOrchestrator.cs index c3283b9..3429436 100644 --- a/Services/VramOrchestrator.cs +++ b/Services/VramOrchestrator.cs @@ -1,18 +1,94 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Net.Http; +using System.Text; using System.Text.Json; using System.Text.Json.Nodes; -using System.Text; +using System.Threading; +using System.Threading.Tasks; +using LocalLLMServerManager.Services; +using LocalLLMServerManager.Shared.Interfaces; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace LocalLLMServerManager; +public record CacheEvictionResult( + bool OllamaUnloaded, + bool ComfyVramFreed, + IReadOnlyList HibernatedEngines, + string Reason +); + public class VramOrchestrator { private readonly HttpClient _httpClient; private readonly ILogger _logger; + private readonly ISettingsService? _settingsService; + private readonly IGpuTelemetryProvider? _telemetryProvider; + private readonly IAiEngineManager? _engineManager; + private readonly TimeProvider _timeProvider; + + private readonly ConcurrentDictionary _lastActivityUtc = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _tier1Evicted = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _tier2Hibernated = new(StringComparer.OrdinalIgnoreCase); public VramOrchestrator(HttpClient httpClient, ILogger logger) + : this(httpClient, logger, null, null, null, null) + { + } + + [ActivatorUtilitiesConstructor] + public VramOrchestrator( + HttpClient httpClient, + ILogger logger, + ISettingsService? settingsService = null, + IGpuTelemetryProvider? telemetryProvider = null, + IAiEngineManager? engineManager = null, + TimeProvider? timeProvider = null) { _httpClient = httpClient; _logger = logger; + _settingsService = settingsService; + _telemetryProvider = telemetryProvider; + _engineManager = engineManager; + _timeProvider = timeProvider ?? TimeProvider.System; + + var now = _timeProvider.GetUtcNow(); + _lastActivityUtc["ollama"] = now; + _lastActivityUtc["comfy"] = now; + _lastActivityUtc["forge"] = now; + _lastActivityUtc["audio"] = now; + } + + public static string NormalizeEngineKey(string? engine) + { + var normalized = engine?.Trim().ToLowerInvariant() ?? ""; + if (normalized == "comfyui" || normalized == "comfy") return "comfy"; + if (normalized == "forge" || normalized == "sdforge") return "forge"; + if (normalized == "audio" || normalized == "kokoro" || normalized == "alltalk" || normalized == "tts") return "audio"; + if (normalized == "ollama") return "ollama"; + return normalized; + } + + public void RecordActivity(string engine) + { + var key = NormalizeEngineKey(engine); + if (!string.IsNullOrEmpty(key)) + { + var now = _timeProvider.GetUtcNow(); + _lastActivityUtc[key] = now; + _tier1Evicted[key] = false; + _tier2Hibernated[key] = false; + _logger.LogDebug("Recorded activity for engine '{Engine}' at {Timestamp}", key, now); + } + } + + public DateTimeOffset? GetLastActivityUtc(string engine) + { + var key = NormalizeEngineKey(engine); + return _lastActivityUtc.TryGetValue(key, out var dt) ? dt : null; } public async Task IsOllamaHealthyAsync(CancellationToken cancellationToken = default) @@ -71,30 +147,32 @@ public async Task EnsureVramForComfyUiAsync() await UnloadOllamaModelsAsync("ComfyUI Workflows (Video / 3D / Image)"); } - public async Task FreeComfyUiVramAsync(string comfyUrl = "http://127.0.0.1:8188") + public async Task FreeComfyUiVramAsync(string comfyUrl = "http://127.0.0.1:8188", CancellationToken cancellationToken = default) { try { var baseUrl = comfyUrl.TrimEnd('/'); var payload = new { free_memory = true, unload_models = true }; var content = new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); - await _httpClient.PostAsync($"{baseUrl}/free", content); + var response = await _httpClient.PostAsync($"{baseUrl}/free", content, cancellationToken); _logger.LogInformation("Triggered ComfyUI VRAM unload / free memory."); + return response.IsSuccessStatusCode; } catch (Exception ex) { _logger.LogError(ex, "Failed to call ComfyUI free memory API."); + return false; } } - private async Task UnloadOllamaModelsAsync(string reason) + public async Task UnloadOllamaModelsAsync(string reason, CancellationToken cancellationToken = default) { try { - var psResponse = await _httpClient.GetAsync("http://127.0.0.1:11434/api/ps"); - if (!psResponse.IsSuccessStatusCode) return; + var psResponse = await _httpClient.GetAsync("http://127.0.0.1:11434/api/ps", cancellationToken); + if (!psResponse.IsSuccessStatusCode) return false; - var psContent = await psResponse.Content.ReadAsStringAsync(); + var psContent = await psResponse.Content.ReadAsStringAsync(cancellationToken); var json = JsonNode.Parse(psContent); var models = json?["models"]?.AsArray(); @@ -109,17 +187,188 @@ private async Task UnloadOllamaModelsAsync(string reason) { var payload = new { model = modelName, keep_alive = 0 }; var content = new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"); - await _httpClient.PostAsync("http://127.0.0.1:11434/api/generate", content); + await _httpClient.PostAsync("http://127.0.0.1:11434/api/generate", content, cancellationToken); _logger.LogInformation($"Unloaded model: {modelName}"); } } - await Task.Delay(1500); // Give the system time to clear VRAM + if (_timeProvider == TimeProvider.System) + { + await Task.Delay(1500, cancellationToken); // Give the system time to clear VRAM + } + + return true; } + + return false; } catch (Exception ex) { _logger.LogError(ex, $"Failed to orchestrate VRAM for {reason}."); + return false; + } + } + + public async Task EvaluateCacheLifetimesAsync(CancellationToken cancellationToken = default) + { + var settings = _settingsService?.LoadSettings() ?? new AppSettings(); + var sysMem = _telemetryProvider?.GetSystemMemoryInfo(); + var now = _timeProvider.GetUtcNow(); + + int modelCacheTimeoutMinutes = settings.ModelCacheIdleTimeoutMinutes; + int engineHibernateTimeoutMinutes = settings.EngineHibernationTimeoutMinutes; + int ramPressureThreshold = settings.RamPressureThresholdPercent; + int ramUsedPct = sysMem?.RamUsedPercent ?? 0; + + bool isRamPressure = ramPressureThreshold > 0 && ramUsedPct >= ramPressureThreshold; + var comfyUrl = string.IsNullOrWhiteSpace(settings.ComfyUiUrl) ? "http://127.0.0.1:8188" : settings.ComfyUiUrl; + + bool ollamaUnloaded = false; + bool comfyVramFreed = false; + var hibernatedEngines = new List(); + + if (isRamPressure) + { + var twoMinutes = TimeSpan.FromMinutes(2); + + var ollamaIdle = now - (_lastActivityUtc.TryGetValue("ollama", out var oLast) ? oLast : now); + if (ollamaIdle >= twoMinutes) + { + ollamaUnloaded = await UnloadOllamaModelsAsync($"RAM pressure ({ramUsedPct}% >= {ramPressureThreshold}%)", cancellationToken); + _tier1Evicted["ollama"] = true; + } + + var comfyIdle = now - (_lastActivityUtc.TryGetValue("comfy", out var cLast) ? cLast : now); + if (comfyIdle >= twoMinutes) + { + comfyVramFreed = await FreeComfyUiVramAsync(comfyUrl, cancellationToken); + _tier1Evicted["comfy"] = true; + } + + if (_engineManager != null) + { + if (comfyIdle >= twoMinutes) + { + await _engineManager.StopComfyUiAsync(_logger); + hibernatedEngines.Add("comfy"); + _tier2Hibernated["comfy"] = true; + } + + var forgeIdle = now - (_lastActivityUtc.TryGetValue("forge", out var fLast) ? fLast : now); + if (forgeIdle >= twoMinutes) + { + await _engineManager.StopForgeAsync(_logger); + hibernatedEngines.Add("forge"); + _tier2Hibernated["forge"] = true; + } + + var audioIdle = now - (_lastActivityUtc.TryGetValue("audio", out var aLast) ? aLast : now); + if (audioIdle >= twoMinutes) + { + await _engineManager.StopAudioEngineAsync(_logger); + hibernatedEngines.Add("audio"); + _tier2Hibernated["audio"] = true; + } + } + + var reason = $"RAM pressure relief ({ramUsedPct}% >= {ramPressureThreshold}%)"; + return new CacheEvictionResult(ollamaUnloaded, comfyVramFreed, hibernatedEngines, reason); + } + + // Tier 1: Model weight unload (default 15m) + if (modelCacheTimeoutMinutes > 0) + { + var tier1Ttl = TimeSpan.FromMinutes(modelCacheTimeoutMinutes); + + var ollamaIdle = now - (_lastActivityUtc.TryGetValue("ollama", out var oLast) ? oLast : now); + if (ollamaIdle >= tier1Ttl && (!_tier1Evicted.TryGetValue("ollama", out var oEvicted) || !oEvicted)) + { + ollamaUnloaded = await UnloadOllamaModelsAsync($"Tier 1 idle timeout ({ollamaIdle.TotalMinutes:F0}m >= {modelCacheTimeoutMinutes}m)", cancellationToken); + _tier1Evicted["ollama"] = true; + } + + var comfyIdle = now - (_lastActivityUtc.TryGetValue("comfy", out var cLast) ? cLast : now); + if (comfyIdle >= tier1Ttl && (!_tier1Evicted.TryGetValue("comfy", out var cEvicted) || !cEvicted)) + { + comfyVramFreed = await FreeComfyUiVramAsync(comfyUrl, cancellationToken); + _tier1Evicted["comfy"] = true; + } + } + + // Tier 2: Child engine process hibernation (default 30m) + if (engineHibernateTimeoutMinutes > 0 && _engineManager != null) + { + var tier2Ttl = TimeSpan.FromMinutes(engineHibernateTimeoutMinutes); + + var comfyIdle = now - (_lastActivityUtc.TryGetValue("comfy", out var cLast) ? cLast : now); + if (comfyIdle >= tier2Ttl && (!_tier2Hibernated.TryGetValue("comfy", out var cHib) || !cHib)) + { + await _engineManager.StopComfyUiAsync(_logger); + hibernatedEngines.Add("comfy"); + _tier2Hibernated["comfy"] = true; + } + + var forgeIdle = now - (_lastActivityUtc.TryGetValue("forge", out var fLast) ? fLast : now); + if (forgeIdle >= tier2Ttl && (!_tier2Hibernated.TryGetValue("forge", out var fHib) || !fHib)) + { + await _engineManager.StopForgeAsync(_logger); + hibernatedEngines.Add("forge"); + _tier2Hibernated["forge"] = true; + } + + var audioIdle = now - (_lastActivityUtc.TryGetValue("audio", out var aLast) ? aLast : now); + if (audioIdle >= tier2Ttl && (!_tier2Hibernated.TryGetValue("audio", out var aHib) || !aHib)) + { + await _engineManager.StopAudioEngineAsync(_logger); + hibernatedEngines.Add("audio"); + _tier2Hibernated["audio"] = true; + } } + + string evictionReason; + if (ollamaUnloaded || comfyVramFreed || hibernatedEngines.Count > 0) + { + var parts = new List(); + if (ollamaUnloaded) parts.Add("Ollama weights unloaded"); + if (comfyVramFreed) parts.Add("ComfyUI VRAM freed"); + if (hibernatedEngines.Count > 0) parts.Add($"Hibernated: {string.Join(", ", hibernatedEngines)}"); + evictionReason = string.Join("; ", parts); + } + else + { + evictionReason = "No eviction needed"; + } + + return new CacheEvictionResult(ollamaUnloaded, comfyVramFreed, hibernatedEngines, evictionReason); + } + + public async Task PurgeAllCachesAsync(bool stopIdleEngines = false, CancellationToken cancellationToken = default) + { + var settings = _settingsService?.LoadSettings() ?? new AppSettings(); + var comfyUrl = string.IsNullOrWhiteSpace(settings.ComfyUiUrl) ? "http://127.0.0.1:8188" : settings.ComfyUiUrl; + + bool ollamaUnloaded = await UnloadOllamaModelsAsync("Manual memory purge", cancellationToken); + bool comfyFreed = await FreeComfyUiVramAsync(comfyUrl, cancellationToken); + + _tier1Evicted["ollama"] = true; + _tier1Evicted["comfy"] = true; + + var hibernatedEngines = new List(); + if (stopIdleEngines && _engineManager != null) + { + await _engineManager.StopComfyUiAsync(_logger); + hibernatedEngines.Add("comfy"); + _tier2Hibernated["comfy"] = true; + + await _engineManager.StopForgeAsync(_logger); + hibernatedEngines.Add("forge"); + _tier2Hibernated["forge"] = true; + + await _engineManager.StopAudioEngineAsync(_logger); + hibernatedEngines.Add("audio"); + _tier2Hibernated["audio"] = true; + } + + return new CacheEvictionResult(ollamaUnloaded, comfyFreed, hibernatedEngines, "Manual cache purge"); } } From 3ded4ccaf81d8671ceee6e25a4ac431b0254ed47 Mon Sep 17 00:00:00 2001 From: Steven Pelech Date: Tue, 27 Oct 2026 14:29:27 -0500 Subject: [PATCH 04/10] fix(orchestration): guard RAM pressure eviction flags and format Reason when no engines eligible (#97, #98) --- .../ServicesAndEngineManagerTests.cs | 9 +++++++++ Services/VramOrchestrator.cs | 14 ++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/LocalLLMServerManager.Tests/ServicesAndEngineManagerTests.cs b/LocalLLMServerManager.Tests/ServicesAndEngineManagerTests.cs index 02f9246..d980237 100644 --- a/LocalLLMServerManager.Tests/ServicesAndEngineManagerTests.cs +++ b/LocalLLMServerManager.Tests/ServicesAndEngineManagerTests.cs @@ -280,6 +280,15 @@ public async Task VramOrchestrator_RamPressureAboveThreshold_EvictsImmediately() Assert.Contains("forge", result.HibernatedEngines); Assert.Contains("audio", result.HibernatedEngines); Assert.Contains("RAM pressure", result.Reason); + Assert.Equal(3, engineManager.StoppedEngines.Count); + + // Second consecutive call without new activity -> does not re-evict or re-stop + var result2 = await orchestrator.EvaluateCacheLifetimesAsync(); + Assert.False(result2.OllamaUnloaded); + Assert.False(result2.ComfyVramFreed); + Assert.Empty(result2.HibernatedEngines); + Assert.Equal("No eviction needed", result2.Reason); + Assert.Equal(3, engineManager.StoppedEngines.Count); } [Fact] diff --git a/Services/VramOrchestrator.cs b/Services/VramOrchestrator.cs index 3429436..c554357 100644 --- a/Services/VramOrchestrator.cs +++ b/Services/VramOrchestrator.cs @@ -232,14 +232,14 @@ public async Task EvaluateCacheLifetimesAsync(CancellationT var twoMinutes = TimeSpan.FromMinutes(2); var ollamaIdle = now - (_lastActivityUtc.TryGetValue("ollama", out var oLast) ? oLast : now); - if (ollamaIdle >= twoMinutes) + if (ollamaIdle >= twoMinutes && (!_tier1Evicted.TryGetValue("ollama", out var oEvicted) || !oEvicted)) { ollamaUnloaded = await UnloadOllamaModelsAsync($"RAM pressure ({ramUsedPct}% >= {ramPressureThreshold}%)", cancellationToken); _tier1Evicted["ollama"] = true; } var comfyIdle = now - (_lastActivityUtc.TryGetValue("comfy", out var cLast) ? cLast : now); - if (comfyIdle >= twoMinutes) + if (comfyIdle >= twoMinutes && (!_tier1Evicted.TryGetValue("comfy", out var cEvicted) || !cEvicted)) { comfyVramFreed = await FreeComfyUiVramAsync(comfyUrl, cancellationToken); _tier1Evicted["comfy"] = true; @@ -247,7 +247,7 @@ public async Task EvaluateCacheLifetimesAsync(CancellationT if (_engineManager != null) { - if (comfyIdle >= twoMinutes) + if (comfyIdle >= twoMinutes && (!_tier2Hibernated.TryGetValue("comfy", out var cHib) || !cHib)) { await _engineManager.StopComfyUiAsync(_logger); hibernatedEngines.Add("comfy"); @@ -255,7 +255,7 @@ public async Task EvaluateCacheLifetimesAsync(CancellationT } var forgeIdle = now - (_lastActivityUtc.TryGetValue("forge", out var fLast) ? fLast : now); - if (forgeIdle >= twoMinutes) + if (forgeIdle >= twoMinutes && (!_tier2Hibernated.TryGetValue("forge", out var fHib) || !fHib)) { await _engineManager.StopForgeAsync(_logger); hibernatedEngines.Add("forge"); @@ -263,7 +263,7 @@ public async Task EvaluateCacheLifetimesAsync(CancellationT } var audioIdle = now - (_lastActivityUtc.TryGetValue("audio", out var aLast) ? aLast : now); - if (audioIdle >= twoMinutes) + if (audioIdle >= twoMinutes && (!_tier2Hibernated.TryGetValue("audio", out var aHib) || !aHib)) { await _engineManager.StopAudioEngineAsync(_logger); hibernatedEngines.Add("audio"); @@ -271,7 +271,9 @@ public async Task EvaluateCacheLifetimesAsync(CancellationT } } - var reason = $"RAM pressure relief ({ramUsedPct}% >= {ramPressureThreshold}%)"; + var reason = (ollamaUnloaded || comfyVramFreed || hibernatedEngines.Count > 0) + ? $"RAM pressure relief ({ramUsedPct}% >= {ramPressureThreshold}%)" + : "No eviction needed"; return new CacheEvictionResult(ollamaUnloaded, comfyVramFreed, hibernatedEngines, reason); } From 9feb96a9eb501d3860c94c95b18fdde089b089bd Mon Sep 17 00:00:00 2001 From: Steven Pelech Date: Tue, 27 Oct 2026 14:48:37 -0500 Subject: [PATCH 05/10] feat(ui): add unified hardware pill, cache lifetime settings & memory purge controls (#97, #98) --- .../Interfaces/ITelemetryService.cs | 10 +- .../Models/EngineStatusCardModel.cs | 6 + .../Services/TelemetryService.cs | 64 +++++++- .../Styles/DesignTokens.axaml | 1 + .../ViewModels/SettingsViewModel.cs | 32 +++- .../ViewModels/TelemetryViewModel.cs | 145 +++++++++++++++++- .../Views/Controls/SettingsTabControl.axaml | 40 +++++ .../Controls/TelemetryRibbonControl.axaml | 37 +++-- .../SettingsViewModelTests.cs | 73 +++++++++ .../TelemetryViewModelTests.cs | 84 ++++++++++ 10 files changed, 470 insertions(+), 22 deletions(-) diff --git a/LocalLLMServerManager.Shared/Interfaces/ITelemetryService.cs b/LocalLLMServerManager.Shared/Interfaces/ITelemetryService.cs index 870cee2..25641e1 100644 --- a/LocalLLMServerManager.Shared/Interfaces/ITelemetryService.cs +++ b/LocalLLMServerManager.Shared/Interfaces/ITelemetryService.cs @@ -7,7 +7,15 @@ public record GpuTelemetryInfo( double UsedVramGb, double TotalVramGb, double FreeVramGb, - int Percent + int Percent, + double UsedRamGb = 0.0, + double TotalRamGb = 32.0, + int RamPercent = 0, + double ProcessWorkingSetMb = 0.0, + double NonPagedPoolMb = 0.0, + int ModelCacheIdleTimeoutMinutes = 15, + int EngineHibernationTimeoutMinutes = 30, + int RamPressureThresholdPercent = 75 ); public interface ITelemetryService diff --git a/LocalLLMServerManager.Shared/Models/EngineStatusCardModel.cs b/LocalLLMServerManager.Shared/Models/EngineStatusCardModel.cs index 12bbdbe..24e6378 100644 --- a/LocalLLMServerManager.Shared/Models/EngineStatusCardModel.cs +++ b/LocalLLMServerManager.Shared/Models/EngineStatusCardModel.cs @@ -84,6 +84,12 @@ public class TelemetryData public double TotalVramGb { get; set; } = 16.0; public int VramPercent { get; set; } + public double UsedRamGb { get; set; } = 0.0; + public double TotalRamGb { get; set; } = 32.0; + public int RamPercent { get; set; } = 0; + public double ProcessWorkingSetMb { get; set; } = 0.0; + public double NonPagedPoolMb { get; set; } = 0.0; + public TelemetryData() { } diff --git a/LocalLLMServerManager.Shared/Services/TelemetryService.cs b/LocalLLMServerManager.Shared/Services/TelemetryService.cs index 9d8de10..20efd92 100644 --- a/LocalLLMServerManager.Shared/Services/TelemetryService.cs +++ b/LocalLLMServerManager.Shared/Services/TelemetryService.cs @@ -16,18 +16,68 @@ public async Task QueryGpuVramAsync(string apiBase, HttpClient if (resp.IsSuccessStatusCode) { var doc = JsonNode.Parse(await resp.Content.ReadAsStringAsync()); - string gpuName = doc?["gpuName"]?.ToString() ?? "GPU Telemetry Active"; - double used = doc?["usedVramGb"]?.GetValue() ?? 0.0; - double total = doc?["totalVramGb"]?.GetValue() ?? 16.0; - double free = doc?["freeVramGb"]?.GetValue() ?? (total - used); - int pct = doc?["usedPercent"]?.GetValue() ?? (int)Math.Round((used / (total > 0 ? total : 1.0)) * 100.0); + string gpuName = doc?["gpuName"]?.ToString() + ?? doc?["GpuName"]?.ToString() + ?? "GPU Telemetry Active"; - return new GpuTelemetryInfo(gpuName, used, total, free, pct); + double totalVram = TryGetDouble(doc, "totalVramGb", "TotalVramGb") + ?? TryGetDouble(doc, "vramGB", "VramGB") + ?? (TryGetDouble(doc, "vramBytes", "VramBytes") is double vb ? Math.Round(vb / (1024.0 * 1024.0 * 1024.0), 2) : 16.0); + + double usedVram = TryGetDouble(doc, "usedVramGb", "UsedVramGb") + ?? (TryGetDouble(doc, "usedVramBytes", "UsedVramBytes") is double uvb ? Math.Round(uvb / (1024.0 * 1024.0 * 1024.0), 2) : 0.0); + + double freeVram = TryGetDouble(doc, "freeVramGb", "FreeVramGb") + ?? Math.Max(0.0, Math.Round(totalVram - usedVram, 2)); + + int pct = TryGetInt(doc, "usedPercent", "UsedPercent") + ?? (int)Math.Round((usedVram / (totalVram > 0 ? totalVram : 1.0)) * 100.0); + + double totalRam = TryGetDouble(doc, "totalRamGb", "TotalRamGb") ?? 32.0; + double usedRam = TryGetDouble(doc, "usedRamGb", "UsedRamGb") ?? 0.0; + int ramPct = TryGetInt(doc, "ramUsedPercent", "RamUsedPercent") + ?? (int)Math.Round((usedRam / (totalRam > 0 ? totalRam : 1.0)) * 100.0); + + double processWs = TryGetDouble(doc, "processWorkingSetMb", "ProcessWorkingSetMb") ?? 0.0; + double nonPaged = TryGetDouble(doc, "nonPagedPoolMb", "NonPagedPoolMb") ?? 0.0; + int modelCacheTimeout = TryGetInt(doc, "modelCacheIdleTimeoutMinutes", "ModelCacheIdleTimeoutMinutes") ?? 15; + int hibernationTimeout = TryGetInt(doc, "engineHibernationTimeoutMinutes", "EngineHibernationTimeoutMinutes") ?? 30; + int ramPressureThreshold = TryGetInt(doc, "ramPressureThresholdPercent", "RamPressureThresholdPercent") ?? 75; + + return new GpuTelemetryInfo( + gpuName, + usedVram, + totalVram, + freeVram, + pct, + usedRam, + totalRam, + ramPct, + processWs, + nonPaged, + modelCacheTimeout, + hibernationTimeout, + ramPressureThreshold + ); } } catch { } - return new GpuTelemetryInfo("GPU Telemetry Active", 0.0, 16.0, 16.0, 0); + return new GpuTelemetryInfo("GPU Telemetry Active", 0.0, 16.0, 16.0, 0, 0.0, 32.0, 0, 0.0, 0.0, 15, 30, 75); + } + + private static double? TryGetDouble(JsonNode? node, string camel, string pascal) + { + var target = node?[camel] ?? node?[pascal]; + if (target == null) return null; + try { return target.GetValue(); } catch { return null; } + } + + private static int? TryGetInt(JsonNode? node, string camel, string pascal) + { + var target = node?[camel] ?? node?[pascal]; + if (target == null) return null; + try { return target.GetValue(); } catch { return null; } } public async Task<(bool Ollama, bool Forge, bool ComfyUi)> CheckServiceHealthAsync(string apiBase, string comfyUrl, HttpClient http) diff --git a/LocalLLMServerManager.Shared/Styles/DesignTokens.axaml b/LocalLLMServerManager.Shared/Styles/DesignTokens.axaml index 4eaccb9..be3f013 100644 --- a/LocalLLMServerManager.Shared/Styles/DesignTokens.axaml +++ b/LocalLLMServerManager.Shared/Styles/DesignTokens.axaml @@ -52,6 +52,7 @@ M21.41 11.58l-9-9C12.05 2.22 11.55 2 11 2H4c-1.1 0-2 .9-2 2v7c0 .55.22 1.05.59 1.42l9 9c.36.36.86.58 1.41.58.55 0 1.05-.22 1.41-.59l7-7c.37-.36.59-.86.59-1.41 0-.55-.23-1.06-.59-1.42zM5.5 7C4.67 7 4 6.33 4 5.5S4.67 4 5.5 4 7 4.67 7 5.5 6.33 7 5.5 7z M21 6H3c-1.1 0-2 .9-2 2v8c0 1.1.9 2 2 2h18c1.1 0 2-.9 2-2V8c0-1.1-.9-2-2-2zm-10 7H8v3H6v-3H3v-2h3V8h2v3h3v2zm4.5 2c-.83 0-1.5-.67-1.5-1.5s.67-1.5 1.5-1.5 1.5.67 1.5 1.5-.67 1.5-1.5 1.5zm3-3c-.83 0-1.5-.67-1.5-1.5s.67-1.5 1.5-1.5 1.5.67 1.5 1.5-.67 1.5-1.5 1.5z M17.65 6.35C16.2 4.9 14.21 4 12 4c-4.42 0-7.99 3.58-7.99 8s3.57 8 7.99 8c3.73 0 6.84-2.55 7.73-6h-2.08c-.82 2.33-3.04 4-5.65 4-3.31 0-6-2.69-6-6s2.69-6 6-6c1.66 0 3.14.69 4.22 1.78L13 11h7V4l-2.35 2.35z + M19,4H15.5L14.5,3H9.5L8.5,4H5V6H19M6,19A2,2 0 0,0 8,21H16A2,2 0 0,0 18,19V7H6V19Z M10 4H4c-1.1 0-1.99.9-1.99 2L2 18c0 1.1.9 2 2 2h16c1.1 0 2-.9 2-2V8c0-1.1-.9-2-2-2h-8l-2-2z M16 1H4c-1.1 0-2 .9-2 2v14h2V3h12V1zm3 4H8c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h11c1.1 0 2-.9 2-2V7c0-1.1-.9-2-2-2zm0 16H8V7h11v14z M17 3H5c-1.11 0-2 .9-2 2v14c0 1.1.89 2 2 2h14c1.1 0 2-.9 2-2V7l-4-4zm-5 16c-1.66 0-3-1.34-3-3s1.34-3 3-3 3 1.34 3 3-1.34 3-3 3zm3-10H5V5h10v4z diff --git a/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs b/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs index 98270b6..5ba2fd3 100644 --- a/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs +++ b/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs @@ -73,6 +73,17 @@ public void SelectCategory(string category) [ObservableProperty] private string _audioEngineUrl = "http://127.0.0.1:8880"; [ObservableProperty] private string _preferredAudioVoice = "af_heart"; + [ObservableProperty] private int _modelCacheIdleTimeoutMinutes = 15; + [ObservableProperty] private int _engineHibernationTimeoutMinutes = 30; + [ObservableProperty] private int _ramPressureThresholdPercent = 75; + + [ObservableProperty] private string _audioPath = ""; + [ObservableProperty] private string _videoModelsPath = ""; + [ObservableProperty] private string _videoOutputPath = ""; + [ObservableProperty] private bool _aiAssistantEnabled; + [ObservableProperty] private string _aiAssistantCustomSystemPrompt = ""; + [ObservableProperty] private string _aiAssistantPromptsDirectory = ""; + [ObservableProperty] private string _aiAssistantEndpoint = "http://127.0.0.1:4000/v1"; [ObservableProperty] private string _aiAssistantApiKey = ""; [ObservableProperty] private string _aiAssistantModel = "google/gemini-2.5-flash"; @@ -885,6 +896,16 @@ public async Task LoadSettingsAsync(string apiBase, HttpClient http) AiAssistantApiKey = settings.AiAssistantApiKey ?? ""; AiAssistantModel = settings.AiAssistantModel ?? "google/gemini-2.5-flash"; + AudioPath = settings.AudioPath ?? ""; + VideoModelsPath = settings.VideoModelsPath ?? ""; + VideoOutputPath = settings.VideoOutputPath ?? ""; + AiAssistantEnabled = settings.AiAssistantEnabled; + AiAssistantCustomSystemPrompt = settings.AiAssistantCustomSystemPrompt ?? ""; + AiAssistantPromptsDirectory = settings.AiAssistantPromptsDirectory ?? ""; + ModelCacheIdleTimeoutMinutes = settings.ModelCacheIdleTimeoutMinutes; + EngineHibernationTimeoutMinutes = settings.EngineHibernationTimeoutMinutes; + RamPressureThresholdPercent = settings.RamPressureThresholdPercent; + if (settings.CustomPresets != null && settings.CustomPresets.Count > 0) { foreach (var preset in settings.CustomPresets) @@ -936,15 +957,24 @@ public async Task SaveSettingsAsync(string apiBase, HttpClient http) ServiceName: this.ServiceName, PublishOutputPath: this.PublishOutputPath, ComfyModelsPath: this.ComfyModelsPath, + AudioPath: this.AudioPath, LanAccessUrl: this.LanAccessUrl, SelectedThemeStyle: this.SelectedThemeStyle, AudioEngineExecutablePath: this.AudioEngineExecutablePath, AudioEngineUrl: this.AudioEngineUrl, PreferredAudioVoice: this.PreferredAudioVoice, + VideoModelsPath: this.VideoModelsPath, + VideoOutputPath: this.VideoOutputPath, CustomPresets: customPresets, + AiAssistantEnabled: this.AiAssistantEnabled, AiAssistantEndpoint: this.AiAssistantEndpoint, AiAssistantApiKey: this.AiAssistantApiKey, - AiAssistantModel: this.AiAssistantModel + AiAssistantModel: this.AiAssistantModel, + AiAssistantCustomSystemPrompt: this.AiAssistantCustomSystemPrompt, + AiAssistantPromptsDirectory: this.AiAssistantPromptsDirectory, + ModelCacheIdleTimeoutMinutes: this.ModelCacheIdleTimeoutMinutes, + EngineHibernationTimeoutMinutes: this.EngineHibernationTimeoutMinutes, + RamPressureThresholdPercent: this.RamPressureThresholdPercent ); var content = new StringContent( diff --git a/LocalLLMServerManager.Shared/ViewModels/TelemetryViewModel.cs b/LocalLLMServerManager.Shared/ViewModels/TelemetryViewModel.cs index a4abec0..e643f11 100644 --- a/LocalLLMServerManager.Shared/ViewModels/TelemetryViewModel.cs +++ b/LocalLLMServerManager.Shared/ViewModels/TelemetryViewModel.cs @@ -14,11 +14,94 @@ public partial class TelemetryViewModel : ObservableObject private readonly ITelemetryService _telemetryService; [ObservableProperty] private bool _isCollapsed = false; - [ObservableProperty] private string _gpuName = "GPU Telemetry Active"; - [ObservableProperty] private double _vramUsedGb = 0.0; - [ObservableProperty] private double _vramTotalGb = 16.0; - [ObservableProperty] private double _vramPercentage = 0.0; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private string _gpuName = "GPU Telemetry Active"; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private double _vramUsedGb = 0.0; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private double _vramTotalGb = 16.0; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private double _vramPercentage = 0.0; + [ObservableProperty] private string _vramStatusText = "0.0 GB / 16.0 GB (0%)"; + [ObservableProperty] private string _vramCompactText = "0.0/16.0G"; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private double _ramUsedGb = 0.0; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private double _ramTotalGb = 32.0; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(RamBarColor))] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private double _ramPercentage = 0.0; + + [ObservableProperty] private string _ramStatusText = "0.0 GB / 32.0 GB (0%)"; + [ObservableProperty] private string _ramCompactText = "0.0/32.0G"; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private double _processWorkingSetMb = 0.0; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private double _nonPagedPoolMb = 0.0; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private int _modelCacheIdleTimeoutMinutes = 15; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private int _engineHibernationTimeoutMinutes = 30; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(RamBarColor))] + [NotifyPropertyChangedFor(nameof(HardwareTooltipText))] + private int _ramPressureThresholdPercent = 75; + + public string RamBarColor => RamPercentage >= 90 + ? "#EF4444" + : (RamPercentage >= RamPressureThresholdPercent ? "#F59E0B" : "#6366F1"); + + public string HardwareTooltipText => + $"{GpuName}\n" + + $"VRAM: {VramUsedGb:0.0} / {VramTotalGb:0.0} GB ({VramPercentage:0}%)\n" + + $"Host RAM: {RamUsedGb:0.0} / {RamTotalGb:0.0} GB ({RamPercentage:0}%)\n" + + $"App + Engine Working Set: {ProcessWorkingSetMb:0.0} MB\n" + + $"Kernel Non-Paged Pool: {NonPagedPoolMb:0.0} MB\n" + + $"Cache Lifetime: {ModelCacheIdleTimeoutMinutes}m Unload / {EngineHibernationTimeoutMinutes}m Hibernate / {RamPressureThresholdPercent}% Pressure"; + + partial void OnVramUsedGbChanged(double value) => UpdateVramStrings(); + partial void OnVramTotalGbChanged(double value) => UpdateVramStrings(); + partial void OnVramPercentageChanged(double value) => UpdateVramStrings(); + + private void UpdateVramStrings() + { + VramStatusText = $"{VramUsedGb:0.0} GB / {VramTotalGb:0.0} GB ({VramPercentage:0}%)"; + VramCompactText = $"{VramUsedGb:0.0}/{VramTotalGb:0.0}G"; + } + + partial void OnRamUsedGbChanged(double value) => UpdateRamStrings(); + partial void OnRamTotalGbChanged(double value) => UpdateRamStrings(); + partial void OnRamPercentageChanged(double value) => UpdateRamStrings(); + + private void UpdateRamStrings() + { + RamStatusText = $"{RamUsedGb:0.0} GB / {RamTotalGb:0.0} GB ({RamPercentage:0}%)"; + RamCompactText = $"{RamUsedGb:0.0}/{RamTotalGb:0.0}G"; + } [ObservableProperty] private EngineStatusCardModel _ollamaCard = new("ollama", "Ollama", 11434); [ObservableProperty] private EngineStatusCardModel _comfyUiCard = new("comfyui", "ComfyUI", 8188); @@ -385,6 +468,17 @@ public void UpdateFromTelemetry(TelemetryData data) VramUsedGb = data.UsedVramGb; VramPercentage = data.VramPercent > 0 ? data.VramPercent : Math.Round((VramUsedGb / VramTotalGb) * 100.0, 1); VramStatusText = $"{VramUsedGb} GB / {VramTotalGb} GB ({VramPercentage}%)"; + VramCompactText = $"{VramUsedGb:0.0}/{VramTotalGb:0.0}G"; + } + if (data.TotalRamGb > 0) + { + RamTotalGb = data.TotalRamGb; + RamUsedGb = data.UsedRamGb; + RamPercentage = data.RamPercent > 0 ? data.RamPercent : Math.Round((RamUsedGb / RamTotalGb) * 100.0, 1); + RamStatusText = $"{RamUsedGb:0.0} GB / {RamTotalGb:0.0} GB ({RamPercentage:0}%)"; + RamCompactText = $"{RamUsedGb:0.0}/{RamTotalGb:0.0}G"; + ProcessWorkingSetMb = data.ProcessWorkingSetMb; + NonPagedPoolMb = data.NonPagedPoolMb; } IsServiceRunning = (OllamaCard?.IsOnline ?? false) || (ComfyUiCard?.IsOnline ?? false) || (ForgeCard?.IsOnline ?? false) || (KokoroCard?.IsOnline ?? false); @@ -445,5 +539,48 @@ public async Task CheckGpuVramAsync(string apiBase, HttpClient http) VramUsedGb = info.UsedVramGb; VramPercentage = info.Percent; VramStatusText = $"{VramUsedGb} GB / {VramTotalGb} GB ({VramPercentage}%)"; + VramCompactText = $"{VramUsedGb:0.0}/{VramTotalGb:0.0}G"; + + RamTotalGb = info.TotalRamGb; + RamUsedGb = info.UsedRamGb; + RamPercentage = info.RamPercent; + RamStatusText = $"{RamUsedGb:0.0} GB / {RamTotalGb:0.0} GB ({RamPercentage:0}%)"; + RamCompactText = $"{RamUsedGb:0.0}/{RamTotalGb:0.0}G"; + ProcessWorkingSetMb = info.ProcessWorkingSetMb; + NonPagedPoolMb = info.NonPagedPoolMb; + ModelCacheIdleTimeoutMinutes = info.ModelCacheIdleTimeoutMinutes; + EngineHibernationTimeoutMinutes = info.EngineHibernationTimeoutMinutes; + RamPressureThresholdPercent = info.RamPressureThresholdPercent; + } + + [RelayCommand] + public async Task PurgeMemoryAsync() + { + await PurgeMemoryAsync(ApiBase, HttpHelper.CreateClient(ApiBase)); + } + + public async Task PurgeMemoryAsync(string apiBase, HttpClient http) + { + try + { + var url = HttpHelper.FormatEndpoint(apiBase, "/api/memory/purge"); + var resp = await http.PostAsync(url, null); + if (resp.IsSuccessStatusCode) + { + ToastService.Instance.Show("Memory caches purged.", ToastType.Success, 2500); + } + else + { + ToastService.Instance.Show("Failed to purge memory caches.", ToastType.Warning, 2500); + } + } + catch (Exception ex) + { + ToastService.Instance.Show($"Error purging memory: {ex.Message}", ToastType.Error, 3000); + } + finally + { + await CheckGpuVramAsync(apiBase, http); + } } } diff --git a/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml b/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml index 38e788c..1f63f4e 100644 --- a/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml +++ b/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml @@ -493,6 +493,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +