From 78151fb6d900654402a9eba9407d3760139f356f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:18:16 +0000 Subject: [PATCH 1/9] Add opt-in QEMU vGPU snapshot fanout --- .github/workflows/qemu-vgpu-runtime.yml | 68 ++ cmd/api/api/instances.go | 4 + cmd/api/api/instances_runtime_test.go | 32 + cmd/api/config/config.go | 5 + config.example.yaml | 1 + lib/hypervisor/config.go | 2 + lib/hypervisor/hypervisor.go | 6 +- lib/hypervisor/qemu/config.go | 13 +- lib/hypervisor/qemu/fork.go | 12 + lib/hypervisor/qemu/process.go | 81 ++- lib/hypervisor/qemu/qemu.go | 31 +- lib/hypervisor/qemu/qmp.go | 21 + lib/hypervisor/qemu/vgpu.go | 231 ++++++ lib/hypervisor/qemu/vgpu_test.go | 111 +++ lib/instances/create.go | 12 +- lib/instances/fork.go | 9 +- lib/instances/lifecycle_noop_test.go | 2 +- lib/instances/manager.go | 7 + lib/instances/restore.go | 28 +- lib/instances/snapshot.go | 3 +- lib/instances/standby.go | 43 +- lib/instances/vgpu_restore_slots_test.go | 33 + lib/instances/vgpu_snapshot.go | 117 +++ lib/instances/vgpu_snapshot_linux_test.go | 122 ++++ lib/oapi/oapi.go | 787 +++++++++++---------- lib/providers/providers.go | 7 +- lib/system/guest_agent/vsock_linux.go | 4 +- lib/system/guest_agent/vsock_linux_test.go | 24 + openapi.yaml | 9 + scripts/qemu-vgpu/Dockerfile | 10 + scripts/qemu-vgpu/README.md | 30 + scripts/qemu-vgpu/build.sh | 59 ++ scripts/qemu-vgpu/test-batched-load.py | 88 +++ scripts/qemu-vgpu/vfio-batched-load.patch | 45 ++ 34 files changed, 1617 insertions(+), 440 deletions(-) create mode 100644 .github/workflows/qemu-vgpu-runtime.yml create mode 100644 cmd/api/api/instances_runtime_test.go create mode 100644 lib/hypervisor/qemu/vgpu.go create mode 100644 lib/hypervisor/qemu/vgpu_test.go create mode 100644 lib/instances/vgpu_restore_slots_test.go create mode 100644 lib/instances/vgpu_snapshot.go create mode 100644 lib/instances/vgpu_snapshot_linux_test.go create mode 100644 lib/system/guest_agent/vsock_linux_test.go create mode 100644 scripts/qemu-vgpu/Dockerfile create mode 100644 scripts/qemu-vgpu/README.md create mode 100755 scripts/qemu-vgpu/build.sh create mode 100644 scripts/qemu-vgpu/test-batched-load.py create mode 100644 scripts/qemu-vgpu/vfio-batched-load.patch diff --git a/.github/workflows/qemu-vgpu-runtime.yml b/.github/workflows/qemu-vgpu-runtime.yml new file mode 100644 index 000000000..ff73847f6 --- /dev/null +++ b/.github/workflows/qemu-vgpu-runtime.yml @@ -0,0 +1,68 @@ +name: Build QEMU vGPU runtime + +on: + pull_request: + paths: + - scripts/qemu-vgpu/** + - lib/hypervisor/qemu/vgpu.go + - .github/workflows/qemu-vgpu-runtime.yml + workflow_dispatch: + inputs: + publish: + description: Publish the immutable runtime release from main + type: boolean + default: false + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + outputs: + version: ${{ steps.package.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Build and test pinned runtime + run: docker build --output type=local,dest=runtime -f scripts/qemu-vgpu/Dockerfile . + - name: Package runtime and corresponding source + id: package + run: | + VERSION=$(python3 -c 'import json,re;v=json.load(open("runtime/manifest.json"))["version"];assert re.fullmatch(r"[0-9][a-z0-9.-]+",v);print(v)') + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + mkdir artifacts + mv runtime/source.tar.gz "artifacts/qemu-vgpu-${VERSION}-source.tar.gz" + tar -czf "artifacts/qemu-vgpu-${VERSION}-linux-amd64.tar.gz" -C runtime . + cd artifacts + sha256sum *.tar.gz > SHA256SUMS + - uses: actions/upload-artifact@v4 + with: + name: qemu-vgpu-runtime + path: artifacts/ + retention-days: 30 + + publish: + if: github.event_name == 'workflow_dispatch' && inputs.publish && github.ref == 'refs/heads/main' + needs: build + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + name: qemu-vgpu-runtime + path: artifacts + - name: Publish a new runtime version without replacing existing assets + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + VERSION: ${{ needs.build.outputs.version }} + COMMIT: ${{ github.sha }} + run: | + gh release create "qemu-vgpu-${VERSION}" artifacts/* \ + --target "$COMMIT" \ + --title "QEMU vGPU runtime ${VERSION}" \ + --notes "Opt-in QEMU runtime. Includes checksums and corresponding patched source. Does not replace the system QEMU or enable GPU snapshot provisioning." diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index d5aa47fcf..93c5792a1 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -354,6 +354,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Devices: deviceRefs, Volumes: volumes, Hypervisor: hvType, + HypervisorVersion: lo.FromPtr(request.Body.HypervisorVersion), GPU: gpuConfig, Entrypoint: entrypoint, Cmd: cmd, @@ -1248,6 +1249,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { _ = json.Unmarshal(b, &oapiInst.Network) } + if inst.HypervisorVersion != "" && inst.HypervisorVersion != "unknown" { + oapiInst.HypervisorVersion = lo.ToPtr(inst.HypervisorVersion) + } if inst.Platform != "" { oapiInst.Platform = lo.ToPtr(inst.Platform) } diff --git a/cmd/api/api/instances_runtime_test.go b/cmd/api/api/instances_runtime_test.go new file mode 100644 index 000000000..7d3673805 --- /dev/null +++ b/cmd/api/api/instances_runtime_test.go @@ -0,0 +1,32 @@ +package api + +import ( + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/qemu" + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/oapi" + "github.com/stretchr/testify/require" +) + +func TestCreateInstanceMapsRequestedRuntimeVersion(t *testing.T) { + svc := newTestService(t) + manager := newCaptureCreateManager(svc.InstanceManager) + svc.InstanceManager = manager + runtime := oapi.CreateInstanceRequestHypervisor(hypervisor.TypeQEMU) + version := qemu.VGPUSnapshotVersion + response, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{Body: &oapi.CreateInstanceRequest{Name: "runtime-version-test", Image: "docker.io/library/alpine:latest", Hypervisor: &runtime, HypervisorVersion: &version}}) + require.NoError(t, err) + require.IsType(t, oapi.CreateInstance201JSONResponse{}, response) + require.Equal(t, version, manager.lastReq.HypervisorVersion) +} + +func TestInstanceRuntimeVersionResponse(t *testing.T) { + inst := instances.Instance{StoredMetadata: instances.StoredMetadata{HypervisorVersion: qemu.VGPUSnapshotVersion}} + response := instanceToOAPI(inst) + require.NotNil(t, response.HypervisorVersion) + require.Equal(t, qemu.VGPUSnapshotVersion, *response.HypervisorVersion) + inst.HypervisorVersion = "" + require.Nil(t, instanceToOAPI(inst).HypervisorVersion) +} diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index cc4a82476..ded87d2b3 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -222,6 +222,7 @@ type HypervisorConfig struct { CloudHypervisorDefaultVersion string `koanf:"cloud_hypervisor_default_version"` FirecrackerBinaryPath string `koanf:"firecracker_binary_path"` FirecrackerMaxConcurrentRestores int `koanf:"firecracker_max_concurrent_restores"` + VGPUMaxConcurrentRestores int `koanf:"vgpu_max_concurrent_restores"` Memory HypervisorMemoryConfig `koanf:"memory"` } @@ -461,6 +462,7 @@ func defaultConfig() *Config { CloudHypervisorDefaultVersion: "", FirecrackerBinaryPath: "", FirecrackerMaxConcurrentRestores: 32, + VGPUMaxConcurrentRestores: 2, Memory: HypervisorMemoryConfig{ Enabled: false, KernelPageInitMode: "hardened", @@ -715,6 +717,9 @@ func (c *Config) Validate() error { } c.Builders.IdleTTLDuration = ttl } + if c.Hypervisor.VGPUMaxConcurrentRestores < 0 { + return fmt.Errorf("hypervisor.vgpu_max_concurrent_restores must be >= 0") + } if c.Hypervisor.FirecrackerMaxConcurrentRestores < 0 { return fmt.Errorf("hypervisor.firecracker_max_concurrent_restores must be >= 0, got %d", c.Hypervisor.FirecrackerMaxConcurrentRestores) } diff --git a/config.example.yaml b/config.example.yaml index 3990cda67..20eb5e5aa 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -40,6 +40,7 @@ data_dir: /var/lib/hypeman # # firecracker_binary_path: /usr/local/bin/firecracker # # Caps concurrent Firecracker snapshot restore/resume work to avoid host overload. # # firecracker_max_concurrent_restores: 32 +# # vgpu_max_concurrent_restores: 2 # memory: # enabled: false # # performance: init_on_alloc=0 init_on_free=0 (better density) diff --git a/lib/hypervisor/config.go b/lib/hypervisor/config.go index da8c41daa..23356e5dd 100644 --- a/lib/hypervisor/config.go +++ b/lib/hypervisor/config.go @@ -26,6 +26,8 @@ type VMConfig struct { // PCI device passthrough (GPU, etc.) PCIDevices []string VGPUDevicePath string + VGPUSnapshot bool + InstanceID string // Boot configuration. Empty BootMode preserves the existing direct-kernel // behavior for Linux callers. diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index 1aff356cd..284a79d13 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -272,8 +272,10 @@ type ForkPrepareRequest struct { VsockCID int64 VsockSocket string - SerialLogPath string - Network *ForkNetworkConfig + SerialLogPath string + Network *ForkNetworkConfig + InstanceID string + VGPUDevicePath string } // ForkPrepareResult describes which optional fork rewrites were actually applied. diff --git a/lib/hypervisor/qemu/config.go b/lib/hypervisor/qemu/config.go index 81c086245..cb1c632f9 100644 --- a/lib/hypervisor/qemu/config.go +++ b/lib/hypervisor/qemu/config.go @@ -25,6 +25,9 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string { // Machine type with KVM acceleration (arch-specific when omitted). machineArg := string(machine) + ",accel=kvm" + if cfg.VGPUSnapshot { + machineArg += ",smbios-entry-point-type=32" + } if cfg.Firmware != nil && cfg.Firmware.SecureBoot { machineArg += ",smm=on" } @@ -47,6 +50,10 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string { // Memory configuration memMB := cfg.MemoryBytes / (1024 * 1024) args = append(args, "-m", fmt.Sprintf("%dM", memMB)) + if cfg.VGPUSnapshot { + id := vgpuInstanceUUID(cfg.InstanceID) + args = append(args, "-mem-prealloc", "-device", "vmgenid,guid=auto", "-uuid", id, "-smbios", "type=1,uuid="+id) + } if cfg.GuestMemory.EnableBalloon { balloonOpts := []string{virtioDevice(microvm, "virtio-balloon")} @@ -144,7 +151,11 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string { } if cfg.VGPUDevicePath != "" { - args = append(args, "-device", fmt.Sprintf("vfio-pci,sysfsdev=%s", cfg.VGPUDevicePath)) + device := fmt.Sprintf("vfio-pci,sysfsdev=%s", cfg.VGPUDevicePath) + if cfg.VGPUSnapshot { + device += ",enable-migration=on" + } + args = append(args, "-device", device) } // Serial console output to file. Use a chardev with append=on so QEMU diff --git a/lib/hypervisor/qemu/fork.go b/lib/hypervisor/qemu/fork.go index 4599c033e..f571d8398 100644 --- a/lib/hypervisor/qemu/fork.go +++ b/lib/hypervisor/qemu/fork.go @@ -32,6 +32,18 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq return hypervisor.ForkPrepareResult{}, fmt.Errorf("select qemu fork machine type: %w", err) } cfg := saved.VMConfig + if cfg.VGPUSnapshot { + if req.InstanceID != "" { + cfg.InstanceID = req.InstanceID + } + if req.VGPUDevicePath != "" { + cfg.VGPUDevicePath = req.VGPUDevicePath + } + if req.SourceDataDir != req.TargetDataDir && req.TargetDataDir != "" { + // A child must claim a new VF before restore can launch it. + cfg.VGPUDevicePath = "" + } + } if req.SourceDataDir != "" && req.TargetDataDir != "" && req.SourceDataDir != req.TargetDataDir { cfg = rewriteQEMUConfigPaths(cfg, req.SourceDataDir, req.TargetDataDir) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index c35ba0197..4e88b856f 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -16,6 +16,7 @@ import ( "syscall" "time" + "github.com/digitalocean/go-qemu/qmp" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/paths" @@ -90,8 +91,12 @@ func (s *Starter) SocketName() string { } // GetBinaryPath returns the path to the QEMU binary. -// QEMU is expected to be installed on the system. +// Unversioned/default requests use the system binary; vGPU snapshots opt in to a managed runtime. func (s *Starter) GetBinaryPath(p *paths.Paths, version string) (string, error) { + if version == VGPUSnapshotVersion { + binary, _, err := loadVGPURuntime(p) + return binary, err + } binaryName, err := qemuBinaryName() if err != nil { return "", err @@ -120,8 +125,11 @@ func (s *Starter) GetVersion(p *paths.Paths) (string, error) { return s.detectVersion(p) } -// ResolveVersion binds QEMU metadata to the one host-installed binary. +// ResolveVersion pins either the system QEMU version or the explicitly selected managed runtime. func (s *Starter) ResolveVersion(p *paths.Paths, requested string) (string, error) { + if requested == VGPUSnapshotVersion { + return s.resolveVGPURuntime(p) + } installed, err := s.GetVersion(p) if err != nil { return "", err @@ -451,6 +459,9 @@ func (s *Starter) validateSnapshotMachineType(stored MachineType) (MachineType, if s.profile.requiresStoredMachineType() && stored == "" { return "", fmt.Errorf("%s snapshot is missing its machine type", s.profile.hypervisorType()) } + if stored == vgpuMachineType && s.profile.hypervisorType() == hypervisor.TypeQEMU { + return stored, nil + } if stored != "" && stored != expected { return "", fmt.Errorf("hypervisor %s cannot use QEMU machine type %q", s.profile.hypervisorType(), stored) } @@ -495,6 +506,23 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s } } + var gpuContract *vgpuSnapshotContract + var runtimeArgs []string + if config.VGPUSnapshot || qemuVersion == VGPUSnapshotVersion { + if !config.VGPUSnapshot || qemuVersion != VGPUSnapshotVersion || s.profile.hypervisorType() != hypervisor.TypeQEMU { + return 0, nil, fmt.Errorf("vGPU snapshots require the managed QEMU runtime and an eligible GPU configuration") + } + gpuContract, err = newVGPUSnapshotContract(p, config) + if err != nil { + return 0, nil, err + } + runtimeArgs, err = vgpuRuntimeArgs(p) + if err != nil { + return 0, nil, err + } + machineType = vgpuMachineType + } + // Some distro QEMU builds may not support newer balloon sub-options. // Retry with progressively more conservative balloon args before failing. attempts := []hypervisor.VMConfig{config} @@ -525,6 +553,7 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s // Build command arguments: QMP socket + VM configuration args := buildQMPArgs(socketPath) args = append(args, buildArgs(attempt, machineType)...) + args = append(args, runtimeArgs...) pid, hv, cu, err = s.startConfiguredProcess(ctx, p, version, socketPath, attempt, args) if err == nil { booted = attempt @@ -558,9 +587,9 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s // compatible across QEMU versions, so stamp the selected binary's version // alongside the device model. instanceDir := filepath.Dir(socketPath) - if err := saveVMConfig(instanceDir, savedVMConfig{VMConfig: booted, MachineType: machineType, QEMUVersion: qemuVersion}); err != nil { - if machineType == MachineTypeMicroVM { - return 0, nil, fmt.Errorf("save qemu-microvm restore contract: %w", err) + if err := saveVMConfig(instanceDir, savedVMConfig{VMConfig: booted, MachineType: machineType, QEMUVersion: qemuVersion, VGPU: gpuContract}); err != nil { + if machineType == MachineTypeMicroVM || config.VGPUSnapshot { + return 0, nil, fmt.Errorf("save QEMU restore contract: %w", err) } // Standard QEMU can continue, but standby restore will be unavailable. log.WarnContext(ctx, "failed to save VM config for restore", "error", err) @@ -654,11 +683,30 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, args := buildQMPArgs(socketPath) args = append(args, buildArgs(config, machineType)...) - // Add incoming migration flag to restore from snapshot - // The "file:" protocol is deprecated in QEMU 7.2+, use "exec:cat < path" instead memoryFile := filepath.Join(snapshotPath, "memory") - incomingURI := "exec:cat < " + memoryFile - args = append(args, "-incoming", incomingURI) + if config.VGPUSnapshot { + if saved.QEMUVersion != VGPUSnapshotVersion || version != VGPUSnapshotVersion || machineType != vgpuMachineType { + return 0, nil, fmt.Errorf("invalid vGPU snapshot runtime contract") + } + current, err := newVGPUSnapshotContract(p, config) + if err != nil { + return 0, nil, err + } + if err := validateVGPUSnapshotContract(saved.VGPU, current); err != nil { + return 0, nil, err + } + runtimeArgs, err := vgpuRuntimeArgs(p) + if err != nil { + return 0, nil, err + } + args = append(args, runtimeArgs...) + args = append(args, "-incoming", "defer", "-S") + } else { + if saved.VGPU != nil || saved.QEMUVersion == VGPUSnapshotVersion { + return 0, nil, fmt.Errorf("vGPU snapshot is missing its device configuration") + } + args = append(args, "-incoming", "exec:cat < "+memoryFile) + } pid, hv, cu, err := s.startConfiguredProcess(ctx, p, version, socketPath, config, args) if err != nil { @@ -666,6 +714,15 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, } defer cu.Clean() + if config.VGPUSnapshot { + if err := hv.client.configureVGPUMigration(); err != nil { + return 0, nil, err + } + if _, err := hv.client.Run(qmp.Command{Execute: "migrate-incoming", Args: map[string]string{"uri": "file:" + memoryFile}}); err != nil { + return 0, nil, fmt.Errorf("load vGPU snapshot: %w", err) + } + } + // Wait for VM to be ready after loading migration data // QEMU transitions from "inmigrate" to "paused" when loading completes migrationWaitStart := time.Now() @@ -674,7 +731,7 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, } log.DebugContext(ctx, "VM ready", "duration_ms", time.Since(migrationWaitStart).Milliseconds()) - if err := saveVMConfig(filepath.Dir(socketPath), savedVMConfig{VMConfig: config, MachineType: machineType, QEMUVersion: qemuVersion}); err != nil { + if err := saveVMConfig(filepath.Dir(socketPath), savedVMConfig{VMConfig: config, MachineType: machineType, QEMUVersion: qemuVersion, VGPU: saved.VGPU}); err != nil { return 0, nil, fmt.Errorf("save restored vm config: %w", err) } @@ -684,6 +741,9 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, } func (s *Starter) validateRestoreVersion(p *paths.Paths, storedVersion string) (string, error) { + if storedVersion == VGPUSnapshotVersion { + return s.resolveVGPURuntime(p) + } currentVersion, err := s.GetVersion(p) if err != nil { return "", fmt.Errorf("get QEMU version for restore: %w", err) @@ -711,6 +771,7 @@ type savedVMConfig struct { hypervisor.VMConfig MachineType MachineType QEMUVersion string + VGPU *vgpuSnapshotContract `json:",omitempty"` } // saveVMConfig saves QEMU's restore contract to the instance directory. diff --git a/lib/hypervisor/qemu/qemu.go b/lib/hypervisor/qemu/qemu.go index b36a2fc94..ab7e4ce7c 100644 --- a/lib/hypervisor/qemu/qemu.go +++ b/lib/hypervisor/qemu/qemu.go @@ -133,10 +133,29 @@ func (q *QEMU) Resume(ctx context.Context) error { // The VM state is saved to destPath/memory file. // The VM config is copied to destPath for restore (QEMU requires exact arg match). func (q *QEMU) Snapshot(ctx context.Context, destPath string) error { - // QEMU uses migrate to file for snapshots - // The "file:" protocol is deprecated in QEMU 7.2+, use "exec:cat > path" instead - memoryFile := destPath + "/memory" + instanceDir := filepath.Dir(q.socketPath) + saved, err := loadVMConfig(instanceDir) + if err != nil { + return fmt.Errorf("load snapshot configuration: %w", err) + } + memoryFile := filepath.Join(destPath, "memory") uri := "exec:cat > " + memoryFile + if saved.VGPUSnapshot { + if saved.VGPU == nil || saved.QEMUVersion != VGPUSnapshotVersion { + return fmt.Errorf("missing vGPU snapshot compatibility contract") + } + state, err := q.client.Status() + if err != nil { + return err + } + if state != qemu.StatusRunning { + return fmt.Errorf("vGPU snapshot requires a running guest") + } + if err := q.client.configureVGPUMigration(); err != nil { + return err + } + uri = "file:" + memoryFile + } if err := q.client.Migrate(uri); err != nil { removeClient(q) return fmt.Errorf("migrate: %w", err) @@ -144,13 +163,17 @@ func (q *QEMU) Snapshot(ctx context.Context, destPath string) error { // Wait for migration to complete if err := q.client.WaitMigration(ctx, migrationTimeout); err != nil { + if saved.VGPUSnapshot { + if cancelErr := q.client.cancelMigration(); cancelErr != nil { + return fmt.Errorf("snapshot failed: %w; cancel migration: %v", err, cancelErr) + } + } removeClient(q) return fmt.Errorf("wait migration: %w", err) } // Copy VM config from instance dir to snapshot dir // QEMU restore requires exact same command-line args as when snapshot was taken - instanceDir := filepath.Dir(q.socketPath) srcConfig := filepath.Join(instanceDir, vmConfigFile) dstConfig := filepath.Join(destPath, vmConfigFile) diff --git a/lib/hypervisor/qemu/qmp.go b/lib/hypervisor/qemu/qmp.go index 29f9eb404..90e239667 100644 --- a/lib/hypervisor/qemu/qmp.go +++ b/lib/hypervisor/qemu/qmp.go @@ -131,6 +131,27 @@ func (c *Client) QueryMigration() (raw.MigrationInfo, error) { return c.raw.QueryMigrate() } +func (c *Client) cancelMigration() error { + if _, err := c.Run(qmp.Command{Execute: "migrate_cancel"}); err != nil { + return err + } + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + info, err := c.QueryMigration() + if err != nil { + return err + } + if info.Status != nil { + switch *info.Status { + case raw.MigrationStatusCancelled, raw.MigrationStatusCompleted, raw.MigrationStatusFailed, raw.MigrationStatusNone: + return nil + } + } + time.Sleep(qmpPollInterval) + } + return fmt.Errorf("migration did not stop after cancellation") +} + // WaitMigration polls until an outgoing migration completes or times out. // Used for snapshot/standby operations where we initiate the migration. // Returns nil if migration completed successfully, error otherwise. diff --git a/lib/hypervisor/qemu/vgpu.go b/lib/hypervisor/qemu/vgpu.go new file mode 100644 index 000000000..7d2ab4042 --- /dev/null +++ b/lib/hypervisor/qemu/vgpu.go @@ -0,0 +1,231 @@ +package qemu + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/digitalocean/go-qemu/qmp" + "github.com/google/uuid" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" +) + +// VGPUSnapshotVersion selects the separately installed, snapshot-capable runtime. +const VGPUSnapshotVersion = "11.1.3-vgpu.1" + +const ( + vgpuUpstreamVersion = "11.1.3" + vgpuSourceCommit = "005c61c0ef6f6ae615283440b4bb7be51b2c6ddd" + vgpuMachineType MachineType = "pc-q35-8.2" +) + +type vgpuRuntimeManifest struct { + Version string `json:"version"` + SourceCommit string `json:"source_commit"` + BinarySHA256 string `json:"binary_sha256"` + FirmwareSHA256 string `json:"firmware_sha256"` +} + +type vgpuSnapshotContract struct { + Runtime vgpuRuntimeManifest + DriverVersion string + VendorID string + DeviceID string + ProfileType string +} + +func vgpuRuntimeDir(p *paths.Paths) (string, error) { + if p == nil || runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + return "", fmt.Errorf("vGPU snapshots require a managed Linux amd64 runtime") + } + return filepath.Join(p.DataDir(), "system", "binaries", "qemu", VGPUSnapshotVersion, "x86_64"), nil +} + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func firmwareSHA256(dir string) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", err + } + if len(entries) == 0 { + return "", fmt.Errorf("empty runtime firmware directory") + } + h := sha256.New() + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return "", err + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("runtime firmware must contain regular files") + } + h.Write([]byte(entry.Name() + "\x00")) + if err := binary.Write(h, binary.LittleEndian, uint64(info.Size())); err != nil { + return "", err + } + f, err := os.Open(filepath.Join(dir, entry.Name())) + if err != nil { + return "", err + } + _, err = io.Copy(h, f) + f.Close() + if err != nil { + return "", err + } + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func loadVGPURuntime(p *paths.Paths) (string, vgpuRuntimeManifest, error) { + dir, err := vgpuRuntimeDir(p) + if err != nil { + return "", vgpuRuntimeManifest{}, err + } + var manifest vgpuRuntimeManifest + data, err := os.ReadFile(filepath.Join(dir, "manifest.json")) + if err != nil { + return "", manifest, fmt.Errorf("read managed QEMU runtime manifest: %w", err) + } + if err := json.Unmarshal(data, &manifest); err != nil { + return "", manifest, fmt.Errorf("decode managed QEMU runtime manifest: %w", err) + } + if manifest.Version != VGPUSnapshotVersion || manifest.SourceCommit != vgpuSourceCommit { + return "", manifest, fmt.Errorf("unsupported managed QEMU runtime manifest") + } + for name, expected := range map[string]string{"qemu-system-x86_64": manifest.BinarySHA256, "firmware": manifest.FirmwareSHA256} { + var actual string + var err error + if name == "firmware" { + actual, err = firmwareSHA256(filepath.Join(dir, name)) + } else { + actual, err = fileSHA256(filepath.Join(dir, name)) + } + if err != nil { + return "", manifest, fmt.Errorf("verify managed runtime %s: %w", name, err) + } + if len(expected) != 64 || actual != expected { + return "", manifest, fmt.Errorf("managed runtime %s checksum mismatch", name) + } + } + binary := filepath.Join(dir, "qemu-system-x86_64") + if err := validateExecutable(binary); err != nil { + return "", manifest, fmt.Errorf("managed QEMU binary: %w", err) + } + return binary, manifest, nil +} + +func (s *Starter) resolveVGPURuntime(p *paths.Paths) (string, error) { + if s.profile.hypervisorType() != hypervisor.TypeQEMU { + return "", fmt.Errorf("vGPU snapshot runtime requires standard QEMU") + } + binary, _, err := loadVGPURuntime(p) + if err != nil { + return "", err + } + ctx, cancel := context.WithTimeout(context.Background(), versionProbeTimeout) + defer cancel() + version, err := versionFromBinary(ctx, binary) + if err != nil { + return "", err + } + if version != vgpuUpstreamVersion { + return "", fmt.Errorf("managed QEMU version %s does not match %s", version, vgpuUpstreamVersion) + } + return VGPUSnapshotVersion, nil +} + +func newVGPUSnapshotContract(p *paths.Paths, config hypervisor.VMConfig) (*vgpuSnapshotContract, error) { + if config.VGPUDevicePath == "" || len(config.PCIDevices) != 0 || config.EffectiveBootMode() != hypervisor.BootModeDirect || config.InstanceID == "" { + return nil, fmt.Errorf("vGPU snapshots require an identified direct-boot guest with one vendor VFIO vGPU and no other PCI assignments") + } + if !strings.HasPrefix(config.VGPUDevicePath, "/sys/bus/pci/devices/") { + return nil, fmt.Errorf("vGPU snapshots require vendor VFIO") + } + _, manifest, err := loadVGPURuntime(p) + if err != nil { + return nil, err + } + read := func(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimSpace(string(data)) + if value == "" { + return "", fmt.Errorf("empty compatibility field %s", path) + } + return value, nil + } + contract := &vgpuSnapshotContract{Runtime: manifest} + if contract.DriverVersion, err = read("/sys/module/nvidia/version"); err != nil { + return nil, fmt.Errorf("read NVIDIA host driver: %w", err) + } + if contract.VendorID, err = read(filepath.Join(config.VGPUDevicePath, "vendor")); err != nil { + return nil, err + } + if contract.DeviceID, err = read(filepath.Join(config.VGPUDevicePath, "device")); err != nil { + return nil, err + } + if contract.ProfileType, err = read(filepath.Join(config.VGPUDevicePath, "nvidia", "current_vgpu_type")); err != nil { + return nil, err + } + if contract.ProfileType == "0" { + return nil, fmt.Errorf("vGPU snapshot requires an allocated profile") + } + if contract.VendorID != "0x10de" { + return nil, fmt.Errorf("vGPU snapshots require an NVIDIA device") + } + return contract, nil +} + +func validateVGPUSnapshotContract(saved, current *vgpuSnapshotContract) error { + if saved == nil || current == nil || *saved != *current { + return fmt.Errorf("vGPU snapshot runtime, firmware or host driver/device changed; rebuild the template") + } + return nil +} + +func vgpuRuntimeArgs(p *paths.Paths) ([]string, error) { + dir, err := vgpuRuntimeDir(p) + if err != nil { + return nil, err + } + return []string{"-L", filepath.Join(dir, "firmware"), "-bios", filepath.Join(dir, "firmware", "bios-256k.bin")}, nil +} + +func vgpuInstanceUUID(instanceID string) string { + return uuid.NewSHA1(uuid.NameSpaceOID, []byte(instanceID)).String() +} + +func (c *Client) configureVGPUMigration() error { + for _, command := range []qmp.Command{ + {Execute: "migrate-set-capabilities", Args: map[string]any{"capabilities": []map[string]any{{"capability": "mapped-ram", "state": true}, {"capability": "multifd", "state": true}}}}, + {Execute: "migrate-set-parameters", Args: map[string]any{"multifd-channels": 4, "max-bandwidth": int64(1 << 30), "downtime-limit": 10000}}, + } { + if _, err := c.Run(command); err != nil { + return fmt.Errorf("configure GPU migration: %w", err) + } + } + return nil +} diff --git a/lib/hypervisor/qemu/vgpu_test.go b/lib/hypervisor/qemu/vgpu_test.go new file mode 100644 index 000000000..bcb48f715 --- /dev/null +++ b/lib/hypervisor/qemu/vgpu_test.go @@ -0,0 +1,111 @@ +package qemu + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func managedVGPUFixture(t *testing.T) (*paths.Paths, string, vgpuRuntimeManifest) { + t.Helper() + p := paths.New(t.TempDir()) + dir, err := vgpuRuntimeDir(p) + if err != nil { + t.Skip(err) + } + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "qemu-system-x86_64"), []byte("#!/bin/sh\necho 'QEMU emulator version 11.1.3'\n"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "firmware"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "firmware", "bios-256k.bin"), []byte("firmware"), 0o644)) + binaryHash, err := fileSHA256(filepath.Join(dir, "qemu-system-x86_64")) + require.NoError(t, err) + firmwareHash, err := firmwareSHA256(filepath.Join(dir, "firmware")) + require.NoError(t, err) + manifest := vgpuRuntimeManifest{Version: VGPUSnapshotVersion, SourceCommit: vgpuSourceCommit, BinarySHA256: binaryHash, FirmwareSHA256: firmwareHash} + data, err := json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.json"), data, 0o644)) + return p, dir, manifest +} + +func TestManagedVGPURuntimeIsExplicitAndChecksumPinned(t *testing.T) { + p, dir, _ := managedVGPUFixture(t) + starter := NewStarter() + version, err := starter.ResolveVersion(p, VGPUSnapshotVersion) + require.NoError(t, err) + require.Equal(t, VGPUSnapshotVersion, version) + binary, err := starter.GetBinaryPath(p, version) + require.NoError(t, err) + require.Equal(t, filepath.Join(dir, "qemu-system-x86_64"), binary) + _, err = NewMicroVMStarter().ResolveVersion(p, version) + require.ErrorContains(t, err, "standard QEMU") + require.NoError(t, os.WriteFile(filepath.Join(dir, "firmware", "bios-256k.bin"), []byte("different firmware"), 0o644)) + _, err = starter.ResolveVersion(p, version) + require.ErrorContains(t, err, "checksum mismatch") +} + +func TestManagedVGPURuntimeMissingDoesNotUseSystemQEMU(t *testing.T) { + _, err := NewStarter().ResolveVersion(paths.New(t.TempDir()), VGPUSnapshotVersion) + require.Error(t, err) +} + +func TestVGPUSnapshotContractRejectsCompatibilityChanges(t *testing.T) { + _, _, runtime := managedVGPUFixture(t) + base := vgpuSnapshotContract{Runtime: runtime, DriverVersion: "driver-a", VendorID: "0x10de", DeviceID: "device-a"} + require.NoError(t, validateVGPUSnapshotContract(&base, &base)) + for _, change := range []func(*vgpuSnapshotContract){ + func(v *vgpuSnapshotContract) { v.Runtime.BinarySHA256 = strings.Repeat("0", 64) }, + func(v *vgpuSnapshotContract) { v.Runtime.FirmwareSHA256 = strings.Repeat("0", 64) }, + func(v *vgpuSnapshotContract) { v.DriverVersion = "driver-b" }, + func(v *vgpuSnapshotContract) { v.DeviceID = "device-b" }, + func(v *vgpuSnapshotContract) { v.ProfileType = "profile-b" }, + } { + other := base + change(&other) + require.Error(t, validateVGPUSnapshotContract(&base, &other)) + } + require.Error(t, validateVGPUSnapshotContract(nil, &base)) +} + +func TestVGPUForkClearsSourceDeviceAndBindsFreshIdentity(t *testing.T) { + dir := t.TempDir() + oldDevice := "/sys/bus/pci/devices/0000:01:00.1" + newDevice := "/sys/bus/pci/devices/0000:01:00.2" + saved := savedVMConfig{VMConfig: hypervisor.VMConfig{VGPUSnapshot: true, InstanceID: "source", VGPUDevicePath: oldDevice, VsockCID: 40}, MachineType: vgpuMachineType, QEMUVersion: VGPUSnapshotVersion} + require.NoError(t, saveVMConfig(dir, saved)) + starter := NewStarter() + _, err := starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{SnapshotConfigPath: filepath.Join(dir, "config.json"), SourceDataDir: "/source", TargetDataDir: "/child", InstanceID: "child", VsockCID: 41}) + require.NoError(t, err) + fork, err := loadVMConfig(dir) + require.NoError(t, err) + require.Empty(t, fork.VGPUDevicePath) + require.Equal(t, "child", fork.InstanceID) + require.EqualValues(t, 41, fork.VsockCID) + _, err = starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{SnapshotConfigPath: filepath.Join(dir, "config.json"), VGPUDevicePath: newDevice}) + require.NoError(t, err) + fork, err = loadVMConfig(dir) + require.NoError(t, err) + require.Equal(t, newDevice, fork.VGPUDevicePath) + require.NotEqual(t, vgpuInstanceUUID("source"), vgpuInstanceUUID(fork.InstanceID)) +} + +func TestVGPUArgumentsAreOptIn(t *testing.T) { + cfg := hypervisor.VMConfig{VGPUSnapshot: true, InstanceID: "child", VGPUDevicePath: "/sys/bus/pci/devices/0000:01:00.1", MemoryBytes: 6 << 30, VCPUs: 4} + args := strings.Join(buildArgs(cfg, vgpuMachineType), " ") + require.Contains(t, args, "pc-q35-8.2,accel=kvm,smbios-entry-point-type=32") + require.Contains(t, args, "-mem-prealloc") + require.Contains(t, args, "vmgenid,guid=auto") + require.Contains(t, args, "enable-migration=on") + cfg.VGPUSnapshot = false + args = strings.Join(buildArgs(cfg, MachineTypeQ35), " ") + require.NotContains(t, args, "enable-migration") + require.NotContains(t, args, "-mem-prealloc") + require.NotContains(t, args, "vmgenid") +} diff --git a/lib/instances/create.go b/lib/instances/create.go index 9477b2202..6ff0bc91f 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -272,6 +272,10 @@ func (m *manager) createInstance( return nil, err } + if err := validateVGPUSnapshotRequest(req, hvType, hvVersion); err != nil { + return nil, err + } + // 10. Validate, resolve, and auto-bind devices (GPU passthrough) // Track devices we've marked as attached for cleanup on error. // The cleanup closure captures this slice by reference, so it will see @@ -830,7 +834,11 @@ func (m *manager) startAndBootVM( // Version-locked snapshot backends must refresh metadata on every cold // start so host upgrades do not leave the reported version stale. if refreshHostVersion && requiresHostSnapshotVersion(stored.HypervisorType) { - resolvedVersion, err := starter.ResolveVersion(m.paths, "") + requested := "" + if usesVGPUSnapshot(stored) { + requested = stored.HypervisorVersion + } + resolvedVersion, err := starter.ResolveVersion(m.paths, requested) if err != nil { return fmt.Errorf("resolve version for hypervisor %s start: %w", stored.HypervisorType, err) } @@ -1026,6 +1034,8 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima VsockSocket: inst.VsockSocket, PCIDevices: pciDevices, VGPUDevicePath: storedVGPUDevicePath(&inst.StoredMetadata), + VGPUSnapshot: usesVGPUSnapshot(&inst.StoredMetadata), + InstanceID: inst.Id, KernelPath: kernelPath, InitrdPath: initrdPath, KernelArgs: m.kernelArgs(inst.HypervisorType), diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 94e2a2132..e13a4ba68 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -311,11 +311,9 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin forkMeta.Phases.Record(phasetracking.PhaseStopped, now) } - // Keep the original CID for snapshot-based forks. Rewriting CID in restored - // memory snapshots is not reliable across hypervisors. Concurrent standby - // fork prepare is currently enabled only for Firecracker, whose host vsock - // dialer routes through the per-VM UDS path rather than this metadata CID. - if source.State == StateStandby { + // Preserve existing backends' CID behavior. Managed QEMU GPU forks use a + // transport reset and the guest agent's CID-independent listening socket. + if source.State == StateStandby && !usesVGPUSnapshot(&forkMeta) { forkMeta.VsockCID = stored.VsockCID } else { forkMeta.VsockCID = generateVsockCID(forkID) @@ -342,6 +340,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin SnapshotConfigPath: snapshotConfigPath, SourceDataDir: stored.DataDir, TargetDataDir: forkMeta.DataDir, + InstanceID: forkMeta.Id, VsockCID: forkMeta.VsockCID, VsockSocket: forkMeta.VsockSocket, SerialLogPath: m.paths.InstanceAppLog(forkID), diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 4926894cf..13cfd6cc0 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -384,7 +384,7 @@ func TestLifecycleNoopStandbyRejectsVendorVFIOVGPU(t *testing.T) { _, err = m.StandbyInstance(context.Background(), id, StandbyInstanceRequest{}) require.ErrorIs(t, err, ErrInvalidState) - assert.ErrorContains(t, err, "standby is not supported for instances with vGPU attached") + assert.ErrorContains(t, err, "vGPU snapshots require the opt-in managed QEMU runtime") } func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) { diff --git a/lib/instances/manager.go b/lib/instances/manager.go index ed85f22f4..88fbd9d6d 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -90,6 +90,7 @@ type ManagerConfig struct { MacOSGuestSubnet netip.Prefix // pool for macOS guest networks; zero uses DefaultMacOSGuestSubnet LifecycleEventBufferSize int MaxConcurrentRestoresByHypervisor map[hypervisor.Type]int + MaxConcurrentVGPURestores int } const defaultManagerMaxConcurrentRestores = 4 @@ -97,12 +98,16 @@ const defaultManagerMaxConcurrentRestores = 4 type restoreSlotPoolKey struct { hypervisorType hypervisor.Type limit int + gpuOnly bool } var restoreSlotPools sync.Map // map[restoreSlotPoolKey]chan struct{} // Normalize applies defaults to manager config values. func (c ManagerConfig) Normalize() ManagerConfig { + if c.MaxConcurrentVGPURestores <= 0 { + c.MaxConcurrentVGPURestores = 2 + } if c.LifecycleEventBufferSize <= 0 { c.LifecycleEventBufferSize = defaultLifecycleEventBufferSize } @@ -234,6 +239,7 @@ type manager struct { defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request guestMemoryPolicy guestmemory.Policy restoreSlotsByHypervisor map[hypervisor.Type]chan struct{} + vgpuRestoreSlots chan struct{} } // platformStarters is populated by platform-specific init functions. @@ -287,6 +293,7 @@ func NewManagerWithConfig(p *paths.Paths, imageManager images.Manager, systemMan tracer: tracer, guestMemoryPolicy: policy, restoreSlotsByHypervisor: sharedRestoreSlotsByHypervisor(managerConfig.MaxConcurrentRestoresByHypervisor), + vgpuRestoreSlots: sharedVGPURestoreSlots(managerConfig.MaxConcurrentVGPURestores), snapshotDefaults: snapshotDefaults, compressionJobs: make(map[string]*compressionJob), nativeCodecPaths: make(map[string]string), diff --git a/lib/instances/restore.go b/lib/instances/restore.go index ef85b5b14..8b9eca507 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -53,6 +53,10 @@ func (m *manager) restoreInstance( inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + var gpuRollback StoredMetadata + if usesVGPUSnapshot(stored) { + gpuRollback = cloneStoredMetadataWithoutPendingStandbyCompression(*stored) + } ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State, "has_snapshot", inst.HasSnapshot) @@ -282,7 +286,7 @@ func (m *manager) restoreInstance( attribute.String("hypervisor", string(stored.HypervisorType)), attribute.String("operation", "restore_capacity_wait"), ) - releaseRestoreSlot, err := m.acquireRestoreSlot(restoreSlotCtx, stored.HypervisorType) + releaseRestoreSlot, err := m.acquireInstanceRestoreSlot(restoreSlotCtx, stored) restoreSlotSpanEnd(err) if err != nil { releaseNetwork() @@ -297,6 +301,17 @@ func (m *manager) restoreInstance( } defer releaseRestoreSlotOnce() + cleanupGPU, err := m.prepareVGPUSnapshotRestore(ctx, meta, gpuRollback) + if err != nil { + releaseNetwork() + return nil, err + } + defer func() { + if retErr != nil { + cleanupGPU() + } + }() + // 5. Transition: Standby → Paused (start hypervisor + restore) restoreCtx, restoreSpanEnd := m.startLifecycleStep(ctx, "restore_from_snapshot", attribute.String("instance_id", id), @@ -313,6 +328,13 @@ func (m *manager) restoreInstance( releaseNetwork() return nil, err } + if usesVGPUSnapshot(stored) { + if err := m.saveMetadata(meta); err != nil { + _ = m.shutdownFailedRestore(ctx, stored, hv) + releaseNetwork() + return nil, fmt.Errorf("record restored vGPU process: %w", err) + } + } // A macOS guest allocates no host network here, so there is none to release. if stored.MacOS != nil { if err := m.invalidateMacOSCheckpoint(id); err != nil { @@ -332,7 +354,7 @@ func (m *manager) restoreInstance( resumeSpanEnd(err) log.ErrorContext(ctx, "failed to resume VM", "instance_id", id, "error", err) // Cleanup on failure - hv.Shutdown(ctx) + _ = m.shutdownFailedRestore(ctx, stored, hv) releaseNetwork() return nil, fmt.Errorf("resume vm failed: %w", err) } @@ -359,7 +381,7 @@ func (m *manager) restoreInstance( if err := reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet); err != nil { reconfigureSpanEnd(err) log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", err) - _ = hv.Shutdown(ctx) + _ = m.shutdownFailedRestore(ctx, stored, hv) m.rollbackAdmissionAllocationActive(stored) releaseNetwork() return nil, fmt.Errorf("configure guest network after restore: %w", err) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index dd6f6aead..7ca6fed72 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -518,7 +518,7 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS forkMeta.ExitMessage = "" forkMeta.RestartStatus = restartpolicy.Status{} clearStoredVGPUDevice(&forkMeta) - if rec.Snapshot.Kind == SnapshotKindStandby { + if rec.Snapshot.Kind == SnapshotKindStandby && !usesVGPUSnapshot(&forkMeta) { forkMeta.VsockCID = rec.StoredMetadata.VsockCID } else { forkMeta.VsockCID = generateVsockCID(forkID) @@ -537,6 +537,7 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS SnapshotConfigPath: m.paths.InstanceSnapshotConfig(forkID), SourceDataDir: rec.StoredMetadata.DataDir, TargetDataDir: forkMeta.DataDir, + InstanceID: forkMeta.Id, VsockCID: forkMeta.VsockCID, VsockSocket: forkMeta.VsockSocket, SerialLogPath: m.paths.InstanceAppLog(forkID), diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 5e07ba80e..40ea60931 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -65,10 +65,10 @@ func (m *manager) standbyInstance( return nil, fmt.Errorf("%w: cannot standby from state %s", ErrInvalidState, inst.State) } - // 2b. Block standby for vGPU instances (driver limitation - NVIDIA vGPU doesn't support snapshots) if inst.GPUMdevUUID != "" || inst.GPUProfile != "" { - log.ErrorContext(ctx, "standby not supported for vGPU instances", "instance_id", id, "gpu_profile", inst.GPUProfile) - return nil, fmt.Errorf("%w: standby is not supported for instances with vGPU attached (driver limitation)", ErrInvalidState) + if err := validateVGPUSnapshotCapture(stored); err != nil { + return nil, err + } } // Resolve/validate compression policy early so invalid request/config @@ -124,24 +124,26 @@ func (m *manager) standbyInstance( return nil, err } - // 6. Transition: Running → Paused - log.DebugContext(ctx, "pausing VM", "instance_id", id) - pauseCtx, pauseSpanEnd := m.startLifecycleStep(ctx, "pause_vm", - attribute.String("instance_id", id), - attribute.String("hypervisor", string(stored.HypervisorType)), - attribute.String("operation", "pause_vm"), - ) - if err := hv.Pause(pauseCtx); err != nil { - pauseSpanEnd(err) - log.ErrorContext(ctx, "failed to pause VM", "instance_id", id, "error", err) - if stored.MacOS != nil { - if resumeErr := resumeSnapshotAfterError(ctx, stored, hv); resumeErr != nil { - log.ErrorContext(ctx, "failed to resume VM after pause error", "instance_id", id, "error", resumeErr) + // VFIO precopy needs the guest driver running until migration stops it. + if !usesVGPUSnapshot(stored) { + log.DebugContext(ctx, "pausing VM", "instance_id", id) + pauseCtx, pauseSpanEnd := m.startLifecycleStep(ctx, "pause_vm", + attribute.String("instance_id", id), + attribute.String("hypervisor", string(stored.HypervisorType)), + attribute.String("operation", "pause_vm"), + ) + if err := hv.Pause(pauseCtx); err != nil { + pauseSpanEnd(err) + log.ErrorContext(ctx, "failed to pause VM", "instance_id", id, "error", err) + if stored.MacOS != nil { + if resumeErr := resumeSnapshotAfterError(ctx, stored, hv); resumeErr != nil { + log.ErrorContext(ctx, "failed to resume VM after pause error", "instance_id", id, "error", resumeErr) + } } + return nil, fmt.Errorf("pause vm failed: %w", err) } - return nil, fmt.Errorf("pause vm failed: %w", err) + pauseSpanEnd(nil) } - pauseSpanEnd(nil) // 7. Create snapshot snapshotDir := m.paths.InstanceSnapshotLatest(id) @@ -260,6 +262,11 @@ func (m *manager) standbyInstance( } } stored.Phases.Record(phasetracking.PhaseStandby, now) + if usesVGPUSnapshot(stored) { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { + return &Instance{StoredMetadata: *stored}, fmt.Errorf("release vGPU after snapshot: %w", err) + } + } meta = &metadata{StoredMetadata: *stored} if err := m.saveMetadata(meta); err != nil { diff --git a/lib/instances/vgpu_restore_slots_test.go b/lib/instances/vgpu_restore_slots_test.go new file mode 100644 index 000000000..fec631dd3 --- /dev/null +++ b/lib/instances/vgpu_restore_slots_test.go @@ -0,0 +1,33 @@ +package instances + +import ( + "context" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/qemu" + "github.com/stretchr/testify/require" +) + +func TestVGPURestoreSlotsDoNotThrottleCPUQEMU(t *testing.T) { + m := &manager{vgpuRestoreSlots: make(chan struct{}, 1)} + gpu := &StoredMetadata{HypervisorType: hypervisor.TypeQEMU, HypervisorVersion: qemu.VGPUSnapshotVersion, GPUProfile: "NVIDIA L40S-3Q"} + release, err := m.acquireInstanceRestoreSlot(t.Context(), gpu) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond) + defer cancel() + cpuRelease, err := m.acquireInstanceRestoreSlot(ctx, &StoredMetadata{HypervisorType: hypervisor.TypeQEMU}) + require.NoError(t, err) + cpuRelease() + _, err = m.acquireInstanceRestoreSlot(ctx, gpu) + require.ErrorIs(t, err, context.DeadlineExceeded) + release() + release, err = m.acquireInstanceRestoreSlot(t.Context(), gpu) + require.NoError(t, err) + release() +} + +func TestVGPURestoreSlotsDoNotReuseTheGenericQEMUPool(t *testing.T) { + require.NotEqual(t, sharedRestoreSlots(hypervisor.TypeQEMU, 2), sharedVGPURestoreSlots(2)) +} diff --git a/lib/instances/vgpu_snapshot.go b/lib/instances/vgpu_snapshot.go new file mode 100644 index 000000000..de4c8583e --- /dev/null +++ b/lib/instances/vgpu_snapshot.go @@ -0,0 +1,117 @@ +package instances + +import ( + "context" + "fmt" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/qemu" +) + +func sharedVGPURestoreSlots(limit int) chan struct{} { + key := restoreSlotPoolKey{hypervisorType: hypervisor.TypeQEMU, limit: limit, gpuOnly: true} + slots, _ := restoreSlotPools.LoadOrStore(key, make(chan struct{}, limit)) + return slots.(chan struct{}) +} + +func (m *manager) acquireInstanceRestoreSlot(ctx context.Context, stored *StoredMetadata) (func(), error) { + release, err := m.acquireRestoreSlot(ctx, stored.HypervisorType) + if err != nil { + return nil, err + } + if !usesVGPUSnapshot(stored) || m.vgpuRestoreSlots == nil { + return release, nil + } + select { + case m.vgpuRestoreSlots <- struct{}{}: + return func() { <-m.vgpuRestoreSlots; release() }, nil + case <-ctx.Done(): + release() + return nil, ctx.Err() + } +} + +func (m *manager) shutdownFailedRestore(ctx context.Context, stored *StoredMetadata, hv hypervisor.Hypervisor) error { + if !usesVGPUSnapshot(stored) { + return hv.Shutdown(ctx) + } + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return m.shutdownHypervisor(cleanupCtx, &Instance{StoredMetadata: *stored}) +} + +func usesVGPUSnapshot(stored *StoredMetadata) bool { + return stored.HypervisorType == hypervisor.TypeQEMU && stored.HypervisorVersion == qemu.VGPUSnapshotVersion && stored.GPUProfile != "" +} + +func validateVGPUSnapshotRequest(req CreateInstanceRequest, hvType hypervisor.Type, version string) error { + if version != qemu.VGPUSnapshotVersion { + return nil + } + if hvType != hypervisor.TypeQEMU || req.GPU == nil || req.GPU.Profile != "NVIDIA L40S-3Q" || req.SkipGuestAgent || len(req.Devices) != 0 { + return fmt.Errorf("%w: the managed vGPU snapshot runtime requires an L40S-3Q guest with its guest agent and no additional PCI devices", ErrInvalidRequest) + } + return nil +} + +func validateVGPUSnapshotCapture(stored *StoredMetadata) error { + if !usesVGPUSnapshot(stored) || stored.GPUFramework != devices.VGPUFrameworkVendorVFIO || stored.GPUMdevUUID != "" || stored.GPUProfile != "NVIDIA L40S-3Q" || stored.SkipGuestAgent { + return fmt.Errorf("%w: vGPU snapshots require the opt-in managed QEMU runtime and vendor VFIO L40S-3Q", ErrInvalidState) + } + return nil +} + +// prepareVGPUSnapshotRestore persists a fresh claim before QEMU can open it. +// The returned rollback must only release a VF after the restoring VMM has exited. +func (m *manager) prepareVGPUSnapshotRestore(ctx context.Context, meta *metadata, original StoredMetadata) (func(), error) { + stored := &meta.StoredMetadata + if !usesVGPUSnapshot(stored) { + return func() {}, nil + } + if storedVGPUDevicePath(stored) != "" { + if m.vgpuCleanupGuard(ctx, stored) == "" { + return nil, fmt.Errorf("cannot replace an uncertain live vGPU claim") + } + if err := m.releaseStoredVGPUPersisted(ctx, meta); err != nil { + return nil, fmt.Errorf("release previous vGPU claim: %w", err) + } + } + rollback := metadata{StoredMetadata: cloneStoredMetadataWithoutPendingStandbyCompression(original)} + clearStoredVGPUDevice(&rollback.StoredMetadata) + device, err := m.claimVGPU(ctx, meta, stored.GPUProfile) + if err != nil { + return nil, fmt.Errorf("claim vGPU for snapshot restore: %w", err) + } + setStoredVGPUDevice(stored, device) + cleanup := func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + m.cleanupStartVGPU(cleanupCtx, stored, rollback) + } + if err := validateVGPUSnapshotCapture(stored); err != nil { + cleanup() + return nil, err + } + if err := m.configureClaimedVGPU(ctx, device); err != nil { + cleanup() + return nil, err + } + starter, err := m.getVMStarter(stored.HypervisorType) + if err != nil { + cleanup() + return nil, err + } + _, err = starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: m.paths.InstanceSnapshotConfig(stored.Id), + InstanceID: stored.Id, + VGPUDevicePath: storedVGPUDevicePath(stored), + VsockCID: stored.VsockCID, + }) + if err != nil { + cleanup() + return nil, fmt.Errorf("bind snapshot to fresh vGPU: %w", err) + } + return cleanup, nil +} diff --git a/lib/instances/vgpu_snapshot_linux_test.go b/lib/instances/vgpu_snapshot_linux_test.go new file mode 100644 index 000000000..a2da48dfe --- /dev/null +++ b/lib/instances/vgpu_snapshot_linux_test.go @@ -0,0 +1,122 @@ +//go:build linux + +package instances + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/qemu" + "github.com/stretchr/testify/require" +) + +func gpuSnapshotManager(t *testing.T) (*manager, *metadata) { + t.Helper() + vfs := []devices.VirtualFunction{{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}} + m := newVGPUAllocationManager(t, vfs) + m.vmStarters = map[hypervisor.Type]hypervisor.VMStarter{hypervisor.TypeQEMU: qemu.NewStarter()} + m.vendorVFIOProfiles = func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + out := make(map[string][]devices.VGPUProfileType) + for _, vf := range vfs { + out[vf.PCIAddress] = []devices.VGPUProfileType{{Name: "NVIDIA L40S-3Q", TypeName: "profile-3q", FramebufferMB: 3072}} + } + return out, nil + } + m.pickVFIndex = pickFirst + meta := saveTestVGPUInstance(t, m, "child") + meta.GPUProfile = "NVIDIA L40S-3Q" + meta.HypervisorType = hypervisor.TypeQEMU + meta.HypervisorVersion = qemu.VGPUSnapshotVersion + meta.VsockCID = 71 + require.NoError(t, m.saveMetadata(meta)) + return m, meta +} + +func writeGPUSnapshotConfig(t *testing.T, m *manager, id string) { + t.Helper() + dir := m.paths.InstanceSnapshotLatest(id) + require.NoError(t, os.MkdirAll(dir, 0o755)) + data, err := json.Marshal(map[string]any{"VGPUSnapshot": true, "InstanceID": "source", "VGPUDevicePath": testVFDevicePath, "VsockCID": 70, "MachineType": "pc-q35-8.2", "QEMUVersion": qemu.VGPUSnapshotVersion}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "qemu-config.json"), data, 0o600)) +} + +func TestVGPUSnapshotRestoreClaimsANewVFAndRollsBack(t *testing.T) { + m, meta := gpuSnapshotManager(t) + source := saveTestVGPUInstance(t, m, "source") + source.GPUFramework = devices.VGPUFrameworkVendorVFIO + source.GPUDevicePath = testVFDevicePath + require.NoError(t, m.saveMetadata(source)) + writeGPUSnapshotConfig(t, m, meta.Id) + reset := "" + m.destroyVGPU = func(_ context.Context, a devices.VGPUAssignment) error { reset = a.DevicePath; return nil } + original := meta.StoredMetadata + meta.IP = "10.0.0.25" + cleanup, err := m.prepareVGPUSnapshotRestore(context.Background(), meta, original) + require.NoError(t, err) + require.Equal(t, "/sys/bus/pci/devices/0000:82:00.5", meta.GPUDevicePath) + persisted, err := m.loadMetadata(meta.Id) + require.NoError(t, err) + require.Equal(t, meta.GPUDevicePath, persisted.GPUDevicePath) + data, err := os.ReadFile(filepath.Join(m.paths.InstanceSnapshotLatest(meta.Id), "qemu-config.json")) + require.NoError(t, err) + var cfg hypervisor.VMConfig + require.NoError(t, json.Unmarshal(data, &cfg)) + require.Equal(t, meta.GPUDevicePath, cfg.VGPUDevicePath) + require.Equal(t, meta.Id, cfg.InstanceID) + require.Equal(t, meta.VsockCID, cfg.VsockCID) + cleanup() + require.Equal(t, meta.GPUDevicePath, reset) + persisted, err = m.loadMetadata(meta.Id) + require.NoError(t, err) + require.Empty(t, persisted.GPUDevicePath) + require.Empty(t, persisted.IP, "failed restore must not retain the released network allocation") + sourceAfter, err := m.loadMetadata(source.Id) + require.NoError(t, err) + require.Equal(t, testVFDevicePath, sourceAfter.GPUDevicePath) +} + +func TestVGPUSnapshotRestoreFailureReleasesClaim(t *testing.T) { + m, meta := gpuSnapshotManager(t) + reset := false + m.destroyVGPU = func(context.Context, devices.VGPUAssignment) error { reset = true; return nil } + _, err := m.prepareVGPUSnapshotRestore(context.Background(), meta, meta.StoredMetadata) + require.ErrorContains(t, err, "bind snapshot") + require.True(t, reset) + persisted, err := m.loadMetadata(meta.Id) + require.NoError(t, err) + require.Empty(t, persisted.GPUDevicePath) +} + +func TestVGPUSnapshotRollbackPreservesLiveClaim(t *testing.T) { + m, meta := gpuSnapshotManager(t) + writeGPUSnapshotConfig(t, m, meta.Id) + m.destroyVGPU = func(context.Context, devices.VGPUAssignment) error { t.Fatal("live VF must not be reset"); return nil } + cleanup, err := m.prepareVGPUSnapshotRestore(context.Background(), meta, meta.StoredMetadata) + require.NoError(t, err) + meta.HypervisorProcessIdentity.Set(os.Getpid()) + require.NoError(t, m.saveMetadata(meta)) + cleanup() + persisted, err := m.loadMetadata(meta.Id) + require.NoError(t, err) + require.Equal(t, meta.GPUDevicePath, persisted.GPUDevicePath) +} + +func TestVGPUSnapshotRequiresExplicitSupportedRuntime(t *testing.T) { + _, meta := gpuSnapshotManager(t) + require.Error(t, validateVGPUSnapshotCapture(&meta.StoredMetadata)) + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + require.NoError(t, validateVGPUSnapshotCapture(&meta.StoredMetadata)) + meta.HypervisorVersion = "8.2.0" + require.False(t, usesVGPUSnapshot(&meta.StoredMetadata)) + require.Error(t, validateVGPUSnapshotCapture(&meta.StoredMetadata)) + req := CreateInstanceRequest{GPU: &GPUConfig{Profile: "NVIDIA L40S-3Q"}} + require.NoError(t, validateVGPUSnapshotRequest(req, hypervisor.TypeQEMU, qemu.VGPUSnapshotVersion)) + req.SkipGuestAgent = true + require.ErrorIs(t, validateVGPUSnapshotRequest(req, hypervisor.TypeQEMU, qemu.VGPUSnapshotVersion), ErrInvalidRequest) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 4c604c755..63ae42040 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -720,6 +720,9 @@ type CreateInstanceRequest struct { // Hypervisor Hypervisor backend to use for this instance. qemu uses the architecture-native standard board; qemu-microvm uses QEMU's minimal Linux amd64 board and does not support PCI devices, hotplug memory, or more than eight virtio-mmio devices. Defaults to server configuration. Hypervisor *CreateInstanceRequestHypervisor `json:"hypervisor,omitempty"` + // HypervisorVersion Optional installed runtime version. The backend validates availability and snapshot compatibility. Omit to use the backend default. + HypervisorVersion *string `json:"hypervisor_version,omitempty"` + // Image OCI image reference Image string `json:"image"` @@ -1275,6 +1278,9 @@ type Instance struct { // Hypervisor Hypervisor backend running this instance Hypervisor *InstanceHypervisor `json:"hypervisor,omitempty"` + // HypervisorVersion Runtime version selected for this instance. + HypervisorVersion *string `json:"hypervisor_version,omitempty"` + // Id Auto-generated unique identifier (CUID2 format) Id string `json:"id"` @@ -19327,396 +19333,397 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st var swaggerSpec = []string{ "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV18Uyr1Hcd2HO1EsbZle749oQ8FdoMkRt1AB0DTZlKu", - "On/NA0ydJ5wnObUWgL4RTbZkS7InrqmayGw0GpeFhXX9rT96kUwzKZgwunf8R09HC5ZS/POxMTRavJFJ", - "nrKX7LecaQM/Z0pmTBnOsFEqc2EmGTUL+FfMdKR4ZrgUvePeGTUL8m7BFCNL7IXohcyTmEwZwfdY3Ov3", - "2HuaZgnrHff2UmH2Ympor98zqwx+0kZxMe996PcUo7EUycp+ZkbzxPSOZzTRrN/47Cl0Tagm8MoA3yn6", - "m0qZMCp6H7DH33KuWNw7/rU6jbdFYzn9O4sMfPxxbuS5oSKers5kwqPV+mR/5iJ/j18jNDcypYZHRNt3", - "SIYvkSnVLCZSEBoZvmSEi6nMRUxePTkjkRSCRdCZHgs51UwtWUxmSqbELBhZSG2wjVE0uiSGThM2HIte", - "v7EfTMCTePsq/XXBzIKpwGC5Jq4XMpOKmAXXhAt4GrFhdcOMytn6yvZ7PE7YxPCUydysL9SP8h1JpJjj", - "tHy/JM21IQu6ZOR3piT5LacJn624mLcv0pTNpGLkx1XGUipIltCIacIN4cJIPxu7RiWN3UtDxMXnQio2", - "iZk2XFDof5JJZfT66F/gHzQhlbY4NGxPzIIaT+VCGnLJWFafKH1HL+vL+OvBQf/RaDR62+9xw1J7rOh7", - "nuZp7/j+vXuH9/q9lAv77/1i9FwYNmeq96H4hSpFV5XpaJmriE0iHqtNM4kSzoQhT06evrzmBHr7oyH+", - "b+9hr9/bf3Qw3L//EP+9f79XndbawtdH/mHz0Ts31OR6nQfZ0zRxhDKpEMn6rH/J0ylTRM5IlCvFhElW", - "BI8UizsQXW3ao9BWRFLM+DxXLF7/tj9yteVcUE2osExj0OAXZWedzl0ETCyW78REsZRyAWu8NoiX/hGB", - "E0rcIYIhRVIYJZMEmIIxLM2M9qeoD2xcEJplCY+Q9dQO1VE60r1+T+RJQqdrIyx3myV8zrFBp6XhurJJ", - "/l1iJGHCMFWc8C5LU2OLbR8ulzu4GyVf7PTJhUziSS4MT9a/+hp+dmta/RLXZMGSmMjZrE/4jFACvcDP", - "lsbry34wOrg/GB0NRvdf7T84Hj06Ht37W6/fm0mVUtM77sXUsAHscpe9QaatuYjCuyOaJJLChaRYZHen", - "uLBqGzhlkUwZga43jvzo40feet08KTcVGhLXsLxZA1s9bFwZ3ag7odoUTAj3i5vVhAbG9IqnTBuaZsCH", - "YAyVxWzjQr7D5j74ld+4wPsftcCCvTcTt0LB+YTog73PWAQ3ovTcpBAwoD9H3gXLuhUaV4xqKayYBJfp", - "r71c6DyDq5vFkyyhBvrt9XuWDCYp1xpeLX6IubZ8pN/zRD4R0kxULoRtKJh5J9VltaXrZcKzXr+3oHqy", - "nGd5r7/p2qoTNX6CJTTT2J/bcTVhSknVs6LxajKTym9S7211CTd0tbZCurhiAyvU6/dqC1Cwcz8XP+5i", - "V4OD6/fcXxNltQrszk5mfeDVvtaHWwxtM2O3t4gVov02E/eyrnOAmNO5kNrwSHdi8yg8wPamMg6wzqdF", - "d4THTBg+40w5uZoRBfdDyojvhEAnhAuS68Y5KET/CVsyYfRkeTQxUba+KA3Fprp5FdmkvBErt3Kx/cVJ", - "2UKk9bkHFacl5Xgmn7Ilj9i67Oa2ZhIrvmQqwL4LAcCyQtuO7MBZBxYipGC7tZUSSx5z2oUdxDimCQ9Q", - "z9mTE2Ifk5OnZGfB3tc/cvBg+rDX3qWgaYAWfsxTKgZwIGBYvn9sW+3756NQz1ymaT6ZK5ln6z2fvDg9", - "fU3wIREo4VZ7fHgQklSziE9oHCumdXj+/mF1bKPRaHRMD45Ho+EoNMolE7FUrUtqH4eXdH8Usw1ddlpS", - "1//akv7y5uTpyWPyRKpMKtTZth6c6vJU51Ulm/quhOj/+5wn8TrVT+Fn1rJOTGmuDROGfG+bEcWsKkfe", - "LaRmJKLRgpGpVVpQPcf+upB88WF/e4VG8JTrTCKfJ29OS2kOVUP2nkW5qX33WxJzbbiInFBVmVyHEYH5", - "KWFw/4YkC1wB4tpwKYjxotO1hYBIMbrlc9Ci08fWz39uyWuS6rbefRPCBUl5knDNIiliXf0GF+b+Uftk", - "KqfYXptrn3oGP5OUaU3njOygWQrVDcvhCddkRnnC4t1uEnbbZP4up5V7rXbmkAwGdBrtHxwGGVpK52wS", - "87mzKzZpEH4H+Rj6MYSnrRNBCaPbPPCTis3Wv/cD3if4EcVmTDERffTnMiWXTFCnUv0Hfrf3f+2VBtc9", - "Z23dw8U8K5t/6Pd+y1nOJpnU3I5wjU24J0BGuNQE3wiPGR/Fu50oShuqNp8PbPEJTmIpbG5dG2f6AXmL", - "zre+8graNBk6sqNCwKlwgVa+/QwkrYDIIoVhIrA6P8s5SbiwahATxu0FCnurjH2XyPlu75OtQ7H864cf", - "xn0N5mV/aOkNnvULrSCR8+pqLhhVZspqi9lyr7qOytG1Lv9Z7fg0LlCq2WQzBznjQrAYbe7uYNuWIFsH", - "dR88RZfcTJZwA4fOHA7rJ26Ia9HaVSKjyxlP2GRB9cKOmMYxtwbXs9pMAiJkzZlB0UjgO0TRBpXq8x8f", - "H9y7T9wHAmvorL/QYH0mlbehe9uWGKqmNEmCtNFOble/o9cpJEwBpcG37e4pKNATpuV0PbebTnnP9cL+", - "hbwbRoV3X6/fi4C8Evh7jXD7vfcD6HWwpApXHbqvDOu//Zcqv31ffrTy61nx/cqPL91QKj/94EdV+e1J", - "OUC/KkytnwenpuGkw1Kl5MIMuMAtAAHcWZ6mdvMKLctZMogUFQmPqS7sfJNg9WRNpCI7L394cnh4+Gi3", - "u3DF9eVE89/ZZD7dKDe7QTtBGV4j8Brhgsz5nE5XhukheUKFkAackdGCijmLCZ0ZeMuNtaaA3xttc8K0", - "CklMtYlI5vcj+ujh+/fUPLrP3+lHv6dTNf/7Ie21WRdz3bK+aIGjbjehKVFUBDbxehd1St+D5m+pJOT8", - "PLXuKlI0cnugm0MYklKhn/H3sOaG7NdWOri4Ye2vcGEJKQa54L/luNtZQlfrOqBhNB0gRYSW1/INe4D0", - "9uOj4fzYd/xM31Fu4OQUPlM35T6RSQz3zowrbXrdvWFXEgyY+lQiUu2UhSWmdYpoLmDrpV6Ms/WsJHzG", - "olWUsHXm7tl3pnJnbo0Z3Dn4Z2FA7MrGmQpwYv/jWfGJ2s9Py+/Vfn9mP/6h33tCMzrlCfecuc6nnU9+", - "4gx/23ap2tlT++pL9+aHfm/GqMkVC6zlOcYHEGsnHiRsyRLiWgPxHpPHZydE52qGLnOkVtsWzPZSMz0W", - "is2pihOmkdTfLXi0KMyVYHrQhIoyLIA8Tt7RlSaZYpoJczwW48JCrse9Phlb5cv9banE/m1DQ3wjMVdM", - "u3+ArWHc60NfiZzrcW9IfpTaDEBddrLUMRk7e4we98iO/ZNkVGuzUDKfL8YipYLOWcqE6RMM0kAzMHCm", - "ZLVLqIjJuKekZsbQAUvzBHk/dPY4yxI2Fuc84ZEUJKXRi3P37jtuFuSlfalyeeKM4bLuk8yZnjVN2VhQ", - "awaFnVyRTMkpIwnNRbRgmjAxkypiu0NyxtTAL7HfW0IVg82wRlySCzgijJaboX99S5gwykU6dGcuMJOr", - "0B8sfaFE66u8eWLf+FB4R67y8i/uFWBVbsZBHUWtSmt6hZ7tjViY+v11VPME9McCVxR3lUOrd4LUtmyW", - "0DmSSuUQWWLUYxHJNEVFz32FaLZkiibFDsEdJ9GeAHeDu5HGIkpkHg8Wq4ypJddS9eGGYBEGI6g++Y2l", - "eR8/Cn8NUh4puUyJFGORABnv0TS+f7Q7JI9JwjUQh58/t8TtSAw5AdoDOIx23KPeJj/u2YlxTeDiH5Jn", - "wihuaY5oS3BTe5M2iKvr5lWY1drdhvtzlc7O7RtrHg77syPo/hqLrdBNSYAFGVe4aOjO2sCA12Vwv66b", - "nVFufMVuRVS4napR5zHhhtAsY1RpwsVYlNQkYqRS91KmGC6H5sZtXcoM2UmpQfYCVtuxQA7xjSa13d8d", - "EpQbbXxYyZHgFaJYsrIDYmPhx/yOJwkaDkmOEQzI4exBQ8tYJYICDxMVBe9LSm+Xc345v2htArUjYr/9", - "389OXxO90oalZMoFVatdwnXJay1prjvmwiKj2z3U3FFUq886lhj1BO8SKeoCZPO8bnUhOBG0pIttFPaj", - "DIVYUhUtQmF02pAnZ68JPOaGRUDEtfFSld4PupKkbukOvkpRiLXrXesOmc7WOUs4UjjibZM9Ke6RsIhU", - "uOPXvV3wZsWPyxIWAa+y8TRWf2PEOZeJTJGlC7Y+m73WJfKd621f12RH6j2Y8G7lZoEzrXIxJC8EQTGC", - "1KSIscCmXERJHjNNKvyclOZbL1/UwqAKsh+Lf/2//5+VJWKy5LSQNsgbrgzEkPxutdiZoikDple/0qIF", - "iy7HoimGEOhVS0L9pVIsM9fVC0Xx+cIQId8Nx+LEjqkY8I6WM/OOKpZnoGaSwcCNeuDkrF0bxIUdcdMI", - "0fy1sTeVxbliLGHNnVdsaH+dwraR6i+l2FKn1Tk17B0NKMTPgfQGS64xZA1pwrUlJ2dDgo/RlRAt7FPg", - "oiDFkh1LFVzMxwKfOJF4lziJ1pKZc00OyYuUm4L4hSyuF3fTjQUEGE4ZE+hFTJY2ALkk1RVrbgBEc+6D", - "g3e4Hzoacxj7xMjJPA/aXf1dF3PFIkPenA6MHLw5JUbR2YxHQEcZU27U9oJpDjrIzyHGIGlZaf+iD6FI", - "hqBkKB7PQciB5XKiv/3NXk4ZUzCuV4/P4ILD1f+WjHuCGvtOyekH4MnhMYvJL49fwdUavTjfdavmVFPb", - "cQ+unZARvN/T+VQw0zZ++xRjcMNbgaG025ivXaK1LdpG3h8n0cDFbzv4ZoNMUnIwkE5AiC1IcEh+rsmv", - "9oUicAVE0HWJvujBH+JvSTHa71CgISmjQhPqZP3KiAjXY+Glj5rAAZI2cSsKr6pcWC5l78NBhfuiWGLl", - "EZTIiiD9Jfz/YKlldOlCPKyWWXBaQVM83dWlW5OsuOkk5Gw1AVQUlsCS+q/XBE/QEbSgmV5Ig1q1/8ee", - "YtpIxXZBOXfxUvA8o7lm5C8kZalUq7Hw7fv2nCXU2BiG4t0ZMKYe2YkSKVjVjmADCOhYaCOzjMXOMfEt", - "mbnDTYsgQqkKW7VtNBbuE3rPSgCaFLPADSg9MMmq3OTKTHbR1ICzKaYFc85TO+yFNFmSzwdunj2yk/Al", - "c9OGCfLfme0DvChSioGLh4Smpe0EHxFD1ZwZZ3rW2D/SzLg3Fjt4bgn+GxwraS54hLc5tptn+aBq4uiR", - "nednr/fKmKWxqDzGV8CoN+BykPCUA3vbgR/Iyd4LouCGxp+5mNvRY2M7nWKS2N5NcUje0CR35xSpHdZ3", - "LLz0jyMdGJWbxSxPjsnydyeE1fej2MlCKto/7DtLU4IWplZJ5s0p0XTJPEHiW6BSawPHiMZLpgzXzhp7", - "SXYcOQ2cnwvpThPBWEyEHItqX3CQkUxhiE2Rn1hDjCaNRRqLUmxrRJL7q827DAtThBd8ultsNms0LT6G", - "j9darqAinxdKfeMiyXi7VxVskXhQaFQuU0V/ZeRFxgS0imWUp0wYwsVM+qb1KLLR8KAtMq3l8+fVa6X4", - "vF5IZcicG6IYiHFSOG427g1irgwcf53PZvw9apE5WoGsTIM96T5wqHEvF5dCvhPjXn2YD+7Tg1k8Pdy6", - "G37Y/doaBvcBmZ6zSrfm6232oT3Z6DADUZMYCbpzVfN3FP4J3GVlShJcemoAZyXhLK7Qdt9G686ZYMC5", - "YhsDaoiX0FpcPoOgKHvjXqUreWJadtRGzrZu6JY5zBM5xQvPT6QMOh2Sk/ra9QnFBygw11e5yJaoBIaS", - "HTacD+F2ySI+gMjQAT0YjEaDUZPak6OBDXnPqDFMwQD/n1/p4PfHg7+NBo/eln9OhoO3f/mPoELeMVrV", - "+7XdPHf8WeoTP9hqCGtzoNvCW6/tWasOv/3wommhdacjxfAY0GTrGCDi4EmleTutPzlZj3qzKxVLsEMP", - "udxL+FRRtdoTcy7eH4M4p0193Ta33WRgCSTHWMHINyBUE2dh+XVvSRWnwry1YyTjqnEA5DeCtwbc+09x", - "RGQw8P049uUj72sOgCH5RRpCnUcaVLpjbLT+3bmiaUrR2rmkCY/xcDg3m+Yxs8ZZgc/I0kpJiplcCXI0", - "Gvknk8qgQG6Q2k9l3HOGUxUtiDMIKYKmEJshE7EMPmlkTBtpSpWlWD9pcMz+slf8teP/HL79y+7//o9P", - "SupIaBto3JozrsjPGnHe0IjsJPIdUxHVjCQMZqv7JOZzbrQV3mIMjUKD2rckssEeNnRRKsKEN1Fjuzo5", - "p6sBzfjAWV6su/tnJuZm0Tu+fxhc3B33x+Dtf/qfWhZW5UlIX3spc7S84uNq4rUfQyfni1/dPEGnS8rF", - "iX1tf12c/Lgd9hPZtNNWqWvdarhjiqygLQNZz8CHiKM0JEQsmVJwFC1fe3L6FFSYS+YYGmiNZJyPRocR", - "NsA/mfsFhDhQZfC33brEY7sLCTu/9li0kBgGmSTyKkbKfpOpt0UJbvSRhVa75P7rMYXoRLducQx2dw3J", - "VMlLptAlA0vMmSaXbGXNBfOaQZOJJVlS5V2DrzAtAZv4R+gNYqgegxKLyfmQy8qUY4tOZ7NpUIwmJbdM", - "Kbi7UK/UkcwYMFTXDKdGLphYXpCUZlatV8wazVNqmOKgLpaGxZTFHHj0WDA8GCSjcOajSCowscLeog22", - "XIVvNLmwauIFdn/BBZD1hT2YDVPpH70Xr199/+L1L08nL86e/fL4ZPLTs/+Bn+1LveNf/+hZMI0i8Op7", - "RhVT5D/+wPl+sMGzGNMHqe4LqZy26130GK9CMz6UGROUDyOZ9vq9/6z+8+2Htz7c02buLXvHwYF9CIqZ", - "zvYZiMH3CVCaSFVGnlLEBUEW5c0OFXNDw5RvJb8rHQlUUbicTDMdzE1p2i1KOXR/NDr9fs9Fj9zz/9gd", - "Eue6xeEDD5LKicd6AeRT4HKgPy1JZOQyNGZtKfj+UyEGj77WTHJhtjKnsuk6jxoMyqdXYEV7Uy72NGzD", - "ILrauiPdXDtQ+ZlYciUFqucoLk0Tputn5ZcXT59Nnv3ypncMF0GcYx5hr987e/HyVe+4dzgajXohAmXv", - "M66YDgY+Pp5qmeSGEWxUxpgOyWmuMbjTMY9ZjnZQlM80SXMwLCUrwt5HSa6BT6E0YEwSyj9+ODjYf7X/", - "8Hg0Oh6N/tY5YBVIfwvzfn722mao43lnNDGLCbrmtr34I7Z9gk0/9L1tElX7wCoVe+qtlXB63TtkZ1GX", - "ruzkCBLkuHf4/Ht7oPafj3sVQnSpwEUvtuPG6j3/PrQuFWvUuqRXPLMZbiL2VL8ORmPt9bl2ZqKqJ3wg", - "KELsQMuYqphMJVXxt/VYGnwTDPnfaILWCpo4P5EVvfEdK0tKplFX9jE+pbFV9xtLgJaf1JoSqSAMPaVL", - "DixnkKZc+vfqbMkZU2o21GHFwRSw41XChXr9HszM/cdPsNfvLX+v51RvNwe60JhOuuIWJZAmGRdsgxb4", - "mQj7YFNOJI0H+59Y1q/EuzVgaOyD+nYXyqkn8DV0pykV8Tsem8UEMuJhyIEb0j0hRePimnwPM6HJv/7x", - "zzenpUFk//k0c3fm/sG9j7wzG7ckdB3MtSkmkmfhabzOwpN4c/qvf/zTz+RuJ2FFymuJ527/n9kemgA1", - "DrfLRuO3+F0LOazMz5UtvvQtGAahK1cumUroqnKduDH19kfI0xujUty6G917FRPy5ssFevPiWvV6ATY/", - "4yyxMVo29Bb1C+09RNZ3iKcFeNI32mcgMTKV0tgRwNowuoaNEr6VPs4kNXQ3RySFoVww5cZb3Bj2SrEa", - "S3r/aEievc+Y4iAw0YTEVL3jwga9+Dedzk2oAECghAs24Knz4NolsZNEngwdg6PNBxxpG3C0brVHY3VM", - "3P1oX5aqYRC7ScuSYsiYJ1mBpLfp+Ly0rUulP0CSAYr8nurCM9uFDgsy3D84dX8eWIKsEl9xuOxZ46og", - "Pvupb0EMBdXZ4Ee1hbWbMgJfYlQbckSe8+8JF5DKnzByyr8n1N1OXJA3f7P7gLxOd5Kl9CXPbJzHhM6L", - "tNRNyH/nlzxz+jy+4aMM7B0U59AznqHhWFhkH2AbSDsQke99twaSCLQNXIDbFu+zdRkNrAPljWabWzet", - "ykWfTHMDKr80jDjji4MYgLFg4ykjeRlE2lDA3QTXAyNwWS6ZEiyZWAVbd1wZ+xJxL7UuDk51RrVDIlMm", - "z+rr9fSn03Oy83QlaMoj8pPt9VTGOUQDWp6wW1+9/lhkCuFd4CNwFrj7rpwRmZuBnA2MYswPMcXOCj+M", - "A7lYPj977WBSNAQpvWSwsEzELuDVCztFxKv4xlhXeK3b6vcbi94GhePd+x1P9rlrXh7t7hbJfs+YJARu", - "l1i2BklENo8CHFcF9fnUvj6cwOeyxGKw3GBIXmvgBiMIe5EKoxbIVJpFVbvEa0lXNaASQrNsBvpnWMUs", - "tdnG+T5aBP3WUZbXSfeg34ppuLQBEyDO6K6cawmNK4wLAZcCnOugI6MKgs24JJ/1HbPwslW7kh1guWfU", - "1AE5uprCbc8IBrs1OtRZtHFJulu0KzbWNdu2NwD+0TFpYnP/J8IPZKNFv7QBfsS3zm0nzSVyfff9zK6x", - "SifFmjT8AJ9meR7ris20E26rlXmsPKPJzgWYWR0dg2H1ok8u/rP2A/BTL0+BuvCO2NVAHi3gp2r/ei2y", - "eYsd99rRzVRffz8e61bULbLcJ0ZRoW2c/YJmbEh+xIuRGJaCtAg3Fa+GFgr57lsirY7iXx0LGJqux1p6", - "a77mcwGhZxg0Dwwnjq3J31nsfPyNDsEdezvcmsRuR+dAi5CXuWh7cuFN7xd1/rtumF+3WzlL/ZrBAn+3", - "hgo05Zi9NDfweau9OJ+0zI0FMXNTbwh4dfP/tsAgN5bCFHmN/T8v2EUTQ3oZsFjA5OSs9NdUHEdt/hkn", - "/IV9R5dshVvu/UR0zVNUdRGFHTkupN2JMpVX0XZorxKuK/4l6ylyziEMPq4f0aDbZNtWwHp1Xv665h/M", - "wxn4yZYU45R5j0/quRBMzn4PjY6a4eKjJeHYpnVd9K3pg6F9kQgglsTH5je7x7BkDGK8cL8MXW8XRZLg", - "p8AcL7M/qltLKjvrRWnsxotihsX9umwA6NR6+6RseOqixe2p2DvFPSPzAJcdRV6XNZM6xevj7EDPKp1t", - "CAjr2sV6lqxd38qYHT1ZYGefRGL3A/1fNcTjJvB/3FDDLXJM/ZMXNEkuyI5rtEsU+zuiwrq9ggi7YvsB", - "5daRQBFu9ua0PxaWC1wsjMkm8H96Aqf4otmZe9ef8BKU/eEIRdejo8N6zoYdcKPburk8iIbTvjUQd3VD", - "8VstRnmPfo+PixsIEG/IjqFzODoO9gjuid2PNdhzoVmUK7b9rD9G8chB7+CYEsrF4MdXr86IYnOuMdU4", - "mNRgQ+NbMNqcsaAMV/PTNbJutT04PLp3/8HDR8P4Ug1ZpIa5HoAiM9gf0pT+LgV9p0Hs2ktXNMuOl/tb", - "+bzdgmJ87Szfa7btpCBTYMk+ILmLlvykfKV0GF5yEXft4Cdouz3o1SvxN+12yRQb5Nlc0Zh9YqfLteOa", - "cDXbd3VLJZTNAc25NmCIKCGAdzbHMdcXaymTQUwNvUoYsx3uesRyurJdWZU81F9rePh5Mya8htsTrIfw", - "sUFmfiyhbWnDFLaGBBZPjAyk73rZ4eQprLlv2wk8FSGQjZwsZ1xuS/KHodVTd/F1b96ALgZZxJ2lzns1", - "eGH1t8a6N6fV2JrhWAxQCjsmT4sPFN0WXVJUMWhsvfw7UtXyh9ERMAXoE/LmdEheFaP9RnungB0TKTJH", - "c3QoolA0sCJZdQA5Jtpw03zdmSUtHvMuhhBJ92xYVG1BA2hhQEN72ZQ35mPzunGj3F1ShZ7paBDehE72", - "0t5HmxDKahEh9waj/cH+vVf7o6tGhNwA5HSor8d13uJi/6vc58nrk6cHzuy5e21Qsk8OSh1mZU/LpAWy", - "k2tWSciVKpiqUMkIaElFuHaGwQ0lDJTwm5va2pV4BS1vAm07hAeGTfrXwMNuMsytoKuVya0bdFYZmh/K", - "U1IR5O2OZhEPZl5DyOD3itFLrCKwfm8jDsXE3mfheEMEC5muEBwLjVxKSjPTpMCQKY3f+0cPjh4e3j96", - "OBoF8JzXCV5GfIIe2U4DgBCchK4KaMMdDGOIyTSR0zqh3zu8//DB6NH+QddxWMd9t3Uo9G//FtlxK/IX", - "X+nNP6kN6uDgwf3Dw8PR/fsHR51GZTvrNijXti6SPDh8cLT/8OCo0yqE9LpnHl+7KcCHij08tvWR4F8D", - "nbGIz3hEEJeOwAuIHLAAN773QtfP5JTGviRG+O4wlCd6Y3Sf/Zhrae2taZ4YniXMPsMN6eSSwJk/xZ6C", - "NcaEKEqQXK0nh0q+Ne7Lz6VoguJMzKb5fO4g/4qlO3XZ/KXwxFkSH5NCZdvI53A3y4G9baMDN4eO1PCz", - "fFeA71WIwF5dFqlCMVLQid202qx8uhAXWR4kidal/CFXFoQDOyV0KnNjPfiuZkz5EYtnI9CjnYu4m7nj", - "B6kut+Z3wE1clMbpaDDwif4zBAJopPhXhb7C027jEdxzTV7aN6yhsPw5y+t1Afv4JWdQFB6dIC7sxq6b", - "rtJlWG5Bm7kP6rPfK3nnLUU0DmY2COzTathqzrAklNkqsQCl2Pipc2zeOV0MXtxqSOmw7oK9u41Fx4TH", - "AZDtAEwoN7PimwKnC1tD2YhIl24A+Jww87gCL9U4aececcMt+XAsXL5V8ZMLCoAX7TrYQAKp+JzXP1y3", - "s95i6PJVKNMT17Wps/piW8ygLtyAIR5gAadnNhLYRi9W8W7dnvT6vfOidpZjTPWl8T8GVqRMMFiHHjp7", - "fdUA5EzJGQ/hAWHUkXvqFDUfmvvz0eh8sP/fNroTyA8lNi5spBL4EYaN+kfYvttF9Pzs9VnbmIriU6Q6", - "urU5bUA5KiNryqyYYnFcrjnXvudKoV4qiKGXDCxMeOJqzSx8F4uH5ImNtUG5hs/Qom18MwuvAgzIfnvJ", - "jmtQORATJljRqXWlxjlQVRXnDZqBKKC91xMx4p6fva7jNTwKid8IuDLNYWCTNGAP/AGeE9ugAIw4/b4u", - "gh8chboOM+6zGgGhpj+jEUQHdKaQgA2xMY1tiIzPz16/dAWX2rCqgZyKokwWpnpIfilKksHy6hKPahgw", - "LjZI0IaeQ4OJTmQogOUHxRjBZ2UpVKAbdHpbdOMi0IDGiG4lhS/ICLH/Dmvwt5wqKgwXLCZvftDfkpHz", - "sr/5gdj8I8ezuK7GXdaVywehLW3NIzxbrDTY9uyy2KIoXFQNm8gFOmsmZ+WLzgQc0E/C5fg8xyE7UIQR", - "F/D85eDkxZu9NGbLfm1M8NBG6T4/e71bYctLD2ZRtK1z42WbhclvVVEdclJWpGweBMTRdqbPJnXA3ozI", - "lEXUB3U3Ny8qCjhPMVwDDWXg5jZMEEWdnZqKWifIuiDwPk+SoeOZfsgELdMzZN0V7Fd7uzmmplhGUfkq", - "JK/3Bj/KiM6jiLG4EfHSHHWF3o5JLlKq9IImzckdFwAL0YIqGhmmyDfvv+mFKzAhK+16N1Sos7icOpNl", - "5SoK0GPl3LUd8Tc/1I+nlQ5sWrWzqXPBDcLEITb4TjFIi2Y+JKd05UwvGV6zjl7KHaOzmQVLLe4RxRJG", - "NVZO1+TND9tjOY00NGmbwyt46PjUDkwIThEMs0+yGhuA36soZdXP3g9eGeV8Ap4p/GDTzTEkJyLM9mzA", - "EOIMWsgwrflcgDqoXcwQ3Kd+Jdf2rn53Bi1JTVzGXn3hatPpBy6AEMWEbiubB7quGa1XcpWXdTYlL7dX", - "D7WdtH/3iU9VbbjCnPrpzy1mtPrgJWJfdRzDGlQ1y6iihiUrq48U1FmvBsHW41AgW+AKubLPoPkHWwEo", - "V2xiFoppiBmrGSgO++tVpDVmJSyZK5xn51Rx1xlJUqou8ZR59Zvkwq5APS/2cBs41sKY7AqTgqAKa5Mz", - "TC1p0szY0mtxOU8ZZGlNmXnHmPBToZrQalR8M+NdtxQvg1weprisr2HvMPDdc5spQeaKRozYt3wN+oKt", - "wdHrupS9w5ZhwYWjdcv+7m/aX/fqLE+67XFoWPvbdthEV9ngV0/OfBGuosi5X+aD9VWGQhL2yLk3tmzt", - "gd5cDs5/SlgU7TWZfcqwPpzLNatmsfuoRKx/JzBjr5I5XmEOuprr7L6Dp8AuVd+e87eddMLmcQ+F30Cm", - "UahYPD5wWXpzBBvE3ASVo+bFYysp2Tu5cj+7TCqQtLgAXOZ63HeUq6TX7w1mblbHe3vA7xPEWz063H+4", - "tzn4e2PUvwtynMR8k1XItXLBch4CwIJV4qTrJLFHs6yD3Rzf3HY/IHta2wRbIB7utoqS5VNcwF/TCF58", - "TyPjS1qiIb0WKEGrxxZR0euCDHSY2ipVoO0+qp7PUTByBcJQ6+S/t0b7MDMbiQw0Yo2UjXW0RP57UDqV", - "KlS3UCrj0pWnzCvtxX3oA4E9/nfNpT56WJ3l/Xv3Du9t40PIbBrH3J27wFRt6zogA6sstzu/rgP4j67L", - "HO7JZrLCddlCU6+eBEhq84LKjIkrree9o8ODq61n14mc+GDOBl8K4VkBgBXKRGVmccoMjamhdSaDFmjg", - "MlgYi7IUEwdn325mLS1RT1WAqutW9ftUMTMtBT5fejz7lAo+QyXJtqx+WS/owb37x7YMccxmR/fuD4fD", - "q8L2PCuedduKPRtaW0HwGerFx+3DDaDzdJnLH72zx69+BEaWa2UvrT095eK48u/in+UD/MP+c8pFGNWn", - "S+VqPlurWF3b3ix3sOksPoaZCJeM5SWlDlF9LS4kzClAKLEgRmYttvrmwDALGuf1midXAMXckMoP4gpY", - "5FvX5tpFqEVhCzWV4tNVu0KHQtQQ+rkp8sNbnLGN+6a1dBQ1utfjYa5VZV1vLES7VqcwY6IoPZsk9q9I", - "CkQlD9WhrV2R/lmHMoZ4jbh6hcUnqz8WX6/8+KQ6kMrvvgxi5SdXkPbtFQPZNgqkf12XQ7dzIS+ObjnM", - "YfN/cSt0Lf7tQDSDGRx3fBdeJ9iz/vUX8//67f/oswd/3//t5zdv/mf5/L+e/sL/501y9uKjAJo246fe", - "KQjqJ8M9xQjHGvhpV1I6Bc/MOj2B+teywu6JtTiYaIHlismUHUMI9s/cMGWLXzZSlqFeA0NNCd8iUmCV", - "QodmsQsvn9k4BHj5Dy8Gf2j2ETvYCuU2pABK0vk0linlYncsxsL1RfxENOoF8FdMIprZCpZcgGEWArsV", - "lht1juHy433yB82yD7tQEoUawt7bUgAZxYqNs8I3IXxYhxuVDV53zVmBt2zziMeiuK0LAFEbHTAsq41w", - "ljQz/VoWZbP+5jSnh6MQ1CpmocFGYkUtVEEKygYyKtLjyMPR7ro+t0XHKGhoA/m58BmbpgxpxgFSbElt", - "fspiHiFf8dm9tmUlsdpSmjPiZUq+X+HevLQppzGhuVkwYVxgZJ9EUl5y1sct7RNwzmK8Fr5po3AWMhtM", - "V4OFzAoYlsIXbuNY6kr2/xm4iQ7eMMVn7ktBNA0gkYDQiUfGzQzblNaFtYm9ssWIhEanvGtuq1ppW8jI", - "JmCYXAlfa4dhxV0EGyqoj4Vk8m9JlHC0OukF+vAWiBZqoJsQ4GdvVCS60WkUs1nz3/VApIN791GD9f8+", - "POicZ26XbhOV5UlAp0096+vAsS2bxAFY8WDiDeFbQgfhBrScmKCdwkj87znxHZUnrmAk1tNnM1+1K92S", - "6ErO624wOdEdAwcqNKG56Tq79VNYy3Xs0IONKcLXkg5wRs9wHuTVz+fEMJV61IudCHYHT4nFrhlwrXNX", - "RfDxk9Nnu8NeECWu5tLCrdqYC1kfdAAhxZ6D1tCp0kZDU9YnJ08xx91dK6UuhklJP0hFEnsrlpfRMcL5", - "1M091NYfPXnqBNBkVQYqWbFl3Nv1PWbN6+2YvPSfJbQYSpGtXNKW77K8TLBbF7ZqM6bWem8kt6N/zKl/", - "7j7G/Cgszmp5MSLdtt5f3W2ObsXhYdNCduULqfJiu/2rsvefGhP+08voh1eT0Z0XepItqA5R96Lq1cRG", - "a/tedWTX2b1qj/SrfXeiefBs/dWXqKpcQ4bBPVd7PQSCfG+wv/9q/+jq5rurwnnXkfoqGJ8Fond3KO6b", - "gLQOAFRzM2lNCSHw2CWAeLvIm1PIrgTkNXjYsI7sHz7oYpTAr3ZNpqg8BJEah1RwKY+8VyQBWADES54k", - "VoDRfC5oQh6RnfOT5z+d/PzzLhmQFy9Om1ux6Y3g/lwD2RtvAVxHGxwVAF+rAXmQIuP31auf4UmUMEya", - "snL45fXxvreaFjvgf/vBPT97DW8sqJ74cOv2DGNaZumz91wbvY672Clr4WPwxu2rpWGsyyRtHy6ocito", - "+Y81VO0giufuDaCN+5STteW8AyDuu0zt/fxAwDfCdn8s9razM9wQ9HbrlRaCrW7gmNxru92uD6J9I8Op", - "wUCF2FZVwvG4C9dGre73eCDn/LGL4yMnZ2WZutIZ4btvzOnRwXD//kOstLw/6sLYUxpt+Pbp4yfdPz46", - "sLfMMZ0eR/ExKuzX9Vk5wrYqCE3eQWbu2CuJ4569MCvabeXY2jbdci7WwcGvhwXeFONaYLlRnPWBS3qS", - "biyU1CGpuAl1mOYONDblScI1i6SIdV1GXlBNdGaxkm3BoEKCHwscYJ8UddtRSiE0ilRemh6ddO3k/Txz", - "dI9FizMpNHNVS35iK01Sjk7Q4vMY+qhJkbsWj8WO8nmORUIj1iuO4QfMGuoTpwr1MWoYCyPBC2OhF7kB", - "JrYLCSgCyhsrZ5UlU44eo12ic6vS4nhxNVbAMDWPmRoLaBZAY/6jUE+O7wMyQr9XaHJQ3GU0ClHTjTo/", - "hw52HXvAFBgqXDk9jHZXuSC5iJkiL6VmxlDCLDk0Q+Su6Dj9SIxz/3o38cq9XspV4YO5DRK9kzD10RjP", - "ONQW/RyjQ6+hnN+7vojeKVPQy68uR9C9NblKBINLyQCNbwq3nTXIsdiZITVzuR5F7s1rWzi4PnUXemwk", - "ONXVirw5Pa2FPbgMjW4TRy7Rsg8yu9I2HGyxkWwdzVXcyxVE7O0o2B+PTN2UVCoS4ifHoa56GD10gaXQ", - "mmGrpjhvsq+BVhnMgeLC7hMQzYYJNmpHW6CXMjCy2qte6ZlDZXWitM+pcIj0BWhL6dTXK703zfVeFvE9", - "+1zvIaLOQ0TUOQpCHsRsOcnzkGoEj9x3yOvXAHuDvyAitE2QqX2e0vv7D0cPHw0eTvfvD47i0f6A7h/e", - "Hxzco6PZYfTgcP/gcENSUYcc2OuntQY15kAccxG1PvHR86Gg5rbchYZs4uKx33ERy3e16y8YIFv9ugu+", - "3fb59dD6zkMIpgQlVBtrvmjhZKd4ybMI+sZ2Ps+6qBMXNnTefzXa/1jrDw6u5Y54pXJh3arYoHQhpJUB", - "VzerPs7rsXwckE982bZa1Y93X7TR8b1Hx/c+dtF88sa2MTbJ6RY3ty0izCOnN7JDfJJwxY7kDZQ9JxNZ", - "q75tvur1e7mo/o3CQCOWunjcKYmr7cD2w2xk07XSAnlwUtNXMFLFImfGxyCpeH0Ea74UwBogAj0BWxip", - "2OIskCD64U4qugt0g24xZ6Kzics2GQMzI7l2+XxcACNG/yN04vARjslzbIuPaGrVOjcIW7i76nqj8crG", - "y8D58p+2StbmIZ87/QrfAWWLwL9w2rAMzmS7uQsrnR2TXyS+U2h7QjZtv7Y5qlnrzRttyY4D2fd4N/gx", - "J2oekx8K8bIQUJ1AuqOZ+3PiGFYJM7VbA/twOw6BhJWdqwBX9Ht2RXv9nl8o+DMAdfG6pPq181clxVAg", - "GaMJnuUyTT83PHHg+jgTrg2PtEsegc1tE3tcbiaLJ1Z5aotJtamvtg0pXvJS1ZtTsoP4qX8hzrAN/9ot", - "4ldrd93Bo6NH9x8cPLrfCSWtHOB20fgJIhOsD26rnBxl+cTZRtqmDkX0cM0ja1UoYl/A21WSeaYksB6Y", - "ue+w+vFHw0dVcLhY5sCHiyE5JEmLRY0bFsQ/LHhRSxzkbzxZ8tlM/PZ7dHnwd8XT/ff39cF0vwX02n4o", - "bHY7qQYXrNmo2XRgS3uG8buQoJRuhbh7yTTOgJwzQ5B+BoRGqN4U+dSO5HJdW/EgYR0dHh4+fHDvoBNd", - "udFVDs4EjXCBS9mNoGxJsCXZeXl+TvYqBGf79EgwWQDmoHHOiCvQPqoLpMP90WGISlou7pJqXN/LtHXJ", - "3zj10U3KLTombxeq5dopD6724eHowdG9h/e6HWNnHp6o95s5jGtnv+fLZ1R3fgelyVePzwj0rmY0glr2", - "j609eNRaUTHXjPzy+JWFdhKSZEwNyhvR91RPc3WxZlean7nS/LCIjC3+sGWKtUiVB/che2u/G+pjKHzF", - "4ZnWjn6dCwaOb4C8AvsaWIp1Jt5vu3dCgpkl1ZcsSihPH0c+WaNxj9kiDxNlm5Wb0OWKcbr82hXY4d1O", - "ZrLC+GVTffAdIhXJRVFaaLjduftJfLTt/N9eNNvvh1CCj4DVc/BktlzjNZYyU2zJZa4/QUfS2OzbWSKl", - "utK7barPS6bBg+dBsd6cfqOJozWiDcvq2pijxg0gbtec3JXOc41EwkTetliddqPL1m+acL/l1PY3QXPU", - "uEErdGIMnCsX28NIn9AkyrGYFi32E2aFFwWCC2RZsrJZAkkipSDRggp0tyhXuk/MCSWAwjAMxtTCk8ks", - "GJ8h35FEWtB3KPbj6kzZQcBrIAyBZWmnWs3RklKj/PO91DIZV0moTo330nCl2jA6U5FUD+tJjazgkdtX", - "ajbVRM41qpcG8x+GzTIYGVU2rYEKWzdtmVqtNBC6HRhig5mHblR8CAfTqspOeMGUdbuSNFJSV4DF3pzq", - "YdfsuSIfentoaH2wHUjXukYDVxlecbpzecXQ/RjIDPqYGxJpGLMPNwRdejNnSkWOlacqhOxM+sPOgZVg", - "154UgFlXHKw2EywnA6bAwgdW5O8XliXfJngvetZ2neVyEczXenuNqsJdtQ2wnacGVzS8Wv2CBkNkvI5Q", - "txEUr4QKbMJiXQWssqw/wjX2yisYhGQHs2cqbKmCJbnbJdwmrPzCd9b0XleJGDweXTEaN0MynlGzOBEz", - "uc4JruKD9TZtF/eaMeVhEmMmOIu9Vlo4Y53RjGtXhy7OmVs5/GwNSA+DJShqTs7aBslCdV7f/GAXQ7Md", - "w+ZqM/hd17DDTnIdTs19pXJcKyQgIJ4ySbdTOCnXk7BjbL1jxeZ5QhVCB3Ycsl6lCReXXXrXq3QqEx4R", - "eKHpYZ9JSCGbwCP9Hc5lt9Ps4IVJW2myczs4l2FoN6Tx3XIK38Esdxv5zWjT2bPv78H7neLQgjHnP/CE", - "OZTL14K/rxB6vRzD0cGoLe++pdNaxv06zOtVObcj2eCJz3UgS3GjlOOLqrHY1cjA1lgcDo9ah1vJgzB7", - "Z+L1fEP1FJSPAzl5Yvl1A+KETBlmEPmprXXXhS12mUqwtAxU2/u7nNZNq10DiAMFC3dECbah2CyYKYA7", - "utG0bVusrUlld6+CZoFsFSaKL10RJGJbaccyUquNn7xcq3K4YG7JuJ+jrXjYoYKPjyQpgBDcV7sjIjTr", - "UQYinxHvRpsVVnbGclqrSpFVKFOmFCLQg4QjhZ8NArhIZdfaI2hBXveKKJZSLsaCi8LcirBsjAi2ZKqS", - "bysVKFlzSPb4a0XFg99ZmpmVKwaBZvhvNAFnnB/jWFQHCZ2DGX1IHgtrWVR5ZmrlYlnstD4gFF8kXzGj", - "sH4qNwsyU0wvqnMP1cwFGe+dVHFrMbIV8U1gGhS9tcTISyaqrKzoJqga2o4m9q31eEBb8BqfOv2T1GpQ", - "k2aN6c3fyzVTYSGxmFLRpFMQTOWoVJQTPJAwAFdb1P1lWXyBp9IBPaXs/r99l+VPZ0Xn9d8azSoIKR4z", - "/LE12wZNsFGWr69Owye3NegFE+o24eKseyXIjg/G9gWa9G5LoaRO92S3nL5m2oEfzZ5mUf3rRw/vPbjf", - "sVLVR7n9LA7Yp3byLdMNzr2WnTrt4kF6eO/ho0eHR/ceHVzJw+IzVFr2py1Lpbo/ACNi4LAm//rHP9+c", - "1qnl4B5Gc4+uNKg8ax/S66zDgN6c/usf//SjuvaAQoxmHe6/JQKgNd4nqe6kDzmoOwO7Ock26PePa0YC", - "WrAZssMQ1BvCd+y6DcrBNABFuknBNKMRN6sAI6fvfIEL16SB2t2h98ZgQyKv7dvhqwLn0vm0aAExLK7B", - "f1ovc4MWHnYueKfzaZtH+0Xzq9iOlF6LarREh2AFSxFhA3cxn3dU10LDY2b9Dv0iV209b8e22Izf20yq", - "gKEgJG+tryhUtaAhT7qXqtvf2M6K37Jm1mmu+NsN57D9CF7J6hu4kQNG3yjLu3ZUyu1wAV7vrcm0Wopy", - "Y63PWt3K8ta9+nc75CGvFz0pbrCrf6+SenmVFxu0ZunRjcEtedl3v0YSLdRUyaoJGNBkwgaVMgjYmujc", - "egThzDvA+kAuaHQpZ7M6au69dpR1BBDCtDH/FWoMaCZ9sPo7m0UTotuiBY179/S4R6SClMx03Gu4rYKJ", - "mCl9P3EfqKPEjDbBnpeJ9I1Baj+DaSKjS1tlEYv3D8mIpIwKTXKBh7/hVdsfbfYO9XtZZW/8OqJyt7aU", - "bjPJlC3okmNlFudTmddCOtl7bjSGnmI/xySWFjeqVmLazRCa2TTJ43LSeOlQsXIdQ4fQTgofG1u2RQPf", - "DAtbi9+Zkn0HewAc+8WL074NYMAgRjuwWqSkn6gdQa/fKz/RKNRQ/h6ORAZnBY67Cfyfrq9jNbsdPauK", - "aWa0QwIvyaFBBJC+JEyzIkDaTZGrJ6itX0m5wLBBF/4BC+q/bgmExCzCE6nXz2Kd0K9B3I0MBLfSoRSE", - "wxAJ46HYUMnnpXMINwdgjQ2V6vC2n2qEuPUSTrSRWa9+qifsfcRY3IQODTfpGnXv3gxG3f9MHdpQUbjd", - "tcbI6fXZDW8uVQzH2rba1ewAIcUATl2xpfZwWuQ6j3pTJ7Tq6lfBMSYhoNZQgw7jF+z95rX+hb03iLQe", - "5wmL19e2OJKWVbnLaNuKXztHsu1AS8W2lue8gbKVNnL9WoUr7avsLmpXulY3Uq9ybXfOmfFtzx0Zte5Q", - "vWRMzaXluiC+ST3GxpJSn7gLnuynuw0SPFqErSIO3rdjtifYW8D5PuPvNxCPbWAV4zpASnmQPDUQi1S6", - "k9L35OgBFhHTjbELPl+YZFUPwDkKoDJ9VFFXxQwT3lDYZefL3fQvrke7ue2s9h4Sjs8rIENrxVGcSDrZ", - "hMD9pGjmrfMZXaEVp9VJ+ODwaDQ6PBhdC4LbD+sKy/WkfMWVQK3305acV3mPsKrHzHKgag9FuvZ6XW1b", - "+K5cTaKNYjQ9xhSejEaMJGyGcHtFaux2z2Lz05sH7wQql49b0L/fKLdv3gdfL75TfErMK/+ypg+cax3N", - "qPp8i0O0hc1Ea+B8gey9w8Ho/qv9w+N794/3928CNrtYpLa8kQe/7797kBzQ2VHycPXgt/3Fg/lBehjU", - "wy65iLvS6k/QtjXKprwk66hINZZGdtwcMqaaBdPrViNAc064YANd5FptT3jcwAus/33r+b+and/OYKPs", - "cF6fZFWEoIboIGXdDZKXm8xG30VzNidPN8/iWrlMzYGE6a05FGjVcTBY62K/95EYD7noeA29rjTsfBFt", - "zK/bdhWFPOx40oO73LLiIfJuQDxUZr3pAl+/5AK207lU3CzSzbdF0awAJMe46d+1ievIUUNyMhdSsfrP", - "RZhcVYmCl3v9XvL7Uf3MuN+7Y4g5LOOCAN1WV6WCDmFkCVuyZPMqYJNS8VA2kp0qhgvx3f5g/xHGISS/", - "H303GjyqRxz07WpVl2/ft679OuqyhtVigr4I1f6jK0Vc+/XcREE/8VApvPJedijHjsbL2vT+6vCpu7UN", - "Lh+v7XEDE6hVAP1YSc9dbpOq0BSDRTaEcl8x1OqG9lglMjJlcy50F7vt4agw3N4Dwy157KDGbUF4WXyn", - "2j3QSpVOeJqymFODo1JmQwbDQUdbXFOXuFqVE/9WQFobhsW1R9vBFrYlXG27Jocfkdn7UdpvN413Ew4I", - "2tW8iopoZNiwT/gMbND1/n3Za6t0YGIkxqsde8i3kmQdD9ClHOjtJH0yl4aUyfgd7W25aLcLFuOHGp35", - "RvQNSxAHnwRapYAi45vY18lTkikZ51GZP5rgoEvsEJWLYVchf+vQbtS+gSneYOTYbt9oM2h0s0+27XfD", - "NgkE277V+6PtW30jRpF+L8/i7TzMNurGwa6EAb8lBTFgoqkve0MSrEzmbQeO/rK6gus6r7UlRyAS5Zl3", - "sABNrVNSwN2CLoZQXO9TljDDQp0QmcRllgTXJRfdzlL37z9ctLk40SO1PpCfGMsINRZJCb+XgjcvNDBf", - "wLS4S3ZGHjdcW4fXwBY+cqtVH9yDrZJY61ZVTbhtxRgsl2/YvC3yymXF/O1nUxPN1rFUPMOvCWkv24sJ", - "uDd92Fur/fgmzHJ3KaS9cK6HBgCshycucNb998tYYBDr6sR7FHLPh8jiFZ1j1b0tfpRWUglVqEQnByWG", - "ujJZWKoT/kmwHqn20C9eQMDXfUavK1tIuCF0Tps1KNoKXFpP9PYYovaSMa/ofCO6bzMdtmp+fzz4mzW3", - "k8nweO+7v/zfg7f/GTS7NwwImqlBzGYYcXXJVgNbzwmMFcM6ti0Wk+hpQ+fuzDCaojENceMdV6qO996o", - "4J6rX2i6NgUMVasUY9rfOqG//Ed7oFdlGV/jhbH17H50rZWbqElrpL+Xd1Km5j6o3mfU7Q7HApP0LtlK", - "k0qJt+mqdmK/0cUrlVB8cmHJfcjE8oJMOdbM1GMB6j2NIpaBWuWqBnEci5LIhhWjSbUfV2rOnxfnmbWB", - "FZA+vQaM/OL1q+9fvP7l6eTF2bNfHp9Mfnr2P73jnmDvBvYL8QBo7+jefVfuvbqS+8GSI1evnDEkpy5f", - "wcU8zHLU7BH6DHykkIKWrAh7HyW5hrAJXHpjkuvXyFjPWr5+zYmPBFU2JgmFZzjwbwD0hjHbe9VFF3Ht", - "iZFrrJPvrDxckHXRxRLOuIdXipFekghWKIGtCK92ubH1Rb+/6HaXWLjZwGHH1F84bAEr6s9cI3CHD4Kp", - "NCY7mELji/naJzYM9Qrwt4+LDoMhmJ+4ZtPo0aeoq/p6YyHVpUwGoOe1FJ8ImtXtWgRTCLArm5rRa/O+", - "zKcBZcbZuOd8TgMOl5Bj5ZPUP/UD2po6trb/rYXgwgkdT5uVOeyxtEvVqCTRsJZoM2jP90hBvG/BMkaw", - "ZpvEyytBhrXu91Jh9lyd4hBwSCwRH35T2nZ5yjzg5ABf2p6NvFHBrMysMpL2vTmVeWhbNi0Q5OMDj1Ss", - "shH4AouvuWQuQakDHA3SWx2dzL2M4jn63TXZKUxgfgmKtOt1v8Dmihan9H3xBWgB2kTTEYvzKCtq7T//", - "HqsPvHS7BOzTdYHDaGi5Yaz9OhVtWhNPVeubUaWq9Xnb9sGD53jVBu7XdrYaxFl+o0aaIXr8K+XmB6lQ", - "L24Hf7lxyH68/GOmEFavCcjfCc2epyyeyNxsPv/QDKRHe+UXlWbLSsXeBkCRiKNaXnMbL/DwJOUY1lca", - "loNFueJmBUYjp5lMMR/UlwfGhcQP4c/lh7Ek64cPaD2fBTJnnjPBFI+w4K2FGhSoMQFsX1n30JbAXIPA", - "RRHoxZMTZ3fxKMqoR3ODpOcDUB+fnYAznilre+iNhofDER7mjAma8d5x73C4Pxz1UKta4BT3plCw3yWS", - "O4260OBPYicJfe8bwZuKpszgG78GIBEwANM1RxWEzitKZEa5clpklsi4IBgOb2MFB3+hHttbuW+XvbPx", - "GPOtMQ2IZS/c5r5FQRnPDk7zYDRyWPHGXb+YxGQzJ/b+7sJoy+92kurcEgUKGqypeV62LJb+Q793NNq/", - "0pg2DQXPbujDrwV12cwMtfN7o9HNf/RE2PxEl23u4sKqJw4JqXrWfn0Le6bzNKVq5ResulqZ1G2CMQO7", - "lmtr9TijSQSsAssuAaANs8/BJEptCLfKBVar9i8ChdZPge3bb3KB1vS9jFefbAlr3/A2ig91dgbH5cMa", - "PX862inIeH0j3SMPWm6p9hYI6HtalFq/s5NyNHp08x8F22/CI8hW9QTsArO5xtinBCHYPQiTVOS3XBpK", - "iryGL+hIO5l1WpBbv7yK9v7g8Qd7vBMW8gecMZVSYbNEbJsth37tOFvfTHmcN95qnvBPnvbcTeXRiOxF", - "hYJc/YhWr62mMLh+HR31jtu+aacX3yHhH93CCXeTLart3uWRwxqn4Ff+ko6T8zVOSyEkKMs9Z+ZzofnR", - "bV5Zri7Dn/AUfSkE/JwVEl65W2uXwl6mcmEV4KAE+LLM3HTvfVMX/l6VTyrhQujXgK6xQoixDmYar4bE", - "r6lV+sHoqy1Ev1racLP6+TqD4X0uJ+zgNk4YzrjwFH29pr5eU5tOuaUWPwU8mJVT3sEGcSULxJ/P/nBl", - "68NX20N320Mny4Ng72xrwIMcEheaG8mYEb2AeqfgbLbATz4Ix1A1nP9OqIoWfMkQ3Q/r3uWJ4RlVGGKT", - "EvASWB96q2Fio1mi6G4Puhv4gMxygZuAHppNMARm0gbEWYZiciHAWEu1j5opcRXXKrTbsx80sBcdllcj", - "ebeQmhXAhsJUbnPM89ZWO8Zuh2PxyiHewgJiVLnnNZoliNu7wf4jBaFj4V741rMQHxGnaVpyLqoQPJFb", - "iE67LWtTxZFOdCRDoEOvmKDCDHTGIj6D6EOc1iVbucDWYIedSlnBgP04oS6MD9Q62A0D12HoUhil+Gnx", - "jDhKqvtvhDSuyHfp5PJYSlRB7YNghZJ5Iqc0mdj1uWQBn+BzbOEWpXS4lN4kIWNmq/JnK7OQwv6dT3Nh", - "cvv3VMl3minwOY0FZqS4tYZohUJAJO+wNl6aSWVsnAN+c88Oce+PS7b6MByLx3HKhX1D21dooiW4KeA9", - "Iz14iOVeLfRgT1PYD/4k10amVQhYT3d2mDI3WW5cao1mph+CPx0LI8kfHuTyw94f5Rc/oLOY0RjopNLE", - "Tgll67ZR6wmF2U+wacDdznABxj24SG2Yx1xRYXBMJUonmVe3dKcoE4FFaJsrHFFBMpnZEhtIVAsKJFfr", - "A0EraJIQg0fJvwuCO+5ky3wcBmE6bQUgtIhxjWPEBTn9vnKYRkcPw+dJs0ixUETJf52/+IXgrQx7YJuV", - "4Vq4RlyAwABRNFg7xPG0ZwCTZB1VWJ9x3OPxuFe4c+NdHGuuXbjMYIA+xe9gaN/Zz/R5/N1wCF1Zd+Ux", - "+fUP28sxnKUstYCo496HPqk8mHOzyKfFs7fhBW3DcTuvMQKyY6+5XeQklCPkTuXGt1ckmO6kuwUAQJCU", - "HKgauDLlgqrVpozKwNK7FZQzG8lYWYw/xhi5OO4dj33s4rjXH/eYWOJvLsBx3PsQXgHntWwvBoj3WeHc", - "LIjo/mi0ux0S3K1vwGfZwTHwiXXAVq2oqGQKO+jwaP9c/oF/a/3z+0JjE/DxEqPJKv7e+H7Xcj18/RY2", - "/xcQe4B1pAwjSgbVmmxwU2AtQbzasOjAtTwjFVWiqiIHfCMNfYCKiCVeH9hugbp9rwZQUcSS5LZPzl2d", - "m8JvV9RS+KIcdbhZ5fne6Fe4Y4ob3dZtV/Mn3A39fnGG/YBZ3xn12dLHYIcrySBKkNPxiW0MlpNzHNPg", - "nAlDnuGvQ/dfr5Qi6uVFIucXx9amAEURSMKFS1SoRFCD3OLWEl+yQEHFe/afRRm/HSvi/Osf/8RBcTH/", - "1z/+6dD3//WPf+Jx37MAeFhP/GLBqDJTRs3FMYF0vAFNIL7dTQYr/rIlUytyOHLOCHwEQxIsKqK9ZQ6I", - "mWPxkplciUpCh62op12HzocB8+EiZ9oBLUFDPnPlfmzQZcCg5M+yXcpbPdH9AGAlzqAyAbgVPQ0g2h+3", - "RdWditwL23LtnGvW3Gb86FoU4Xb+Yth7Y6l3YAd4RQaDSxw6d/jATZrsnJ8/2x0SVAMtVWBJJ1Rqym6c", - "fjP8ypO28yTLUeoMBVfZ8qaIZnTKE+5toS31aOwRTGm0AOCoIvC5QIP3XRz7kQKPgRhQF6HZx6Zj8eJ8", - "D22/hkUmV6zvOIFyGLBlwTrpEnDwC8i/uMGwtYFrOxYzRqEPcvLUMoEKTHqRsVl0LBBqBYNvuanVxuuP", - "hcX6ddjScPBAkk3wJfz+nBr2jq76pCL52vo1CTWgqes+NB4LmwPp1mCAYDKkMswh8jM7pIEPMXbJZIrN", - "wGqLE3El3vHbO2DwcKHX/TLh1X/OpsHaYRHpxHGY3xxVVGkNldjTi3O/G7t9oiWJEo7UEFExFnOMUPLw", - "ylLUdrXIdFtQFQ8iCZdAFW4LQqMTFs/beOyTSl+9G5Rkat8JHKcfm+T6pQkXi/UJwCG2EIKbXYpPXZtu", - "PkXX45/JqWgX6CpeRWtaZpbf2NX96mHs4GEMr5v3NoZcfk89RubNhSLbT9xRJLKnvfU1t08qS3YXpkay", - "48GH0F0jFTl7ckJoHCum9e6/tyESZmqptJT/4H4EVnwXMTFuLFLZvfD2ljqBfCns4KUbNaF+Xs0KyNX7", - "ba9WHqn1pisqJZVX3s3fHo2PXuUaKYXekta+3iRbo4C5jiCnskotAxSNElaIL8U5rVLRNquyjS8urpyN", - "4pJtBcEb7kDenn3ZfToXzbvhFpji0wZDvENGWM8Br9Y1/5Ko+XWxi25em8zPnxdpjm5PCrptU3SIzL8k", - "dTFuLBtwQYvA0nqBPmfG4q7cpJ7uvhCYONi53am2A13ZWRfTsq8SCyCDE0JLzGbd98Q26ab62v7+TJov", - "Ls9VJBa35F9FlA7KbrlWmxTcE1ek++b02xrE3C3H0zgCCywyPHC2acRrgrqXVK9EtPs1pOaTUzQudkWJ", - "VX7eLC4s2fi81LNuS66DgutlRXiuC72WCzJLAL7WOgFiPsMgQlOtsI6jPLiFURaVzBU1zMVOfokJyWd5", - "Yr0bkRRLpioQktUrde8PjKbdrip55rXxdn398ucBE5GMC+dJu0zqnnxihcnSfy3J+PZP3ReYaMu9eNAm", - "MH7E/tsod1Jgiv6vgx8cquj/OviBJhkX7H8dPk6oYdrs3hixjG7rprttBeYLJj7QX3h90dZY056h82qC", - "bJ0yPa7uVSizhofrjd1Fhs8no75PL+Q1QYQ7SXi3RvegSd02lMqJr5dgt1QqD6vVwEr+9xb6zmsEfev2", - "utrnucYRoGD3BeK3SOXhYKnD0+aeuB1vEnPFtN5mjChadbRH+PZ/KpOEnfSVjBLFun61S3SxS1SXa6Np", - "wja8WeOE/cYded8LYgutNj76igN2Cw4dR5EVHLCah7tEAltIbfDR5wEK8e+UicOLo1C9zzq6TEtOsfFe", - "82fq5Gkfd7gPe3rytEwGvyUHqh/HrVsE3HdvXxp7nE75PJe5ruaZp9REC6YdBkPC6jfDl2arKOWGVmvF", - "Z0ylo9u8027dGPGV7m/ITNLcUMu8bRTENmXEt+qqjLj2oIxYfGqHU+GqSfV9pcHdluwUj1DdlYxrQN4f", - "+l3GFVKSoDqHJqUeQ1C1OR6L/w2K0a+G0fTtdz4hPh+NDu7j70ws337nc+LFqScVJozirjDs41+eYujJ", - "HLMLsHZsCb/RHAdJc20s6fkiMf92mlsZfdNddfNU+FV166S6VZZrs+pmG96w7lYvNHXrypunt9CC22d/", - "UvXtT+5TrqmWOp/NeMSZwHJdaJXUa0HUVsX86k6+nn2UChfEUYnArEkindXIgmttkdCLGtCfNMSx31q1", - "QxJqDEszA7BPEYPqwTZ1UC9yE8t3whfxwAn6enC8nE/oevddTXwn4czdcFH7rppuUcjytlVd9+EvNHVW", - "Zq4mu1MuS9GmXbu8W+K9WZ2yw1V7+1rll0xiVn1bX7oMNIRAUTrrf0pzm2dcvFniWQ7Jq1c/+5xiUE+U", - "L3FopK9r6Gtfj0W1ruGQPCsLRtoGvgdQH1jsMAjgfV8pMGY0TrhgmITBdCj9t16N9E6PxaeXgMOlVm/b", - "997lWNoNvUMJ+M5Ywa3ImkWwgXGe4uJoVqqwFqfFy5t4ar4ofuUYUIDxhGS9PZobOXAoBXsLaTE1w7DC", - "ZwmNEFUYmlnASwcMYxFuq10h2ouSScKUBTLNcuPFLcAUdoPjwsiiVLeTzC6g+0kuDE8u+jYGEtommkCZ", - "eAt1Mxa1jzmZD8EbEJgER6hYZkfcqDsMg+ZgaoRWiLNQ/SShyTu60mPh4B7s61isXrHIYv4myZD8KBFp", - "x9bfrjBeW/z2Gz0WFzxO2MQB5VwQroleSGWYYDFJ5ZLp+ncZVQlnCifxhMLKaZLSFUJpWlRhuz4yYxau", - "sgbHI+HfVMQcy6jCl4spH48FJQejEUkZFdqBa2g6wwvH9UFwELUBfUsoORo9cm819g3h3v3y78BpUoot", - "ZUSnUDEXqNiWJ9/FDUxdWWNbJx+2b8aVtvtV2DddvcraxnLtq/TGfZKLEj4Ebf0V7xpsl8mVwHk69yTj", - "qrgGHWLSlEUU1lPI+ncQRFdGUa5CFyRsdaW+9r+j4FiZ3jkuVRicI0GTQQQ5ABKRHxd4piUepd1vW6iq", - "JKo/x0UTPCRSEUoqdF1aNFiUI2vcQdDZi7JYrPDF3y92v/VnB46vYwT++Iv5l3Q/IRHJ2ax2ALdfTfYA", - "b0qKWyfhP+s5feKYXo3FxZzOhdSGR54ZekzvqtL8VSHspBBuXtkgNc+kumyPhP5BqsuuGpiP2PyyFLHq", - "DD9DRwQMD4N07t4fgdZwq6wA0dy6ktakr+KUotDFjS5ilUkixZypqlX+1t0GVa1uxyJtwmWqrLO7wEUD", - "JWTifrSFxivx5uhiiFyvf9pgNSQ22OwyZm26sqpHiSR3NV4Jp6oK+GB1QW3jD/peHEK68hu2g9L7+nYF", - "mSoAjW5Fai0+7mFJA1CtY/Fa2+IOF9b1dEEKHgwCrS3YQt4teLSAfvA37N+iutIsuyig9HePyXM8yJVt", - "sx/fsWVagNa0TJhFY12m6cXxeqntN6en+BK2Wdii2hfHxJfXLu4PDa2qMKwwi4RqQ35x4LI7hTKOO3oB", - "NszK/HYdQGtZ6mAsQmCtgHVqO+QzclHBbb1oARX0/PZnOdefjauoLFBj52Ikcaoj0iYTca8tyIMnYcfP", - "/mgUqpvQET7WDuOG0WPXBvOznBfFcWqkTLOsK/m6YSIVL9N0Aw2TnZKDEG1imZu/aBMzpfBlR91txE12", - "aGT/YeglEKpDHvUHe3csWpbKzjC8VMBUe/0eE3naO/7V/WuZpr1+z40H3ntnsrRS7eIKQv4WON5mhx/6", - "oR2qYO7+icXzO7sSXSGl4qjEktnoSJR4nAYFUK54XnBLrwP9W7+hKti/jevO2QDa1YiXtsGf3sXpDY23", - "Jr47jaYhPgXp485OMYpZDVv45yTJV8Zbt5sJWaAq48ge3cbIEHqZRInUrOYn+3KAG529sCF6t9vb/BoP", - "YHhx7kvwdQkEOnevnvs3PwMTxraQGz9m4qd767E36yP4kkEo9NpsgP010P62BeV89oT06bZkbapdKOQr", - "bV7dWNuJMEHdWmcR7oXYlmSluZEpNTzCcoDRQkpdIfsCmt8W7nQ2+IIy0UJljQUuEeMCSPXCWfMvnIh5", - "7CyPhFYfuW8M8XWXvhF+wz8q3/ihYlwpOH7fa1BYmUZDyTTF2YxkNNcM5MwcLGKrKGG+/iMDv3pEM5Mr", - "hqVtGUm54GmeVmsuwI4tKeJDXeynF30yzQ1JwMqmioc+ZimSacpEbOWksVgwuuRMwaIk1DARrQaaYUn8", - "JSNQ9iGRNEZLTRZTdJhhIqViQIFYwCJlhsbUUBR0LuDET2wu2EVRJd9aRwR7X1ID+I5z8a2tpgPdXviB", - "XhCG5SK4XhTVlCMaMxEFyyicf95s7NOb9M+ZaU70jgKsrsVL7zLiqmq69sP5PIKxvljIjw5sfoPQq9uV", - "6noSjSejf88jbefq53hHfrpiiTed4s/DQVcQ3WfjpLt7L5xUJM7t5yqnEsn8z2pHLBhKDRECEnTtNl7X", - "v1ZUZy2W+Uo8b+8P/+fJNayLnwkn7Lcq9m11AMtJfw4s163qtXjuHZlVnS2papO7OxbsBnV34pNUFS73", - "pRhba8h7Bd+uciejKGpfUnxl20227eJGrsu2vW12LTKiwsi5GGCobZiDOzNuK6t2poN/06SexuwqLPPO", - "WWTpubh1GE3PGjO6AlPJnyHWeoP/KJJKWRQRatiXBf1dsRpWsyzQNlcWGO37pNc3p6e7bVxCmY08Qpkv", - "mENUMpvgtTReN+C+WDKleOzhTZ+cPi19sioXQ/Ii5YYYSS4Zy8rEIEzOBNTOAk+lMewmcEq/x4RRq0xy", - "YbaOomx6M4Mpf5AOVuYzFCVdLYmv7vDO7nC07H957Ay5DKHFBDZrpoaarTWuuZhJlVq5jE5lDr0DD4Jl", - "gv20wRYznjC90oalNrgToCDguGFdIld73r1nd7mPoc1wcmzWYcZUyrXmUuixcCk3GVPwbXgd+q/EqQUd", - "AoYW/PXMMsnPIwYSBmPD/qhpWzVEvsKa1r3j3h7Nsr2YGtoSZ+eG9xFD+gGDGolepVNIoISoyEtNdhJ+", - "adUTstQkgT92N0ZFTvC9T11Z//onC1b6RMxksG6ppdmCmP9UyXGOrXnH5GfB1u5Iz/zoKMLnrHpwPS+c", - "yVYWq7fyWMVoMsC6/B6LieSGJ/x3y3ahE64Nj2wWGS328c1pweCHY3HKjOKR9fdGEH+OIBWo6e5lSkZ7", - "gON3GGUcAf0OGQ4OmW/74xS/+OTsNbZLWSrVqj8WVLiOXz0+s57mGY1Yc6CCGfAkk5O9F1ti1s9xmf6N", - "4yftBDcCUgQ3/Kt78uqwMa1nSLccUZltUttk9qcP8HXS5Fcbx5dp4wAaLmezU2C1eYC1sD1jKZM8hX/Y", - "P062QdUZGi3eYNPPRvK2w9n6GT/BL+JQujnFzNZ4vhMHjF2wLzV+FhbOTwGFmFpkYvAWeGz+jNT96V0J", - "1XX8DF2vbkV9/fTP5mzd9s3nxuBBU6rr8aUcc0tpfiZGbraEvaO83RL2fSKjS+3QdaomTKIYRch8+LGE", - "OHfuShQTMNmXOFQqi33GdH8sGsZQC+KkCSWGqZQLmuzhnG0nCNbuLWp0KTnm3EeYMzPQYOrOZJIgIjsi", - "GsJsoMXMd1DxLmtXYbLapuoYNRJgm2TKPID9bkh1+yvl5gep6mj0nwtffFVZf0R5pBxt/1sA+Nu/+FGA", - "/Kf0PYZtx7lzbvsR7TyX5Y/WLNUnuDfj3uFIj3t9Mu4dpOMe7MATiuZcasg9knKRG0C2fGptbZhVfX9E", - "NIukiLXH0ffWxMORbpm5I8uWhN37+N5tij2OqnApX7qPhNgDtCPwPiYQkZ3qgXNnMu7joYuJzI11Pbhz", - "5VrFzKB5ZPfW/caVM/JVt+/Cyf/qjm+NR+EuA7usbL3l7FmuF6zd5PazLZqVmymsQ1FkWy/I3+VU9yHr", - "wFrmATFvje/B22f2A7dROwI+dZW6EW7uX4tGdCgaUa5VGH/TBnvCleypw4JwsveZVAZ+d7gFjoZQk0Aw", - "ECyF+eLJyVhEwIosWqRiqUTu5CDu7S38+K/n5NmTl33yFAs+kx/z6e6QvIBMG+xVO3/RWFhJzDKviAKw", - "up1BHLqe7diRem4ycB0+cCUp/uDTeXrwZAS8PH6vfMB6v7dgNEaJ5I/ez9J+LAAk/fJnOECwf+7NYtt7", - "G4UPsOGr1eDxzDC13u2py9kSBQyKu6Q9qqAT3PDD+EHtwfTKb1rZwKKdHB70AuAnH77W8bj5QuG347HD", - "77rSjpDN96XWCMaDWDDHEAusXtdFJYy2jGXHyzYqGPjJk6efRKm4US/1Jt5VKxfw73q6cKZfrKMpq+0T", - "EHFRQWerp9cnKi8swrV9jUQ0oxE3qz6hSeLuKHcTFNExg0L8nSpGL8FuPwQUMvdln1wMvtq+d9SSmOtL", - "24PzxQ4JhMfpfFoMjuBBs15jXHNI0TWSRDSJ8oQaRthsxiLMC8aSPLrFl1sMpXeDZ6f8SBAWphJhn39x", - "ZQvDNIG7V5JFk+L27FbvKRYllKftePJOUMPgRww1mEKnUkDsQmIpjkZKak1cVwOW8DmHshoYKQGVNMDM", - "RVM2FllChWCK5NpG6MPQB5liWueK+Q7g7nIU1SclVmSmpHGhCYmUSo+Fp/A3p0Qblm0gs5e251Oc8w3J", - "trZz96U7MlI3xtBuCnFNCGyIpRS74EBHYBq6g7B4O6C7lhK/lIP/SvE5xL5mSlLLZG1ckj3Wfjntoa9l", - "T7eWMD0vWnUrYVr0WsmQrGQPbsT6m5Tw6XHvahGIgY9f8lY4SPfoahnNP8FLHb9dz5wND8I9+shZhqqx", - "/jsWPj2vJCx2NWCVFP6lmZMqI68d1VrS73aIr85ZvjeZddsZy+vOILy+ZOQuWkvlbVN4Pz9CGN0u4sRt", - "1837smmrhrxV001b4Ae2F0j4LCjwZioj3DHiyjUqI3xWGABAO3eIxRI8qHeV01/zPUv1tbjBjaby2woH", - "CA3XlspvuZ4LXt2oKL1xbbqpSa7HP5MEbxfoKvK7X/avWn8HlaGyWNtc0EDwLM3Myr3kfZVl0Jnmv7Nh", - "iyO4iFu9OVfwNUI6Px15eDptDei8Q/SyO3Tk3EnMqKsGyTWE+ZVV93xi9xeGd1g9c7WLZQ9unQFV0YIv", - "WbvRvX6C3RJlig0ymaFzJbYL5tbD32WGquH8d+K6d/iv7l9YThQLGbCYxFyxyEC0iDASOYL9xjeaKCmN", - "ey7Vqj1KxB6RH5RMH7vZbLkP3ZlyxrAyzjBdDWJq6GDpuc0GE9pHRHf6eEpgeIQL8vx7ssPeG2WLlpAZ", - "aD6Qbe6XlL2PGIs10uRudcD7o/AQoePJfNpllBvKz7zAP2hColwbmfq9P3lKdrCc3ZwJ2AsQ9WcoyWZK", - "LnnM4toYIXjfrup+y4Je1e4KQkVRi9ArF3ZwdyLDdLmQ5r/zrM4WipCYKRcUB7e1wEv9TFlAAfge5cIH", - "4Lg98qP4eoU5zW/HKztAiVha1S2ikdLCTe9+vea+5Guumgzl77TabefDczYbr7vlR3VMW7qJIhRF7tzt", - "mq3ffD4pPVx/kdk8znS+LBTSNrP550WCo9u7H27bXP7mC04BBWP5srFstgO1DBPMzxjTHbMlS2SWYol7", - "bNvr93KV9I57C2Oy4709jP1eSG2Ojx49OOx9ePvh/x8AJoyqeVL+AQA=", + "On/NA0ydJ5wnObUWgL4RTbZkS7InrqmayGzcsbCwsC6/9UcvkmkmBRNG947/6OlowVKKfz42hkaLNzLJ", + "U/aS/ZYzbeDnTMmMKcMZFkplLswko2YB/4qZjhTPDJeid9w7o2ZB3i2YYmSJrRC9kHkSkykjWI/FvX6P", + "vadplrDecW8vFWYvpob2+j2zyuAnbRQX896Hfk8xGkuRrGw3M5onpnc8o4lm/Ua3p9A0oZpAlQHWKdqb", + "SpkwKnofsMXfcq5Y3Dv+tTqNt0VhOf07iwx0/jg38txQEU9XZzLh0Wp9sj9zkb/H3gjNjUyp4RHRtg7J", + "sBKZUs1iIgWhkeFLRriYylzE5NWTMxJJIVgEjemxkFPN1JLFZKZkSsyCkYXUBssYRaNLYug0YcOx6PUb", + "+8EEfIm3r9JfF8wsmAoMlmviWiEzqYhZcE24gK8RG1Y3zKicra9sv8fjhE0MT5nMzfpC/SjfkUSKOU7L", + "t0vSXBuyoEtGfmdKkt9ymvDZiot5+yJN2UwqRn5cZSylgmQJjZgm3BAujPSzsWtU0ti9NERcfC6kYpOY", + "acMFhfYnmVRGr4/+Bf5BE1Ipi0PD8sQsqPFULqQhl4xl9YnSd/Syvoy/Hhz0H41Go7f9HjcstceKvudp", + "nvaO79+7d3iv30u5sP/eL0bPhWFzpnofil+oUnRVmY6WuYrYJOKx2jSTKOFMGPLk5OnLa06gtz8a4v/2", + "Hvb6vf1HB8P9+w/x3/v3e9VprS18feQfNh+9c0NNrtd5kD1NE0cokwqRrM/6lzydMkXkjES5UkyYZEXw", + "SLG4A9HVpj0KbUUkxYzPc8Xi9b79kast54JqQoVlGoMGvygb63TuImBisXwnJoqllAtY47VBvPSfCJxQ", + "4g4RDCmSwiiZJMAUjGFpZrQ/RX1g44LQLEt4hKyndqiO0pHu9XsiTxI6XRthudss4XOOBTotDdeVTfJ1", + "iZGECcNUccK7LE2NLbZ1XC53cDdKvtipy4VM4kkuDE/We30NP7s1rfbENVmwJCZyNusTPiOUQCvws6Xx", + "+rIfjA7uD0ZHg9H9V/sPjkePjkf3/tbr92ZSpdT0jnsxNWwAu9xlb5Bpay6i8O6IJomkcCEpFtndKS6s", + "2gZOWSRTRqDpjSM/+viRt143T8pNhYLEFSxv1sBWDxtXRjfqTqg2BRPC/eJmNaGBMb3iKdOGphnwIRhD", + "ZTHbuJBvsLkPfuU3LvD+Ry2wYO/NxK1QcD4h+mDvMxbBjSg9NykEDGjPkXfBsm6FxhWjWgorJsFl+msv", + "FzrP4Opm8SRLqIF2e/2eJYNJyrWGqsUPMdeWj/R7nsgnQpqJyoWwBQUz76S6rJZ0rUx41uv3FlRPlvMs", + "7/U3XVt1osYuWEIzje25HVcTppRUPSsaryYzqfwm9d5Wl3BDU2srpIsrNrBCvX6vtgAFO/dz8eMudjU4", + "uH7P/TVR9lWBzdnJrA+82tb6cIuhbWbs9haxQrTfZuIq6zoHiDmdC6kNj3QnNo/CA2xvKuMA63xaNEd4", + "zIThM86Uk6sZUXA/pIz4Rgg0QrgguW6cg0L0n7AlE0ZPlkcTE2Xri9J42FQ3ryKblDdi5VYutr84KVuI", + "tD734MNpSTmeyadsySO2Lru5rZnEii+ZCrDvQgCwrNCWIztw1oGFCCnYbm2lxJLHnHZhBzGOacID1HP2", + "5ITYz+TkKdlZsPf1Tg4eTB/22psUNA3Qwo95SsUADgQMy7ePZatt/3wUapnLNM0ncyXzbL3lkxenp68J", + "fiQCJdxqiw8PQpJqFvEJjWPFtA7P33+sjm00Go2O6cHxaDQchUa5ZCKWqnVJ7efwku6PYrahyU5L6tpf", + "W9Jf3pw8PXlMnkiVSYVvtq0Hp7o81XlVyaa+KyH6/z7nSbxO9VP4mbWsE1Oaa8OEId/bYkQx+5Qj7xZS", + "MxLRaMHI1D5a8HmO7XUh+aJjf3uFRvCU60winydvTktpDp+G7D2LclPr91sSc224iJxQVZlchxGB+ilh", + "cP+GJAtcAeLKcCmI8aLTtYWASDG6pTso0amz9fOfW/KapLqtdV+EcEFSniRcs0iKWFf74MLcP2qfTOUU", + "22tzratn8DNJmdZ0zsgOqqXwuWE5POGazChPWLzbTcJum8zf5bRyr9XOHJLBgE6j/YPDIENL6ZxNYj53", + "esUmDcLvIB9DO4bwtHUiKGF0mwd2qdhsvb8f8D7BThSbMcVE9NHdZUoumaDuSfUf2G/v/9orFa57Ttu6", + "h4t5Vhb/0O/9lrOcTTKpuR3hGptwX4CMcKkJ1giPGT/Fu50oShuqNp8PLPEJTmIpbG5dG6f6AXmLzrdW", + "eQVlmgwd2VEh4FS4QCvffgaSVkBkkcIwEVidn+WcJFzYZxATxu0FCnurjH2XyPlu75OtQ7H864cfxn0N", + "5mV/aGkNvvWLV0Ei59XVXDCqzJTVFrPlXnUNlaNrXf6z2vFpXKBUs8lmDnLGhWAx6tzdwbYlQbYOvn3w", + "FF1yM1nCDRw6czisn7ghrkRrU4mMLmc8YZMF1Qs7YhrH3Cpcz2ozCYiQNWMGRSWBbxBFG3xUn//4+ODe", + "feI6CKyh0/5CgfWZVGpD87YsMVRNaZIEaaOd3K5+R69TSJgCSoVv291TUKAnTMvpem433eM91wv7F/Ju", + "GBXefb1+LwLySuDvNcLt994PoNXBkipcdWi+Mqz/9j1Vfvu+7LTy61nRf+XHl24olZ9+8KOq/PakHKBf", + "FabWz4N7puGkw1Kl5MIMuMAtAAHcaZ6mdvOKV5bTZBApKhIeU13Y+SbB6smaSEV2Xv7w5PDw8NFud+GK", + "68uJ5r+zyXy6UW52g3aCMlQjUI1wQeZ8Tqcrw/SQPKFCSAPGyGhBxZzFhM4M1HJjrT3A7422GWFahSSm", + "2kQk8/sRffTw/XtqHt3n7/Sj39Opmv/9kPbatIu5bllf1MBRt5tQlCgqApt4vYs6pe/h5W+pJGT8PLXm", + "KlIUcnugm0MYkvJBP+PvYc0N2a+tdHBxw6+/woQlpBjkgv+W425nCV2tvwENo+kAKSK0vJZv2AOktx8f", + "DefH1vEzfUe5gZNT2EzdlPtEJjHcOzOutOl1t4ZdSTBg6lOJSLVTFpaY1imiuYCtl3oxztazkvAZi1ZR", + "wtaZu2ffmcqdujVmcOfgn4UCsSsbZyrAif2PZ0UXtZ+flv3Vfn9mO//Q7z2hGZ3yhHvOXOfTziY/cYq/", + "bbtUbeyprfrS1fzQ780YNbligbU8R/8AYvXEg4QtWUJcaSDeY/L47IToXM3QZI7UasuC2l5qpsdCsTlV", + "ccI0kvq7BY8WhboSVA+aUFG6BZDHyTu60iRTTDNhjsdiXGjI9bjXJ2P7+HJ/Wyqxf1vXEF9IzBXT7h+g", + "axj3+tBWIud63BuSH6U2A3guO1nqmIydPkaPe2TH/kkyqrVZKJnPF2ORUkHnLGXC9Ak6aaAaGDhTstol", + "VMRk3FNSM2PogKV5grwfGnucZQkbi3Oe8EgKktLoxbmr+46bBXlpK1UuT5wxXNZ9kjnVs6YpGwtq1aCw", + "kyuSKTllJKG5iBZMEyZmUkVsd0jOmBr4JfZ7S6hisBlWiUtyAUeE0XIz9K9vCRNGOU+H7swFZnIV+oOl", + "Lx7R+io1T2yND4V15CqVf3FVgFW5GQffKGpVatMr9GxvxELV76+jmiWgPxa4orirHEq9E6S2ZbOEzpFU", + "KofIEqMei0imKT70XC9EsyVTNCl2CO44ifoEuBvcjTQWUSLzeLBYZUwtuZaqDzcEi9AZQfXJbyzN+9gp", + "/DVIeaTkMiVSjEUCZLxH0/j+0e6QPCYJ10Acfv7cErcjMeQEqA/gMNpxj3qd/LhnJ8Y1gYt/SJ4Jo7il", + "OaItwU3tTdogrq6bV2FWa3cb7s9VGju3NdYsHPZnR9D9NRZboZuSAAsyrnDR0J21gQGvy+B+XTcbo9z4", + "it2KqHA7VaPOY8INoVnGqNKEi7EoqUnESKWuUqYYLofmxm1dygzZSalB9gJa27FADvGNJrXd3x0SlBut", + "f1jJkaAKUSxZ2QGxsfBjfseTBBWHJEcPBuRw9qChZqziQYGHiYqC9yWltcsZv5xdtDaB2hGxff/3s9PX", + "RK+0YSmZckHVapdwXfJaS5rrhrmwyOh2D1/uKKrVZx1L9HqCukSKugDZPK9bTQhOBC3pYhuF/ShDLpZU", + "RYuQG5025MnZawKfuWEREHFtvFSl94OmJKlbmoNeKQqxdr1rzSHT2TpnCUcKR7xtsifFPRIWkQpz/Lq1", + "C2pW7LgsYRHwKutPY99vjDjjMpEpsnTB1mez17pEvnG9rXdNdqTegwnvVm4WONMqF0PyQhAUI0hNihgL", + "LMpFlOQx06TCz0mpvvXyRc0NqiD7sfjX//v/WVkiJktOC2mDvOHKgA/J7/YVO1M0ZcD06ldatGDR5Vg0", + "xRACrWpJqL9UimXmunqhKD5fGCLku+FYnNgxFQPe0XJm3lHF8gyemWQwcKMeODlr1zpxYUPcNFw0f23s", + "TWVxruhLWDPnFRvaX6ewbaT6Sym21Gl1Tg17RwMP4udAeoMl1+iyhjThypKTsyHBz2hKiBb2K3BRkGLJ", + "jqUKLuZjgV+cSLxLnERrycyZJofkRcpNQfxCFteLu+nGAhwMp4wJtCImS+uAXJLqijU3ALw598HAO9wP", + "HY05jH1i5GSeB/Wu/q6LuWKRIW9OB0YO3pwSo+hsxiOgo4wpN2p7wTQHHeTn4GOQtKy0r+hdKJIhPDIU", + "j+cg5MByOdHf/mYvp4wpGNerx2dwweHqf0vGPUGNrVNy+gFYcnjMYvLL41dwtUYvznfdqrmnqW24B9dO", + "SAne7+l8KphpG7/9ij644a1AV9ptzNcu0doWbSPvj5No4OK3DXyzQSYpORhIJyDEFiQ4JD/X5FdboXBc", + "ARF0XaIvWvCH+FtSjPY7FGhIyqjQhDpZvzIiwvVYeOmjJnCApE3cikJVlQvLpex9OKhwXxRLrDyCElnh", + "pL+E/x8stYwunYuHfWUWnFbQFE93denWJCtuOgk5W1UAlQdLYEl97zXBE94IWtBML6TBV7X/x55i2kjF", + "duFx7vyl4HtGc83IX0jKUqlWY+HL9+05S6ixPgxF3Rkwph7ZiRIpWFWPYB0I6FhoI7OMxc4w8S2ZucNN", + "CydCqQpdtS00Fq4LvWclAE2KWeAGlBaYZFVucmUmu6hqwNkU04I556kd9kKaLMnnAzfPHtlJ+JK5acME", + "+e/MtgFWFCnFwPlDQtFSd4KfiKFqzoxTPWtsH2lm3BuLHTy3BP8NhpU0FzzC2xzLzbN8UFVx9MjO87PX", + "e6XP0lhUPmMVUOoNuBwkPOXA3nbgB3Ky94IouKHxZy7mdvRY2E6nmCSWd1Mckjc0yd05RWqH9R0LL/3j", + "SAdG5WYxy5NjsvzdCWH1/Sh2spCK9g/7TtOUoIapVZJ5c0o0XTJPkFgLntTawDGi8ZIpw7XTxl6SHUdO", + "A2fnQrrTRDAWEyHHotoWHGQkUxhiU+QnVhGjSWORxqIU2xqe5P5q8ybDQhXhBZ/uGpvNL5oWG8PHv1qu", + "8EQ+Lx71jYsk4+1WVdBF4kGhUblMlfcrIy8yJqBULKM8ZcIQLmbSF617kY2GB22eaS3dn1evlaJ7vZDK", + "kDk3RDEQ46Rw3GzcG8RcGTj+Op/N+Ht8ReaoBbIyDbak+8Chxr1cXAr5Tox79WE+uE8PZvH0cOtu+GH3", + "a2sY3Adkek4r3Rqvt9mG9mSjwQxETWIkvJ2rL39H4Z/AXFaGJMGlpwZwVhLO4gpt96237pwJBpwrtj6g", + "hngJrcXkMwiKsjduVbqSJaZlR63nbOuGbpnDPJFTvPD8REqn0yE5qa9dn1D8gAJzfZWLaImKYyjZYcP5", + "EG6XLOID8Awd0IPBaDQYNak9ORpYl/eMGsMUDPD/+ZUOfn88+Nto8Oht+edkOHj7l/8IPsg7eqt6u7ab", + "544/S33iB1t1YW0OdJt767Uta9Xhtx9eVC207nSkGB4DmmwdA3gcPKkUb6f1JyfrXm92pWIJeughl3sJ", + "nyqqVntizsX7YxDntKmv2+aymxQsgeAYKxj5AoRq4jQsv+4tqeJUmLd2jGRcVQ6A/Ebw1oB7/ymOiAwG", + "vh3Hvrznfc0AMCS/SEOos0jDk+4YC633O1c0TSlqO5c04TEeDmdm0zxmVjkr8BtZWilJMZMrQY5GI/9l", + "UhkUyA1S+6mMe05xqqIFcQohRVAVYiNkIpZBl0bGtBGmVFmK9ZMGx+wve8VfO/7P4du/7P7v//ikpI6E", + "toHGrTrjivys4ecNhchOIt8xFVHNSMJgtrpPYj7nRlvhLUbXKFSofUsi6+xhXRelIkx4FTWWq5NzuhrQ", + "jA+c5sWau39mYm4WveP7h8HF3XF/DN7+p/+pZWFVnoTeay9ljppX/FwNvPZj6GR88aubJ2h0Sbk4sdX2", + "18XJj9thP5FNO20fda1bDXdMERW0ZSDrEfjgcZSGhIglUwqOouVrT06fwhPmkjmGBq9GMs5Ho8MIC+Cf", + "zP0CQhw8ZfC33brEY5sLCTu/9li0kOgGmSTyKkrKfpOpt3kJbrSRhVa75P7rPoVoRLdmcXR2dwXJVMlL", + "ptAkA0vMmSaXbGXVBfOaQpOJJVlS5U2DrzAsAYv4T2gNYvg8hkcsBudDLCtTji26N5sNg2I0KbllSsHc", + "he9KHcmMAUN1xXBq5IKJ5QVJaWaf9YpZpXlKDVMcnoulYjFlMQcePRYMDwbJKJz5KJIKVKywt6iDLVfh", + "G00u7DPxApu/4ALI+sIezIaq9I/ei9evvn/x+penkxdnz355fDL56dn/wM+2Uu/41z96FkyjcLz6nlHF", + "FPmPP3C+H6zzLPr0Qaj7Qir32vUmevRXoRkfyowJyoeRTHv93n9W//n2w1vv7mkj95a94+DAPgTFTKf7", + "DPjg+wAoTaQqPU8p4oIgi/Jqh4q6oaHKt5LflY4EPlG4nEwzHYxNaeotSjl0fzQ6/X7PeY/c8//YHRJn", + "usXhAw+SyonHegHkU+ByoD0tSWTkIjRmbSH4vqsQg0dbaya5MFuZU1l0nUcNBuXXK7CivSkXexq2YRBd", + "bd2Rbq7tqPxMLLmSAp/nKC5NE6brZ+WXF0+fTZ798qZ3DBdBnGMcYa/fO3vx8lXvuHc4Go16IQJl7zOu", + "mA46Pj6eapnkhhEsVPqYDslprtG50zGPWY56UJTPNElzUCwlK8LeR0mugU+hNGBMEoo/fjg42H+1//B4", + "NDoejf7W2WEVSH8L835+9tpGqON5ZzQxiwma5rZV/BHLPsGiH/peN4lP+8AqFXvqtZVwel0dsrOoS1d2", + "cgQJctw7fP69PVD7z8e9CiG6UOCiFdtwY/Wefx9al4o2al3SK77ZCDcRe6pfB6Ox+vpcOzVR1RI+EBQh", + "dqBkTFVMppKq+Nu6Lw3WBEX+N5qgtoImzk5kRW+sY2VJyTS+lb2PT6ls1f3GEqDmJ7WqRCoIQ0vpkgPL", + "GaQpl75enS05ZUpNhzqsGJgCeryKuxA4YLI0d//xE+z1e8vf6zHV29WB1e1pV9oVCobCRFEYE1ydIXnl", + "oxRFXLyWdN0KjXpgpxXGEAJquP20rm7yTYX4Xm9/f7g/PBxAaP1wvy6x7x88bIsF6/Qg3vLSpUnGBdvw", + "1P1MXjSgOE8kjQf7n/hBU3Hqa2Dt2A91mi5e4P4Ur0FYTamI3/HYLCYQ9g9DDogB7gspCheywHuYCU3+", + "9Y9/vjkttT77z6eZEwz2D+59pGDQID1oOhhQVEwkz8LTeJ2FJ/Hm9F//+Kefyd1OwsrN13qDuP1/Zlto", + "ovA4cDIbctBiXC6EzTIIWbY4DGwBagjJFXLJVEJXlTvTjam3P8KLqzEqxa1N1dWr6Mk336DQmpdJq3co", + "3GUzzhLriGb9i/ERpb0ZzBpI8bQAT/pG+zArRqZSGjsCWBtG1wBgwlfvx+ndhu56jKQwlAum3HiLa9He", + "m/ZZlt4/GpJn7zOmOEiFNCExVe+4sJ49vqZTLBAqAPUo4YINeOrM1HZJ7CSRJ0PDYE30XlXaelWt3xWo", + "kY+JEwJsZakaWr+bVJ8phox5khVwgZuOz0tbutRsBEgyQJHfU12Yn7vQYUGG+wen7s8DS5BV4isOlz1r", + "XBXEZ7v6FmTthEfcYKfaYvdNGYGeGNWGHJHn/HvCBeAVJIyc8u8JdbcTF+TN3+w+IK/TnQRGfckz68wy", + "ofMi9nYTvOH5Jc+c0gJreDnF3kFxDi3jGRqOhYUvAraBtANhB95AbSBSQlvvDLht8T5bF0RBBVLeaLa4", + "tUWrXPTJNDdEsVQaRpyGyeEowFiw8JSRvPSUbWgZ3ATXvT9wWS6ZEiyZWC2C7rgythJxlVoXB6c6o9rB", + "rSmTZ/X1evrT6TnZeboSNOUR+cm2eirjHFweLU/Yra9efywyhRg20AmcBe76lTMiczOQs4FRjPkhpthY", + "YWxySB7L52evHRaMBk+slwwWlonYefV6Yadw6xXfGGvvrzVb7b+x6G14P15a7Xiyz13x8mh3V7v2e8Yk", + "IQS/xLI1iJSywSJgnSuoz8cv9uEEPpcl4ITlBkPyWgM3GIFvj1TomkGm0iyqT2i8lnT1mVfihJbF4JEd", + "fkeXT/bG+T5aBI3zUZbXSfeg3wrcuLReISDO6K6cawmFK4wLUaUCnOugI6MKIuq4SKb1HbMYulXlmR1g", + "uWfU1FFHuur7bcuIeLvVBdap7XFJuqvtK4rkNQW+13L+0TEyZHP7J8IPZKPZolR0fkRf57aR5hK5tvt+", + "ZtdYpZNiTRrGjk+zPI91RTHcCZzWyjxWntFk5wJ0yY6OQXt80ScX/1n7Afipl6fgufCO2NVAHi3gp2r7", + "es19e4uy+tou3FRffz8e61ZoMbLcJ0ZRoW0wwYJmbEh+xIuRGJaCtAg3Fa/6Twr57lsi7RvFVx0LGJqu", + "O5R6k4XmcwH+dRgZAAwnjq1dw6klvZORDmE6e2XjmsRuR+eQmZCXuZACcuHtCxd1/rtufVjX/jhzxJrC", + "An+3igrUV5m9NDfQvX29OMO7zI1FanNTbwh4dRvHNu8nN5ZC33qN/T8v2EUTKHsZ0FjA5OSsNEpVrGNt", + "Rign/IUNZJdshVvujWF0zRxWtYOFrVXOb9+JMpWqqBGzVwnXFSOaNYc5Cxh6WNePaNA2tG0rYL06L3/9", + "5R8MNhr4yZYU4x7zHoTVcyGYnO0PNaua4eKjJuHYxq5d9K3qg6ESlQgglsQHIDSbR99r9NS8cL8MXWsX", + "RSTkpwBWL0NcqltLKjvrRWlsxotihsX9umwAENx6+6SsD+6ixbar2DvFPSPzKJ4dRV4XGpS6h9fH6YGe", + "VRrb4PXWtYn1UGC7vpUxO3qy6NU+UsbuBxr5arDOzewGceMZbuFx6l1e0CS5IDuu0C5R7O8Ifev2CtwI", + "i+0HKF9HAoVP3ZvT/lhYLnCxMCabwP/pCZzii2Zjrq4/4SXy/MMRiq5HR4f1wBQ74EazdZtAEPKnfWvA", + "ueyGnNRalPIe4h8/FzcQwPqQHUPncHQcthPcE7sfq7DnQrMoV2z7WX+M4pHDF8IxJZSLwY+vXp0RxeZc", + "Yzx1MHLD+v+3ANE5ZUHpk+ena2Rda3tweHTv/oOHj4bxpRqySA1zPYCHzGB/SFP6uxT0nQaxay9d0Sw7", + "Xu5v5fN2C4rxtbN8/7JtJwWZAkv29qMur+QnZZXSKnrJRdy1gZ+g7HbP3sLkdMNml0yxQZ7NFY3ZJza6", + "XNt5C1ezfVe3pHvZ7LWdawOKiBLneGezs3Z9sZYyGcTU0Kv4atvhrrtlpyvblH2Sh9pr9YE/bzq+18CJ", + "gkkfPtaTzo8ltC1twMlWkcDiiZGBGGUvO5w8hTX3ZTshxCLOs5GT5YzLbUgGMLR6fDJW9+oNaGKQRdxp", + "6rxVgxdaf6use3NadSAajsUApbBj8rTooGi2aJLiE4PG1pVhR6pakDQaAqaA70LenFpLtB3tN9obBeyY", + "SBEem6NBEYWigRXJqgPIMZqIm2Z1p5a0oNO76Ccl3bdhkZoGFaCFAg31ZVPemI8NXseNcndJFV+no0J4", + "EwTbS3sfbYJhq7m93BuM9gf7917tj67q9nIDuNqhth7XeYsLcKhynyevT54eOLXn7rWR1z458naYlT0t", + "IzPITq5ZJepYqmA8RiXsoSXe4tphFDcUFVFijG4qa1fiFZS8CUjxEOgZFulfA/S7yTC3IstWJreu0Fll", + "qH4oT0lFkLc7mkU8GF4OfpHfK0YvMVXC+r2NYBsTe5+FnSoREWW6QgQwVHIpKc1MkwIop1R+7x89OHp4", + "eP/o4WgUAK1eJ3gZ8QlaZDsNAFxwEroq8Bt30I0hJtNETuuEfu/w/sMHo0f7B13HYQ333daheH/7WmTH", + "rchffDo7/6U2qIODB/cPDw9H9+8fHHUalW2s26Bc2bpI8uDwwdH+w4OjTqsQetc98yDiTQE+lNHisU0C", + "Bf8a6IxFfMYjguB7BCogPMICzPjeCl0/k1Ma+7wf4bvDUJ7ojS6MtjNX0upb0zwxPEuY/YYb0skkgTN/", + "ii0FE6kJUeRZuVpLDnp9q9+Xn0tRBMWZmE3z+dzhGhZLd+ogC0rhibMkPibFk20jn8PdLAf2to0O3Bw6", + "UsPP8l2BMFghAnt1WTgOxUhBJ3bTarPyMVFcZHmQJFqX8odcWaQRbJTQqcyNteC7xDhlJxa0R6BFOxdx", + "N3XHD1Jdbg1igZu4yP/TUWHg0QxmiHbQwDGoCn2Fpd36I7jvmry0NayisPw5y+vJD/vYk1MoCg/BEBd6", + "Y9dMV+kyLLegztw79dn+St55Sx6Ng5l1Avu0L2w1Z5j3ymyVWIBSrP/UORbvHBMHFbcqUjqsu2DvbmPR", + "MapzAGQ7ABXKzaz4Ju/wQtdQFiLSxVQACCnMPK5gaDVO2rmHFXFLPhwLF1RW/OScAqCiXQfrSCAVn/N6", + "x3U96y36Z1+FMj1xXZs6qxXbfAZ1YQYM8QCLqj2znsDWe7EK6uv2pNfvnRcJwhxjqi+N/zGwImUUxTq+", + "0tnrqzogZ0rOeAj0CL2O3Ff3UPOuuT8fjc4H+/9tvTuB/FBi48J6KoEdYdhI8oTlu11Ez89en7WNqciw", + "RaqjW5vTBiin0rOmDP0pFscF1HPtW65kI6aCGHrJQMOEJ65WzGKUsXhInlhfG5Rr+Aw12sYXsxgywIBs", + "30t2XMMDAp8wwYpGrSk1zoGqqmEEUAxEAe2tngiE9/zsdR2U4lFI/EZUmWkOA5ukAX3gD/Cd2AIFKsbp", + "93UR/OAo1HSYcZ/VCAhf+jMagXdAZwoJ6BAb09gGO/n87PVLl1WqDZAbyKnIPGWxuIfklyLvGiyvLkG3", + "hgHlYoMEres5FJjoRIYcWH5QjBH8VuZ7BbpBo7eFcC4cDWiMEF5S+KyT4PvvABV/y6miwnDBYvLmB/0t", + "GTkr+5sfiA2ycjyL66rfZf1x+SC0pa3BkmeLlQbdnl0Wm/mFi6piE7lA55fJWVnRqYAD75NwzkHPccgO", + "hMPgAp6/HJy8eLOXxmzZr40JPlov3ednr3crbHnpETuKsnVuvGzTMPmtKlJgTsq0m82DgGDhTvXZpA7Y", + "mxGZsoh6p+7m5kVFluopumugogzM3IYJoqjTU1NRawRZFzje50kydDzTD5mgZnqGrLsCcGtvN8fUFMso", + "Pr4Kyeu9wU4Z0XkUMRY3PF6ao67Q2zHJRUqVXtCkObnjAkUiWlBFI8MU+eb9N71wmilkpV3vhgp1FpdT", + "Z7KsXEUBeqycu7Yj/uaH+vG00oGNHXc6dS64QSw8BEDfKQZpIduH5JSunOolw2vW0Uu5Y3Q2s4iwxT2i", + "WMKoxvTwmrz5Ybsvp5GGJm1zeAUfHZ/agQnBKYJh9klWYwPwexWKrdrt/eCVUc4nYJnCDptmjiE5EWG2", + "Zx2GEEzR4qJpzecCnoPa+QzBfepXcm3v6ndnUJPUBJ/s1ReuNp1+4AIIUUzotrLBrusvo/V0tfKyzqbk", + "5fYUqbaR9n6f+HjchinMPT/9ucWwXe+8RGxVxzGsQlWzjCpqWLKy75GCOuspL9i6HwpEC1whIPgZFP9g", + "0xzlik3MQjENPmM1BcVhfz1VtsaohCVz2QHtnCrmOiNJStUlnjL//Ca5sCtQD/493IYAtjAmu8KkwKnC", + "6uQMU0uaNCO29JpfzlMGUVpTZt4xJvxUqCa06hXfDOvXLRnaIJaHKS7ra9g7DPR7biMlyFzRiBFbyyfa", + "L9gaHL2uS9k7bBkWXDhat+zv/qb9dVVnedJtj0PD2t+2wya6yga/enLmM40Vmdz9Mh+srzJky7BHztXY", + "srUHenPOO9+VsFDhazL7lGESPBdrVg3V916JmORPYMReJTy+whx0NaDb9YOnwC5V357zt53ehM3jHnK/", + "gUijUEZ8/OCi9OaIqIixCSrHlxePraRk7+TK/ewiqUDS4gLAp+t+31Gukl6/N5i5WR3v7QG/TxBU9uhw", + "/+HeZufvjV7/zslxEvNNWiFXyjnLeZwDi8iJk66TxB7Nsg56c6y57X5A9rS2CTYLPtxtlUeWD3EBe03D", + "efE9jYzP24mK9JqjBK0eW4R+rwsy0GBqU3HBa/dR9XyOgp4r4IZaJ/+9NdqHmVlPZKARq6RsrKMl8t+D", + "0qlUoeSMUhkXrjxl/tFe3IfeEdiDnNdM6qOH1Vnev3fv8N42PoTMpnHM3bkLTNWWrqNOsMpyu/PrGoD/", + "6LrM4b5sJitcly009epJgKQ2L6jMmLjSet47Ojy42np2nciJd+Zs8KUQaBegdKFMVEYWp8zQmBpaZzKo", + "gQYug9m/KEsxcHD27WbW0uL1VEXhum7qwk/lM9OSxfSlB+1PqeAzfCTZktWe9YIe3Lt/bHMtx2x2dO/+", + "cDi8KjbRs+Jbt63Ys661FZiioV583D7cAARRl7n80Tt7/OpHYGS5VvbS2tNTLo4r/y7+WX7AP+w/p1yE", + "oYu6pOfms7W03LXtzXKHDc/iY5iJcMFYXlLq4NXXYkLCmALESwsCgdZ8q28O8bOgcV5P7HIF5M8Nofwg", + "roBGvnVtrp1pWxS6UFPJsF3VK3TItg2un5s8P7zGGcu4Pq2mo0hEvu4Pc61U8npjtt21ZIwZE0V+3SSx", + "f0VSIPR6KNlu7Yr03zrkasRrxCVlLLqs/lj0XvnxSXUgld99rsfKTy7r7tsrOrJtFEj/ui6HbudCXhzd", + "cpjD6v/iVuia4dwhhQYjOO74LryOs2e99xfz//rt/+izB3/f/+3nN2/+Z/n8v57+wv/nTXL24qMAmjaD", + "xN4p0usnA3dFD8cawmtXUjoFy8w6PcHzr2WF3RercTDRAnMykyk7Bhfsn7lhymb4bIQsQ1IKhi8lrEWk", + "wFSMDs1iFyqfWT8EqPyHF4M/NNuIHWyFchtSACXpfBrLlHKxOxZj4doifiIa3wXwV0wimtk0nVyAYhYc", + "uxXmVHWG4bLzPvmDZtmHXcj7Qg1h722+g4xiWspZYZsQ3q3Djco6r7virACVtnHEY1Hc1gVKqvUOGJYp", + "VThLmpF+LYuy+f3mXk4PRyE8WYxCg43EtGH4BCkoG8ioCI8jD0e76++5LW+MgoY2kJ9zn7FhyhBmHCDF", + "ltDmpyzmEfIVH91rS1YCqy2lOSVepuT7Fe7NSxtyGhOamwUTxjlG9kkk5SVnfdzSPgHjLPprYU3rhbOQ", + "2WC6GixkVsCwFLZw68dSf2T/n4Gb6OANU3zmegqiaQCJBIROPDJuZlim1C6sTeyVzbgkNBrlXXGbukvb", + "bE02AMPkSviEQgzTCiPYUEF9LCSTf0uihKPWSS/QhrdASFQDzQTR/UZFoBudRjGbNf9dd0Q6uHcfX7D+", + "34cHnePM7dJtorI8CbxpU8/6OnBsyyZxAFY8mHhF+BbXQbgBLScmqKcwEv97TnxD5YkrGIm19NnIV+3y", + "0yS6EvO6GwxOdMfAgQpNaG66zm79FNZiHTu0YH2KsFrSAc7oGc6DvPr5nBimUo96sRPB7uApsdg1A651", + "7lIlPn5y+mx32AuixNVMWrhVG2Mh64MOIKTYc9DqOlXqaGjK+uTkKca4u2ulfIthUNIPUpHE3orlZXSM", + "cD51dQ+1SVZPnjoBNFmVjkpWbBn3dn2LWfN6OyYvfbeEFkMpopVL2vJNlpcJNuvcVm3E1FrrjeB2tI+5", + "55+7jzE+CjPQWl6McL6t91d3naNbcfjY1JBd+UKqVGzXf1X2/lMD3396Gf3wajK6s0JPsgXVIepeVK2a", + "WGht36uG7Dq7V+2efrV+J5oHz9ZffR6uyjVkGNxzteohpOd7g/39V/tHV1ffXRWzvI7UV8H4LGDLu+ON", + "3wRudwCFm5tJa0gIgc8uAMTrRd6cQnQlIK/Bx4Z2ZP/wQRelBPbaNZii8hFEahxSwaU88l4RBGABEC85", + "gheDBzCfC5qQR2Tn/OT5Tyc//7xLBuTFi9PmVmyqEdyfa8CX4y2A62idowLgazUgD1JE/L569TN8iRKG", + "QVNWDr+8Pqj5VtViB5BzP7jnZ6+hxoLqiXe3bo8wpmWUPnvPtdHruIudohY+BlTdVi0VY10madtwTpVb", + "kdl/rEGHB1E8d28AUt2HnKwt5+eJNv6yDi1epvxeh+LciAr+ecUVf34I5Bsxwz8W+NspOW4I97v1Pg1h", + "ZjdAVO61Xa3XR/C+keHUMKhCPLMqXnnQh2tDZvd7PBDw/tg5EZKTszIRYGkJ8c035vToYLh//yHmst4f", + "dblVUhpt6Pv08ZPunY8O7BV3TKfHUXyM2oLrGswcYdv3D03eQVjw2L9Qxz17W1ee1pVja8t0C/hYRya/", + "HhB5U4ZswQRHWdp7TelJujEVVYeI5ibOYpo7xNqUJwnXLJIi1nUBfUE10ZkFarYpmYrnw1jgAPukyIyP", + "IhKhUaTyUu/pRHv32MgzR/eYFjqTQjOXF+YnttIk5WiBLbpHv0tNisC5eCx2lA+yLKIpMSN0DD9gyFKf", + "uHdYH12WMfUUVBgLvcgNMLFdiH4RkEBaOZUwmXI0V+0Sndv3NI4XV2MFDFPzmKmxgGIBKOg/irfR8X2A", + "Zej3imckpM8ZjULUdKOW16HDfMcWMP6GCpewEF3tVS5ILmKmyEupmTGUMEsOTf+8K1ptPxJg3VfvJtu5", + "6qVQFz6Y2/DYO0lyHw0wjUNtUQ6ga+o1NAP3rv8+6BSm6IVnF6Doak2u4j7h4kHguTmF285qA1nsdKCa", + "uUCTIvDntU3NXJ+683s2Eiz6akXenJ7WfC5ceEi3iSOXaNkHmV1pGw62KGi2juYqtu0KHPd2CO6Ph8Vu", + "SioVCfGTg2BXzZseN8FSaE2rVnu1b1LuwZM2GIDFhd0nIJoNE2xk57YoM6VXZrVVvdIzBwnrRGkf0OHg", + "8AvEmNKjQK/03jTXe1nE9+x3vYdwPg8RzucoiLcQs+Ukz0NPI/jk+iGvXwPmDv6CcNQ2OqfWPaX39x+O", + "Hj4aPJzu3x8cxaP9Ad0/vD84uEdHs8PoweH+weGGiKYOAbjXj6kNPtcDTtSFy/zEu+6HPKrbAicasolz", + "Bn/HRSzf1a6/oHdutXfn+but+3W//s5DCMYjJVQbqztp4WSneMmzCNrGcj7Iu8jEF9ay3n812v9Y1RMO", + "ruWOeKVyYW26WKC0X6SVAVc3qz7O67F8HJCPutm2WtXOuy/a6Pjeo+N7H7toPnJk2xib5HSLm9vmjuZh", + "2xuhKT5CuaLE8trRnpOJrEnBFl/1+r1cVP9GYaDhyF187hRB1nZg+2E2sulaacFbOKm9V9BNxsJ2xscg", + "qfj3CCacKVA9QAR6Aoo4UlEEWhRDNAKeVN4u0Aza5Jx+0EZN20gQDMvk2gUTcgGMGI2f0IgDZzgmz7Es", + "fqKpfda5QdjU6FW7H41X1lkHzpfv2j6yNg/53L2vsA48tgj8C6cNy+D0xZubsNLZMflFYp3itSdkU/Fs", + "i+Mza714oyzZcQj/HmwHO3Oi5jH5oRAvCwHVCaQ7mrk/J45hlRhXuzWkEbfj4MVY2bkKaka/Z1e01+/5", + "hYI/Azgbr0uqXzt/VVIMebExmuBZLjECcsMTh+yPM+Ha8Ei7yBXY3DaxxwWGsnhiH09tDrE27taWIUUl", + "L1W9OSU7CN76F+K06vCv3cJ5tnbXHTw6enT/wcGj+50g2soBbheNnyAswvrgtsrJUZZPnG6kbeqQwQ/X", + "PLJahcLxBkxtJZlnSgLrgZn7BqudPxo+qiLTxTIHPlwMycFYWiBs3LAg+GLBi1qcMH/jyZLPZuK336PL", + "g78rnu6/v68PpvstiNu2o7Da7aTq2bCmo2bTgU2eGgYPQ4JSuhVf7yXTOANyzgxB+hkQGuHzpgjmdiSX", + "69qKBwnr6PDw8OGDewed6MqNrnJwJqiEC1zKbgRlSYIlyc7L83OyVyE426aHockCGAuNc0ZcCvxRXSAd", + "7o8OQ1TScnGXVOPaXqatS/7GPR/dpNyiY+R48bRcO+XB1T48HD04uvfwXrdj7NTDE/V+M4dx5Wx/PndH", + "ded3UJp89fiMQOtqRiO2C/BTqA8etaZzzDUjvzx+ZXGlhCQZU4PyRvQt1WNsnaPbleZnrjQ/zGBjM09s", + "mWLNTebBfQgd2+8GORnynXFgqrWjX+eCgeMbIK/AvgaWYp2J99vunZBgZkn1JYsSytPHkY8UadxjNsPE", + "RNli5SZ0uWLcW37tCuxQt5OarFB+2TgjrEOkIrko8hoNt1uWP4mBuJ3/24tm+/0Qii4SsHoOG83mirzG", + "UmaKLbnM9SdoSBob+jtLpFRXqtv29HnJNFjwPCLXm9NvNHG0RrRhWf015qhxA4LcNSd3pfNcI5Ewkbct", + "Vqfd6LL1mybcbzm1/U24IDVu0IrbGAPnysV2H9YnNIlyzORFi/2EWeFFgcgGWZasbIhCkkgpSLSgAs0t", + "yuUNFHNCCUBADIMOvfBlMgs6h8h3JJEWcR4yDbkkV3YQUA2EIdAs7VRTSVpSauSevpdaJuPSGNWp8V4a", + "TpMbhoYqIvphPamRFTB0W6WmU03kXOPz0mDwxbCZgyOjysZUUGGTti1T+yoN+I0Hhthg5qEbFT/CwbRP", + "ZSe8YLy8XUkaKakrqGZvTvWwa+heEYy93S+1PtgOpGtNo4GrDK843Tm3Y+h+DIQlfcwNiTSMoY8bPD69", + "mjOlIse0VxVCdir9YWevTtBrTwq0risOVpsJ5rIBVWBhAyvAAwrNki8TvBc9a7vOcjn36WvVXqOqcFNt", + "A2znqcEVDa9Wv6DBEBmvw+NtROQrcQqbmFxXQcosk59wja3yCgAi2cHQnQpbqgBZ7nZxtwk/fqGftXev", + "S4MMFo+uAJGb8SDPqFmciJlc5wRXscF6nbZzus2Y8hiNMROcxf5VWhhjndKMa5cEL86ZWznstobih84S", + "FF9OTtsGkUp1Xt/ssIui2Y5hc6ob7NcV7LCTXIfjgl+pHNcKCQiIp4wQ7uTLyvUkbBhbb1ixeZ5QhbiF", + "HYesV2nCxWWX1vUqncqERwQqNC3sMwnxaxP4pL/Duex2mh1UmLTlRTu3g3PhjXZDGv2WU/gOZrnbCK5G", + "nc6erb8H9Tv5oQUd3n/gCXMQm68Ff18h9HouiKODUVvQf0ujtXD/dYzZq3JuR7LBE5/rQIjkRinHZ3Rj", + "sUvQgaUxMx0etQ63kkeA9sbE69mG6vEvH4ew8sTy6wa+CpkyDF/yU1trrgtb7DKVYF4bSPX3dzmtq1a7", + "OhAHsiXuiBLpQ7FZMEwBd3SjatuWWFuTyu5eBUoD2SpMFCtdEaFiW17J0lOrjZ+8XEuxuGBuybifo023", + "2CF9kPckKVAYXK/d4RiayTADns8ItqPNCtNKYy6vVSXDK+RIUwrh70HCkcLPBtFjpLJr7eG7IKh8RRRL", + "KRdjwUWhbkVMOEYEWzJVCfaVCh5Zc4g0+WvliQe/szQzK5eJAtXw32gCxjg/xrGoDhIaBzX6kDwWVrOo", + "8szUctWy2L36gFB8hn7FjMLkrdwsyEwxvajOPZSwF2S8d1LFrZnQVsQXgWlQtNYSIy+ZqLKyopng09A2", + "NLG11v0BbbZt/Oren6SWAJs0E1xv7i/XTIWFxGJKRZFOTjCVo1J5nOCBhAG4xKbuL8viCzCXDtAtZfP/", + "7ZssfzorGq//1ihWgWfxgOWPrdo2qIKNsnx9dRo2ua1OLxjNtwmUZ90qQXa8M7bPDqV3W7I0dbonuwUU", + "NsMO/Gj2NIvqvR89vPfgfsc0WR9l9rMgZJ/ayLdMNxj3WnbqtIsF6eG9h48eHR7de3RwJQuLj1Bp2Z+2", + "KJXq/gCGiYHDmvzrH/98c1qnloN76M09utKg8qx9SK+zDgN6c/qvf/zTj+raAwoxmvVcAy0eAK3+Pkl1", + "J73LQd0Y2M1ItuF9/7imJKAFmyE7DBHFwX3HrtugHEwDzaSbFEwzGnGzCjBy+s5n13BFGpDhHVpvDDYk", + "8tq2HbgrcC6dT4sS4MPiCvyntTI3aOFh52x7Op+2WbRfNHvFcqS0WlS9JTo4K1iKCCu4i/m8o7rmGh4z", + "a3foF7Fq63E7tsRm8OBmUAUMBfGAa21FoZQJDXnSVapuf2M7K3bLmlqnueJvN5zD9iN4Ja1v4EYOKH2j", + "LO/aUCm3wwV4vVqTaTUP5sZEo7WkmeWte/V+OwRBr2dcKW6wq/dXCb28SsUGrVl6dGNwS1623a+RRAs1", + "VaJqAgo0mbBBJQcDliY6txZBOPMOLT8QCxpdytmsDtl7rx3iHdGLMGzM90KNgZdJH7T+TmfRxAe3UEXj", + "3j097hGpICQzHfcaZqtgIGZK309cB3WImtEmzPUyir8xSO1nME1kdGlTPBrFmR6SEUkZFZrkAg9/w6q2", + "P9psHer3ssre+HXEx93aUrrNJFO2oEuOaWGcTWVec+lk77nR6HqK7RyTWFrQqlp+azdDKGbDJI/LSeOl", + "Q8XKNQwNQjkpvG9sWRYVfDPMqi1+Z0r2HeYCcOwXL0771oEBnRjtwGqekn6idgS9fq/sopElovw97IkM", + "xgocdzPrQLq+jtXQerSsKqaZ0Q6GvCSHBhFA+JIwzXQEabeHXD1ArTV+3rl/wIL63i2BkJhFeCL1+lms", + "E/o1iLsRgeBWOhSCcBgiYTwUG9IIvXQG4eYArLKhkpretlP1ELdWwok2MuvVT/WEvY8Yi5u4peEiXb3u", + "Xc2g1/3P1EEdFVnjXWn0nF6f3fDmQsVwrG2rXY0OEFIM4NQVW2oPp4XN85A7dUKrrn4VmWMSQokNFegw", + "fsHeb17rX9h7gzDvcZ6weH1tiyNpWZW7jLat+LVjJNsOtFRsa27QG8iZaT3Xr5U101Zld5E405W6kWSZ", + "a7tzzowve+7IqHWH6vlqaiYt1wTxReo+NpaU+sRd8GQ/3W2Q4NEirBVx2MIdoz1B3wLG9xl/v4F4bAH7", + "MK4DpJQHyVMDsTCpOyl9T44eYAYz3Ri74POFSVZ1B5yjACTUR2WUVcww4RWFXXa+3E1fcd3bzW1ntfWQ", + "cHxeQThay8ziRNLJJvjvJ0Uxr53P6Aq1OK1GwgeHR6PR4cHoWvjfflhXWK4nZRWXf7XeTltwXqUeYVWL", + "meVA1RaKcO31pN426165mkQbxWh6jCE8GY0YSdgMsf6K0NjtlsVm15sH7wQqF49b0L/fKLdv3gZfz/xT", + "dCXmlX9Z1QfOtQ6lVP2+xSDawmaiNWTAQPTe4WB0/9X+4fG9+8f7+zeB2V0sUlvcyIPf9989SA7o7Ch5", + "uHrw2/7iwfwgPQy+wy65iLvS6k9QttXLprwk66hINZZGdtwcMqaa2drrWiOAkk64YANdxFptD3jcwAus", + "/X3r+b+ant/OYKPscF6fZFWEoIboIGXdDYyYm8xG20VzNidPN8/iWrFMzYGE6a05FCjVcTCYaGO/95EY", + "D7noeA29rhTsfBFtjK/bdhWFLOx40oO73LLiIfJuQDxUZr3pAl+/5AK607lU3CzSzbdFUaxAQ0e/6d+1", + "ievIUUNyMhdSsfrPhZtc9REFlXv9XvL7Uf3MuN+7Y4g5IOWCAN1WV6WCDm5kCVuyZPMqYJHy4aGsJztV", + "DBfiu/3B/iP0Q0h+P/puNHhU9zjo29WqLt++L137ddRlDauZDH0GrP1HV/K49uu5iYJ+4qE8fOW97CCW", + "HY2XifH91eFDd2sbXH5e2+MGJlCrAPqxkp673CZVoSkGjWwIYr+iqNWN12OVyMiUzbnQXfS2h6NCcXsP", + "FLfkscM5t9noZdFPtXmglSqd8DRlMacGR6XMhgiGg466uOZb4mopVnytgLQ2DItrj7aDLWwLuNp2TQ4/", + "IrL3o16/3V68m3BAUK/mn6iIRoYF+4TPQAddb9/n3LaPDgyMRH+1Yw/5VpKs4wG6lAO9nqRP5tKQMhi/", + "o74tF+16wWL8kCA034i+YQni4JNAqxRQZHwT+zp5SjIl4zwq40cTHHSJHaJyMewq5G8d2o3qNzDEG5Qc", + "2/UbbQqNbvrJtv1u6CaBYNu3en+0fatvRCnS7+VZvJ2H2ULdONiVAOi3hCAGVDT1ZW9IgpXJvO3A0V9W", + "V3D9zWt1yRGIRHnmDSxAU+uUFDC3oIkh5Nf7lCXMsFAjRCZxGSXBdclFt7PU/fsPF20mTrRIrQ/kJ8Yy", + "eKsgkhL2l4I1LzQwnz21uEt2Rh60XFuD18BmXXKrVR/cg62SWOtWVVW4bZkgLJdv6Lwt8splRf3tZ1MT", + "zdaxVDzDrwlpL9szGbia3u2tVX98E2q5uxTSXjjTQwMA1sMTF4jevv/SFxjEujrxHoXM8yGyeEXnmPJv", + "ix2llVRC6THRyEGJoS5HF+YJhX8STIaqPfSLFxCwuo/odTkTCTeEzmkzAUZbdk1rid7uQ9Ser+YVnW9E", + "922Gw1bV748Hf7PqdjIZHu9995f/e/D2P4Nq94YCQTM1iNkMPa4u2Wpgk0mBsmJYx7bFTBY9bejcnRlG", + "U1SmIWi940rV8d4bFdxz9QtN16aArmqVTFD7Wyf0l/9od/SqLONrvDC2nt2PTvRyEwlxjfT38k7K1Nw7", + "1fuIut3hWGCQ3iVbaVLJLzdd1U7sN7qoUnHFJxeW3IdMLC/IlGPCTj0W8LynUcQyeFa5lEUcx6IksmHF", + "aFJtx+W58+fFWWatYwWET68BI794/er7F69/eTp5cfbsl8cnk5+e/U/vuCfYu4HtIR4A7R3du+9yzVdX", + "cj+Y7+TqaTuG5NTFKzifh1mOL3uEPgMbKYSgJSvC3kdJrsFtApfemOT6CTrWo5avn/DiI0GVjUlC7hkO", + "/BsAvWHM9l513kVce2LkGpP0Oy0PF2RddLGEM+7hlWKklySC6VFgK8KrXW5sfdHvL7rdJRZuNnDYMfQX", + "DltAi/oz1wjc4Z1gKoXJDobQ+EzC9ot1Q70C/O3josGgC+YnThg1evQpkrq+3pjFdSmTAbzzWpJPBNXq", + "di2CIQTYlA3N6LVZX+bTwGPG6bjnfE4DBpeQYeWTJF/1A9oaOra2/61Z6MIBHU+bmTnssbRL1cgk0dCW", + "aDNoj/dIQbxvwTJGsGYbxMsrToa15vdSYfZckuQQcEgsER9+U9h2eco84OQAK22PRt74wKzMrDKS9r05", + "lXloWzYtEMTjA49UrLIRWIHF11wyF6DUAY4G6a2OTuYqo3iOdndNdgoVmF+CIux63S6wOaPFKX1f9AAl", + "4DXRNMTiPMp0XvvPv8fsAy/dLgH7dE3gMBqv3DDWfp2KNq2Jp6r1zahS1fq8bfngwXO8agP3aztbDeIs", + "+6iRZoge/0q5+UEqfBe3g7/cOGQ/Xv4xUwir1wTk74Rmz1MWT2RuNp9/KAbSo73yizS3ZZpkrwOgSMRR", + "La65jRd4eJJyDOsrDcvBolxxswKlkXuZTDEe1OcmxoXEjvDnsmPMB/vhA2rPZ4HImedMMMUjzLZroQYF", + "vpgAtq9Mumjzb65B4KII9OLJidO7eBRlfEdzg6TnHVAfn52AMd4n5eqNhofDER7mjAma8d5x73C4Pxz1", + "8FW1wCnuTXMOyij8h3tRFy/4k9hJQt/7QlBT0ZQZrPFrABIBHTBdcXyC0HnlEZlRrtwrMktkXBAMh9qY", + "wcFfqMf2Vu7bZe+sPMZ4awwDYtkLt7lvUVDGs4PTPBiNHFa8cdcvBjHZyIm9vzs32rLfTlKdW6JAQoO1", + "Z56XLYul/9DvHY32rzSmTUPBsxvq+LWgLpqZ4ev83mh0852eCBuf6KLNnV9Y9cQhIVXP2q9vYc90nqZU", + "rfyCVVcrk7pNMGag13Jl7TvOaBIBq8C0SwBow+x3UIlS68KtcoGpsn1FoND6KbBt+00u0Jq+l/Hqky1h", + "rQ+vo/hQZ2dwXD6s0fOno52CjNc30n3yoOWWam+BgL6nRZ73OzspR6NHN98p6H4THkG0qidg55jNNfo+", + "JQjB7kGYpCK/5dJQUsQ1fEFH2sms04Lc+uVVtPcHjz/Y452wkD3gjKmUChslYstsOfRrx9naZsrjvPFW", + "84R/8rTnbiqPRmQvKhTk6ke0em01hcH16+iod9zWp51efIeEf3QLJ9xNtkj1e5dHDpOTgl35SzpOztY4", + "LYWQoCz3nJnPheZHt3llubwMf8JT9KUQ8HNWSHjlbq1dCnuZyoV9AAclwJdl5Kar901d+HtVfqm4C6Fd", + "A5rGDCHGGphpvBoSv6b20Q9KX20h+tXSupvVz9cZDO9zOWEHt3HCcMaFpejrNfX1mtp0yi21+Cngwayc", + "8g46iCtpIP58+ocrax++6h666x46aR4Ee2dLAx7kkDjX3EjGjOgF5DsFY7MFfvJOOIaq4fx3QlW04EuG", + "6H6Y9y5PDM+oQheblICVwNrQWxUTG9USRXN70NzAO2SWC9wE9NBsgi4wkzYgztIVkwsBylqqvddMiau4", + "lqHdnv2ggr1osLwaybuF1KwANhSmcptjnLe2r2NsdjgWrxziLSwgepV7XqNZgri9G/Q/UhA6Fq7Ct56F", + "eI84TdOSc1GF4IncQnTabVmbKo50oiMZAh16xQQVZqAzFvEZeB/itC7Zyjm2BhvslMoKBuzHCXlhvKPW", + "wW4YuA5dl8IoxU+Lb8RRUt1+I6RxSb5LI5fHUqIKch8EM5TMEzmlycSuzyUL2ASfYwm3KKXBpbQmCRkz", + "m5U/W5mFFPbvfJoLk9u/p0q+00yBzWksMCLFrTV4KxQCInmHufHSTCpj/Rywzz07xL0/Ltnqw3AsHscp", + "F7aGtlVooiWYKaCekR48xHKvFnqwpylsB3+SayPTKgSspzs7TJmbLDcutEYz0w/Bn46FkeQPD3L5Ye+P", + "sscPaCxmNAY6qRSxU0LZum3UekJh9hMsGjC3M1yAcQ8uUuvmMVdUGBxTidJJ5tUt3SnSRGAS2uYKR1SQ", + "TGY2xQYS1YICydXaQNAKmiTE4FHydUFwx51smY/DIEynrQCEFjGucYy4IKffVw7T6Ohh+DxpFikW8ij5", + "r/MXvxC8lWEPbLHSXQvXiAsQGMCLBnOHOJ72DGCSrKEK8zOOezwe9wpzbryLY821c5cZDNCm+B0M7Tvb", + "TZ/H3w2H0JQ1Vx6TX/+wrRzDWcpSC4g67n3ok8qHOTeLfFp8exte0DYct/MaIyA79prbRU5COULuVG58", + "e0WC6k66WwAABEnJgaqOK1MuqFptiqgMLL1bQTmznoyVxfhjjJ6L497x2Psujnv9cY+JJf7mHBzHvQ/h", + "FXBWy/ZkgHifFcbNgojuj0a72yHB3foGbJYdDAOf+A3Y+ioqMpnCDjo82j+XfeDf+v35ffFiE9B5idFk", + "H/5e+X7Xcj30fgub/wuIPcA6UoYeJYNqTja4KTCXIF5tmHTgWpaRylOi+kQO2EYa7wEqIpb498B2DdTt", + "WzWAiiKWJLd9cu7q3BR2uyKXwhdlqMPNKs/3RrvCHVPc6LZuu5o94W7o94tT7AfU+k6pz5beBzucSQZR", + "gtwbn9jCoDk5xzENzpkw5Bn+OnT/9Y9SRL28SOT84tjqFCApAkm4cIEKFQ9qkFvcWmIlCxRU1LP/LNL4", + "7VgR51//+CcOiov5v/7xT4e+/69//BOP+54FwMN84hcLRpWZMmoujgmE4w1oAv7tbjKY8ZctmVqRw5Ez", + "RuAnGJJgUeHtLXNAzByLl8zkSlQCOmxGPe0adDYMmA8XOdMOaAkK8plL92OdLgMKJX+W7VLe6onuBwAr", + "cQaVCcCt6GkA0f64Tarunsi9sC7XzrmmzW36j655EW7nL4a9N5Z6B3aAV2QwuMShc4cf3KTJzvn5s90h", + "wWegpQpM6YSPmrIZ974ZfuVJ23mS5Sh1hoKrbHlTRDM65Qn3utCWfDT2CKY0WgBwVOH4XKDB+yaO/UiB", + "x4APqPPQ7GPRsXhxvoe6X8MikyvWd5xAOQzYMmGddAE42APyL27QbW3gyo7FjFFog5w8tUygApNeRGwW", + "DQuEWkHnW25qufH6Y2Gxfh22NBw8kGQTrIT9z6lh7+iqTyqSr81fk1ADL3Xdh8JjYWMg3RoMEEyGVIY5", + "RH5mhzTwLsYumEyxGWhtcSIuxTv2vQMKD+d63S8DXn13NgzWDotIJ47D/Ob4RJVWUYktvTj3u7HbJ1qS", + "KOFIDREVYzFHDyUPryxFbVeLSLcFVfEgknAJVOG2wDU6YfG8jcc+qbTVu0FJptZP4Dj92CTXL024WKxP", + "AA6xhRDcbFJ86sp0sym6Fv9MRkW7QFexKlrVMrP8xq7uVwtjBwtjeN28tTFk8nvqMTJvzhXZdnFHnsie", + "9tbX3H6pLNldqBrJjgcfQnONVOTsyQmhcayY1rv/3opImKml0lL+g/sRWPFd+MS4sUhl98LrW+oE8qWw", + "g5du1IT6eTUzIFfvt71aeqTWm67IlFReeTd/ezQ6vco1Ugq9Ja19vUm2egFzHUFMZZVaBigaJawQX4pz", + "WqWibVpl619cXDkbxSVbCpw33IG8Pf2y6zoXzbvhFpji0wZDvENGWI8Br+Y1/5Ko+XWxi25em9TPnxdp", + "jm5PCrptVXSIzL+k52LcWDbgghaBpfUCfc6MxV25yXe66yEwcdBzu1NtB7qysy6mZasSCyCDE0JNzOa3", + "74kt0u3pa9v7M718cXmuIrG4Jf8qonR47JZrtemBe+KSdN/c+7YGMXfL/jSOwAKLDB+cbhrxmiDvJdUr", + "Ee1+dan55BSNi115xCo/bxYXmmz8Xr6zbkuug4TrZUZ4rot3LRdklgB8rTUCxHyGToSmmmEdR3lwC6Ms", + "MpkrapjznfwSA5LP8sRaNyIplkxVICSrV+reH+hNu/2p5JnXxtv19cufB0xEMi6MJ+0yqfvyiR9Mlv5r", + "Qca3f+q+wEBb7sWDNoHxI/bfermTAlP0fx384FBF/9fBDzTJuGD/6/BxQg3TZvfGiGV0WzfdbT9gvmDi", + "g/cLry/aGmvaM3ReDZCtU6bH1b0KZdbwcL2yu4jw+WTU9+mFvCaIcCcJ79boHl5Stw2lcuLzJdgtlcrD", + "ajWwkv+9hb7zGkHfur6u1j3XOAIU7L5A/BapPBwsdXja3BO3401irpjW25QRRamO+ghf/k+lkrCTvpJS", + "oljXr3qJLnqJ6nJtVE3YgjernLB93JH1vSC20Grjp684YLdg0HEUWcEBq1m4SySwhdQGP30eoBD/TpE4", + "vDgK1fuso8m05BQb7zV/pk6e9nGH+7CnJ0/LYPBbMqD6cdy6RsD1e/vS2ON0yue5zHU1zjylJlow7TAY", + "Ela/Gb40XUUpN7RqKz5jKh3d5p1268qIr3R/Q2qS5oZa5m29ILY9Rnypro8RVx4eIxaf2uFUuGxSfZ9p", + "cLclOsUjVHcl4xqQ94d+l3GFHkmQnUOT8h1D8GlzPBb/Gx5GvxpG07ff+YD4fDQ6uI+/M7F8+52PiRen", + "nlSYMIq7xLCPf3mKridzjC7A3LEl/EZzHCTNtbGk55PE/Nu93Ervm+5PN0+FX59unZ5uleXa/HSzBW/4", + "7VZPNHXrjzdPb6EFt9/+pM+3P7lNufa01PlsxiPOBKbrQq2kXnOitk/Mr+bk6+lHqXBOHBUPzJok0vkZ", + "WXCtLRJ6kQP6k7o49luzdkhCjWFpZgD2KWKQPdiGDupFbmL5TvgkHjhBnw+Ol/MJXe++qYlvJBy5G05q", + "3/WlWySyvO2nruv4Cw2dlZnLye4el6Vo0/66vFvivdk3ZYer9vZflV8yidnn2/rSZfBCCCSls/anNLdx", + "xkXNEs9ySF69+tnHFMPzRPkUh0b6vIY+9/VYVPMaDsmzMmGkLeBbgOcDix0GAdT3mQJjRuOEC4ZBGEyH", + "wn/r2Ujv9Fh8egk4nGr1tm3vXY6l3dA7lIDvjBXciqxZOBsYZykujmYlC2txWry8iafmi+JXjgEFGE9I", + "1tujuZEDh1Kwt5AWUzMMK3yW0AhRhaGYBbx0wDAW4bbaFKK9KJkkTFkg0yw3XtwCTGE3OC6MLFJ1O8ns", + "Apqf5MLw5KJvfSChbKIJpIm3UDdjUevMyXwI3oDAJDhCxTI74kbeYRg0B1UjlEKchWqXhCbv6EqPhYN7", + "sNUxWb1ikcX8TZIh+VEi0o7Nv11hvDb57Td6LC54nLCJA8q5IFwTvZDKMMFiksol0/V+GVUJZwon8YTC", + "ymmS0hVCaVpUYbs+MmMWrrIGxyPh31TEHNOoQs/FlI/HgpKD0YikjArtwDU0neGF49ogOIjagL4llByN", + "HrlajX1DuHe//DtwmpRiSxnRKWTMBSq26cl3cQNTl9bY5smH7Ztxpe1+FfpNl6+ytrFc+yy9cZ/kooQP", + "QV1/xboG22VyJXCezjzJuCquQYeYNGURhfUUst4PgujKKMpV6IKEra7k1/53FBwr0zvHpQqDcySoMogg", + "BkAi8uMCz7TEo7T7bQtVlUT157hogodEKkJJha5LjQaLcmSNOwg6e1EmixU++fvF7rf+7MDxdYzAH38x", + "/5LuJyQiOZvVDuD2q8ke4E1Bcesk/Gc9p08c06uxuJjTuZDa8MgzQ4/pXX00f30QdnoQbl7ZIDXPpLps", + "94T+QarLri8w77H5ZT3EqjP8DA0RMDx00rl7ewRqw+1jBYjm1h9pTfoqTikKXdzowleZJFLMmapq5W/d", + "bFB91e1YpE24TJU1dhe4aPAImbgfbaLxir85mhgi1+qf1lkNiQ02u/RZm67s06NEkrsar4RTVQV8sG9B", + "bf0P+l4cQrryG7aD0vv6dgWZKgCNbkVqLTr3sKQBqNaxeK1tcocLa3q6IAUPBoHWJmwh7xY8WkA7+Bu2", + "b1FdaZZdFFD6u8fkOR7kyrbZzndsmhagNS0TZtFYl2l6cbyeavvN6SlWwjILm1T74pj49NrF/aGhVBWG", + "FWaRUG3ILw5cdqd4jOOOXoAOszK/XQfQWqY6GIsQWCtgndoG+YxcVHBbL1pABT2//VnO9WdjKioT1Ni5", + "GEnc0xFpk4m41+bkwZOw4Wd/NArlTegIH2uHccPosWuD+VnOi+Q4NVKmWdaVfN0wkYqXabqBhslOyUGI", + "NrHMzV+0iZlSWNlRdxtxkx0a2X8YegmE6pBH/cHeHYuWpbIzDC8VMNVev8dEnvaOf3X/WqZpr99z44F6", + "70yWVrJdXEHI3wLH22zwQz+0QxXM3T+xeH5nV6JLpFQclVgy6x2JEo97QQGUK54X3NLrQP/Wb6gK9m/j", + "unM6gPZnxEtb4E9v4vSKxlsT392LpiE+Benjzk4xilkNXfjnJMlXxlvXmwlZoCrjyB7dxsgQeplEidSs", + "Zif7coAbnb6wIXq369v8Gg9geHHuU/B1cQQ6d1XPfc3PQIWxzeXGj5n46d667836CL5kEAq9Nhtgfw20", + "v21OOZ89IX26LVmbahcK+UqbV1fWdiJMeG6tswhXIbYpWWluZEoNjzAdYLSQUlfIvoDmt4k7nQ6+oEzU", + "UFllgQvEuABSvXDa/AsnYh47zSOh1U+ujyFWd+Eb4Rr+U1njh4pypeD4ff+Cwsw0GlKmKc5mJKO5ZiBn", + "5qARW0UJ8/kfGdjVI5qZXDFMbctIygVP87SacwF2bEkRH+piP73ok2luSAJaNlV89D5LkUxTJmIrJ43F", + "gtElZwoWJaGGiWg10AxT4i8ZgbQPiaQxamqymKLBDAMpFQMKxAQWKTM0poaioHMBJ35iY8Euiiz5Vjsi", + "2PuSGsB2nItvbTYdaPbCD/SCMEwXwfWiyKYc0ZiJKJhG4fzzZmOfXqV/zkxzonfkYHUtXnqXHldV1bUf", + "zufhjPXFQn50YPMbhF7d/qiuB9F4Mvr3PNJ2rn6Od2SnK5Z40yn+PAx0BdF9Nka6u7fCSUXi3HZXOZVI", + "5n9WPWLBUGqIEBCga7fxuva1IjtrscxX4nl7f/g/T66hXfxMOGG/9WHflgewnPTnwHLdql6L596RWtXp", + "kqo6ubtjwW5Qdyc+SVXhcl+KsrWGvFfw7Sp3Mori60uKr2y7ybad38h12bbXza55RlQYORcDdLUNc3Cn", + "xm1l1U518G8a1NOYXYVl3jmLLC0Xtw6j6VljRlegKvkz+FpvsB9FUimLIkIN+7Kgvytaw2qUBermygSj", + "fR/0+ub0dLeNSyizkUco8wVziEpkE1RL43UF7oslU4rHHt70yenT0iarcjEkL1JuiJHkkrGsDAzC4ExA", + "7SzwVBrDbgKn9HtMGLXKJBdm6yjKojczmPIH6WBlPkNR0uWS+GoO72wOR83+l8fOkMsQWkxg88vUULM1", + "xzUXM6lSK5fRqcyhdeBBsEywn9bZYsYTplfasNQ6dwIUBBw3zEvkcs+7enaX++jaDCfHRh1mTKVcay6F", + "HgsXcpMxBX1DdWi/4qcWNAgYWvDXM8skPw8fSBiMdfujpm3VEPkKc1r3jnt7NMv2Ympoi5+dG95HDOkH", + "dGokepVOIYASvCIvNdlJ+KV9npClJgn8sbvRK3KC9T51Zv3rnyxY6RMxk8G8pZZmC2L+UwXHObbmDZOf", + "BVu7o3fmR3sRPmfVg+t54Uy2sli9lccqRpMB5uX3WEwkNzzhv1u2C41wbXhko8hosY9vTgsGPxyLU2YU", + "j6y9NwL/cwSpwJfuXqZktAc4fodRxhHQ75Dh4JD5tn9OsccnZ6+xXMpSqVb9saDCNfzq8Zm1NM9oxJoD", + "FcyAJZmc7L3Y4rN+jsv0b+w/aSe4EZAiuOFfzZNXh41pPUO65YjKbNOzTWZ/egdfJ01+1XF8mToOoOFy", + "NjsFVpsHWAvrM5YyyVP4h/3jZBtUnaHR4g0W/Wwkbzucrd34CX4Rh9LNKWY2x/OdGGDsgn2p/rOwcH4K", + "KMTUPBODt8Bj82ek7k9vSqiu42doenUr6vOnfzZn67ZvPjcGD5pSXY8v5ZhbSvMzMXKzJuwd5e2asO8T", + "GV1qh65TVWESxShC5sOPJcS5M1eimIDBvsShUlnsM6b7Y9FQhloQJ00oMUylXNBkD+dsG0Gwdq9Ro0vJ", + "MeY+wpiZgQZVdyaTBBHZEdEQZgMlZr6BinVZuwyT1TJVw6iRANskU+YB7HdDT7e/Um5+kKqORv+58MVX", + "lfVHlEfKUfe/BYC/vcePAuQ/pe/RbTvOnXHbj2jnuSx/tGqpPsG9GfcOR3rc65Nx7yAd92AHnlBU51JD", + "7pGUi9wAsuVTq2vDqOr7I6JZJEWsPY6+1yYejnTLzB1ZtgTs3sd6tyn2OKrCpXzpOgmxByhHoD4GEJGd", + "6oFzZzLu46GLicyNNT24c+VKxcygemT31u3GlTPy9W3fhZP/1R3fGo/CXQZ2Wdl6y9mzXC9Yu8rtZ5s0", + "KzdTWIciybZekL/Lqe5D1IHVzANi3hrfg9pntoPbyB0BXV0lb4Sb+9ekER2SRpRrFcbftM6ecCV76rAg", + "nOx9JpWB3x1ugaMhfEkgGAimwnzx5GQsImBFFi1SsVQid3IQ9/YWfvzXc/Lsycs+eYoJn8mP+XR3SF5A", + "pA22qp29aCysJGaZV0QBWN3OIA5dz3bsSD036bgOHVxJij/4dJYePBkBK4/fK++w3u8tGI1RIvmj97O0", + "nQWApF/+DAcI9s/VLLa9t1H4AB2+Wg0ezwxT682eupgtUcCguEvaowo6wQ07xg61B9Mr+7SygUU7OTzo", + "BcBPPnzN43HzicJvx2KH/brUjhDN96XmCMaDWDDHEAusXtdFJoy2iGXHyzY+MLDLk6ef5FFxo1bqTbyr", + "li7g3/V04Uy/WENTVtsnIOIig85WS68PVF5YhGtbjUQ0oxE3qz6hSeLuKHcTFN4xg0L8nSpGL0FvPwQU", + "MtezDy4GW23fG2pJzPWlbcHZYocE3ON0Pi0GR/CgWasxrjmE6BpJIppEeUINI2w2YxHGBWNKHt1iyy2G", + "0rvBs1N2EoSFqXjY519c2sIwTeDulWTRpLg9u9V7ikUJ5Wk7nrwT1ND5EV0NptCoFOC7kFiKo5GSWhPX", + "1IAlfM4hrQZ6SkAmDVBz0ZSNRZZQIZgiubYe+jD0QaaY1rlivgG4uxxF9UmJFZkpaZxrQiKl0mPhKfzN", + "KdGGZRvI7KVt+RTnfEOyrW3c9XRHSurGGNpVIa4IgQ2xlGIXHOgIVEN34BZvB3TXUuKXcvBfKT4H39dM", + "SWqZrPVLssfaL6c99LXo6dYUpudFqW4pTItWKxGSlejBjVh/kxI+Pe5dzQMx0Pklb4WDdJ+uFtH8E1Tq", + "2Hc9cjY8CPfpI2cZysb675j49LwSsNhVgVVS+JemTqqMvHZUa0G/2yG+Okf53mTUbWcsrzuD8PqSkbto", + "LZS37cH7+RHC6HYRJ247b96XTVs15K3a27QFfmB7goTPggJvJjPCHSOuXCMzwmeFAQC0c4dYLMGDelcx", + "/TXbs1RfkxvcaCi/zXCA0HBtofyW6znn1Y0PpTeuTLdnkmvxzyTB2wW6ivzul/3rq7/Dk6GyWNtM0EDw", + "LM3MylXytsrS6Uzz39mwxRBc+K3enCn4Gi6dn448PJ22OnTeIXrZHRpy7sRn1GWD5Brc/Mqsez6w+wvD", + "O6yeudrFsge3zoCqaMGXrF3pXj/BbokyxQaZzNC4EtsFc+vh7zJD1XD+O3HNO/xX9y9MJ4qJDFhMYq5Y", + "ZMBbRBiJHMH28Y0mSkrjvku1avcSsUfkByXTx242W+5Dd6acMqz0M0xXg5gaOlh6brNBhfYR3p3enxIY", + "HuGCPP+e7LD3RtmkJWQGLx+INvdLyt5HjMUaaXK3OuD9UXiI0PBkPu0yyg3pZ17gHzQhUa6NTP3enzwl", + "O5jObs4E7AWI+jOUZDMllzxmcW2M4LxvV3W/ZUGvqncFoaLIRegfF3ZwdyLDdLmQ5r/zrM4WCpeYKRcU", + "B7c1wUv9TFlAAeiPcuEdcNwe+VF8vcLcy2/HP3aAEjG1qltEI6WFm979es19yddcNRjK32m1286752xW", + "XneLj+oYtnQTSSiK2LnbVVu/+XxCerj+IqN5nOp8WTxI29TmnxcJjm7vfrhtdfmbLzgEFJTly8ay2QbU", + "MkwwP6NPd8yWLJFZiinusWyv38tV0jvuLYzJjvf20Pd7IbU5Pnr04LD34e2H/38AIrS4drT/AQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index b7bad52a5..a3e6d8a66 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -146,9 +146,10 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima return nil, fmt.Errorf("failed to parse network.macos_subnet_cidr %q: %w", cfg.Network.MacOSSubnetCIDR, err) } managerConfig := instances.ManagerConfig{ - MacOSOnly: cfg.MacOSOnly, - MacOSGuestSubnet: macOSGuestSubnet, - LifecycleEventBufferSize: cfg.Instances.LifecycleEventBufferSize, + MacOSOnly: cfg.MacOSOnly, + MacOSGuestSubnet: macOSGuestSubnet, + LifecycleEventBufferSize: cfg.Instances.LifecycleEventBufferSize, + MaxConcurrentVGPURestores: cfg.Hypervisor.VGPUMaxConcurrentRestores, MaxConcurrentRestoresByHypervisor: map[hypervisor.Type]int{ hypervisor.TypeFirecracker: cfg.Hypervisor.FirecrackerMaxConcurrentRestores, }, diff --git a/lib/system/guest_agent/vsock_linux.go b/lib/system/guest_agent/vsock_linux.go index 3de52ddc5..747fe6af3 100644 --- a/lib/system/guest_agent/vsock_linux.go +++ b/lib/system/guest_agent/vsock_linux.go @@ -6,8 +6,10 @@ import ( "net" "github.com/mdlayher/vsock" + "golang.org/x/sys/unix" ) func listenVsock(port uint32) (net.Listener, error) { - return vsock.Listen(port, nil) + // A restored guest may receive a new CID; keep its listening socket valid. + return vsock.ListenContextID(unix.VMADDR_CID_ANY, port, nil) } diff --git a/lib/system/guest_agent/vsock_linux_test.go b/lib/system/guest_agent/vsock_linux_test.go new file mode 100644 index 000000000..afe45cb6b --- /dev/null +++ b/lib/system/guest_agent/vsock_linux_test.go @@ -0,0 +1,24 @@ +//go:build linux + +package main + +import ( + "errors" + "testing" + + "github.com/mdlayher/vsock" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestVsockListenerDoesNotPinTheBootCID(t *testing.T) { + listener, err := listenVsock(0) + if errors.Is(err, unix.EAFNOSUPPORT) || errors.Is(err, unix.ENODEV) || errors.Is(err, unix.EPERM) || errors.Is(err, unix.EACCES) { + t.Skipf("vsock is unavailable: %v", err) + } + require.NoError(t, err) + defer listener.Close() + addr, ok := listener.Addr().(*vsock.Addr) + require.True(t, ok) + require.Equal(t, uint32(unix.VMADDR_CID_ANY), addr.ContextID) +} diff --git a/openapi.yaml b/openapi.yaml index 73aa90cde..70c226479 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -656,6 +656,11 @@ components: enum: [cloud-hypervisor, firecracker, qemu, qemu-microvm, vz] description: Hypervisor backend to use for this instance. qemu uses the architecture-native standard board; qemu-microvm uses QEMU's minimal Linux amd64 board and does not support PCI devices, hotplug memory, or more than eight virtio-mmio devices. Defaults to server configuration. example: cloud-hypervisor + hypervisor_version: + type: string + maxLength: 128 + description: Optional installed runtime version. The backend validates availability and snapshot compatibility. Omit to use the backend default. + example: 11.1.3-vgpu.1 snapshot_policy: description: Snapshot policy for this instance. Controls compression settings applied when creating snapshots or entering standby, plus any default standby-only compression delay. $ref: "#/components/schemas/SnapshotPolicy" @@ -1126,6 +1131,10 @@ components: enum: [cloud-hypervisor, firecracker, qemu, qemu-microvm, vz] description: Hypervisor backend running this instance example: cloud-hypervisor + hypervisor_version: + type: string + description: Runtime version selected for this instance. + example: 11.1.3-vgpu.1 snapshot_policy: $ref: "#/components/schemas/SnapshotPolicy" auto_standby: diff --git a/scripts/qemu-vgpu/Dockerfile b/scripts/qemu-vgpu/Dockerfile new file mode 100644 index 000000000..c9754b764 --- /dev/null +++ b/scripts/qemu-vgpu/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:24.04 AS build +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential ca-certificates git python3 python3-venv python3-pip \ + ninja-build pkg-config libglib2.0-dev libpixman-1-dev zlib1g-dev libaio-dev \ + && rm -rf /var/lib/apt/lists/* +COPY scripts/qemu-vgpu /src +RUN bash /src/build.sh /runtime + +FROM scratch AS artifact +COPY --from=build /runtime/ / diff --git a/scripts/qemu-vgpu/README.md b/scripts/qemu-vgpu/README.md new file mode 100644 index 000000000..c27851222 --- /dev/null +++ b/scripts/qemu-vgpu/README.md @@ -0,0 +1,30 @@ +# Opt-in QEMU vGPU snapshot runtime + +This package builds QEMU 11.1.3 from a pinned source commit with the bounded VFIO state-load batching patch. It does not replace the system QEMU. Linux amd64 vendor-VFIO L40S-3Q is the initial supported configuration. + +Build on Ubuntu 24.04 (glibc 2.39), or use the container build so binaries also run on Ubuntu 24.04 hosts: + +```sh +docker build --output type=local,dest=./qemu-vgpu-runtime \ + -f scripts/qemu-vgpu/Dockerfile . +``` + +The `Build QEMU vGPU runtime` workflow builds pull-request artifacts. A manual run from main with `publish=true` creates a new `qemu-vgpu-` release with the runtime archive, corresponding patched source archive, and `SHA256SUMS`; it does not replace an existing release or trigger the API's `v*` release workflow. Install only an explicitly pinned archive checksum. + +The build runs the patched load routine against partial writes, EINTR, truncated reads, write errors, zero writes and multi-buffer payloads under ASan/UBSan. `build.sh OUTPUT_DIRECTORY` is also available on a compatible build host with the Dockerfile's dependencies installed. Output directories must not already exist. + +Install the resulting directory, preserving its manifest and firmware, at: + +```text +/system/binaries/qemu/11.1.3-vgpu.1/x86_64/ +``` + +Select it explicitly on a new instance with `hypervisor: qemu`, `hypervisor_version: 11.1.3-vgpu.1`, `gpu.profile: NVIDIA L40S-3Q`, and the guest agent enabled. Ordinary QEMU instances keep using the system binary. Never overwrite an installed runtime that has live snapshots: change the runtime version and rebuild templates instead. + +Snapshots pin the runtime binary and BIOS checksums, machine type, NVIDIA host-driver version and PCI device identity. Restores reject mismatches. A child claims its own VF and receives a separate vsock CID; the captured source device assignment is not reused. RAM is eagerly allocated privately per child, not shared through CPU-style copy-on-write. `hypervisor.vgpu_max_concurrent_restores` defaults to two. + +Capture uses live VFIO precopy: do not pause the guest before requesting a standby snapshot. Only after migration completes and the source VMM exits is its VF released. Restores load the mapped-RAM stream paused before guest network reconfiguration and application identity handoff. + +The runtime does not make arbitrary application state safe to clone. Browser templates must contain no customer state, license token, trusted license cache or active lease. The companion browser image keeps licensing deferred until the child has its own MAC/license identity, mixes fresh host entropy, and restarts Chromium before returning readiness. Licensing then proceeds asynchronously with normal health checks. Host/runtime support must deploy before enabling GPU fanout in the browser control plane. + +Keep GPU fanout disabled until the complete template/identity/licensing flow, concurrent forks, cancellation, failed restore cleanup, old-template behavior and neighboring workload impact have been qualified. Retain the existing cold-boot path for fallback and the previous runtime for existing snapshots. diff --git a/scripts/qemu-vgpu/build.sh b/scripts/qemu-vgpu/build.sh new file mode 100755 index 000000000..3f2b7638b --- /dev/null +++ b/scripts/qemu-vgpu/build.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +version=11.1.3-vgpu.1 +commit=005c61c0ef6f6ae615283440b4bb7be51b2c6ddd +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +output=$(realpath -m -- "${1:?usage: build.sh OUTPUT_DIRECTORY}") +if [[ -e "$output" ]]; then + echo "refusing to replace an existing runtime directory: $output" >&2 + exit 1 +fi +if [[ $(uname -s) != Linux || $(uname -m) != x86_64 ]]; then + echo "the vGPU runtime requires Linux x86_64" >&2 + exit 1 +fi +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +git init -q "$work/source" +git -C "$work/source" remote add origin https://github.com/qemu/qemu.git +git -C "$work/source" fetch --depth=1 origin "$commit" +git -C "$work/source" checkout --detach FETCH_HEAD +test "$(git -C "$work/source" rev-parse HEAD)" = "$commit" +git -C "$work/source" apply "$script_dir/vfio-batched-load.patch" +python3 "$script_dir/test-batched-load.py" "$work/source" +mkdir "$work/build" +cd "$work/build" +"$work/source/configure" --prefix=/usr --target-list=x86_64-softmmu \ + --enable-kvm --disable-tcg --disable-docs --disable-tools --disable-user \ + --disable-werror --disable-gtk --disable-sdl --disable-vnc --disable-opengl \ + --disable-virglrenderer --enable-trace-backends=log +ninja -j"${JOBS:-4}" qemu-system-x86_64 + +mkdir -p -- "$(dirname -- "$output")" +stage=$(mktemp -d "${output}.tmp.XXXXXX") +trap 'rm -rf "$work" "$stage"' EXIT +install -m 0755 qemu-system-x86_64 "$stage/qemu-system-x86_64" +mkdir "$stage/firmware" +install -m 0644 "$work/source"/pc-bios/*.bin "$work/source"/pc-bios/*.rom "$stage/firmware/" +install -m 0644 "$work/source/COPYING" "$stage/COPYING" +tar --exclude=.git -czf "$stage/source.tar.gz" -C "$work/source" . +python3 - "$stage" "$version" "$commit" <<'PY' +import hashlib, json, pathlib, struct, sys +root = pathlib.Path(sys.argv[1]) +firmware = hashlib.sha256() +for path in sorted((root / 'firmware').iterdir()): + firmware.update(path.name.encode() + b'\0') + firmware.update(struct.pack(' +#include +#include +#include +#include +#include +#define VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE (1024 * 1024) +typedef struct { unsigned char *data; size_t length, offset; int error; } QEMUFile; +typedef struct { int data_fd; } VFIOMigration; +typedef struct { VFIOMigration *migration; const char *name; } VFIODevice; +static unsigned char output[3 * 1024 * 1024]; +static size_t used, calls, max_write; +static int interrupt_once, fail_errno, zero_once; +static size_t qemu_get_buffer(QEMUFile *f, uint8_t *buffer, size_t size) { + size_t amount = MIN(size, f->length - f->offset); + memcpy(buffer, f->data + f->offset, amount); f->offset += amount; + return amount; +} +static int qemu_file_get_error(QEMUFile *f) { return f->error; } +static ssize_t fake_write(int fd, const void *data, size_t size) { + (void)fd; calls++; + if (interrupt_once) { interrupt_once = 0; errno = EINTR; return -1; } + if (fail_errno) { errno = fail_errno; return -1; } + if (zero_once) { zero_once = 0; return 0; } + size = MIN(size, max_write); + assert(used + size <= sizeof(output)); + memcpy(output + used, data, size); used += size; return size; +} +#define write fake_write +#define trace_vfio_load_state_device_data(...) ((void)0) +''' +suffix = r''' +static void reset(void) { + used = calls = 0; max_write = sizeof(output); + interrupt_once = fail_errno = zero_once = 0; +} +int main(void) { + unsigned char *input = g_malloc(sizeof(output)); + for (size_t i = 0; i < sizeof(output); i++) input[i] = (i * 37) % 251; + VFIOMigration migration = {.data_fd = 7}; + VFIODevice device = {.migration = &migration, .name = "test"}; + size_t sizes[] = {0, 1, 32768, 1048576, 1048593, sizeof(output)}; + for (size_t i = 0; i < G_N_ELEMENTS(sizes); i++) { + reset(); QEMUFile f = {.data = input, .length = sizes[i]}; + assert(vfio_load_buffer(&f, &device, sizes[i]) == 0); + assert(used == sizes[i]); assert(!memcmp(output, input, used)); + assert(calls == (sizes[i] + 1048575) / 1048576); + } + reset(); max_write = 73; + QEMUFile partial = {.data = input, .length = 1048593}; + assert(vfio_load_buffer(&partial, &device, partial.length) == 0); + assert(used == partial.length && !memcmp(output, input, used)); + reset(); interrupt_once = 1; + QEMUFile interrupt = {.data = input, .length = 1048593}; + assert(vfio_load_buffer(&interrupt, &device, interrupt.length) == 0); + assert(used == interrupt.length && !memcmp(output, input, used)); + reset(); zero_once = 1; + QEMUFile zero = {.data = input, .length = 128}; + assert(vfio_load_buffer(&zero, &device, zero.length) == -EIO); + reset(); fail_errno = ENOSPC; + QEMUFile failure = {.data = input, .length = 128}; + assert(vfio_load_buffer(&failure, &device, failure.length) == -ENOSPC); + reset(); QEMUFile truncated = {.data = input, .length = 127}; + assert(vfio_load_buffer(&truncated, &device, 128) == -EIO && used == 0); + reset(); QEMUFile read_error = {.data = input, .length = 127, .error = -EINVAL}; + assert(vfio_load_buffer(&read_error, &device, 128) == -EINVAL && used == 0); + g_free(input); + return 0; +} +''' +with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / 'test.c').write_text(prefix + function + suffix) + flags = subprocess.check_output(['pkg-config', '--cflags', '--libs', 'glib-2.0'], text=True).split() + subprocess.run(['cc', '-Wall', '-Wextra', '-Werror', '-fsanitize=address,undefined', '-g', str(root / 'test.c'), '-o', str(root / 'test'), *flags], check=True) + subprocess.run([str(root / 'test')], check=True) +print('passed: exact batching, byte integrity, partial writes, EINTR, zero write, write error, truncated input and read error (ASan/UBSan)') diff --git a/scripts/qemu-vgpu/vfio-batched-load.patch b/scripts/qemu-vgpu/vfio-batched-load.patch new file mode 100644 index 000000000..dc2698162 --- /dev/null +++ b/scripts/qemu-vgpu/vfio-batched-load.patch @@ -0,0 +1,45 @@ +diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c +index 73c49d8..5302b41 100644 +--- a/hw/vfio/migration.c ++++ b/hw/vfio/migration.c +@@ -252,7 +252,39 @@ static int vfio_load_buffer(QEMUFile *f, VFIODevice *vbasedev, + VFIOMigration *migration = vbasedev->migration; + int ret; + +- ret = qemu_file_get_to_fd(f, migration->data_fd, data_size); ++ g_autofree uint8_t *buffer = g_malloc(MIN(data_size, ++ VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE)); ++ uint64_t remaining = data_size; ++ ++ ret = 0; ++ while (remaining) { ++ size_t chunk = MIN(remaining, VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE); ++ size_t offset = 0; ++ ++ if (qemu_get_buffer(f, buffer, chunk) != chunk) { ++ ret = qemu_file_get_error(f); ++ if (!ret) { ++ ret = -EIO; ++ } ++ break; ++ } ++ while (offset < chunk) { ++ ssize_t written = write(migration->data_fd, buffer + offset, ++ chunk - offset); ++ if (written < 0 && errno == EINTR) { ++ continue; ++ } ++ if (written <= 0) { ++ ret = written < 0 ? -errno : -EIO; ++ break; ++ } ++ offset += written; ++ } ++ if (ret) { ++ break; ++ } ++ remaining -= chunk; ++ } + trace_vfio_load_state_device_data(vbasedev->name, data_size, ret); + + return ret; From a0fbfaa4a2f459a748f23c5aaf5d4781690c64a4 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:18:28 +0000 Subject: [PATCH 2/9] Cache runtime verification and share fork snapshot memory --- lib/hypervisor/qemu/fork.go | 5 +- lib/hypervisor/qemu/process.go | 4 +- lib/hypervisor/qemu/qemu.go | 7 +- lib/hypervisor/qemu/vgpu.go | 176 +++++++++++++++++----- lib/hypervisor/qemu/vgpu_test.go | 52 +++++++ lib/instances/firecracker_memfile.go | 10 +- lib/instances/firecracker_memfile_test.go | 37 +++++ lib/instances/fork.go | 2 +- lib/instances/snapshot.go | 2 +- lib/instances/standby.go | 6 +- lib/instances/vgpu_snapshot.go | 12 +- lib/paths/paths.go | 5 + 12 files changed, 263 insertions(+), 55 deletions(-) diff --git a/lib/hypervisor/qemu/fork.go b/lib/hypervisor/qemu/fork.go index f571d8398..39c5c010b 100644 --- a/lib/hypervisor/qemu/fork.go +++ b/lib/hypervisor/qemu/fork.go @@ -39,8 +39,9 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq if req.VGPUDevicePath != "" { cfg.VGPUDevicePath = req.VGPUDevicePath } - if req.SourceDataDir != req.TargetDataDir && req.TargetDataDir != "" { - // A child must claim a new VF before restore can launch it. + if req.TargetDataDir != "" && req.TargetDataDir != req.SourceDataDir { + // A fork never opens its source's VF: restore claims a fresh one + // and binds it here before launching QEMU. cfg.VGPUDevicePath = "" } } diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index 4e88b856f..ccfcddf9d 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -94,8 +94,8 @@ func (s *Starter) SocketName() string { // Unversioned/default requests use the system binary; vGPU snapshots opt in to a managed runtime. func (s *Starter) GetBinaryPath(p *paths.Paths, version string) (string, error) { if version == VGPUSnapshotVersion { - binary, _, err := loadVGPURuntime(p) - return binary, err + rt, err := loadVGPURuntime(p) + return rt.binary, err } binaryName, err := qemuBinaryName() if err != nil { diff --git a/lib/hypervisor/qemu/qemu.go b/lib/hypervisor/qemu/qemu.go index ab7e4ce7c..6029cc9f7 100644 --- a/lib/hypervisor/qemu/qemu.go +++ b/lib/hypervisor/qemu/qemu.go @@ -151,6 +151,9 @@ func (q *QEMU) Snapshot(ctx context.Context, destPath string) error { if state != qemu.StatusRunning { return fmt.Errorf("vGPU snapshot requires a running guest") } + if err := removeSnapshotMemory(memoryFile); err != nil { + return err + } if err := q.client.configureVGPUMigration(); err != nil { return err } @@ -164,8 +167,10 @@ func (q *QEMU) Snapshot(ctx context.Context, destPath string) error { // Wait for migration to complete if err := q.client.WaitMigration(ctx, migrationTimeout); err != nil { if saved.VGPUSnapshot { + // The guest was never paused, so stop the stream rather than + // leave it migrating into a file nobody will load. if cancelErr := q.client.cancelMigration(); cancelErr != nil { - return fmt.Errorf("snapshot failed: %w; cancel migration: %v", err, cancelErr) + err = fmt.Errorf("%w; cancel migration: %v", err, cancelErr) } } removeClient(q) diff --git a/lib/hypervisor/qemu/vgpu.go b/lib/hypervisor/qemu/vgpu.go index 7d2ab4042..57e76a581 100644 --- a/lib/hypervisor/qemu/vgpu.go +++ b/lib/hypervisor/qemu/vgpu.go @@ -12,6 +12,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "github.com/digitalocean/go-qemu/qmp" "github.com/google/uuid" @@ -26,6 +27,10 @@ const ( vgpuUpstreamVersion = "11.1.3" vgpuSourceCommit = "005c61c0ef6f6ae615283440b4bb7be51b2c6ddd" vgpuMachineType MachineType = "pc-q35-8.2" + + vgpuRuntimeBinaryName = "qemu-system-x86_64" + vgpuRuntimeFirmwareDir = "firmware" + vgpuRuntimeManifestName = "manifest.json" ) type vgpuRuntimeManifest struct { @@ -35,6 +40,14 @@ type vgpuRuntimeManifest struct { FirmwareSHA256 string `json:"firmware_sha256"` } +// vgpuRuntime is a managed QEMU installation whose contents matched its manifest. +type vgpuRuntime struct { + binary string + firmware string + manifest vgpuRuntimeManifest +} + +// vgpuSnapshotContract is the host state a vGPU snapshot can only be loaded on. type vgpuSnapshotContract struct { Runtime vgpuRuntimeManifest DriverVersion string @@ -43,11 +56,27 @@ type vgpuSnapshotContract struct { ProfileType string } +// vgpuRuntimeCache holds the last verified runtime per installation +// directory. Hashing the binary and firmware costs tens of milliseconds and +// every start and restore resolves the runtime several times, so a +// verification is reused while each file's size and modification time are +// unchanged. Installations are immutable once placed; a replaced file is +// hashed again. +var vgpuRuntimeCache struct { + sync.Mutex + entries map[string]vgpuRuntimeCacheEntry +} + +type vgpuRuntimeCacheEntry struct { + fingerprint string + runtime vgpuRuntime +} + func vgpuRuntimeDir(p *paths.Paths) (string, error) { if p == nil || runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { return "", fmt.Errorf("vGPU snapshots require a managed Linux amd64 runtime") } - return filepath.Join(p.DataDir(), "system", "binaries", "qemu", VGPUSnapshotVersion, "x86_64"), nil + return p.QEMURuntimeDir(VGPUSnapshotVersion, "x86_64"), nil } func fileSHA256(path string) (string, error) { @@ -63,6 +92,8 @@ func fileSHA256(path string) (string, error) { return hex.EncodeToString(h.Sum(nil)), nil } +// firmwareSHA256 hashes every firmware file with its name and length, in +// name order, so a renamed, added or removed blob changes the digest. func firmwareSHA256(dir string) (string, error) { entries, err := os.ReadDir(dir) if err != nil { @@ -97,60 +128,113 @@ func firmwareSHA256(dir string) (string, error) { return hex.EncodeToString(h.Sum(nil)), nil } -func loadVGPURuntime(p *paths.Paths) (string, vgpuRuntimeManifest, error) { - dir, err := vgpuRuntimeDir(p) +// vgpuRuntimeFingerprint summarizes the installed files without reading them. +func vgpuRuntimeFingerprint(dir string) (string, error) { + var b strings.Builder + add := func(path string) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + fmt.Fprintf(&b, "%s\x00%d\x00%d\x00%o\n", path, info.Size(), info.ModTime().UnixNano(), info.Mode()) + return nil + } + for _, name := range []string{vgpuRuntimeManifestName, vgpuRuntimeBinaryName} { + if err := add(filepath.Join(dir, name)); err != nil { + return "", err + } + } + firmware := filepath.Join(dir, vgpuRuntimeFirmwareDir) + entries, err := os.ReadDir(firmware) if err != nil { - return "", vgpuRuntimeManifest{}, err + return "", err } - var manifest vgpuRuntimeManifest - data, err := os.ReadFile(filepath.Join(dir, "manifest.json")) + for _, entry := range entries { + if err := add(filepath.Join(firmware, entry.Name())); err != nil { + return "", err + } + } + return b.String(), nil +} + +func loadVGPURuntime(p *paths.Paths) (vgpuRuntime, error) { + dir, err := vgpuRuntimeDir(p) if err != nil { - return "", manifest, fmt.Errorf("read managed QEMU runtime manifest: %w", err) + return vgpuRuntime{}, err } - if err := json.Unmarshal(data, &manifest); err != nil { - return "", manifest, fmt.Errorf("decode managed QEMU runtime manifest: %w", err) + fingerprint, err := vgpuRuntimeFingerprint(dir) + if err != nil { + return vgpuRuntime{}, fmt.Errorf("inspect managed QEMU runtime: %w", err) } - if manifest.Version != VGPUSnapshotVersion || manifest.SourceCommit != vgpuSourceCommit { - return "", manifest, fmt.Errorf("unsupported managed QEMU runtime manifest") + vgpuRuntimeCache.Lock() + defer vgpuRuntimeCache.Unlock() + if entry, ok := vgpuRuntimeCache.entries[dir]; ok && entry.fingerprint == fingerprint { + return entry.runtime, nil } - for name, expected := range map[string]string{"qemu-system-x86_64": manifest.BinarySHA256, "firmware": manifest.FirmwareSHA256} { - var actual string - var err error - if name == "firmware" { - actual, err = firmwareSHA256(filepath.Join(dir, name)) - } else { - actual, err = fileSHA256(filepath.Join(dir, name)) - } - if err != nil { - return "", manifest, fmt.Errorf("verify managed runtime %s: %w", name, err) - } - if len(expected) != 64 || actual != expected { - return "", manifest, fmt.Errorf("managed runtime %s checksum mismatch", name) - } + rt, err := verifyVGPURuntime(dir) + if err != nil { + delete(vgpuRuntimeCache.entries, dir) + return vgpuRuntime{}, err } - binary := filepath.Join(dir, "qemu-system-x86_64") - if err := validateExecutable(binary); err != nil { - return "", manifest, fmt.Errorf("managed QEMU binary: %w", err) + if vgpuRuntimeCache.entries == nil { + vgpuRuntimeCache.entries = make(map[string]vgpuRuntimeCacheEntry) } - return binary, manifest, nil + vgpuRuntimeCache.entries[dir] = vgpuRuntimeCacheEntry{fingerprint: fingerprint, runtime: rt} + return rt, nil } -func (s *Starter) resolveVGPURuntime(p *paths.Paths) (string, error) { - if s.profile.hypervisorType() != hypervisor.TypeQEMU { - return "", fmt.Errorf("vGPU snapshot runtime requires standard QEMU") +// verifyVGPURuntime checks the installation against its manifest and confirms +// the binary reports the pinned upstream version. +func verifyVGPURuntime(dir string) (vgpuRuntime, error) { + rt := vgpuRuntime{ + binary: filepath.Join(dir, vgpuRuntimeBinaryName), + firmware: filepath.Join(dir, vgpuRuntimeFirmwareDir), } - binary, _, err := loadVGPURuntime(p) + data, err := os.ReadFile(filepath.Join(dir, vgpuRuntimeManifestName)) if err != nil { - return "", err + return vgpuRuntime{}, fmt.Errorf("read managed QEMU runtime manifest: %w", err) + } + if err := json.Unmarshal(data, &rt.manifest); err != nil { + return vgpuRuntime{}, fmt.Errorf("decode managed QEMU runtime manifest: %w", err) + } + if rt.manifest.Version != VGPUSnapshotVersion || rt.manifest.SourceCommit != vgpuSourceCommit { + return vgpuRuntime{}, fmt.Errorf("unsupported managed QEMU runtime manifest") + } + binarySum, err := fileSHA256(rt.binary) + if err != nil { + return vgpuRuntime{}, fmt.Errorf("verify managed runtime binary: %w", err) + } + if binarySum != rt.manifest.BinarySHA256 { + return vgpuRuntime{}, fmt.Errorf("managed runtime binary checksum mismatch") + } + firmwareSum, err := firmwareSHA256(rt.firmware) + if err != nil { + return vgpuRuntime{}, fmt.Errorf("verify managed runtime firmware: %w", err) + } + if firmwareSum != rt.manifest.FirmwareSHA256 { + return vgpuRuntime{}, fmt.Errorf("managed runtime firmware checksum mismatch") + } + if err := validateExecutable(rt.binary); err != nil { + return vgpuRuntime{}, fmt.Errorf("managed QEMU binary: %w", err) } ctx, cancel := context.WithTimeout(context.Background(), versionProbeTimeout) defer cancel() - version, err := versionFromBinary(ctx, binary) + version, err := versionFromBinary(ctx, rt.binary) if err != nil { - return "", err + return vgpuRuntime{}, err } if version != vgpuUpstreamVersion { - return "", fmt.Errorf("managed QEMU version %s does not match %s", version, vgpuUpstreamVersion) + return vgpuRuntime{}, fmt.Errorf("managed QEMU version %s does not match %s", version, vgpuUpstreamVersion) + } + return rt, nil +} + +func (s *Starter) resolveVGPURuntime(p *paths.Paths) (string, error) { + if s.profile.hypervisorType() != hypervisor.TypeQEMU { + return "", fmt.Errorf("vGPU snapshot runtime requires standard QEMU") + } + if _, err := loadVGPURuntime(p); err != nil { + return "", err } return VGPUSnapshotVersion, nil } @@ -162,7 +246,7 @@ func newVGPUSnapshotContract(p *paths.Paths, config hypervisor.VMConfig) (*vgpuS if !strings.HasPrefix(config.VGPUDevicePath, "/sys/bus/pci/devices/") { return nil, fmt.Errorf("vGPU snapshots require vendor VFIO") } - _, manifest, err := loadVGPURuntime(p) + rt, err := loadVGPURuntime(p) if err != nil { return nil, err } @@ -177,7 +261,7 @@ func newVGPUSnapshotContract(p *paths.Paths, config hypervisor.VMConfig) (*vgpuS } return value, nil } - contract := &vgpuSnapshotContract{Runtime: manifest} + contract := &vgpuSnapshotContract{Runtime: rt.manifest} if contract.DriverVersion, err = read("/sys/module/nvidia/version"); err != nil { return nil, fmt.Errorf("read NVIDIA host driver: %w", err) } @@ -207,11 +291,21 @@ func validateVGPUSnapshotContract(saved, current *vgpuSnapshotContract) error { } func vgpuRuntimeArgs(p *paths.Paths) ([]string, error) { - dir, err := vgpuRuntimeDir(p) + rt, err := loadVGPURuntime(p) if err != nil { return nil, err } - return []string{"-L", filepath.Join(dir, "firmware"), "-bios", filepath.Join(dir, "firmware", "bios-256k.bin")}, nil + return []string{"-L", rt.firmware, "-bios", filepath.Join(rt.firmware, "bios-256k.bin")}, nil +} + +// removeSnapshotMemory unlinks a previous snapshot's memory file. QEMU writes +// mapped RAM into an existing file in place, and forks hardlink the previous +// snapshot's memory, so a new snapshot must start from a new inode. +func removeSnapshotMemory(path string) error { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove previous snapshot memory: %w", err) + } + return nil } func vgpuInstanceUUID(instanceID string) string { diff --git a/lib/hypervisor/qemu/vgpu_test.go b/lib/hypervisor/qemu/vgpu_test.go index bcb48f715..48770c385 100644 --- a/lib/hypervisor/qemu/vgpu_test.go +++ b/lib/hypervisor/qemu/vgpu_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/paths" @@ -51,6 +52,42 @@ func TestManagedVGPURuntimeIsExplicitAndChecksumPinned(t *testing.T) { require.ErrorContains(t, err, "checksum mismatch") } +func TestManagedVGPURuntimeVerificationIsReusedUntilFilesChange(t *testing.T) { + p, dir, _ := managedVGPUFixture(t) + binary := filepath.Join(dir, "qemu-system-x86_64") + info, err := os.Stat(binary) + require.NoError(t, err) + _, err = loadVGPURuntime(p) + require.NoError(t, err) + + // Same size and modification time: the earlier verification is reused. + tampered := []byte(strings.Repeat("x", int(info.Size()))) + require.NoError(t, os.WriteFile(binary, tampered, 0o755)) + require.NoError(t, os.Chtimes(binary, info.ModTime(), info.ModTime())) + _, err = loadVGPURuntime(p) + require.NoError(t, err) + + // Any change to a file's metadata forces the checksums to be recomputed. + later := info.ModTime().Add(time.Second) + require.NoError(t, os.Chtimes(binary, later, later)) + _, err = loadVGPURuntime(p) + require.ErrorContains(t, err, "binary checksum mismatch") +} + +func TestManagedVGPURuntimeRejectsUnexpectedUpstreamVersion(t *testing.T) { + p, dir, manifest := managedVGPUFixture(t) + binary := filepath.Join(dir, "qemu-system-x86_64") + require.NoError(t, os.WriteFile(binary, []byte("#!/bin/sh\necho 'QEMU emulator version 8.2.2'\n"), 0o755)) + var err error + manifest.BinarySHA256, err = fileSHA256(binary) + require.NoError(t, err) + data, err := json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.json"), data, 0o644)) + _, err = NewStarter().ResolveVersion(p, VGPUSnapshotVersion) + require.ErrorContains(t, err, "does not match") +} + func TestManagedVGPURuntimeMissingDoesNotUseSystemQEMU(t *testing.T) { _, err := NewStarter().ResolveVersion(paths.New(t.TempDir()), VGPUSnapshotVersion) require.Error(t, err) @@ -96,6 +133,21 @@ func TestVGPUForkClearsSourceDeviceAndBindsFreshIdentity(t *testing.T) { require.NotEqual(t, vgpuInstanceUUID("source"), vgpuInstanceUUID(fork.InstanceID)) } +func TestRemoveSnapshotMemoryLeavesForkLinksIntact(t *testing.T) { + dir := t.TempDir() + memory := filepath.Join(dir, "memory") + forkLink := filepath.Join(dir, "fork-memory") + require.NoError(t, os.WriteFile(memory, []byte("template memory"), 0o644)) + require.NoError(t, os.Link(memory, forkLink)) + + require.NoError(t, removeSnapshotMemory(memory)) + require.NoFileExists(t, memory) + content, err := os.ReadFile(forkLink) + require.NoError(t, err) + require.Equal(t, "template memory", string(content)) + require.NoError(t, removeSnapshotMemory(memory), "a missing file is not an error") +} + func TestVGPUArgumentsAreOptIn(t *testing.T) { cfg := hypervisor.VMConfig{VGPUSnapshot: true, InstanceID: "child", VGPUDevicePath: "/sys/bus/pci/devices/0000:01:00.1", MemoryBytes: 6 << 30, VCPUs: 4} args := strings.Join(buildArgs(cfg, vgpuMachineType), " ") diff --git a/lib/instances/firecracker_memfile.go b/lib/instances/firecracker_memfile.go index 628b40c94..e6f5a9be7 100644 --- a/lib/instances/firecracker_memfile.go +++ b/lib/instances/firecracker_memfile.go @@ -10,6 +10,7 @@ import ( "time" "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -19,13 +20,20 @@ func firecrackerSnapshotMemoryPathInGuestDir(guestDir string) string { return filepath.Join(guestDir, firecrackerSnapshotMemoryRelPath) } +// forkSharesSnapshotMemory reports whether forks of a standby snapshot +// hardlink its mem-file instead of copying it. +func forkSharesSnapshotMemory(stored *StoredMetadata) bool { + return stored.HypervisorType == hypervisor.TypeFirecracker || usesVGPUSnapshot(stored) +} + // linkForkFirecrackerMemFile hardlinks the source's snapshot mem-file into the // fork's guest dir so all forks of a snapshot share one inode: fanout costs no // copy I/O, and reads hit the kernel page cache warmed by sibling forks. Falls // back to a reflink/sparse copy when linking fails. // Sharing an inode is safe because Firecracker mmaps the mem-file MAP_PRIVATE // and the only file writer, the standby diff snapshot, unshares first via -// ensureExclusiveSnapshotMemoryOwnership. +// ensureExclusiveSnapshotMemoryOwnership. Managed QEMU vGPU restores open the +// file read-only, and their snapshots always write a newly created file. func (m *manager) linkForkFirecrackerMemFile(ctx context.Context, srcGuestDir, dstGuestDir string) error { srcMem := firecrackerSnapshotMemoryPathInGuestDir(srcGuestDir) dstMem := firecrackerSnapshotMemoryPathInGuestDir(dstGuestDir) diff --git a/lib/instances/firecracker_memfile_test.go b/lib/instances/firecracker_memfile_test.go index 57b2450a8..19c555a89 100644 --- a/lib/instances/firecracker_memfile_test.go +++ b/lib/instances/firecracker_memfile_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "testing" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/qemu" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -62,3 +64,38 @@ func TestEnsureExclusiveSnapshotMemoryOwnershipSkipsPrivateMemory(t *testing.T) require.NoError(t, ensureExclusiveSnapshotMemoryOwnership(context.Background(), filepath.Join(t.TempDir(), "missing"))) } + +func TestForkSharesSnapshotMemory(t *testing.T) { + t.Parallel() + + gpu := &StoredMetadata{HypervisorType: hypervisor.TypeQEMU, HypervisorVersion: qemu.VGPUSnapshotVersion, GPUProfile: "NVIDIA L40S-3Q"} + assert.True(t, forkSharesSnapshotMemory(&StoredMetadata{HypervisorType: hypervisor.TypeFirecracker})) + assert.True(t, forkSharesSnapshotMemory(gpu)) + // System QEMU streams its snapshot through a pipe and keeps private copies. + assert.False(t, forkSharesSnapshotMemory(&StoredMetadata{HypervisorType: hypervisor.TypeQEMU})) + assert.False(t, forkSharesSnapshotMemory(&StoredMetadata{HypervisorType: hypervisor.TypeCloudHypervisor})) +} + +func TestCloneGuestDirectoryForForkHardlinksSharedMemory(t *testing.T) { + t.Parallel() + + root := t.TempDir() + src := filepath.Join(root, "template") + dst := filepath.Join(root, "fork") + srcMem := firecrackerSnapshotMemoryPathInGuestDir(src) + require.NoError(t, os.MkdirAll(filepath.Dir(srcMem), 0755)) + require.NoError(t, os.WriteFile(srcMem, []byte("guest memory"), 0644)) + srcConfig := filepath.Join(filepath.Dir(srcMem), "qemu-config.json") + require.NoError(t, os.WriteFile(srcConfig, []byte("{}"), 0644)) + + m := &manager{} + require.NoError(t, m.cloneGuestDirectoryForFork(context.Background(), src, dst, true)) + + dstMem := firecrackerSnapshotMemoryPathInGuestDir(dst) + srcInfo, err := os.Stat(srcMem) + require.NoError(t, err) + dstInfo, err := os.Stat(dstMem) + require.NoError(t, err) + assert.True(t, os.SameFile(srcInfo, dstInfo), "fork must share the template's memory inode") + assertDifferentInode(t, srcConfig, filepath.Join(filepath.Dir(dstMem), "qemu-config.json")) +} diff --git a/lib/instances/fork.go b/lib/instances/fork.go index e13a4ba68..4dc6e37ae 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -263,7 +263,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin srcDir := m.paths.InstanceDir(id) dstDir := m.paths.InstanceDir(forkID) - shareMemFile := stored.HypervisorType == hypervisor.TypeFirecracker && source.State == StateStandby + shareMemFile := source.State == StateStandby && forkSharesSnapshotMemory(stored) cu := cleanup.Make(func() { _ = os.RemoveAll(dstDir) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 7ca6fed72..c2fd811a2 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -495,7 +495,7 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if target != nil && target.State == compressionJobStateRunning { m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionForkSnapshot, target.Target) } - shareMemFile := rec.StoredMetadata.HypervisorType == hypervisor.TypeFirecracker && rec.Snapshot.Kind == SnapshotKindStandby + shareMemFile := rec.Snapshot.Kind == SnapshotKindStandby && forkSharesSnapshotMemory(&rec.StoredMetadata) if err := m.copySnapshotGuestDirectoryForFork(ctx, snapshotID, rec.StoredMetadata.HypervisorType, dstDir, shareMemFile); err != nil { return nil, err } diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 40ea60931..5c0becc27 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -262,9 +262,11 @@ func (m *manager) standbyInstance( } } stored.Phases.Record(phasetracking.PhaseStandby, now) - if usesVGPUSnapshot(stored) { + // The VMM has exited, so its vGPU can serve another guest; restore claims + // a fresh one. A failed release keeps the claim for the reconciler. + if path := storedVGPUDevicePath(stored); path != "" { if err := m.releaseStoredVGPU(ctx, stored); err != nil { - return &Instance{StoredMetadata: *stored}, fmt.Errorf("release vGPU after snapshot: %w", err) + log.WarnContext(ctx, "failed to release vGPU after standby; retaining assignment metadata", "instance_id", id, "device_path", path, "error", err) } } diff --git a/lib/instances/vgpu_snapshot.go b/lib/instances/vgpu_snapshot.go index de4c8583e..557142c1a 100644 --- a/lib/instances/vgpu_snapshot.go +++ b/lib/instances/vgpu_snapshot.go @@ -42,6 +42,10 @@ func (m *manager) shutdownFailedRestore(ctx context.Context, stored *StoredMetad return m.shutdownHypervisor(cleanupCtx, &Instance{StoredMetadata: *stored}) } +// vgpuSnapshotProfile is the only vGPU profile whose device state has been +// qualified for capture and restore. +const vgpuSnapshotProfile = "NVIDIA L40S-3Q" + func usesVGPUSnapshot(stored *StoredMetadata) bool { return stored.HypervisorType == hypervisor.TypeQEMU && stored.HypervisorVersion == qemu.VGPUSnapshotVersion && stored.GPUProfile != "" } @@ -50,15 +54,15 @@ func validateVGPUSnapshotRequest(req CreateInstanceRequest, hvType hypervisor.Ty if version != qemu.VGPUSnapshotVersion { return nil } - if hvType != hypervisor.TypeQEMU || req.GPU == nil || req.GPU.Profile != "NVIDIA L40S-3Q" || req.SkipGuestAgent || len(req.Devices) != 0 { - return fmt.Errorf("%w: the managed vGPU snapshot runtime requires an L40S-3Q guest with its guest agent and no additional PCI devices", ErrInvalidRequest) + if hvType != hypervisor.TypeQEMU || req.GPU == nil || req.GPU.Profile != vgpuSnapshotProfile || req.SkipGuestAgent || len(req.Devices) != 0 { + return fmt.Errorf("%w: the managed vGPU snapshot runtime requires a %s guest with its guest agent and no additional PCI devices", ErrInvalidRequest, vgpuSnapshotProfile) } return nil } func validateVGPUSnapshotCapture(stored *StoredMetadata) error { - if !usesVGPUSnapshot(stored) || stored.GPUFramework != devices.VGPUFrameworkVendorVFIO || stored.GPUMdevUUID != "" || stored.GPUProfile != "NVIDIA L40S-3Q" || stored.SkipGuestAgent { - return fmt.Errorf("%w: vGPU snapshots require the opt-in managed QEMU runtime and vendor VFIO L40S-3Q", ErrInvalidState) + if !usesVGPUSnapshot(stored) || stored.GPUFramework != devices.VGPUFrameworkVendorVFIO || stored.GPUMdevUUID != "" || stored.GPUProfile != vgpuSnapshotProfile || stored.SkipGuestAgent { + return fmt.Errorf("%w: vGPU snapshots require the opt-in managed QEMU runtime and a vendor VFIO %s", ErrInvalidState, vgpuSnapshotProfile) } return nil } diff --git a/lib/paths/paths.go b/lib/paths/paths.go index b8cc331b6..ef5d05787 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -115,6 +115,11 @@ func (p *Paths) FirecrackerBinary(version, arch string) string { return filepath.Join(p.dataDir, "system", "binaries", "firecracker", version, arch, "firecracker") } +// QEMURuntimeDir returns the directory of a managed QEMU runtime installation. +func (p *Paths) QEMURuntimeDir(version, arch string) string { + return filepath.Join(p.dataDir, "system", "binaries", "qemu", version, arch) +} + // Image path methods // ImageContentDir returns the directory for content-addressed image data. From 24fe8913787e2b24bd1fbdfa1042f8fe63fb8a3c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:18:29 +0000 Subject: [PATCH 3/9] Populate vGPU guest RAM with parallel prealloc threads --- lib/hypervisor/qemu/config.go | 10 ++++++++-- lib/hypervisor/qemu/vgpu.go | 6 ++++++ lib/hypervisor/qemu/vgpu_test.go | 6 +++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/hypervisor/qemu/config.go b/lib/hypervisor/qemu/config.go index cb1c632f9..4240890a2 100644 --- a/lib/hypervisor/qemu/config.go +++ b/lib/hypervisor/qemu/config.go @@ -26,7 +26,7 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string { // Machine type with KVM acceleration (arch-specific when omitted). machineArg := string(machine) + ",accel=kvm" if cfg.VGPUSnapshot { - machineArg += ",smbios-entry-point-type=32" + machineArg += ",smbios-entry-point-type=32,memory-backend=" + vgpuRAMBackendID } if cfg.Firmware != nil && cfg.Firmware.SecureBoot { machineArg += ",smm=on" @@ -51,8 +51,14 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string { memMB := cfg.MemoryBytes / (1024 * 1024) args = append(args, "-m", fmt.Sprintf("%dM", memMB)) if cfg.VGPUSnapshot { + // VFIO pins every guest page when the device attaches. Populating RAM + // first, in parallel, is much cheaper than faulting it in during the + // pin; the default backend would only use one thread per vCPU. + args = append(args, "-object", fmt.Sprintf( + "memory-backend-ram,id=%s,size=%dM,prealloc=on,prealloc-threads=%d,x-use-canonical-path-for-ramblock-id=off", + vgpuRAMBackendID, memMB, vgpuPreallocThreads)) id := vgpuInstanceUUID(cfg.InstanceID) - args = append(args, "-mem-prealloc", "-device", "vmgenid,guid=auto", "-uuid", id, "-smbios", "type=1,uuid="+id) + args = append(args, "-device", "vmgenid,guid=auto", "-uuid", id, "-smbios", "type=1,uuid="+id) } if cfg.GuestMemory.EnableBalloon { diff --git a/lib/hypervisor/qemu/vgpu.go b/lib/hypervisor/qemu/vgpu.go index 57e76a581..6195f7f1f 100644 --- a/lib/hypervisor/qemu/vgpu.go +++ b/lib/hypervisor/qemu/vgpu.go @@ -28,6 +28,12 @@ const ( vgpuSourceCommit = "005c61c0ef6f6ae615283440b4bb7be51b2c6ddd" vgpuMachineType MachineType = "pc-q35-8.2" + // vgpuRAMBackendID is q35's default RAM block id, so the explicit backend + // keeps the name an implicit one would give guest RAM in the stream. + vgpuRAMBackendID = "pc.ram" + // vgpuPreallocThreads matches QEMU's own upper bound for populating RAM. + vgpuPreallocThreads = 16 + vgpuRuntimeBinaryName = "qemu-system-x86_64" vgpuRuntimeFirmwareDir = "firmware" vgpuRuntimeManifestName = "manifest.json" diff --git a/lib/hypervisor/qemu/vgpu_test.go b/lib/hypervisor/qemu/vgpu_test.go index 48770c385..b2e6eea5e 100644 --- a/lib/hypervisor/qemu/vgpu_test.go +++ b/lib/hypervisor/qemu/vgpu_test.go @@ -151,13 +151,13 @@ func TestRemoveSnapshotMemoryLeavesForkLinksIntact(t *testing.T) { func TestVGPUArgumentsAreOptIn(t *testing.T) { cfg := hypervisor.VMConfig{VGPUSnapshot: true, InstanceID: "child", VGPUDevicePath: "/sys/bus/pci/devices/0000:01:00.1", MemoryBytes: 6 << 30, VCPUs: 4} args := strings.Join(buildArgs(cfg, vgpuMachineType), " ") - require.Contains(t, args, "pc-q35-8.2,accel=kvm,smbios-entry-point-type=32") - require.Contains(t, args, "-mem-prealloc") + require.Contains(t, args, "pc-q35-8.2,accel=kvm,smbios-entry-point-type=32,memory-backend=pc.ram") + require.Contains(t, args, "-object memory-backend-ram,id=pc.ram,size=6144M,prealloc=on,prealloc-threads=16,x-use-canonical-path-for-ramblock-id=off") require.Contains(t, args, "vmgenid,guid=auto") require.Contains(t, args, "enable-migration=on") cfg.VGPUSnapshot = false args = strings.Join(buildArgs(cfg, MachineTypeQ35), " ") require.NotContains(t, args, "enable-migration") - require.NotContains(t, args, "-mem-prealloc") + require.NotContains(t, args, "memory-backend") require.NotContains(t, args, "vmgenid") } From 717d93d5539237c9f46b0968c39047fd3d9c282f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:18:29 +0000 Subject: [PATCH 4/9] Disable self-announce for vGPU snapshot restores --- lib/hypervisor/qemu/vgpu.go | 27 +++++++++++++++++++++++---- lib/hypervisor/qemu/vgpu_test.go | 12 ++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/lib/hypervisor/qemu/vgpu.go b/lib/hypervisor/qemu/vgpu.go index 6195f7f1f..76026fc80 100644 --- a/lib/hypervisor/qemu/vgpu.go +++ b/lib/hypervisor/qemu/vgpu.go @@ -318,11 +318,30 @@ func vgpuInstanceUUID(instanceID string) string { return uuid.NewSHA1(uuid.NameSpaceOID, []byte(instanceID)).String() } +// vgpuMigrationCommands configures either side of a vGPU snapshot stream. +// +// Self-announcement is disabled: a restored guest is given a new MAC and +// address straight away, and each announce round raises a virtio-net config +// interrupt whose handler takes rtnl in the guest. Linux's virtio-net waits +// for that handler while closing the interface with rtnl held, so an announce +// landing during the guest agent's link-down deadlocks the guest's networking. +func vgpuMigrationCommands() []qmp.Command { + return []qmp.Command{ + {Execute: "migrate-set-capabilities", Args: map[string]any{"capabilities": []map[string]any{ + {"capability": "mapped-ram", "state": true}, + {"capability": "multifd", "state": true}, + }}}, + {Execute: "migrate-set-parameters", Args: map[string]any{ + "multifd-channels": 4, + "max-bandwidth": int64(1 << 30), + "downtime-limit": 10000, + "announce-rounds": 0, + }}, + } +} + func (c *Client) configureVGPUMigration() error { - for _, command := range []qmp.Command{ - {Execute: "migrate-set-capabilities", Args: map[string]any{"capabilities": []map[string]any{{"capability": "mapped-ram", "state": true}, {"capability": "multifd", "state": true}}}}, - {Execute: "migrate-set-parameters", Args: map[string]any{"multifd-channels": 4, "max-bandwidth": int64(1 << 30), "downtime-limit": 10000}}, - } { + for _, command := range vgpuMigrationCommands() { if _, err := c.Run(command); err != nil { return fmt.Errorf("configure GPU migration: %w", err) } diff --git a/lib/hypervisor/qemu/vgpu_test.go b/lib/hypervisor/qemu/vgpu_test.go index b2e6eea5e..e1864ea60 100644 --- a/lib/hypervisor/qemu/vgpu_test.go +++ b/lib/hypervisor/qemu/vgpu_test.go @@ -148,6 +148,18 @@ func TestRemoveSnapshotMemoryLeavesForkLinksIntact(t *testing.T) { require.NoError(t, removeSnapshotMemory(memory), "a missing file is not an error") } +func TestVGPUMigrationNeverSelfAnnounces(t *testing.T) { + // An announce round arriving while the guest agent takes the restored + // interface down deadlocks the guest's network stack. + var parameters map[string]any + for _, command := range vgpuMigrationCommands() { + if command.Execute == "migrate-set-parameters" { + parameters = command.Args.(map[string]any) + } + } + require.Equal(t, 0, parameters["announce-rounds"]) +} + func TestVGPUArgumentsAreOptIn(t *testing.T) { cfg := hypervisor.VMConfig{VGPUSnapshot: true, InstanceID: "child", VGPUDevicePath: "/sys/bus/pci/devices/0000:01:00.1", MemoryBytes: 6 << 30, VCPUs: 4} args := strings.Join(buildArgs(cfg, vgpuMachineType), " ") From 4bda16e8329d1f4c981a33d7c3ac33fbde7ae2b8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:19:17 +0000 Subject: [PATCH 5/9] Bound the guest agent wait on vGPU snapshot restores --- lib/instances/restore.go | 5 ++++- lib/instances/vgpu_restore_slots_test.go | 14 ++++++++++++++ lib/instances/vgpu_snapshot.go | 13 +++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 8b9eca507..4bdc0ad17 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -378,7 +378,10 @@ func (m *manager) restoreInstance( attribute.String("hypervisor", string(stored.HypervisorType)), attribute.String("operation", "reconfigure_guest_network"), ) - if err := reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet); err != nil { + reconfigureCtx, cancelReconfigure := guestReconfigureContext(reconfigureCtx, stored) + err := reconfigureGuestNetwork(reconfigureCtx, stored, allocatedNet) + cancelReconfigure() + if err != nil { reconfigureSpanEnd(err) log.ErrorContext(ctx, "failed to configure guest network after restore", "instance_id", id, "error", err) _ = m.shutdownFailedRestore(ctx, stored, hv) diff --git a/lib/instances/vgpu_restore_slots_test.go b/lib/instances/vgpu_restore_slots_test.go index fec631dd3..dffcf265b 100644 --- a/lib/instances/vgpu_restore_slots_test.go +++ b/lib/instances/vgpu_restore_slots_test.go @@ -31,3 +31,17 @@ func TestVGPURestoreSlotsDoNotThrottleCPUQEMU(t *testing.T) { func TestVGPURestoreSlotsDoNotReuseTheGenericQEMUPool(t *testing.T) { require.NotEqual(t, sharedRestoreSlots(hypervisor.TypeQEMU, 2), sharedVGPURestoreSlots(2)) } + +func TestGuestReconfigureIsBoundedOnlyForVGPURestores(t *testing.T) { + gpu := &StoredMetadata{HypervisorType: hypervisor.TypeQEMU, HypervisorVersion: qemu.VGPUSnapshotVersion, GPUProfile: "NVIDIA L40S-3Q"} + ctx, cancel := guestReconfigureContext(t.Context(), gpu) + defer cancel() + deadline, ok := ctx.Deadline() + require.True(t, ok) + require.WithinDuration(t, time.Now().Add(vgpuRestoreAgentTimeout), deadline, time.Second) + + ctx, cancel = guestReconfigureContext(t.Context(), &StoredMetadata{HypervisorType: hypervisor.TypeFirecracker}) + defer cancel() + _, ok = ctx.Deadline() + require.False(t, ok) +} diff --git a/lib/instances/vgpu_snapshot.go b/lib/instances/vgpu_snapshot.go index 557142c1a..9c41da604 100644 --- a/lib/instances/vgpu_snapshot.go +++ b/lib/instances/vgpu_snapshot.go @@ -42,6 +42,19 @@ func (m *manager) shutdownFailedRestore(ctx context.Context, stored *StoredMetad return m.shutdownHypervisor(cleanupCtx, &Instance{StoredMetadata: *stored}) } +// vgpuRestoreAgentTimeout bounds how long a restored vGPU guest's agent may +// take to apply its new network identity. The agent was already running when +// the snapshot was taken, so it answers in milliseconds or not at all; failing +// here leaves the caller time to boot a fresh guest instead. +const vgpuRestoreAgentTimeout = 10 * time.Second + +func guestReconfigureContext(ctx context.Context, stored *StoredMetadata) (context.Context, context.CancelFunc) { + if !usesVGPUSnapshot(stored) { + return ctx, func() {} + } + return context.WithTimeout(ctx, vgpuRestoreAgentTimeout) +} + // vgpuSnapshotProfile is the only vGPU profile whose device state has been // qualified for capture and restore. const vgpuSnapshotProfile = "NVIDIA L40S-3Q" From b4c33a25f34408ed1ae007b1f8b22a02843b90c7 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 10 Nov 2026 03:21:57 +0000 Subject: [PATCH 6/9] Re-verify rewritten runtime files and queue vGPU restores on their own slot first --- cmd/api/config/config.go | 2 +- config.example.yaml | 1 + lib/hypervisor/config.go | 8 ++++-- lib/hypervisor/hypervisor.go | 7 ++++-- lib/hypervisor/qemu/fork.go | 8 +++--- lib/hypervisor/qemu/process.go | 2 ++ lib/hypervisor/qemu/qemu.go | 4 +++ lib/hypervisor/qemu/vgpu.go | 8 +++--- lib/hypervisor/qemu/vgpu_stat_linux.go | 18 ++++++++++++++ lib/hypervisor/qemu/vgpu_stat_other.go | 10 ++++++++ lib/hypervisor/qemu/vgpu_test.go | 34 ++++++++++++++++++-------- lib/instances/fork.go | 8 ++++-- lib/instances/standby.go | 3 ++- lib/instances/vgpu_snapshot.go | 16 ++++++------ 14 files changed, 96 insertions(+), 33 deletions(-) create mode 100644 lib/hypervisor/qemu/vgpu_stat_linux.go create mode 100644 lib/hypervisor/qemu/vgpu_stat_other.go diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index ded87d2b3..0e3c5e768 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -718,7 +718,7 @@ func (c *Config) Validate() error { c.Builders.IdleTTLDuration = ttl } if c.Hypervisor.VGPUMaxConcurrentRestores < 0 { - return fmt.Errorf("hypervisor.vgpu_max_concurrent_restores must be >= 0") + return fmt.Errorf("hypervisor.vgpu_max_concurrent_restores must be >= 0, got %d", c.Hypervisor.VGPUMaxConcurrentRestores) } if c.Hypervisor.FirecrackerMaxConcurrentRestores < 0 { return fmt.Errorf("hypervisor.firecracker_max_concurrent_restores must be >= 0, got %d", c.Hypervisor.FirecrackerMaxConcurrentRestores) diff --git a/config.example.yaml b/config.example.yaml index 20eb5e5aa..b21063cf0 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -40,6 +40,7 @@ data_dir: /var/lib/hypeman # # firecracker_binary_path: /usr/local/bin/firecracker # # Caps concurrent Firecracker snapshot restore/resume work to avoid host overload. # # firecracker_max_concurrent_restores: 32 +# # Caps concurrent vGPU snapshot restores, which each pin all guest memory. # # vgpu_max_concurrent_restores: 2 # memory: # enabled: false diff --git a/lib/hypervisor/config.go b/lib/hypervisor/config.go index 23356e5dd..230099ca8 100644 --- a/lib/hypervisor/config.go +++ b/lib/hypervisor/config.go @@ -26,8 +26,12 @@ type VMConfig struct { // PCI device passthrough (GPU, etc.) PCIDevices []string VGPUDevicePath string - VGPUSnapshot bool - InstanceID string + // VGPUSnapshot launches the guest so its vGPU state can be captured and + // restored. Only QEMU's managed vGPU runtime supports it. + VGPUSnapshot bool + // InstanceID identifies the guest to backends that derive guest-visible + // identity from it. + InstanceID string // Boot configuration. Empty BootMode preserves the existing direct-kernel // behavior for Linux callers. diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index 284a79d13..d5d902384 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -272,8 +272,11 @@ type ForkPrepareRequest struct { VsockCID int64 VsockSocket string - SerialLogPath string - Network *ForkNetworkConfig + SerialLogPath string + Network *ForkNetworkConfig + + // InstanceID is the fork's own id. VGPUDevicePath, when set, replaces the + // vGPU the snapshot was captured with. InstanceID string VGPUDevicePath string } diff --git a/lib/hypervisor/qemu/fork.go b/lib/hypervisor/qemu/fork.go index 39c5c010b..9427a5c40 100644 --- a/lib/hypervisor/qemu/fork.go +++ b/lib/hypervisor/qemu/fork.go @@ -32,16 +32,16 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq return hypervisor.ForkPrepareResult{}, fmt.Errorf("select qemu fork machine type: %w", err) } cfg := saved.VMConfig + if req.InstanceID != "" { + cfg.InstanceID = req.InstanceID + } if cfg.VGPUSnapshot { - if req.InstanceID != "" { - cfg.InstanceID = req.InstanceID - } if req.VGPUDevicePath != "" { cfg.VGPUDevicePath = req.VGPUDevicePath } if req.TargetDataDir != "" && req.TargetDataDir != req.SourceDataDir { // A fork never opens its source's VF: restore claims a fresh one - // and binds it here before launching QEMU. + // and records it with a second PrepareFork before launching QEMU. cfg.VGPUDevicePath = "" } } diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index ccfcddf9d..9d591d978 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -705,6 +705,8 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, if saved.VGPU != nil || saved.QEMUVersion == VGPUSnapshotVersion { return 0, nil, fmt.Errorf("vGPU snapshot is missing its device configuration") } + // Add incoming migration flag to restore from snapshot + // The "file:" protocol is deprecated in QEMU 7.2+, use "exec:cat < path" instead args = append(args, "-incoming", "exec:cat < "+memoryFile) } diff --git a/lib/hypervisor/qemu/qemu.go b/lib/hypervisor/qemu/qemu.go index 6029cc9f7..fcb276efc 100644 --- a/lib/hypervisor/qemu/qemu.go +++ b/lib/hypervisor/qemu/qemu.go @@ -138,6 +138,8 @@ func (q *QEMU) Snapshot(ctx context.Context, destPath string) error { if err != nil { return fmt.Errorf("load snapshot configuration: %w", err) } + // QEMU uses migrate to file for snapshots + // The "file:" protocol is deprecated in QEMU 7.2+, use "exec:cat > path" instead memoryFile := filepath.Join(destPath, "memory") uri := "exec:cat > " + memoryFile if saved.VGPUSnapshot { @@ -157,6 +159,8 @@ func (q *QEMU) Snapshot(ctx context.Context, destPath string) error { if err := q.client.configureVGPUMigration(); err != nil { return err } + // Mapped RAM needs a seekable target, which the managed runtime's + // "file:" transport provides. uri = "file:" + memoryFile } if err := q.client.Migrate(uri); err != nil { diff --git a/lib/hypervisor/qemu/vgpu.go b/lib/hypervisor/qemu/vgpu.go index 76026fc80..60e7365f3 100644 --- a/lib/hypervisor/qemu/vgpu.go +++ b/lib/hypervisor/qemu/vgpu.go @@ -65,9 +65,8 @@ type vgpuSnapshotContract struct { // vgpuRuntimeCache holds the last verified runtime per installation // directory. Hashing the binary and firmware costs tens of milliseconds and // every start and restore resolves the runtime several times, so a -// verification is reused while each file's size and modification time are -// unchanged. Installations are immutable once placed; a replaced file is -// hashed again. +// verification is reused until a file is replaced or written: either one +// changes its inode or change time, which a caller cannot set. var vgpuRuntimeCache struct { sync.Mutex entries map[string]vgpuRuntimeCacheEntry @@ -142,7 +141,8 @@ func vgpuRuntimeFingerprint(dir string) (string, error) { if err != nil { return err } - fmt.Fprintf(&b, "%s\x00%d\x00%d\x00%o\n", path, info.Size(), info.ModTime().UnixNano(), info.Mode()) + inode, changed := fileChangeIdentity(info) + fmt.Fprintf(&b, "%s\x00%d\x00%d\x00%o\x00%d\x00%d\n", path, info.Size(), info.ModTime().UnixNano(), info.Mode(), inode, changed) return nil } for _, name := range []string{vgpuRuntimeManifestName, vgpuRuntimeBinaryName} { diff --git a/lib/hypervisor/qemu/vgpu_stat_linux.go b/lib/hypervisor/qemu/vgpu_stat_linux.go new file mode 100644 index 000000000..85b26cc7d --- /dev/null +++ b/lib/hypervisor/qemu/vgpu_stat_linux.go @@ -0,0 +1,18 @@ +//go:build linux + +package qemu + +import ( + "os" + "syscall" +) + +// fileChangeIdentity returns the inode and change time of a file. Unlike the +// modification time, neither can be restored after the file is rewritten. +func fileChangeIdentity(info os.FileInfo) (inode uint64, changedNanos int64) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0 + } + return stat.Ino, stat.Ctim.Nano() +} diff --git a/lib/hypervisor/qemu/vgpu_stat_other.go b/lib/hypervisor/qemu/vgpu_stat_other.go new file mode 100644 index 000000000..6320af5f9 --- /dev/null +++ b/lib/hypervisor/qemu/vgpu_stat_other.go @@ -0,0 +1,10 @@ +//go:build !linux + +package qemu + +import "os" + +// The managed runtime only exists on Linux. +func fileChangeIdentity(os.FileInfo) (inode uint64, changedNanos int64) { + return 0, 0 +} diff --git a/lib/hypervisor/qemu/vgpu_test.go b/lib/hypervisor/qemu/vgpu_test.go index e1864ea60..4e171d20e 100644 --- a/lib/hypervisor/qemu/vgpu_test.go +++ b/lib/hypervisor/qemu/vgpu_test.go @@ -53,24 +53,38 @@ func TestManagedVGPURuntimeIsExplicitAndChecksumPinned(t *testing.T) { } func TestManagedVGPURuntimeVerificationIsReusedUntilFilesChange(t *testing.T) { - p, dir, _ := managedVGPUFixture(t) + p, dir, manifest := managedVGPUFixture(t) binary := filepath.Join(dir, "qemu-system-x86_64") - info, err := os.Stat(binary) + probes := filepath.Join(t.TempDir(), "probes") + script := "#!/bin/sh\necho probe >> " + probes + "\necho 'QEMU emulator version 11.1.3'\n" + require.NoError(t, os.WriteFile(binary, []byte(script), 0o755)) + var err error + manifest.BinarySHA256, err = fileSHA256(binary) require.NoError(t, err) - _, err = loadVGPURuntime(p) + data, err := json.Marshal(manifest) require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.json"), data, 0o644)) + verifications := func() int { + content, err := os.ReadFile(probes) + require.NoError(t, err) + return strings.Count(string(content), "probe") + } + + for range 3 { + _, err = loadVGPURuntime(p) + require.NoError(t, err) + } + require.Equal(t, 1, verifications(), "an unchanged installation is verified once") - // Same size and modification time: the earlier verification is reused. + // Rewriting a file with the same size and modification time still + // forces the checksums to be recomputed. + info, err := os.Stat(binary) + require.NoError(t, err) + time.Sleep(10 * time.Millisecond) tampered := []byte(strings.Repeat("x", int(info.Size()))) require.NoError(t, os.WriteFile(binary, tampered, 0o755)) require.NoError(t, os.Chtimes(binary, info.ModTime(), info.ModTime())) _, err = loadVGPURuntime(p) - require.NoError(t, err) - - // Any change to a file's metadata forces the checksums to be recomputed. - later := info.ModTime().Add(time.Second) - require.NoError(t, os.Chtimes(binary, later, later)) - _, err = loadVGPURuntime(p) require.ErrorContains(t, err, "binary checksum mismatch") } diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 4dc6e37ae..fd6079a91 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -311,8 +311,12 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin forkMeta.Phases.Record(phasetracking.PhaseStopped, now) } - // Preserve existing backends' CID behavior. Managed QEMU GPU forks use a - // transport reset and the guest agent's CID-independent listening socket. + // Keep the original CID for snapshot-based forks. Rewriting CID in restored + // memory snapshots is not reliable across hypervisors. Concurrent standby + // fork prepare is currently enabled only for Firecracker, whose host vsock + // dialer routes through the per-VM UDS path rather than this metadata CID. + // Managed QEMU vGPU forks do get a fresh CID: the device resets its + // transport on restore and the guest agent listens on every CID. if source.State == StateStandby && !usesVGPUSnapshot(&forkMeta) { forkMeta.VsockCID = stored.VsockCID } else { diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 5c0becc27..c5c52bcf2 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -124,7 +124,8 @@ func (m *manager) standbyInstance( return nil, err } - // VFIO precopy needs the guest driver running until migration stops it. + // 6. Transition: Running → Paused. A vGPU guest keeps running instead: + // VFIO precopy needs its driver until the migration itself stops it. if !usesVGPUSnapshot(stored) { log.DebugContext(ctx, "pausing VM", "instance_id", id) pauseCtx, pauseSpanEnd := m.startLifecycleStep(ctx, "pause_vm", diff --git a/lib/instances/vgpu_snapshot.go b/lib/instances/vgpu_snapshot.go index 9c41da604..2e4f80171 100644 --- a/lib/instances/vgpu_snapshot.go +++ b/lib/instances/vgpu_snapshot.go @@ -16,21 +16,23 @@ func sharedVGPURestoreSlots(limit int) chan struct{} { return slots.(chan struct{}) } +// acquireInstanceRestoreSlot takes the vGPU slot before the hypervisor-wide +// one, so queued vGPU restores do not hold capacity other guests could use. func (m *manager) acquireInstanceRestoreSlot(ctx context.Context, stored *StoredMetadata) (func(), error) { - release, err := m.acquireRestoreSlot(ctx, stored.HypervisorType) - if err != nil { - return nil, err - } if !usesVGPUSnapshot(stored) || m.vgpuRestoreSlots == nil { - return release, nil + return m.acquireRestoreSlot(ctx, stored.HypervisorType) } select { case m.vgpuRestoreSlots <- struct{}{}: - return func() { <-m.vgpuRestoreSlots; release() }, nil case <-ctx.Done(): - release() return nil, ctx.Err() } + release, err := m.acquireRestoreSlot(ctx, stored.HypervisorType) + if err != nil { + <-m.vgpuRestoreSlots + return nil, err + } + return func() { release(); <-m.vgpuRestoreSlots }, nil } func (m *manager) shutdownFailedRestore(ctx context.Context, stored *StoredMetadata, hv hypervisor.Hypervisor) error { From a1c45432d5c9291903cbed5a714a7e6023ea0b28 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 10 Nov 2026 03:26:02 +0000 Subject: [PATCH 7/9] Scope the vGPU runtime README to Hypeman --- scripts/qemu-vgpu/README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/qemu-vgpu/README.md b/scripts/qemu-vgpu/README.md index c27851222..73ae94a8b 100644 --- a/scripts/qemu-vgpu/README.md +++ b/scripts/qemu-vgpu/README.md @@ -21,10 +21,8 @@ Install the resulting directory, preserving its manifest and firmware, at: Select it explicitly on a new instance with `hypervisor: qemu`, `hypervisor_version: 11.1.3-vgpu.1`, `gpu.profile: NVIDIA L40S-3Q`, and the guest agent enabled. Ordinary QEMU instances keep using the system binary. Never overwrite an installed runtime that has live snapshots: change the runtime version and rebuild templates instead. -Snapshots pin the runtime binary and BIOS checksums, machine type, NVIDIA host-driver version and PCI device identity. Restores reject mismatches. A child claims its own VF and receives a separate vsock CID; the captured source device assignment is not reused. RAM is eagerly allocated privately per child, not shared through CPU-style copy-on-write. `hypervisor.vgpu_max_concurrent_restores` defaults to two. +Snapshots record the runtime binary and firmware checksums, the machine type, the NVIDIA host-driver version, the PCI vendor and device IDs and the vGPU profile type. Restores reject any mismatch. A fork claims its own VF and gets its own vsock CID and machine UUID; the source's device assignment is never reused. Forks hardlink the template's memory file instead of copying it, and each fork's guest RAM is private and populated up front rather than shared copy-on-write. `hypervisor.vgpu_max_concurrent_restores` (default 2) caps concurrent restores. -Capture uses live VFIO precopy: do not pause the guest before requesting a standby snapshot. Only after migration completes and the source VMM exits is its VF released. Restores load the mapped-RAM stream paused before guest network reconfiguration and application identity handoff. +Capture uses live VFIO precopy: do not pause the guest before requesting a standby snapshot. The source's VF is released only after migration completes and its VMM exits. A restore loads the stream into a paused guest, resumes it, and has the guest agent apply the fork's network identity; a guest whose agent does not answer within ten seconds fails the restore. QEMU's post-migration self-announcement is disabled for these restores. -The runtime does not make arbitrary application state safe to clone. Browser templates must contain no customer state, license token, trusted license cache or active lease. The companion browser image keeps licensing deferred until the child has its own MAC/license identity, mixes fresh host entropy, and restarts Chromium before returning readiness. Licensing then proceeds asynchronously with normal health checks. Host/runtime support must deploy before enabling GPU fanout in the browser control plane. - -Keep GPU fanout disabled until the complete template/identity/licensing flow, concurrent forks, cancellation, failed restore cleanup, old-template behavior and neighboring workload impact have been qualified. Retain the existing cold-boot path for fallback and the previous runtime for existing snapshots. +A snapshot clones guest state as it is. Capture templates without per-instance secrets, and have a restored guest refresh whatever must be unique per instance (machine identity, entropy, licenses, sessions) before it serves work. Keep a cold-boot path as the fallback, and keep the runtime that existing snapshots were captured with installed until those snapshots are gone. From 6525ea0ead6bca98c145b53b9a060f9e2eb6c28f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 10 Nov 2026 05:51:55 +0000 Subject: [PATCH 8/9] Fail a QEMU restore as soon as the VMM exits A restore polls the monitor until the incoming migration has loaded and treats monitor errors as transient. When QEMU died while loading, every poll failed and the restore only gave up after the 30 s migration timeout. The process reaper now closes a channel that the wait watches, so the restore fails at once with the VMM log attached. --- lib/hypervisor/qemu/process.go | 57 +++++++++++++++++------------ lib/hypervisor/qemu/process_test.go | 14 +++++++ lib/hypervisor/qemu/qmp.go | 17 ++++++++- lib/hypervisor/qemu/qmp_test.go | 47 ++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 26 deletions(-) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index 9d591d978..db573cbc9 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -240,6 +240,9 @@ type startedProcess struct { waitDone chan error waitConsumed bool waitErr error + // exited is closed once the process has been reaped. Unlike waitDone, any + // number of goroutines can watch it. + exited chan struct{} } func startManagedProcess(cmd *exec.Cmd, socketPath string) (*startedProcess, error) { @@ -251,11 +254,13 @@ func startManagedProcess(cmd *exec.Cmd, socketPath string) (*startedProcess, err pid: cmd.Process.Pid, socketPath: socketPath, waitDone: make(chan error, 1), + exited: make(chan struct{}), } go func() { err := cmd.Wait() _ = os.Remove(socketPath) proc.waitDone <- err + close(proc.exited) }() return proc, nil @@ -325,9 +330,9 @@ func vfioTermGraceFor(cfg hypervisor.VMConfig) time.Duration { } // startQEMUProcess handles the common QEMU process startup logic. -// Returns the PID, hypervisor client, and a cleanup function. +// Returns the started process, hypervisor client, and a cleanup function. // The cleanup function must be called on error; call cleanup.Release() on success. -func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string, termGrace time.Duration) (int, *QEMU, *cleanup.Cleanup, error) { +func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string, termGrace time.Duration) (*startedProcess, *QEMU, *cleanup.Cleanup, error) { log := logger.FromContext(ctx) processAttrs := hypervisor.TraceAttributesFromContext(ctx) processAttrs = append(processAttrs, @@ -342,13 +347,13 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version if err != nil { processSpan.RecordError(err) processSpan.SetStatus(codes.Error, err.Error()) - return 0, nil, nil, fmt.Errorf("get binary: %w", err) + return nil, nil, nil, fmt.Errorf("get binary: %w", err) } // Check if socket is already in use if isSocketInUse(socketPath) { processSpan.SetStatus(codes.Error, "socket already in use") - return 0, nil, nil, fmt.Errorf("socket already in use, QEMU may be running at %s", socketPath) + return nil, nil, nil, fmt.Errorf("socket already in use, QEMU may be running at %s", socketPath) } // A graceful guest shutdown can leave the old QMP client pooled even though @@ -370,7 +375,7 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version if err := os.MkdirAll(logsDir, 0755); err != nil { processSpan.RecordError(err) processSpan.SetStatus(codes.Error, err.Error()) - return 0, nil, nil, fmt.Errorf("create logs directory: %w", err) + return nil, nil, nil, fmt.Errorf("create logs directory: %w", err) } vmmLogFile, err := os.OpenFile( @@ -381,7 +386,7 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version if err != nil { processSpan.RecordError(err) processSpan.SetStatus(codes.Error, err.Error()) - return 0, nil, nil, fmt.Errorf("create vmm log: %w", err) + return nil, nil, nil, fmt.Errorf("create vmm log: %w", err) } defer vmmLogFile.Close() @@ -393,7 +398,7 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version if err != nil { processSpan.RecordError(err) processSpan.SetStatus(codes.Error, err.Error()) - return 0, nil, nil, fmt.Errorf("start qemu: %w", err) + return nil, nil, nil, fmt.Errorf("start qemu: %w", err) } pid := proc.pid @@ -413,7 +418,7 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version // Diagnostic (log-only): if this was a vsock guest-CID collision, capture // the colliding CID and which host process is holding it. logVsockCIDConflict(processCtx, guestCIDFromArgs(args), wrapped.Error()) - return 0, nil, nil, wrapped + return nil, nil, nil, wrapped } log.DebugContext(processCtx, "QMP socket ready", "duration_ms", time.Since(socketWaitStart).Milliseconds()) @@ -431,7 +436,7 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version // Diagnostic (log-only): if this was a vsock guest-CID collision, // capture the colliding CID and which host process is holding it. logVsockCIDConflict(processCtx, guestCIDFromArgs(args), wrapped.Error()) - return 0, nil, nil, wrapped + return nil, nil, nil, wrapped } hv, err = NewForType(socketPath, s.profile.hypervisorType()) @@ -442,13 +447,13 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version processSpan.RecordError(err) processSpan.SetStatus(codes.Error, err.Error()) cu.Clean() - return 0, nil, nil, appendVMMLog(fmt.Errorf("create client: %w", err), logsDir) + return nil, nil, nil, appendVMMLog(fmt.Errorf("create client: %w", err), logsDir) } time.Sleep(socketPollInterval) } processSpan.SetStatus(codes.Ok, "") - return pid, hv, &cu, nil + return proc, hv, &cu, nil } func (s *Starter) validateSnapshotMachineType(stored MachineType) (MachineType, error) { @@ -468,23 +473,23 @@ func (s *Starter) validateSnapshotMachineType(stored MachineType) (MachineType, return expected, nil } -func (s *Starter) startConfiguredProcess(ctx context.Context, p *paths.Paths, version, socketPath string, config hypervisor.VMConfig, args []string) (int, *QEMU, *cleanup.Cleanup, error) { +func (s *Starter) startConfiguredProcess(ctx context.Context, p *paths.Paths, version, socketPath string, config hypervisor.VMConfig, args []string) (*startedProcess, *QEMU, *cleanup.Cleanup, error) { cu := cleanup.Make(func() {}) tpmProcess, err := startSWTPM(config.TPM, filepath.Dir(socketPath)) if err != nil { - return 0, nil, nil, err + return nil, nil, nil, err } if tpmProcess != nil { cu.Add(tpmProcess.cleanup) } - pid, hv, qemuCleanup, err := s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(config)) + proc, hv, qemuCleanup, err := s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(config)) if err != nil { cu.Clean() - return 0, nil, nil, err + return nil, nil, nil, err } cu.Add(qemuCleanup.Clean) - return pid, hv, &cu, nil + return proc, hv, &cu, nil } // StartVM launches QEMU with the VM configuration and returns a Hypervisor client. @@ -541,7 +546,7 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s } var ( - pid int + proc *startedProcess hv *QEMU cu *cleanup.Cleanup booted hypervisor.VMConfig @@ -554,7 +559,7 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s args := buildQMPArgs(socketPath) args = append(args, buildArgs(attempt, machineType)...) args = append(args, runtimeArgs...) - pid, hv, cu, err = s.startConfiguredProcess(ctx, p, version, socketPath, attempt, args) + proc, hv, cu, err = s.startConfiguredProcess(ctx, p, version, socketPath, attempt, args) if err == nil { booted = attempt started = true @@ -596,7 +601,7 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s } cu.Release() - return pid, hv, nil + return proc.pid, hv, nil } func appendVMMLog(err error, logsDir string) error { @@ -710,7 +715,7 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, args = append(args, "-incoming", "exec:cat < "+memoryFile) } - pid, hv, cu, err := s.startConfiguredProcess(ctx, p, version, socketPath, config, args) + proc, hv, cu, err := s.startConfiguredProcess(ctx, p, version, socketPath, config, args) if err != nil { return 0, nil, err } @@ -728,8 +733,12 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, // Wait for VM to be ready after loading migration data // QEMU transitions from "inmigrate" to "paused" when loading completes migrationWaitStart := time.Now() - if err := hv.client.WaitVMReady(ctx, migrationTimeout); err != nil { - return 0, nil, fmt.Errorf("wait for vm ready: %w", err) + if err := hv.client.WaitVMReady(ctx, migrationTimeout, proc.exited); err != nil { + err = fmt.Errorf("wait for vm ready: %w", err) + if errors.Is(err, errVMMExited) { + err = appendVMMLog(err, filepath.Join(filepath.Dir(socketPath), "logs")) + } + return 0, nil, err } log.DebugContext(ctx, "VM ready", "duration_ms", time.Since(migrationWaitStart).Milliseconds()) @@ -738,8 +747,8 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, } cu.Release() - log.DebugContext(ctx, "QEMU restore complete", "pid", pid, "total_duration_ms", time.Since(startTime).Milliseconds()) - return pid, hv, nil + log.DebugContext(ctx, "QEMU restore complete", "pid", proc.pid, "total_duration_ms", time.Since(startTime).Milliseconds()) + return proc.pid, hv, nil } func (s *Starter) validateRestoreVersion(p *paths.Paths, storedVersion string) (string, error) { diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index ad08aea24..f208e39e3 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -394,6 +394,20 @@ func TestStartManagedProcessCleanupRemovesSocketAndReapsExitedProcess(t *testing assert.True(t, cmd.ProcessState.Exited()) } +func TestStartManagedProcessClosesExitedOnceProcessIsReaped(t *testing.T) { + cmd := exec.Command("sh", "-c", "exit 0") + proc, err := startManagedProcess(cmd, filepath.Join(t.TempDir(), "qemu.sock")) + require.NoError(t, err) + + select { + case <-proc.exited: + case <-time.After(5 * time.Second): + t.Fatal("exited was not closed after the process ended") + } + _, exited := proc.checkExited() + assert.True(t, exited) +} + func TestWaitForSocketOrExitReturnsEarlyWhenProcessDies(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "qemu.sock") diff --git a/lib/hypervisor/qemu/qmp.go b/lib/hypervisor/qemu/qmp.go index 90e239667..e26915616 100644 --- a/lib/hypervisor/qemu/qmp.go +++ b/lib/hypervisor/qemu/qmp.go @@ -2,6 +2,7 @@ package qemu import ( "context" + "errors" "fmt" "time" @@ -199,21 +200,33 @@ func (c *Client) WaitMigration(ctx context.Context, timeout time.Duration) error return fmt.Errorf("migration timeout after %v", timeout) } +// errVMMExited reports that QEMU exited before it finished loading migration data. +var errVMMExited = errors.New("qemu exited while loading migration data") + // WaitVMReady polls until the VM is ready after an incoming migration. // Used for restore operations where QEMU was started with -incoming. // The VM transitions from "inmigrate" to "paused" when migration data is loaded. // Returns nil when VM is ready for resume, error on timeout or failure. -func (c *Client) WaitVMReady(ctx context.Context, timeout time.Duration) error { +// exited, when not nil, is closed if the QEMU process exits: the monitor of a +// dead process only ever returns errors, which would otherwise be polled as +// transient until the timeout. +func (c *Client) WaitVMReady(ctx context.Context, timeout time.Duration, exited <-chan struct{}) error { + return waitVMReady(ctx, timeout, exited, c.Status) +} + +func waitVMReady(ctx context.Context, timeout time.Duration, exited <-chan struct{}, queryStatus func() (qemu.Status, error)) error { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { select { case <-ctx.Done(): return ctx.Err() + case <-exited: + return errVMMExited default: } - status, err := c.Status() + status, err := queryStatus() if err != nil { // Ignore transient errors, keep polling time.Sleep(qmpPollInterval) diff --git a/lib/hypervisor/qemu/qmp_test.go b/lib/hypervisor/qemu/qmp_test.go index 16d8e2a23..59b6a59c7 100644 --- a/lib/hypervisor/qemu/qmp_test.go +++ b/lib/hypervisor/qemu/qmp_test.go @@ -1,13 +1,60 @@ package qemu import ( + "context" + "errors" "testing" + "time" "github.com/digitalocean/go-qemu/qemu" "github.com/digitalocean/go-qemu/qmp/raw" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestWaitVMReadyStopsWhenVMMExits(t *testing.T) { + exited := make(chan struct{}) + polls := 0 + queryStatus := func() (qemu.Status, error) { + polls++ + if polls == 2 { + close(exited) + } + if polls >= 2 { + return 0, errors.New("monitor closed") + } + return qemu.StatusInMigrate, nil + } + + start := time.Now() + err := waitVMReady(context.Background(), 30*time.Second, exited, queryStatus) + + require.ErrorIs(t, err, errVMMExited) + assert.Equal(t, 2, polls) + assert.Less(t, time.Since(start), 5*time.Second) +} + +func TestWaitVMReadyPollsThroughMonitorErrorsWhileVMMRuns(t *testing.T) { + polls := 0 + queryStatus := func() (qemu.Status, error) { + polls++ + switch polls { + case 1: + return 0, errors.New("monitor busy") + case 2: + return qemu.StatusInMigrate, nil + } + return qemu.StatusPaused, nil + } + + require.NoError(t, waitVMReady(context.Background(), 30*time.Second, make(chan struct{}), queryStatus)) + assert.Equal(t, 3, polls) + + require.ErrorContains(t, waitVMReady(context.Background(), 30*time.Second, nil, func() (qemu.Status, error) { + return qemu.StatusShutdown, nil + }), "shut down during migration") +} + func TestStatusMapping(t *testing.T) { // Test that qemu.Status values are properly defined tests := []struct { From 07ad5f4b0d2966a6029a7dfab9ff06adfff927f5 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:31:30 +0000 Subject: [PATCH 9/9] Pin the runtime workflow's actions to commit SHAs The publish job can write releases, and the build job produces the artifact it publishes, so neither should run an action through a tag that can be moved. --- .github/workflows/qemu-vgpu-runtime.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/qemu-vgpu-runtime.yml b/.github/workflows/qemu-vgpu-runtime.yml index ff73847f6..f82cf75bb 100644 --- a/.github/workflows/qemu-vgpu-runtime.yml +++ b/.github/workflows/qemu-vgpu-runtime.yml @@ -23,7 +23,7 @@ jobs: outputs: version: ${{ steps.package.outputs.version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - name: Build and test pinned runtime @@ -38,7 +38,7 @@ jobs: tar -czf "artifacts/qemu-vgpu-${VERSION}-linux-amd64.tar.gz" -C runtime . cd artifacts sha256sum *.tar.gz > SHA256SUMS - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: qemu-vgpu-runtime path: artifacts/ @@ -51,7 +51,7 @@ jobs: permissions: contents: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: qemu-vgpu-runtime path: artifacts