Add opt-in vGPU snapshots and forks on a managed QEMU runtime

.github/workflows/qemu-vgpu-runtime.yml

@@ -0,0 +1,68 @@
+name: Build QEMU vGPU runtime
+
+on:
+  pull_request:
+    paths:
+      - scripts/qemu-vgpu/**
+      - lib/hypervisor/qemu/vgpu.go
+      - .github/workflows/qemu-vgpu-runtime.yml
+  workflow_dispatch:
+    inputs:
+      publish:
+        description: Publish the immutable runtime release from main
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    outputs:
+      version: ${{ steps.package.outputs.version }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+      - name: Build and test pinned runtime
+        run: docker build --output type=local,dest=runtime -f scripts/qemu-vgpu/Dockerfile .
+      - name: Package runtime and corresponding source
+        id: package
+        run: |
+          VERSION=$(python3 -c 'import json,re;v=json.load(open("runtime/manifest.json"))["version"];assert re.fullmatch(r"[0-9][a-z0-9.-]+",v);print(v)')
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          mkdir artifacts
+          mv runtime/source.tar.gz "artifacts/qemu-vgpu-${VERSION}-source.tar.gz"
+          tar -czf "artifacts/qemu-vgpu-${VERSION}-linux-amd64.tar.gz" -C runtime .
+          cd artifacts
+          sha256sum *.tar.gz > SHA256SUMS
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: qemu-vgpu-runtime
+          path: artifacts/
+          retention-days: 30
+
+  publish:
+    if: github.event_name == 'workflow_dispatch' && inputs.publish && github.ref == 'refs/heads/main'
+    needs: build
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: qemu-vgpu-runtime
+          path: artifacts
+      - name: Publish a new runtime version without replacing existing assets
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          VERSION: ${{ needs.build.outputs.version }}
+          COMMIT: ${{ github.sha }}
+        run: |
+          gh release create "qemu-vgpu-${VERSION}" artifacts/* \
+            --target "$COMMIT" \
+            --title "QEMU vGPU runtime ${VERSION}" \
+            --notes "Opt-in QEMU runtime. Includes checksums and corresponding patched source. Does not replace the system QEMU or enable GPU snapshot provisioning."

cmd/api/api/instances.go

@@ -354,6 +354,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		Devices:                  deviceRefs,
 		Volumes:                  volumes,
 		Hypervisor:               hvType,
+		HypervisorVersion:        lo.FromPtr(request.Body.HypervisorVersion),
 		GPU:                      gpuConfig,
 		Entrypoint:               entrypoint,
 		Cmd:                      cmd,
@@ -1248,6 +1249,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance {
 		_ = json.Unmarshal(b, &oapiInst.Network)
 	}
 
+	if inst.HypervisorVersion != "" && inst.HypervisorVersion != "unknown" {
+		oapiInst.HypervisorVersion = lo.ToPtr(inst.HypervisorVersion)
+	}
 	if inst.Platform != "" {
 		oapiInst.Platform = lo.ToPtr(inst.Platform)
 	}

cmd/api/api/instances_runtime_test.go

@@ -0,0 +1,32 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/kernel/hypeman/lib/hypervisor"
+	"github.com/kernel/hypeman/lib/hypervisor/qemu"
+	"github.com/kernel/hypeman/lib/instances"
+	"github.com/kernel/hypeman/lib/oapi"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCreateInstanceMapsRequestedRuntimeVersion(t *testing.T) {
+	svc := newTestService(t)
+	manager := newCaptureCreateManager(svc.InstanceManager)
+	svc.InstanceManager = manager
+	runtime := oapi.CreateInstanceRequestHypervisor(hypervisor.TypeQEMU)
+	version := qemu.VGPUSnapshotVersion
+	response, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{Body: &oapi.CreateInstanceRequest{Name: "runtime-version-test", Image: "docker.io/library/alpine:latest", Hypervisor: &runtime, HypervisorVersion: &version}})
+	require.NoError(t, err)
+	require.IsType(t, oapi.CreateInstance201JSONResponse{}, response)
+	require.Equal(t, version, manager.lastReq.HypervisorVersion)
+}
+
+func TestInstanceRuntimeVersionResponse(t *testing.T) {
+	inst := instances.Instance{StoredMetadata: instances.StoredMetadata{HypervisorVersion: qemu.VGPUSnapshotVersion}}
+	response := instanceToOAPI(inst)
+	require.NotNil(t, response.HypervisorVersion)
+	require.Equal(t, qemu.VGPUSnapshotVersion, *response.HypervisorVersion)
+	inst.HypervisorVersion = ""
+	require.Nil(t, instanceToOAPI(inst).HypervisorVersion)
+}

cmd/api/config/config.go

@@ -222,6 +222,7 @@ type HypervisorConfig struct {
 	CloudHypervisorDefaultVersion    string                 `koanf:"cloud_hypervisor_default_version"`
 	FirecrackerBinaryPath            string                 `koanf:"firecracker_binary_path"`
 	FirecrackerMaxConcurrentRestores int                    `koanf:"firecracker_max_concurrent_restores"`
+	VGPUMaxConcurrentRestores        int                    `koanf:"vgpu_max_concurrent_restores"`
 	Memory                           HypervisorMemoryConfig `koanf:"memory"`
 }
 
@@ -461,6 +462,7 @@ func defaultConfig() *Config {
 			CloudHypervisorDefaultVersion:    "",
 			FirecrackerBinaryPath:            "",
 			FirecrackerMaxConcurrentRestores: 32,
+			VGPUMaxConcurrentRestores:        2,
 			Memory: HypervisorMemoryConfig{
 				Enabled:            false,
 				KernelPageInitMode: "hardened",
@@ -715,6 +717,9 @@ func (c *Config) Validate() error {
 		}
 		c.Builders.IdleTTLDuration = ttl
 	}
+	if c.Hypervisor.VGPUMaxConcurrentRestores < 0 {
+		return fmt.Errorf("hypervisor.vgpu_max_concurrent_restores must be >= 0, got %d", c.Hypervisor.VGPUMaxConcurrentRestores)
+	}
 	if c.Hypervisor.FirecrackerMaxConcurrentRestores < 0 {
 		return fmt.Errorf("hypervisor.firecracker_max_concurrent_restores must be >= 0, got %d", c.Hypervisor.FirecrackerMaxConcurrentRestores)
 	}

config.example.yaml

@@ -40,6 +40,8 @@ data_dir: /var/lib/hypeman
 #   # firecracker_binary_path: /usr/local/bin/firecracker
 #   # Caps concurrent Firecracker snapshot restore/resume work to avoid host overload.
 #   # firecracker_max_concurrent_restores: 32
+#   # Caps concurrent vGPU snapshot restores, which each pin all guest memory.
+#   # vgpu_max_concurrent_restores: 2
 #   memory:
 #     enabled: false
 #     # performance: init_on_alloc=0 init_on_free=0 (better density)

lib/hypervisor/config.go

@@ -26,6 +26,12 @@ type VMConfig struct {
 	// PCI device passthrough (GPU, etc.)
 	PCIDevices     []string
 	VGPUDevicePath string
+	// VGPUSnapshot launches the guest so its vGPU state can be captured and
+	// restored. Only QEMU's managed vGPU runtime supports it.
+	VGPUSnapshot bool
+	// InstanceID identifies the guest to backends that derive guest-visible
+	// identity from it.
+	InstanceID string
 
 	// Boot configuration. Empty BootMode preserves the existing direct-kernel
 	// behavior for Linux callers.

lib/hypervisor/hypervisor.go

@@ -274,6 +274,11 @@ type ForkPrepareRequest struct {
 
 	SerialLogPath string
 	Network       *ForkNetworkConfig
+
+	// InstanceID is the fork's own id. VGPUDevicePath, when set, replaces the
+	// vGPU the snapshot was captured with.
+	InstanceID     string
+	VGPUDevicePath string
 }
 
 // ForkPrepareResult describes which optional fork rewrites were actually applied.

lib/hypervisor/qemu/config.go

@@ -25,6 +25,9 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string {
 
 	// Machine type with KVM acceleration (arch-specific when omitted).
 	machineArg := string(machine) + ",accel=kvm"
+	if cfg.VGPUSnapshot {
+		machineArg += ",smbios-entry-point-type=32,memory-backend=" + vgpuRAMBackendID
+	}
 	if cfg.Firmware != nil && cfg.Firmware.SecureBoot {
 		machineArg += ",smm=on"
 	}
@@ -47,6 +50,16 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string {
 	// Memory configuration
 	memMB := cfg.MemoryBytes / (1024 * 1024)
 	args = append(args, "-m", fmt.Sprintf("%dM", memMB))
+	if cfg.VGPUSnapshot {
+		// VFIO pins every guest page when the device attaches. Populating RAM
+		// first, in parallel, is much cheaper than faulting it in during the
+		// pin; the default backend would only use one thread per vCPU.
+		args = append(args, "-object", fmt.Sprintf(
+			"memory-backend-ram,id=%s,size=%dM,prealloc=on,prealloc-threads=%d,x-use-canonical-path-for-ramblock-id=off",
+			vgpuRAMBackendID, memMB, vgpuPreallocThreads))
+		id := vgpuInstanceUUID(cfg.InstanceID)
+		args = append(args, "-device", "vmgenid,guid=auto", "-uuid", id, "-smbios", "type=1,uuid="+id)
+	}
 
 	if cfg.GuestMemory.EnableBalloon {
 		balloonOpts := []string{virtioDevice(microvm, "virtio-balloon")}
@@ -144,7 +157,11 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string {
 	}
 
 	if cfg.VGPUDevicePath != "" {
-		args = append(args, "-device", fmt.Sprintf("vfio-pci,sysfsdev=%s", cfg.VGPUDevicePath))
+		device := fmt.Sprintf("vfio-pci,sysfsdev=%s", cfg.VGPUDevicePath)
+		if cfg.VGPUSnapshot {
+			device += ",enable-migration=on"
+		}
+		args = append(args, "-device", device)
 	}
 
 	// Serial console output to file. Use a chardev with append=on so QEMU

lib/hypervisor/qemu/fork.go

@@ -32,6 +32,19 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq
 		return hypervisor.ForkPrepareResult{}, fmt.Errorf("select qemu fork machine type: %w", err)
 	}
 	cfg := saved.VMConfig
+	if req.InstanceID != "" {
+		cfg.InstanceID = req.InstanceID
+	}
+	if cfg.VGPUSnapshot {
+		if req.VGPUDevicePath != "" {
+			cfg.VGPUDevicePath = req.VGPUDevicePath
+		}
+		if req.TargetDataDir != "" && req.TargetDataDir != req.SourceDataDir {
+			// A fork never opens its source's VF: restore claims a fresh one
+			// and records it with a second PrepareFork before launching QEMU.
+			cfg.VGPUDevicePath = ""
+		}
+	}
 
 	if req.SourceDataDir != "" && req.TargetDataDir != "" && req.SourceDataDir != req.TargetDataDir {
 		cfg = rewriteQEMUConfigPaths(cfg, req.SourceDataDir, req.TargetDataDir)