From da0bb0d3aea910b198bde4f7613558539d81177b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:41:49 +0000 Subject: [PATCH 01/12] Release QEMU QMP client when a VM is deleted or stopped --- lib/hypervisor/hypervisor.go | 21 ++++++++++ lib/hypervisor/qemu/pool_test.go | 56 ++++++++++++++++++++++++++ lib/hypervisor/qemu/process.go | 1 + lib/instances/delete.go | 4 ++ lib/instances/release_client_test.go | 60 ++++++++++++++++++++++++++++ lib/instances/stop.go | 9 +++-- 6 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 lib/instances/release_client_test.go diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index 1aff356cd..8dc62a11b 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -468,3 +468,24 @@ func NewClient(hvType Type, socketPath string) (Hypervisor, error) { } return WrapHypervisor(hvType, client), nil } + +// ClientReleaser closes and forgets a cached client for a socket whose VMM has exited. +type ClientReleaser func(socketPath string) + +// clientReleasers maps hypervisor types that cache clients to their releasers. +var clientReleasers = make(map[Type]ClientReleaser) + +// RegisterClientReleaser registers the releaser for a hypervisor type that +// caches clients per socket. +func RegisterClientReleaser(t Type, releaser ClientReleaser) { + clientReleasers[t] = releaser +} + +// ReleaseClient closes and forgets any cached client for socketPath. Call it +// only once the VMM behind socketPath is confirmed gone. Hypervisor types that +// do not cache clients ignore it. +func ReleaseClient(hvType Type, socketPath string) { + if release, ok := clientReleasers[hvType]; ok { + release(socketPath) + } +} diff --git a/lib/hypervisor/qemu/pool_test.go b/lib/hypervisor/qemu/pool_test.go index 536713f39..160b52f32 100644 --- a/lib/hypervisor/qemu/pool_test.go +++ b/lib/hypervisor/qemu/pool_test.go @@ -1,7 +1,12 @@ package qemu import ( + "bufio" + "io" + "net" + "path/filepath" "testing" + "time" "github.com/kernel/hypeman/lib/hypervisor" "github.com/stretchr/testify/require" @@ -47,3 +52,54 @@ func TestGetOrCreateForTypeRejectsCachedBackendMismatch(t *testing.T) { clientPool.RUnlock() require.Equal(t, hypervisor.TypeQEMU, cached.profile.hypervisorType(), "a stale caller must not evict the live client") } + +func TestReleaseClientClosesPooledConnection(t *testing.T) { + for _, hvType := range []hypervisor.Type{hypervisor.TypeQEMU, hypervisor.TypeQEMUMicroVM} { + t.Run(string(hvType), func(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + closed := serveFakeQMP(t, socketPath) + + _, err := GetOrCreateForType(socketPath, hvType) + require.NoError(t, err) + + hypervisor.ReleaseClient(hvType, socketPath) + + clientPool.RLock() + _, pooled := clientPool.clients[socketPath] + clientPool.RUnlock() + require.False(t, pooled) + select { + case <-closed: + case <-time.After(5 * time.Second): + t.Fatal("released client did not close its QMP connection") + } + }) + } +} + +// serveFakeQMP accepts one QMP connection on socketPath, completes the +// capabilities handshake, and closes the returned channel when the client +// disconnects. +func serveFakeQMP(t *testing.T, socketPath string) <-chan struct{} { + t.Helper() + ln, err := net.Listen("unix", socketPath) + require.NoError(t, err) + t.Cleanup(func() { ln.Close() }) + + closed := make(chan struct{}) + go func() { + defer close(closed) + conn, err := ln.Accept() + if err != nil { + return + } + defer conn.Close() + _, _ = io.WriteString(conn, `{"QMP":{"version":{"qemu":{"major":8,"minor":2,"micro":0},"package":""},"capabilities":[]}}`+"\n") + if _, err := bufio.NewReader(conn).ReadBytes('\n'); err != nil { + return + } + _, _ = io.WriteString(conn, `{"return":{}}`+"\n") + _, _ = io.Copy(io.Discard, conn) + }() + return closed +} diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index c5ac61ea2..5e8614ca6 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -55,6 +55,7 @@ func init() { hypervisor.RegisterClientFactory(t, func(socketPath string) (hypervisor.Hypervisor, error) { return NewForType(socketPath, t) }) + hypervisor.RegisterClientReleaser(t, Remove) } } diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 37fd908ca..5489b2a35 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -143,6 +143,10 @@ func (m *manager) deleteInstanceWithOptions( } } + // The VMM is gone. Close its cached control connection, which would + // otherwise stay open for the life of the process. + hypervisor.ReleaseClient(stored.HypervisorType, inst.SocketPath) + // Release before deleting metadata so a failed release can be retried safely. if storedVGPUDevicePath(stored) != "" { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) diff --git a/lib/instances/release_client_test.go b/lib/instances/release_client_test.go new file mode 100644 index 000000000..408fd7699 --- /dev/null +++ b/lib/instances/release_client_test.go @@ -0,0 +1,60 @@ +package instances + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/stretchr/testify/require" +) + +const releaseClientTestHypervisorType hypervisor.Type = "release-client-test" + +var releasedClientSockets sync.Map + +func init() { + hypervisor.RegisterClientFactory(releaseClientTestHypervisorType, func(socketPath string) (hypervisor.Hypervisor, error) { + state, ok := lifecycleNoopHypervisorStates.Load(socketPath) + if !ok { + return nil, errors.New("missing fake hypervisor state") + } + return lifecycleNoopHypervisor{state: state.(hypervisor.VMState)}, nil + }) + hypervisor.RegisterVsockDialerFactory(releaseClientTestHypervisorType, func(socketPath string, _ int64) hypervisor.VsockDialer { + return lifecycleNoopVsockDialer{socketPath: socketPath} + }) + hypervisor.RegisterClientReleaser(releaseClientTestHypervisorType, func(socketPath string) { + releasedClientSockets.Store(socketPath, true) + }) +} + +// A deleted or stopped VM's cached hypervisor client must be released, or its +// connection stays open for the life of the process. +func TestDeleteAndStopReleaseHypervisorClient(t *testing.T) { + for _, tc := range []struct { + name string + action func(context.Context, *manager, string) error + }{ + {name: "delete", action: func(ctx context.Context, m *manager, id string) error { return m.DeleteInstance(ctx, id) }}, + {name: "stop", action: func(ctx context.Context, m *manager, id string) error { + _, err := m.StopInstance(ctx, id) + return err + }}, + } { + t.Run(tc.name, func(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.HypervisorType = releaseClientTestHypervisorType + require.NoError(t, m.saveMetadata(meta)) + + require.NoError(t, tc.action(context.Background(), m, id)) + + _, released := releasedClientSockets.Load(meta.SocketPath) + require.True(t, released) + }) + } +} diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 8900f73fd..4c8f7b02a 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -261,13 +261,14 @@ func (m *manager) stopInstance( } } - // 8. Drop the guest-agent connection for this VM incarnation and remove - // stale runtime sockets after process exit. The hypervisor shutdown closes - // the live peer, but the keyed gRPC ClientConn must not survive into a later - // start that reuses the same per-instance socket path. + // 8. Drop the guest-agent and hypervisor connections for this VM incarnation + // and remove stale runtime sockets after process exit. The hypervisor + // shutdown closes the live peer, but the keyed gRPC ClientConn must not + // survive into a later start that reuses the same per-instance socket path. if dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID); err == nil { guest.CloseConn(dialer.Key()) } + hypervisor.ReleaseClient(inst.HypervisorType, inst.SocketPath) // If graceful guest shutdown exits before shutdownHypervisor() is called, these // files may still exist and cause state derivation as Unknown or bind conflicts. _ = os.Remove(inst.SocketPath) From b959addc5c05871c7fe6ebfb3e302e27dd6e868a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:12:48 +0000 Subject: [PATCH 02/12] Use a short socket path in the QMP release test t.TempDir() on macOS produces a path longer than the unix socket limit, so binding the fake QMP socket failed there. --- lib/hypervisor/qemu/pool_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/hypervisor/qemu/pool_test.go b/lib/hypervisor/qemu/pool_test.go index 160b52f32..f153b9f6a 100644 --- a/lib/hypervisor/qemu/pool_test.go +++ b/lib/hypervisor/qemu/pool_test.go @@ -4,6 +4,7 @@ import ( "bufio" "io" "net" + "os" "path/filepath" "testing" "time" @@ -56,10 +57,14 @@ func TestGetOrCreateForTypeRejectsCachedBackendMismatch(t *testing.T) { func TestReleaseClientClosesPooledConnection(t *testing.T) { for _, hvType := range []hypervisor.Type{hypervisor.TypeQEMU, hypervisor.TypeQEMUMicroVM} { t.Run(string(hvType), func(t *testing.T) { - socketPath := filepath.Join(t.TempDir(), "qemu.sock") + // t.TempDir() overflows the unix socket path limit on macOS. + dir, err := os.MkdirTemp("/tmp", "qmp") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + socketPath := filepath.Join(dir, "qemu.sock") closed := serveFakeQMP(t, socketPath) - _, err := GetOrCreateForType(socketPath, hvType) + _, err = GetOrCreateForType(socketPath, hvType) require.NoError(t, err) hypervisor.ReleaseClient(hvType, socketPath) From 2dd7bb5356950f309fc56226e43f5dc69b389339 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:12:48 +0000 Subject: [PATCH 03/12] Close the guest-agent connection after delete's graceful shutdown Delete closes the pooled guest-agent connection before the graceful shutdown RPC, and the RPC opens it again. Nothing closed that second connection, so each delete of a running instance left a gRPC client and its goroutines in the pool. --- lib/instances/delete.go | 8 ++- lib/instances/delete_guest_conn_linux_test.go | 62 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 lib/instances/delete_guest_conn_linux_test.go diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 5489b2a35..0f3224de7 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -143,9 +143,13 @@ func (m *manager) deleteInstanceWithOptions( } } - // The VMM is gone. Close its cached control connection, which would - // otherwise stay open for the life of the process. + // The VMM is gone. Close its cached control connection, and the guest-agent + // connection that the graceful shutdown RPC opened again after step 3. Both + // would otherwise stay open for the life of the process. hypervisor.ReleaseClient(stored.HypervisorType, inst.SocketPath) + if dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID); err == nil { + guest.CloseConn(dialer.Key()) + } // Release before deleting metadata so a failed release can be retried safely. if storedVGPUDevicePath(stored) != "" { diff --git a/lib/instances/delete_guest_conn_linux_test.go b/lib/instances/delete_guest_conn_linux_test.go new file mode 100644 index 000000000..ea6f4d981 --- /dev/null +++ b/lib/instances/delete_guest_conn_linux_test.go @@ -0,0 +1,62 @@ +package instances + +import ( + "bufio" + "context" + "os" + "os/exec" + "testing" + "time" + + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" +) + +// Delete closes the pooled guest-agent connection before the graceful shutdown +// RPC, and the RPC opens it again. Delete must close that one too, or it stays +// pooled for the life of the process. +func TestDeleteClosesGuestConn(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + + // The graceful shutdown RPC is only sent to a live hypervisor, so a real + // process has to own the instance socket. + require.NoError(t, os.Remove(meta.SocketPath)) + owner := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") + owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+meta.SocketPath) + stdin, err := owner.StdinPipe() + require.NoError(t, err) + stdout, err := owner.StdoutPipe() + require.NoError(t, err) + require.NoError(t, owner.Start()) + t.Cleanup(func() { + _ = stdin.Close() + _ = owner.Process.Kill() + _ = owner.Wait() + }) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + meta.HypervisorProcessIdentity.Set(owner.Process.Pid) + require.NoError(t, m.saveMetadata(meta)) + + var conn *grpc.ClientConn + m.shutdownGuestFn = func(ctx context.Context, dialer hypervisor.VsockDialer, _ int32) error { + conn, err = guest.GetOrCreateConn(ctx, dialer) + if err != nil { + return err + } + // The owner exits when its stdin closes, like a guest powering off. + return stdin.Close() + } + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + require.NotNil(t, conn, "graceful shutdown RPC was not sent") + require.Eventually(t, func() bool { + return conn.GetState() == connectivity.Shutdown + }, 5*time.Second, 10*time.Millisecond) +} From 783efce246d0a048c1c3827b2554ce2e70688ab2 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:12:48 +0000 Subject: [PATCH 04/12] Drop the pooled QMP client when a QEMU start is cleaned up A start that fails after connecting to QMP kills the process but left its client in the pool. --- lib/hypervisor/qemu/process.go | 3 +++ lib/hypervisor/qemu/process_test.go | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index 5e8614ca6..e6a9a2385 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -308,6 +308,9 @@ func (p *startedProcess) cleanup() { } } _ = os.Remove(p.socketPath) + // A start can fail after its QMP client is pooled, so drop the client + // along with the process. + resetClient(p.socketPath) } func vfioTermGraceFor(cfg hypervisor.VMConfig) time.Duration { diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index ad08aea24..319676654 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -394,6 +394,28 @@ func TestStartManagedProcessCleanupRemovesSocketAndReapsExitedProcess(t *testing assert.True(t, cmd.ProcessState.Exited()) } +func TestCleanupDropsPooledClient(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + clientPool.Lock() + clientPool.clients[socketPath] = &QEMU{socketPath: socketPath, profile: StandardProfile{}} + clientPool.Unlock() + t.Cleanup(func() { + clientPool.Lock() + delete(clientPool.clients, socketPath) + clientPool.Unlock() + }) + + proc, err := startManagedProcess(exec.Command("sleep", "30"), socketPath) + require.NoError(t, err) + + proc.cleanup() + + clientPool.RLock() + _, pooled := clientPool.clients[socketPath] + clientPool.RUnlock() + require.False(t, pooled) +} + func TestWaitForSocketOrExitReturnsEarlyWhenProcessDies(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "qemu.sock") From a7270849e16f37eb99aa8bf1603be93adda6b97d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:38:35 +0000 Subject: [PATCH 05/12] Forget the balloon target when a QEMU client is released Shutdown and DeleteVM clear the cached balloon target, but a VM that was deleted or shut down by its guest kept it. The entries stayed for the life of the process, and a later start on the same socket path read the previous boot's target back. --- lib/hypervisor/qemu/pool.go | 8 ++++++++ lib/hypervisor/qemu/pool_test.go | 3 +++ lib/hypervisor/qemu/process.go | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/hypervisor/qemu/pool.go b/lib/hypervisor/qemu/pool.go index 17feed9e1..808a25fc8 100644 --- a/lib/hypervisor/qemu/pool.go +++ b/lib/hypervisor/qemu/pool.go @@ -88,6 +88,14 @@ func Remove(socketPath string) { closeClientAsync(client) } +// releaseClient drops what is cached for a socket whose QEMU has exited. Unlike +// Remove, it also forgets the balloon target, which has to outlive a reconnect +// to a QEMU that is still running. +func releaseClient(socketPath string) { + Remove(socketPath) + clearBalloonTargetCache(socketPath) +} + // takeClient removes the current client. When expected is non-nil, removal is // conditional on pointer identity to protect against socket-path reuse. func takeClient(socketPath string, expected *QEMU) *QEMU { diff --git a/lib/hypervisor/qemu/pool_test.go b/lib/hypervisor/qemu/pool_test.go index f153b9f6a..5ef3e6917 100644 --- a/lib/hypervisor/qemu/pool_test.go +++ b/lib/hypervisor/qemu/pool_test.go @@ -66,6 +66,7 @@ func TestReleaseClientClosesPooledConnection(t *testing.T) { _, err = GetOrCreateForType(socketPath, hvType) require.NoError(t, err) + balloonTargetCache.Store(socketPath, 512<<20) hypervisor.ReleaseClient(hvType, socketPath) @@ -73,6 +74,8 @@ func TestReleaseClientClosesPooledConnection(t *testing.T) { _, pooled := clientPool.clients[socketPath] clientPool.RUnlock() require.False(t, pooled) + _, cached := balloonTargetCache.Load(socketPath) + require.False(t, cached, "balloon target must not outlive the VM") select { case <-closed: case <-time.After(5 * time.Second): diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index e6a9a2385..feac1d71e 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -55,7 +55,7 @@ func init() { hypervisor.RegisterClientFactory(t, func(socketPath string) (hypervisor.Hypervisor, error) { return NewForType(socketPath, t) }) - hypervisor.RegisterClientReleaser(t, Remove) + hypervisor.RegisterClientReleaser(t, releaseClient) } } From f41e3da4b4bdeacf136b09837bfafac99e3f1ec7 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:38:35 +0000 Subject: [PATCH 06/12] Reset the QMP pool from the QEMU start cleanup only startedProcess is also used for swtpm, where the reset ran against a socket path that is never pooled. --- lib/hypervisor/qemu/process.go | 12 ++++++++---- lib/hypervisor/qemu/process_test.go | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index feac1d71e..05228c9ae 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -308,9 +308,13 @@ func (p *startedProcess) cleanup() { } } _ = os.Remove(p.socketPath) - // A start can fail after its QMP client is pooled, so drop the client - // along with the process. - resetClient(p.socketPath) +} + +// cleanupFailedStart kills a QEMU process whose start failed. The failure can +// come after the QMP client is pooled, so the client is dropped with it. +func cleanupFailedStart(proc *startedProcess) { + proc.cleanup() + resetClient(proc.socketPath) } func vfioTermGraceFor(cfg hypervisor.VMConfig) time.Duration { @@ -398,7 +402,7 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) // Setup cleanup to kill, reap, and remove the socket if subsequent steps fail. - cu := cleanup.Make(proc.cleanup) + cu := cleanup.Make(func() { cleanupFailedStart(proc) }) // Wait for socket to be ready socketWaitStart := time.Now() diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index 319676654..7b4078751 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -394,7 +394,7 @@ func TestStartManagedProcessCleanupRemovesSocketAndReapsExitedProcess(t *testing assert.True(t, cmd.ProcessState.Exited()) } -func TestCleanupDropsPooledClient(t *testing.T) { +func TestCleanupFailedStartDropsPooledClient(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "qemu.sock") clientPool.Lock() clientPool.clients[socketPath] = &QEMU{socketPath: socketPath, profile: StandardProfile{}} @@ -408,7 +408,7 @@ func TestCleanupDropsPooledClient(t *testing.T) { proc, err := startManagedProcess(exec.Command("sleep", "30"), socketPath) require.NoError(t, err) - proc.cleanup() + cleanupFailedStart(proc) clientPool.RLock() _, pooled := clientPool.clients[socketPath] From dd573153e7826ec3684d375f4f139d4c8fb39c0b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:38:35 +0000 Subject: [PATCH 07/12] Use inst.HypervisorType for the delete release and a local err in the test --- lib/instances/delete.go | 2 +- lib/instances/delete_guest_conn_linux_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 0f3224de7..09b7edd31 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -146,7 +146,7 @@ func (m *manager) deleteInstanceWithOptions( // The VMM is gone. Close its cached control connection, and the guest-agent // connection that the graceful shutdown RPC opened again after step 3. Both // would otherwise stay open for the life of the process. - hypervisor.ReleaseClient(stored.HypervisorType, inst.SocketPath) + hypervisor.ReleaseClient(inst.HypervisorType, inst.SocketPath) if dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID); err == nil { guest.CloseConn(dialer.Key()) } diff --git a/lib/instances/delete_guest_conn_linux_test.go b/lib/instances/delete_guest_conn_linux_test.go index ea6f4d981..fde286440 100644 --- a/lib/instances/delete_guest_conn_linux_test.go +++ b/lib/instances/delete_guest_conn_linux_test.go @@ -45,10 +45,11 @@ func TestDeleteClosesGuestConn(t *testing.T) { var conn *grpc.ClientConn m.shutdownGuestFn = func(ctx context.Context, dialer hypervisor.VsockDialer, _ int32) error { - conn, err = guest.GetOrCreateConn(ctx, dialer) + c, err := guest.GetOrCreateConn(ctx, dialer) if err != nil { return err } + conn = c // The owner exits when its stdin closes, like a guest powering off. return stdin.Close() } From 493710948989f56b67de648454ffabc4a91b6f21 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:05:37 +0000 Subject: [PATCH 08/12] Forget a stale balloon target when a QEMU process starts A guest that powers off or crashes by itself never goes through stop or delete, so its balloon target stayed cached. The next start on the same socket path reported that target for a process whose balloon was never set. The start path now drops the target with the pooled client, and resetClient, which only dropped the client, is no longer needed. --- lib/hypervisor/qemu/pool.go | 8 -------- lib/hypervisor/qemu/process.go | 8 ++++---- lib/hypervisor/qemu/process_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/lib/hypervisor/qemu/pool.go b/lib/hypervisor/qemu/pool.go index 808a25fc8..1a686132a 100644 --- a/lib/hypervisor/qemu/pool.go +++ b/lib/hypervisor/qemu/pool.go @@ -63,14 +63,6 @@ func poolTypeMismatchError(socketPath string, cached, requested hypervisor.Type) return fmt.Errorf("QEMU client for %s is pooled as hypervisor %s, not %s", socketPath, cached, requested) } -// resetClient drops a pooled connection before a new QEMU process reuses the -// same socket path. Disconnect happens outside the pool lock and stale users -// can only remove their own client generation. -func resetClient(socketPath string) { - client := takeClient(socketPath, nil) - closeClientAsync(client) -} - // removeClient removes client only if it is still the current generation for // its socket path. This prevents a late error from an old QEMU process from // evicting the replacement process's client. diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index 05228c9ae..c914f79eb 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -314,7 +314,7 @@ func (p *startedProcess) cleanup() { // come after the QMP client is pooled, so the client is dropped with it. func cleanupFailedStart(proc *startedProcess) { proc.cleanup() - resetClient(proc.socketPath) + releaseClient(proc.socketPath) } func vfioTermGraceFor(cfg hypervisor.VMConfig) time.Duration { @@ -351,9 +351,9 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version return 0, nil, nil, fmt.Errorf("socket already in use, QEMU may be running at %s", socketPath) } - // A graceful guest shutdown can leave the old QMP client pooled even though - // the process exited. Drop it before this process reuses the socket path. - resetClient(socketPath) + // A QEMU that exited on its own leaves its QMP client and balloon target + // cached. Drop both before this process reuses the socket path. + releaseClient(socketPath) os.Remove(socketPath) // Create command diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index 7b4078751..3247c61b9 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -416,6 +416,30 @@ func TestCleanupFailedStartDropsPooledClient(t *testing.T) { require.False(t, pooled) } +// A QEMU that exits on its own never goes through stop or delete, so the next +// start on the same socket path has to drop what the old process left cached. +func TestStartQEMUProcessDropsStaleBalloonTarget(t *testing.T) { + starter := NewStarter() + if _, err := starter.GetBinaryPath(nil, ""); err != nil { + t.Skipf("QEMU not available: %v", err) + } + + // t.TempDir() overflows the unix socket path limit on macOS. + dir, err := os.MkdirTemp("/tmp", "qmp") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + socketPath := filepath.Join(dir, "qemu.sock") + balloonTargetCache.Store(socketPath, 512<<20) + + args := append(buildQMPArgs(socketPath), "-machine", "none", "-nodefaults", "-display", "none") + _, _, cu, err := starter.startQEMUProcess(t.Context(), paths.New(dir), "", socketPath, args, 0) + require.NoError(t, err) + t.Cleanup(cu.Clean) + + _, cached := balloonTargetCache.Load(socketPath) + require.False(t, cached) +} + func TestWaitForSocketOrExitReturnsEarlyWhenProcessDies(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "qemu.sock") From d3218af894372145770c45c0aff42419ae5fb8c7 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:05:37 +0000 Subject: [PATCH 09/12] Forget the Cloud Hypervisor balloon target with its VMM Cloud Hypervisor keeps the same per-socket balloon target cache as QEMU and only cleared it in Shutdown and DeleteVM. Register a releaser so stop and delete clear it, and clear it when a new process starts so a VMM that exited by itself does not hand its target to the next boot. --- .../cloudhypervisor/cloudhypervisor_test.go | 16 ++++++++++++++++ lib/hypervisor/cloudhypervisor/process.go | 9 +++++++-- lib/hypervisor/hypervisor.go | 13 +++++++------ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/lib/hypervisor/cloudhypervisor/cloudhypervisor_test.go b/lib/hypervisor/cloudhypervisor/cloudhypervisor_test.go index 2e2596fb1..5701965f5 100644 --- a/lib/hypervisor/cloudhypervisor/cloudhypervisor_test.go +++ b/lib/hypervisor/cloudhypervisor/cloudhypervisor_test.go @@ -3,6 +3,7 @@ package cloudhypervisor import ( "testing" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/vmm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -34,6 +35,21 @@ func TestAssignedGuestMemoryBytes(t *testing.T) { }) } +func TestReleaseClientForgetsBalloonTarget(t *testing.T) { + t.Parallel() + + socketPath := t.TempDir() + "/cloud-hypervisor.sock" + balloonTargetCache.Store(socketPath, int64(384)) + t.Cleanup(func() { + clearBalloonTargetCache(socketPath) + }) + + hypervisor.ReleaseClient(hypervisor.TypeCloudHypervisor, socketPath) + + _, cached := balloonTargetCache.Load(socketPath) + assert.False(t, cached) +} + func TestGetTargetGuestMemoryBytesUsesWarmCacheBeforeVMInfo(t *testing.T) { t.Parallel() diff --git a/lib/hypervisor/cloudhypervisor/process.go b/lib/hypervisor/cloudhypervisor/process.go index 946eb40db..86916f015 100644 --- a/lib/hypervisor/cloudhypervisor/process.go +++ b/lib/hypervisor/cloudhypervisor/process.go @@ -59,6 +59,7 @@ func init() { hypervisor.RegisterClientFactory(hypervisor.TypeCloudHypervisor, func(socketPath string) (hypervisor.Hypervisor, error) { return New(socketPath) }) + hypervisor.RegisterClientReleaser(hypervisor.TypeCloudHypervisor, clearBalloonTargetCache) } // Starter implements hypervisor.VMStarter for Cloud Hypervisor. @@ -128,7 +129,9 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s return 0, nil, fmt.Errorf("start serial reader: %w", err) } - // 1. Start the Cloud Hypervisor process + // 1. Start the Cloud Hypervisor process. One that exited on its own leaves + // its balloon target cached, so drop it before the socket path is reused. + clearBalloonTargetCache(socketPath) processCtx, processSpan := hypervisor.StartProcessSpan(ctx, hypervisor.TypeCloudHypervisor) pid, err := vmm.StartProcess(processCtx, p, chVersion, socketPath) hypervisor.FinishTraceSpan(processSpan, err) @@ -209,7 +212,9 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, return 0, nil, fmt.Errorf("rewrite snapshot serial config: %w", err) } - // 1. Start the Cloud Hypervisor process + // 1. Start the Cloud Hypervisor process. One that exited on its own leaves + // its balloon target cached, so drop it before the socket path is reused. + clearBalloonTargetCache(socketPath) processStartTime := time.Now() processCtx, processSpan := hypervisor.StartProcessSpan(ctx, hypervisor.TypeCloudHypervisor) pid, err := vmm.StartProcess(processCtx, p, chVersion, socketPath) diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index 8dc62a11b..20ce5e2c9 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -469,21 +469,22 @@ func NewClient(hvType Type, socketPath string) (Hypervisor, error) { return WrapHypervisor(hvType, client), nil } -// ClientReleaser closes and forgets a cached client for a socket whose VMM has exited. +// ClientReleaser drops what a hypervisor type caches for a socket whose VMM has +// exited, such as a pooled client or the last balloon target. type ClientReleaser func(socketPath string) -// clientReleasers maps hypervisor types that cache clients to their releasers. +// clientReleasers maps hypervisor types that cache per-socket state to their releasers. var clientReleasers = make(map[Type]ClientReleaser) // RegisterClientReleaser registers the releaser for a hypervisor type that -// caches clients per socket. +// caches state per socket. func RegisterClientReleaser(t Type, releaser ClientReleaser) { clientReleasers[t] = releaser } -// ReleaseClient closes and forgets any cached client for socketPath. Call it -// only once the VMM behind socketPath is confirmed gone. Hypervisor types that -// do not cache clients ignore it. +// ReleaseClient drops any state cached for socketPath, closing a cached client. +// Call it only once the VMM behind socketPath is confirmed gone. Hypervisor +// types that cache nothing ignore it. func ReleaseClient(hvType Type, socketPath string) { if release, ok := clientReleasers[hvType]; ok { release(socketPath) From d120537a085b552363d0d7ba1ec6778c32ab03c3 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:47:59 +0000 Subject: [PATCH 10/12] Rename the release hook to ReleaseVMMState For Cloud Hypervisor the hook only clears a balloon target, so there is no client involved. --- .../cloudhypervisor/cloudhypervisor_test.go | 4 +-- lib/hypervisor/cloudhypervisor/process.go | 2 +- lib/hypervisor/hypervisor.go | 26 +++++++++---------- lib/hypervisor/qemu/pool.go | 8 +++--- lib/hypervisor/qemu/pool_test.go | 4 +-- lib/hypervisor/qemu/process.go | 6 ++--- lib/instances/delete.go | 2 +- ...ient_test.go => release_vmm_state_test.go} | 22 ++++++++-------- lib/instances/stop.go | 2 +- 9 files changed, 38 insertions(+), 38 deletions(-) rename lib/instances/{release_client_test.go => release_vmm_state_test.go} (58%) diff --git a/lib/hypervisor/cloudhypervisor/cloudhypervisor_test.go b/lib/hypervisor/cloudhypervisor/cloudhypervisor_test.go index 5701965f5..14b5137f0 100644 --- a/lib/hypervisor/cloudhypervisor/cloudhypervisor_test.go +++ b/lib/hypervisor/cloudhypervisor/cloudhypervisor_test.go @@ -35,7 +35,7 @@ func TestAssignedGuestMemoryBytes(t *testing.T) { }) } -func TestReleaseClientForgetsBalloonTarget(t *testing.T) { +func TestReleaseVMMStateForgetsBalloonTarget(t *testing.T) { t.Parallel() socketPath := t.TempDir() + "/cloud-hypervisor.sock" @@ -44,7 +44,7 @@ func TestReleaseClientForgetsBalloonTarget(t *testing.T) { clearBalloonTargetCache(socketPath) }) - hypervisor.ReleaseClient(hypervisor.TypeCloudHypervisor, socketPath) + hypervisor.ReleaseVMMState(hypervisor.TypeCloudHypervisor, socketPath) _, cached := balloonTargetCache.Load(socketPath) assert.False(t, cached) diff --git a/lib/hypervisor/cloudhypervisor/process.go b/lib/hypervisor/cloudhypervisor/process.go index 86916f015..5f20cf648 100644 --- a/lib/hypervisor/cloudhypervisor/process.go +++ b/lib/hypervisor/cloudhypervisor/process.go @@ -59,7 +59,7 @@ func init() { hypervisor.RegisterClientFactory(hypervisor.TypeCloudHypervisor, func(socketPath string) (hypervisor.Hypervisor, error) { return New(socketPath) }) - hypervisor.RegisterClientReleaser(hypervisor.TypeCloudHypervisor, clearBalloonTargetCache) + hypervisor.RegisterVMMStateReleaser(hypervisor.TypeCloudHypervisor, clearBalloonTargetCache) } // Starter implements hypervisor.VMStarter for Cloud Hypervisor. diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index 20ce5e2c9..e7c5f068f 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -469,24 +469,24 @@ func NewClient(hvType Type, socketPath string) (Hypervisor, error) { return WrapHypervisor(hvType, client), nil } -// ClientReleaser drops what a hypervisor type caches for a socket whose VMM has -// exited, such as a pooled client or the last balloon target. -type ClientReleaser func(socketPath string) +// VMMStateReleaser drops what a hypervisor type caches for a socket whose VMM +// has exited, such as a pooled client or the last balloon target. +type VMMStateReleaser func(socketPath string) -// clientReleasers maps hypervisor types that cache per-socket state to their releasers. -var clientReleasers = make(map[Type]ClientReleaser) +// vmmStateReleasers maps hypervisor types that cache per-socket state to their releasers. +var vmmStateReleasers = make(map[Type]VMMStateReleaser) -// RegisterClientReleaser registers the releaser for a hypervisor type that +// RegisterVMMStateReleaser registers the releaser for a hypervisor type that // caches state per socket. -func RegisterClientReleaser(t Type, releaser ClientReleaser) { - clientReleasers[t] = releaser +func RegisterVMMStateReleaser(t Type, releaser VMMStateReleaser) { + vmmStateReleasers[t] = releaser } -// ReleaseClient drops any state cached for socketPath, closing a cached client. -// Call it only once the VMM behind socketPath is confirmed gone. Hypervisor -// types that cache nothing ignore it. -func ReleaseClient(hvType Type, socketPath string) { - if release, ok := clientReleasers[hvType]; ok { +// ReleaseVMMState drops any state cached for socketPath, closing a cached +// client. Call it only once the VMM behind socketPath is confirmed gone. +// Hypervisor types that cache nothing ignore it. +func ReleaseVMMState(hvType Type, socketPath string) { + if release, ok := vmmStateReleasers[hvType]; ok { release(socketPath) } } diff --git a/lib/hypervisor/qemu/pool.go b/lib/hypervisor/qemu/pool.go index 1a686132a..b16018ee1 100644 --- a/lib/hypervisor/qemu/pool.go +++ b/lib/hypervisor/qemu/pool.go @@ -80,10 +80,10 @@ func Remove(socketPath string) { closeClientAsync(client) } -// releaseClient drops what is cached for a socket whose QEMU has exited. Unlike -// Remove, it also forgets the balloon target, which has to outlive a reconnect -// to a QEMU that is still running. -func releaseClient(socketPath string) { +// releaseVMMState drops what is cached for a socket whose QEMU has exited. +// Unlike Remove, it also forgets the balloon target, which has to outlive a +// reconnect to a QEMU that is still running. +func releaseVMMState(socketPath string) { Remove(socketPath) clearBalloonTargetCache(socketPath) } diff --git a/lib/hypervisor/qemu/pool_test.go b/lib/hypervisor/qemu/pool_test.go index 5ef3e6917..b78eb56f4 100644 --- a/lib/hypervisor/qemu/pool_test.go +++ b/lib/hypervisor/qemu/pool_test.go @@ -54,7 +54,7 @@ func TestGetOrCreateForTypeRejectsCachedBackendMismatch(t *testing.T) { require.Equal(t, hypervisor.TypeQEMU, cached.profile.hypervisorType(), "a stale caller must not evict the live client") } -func TestReleaseClientClosesPooledConnection(t *testing.T) { +func TestReleaseVMMStateClosesPooledConnection(t *testing.T) { for _, hvType := range []hypervisor.Type{hypervisor.TypeQEMU, hypervisor.TypeQEMUMicroVM} { t.Run(string(hvType), func(t *testing.T) { // t.TempDir() overflows the unix socket path limit on macOS. @@ -68,7 +68,7 @@ func TestReleaseClientClosesPooledConnection(t *testing.T) { require.NoError(t, err) balloonTargetCache.Store(socketPath, 512<<20) - hypervisor.ReleaseClient(hvType, socketPath) + hypervisor.ReleaseVMMState(hvType, socketPath) clientPool.RLock() _, pooled := clientPool.clients[socketPath] diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index c914f79eb..b6da760cc 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -55,7 +55,7 @@ func init() { hypervisor.RegisterClientFactory(t, func(socketPath string) (hypervisor.Hypervisor, error) { return NewForType(socketPath, t) }) - hypervisor.RegisterClientReleaser(t, releaseClient) + hypervisor.RegisterVMMStateReleaser(t, releaseVMMState) } } @@ -314,7 +314,7 @@ func (p *startedProcess) cleanup() { // come after the QMP client is pooled, so the client is dropped with it. func cleanupFailedStart(proc *startedProcess) { proc.cleanup() - releaseClient(proc.socketPath) + releaseVMMState(proc.socketPath) } func vfioTermGraceFor(cfg hypervisor.VMConfig) time.Duration { @@ -353,7 +353,7 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version // A QEMU that exited on its own leaves its QMP client and balloon target // cached. Drop both before this process reuses the socket path. - releaseClient(socketPath) + releaseVMMState(socketPath) os.Remove(socketPath) // Create command diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 09b7edd31..c89f38584 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -146,7 +146,7 @@ func (m *manager) deleteInstanceWithOptions( // The VMM is gone. Close its cached control connection, and the guest-agent // connection that the graceful shutdown RPC opened again after step 3. Both // would otherwise stay open for the life of the process. - hypervisor.ReleaseClient(inst.HypervisorType, inst.SocketPath) + hypervisor.ReleaseVMMState(inst.HypervisorType, inst.SocketPath) if dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID); err == nil { guest.CloseConn(dialer.Key()) } diff --git a/lib/instances/release_client_test.go b/lib/instances/release_vmm_state_test.go similarity index 58% rename from lib/instances/release_client_test.go rename to lib/instances/release_vmm_state_test.go index 408fd7699..cc30f54b2 100644 --- a/lib/instances/release_client_test.go +++ b/lib/instances/release_vmm_state_test.go @@ -11,29 +11,29 @@ import ( "github.com/stretchr/testify/require" ) -const releaseClientTestHypervisorType hypervisor.Type = "release-client-test" +const releaseVMMStateTestHypervisorType hypervisor.Type = "release-vmm-state-test" -var releasedClientSockets sync.Map +var releasedSockets sync.Map func init() { - hypervisor.RegisterClientFactory(releaseClientTestHypervisorType, func(socketPath string) (hypervisor.Hypervisor, error) { + hypervisor.RegisterClientFactory(releaseVMMStateTestHypervisorType, func(socketPath string) (hypervisor.Hypervisor, error) { state, ok := lifecycleNoopHypervisorStates.Load(socketPath) if !ok { return nil, errors.New("missing fake hypervisor state") } return lifecycleNoopHypervisor{state: state.(hypervisor.VMState)}, nil }) - hypervisor.RegisterVsockDialerFactory(releaseClientTestHypervisorType, func(socketPath string, _ int64) hypervisor.VsockDialer { + hypervisor.RegisterVsockDialerFactory(releaseVMMStateTestHypervisorType, func(socketPath string, _ int64) hypervisor.VsockDialer { return lifecycleNoopVsockDialer{socketPath: socketPath} }) - hypervisor.RegisterClientReleaser(releaseClientTestHypervisorType, func(socketPath string) { - releasedClientSockets.Store(socketPath, true) + hypervisor.RegisterVMMStateReleaser(releaseVMMStateTestHypervisorType, func(socketPath string) { + releasedSockets.Store(socketPath, true) }) } -// A deleted or stopped VM's cached hypervisor client must be released, or its -// connection stays open for the life of the process. -func TestDeleteAndStopReleaseHypervisorClient(t *testing.T) { +// What the hypervisor caches for a deleted or stopped VM must be released, or +// it stays for the life of the process. +func TestDeleteAndStopReleaseVMMState(t *testing.T) { for _, tc := range []struct { name string action func(context.Context, *manager, string) error @@ -48,12 +48,12 @@ func TestDeleteAndStopReleaseHypervisorClient(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC()) meta, err := m.loadMetadata(id) require.NoError(t, err) - meta.HypervisorType = releaseClientTestHypervisorType + meta.HypervisorType = releaseVMMStateTestHypervisorType require.NoError(t, m.saveMetadata(meta)) require.NoError(t, tc.action(context.Background(), m, id)) - _, released := releasedClientSockets.Load(meta.SocketPath) + _, released := releasedSockets.Load(meta.SocketPath) require.True(t, released) }) } diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 4c8f7b02a..31c7fec26 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -268,7 +268,7 @@ func (m *manager) stopInstance( if dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID); err == nil { guest.CloseConn(dialer.Key()) } - hypervisor.ReleaseClient(inst.HypervisorType, inst.SocketPath) + hypervisor.ReleaseVMMState(inst.HypervisorType, inst.SocketPath) // If graceful guest shutdown exits before shutdownHypervisor() is called, these // files may still exist and cause state derivation as Unknown or bind conflicts. _ = os.Remove(inst.SocketPath) From 85be5b526d4f1d921b19e89f9d03c2e9aa7e70b1 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:47:59 +0000 Subject: [PATCH 11/12] Do not hand a recreated socket the previous balloon target The cache is keyed by socket path, device and inode, so a recreated socket should miss. On a miss Load fell back to the key last indexed for the path and returned the old process's target anyway. Keep that fallback for the case it is needed for, a socket file that is missing while its VMM is still running. --- lib/hypervisor/balloon_target_cache.go | 14 ++++-- lib/hypervisor/balloon_target_cache_test.go | 51 +++++++++++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/lib/hypervisor/balloon_target_cache.go b/lib/hypervisor/balloon_target_cache.go index ccfd43a68..7e3ae1795 100644 --- a/lib/hypervisor/balloon_target_cache.go +++ b/lib/hypervisor/balloon_target_cache.go @@ -30,10 +30,16 @@ func (c *BalloonTargetCache) Load(socketPath string) (int64, bool) { return value, true } - if indexedKey, ok := c.keys.Load(socketPath); ok { - if keyString, ok := indexedKey.(string); ok { - if value, ok := c.loadKey(keyString); ok { - return value, true + // SocketCacheKey falls back to the bare path when the socket file is + // missing, which can happen while its VMM is still running. A socket that + // exists under another key belongs to a newer process, which must not + // inherit the old target. + if key == socketPath { + if indexedKey, ok := c.keys.Load(socketPath); ok { + if keyString, ok := indexedKey.(string); ok { + if value, ok := c.loadKey(keyString); ok { + return value, true + } } } } diff --git a/lib/hypervisor/balloon_target_cache_test.go b/lib/hypervisor/balloon_target_cache_test.go index 7ade8e37a..9a83e208b 100644 --- a/lib/hypervisor/balloon_target_cache_test.go +++ b/lib/hypervisor/balloon_target_cache_test.go @@ -47,6 +47,57 @@ func TestBalloonTargetCacheDeleteClearsIndexedKeyAfterSocketRemoval(t *testing.T assert.False(t, ok) } +func TestBalloonTargetCacheSurvivesMissingSocket(t *testing.T) { + t.Parallel() + + socketPath := testSocketPath(t) + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + + var cache BalloonTargetCache + cache.Store(socketPath, 512) + + require.NoError(t, listener.Close()) + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + require.NoError(t, err) + } + + value, ok := cache.Load(socketPath) + require.True(t, ok) + assert.Equal(t, int64(512), value) +} + +func TestBalloonTargetCacheMissesForRecreatedSocket(t *testing.T) { + t.Parallel() + + socketPath := testSocketPath(t) + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + + var cache BalloonTargetCache + cache.Store(socketPath, 512) + oldKey := SocketCacheKey(socketPath) + + require.NoError(t, listener.Close()) + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + require.NoError(t, err) + } + // Some filesystems hand a freed inode straight back, so occupy it. + placeholder := socketPath + ".hold" + require.NoError(t, os.WriteFile(placeholder, nil, 0o600)) + t.Cleanup(func() { _ = os.Remove(placeholder) }) + + listener, err = net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + if SocketCacheKey(socketPath) == oldKey { + t.Skip("filesystem reused the socket inode") + } + + _, ok := cache.Load(socketPath) + assert.False(t, ok) +} + func testSocketPath(t *testing.T) string { t.Helper() From 4a4c5851eef887d327ec6dfd20ac54a476cc9d93 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:47:59 +0000 Subject: [PATCH 12/12] Make the QEMU cache tests clean up and not depend on inode reuse The start test now reads the target back with the socket file removed, which finds a leftover on any filesystem. Both tests drop what they cached when they fail early. --- lib/hypervisor/qemu/pool_test.go | 1 + lib/hypervisor/qemu/process_test.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/lib/hypervisor/qemu/pool_test.go b/lib/hypervisor/qemu/pool_test.go index b78eb56f4..345b4ea5c 100644 --- a/lib/hypervisor/qemu/pool_test.go +++ b/lib/hypervisor/qemu/pool_test.go @@ -62,6 +62,7 @@ func TestReleaseVMMStateClosesPooledConnection(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = os.RemoveAll(dir) }) socketPath := filepath.Join(dir, "qemu.sock") + t.Cleanup(func() { releaseVMMState(socketPath) }) closed := serveFakeQMP(t, socketPath) _, err = GetOrCreateForType(socketPath, hvType) diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index 3247c61b9..4411f3d9c 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -430,12 +430,17 @@ func TestStartQEMUProcessDropsStaleBalloonTarget(t *testing.T) { t.Cleanup(func() { _ = os.RemoveAll(dir) }) socketPath := filepath.Join(dir, "qemu.sock") balloonTargetCache.Store(socketPath, 512<<20) + t.Cleanup(func() { clearBalloonTargetCache(socketPath) }) args := append(buildQMPArgs(socketPath), "-machine", "none", "-nodefaults", "-display", "none") _, _, cu, err := starter.startQEMUProcess(t.Context(), paths.New(dir), "", socketPath, args, 0) require.NoError(t, err) t.Cleanup(cu.Clean) + // Whether a leftover target is still reachable through the new socket + // depends on the filesystem reusing its inode. With the socket file gone + // the lookup is by path alone, so it finds a leftover on any filesystem. + require.NoError(t, os.Remove(socketPath)) _, cached := balloonTargetCache.Load(socketPath) require.False(t, cached) }