From ca1157d8ac9fae8251faa801784032433fb50c98 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 19:05:18 +0000
Subject: [PATCH 01/38] Guard vGPU releases with live-instance claims

A vGPU assignment goes stale when its release succeeds but the
metadata save does not (or start fails between the release and its
first save). The backend's owner map only covers assignments created
since the last restart and the VFIO handle scan only covers VMs that
have opened the device, so after a restart a stale release could still
clear a VF during another live instance's pre-open boot window.
Consult live instance metadata on every release: when another instance
with a live hypervisor process claims the same device path, drop the
stale metadata without touching the device.

Tag assignments with the owning instance ID, persist the assignment
before booting a started instance, and retain assignment metadata when
rollback release fails in create and start so later release paths can
still find the device.
---
 lib/instances/create.go              | 41 ++++++++++++++++++++-
 lib/instances/delete.go              |  2 +-
 lib/instances/lifecycle_noop_test.go | 55 ++++++++++++++++++++++++++++
 lib/instances/start.go               |  9 ++++-
 lib/instances/stop.go                |  2 +-
 lib/instances/vgpu.go                | 44 +++++++++++++++++-----
 lib/instances/vgpu_test.go           | 50 ++++++++++++++++++++-----
 7 files changed, 180 insertions(+), 23 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index f24519a9..ee32ce50 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -262,11 +262,13 @@ func (m *manager) createInstance(
 	var gpuFramework devices.VGPUFramework
 	var gpuDevicePath string
 	var gpuMdevUUID string
+	var stored *StoredMetadata
+	var retainedVGPU *StoredMetadata
 
 	// Setup cleanup stack early so device attachment errors trigger cleanup
 	cu := cleanup.Make(func() {
 		log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id)
-		m.deleteInstanceData(id)
+		m.cleanupFailedCreate(ctx, id, retainedVGPU)
 	})
 	defer cu.Clean()
 
@@ -305,6 +307,25 @@ func (m *manager) createInstance(
 			}
 			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
 				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err)
+				retainedVGPU = stored
+				if retainedVGPU == nil {
+					retainedVGPU = &StoredMetadata{
+						Id:                id,
+						Name:              req.Name,
+						Image:             req.Image,
+						ResolvedImage:     resolvedImageRef,
+						Platform:          imageInfo.Platform,
+						CreatedAt:         time.Now(),
+						HypervisorType:    hvType,
+						HypervisorVersion: hvVersion,
+						SocketPath:        m.paths.InstanceSocket(id, starter.SocketName()),
+						DataDir:           m.paths.InstanceDir(id),
+						GPUProfile:        gpuDevice.ProfileName,
+						GPUFramework:      gpuDevice.Framework,
+						GPUDevicePath:     gpuDevice.SysfsPath,
+						GPUMdevUUID:       gpuDevice.MdevUUID,
+					}
+				}
 			}
 		})
 	}
@@ -340,7 +361,7 @@ func (m *manager) createInstance(
 	}
 
 	// 11. Create instance metadata
-	stored := &StoredMetadata{
+	stored = &StoredMetadata{
 		Id:                       id,
 		Name:                     req.Name,
 		Image:                    req.Image,
@@ -574,6 +595,22 @@ func (m *manager) createInstance(
 	return &finalInst, nil
 }
 
+func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) {
+	if retainedVGPU == nil {
+		m.deleteInstanceData(id)
+		return
+	}
+
+	log := logger.FromContext(ctx)
+	if err := m.ensureDirectories(id); err != nil {
+		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
+		return
+	}
+	if err := m.saveMetadata(&metadata{StoredMetadata: *retainedVGPU}); err != nil {
+		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
+	}
+}
+
 // validateCreateRequest validates the create instance request.
 // The request is mutated in-place to persist normalized egress/credential policy fields.
 func validateCreateRequest(req *CreateInstanceRequest) error {
diff --git a/lib/instances/delete.go b/lib/instances/delete.go
index 9dc83711..cd7a0f19 100644
--- a/lib/instances/delete.go
+++ b/lib/instances/delete.go
@@ -152,7 +152,7 @@ func (m *manager) deleteInstanceWithOptions(
 	if hadVGPUAssignment {
 		log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID)
 	}
-	if err := releaseStoredVGPU(ctx, stored); err != nil {
+	if err := m.releaseStoredVGPU(ctx, stored); err != nil {
 		// Log error but continue with cleanup.
 		log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err)
 	} else if hadVGPUAssignment {
diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go
index a9b918df..89468d97 100644
--- a/lib/instances/lifecycle_noop_test.go
+++ b/lib/instances/lifecycle_noop_test.go
@@ -3,6 +3,7 @@ package instances
 import (
 	"context"
 	"errors"
+	"net"
 	"os"
 	"path/filepath"
 	"sync"
@@ -195,6 +196,46 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) {
 	assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason)
 }
 
+func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) {
+	now := time.Now().UTC()
+	m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now)
+	meta, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	meta.GPUProfile = "NVIDIA L40S-2Q"
+	meta.GPUFramework = devices.VGPUFramework("future-framework")
+	meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4"
+	require.NoError(t, m.saveMetadata(meta))
+
+	claimantID := "inst-live-claimant"
+	require.NoError(t, m.ensureDirectories(claimantID))
+	pid := os.Getpid()
+	socketPath := m.paths.InstanceSocket(claimantID, "noop.sock")
+	listener, err := net.Listen("unix", socketPath)
+	require.NoError(t, err)
+	defer listener.Close()
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:             claimantID,
+		Name:           claimantID,
+		Image:          "test-image",
+		CreatedAt:      now,
+		HypervisorType: lifecycleNoopHypervisorType,
+		HypervisorPID:  &pid,
+		SocketPath:     socketPath,
+		DataDir:        m.paths.InstanceDir(claimantID),
+		GPUProfile:     "NVIDIA L40S-2Q",
+		GPUFramework:   devices.VGPUFramework("future-framework"),
+		GPUDevicePath:  "/sys/bus/pci/devices/0000:82:00.4",
+	}}))
+
+	require.NoError(t, m.DeleteInstance(context.Background(), id))
+
+	_, err = m.loadMetadata(id)
+	require.Error(t, err, "deleted instance metadata should be gone")
+	claimant, err := m.loadMetadata(claimantID)
+	require.NoError(t, err)
+	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", claimant.GPUDevicePath, "live claimant keeps its assignment")
+}
+
 func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) {
 	m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC())
 	deviceManager := &recordingDeviceManager{}
@@ -300,6 +341,20 @@ func (m *recordingDeviceManager) UnbindFromVFIO(ctx context.Context, id string)
 	return nil
 }
 
+func TestLifecycleNoopStandbyRejectsVendorVFIOVGPU(t *testing.T) {
+	m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC())
+	meta, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	meta.GPUProfile = "NVIDIA L40S-2Q"
+	meta.GPUFramework = devices.VGPUFrameworkVendorVFIO
+	meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4"
+	require.NoError(t, m.saveMetadata(meta))
+
+	_, err = m.StandbyInstance(context.Background(), id, StandbyInstanceRequest{})
+	require.ErrorIs(t, err, ErrInvalidState)
+	assert.ErrorContains(t, err, "standby is not supported for instances with vGPU attached")
+}
+
 func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) {
 	t.Helper()
 
diff --git a/lib/instances/start.go b/lib/instances/start.go
index a6c83245..ff8e94ea 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -54,7 +54,7 @@ func (m *manager) startInstance(
 	// cannot leave on-disk metadata pointing at a device that is already
 	// gone (matching releaseRetainedVGPULocked).
 	if storedVGPUDevicePath(stored) != "" {
-		if err := releaseStoredVGPU(ctx, stored); err != nil {
+		if err := m.releaseStoredVGPU(ctx, stored); err != nil {
 			log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err)
 			return nil, fmt.Errorf("release stale vGPU before start: %w", err)
 		}
@@ -181,8 +181,15 @@ func (m *manager) startInstance(
 			}
 			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
 				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err)
+				if saveErr := m.saveMetadata(meta); saveErr != nil {
+					log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr)
+				}
 			}
 		})
+		if err := m.saveMetadata(meta); err != nil {
+			log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err)
+			return nil, fmt.Errorf("save metadata after vGPU creation: %w", err)
+		}
 	}
 
 	// 5. Regenerate config disk with new network configuration
diff --git a/lib/instances/stop.go b/lib/instances/stop.go
index b23ff91c..9276b00d 100644
--- a/lib/instances/stop.go
+++ b/lib/instances/stop.go
@@ -262,7 +262,7 @@ func (m *manager) stopInstance(
 	// 7. Release the vGPU assignment if present (frees the vGPU slot for other VMs).
 	if storedVGPUDevicePath(stored) != "" {
 		log.InfoContext(ctx, "destroying vGPU on stop", "instance_id", id, "uuid", stored.GPUMdevUUID)
-		if err := releaseStoredVGPU(ctx, stored); err != nil {
+		if err := m.releaseStoredVGPU(ctx, stored); err != nil {
 			// Log error but continue - vGPU cleanup is best-effort
 			log.WarnContext(ctx, "failed to destroy vGPU on stop; retaining assignment metadata", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err)
 		}
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index a8ca6ace..b7d27542 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -20,23 +20,49 @@ func clearStoredVGPUDevice(stored *StoredMetadata) {
 	stored.GPUMdevUUID = ""
 }
 
-func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error {
+func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error {
 	path := storedVGPUDevicePath(stored)
 	if path != "" {
-		assignment := devices.VGPUAssignment{
-			Framework:  stored.GPUFramework,
-			DevicePath: path,
-			MdevUUID:   stored.GPUMdevUUID,
-			InstanceID: stored.Id,
-		}
-		if err := devices.DestroyVGPU(ctx, assignment); err != nil {
+		claimed, err := m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path)
+		if err != nil {
 			return err
 		}
+		if claimed {
+			logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance",
+				"instance_id", stored.Id, "device_path", path)
+		} else {
+			assignment := devices.VGPUAssignment{
+				Framework:  stored.GPUFramework,
+				DevicePath: path,
+				MdevUUID:   stored.GPUMdevUUID,
+				InstanceID: stored.Id,
+			}
+			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
+				return err
+			}
+		}
 	}
 	clearStoredVGPUDevice(stored)
 	return nil
 }
 
+func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) {
+	instances, err := m.listInstances(ctx)
+	if err != nil {
+		return false, fmt.Errorf("list instances for vGPU release check: %w", err)
+	}
+	for i := range instances {
+		inst := &instances[i]
+		if inst.Id == excludeID || inst.GPUDevicePath != devicePath || inst.HypervisorPID == nil {
+			continue
+		}
+		if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 // releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped
 // instance after a failed release during the original stop. It is a no-op
 // when no assignment is retained, and a failed retry only logs so the
@@ -52,7 +78,7 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) {
 	if storedVGPUDevicePath(stored) == "" {
 		return
 	}
-	if err := releaseStoredVGPU(ctx, stored); err != nil {
+	if err := m.releaseStoredVGPU(ctx, stored); err != nil {
 		log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err)
 		return
 	}
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 6f2c4681..2b7d84a9 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -5,14 +5,47 @@ import (
 	"testing"
 
 	"github.com/kernel/hypeman/lib/devices"
+	"github.com/kernel/hypeman/lib/hypervisor"
+	"github.com/kernel/hypeman/lib/paths"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+func TestValidateVGPUHypervisor(t *testing.T) {
+	t.Parallel()
+
+	assert.NoError(t, validateVGPUHypervisor(hypervisor.TypeQEMU))
+	assert.EqualError(t, validateVGPUHypervisor(hypervisor.TypeCloudHypervisor), "vGPU is only supported with qemu, got cloud-hypervisor")
+}
+
+func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	stored := &StoredMetadata{
+		Id:             "failed-create",
+		Name:           "failed-create",
+		GPUProfile:     "NVIDIA L40S-2Q",
+		GPUFramework:   devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath:  "/sys/bus/pci/devices/0000:82:00.4",
+		HypervisorType: "qemu",
+		DataDir:        m.paths.InstanceDir("failed-create"),
+	}
+
+	m.cleanupFailedCreate(context.Background(), stored.Id, stored)
+
+	retained, err := m.loadMetadata(stored.Id)
+	require.NoError(t, err)
+	assert.Equal(t, stored.GPUProfile, retained.GPUProfile)
+	assert.Equal(t, stored.GPUFramework, retained.GPUFramework)
+	assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath)
+}
+
 func TestStoredVGPUDevicePath(t *testing.T) {
 	t.Parallel()
 
-	assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", storedVGPUDevicePath(&StoredMetadata{
-		GPUDevicePath: "/sys/bus/mdev/devices/new-uuid",
+	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", storedVGPUDevicePath(&StoredMetadata{
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
 		GPUMdevUUID:   "legacy-uuid",
 	}))
 	assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{
@@ -24,11 +57,12 @@ func TestStoredVGPUDevicePath(t *testing.T) {
 func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) {
 	t.Parallel()
 
+	m := &manager{paths: paths.New(t.TempDir())}
 	stored := &StoredMetadata{
 		GPUFramework:  devices.VGPUFramework("future-framework"),
 		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
 	}
-	err := releaseStoredVGPU(context.Background(), stored)
+	err := m.releaseStoredVGPU(context.Background(), stored)
 	assert.Error(t, err)
 	assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework)
 	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath)
@@ -39,13 +73,11 @@ func TestSetAndClearStoredVGPUDevice(t *testing.T) {
 
 	stored := &StoredMetadata{}
 	setStoredVGPUDevice(stored, &devices.VGPUDevice{
-		Framework: devices.VGPUFrameworkMdev,
-		SysfsPath: "/sys/bus/mdev/devices/new-uuid",
-		MdevUUID:  "new-uuid",
+		Framework: devices.VGPUFrameworkVendorVFIO,
+		SysfsPath: "/sys/bus/pci/devices/0000:82:00.4",
 	})
-	assert.Equal(t, devices.VGPUFrameworkMdev, stored.GPUFramework)
-	assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", stored.GPUDevicePath)
-	assert.Equal(t, "new-uuid", stored.GPUMdevUUID)
+	assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework)
+	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath)
 
 	clearStoredVGPUDevice(stored)
 	assert.Empty(t, stored.GPUFramework)

From 91862b5f6cdcfe867b74921270d7872aad6982ee Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 19:05:42 +0000
Subject: [PATCH 02/38] Reconcile vendor VFIO vGPUs against a fail-closed
 instance inventory

Startup reconciliation protects the VFs of instances whose hypervisor
survived the restart, verified by socket ownership so a reused PID
cannot hold a VF. The inventory behind that protected set must not
silently skip unreadable metadata: a skipped live claimant would leave
its VF unprotected during the pre-VFIO-open boot window. Add
ListInstancesForReconcile, which fails on any unreadable metadata, and
skip vendor VFIO reconciliation when the inventory is unavailable
while keeping mdev reconciliation running.
---
 cmd/api/main.go             | 33 ++++++++++++++++++++++++++++-----
 lib/builds/manager_test.go  |  4 ++++
 lib/instances/manager.go    |  6 ++++++
 lib/instances/query.go      | 13 +++++++++++--
 lib/instances/query_test.go | 22 ++++++++++++++++++++++
 lib/instances/storage.go    |  8 +++++++-
 lib/instances/wait_test.go  |  3 +++
 7 files changed, 81 insertions(+), 8 deletions(-)

diff --git a/cmd/api/main.go b/cmd/api/main.go
index ea7376b4..ae67fc7f 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -185,6 +185,24 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta
 	}, logger), nil
 }
 
+func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, error) {
+	allInstances, err := instanceManager.ListInstancesForReconcile(ctx)
+	if err != nil {
+		return nil, err
+	}
+	protected := make(map[string]struct{})
+	for _, inst := range allInstances {
+		if inst.GPUDevicePath == "" || inst.HypervisorPID == nil {
+			continue
+		}
+		if !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
+			continue
+		}
+		protected[inst.GPUDevicePath] = struct{}{}
+	}
+	return protected, nil
+}
+
 func run() error {
 	startupStarted := time.Now()
 	slog.Info("starting hypeman initialization")
@@ -389,11 +407,16 @@ func run() error {
 		return fmt.Errorf("reconcile device state: %w", err)
 	}
 
-	// Reconcile mdev devices (clears orphaned vGPUs from previous runs)
-	logger.Info("Reconciling mdev devices...")
-	if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil {
-		// Log but don't fail - mdev cleanup is best-effort
-		logger.Warn("failed to reconcile mdev devices", "error", err)
+	// Reconcile vGPU devices (clears orphaned vGPUs from previous runs)
+	logger.Info("Reconciling vGPU devices...")
+	protected, err := liveInstanceVGPUDevicePaths(app.Ctx, app.InstanceManager)
+	if err != nil {
+		logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err)
+		protected = nil
+	}
+	if err := devices.ReconcileVGPUs(app.Ctx, protected); err != nil {
+		// Log but don't fail - vGPU cleanup is best-effort
+		logger.Warn("failed to reconcile vGPU devices", "error", err)
 	}
 
 	// Wire up resource validator for aggregate limit checking
diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go
index 7be58b0f..b2838388 100644
--- a/lib/builds/manager_test.go
+++ b/lib/builds/manager_test.go
@@ -51,6 +51,10 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc
 	return result, nil
 }
 
+func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) {
+	return m.ListInstances(ctx, nil)
+}
+
 func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) {
 	return nil, nil
 }
diff --git a/lib/instances/manager.go b/lib/instances/manager.go
index 85f75975..d4206c91 100644
--- a/lib/instances/manager.go
+++ b/lib/instances/manager.go
@@ -27,6 +27,7 @@ import (
 
 type Manager interface {
 	ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error)
+	ListInstancesForReconcile(ctx context.Context) ([]Instance, error)
 	ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error)
 	GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error)
 	CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error)
@@ -696,6 +697,11 @@ func (m *manager) UpdateInstance(ctx context.Context, id string, req UpdateInsta
 	return inst, err
 }
 
+// ListInstancesForReconcile returns every instance or an invalid metadata error.
+func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) {
+	return m.loadInstances(ctx, false)
+}
+
 // ListInstances returns instances, optionally filtered by the given criteria.
 // Pass nil to return all instances.
 func (m *manager) ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) {
diff --git a/lib/instances/query.go b/lib/instances/query.go
index ff6d9d6a..a7556035 100644
--- a/lib/instances/query.go
+++ b/lib/instances/query.go
@@ -1018,14 +1018,18 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) {
 	return time.Time{}, false
 }
 
-// listInstances returns all instances
+// listInstances returns all instances, skipping metadata files that cannot be loaded.
 func (m *manager) listInstances(ctx context.Context) ([]Instance, error) {
+	return m.loadInstances(ctx, true)
+}
+
+func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instance, error) {
 	ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata")
 	defer span.End()
 	log := logger.FromContext(ctx)
 	log.DebugContext(ctx, "listing all instances")
 
-	files, err := m.listMetadataFiles()
+	files, err := m.listMetadataFilesWithStatErrors(!skipInvalid)
 	if err != nil {
 		log.ErrorContext(ctx, "failed to list metadata files", "error", err)
 		return nil, err
@@ -1043,6 +1047,11 @@ func (m *manager) listInstances(ctx context.Context) ([]Instance, error) {
 		)
 		meta, err := m.loadMetadata(id)
 		if err != nil {
+			if !skipInvalid {
+				hydrateSpan.RecordError(err)
+				hydrateSpan.End()
+				return nil, fmt.Errorf("load metadata for instance %s: %w", id, err)
+			}
 			// Skip instances with invalid metadata
 			log.WarnContext(hydrateCtx, "skipping instance with invalid metadata", "instance_id", id, "error", err)
 			hydrateSpan.End()
diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go
index 8bb0bb46..41aba54e 100644
--- a/lib/instances/query_test.go
+++ b/lib/instances/query_test.go
@@ -14,6 +14,28 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) {
+	m := &manager{paths: paths.New(t.TempDir())}
+
+	require.NoError(t, m.ensureDirectories("valid"))
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:        "valid",
+		Name:      "valid",
+		CreatedAt: time.Now(),
+		DataDir:   m.paths.InstanceDir("valid"),
+	}}))
+	require.NoError(t, m.ensureDirectories("invalid"))
+	require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid"), []byte("{"), 0644))
+
+	listed, err := m.ListInstances(context.Background(), nil)
+	require.NoError(t, err)
+	require.Len(t, listed, 1)
+
+	_, err = m.ListInstancesForReconcile(context.Background())
+	require.Error(t, err)
+	assert.ErrorContains(t, err, "load metadata for instance invalid")
+}
+
 func TestParseExitSentinelLine(t *testing.T) {
 	t.Parallel()
 	tests := []struct {
diff --git a/lib/instances/storage.go b/lib/instances/storage.go
index a293fc6e..dd932d41 100644
--- a/lib/instances/storage.go
+++ b/lib/instances/storage.go
@@ -187,8 +187,12 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti
 	}
 }
 
-// listMetadataFiles returns paths to all instance metadata files
+// listMetadataFiles returns paths to all instance metadata files.
 func (m *manager) listMetadataFiles() ([]string, error) {
+	return m.listMetadataFilesWithStatErrors(false)
+}
+
+func (m *manager) listMetadataFilesWithStatErrors(failOnStatError bool) ([]string, error) {
 	guestsDir := m.paths.GuestsDir()
 
 	// Ensure guests directory exists
@@ -210,6 +214,8 @@ func (m *manager) listMetadataFiles() ([]string, error) {
 		metaPath := filepath.Join(guestsDir, entry.Name(), "metadata.json")
 		if _, err := os.Stat(metaPath); err == nil {
 			metaFiles = append(metaFiles, metaPath)
+		} else if failOnStatError && !os.IsNotExist(err) {
+			return nil, fmt.Errorf("stat metadata for instance %s: %w", entry.Name(), err)
 		}
 	}
 
diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go
index bab6f06d..a4246479 100644
--- a/lib/instances/wait_test.go
+++ b/lib/instances/wait_test.go
@@ -32,6 +32,9 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er
 func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) {
 	return nil, nil
 }
+func (s *stubManager) ListInstancesForReconcile(context.Context) ([]Instance, error) {
+	return nil, nil
+}
 func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) {
 	return nil, nil
 }

From 62cd6f49335fa105b045532351a035e5d1801d34 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 20:19:26 +0000
Subject: [PATCH 03/38] Fail closed on vGPU claim checks

---
 lib/instances/vgpu.go      |  2 +-
 lib/instances/vgpu_test.go | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index b7d27542..29edfeab 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -47,7 +47,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata)
 }
 
 func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) {
-	instances, err := m.listInstances(ctx)
+	instances, err := m.ListInstancesForReconcile(ctx)
 	if err != nil {
 		return false, fmt.Errorf("list instances for vGPU release check: %w", err)
 	}
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 2b7d84a9..29341d23 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -2,6 +2,7 @@ package instances
 
 import (
 	"context"
+	"os"
 	"testing"
 
 	"github.com/kernel/hypeman/lib/devices"
@@ -41,6 +42,17 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 	assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath)
 }
 
+func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	require.NoError(t, m.ensureDirectories("invalid-instance"))
+	require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644))
+
+	_, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4")
+	require.Error(t, err)
+}
+
 func TestStoredVGPUDevicePath(t *testing.T) {
 	t.Parallel()
 

From 98b4d3d68bd90b736828e906ce15ee47579793a6 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 20:19:50 +0000
Subject: [PATCH 04/38] Retain only vGPU assignment after failed create

---
 lib/instances/create.go    |  8 +++++++-
 lib/instances/vgpu_test.go | 13 ++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index ee32ce50..c84d0534 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -606,7 +606,13 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
 		return
 	}
-	if err := m.saveMetadata(&metadata{StoredMetadata: *retainedVGPU}); err != nil {
+	retained := StoredMetadata{
+		Id:            id,
+		GPUFramework:  retainedVGPU.GPUFramework,
+		GPUDevicePath: retainedVGPU.GPUDevicePath,
+		GPUMdevUUID:   retainedVGPU.GPUMdevUUID,
+	}
+	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
 	}
 }
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 29341d23..b408ccaf 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -29,6 +29,10 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 		GPUProfile:     "NVIDIA L40S-2Q",
 		GPUFramework:   devices.VGPUFrameworkVendorVFIO,
 		GPUDevicePath:  "/sys/bus/pci/devices/0000:82:00.4",
+		GPUMdevUUID:    "mdev-uuid",
+		NetworkEnabled: true,
+		IP:             "192.0.2.1",
+		Volumes:        []VolumeAttachment{{VolumeID: "volume"}},
 		HypervisorType: "qemu",
 		DataDir:        m.paths.InstanceDir("failed-create"),
 	}
@@ -37,9 +41,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 
 	retained, err := m.loadMetadata(stored.Id)
 	require.NoError(t, err)
-	assert.Equal(t, stored.GPUProfile, retained.GPUProfile)
+	assert.Equal(t, stored.Id, retained.Id)
 	assert.Equal(t, stored.GPUFramework, retained.GPUFramework)
 	assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath)
+	assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID)
+	assert.Empty(t, retained.Name)
+	assert.Empty(t, retained.GPUProfile)
+	assert.False(t, retained.NetworkEnabled)
+	assert.Empty(t, retained.IP)
+	assert.Empty(t, retained.Volumes)
+	assert.Empty(t, retained.DataDir)
 }
 
 func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) {

From 44b821869f91f1dcaa7e96cf58c9dc6b057538a2 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 20:20:05 +0000
Subject: [PATCH 05/38] Clear released vGPU assignment on start rollback

---
 lib/instances/start.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/instances/start.go b/lib/instances/start.go
index ff8e94ea..d9c8c9aa 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -184,6 +184,11 @@ func (m *manager) startInstance(
 				if saveErr := m.saveMetadata(meta); saveErr != nil {
 					log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr)
 				}
+			} else {
+				clearStoredVGPUDevice(stored)
+				if saveErr := m.saveMetadata(meta); saveErr != nil {
+					log.ErrorContext(ctx, "failed to save metadata after vGPU cleanup", "instance_id", id, "error", saveErr)
+				}
 			}
 		})
 		if err := m.saveMetadata(meta); err != nil {

From 99ffa312c5747fd0915561f81e8386c8d8a9024d Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 20:50:00 +0000
Subject: [PATCH 06/38] Test start rollback vGPU cleanup

---
 lib/instances/vgpu_test.go | 70 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index b408ccaf..d9aa02f0 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -3,7 +3,10 @@ package instances
 import (
 	"context"
 	"os"
+	"path/filepath"
+	"sync"
 	"testing"
+	_ "unsafe"
 
 	"github.com/kernel/hypeman/lib/devices"
 	"github.com/kernel/hypeman/lib/hypervisor"
@@ -53,6 +56,73 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 	assert.Empty(t, retained.DataDir)
 }
 
+//go:linkname hostVendorVFIO github.com/kernel/hypeman/lib/devices.hostVendorVFIO
+var hostVendorVFIO vendorVFIOSysfs
+
+type vendorVFIOSysfs struct {
+	pciDevicesPath  string
+	procPath        string
+	vfioDevicesPath string
+	owners          map[string]string
+}
+
+func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {
+	root := t.TempDir()
+	pciDevicesPath := filepath.Join(root, "sys", "bus", "pci", "devices")
+	vfAddress := "0000:82:00.4"
+	nvidiaPath := filepath.Join(pciDevicesPath, vfAddress, "nvidia")
+	require.NoError(t, os.MkdirAll(nvidiaPath, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte("0"), 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte("ID    : vGPU Name\n1148  : NVIDIA L40S-2Q\n"), 0o644))
+
+	originalVendorVFIO := hostVendorVFIO
+	hostVendorVFIO = vendorVFIOSysfs{
+		pciDevicesPath:  pciDevicesPath,
+		procPath:        filepath.Join(root, "proc"),
+		vfioDevicesPath: filepath.Join(root, "dev", "vfio", "devices"),
+		owners:          make(map[string]string),
+	}
+	t.Cleanup(func() { hostVendorVFIO = originalVendorVFIO })
+	require.NoError(t, os.MkdirAll(hostVendorVFIO.procPath, 0o755))
+
+	m := &manager{
+		paths:           paths.New(t.TempDir()),
+		imageManager:    readyFixtureImageManager{name: "test-image"},
+		instanceLocks:   sync.Map{},
+		bootMarkerScans: sync.Map{},
+	}
+	const id = "start-rollback"
+	require.NoError(t, m.ensureDirectories(id))
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:             id,
+		Name:           id,
+		Image:          "test-image",
+		GPUProfile:     "NVIDIA L40S-2Q",
+		HypervisorType: lifecycleNoopHypervisorType,
+		SocketPath:     m.paths.InstanceSocket(id, "noop.sock"),
+		DataDir:        m.paths.InstanceDir(id),
+	}}))
+
+	t.Setenv("TMPDIR", filepath.Join(root, "missing"))
+	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{})
+	require.Error(t, err)
+
+	stored, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile)
+	assert.Empty(t, stored.GPUFramework)
+	assert.Empty(t, stored.GPUDevicePath)
+	assert.Empty(t, stored.GPUMdevUUID)
+	assertFileContents(t, filepath.Join(nvidiaPath, "current_vgpu_type"), "0")
+}
+
+func assertFileContents(t *testing.T, path, want string) {
+	t.Helper()
+	got, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.Equal(t, want, string(got))
+}
+
 func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) {
 	t.Parallel()
 

From b61c833750cd71fab71bb55dce396789bc329121 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Fri, 7 Aug 2026 15:02:16 +0000
Subject: [PATCH 07/38] Normalize legacy mdev paths in live-claim check

The claim guard compared raw GPUDevicePath, which is empty on records
persisted before the framework migration; a live claimant with only a
legacy GPUMdevUUID was invisible to the check. Normalize the inventory
side with storedVGPUDevicePath, matching the release subject.
---
 lib/instances/vgpu.go      |  2 +-
 lib/instances/vgpu_test.go | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 29edfeab..23ea2d82 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -53,7 +53,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu
 	}
 	for i := range instances {
 		inst := &instances[i]
-		if inst.Id == excludeID || inst.GPUDevicePath != devicePath || inst.HypervisorPID == nil {
+		if inst.Id == excludeID || storedVGPUDevicePath(&inst.StoredMetadata) != devicePath || inst.HypervisorPID == nil {
 			continue
 		}
 		if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index d9aa02f0..79e6fe0c 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -134,6 +134,24 @@ func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T)
 	require.Error(t, err)
 }
 
+func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	require.NoError(t, m.ensureDirectories("legacy-claimant"))
+	pid := os.Getpid()
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            "legacy-claimant",
+		Name:          "legacy-claimant",
+		GPUMdevUUID:   "legacy-uuid",
+		HypervisorPID: &pid,
+	}}))
+
+	claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid")
+	require.NoError(t, err)
+	assert.True(t, claimed)
+}
+
 func TestStoredVGPUDevicePath(t *testing.T) {
 	t.Parallel()
 

From 4e2b4b1eab4167f0198c56548c20be37e0c31805 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Fri, 7 Aug 2026 20:45:49 +0000
Subject: [PATCH 08/38] Bind the live-claimant test socket under /tmp for macOS

---
 lib/instances/lifecycle_noop_test.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go
index 89468d97..f3baa9da 100644
--- a/lib/instances/lifecycle_noop_test.go
+++ b/lib/instances/lifecycle_noop_test.go
@@ -209,7 +209,14 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) {
 	claimantID := "inst-live-claimant"
 	require.NoError(t, m.ensureDirectories(claimantID))
 	pid := os.Getpid()
-	socketPath := m.paths.InstanceSocket(claimantID, "noop.sock")
+	// Bind under /tmp: a t.TempDir()-derived path exceeds the macOS AF_UNIX
+	// path limit.
+	socketDir, err := os.MkdirTemp("/tmp", "hypeman-claimant-socket-")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = os.RemoveAll(socketDir)
+	})
+	socketPath := filepath.Join(socketDir, "noop.sock")
 	listener, err := net.Listen("unix", socketPath)
 	require.NoError(t, err)
 	defer listener.Close()

From a57d3a929b9129bd256e98b6315ca7e545a5a7da Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Fri, 7 Aug 2026 20:45:49 +0000
Subject: [PATCH 09/38] Surface retained vGPU cleanup through a typed create
 error and manager seam

Replace the go:linkname shadow of devices.hostVendorVFIO with createVGPU/destroyVGPU manager fields, and wrap failed creates whose rollback release also failed in VGPUCleanupPendingError so the API can point callers at the retained instance record.
---
 cmd/api/api/instances.go   |  7 +++
 lib/instances/create.go    | 28 +++++++++---
 lib/instances/manager.go   |  4 ++
 lib/instances/start.go     |  8 ++--
 lib/instances/vgpu.go      | 40 ++++++++++++++++-
 lib/instances/vgpu_test.go | 92 ++++++++++++++++++++++++--------------
 6 files changed, 134 insertions(+), 45 deletions(-)

diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go
index adcaf05f..a55bed0f 100644
--- a/cmd/api/api/instances.go
+++ b/cmd/api/api/instances.go
@@ -342,6 +342,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 
 	inst, err := s.InstanceManager.CreateInstance(ctx, domainReq)
 	if err != nil {
+		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
 		case errors.Is(err, instances.ErrImageNotReady):
 			return oapi.CreateInstance400JSONResponse{
@@ -388,6 +389,12 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 				Code:    "not_found",
 				Message: err.Error(),
 			}, nil
+		case errors.As(err, &vgpuPending):
+			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
+			return oapi.CreateInstance500JSONResponse{
+				Code:    "vgpu_cleanup_pending",
+				Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID),
+			}, nil
 		default:
 			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
 			return oapi.CreateInstance500JSONResponse{
diff --git a/lib/instances/create.go b/lib/instances/create.go
index c84d0534..eea01fe2 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -265,10 +265,20 @@ func (m *manager) createInstance(
 	var stored *StoredMetadata
 	var retainedVGPU *StoredMetadata
 
-	// Setup cleanup stack early so device attachment errors trigger cleanup
+	// Setup cleanup stack early so device attachment errors trigger cleanup.
+	// When rollback retains a vGPU assignment, surface the retained instance
+	// ID to the caller so the record is discoverable and can be deleted to
+	// retry the release. The wrapping defer is registered first so it runs
+	// after cu.Clean has decided whether metadata was retained.
+	vgpuRetained := false
+	defer func() {
+		if retErr != nil && vgpuRetained {
+			retErr = &VGPUCleanupPendingError{InstanceID: id, Err: retErr}
+		}
+	}()
 	cu := cleanup.Make(func() {
 		log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id)
-		m.cleanupFailedCreate(ctx, id, retainedVGPU)
+		vgpuRetained = m.cleanupFailedCreate(ctx, id, retainedVGPU)
 	})
 	defer cu.Clean()
 
@@ -285,7 +295,7 @@ func (m *manager) createInstance(
 	// Handle vGPU profile request
 	if req.GPU != nil && req.GPU.Profile != "" {
 		log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile)
-		gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id)
+		gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id)
 		if err != nil {
 			log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err)
 			return nil, wrapCreateVGPUErr(req.GPU.Profile, err)
@@ -305,7 +315,7 @@ func (m *manager) createInstance(
 				MdevUUID:   gpuDevice.MdevUUID,
 				InstanceID: id,
 			}
-			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
+			if err := m.destroyVGPUAssignment(ctx, assignment); err != nil {
 				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err)
 				retainedVGPU = stored
 				if retainedVGPU == nil {
@@ -595,16 +605,18 @@ func (m *manager) createInstance(
 	return &finalInst, nil
 }
 
-func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) {
+// cleanupFailedCreate reports whether it retained instance metadata for a
+// vGPU assignment whose release failed during rollback.
+func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool {
 	if retainedVGPU == nil {
 		m.deleteInstanceData(id)
-		return
+		return false
 	}
 
 	log := logger.FromContext(ctx)
 	if err := m.ensureDirectories(id); err != nil {
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
-		return
+		return false
 	}
 	retained := StoredMetadata{
 		Id:            id,
@@ -614,7 +626,9 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
+		return false
 	}
+	return true
 }
 
 // validateCreateRequest validates the create instance request.
diff --git a/lib/instances/manager.go b/lib/instances/manager.go
index d4206c91..ebb98081 100644
--- a/lib/instances/manager.go
+++ b/lib/instances/manager.go
@@ -180,6 +180,8 @@ type manager struct {
 	tracer                    trace.Tracer
 	now                       func() time.Time
 	writeFile                 func(string, []byte, os.FileMode) error
+	createVGPU                func(context.Context, string, string) (*devices.VGPUDevice, error)
+	destroyVGPU               func(context.Context, devices.VGPUAssignment) error
 	deleteSnapshotFn          func(context.Context, string) error
 	egressProxy               *egressproxy.Service
 	egressProxyServiceOptions egressproxy.ServiceOptions
@@ -278,6 +280,8 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa
 		defaultHypervisor:                defaultHypervisor,
 		now:                              time.Now,
 		writeFile:                        os.WriteFile,
+		createVGPU:                       devices.CreateVGPU,
+		destroyVGPU:                      devices.DestroyVGPU,
 		meter:                            meter,
 		tracer:                           tracer,
 		guestMemoryPolicy:                policy,
diff --git a/lib/instances/start.go b/lib/instances/start.go
index d9c8c9aa..b830bafb 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -162,11 +162,11 @@ func (m *manager) startInstance(
 	// 4b. Recreate the vGPU if this instance had a GPU profile
 	// Note: GPU availability was already validated in step 2b
 	if stored.GPUProfile != "" {
-		log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile)
-		device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id)
+		log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile)
+		device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id)
 		if err != nil {
 			log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err)
-			return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err)
+			return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err)
 		}
 		setStoredVGPUDevice(stored, device)
 		log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID)
@@ -179,7 +179,7 @@ func (m *manager) startInstance(
 				MdevUUID:   device.MdevUUID,
 				InstanceID: id,
 			}
-			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
+			if err := m.destroyVGPUAssignment(ctx, assignment); err != nil {
 				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err)
 				if saveErr := m.saveMetadata(meta); saveErr != nil {
 					log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr)
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 23ea2d82..ae100a39 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -5,9 +5,47 @@ import (
 	"path/filepath"
 
 	"github.com/kernel/hypeman/lib/devices"
+	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/logger"
 )
 
+func validateVGPUHypervisor(hvType hypervisor.Type) error {
+	if hvType != hypervisor.TypeQEMU {
+		return fmt.Errorf("vGPU is only supported with qemu, got %s", hvType)
+	}
+	return nil
+}
+
+// VGPUCleanupPendingError reports a failed create whose vGPU release also
+// failed during rollback. The instance record identified by InstanceID is
+// retained so the release can be retried; deleting the instance retries it.
+type VGPUCleanupPendingError struct {
+	InstanceID string
+	Err        error
+}
+
+func (e *VGPUCleanupPendingError) Error() string {
+	return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID)
+}
+
+func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err }
+
+func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) {
+	create := m.createVGPU
+	if create == nil {
+		create = devices.CreateVGPU
+	}
+	return create(ctx, profileName, instanceID)
+}
+
+func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error {
+	destroy := m.destroyVGPU
+	if destroy == nil {
+		destroy = devices.DestroyVGPU
+	}
+	return destroy(ctx, assignment)
+}
+
 func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) {
 	stored.GPUFramework = device.Framework
 	stored.GPUDevicePath = device.SysfsPath
@@ -37,7 +75,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata)
 				MdevUUID:   stored.GPUMdevUUID,
 				InstanceID: stored.Id,
 			}
-			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
+			if err := m.destroyVGPUAssignment(ctx, assignment); err != nil {
 				return err
 			}
 		}
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 79e6fe0c..da8426ef 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -2,11 +2,11 @@ package instances
 
 import (
 	"context"
+	"errors"
 	"os"
 	"path/filepath"
 	"sync"
 	"testing"
-	_ "unsafe"
 
 	"github.com/kernel/hypeman/lib/devices"
 	"github.com/kernel/hypeman/lib/hypervisor"
@@ -40,7 +40,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 		DataDir:        m.paths.InstanceDir("failed-create"),
 	}
 
-	m.cleanupFailedCreate(context.Background(), stored.Id, stored)
+	assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored))
 
 	retained, err := m.loadMetadata(stored.Id)
 	require.NoError(t, err)
@@ -56,40 +56,43 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 	assert.Empty(t, retained.DataDir)
 }
 
-//go:linkname hostVendorVFIO github.com/kernel/hypeman/lib/devices.hostVendorVFIO
-var hostVendorVFIO vendorVFIOSysfs
+func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	require.NoError(t, m.ensureDirectories("failed-create"))
 
-type vendorVFIOSysfs struct {
-	pciDevicesPath  string
-	procPath        string
-	vfioDevicesPath string
-	owners          map[string]string
+	assert.False(t, m.cleanupFailedCreate(context.Background(), "failed-create", nil))
+	_, err := m.loadMetadata("failed-create")
+	require.Error(t, err)
 }
 
-func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {
-	root := t.TempDir()
-	pciDevicesPath := filepath.Join(root, "sys", "bus", "pci", "devices")
-	vfAddress := "0000:82:00.4"
-	nvidiaPath := filepath.Join(pciDevicesPath, vfAddress, "nvidia")
-	require.NoError(t, os.MkdirAll(nvidiaPath, 0o755))
-	require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte("0"), 0o644))
-	require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte("ID    : vGPU Name\n1148  : NVIDIA L40S-2Q\n"), 0o644))
-
-	originalVendorVFIO := hostVendorVFIO
-	hostVendorVFIO = vendorVFIOSysfs{
-		pciDevicesPath:  pciDevicesPath,
-		procPath:        filepath.Join(root, "proc"),
-		vfioDevicesPath: filepath.Join(root, "dev", "vfio", "devices"),
-		owners:          make(map[string]string),
-	}
-	t.Cleanup(func() { hostVendorVFIO = originalVendorVFIO })
-	require.NoError(t, os.MkdirAll(hostVendorVFIO.procPath, 0o755))
+func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) {
+	t.Parallel()
+
+	cause := errors.New("boot failed")
+	err := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause}
+	assert.ErrorIs(t, err, cause)
+	assert.Contains(t, err.Error(), "inst-1")
+}
 
+func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) {
+	t.Helper()
 	m := &manager{
 		paths:           paths.New(t.TempDir()),
 		imageManager:    readyFixtureImageManager{name: "test-image"},
 		instanceLocks:   sync.Map{},
 		bootMarkerScans: sync.Map{},
+		createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) {
+			return &devices.VGPUDevice{
+				Framework:   devices.VGPUFrameworkVendorVFIO,
+				VFAddress:   "0000:82:00.4",
+				ProfileType: "1148",
+				ProfileName: profileName,
+				SysfsPath:   "/sys/bus/pci/devices/0000:82:00.4",
+			}, nil
+		},
+		destroyVGPU: destroy,
 	}
 	const id = "start-rollback"
 	require.NoError(t, m.ensureDirectories(id))
@@ -102,25 +105,48 @@ func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {
 		SocketPath:     m.paths.InstanceSocket(id, "noop.sock"),
 		DataDir:        m.paths.InstanceDir(id),
 	}}))
+	return m, id
+}
+
+func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {
+	var destroyed []devices.VGPUAssignment
+	m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error {
+		destroyed = append(destroyed, assignment)
+		return nil
+	})
 
-	t.Setenv("TMPDIR", filepath.Join(root, "missing"))
+	t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing"))
 	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{})
 	require.Error(t, err)
 
+	require.Len(t, destroyed, 1)
+	assert.Equal(t, devices.VGPUAssignment{
+		Framework:  devices.VGPUFrameworkVendorVFIO,
+		DevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+		InstanceID: id,
+	}, destroyed[0])
+
 	stored, err := m.loadMetadata(id)
 	require.NoError(t, err)
 	assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile)
 	assert.Empty(t, stored.GPUFramework)
 	assert.Empty(t, stored.GPUDevicePath)
 	assert.Empty(t, stored.GPUMdevUUID)
-	assertFileContents(t, filepath.Join(nvidiaPath, "current_vgpu_type"), "0")
 }
 
-func assertFileContents(t *testing.T, path, want string) {
-	t.Helper()
-	got, err := os.ReadFile(path)
+func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) {
+	m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error {
+		return errors.New("destroy failed")
+	})
+
+	t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing"))
+	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{})
+	require.Error(t, err)
+
+	stored, err := m.loadMetadata(id)
 	require.NoError(t, err)
-	assert.Equal(t, want, string(got))
+	assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework)
+	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath)
 }
 
 func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) {

From a7ef20fceb19f4ec6e50aca5836595edd6ea1007 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Fri, 7 Aug 2026 20:45:49 +0000
Subject: [PATCH 10/38] Generalize the create vGPU error text

---
 lib/instances/create.go           | 2 +-
 lib/instances/create_mdev_test.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index eea01fe2..b8806b92 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -56,7 +56,7 @@ func wrapCreateVGPUErr(profile string, err error) error {
 	if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) {
 		return fmt.Errorf("%w: %w", ErrInvalidRequest, err)
 	}
-	return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err)
+	return fmt.Errorf("create vGPU for profile %s: %w", profile, err)
 }
 
 // generateVsockCID converts first 8 chars of instance ID to a unique CID
diff --git a/lib/instances/create_mdev_test.go b/lib/instances/create_mdev_test.go
index e6e4f55c..05b3e9d8 100644
--- a/lib/instances/create_mdev_test.go
+++ b/lib/instances/create_mdev_test.go
@@ -63,7 +63,7 @@ func TestWrapCreateVGPUErr(t *testing.T) {
 		{
 			name:        "other vGPU error",
 			err:         errors.New("boom"),
-			wantMessage: "create vGPU mdev for profile profile: boom",
+			wantMessage: "create vGPU for profile profile: boom",
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {

From b511e9868331eb49ca7d96785c9d36528d29c7b7 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sat, 8 Aug 2026 00:58:24 +0000
Subject: [PATCH 11/38] Scope vGPU claim scan to vendor VFIO and close
 reconcile gaps

---
 cmd/api/api/instances.go             | 14 +++++++------
 cmd/api/api/instances_test.go        | 31 ++++++++++++++++++++++++++++
 cmd/api/main.go                      |  8 +++++--
 cmd/api/main_test.go                 | 30 +++++++++++++++++++++++++++
 lib/instances/lifecycle_noop_test.go |  4 ++--
 lib/instances/vgpu.go                | 15 +++++++++++---
 lib/instances/vgpu_test.go           | 21 +++++++++++++++++++
 7 files changed, 110 insertions(+), 13 deletions(-)

diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go
index a55bed0f..6ae589e5 100644
--- a/cmd/api/api/instances.go
+++ b/cmd/api/api/instances.go
@@ -344,6 +344,14 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 	if err != nil {
 		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
+		// Checked first: it wraps the original create error, so a later
+		// errors.Is case would match the cause and hide the retained instance.
+		case errors.As(err, &vgpuPending):
+			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
+			return oapi.CreateInstance500JSONResponse{
+				Code:    "vgpu_cleanup_pending",
+				Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID),
+			}, nil
 		case errors.Is(err, instances.ErrImageNotReady):
 			return oapi.CreateInstance400JSONResponse{
 				Code:    "image_not_ready",
@@ -389,12 +397,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 				Code:    "not_found",
 				Message: err.Error(),
 			}, nil
-		case errors.As(err, &vgpuPending):
-			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
-			return oapi.CreateInstance500JSONResponse{
-				Code:    "vgpu_cleanup_pending",
-				Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID),
-			}, nil
 		default:
 			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
 			return oapi.CreateInstance500JSONResponse{
diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go
index d23e7c21..c0f804c3 100644
--- a/cmd/api/api/instances_test.go
+++ b/cmd/api/api/instances_test.go
@@ -16,6 +16,7 @@ import (
 	"github.com/kernel/hypeman/lib/instances"
 	"github.com/kernel/hypeman/lib/instances/phasetracking"
 	mw "github.com/kernel/hypeman/lib/middleware"
+	"github.com/kernel/hypeman/lib/network"
 	"github.com/kernel/hypeman/lib/oapi"
 	"github.com/kernel/hypeman/lib/paths"
 	restartpolicy "github.com/kernel/hypeman/lib/restart-policy"
@@ -46,6 +47,36 @@ func TestGetInstance_NotFound(t *testing.T) {
 	require.Error(t, err)
 }
 
+type createErrorInstanceManager struct {
+	instances.Manager
+	err error
+}
+
+func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) {
+	return nil, m.err
+}
+
+// A retained-assignment error must win over the mapping of the create error
+// it wraps, or the response omits the instance the caller has to delete.
+func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
+		InstanceID: "inst-1",
+		Err:        network.ErrNameExists,
+	}}
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
+		Body: &oapi.CreateInstanceRequest{Image: "test-image"},
+	})
+	require.NoError(t, err)
+
+	pending, ok := resp.(oapi.CreateInstance500JSONResponse)
+	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+	assert.Contains(t, pending.Message, "inst-1")
+}
+
 func TestCreateInstance_AutoPullImage(t *testing.T) {
 	t.Parallel()
 	if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) {
diff --git a/cmd/api/main.go b/cmd/api/main.go
index ae67fc7f..aaa30ad8 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -192,10 +192,14 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.
 	}
 	protected := make(map[string]struct{})
 	for _, inst := range allInstances {
-		if inst.GPUDevicePath == "" || inst.HypervisorPID == nil {
+		if inst.GPUDevicePath == "" {
 			continue
 		}
-		if !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
+		// A nil PID does not mean the assignment is orphaned: the PID is
+		// persisted only after the hypervisor starts, so a crash during boot
+		// leaves the device path without one. Only skip protection when the
+		// recorded hypervisor is known to be gone.
+		if inst.HypervisorPID != nil && !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
 			continue
 		}
 		protected[inst.GPUDevicePath] = struct{}{}
diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go
index 34dbba42..573a404c 100644
--- a/cmd/api/main_test.go
+++ b/cmd/api/main_test.go
@@ -2,15 +2,18 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os/exec"
 	"testing"
 	"time"
 
 	"github.com/getkin/kin-openapi/openapi3filter"
 	"github.com/go-chi/chi/v5"
 	"github.com/golang-jwt/jwt/v5"
+	"github.com/kernel/hypeman/lib/instances"
 	mw "github.com/kernel/hypeman/lib/middleware"
 	"github.com/kernel/hypeman/lib/oapi"
 	nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware"
@@ -338,3 +341,30 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) {
 		})
 	}
 }
+
+type vgpuReconcileManagerStub struct {
+	instances.Manager
+	list []instances.Instance
+}
+
+func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) {
+	return s.list, nil
+}
+
+// The hypervisor PID is persisted only after boot, so an assignment without
+// one may belong to a VM that is still starting and must stay protected.
+func TestLiveInstanceVGPUDevicePathsProtectsAssignmentsWithoutPID(t *testing.T) {
+	dead := exec.Command("true")
+	require.NoError(t, dead.Run())
+	deadPID := dead.Process.Pid
+
+	manager := vgpuReconcileManagerStub{list: []instances.Instance{
+		{StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4"}},
+		{StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", HypervisorPID: &deadPID}},
+	}}
+
+	protected, err := liveInstanceVGPUDevicePaths(context.Background(), manager)
+	require.NoError(t, err)
+	assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4")
+	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5")
+}
diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go
index f3baa9da..15f632b3 100644
--- a/lib/instances/lifecycle_noop_test.go
+++ b/lib/instances/lifecycle_noop_test.go
@@ -202,7 +202,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) {
 	meta, err := m.loadMetadata(id)
 	require.NoError(t, err)
 	meta.GPUProfile = "NVIDIA L40S-2Q"
-	meta.GPUFramework = devices.VGPUFramework("future-framework")
+	meta.GPUFramework = devices.VGPUFrameworkVendorVFIO
 	meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4"
 	require.NoError(t, m.saveMetadata(meta))
 
@@ -230,7 +230,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) {
 		SocketPath:     socketPath,
 		DataDir:        m.paths.InstanceDir(claimantID),
 		GPUProfile:     "NVIDIA L40S-2Q",
-		GPUFramework:   devices.VGPUFramework("future-framework"),
+		GPUFramework:   devices.VGPUFrameworkVendorVFIO,
 		GPUDevicePath:  "/sys/bus/pci/devices/0000:82:00.4",
 	}}))
 
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index ae100a39..9b97fddb 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -61,9 +61,18 @@ func clearStoredVGPUDevice(stored *StoredMetadata) {
 func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error {
 	path := storedVGPUDevicePath(stored)
 	if path != "" {
-		claimed, err := m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path)
-		if err != nil {
-			return err
+		// Vendor VFIO VFs are reused across instances, so stale metadata can
+		// point at a path claimed by a live instance and the release must fail
+		// closed on an incomplete inventory. mdev UUIDs are unique and never
+		// reused, so skip the scan there — it would let one unreadable
+		// metadata file block every mdev release on the host.
+		claimed := false
+		if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO {
+			var err error
+			claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path)
+			if err != nil {
+				return err
+			}
 		}
 		if claimed {
 			logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance",
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index da8426ef..6f9a2007 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -178,6 +178,27 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.
 	assert.True(t, claimed)
 }
 
+func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{
+		paths:       paths.New(t.TempDir()),
+		destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil },
+	}
+	require.NoError(t, m.ensureDirectories("invalid-instance"))
+	require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644))
+
+	stored := &StoredMetadata{
+		Id:            "mdev-instance",
+		GPUFramework:  devices.VGPUFrameworkMdev,
+		GPUMdevUUID:   "uuid-1",
+		GPUDevicePath: "/sys/bus/mdev/devices/uuid-1",
+	}
+	require.NoError(t, m.releaseStoredVGPU(context.Background(), stored),
+		"an unreadable metadata file must not block mdev releases")
+	assert.Empty(t, stored.GPUDevicePath)
+}
+
 func TestStoredVGPUDevicePath(t *testing.T) {
 	t.Parallel()
 

From 83733aece75639dc3460edb536387b905caf3a0a Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sun, 9 Aug 2026 06:25:10 +0000
Subject: [PATCH 12/38] Harden the vendor VFIO release path

Enable vendor VFIO dispatch in CreateVGPU now that the lifecycle
persists assignments durably and guards releases.

Protect nil-PID claims in the release guard: the hypervisor PID is only
persisted after the claimant boots, so a matching assignment without a
PID must be treated as live, matching the startup reconcile protection.

Scan raw metadata instead of hydrating instances for the claim check.
Hydration derives state through hypervisor queries for every instance
on the host, which every vendor VFIO release would pay; the guard only
needs the stored assignment, PID, and socket. Unreadable metadata still
fails the release closed.

Report pending vGPU cleanup even when retaining the rollback record
fails: the destroy already failed, so the caller must learn about the
outstanding assignment either way.
---
 integration/vgpu_test.go   |  5 ----
 lib/devices/vgpu_linux.go  |  5 +---
 lib/instances/create.go    | 11 +++++---
 lib/instances/vgpu.go      | 35 ++++++++++++++++++++++----
 lib/instances/vgpu_test.go | 51 ++++++++++++++++++++++++++++++++++++++
 5 files changed, 89 insertions(+), 18 deletions(-)

diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go
index 7873aa35..1f1a8277 100644
--- a/integration/vgpu_test.go
+++ b/integration/vgpu_test.go
@@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) {
 	if framework == devices.VGPUFrameworkNone {
 		return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", ""
 	}
-	if framework == devices.VGPUFrameworkVendorVFIO {
-		// CreateVGPU rejects vendor VFIO until the instance lifecycle
-		// integration lands.
-		return "vGPU test requires the vendor VFIO instance lifecycle integration", ""
-	}
 
 	// Check for available profiles
 	profiles, err := devices.ListGPUProfiles()
diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go
index 72fe3b94..a4ccd2db 100644
--- a/lib/devices/vgpu_linux.go
+++ b/lib/devices/vgpu_linux.go
@@ -73,10 +73,7 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic
 			MdevUUID:    mdev.UUID,
 		}, nil
 	case VGPUFrameworkVendorVFIO:
-		// The instance lifecycle does not yet persist vendor VFIO assignments
-		// durably or guard their release against live claims, so keep the
-		// backend out of the create path until that integration lands.
-		return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle")
+		return hostVendorVFIO.create(ctx, profileName, instanceID)
 	default:
 		return nil, fmt.Errorf("vGPU framework not available")
 	}
diff --git a/lib/instances/create.go b/lib/instances/create.go
index b8806b92..3043bfe5 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -605,8 +605,11 @@ func (m *manager) createInstance(
 	return &finalInst, nil
 }
 
-// cleanupFailedCreate reports whether it retained instance metadata for a
-// vGPU assignment whose release failed during rollback.
+// cleanupFailedCreate reports whether a vGPU assignment is still outstanding
+// after a failed create. The vGPU destroy already failed when retainedVGPU is
+// set, so the pending cleanup is reported even when the retention record
+// cannot be persisted — in that case the assignment is orphaned until the
+// next startup reconcile, and the caller must still surface it.
 func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool {
 	if retainedVGPU == nil {
 		m.deleteInstanceData(id)
@@ -616,7 +619,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	log := logger.FromContext(ctx)
 	if err := m.ensureDirectories(id); err != nil {
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
-		return false
+		return true
 	}
 	retained := StoredMetadata{
 		Id:            id,
@@ -626,7 +629,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
-		return false
+		return true
 	}
 	return true
 }
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 9b97fddb..d62fc0d5 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -93,19 +93,44 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata)
 	return nil
 }
 
+// vgpuAssignmentClaimedByLiveInstance reports whether another instance's
+// stored metadata claims devicePath. It reads raw metadata instead of
+// hydrating full instances: the scan runs on every vendor VFIO release, and
+// deriving state would query the hypervisor of every instance on the host.
+// It fails closed: unreadable metadata is an error, and a matching claim
+// without a persisted PID counts as live because the PID is only persisted
+// after the claimant's hypervisor starts.
 func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) {
-	instances, err := m.ListInstancesForReconcile(ctx)
+	files, err := m.listMetadataFilesWithStatErrors(true)
 	if err != nil {
 		return false, fmt.Errorf("list instances for vGPU release check: %w", err)
 	}
-	for i := range instances {
-		inst := &instances[i]
-		if inst.Id == excludeID || storedVGPUDevicePath(&inst.StoredMetadata) != devicePath || inst.HypervisorPID == nil {
+	for _, file := range files {
+		id := filepath.Base(filepath.Dir(file))
+		if id == excludeID {
 			continue
 		}
-		if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
+		meta, err := m.loadMetadata(id)
+		if err != nil {
+			return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err)
+		}
+		stored := &meta.StoredMetadata
+		if storedVGPUDevicePath(stored) != devicePath {
+			continue
+		}
+		if stored.HypervisorPID == nil {
 			return true, nil
 		}
+		if HypervisorProcessExists(*stored.HypervisorPID, stored.SocketPath) {
+			return true, nil
+		}
+		// The stored PID can be stale after a hypeman restart; a live owner
+		// of the claimant's socket still marks the claim as live.
+		if stored.SocketPath != "" && !ProcessExists(*stored.HypervisorPID) {
+			if owner, _, err := hypervisor.ResolveProcessPID(stored.SocketPath); err == nil && ProcessExists(owner) {
+				return true, nil
+			}
+		}
 	}
 	return false, nil
 }
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 6f9a2007..0253d83d 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -67,6 +67,23 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) {
 	require.Error(t, err)
 }
 
+func TestCleanupFailedCreateReportsPendingWhenRetentionFails(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	// A file at the guests directory path makes ensureDirectories fail even
+	// when running as root.
+	require.NoError(t, os.WriteFile(m.paths.GuestsDir(), nil, 0o644))
+
+	stored := &StoredMetadata{
+		Id:            "failed-create",
+		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+	}
+	assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored),
+		"a failed retention must still report the outstanding vGPU assignment")
+}
+
 func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) {
 	t.Parallel()
 
@@ -178,6 +195,40 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.
 	assert.True(t, claimed)
 }
 
+func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	require.NoError(t, m.ensureDirectories("booting-claimant"))
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            "booting-claimant",
+		Name:          "booting-claimant",
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+	}}))
+
+	claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4")
+	require.NoError(t, err)
+	assert.True(t, claimed, "a matching claim without a persisted PID must be treated as live: the PID is only persisted after the claimant boots")
+}
+
+func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	require.NoError(t, m.ensureDirectories("dead-claimant"))
+	deadPID := 1 << 30
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            "dead-claimant",
+		Name:          "dead-claimant",
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+		HypervisorPID: &deadPID,
+	}}))
+
+	claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4")
+	require.NoError(t, err)
+	assert.False(t, claimed, "a claim whose hypervisor is gone must not block the release")
+}
+
 func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) {
 	t.Parallel()
 

From fc216a7b6bbd7d3d67bcfb935ae565c020de6778 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sun, 9 Aug 2026 07:40:01 +0000
Subject: [PATCH 13/38] Fail closed on retained vGPU cleanup

---
 lib/instances/create.go                      | 26 +++++-------
 lib/instances/process_identity_linux_test.go | 43 ++++++++++++++++++++
 lib/instances/vgpu.go                        | 21 ++++------
 lib/instances/vgpu_test.go                   | 23 ++++++-----
 4 files changed, 76 insertions(+), 37 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index 3043bfe5..7af746ff 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -266,19 +266,18 @@ func (m *manager) createInstance(
 	var retainedVGPU *StoredMetadata
 
 	// Setup cleanup stack early so device attachment errors trigger cleanup.
-	// When rollback retains a vGPU assignment, surface the retained instance
-	// ID to the caller so the record is discoverable and can be deleted to
-	// retry the release. The wrapping defer is registered first so it runs
-	// after cu.Clean has decided whether metadata was retained.
-	vgpuRetained := false
+	// When rollback cannot release a vGPU assignment, report whether its
+	// retention record was persisted. The wrapping defer is registered first
+	// so it runs after cu.Clean has attempted to retain the metadata.
+	vgpuPersisted := false
 	defer func() {
-		if retErr != nil && vgpuRetained {
-			retErr = &VGPUCleanupPendingError{InstanceID: id, Err: retErr}
+		if retErr != nil && retainedVGPU != nil {
+			retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuPersisted, Err: retErr}
 		}
 	}()
 	cu := cleanup.Make(func() {
 		log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id)
-		vgpuRetained = m.cleanupFailedCreate(ctx, id, retainedVGPU)
+		vgpuPersisted = m.cleanupFailedCreate(ctx, id, retainedVGPU)
 	})
 	defer cu.Clean()
 
@@ -605,11 +604,8 @@ func (m *manager) createInstance(
 	return &finalInst, nil
 }
 
-// cleanupFailedCreate reports whether a vGPU assignment is still outstanding
-// after a failed create. The vGPU destroy already failed when retainedVGPU is
-// set, so the pending cleanup is reported even when the retention record
-// cannot be persisted — in that case the assignment is orphaned until the
-// next startup reconcile, and the caller must still surface it.
+// cleanupFailedCreate reports whether the retention record for a vGPU
+// assignment whose release failed during rollback was persisted.
 func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool {
 	if retainedVGPU == nil {
 		m.deleteInstanceData(id)
@@ -619,7 +615,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	log := logger.FromContext(ctx)
 	if err := m.ensureDirectories(id); err != nil {
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
-		return true
+		return false
 	}
 	retained := StoredMetadata{
 		Id:            id,
@@ -629,7 +625,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
-		return true
+		return false
 	}
 	return true
 }
diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go
index a7933c25..41a685d2 100644
--- a/lib/instances/process_identity_linux_test.go
+++ b/lib/instances/process_identity_linux_test.go
@@ -16,7 +16,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/kernel/hypeman/lib/devices"
 	"github.com/kernel/hypeman/lib/hypervisor"
+	"github.com/kernel/hypeman/lib/paths"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -479,6 +481,47 @@ func TestRefreshHypervisorPIDResolvesSocketOwnerWhenStoredPIDIsDead(t *testing.T
 	assert.Equal(t, hostBootID(), stored.HypervisorBootID)
 }
 
+func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) {
+	socketPath := filepath.Join(t.TempDir(), "test.sock")
+	owner := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$")
+	owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath)
+	stdin, err := owner.StdinPipe()
+	require.NoError(t, err)
+	stdout, err := owner.StdoutPipe()
+	require.NoError(t, err)
+	require.NoError(t, owner.Start())
+	t.Cleanup(func() {
+		_ = stdin.Close()
+		_ = owner.Process.Kill()
+		_ = owner.Wait()
+	})
+	_, err = bufio.NewReader(stdout).ReadString('\n')
+	require.NoError(t, err)
+
+	stale := exec.Command("sleep", "30")
+	require.NoError(t, stale.Start())
+	t.Cleanup(func() {
+		_ = stale.Process.Kill()
+		_ = stale.Wait()
+	})
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	const devicePath = "/sys/bus/pci/devices/0000:82:00.4"
+	stalePID := stale.Process.Pid
+	require.NoError(t, m.ensureDirectories("live-claimant"))
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            "live-claimant",
+		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath: devicePath,
+		HypervisorPID: &stalePID,
+		SocketPath:    socketPath,
+	}}))
+
+	claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath)
+	require.NoError(t, err)
+	assert.True(t, claimed)
+}
+
 func TestKillHypervisorSurvivesConcurrentReaper(t *testing.T) {
 	socketPath := filepath.Join(t.TempDir(), "test.sock")
 	process := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$")
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index d62fc0d5..e27cf7a5 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -5,7 +5,6 @@ import (
 	"path/filepath"
 
 	"github.com/kernel/hypeman/lib/devices"
-	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/logger"
 )
 
@@ -17,15 +16,19 @@ func validateVGPUHypervisor(hvType hypervisor.Type) error {
 }
 
 // VGPUCleanupPendingError reports a failed create whose vGPU release also
-// failed during rollback. The instance record identified by InstanceID is
-// retained so the release can be retried; deleting the instance retries it.
+// failed during rollback. When Retained is true, deleting the retained instance
+// retries the release; otherwise startup reconciliation recovers the assignment.
 type VGPUCleanupPendingError struct {
 	InstanceID string
+	Retained   bool
 	Err        error
 }
 
 func (e *VGPUCleanupPendingError) Error() string {
-	return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID)
+	if e.Retained {
+		return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID)
+	}
+	return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", e.Err, e.InstanceID)
 }
 
 func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err }
@@ -121,16 +124,10 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu
 		if stored.HypervisorPID == nil {
 			return true, nil
 		}
-		if HypervisorProcessExists(*stored.HypervisorPID, stored.SocketPath) {
+		pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.SocketPath)
+		if err != nil || pid > 0 {
 			return true, nil
 		}
-		// The stored PID can be stale after a hypeman restart; a live owner
-		// of the claimant's socket still marks the claim as live.
-		if stored.SocketPath != "" && !ProcessExists(*stored.HypervisorPID) {
-			if owner, _, err := hypervisor.ResolveProcessPID(stored.SocketPath); err == nil && ProcessExists(owner) {
-				return true, nil
-			}
-		}
 	}
 	return false, nil
 }
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 0253d83d..a0d3ae9d 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -67,30 +67,33 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) {
 	require.Error(t, err)
 }
 
-func TestCleanupFailedCreateReportsPendingWhenRetentionFails(t *testing.T) {
+func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) {
 	t.Parallel()
 
 	m := &manager{paths: paths.New(t.TempDir())}
-	// A file at the guests directory path makes ensureDirectories fail even
-	// when running as root.
-	require.NoError(t, os.WriteFile(m.paths.GuestsDir(), nil, 0o644))
+	const id = "failed-create"
+	require.NoError(t, m.ensureDirectories(id))
+	require.NoError(t, os.Mkdir(m.paths.InstanceMetadata(id), 0o755))
 
 	stored := &StoredMetadata{
-		Id:            "failed-create",
+		Id:            id,
 		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
 		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
 	}
-	assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored),
-		"a failed retention must still report the outstanding vGPU assignment")
+	assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored))
 }
 
 func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) {
 	t.Parallel()
 
 	cause := errors.New("boot failed")
-	err := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause}
-	assert.ErrorIs(t, err, cause)
-	assert.Contains(t, err.Error(), "inst-1")
+	retained := &VGPUCleanupPendingError{InstanceID: "inst-1", Retained: true, Err: cause}
+	assert.ErrorIs(t, retained, cause)
+	assert.Equal(t, "boot failed; vGPU release failed during rollback, instance inst-1 retains the assignment", retained.Error())
+
+	unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause}
+	assert.ErrorIs(t, unpersisted, cause)
+	assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error())
 }
 
 func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) {

From 2566ca38382d61d6f39c88ce41a0129c09384cad Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sun, 9 Aug 2026 22:02:02 +0000
Subject: [PATCH 14/38] Report surviving vGPU retention metadata

---
 lib/instances/create.go    | 14 ++++++++++++--
 lib/instances/vgpu_test.go | 28 ++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index 7af746ff..06c746ea 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -613,9 +613,19 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 
 	log := logger.FromContext(ctx)
+	retentionFailed := func() bool {
+		meta, err := m.loadMetadata(id)
+		if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" {
+			return true
+		}
+		if err := m.deleteInstanceData(id); err != nil {
+			log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err)
+		}
+		return false
+	}
 	if err := m.ensureDirectories(id); err != nil {
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
-		return false
+		return retentionFailed()
 	}
 	retained := StoredMetadata{
 		Id:            id,
@@ -625,7 +635,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
-		return false
+		return retentionFailed()
 	}
 	return true
 }
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index a0d3ae9d..b6831109 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -81,6 +81,34 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) {
 		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
 	}
 	assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored))
+	_, err := m.loadMetadata(id)
+	require.Error(t, err)
+}
+
+func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("root bypasses directory permissions")
+	}
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	const id = "failed-create"
+	require.NoError(t, m.ensureDirectories(id))
+	stored := &StoredMetadata{
+		Id:            id,
+		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+	}
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: *stored}))
+
+	instanceDir := filepath.Dir(m.paths.InstanceMetadata(id))
+	require.NoError(t, os.Chmod(instanceDir, 0o555))
+	t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) })
+
+	assert.True(t, m.cleanupFailedCreate(context.Background(), id, stored))
+	retained, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	assert.Equal(t, stored.GPUFramework, retained.GPUFramework)
+	assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath)
 }
 
 func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) {

From 85e2a4dc7021784755fc47a9567229b2608c4f38 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sun, 9 Aug 2026 22:02:38 +0000
Subject: [PATCH 15/38] Return accurate vGPU cleanup guidance

---
 cmd/api/api/instances.go      |  8 ++++++--
 cmd/api/api/instances_test.go | 23 +++++++++++++++++++++++
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go
index 6ae589e5..19a42978 100644
--- a/cmd/api/api/instances.go
+++ b/cmd/api/api/instances.go
@@ -345,12 +345,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
 		// Checked first: it wraps the original create error, so a later
-		// errors.Is case would match the cause and hide the retained instance.
+		// errors.Is case would match the cause and hide the pending vGPU cleanup.
 		case errors.As(err, &vgpuPending):
 			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
+			message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID)
+			if !vgpuPending.Retained {
+				message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID)
+			}
 			return oapi.CreateInstance500JSONResponse{
 				Code:    "vgpu_cleanup_pending",
-				Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID),
+				Message: message,
 			}, nil
 		case errors.Is(err, instances.ErrImageNotReady):
 			return oapi.CreateInstance400JSONResponse{
diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go
index c0f804c3..edf9f257 100644
--- a/cmd/api/api/instances_test.go
+++ b/cmd/api/api/instances_test.go
@@ -63,6 +63,7 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T)
 	svc := newTestService(t)
 	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
 		InstanceID: "inst-1",
+		Retained:   true,
 		Err:        network.ErrNameExists,
 	}}
 
@@ -75,6 +76,28 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T)
 	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
 	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
 	assert.Contains(t, pending.Message, "inst-1")
+	assert.Contains(t, pending.Message, "delete it to retry")
+}
+
+func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
+		InstanceID: "inst-1",
+		Err:        network.ErrNameExists,
+	}}
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
+		Body: &oapi.CreateInstanceRequest{Image: "test-image"},
+	})
+	require.NoError(t, err)
+
+	pending, ok := resp.(oapi.CreateInstance500JSONResponse)
+	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+	assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved")
+	assert.Contains(t, pending.Message, "startup reconcile")
+	assert.NotContains(t, pending.Message, "delete")
 }
 
 func TestCreateInstance_AutoPullImage(t *testing.T) {

From 5c183efd5819ddac4370454bd5c41ef68fdb5873 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sun, 9 Aug 2026 22:26:00 +0000
Subject: [PATCH 16/38] Clarify vGPU retention fallback

---
 lib/instances/create.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index 06c746ea..eb973038 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -613,7 +613,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 
 	log := logger.FromContext(ctx)
-	retentionFailed := func() bool {
+	retentionSurvives := func() bool {
 		meta, err := m.loadMetadata(id)
 		if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" {
 			return true
@@ -625,7 +625,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.ensureDirectories(id); err != nil {
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
-		return retentionFailed()
+		return retentionSurvives()
 	}
 	retained := StoredMetadata{
 		Id:            id,
@@ -635,7 +635,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
-		return retentionFailed()
+		return retentionSurvives()
 	}
 	return true
 }

From 9dbf18eab53d5be193aba42c3246395003d007c2 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 13:43:17 +0000
Subject: [PATCH 17/38] Pass hypervisor identity token to vGPU claim check

---
 lib/instances/vgpu.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index e27cf7a5..acc5eb74 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -124,7 +124,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu
 		if stored.HypervisorPID == nil {
 			return true, nil
 		}
-		pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.SocketPath)
+		pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath)
 		if err != nil || pid > 0 {
 			return true, nil
 		}

From e5e95b4ea439ce45c52c3d0c567619982238b450 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 14:12:07 +0000
Subject: [PATCH 18/38] Fail safely on ambiguous vGPU claims

---
 lib/instances/vgpu.go      | 15 +++++++++------
 lib/instances/vgpu_test.go | 38 ++++++++++++++++++++++++++++++++++----
 2 files changed, 43 insertions(+), 10 deletions(-)

diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index acc5eb74..9bca44db 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -96,13 +96,13 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata)
 	return nil
 }
 
-// vgpuAssignmentClaimedByLiveInstance reports whether another instance's
+// vgpuAssignmentClaimedByLiveInstance reports whether another live instance's
 // stored metadata claims devicePath. It reads raw metadata instead of
 // hydrating full instances: the scan runs on every vendor VFIO release, and
 // deriving state would query the hypervisor of every instance on the host.
-// It fails closed: unreadable metadata is an error, and a matching claim
-// without a persisted PID counts as live because the PID is only persisted
-// after the claimant's hypervisor starts.
+// A confirmed live claimant returns true. Unreadable metadata, a missing PID,
+// or unverifiable process ownership returns an error so the requester retains
+// its assignment for a later retry.
 func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) {
 	files, err := m.listMetadataFilesWithStatErrors(true)
 	if err != nil {
@@ -122,10 +122,13 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu
 			continue
 		}
 		if stored.HypervisorPID == nil {
-			return true, nil
+			return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath)
 		}
 		pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath)
-		if err != nil || pid > 0 {
+		if err != nil {
+			return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err)
+		}
+		if pid > 0 {
 			return true, nil
 		}
 	}
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index b6831109..d4e0c648 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -226,7 +226,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.
 	assert.True(t, claimed)
 }
 
-func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) {
+func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) {
 	t.Parallel()
 
 	m := &manager{paths: paths.New(t.TempDir())}
@@ -237,9 +237,9 @@ func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) {
 		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
 	}}))
 
-	claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4")
-	require.NoError(t, err)
-	assert.True(t, claimed, "a matching claim without a persisted PID must be treated as live: the PID is only persisted after the claimant boots")
+	_, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "booting-claimant")
 }
 
 func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) {
@@ -281,6 +281,36 @@ func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) {
 	assert.Empty(t, stored.GPUDevicePath)
 }
 
+func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) {
+	t.Parallel()
+
+	const devicePath = "/sys/bus/pci/devices/0000:82:00.4"
+	m := &manager{
+		paths: paths.New(t.TempDir()),
+		destroyVGPU: func(context.Context, devices.VGPUAssignment) error {
+			t.Fatal("destroyVGPU must not be called for an ambiguous claim")
+			return nil
+		},
+	}
+	require.NoError(t, m.ensureDirectories("ambiguous-claimant"))
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            "ambiguous-claimant",
+		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath: devicePath,
+	}}))
+
+	stored := &StoredMetadata{
+		Id:            "requester",
+		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath: devicePath,
+	}
+	err := m.releaseStoredVGPU(context.Background(), stored)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "ambiguous-claimant")
+	assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework)
+	assert.Equal(t, devicePath, stored.GPUDevicePath)
+}
+
 func TestStoredVGPUDevicePath(t *testing.T) {
 	t.Parallel()
 

From d49e80aad6abb886e88000a7b0e6eb83f2cb88f6 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 14:14:21 +0000
Subject: [PATCH 19/38] Expose retained vGPU instance IDs

---
 cmd/api/api/instances.go      |  6 ++++++
 cmd/api/api/instances_test.go | 10 ++++++++++
 2 files changed, 16 insertions(+)

diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go
index 19a42978..7673d13c 100644
--- a/cmd/api/api/instances.go
+++ b/cmd/api/api/instances.go
@@ -349,12 +349,18 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		case errors.As(err, &vgpuPending):
 			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
 			message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID)
+			innerCode := "vgpu_retained_instance"
 			if !vgpuPending.Retained {
 				message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID)
+				innerCode = "vgpu_unretained_instance"
 			}
 			return oapi.CreateInstance500JSONResponse{
 				Code:    "vgpu_cleanup_pending",
 				Message: message,
+				InnerError: &oapi.ErrorDetail{
+					Code:    lo.ToPtr(innerCode),
+					Message: lo.ToPtr(vgpuPending.InstanceID),
+				},
 			}, nil
 		case errors.Is(err, instances.ErrImageNotReady):
 			return oapi.CreateInstance400JSONResponse{
diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go
index edf9f257..d360c4cb 100644
--- a/cmd/api/api/instances_test.go
+++ b/cmd/api/api/instances_test.go
@@ -77,6 +77,11 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T)
 	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
 	assert.Contains(t, pending.Message, "inst-1")
 	assert.Contains(t, pending.Message, "delete it to retry")
+	require.NotNil(t, pending.InnerError)
+	require.NotNil(t, pending.InnerError.Code)
+	assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code)
+	require.NotNil(t, pending.InnerError.Message)
+	assert.Equal(t, "inst-1", *pending.InnerError.Message)
 }
 
 func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) {
@@ -98,6 +103,11 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(
 	assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved")
 	assert.Contains(t, pending.Message, "startup reconcile")
 	assert.NotContains(t, pending.Message, "delete")
+	require.NotNil(t, pending.InnerError)
+	require.NotNil(t, pending.InnerError.Code)
+	assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code)
+	require.NotNil(t, pending.InnerError.Message)
+	assert.Equal(t, "inst-1", *pending.InnerError.Message)
 }
 
 func TestCreateInstance_AutoPullImage(t *testing.T) {

From 412d3255f3dabd210f9d72cb1d90cf6eaaf9b04f Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 15:31:22 +0000
Subject: [PATCH 20/38] Harden vGPU startup rollback recovery

---
 cmd/api/main.go            | 62 +++++++++++++++++++++++++++----------
 cmd/api/main_test.go       | 18 +++++++----
 lib/instances/create.go    |  4 +++
 lib/instances/start.go     | 25 +++------------
 lib/instances/types.go     |  3 +-
 lib/instances/vgpu.go      | 27 +++++++++++++++-
 lib/instances/vgpu_test.go | 63 ++++++++++++++++++++++++++++++++++++--
 7 files changed, 155 insertions(+), 47 deletions(-)

diff --git a/cmd/api/main.go b/cmd/api/main.go
index aaa30ad8..f64615be 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -185,26 +185,62 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta
 	}, logger), nil
 }
 
-func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, error) {
+const vgpuAssignmentStartupGracePeriod = 5 * time.Minute
+
+func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) {
 	allInstances, err := instanceManager.ListInstancesForReconcile(ctx)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	protected := make(map[string]struct{})
+	var retryAfter time.Duration
 	for _, inst := range allInstances {
 		if inst.GPUDevicePath == "" {
 			continue
 		}
-		// A nil PID does not mean the assignment is orphaned: the PID is
-		// persisted only after the hypervisor starts, so a crash during boot
-		// leaves the device path without one. Only skip protection when the
-		// recorded hypervisor is known to be gone.
-		if inst.HypervisorPID != nil && !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
+		if inst.HypervisorPID != nil {
+			if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.SocketPath) {
+				continue
+			}
+			protected[inst.GPUDevicePath] = struct{}{}
+			continue
+		}
+		if inst.GPUAssignedAt == nil {
+			continue
+		}
+		remaining := vgpuAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt)
+		if remaining <= 0 {
 			continue
 		}
 		protected[inst.GPUDevicePath] = struct{}{}
+		if retryAfter == 0 || remaining < retryAfter {
+			retryAfter = remaining
+		}
 	}
-	return protected, nil
+	return protected, retryAfter, nil
+}
+
+func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) {
+	protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager)
+	if err != nil {
+		logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err)
+		return
+	}
+	if err := devices.ReconcileVGPUs(ctx, protected); err != nil {
+		logger.Warn("failed to reconcile vGPU devices", "error", err)
+	}
+	if retryAfter <= 0 {
+		return
+	}
+	go func() {
+		timer := time.NewTimer(retryAfter)
+		defer timer.Stop()
+		select {
+		case <-ctx.Done():
+		case <-timer.C:
+			reconcileVGPUs(ctx, instanceManager, logger)
+		}
+	}()
 }
 
 func run() error {
@@ -413,15 +449,7 @@ func run() error {
 
 	// Reconcile vGPU devices (clears orphaned vGPUs from previous runs)
 	logger.Info("Reconciling vGPU devices...")
-	protected, err := liveInstanceVGPUDevicePaths(app.Ctx, app.InstanceManager)
-	if err != nil {
-		logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err)
-		protected = nil
-	}
-	if err := devices.ReconcileVGPUs(app.Ctx, protected); err != nil {
-		// Log but don't fail - vGPU cleanup is best-effort
-		logger.Warn("failed to reconcile vGPU devices", "error", err)
-	}
+	reconcileVGPUs(ctx, app.InstanceManager, logger)
 
 	// Wire up resource validator for aggregate limit checking
 	// This enables the instance manager to validate CPU, memory, network, and GPU
diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go
index 573a404c..02909e6a 100644
--- a/cmd/api/main_test.go
+++ b/cmd/api/main_test.go
@@ -351,20 +351,26 @@ func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]
 	return s.list, nil
 }
 
-// The hypervisor PID is persisted only after boot, so an assignment without
-// one may belong to a VM that is still starting and must stay protected.
-func TestLiveInstanceVGPUDevicePathsProtectsAssignmentsWithoutPID(t *testing.T) {
+func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) {
 	dead := exec.Command("true")
 	require.NoError(t, dead.Run())
 	deadPID := dead.Process.Pid
+	recent := time.Now().Add(-time.Minute)
+	stale := time.Now().Add(-vgpuAssignmentStartupGracePeriod - time.Minute)
 
 	manager := vgpuReconcileManagerStub{list: []instances.Instance{
-		{StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4"}},
-		{StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", HypervisorPID: &deadPID}},
+		{StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}},
+		{StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}},
+		{StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}},
+		{StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}},
 	}}
 
-	protected, err := liveInstanceVGPUDevicePaths(context.Background(), manager)
+	protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager)
 	require.NoError(t, err)
+	require.Positive(t, retryAfter)
+	require.LessOrEqual(t, retryAfter, vgpuAssignmentStartupGracePeriod)
 	assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4")
 	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5")
+	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6")
+	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7")
 }
diff --git a/lib/instances/create.go b/lib/instances/create.go
index eb973038..3e352eb7 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -262,6 +262,7 @@ func (m *manager) createInstance(
 	var gpuFramework devices.VGPUFramework
 	var gpuDevicePath string
 	var gpuMdevUUID string
+	var gpuAssignedAt *time.Time
 	var stored *StoredMetadata
 	var retainedVGPU *StoredMetadata
 
@@ -303,6 +304,8 @@ func (m *manager) createInstance(
 		gpuFramework = gpuDevice.Framework
 		gpuDevicePath = gpuDevice.SysfsPath
 		gpuMdevUUID = gpuDevice.MdevUUID
+		assignedAt := m.nowUTC()
+		gpuAssignedAt = &assignedAt
 		log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID)
 
 		// Add vGPU cleanup to stack
@@ -405,6 +408,7 @@ func (m *manager) createInstance(
 		GPUFramework:             gpuFramework,
 		GPUDevicePath:            gpuDevicePath,
 		GPUMdevUUID:              gpuMdevUUID,
+		GPUAssignedAt:            gpuAssignedAt,
 		Entrypoint:               req.Entrypoint,
 		Cmd:                      req.Cmd,
 		SkipKernelHeaders:        req.SkipKernelHeaders,
diff --git a/lib/instances/start.go b/lib/instances/start.go
index b830bafb..77503855 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/kernel/hypeman/lib/devices"
 	"github.com/kernel/hypeman/lib/egressproxy"
 	"github.com/kernel/hypeman/lib/instances/phasetracking"
 	"github.com/kernel/hypeman/lib/logger"
@@ -64,6 +63,8 @@ func (m *manager) startInstance(
 		}
 	}
 
+	rollbackMeta := *meta
+
 	// 2a. Clear stale exit info from previous run and apply command overrides
 	stored.ExitCode = nil
 	stored.ExitMessage = ""
@@ -168,28 +169,12 @@ func (m *manager) startInstance(
 			log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err)
 			return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err)
 		}
-		setStoredVGPUDevice(stored, device)
+		assignedAt := m.nowUTC()
+		setStoredVGPUDevice(stored, device, assignedAt)
 		log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID)
 		// Add vGPU cleanup to stack
 		cu.Add(func() {
-			log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID)
-			assignment := devices.VGPUAssignment{
-				Framework:  device.Framework,
-				DevicePath: device.SysfsPath,
-				MdevUUID:   device.MdevUUID,
-				InstanceID: id,
-			}
-			if err := m.destroyVGPUAssignment(ctx, assignment); err != nil {
-				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err)
-				if saveErr := m.saveMetadata(meta); saveErr != nil {
-					log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr)
-				}
-			} else {
-				clearStoredVGPUDevice(stored)
-				if saveErr := m.saveMetadata(meta); saveErr != nil {
-					log.ErrorContext(ctx, "failed to save metadata after vGPU cleanup", "instance_id", id, "error", saveErr)
-				}
-			}
+			m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta)
 		})
 		if err := m.saveMetadata(meta); err != nil {
 			log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err)
diff --git a/lib/instances/types.go b/lib/instances/types.go
index ae810414..f9f33c3a 100644
--- a/lib/instances/types.go
+++ b/lib/instances/types.go
@@ -154,7 +154,8 @@ type StoredMetadata struct {
 	GPUProfile    string // vGPU profile name (e.g., "L40S-1Q")
 	GPUFramework  devices.VGPUFramework
 	GPUDevicePath string
-	GPUMdevUUID   string // populated for mdev-backed vGPUs
+	GPUMdevUUID   string     // populated for mdev-backed vGPUs
+	GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection
 
 	// Command overrides (like docker run <image> <command>)
 	Entrypoint []string // Override image entrypoint (nil = use image default)
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 9bca44db..3e7dc350 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -3,6 +3,7 @@ package instances
 import (
 	"context"
 	"path/filepath"
+	"time"
 
 	"github.com/kernel/hypeman/lib/devices"
 	"github.com/kernel/hypeman/lib/logger"
@@ -49,16 +50,40 @@ func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.
 	return destroy(ctx, assignment)
 }
 
-func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) {
+func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) {
 	stored.GPUFramework = device.Framework
 	stored.GPUDevicePath = device.SysfsPath
 	stored.GPUMdevUUID = device.MdevUUID
+	stored.GPUAssignedAt = &assignedAt
 }
 
 func clearStoredVGPUDevice(stored *StoredMetadata) {
 	stored.GPUFramework = devices.VGPUFrameworkNone
 	stored.GPUDevicePath = ""
 	stored.GPUMdevUUID = ""
+	stored.GPUAssignedAt = nil
+}
+
+func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) {
+	assignment := devices.VGPUAssignment{
+		Framework:  device.Framework,
+		DevicePath: device.SysfsPath,
+		MdevUUID:   device.MdevUUID,
+		InstanceID: instanceID,
+	}
+	cleanupMeta := rollbackMeta
+	releaseErr := m.destroyVGPUAssignment(ctx, assignment)
+	if releaseErr != nil {
+		logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr)
+		setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt)
+	}
+	if err := m.saveMetadata(&cleanupMeta); err != nil {
+		message := "failed to save metadata after vGPU cleanup"
+		if releaseErr != nil {
+			message = "failed to retain vGPU assignment metadata after cleanup failure"
+		}
+		logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err)
+	}
 }
 
 func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error {
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index d4e0c648..366e343f 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/kernel/hypeman/lib/devices"
 	"github.com/kernel/hypeman/lib/hypervisor"
@@ -188,13 +189,68 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) {
 	})
 
 	t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing"))
-	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{})
+	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}})
 	require.Error(t, err)
 
 	stored, err := m.loadMetadata(id)
 	require.NoError(t, err)
 	assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework)
 	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath)
+	assert.NotNil(t, stored.GPUAssignedAt)
+	assert.Empty(t, stored.Entrypoint)
+}
+
+func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) {
+	m := &manager{
+		paths: paths.New(t.TempDir()),
+		destroyVGPU: func(context.Context, devices.VGPUAssignment) error {
+			return nil
+		},
+	}
+	const id = "failed-start"
+	require.NoError(t, m.ensureDirectories(id))
+
+	previousStart := time.Now().Add(-time.Hour).UTC()
+	previousProgramStart := previousStart.Add(time.Second)
+	exitCode := 1
+	rollbackMeta := metadata{StoredMetadata: StoredMetadata{
+		Id:               id,
+		GPUProfile:       "NVIDIA L40S-2Q",
+		Entrypoint:       []string{"old-entrypoint"},
+		Cmd:              []string{"old-command"},
+		StartedAt:        &previousStart,
+		ProgramStartedAt: &previousProgramStart,
+		ExitCode:         &exitCode,
+		ExitMessage:      "previous exit",
+	}}
+
+	partial := rollbackMeta
+	partial.Entrypoint = []string{"new-entrypoint"}
+	partial.Cmd = []string{"new-command"}
+	partial.StartedAt = ptr(time.Now().UTC())
+	partial.ProgramStartedAt = nil
+	partial.ExitCode = nil
+	partial.ExitMessage = ""
+	assignedAt := time.Now().UTC()
+	device := &devices.VGPUDevice{
+		Framework: devices.VGPUFrameworkVendorVFIO,
+		SysfsPath: "/sys/bus/pci/devices/0000:82:00.4",
+	}
+	setStoredVGPUDevice(&partial.StoredMetadata, device, assignedAt)
+	require.NoError(t, m.saveMetadata(&partial))
+
+	m.cleanupStartVGPU(context.Background(), id, device, assignedAt, rollbackMeta)
+
+	stored, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint)
+	assert.Equal(t, rollbackMeta.Cmd, stored.Cmd)
+	assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt)
+	assert.Equal(t, rollbackMeta.ProgramStartedAt, stored.ProgramStartedAt)
+	assert.Equal(t, rollbackMeta.ExitCode, stored.ExitCode)
+	assert.Equal(t, rollbackMeta.ExitMessage, stored.ExitMessage)
+	assert.Empty(t, stored.GPUDevicePath)
+	assert.Nil(t, stored.GPUAssignedAt)
 }
 
 func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) {
@@ -341,16 +397,19 @@ func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) {
 func TestSetAndClearStoredVGPUDevice(t *testing.T) {
 	t.Parallel()
 
+	assignedAt := time.Now().UTC()
 	stored := &StoredMetadata{}
 	setStoredVGPUDevice(stored, &devices.VGPUDevice{
 		Framework: devices.VGPUFrameworkVendorVFIO,
 		SysfsPath: "/sys/bus/pci/devices/0000:82:00.4",
-	})
+	}, assignedAt)
 	assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework)
 	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath)
+	assert.Equal(t, assignedAt, *stored.GPUAssignedAt)
 
 	clearStoredVGPUDevice(stored)
 	assert.Empty(t, stored.GPUFramework)
 	assert.Empty(t, stored.GPUDevicePath)
 	assert.Empty(t, stored.GPUMdevUUID)
+	assert.Nil(t, stored.GPUAssignedAt)
 }

From 1d7d3f341f67ec6ac14f7f39ed81a5b1d0c1ce85 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 16:12:47 +0000
Subject: [PATCH 21/38] Preserve the create failure cause in vGPU cleanup
 errors

The vgpu_cleanup_pending response replaced the original create error
with cleanup guidance, leaving the cause only in server logs. Prefix
the message with the wrapped error so callers see why creation failed
as well as how to recover.
---
 cmd/api/api/instances.go      | 4 ++--
 cmd/api/api/instances_test.go | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go
index 7673d13c..0efa1c76 100644
--- a/cmd/api/api/instances.go
+++ b/cmd/api/api/instances.go
@@ -348,10 +348,10 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		// errors.Is case would match the cause and hide the pending vGPU cleanup.
 		case errors.As(err, &vgpuPending):
 			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
-			message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID)
+			message := fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.Err, vgpuPending.InstanceID)
 			innerCode := "vgpu_retained_instance"
 			if !vgpuPending.Retained {
-				message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID)
+				message = fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID)
 				innerCode = "vgpu_unretained_instance"
 			}
 			return oapi.CreateInstance500JSONResponse{
diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go
index d360c4cb..8e0512bd 100644
--- a/cmd/api/api/instances_test.go
+++ b/cmd/api/api/instances_test.go
@@ -76,6 +76,8 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T)
 	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
 	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
 	assert.Contains(t, pending.Message, "inst-1")
+	assert.Contains(t, pending.Message, network.ErrNameExists.Error(),
+		"the underlying create failure must survive the cleanup guidance")
 	assert.Contains(t, pending.Message, "delete it to retry")
 	require.NotNil(t, pending.InnerError)
 	require.NotNil(t, pending.InnerError.Code)
@@ -101,6 +103,8 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(
 	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
 	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
 	assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved")
+	assert.Contains(t, pending.Message, network.ErrNameExists.Error(),
+		"the underlying create failure must survive the cleanup guidance")
 	assert.Contains(t, pending.Message, "startup reconcile")
 	assert.NotContains(t, pending.Message, "delete")
 	require.NotNil(t, pending.InnerError)

From 3d0dc135b81e0d8e00171b57654b5067c2a6f95f Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 17:33:44 +0000
Subject: [PATCH 22/38] Fix vGPU reconciliation edge cases

---
 cmd/api/main.go            |  9 ++++-----
 cmd/api/main_test.go       |  4 ++--
 lib/instances/create.go    |  2 ++
 lib/instances/vgpu.go      | 16 ++++++++++++----
 lib/instances/vgpu_test.go | 27 ++++++++++++++++++++++++++-
 5 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/cmd/api/main.go b/cmd/api/main.go
index f64615be..67214b13 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -185,8 +185,6 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta
 	}, logger), nil
 }
 
-const vgpuAssignmentStartupGracePeriod = 5 * time.Minute
-
 func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) {
 	allInstances, err := instanceManager.ListInstancesForReconcile(ctx)
 	if err != nil {
@@ -208,7 +206,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.
 		if inst.GPUAssignedAt == nil {
 			continue
 		}
-		remaining := vgpuAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt)
+		remaining := instances.VGPUAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt)
 		if remaining <= 0 {
 			continue
 		}
@@ -223,8 +221,9 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.
 func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) {
 	protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager)
 	if err != nil {
-		logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err)
-		return
+		logger.Warn("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err)
+		protected = nil
+		retryAfter = 0
 	}
 	if err := devices.ReconcileVGPUs(ctx, protected); err != nil {
 		logger.Warn("failed to reconcile vGPU devices", "error", err)
diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go
index 02909e6a..09a4a241 100644
--- a/cmd/api/main_test.go
+++ b/cmd/api/main_test.go
@@ -356,7 +356,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) {
 	require.NoError(t, dead.Run())
 	deadPID := dead.Process.Pid
 	recent := time.Now().Add(-time.Minute)
-	stale := time.Now().Add(-vgpuAssignmentStartupGracePeriod - time.Minute)
+	stale := time.Now().Add(-instances.VGPUAssignmentStartupGracePeriod - time.Minute)
 
 	manager := vgpuReconcileManagerStub{list: []instances.Instance{
 		{StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}},
@@ -368,7 +368,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) {
 	protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager)
 	require.NoError(t, err)
 	require.Positive(t, retryAfter)
-	require.LessOrEqual(t, retryAfter, vgpuAssignmentStartupGracePeriod)
+	require.LessOrEqual(t, retryAfter, instances.VGPUAssignmentStartupGracePeriod)
 	assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4")
 	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5")
 	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6")
diff --git a/lib/instances/create.go b/lib/instances/create.go
index 3e352eb7..2cff7931 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -336,6 +336,7 @@ func (m *manager) createInstance(
 						GPUFramework:      gpuDevice.Framework,
 						GPUDevicePath:     gpuDevice.SysfsPath,
 						GPUMdevUUID:       gpuDevice.MdevUUID,
+						GPUAssignedAt:     gpuAssignedAt,
 					}
 				}
 			}
@@ -636,6 +637,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 		GPUFramework:  retainedVGPU.GPUFramework,
 		GPUDevicePath: retainedVGPU.GPUDevicePath,
 		GPUMdevUUID:   retainedVGPU.GPUMdevUUID,
+		GPUAssignedAt: retainedVGPU.GPUAssignedAt,
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 3e7dc350..0d12e532 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/kernel/hypeman/lib/devices"
+	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/logger"
 )
 
@@ -16,6 +17,10 @@ func validateVGPUHypervisor(hvType hypervisor.Type) error {
 	return nil
 }
 
+// VGPUAssignmentStartupGracePeriod bounds how long an assignment without a
+// persisted hypervisor PID is treated as potentially live.
+const VGPUAssignmentStartupGracePeriod = 5 * time.Minute
+
 // VGPUCleanupPendingError reports a failed create whose vGPU release also
 // failed during rollback. When Retained is true, deleting the retained instance
 // retries the release; otherwise startup reconciliation recovers the assignment.
@@ -125,9 +130,9 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata)
 // stored metadata claims devicePath. It reads raw metadata instead of
 // hydrating full instances: the scan runs on every vendor VFIO release, and
 // deriving state would query the hypervisor of every instance on the host.
-// A confirmed live claimant returns true. Unreadable metadata, a missing PID,
-// or unverifiable process ownership returns an error so the requester retains
-// its assignment for a later retry.
+// A confirmed live claimant returns true. Unreadable metadata, a recent
+// assignment without a PID, or unverifiable process ownership returns an error
+// so the requester retains its assignment for a later retry.
 func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) {
 	files, err := m.listMetadataFilesWithStatErrors(true)
 	if err != nil {
@@ -147,7 +152,10 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu
 			continue
 		}
 		if stored.HypervisorPID == nil {
-			return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath)
+			if stored.GPUAssignedAt == nil || time.Since(*stored.GPUAssignedAt) >= VGPUAssignmentStartupGracePeriod {
+				continue
+			}
+			return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath)
 		}
 		pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath)
 		if err != nil {
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 366e343f..7eed1744 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -27,6 +27,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 	t.Parallel()
 
 	m := &manager{paths: paths.New(t.TempDir())}
+	assignedAt := time.Now().UTC()
 	stored := &StoredMetadata{
 		Id:             "failed-create",
 		Name:           "failed-create",
@@ -34,6 +35,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 		GPUFramework:   devices.VGPUFrameworkVendorVFIO,
 		GPUDevicePath:  "/sys/bus/pci/devices/0000:82:00.4",
 		GPUMdevUUID:    "mdev-uuid",
+		GPUAssignedAt:  &assignedAt,
 		NetworkEnabled: true,
 		IP:             "192.0.2.1",
 		Volumes:        []VolumeAttachment{{VolumeID: "volume"}},
@@ -49,6 +51,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 	assert.Equal(t, stored.GPUFramework, retained.GPUFramework)
 	assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath)
 	assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID)
+	assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt)
 	assert.Empty(t, retained.Name)
 	assert.Empty(t, retained.GPUProfile)
 	assert.False(t, retained.NetworkEnabled)
@@ -282,15 +285,17 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.
 	assert.True(t, claimed)
 }
 
-func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) {
+func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnRecentNilPIDClaim(t *testing.T) {
 	t.Parallel()
 
 	m := &manager{paths: paths.New(t.TempDir())}
 	require.NoError(t, m.ensureDirectories("booting-claimant"))
+	assignedAt := time.Now().UTC()
 	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
 		Id:            "booting-claimant",
 		Name:          "booting-claimant",
 		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+		GPUAssignedAt: &assignedAt,
 	}}))
 
 	_, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4")
@@ -298,6 +303,24 @@ func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) {
 	assert.Contains(t, err.Error(), "booting-claimant")
 }
 
+func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	require.NoError(t, m.ensureDirectories("stale-claimant"))
+	assignedAt := time.Now().Add(-VGPUAssignmentStartupGracePeriod - time.Minute)
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            "stale-claimant",
+		Name:          "stale-claimant",
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+		GPUAssignedAt: &assignedAt,
+	}}))
+
+	claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4")
+	require.NoError(t, err)
+	assert.False(t, claimed)
+}
+
 func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) {
 	t.Parallel()
 
@@ -349,10 +372,12 @@ func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) {
 		},
 	}
 	require.NoError(t, m.ensureDirectories("ambiguous-claimant"))
+	assignedAt := time.Now().UTC()
 	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
 		Id:            "ambiguous-claimant",
 		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
 		GPUDevicePath: devicePath,
+		GPUAssignedAt: &assignedAt,
 	}}))
 
 	stored := &StoredMetadata{

From 3518c107826440098627694ff7d9c3128f002f3f Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 18:26:01 +0000
Subject: [PATCH 23/38] Use boot-scoped hypervisor identities for vGPUs

---
 cmd/api/main.go       | 2 +-
 lib/instances/vgpu.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/api/main.go b/cmd/api/main.go
index 67214b13..e68fb9bd 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -197,7 +197,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.
 			continue
 		}
 		if inst.HypervisorPID != nil {
-			if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.SocketPath) {
+			if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) {
 				continue
 			}
 			protected[inst.GPUDevicePath] = struct{}{}
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 0d12e532..772d93e1 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -157,7 +157,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu
 			}
 			return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath)
 		}
-		pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath)
+		pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.HypervisorBootID, stored.SocketPath)
 		if err != nil {
 			return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err)
 		}

From a473558e83c9a69d887cb351db91639565eeaafc Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 18:34:19 +0000
Subject: [PATCH 24/38] Run vGPU rollback tests with QEMU

---
 lib/instances/vgpu_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 7eed1744..56c6f857 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -153,7 +153,7 @@ func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, dev
 		Name:           id,
 		Image:          "test-image",
 		GPUProfile:     "NVIDIA L40S-2Q",
-		HypervisorType: lifecycleNoopHypervisorType,
+		HypervisorType: hypervisor.TypeQEMU,
 		SocketPath:     m.paths.InstanceSocket(id, "noop.sock"),
 		DataDir:        m.paths.InstanceDir(id),
 	}}))

From 964805ea3e198c6f7ba9ed2f6dbb3b980c7d6cc0 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 19:07:05 +0000
Subject: [PATCH 25/38] Protect new vGPU assignments from stale PIDs

---
 cmd/api/main.go            |  5 +----
 cmd/api/main_test.go       |  4 +++-
 lib/instances/create.go    |  6 ++++++
 lib/instances/start.go     |  9 +++++++++
 lib/instances/vgpu.go      |  1 +
 lib/instances/vgpu_test.go | 12 +++++++++++-
 6 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/cmd/api/main.go b/cmd/api/main.go
index e68fb9bd..e4e79451 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -196,10 +196,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.
 		if inst.GPUDevicePath == "" {
 			continue
 		}
-		if inst.HypervisorPID != nil {
-			if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) {
-				continue
-			}
+		if inst.HypervisorPID != nil && instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) {
 			protected[inst.GPUDevicePath] = struct{}{}
 			continue
 		}
diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go
index 09a4a241..217e9db8 100644
--- a/cmd/api/main_test.go
+++ b/cmd/api/main_test.go
@@ -351,7 +351,7 @@ func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]
 	return s.list, nil
 }
 
-func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) {
+func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) {
 	dead := exec.Command("true")
 	require.NoError(t, dead.Run())
 	deadPID := dead.Process.Pid
@@ -363,6 +363,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) {
 		{StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}},
 		{StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}},
 		{StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}},
+		{StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorPID: &deadPID, GPUAssignedAt: &recent}},
 	}}
 
 	protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager)
@@ -373,4 +374,5 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) {
 	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5")
 	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6")
 	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7")
+	assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8")
 }
diff --git a/lib/instances/create.go b/lib/instances/create.go
index 2cff7931..fe235f4f 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -103,6 +103,11 @@ func (m *manager) createInstance(
 	if hvType == "" {
 		hvType = m.defaultHypervisor
 	}
+	if req.GPU != nil && req.GPU.Profile != "" {
+		if err := validateVGPUHypervisor(hvType); err != nil {
+			return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
+		}
+	}
 	starter, starterErr := m.getVMStarter(hvType)
 	if starterErr == nil {
 		if err := m.validateCreateVMConfig(starter, req, hvType); err != nil {
@@ -110,6 +115,7 @@ func (m *manager) createInstance(
 		}
 	}
 
+
 	// 2. Validate image exists and is ready; auto-pull if not found
 	log.DebugContext(ctx, "validating image", "image", req.Image)
 	imageCtx, imageSpanEnd := m.startLifecycleStep(ctx, "resolve_image",
diff --git a/lib/instances/start.go b/lib/instances/start.go
index 77503855..ecb33f11 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -48,6 +48,11 @@ func (m *manager) startInstance(
 		return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State)
 	}
 
+	if stored.GPUProfile != "" {
+		if err := validateVGPUHypervisor(stored.HypervisorType); err != nil {
+			return nil, fmt.Errorf("%w: %w", ErrInvalidState, err)
+		}
+	}
 	// Release any assignment retained by an earlier failed release and
 	// persist the cleared fields immediately, so a failure later in start
 	// cannot leave on-disk metadata pointing at a device that is already
@@ -63,6 +68,10 @@ func (m *manager) startInstance(
 		}
 	}
 
+	// Do not persist the previous VMM's identity with a new vGPU assignment.
+	stored.HypervisorPID = nil
+	stored.HypervisorStartTime = 0
+	stored.HypervisorBootID = ""
 	rollbackMeta := *meta
 
 	// 2a. Clear stale exit info from previous run and apply command overrides
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 772d93e1..6fd02352 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -2,6 +2,7 @@ package instances
 
 import (
 	"context"
+	"fmt"
 	"path/filepath"
 	"time"
 
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 56c6f857..4491bdd1 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -190,9 +190,16 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) {
 	m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error {
 		return errors.New("destroy failed")
 	})
+	meta, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	stalePID := os.Getpid()
+	meta.HypervisorPID = &stalePID
+	meta.HypervisorStartTime = 1
+	meta.HypervisorBootID = "previous-boot"
+	require.NoError(t, m.saveMetadata(meta))
 
 	t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing"))
-	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}})
+	_, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}})
 	require.Error(t, err)
 
 	stored, err := m.loadMetadata(id)
@@ -200,6 +207,9 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) {
 	assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework)
 	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath)
 	assert.NotNil(t, stored.GPUAssignedAt)
+	assert.Nil(t, stored.HypervisorPID)
+	assert.Zero(t, stored.HypervisorStartTime)
+	assert.Empty(t, stored.HypervisorBootID)
 	assert.Empty(t, stored.Entrypoint)
 }
 

From c08ec8d89b5865c3dbaf380dff7fa2e5c707b953 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 19:30:28 +0000
Subject: [PATCH 26/38] Persist vGPU assignments after create rollback failure

---
 lib/instances/create.go    |  1 +
 lib/instances/start.go     |  8 ++++++
 lib/instances/vgpu.go      | 23 ++++++++++++++++
 lib/instances/vgpu_test.go | 56 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 88 insertions(+)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index fe235f4f..97f236e1 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -303,6 +303,7 @@ func (m *manager) createInstance(
 		log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile)
 		gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id)
 		if err != nil {
+			retainedVGPU = retainedVGPUFromCreateError(id, m.nowUTC(), err)
 			log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err)
 			return nil, wrapCreateVGPUErr(req.GPU.Profile, err)
 		}
diff --git a/lib/instances/start.go b/lib/instances/start.go
index ecb33f11..39418ad7 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -175,6 +175,14 @@ func (m *manager) startInstance(
 		log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile)
 		device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id)
 		if err != nil {
+			if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok {
+				assignedAt := m.nowUTC()
+				setStoredVGPUDevice(stored, pendingDevice, assignedAt)
+				if saveErr := m.saveMetadata(meta); saveErr != nil {
+					log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr)
+					return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr)
+				}
+			}
 			log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err)
 			return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err)
 		}
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 6fd02352..a52cfa5d 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -2,6 +2,7 @@ package instances
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"path/filepath"
 	"time"
@@ -48,6 +49,28 @@ func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID
 	return create(ctx, profileName, instanceID)
 }
 
+func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) {
+	var pending *devices.VGPUCreateCleanupPendingError
+	if !errors.As(err, &pending) {
+		return nil, false
+	}
+	return &pending.Device, true
+}
+
+func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err error) *StoredMetadata {
+	device, ok := vgpuDevicePendingCleanup(err)
+	if !ok {
+		return nil
+	}
+	return &StoredMetadata{
+		Id:            instanceID,
+		GPUFramework:  device.Framework,
+		GPUDevicePath: device.SysfsPath,
+		GPUMdevUUID:   device.MdevUUID,
+		GPUAssignedAt: &assignedAt,
+	}
+}
+
 func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error {
 	destroy := m.destroyVGPU
 	if destroy == nil {
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 4491bdd1..3cfd0443 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -3,6 +3,7 @@ package instances
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"sync"
@@ -128,6 +129,35 @@ func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) {
 	assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error())
 }
 
+func TestVGPUDevicePendingCleanup(t *testing.T) {
+	t.Parallel()
+
+	device := devices.VGPUDevice{
+		Framework: devices.VGPUFrameworkVendorVFIO,
+		SysfsPath: "/sys/bus/pci/devices/0000:82:00.4",
+	}
+	cause := errors.New("rollback failed")
+	pending := &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause}
+
+	wrapped := fmt.Errorf("create failed: %w", pending)
+	actual, ok := vgpuDevicePendingCleanup(wrapped)
+	require.True(t, ok)
+	assert.Equal(t, device, *actual)
+
+	assignedAt := time.Now().UTC()
+	retained := retainedVGPUFromCreateError("inst-1", assignedAt, wrapped)
+	require.NotNil(t, retained)
+	assert.Equal(t, "inst-1", retained.Id)
+	assert.Equal(t, device.Framework, retained.GPUFramework)
+	assert.Equal(t, device.SysfsPath, retained.GPUDevicePath)
+	assert.Equal(t, assignedAt, *retained.GPUAssignedAt)
+
+	actual, ok = vgpuDevicePendingCleanup(cause)
+	assert.False(t, ok)
+	assert.Nil(t, actual)
+	assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause))
+}
+
 func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) {
 	t.Helper()
 	m := &manager{
@@ -160,6 +190,32 @@ func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, dev
 	return m, id
 }
 
+func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) {
+	m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error {
+		return nil
+	})
+	device := devices.VGPUDevice{
+		Framework:   devices.VGPUFrameworkVendorVFIO,
+		VFAddress:   "0000:82:00.4",
+		ProfileType: "1148",
+		ProfileName: "NVIDIA L40S-2Q",
+		SysfsPath:   "/sys/bus/pci/devices/0000:82:00.4",
+	}
+	cause := errors.New("create verification and rollback failed")
+	m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) {
+		return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause}
+	}
+
+	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{})
+	require.ErrorIs(t, err, cause)
+
+	stored, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	assert.Equal(t, device.Framework, stored.GPUFramework)
+	assert.Equal(t, device.SysfsPath, stored.GPUDevicePath)
+	assert.NotNil(t, stored.GPUAssignedAt)
+}
+
 func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {
 	var destroyed []devices.VGPUAssignment
 	m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error {

From 8e3acf263f9d4d2699ceea289d5c256b18bdb660 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 19:52:26 +0000
Subject: [PATCH 27/38] Preserve vGPU lifecycle compatibility

---
 lib/instances/create.go    |  6 ---
 lib/instances/start.go     | 10 ++---
 lib/instances/vgpu.go      |  8 ----
 lib/instances/vgpu_test.go | 76 ++++++++++++++++++++++++++++++++++----
 4 files changed, 71 insertions(+), 29 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index 97f236e1..203cc2aa 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -103,11 +103,6 @@ func (m *manager) createInstance(
 	if hvType == "" {
 		hvType = m.defaultHypervisor
 	}
-	if req.GPU != nil && req.GPU.Profile != "" {
-		if err := validateVGPUHypervisor(hvType); err != nil {
-			return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
-		}
-	}
 	starter, starterErr := m.getVMStarter(hvType)
 	if starterErr == nil {
 		if err := m.validateCreateVMConfig(starter, req, hvType); err != nil {
@@ -115,7 +110,6 @@ func (m *manager) createInstance(
 		}
 	}
 
-
 	// 2. Validate image exists and is ready; auto-pull if not found
 	log.DebugContext(ctx, "validating image", "image", req.Image)
 	imageCtx, imageSpanEnd := m.startLifecycleStep(ctx, "resolve_image",
diff --git a/lib/instances/start.go b/lib/instances/start.go
index 39418ad7..f9596769 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -48,11 +48,6 @@ func (m *manager) startInstance(
 		return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State)
 	}
 
-	if stored.GPUProfile != "" {
-		if err := validateVGPUHypervisor(stored.HypervisorType); err != nil {
-			return nil, fmt.Errorf("%w: %w", ErrInvalidState, err)
-		}
-	}
 	// Release any assignment retained by an earlier failed release and
 	// persist the cleared fields immediately, so a failure later in start
 	// cannot leave on-disk metadata pointing at a device that is already
@@ -177,8 +172,9 @@ func (m *manager) startInstance(
 		if err != nil {
 			if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok {
 				assignedAt := m.nowUTC()
-				setStoredVGPUDevice(stored, pendingDevice, assignedAt)
-				if saveErr := m.saveMetadata(meta); saveErr != nil {
+				retentionMeta := rollbackMeta
+				setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt)
+				if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil {
 					log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr)
 					return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr)
 				}
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index a52cfa5d..290c474b 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -8,17 +8,9 @@ import (
 	"time"
 
 	"github.com/kernel/hypeman/lib/devices"
-	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/logger"
 )
 
-func validateVGPUHypervisor(hvType hypervisor.Type) error {
-	if hvType != hypervisor.TypeQEMU {
-		return fmt.Errorf("vGPU is only supported with qemu, got %s", hvType)
-	}
-	return nil
-}
-
 // VGPUAssignmentStartupGracePeriod bounds how long an assignment without a
 // persisted hypervisor PID is treated as potentially live.
 const VGPUAssignmentStartupGracePeriod = 5 * time.Minute
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 3cfd0443..1ff75936 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -12,18 +12,12 @@ import (
 
 	"github.com/kernel/hypeman/lib/devices"
 	"github.com/kernel/hypeman/lib/hypervisor"
+	"github.com/kernel/hypeman/lib/network"
 	"github.com/kernel/hypeman/lib/paths"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestValidateVGPUHypervisor(t *testing.T) {
-	t.Parallel()
-
-	assert.NoError(t, validateVGPUHypervisor(hypervisor.TypeQEMU))
-	assert.EqualError(t, validateVGPUHypervisor(hypervisor.TypeCloudHypervisor), "vGPU is only supported with qemu, got cloud-hypervisor")
-}
-
 func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 	t.Parallel()
 
@@ -158,6 +152,22 @@ func TestVGPUDevicePendingCleanup(t *testing.T) {
 	assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause))
 }
 
+type startRetentionNetworkManager struct {
+	network.Manager
+	config       network.NetworkConfig
+	releaseCalls int
+}
+
+func (m *startRetentionNetworkManager) CreateAllocation(context.Context, network.AllocateRequest) (*network.NetworkConfig, error) {
+	config := m.config
+	return &config, nil
+}
+
+func (m *startRetentionNetworkManager) ReleaseAllocation(context.Context, *network.Allocation) error {
+	m.releaseCalls++
+	return nil
+}
+
 func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) {
 	t.Helper()
 	m := &manager{
@@ -194,6 +204,27 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) {
 	m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error {
 		return nil
 	})
+	networkManager := &startRetentionNetworkManager{config: network.NetworkConfig{
+		IP:        "192.0.2.20",
+		MAC:       "02:00:00:00:00:20",
+		TAPDevice: "tap-new",
+	}}
+	m.networkManager = networkManager
+
+	previousProgramStart := time.Now().Add(-time.Hour).UTC()
+	previousExitCode := 23
+	meta, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	meta.NetworkEnabled = true
+	meta.IP = "192.0.2.10"
+	meta.MAC = "02:00:00:00:00:10"
+	meta.Entrypoint = []string{"old-entrypoint"}
+	meta.Cmd = []string{"old-command"}
+	meta.ProgramStartedAt = &previousProgramStart
+	meta.ExitCode = &previousExitCode
+	meta.ExitMessage = "previous exit"
+	require.NoError(t, m.saveMetadata(meta))
+
 	device := devices.VGPUDevice{
 		Framework:   devices.VGPUFrameworkVendorVFIO,
 		VFAddress:   "0000:82:00.4",
@@ -206,7 +237,10 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) {
 		return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause}
 	}
 
-	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{})
+	_, err = m.startInstance(context.Background(), id, StartInstanceRequest{
+		Entrypoint: []string{"new-entrypoint"},
+		Cmd:        []string{"new-command"},
+	})
 	require.ErrorIs(t, err, cause)
 
 	stored, err := m.loadMetadata(id)
@@ -214,6 +248,32 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) {
 	assert.Equal(t, device.Framework, stored.GPUFramework)
 	assert.Equal(t, device.SysfsPath, stored.GPUDevicePath)
 	assert.NotNil(t, stored.GPUAssignedAt)
+	assert.Equal(t, []string{"old-entrypoint"}, stored.Entrypoint)
+	assert.Equal(t, []string{"old-command"}, stored.Cmd)
+	assert.Equal(t, previousProgramStart, *stored.ProgramStartedAt)
+	assert.Equal(t, previousExitCode, *stored.ExitCode)
+	assert.Equal(t, "previous exit", stored.ExitMessage)
+	assert.Equal(t, "192.0.2.10", stored.IP)
+	assert.Equal(t, "02:00:00:00:00:10", stored.MAC)
+	assert.Equal(t, 1, networkManager.releaseCalls)
+}
+
+func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) {
+	m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error {
+		return nil
+	})
+	meta, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	meta.HypervisorType = hypervisor.TypeCloudHypervisor
+	require.NoError(t, m.saveMetadata(meta))
+
+	cause := errors.New("create failed")
+	m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) {
+		return nil, cause
+	}
+
+	_, err = m.startInstance(context.Background(), id, StartInstanceRequest{})
+	assert.ErrorIs(t, err, cause)
 }
 
 func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {

From 79a6fd6b681f5a39f56070331486d4257faaa790 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 21:11:08 +0000
Subject: [PATCH 28/38] Reconcile vGPU protection from raw metadata and restore
 GPUAssignedAt

ListInstancesForReconcile hydrated every instance (socket stat, UFFD
health, /vm.info per instance) before the API served and again on each
grace retry, while the protected-set scan only reads stored metadata
fields. List raw metadata fail-closed instead, matching the release
claim scan, and drop the now-unused loadInstances parameterization.

Snapshot restore preserved the source's vGPU assignment path fields but
not GPUAssignedAt, so a retained assignment lost its crash-recovery
grace timestamp across a restore. Carry the timestamp with the rest of
the assignment.
---
 lib/instances/manager.go       | 21 +++++++++++++++++++--
 lib/instances/query.go         | 11 +----------
 lib/instances/query_test.go    |  6 ++++++
 lib/instances/snapshot.go      |  1 +
 lib/instances/snapshot_test.go |  4 ++++
 5 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/lib/instances/manager.go b/lib/instances/manager.go
index ebb98081..16a6b419 100644
--- a/lib/instances/manager.go
+++ b/lib/instances/manager.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -701,9 +702,25 @@ func (m *manager) UpdateInstance(ctx context.Context, id string, req UpdateInsta
 	return inst, err
 }
 
-// ListInstancesForReconcile returns every instance or an invalid metadata error.
+// ListInstancesForReconcile returns every instance's stored metadata or an
+// invalid metadata error. It does not derive state: reconcile protection only
+// needs raw metadata fields, and hydration would query the hypervisor of
+// every instance on the host before the API serves.
 func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) {
-	return m.loadInstances(ctx, false)
+	files, err := m.listMetadataFilesWithStatErrors(true)
+	if err != nil {
+		return nil, err
+	}
+	result := make([]Instance, 0, len(files))
+	for _, file := range files {
+		id := filepath.Base(filepath.Dir(file))
+		meta, err := m.loadMetadata(id)
+		if err != nil {
+			return nil, fmt.Errorf("load metadata for instance %s: %w", id, err)
+		}
+		result = append(result, Instance{StoredMetadata: meta.StoredMetadata})
+	}
+	return result, nil
 }
 
 // ListInstances returns instances, optionally filtered by the given criteria.
diff --git a/lib/instances/query.go b/lib/instances/query.go
index a7556035..84e61038 100644
--- a/lib/instances/query.go
+++ b/lib/instances/query.go
@@ -1020,16 +1020,12 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) {
 
 // listInstances returns all instances, skipping metadata files that cannot be loaded.
 func (m *manager) listInstances(ctx context.Context) ([]Instance, error) {
-	return m.loadInstances(ctx, true)
-}
-
-func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instance, error) {
 	ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata")
 	defer span.End()
 	log := logger.FromContext(ctx)
 	log.DebugContext(ctx, "listing all instances")
 
-	files, err := m.listMetadataFilesWithStatErrors(!skipInvalid)
+	files, err := m.listMetadataFiles()
 	if err != nil {
 		log.ErrorContext(ctx, "failed to list metadata files", "error", err)
 		return nil, err
@@ -1047,11 +1043,6 @@ func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instan
 		)
 		meta, err := m.loadMetadata(id)
 		if err != nil {
-			if !skipInvalid {
-				hydrateSpan.RecordError(err)
-				hydrateSpan.End()
-				return nil, fmt.Errorf("load metadata for instance %s: %w", id, err)
-			}
 			// Skip instances with invalid metadata
 			log.WarnContext(hydrateCtx, "skipping instance with invalid metadata", "instance_id", id, "error", err)
 			hydrateSpan.End()
diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go
index 41aba54e..ab3db29c 100644
--- a/lib/instances/query_test.go
+++ b/lib/instances/query_test.go
@@ -34,6 +34,12 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) {
 	_, err = m.ListInstancesForReconcile(context.Background())
 	require.Error(t, err)
 	assert.ErrorContains(t, err, "load metadata for instance invalid")
+
+	require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid")))
+	listed, err = m.ListInstancesForReconcile(context.Background())
+	require.NoError(t, err)
+	require.Len(t, listed, 1)
+	assert.Equal(t, "valid", listed[0].Id)
 }
 
 func TestParseExitSentinelLine(t *testing.T) {
diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go
index 4ad2065e..c54b91fd 100644
--- a/lib/instances/snapshot.go
+++ b/lib/instances/snapshot.go
@@ -313,6 +313,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str
 	restored.GPUFramework = sourceMeta.GPUFramework
 	restored.GPUDevicePath = sourceMeta.GPUDevicePath
 	restored.GPUMdevUUID = sourceMeta.GPUMdevUUID
+	restored.GPUAssignedAt = sourceMeta.GPUAssignedAt
 	restored.HypervisorType = targetHypervisor
 	restored.HypervisorVersion = targetHypervisorVersion
 	restored.SocketPath = m.paths.InstanceSocket(id, starter.SocketName())
diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go
index 1549133b..a42bb1d1 100644
--- a/lib/instances/snapshot_test.go
+++ b/lib/instances/snapshot_test.go
@@ -113,6 +113,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) {
 	meta.GPUFramework = devices.VGPUFramework("future-framework")
 	meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4"
 	meta.GPUMdevUUID = "retained-uuid"
+	assignedAt := time.Now().UTC().Truncate(time.Second)
+	meta.GPUAssignedAt = &assignedAt
 	require.NoError(t, mgr.saveMetadata(meta))
 
 	_, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{
@@ -126,6 +128,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) {
 	assert.Equal(t, devices.VGPUFramework("future-framework"), restored.GPUFramework)
 	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", restored.GPUDevicePath)
 	assert.Equal(t, "retained-uuid", restored.GPUMdevUUID)
+	require.NotNil(t, restored.GPUAssignedAt)
+	assert.True(t, assignedAt.Equal(*restored.GPUAssignedAt))
 }
 
 func TestStoppedSnapshotLifecycleAndForkAfterSourceDeletion(t *testing.T) {

From 6ef15b3dc4b683d858177392acf2ce04e2b2b97b Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Tue, 11 Aug 2026 14:46:50 +0000
Subject: [PATCH 29/38] Surface pending vGPU cleanup from start as a typed
 error
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When start's vGPU create fails with a pending device-layer cleanup, the
error was returned untyped, so the API mapped it to a generic
internal_error. Create already wraps the same condition in
VGPUCleanupPendingError and surfaces vgpu_cleanup_pending with
retained/unretained guidance.

Wrap start's pending-cleanup error the same way — Retained reflects
whether the retention record was persisted — and map it in the
StartInstance handler ahead of the errors.Is cases so the wrapped cause
cannot hide the pending cleanup.
---
 cmd/api/api/instances.go      | 19 +++++++++++
 cmd/api/api/instances_test.go | 62 +++++++++++++++++++++++++++++++++++
 lib/instances/start.go        |  6 ++--
 lib/instances/vgpu_test.go    | 39 ++++++++++++++++++++++
 4 files changed, 124 insertions(+), 2 deletions(-)

diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go
index 0efa1c76..dc060399 100644
--- a/cmd/api/api/instances.go
+++ b/cmd/api/api/instances.go
@@ -817,7 +817,26 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan
 
 	result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq)
 	if err != nil {
+		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
+		// Checked first: it wraps the original start error, so a later
+		// errors.Is case would match the cause and hide the pending vGPU cleanup.
+		case errors.As(err, &vgpuPending):
+			log.ErrorContext(ctx, "failed to start instance", "error", err)
+			message := fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it or retry start to release it", vgpuPending.Err, vgpuPending.InstanceID)
+			innerCode := "vgpu_retained_instance"
+			if !vgpuPending.Retained {
+				message = fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID)
+				innerCode = "vgpu_unretained_instance"
+			}
+			return oapi.StartInstance500JSONResponse{
+				Code:    "vgpu_cleanup_pending",
+				Message: message,
+				InnerError: &oapi.ErrorDetail{
+					Code:    lo.ToPtr(innerCode),
+					Message: lo.ToPtr(vgpuPending.InstanceID),
+				},
+			}, nil
 		case errors.Is(err, instances.ErrInvalidState):
 			return oapi.StartInstance409JSONResponse{
 				Code:    "invalid_state",
diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go
index 8e0512bd..4f04692e 100644
--- a/cmd/api/api/instances_test.go
+++ b/cmd/api/api/instances_test.go
@@ -847,6 +847,68 @@ func (m *errActionInstanceManager) RestoreSnapshot(context.Context, string, stri
 	return nil, m.err
 }
 
+// A retained-assignment error must win over the mapping of the start error
+// it wraps, or the response omits the pending vGPU cleanup the caller has to
+// resolve.
+func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) {
+	t.Parallel()
+
+	resolved := &instances.Instance{
+		StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"},
+		State:          instances.StateStopped,
+	}
+
+	t.Run("retained", func(t *testing.T) {
+		t.Parallel()
+		svc := newTestService(t)
+		svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{
+			InstanceID: "inst-1",
+			Retained:   true,
+			Err:        fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources),
+		}}
+
+		resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id})
+		require.NoError(t, rerr)
+
+		pending, ok := resp.(oapi.StartInstance500JSONResponse)
+		require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+		assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+		assert.Contains(t, pending.Message, "inst-1")
+		assert.Contains(t, pending.Message, instances.ErrInsufficientResources.Error(),
+			"the underlying start failure must survive the cleanup guidance")
+		assert.Contains(t, pending.Message, "delete it or retry start")
+		require.NotNil(t, pending.InnerError)
+		require.NotNil(t, pending.InnerError.Code)
+		assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code)
+		require.NotNil(t, pending.InnerError.Message)
+		assert.Equal(t, "inst-1", *pending.InnerError.Message)
+	})
+
+	t.Run("unretained", func(t *testing.T) {
+		t.Parallel()
+		svc := newTestService(t)
+		svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{
+			InstanceID: "inst-1",
+			Err:        fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources),
+		}}
+
+		resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id})
+		require.NoError(t, rerr)
+
+		pending, ok := resp.(oapi.StartInstance500JSONResponse)
+		require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+		assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+		assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved")
+		assert.Contains(t, pending.Message, "startup reconcile")
+		assert.NotContains(t, pending.Message, "delete")
+		require.NotNil(t, pending.InnerError)
+		require.NotNil(t, pending.InnerError.Code)
+		assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code)
+		require.NotNil(t, pending.InnerError.Message)
+		assert.Equal(t, "inst-1", *pending.InnerError.Message)
+	})
+}
+
 func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) {
 	t.Parallel()
 
diff --git a/lib/instances/start.go b/lib/instances/start.go
index f9596769..2b7712af 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -170,16 +170,18 @@ func (m *manager) startInstance(
 		log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile)
 		device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id)
 		if err != nil {
+			log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err)
 			if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok {
 				assignedAt := m.nowUTC()
 				retentionMeta := rollbackMeta
 				setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt)
+				wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err)
 				if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil {
 					log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr)
-					return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr)
+					return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)}
 				}
+				return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped}
 			}
-			log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err)
 			return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err)
 		}
 		assignedAt := m.nowUTC()
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 1ff75936..d9e599d9 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -242,6 +242,10 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) {
 		Cmd:        []string{"new-command"},
 	})
 	require.ErrorIs(t, err, cause)
+	var pending *VGPUCleanupPendingError
+	require.ErrorAs(t, err, &pending)
+	assert.Equal(t, id, pending.InstanceID)
+	assert.True(t, pending.Retained)
 
 	stored, err := m.loadMetadata(id)
 	require.NoError(t, err)
@@ -258,6 +262,41 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) {
 	assert.Equal(t, 1, networkManager.releaseCalls)
 }
 
+func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("root bypasses directory permissions")
+	}
+
+	m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error {
+		return nil
+	})
+	device := devices.VGPUDevice{
+		Framework:   devices.VGPUFrameworkVendorVFIO,
+		VFAddress:   "0000:82:00.4",
+		ProfileType: "1148",
+		ProfileName: "NVIDIA L40S-2Q",
+		SysfsPath:   "/sys/bus/pci/devices/0000:82:00.4",
+	}
+	cause := errors.New("create verification and rollback failed")
+	m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) {
+		instanceDir := filepath.Dir(m.paths.InstanceMetadata(id))
+		require.NoError(t, os.Chmod(instanceDir, 0o555))
+		t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) })
+		return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause}
+	}
+
+	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{})
+	require.ErrorIs(t, err, cause)
+	var pending *VGPUCleanupPendingError
+	require.ErrorAs(t, err, &pending)
+	assert.Equal(t, id, pending.InstanceID)
+	assert.False(t, pending.Retained)
+
+	stored, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded")
+}
+
 func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) {
 	m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error {
 		return nil

From f3dd2f9890cc87f0fe3264313164ae33dbb01120 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Tue, 11 Aug 2026 15:41:14 +0000
Subject: [PATCH 30/38] Cover retained-stub delete recovery and flag reconcile
 inventory failures

The vgpu_cleanup_pending guidance tells callers to delete the retained
instance to retry a failed vGPU release, but no test exercised delete
against the minimal GPU-fields-only stub cleanupFailedCreate writes.
Add one.

Losing the reconcile inventory disables vendor VFIO reconciliation
host-wide while releases fail closed on the same inventory, so log it
at error level instead of warn. Also document the wholesale-restore
assumption in cleanupStartVGPU.
---
 cmd/api/main.go                      |  5 +++-
 lib/instances/lifecycle_noop_test.go | 37 ++++++++++++++++++++++++++++
 lib/instances/vgpu.go                |  4 +++
 3 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/cmd/api/main.go b/cmd/api/main.go
index e4e79451..3c9c9e88 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -218,7 +218,10 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.
 func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) {
 	protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager)
 	if err != nil {
-		logger.Warn("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err)
+		// Operator-actionable: vendor VFIO reconciliation stays disabled
+		// host-wide (and releases fail closed on the same inventory) until
+		// the unreadable instance metadata is repaired.
+		logger.Error("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err)
 		protected = nil
 		retryAfter = 0
 	}
diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go
index 15f632b3..c74d352f 100644
--- a/lib/instances/lifecycle_noop_test.go
+++ b/lib/instances/lifecycle_noop_test.go
@@ -196,6 +196,43 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) {
 	assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason)
 }
 
+// A failed create whose vGPU release also failed retains a minimal
+// GPU-fields-only stub, and the API tells the caller to delete it to retry
+// the release. Exercise that recovery path against the exact stub shape
+// cleanupFailedCreate writes.
+func TestDeleteReleasesRetainedCreateStub(t *testing.T) {
+	p := paths.New(t.TempDir())
+	var destroyed []devices.VGPUAssignment
+	m := &manager{
+		paths:           p,
+		instanceLocks:   sync.Map{},
+		bootMarkerScans: sync.Map{},
+		now:             time.Now,
+		lifecycleEvents: newLifecycleSubscribers(),
+		destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error {
+			destroyed = append(destroyed, assignment)
+			return nil
+		},
+	}
+	const id = "retained-stub"
+	require.NoError(t, m.ensureDirectories(id))
+	assignedAt := time.Now().UTC()
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            id,
+		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+		GPUAssignedAt: &assignedAt,
+	}}))
+
+	require.NoError(t, m.DeleteInstance(context.Background(), id))
+
+	require.Len(t, destroyed, 1)
+	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", destroyed[0].DevicePath)
+	assert.Equal(t, id, destroyed[0].InstanceID)
+	_, err := m.loadMetadata(id)
+	require.Error(t, err, "retained stub must be fully deleted")
+}
+
 func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) {
 	now := time.Now().UTC()
 	m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now)
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 290c474b..f25c7c63 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -85,6 +85,10 @@ func clearStoredVGPUDevice(stored *StoredMetadata) {
 	stored.GPUAssignedAt = nil
 }
 
+// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. That
+// is safe while the instance lock serializes start and no cleanup registered
+// after the vGPU one persists metadata; a future cleanup that writes metadata
+// must switch this to targeted field restores.
 func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) {
 	assignment := devices.VGPUAssignment{
 		Framework:  device.Framework,

From 0cbcb36ee7eca1990a48e022496e0bcc289535ae Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Tue, 11 Aug 2026 17:16:52 +0000
Subject: [PATCH 31/38] Reject vendor VFIO vGPUs on Cloud Hypervisor and
 improve wedge forensics

Vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor
(upstream cloud-hypervisor#7572), and the wedged VM then blocks the VF
release until startup reconcile. Reject the combination at create and
start after the rollback handler is registered, so the rejected device
is released through the normal cleanup path. Hypervisor selection
otherwise stays caller policy and mdev on Cloud Hypervisor keeps
working.

Retain identity fields (name, image, hypervisor, data dir) on the
failed-create retention record so it lists as a recognizable, deletable
instance instead of a nameless phantom; resource claims released by
rollback stay dropped.

Expose the assigned vGPU device_path in the instance API - on vendor
VFIO hosts mdev_uuid is empty and the sysfs path is the identity an
operator needs when a release wedges.
---
 cmd/api/api/instances.go   |   3 +
 lib/instances/create.go    |  29 ++-
 lib/instances/start.go     |   6 +
 lib/instances/vgpu.go      |  24 ++-
 lib/instances/vgpu_test.go |  39 +++-
 lib/oapi/oapi.go           | 390 +++++++++++++++++++------------------
 openapi.yaml               |   6 +-
 7 files changed, 291 insertions(+), 206 deletions(-)

diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go
index dc060399..8ff70223 100644
--- a/cmd/api/api/instances.go
+++ b/cmd/api/api/instances.go
@@ -1243,6 +1243,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance {
 		if inst.GPUMdevUUID != "" {
 			gpu.MdevUuid = lo.ToPtr(inst.GPUMdevUUID)
 		}
+		if inst.GPUDevicePath != "" {
+			gpu.DevicePath = lo.ToPtr(inst.GPUDevicePath)
+		}
 		oapiInst.Gpu = gpu
 	}
 
diff --git a/lib/instances/create.go b/lib/instances/create.go
index 203cc2aa..66f6ebb3 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -342,6 +342,12 @@ func (m *manager) createInstance(
 				}
 			}
 		})
+		// Checked after the cleanup handler is registered so rejection
+		// releases the device through the normal rollback.
+		if err := validateVGPUHypervisorCompat(gpuDevice.Framework, hvType); err != nil {
+			log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", gpuDevice.Framework, "hypervisor", hvType)
+			return nil, err
+		}
 	}
 
 	if len(req.Devices) > 0 && m.deviceManager != nil {
@@ -633,12 +639,25 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
 		return retentionSurvives()
 	}
+	// Retain identity fields so the instance lists as a recognizable,
+	// deletable record rather than a nameless phantom, but drop resource
+	// claims (network, volumes, devices) that rollback already released.
 	retained := StoredMetadata{
-		Id:            id,
-		GPUFramework:  retainedVGPU.GPUFramework,
-		GPUDevicePath: retainedVGPU.GPUDevicePath,
-		GPUMdevUUID:   retainedVGPU.GPUMdevUUID,
-		GPUAssignedAt: retainedVGPU.GPUAssignedAt,
+		Id:                id,
+		Name:              retainedVGPU.Name,
+		Image:             retainedVGPU.Image,
+		ResolvedImage:     retainedVGPU.ResolvedImage,
+		Platform:          retainedVGPU.Platform,
+		CreatedAt:         retainedVGPU.CreatedAt,
+		HypervisorType:    retainedVGPU.HypervisorType,
+		HypervisorVersion: retainedVGPU.HypervisorVersion,
+		SocketPath:        retainedVGPU.SocketPath,
+		DataDir:           retainedVGPU.DataDir,
+		GPUProfile:        retainedVGPU.GPUProfile,
+		GPUFramework:      retainedVGPU.GPUFramework,
+		GPUDevicePath:     retainedVGPU.GPUDevicePath,
+		GPUMdevUUID:       retainedVGPU.GPUMdevUUID,
+		GPUAssignedAt:     retainedVGPU.GPUAssignedAt,
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
diff --git a/lib/instances/start.go b/lib/instances/start.go
index 2b7712af..b44a68d5 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -191,6 +191,12 @@ func (m *manager) startInstance(
 		cu.Add(func() {
 			m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta)
 		})
+		// Checked after the cleanup handler is registered so rejection
+		// releases the device through the normal rollback.
+		if err := validateVGPUHypervisorCompat(device.Framework, stored.HypervisorType); err != nil {
+			log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", device.Framework, "hypervisor", stored.HypervisorType)
+			return nil, err
+		}
 		if err := m.saveMetadata(meta); err != nil {
 			log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err)
 			return nil, fmt.Errorf("save metadata after vGPU creation: %w", err)
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index f25c7c63..8c635174 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/kernel/hypeman/lib/devices"
+	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/logger"
 )
 
@@ -63,6 +64,18 @@ func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err er
 	}
 }
 
+// validateVGPUHypervisorCompat rejects the one proven-broken combination:
+// vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor (upstream
+// cloud-hypervisor#7572), and the wedged VM then blocks the VF release until
+// startup reconcile. Hypervisor selection otherwise remains caller policy;
+// mdev on Cloud Hypervisor keeps working. See lib/devices/GPU.md.
+func validateVGPUHypervisorCompat(framework devices.VGPUFramework, hvType hypervisor.Type) error {
+	if framework == devices.VGPUFrameworkVendorVFIO && hvType == hypervisor.TypeCloudHypervisor {
+		return fmt.Errorf("%w: vendor VFIO vGPUs are not functional on cloud-hypervisor, use qemu", ErrInvalidRequest)
+	}
+	return nil
+}
+
 func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error {
 	destroy := m.destroyVGPU
 	if destroy == nil {
@@ -85,10 +98,13 @@ func clearStoredVGPUDevice(stored *StoredMetadata) {
 	stored.GPUAssignedAt = nil
 }
 
-// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. That
-// is safe while the instance lock serializes start and no cleanup registered
-// after the vGPU one persists metadata; a future cleanup that writes metadata
-// must switch this to targeted field restores.
+// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. The
+// cleanup stack is LIFO, so cleanups registered after this one run before it
+// and this restore would clobber anything they persisted; it is safe only
+// while no such cleanup writes metadata and the instance lock serializes
+// start. The snapshot is also a shallow copy (Phases shares its map), so it
+// must be persisted before any Phases.Record on the live struct. Violating
+// either invariant requires switching to targeted field restores.
 func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) {
 	assignment := devices.VGPUAssignment{
 		Framework:  device.Framework,
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index d9e599d9..d466f188 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -47,12 +47,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 	assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath)
 	assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID)
 	assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt)
-	assert.Empty(t, retained.Name)
-	assert.Empty(t, retained.GPUProfile)
+	// Identity fields survive so the retained record lists as a
+	// recognizable, deletable instance instead of a nameless phantom.
+	assert.Equal(t, stored.Name, retained.Name)
+	assert.Equal(t, stored.GPUProfile, retained.GPUProfile)
+	assert.Equal(t, stored.HypervisorType, retained.HypervisorType)
+	assert.Equal(t, stored.DataDir, retained.DataDir)
+	// Resource claims released by rollback stay dropped.
 	assert.False(t, retained.NetworkEnabled)
 	assert.Empty(t, retained.IP)
 	assert.Empty(t, retained.Volumes)
-	assert.Empty(t, retained.DataDir)
 }
 
 func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) {
@@ -315,6 +319,35 @@ func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) {
 	assert.ErrorIs(t, err, cause)
 }
 
+func TestValidateVGPUHypervisorCompat(t *testing.T) {
+	t.Parallel()
+
+	err := validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeCloudHypervisor)
+	require.ErrorIs(t, err, ErrInvalidRequest)
+	assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeQEMU))
+	assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkMdev, hypervisor.TypeCloudHypervisor))
+}
+
+func TestStartRejectsVendorVFIOOnCloudHypervisor(t *testing.T) {
+	var destroyed []devices.VGPUAssignment
+	m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error {
+		destroyed = append(destroyed, assignment)
+		return nil
+	})
+	meta, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	meta.HypervisorType = hypervisor.TypeCloudHypervisor
+	require.NoError(t, m.saveMetadata(meta))
+
+	_, err = m.startInstance(context.Background(), id, StartInstanceRequest{})
+	require.ErrorIs(t, err, ErrInvalidRequest)
+
+	require.Len(t, destroyed, 1, "the rejected vGPU must be released by rollback")
+	stored, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	assert.Empty(t, stored.GPUDevicePath, "no assignment may be persisted for a rejected combination")
+}
+
 func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {
 	var destroyed []devices.VGPUAssignment
 	m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error {
diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go
index 8f02bcf8..6e3e31d8 100644
--- a/lib/oapi/oapi.go
+++ b/lib/oapi/oapi.go
@@ -1214,7 +1214,10 @@ type InstanceHypervisor string
 
 // InstanceGPU GPU information attached to the instance
 type InstanceGPU struct {
-	// MdevUuid mdev device UUID
+	// DevicePath sysfs path of the assigned vGPU device
+	DevicePath *string `json:"device_path,omitempty"`
+
+	// MdevUuid mdev device UUID (mdev hosts only)
 	MdevUuid *string `json:"mdev_uuid,omitempty"`
 
 	// Profile vGPU profile name
@@ -18643,198 +18646,199 @@ var swaggerSpec = []string{
 	"AjUWWmBKGhIxZPo1D0jnx0x/398bDAYDuJMwps62/vfAx013ejvYtyC0JikW4PCYhaMFnD2RMpSykAh0",
 	"xiVRCiNi2KEaQ3bDm8XPRJ91n7dTr+znuV7l35irwGrbodB+LvTnskLV5+US1a2t191/fFY1a9JWf7Xp",
 	"OPar0U2u+AkKeBqF2iQa69POeKxIaP10kqi8+jcckG/ZJdN7tDR1G5urOPozJWKB3p2clOICBJnY4sYt",
-	"Jg5SomEdeHKjZdha4URYOZpbAsLeBwhsVVMpaIhfHPK1eAXnsoQNh7a4issNSW9mBWVmaTSfLJlTxb0d",
-	"kvkoTX2GiH7ksEHevj0+KjEHxnubTwdPn/Wejjf3ejvhYLOHN7f3elu7eDDZDp5sb25tL8nCaJHKdfvs",
-	"LK816qsX70KmRy502xdR2xQ4Xzn3bTDwFWUhvyodLd7ozGLvNvJzVff1uO7WQ/Bmg0DpYmipQUqcwAFK",
-	"At22iYauVBNv8LLtvRlsrvCyrZQXMLgG+ftGpMzc6ZkE98x/HRcGXFys8jhvJ05hQC7rYhW1ip23J9pg",
-	"f/fZ/u7nEs1lDqwaY5Wd7nFxm8KRHABwJTXBpccVfDQOMLNj9Q3jUraZDJ1uJ0u2gL/hoK0E8maPW2UQ",
-	"NW3Yrl+MLJPfDZm7xyVbAMIkDABcuK+1AKfrA8x+lh+u1YvDiKchKvi5DB4WXAIdF+wC3QzcyVj3l8G3",
-	"NJkA2n4AQGWA86dMC2K4/NKN2DTfffQS3oVHODYmkx2EKSpRvPfB4cIEa+j95bo2BszyIZ9b2wW+0YYM",
-	"0v+CaWsyWHfo8iaM5rOPfuPwTWZJMV71q5rXwYSpv171wa5ZrGgH2wCdWTVuH/2cqW6Z8meVvTVJ7J8j",
-	"K7BytJT1Us66XfGO5pZ85Qr5192OoWin23GEgjztesb225zra/uvyIq+KCaCI1NRPUtQTRWNLEY0zIRK",
-	"RQNpMxf04jbpF7beDQlHxjBpCog0WY/WeMk+curLuxO0BjCAf0PWaaz/tZ4FT5bOuq1nO8/2nmw922sF",
-	"9pMPcLXaeQg5ufXBrdRBgyQdWb9D09QPT98av0JgLPYs8OLdSRFbIRFcix49c9dgsfNn/WdFjKOQp+Oo",
-	"cKtlAdEMpCosmBfGK5NFDUF4f9JoTicT9ueH4HLrD0Hjzes9uTXebMBuNR35XVrHxZvtmv+XjHumdo0f",
-	"hgYYSshGpKYzImEG6JwoBPzTQzgA0yFLpbUs5/CcLMW9jLWzvb399MnuViu+sqMrbJwROLg8h7IdQWGL",
-	"wZto7ez8HG0UGM606QANAF6bWbPSv8+Qrd86KCuk/c3Bto9LGg7unGts2/O4keTvrGlmJ2WJDhnBmdlW",
-	"2+Veam9vD57s7D7dbbeNret1JK6XSxiXL2PIY1Hgiyu/Btrkm4NTBNmoExyU/SYuPOlGo1I3GhVUMDDI",
-	"4zcY2NMne7s721ub7SDHfBEPFkyvtGHLssuz6TxM4VkNDynqorfbdFr41CnDYGckiDCNDwIX3185fQzC",
-	"+EiY1/JFaHMwWG9/7eBq8W0rx1HmDjLZIUY14AKlLKtr0V993flFbi2bpbY5HlZLdV9OCNPUs9g4pq7V",
-	"LUiZCDKnPJVfoCGuTMLmJOJc3OjbJoPljMg0UuZqkUr07uQHkCma15BUJCnbUJYblyAI3XJyN9rPJRbx",
-	"M3kTsVqtRpulXzbhbsOu7S5DcyhJg0bcrlBLrpStjjw8xFGQQiUXnK2nnhUAUEE+epJECxNYHkWcMxTM",
-	"MIMLCFGA20EzHoV9bximfjKaeCMW+BWKuEEcviQksUVOzCD0Z1qFoXOC1gr58MiwUqUY5W5shIwtY1Hm",
-	"xt3YX1UPS1+mVJaHremJFS+A4ZpPSi7HiE8lGIUKQub7VQz2BAsTCY+ZKdozj40t6Yn29QyxIsx9J6o5",
-	"SfnEGrhW5YAsZ0NJHAguJSIRnUKBmHcnleTZJQlXWQrt6mjC8mBbsK65LPQcZQaKqXVtL9/56Ekm+ZwT",
-	"EngYEtaWxOk552SMWQplTwqMTK4TKgx7tIvFm3GpRhko0g0HK9UIahmkguRQbVnKd+YPcu94z0Un2m5D",
-	"Lhv0equva1zlb6ppgM0y1UtRP7W6GQ/62LgOC7UUiSqHtqriGN0EKS0Hv6cSWqUFzCy0BgkXBbFUwD9b",
-	"bxOA4jdZdT81a9WWbHy1Mzhviym2HELsFKvZMZtwD/DEDW4lnSfahkomRMQUqnmgkDBKQmdLZteT1tUF",
-	"2cqRJChMiaWc0U8FtgTHZnsDeARzPjLKphVZX+2wjXvYjGF5qQPo177YJpRI+rM534gUaGViASXCeV5n",
-	"qwBLKkf+66x6w4JM0wgLZNEA2wxZLuKIsss2rctFPOYRDZD+oHrnPOFRxK9G+pH8Ceay3mp2+oNRU12c",
-	"czM4m5RmFqTSbz6Fn/Qs1yspseCJ2TDfbwBcSZvILG+Y8s80IhZa7i2j1wVGL2OB72wNmlK1GxotJWnX",
-	"YQlvKrkty3p3fCo9iW1LtRxX0YeEFqDdqD1JKk1xkRankkMAdVeAt7vRKWctfB4uxqGR1xVUDDQmkHTi",
-	"plaXGi3EYpupeOsapHKG/uDjskO0bUitp1rWGsvxGQSZeIPLYUWXOqTNGzWaFFb3JgAIIFb1ROGjG+IK",
-	"rKorlscuNcmTs1qJrRmxJKNujqbcVovyES62Isudt722T6KvFkPzxAIDRIpUCygrCrVcFoUKfxKNuRAA",
-	"f6w1HM7cbADzQ+s8mtYOdAm9mZEFEiTGlA0ZZZmTFJC8CGJkTkQhRZMLbWRNSdhHfy+YeAAYHSdqYZHI",
-	"wXn+g0T8imVjHLLiIHXjqdTtHDDjWRQpVHDPX9LNgtWnGQVSZuESTAko3kfVDE0EkbPi3H0FG7WOd8VF",
-	"2FgJZ4HcK1BgBe5YkeKXhBVFWdaM1zQ0DY3MV/UIOVNtFZ5a+xOVCqCiaoHT5f2lkgi/kphNKXulVehK",
-	"YasUjBODQAJwHFDYzv5lRHwGwdECcCNv/n9ck/lPp1nj5d8qrxVANRzG7YFx23pdsIFJkakE+5Rv0laG",
-	"qkAO1jIolfpdAlpz4cmuOkhZEyhU6Wh1TrZLA6sG4rvRbEgSlHvfebr7ZK9lmZTPuqwz0FFf+mpuHi+5",
-	"kmtYqZM29z5Pd58+e7a9s/ts60Y3LC5no2F9mvI2iuuD1si10ps1+vc///XupHLrswvxzYMbDcpkbfiH",
-	"1JC5UR7Qu5N///NfblS3HpBP0NThqRvu7RujdKLiSrpAgfIVXrtLsiX2/UHJSYAzMYPWyGRCwA06MnTr",
-	"5YOpYFC004JxggOqFh5Bjq9MJHn2SgVmuc11UHmwPpXXtG0hObXkkuk4z3hcc52jv5q74QovPG1dbUmm",
-	"46Z76NfVXs0tdH5rUYxxaBFiILNC33UHdzafKyxLwdL67wByGlz2Vj2TxbyxHPK1mmYAUSy2qFghFNAH",
-	"FV7RJ+1HxeWvLGfh3rLk1qlS/P2Sfdi8BW/k9fWcyB6nb7A6bbUiH+wBeLuvRuNiHbSlheZKRdPyU/fm",
-	"/bbIzK2D9Gcn2M37KyQj3uTDKiAt8KMdgyV53na3xBIN3FTIM/E40HhEelmgnk1CQTI1N4J6z1uMc092",
-	"ZHDJJ5My0OpuMzA3YM5AIpXrBSulLZMuItfOZ1FFdTYAM8POrhx2tAkw7GzGw07l2sqbmhjj65HtoAws",
-	"MliGlJ1VAK0OUroZjCMeXJoSX1A5uo8GKCaYSZQy2PyVW7XNwfLboW4nKaxNhktNTIhTTWzBmMZkhucU",
-	"yiHYO5VpKRCTXFMlIWAU2tlHITdQQ6X6pnaG+jWTOLifTxoOHcwWtmHdoH6PMxfRmr8LDr4JVFVlH4jg",
-	"XZspryX269cnXRPAAKGHZmCl+EY3UTMCLSCzLirY/vnv/vjhcURGMO4qVnxcp2Mx3xtuVgWRREkLHp2z",
-	"Q4UJUMBTpqog8nE7Q66cslU/klIGwX42/ANAwWzvtjh+SALYkbK+F8uMfgvmruQNWEr7Ege2fSwMmwLu",
-	"kvx3xWf2Qrg6AONsKJQmNu0U47rNLeFIKm5rWWW7ekSuA0LCKtqk/5W2sfL2S2+s/CtsAWqyqsH2bYh3",
-	"rs+uf3fJUzDWJmoXY/oZZz2AxnBLamEsDC6dBUopM1oJ97oAFzHyYXv6XmiTzUyul9P6N3KtAJw7TCOD",
-	"uOZnXSuq7GG0iuK3zhps2tBcrC6yfwc100y8+a2qptlQ9YconGbfupNiabXVOSfKvXtu2ai5ZH6pykjp",
-	"SssF/LtXyjE2hpW6yB7waDNer7DgzszvFbGIsC3zHxmOySgRZEKvlzCPecEYxmXIkHwjZRkMBtxyLcbX",
-	"aOcJCmZYyMrYGZ3OVLQoB+DseIB8PquioCCKMOcobLPy+Wq6D+vRbnY5i637lOPzAuxOrZ6GVUlHy0Cb",
-	"D/PbNuudT/ACvDiNl4RPtncGg+2twa1Qm92wbkCuw/wTW3+v3E5TSl3hO3vRX4pSLbaQJTDXi7peCQp5",
-	"0BmZpBIEx/uQeJPggKCITAChLSumvfpmsdr18sFbhcqCvWT87xbKrpu7gy/Xa8m6soDXbhodd7lYxvcp",
-	"Pl9xIdogZoIanpsn5267N9h7s7m9v7u3v7l5F0jLGZGasj2efNi8ehJt4clO9HTx5M/N2ZPpVrzttcMu",
-	"qSlL04ZXf9XvNkbZ5IdkGSeoJNLQmp1DQkS1Wm+1yrUkEWWkJ7MMqdVpiktkgbl/X7n/b+bnNzNYqjuc",
-	"lydZVCGwyolT4qyHwbayk1l6d1GdzfHR8lncKgOpOhA/v1WHAuzVbjBQHmGz85moBylreQy9LbzY+iBa",
-	"mhW36ijy3bDDTveucgPFfexdkpOlDbfsAK8fch7f6ZQLqmbx8tMiey3DsIa46Q9ShWUspT46njIo1V38",
-	"OQuTKxpR+uNOtxN92CnvGft7e1QtC3+bMaBd6qJW0CKMDCrBL6cCvJIbHsJEsmtbXY/5p83e5jOIQ4g+",
-	"7Pw06D0rRxx0DbWK5Nt0b5d+HbShYbH+nKtbtPnsRhHXjp7LOOhX6quelp/LFhjX8nheGNkdHS7htrTA",
-	"+ePaGldQchoV0M/V9OzhNioqTSGJ8MIHjF5w1MqK9VhkMjQmU8pkG7/t9iBz3O7Gw04fHVh0arBl8zL4",
-	"peahAHqBT2gck5BqHdOY/s0ZDFstfXFVW+JmhTHcVx5tre9X156thkhYlXC16pjsf0Y+7mdZv+0s3mXo",
-	"HeBXcyYq4HPBi11EJwizSnVMyuY4oqFNpIfESIhX23cgaDnLWhkgcz3Q+Um6aMoVylPoW/rbUtbsF8zG",
-	"T67B37oEM8MwxNYXAUTJwLnoMvF1fIQSwcM0yPNHIxh0jvgh0gr82RIlf3VI7l36NyAxe8IFWu3faHJo",
-	"tPNPNq13xTepGbZ5qTcHq5f6Tpwi3U6ahKtlmHmpnQS7EWz4ihREj4umTPaKJliYzPsWEv2sSMG6zWt8",
-	"yYFWidLEXbBonqpzkue6Ba4YfHG9RyQi+piqN4J4FOZZElTmUnS1SN3cezpruuKEG6n6QH4lJNG2CuAf",
-	"QX8xZgvvwFzNy+wsWRu4erbSXHj1TK0cS63y4J6s1MQal6rowm3C7zdSvuLzNngplwX3d4afXVTN6ggo",
-	"TuCXlLSzZvx5+6ULe2v0H9+FW+4hlbTX9uqhAonqAHsz5HHXfx4LrNW6MvPu+K7nfWzxxlrGTTCv1SzQ",
-	"otf5oPcP42VGo/7+xk9/+3977//q9TZX7GZJRC8kEwg0uiSLnql8o230fhnkFGD3tTI9taxCcAw+JAAQ",
-	"t5uxON7dQSY0Fr/huDYFiNAqlK3ZXDmhv/2lOb6pQMa3ICdXsuxnV6W4i+qdirvjaC0mYupiyV0i2Xp/",
-	"yCA37ZIsJCoUw7IqjWPUH2T2SSECHV0YNbBP2PwCjSlUF5RDpq1aHAQk0daEra9CTYlsDtJHEBwV27FF",
-	"uVzit72QNPEEBL07qSHkvn775vnrt78djV6fvvjt4Hj064v/hRCPq57pIexp3tvZ3bOFsYuU3PQs8Wdg",
-	"+H8WSK2P3QzUpYe/IMkSqo17FGYqASLBhRsUXkZrkKzgKm26XM31m0FvHmQNeoPdvnBBlcGzL1H08O3S",
-	"KodzHvW0Rt0AfO91YBpaeIO1oSkTBN9p8nNPxx610XoTp3SKPa5tnwv7ixQndANamaRTW//GKk3+0Pmj",
-	"alUAIw0MqSoo9hW7VKpec2R9rBWpUV6BvRyvkTKbLkkL4Vzl3MiYqQ1bRNQH0RBywKZeliCb7zIHyNeD",
-	"j1bnfS5V5QszK4ykeW1OnMZa0amXEOhUk+ZqRgQpLAR8kKOt35BkNhWkBfCHKcKWEJGHSbo8Eq0IwQ2n",
-	"RGuZs8GRIEtwrXtgl6Ppn+DrrAfw3mNZu/KCeeTlbjZfPgfk8zNX4o9OXBMwjIo94cf5LnPRMpo4rqov",
-	"RpGr6vM273s3npVVS6Rf096qMGfeR4k1ffz4d0zVz1yABdIMs3HncOFg3YREAOxYFQy8FZI2jUk44qla",
-	"vv9tBXGLsZGVgczLiDprCwMTB6UM0iZZ4IAg8jHUKa3JQYJUULXQ5rlVhseQeedqdwIhoSP4Oe8Y6iV+",
-	"+gR+yoknR+ElYUTQAKpR6v0YYwZKOnp3UihKZurT1SBCQb18fXhsLVyHMgsWC1XAei7U7+D0uNPtzIkw",
-	"Vl5n0N/uD2AzJ4ThhHb2O9v9zf6gA4r8DKa4ARXEbcquTXHNbKXj0GpCz91L+kuBY6Lgi989yecQ6mZf",
-	"B60XTwt2S4KpsIZLEkFSvGEYqr8G9Hh3oO6bU7lryN7aTQeZrZBwQZLXdnHfg1IJewemuTUYWCxtZY9f",
-	"SBcxMeobf9iAxbzfVlqdJZEHTL1mWTjdMiP9p25nZ7B5ozEtGwrsXV/Hbxm2eaMEDMLdGxLiVp0eM5MJ",
-	"ZvN6bQROcccBIxX32u/v9ZrJNI6xWDiCFamVcNmkGBOJsHvXQPwqiQItKqDkSx+9ZsSWzccKYRMsK1IG",
-	"pWTdh5pDy7vAtO0WOcPFec7DxRcjYakPZxZ/KoszvV0+1fj5y/FOxsb1hbSPHKiz4dp7YKDnOKuD/GA7",
-	"ZWfw7O47PeRsEtFAoV7GwDYElkqIMokAotrB3XCB/ky5wiiLIH9EW9rqrOOM3br5UbTxkYafzPaOiM/z",
-	"ekpEjJmJxzfvrNj0te1svOD5dl56qjnGh2oScFI53BdzUIEiV96ixWOrqgzWj6MdT9K/7dNML3xAxt+5",
-	"hx1uJ5uVwnzILQf1FVEqyWPaTvZWZ5wrIV5d7iVRXwvPD+7zyLK49d/gLnosDPySZBpevlq1Q2EjESkz",
-	"BrBXAzzLc+Tsdz+Ulb83+ZNCYAa40nXTUEFBmas8HC76yNHUGP1qAag+gsA8w/qxcqqH97XssK372GEw",
-	"4+xy4vsx9f2YWrbLDbe4KcDGLOzyFj6IG3kgvj3/w429D999D+19D608D4xcWe/CH3zcRzYIEurQyxlP",
-	"oxCNCTIQOy7cQWHRn35AWAQzOieAowZ1wdJI0QQLCGaIUYgVNte2jY6JpW6JrLkN3VzPhb7lBK5CJ0gy",
-	"Aui3URPkYR70RhkjIdKfWLS4HMGuVh3a7H2vgz1rMD8a0dWMS5JByDFVOM0ho1Ya6xia7Q/ZG4stqgkI",
-	"8btO1kgSAULqEv8PZwgPmf3gRydCXOyRxHEuubAAmDpqwBDNstSzqfRIRzLgPniXN4RhpnoyIQGd0MBO",
-	"65IsbAiht8FWpX70gN04351kOQJoa90PEQaIgH482KPsGbKcVL6/YRB3G0RpmF9yOdQaLMY4iry1IKYR",
-	"H+NoZOhzSTx3gi/hDUuUYhV5d5vEeEhMRfBkoWacmb/TccpUav4eC34liRh21vtDBrH/ltYk7OYKIrqC",
-	"2mFxwvU+Ezw2fW6YIW58vCSLT/0hOwhjyhxHwCc4khyRa/gOSioBTIORXg38YHaT/x78MJWKx0WwTcd3",
-	"Zpg8VUmqbBKDJKrrA5ocMsXRRwcn+GnjY97jJ7gsJjjUfFJ4xUwJdOumUcsR1rMfwaue63YCBBh29EE6",
-	"7Oi/pwIzZZAiMzxENC0u6VoGyK836XqVwgFmKOGJKWYATDXDmuVKbQA8AI4ipGAruW+14g4r2TAfi/YW",
-	"jxuh3gw2V2UbUYZOnhc202DnqX8/SRII4oso+e/z178hOJX1GpjX8gghk0XAtMKAwhSuTp1Me4GDGTIX",
-	"VVC/btih4bCTXeeG6zDWVNqE+V4P7hR/0kP7yXTTpeFP/b5uylxX7qPfP5pW9vVeSmIDPTnsfOqiwoMp",
-	"VbN0nD177ydoE2LWeUkQoDVzzK2DJMEUwE0KJ745IjELEbenQLRAGOUSqBi4MqYMi8Wy3DUP6S0F+cQE",
-	"zxWI8XEIwXLDzv7QhcsNO91hh7A5/GZj6oadT34K2FvL5mJpcJ5ll5sZE+0NBuurwZctfT13li0uBr6w",
-	"DdhoFWWVHvUKWuTPb+t+4D/a/syufjDTnedoOMbwd873R3gBUdDYi5ao5wqionZjFpDIqd2rHT33f3mg",
-	"FysgUXTfDPpQ7Jldj2Xg8I+KHWGx8m201H3/wBw3uK9DpeS2fxj+fXT+c4/33PrOydyFOvtLYwDsiTWl",
-	"kXkZYYnOYUy9c218v4Bf+/a/zvYDGL+LiE8v9o3pjiI+RRFlNgS9EKis1QNLS/jIIJ9k31kgFFeXbM1o",
-	"Ev/+579gUJRN//3Pf1k48X//81+w3TcMoheUNb6YESzUmGB1sY9+JSTp4YjOiZsMFB4lcyIWaHtgff7w",
-	"CBWqq1stTQ7ZkJ0RlQpWCNU3JcKkbdBeFej5UJYSaZFj9It0YuuXmNhGj9/G7WVDynvd0V0PAh/MoDAB",
-	"fSo6HgD4MmpqO1tLtON3mZo5l5ym1TDNWrDeavmiyLUy3NszA7yhgAES+/YdPLCTRmvn5y/W+wisLcMV",
-	"UKMGbIe8GWtG9L/LpNUyyUiUskABKhvZZOCUljv9j+w77bz+tsVvye1vy5bdwO9vnD8AquhW4PsdQIs7",
-	"AD/d3H2Azyl/5PDC7i5Y0HTxQLGCjvfqNDdPCiR7CGcAWnNADOBQ5QKdHh4jHIaCSLn+n+0q0DM1XJof",
-	"HYgzKArwELfWdixcWIgqa6qVGeSxiIMzO2qE3byq1SCL59tGqVRE40mXVY3Ij7y7Pz0qnd7kGMkrVua8",
-	"9v0kWRmnR2XAoShWzi29ACdASKe+ZPu0yEWrHFImAjA7cpaqS1Y8Hx+5DXl/rinbdcqqZ8M9CMWjikB8",
-	"QEFYztIs1nh9TNz8NltFh426xHP1dbHm4P60oPv2YvnY/DG5scIK2bQUNHgCjQfoS6IMikDnDhfa9uCZ",
-	"+DkRble7ktww62xa5lNk4BBgQnA1v9z2PTavtDN9TXvfkuUL5LmJxmJJ/l1FaWHs5rRaZuAe24Kld2ff",
-	"Qg83Mm+/3I23ZTAPkSHsZuw81kKREK1huWDB+vdL7y/O0SYkKjdihZs3CVESYQXRkQDEktlZ96XXHbBi",
-	"dVyt01m7ljI0ieh0psz9R0gnEOajitVmYZRb9zDKrKqrwIrY6KbHmDJ4qolsL5DmRCj0+vDY0L94pG58",
-	"hHi31aaSE15LT9e3Z696hAUcAhyz4Dy/TmqffGGDyfB/KQ3w/nfdI0yFo049aFIYP2P9TRwqMqGzfcr/",
-	"a+vniI4FFov/2voZRwll5L+2DyKsiFTrd8Ysg/s66e7bgHnEzKftF1omGogmNgWAwxUKf/ZWS53fvf9N",
-	"qf1m0jdS/DO6ftf92+j+RXItVf/tUtypAWD6eKAbrozZfNSGR9/RMO7BaWo5soCGUbpFyvEwZlwqePT4",
-	"UiNtPCrNOK54bLT0/ucbcunx4Vj3+KgLhIT6p4C/bjOP7ukuwI3j3pVb2+/9XwQcxGM6TXkqi0lNMVbB",
-	"jEib8BeRsgB+bGp3fjw3Kt5fMZcO7vPouHe9+jvf35HGX11QI7zNhd4qnd+91Vbnt+9rnd+AIdqkSAsS",
-	"33UFRNYbYjQdHGJbNi6hRtZjR33j8tki6K02VHJzAYEFsT9k/0fbH78rguP3P7nsq3Qw2NqD3wmbv//J",
-	"JWCxE8cqhEFlc0jEPfjtCG5Rp4AjCSWh8lzP6jhMhVlgPQeC/R9nIOUXye0tJMeF3y2kVhZSgVzLLSS7",
-	"FndrIpWB9O/dRnL85iO4hSP+Nq2kb/x6pGTByXQyoQElDMoRQE6rrMUDGkvu+83ILXMZmb2PLAQTlTSR",
-	"1mZkJrVWaOh5BdR7DyQ7zku+3Lf16IqtPs6cDJ7Y6oXWXsu1hWaD7Wvjh8H9nl73b6g9ZhYzFlGddIlW",
-	"uj1FRUw5nThVEASbwxBBlDESxqzJWuyjwyz7XKZJwoWSpiQPWAimaOdMWwi+8j3lijy+EjxQdoYS2R0y",
-	"KMqqHxsUjY1LsjAFdyhnWW2dbKa2bo0v169c8OhBt9GXV0L91ZxaKaH3vI1tfb6HU0IfTHTci7p3XCp7",
-	"upZtDLC4xyTbyTxLJqUfKJuuP6qIZyOssrkVQNM8qtYGThV3xfs3ZtzgJ/kh5E4jHACCnH7NgBvZ7GSD",
-	"ZlZsClKOBY8iIgxoVZIqV91ryLLBUVaoXmxLaVzo5kcpUzS66JpoGkAekAizhUVuGbJSZ1gpEidasFks",
-	"IhihIIkZcaWsmR405amEt7pI8lKXCEdXeCGHTJBJRAI7NygBKUhg8N2iqI9+4ZDujfAUU2YzkPWbpijY",
-	"D3LILmgYkZHN1r5AVCI540IRRkIU8zmR5X4JFhElAiZxiDXlJIrxAmCTDIKcoQ9PiIEmKuWEc/1vzEIK",
-	"JbN0z9mU94cMo63BAMUEM4kopA1LPCH6K9sGgkGUBvQjwmhn8Mx+VVk3gPZ05F/T+0UIMucBHkcLRDQX",
-	"w4mo1mEBs/0F1Sf18k2okGa9MveirU1UWlgqXRXNsItSFkCRx1Tof3GBUmaPV92igEx4mKe9hCNUZMXR",
-	"bNr+mARY05Pxcj8AmMaDIBW+w1EvdaF833+iklmY3jmQyiedNB0Q7KkQ1pxxNYM9zWErrf/YwFU5U30b",
-	"h4x3k3CBMCrwde5QgErXbIrWAGDsIi8MxlxtyYv1H93e0dvXCgK3/Q3E12M5n4CJ+GRS2oCrjyazgZel",
-	"V9RZ+Fvdp4euImRRxIUUTxmXigZOGFZrFn83Hlsbj8sp6+XmCReXRd2qzL8/c3HZ1vo6d4X4H5URVpzh",
-	"V3gPoIcHELEPfx0AzmhjqGimuXcDrcpf2S4FpYsq6eKMOYo4m+pdlDvF791rX7HogigFtdyZcs4Joo2Q",
-	"kf3RFJXUk7El+8DDH9hWH1oW6d7v4S7oN64QjZOIxASKTvYMs+nFzrRqUxOaSjTLqv3dTFbqXVVMHTa2",
-	"oDTX/12nDgFfuQVbA+29vlxeoRrx6Wq4sKxzh43lwQsbMlOzmrgC1xcok8FaoTXg3OhqRoMZYIeB3arb",
-	"N9BiOEkuMtjU9X30EjZyET0WOl8zkNya1ySPiIEEm8fxxX69rOK7kxP4yMCGmQKKF/vIlVLMzg+p3ypi",
-	"gelZRFgq9JtFOFvLjHFY0QuFtb2ZzW/dooTlsLZD5kMMY+TKNkgn6KIAHnbRgB7m5O0rPn0wZazbDEZu",
-	"5qI4sqYj8CZhYacpxoJGftywzcHAh5HbEsPMDOOOIcxqg3nFpxkQeomVcZK0ZV87TODieRwv4WG0lksQ",
-	"JFXIU/U3qUIiBHxsubuJudEaDmwRHHypGZUZqeQ29jqwnzeSyCATe0mlhWqn2yEsjTv7v9t/zeO40+3Y",
-	"8RQQjW+g3K/Agqs2WI940StTAHz7rpbfBMqtLOwLWG6Vk8Oa080a+Zl54Zu/WXQ+uwdkQ9APKk7cr0kF",
-	"LYy37PBhHEmGEznj6nGhR1lXU0Vra3bVuFn29PDC1FXqaBPCcW4/PXdffgXW76rIDjdm5KZ77yEe9RE8",
-	"5kxYWZvNhIsq5NCq2I+vnpG+3JLUptqGQ77z5s39fK0YU2vqdRFhPwhN5SacKh5jRQOoGhLMOJcFth+T",
-	"GZ5Tbq9K3Z1Vxpng3DB2pg2hv9CsemEdwRdWkd+3TiuEi49sH3343Abe+79wj/Ivfi7Y5ZnE7zrlG5C1",
-	"oayxoGSCEpxKovWqNCYoWARaKpoyMQQHMxTgRKWCQAUsgmLKaJzGBVeDNpzEHANIxcVmfNFF41ShCIsp",
-	"2EXmoQmnFyTgcUxYSMBDNmQzgudUG3UCRVgRFix6kkDlzDlBV1xcRhyHYOTbKBxTeUsQzYGUsy6KicIh",
-	"VhhUjQu940cmi+ciK6ZpDGtGrnNuCIdMpOxHgwaum71wA71ARCo8jqicZUXXAhwSFnihts+/bjH25b3B",
-	"50RVJ/pAcTm3kqUPGahT9Hq64XwdMTyPLBiZC7uMbcT8EqVXNhuR5fQHx0b/mVvazNXN8YGueDISL9vF",
-	"X8fdTsZ0X839zsNf4HCBwtR0V9iVwObf6q1MJlCK4U6QWmmW8bZXM1l1qYzMN5J5Gx/dn8e38KZ9JZKw",
-	"22jYN9UxySf9NYhcS9VbydwHciNaX1LBK/aAItjFVD2Y+sRFQco9FnenFdhma2ZyuyidlMBgfXH2XWxX",
-	"xbYNObit2Ha+2dqlekGQU9aDKE2/BLdu3EZRbV0H/6G5IJXZFUTmg4vI/O7g3sTicSYIjWhM8CLiOPwW",
-	"wnSX3OAEXAiD/wCIEo8Jf7TgNSwG6INvrptJiK7LrXx3crLeJCWEWiojhHrEEqKQFKM/i32l/udECBq6",
-	"EuaHJ0c2YJZKJFLWR69jCnXFLwlJ8pwSAPLo6/k5JIx6MeYS5EW3Q5gSi4RTplaOIn/1bgbz6VYlnO9Z",
-	"TlpA6+8X0q0vpMGz//jEGUgZyJowE1humSqsGkMBXWgcZaZCu9bL8JinunUtgzSZ9HpO4RSc0IjIhVQk",
-	"NnGBkzSC7QbFEWztTPudWeUuRMXqnWMS1hIiYiol5UwOmc3WSIjQfevPdfuFECfvhYDCmXw9NULy6wif",
-	"04MxEWNYNVENMIugcn1nv7OBk2QjxAo3hGjZ4X3GkH6GeDgkF/GYRzRAEWWXEq1F9NKYJ2guUaT/WF8a",
-	"UDeC7750ZdDb7yxN6WM24d7iaYZnM2b+pvKqrFhzF5OPTqy9JMXN4uQPLLRfrMmVck0QHPUUjUmGXINS",
-	"RSP6wYg63QiVigYm6SeHLHh3kqMWDNkJUUK/gyG5LIpIoJzDZiMRPNgYpoPBdpBQgD/bJjA4EHjNj2Po",
-	"8fD0rUkEJTEXi+6Q6X9Aw28OTs3t7gRbb0JhoIyoKy4u0fHG6xUhxudApv/gGD0zwaXYAd4F/34leHNE",
-	"kMY9JBu2KE+WmUo8+eaDSK0G992v8Dj9CgDJlM1mbSpwAEqxnKUq5FfM70OY8yiN9T/MH8ergL0UDmbv",
-	"4NWvRts1w1nZjZvgo9iUdk4hMcUdH+TSwxDsscasasK5KYASU4oG9J4CB+pb5O4v774v0vErvO60FHWF",
-	"U7+avXXfJ58dg8O4KNLjsWxzw2luJoov9z5dYdrsfXoe8eBSWjCUottQ220AMK5/zAGh7RUhqAmQm4ks",
-	"iBAi1wkVgPxWcUAazB2JMFJExJThaAPmbBoBaGvnxcJzTiFFOogoJKnREFCLIkCnu5oRhvRswFHlGijc",
-	"6EpbWqr4TvEyUnE0JgGPiYP7XveZbn/HVP3MRRm7+2uRi28K9Nfz0VPV81wBV97c42fBl5/gawiVDlN7",
-	"oexGtPaS5z8aV1AXwdoMO9sDOex00bCzFQ87egUOMbhQsUK7KKYsVUT20ZHxb0ES7N4ASRJwFkqHOu48",
-	"eNsD2ZQSa9iyIb9yD767T7XHchWQ8sx24hMP+j2kv4ekHbRW3HB2T4Zd2HQh4qky7n67r+xbIVHgHlm/",
-	"97vawh75btu3keR/t9u3JKNglbW4LCy9kexJKmek2eX2ylTySdUY0KxddU05Q3/wsewiRq6MN1xI1a/J",
-	"Pf31qengPpD2dVc3Qdm3c/8Osd8CYj+nlR8u0QRY6iPZcYfBTCTXBhEWu7R3y0NgSQB2Aw9whF4fHg9Z",
-	"oEWRAfcTJOYgnSwguDmFD/5+jl4cnnXREVR6RL+k4/U+es2ihau3be5ohsxoYkZ4BZihseFaEvqOZzN2",
-	"4J67DBbXHTxQ6WSzMzw3K26tXJB4tzMjOASN5GPnFTedeTCCz17pDaTXz36ZLXtnqfLROSNKLHoHE0VE",
-	"vdkTmyfFMtQKe0g7EDiruBnoSd2hdNhneZ9GNzDgFNtbHQ9WxafvVQ/uvkLo/dySmTgRU29unALWJ4Mk",
-	"AxwuHlcsk5yhTDj6RGDxuM7qBjRlCVtZttTAgC6bIr+/Ipf7UtlVQoL/T91dMNNHe9GUlNZJM3FWb2Tl",
-	"Ta9LDp4ZQGJ7URXgBAdULboIR5E9o+xJkEWk9DL1dywIvgz5FesP2VlW6cQm9KLD07ddd1GLQiovTQv2",
-	"LraPXs+JkOk4GxyCjWZujYHmJBwyxVGAoyCNtLpBJhMSQC4uFDCRDXe52VA6d7h38k681VYKUe3poyvy",
-	"5ucJWL2cLaoct2GWekOQIMI0bob/tooaBBxCqMFYN8oZomwS2ZCqQHApkW2qRyI6pePIBgjJPnozI0ji",
-	"mAxZEmHGiECpNFHxeui9RBApU5PgrRsAmFzDUV2UQ/slgisbmhBxLqSJJtAc/u4ESUWSJWx2Zlo+gTnf",
-	"kW5rGrc9PZCTujKGZleIfQXpBTGcYgiu+SiNXADjvYaimwE9tJb4WDb+G0GnUyL0rsBGyJpwPLOtHTnN",
-	"pi9lLDcWfDzP3mpX8DFrtZCVWMjYWwrNNsrRrsPOzaL+PJ1f0kb0PvvoZlnEv+qPWvZdzlb1D8I++sxZ",
-	"fit19M8LSYJtHVg5hz82d1Jh5KWtWkq0XQ2r1Tqz9i4zXVvjZz0YbNZjRsvCpfTZJoP362OEwf2iPNx3",
-	"SbTHzVsltKuSbdqQ8r8az/6r4MC7AbJ/YJSTWwDZf1V594A0/nD4J96N+lB59KW7Z1dt9pvHor+r9HkD",
-	"SA9wbE3p80bq2eDVpYbSO/tOOzPJtvgtafA23vEG+rsj+3erv4XJUCDWqitozfAkTtTCBbTZu8o86EzS",
-	"D6TfcBGcxa3e3VXwLUI6vxx7OD5tDOj8NovDP0jMqC3eRyU6PvJUXX9kGIPFPVc6WDb0qdPDIpjROWl2",
-	"upd3sCVRIkgv4QlcroSGYJYe7ixTWPSnH5Bt3mKu2n9B9UcAyychCqkggYoWphKnlgimjx8kElxbAvCc",
-	"i0VzlIjZIj8LHh/Y2aw4D+2ess6wPM4wXvRCrHBv7qTNEhfaZ0R3unhKLfAQZejlc7RGrpUwNSbQRFs+",
-	"iE4ykppy+xJ4cr044M1Bg2eTfiCj6bjNKJdUC3ltq7GgIJWKx27tj4/QGlQfmxKm10Kr+hPQZBPB5zQk",
-	"YWmMnTmPDFU3Gwh6U7+rViqy0nHOuDCDexAdps2BNP1Ak7JYyEJixpRhGNzKuhzlPWWS+HV/mDIXgGPX",
-	"yI3i+xFmLb81Z+xoToRKmJaIinMD8bz+/Zh7zMdcMRnKnWml086F5yx3XrfLj2qZtnQXhR+y3Ln7dVu/",
-	"+3pSeqh8lNk81nU+zwzSJrf518WCg/s7H+7bXf7uEaeAviTO+C64yqEB3aKPYV5BTHdI5iTiSQwVyeHd",
-	"TreTiqiz35kplexvbEDs94xLtb/z7Ml259P7T/9/AAAA///ULxEss9ABAA==",
+	"Jg5SomEdeHKjZdha4URYOZpbAsLeBwhsVVMpaIhfHPK1eAXnsoQNh7a4issNSW9mBWVmaTSfLJmTL21k",
+	"lAcLFluVCzmxmIdWe8ZOBTLYxBkkQn7RLRdyY5zKjSSgGzYnZQPwKp4CXsWON6E4JPNRmvqsIf3IAZS8",
+	"fXt8hNbgF8BbhbTCMs9ivLf5dPD0We/peHOvtxMONnt4c3uvt7WLB5Pt4Mn25tb2kuSQFhlmt08a8xrJ",
+	"vjL2LpJ75CLKfYG+TfH8FXXExihfURbyq9KJ5w0aLfZuA1JXdV8PN289BG+SClRUhpYahNcJnOsk0G2b",
+	"IO1KkfMG59/em8HmCuffSjEGg2s4Ft6IlJmrRpN3n7nV48KAi4tVHuftpDwMyCWDrKJWsfP2RBvs7z7b",
+	"3/1cormEhlVjrLLTPS5uU5SUwyWuZEy4rL2C68jheHasGmQ83TbBotPtZDkg8Dec/5X44uxxq8Smpg3b",
+	"9YuRZcdKQ0LxcclEgegNg0sX7mvlxJkggP6fpa1rrecw4mmICu43A9MFd1PHBXNFNwNXRdYrZ2A3TYKC",
+	"NmsA5xmqDFCmBTHcyelGbPbxPnoJ78IjHBtLzg7C1LooXkfhcGFiSPT+cl0bu2r5kM+tSQXfaPsK6X/B",
+	"tDUZrJd2eRNGIdtHv3H4JjPwGK+6e83rYFnVX6+6htcshLVDk4DOrHa5j37ONMpMJ7U66Jok9s+RFVg5",
+	"iMt6KZXernhHc0u+coW08G7HULTT7ThCQfp4PZH8bc71tf1XZEVfcBXBkSn0nuXNpopGFroaZkKlooG0",
+	"CRV6cZvUHluGh4QjYy81xWmaZExrU2UfOa3q3QlaA3TCvyHry9b/Ws9iOktn3daznWd7T7ae7bXCIMoH",
+	"uFobPoRU4frgVqrGQZKOrDukaeqHp2+NuyMwjoQsHuTdSRHyIRFcix49c9dgsfNn/WdF6KWQp+OocNlm",
+	"cdoM0issmBddLJNFDbGBf9JoTicT9ueH4HLrD0Hjzes9uTXebICUNR35PW3HxQv3mluajHumpI4fHQcY",
+	"SshGAKkzImEG6JwoBPzTQzgAiybL8LUs52CmLMW9jLWzvb399MnuViu+sqMrbJwR+N08h7IdQWGLwZto",
+	"7ez8HG0UGM606XAWAPWbWWvXv8+QLSs7KCuk/c3Bto9LGg7unGts2/O4keTvrMVoJ2WJDonKmTVZ2+Ve",
+	"am9vD57s7D7dbbeNrUd4JK6XSxiXxmPIY8Hpiyu/Btrkm4NTBEmyExyU3TkuaupGo1I3GhUUVjCA6DcY",
+	"2NMne7s721ub7ZDQfIEYFuOvtGHLssuz6TxM4VkNDynqorfbdFr41CnDYGckiDCNDwKXdlA5fQzw+UiY",
+	"1/JFaHMwWAu8dnC1+LaVPyvzUpmkFaMacIFSlpXb6K++hf0il6nNUtscD6ului9VhWnqWcgeU27rFqRM",
+	"BJlTnsov0BBXJo90EnEubvRtk8FyRmQaKeOzoRK9O/kBZIrmNSQVSco2lOXGJcBGt5zcjfZziUX8TN5E",
+	"rFar0Wbpl02427Bru8tAJkrSoBFOLNSSK2WrAyIPcRSkUGAGZ+upZwW4WJAmnyTRwsS7RxHnDAUzzOBe",
+	"RBRQgNCMR2HfGx2qn4wm3kAKfoUiboCQLwlJbO0VMwj9mVZh6JygtUKaPjKsVKmRuRsbIWOra5S5cTf2",
+	"F/vD0pfAlaWHa3pixQsYveaTkic04lMJRqGCSP5+FRo+wcIE6GNmagnNY2NLeoKQPUOsCHPfiWpOUj6x",
+	"Bq5VOSD52lASB4JLiUhEp1C35t1JJad3SR5Yltm7OsixPNgWrGvuMD1HmUGIal1yzHc+enJcPueEBB6G",
+	"PLol4YPOORljlkI1lgIjk+uECsMe7UIEZ1yqUYbVdMPBSjWCEgupIDmCXJaJnvmD3Dvec9GJttuQy8bi",
+	"3urrGlf5m2oaYLNM9VLUT61uxoM+Nq6jVS0FyMoRt6rwSjcBcMsx+amEVmkBygutQR5IQSwVYNnW28TF",
+	"+E1W3U/NWrWVJF/tDM7bQp0tRzY7xWp2zCbcg4dxg8tS54m2EZwJETGFIiMoJIyS0NmS2a2pdXVBEnUk",
+	"CQpTYiln9FOBLcGx2d5wZ8Wcj4yyaUXWVzts4x42Y1hegQH6tS+2iXCS/iTTNyIFWpkQRYlwnm7aKu6T",
+	"ypH/OqvesCDTNMICWZDCNkOWizii7LJN63IRj3lEA6Q/qF6FT3gU8auRfiR/grmst5qd/mDUVK7n3AzO",
+	"5sqZBan0m0/hJz3L9UqmLnhiNsz3G3Ax2iZgzBs9/TONiEW8e8vodYHRyxDlO1uDpgzyhkZLueN1tMSb",
+	"Sm7Lst4dn0pPvt1SLccVGiKhxY03ak+SSlPzpMWp5IBJ3RXg7W50yskUnwfXcWjkdQWsA40J5MK4qdWl",
+	"Rgux2GYq3nILqZyhP/i47BBtG+nrKeK1xnLYCEEm3ph3WNGlDmnzRo0mhdW9CS4DiFU9UfjohnAHq8qd",
+	"5SFVTfLkrFb5a0Ysyaibo6kC1qKqhQv5yFL6ba/tc/urNdo8IcqA3CLVAqqdQomZRaHwoERjLgSgMmsN",
+	"hzM3G4Ai0TqPprXDgkJvZmSBBIkxZUNGWeYkBYAxghiZE1HIHOVCG1lTEvbR3wsmHuBYx4laWIB0cJ7/",
+	"IBG/YtkYh6w4SN14KnU7B8x4FkUKheXzl3SzYPVpRoFMXrgEUwJqClI1QxNB5Kw4d18dSa3jXXERNhbo",
+	"WSD3CtR9gTtWpPglYUVRljXjNQ1NQyPzVT1wzxSBhafW/kSluqyoWnd1eX+pJMKvJGZTyl5pFbpS2CoF",
+	"48QAowBKCNTbs38ZEZ8hg7TAAcmb/x/XZP7TadZ4+bfKawWsDwe9e2Dctl4XbGAydyrBPuWbtJWhKpAa",
+	"tgzhpX6XgNZc1LQrWlLWBArFQ1qdk+2y06r5AW40G5IE5d53nu4+2WtZveWzLusMotWXvpqbx0uu5BpW",
+	"6qTNvc/T3afPnm3v7D7butENi0slaVifpnSS4vqgNXKt9GaN/v3Pf707qdz67ELY9eBGgzLJJP4hNSSU",
+	"lAf07uTf//yXG9WtB+QTNHXU7IZ7+8Yonai4ki5QoHyF1+6SbIl9f1ByEuBMzKA1MpkQcIOODN16+WAq",
+	"0BjttGCc4ICqhUeQ4ysT4J69UkF/bnMdVB6sT+U1bVukUC25ZDrOEzHXXOfor+ZuuMILT1sXgZLpuOke",
+	"+nW1V3MLnd9aFGMcWoQYyKz+eN3Bnc3nCstSDLf+O4BUC5dUVk+wMW8sR6KtZj9AFIutdVYIBfQhmFf0",
+	"SftRcfkry1m4tyy5daoUf79kHzZvwRt5fT0nssfpG6zOpq3IB3sA3u6r0bhYnm1p/btSLbf81L15vy0S",
+	"huu1A7IT7Ob9FXIkb/JhFScX+NGOwZI8b7tbYokGbiqkv3gcaDwivSxQz+bGIJmaG0G95y30uidpM7jk",
+	"k0kZ/3W3GS8coHAgv8v1gpXSlkkXkWvns6iCTRvcm2FnVw472gQYdjbjYadybeXNmIzx9ch2UMY7GSwD",
+	"8M4Kk1YHKd0MxhEPLk3lMSho3UcDFBPMJEoZbP7KrdrmYPntULeTFNYmg8smJsSpJrZgTGMyw3MKVRrs",
+	"ncq0FIhJrqmSEDAK7eyjkBsEpFLZVTtD/ZrJZ9zPJw2HDmYL27BuUL/HmYtozd8FB98Eir2yD0Twrk3g",
+	"1xL79euTrglggNBDM7BSfKObqBmBFpBZF5WSA/nv/vjhcURGMO4qhH1cp2MxDR1uVgWRREmLaZ2zQ4UJ",
+	"UMBTpqrY9nE7Q66cSVY/klIGwX42/AOwymzvtmZ/SALYkbK+F8uMfgvmruQNWEr7Ege2fSwMmwLukvx3",
+	"xWf2Qrg6AONsKFRMNu0U47rNLeFIKm5LbGW7ekSuA0LCKgim/5W2sfL2S2+s/CtscXOyYsb2bYh3rs+u",
+	"f3c5XTDWJmoXY/oZZz1A7HBLatE1DFyexW8pM1oJjruAYjHyQY76XmiTZE2ul9P6N3KtADM8TCMDBOdn",
+	"XSuq7GG0iuK3TmZs2tBcrK79fwel3Ey8+a2KudlQ9Yeo52bfupMabrXVOSfKvXtu2ai5kn+p+EnpSssF",
+	"/LtXyjE2hpW6yB7waDNer7DgzszvFbFAtS3TMhmOySgRZEKvlzCPecEYxmUkk3wjZRkMBnNzLcbXaOcJ",
+	"CmZYyMrYGZ3OVLQoB+DsePCFPqvQoSCKMOcobLPy+Wq6D+vRbnY5i637lOPzAhpQrcyHVUlHy7CkD/Pb",
+	"NuudT/ACvDiNl4RPtncGg+2twa3ApN2wbkCuw/wTWxaw3E5TSl3hO3vRX4pSLbaQ5VXXa81eCQrp2RmZ",
+	"pBIEx/uQeJPggKCITAA4LktoXX2zWO16+eCtQmWzaDP+dwtl183dwZfLyGRdWRxuN42Ou1wsww4Vn6+4",
+	"EG0QM0ENZs6Tc7fdG+y92dze393b39y8CwDojEhN2R5PPmxePYm28GQnerp48ufm7Ml0K9722mGX1FTL",
+	"acOrv+p3G6Ns8kOyDF9UEmlozc4hIaJaRLhafFuSiDLSk1mG1Oo0xSWywNy/r9z/N/Pzmxks1R3Oy5Ms",
+	"qhBY5cQpcdbDQG7ZySy9u6jO5vho+SxulYFUHYif36pDAfZqNxio2rDZ+UwwhpS1PIbeFl5sfRAtzYpb",
+	"dRT5bthhp3tXuYHiPvYuycnShlt2gNcPOY/vdMoFVbN4+WmRvZZBa0Pc9AepwjLEUx8dTxlUEC/+nIXJ",
+	"FY0o/XGn24k+7JT3jP29PdiXReXNGNAudVEraBFGBgXql1MBXskND2Ei2bWtrsf802Zv8xnEIUQfdn4a",
+	"9J6VIw66hlpF8m26t0u/DtrQsFgWz5VT2nx2o4hrR89lHPQr9RV1y89li9dreTyv1+yODpdwW1rg/HFt",
+	"jSvgPY0K6OdqevZwGxWVppBEeOHDay84amXFeiwyGRqTKWWyjd92e5A5bnfjYaePDixoNtiyeXX+UvNQ",
+	"l73AJzSOSUi1jmlM/+YMhq2WvriqLXGzeh3uK4+21vera89WQySsSrhadUz2PyMf97Os33YW7zL0DvCr",
+	"ORMVYMPgxS6iE4RZpWgnZXMc0dAm0kNiJMSr7TtstpxlrQyQuR7o/CRdNOUK5Sn0Lf1tKWv2C2bjJ9fg",
+	"b12CmWEYYuuLAKJkmGF0mfg6PkKJ4GEa5PmjEQw6R/wQaQWVbYmSvzok9y79G5CYPeECrfZvNDk02vkn",
+	"m9a74pvUDNu81JuD1Ut9J06RbidNwtUyzLzUToLdCM18RQqix0VTJntFEyxM5n0LiX5WpGDd5jW+5ECr",
+	"RGniLlg0T9U5yXPdAlcMvrjeIxIRfUzVG0E8CvMsCSpzKbpapG7uPZ01XXHCjVR9IL8SkmhbBfCPoL8Y",
+	"s4V3YK4UZ3aWrA1cmV1pLrx6poSPpVZ5cE9WamKNS1V04TaVFTBSvuLzNngplwX3dwbrXVTN6ggoTuCX",
+	"lLSzZlh8+6ULe2v0H9+FW+4hlbTX9uqhgtTqcIQzQHTXfx4LrNW6MvPu+K7nfWzxxlrGTeiz1SzQotf5",
+	"oPcP42VGo/7+xk9/+3977//q9TZX7GZJRC8kEwg0uiSLninIo230fhl7FaoBaGV6almF4Bh8SIBrbjdj",
+	"cby7g0xoLH7DcW0KEKFVqKazuXJCf/tLc3xTgYxvQU6uZNnPLpZxF0VFFXfH0VpMxNTFkrtEsvX+kEFu",
+	"2iVZSFSo0WVVGseoP8jsk0IEOrowamCfsPkFGlMoeiiHTFu1OAhIoq0JW/aFmsrdHKSPIDgqtmNrhbnE",
+	"b3shaeIJCHp3UgPuff32zfPXb387Gr0+ffHbwfHo1xf/CyEeVz3TQ9jTvLezu2frdRcpuelZ4s8oLfBZ",
+	"2Lk+djMInB7+giRLKILuUZipBIgEF25QeBmtQbKCKwDqcjXXb4YIepA16A12+8J1XgbPvkQtxrdLiy/O",
+	"edTTGnUDHr/XgWlo4Q3WhqZMEHynyc89HXvURutNnNIp9ri2fS7sL1Iz0Q1oZZJObf0bi0f5Q+ePqsUK",
+	"jDQwpKqA61fsUql6zZH1sVakGrBeAb/WpEvSQjhXOTcyZmrD1jb1QTSEHCCzlyXI5rvMAfL14KPVeZ9L",
+	"VfnCzAojaV6bE6exVnTqJQQ61aS5mhFBCgsBH+Qg8DckmU0FaQH8YWrDJUTkYZIuj0QrQnDDKdFa5mxw",
+	"JMgSXOse2OUg/yf4OusBvPdY1q68YB55FZ7Nl88BkP3MVR6kE9cEDKNiT/jhx8tctIwmjqvqi1Hkqvq8",
+	"zfvejWdl1RLp17S3KsyZ91FiTR8//h1T9TMXYIE0w2zcOYo5WDchEQA7VsUobwXwTWMSjniqlu9/W9jc",
+	"Ymxk1Snz6qbO2sLAxEEpg7RJFjggiHwMdUprcpAgFVQttHluleExZN65kqJASOgIfs47hjKOnz6Bn3Li",
+	"yVF4SRgRNIAimXo/xpiBko7enRRqpZmyeTWIUFAvXx8eWwvXocyCxUIVsJ4L9Ts4Pe50O3MijJXXGfS3",
+	"+wPYzAlhOKGd/c52f7M/6IAiP4MpbkBhc5uya1NcM1vpOLSa0HP3kv5S4Jgo+OJ3T/I5hLrZ10HrxdOC",
+	"3ZJgKqzhkkSQFG8YhuqvAdTeHaj75lTuGrK3dtNBZiskXJDktV3c96BUwt6BaW4NBhZLW9njF9JFTIz6",
+	"xh82YDHvt5VWZ0nkwXivWRZOt8xI/6nb2Rls3mhMy4YCe9fX8VuGbd4oAYNw94aEuFWnx8xkgtm8XhuB",
+	"U9xxwEjFvfb7e71mMo1jLBaOYEVqJVw2KcZEIuzeNRC/SqJAiwqoRNNHrxmx1fyxQtgEy4qUQYVb96Hm",
+	"0PIuMG27Rc5wcZ7zcPHFSFjqw5nFn8riTG+XTzV+/nK8k7FxfSHtIwfqbLj2HhjoOc7KMz/YTtkZPLv7",
+	"Tg85m0Q0UKiXMbANgaUSokwigKh2cDdcoD9TrjDKIsgf0Za2Ous4Y7dufhRtfKThJ7O9I+LzvJ4SEWNm",
+	"4vHNOys2fW07Gy94vp2XnmqO8Y+POvakcrgv5qACRa68RYvHVlUZrB9HO56kf9unmV74gIy/cw873E42",
+	"q9D5kFsOyj6iVJLHtJ3src44V0K8utxLor4Wnh/c55Flceu/wV30WBj4Jck0vHy1aofCRiJSZgxgrwZ4",
+	"lufI2e9+KCt/b/InhcAMcKXrpqGCgjJXeThc9JGjqTH61QJQfQSBeYb1Y+VUD+9r2WFb97HDYMbZ5cT3",
+	"Y+r7MbVslxtucVOAjVnY5S18EDfyQHx7/ocbex+++x7a+x5aeR4YubLehT/4uI9sECSUx5cznkYhGhNk",
+	"IHZcuIPCoj/9gLAIZnROAEcN6oKlkaIJFhDMEKMQK2yubRsdE0vdEllzG7q5ngt9ywlchU6QZATQb6Mm",
+	"yMM86I0yRkKkP7FocTmCXa1otdn7Xgd71mB+NKKrGZckg5BjqnCaQ0atNNYxNNsfsjcWW1QTEOJ3nayR",
+	"JAKE1CX+H84QHjL7wY9OhLjYI4njXHJhATB11IAhmmWpZ1PpkY5kwH3wLm8Iw0z1ZEICOqGBndYlWdgQ",
+	"Qm+DrUr96AG7cb47yXIE0Na6HyIMEAH9eLBH2TNkOal8f8Mg7jaI0jC/5HKoNViMcRR5a0FMIz7G0cjQ",
+	"55J47gRfwhuWKMXi9u42ifGQmELlyULNODN/p+OUqdT8PRb8ShIx7Kz3hwxi/y2tSdjNFUR0BbXD4oTr",
+	"fSZ4bPrcMEPc+HhJFp/6Q3YQxpQ5joBPcCQ5ItfwHZRUApgGI70a+MHsJv89+GEqFY+LYJuO78wweaqS",
+	"VNkkBklU1wc0OWSKo48OTvDTxse8x09wWUxwqPmk8IqZEujWTaOWI6xnP4JXPdftBAgw7OiDdNjRf08F",
+	"ZsogRWZ4iGhaXNK1DJAfinRWKRxghhKemGIGwFQzrFmu1AbAA+AoQgq2kvtWK+6wkg3zsWhv8bgR6s1g",
+	"c1W2EWXo5HlhMw12nvr3kySBIL6Ikv8+f/0bglNZr4F5LY8QMlkETCsMKEzh6tTJtBc4mCFzUQX164Yd",
+	"Gg472XVuuA5jTaVNmO/14E7xJz20n0w3XRr+1O/rpsx15T76/aNpZV/vpSQ20JPDzqcuKjyYUjVLx9mz",
+	"936CNiFmnZcEAVozx9w6SBJMAdykcOKbIxKzEHF7CkQLhFEugYqBK2PKsFgsy13zkN5SkE9M8FyBGB+H",
+	"ECw37OwPXbjcsNMddgibw282pm7Y+eSngL21bC6WBudZdrmZMdHeYLC+GnzZ0tdzZ9niYuAL24CNVlFW",
+	"6VGvoEX+/LbuB/6j7c/s6gcz3XmOhmMMf+d8f4QXEAWNvWiJeq4gKmo3ZgGJnNq92tFz/5cHerECEkX3",
+	"zaAPxZ7Z9VgGDv+o2BEWK99GS933D8xxg/s6VEpu+4fh30fnP/d4z63vnMxdqLO/NAbAnlhTGpmXEZbo",
+	"HMbUO9fG9wv4tW//62w/gPG7iPj0Yt+Y7ijiUxRRZkPQC4HKWj2wtISPDPJJ9p0FQnF1ydaMJvHvf/4L",
+	"BkXZ9N///JeFE//3P/8F233DIHpBWeOLGcFCjQlWF/voV0KSHo7onLjJQOFRMidigbYH1ucPj1ChurrV",
+	"0uSQDdkZUalghVB9UyJM2gbtVYGeD2UpkRY5Rr9IJ7Z+iYlt9Pht3F42pLzXHd31IPDBDAoT0Kei4wGA",
+	"L6OmtrO1RDt+l6mZc8lpWg3TrAXrrZYvilwrw709M8AbChggsW/fwQM7abR2fv5ivY/A2jJcATVqwHbI",
+	"m7FmRP+7TFotk4xEKQsUoLKRTQZOabnT/8i+087rb1v8ltz+tmzZDfz+xvkDoIpuBb7fAbS4A/DTzd0H",
+	"+JzyRw4v7O6CBU0XDxQr6HivTnPzpECyh3AGoDUHxAAOVS7Q6eExwmEoiJTr/9muAj1Tw6X50YE4g6IA",
+	"D3FrbcfChYWosqZamUEeizg4s6NG2M2rWg2yeL5tlEpFNJ50WdWI/Mi7+9Oj0ulNjpG8YmXOa99PkpVx",
+	"elQGHIpi5dzSC3AChHTqS7ZPi1y0yiFlIgCzI2epumTF8/GR25D355qyXaesejbcg1A8qgjEBxSE5SzN",
+	"Yo3Xx8TNb7NVdNioSzxXXxdrDu5PC7pvL5aPzR+TGyuskE1LQYMn0HiAviTKoAh07nChbQ+eiZ8T4Xa1",
+	"K8kNs86mZT5FBg4BJgRX88tt32PzSjvT17T3LVm+QJ6baCyW5N9VlBbGbk6rZQbusS1Yenf2LfRwI/P2",
+	"y914WwbzEBnCbsbOYy0UCdEalgsWrH+/9P7iHG1ConIjVrh5kxAlEVYQHQlALJmddV963QErVsfVOp21",
+	"aylDk4hOZ8rcf4R0AmE+qlhtFka5dQ+jzKq6CqyIjW56jCmDp5rI9gJpToRCrw+PDf2LR+rGR4h3W20q",
+	"OeG19HR9e/aqR1jAIcAxC87z66T2yRc2mAz/l9IA73/XPcJUOOrUgyaF8TPW38ShIhM626f8v7Z+juhY",
+	"YLH4r62fcZRQRv5r+yDCiki1fmfMMrivk+6+DZhHzHzafqFlooFoYlMAOFyh8GdvtdT53fvflNpvJn0j",
+	"xT+j63fdv43uXyTXUvXfLsWdGgCmjwe64cqYzUdtePQdDeMenKaWIwtoGKVbpBwPY8algkePLzXSxqPS",
+	"jOOKx0ZL73++IZceH451j4+6QEiofwr46zbz6J7uAtw47l25tf3e/0XAQTym05SnspjUFGMVzIi0CX8R",
+	"KQvgx6Z258dzo+L9FXPp4D6PjnvXq7/z/R1p/NUFNcLbXOit0vndW211fvu+1vkNGKJNirQg8V1XQGS9",
+	"IUbTwSG2ZeMSamQ9dtQ3Lp8tgt5qQyU3FxBYEPtD9n+0/fG7Ijh+/5PLvkoHg609+J2w+fufXAIWO3Gs",
+	"QhhUNodE3IPfjuAWdQo4klASKs/1rI7DVJgF1nMg2P9xBlJ+kdzeQnJc+N1CamUhFci13EKya3G3JlIZ",
+	"SP/ebSTHbz6CWzjib9NK+savR0oWnEwnExpQwqAcAeS0ylo8oLHkvt+M3DKXkdn7yEIwUUkTaW1GZlJr",
+	"hYaeV0C990Cy47zky31bj67Y6uPMyeCJrV5o7bVcW2g22L42fhjc7+l1/4baY2YxYxHVSZdopdtTVMSU",
+	"04lTBUGwOQwRRBkjYcyarMU+Osyyz2WaJFwoaUrygIVginbOtIXgK99TrsjjK8EDZWcokd0hg6Ks+rFB",
+	"0di4JAtTcIdyltXWyWZq69b4cv3KBY8edBt9eSXUX82plRJ6z9vY1ud7OCX0wUTHvah7x6Wyp2vZxgCL",
+	"e0yyncyzZFL6gbLp+qOKeDbCKptbATTNo2pt4FRxV7x/Y8YNfpIfQu40wgEgyOnXDLiRzU42aGbFpiDl",
+	"WPAoIsKAViWpctW9hiwbHGWF6sW2lMaFbn6UMkWji66JpgHkAYkwW1jkliErdYaVInGiBZvFIoIRCpKY",
+	"EVfKmulBU55KeKuLJC91iXB0hRdyyASZRCSwc4MSkIIEBt8tivroFw7p3ghPMWU2A1m/aYqC/SCH7IKG",
+	"ERnZbO0LRCWSMy4UYSREMZ8TWe6XYBFRImASh1hTTqIYLwA2ySDIGfrwhBhoolJOONf/xiykUDJL95xN",
+	"eX/IMNoaDFBMMJOIQtqwxBOiv7JtIBhEaUA/Iox2Bs/sV5V1A2hPR/41vV+EIHMe4HG0QERzMZyIah0W",
+	"MNtfUH1SL9+ECmnWK3Mv2tpEpYWl0lXRDLsoZQEUeUyF/hcXKGX2eNUtCsiEh3naSzhCRVYczabtj0mA",
+	"NT0ZL/cDgGk8CFLhOxz1UhfK9/0nKpmF6Z0DqXzSSdMBwZ4KYc0ZVzPY0xy20vqPDVyVM9W3cch4NwkX",
+	"CKMCX+cOBah0zaZoDQDGLvLCYMzVlrxY/9HtHb19rSBw299AfD2W8wmYiE8mpQ24+mgyG3hZekWdhb/V",
+	"fXroKkIWRVxI8ZRxqWjghGG1ZvF347G18bicsl5unnBxWdStyvz7MxeXba2vc1eI/1EZYcUZfoX3AHp4",
+	"ABH78NcB4Iw2hopmmns30Kr8le1SULqoki7OmKOIs6neRblT/N699hWLLohSUMudKeecINoIGdkfTVFJ",
+	"PRlbsg88/IFt9aFlke79Hu6CfuMK0TiJSEyg6GTPMJte7EyrNjWhqUSzrNrfzWSl3lXF1GFjC0pz/d91",
+	"6hDwlVuwNdDe68vlFaoRn66GC8s6d9hYHrywITM1q4krcH2BMhmsFVoDzo2uZjSYAXYY2K26fQMthpPk",
+	"IoNNXd9HL2EjF9FjofM1A8mteU3yiBhIsHkcX+zXyyq+OzmBjwxsmCmgeLGPXCnF7PyQ+q0iFpieRYSl",
+	"Qr9ZhLO1zBiHFb1QWNub2fzWLUpYDms7ZD7EMEaubIN0gi4K4GEXDehhTt6+4tMHU8a6zWDkZi6KI2s6",
+	"Am8SFnaaYixo5McN2xwMfBi5LTHMzDDuGMKsNphXfJoBoZdYGSdJW/a1wwQunsfxEh5Ga7kEQVKFPFV/",
+	"kyokQsDHlrubmBut4cAWwcGXmlGZkUpuY68D+3kjiQwysZdUWqh2uh3C0riz/7v91zyOO92OHU8B0fgG",
+	"yv0KLLhqg/WIF70yBcC372r5TaDcysK+gOVWOTmsOd2skZ+ZF775m0Xns3tANgT9oOLE/ZpU0MJ4yw4f",
+	"xpFkOJEzrh4XepR1NVW0tmZXjZtlTw8vTF2ljjYhHOf203P35Vdg/a6K7HBjRm669x7iUR/BY86ElbXZ",
+	"TLioQg6tiv346hnpyy1JbaptOOQ7b97cz9eKMbWmXhcR9oPQVG7CqeIxVjSAqiHBjHNZYPsxmeE55faq",
+	"1N1ZZZwJzg1jZ9oQ+gvNqhfWEXxhFfl967RCuPjI9tGHz23gvf8L9yj/4ueCXZ5J/K5TvgFZG8oaC0om",
+	"KMGpJFqvSmOCgkWgpaIpE0NwMEMBTlQqCFTAIiimjMZpXHA1aMNJzDGAVFxsxhddNE4VirCYgl1kHppw",
+	"ekECHseEhQQ8ZEM2I3hOtVEnUIQVYcGiJwlUzpwTdMXFZcRxCEa+jcIxlbcE0RxIOeuimCgcYoVB1bjQ",
+	"O35ksngusmKaxrBm5DrnhnDIRMp+NGjgutkLN9ALRKTC44jKWVZ0LcAhYYEXavv86xZjX94bfE5UdaIP",
+	"FJdzK1n6kIE6Ra+nG87XEcPzyIKRubDL2EbML1F6ZbMRWU5/cGz0n7mlzVzdHB/oiicj8bJd/HXc7WRM",
+	"99Xc7zz8BQ4XKExNd4VdCWz+rd7KZAKlGO4EqZVmGW97NZNVl8rIfCOZt/HR/Xl8C2/aVyIJu42GfVMd",
+	"k3zSX4PItVS9lcx9IDei9SUVvGIPKIJdTNWDqU9cFKTcY3F3WoFttmYmt4vSSQkM1hdn38V2VWzbkIPb",
+	"im3nm61dqhcEOWU9iNL0S3Drxm0U1dZ18B+aC1KZXUFkPriIzO8O7k0sHmeC0IjGBC8ijsNvIUx3yQ1O",
+	"wIUw+A+AKPGY8EcLXsNigD745rqZhOi63Mp3JyfrTVJCqKUyQqhHLCEKSTH6s9hX6n9OhKChK2F+eHJk",
+	"A2apRCJlffQ6plBX/JKQJM8pASCPvp6fQ8KoF2MuQV50O4QpsUg4ZWrlKPJX72Ywn25Vwvme5aQFtP5+",
+	"Id36Qho8+49PnIGUgawJM4HllqnCqjEU0IXGUWYqtGu9DI95qlvXMkiTSa/nFE7BCY2IXEhFYhMXOEkj",
+	"2G5QHMHWzrTfmVXuQlSs3jkmYS0hIqZSUs7kkNlsjYQI3bf+XLdfCHHyXggonMnXUyMkv47wOT0YEzGG",
+	"VRPVALMIKtd39jsbOEk2QqxwQ4iWHd5nDOlniIdDchGPeUQDFFF2KdFaRC+NeYLmEkX6j/WlAXUj+O5L",
+	"Vwa9/c7SlD5mE+4tnmZ4NmPmbyqvyoo1dzH56MTaS1LcLE7+wEL7xZpcKdcEwVFP0ZhkyDUoVTSiH4yo",
+	"041QqWhgkn5yyIJ3JzlqwZCdECX0OxiSy6KIBMo5bDYSwYONYToYbAcJBfizbQKDA4HX/DiGHg9P35pE",
+	"UBJzsegOmf4HNPzm4NTc7k6w9SYUBsqIuuLiEh1vvF4RYnwOZPoPjtEzE1yKHeBd8O9XgjdHBGncQ7Jh",
+	"i/JkmanEk28+iNRqcN/9Co/TrwCQTNls1qYCB6AUy1mqQn7F/D6EOY/SWP/D/HG8CthL4WD2Dl79arRd",
+	"M5yV3bgJPopNaecUElPc8UEuPQzBHmvMqiacmwIoMaVoQO8pcKC+Re7+8u77Ih2/wutOS1FXOPWr2Vv3",
+	"ffLZMTiMiyI9Hss2N5zmZqL4cu/TFabN3qfnEQ8upQVDKboNtd0GAOP6xxwQ2l4RgpoAuZnIggghcp1Q",
+	"AchvFQekwdyRCCNFREwZjjZgzqYRgLZ2Xiw85xRSpIOIQpIaDQG1KAJ0uqsZYUjPBhxVroHCja60paWK",
+	"7xQvIxVHYxLwmDi473Wf6fZ3TNXPXJSxu78WufimQH89Hz1VPc8VcOXNPX4WfPkJvoZQ6TC1F8puRGsv",
+	"ef6jcQV1EazNsLM9kMNOFw07W/Gwo1fgEIMLFSu0i2LKUkVkHx0Z/xYkwe4NkCQBZ6F0qOPOg7c9kE0p",
+	"sYYtG/Ir9+C7+1R7LFcBKc9sJz7xoN9D+ntI2kFrxQ1n92TYhU0XIp4q4+63+8q+FRIF7pH1e7+rLeyR",
+	"77Z9G0n+d7t9SzIKVlmLy8LSG8mepHJGml1ur0wln1SNAc3aVdeUM/QHH8suYuTKeMOFVP2a3NNfn5oO",
+	"7gNpX3d1E5R9O/fvEPstIPZzWvnhEk2ApT6SHXcYzERybRBhsUt7tzwElgRgN/AAR+j14fGQBVoUGXA/",
+	"QWIO0skCgptT+ODv5+jF4VkXHUGlR/RLOl7vo9csWrh62+aOZsiMJmaEV4AZGhuuJaHveDZjB+65y2Bx",
+	"3cEDlU42O8Nzs+LWygWJdzszgkPQSD52XnHTmQcj+OyV3kB6/eyX2bJ3liofnTOixKJ3MFFE1Js9sXlS",
+	"LEOtsIe0A4GzipuBntQdSod9lvdpdAMDTrG91fFgVXz6XvXg7iuE3s8tmYkTMfXmxilgfTJIMsDh4nHF",
+	"MskZyoSjTwQWj+usbkBTlrCVZUsNDOiyKfL7K3K5L5VdJST4/9TdBTN9tBdNSWmdNBNn9UZW3vS65OCZ",
+	"ASS2F1UBTnBA1aKLcBTZM8qeBFlESi9Tf8eC4MuQX7H+kJ1llU5sQi86PH3bdRe1KKTy0rRg72L76PWc",
+	"CJmOs8Eh2Gjm1hhoTsIhUxwFOArSSKsbZDIhAeTiQgET2XCXmw2lc4d7J+/EW22lENWeProib36egNXL",
+	"2aLKcRtmqTcECSJM42b4b6uoQcAhhBqMdaOcIcomkQ2pCgSXEtmmeiSiUzqObICQ7KM3M4IkjsmQJRFm",
+	"jAiUShMVr4feSwSRMjUJ3roBgMk1HNVFObRfIriyoQkR50KaaALN4e9OkFQkWcJmZ6blE5jzHem2pnHb",
+	"0wM5qStjaHaF2FeQXhDDKYbgmo/SyAUw3msouhnQQ2uJj2XjvxF0OiVC7wpshKwJxzPb2pHTbPpSxnJj",
+	"wcfz7K12BR+zVgtZiYWMvaXQbKMc7Trs3Czqz9P5JW1E77OPbpZF/Kv+qGXf5WxV/yDso8+c5bdSR/+8",
+	"kCTY1oGVc/hjcycVRl7aqqVE29WwWq0za+8y07U1ftaDwWY9ZrQsXEqfbTJ4vz5GGNwvysN9l0R73LxV",
+	"Qrsq2aYNKf+r8ey/Cg68GyD7B0Y5uQWQ/VeVdw9I4w+Hf+LdqA+VR1+6e3bVZr95LPq7Sp83gPQAx9aU",
+	"Pm+kng1eXWoovbPvtDOTbIvfkgZv4x1voL87sn+3+luYDAVirbqC1gxP4kQtXECbvavMg84k/UD6DRfB",
+	"Wdzq3V0F3yKk88uxh+PTxoDOb7M4/IPEjNrifVSi4yNP1fVHhjFY3HOlg2VDnzo9LIIZnZNmp3t5B1sS",
+	"JYL0Ep7A5UpoCGbp4c4yhUV/+gHZ5i3mqv0XVH8EsHwSopAKEqhoYSpxaolg+vhBIsG1JQDPuVg0R4mY",
+	"LfKz4PGBnc2K89DuKesMy+MM40UvxAr35k7aLHGhfUZ0p4un1AIPUYZePkdr5FoJU2MCTbTlg+gkI6kp",
+	"ty+BJ9eLA94cNHg26Qcymo7bjHJJtZDXthoLClKpeOzW/vgIrUH1sSlhei20qj8BTTYRfE5DEpbG2Jnz",
+	"yFB1s4GgN/W7aqUiKx3njAszuAfRYdocSNMPNCmLhSwkZkwZhsGtrMtR3lMmiV/3hylzATh2jdwovh9h",
+	"1vJbc8aO5kSohGmJqDg3EM/r34+5x3zMFZOh3JlWOu1ceM5y53W7/KiWaUt3Ufghy527X7f1u68npYfK",
+	"R5nNY13n88wgbXKbf10sOLi/8+G+3eXvHnEK6EvijO+Cqxwa0C36GOYVxHSHZE4insRQkRze7XQ7qYg6",
+	"+52ZUsn+xgbEfs+4VPs7z55sdz69//T/BwAA//8MF3pPStEBAA==",
 }
 
 // GetSwagger returns the content of the embedded swagger specification file
diff --git a/openapi.yaml b/openapi.yaml
index f7d051c5..deee103e 100644
--- a/openapi.yaml
+++ b/openapi.yaml
@@ -1567,8 +1567,12 @@ components:
           example: "L40S-1Q"
         mdev_uuid:
           type: string
-          description: mdev device UUID
+          description: mdev device UUID (mdev hosts only)
           example: "aa618089-8b16-4d01-a136-25a0f3c73123"
+        device_path:
+          type: string
+          description: sysfs path of the assigned vGPU device
+          example: "/sys/bus/pci/devices/0000:82:00.4"
     
     GPUProfile:
       type: object

From c196c8753d1931328cf7f4be7d17a6397f3ddea7 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Tue, 11 Aug 2026 17:53:21 +0000
Subject: [PATCH 32/38] Surface retained rollback assignments from start as
 vgpu_cleanup_pending

When a later start step failed and rollback could not destroy the
freshly created vGPU, cleanupStartVGPU retained the assignment on disk
but startInstance returned the original failure untyped, so the API
reported a generic error instead of vgpu_cleanup_pending with the
retained-assignment guidance. Mirror create's named-return wrap:
cleanupStartVGPU reports retention state and start wraps the returned
error in VGPUCleanupPendingError.
---
 lib/instances/start.go     | 12 +++++++++++-
 lib/instances/vgpu.go      | 10 +++++++++-
 lib/instances/vgpu_test.go |  4 ++++
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/lib/instances/start.go b/lib/instances/start.go
index b44a68d5..1045829a 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -116,6 +116,16 @@ func (m *manager) startInstance(
 	}
 
 	// Setup cleanup stack for automatic rollback on errors
+	// Registered before cu.Clean so it runs after cleanup and can report a
+	// vGPU assignment that rollback failed to destroy, matching create's
+	// vgpu_cleanup_pending contract.
+	vgpuRetained := false
+	vgpuRetentionPersisted := false
+	defer func() {
+		if retErr != nil && vgpuRetained {
+			retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuRetentionPersisted, Err: retErr}
+		}
+	}()
 	cu := cleanup.Make(func() {})
 	defer cu.Clean()
 
@@ -189,7 +199,7 @@ func (m *manager) startInstance(
 		log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID)
 		// Add vGPU cleanup to stack
 		cu.Add(func() {
-			m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta)
+			vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta)
 		})
 		// Checked after the cleanup handler is registered so rejection
 		// releases the device through the normal rollback.
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 8c635174..1e3dcc35 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -105,7 +105,12 @@ func clearStoredVGPUDevice(stored *StoredMetadata) {
 // start. The snapshot is also a shallow copy (Phases shares its map), so it
 // must be persisted before any Phases.Record on the live struct. Violating
 // either invariant requires switching to targeted field restores.
-func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) {
+//
+// It reports whether the assignment was retained after a failed destroy and
+// whether that retention record was persisted, so start can surface the
+// pending cleanup as a typed error like create does.
+func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) {
+	logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID)
 	assignment := devices.VGPUAssignment{
 		Framework:  device.Framework,
 		DevicePath: device.SysfsPath,
@@ -117,6 +122,7 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic
 	if releaseErr != nil {
 		logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr)
 		setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt)
+		retained = true
 	}
 	if err := m.saveMetadata(&cleanupMeta); err != nil {
 		message := "failed to save metadata after vGPU cleanup"
@@ -124,7 +130,9 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic
 			message = "failed to retain vGPU assignment metadata after cleanup failure"
 		}
 		logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err)
+		return retained, false
 	}
+	return retained, retained
 }
 
 func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error {
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index d466f188..7668ff90 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -389,6 +389,10 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) {
 	t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing"))
 	_, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}})
 	require.Error(t, err)
+	var pending *VGPUCleanupPendingError
+	require.ErrorAs(t, err, &pending, "a retained rollback assignment must surface as vgpu_cleanup_pending")
+	assert.Equal(t, id, pending.InstanceID)
+	assert.True(t, pending.Retained)
 
 	stored, err := m.loadMetadata(id)
 	require.NoError(t, err)

From e868222bcbad67a3b18bc36e90364d99e5fc8896 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Tue, 11 Aug 2026 18:07:37 +0000
Subject: [PATCH 33/38] Report retention as persisted when the mid-start save
 survives

When start rollback fails to destroy a vGPU and the cleanup metadata
save also fails, the assignment may still be on disk from the mid-start
save. Reporting Retained: false then misdirects callers to wait for
startup reconcile when delete or a retried start can already release
it. Check whether the surviving record still points at the device,
matching create's retention-survives check.
---
 lib/instances/vgpu.go      | 12 +++++++++++-
 lib/instances/vgpu_test.go | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 1e3dcc35..330e2d82 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -130,7 +130,17 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic
 			message = "failed to retain vGPU assignment metadata after cleanup failure"
 		}
 		logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err)
-		return retained, false
+		if !retained {
+			return false, false
+		}
+		// The mid-start save may already have persisted this assignment, in
+		// which case the on-disk record still points at the device and
+		// delete or a retried start can release it (matching create's
+		// retention-survives check).
+		if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath {
+			return true, true
+		}
+		return true, false
 	}
 	return retained, retained
 }
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 7668ff90..b34fe242 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -405,6 +405,38 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) {
 	assert.Empty(t, stored.Entrypoint)
 }
 
+func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("root bypasses directory permissions")
+	}
+
+	m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error {
+		return errors.New("destroy failed")
+	})
+	device := devices.VGPUDevice{
+		Framework: devices.VGPUFrameworkVendorVFIO,
+		SysfsPath: "/sys/bus/pci/devices/0000:82:00.4",
+	}
+	assignedAt := time.Now().UTC()
+
+	// The mid-start save already persisted the assignment.
+	meta, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	rollbackMeta := *meta
+	setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt)
+	require.NoError(t, m.saveMetadata(meta))
+
+	// The cleanup save fails, but the surviving on-disk record still points
+	// at the device, so retention must be reported as persisted.
+	instanceDir := filepath.Dir(m.paths.InstanceMetadata(id))
+	require.NoError(t, os.Chmod(instanceDir, 0o555))
+	t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) })
+
+	retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta)
+	assert.True(t, retained)
+	assert.True(t, persisted, "a surviving mid-start save keeps the assignment recoverable via delete")
+}
+
 func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) {
 	m := &manager{
 		paths: paths.New(t.TempDir()),

From fc896f139483f11b9eb50c3cb3db9902e2663f88 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Tue, 11 Aug 2026 19:07:40 +0000
Subject: [PATCH 34/38] Leave vGPU hypervisor selection to callers

Drop the vendor-VFIO-on-Cloud-Hypervisor rejection from create and
start, restoring the phase-0 decision that hypervisor selection is
caller policy: production callers pin vGPU instances to QEMU, and the
Cloud Hypervisor limitation stays documented in lib/devices/GPU.md.
---
 lib/instances/create.go    |  6 ------
 lib/instances/start.go     |  6 ------
 lib/instances/vgpu.go      | 13 -------------
 lib/instances/vgpu_test.go | 29 -----------------------------
 4 files changed, 54 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index 66f6ebb3..510f179a 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -342,12 +342,6 @@ func (m *manager) createInstance(
 				}
 			}
 		})
-		// Checked after the cleanup handler is registered so rejection
-		// releases the device through the normal rollback.
-		if err := validateVGPUHypervisorCompat(gpuDevice.Framework, hvType); err != nil {
-			log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", gpuDevice.Framework, "hypervisor", hvType)
-			return nil, err
-		}
 	}
 
 	if len(req.Devices) > 0 && m.deviceManager != nil {
diff --git a/lib/instances/start.go b/lib/instances/start.go
index 1045829a..3e164b39 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -201,12 +201,6 @@ func (m *manager) startInstance(
 		cu.Add(func() {
 			vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta)
 		})
-		// Checked after the cleanup handler is registered so rejection
-		// releases the device through the normal rollback.
-		if err := validateVGPUHypervisorCompat(device.Framework, stored.HypervisorType); err != nil {
-			log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", device.Framework, "hypervisor", stored.HypervisorType)
-			return nil, err
-		}
 		if err := m.saveMetadata(meta); err != nil {
 			log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err)
 			return nil, fmt.Errorf("save metadata after vGPU creation: %w", err)
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 330e2d82..88ae217d 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/kernel/hypeman/lib/devices"
-	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/logger"
 )
 
@@ -64,18 +63,6 @@ func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err er
 	}
 }
 
-// validateVGPUHypervisorCompat rejects the one proven-broken combination:
-// vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor (upstream
-// cloud-hypervisor#7572), and the wedged VM then blocks the VF release until
-// startup reconcile. Hypervisor selection otherwise remains caller policy;
-// mdev on Cloud Hypervisor keeps working. See lib/devices/GPU.md.
-func validateVGPUHypervisorCompat(framework devices.VGPUFramework, hvType hypervisor.Type) error {
-	if framework == devices.VGPUFrameworkVendorVFIO && hvType == hypervisor.TypeCloudHypervisor {
-		return fmt.Errorf("%w: vendor VFIO vGPUs are not functional on cloud-hypervisor, use qemu", ErrInvalidRequest)
-	}
-	return nil
-}
-
 func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error {
 	destroy := m.destroyVGPU
 	if destroy == nil {
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index b34fe242..1bdb0fb9 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -319,35 +319,6 @@ func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) {
 	assert.ErrorIs(t, err, cause)
 }
 
-func TestValidateVGPUHypervisorCompat(t *testing.T) {
-	t.Parallel()
-
-	err := validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeCloudHypervisor)
-	require.ErrorIs(t, err, ErrInvalidRequest)
-	assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeQEMU))
-	assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkMdev, hypervisor.TypeCloudHypervisor))
-}
-
-func TestStartRejectsVendorVFIOOnCloudHypervisor(t *testing.T) {
-	var destroyed []devices.VGPUAssignment
-	m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error {
-		destroyed = append(destroyed, assignment)
-		return nil
-	})
-	meta, err := m.loadMetadata(id)
-	require.NoError(t, err)
-	meta.HypervisorType = hypervisor.TypeCloudHypervisor
-	require.NoError(t, m.saveMetadata(meta))
-
-	_, err = m.startInstance(context.Background(), id, StartInstanceRequest{})
-	require.ErrorIs(t, err, ErrInvalidRequest)
-
-	require.Len(t, destroyed, 1, "the rejected vGPU must be released by rollback")
-	stored, err := m.loadMetadata(id)
-	require.NoError(t, err)
-	assert.Empty(t, stored.GPUDevicePath, "no assignment may be persisted for a rejected combination")
-}
-
 func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {
 	var destroyed []devices.VGPUAssignment
 	m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error {

From 91556d15acdba2c8b6b46093ac962fec9f46aaaa Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Tue, 11 Aug 2026 19:21:39 +0000
Subject: [PATCH 35/38] Carry identity fields into the create-pending retention
 stub

retainedVGPUFromCreateError built a GPU-fields-only stub, so the
retained record from a failed device-layer cleanup listed nameless and,
with GPUProfile empty, the API hid its gpu block including device_path.
The caller now supplies the identity fields and the stub picks up the
pending device's profile.
---
 lib/instances/create.go    | 16 +++++++++++++++-
 lib/instances/vgpu.go      | 19 +++++++++++--------
 lib/instances/vgpu_test.go |  6 ++++--
 3 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index 510f179a..08d699f8 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -297,7 +297,21 @@ func (m *manager) createInstance(
 		log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile)
 		gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id)
 		if err != nil {
-			retainedVGPU = retainedVGPUFromCreateError(id, m.nowUTC(), err)
+			stub := StoredMetadata{
+				Id:                id,
+				Name:              req.Name,
+				Image:             req.Image,
+				ResolvedImage:     resolvedImageRef,
+				Platform:          imageInfo.Platform,
+				CreatedAt:         time.Now(),
+				HypervisorType:    hvType,
+				HypervisorVersion: hvVersion,
+				DataDir:           m.paths.InstanceDir(id),
+			}
+			if starterErr == nil {
+				stub.SocketPath = m.paths.InstanceSocket(id, starter.SocketName())
+			}
+			retainedVGPU = retainedVGPUFromCreateError(stub, m.nowUTC(), err)
 			log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err)
 			return nil, wrapCreateVGPUErr(req.GPU.Profile, err)
 		}
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 88ae217d..4caa365b 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -49,18 +49,21 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) {
 	return &pending.Device, true
 }
 
-func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err error) *StoredMetadata {
+// retainedVGPUFromCreateError fills stub with the pending device's assignment
+// fields when err carries a failed device-layer cleanup. The caller provides
+// identity fields on stub so the retained record lists as a recognizable,
+// deletable instance.
+func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata {
 	device, ok := vgpuDevicePendingCleanup(err)
 	if !ok {
 		return nil
 	}
-	return &StoredMetadata{
-		Id:            instanceID,
-		GPUFramework:  device.Framework,
-		GPUDevicePath: device.SysfsPath,
-		GPUMdevUUID:   device.MdevUUID,
-		GPUAssignedAt: &assignedAt,
-	}
+	stub.GPUProfile = device.ProfileName
+	stub.GPUFramework = device.Framework
+	stub.GPUDevicePath = device.SysfsPath
+	stub.GPUMdevUUID = device.MdevUUID
+	stub.GPUAssignedAt = &assignedAt
+	return &stub
 }
 
 func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error {
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 1bdb0fb9..e5694ef3 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -143,9 +143,11 @@ func TestVGPUDevicePendingCleanup(t *testing.T) {
 	assert.Equal(t, device, *actual)
 
 	assignedAt := time.Now().UTC()
-	retained := retainedVGPUFromCreateError("inst-1", assignedAt, wrapped)
+	retained := retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped)
 	require.NotNil(t, retained)
 	assert.Equal(t, "inst-1", retained.Id)
+	assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub")
+	assert.Equal(t, "img", retained.Image)
 	assert.Equal(t, device.Framework, retained.GPUFramework)
 	assert.Equal(t, device.SysfsPath, retained.GPUDevicePath)
 	assert.Equal(t, assignedAt, *retained.GPUAssignedAt)
@@ -153,7 +155,7 @@ func TestVGPUDevicePendingCleanup(t *testing.T) {
 	actual, ok = vgpuDevicePendingCleanup(cause)
 	assert.False(t, ok)
 	assert.Nil(t, actual)
-	assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause))
+	assert.Nil(t, retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1"}, assignedAt, cause))
 }
 
 type startRetentionNetworkManager struct {

From b6a39019e3a978aa6c98c49533fd10562844e79f Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Tue, 11 Aug 2026 19:39:32 +0000
Subject: [PATCH 36/38] Grace recent dead-PID claims in the release scan like
 reconcile does

Startup reconcile protects an assignment whose PID is absent or stale
for a bounded grace window, but the release-side claim scan treated a
dead PID as unclaimed immediately. Align the two guards: a recent
assignment whose recorded hypervisor is not running fails the scan
closed so the requester retains and retries, and past the grace window
the dead claim no longer blocks the release.
---
 lib/instances/vgpu.go      |  8 +++++++-
 lib/instances/vgpu_test.go | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 4caa365b..2f9cfa7d 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -110,7 +110,7 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic
 	cleanupMeta := rollbackMeta
 	releaseErr := m.destroyVGPUAssignment(ctx, assignment)
 	if releaseErr != nil {
-		logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr)
+		logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr)
 		setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt)
 		retained = true
 	}
@@ -208,6 +208,12 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu
 		if pid > 0 {
 			return true, nil
 		}
+		// A dead PID with a recent assignment gets the same bounded grace as
+		// startup reconcile protection, so the two guards agree in the
+		// fail-closed direction while a mid-boot claimant hydrates.
+		if stored.GPUAssignedAt != nil && time.Since(*stored.GPUAssignedAt) < VGPUAssignmentStartupGracePeriod {
+			return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath)
+		}
 	}
 	return false, nil
 }
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index e5694ef3..d5e988a4 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -528,6 +528,38 @@ func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T
 	assert.False(t, claimed)
 }
 
+func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing.T) {
+	m := &manager{paths: paths.New(t.TempDir())}
+	claimantID := "claimant-dead-pid"
+	require.NoError(t, m.ensureDirectories(claimantID))
+	deadPID := 1<<22 - 1
+	require.False(t, ProcessExists(deadPID))
+	assignedAt := time.Now().UTC()
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            claimantID,
+		HypervisorPID: &deadPID,
+		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+		GPUAssignedAt: &assignedAt,
+	}}))
+
+	// Same bounded grace as startup reconcile: a recent claim whose PID is
+	// dead fails closed instead of being treated as unclaimed.
+	_, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4")
+	require.Error(t, err)
+
+	// Past the grace period the dead claim no longer blocks the release.
+	stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod)
+	meta, err := m.loadMetadata(claimantID)
+	require.NoError(t, err)
+	meta.GPUAssignedAt = &stale
+	require.NoError(t, m.saveMetadata(meta))
+
+	claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4")
+	require.NoError(t, err)
+	assert.False(t, claimed)
+}
+
 func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) {
 	t.Parallel()
 

From 0f124e986b29879e7233912d9d9fef47f0f1dffc Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 13 Aug 2026 18:57:32 +0000
Subject: [PATCH 37/38] Reject start on vGPU retention records

A failed create whose vGPU release also failed persists a delete-only
retention stub with no boot configuration. The stub derives as Stopped,
so start would release the retained VF and then try to boot the
incomplete record. Mark the stub with GPURetainedForCleanup and reject
start with invalid_state guidance pointing at delete, which retries the
release.
---
 lib/instances/create.go              | 31 ++++++++++++++--------------
 lib/instances/lifecycle_noop_test.go | 20 ++++++++++++++++++
 lib/instances/start.go               |  7 +++++++
 lib/instances/types.go               |  4 ++++
 lib/instances/vgpu_test.go           |  2 ++
 5 files changed, 49 insertions(+), 15 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index 08d699f8..8a3dc262 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -651,21 +651,22 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	// deletable record rather than a nameless phantom, but drop resource
 	// claims (network, volumes, devices) that rollback already released.
 	retained := StoredMetadata{
-		Id:                id,
-		Name:              retainedVGPU.Name,
-		Image:             retainedVGPU.Image,
-		ResolvedImage:     retainedVGPU.ResolvedImage,
-		Platform:          retainedVGPU.Platform,
-		CreatedAt:         retainedVGPU.CreatedAt,
-		HypervisorType:    retainedVGPU.HypervisorType,
-		HypervisorVersion: retainedVGPU.HypervisorVersion,
-		SocketPath:        retainedVGPU.SocketPath,
-		DataDir:           retainedVGPU.DataDir,
-		GPUProfile:        retainedVGPU.GPUProfile,
-		GPUFramework:      retainedVGPU.GPUFramework,
-		GPUDevicePath:     retainedVGPU.GPUDevicePath,
-		GPUMdevUUID:       retainedVGPU.GPUMdevUUID,
-		GPUAssignedAt:     retainedVGPU.GPUAssignedAt,
+		Id:                    id,
+		Name:                  retainedVGPU.Name,
+		Image:                 retainedVGPU.Image,
+		ResolvedImage:         retainedVGPU.ResolvedImage,
+		Platform:              retainedVGPU.Platform,
+		CreatedAt:             retainedVGPU.CreatedAt,
+		HypervisorType:        retainedVGPU.HypervisorType,
+		HypervisorVersion:     retainedVGPU.HypervisorVersion,
+		SocketPath:            retainedVGPU.SocketPath,
+		DataDir:               retainedVGPU.DataDir,
+		GPUProfile:            retainedVGPU.GPUProfile,
+		GPUFramework:          retainedVGPU.GPUFramework,
+		GPUDevicePath:         retainedVGPU.GPUDevicePath,
+		GPUMdevUUID:           retainedVGPU.GPUMdevUUID,
+		GPUAssignedAt:         retainedVGPU.GPUAssignedAt,
+		GPURetainedForCleanup: true,
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go
index c74d352f..e440e7e4 100644
--- a/lib/instances/lifecycle_noop_test.go
+++ b/lib/instances/lifecycle_noop_test.go
@@ -324,6 +324,26 @@ func TestStartPersistsStaleVGPUReleaseImmediately(t *testing.T) {
 	assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start")
 }
 
+func TestStartRejectsVGPURetentionRecord(t *testing.T) {
+	m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC())
+	meta, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	meta.GPUProfile = "NVIDIA L40S-2Q"
+	meta.GPUFramework = devices.VGPUFrameworkNone
+	meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4"
+	meta.GPURetainedForCleanup = true
+	require.NoError(t, m.saveMetadata(meta))
+
+	_, err = m.StartInstance(context.Background(), id, StartInstanceRequest{})
+	require.ErrorIs(t, err, ErrInvalidState)
+	require.ErrorContains(t, err, "delete it to release the assignment")
+
+	// The retained assignment must survive the rejected start for delete.
+	stored, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath)
+}
+
 func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) {
 	m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC())
 	meta, err := m.loadMetadata(id)
diff --git a/lib/instances/start.go b/lib/instances/start.go
index 3e164b39..8914f2ae 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -47,6 +47,13 @@ func (m *manager) startInstance(
 		log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State)
 		return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State)
 	}
+	if stored.GPURetainedForCleanup {
+		// A delete-only retention stub from a failed create: it carries no
+		// boot configuration, so starting it would release the retained VF
+		// and then boot an incomplete record. Delete retries the release.
+		log.ErrorContext(ctx, "refusing to start vGPU retention record", "instance_id", id)
+		return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState)
+	}
 
 	// Release any assignment retained by an earlier failed release and
 	// persist the cleared fields immediately, so a failure later in start
diff --git a/lib/instances/types.go b/lib/instances/types.go
index f9f33c3a..9eb33360 100644
--- a/lib/instances/types.go
+++ b/lib/instances/types.go
@@ -156,6 +156,10 @@ type StoredMetadata struct {
 	GPUDevicePath string
 	GPUMdevUUID   string     // populated for mdev-backed vGPUs
 	GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection
+	// GPURetainedForCleanup marks a delete-only retention stub written when a
+	// failed create could not release its vGPU: the record has no boot
+	// configuration, so only delete (which retries the release) may act on it.
+	GPURetainedForCleanup bool
 
 	// Command overrides (like docker run <image> <command>)
 	Entrypoint []string // Override image entrypoint (nil = use image default)
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index d5e988a4..a6fe4e49 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -57,6 +57,8 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 	assert.False(t, retained.NetworkEnabled)
 	assert.Empty(t, retained.IP)
 	assert.Empty(t, retained.Volumes)
+	// The stub has no boot configuration, so it is marked delete-only.
+	assert.True(t, retained.GPURetainedForCleanup)
 }
 
 func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) {

From ce4032367797302ff268a94fd72bdfe7557617f8 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 13 Aug 2026 19:05:38 +0000
Subject: [PATCH 38/38] Make vGPU retention records fully delete-only

A fork or snapshot of a failed-create retention stub could never boot:
the stub has no boot configuration, and clearing the delete-only marker
on the child would only produce a startable-but-broken record that
recreates a vGPU from GPUProfile with incomplete metadata. Reject fork
and snapshot of retention stubs with the same invalid_state guidance as
start, so delete (which retries the release) is the only action on
them.
---
 lib/instances/fork.go          |  6 ++++++
 lib/instances/fork_test.go     | 25 +++++++++++++++++++++++++
 lib/instances/snapshot.go      |  7 +++++++
 lib/instances/snapshot_test.go | 25 +++++++++++++++++++++++++
 4 files changed, 63 insertions(+)

diff --git a/lib/instances/fork.go b/lib/instances/fork.go
index 7eaf7c0c..514e7560 100644
--- a/lib/instances/fork.go
+++ b/lib/instances/fork.go
@@ -219,6 +219,12 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin
 	default:
 		return nil, false, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State)
 	}
+	if stored.GPURetainedForCleanup {
+		// A delete-only retention stub from a failed create has no boot
+		// configuration, so a fork of it could never boot. Delete the stub to
+		// release its retained vGPU assignment.
+		return nil, false, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState)
+	}
 
 	if !supportValidated {
 		if err := m.validateForkSupport(ctx, stored.HypervisorType); err != nil {
diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go
index d9cace3f..8e2142b1 100644
--- a/lib/instances/fork_test.go
+++ b/lib/instances/fork_test.go
@@ -63,6 +63,31 @@ func TestForkInstanceClearsVGPUAssignment(t *testing.T) {
 	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath)
 }
 
+func TestForkInstanceRejectsVGPURetentionRecord(t *testing.T) {
+	manager, _ := setupTestManager(t)
+	ctx := context.Background()
+	hvType := hypervisor.Type("fork-vgpu-retention-test")
+	hypervisor.RegisterCapabilities(hvType, hypervisor.Capabilities{SupportsConcurrentForkPrepare: true})
+	manager.vmStarters[hvType] = concurrentForkPrepareTestStarter{}
+
+	sourceID := "fork-vgpu-retention-source"
+	createStoppedSnapshotSourceFixture(t, manager, sourceID, sourceID, hvType)
+
+	meta, err := manager.loadMetadata(sourceID)
+	require.NoError(t, err)
+	meta.GPUProfile = "NVIDIA L40S-2Q"
+	meta.GPUFramework = devices.VGPUFramework("future-framework")
+	meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4"
+	meta.GPURetainedForCleanup = true
+	require.NoError(t, manager.saveMetadata(meta))
+
+	// The delete-only retention stub has no boot configuration, so a fork of
+	// it could never boot; only delete may act on it.
+	_, err = manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vgpu-retention-copy"})
+	require.ErrorIs(t, err, ErrInvalidState)
+	require.ErrorContains(t, err, "delete it to release the assignment")
+}
+
 func TestForkInstance_VZStoppedSourceSupported(t *testing.T) {
 	t.Parallel()
 	manager, _ := setupTestManager(t)
diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go
index c54b91fd..6e11259d 100644
--- a/lib/instances/snapshot.go
+++ b/lib/instances/snapshot.go
@@ -66,6 +66,13 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps
 	inst := m.toInstance(ctx, meta)
 	stored := &meta.StoredMetadata
 
+	if stored.GPURetainedForCleanup {
+		// A delete-only retention stub from a failed create has no boot
+		// configuration, so a snapshot of it could never be restored or
+		// forked into a bootable instance. Delete the stub to release its
+		// retained vGPU assignment.
+		return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState)
+	}
 	if err := validateForkVolumeSafety(stored.Volumes); err != nil {
 		return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err)
 	}
diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go
index a42bb1d1..4e3ff48d 100644
--- a/lib/instances/snapshot_test.go
+++ b/lib/instances/snapshot_test.go
@@ -52,6 +52,31 @@ func TestForkSnapshotClearsVGPUAssignment(t *testing.T) {
 	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath)
 }
 
+func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) {
+	mgr, _ := setupTestManager(t)
+	ctx := context.Background()
+
+	sourceID := "snapshot-vgpu-retention"
+	createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor)
+
+	meta, err := mgr.loadMetadata(sourceID)
+	require.NoError(t, err)
+	meta.GPUProfile = "NVIDIA L40S-2Q"
+	meta.GPUFramework = devices.VGPUFramework("future-framework")
+	meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4"
+	meta.GPURetainedForCleanup = true
+	require.NoError(t, mgr.saveMetadata(meta))
+
+	// The delete-only retention stub has no boot configuration, so a snapshot
+	// of it could never be restored or forked into a bootable instance.
+	_, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{
+		Kind: SnapshotKindStopped,
+		Name: "snapshot-vgpu-retention",
+	})
+	require.ErrorIs(t, err, ErrInvalidState)
+	require.ErrorContains(t, err, "delete it to release the assignment")
+}
+
 func TestRestoreSnapshotDoesNotResurrectStaleVGPUAssignment(t *testing.T) {
 	mgr, _ := setupTestManager(t)
 	ctx := context.Background()