Experiment with CPU and memory hotplug snapshots

lib/hypervisor/cloudhypervisor/cloudhypervisor.go

@@ -208,8 +208,33 @@ func (c *CloudHypervisor) Snapshot(ctx context.Context, destPath string) error {
 	return nil
 }
 
+func (c *CloudHypervisor) ResizeVCPUs(ctx context.Context, vcpus int) error {
+	resp, err := c.client.PutVmResizeWithResponse(ctx, vmm.VmResize{DesiredVcpus: &vcpus})
+	if err != nil {
+		return fmt.Errorf("resize vCPUs: %w", err)
+	}
+	if resp.StatusCode() != 204 {
+		return fmt.Errorf("resize vCPUs failed with status %d", resp.StatusCode())
+	}
+	return nil
+}
+
 // ResizeMemory changes the VM's memory allocation.
 func (c *CloudHypervisor) ResizeMemory(ctx context.Context, bytes int64) error {
+	if ExperimentalHotplugOverlayEnabled() {
+		resp, err := c.client.PutVmResizeZoneWithResponse(ctx, vmm.VmResizeZone{
+			Id:         ptr(kernelPagingMemoryZoneID),
+			DesiredRam: ptr(bytes),
+		})
+		if err != nil {
+			return fmt.Errorf("resize memory zone: %w", err)
+		}
+		if resp.StatusCode() != 204 {
+			return fmt.Errorf("resize memory zone failed with status %d", resp.StatusCode())
+		}
+		return nil
+	}
+
 	resizeConfig := vmm.VmResize{DesiredRam: &bytes}
 	resp, err := c.client.PutVmResizeWithResponse(ctx, resizeConfig)
 	if err != nil {

lib/hypervisor/cloudhypervisor/config.go

@@ -37,13 +37,17 @@ func ToVMConfig(cfg hypervisor.VMConfig) vmm.VmConfig {
 	}
 
 	// CPU configuration
+	maxVCPUs := cfg.VCPUs
+	if experimentalMaxVCPUs := ExperimentalMaxVCPUs(); experimentalMaxVCPUs > maxVCPUs {
+		maxVCPUs = experimentalMaxVCPUs
+	}
 	cpus := vmm.CpusConfig{
 		BootVcpus: cfg.VCPUs,
-		MaxVcpus:  cfg.VCPUs,
+		MaxVcpus:  maxVCPUs,
 	}
 
-	// Add topology if provided
-	if cfg.Topology != nil {
+	// Add topology if provided.
+	if cfg.Topology != nil && maxVCPUs == cfg.VCPUs {
 		cpus.Topology = &vmm.CpuTopology{
 			ThreadsPerCore: ptr(cfg.Topology.ThreadsPerCore),
 			CoresPerDie:    ptr(cfg.Topology.CoresPerDie),
@@ -55,21 +59,26 @@ func ToVMConfig(cfg hypervisor.VMConfig) vmm.VmConfig {
 	// Memory configuration
 	memory := vmm.MemoryConfig{Size: cfg.MemoryBytes}
 	if cfg.MemoryBackingFile != "" {
-		zones := []vmm.MemoryZoneConfig{{
+		zone := vmm.MemoryZoneConfig{
 			Id:        kernelPagingMemoryZoneID,
 			Size:      cfg.MemoryBytes,
 			File:      ptr(cfg.MemoryBackingFile),
 			Shared:    ptr(false),
 			Prefault:  ptr(false),
 			Hugepages: ptr(false),
-		}}
+		}
+		if cfg.HotplugBytes > 0 && ExperimentalHotplugOverlayEnabled() {
+			zone.HotplugSize = &cfg.HotplugBytes
+			memory.HotplugMethod = ptr("VirtioMem")
+		}
+		zones := []vmm.MemoryZoneConfig{zone}
 		memory.Size = 0
 		memory.Shared = ptr(false)
 		memory.Prefault = ptr(false)
 		memory.Hugepages = ptr(false)
 		memory.Zones = &zones
 	}
-	if cfg.HotplugBytes > 0 {
+	if cfg.HotplugBytes > 0 && (cfg.MemoryBackingFile == "" || !ExperimentalHotplugOverlayEnabled()) {
 		memory.HotplugSize = &cfg.HotplugBytes
 		memory.HotplugMethod = ptr("VirtioMem")
 	}

lib/hypervisor/cloudhypervisor/config_test.go

@@ -34,6 +34,47 @@ func TestToVMConfigFileBackedMemory(t *testing.T) {
 	assert.False(t, *zone.Hugepages)
 }
 
+func TestToVMConfigCPUHotplugEnvelope(t *testing.T) {
+	t.Setenv(experimentalMaxVCPUsEnv, "8")
+
+	vmCfg := ToVMConfig(hypervisor.VMConfig{
+		VCPUs:       4,
+		MemoryBytes: 1024 * 1024 * 1024,
+		Topology: &hypervisor.CPUTopology{
+			ThreadsPerCore: 2,
+			CoresPerDie:    2,
+			DiesPerPackage: 1,
+			Packages:       1,
+		},
+	})
+
+	require.NotNil(t, vmCfg.Cpus)
+	assert.Equal(t, 4, vmCfg.Cpus.BootVcpus)
+	assert.Equal(t, 8, vmCfg.Cpus.MaxVcpus)
+	assert.Nil(t, vmCfg.Cpus.Topology)
+}
+
+func TestToVMConfigFileBackedHotplugMemory(t *testing.T) {
+	t.Setenv(experimentalHotplugOverlayEnv, "true")
+
+	vmCfg := ToVMConfig(hypervisor.VMConfig{
+		VCPUs:             2,
+		MemoryBytes:       4 * 1024 * 1024 * 1024,
+		HotplugBytes:      4 * 1024 * 1024 * 1024,
+		MemoryBackingFile: "/var/lib/hypeman/memory.raw",
+	})
+
+	require.NotNil(t, vmCfg.Memory)
+	require.NotNil(t, vmCfg.Memory.Zones)
+	require.Len(t, *vmCfg.Memory.Zones, 1)
+	zone := (*vmCfg.Memory.Zones)[0]
+	require.NotNil(t, zone.HotplugSize)
+	assert.Equal(t, int64(4*1024*1024*1024), *zone.HotplugSize)
+	assert.Nil(t, vmCfg.Memory.HotplugSize)
+	require.NotNil(t, vmCfg.Memory.HotplugMethod)
+	assert.Equal(t, "VirtioMem", *vmCfg.Memory.HotplugMethod)
+}
+
 func TestToVMConfig_GuestMemoryBalloon(t *testing.T) {
 	cfg := hypervisor.VMConfig{
 		VCPUs:       1,

lib/hypervisor/cloudhypervisor/diff_snapshot_linux.go

@@ -50,7 +50,7 @@ func mergeCloudHypervisorDiff(snapshotDir string) (diffMergeStats, error) {
 	dstFD := int(dst.Fd())
 	size := srcInfo.Size()
 	offset := int64(0)
-	cloneRanges := true
+	cloneRanges := !ExperimentalHotplugOverlayEnabled()
 	buf := make([]byte, 1<<20)
 
 	for offset < size {

lib/hypervisor/cloudhypervisor/hotplug_overlay.go

@@ -0,0 +1,29 @@
+package cloudhypervisor
+
+import (
+	"os"
+	"strconv"
+	"strings"
+)
+
+const (
+	experimentalHotplugOverlayEnv = "HYPEMAN_EXPERIMENTAL_CH_HOTPLUG_OVERLAY"
+	experimentalMaxVCPUsEnv       = "HYPEMAN_EXPERIMENTAL_CH_MAX_VCPUS"
+)
+
+func ExperimentalMaxVCPUs() int {
+	value, err := strconv.Atoi(strings.TrimSpace(os.Getenv(experimentalMaxVCPUsEnv)))
+	if err != nil || value <= 0 {
+		return 0
+	}
+	return value
+}
+
+func ExperimentalHotplugOverlayEnabled() bool {
+	switch strings.ToLower(strings.TrimSpace(os.Getenv(experimentalHotplugOverlayEnv))) {
+	case "1", "true", "yes", "on":
+		return true
+	default:
+		return false
+	}
+}

lib/hypervisor/cloudhypervisor/kernel_paging.go

@@ -80,12 +80,16 @@ func prepareSnapshotForKernelPaging(snapshotDir string) (bool, error) {
 		return false, nil
 	}
 
-	fileOffsets, totalSize, ok := snapshotMappingFileOffsets(ranges, mappings)
-	if !ok || totalSize != uint64(memoryInfo.Size()) {
-		return false, nil
-	}
-	for i := range mappings {
-		mappings[i].FileOffset = fileOffsets[i]
+	fixedLayout := ExperimentalHotplugOverlayEnabled() &&
+		fixedSnapshotMappingLayout(ranges, mappings, uint64(memoryInfo.Size()))
+	if !fixedLayout {
+		fileOffsets, totalSize, ok := snapshotMappingFileOffsets(ranges, mappings)
+		if !ok || totalSize != uint64(memoryInfo.Size()) {
+			return false, nil
+		}
+		for i := range mappings {
+			mappings[i].FileOffset = fileOffsets[i]
+		}
 	}
 
 	updatedConfig, ok, err := kernelPagingConfig(configData, memoryPath, mappings)
@@ -256,6 +260,55 @@ func snapshotMappingFileOffsets(
 	return mappingOffsets, offset, true
 }
 
+func fixedSnapshotMappingLayout(
+	ranges []snapshotMemoryRange,
+	mappings []snapshotGuestRAMMapping,
+	fileSize uint64,
+) bool {
+	if len(ranges) == 0 || len(mappings) == 0 {
+		return false
+	}
+
+	hasVirtioMem := false
+	var layoutSize uint64
+	for _, mapping := range mappings {
+		if mapping.Size == 0 || mapping.FileOffset > ^uint64(0)-mapping.Size {
+			return false
+		}
+		end := mapping.FileOffset + mapping.Size
+		if end > layoutSize {
+			layoutSize = end
+		}
+		hasVirtioMem = hasVirtioMem || mapping.VirtioMem
+	}
+	if !hasVirtioMem || layoutSize != fileSize {
+		return false
+	}
+
+	for _, r := range ranges {
+		if r.Length == 0 || r.GPA > ^uint64(0)-r.Length {
+			return false
+		}
+		rangeEnd := r.GPA + r.Length
+		found := false
+		for _, mapping := range mappings {
+			if mapping.GPA > ^uint64(0)-mapping.Size {
+				return false
+			}
+			if r.GPA >= mapping.GPA && rangeEnd <= mapping.GPA+mapping.Size {
+				if found {
+					return false
+				}
+				found = true
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+	return true
+}
+
 func kernelPagingConfig(
 	configData []byte,
 	memoryPath string,
@@ -301,16 +354,17 @@ func kernelPagingConfig(
 	}
 
 	zone := zones[0]
+	zoneHasHotplug := hasConfiguredDevice(zone["hotplug_size"]) || hasConfiguredDevice(zone["hotplugged_size"])
 	if rawBool(zone["shared"]) || rawBool(zone["hugepages"]) ||
-		hasConfiguredDevice(zone["hotplug_size"]) || hasConfiguredDevice(zone["hotplugged_size"]) {
+		(zoneHasHotplug && !ExperimentalHotplugOverlayEnabled()) {
 		return nil, false, nil
 	}
 	var zoneID string
 	if err := json.Unmarshal(zone["id"], &zoneID); err != nil || zoneID != kernelPagingMemoryZoneID {
 		return nil, false, nil
 	}
 	for _, mapping := range mappings {
-		if mapping.ZoneID != zoneID {
+		if mapping.ZoneID != zoneID || (mapping.VirtioMem && !ExperimentalHotplugOverlayEnabled()) {
 			return nil, false, nil
 		}
 	}

lib/hypervisor/cloudhypervisor/kernel_paging_test.go

@@ -70,6 +70,59 @@ func TestPrepareSnapshotForKernelPaging(t *testing.T) {
 	assert.False(t, optimized)
 }
 
+func TestPrepareSnapshotForKernelPagingWithHotplugOverlay(t *testing.T) {
+	t.Setenv(experimentalHotplugOverlayEnv, "true")
+
+	dir := writeKernelPagingSnapshot(t, kernelPagingSnapshotFixture{
+		Ranges: []snapshotMemoryRange{
+			{GPA: 0, Length: 4096},
+			{GPA: 1 << 32, Length: 4096},
+		},
+		Mappings: []snapshotGuestRAMMapping{
+			{Slot: 0, GPA: 0, Size: 4096, ZoneID: kernelPagingMemoryZoneID},
+			{Slot: 1, GPA: 1 << 32, Size: 8192, ZoneID: kernelPagingMemoryZoneID, VirtioMem: true, FileOffset: 4096},
+		},
+		MemorySize:     0,
+		MemoryFileSize: 12288,
+		Zones: []map[string]any{{
+			"id":              kernelPagingMemoryZoneID,
+			"size":            4096,
+			"file":            "/old/memory.raw",
+			"hotplug_size":    8192,
+			"hotplugged_size": 4096,
+		}},
+	})
+
+	optimized, err := prepareSnapshotForKernelPaging(dir)
+	require.NoError(t, err)
+	require.True(t, optimized)
+
+	stateData, err := os.ReadFile(filepath.Join(dir, cloudHypervisorStateFile))
+	require.NoError(t, err)
+	_, _, ranges, mappings, err := decodeSnapshotMemoryState(stateData)
+	require.NoError(t, err)
+	assert.Empty(t, ranges)
+	require.Len(t, mappings, 2)
+	assert.Equal(t, uint64(4096), mappings[1].FileOffset)
+
+	configData, err := os.ReadFile(filepath.Join(dir, cloudHypervisorConfigFile))
+	require.NoError(t, err)
+	var config struct {
+		Memory struct {
+			Zones []struct {
+				File           string `json:"file"`
+				HotplugSize    uint64 `json:"hotplug_size"`
+				HotpluggedSize uint64 `json:"hotplugged_size"`
+			} `json:"zones"`
+		} `json:"memory"`
+	}
+	require.NoError(t, json.Unmarshal(configData, &config))
+	require.Len(t, config.Memory.Zones, 1)
+	assert.Equal(t, filepath.Join(dir, cloudHypervisorMemoryFile), config.Memory.Zones[0].File)
+	assert.Equal(t, uint64(8192), config.Memory.Zones[0].HotplugSize)
+	assert.Equal(t, uint64(4096), config.Memory.Zones[0].HotpluggedSize)
+}
+
 func TestPrepareSnapshotForKernelPagingIgnoresUnmarkedMemoryZones(t *testing.T) {
 	t.Parallel()
 
@@ -177,23 +230,26 @@ func TestPrepareSnapshotForKernelPagingLeavesUnsupportedSnapshotsUnchanged(t *te
 }
 
 type kernelPagingSnapshotFixture struct {
-	Ranges     []snapshotMemoryRange
-	Mappings   []snapshotGuestRAMMapping
-	MemorySize uint64
-	Zones      []map[string]any
-	Balloon    any
-	Hugepages  bool
-	Net        any
-	Devices    any
+	Ranges         []snapshotMemoryRange
+	Mappings       []snapshotGuestRAMMapping
+	MemorySize     uint64
+	MemoryFileSize int64
+	Zones          []map[string]any
+	Balloon        any
+	Hugepages      bool
+	Net            any
+	Devices        any
 }
 
 func writeKernelPagingSnapshot(t *testing.T, fixture kernelPagingSnapshotFixture) string {
 	t.Helper()
 
 	dir := t.TempDir()
-	memorySize := int64(0)
-	for _, r := range fixture.Ranges {
-		memorySize += int64(r.Length)
+	memorySize := fixture.MemoryFileSize
+	if memorySize == 0 {
+		for _, r := range fixture.Ranges {
+			memorySize += int64(r.Length)
+		}
 	}
 	require.NoError(t, os.WriteFile(filepath.Join(dir, cloudHypervisorMemoryFile), nil, 0600))
 	require.NoError(t, os.Truncate(filepath.Join(dir, cloudHypervisorMemoryFile), memorySize))

lib/hypervisor/cloudhypervisor/process.go

@@ -80,7 +80,7 @@ func (s *Starter) ValidateConfig(config hypervisor.VMConfig) error {
 	if config.MemoryBytes <= 0 {
 		return fmt.Errorf("file-backed memory requires a positive memory size")
 	}
-	if config.HotplugBytes > 0 {
+	if config.HotplugBytes > 0 && !ExperimentalHotplugOverlayEnabled() {
 		return fmt.Errorf("file-backed memory does not support hotplug memory")
 	}
 	if config.GuestMemory.EnableBalloon {
@@ -138,7 +138,11 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s
 	if err := s.ValidateConfig(config); err != nil {
 		return 0, nil, err
 	}
-	if err := prepareMemoryBackingFile(config.MemoryBackingFile, config.MemoryBytes); err != nil {
+	memoryBackingSize := config.MemoryBytes
+	if config.MemoryBackingFile != "" && ExperimentalHotplugOverlayEnabled() {
+		memoryBackingSize += config.HotplugBytes
+	}
+	if err := prepareMemoryBackingFile(config.MemoryBackingFile, memoryBackingSize); err != nil {
 		return 0, nil, fmt.Errorf("prepare memory backing file: %w", err)
 	}